From 2aa8956925b3ef11d3a6f16347bfeb48870c62e0 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 11 May 2022 13:23:37 -0700 Subject: [PATCH 01/18] chore(v2): setup the v2 release given the expected breaking changes, it's time to setup the v2 release BREAKING CHANGE: v2 modules --- CONTRIBUTING.md | 2 +- README.md | 6 +++--- benchmarks/benchmark_test.go | 8 ++++---- benchmarks/testinstance/testinstance.go | 12 ++++++------ benchmarks/testnet/interface.go | 2 +- benchmarks/testnet/peernet.go | 2 +- channelmonitor/channelmonitor.go | 4 ++-- channelmonitor/channelmonitor_test.go | 2 +- channels/block_index_cache.go | 2 +- channels/channel_state.go | 4 ++-- channels/channels.go | 8 ++++---- channels/channels_fsm.go | 4 ++-- channels/channels_test.go | 8 ++++---- channels/internal/internalchannel.go | 2 +- channels/internal/internalchannel_cbor_gen.go | 2 +- encoding/encoding_test.go | 4 ++-- go.mod | 2 +- impl/environment.go | 2 +- impl/events.go | 6 +++--- impl/impl.go | 16 ++++++++-------- impl/initiating_test.go | 10 +++++----- impl/integration_test.go | 18 +++++++++--------- impl/receiver.go | 4 ++-- impl/responding_test.go | 10 +++++----- impl/restart.go | 8 ++++---- impl/restart_integration_test.go | 6 +++--- impl/utils.go | 6 +++--- message.go | 2 +- message/message.go | 2 +- message/message1_1/message.go | 6 +++--- message/message1_1/message_test.go | 6 +++--- message/message1_1/transfer_message.go | 2 +- message/message1_1/transfer_request.go | 6 +++--- .../message1_1/transfer_request_cbor_gen.go | 2 +- message/message1_1/transfer_request_test.go | 6 +++--- message/message1_1/transfer_response.go | 6 +++--- .../message1_1/transfer_response_cbor_gen.go | 2 +- message/message1_1/transfer_response_test.go | 6 +++--- message/message1_1prime/message.go | 6 +++--- message/message1_1prime/message_test.go | 8 ++++---- message/message1_1prime/transfer_message.go | 2 +- message/message1_1prime/transfer_request.go | 6 +++--- .../message1_1prime/transfer_request_test.go | 6 +++--- message/message1_1prime/transfer_response.go | 6 +++--- .../message1_1prime/transfer_response_test.go | 6 +++--- network/interface.go | 2 +- network/libp2p_impl.go | 4 ++-- network/libp2p_impl_test.go | 8 ++++---- registry/registry.go | 4 ++-- registry/registry_test.go | 4 ++-- scripts/fiximports | 2 +- testutil/fakedttype.go | 4 ++-- testutil/fakegraphsync.go | 6 +++--- testutil/faketransport.go | 2 +- testutil/gstestdata.go | 8 ++++---- testutil/message.go | 4 ++-- testutil/stubbedvalidator.go | 2 +- testutil/testnet.go | 4 ++-- testutil/testutil.go | 2 +- tracing/tracing.go | 2 +- tracing/tracing_test.go | 6 +++--- transport/graphsync/extension/gsextension.go | 4 ++-- transport/graphsync/graphsync.go | 4 ++-- transport/graphsync/graphsync_test.go | 10 +++++----- transport/graphsync/gskeychidmap_test.go | 2 +- types.go | 2 +- 66 files changed, 166 insertions(+), 166 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eb4746b1..a235287a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,7 +59,7 @@ import ( "github.com/filecoin-project/go-statemachine" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) ``` diff --git a/README.md b/README.md index ff5738ac..f2add43b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ This module encapsulates protocols for exchanging piece data between storage cli **Requires go 1.13** -Install the module in your package or app with `go get "github.com/filecoin-project/go-data-transfer/datatransfer"` +Install the module in your package or app with `go get "github.com/filecoin-project/go-data-transfer/v2/datatransfer"` ### Initialize a data transfer module @@ -31,8 +31,8 @@ Install the module in your package or app with `go get "github.com/filecoin-proj import ( gsimpl "github.com/ipfs/go-graphsync/impl" - datatransfer "github.com/filecoin-project/go-data-transfer/impl" - gstransport "github.com/filecoin-project/go-data-transfer/transport/graphsync" + datatransfer "github.com/filecoin-project/go-data-transfer/v2/impl" + gstransport "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" "github.com/libp2p/go-libp2p-core/host" ) diff --git a/benchmarks/benchmark_test.go b/benchmarks/benchmark_test.go index 41d78913..357f80f1 100644 --- a/benchmarks/benchmark_test.go +++ b/benchmarks/benchmark_test.go @@ -30,10 +30,10 @@ import ( mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/benchmarks/testinstance" - tn "github.com/filecoin-project/go-data-transfer/benchmarks/testnet" - "github.com/filecoin-project/go-data-transfer/testutil" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/benchmarks/testinstance" + tn "github.com/filecoin-project/go-data-transfer/v2/benchmarks/testnet" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) const stdBlockSize = 8000 diff --git a/benchmarks/testinstance/testinstance.go b/benchmarks/testinstance/testinstance.go index 6732610e..c720f44b 100644 --- a/benchmarks/testinstance/testinstance.go +++ b/benchmarks/testinstance/testinstance.go @@ -18,12 +18,12 @@ import ( "github.com/ipld/go-ipld-prime" peer "github.com/libp2p/go-libp2p-core/peer" - datatransfer "github.com/filecoin-project/go-data-transfer" - tn "github.com/filecoin-project/go-data-transfer/benchmarks/testnet" - dtimpl "github.com/filecoin-project/go-data-transfer/impl" - dtnet "github.com/filecoin-project/go-data-transfer/network" - "github.com/filecoin-project/go-data-transfer/testutil" - gstransport "github.com/filecoin-project/go-data-transfer/transport/graphsync" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + tn "github.com/filecoin-project/go-data-transfer/v2/benchmarks/testnet" + dtimpl "github.com/filecoin-project/go-data-transfer/v2/impl" + dtnet "github.com/filecoin-project/go-data-transfer/v2/network" + "github.com/filecoin-project/go-data-transfer/v2/testutil" + gstransport "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" ) // TempDirGenerator is any interface that can generate temporary directories diff --git a/benchmarks/testnet/interface.go b/benchmarks/testnet/interface.go index 8a53d040..dc5cf55c 100644 --- a/benchmarks/testnet/interface.go +++ b/benchmarks/testnet/interface.go @@ -4,7 +4,7 @@ import ( gsnet "github.com/ipfs/go-graphsync/network" "github.com/libp2p/go-libp2p-core/peer" - dtnet "github.com/filecoin-project/go-data-transfer/network" + dtnet "github.com/filecoin-project/go-data-transfer/v2/network" ) // Network is an interface for generating graphsync network interfaces diff --git a/benchmarks/testnet/peernet.go b/benchmarks/testnet/peernet.go index 94e469eb..c6ca1778 100644 --- a/benchmarks/testnet/peernet.go +++ b/benchmarks/testnet/peernet.go @@ -7,7 +7,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" mockpeernet "github.com/libp2p/go-libp2p/p2p/net/mock" - dtnet "github.com/filecoin-project/go-data-transfer/network" + dtnet "github.com/filecoin-project/go-data-transfer/v2/network" ) type peernet struct { diff --git a/channelmonitor/channelmonitor.go b/channelmonitor/channelmonitor.go index ed3b0a3c..6a5863e7 100644 --- a/channelmonitor/channelmonitor.go +++ b/channelmonitor/channelmonitor.go @@ -11,8 +11,8 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/channels" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channels" ) var log = logging.Logger("dt-chanmon") diff --git a/channelmonitor/channelmonitor_test.go b/channelmonitor/channelmonitor_test.go index bb4857f0..067d513a 100644 --- a/channelmonitor/channelmonitor_test.go +++ b/channelmonitor/channelmonitor_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) var ch1 = datatransfer.ChannelID{ diff --git a/channels/block_index_cache.go b/channels/block_index_cache.go index 490f77fd..4c16e007 100644 --- a/channels/block_index_cache.go +++ b/channels/block_index_cache.go @@ -4,7 +4,7 @@ import ( "sync" "sync/atomic" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) type readOriginalFn func(datatransfer.ChannelID) (int64, error) diff --git a/channels/channel_state.go b/channels/channel_state.go index bf89b85a..5a477486 100644 --- a/channels/channel_state.go +++ b/channels/channel_state.go @@ -10,8 +10,8 @@ import ( peer "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/channels/internal" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channels/internal" ) // channelState is immutable channel data plus mutable state diff --git a/channels/channels.go b/channels/channels.go index dd2ad45d..f4f6940c 100644 --- a/channels/channels.go +++ b/channels/channels.go @@ -17,10 +17,10 @@ import ( "github.com/filecoin-project/go-statemachine" "github.com/filecoin-project/go-statemachine/fsm" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/channels/internal" - "github.com/filecoin-project/go-data-transfer/channels/internal/migrations" - "github.com/filecoin-project/go-data-transfer/encoding" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channels/internal" + "github.com/filecoin-project/go-data-transfer/v2/channels/internal/migrations" + "github.com/filecoin-project/go-data-transfer/v2/encoding" ) type DecoderByTypeFunc func(identifier datatransfer.TypeIdentifier) (encoding.Decoder, bool) diff --git a/channels/channels_fsm.go b/channels/channels_fsm.go index 6fd51967..005cdcd3 100644 --- a/channels/channels_fsm.go +++ b/channels/channels_fsm.go @@ -6,8 +6,8 @@ import ( "github.com/filecoin-project/go-statemachine/fsm" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/channels/internal" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channels/internal" ) var log = logging.Logger("data-transfer") diff --git a/channels/channels_test.go b/channels/channels_test.go index c6fd7478..f80b433f 100644 --- a/channels/channels_test.go +++ b/channels/channels_test.go @@ -14,10 +14,10 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/channels" - "github.com/filecoin-project/go-data-transfer/encoding" - "github.com/filecoin-project/go-data-transfer/testutil" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channels" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) func TestChannels(t *testing.T) { diff --git a/channels/internal/internalchannel.go b/channels/internal/internalchannel.go index f6cf916b..6e5fba9f 100644 --- a/channels/internal/internalchannel.go +++ b/channels/internal/internalchannel.go @@ -7,7 +7,7 @@ import ( peer "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) //go:generate cbor-gen-for --map-encoding ChannelState EncodedVoucher EncodedVoucherResult diff --git a/channels/internal/internalchannel_cbor_gen.go b/channels/internal/internalchannel_cbor_gen.go index 5a6cfb19..afedd3a2 100644 --- a/channels/internal/internalchannel_cbor_gen.go +++ b/channels/internal/internalchannel_cbor_gen.go @@ -7,7 +7,7 @@ import ( "io" "sort" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" cid "github.com/ipfs/go-cid" peer "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" diff --git a/encoding/encoding_test.go b/encoding/encoding_test.go index 43a66f5d..3d751f7a 100644 --- a/encoding/encoding_test.go +++ b/encoding/encoding_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/require" - "github.com/filecoin-project/go-data-transfer/encoding" - "github.com/filecoin-project/go-data-transfer/encoding/testdata" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + "github.com/filecoin-project/go-data-transfer/v2/encoding/testdata" ) func TestRoundTrip(t *testing.T) { diff --git a/go.mod b/go.mod index 705e6742..c700d560 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/filecoin-project/go-data-transfer +module github.com/filecoin-project/go-data-transfer/v2 go 1.16 diff --git a/impl/environment.go b/impl/environment.go index 102e1441..2e774f12 100644 --- a/impl/environment.go +++ b/impl/environment.go @@ -3,7 +3,7 @@ package impl import ( "github.com/libp2p/go-libp2p-core/peer" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) type channelEnvironment struct { diff --git a/impl/events.go b/impl/events.go index 2c6860fb..710a6b26 100644 --- a/impl/events.go +++ b/impl/events.go @@ -13,9 +13,9 @@ import ( "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/encoding" - "github.com/filecoin-project/go-data-transfer/registry" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + "github.com/filecoin-project/go-data-transfer/v2/registry" ) // OnChannelOpened is called when we send a request for data to the other diff --git a/impl/impl.go b/impl/impl.go index c3f511d4..0a3c09e5 100644 --- a/impl/impl.go +++ b/impl/impl.go @@ -19,14 +19,14 @@ import ( "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/channelmonitor" - "github.com/filecoin-project/go-data-transfer/channels" - "github.com/filecoin-project/go-data-transfer/encoding" - "github.com/filecoin-project/go-data-transfer/message" - "github.com/filecoin-project/go-data-transfer/network" - "github.com/filecoin-project/go-data-transfer/registry" - "github.com/filecoin-project/go-data-transfer/tracing" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channelmonitor" + "github.com/filecoin-project/go-data-transfer/v2/channels" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/network" + "github.com/filecoin-project/go-data-transfer/v2/registry" + "github.com/filecoin-project/go-data-transfer/v2/tracing" ) var log = logging.Logger("dt-impl") diff --git a/impl/initiating_test.go b/impl/initiating_test.go index 50401014..6740f69f 100644 --- a/impl/initiating_test.go +++ b/impl/initiating_test.go @@ -15,11 +15,11 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/channels" - . "github.com/filecoin-project/go-data-transfer/impl" - "github.com/filecoin-project/go-data-transfer/message" - "github.com/filecoin-project/go-data-transfer/testutil" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channels" + . "github.com/filecoin-project/go-data-transfer/v2/impl" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) func TestDataTransferInitiating(t *testing.T) { diff --git a/impl/integration_test.go b/impl/integration_test.go index 6142c7df..9c2eee17 100644 --- a/impl/integration_test.go +++ b/impl/integration_test.go @@ -33,15 +33,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/channelmonitor" - "github.com/filecoin-project/go-data-transfer/encoding" - . "github.com/filecoin-project/go-data-transfer/impl" - "github.com/filecoin-project/go-data-transfer/message" - "github.com/filecoin-project/go-data-transfer/network" - "github.com/filecoin-project/go-data-transfer/testutil" - tp "github.com/filecoin-project/go-data-transfer/transport/graphsync" - "github.com/filecoin-project/go-data-transfer/transport/graphsync/extension" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channelmonitor" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + . "github.com/filecoin-project/go-data-transfer/v2/impl" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/network" + "github.com/filecoin-project/go-data-transfer/v2/testutil" + tp "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" ) const loremFile = "lorem.txt" diff --git a/impl/receiver.go b/impl/receiver.go index 6f861704..a33cd3a7 100644 --- a/impl/receiver.go +++ b/impl/receiver.go @@ -9,8 +9,8 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/channels" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channels" ) type receiver struct { diff --git a/impl/responding_test.go b/impl/responding_test.go index 2566886f..0625135e 100644 --- a/impl/responding_test.go +++ b/impl/responding_test.go @@ -17,11 +17,11 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/channels" - . "github.com/filecoin-project/go-data-transfer/impl" - "github.com/filecoin-project/go-data-transfer/message" - "github.com/filecoin-project/go-data-transfer/testutil" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channels" + . "github.com/filecoin-project/go-data-transfer/v2/impl" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) func TestDataTransferResponding(t *testing.T) { diff --git a/impl/restart.go b/impl/restart.go index 697985cd..fa1f79ec 100644 --- a/impl/restart.go +++ b/impl/restart.go @@ -8,10 +8,10 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/channels" - "github.com/filecoin-project/go-data-transfer/encoding" - "github.com/filecoin-project/go-data-transfer/message" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channels" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + "github.com/filecoin-project/go-data-transfer/v2/message" ) // ChannelDataTransferType identifies the type of a data transfer channel for the purposes of a restart diff --git a/impl/restart_integration_test.go b/impl/restart_integration_test.go index c3f4e389..9b06a013 100644 --- a/impl/restart_integration_test.go +++ b/impl/restart_integration_test.go @@ -15,9 +15,9 @@ import ( "go.uber.org/atomic" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - . "github.com/filecoin-project/go-data-transfer/impl" - "github.com/filecoin-project/go-data-transfer/testutil" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + . "github.com/filecoin-project/go-data-transfer/v2/impl" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) const totalIncrements = 204 diff --git a/impl/utils.go b/impl/utils.go index 0b4c4e6c..d02a4442 100644 --- a/impl/utils.go +++ b/impl/utils.go @@ -8,9 +8,9 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/message" - "github.com/filecoin-project/go-data-transfer/registry" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/registry" ) type statusList []datatransfer.Status diff --git a/message.go b/message.go index 73c8767c..3fa16b96 100644 --- a/message.go +++ b/message.go @@ -8,7 +8,7 @@ import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/libp2p/go-libp2p-core/protocol" - "github.com/filecoin-project/go-data-transfer/encoding" + "github.com/filecoin-project/go-data-transfer/v2/encoding" ) var ( diff --git a/message/message.go b/message/message.go index ff4a988b..60b183fe 100644 --- a/message/message.go +++ b/message/message.go @@ -1,7 +1,7 @@ package message import ( - message1_1 "github.com/filecoin-project/go-data-transfer/message/message1_1prime" + message1_1 "github.com/filecoin-project/go-data-transfer/v2/message/message1_1prime" ) var NewRequest = message1_1.NewRequest diff --git a/message/message1_1/message.go b/message/message1_1/message.go index c07544c3..c52471f1 100644 --- a/message/message1_1/message.go +++ b/message/message1_1/message.go @@ -11,9 +11,9 @@ import ( cborgen "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/encoding" - "github.com/filecoin-project/go-data-transfer/message/types" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + "github.com/filecoin-project/go-data-transfer/v2/message/types" ) // NewRequest generates a new request for the data transfer protocol diff --git a/message/message1_1/message_test.go b/message/message1_1/message_test.go index 7cb2ac24..9a4e0ef4 100644 --- a/message/message1_1/message_test.go +++ b/message/message1_1/message_test.go @@ -14,9 +14,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/message/message1_1" - "github.com/filecoin-project/go-data-transfer/testutil" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message/message1_1" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) func TestNewRequest(t *testing.T) { diff --git a/message/message1_1/transfer_message.go b/message/message1_1/transfer_message.go index f8866fd6..f4e04f32 100644 --- a/message/message1_1/transfer_message.go +++ b/message/message1_1/transfer_message.go @@ -8,7 +8,7 @@ import ( "github.com/ipld/go-ipld-prime/datamodel" basicnode "github.com/ipld/go-ipld-prime/node/basic" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) //go:generate cbor-gen-for --map-encoding TransferMessage1_1 diff --git a/message/message1_1/transfer_request.go b/message/message1_1/transfer_request.go index a2aac438..a751ed31 100644 --- a/message/message1_1/transfer_request.go +++ b/message/message1_1/transfer_request.go @@ -13,9 +13,9 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/encoding" - "github.com/filecoin-project/go-data-transfer/message/types" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + "github.com/filecoin-project/go-data-transfer/v2/message/types" ) //go:generate cbor-gen-for --map-encoding TransferRequest1_1 diff --git a/message/message1_1/transfer_request_cbor_gen.go b/message/message1_1/transfer_request_cbor_gen.go index 16873836..056fe964 100644 --- a/message/message1_1/transfer_request_cbor_gen.go +++ b/message/message1_1/transfer_request_cbor_gen.go @@ -7,7 +7,7 @@ import ( "io" "sort" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" diff --git a/message/message1_1/transfer_request_test.go b/message/message1_1/transfer_request_test.go index da0bccac..7ea148f8 100644 --- a/message/message1_1/transfer_request_test.go +++ b/message/message1_1/transfer_request_test.go @@ -8,9 +8,9 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/message/message1_1" - "github.com/filecoin-project/go-data-transfer/testutil" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message/message1_1" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) func TestRequestMessageForProtocol(t *testing.T) { diff --git a/message/message1_1/transfer_response.go b/message/message1_1/transfer_response.go index 83128feb..5e91ea2f 100644 --- a/message/message1_1/transfer_response.go +++ b/message/message1_1/transfer_response.go @@ -11,9 +11,9 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/encoding" - "github.com/filecoin-project/go-data-transfer/message/types" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + "github.com/filecoin-project/go-data-transfer/v2/message/types" ) //go:generate cbor-gen-for --map-encoding TransferResponse1_1 diff --git a/message/message1_1/transfer_response_cbor_gen.go b/message/message1_1/transfer_response_cbor_gen.go index 29907e2c..0a9badc7 100644 --- a/message/message1_1/transfer_response_cbor_gen.go +++ b/message/message1_1/transfer_response_cbor_gen.go @@ -7,7 +7,7 @@ import ( "io" "sort" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" diff --git a/message/message1_1/transfer_response_test.go b/message/message1_1/transfer_response_test.go index 8e98e668..b6335fb4 100644 --- a/message/message1_1/transfer_response_test.go +++ b/message/message1_1/transfer_response_test.go @@ -6,9 +6,9 @@ import ( "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/message/message1_1" - "github.com/filecoin-project/go-data-transfer/testutil" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message/message1_1" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) func TestResponseMessageForProtocol(t *testing.T) { diff --git a/message/message1_1prime/message.go b/message/message1_1prime/message.go index 19fc084c..2cd2b6aa 100644 --- a/message/message1_1prime/message.go +++ b/message/message1_1prime/message.go @@ -11,9 +11,9 @@ import ( "github.com/ipld/go-ipld-prime/schema" xerrors "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/encoding" - "github.com/filecoin-project/go-data-transfer/message/types" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + "github.com/filecoin-project/go-data-transfer/v2/message/types" ) // NewRequest generates a new request for the data transfer protocol diff --git a/message/message1_1prime/message_test.go b/message/message1_1prime/message_test.go index 2e13a4d5..9e2ba49c 100644 --- a/message/message1_1prime/message_test.go +++ b/message/message1_1prime/message_test.go @@ -16,10 +16,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/encoding" - message1_1 "github.com/filecoin-project/go-data-transfer/message/message1_1prime" - "github.com/filecoin-project/go-data-transfer/testutil" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + message1_1 "github.com/filecoin-project/go-data-transfer/v2/message/message1_1prime" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) func TestNewRequest(t *testing.T) { diff --git a/message/message1_1prime/transfer_message.go b/message/message1_1prime/transfer_message.go index 8c889f7f..fc651220 100644 --- a/message/message1_1prime/transfer_message.go +++ b/message/message1_1prime/transfer_message.go @@ -8,7 +8,7 @@ import ( "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) // TransferMessage1_1 is the transfer message for the 1.1 Data Transfer Protocol. diff --git a/message/message1_1prime/transfer_request.go b/message/message1_1prime/transfer_request.go index 495f0668..5d844d92 100644 --- a/message/message1_1prime/transfer_request.go +++ b/message/message1_1prime/transfer_request.go @@ -10,9 +10,9 @@ import ( "github.com/libp2p/go-libp2p-core/protocol" xerrors "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/encoding" - "github.com/filecoin-project/go-data-transfer/message/types" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + "github.com/filecoin-project/go-data-transfer/v2/message/types" ) // TransferRequest1_1 is a struct for the 1.1 Data Transfer Protocol that fulfills the datatransfer.Request interface. diff --git a/message/message1_1prime/transfer_request_test.go b/message/message1_1prime/transfer_request_test.go index 96d43d29..093653ed 100644 --- a/message/message1_1prime/transfer_request_test.go +++ b/message/message1_1prime/transfer_request_test.go @@ -8,9 +8,9 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - message1_1 "github.com/filecoin-project/go-data-transfer/message/message1_1prime" - "github.com/filecoin-project/go-data-transfer/testutil" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + message1_1 "github.com/filecoin-project/go-data-transfer/v2/message/message1_1prime" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) func TestRequestMessageForProtocol(t *testing.T) { diff --git a/message/message1_1prime/transfer_response.go b/message/message1_1prime/transfer_response.go index c468906d..902944fd 100644 --- a/message/message1_1prime/transfer_response.go +++ b/message/message1_1prime/transfer_response.go @@ -9,9 +9,9 @@ import ( "github.com/libp2p/go-libp2p-core/protocol" xerrors "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/encoding" - "github.com/filecoin-project/go-data-transfer/message/types" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/encoding" + "github.com/filecoin-project/go-data-transfer/v2/message/types" ) // TransferResponse1_1 is a private struct that satisfies the datatransfer.Response interface diff --git a/message/message1_1prime/transfer_response_test.go b/message/message1_1prime/transfer_response_test.go index fb4b6fa6..cffe5122 100644 --- a/message/message1_1prime/transfer_response_test.go +++ b/message/message1_1prime/transfer_response_test.go @@ -6,9 +6,9 @@ import ( "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - message1_1 "github.com/filecoin-project/go-data-transfer/message/message1_1prime" - "github.com/filecoin-project/go-data-transfer/testutil" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + message1_1 "github.com/filecoin-project/go-data-transfer/v2/message/message1_1prime" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) func TestResponseMessageForProtocol(t *testing.T) { diff --git a/network/interface.go b/network/interface.go index 0cb3f528..965a337d 100644 --- a/network/interface.go +++ b/network/interface.go @@ -6,7 +6,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) // DataTransferNetwork provides network connectivity for GraphSync. diff --git a/network/libp2p_impl.go b/network/libp2p_impl.go index 30437717..1408ab2e 100644 --- a/network/libp2p_impl.go +++ b/network/libp2p_impl.go @@ -18,8 +18,8 @@ import ( "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/message" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message" ) var log = logging.Logger("data_transfer_network") diff --git a/network/libp2p_impl_test.go b/network/libp2p_impl_test.go index 934f3f6c..dc71c641 100644 --- a/network/libp2p_impl_test.go +++ b/network/libp2p_impl_test.go @@ -18,10 +18,10 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/message" - "github.com/filecoin-project/go-data-transfer/network" - "github.com/filecoin-project/go-data-transfer/testutil" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/network" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) // Receiver is an interface for receiving messages from the DataTransferNetwork. diff --git a/registry/registry.go b/registry/registry.go index 3386d1da..4237cec7 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -5,8 +5,8 @@ import ( "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/encoding" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/encoding" ) // Processor is an interface that processes a certain type of encodable objects diff --git a/registry/registry_test.go b/registry/registry_test.go index f3eaeb96..63adf363 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/require" - "github.com/filecoin-project/go-data-transfer/registry" - "github.com/filecoin-project/go-data-transfer/testutil" + "github.com/filecoin-project/go-data-transfer/v2/registry" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) func TestRegistry(t *testing.T) { diff --git a/scripts/fiximports b/scripts/fiximports index e51d54ab..5e2d3e2d 100755 --- a/scripts/fiximports +++ b/scripts/fiximports @@ -8,5 +8,5 @@ find . -type f -name \*.go -not -name \*_cbor_gen.go | xargs -I '{}' sed -i.bak }' '{}' git clean -fd find . -type f -name \*.go -not -name \*_cbor_gen.go | xargs -I '{}' goimports -w -local "github.com/filecoin-project" '{}' -find . -type f -name \*.go -not -name \*_cbor_gen.go | xargs -I '{}' goimports -w -local "github.com/filecoin-project/go-data-transfer" '{}' +find . -type f -name \*.go -not -name \*_cbor_gen.go | xargs -I '{}' goimports -w -local "github.com/filecoin-project/go-data-transfer/v2" '{}' diff --git a/testutil/fakedttype.go b/testutil/fakedttype.go index 9fde52f0..20cc219e 100644 --- a/testutil/fakedttype.go +++ b/testutil/fakedttype.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/encoding" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/encoding" ) //go:generate cbor-gen-for FakeDTType diff --git a/testutil/fakegraphsync.go b/testutil/fakegraphsync.go index c6477522..42b30873 100644 --- a/testutil/fakegraphsync.go +++ b/testutil/fakegraphsync.go @@ -16,9 +16,9 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/message" - "github.com/filecoin-project/go-data-transfer/transport/graphsync/extension" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" ) func matchDtMessage(t *testing.T, extensions []graphsync.ExtensionData) datatransfer.Message { diff --git a/testutil/faketransport.go b/testutil/faketransport.go index ecada472..92441364 100644 --- a/testutil/faketransport.go +++ b/testutil/faketransport.go @@ -6,7 +6,7 @@ import ( "github.com/ipld/go-ipld-prime" "github.com/libp2p/go-libp2p-core/peer" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) // OpenedChannel records a call to open a channel diff --git a/testutil/gstestdata.go b/testutil/gstestdata.go index 40868d42..ea7c1f69 100644 --- a/testutil/gstestdata.go +++ b/testutil/gstestdata.go @@ -38,10 +38,10 @@ import ( mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/network" - gstransport "github.com/filecoin-project/go-data-transfer/transport/graphsync" - "github.com/filecoin-project/go-data-transfer/transport/graphsync/extension" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/network" + gstransport "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" ) var allSelector ipld.Node diff --git a/testutil/message.go b/testutil/message.go index eefc7551..c3745be9 100644 --- a/testutil/message.go +++ b/testutil/message.go @@ -7,8 +7,8 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/message" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message" ) // NewDTRequest makes a new DT Request message diff --git a/testutil/stubbedvalidator.go b/testutil/stubbedvalidator.go index 8cef087e..798d0fad 100644 --- a/testutil/stubbedvalidator.go +++ b/testutil/stubbedvalidator.go @@ -9,7 +9,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) // NewStubbedValidator returns a new instance of a stubbed validator diff --git a/testutil/testnet.go b/testutil/testnet.go index 9b99c9b6..88d30a07 100644 --- a/testutil/testnet.go +++ b/testutil/testnet.go @@ -6,8 +6,8 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/network" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/network" ) // FakeSentMessage is a recording of a message sent on the FakeNetwork diff --git a/testutil/testutil.go b/testutil/testutil.go index 731ee3f8..66600730 100644 --- a/testutil/testutil.go +++ b/testutil/testutil.go @@ -17,7 +17,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) var blockGenerator = blocksutil.NewBlockGenerator() diff --git a/tracing/tracing.go b/tracing/tracing.go index 2ba36800..204793e6 100644 --- a/tracing/tracing.go +++ b/tracing/tracing.go @@ -8,7 +8,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) type SpansIndex struct { diff --git a/tracing/tracing_test.go b/tracing/tracing_test.go index f8f8c00f..c342e24d 100644 --- a/tracing/tracing_test.go +++ b/tracing/tracing_test.go @@ -7,9 +7,9 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/exp/rand" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/testutil" - "github.com/filecoin-project/go-data-transfer/tracing" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/testutil" + "github.com/filecoin-project/go-data-transfer/v2/tracing" ) func TestSpansIndex(t *testing.T) { diff --git a/transport/graphsync/extension/gsextension.go b/transport/graphsync/extension/gsextension.go index 356ac41c..a8c34c35 100644 --- a/transport/graphsync/extension/gsextension.go +++ b/transport/graphsync/extension/gsextension.go @@ -7,8 +7,8 @@ import ( "github.com/ipld/go-ipld-prime/datamodel" "github.com/libp2p/go-libp2p-core/protocol" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/message" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message" ) const ( diff --git a/transport/graphsync/graphsync.go b/transport/graphsync/graphsync.go index 42df4991..2a78a003 100644 --- a/transport/graphsync/graphsync.go +++ b/transport/graphsync/graphsync.go @@ -15,8 +15,8 @@ import ( "golang.org/x/sync/errgroup" "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/transport/graphsync/extension" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" ) var log = logging.Logger("dt_graphsync") diff --git a/transport/graphsync/graphsync_test.go b/transport/graphsync/graphsync_test.go index 82637593..2cc40689 100644 --- a/transport/graphsync/graphsync_test.go +++ b/transport/graphsync/graphsync_test.go @@ -19,11 +19,11 @@ import ( "github.com/libp2p/go-libp2p-core/protocol" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-data-transfer/message" - "github.com/filecoin-project/go-data-transfer/testutil" - . "github.com/filecoin-project/go-data-transfer/transport/graphsync" - "github.com/filecoin-project/go-data-transfer/transport/graphsync/extension" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/testutil" + . "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" ) func TestManager(t *testing.T) { diff --git a/transport/graphsync/gskeychidmap_test.go b/transport/graphsync/gskeychidmap_test.go index 0962eb7d..61b5fbd7 100644 --- a/transport/graphsync/gskeychidmap_test.go +++ b/transport/graphsync/gskeychidmap_test.go @@ -6,7 +6,7 @@ import ( "github.com/ipfs/go-graphsync" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) func TestRequestIDToChannelIDMap(t *testing.T) { diff --git a/types.go b/types.go index cd970e0d..f7ddff7f 100644 --- a/types.go +++ b/types.go @@ -9,7 +9,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/go-data-transfer/encoding" + "github.com/filecoin-project/go-data-transfer/v2/encoding" ) //go:generate cbor-gen-for ChannelID ChannelStages ChannelStage Log From a4c58c550b1db479235331a4993f1116333a6ce8 Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Wed, 11 May 2022 15:28:05 -0700 Subject: [PATCH 02/18] Refactor revalidators (#308) * feat(revalidators): initial refactor refactor revalidation process -- removing independent revalidations, making validations results more clear * refactor(rename): RevalidateToComplete -> RequiresFinalization * refactor(datatransfer): enhance validator comments * refactor(message): revert message response changes * refactor(impl): comment and refactor on events add comments to event processing and also extract functions for receiving requests to a new file * refactor(message): s/IsVoucherResult/IsValidationResult rename the IsVoucherResult to is 'IsValidationResult' also add a method for generating messages from validation results * feat(events): only dispatch events on change Only dispatch Pause, Resume, SetDataLimit, and RequiresFinalization on change * style(imports): fix imports * feat(events): add DataLimitExceeded event * Update channels/channel_state.go Co-authored-by: dirkmc * Update manager.go Co-authored-by: dirkmc * Update manager.go Co-authored-by: dirkmc * Update statuses.go Co-authored-by: Rod Vagg Co-authored-by: dirkmc Co-authored-by: Rod Vagg --- channelmonitor/channelmonitor_test.go | 132 +---- channels/block_index_cache.go | 63 --- channels/caches.go | 133 +++++ channels/channel_state.go | 143 ++--- channels/channels.go | 107 +++- channels/channels_fsm.go | 23 +- channels/channels_test.go | 85 ++- channels/internal/internalchannel.go | 6 + channels/internal/internalchannel_cbor_gen.go | 67 ++- events.go | 17 + impl/events.go | 513 ++++-------------- impl/impl.go | 65 ++- impl/initiating_test.go | 77 ++- impl/integration_test.go | 275 ++++------ impl/receiver.go | 1 - impl/receiving_requests.go | 352 ++++++++++++ impl/responding_test.go | 435 +++++++++------ impl/restart.go | 35 +- impl/restart_integration_test.go | 8 + impl/utils.go | 32 +- manager.go | 79 ++- message.go | 2 +- message/message.go | 7 + message/message1_1/message_test.go | 2 +- message/message1_1/transfer_response.go | 2 +- message/message1_1/transfer_response_test.go | 2 +- message/message1_1prime/message.go | 27 + message/message1_1prime/message_test.go | 2 +- message/message1_1prime/transfer_response.go | 2 +- .../message1_1prime/transfer_response_test.go | 2 +- network/libp2p_impl_test.go | 3 +- statuses.go | 12 + testutil/mockchannelstate.go | 170 ++++++ testutil/stubbedvalidator.go | 292 ++-------- transport.go | 2 - transport/graphsync/graphsync_test.go | 117 +--- types.go | 8 + 37 files changed, 1759 insertions(+), 1541 deletions(-) delete mode 100644 channels/block_index_cache.go create mode 100644 channels/caches.go create mode 100644 impl/receiving_requests.go create mode 100644 testutil/mockchannelstate.go diff --git a/channelmonitor/channelmonitor_test.go b/channelmonitor/channelmonitor_test.go index 067d513a..af91142e 100644 --- a/channelmonitor/channelmonitor_test.go +++ b/channelmonitor/channelmonitor_test.go @@ -7,13 +7,12 @@ import ( "testing" "time" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/testutil" ) var ch1 = datatransfer.ChannelID{ @@ -41,7 +40,7 @@ func TestChannelMonitorAutoRestart(t *testing.T) { runTest := func(name string, isPush bool) { for _, tc := range testCases { t.Run(name+": "+tc.name, func(t *testing.T) { - ch := &mockChannelState{chid: ch1} + ch := testutil.NewMockChannelState(testutil.MockChannelStateParams{ChannelID: ch1}) mockAPI := newMockMonitorAPI(ch, tc.errReconnect, tc.errSendRestartMsg) triggerErrorEvent := func() { @@ -115,7 +114,7 @@ func TestChannelMonitorAutoRestart(t *testing.T) { func TestChannelMonitorMaxConsecutiveRestarts(t *testing.T) { runTest := func(name string, isPush bool) { t.Run(name, func(t *testing.T) { - ch := &mockChannelState{chid: ch1} + ch := testutil.NewMockChannelState(testutil.MockChannelStateParams{ChannelID: ch1}) mockAPI := newMockMonitorAPI(ch, false, false) triggerErrorEvent := func() { @@ -198,7 +197,7 @@ func awaitRestartComplete(mch *monitoredChannel) error { func TestChannelMonitorQueuedRestart(t *testing.T) { runTest := func(name string, isPush bool) { t.Run(name, func(t *testing.T) { - ch := &mockChannelState{chid: ch1} + ch := testutil.NewMockChannelState(testutil.MockChannelStateParams{ChannelID: ch1}) mockAPI := newMockMonitorAPI(ch, false, false) triggerErrorEvent := func() { @@ -285,7 +284,7 @@ func TestChannelMonitorTimeouts(t *testing.T) { runTest := func(name string, isPush bool) { for _, tc := range testCases { t.Run(name+": "+tc.name, func(t *testing.T) { - ch := &mockChannelState{chid: ch1} + ch := testutil.NewMockChannelState(testutil.MockChannelStateParams{ChannelID: ch1}) mockAPI := newMockMonitorAPI(ch, false, false) verifyClosedAndShutdown := func(chCtx context.Context, timeout time.Duration) { @@ -370,7 +369,7 @@ func verifyChannelShutdown(t *testing.T, shutdownCtx context.Context) { } type mockMonitorAPI struct { - ch *mockChannelState + ch *testutil.MockChannelState connectErrors bool restartErrors bool restartMessages chan struct{} @@ -380,7 +379,7 @@ type mockMonitorAPI struct { subscribers map[int]datatransfer.Subscriber } -func newMockMonitorAPI(ch *mockChannelState, errOnReconnect, errOnRestart bool) *mockMonitorAPI { +func newMockMonitorAPI(ch *testutil.MockChannelState, errOnReconnect, errOnRestart bool) *mockMonitorAPI { return &mockMonitorAPI{ ch: ch, connectErrors: errOnReconnect, @@ -482,17 +481,17 @@ func (m *mockMonitorAPI) accept() { } func (m *mockMonitorAPI) dataQueued(n uint64) { - m.ch.queued = n + m.ch.SetQueued(n) m.fireEvent(datatransfer.Event{Code: datatransfer.DataQueued}, m.ch) } func (m *mockMonitorAPI) dataSent(n uint64) { - m.ch.sent = n + m.ch.SetSent(n) m.fireEvent(datatransfer.Event{Code: datatransfer.DataSent}, m.ch) } func (m *mockMonitorAPI) dataReceived(n uint64) { - m.ch.received = n + m.ch.SetReceived(n) m.fireEvent(datatransfer.Event{Code: datatransfer.DataReceived}, m.ch) } @@ -501,7 +500,7 @@ func (m *mockMonitorAPI) finishTransfer() { } func (m *mockMonitorAPI) completed() { - m.ch.complete = true + m.ch.SetComplete(true) m.fireEvent(datatransfer.Event{Code: datatransfer.Complete}, m.ch) } @@ -512,112 +511,3 @@ func (m *mockMonitorAPI) sendDataErrorEvent() { func (m *mockMonitorAPI) receiveDataErrorEvent() { m.fireEvent(datatransfer.Event{Code: datatransfer.ReceiveDataError}, m.ch) } - -type mockChannelState struct { - chid datatransfer.ChannelID - queued uint64 - sent uint64 - received uint64 - complete bool -} - -var _ datatransfer.ChannelState = (*mockChannelState)(nil) - -func (m *mockChannelState) Queued() uint64 { - return m.queued -} - -func (m *mockChannelState) Sent() uint64 { - return m.sent -} - -func (m *mockChannelState) Received() uint64 { - return m.received -} - -func (m *mockChannelState) ChannelID() datatransfer.ChannelID { - return m.chid -} - -func (m *mockChannelState) Status() datatransfer.Status { - if m.complete { - return datatransfer.Completed - } - return datatransfer.Ongoing -} - -func (m *mockChannelState) TransferID() datatransfer.TransferID { - panic("implement me") -} - -func (m *mockChannelState) BaseCID() cid.Cid { - panic("implement me") -} - -func (m *mockChannelState) Selector() ipld.Node { - panic("implement me") -} - -func (m *mockChannelState) Voucher() datatransfer.Voucher { - panic("implement me") -} - -func (m *mockChannelState) Sender() peer.ID { - panic("implement me") -} - -func (m *mockChannelState) Recipient() peer.ID { - panic("implement me") -} - -func (m *mockChannelState) TotalSize() uint64 { - panic("implement me") -} - -func (m *mockChannelState) IsPull() bool { - panic("implement me") -} - -func (m *mockChannelState) OtherPeer() peer.ID { - panic("implement me") -} - -func (m *mockChannelState) SelfPeer() peer.ID { - panic("implement me") -} - -func (m *mockChannelState) Message() string { - panic("implement me") -} - -func (m *mockChannelState) Vouchers() []datatransfer.Voucher { - panic("implement me") -} - -func (m *mockChannelState) VoucherResults() []datatransfer.VoucherResult { - panic("implement me") -} - -func (m *mockChannelState) LastVoucher() datatransfer.Voucher { - panic("implement me") -} - -func (m *mockChannelState) LastVoucherResult() datatransfer.VoucherResult { - panic("implement me") -} - -func (m *mockChannelState) Stages() *datatransfer.ChannelStages { - panic("implement me") -} - -func (m *mockChannelState) ReceivedCidsTotal() int64 { - panic("implement me") -} - -func (m *mockChannelState) QueuedCidsTotal() int64 { - panic("implement me") -} - -func (m *mockChannelState) SentCidsTotal() int64 { - panic("implement me") -} diff --git a/channels/block_index_cache.go b/channels/block_index_cache.go deleted file mode 100644 index 4c16e007..00000000 --- a/channels/block_index_cache.go +++ /dev/null @@ -1,63 +0,0 @@ -package channels - -import ( - "sync" - "sync/atomic" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" -) - -type readOriginalFn func(datatransfer.ChannelID) (int64, error) - -type blockIndexKey struct { - evt datatransfer.EventCode - chid datatransfer.ChannelID -} -type blockIndexCache struct { - lk sync.RWMutex - values map[blockIndexKey]*int64 -} - -func newBlockIndexCache() *blockIndexCache { - return &blockIndexCache{ - values: make(map[blockIndexKey]*int64), - } -} - -func (bic *blockIndexCache) getValue(evt datatransfer.EventCode, chid datatransfer.ChannelID, readFromOriginal readOriginalFn) (*int64, error) { - idxKey := blockIndexKey{evt, chid} - bic.lk.RLock() - value := bic.values[idxKey] - bic.lk.RUnlock() - if value != nil { - return value, nil - } - bic.lk.Lock() - defer bic.lk.Unlock() - value = bic.values[idxKey] - if value != nil { - return value, nil - } - newValue, err := readFromOriginal(chid) - if err != nil { - return nil, err - } - bic.values[idxKey] = &newValue - return &newValue, nil -} - -func (bic *blockIndexCache) updateIfGreater(evt datatransfer.EventCode, chid datatransfer.ChannelID, newIndex int64, readFromOriginal readOriginalFn) (bool, error) { - value, err := bic.getValue(evt, chid, readFromOriginal) - if err != nil { - return false, err - } - for { - currentIndex := atomic.LoadInt64(value) - if newIndex <= currentIndex { - return false, nil - } - if atomic.CompareAndSwapInt64(value, currentIndex, newIndex) { - return true, nil - } - } -} diff --git a/channels/caches.go b/channels/caches.go new file mode 100644 index 00000000..2e2d75a9 --- /dev/null +++ b/channels/caches.go @@ -0,0 +1,133 @@ +package channels + +import ( + "sync" + "sync/atomic" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" +) + +type readIndexFn func(datatransfer.ChannelID) (int64, error) + +type cacheKey struct { + evt datatransfer.EventCode + chid datatransfer.ChannelID +} + +type blockIndexCache struct { + lk sync.RWMutex + values map[cacheKey]*int64 +} + +func newBlockIndexCache() *blockIndexCache { + return &blockIndexCache{ + values: make(map[cacheKey]*int64), + } +} + +func (bic *blockIndexCache) getValue(evt datatransfer.EventCode, chid datatransfer.ChannelID, readFromOriginal readIndexFn) (*int64, error) { + idxKey := cacheKey{evt, chid} + bic.lk.RLock() + value := bic.values[idxKey] + bic.lk.RUnlock() + if value != nil { + return value, nil + } + bic.lk.Lock() + defer bic.lk.Unlock() + value = bic.values[idxKey] + if value != nil { + return value, nil + } + newValue, err := readFromOriginal(chid) + if err != nil { + return nil, err + } + bic.values[idxKey] = &newValue + return &newValue, nil +} + +func (bic *blockIndexCache) updateIfGreater(evt datatransfer.EventCode, chid datatransfer.ChannelID, newIndex int64, readFromOriginal readIndexFn) (bool, error) { + value, err := bic.getValue(evt, chid, readFromOriginal) + if err != nil { + return false, err + } + for { + currentIndex := atomic.LoadInt64(value) + if newIndex <= currentIndex { + return false, nil + } + if atomic.CompareAndSwapInt64(value, currentIndex, newIndex) { + return true, nil + } + } +} + +type progressState struct { + dataLimit uint64 + progress *uint64 +} + +type readProgressFn func(datatransfer.ChannelID) (dataLimit uint64, progress uint64, err error) + +type progressCache struct { + lk sync.RWMutex + values map[datatransfer.ChannelID]progressState +} + +func newProgressCache() *progressCache { + return &progressCache{ + values: make(map[datatransfer.ChannelID]progressState), + } +} + +func (pc *progressCache) getValue(chid datatransfer.ChannelID, readProgress readProgressFn) (progressState, error) { + pc.lk.RLock() + value, ok := pc.values[chid] + pc.lk.RUnlock() + if ok { + return value, nil + } + pc.lk.Lock() + defer pc.lk.Unlock() + value, ok = pc.values[chid] + if ok { + return value, nil + } + dataLimit, progress, err := readProgress(chid) + if err != nil { + return progressState{}, err + } + newValue := progressState{ + dataLimit: dataLimit, + progress: &progress, + } + pc.values[chid] = newValue + return newValue, nil +} + +func (pc *progressCache) progress(chid datatransfer.ChannelID, additionalData uint64, readFromOriginal readProgressFn) (bool, error) { + state, err := pc.getValue(chid, readFromOriginal) + if err != nil { + return false, err + } + total := atomic.AddUint64(state.progress, additionalData) + return state.dataLimit != 0 && total >= state.dataLimit, nil +} + +func (pc *progressCache) setDataLimit(chid datatransfer.ChannelID, newLimit uint64) { + pc.lk.RLock() + value, ok := pc.values[chid] + pc.lk.RUnlock() + if !ok { + return + } + pc.lk.Lock() + defer pc.lk.Unlock() + value, ok = pc.values[chid] + if !ok { + return + } + value.dataLimit = newLimit + pc.values[chid] = value +} diff --git a/channels/channel_state.go b/channels/channel_state.go index 5a477486..a24a7feb 100644 --- a/channels/channel_state.go +++ b/channels/channel_state.go @@ -8,7 +8,6 @@ import ( "github.com/ipld/go-ipld-prime/codec/dagcbor" basicnode "github.com/ipld/go-ipld-prime/node/basic" peer "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channels/internal" @@ -16,79 +15,39 @@ import ( // channelState is immutable channel data plus mutable state type channelState struct { - // peerId of the manager peer - selfPeer peer.ID - // an identifier for this channel shared by request and responder, set by requester through protocol - transferID datatransfer.TransferID - // base CID for the piece being transferred - baseCid cid.Cid - // portion of Piece to return, specified by an IPLD selector - selector *cbg.Deferred - // the party that is sending the data (not who initiated the request) - sender peer.ID - // the party that is receiving the data (not who initiated the request) - recipient peer.ID - // expected amount of data to be transferred - totalSize uint64 - // current status of this deal - status datatransfer.Status - // isPull indicates if this is a push or pull request - isPull bool - // total bytes read from this node and queued for sending (0 if receiver) - queued uint64 - // total bytes sent from this node (0 if receiver) - sent uint64 - // total bytes received by this node (0 if sender) - received uint64 - // number of blocks that have been received, including blocks that are - // present in more than one place in the DAG - receivedBlocksTotal int64 - // Number of blocks that have been queued, including blocks that are - // present in more than one place in the DAG - queuedBlocksTotal int64 - // Number of blocks that have been sent, including blocks that are - // present in more than one place in the DAG - sentBlocksTotal int64 - // more informative status on a channel - message string - // additional vouchers - vouchers []internal.EncodedVoucher + ic internal.ChannelState + // additional voucherResults - voucherResults []internal.EncodedVoucherResult voucherResultDecoder DecoderByTypeFunc voucherDecoder DecoderByTypeFunc - - // stages tracks the timeline of events related to a data transfer, for - // traceability purposes. - stages *datatransfer.ChannelStages } // EmptyChannelState is the zero value for channel state, meaning not present var EmptyChannelState = channelState{} // Status is the current status of this channel -func (c channelState) Status() datatransfer.Status { return c.status } +func (c channelState) Status() datatransfer.Status { return c.ic.Status } // Received returns the number of bytes received -func (c channelState) Queued() uint64 { return c.queued } +func (c channelState) Queued() uint64 { return c.ic.Queued } // Sent returns the number of bytes sent -func (c channelState) Sent() uint64 { return c.sent } +func (c channelState) Sent() uint64 { return c.ic.Sent } // Received returns the number of bytes received -func (c channelState) Received() uint64 { return c.received } +func (c channelState) Received() uint64 { return c.ic.Received } // TransferID returns the transfer id for this channel -func (c channelState) TransferID() datatransfer.TransferID { return c.transferID } +func (c channelState) TransferID() datatransfer.TransferID { return c.ic.TransferID } // BaseCID returns the CID that is at the root of this data transfer -func (c channelState) BaseCID() cid.Cid { return c.baseCid } +func (c channelState) BaseCID() cid.Cid { return c.ic.BaseCid } // Selector returns the IPLD selector for this data transfer (represented as // an IPLD node) func (c channelState) Selector() ipld.Node { builder := basicnode.Prototype.Any.NewBuilder() - reader := bytes.NewReader(c.selector.Raw) + reader := bytes.NewReader(c.ic.Selector.Raw) err := dagcbor.Decode(builder, reader) if err != nil { log.Error(err) @@ -98,60 +57,60 @@ func (c channelState) Selector() ipld.Node { // Voucher returns the voucher for this data transfer func (c channelState) Voucher() datatransfer.Voucher { - if len(c.vouchers) == 0 { + if len(c.ic.Vouchers) == 0 { return nil } - decoder, _ := c.voucherDecoder(c.vouchers[0].Type) - encodable, _ := decoder.DecodeFromCbor(c.vouchers[0].Voucher.Raw) + decoder, _ := c.voucherDecoder(c.ic.Vouchers[0].Type) + encodable, _ := decoder.DecodeFromCbor(c.ic.Vouchers[0].Voucher.Raw) return encodable.(datatransfer.Voucher) } // ReceivedCidsTotal returns the number of (non-unique) cids received so far // on the channel - note that a block can exist in more than one place in the DAG func (c channelState) ReceivedCidsTotal() int64 { - return c.receivedBlocksTotal + return c.ic.ReceivedBlocksTotal } // QueuedCidsTotal returns the number of (non-unique) cids queued so far // on the channel - note that a block can exist in more than one place in the DAG func (c channelState) QueuedCidsTotal() int64 { - return c.queuedBlocksTotal + return c.ic.QueuedBlocksTotal } // SentCidsTotal returns the number of (non-unique) cids sent so far // on the channel - note that a block can exist in more than one place in the DAG func (c channelState) SentCidsTotal() int64 { - return c.sentBlocksTotal + return c.ic.SentBlocksTotal } // Sender returns the peer id for the node that is sending data -func (c channelState) Sender() peer.ID { return c.sender } +func (c channelState) Sender() peer.ID { return c.ic.Sender } // Recipient returns the peer id for the node that is receiving data -func (c channelState) Recipient() peer.ID { return c.recipient } +func (c channelState) Recipient() peer.ID { return c.ic.Recipient } // TotalSize returns the total size for the data being transferred -func (c channelState) TotalSize() uint64 { return c.totalSize } +func (c channelState) TotalSize() uint64 { return c.ic.TotalSize } // IsPull returns whether this is a pull request based on who initiated it func (c channelState) IsPull() bool { - return c.isPull + return c.ic.Initiator == c.ic.Recipient } func (c channelState) ChannelID() datatransfer.ChannelID { - if c.isPull { - return datatransfer.ChannelID{ID: c.transferID, Initiator: c.recipient, Responder: c.sender} + if c.IsPull() { + return datatransfer.ChannelID{ID: c.ic.TransferID, Initiator: c.ic.Recipient, Responder: c.ic.Sender} } - return datatransfer.ChannelID{ID: c.transferID, Initiator: c.sender, Responder: c.recipient} + return datatransfer.ChannelID{ID: c.ic.TransferID, Initiator: c.ic.Sender, Responder: c.ic.Recipient} } func (c channelState) Message() string { - return c.message + return c.ic.Message } func (c channelState) Vouchers() []datatransfer.Voucher { - vouchers := make([]datatransfer.Voucher, 0, len(c.vouchers)) - for _, encoded := range c.vouchers { + vouchers := make([]datatransfer.Voucher, 0, len(c.ic.Vouchers)) + for _, encoded := range c.ic.Vouchers { decoder, _ := c.voucherDecoder(encoded.Type) encodable, _ := decoder.DecodeFromCbor(encoded.Voucher.Raw) vouchers = append(vouchers, encodable.(datatransfer.Voucher)) @@ -160,20 +119,20 @@ func (c channelState) Vouchers() []datatransfer.Voucher { } func (c channelState) LastVoucher() datatransfer.Voucher { - decoder, _ := c.voucherDecoder(c.vouchers[len(c.vouchers)-1].Type) - encodable, _ := decoder.DecodeFromCbor(c.vouchers[len(c.vouchers)-1].Voucher.Raw) + decoder, _ := c.voucherDecoder(c.ic.Vouchers[len(c.ic.Vouchers)-1].Type) + encodable, _ := decoder.DecodeFromCbor(c.ic.Vouchers[len(c.ic.Vouchers)-1].Voucher.Raw) return encodable.(datatransfer.Voucher) } func (c channelState) LastVoucherResult() datatransfer.VoucherResult { - decoder, _ := c.voucherResultDecoder(c.voucherResults[len(c.voucherResults)-1].Type) - encodable, _ := decoder.DecodeFromCbor(c.voucherResults[len(c.voucherResults)-1].VoucherResult.Raw) + decoder, _ := c.voucherResultDecoder(c.ic.VoucherResults[len(c.ic.VoucherResults)-1].Type) + encodable, _ := decoder.DecodeFromCbor(c.ic.VoucherResults[len(c.ic.VoucherResults)-1].VoucherResult.Raw) return encodable.(datatransfer.VoucherResult) } func (c channelState) VoucherResults() []datatransfer.VoucherResult { - voucherResults := make([]datatransfer.VoucherResult, 0, len(c.voucherResults)) - for _, encoded := range c.voucherResults { + voucherResults := make([]datatransfer.VoucherResult, 0, len(c.ic.VoucherResults)) + for _, encoded := range c.ic.VoucherResults { decoder, _ := c.voucherResultDecoder(encoded.Type) encodable, _ := decoder.DecodeFromCbor(encoded.VoucherResult.Raw) voucherResults = append(voucherResults, encodable.(datatransfer.VoucherResult)) @@ -182,14 +141,22 @@ func (c channelState) VoucherResults() []datatransfer.VoucherResult { } func (c channelState) SelfPeer() peer.ID { - return c.selfPeer + return c.ic.SelfPeer } func (c channelState) OtherPeer() peer.ID { - if c.sender == c.selfPeer { - return c.recipient + if c.ic.Sender == c.ic.SelfPeer { + return c.ic.Recipient } - return c.sender + return c.ic.Sender +} + +func (c channelState) DataLimit() uint64 { + return c.ic.DataLimit +} + +func (c channelState) RequiresFinalization() bool { + return c.ic.RequiresFinalization } // Stages returns the current ChannelStages object, or an empty object. @@ -198,38 +165,20 @@ func (c channelState) OtherPeer() peer.ID { // // EXPERIMENTAL; subject to change. func (c channelState) Stages() *datatransfer.ChannelStages { - if c.stages == nil { + if c.ic.Stages == nil { // return an empty placeholder; it will be discarded because the caller // is not supposed to mutate the value anyway. return &datatransfer.ChannelStages{} } - return c.stages + return c.ic.Stages } func fromInternalChannelState(c internal.ChannelState, voucherDecoder DecoderByTypeFunc, voucherResultDecoder DecoderByTypeFunc) datatransfer.ChannelState { return channelState{ - selfPeer: c.SelfPeer, - isPull: c.Initiator == c.Recipient, - transferID: c.TransferID, - baseCid: c.BaseCid, - selector: c.Selector, - sender: c.Sender, - recipient: c.Recipient, - totalSize: c.TotalSize, - status: c.Status, - queued: c.Queued, - sent: c.Sent, - received: c.Received, - receivedBlocksTotal: c.ReceivedBlocksTotal, - queuedBlocksTotal: c.QueuedBlocksTotal, - sentBlocksTotal: c.SentBlocksTotal, - message: c.Message, - vouchers: c.Vouchers, - voucherResults: c.VoucherResults, + ic: c, voucherResultDecoder: voucherResultDecoder, voucherDecoder: voucherDecoder, - stages: c.Stages, } } diff --git a/channels/channels.go b/channels/channels.go index f4f6940c..0c575f54 100644 --- a/channels/channels.go +++ b/channels/channels.go @@ -49,6 +49,7 @@ type Channels struct { voucherDecoder DecoderByTypeFunc voucherResultDecoder DecoderByTypeFunc blockIndexCache *blockIndexCache + progressCache *progressCache stateMachines fsm.Group migrateStateMachines func(context.Context) error } @@ -75,6 +76,7 @@ func New(ds datastore.Batching, voucherResultDecoder: voucherResultDecoder, } c.blockIndexCache = newBlockIndexCache() + c.progressCache = newProgressCache() channelMigrations, err := migrations.GetChannelStateMigrations(selfPeer) if err != nil { return nil, err @@ -235,18 +237,35 @@ func (c *Channels) getSentIndex(chid datatransfer.ChannelID) (int64, error) { return chst.SentCidsTotal(), nil } -func (c *Channels) DataSent(chid datatransfer.ChannelID, k cid.Cid, delta uint64, index int64, unique bool) (bool, error) { - return c.fireProgressEvent(chid, datatransfer.DataSent, datatransfer.DataSentProgress, k, delta, index, unique, c.getSentIndex) +func (c *Channels) getQueuedProgress(chid datatransfer.ChannelID) (uint64, uint64, error) { + chst, err := c.GetByID(context.TODO(), chid) + if err != nil { + return 0, 0, err + } + dataLimit := chst.DataLimit() + return dataLimit, chst.Queued(), nil +} + +func (c *Channels) getReceivedProgress(chid datatransfer.ChannelID) (uint64, uint64, error) { + chst, err := c.GetByID(context.TODO(), chid) + if err != nil { + return 0, 0, err + } + dataLimit := chst.DataLimit() + return dataLimit, chst.Received(), nil +} + +func (c *Channels) DataSent(chid datatransfer.ChannelID, k cid.Cid, delta uint64, index int64, unique bool) error { + return c.fireProgressEvent(chid, datatransfer.DataSent, datatransfer.DataSentProgress, delta, index, unique, c.getSentIndex, nil) } -func (c *Channels) DataQueued(chid datatransfer.ChannelID, k cid.Cid, delta uint64, index int64, unique bool) (bool, error) { - return c.fireProgressEvent(chid, datatransfer.DataQueued, datatransfer.DataQueuedProgress, k, delta, index, unique, c.getQueuedIndex) +func (c *Channels) DataQueued(chid datatransfer.ChannelID, k cid.Cid, delta uint64, index int64, unique bool) error { + return c.fireProgressEvent(chid, datatransfer.DataQueued, datatransfer.DataQueuedProgress, delta, index, unique, c.getQueuedIndex, c.getQueuedProgress) } // Returns true if this is the first time the block has been received -func (c *Channels) DataReceived(chid datatransfer.ChannelID, k cid.Cid, delta uint64, index int64, unique bool) (bool, error) { - new, err := c.fireProgressEvent(chid, datatransfer.DataReceived, datatransfer.DataReceivedProgress, k, delta, index, unique, c.getReceivedIndex) - return new, err +func (c *Channels) DataReceived(chid datatransfer.ChannelID, k cid.Cid, delta uint64, index int64, unique bool) error { + return c.fireProgressEvent(chid, datatransfer.DataReceived, datatransfer.DataReceivedProgress, delta, index, unique, c.getReceivedIndex, c.getReceivedProgress) } // PauseInitiator pauses the initator of this channel @@ -354,6 +373,17 @@ func (c *Channels) ReceiveDataError(chid datatransfer.ChannelID, err error) erro return c.send(chid, datatransfer.ReceiveDataError, err) } +// SetDataLimit means a data limit has been set on this channel +func (c *Channels) SetDataLimit(chid datatransfer.ChannelID, dataLimit uint64) error { + c.progressCache.setDataLimit(chid, dataLimit) + return c.send(chid, datatransfer.SetDataLimit, dataLimit) +} + +// SetRequiresFinalization sets the state of whether a data transfer can complete +func (c *Channels) SetRequiresFinalization(chid datatransfer.ChannelID, RequiresFinalization bool) error { + return c.send(chid, datatransfer.SetRequiresFinalization, RequiresFinalization) +} + // HasChannel returns true if the given channel id is being tracked func (c *Channels) HasChannel(chid datatransfer.ChannelID) (bool, error) { return c.stateMachines.Has(chid) @@ -362,29 +392,74 @@ func (c *Channels) HasChannel(chid datatransfer.ChannelID) (bool, error) { // fireProgressEvent fires // - an event for queuing / sending / receiving blocks // - a corresponding "progress" event if the block has not been seen before +// - a DataLimitExceeded event if the progress goes past the data limit // For example, if a block is being sent for the first time, the method will // fire both DataSent AND DataSentProgress. // If a block is resent, the method will fire DataSent but not DataSentProgress. -// Returns true if the block is new (both the event and a progress event were fired). -func (c *Channels) fireProgressEvent(chid datatransfer.ChannelID, evt datatransfer.EventCode, progressEvt datatransfer.EventCode, k cid.Cid, delta uint64, index int64, unique bool, readFromOriginal readOriginalFn) (bool, error) { +// If a block is sent for the first time, and more data has been sent than the data limit, +// the method will fire DataSent AND DataProgress AND DataLimitExceeded AND it will return +// datatransfer.ErrPause as the error +func (c *Channels) fireProgressEvent(chid datatransfer.ChannelID, evt datatransfer.EventCode, progressEvt datatransfer.EventCode, delta uint64, index int64, unique bool, readFromOriginal readIndexFn, readProgress readProgressFn) error { if err := c.checkChannelExists(chid, evt); err != nil { - return false, err + return err } - isNewIndex, err := c.blockIndexCache.updateIfGreater(evt, chid, index, readFromOriginal) + pause, progress, err := c.checkEvents(chid, evt, delta, index, unique, readFromOriginal, readProgress) + if err != nil { - return false, err + return err } - // If the block has not been seen before, fire the progress event - if unique && isNewIndex { + // Fire the progress event if there is progress + if progress { if err := c.stateMachines.Send(chid, progressEvt, delta); err != nil { - return false, err + return err } } // Fire the regular event - return unique && isNewIndex, c.stateMachines.Send(chid, evt, index) + if err := c.stateMachines.Send(chid, evt, index); err != nil { + return err + } + + // fire the pause event if we past our data limit + if pause { + // pause. Data limits only exist on the responder, so we always pause the responder + if err := c.stateMachines.Send(chid, datatransfer.DataLimitExceeded); err != nil { + return err + } + // return a pause error so the transfer knows to pause + return datatransfer.ErrPause + } + return nil +} + +func (c *Channels) checkEvents(chid datatransfer.ChannelID, evt datatransfer.EventCode, delta uint64, index int64, unique bool, readFromOriginal readIndexFn, readProgress readProgressFn) (pause bool, progress bool, err error) { + + // if this is not a unique block, no data progress is made, return + if !unique { + return + } + + // check if data progress is made + progress, err = c.blockIndexCache.updateIfGreater(evt, chid, index, readFromOriginal) + if err != nil { + return false, false, err + } + + // if no data progress, return + if !progress { + return + } + + // if we don't check data limits on this function, return + if readProgress == nil { + return + } + + // check if we're past our data limit + pause, err = c.progressCache.progress(chid, delta, readProgress) + return } func (c *Channels) send(chid datatransfer.ChannelID, code datatransfer.EventCode, args ...interface{}) error { diff --git a/channels/channels_fsm.go b/channels/channels_fsm.go index 005cdcd3..12e2cf0b 100644 --- a/channels/channels_fsm.go +++ b/channels/channels_fsm.go @@ -109,6 +109,18 @@ var ChannelEvents = fsm.Events{ chst.AddLog("") return nil }), + fsm.Event(datatransfer.SetDataLimit).FromAny().ToJustRecord(). + Action(func(chst *internal.ChannelState, dataLimit uint64) error { + chst.DataLimit = dataLimit + chst.AddLog("") + return nil + }), + fsm.Event(datatransfer.SetRequiresFinalization).FromAny().ToJustRecord(). + Action(func(chst *internal.ChannelState, RequiresFinalization bool) error { + chst.RequiresFinalization = RequiresFinalization + chst.AddLog("") + return nil + }), fsm.Event(datatransfer.Disconnected).FromAny().ToNoChange().Action(func(chst *internal.ChannelState, err error) error { chst.Message = err.Error() chst.AddLog("data transfer disconnected: %s", chst.Message) @@ -165,6 +177,14 @@ var ChannelEvents = fsm.Events{ return nil }), + fsm.Event(datatransfer.DataLimitExceeded). + FromMany(datatransfer.Requested, datatransfer.Ongoing).To(datatransfer.ResponderPaused). + From(datatransfer.InitiatorPaused).To(datatransfer.BothPaused). + FromAny().ToJustRecord().Action(func(chst *internal.ChannelState) error { + chst.AddLog("") + return nil + }), + fsm.Event(datatransfer.ResumeInitiator). From(datatransfer.InitiatorPaused).To(datatransfer.Ongoing). From(datatransfer.BothPaused).To(datatransfer.ResponderPaused). @@ -200,7 +220,8 @@ var ChannelEvents = fsm.Events{ fsm.Event(datatransfer.ResponderBeginsFinalization). FromAny().To(datatransfer.ResponderFinalizing). FromMany(datatransfer.Failing, datatransfer.Cancelling).ToJustRecord(). - From(datatransfer.TransferFinished).To(datatransfer.ResponderFinalizingTransferFinished).Action(func(chst *internal.ChannelState) error { + From(datatransfer.TransferFinished).To(datatransfer.ResponderFinalizingTransferFinished). + FromMany(datatransfer.ResponderFinalizing, datatransfer.ResponderFinalizingTransferFinished).ToJustRecord().Action(func(chst *internal.ChannelState) error { chst.AddLog("") return nil }), diff --git a/channels/channels_test.go b/channels/channels_test.go index f80b433f..8b739282 100644 --- a/channels/channels_test.go +++ b/channels/channels_test.go @@ -34,7 +34,7 @@ func TestChannels(t *testing.T) { tid2 := datatransfer.TransferID(1) fv1 := &testutil.FakeDTType{} fv2 := &testutil.FakeDTType{} - cids := testutil.GenerateCids(2) + cids := testutil.GenerateCids(4) selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() peers := testutil.GeneratePeers(4) @@ -141,13 +141,13 @@ func TestChannels(t *testing.T) { require.Equal(t, datatransfer.TransferFinished, state.Status()) // send a data-sent event and ensure it's a no-op - _, err = channelList.DataSent(chid, cids[1], 1, 1, true) + err = channelList.DataSent(chid, cids[1], 1, 1, true) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.DataSent) require.Equal(t, datatransfer.TransferFinished, state.Status()) // send a data-queued event and ensure it's a no-op. - _, err = channelList.DataQueued(chid, cids[1], 1, 1, true) + err = channelList.DataQueued(chid, cids[1], 1, 1, true) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.DataQueued) require.Equal(t, datatransfer.TransferFinished, state.Status()) @@ -168,61 +168,108 @@ func TestChannels(t *testing.T) { require.Equal(t, uint64(0), state.Received()) require.Equal(t, uint64(0), state.Sent()) - isNew, err := channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[0], 50, 1, true) + err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[0], 50, 1, true) require.NoError(t, err) _ = checkEvent(ctx, t, received, datatransfer.DataReceivedProgress) - require.True(t, isNew) state = checkEvent(ctx, t, received, datatransfer.DataReceived) require.Equal(t, uint64(50), state.Received()) require.Equal(t, uint64(0), state.Sent()) - isNew, err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 100, 1, true) + err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 100, 1, true) require.NoError(t, err) _ = checkEvent(ctx, t, received, datatransfer.DataSentProgress) - require.True(t, isNew) state = checkEvent(ctx, t, received, datatransfer.DataSent) require.Equal(t, uint64(50), state.Received()) require.Equal(t, uint64(100), state.Sent()) // send block again has no effect - isNew, err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 100, 1, true) + err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 100, 1, true) require.NoError(t, err) - require.False(t, isNew) state = checkEvent(ctx, t, received, datatransfer.DataSent) require.Equal(t, uint64(50), state.Received()) require.Equal(t, uint64(100), state.Sent()) // errors if channel does not exist - isNew, err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[1], 200, 2, true) + err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[1], 200, 2, true) require.True(t, xerrors.As(err, new(*channels.ErrNotFound))) - require.False(t, isNew) - isNew, err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[1], 200, 2, true) + err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[1], 200, 2, true) require.True(t, xerrors.As(err, new(*channels.ErrNotFound))) - require.False(t, isNew) - isNew, err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 50, 2, true) + err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 50, 2, true) require.NoError(t, err) _ = checkEvent(ctx, t, received, datatransfer.DataReceivedProgress) - require.True(t, isNew) state = checkEvent(ctx, t, received, datatransfer.DataReceived) require.Equal(t, uint64(100), state.Received()) require.Equal(t, uint64(100), state.Sent()) - isNew, err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 25, 2, false) + err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 25, 2, false) require.NoError(t, err) - require.False(t, isNew) state = checkEvent(ctx, t, received, datatransfer.DataSent) require.Equal(t, uint64(100), state.Received()) require.Equal(t, uint64(100), state.Sent()) - isNew, err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[0], 50, 3, false) + err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[0], 50, 3, false) require.NoError(t, err) - require.False(t, isNew) state = checkEvent(ctx, t, received, datatransfer.DataReceived) require.Equal(t, uint64(100), state.Received()) require.Equal(t, uint64(100), state.Sent()) }) + t.Run("data limit", func(t *testing.T) { + ds := dss.MutexWrap(datastore.NewMapDatastore()) + + channelList, err := channels.New(ds, notifier, decoderByType, decoderByType, &fakeEnv{}, peers[0]) + require.NoError(t, err) + err = channelList.Start(ctx) + require.NoError(t, err) + + _, err = channelList.CreateNew(peers[0], tid1, cids[0], selector, fv1, peers[1], peers[0], peers[1]) + require.NoError(t, err) + state := checkEvent(ctx, t, received, datatransfer.Open) + + err = channelList.DataQueued(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[0], 300, 1, true) + require.NoError(t, err) + _ = checkEvent(ctx, t, received, datatransfer.DataQueuedProgress) + state = checkEvent(ctx, t, received, datatransfer.DataQueued) + require.Equal(t, uint64(300), state.Queued()) + + err = channelList.SetDataLimit(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, 400) + require.NoError(t, err) + state = checkEvent(ctx, t, received, datatransfer.SetDataLimit) + require.Equal(t, state.DataLimit(), uint64(400)) + + // send block again has no effect + err = channelList.DataQueued(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[0], 300, 1, true) + require.NoError(t, err) + state = checkEvent(ctx, t, received, datatransfer.DataQueued) + require.Equal(t, uint64(300), state.Queued()) + + err = channelList.DataQueued(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[1], 200, 2, true) + require.EqualError(t, err, datatransfer.ErrPause.Error()) + _ = checkEvent(ctx, t, received, datatransfer.DataQueuedProgress) + _ = checkEvent(ctx, t, received, datatransfer.DataQueued) + state = checkEvent(ctx, t, received, datatransfer.DataLimitExceeded) + require.Equal(t, uint64(500), state.Queued()) + + err = channelList.SetDataLimit(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, 700) + require.NoError(t, err) + state = checkEvent(ctx, t, received, datatransfer.SetDataLimit) + require.Equal(t, state.DataLimit(), uint64(700)) + + err = channelList.DataQueued(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[2], 150, 3, true) + require.NoError(t, err) + _ = checkEvent(ctx, t, received, datatransfer.DataQueuedProgress) + state = checkEvent(ctx, t, received, datatransfer.DataQueued) + require.Equal(t, uint64(650), state.Queued()) + + err = channelList.DataQueued(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[3], 200, 4, true) + require.EqualError(t, err, datatransfer.ErrPause.Error()) + _ = checkEvent(ctx, t, received, datatransfer.DataQueuedProgress) + _ = checkEvent(ctx, t, received, datatransfer.DataQueued) + state = checkEvent(ctx, t, received, datatransfer.DataLimitExceeded) + require.Equal(t, uint64(850), state.Queued()) + }) + t.Run("pause/resume", func(t *testing.T) { state, err := channelList.GetByID(ctx, datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) require.NoError(t, err) diff --git a/channels/internal/internalchannel.go b/channels/internal/internalchannel.go index 6e5fba9f..43a8c05d 100644 --- a/channels/internal/internalchannel.go +++ b/channels/internal/internalchannel.go @@ -69,6 +69,12 @@ type ChannelState struct { // Number of blocks that have been sent, including blocks that are // present in more than one place in the DAG SentBlocksTotal int64 + // DataLimit is the maximum data that can be transferred on this channel before + // revalidation. 0 indicates no limit. + DataLimit uint64 + // RequiresFinalization indicates at the end of the transfer, the channel should + // be left open for a final settlement + RequiresFinalization bool // Stages traces the execution fo a data transfer. // // EXPERIMENTAL; subject to change. diff --git a/channels/internal/internalchannel_cbor_gen.go b/channels/internal/internalchannel_cbor_gen.go index afedd3a2..192ba0cf 100644 --- a/channels/internal/internalchannel_cbor_gen.go +++ b/channels/internal/internalchannel_cbor_gen.go @@ -23,7 +23,7 @@ func (t *ChannelState) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{180}); err != nil { + if _, err := w.Write([]byte{182}); err != nil { return err } @@ -411,6 +411,38 @@ func (t *ChannelState) MarshalCBOR(w io.Writer) error { } } + // t.DataLimit (uint64) (uint64) + if len("DataLimit") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DataLimit\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DataLimit"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("DataLimit")); err != nil { + return err + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DataLimit)); err != nil { + return err + } + + // t.RequiresFinalization (bool) (bool) + if len("RequiresFinalization") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"RequiresFinalization\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("RequiresFinalization"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("RequiresFinalization")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.RequiresFinalization); err != nil { + return err + } + // t.Stages (datatransfer.ChannelStages) (struct) if len("Stages") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Stages\" was too long") @@ -780,6 +812,39 @@ func (t *ChannelState) UnmarshalCBOR(r io.Reader) error { t.SentBlocksTotal = int64(extraI) } + // t.DataLimit (uint64) (uint64) + case "DataLimit": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DataLimit = uint64(extra) + + } + // t.RequiresFinalization (bool) (bool) + case "RequiresFinalization": + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.RequiresFinalization = false + case 21: + t.RequiresFinalization = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } // t.Stages (datatransfer.ChannelStages) (struct) case "Stages": diff --git a/events.go b/events.go index 664579c4..9ccd47c9 100644 --- a/events.go +++ b/events.go @@ -110,6 +110,19 @@ const ( // Opened is fired when a request for data is sent from this node to a peer Opened + + // SetDataLimit is fired when a responder sets a limit for data it will allow + // before pausing the request + SetDataLimit + + // SetRequiresFinalization is fired when a responder sets a limit for data it will allow + // before pausing the request + SetRequiresFinalization + + // DataLimitExceeded is fired when a request exceeds it's data limit. It has the effect of + // pausing the responder, but is distinct from PauseResponder to indicate why the pause + // happened + DataLimitExceeded ) // Events are human readable names for data transfer events @@ -144,6 +157,10 @@ var Events = map[EventCode]string{ ReceiveDataError: "ReceiveDataError", TransferRequestQueued: "TransferRequestQueued", RequestCancelled: "RequestCancelled", + Opened: "Opened", + SetDataLimit: "SetDataLimit", + SetRequiresFinalization: "SetRequiresFinalization", + DataLimitExceeded: "DataLimitExceeded", } // Event is a struct containing information about a data transfer event diff --git a/impl/events.go b/impl/events.go index 710a6b26..ef5c5fb8 100644 --- a/impl/events.go +++ b/impl/events.go @@ -2,20 +2,16 @@ package impl import ( "context" - "errors" - "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/libp2p/go-libp2p-core/peer" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/encoding" - "github.com/filecoin-project/go-data-transfer/v2/registry" + "github.com/filecoin-project/go-data-transfer/v2/message" ) // OnChannelOpened is called when we send a request for data to the other @@ -38,9 +34,8 @@ func (m *manager) OnChannelOpened(chid datatransfer.ChannelID) error { // OnDataReceived is called when the transport layer reports that it has // received some data from the sender. -// It fires an event on the channel, updating the sum of received data and -// calls revalidators so they can pause / resume the channel or send a -// message over the transport. +// It fires an event on the channel, updating the sum of received data and reports +// back a pause to the transport if the data limit is exceeded func (m *manager) OnDataReceived(chid datatransfer.ChannelID, link ipld.Link, size uint64, index int64, unique bool) error { ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) ctx, span := otel.Tracer("data-transfer").Start(ctx, "dataReceived", trace.WithAttributes( @@ -51,52 +46,23 @@ func (m *manager) OnDataReceived(chid datatransfer.ChannelID, link ipld.Link, si )) defer span.End() - isNew, err := m.channels.DataReceived(chid, link.(cidlink.Link).Cid, size, index, unique) - if err != nil { - return err - } - - // If this block has already been received on the channel, take no further - // action (this can happen when the data-transfer channel is restarted) - if !isNew { - return nil - } - - // If this node initiated the data transfer, there's nothing more to do - if chid.Initiator == m.peerID { - return nil - } - - // Check each revalidator to see if they want to pause / resume, or send - // a message over the transport - var result datatransfer.VoucherResult - var handled bool - _ = m.revalidators.Each(func(_ datatransfer.TypeIdentifier, _ encoding.Decoder, processor registry.Processor) error { - revalidator := processor.(datatransfer.Revalidator) - handled, result, err = revalidator.OnPushDataReceived(chid, size) - if handled { - return errors.New("stop processing") + err := m.channels.DataReceived(chid, link.(cidlink.Link).Cid, size, index, unique) + // if this channel is now paused, send the pause message + if err == datatransfer.ErrPause { + msg := message.UpdateResponse(chid.ID, true) + ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) + if err := m.dataTransferNetwork.SendMessage(ctx, chid.Initiator, msg); err != nil { + return err } - return nil - }) - if err != nil || result != nil { - msg, err := m.processRevalidationResult(chid, result, err) - if msg != nil { - ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) - if err := m.dataTransferNetwork.SendMessage(ctx, chid.Initiator, msg); err != nil { - return err - } - } - return err } - return nil + return err } // OnDataQueued is called when the transport layer reports that it has queued // up some data to be sent to the requester. -// It fires an event on the channel, updating the sum of queued data and calls -// revalidators so they can pause / resume or send a message over the transport. +// It fires an event on the channel, updating the sum of queued data and reports +// back a pause to the transport if the data limit is exceeded func (m *manager) OnDataQueued(chid datatransfer.ChannelID, link ipld.Link, size uint64, index int64, unique bool) (datatransfer.Message, error) { // The transport layer reports that some data has been queued up to be sent // to the requester, so fire a DataQueued event on the channels state @@ -110,43 +76,18 @@ func (m *manager) OnDataQueued(chid datatransfer.ChannelID, link ipld.Link, size )) defer span.End() - isNew, err := m.channels.DataQueued(chid, link.(cidlink.Link).Cid, size, index, unique) - if err != nil { - return nil, err - } - - // If this block has already been queued on the channel, take no further - // action (this can happen when the data-transfer channel is restarted) - if !isNew { - return nil, nil + var msg datatransfer.Message + err := m.channels.DataQueued(chid, link.(cidlink.Link).Cid, size, index, unique) + // if this channel is now paused, send the pause message + if err == datatransfer.ErrPause { + msg = message.UpdateResponse(chid.ID, true) } - // If this node initiated the data transfer, there's nothing more to do - if chid.Initiator == m.peerID { - return nil, nil - } - - // Check each revalidator to see if they want to pause / resume, or send - // a message over the transport. - // For example if the data-sender is waiting for the receiver to pay for - // data they may pause the data-transfer. - var result datatransfer.VoucherResult - var handled bool - _ = m.revalidators.Each(func(_ datatransfer.TypeIdentifier, _ encoding.Decoder, processor registry.Processor) error { - revalidator := processor.(datatransfer.Revalidator) - handled, result, err = revalidator.OnPullDataSent(chid, size) - if handled { - return errors.New("stop processing") - } - return nil - }) - if err != nil || result != nil { - return m.processRevalidationResult(chid, result, err) - } - - return nil, nil + return msg, err } +// OnDataSent is called when the transport layer reports that it has finished +// sending data to the requester. func (m *manager) OnDataSent(chid datatransfer.ChannelID, link ipld.Link, size uint64, index int64, unique bool) error { ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) @@ -157,60 +98,61 @@ func (m *manager) OnDataSent(chid datatransfer.ChannelID, link ipld.Link, size u )) defer span.End() - _, err := m.channels.DataSent(chid, link.(cidlink.Link).Cid, size, index, unique) - return err + return m.channels.DataSent(chid, link.(cidlink.Link).Cid, size, index, unique) } +// OnRequestReceived is called when a Request message is received from the initiator +// on the responder func (m *manager) OnRequestReceived(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { + + // if request is restart request, process as restart if request.IsRestart() { return m.receiveRestartRequest(chid, request) } + // if request is new request, process as new if request.IsNew() { return m.receiveNewRequest(chid, request) } + + // if request is cancel request, process as cancel if request.IsCancel() { log.Infof("channel %s: received cancel request, cleaning up channel", chid) m.transport.CleanupChannel(chid) return nil, m.channels.Cancel(chid) } + + // if request contains a new voucher, process updated voucher if request.IsVoucher() { return m.processUpdateVoucher(chid, request) } - if request.IsPaused() { - return nil, m.pauseOther(chid) - } - err := m.resumeOther(chid) - if err != nil { - return nil, err - } - chst, err := m.channels.GetByID(context.TODO(), chid) - if err != nil { - return nil, err - } - if chst.Status() == datatransfer.ResponderPaused || - chst.Status() == datatransfer.ResponderFinalizing { - return nil, datatransfer.ErrPause - } - return nil, nil + + // otherwise process as an "update" message (i.e. a pause or resume) + return m.receiveUpdateRequest(chid, request) } +// OnTransferQueued is called when the transport layer receives a request but has not yet processed it func (m *manager) OnTransferQueued(chid datatransfer.ChannelID) { m.channels.TransferRequestQueued(chid) } +// OnRequestReceived is called when a Response message is received from the responder +// on the initiator func (m *manager) OnResponseReceived(chid datatransfer.ChannelID, response datatransfer.Response) error { - if response.IsComplete() { - log.Infow("received complete response", "chid", chid, "isAccepted", response.Accepted()) - } + // if response is cancel, process as cancel if response.IsCancel() { log.Infof("channel %s: received cancel response, cancelling channel", chid) return m.channels.Cancel(chid) } - if response.IsVoucherResult() { + + // does this response contain a response to a validation attempt? + if response.IsValidationResult() { + + // is there a voucher response in this message? if !response.EmptyVoucherResult() { + // if so decode and save it vresult, err := m.decodeVoucherResult(response) if err != nil { return err @@ -220,59 +162,78 @@ func (m *manager) OnResponseReceived(chid datatransfer.ChannelID, response datat return err } } + + // was the validateion attempt successful? if !response.Accepted() { + // if not, error and fail log.Infof("channel %s: received rejected response, erroring out channel", chid) return m.channels.Error(chid, datatransfer.ErrRejected) } - if response.IsNew() { - log.Infof("channel %s: received new response, accepting channel", chid) - err := m.channels.Accept(chid) - if err != nil { - return err - } + } + + // was this the first response to our initial request + if response.IsNew() { + log.Infof("channel %s: received new response, accepting channel", chid) + // if so, record an accept event (not accepted has already been handled) + err := m.channels.Accept(chid) + if err != nil { + return err } + } - if response.IsRestart() { - log.Infof("channel %s: received restart response, restarting channel", chid) - err := m.channels.Restart(chid) - if err != nil { - return err - } + // was this a response to a restart attempt? + if response.IsRestart() { + log.Infof("channel %s: received restart response, restarting channel", chid) + // if so, record restart + err := m.channels.Restart(chid) + if err != nil { + return err } } - if response.IsComplete() && response.Accepted() { + + // was this response a final status message? + if response.IsComplete() { + // is the responder paused pending final settlement? if !response.IsPaused() { + // if not, mark the responder done and return log.Infow("received complete response,responder not paused, completing channel", "chid", chid) return m.channels.ResponderCompletes(chid) } + // if yes, mark the responder being in final settlement log.Infow("received complete response, responder is paused, not completing channel", "chid", chid) err := m.channels.ResponderBeginsFinalization(chid) if err != nil { - return nil + return err } } + + // handle pause/resume for all response types if response.IsPaused() { return m.pauseOther(chid) } return m.resumeOther(chid) } +// OnRequestCancelled is called when a transport reports a channel is cancelled func (m *manager) OnRequestCancelled(chid datatransfer.ChannelID, err error) error { log.Warnf("channel %+v was cancelled: %s", chid, err) return m.channels.RequestCancelled(chid, err) } +// OnRequestCancelled is called when a transport reports a channel disconnected func (m *manager) OnRequestDisconnected(chid datatransfer.ChannelID, err error) error { log.Warnf("channel %+v has stalled or disconnected: %s", chid, err) return m.channels.Disconnected(chid, err) } +// OnSendDataError is called when a transport has a network error sending data func (m *manager) OnSendDataError(chid datatransfer.ChannelID, err error) error { log.Debugf("channel %+v had transport send error: %s", chid, err) return m.channels.SendDataError(chid, err) } +// OnReceiveDataError is called when a transport has a network error receiving data func (m *manager) OnReceiveDataError(chid datatransfer.ChannelID, err error) error { log.Debugf("channel %+v had transport receive error: %s", chid, err) return m.channels.ReceiveDataError(chid, err) @@ -282,309 +243,59 @@ func (m *manager) OnReceiveDataError(chid datatransfer.ChannelID, err error) err // - by the requester when all data for a transfer has been received // - by the responder when all data for a transfer has been sent func (m *manager) OnChannelCompleted(chid datatransfer.ChannelID, completeErr error) error { - // If the channel completed successfully - if completeErr == nil { - // If the channel was initiated by the other peer - if chid.Initiator != m.peerID { - log.Infow("received OnChannelCompleted, will send completion message to initiator", "chid", chid) - msg, err := m.completeMessage(chid) - if err != nil { - return err - } - if msg != nil { - // Send the other peer a message that the transfer has completed - log.Infow("sending completion message to initiator", "chid", chid) - ctx, _ := m.spansIndex.SpanForChannel(context.Background(), chid) - if err := m.dataTransferNetwork.SendMessage(ctx, chid.Initiator, msg); err != nil { - err := xerrors.Errorf("channel %s: failed to send completion message to initiator: %w", chid, err) - log.Warnw("failed to send completion message to initiator", "chid", chid, "err", err) - return m.OnRequestDisconnected(chid, err) - } - log.Infow("successfully sent completion message to initiator", "chid", chid) - } - if msg.Accepted() { - if msg.IsPaused() { - return m.channels.BeginFinalizing(chid) - } - return m.channels.Complete(chid) - } - return m.channels.Error(chid, err) - } - - // The channel was initiated by this node, so move to the finished state - log.Infof("channel %s: transfer initiated by local node is complete", chid) - return m.channels.FinishTransfer(chid) - } - // There was an error so fire an Error event + // read the channel state chst, err := m.channels.GetByID(context.TODO(), chid) if err != nil { return err } - // send an error, but only if we haven't already errored for some reason - if chst.Status() != datatransfer.Failing && chst.Status() != datatransfer.Failed { - err := xerrors.Errorf("data transfer channel %s failed to transfer data: %w", chid, completeErr) - log.Warnf(err.Error()) - return m.channels.Error(chid, err) - } - return nil -} - -func (m *manager) OnContextAugment(chid datatransfer.ChannelID) func(context.Context) context.Context { - return func(ctx context.Context) context.Context { - ctx, _ = m.spansIndex.SpanForChannel(ctx, chid) - return ctx - } -} - -func (m *manager) receiveRestartRequest(chid datatransfer.ChannelID, incoming datatransfer.Request) (datatransfer.Response, error) { - log.Infof("channel %s: received restart request", chid) - - result, err := m.restartRequest(chid, incoming) - msg, msgErr := m.response(true, false, err, incoming.TransferID(), result) - if msgErr != nil { - return nil, msgErr - } - return msg, err -} - -func (m *manager) receiveNewRequest(chid datatransfer.ChannelID, incoming datatransfer.Request) (datatransfer.Response, error) { - log.Infof("channel %s: received new channel request from %s", chid, chid.Initiator) - - result, err := m.acceptRequest(chid, incoming) - msg, msgErr := m.response(false, true, err, incoming.TransferID(), result) - if msgErr != nil { - return nil, msgErr - } - return msg, err -} - -func (m *manager) restartRequest(chid datatransfer.ChannelID, - incoming datatransfer.Request) (datatransfer.VoucherResult, error) { - - initiator := chid.Initiator - if m.peerID == initiator { - return nil, xerrors.New("initiator cannot be manager peer for a restart request") - } - - if err := m.validateRestartRequest(context.Background(), initiator, chid, incoming); err != nil { - return nil, xerrors.Errorf("restart request for channel %s failed validation: %w", chid, err) - } - - stor, err := incoming.Selector() - if err != nil { - return nil, err - } - - voucher, result, err := m.validateVoucher(true, chid, initiator, incoming, incoming.IsPull(), incoming.BaseCid(), stor) - if err != nil && err != datatransfer.ErrPause { - return result, xerrors.Errorf("failed to validate voucher: %w", err) - } - voucherErr := err - - if result != nil { - err := m.channels.NewVoucherResult(chid, result) - if err != nil { - return result, err - } - } - if err := m.channels.Restart(chid); err != nil { - return result, xerrors.Errorf("failed to restart channel %s: %w", chid, err) - } - processor, has := m.transportConfigurers.Processor(voucher.Type()) - if has { - transportConfigurer := processor.(datatransfer.TransportConfigurer) - transportConfigurer(chid, voucher, m.transport) - } - m.dataTransferNetwork.Protect(initiator, chid.String()) - if voucherErr == datatransfer.ErrPause { - err := m.channels.PauseResponder(chid) - if err != nil { - return result, err - } - } - return result, voucherErr -} - -func (m *manager) acceptRequest(chid datatransfer.ChannelID, incoming datatransfer.Request) (datatransfer.VoucherResult, error) { - - stor, err := incoming.Selector() - if err != nil { - return nil, err - } - - voucher, result, err := m.validateVoucher(false, chid, chid.Initiator, incoming, incoming.IsPull(), incoming.BaseCid(), stor) - if err != nil && err != datatransfer.ErrPause { - return result, err - } - voucherErr := err - - var dataSender, dataReceiver peer.ID - if incoming.IsPull() { - dataSender = m.peerID - dataReceiver = chid.Initiator - } else { - dataSender = chid.Initiator - dataReceiver = m.peerID - } - - log.Infow("data-transfer request validated, will create & start tracking channel", "channelID", chid, "payloadCid", incoming.BaseCid()) - _, err = m.channels.CreateNew(m.peerID, incoming.TransferID(), incoming.BaseCid(), stor, voucher, chid.Initiator, dataSender, dataReceiver) - if err != nil { - log.Errorw("failed to create and start tracking channel", "channelID", chid, "err", err) - return result, err - } - log.Debugw("successfully created and started tracking channel", "channelID", chid) - if result != nil { - err := m.channels.NewVoucherResult(chid, result) - if err != nil { - return result, err - } - } - if err := m.channels.Accept(chid); err != nil { - return result, err - } - processor, has := m.transportConfigurers.Processor(voucher.Type()) - if has { - transportConfigurer := processor.(datatransfer.TransportConfigurer) - transportConfigurer(chid, voucher, m.transport) - } - m.dataTransferNetwork.Protect(chid.Initiator, chid.String()) - if voucherErr == datatransfer.ErrPause { - err := m.channels.PauseResponder(chid) - if err != nil { - return result, err - } - } - return result, voucherErr -} -// validateVoucher converts a voucher in an incoming message to its appropriate -// voucher struct, then runs the validator and returns the results. -// returns error if: -// * reading voucher fails -// * deserialization of selector fails -// * validation fails -func (m *manager) validateVoucher( - isRestart bool, - chid datatransfer.ChannelID, - sender peer.ID, - incoming datatransfer.Request, - isPull bool, - baseCid cid.Cid, - stor ipld.Node, -) (datatransfer.Voucher, datatransfer.VoucherResult, error) { - vouch, err := m.decodeVoucher(incoming, m.validatedTypes) - if err != nil { - return nil, nil, err - } - var validatorFunc func(bool, datatransfer.ChannelID, peer.ID, datatransfer.Voucher, cid.Cid, ipld.Node) (datatransfer.VoucherResult, error) - processor, _ := m.validatedTypes.Processor(vouch.Type()) - validator := processor.(datatransfer.RequestValidator) - if isPull { - validatorFunc = validator.ValidatePull - } else { - validatorFunc = validator.ValidatePush - } - - result, err := validatorFunc(isRestart, chid, sender, vouch, baseCid, stor) - return vouch, result, err -} - -// revalidateVoucher converts a voucher in an incoming message to its appropriate -// voucher struct, then runs the revalidator and returns the results. -// returns error if: -// * reading voucher fails -// * deserialization of selector fails -// * validation fails -func (m *manager) revalidateVoucher(chid datatransfer.ChannelID, - incoming datatransfer.Request) (datatransfer.Voucher, datatransfer.VoucherResult, error) { - vouch, err := m.decodeVoucher(incoming, m.revalidators) - if err != nil { - return nil, nil, err - } - processor, _ := m.revalidators.Processor(vouch.Type()) - validator := processor.(datatransfer.Revalidator) - - result, err := validator.Revalidate(chid, vouch) - return vouch, result, err -} - -func (m *manager) processUpdateVoucher(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { - vouch, result, voucherErr := m.revalidateVoucher(chid, request) - if vouch != nil { - err := m.channels.NewVoucher(chid, vouch) - if err != nil { - return nil, err + // If the transferred errored on completion + if completeErr != nil { + // send an error, but only if we haven't already errored for some reason + if chst.Status() != datatransfer.Failing && chst.Status() != datatransfer.Failed { + err := xerrors.Errorf("data transfer channel %s failed to transfer data: %w", chid, completeErr) + log.Warnf(err.Error()) + return m.channels.Error(chid, err) } + return nil } - return m.processRevalidationResult(chid, result, voucherErr) -} -func (m *manager) revalidationResponse(chid datatransfer.ChannelID, result datatransfer.VoucherResult, resultErr error) (datatransfer.Response, error) { - chst, err := m.channels.GetByID(context.TODO(), chid) - if err != nil { - return nil, err - } - if chst.Status() == datatransfer.Finalizing { - return m.completeResponse(resultErr, chid.ID, result) + // if the channel was initiated by this node, simply record the transfer being finished + if chid.Initiator == m.peerID { + log.Infof("channel %s: transfer initiated by local node is complete", chid) + return m.channels.FinishTransfer(chid) } - return m.response(false, false, resultErr, chid.ID, result) -} -func (m *manager) processRevalidationResult(chid datatransfer.ChannelID, result datatransfer.VoucherResult, resultErr error) (datatransfer.Response, error) { - vresMessage, err := m.revalidationResponse(chid, result, resultErr) + // otherwise, process as responder + log.Infow("received OnChannelCompleted, will send completion message to initiator", "chid", chid) + // generate and send the final status message + msg, err := message.CompleteResponse(chst.TransferID(), true, chst.RequiresFinalization(), datatransfer.EmptyTypeIdentifier, nil) if err != nil { - return nil, err + return err } - if result != nil { - err := m.channels.NewVoucherResult(chid, result) - if err != nil { - return nil, err - } + log.Infow("sending completion message to initiator", "chid", chid) + ctx, _ := m.spansIndex.SpanForChannel(context.Background(), chid) + if err := m.dataTransferNetwork.SendMessage(ctx, chid.Initiator, msg); err != nil { + err := xerrors.Errorf("channel %s: failed to send completion message to initiator: %w", chid, err) + log.Warnw("failed to send completion message to initiator", "chid", chid, "err", err) + return m.OnRequestDisconnected(chid, err) } + log.Infow("successfully sent completion message to initiator", "chid", chid) - if resultErr == nil { - return vresMessage, nil + // set the channel state based on whether its paused final settlement + if chst.RequiresFinalization() { + return m.channels.BeginFinalizing(chid) } - - if resultErr == datatransfer.ErrPause { - err := m.pause(chid) - if err != nil { - return nil, err - } - return vresMessage, datatransfer.ErrPause - } - - if resultErr == datatransfer.ErrResume { - err = m.resume(chid) - if err != nil { - return nil, err - } - return vresMessage, datatransfer.ErrResume - } - return vresMessage, resultErr + return m.channels.Complete(chid) } -func (m *manager) completeMessage(chid datatransfer.ChannelID) (datatransfer.Response, error) { - var result datatransfer.VoucherResult - var resultErr error - var handled bool - _ = m.revalidators.Each(func(_ datatransfer.TypeIdentifier, _ encoding.Decoder, processor registry.Processor) error { - revalidator := processor.(datatransfer.Revalidator) - handled, result, resultErr = revalidator.OnComplete(chid) - if handled { - return errors.New("stop processing") - } - return nil - }) - if result != nil { - err := m.channels.NewVoucherResult(chid, result) - if err != nil { - return nil, err - } +// OnContextAugment provides an oppurtunity for transports to have data transfer add data to their context (i.e. +// to tie into tracing, etc) +func (m *manager) OnContextAugment(chid datatransfer.ChannelID) func(context.Context) context.Context { + return func(ctx context.Context) context.Context { + ctx, _ = m.spansIndex.SpanForChannel(ctx, chid) + return ctx } - - return m.completeResponse(resultErr, chid.ID, result) } diff --git a/impl/impl.go b/impl/impl.go index 0a3c09e5..01b99d72 100644 --- a/impl/impl.go +++ b/impl/impl.go @@ -36,7 +36,6 @@ type manager struct { dataTransferNetwork network.DataTransferNetwork validatedTypes *registry.Registry resultTypes *registry.Registry - revalidators *registry.Registry transportConfigurers *registry.Registry pubSub *pubsub.PubSub readySub *pubsub.PubSub @@ -97,7 +96,6 @@ func NewDataTransfer(ds datastore.Batching, dataTransferNetwork network.DataTran dataTransferNetwork: dataTransferNetwork, validatedTypes: registry.NewRegistry(), resultTypes: registry.NewRegistry(), - revalidators: registry.NewRegistry(), transportConfigurers: registry.NewRegistry(), pubSub: pubsub.New(dispatcher), readySub: pubsub.New(readyDispatcher), @@ -126,11 +124,7 @@ func NewDataTransfer(ds datastore.Batching, dataTransferNetwork network.DataTran } func (m *manager) voucherDecoder(voucherType datatransfer.TypeIdentifier) (encoding.Decoder, bool) { - decoder, has := m.validatedTypes.Decoder(voucherType) - if !has { - return m.revalidators.Decoder(voucherType) - } - return decoder, true + return m.validatedTypes.Decoder(voucherType) } func (m *manager) notifier(evt datatransfer.Event, chst datatransfer.ChannelState) { @@ -300,6 +294,50 @@ func (m *manager) SendVoucher(ctx context.Context, channelID datatransfer.Channe return m.channels.NewVoucher(channelID, voucher) } +func (m *manager) SendVoucherResult(ctx context.Context, channelID datatransfer.ChannelID, voucherResult datatransfer.VoucherResult) error { + chst, err := m.channels.GetByID(ctx, channelID) + if err != nil { + return err + } + ctx, _ = m.spansIndex.SpanForChannel(ctx, channelID) + ctx, span := otel.Tracer("data-transfer").Start(ctx, "sendVoucherResult", trace.WithAttributes( + attribute.String("channelID", channelID.String()), + attribute.String("voucherResultType", string(voucherResult.Type())), + )) + defer span.End() + if channelID.Initiator == m.peerID { + err := errors.New("cannot send voucher result for request we initiated") + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return err + } + + var updateResponse datatransfer.Response + if chst.Status().InFinalization() { + updateResponse, err = message.CompleteResponse(channelID.ID, chst.Status().IsAccepted(), chst.Status().IsResponderPaused(), voucherResult.Type(), voucherResult) + } else { + updateResponse, err = message.VoucherResultResponse(channelID.ID, chst.Status().IsAccepted(), chst.Status().IsResponderPaused(), voucherResult.Type(), voucherResult) + } + + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return err + } + if err := m.dataTransferNetwork.SendMessage(ctx, chst.OtherPeer(), updateResponse); err != nil { + err = fmt.Errorf("Unable to send request: %w", err) + _ = m.OnRequestDisconnected(channelID, err) + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return err + } + return m.channels.NewVoucherResult(channelID, voucherResult) +} + +func (m *manager) SetDataLimit(ctx context.Context, chid datatransfer.ChannelID, totalData uint64, stopAtEnd bool) { + +} + // close an open channel (effectively a cancel) func (m *manager) CloseDataTransferChannel(ctx context.Context, chid datatransfer.ChannelID) error { log.Infof("close channel %s", chid) @@ -461,19 +499,6 @@ func (m *manager) InProgressChannels(ctx context.Context) (map[datatransfer.Chan return m.channels.InProgress() } -// RegisterRevalidator registers a revalidator for the given voucher type -// Note: this is the voucher type used to revalidate. It can share a name -// with the initial validator type and CAN be the same type, or a different type. -// The revalidator can simply be the sampe as the original request validator, -// or a different validator that satisfies the revalidator interface. -func (m *manager) RegisterRevalidator(voucherType datatransfer.Voucher, revalidator datatransfer.Revalidator) error { - err := m.revalidators.Register(voucherType, revalidator) - if err != nil { - return xerrors.Errorf("error registering revalidator type: %w", err) - } - return nil -} - // RegisterVoucherResultType allows deserialization of a voucher result, // so that a listener can read the metadata func (m *manager) RegisterVoucherResultType(resultType datatransfer.VoucherResult) error { diff --git a/impl/initiating_test.go b/impl/initiating_test.go index 6740f69f..34f2d180 100644 --- a/impl/initiating_test.go +++ b/impl/initiating_test.go @@ -445,9 +445,16 @@ func TestDataTransferRestartInitiating(t *testing.T) { }, }, "RestartDataTransferChannel: Manager Peer Receive Push Restart works ": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + }, verify: func(t *testing.T, h *harness) { ctx := context.Background() + + h.voucherValidator.ExpectSuccessPush() + h.voucherValidator.StubResult(datatransfer.ValidationResult{Accepted: true}) + // receive a push request h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) require.Len(t, h.transport.OpenedChannels, 1) @@ -455,19 +462,16 @@ func TestDataTransferRestartInitiating(t *testing.T) { require.Len(t, h.voucherValidator.ValidationsReceived, 1) // restart the push request received above and validate it + h.voucherValidator.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} require.NoError(t, h.dt.RestartDataTransferChannel(ctx, chid)) - require.Len(t, h.voucherValidator.ValidationsReceived, 2) + require.Len(t, h.voucherValidator.RevalidationsReceived, 1) require.Len(t, h.transport.OpenedChannels, 1) require.Len(t, h.network.SentMessages, 1) // assert validation on restart - vmsg := h.voucherValidator.ValidationsReceived[1] - require.Equal(t, h.voucher, vmsg.Voucher) - require.False(t, vmsg.IsPull) - require.Equal(t, h.stor, vmsg.Selector) - require.Equal(t, h.baseCid, vmsg.BaseCid) - require.Equal(t, h.peers[1], vmsg.Other) + vmsg := h.voucherValidator.RevalidationsReceived[0] + require.Equal(t, channelID(h.id, h.peers), vmsg.ChannelID) // assert req was sent correctly req := h.network.SentMessages[0] @@ -480,35 +484,36 @@ func TestDataTransferRestartInitiating(t *testing.T) { achId, err := receivedRequest.RestartChannelId() require.NoError(t, err) require.Equal(t, chid, achId) - - h.voucherValidator.ExpectSuccessPush() }, }, "RestartDataTransferChannel: Manager Peer Receive Pull Restart works ": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + }, verify: func(t *testing.T, h *harness) { ctx := context.Background() // receive a pull request + h.voucherValidator.ExpectSuccessPull() + h.voucherValidator.StubResult(datatransfer.ValidationResult{Accepted: true}) + h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pullRequest) require.Len(t, h.transport.OpenedChannels, 0) require.Len(t, h.network.SentMessages, 1) require.Len(t, h.voucherValidator.ValidationsReceived, 1) // restart the pull request received above - h.voucherValidator.ExpectSuccessPull() + h.voucherValidator.ExpectSuccessRevalidation() + h.voucherValidator.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pullRequest.TransferID()} require.NoError(t, h.dt.RestartDataTransferChannel(ctx, chid)) require.Len(t, h.transport.OpenedChannels, 0) require.Len(t, h.network.SentMessages, 2) - require.Len(t, h.voucherValidator.ValidationsReceived, 2) + require.Len(t, h.voucherValidator.RevalidationsReceived, 1) // assert validation on restart - vmsg := h.voucherValidator.ValidationsReceived[1] - require.Equal(t, h.voucher, vmsg.Voucher) - require.True(t, vmsg.IsPull) - require.Equal(t, h.stor, vmsg.Selector) - require.Equal(t, h.baseCid, vmsg.BaseCid) - require.Equal(t, h.peers[1], vmsg.Other) + vmsg := h.voucherValidator.RevalidationsReceived[0] + require.Equal(t, channelID(h.id, h.peers), vmsg.ChannelID) // assert req was sent correctly req := h.network.SentMessages[1] @@ -524,35 +529,47 @@ func TestDataTransferRestartInitiating(t *testing.T) { }, }, "RestartDataTransferChannel: Manager Peer Receive Pull Restart fails if validation fails ": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + }, verify: func(t *testing.T, h *harness) { ctx := context.Background() // receive a pull request + h.voucherValidator.ExpectSuccessPull() + h.voucherValidator.StubResult(datatransfer.ValidationResult{Accepted: true}) h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pullRequest) require.Len(t, h.transport.OpenedChannels, 0) require.Len(t, h.network.SentMessages, 1) require.Len(t, h.voucherValidator.ValidationsReceived, 1) // restart the pull request received above - h.voucherValidator.ExpectErrorPull() + h.voucherValidator.ExpectSuccessRevalidation() + h.voucherValidator.StubRevalidationResult(datatransfer.ValidationResult{Accepted: false}) chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pullRequest.TransferID()} - require.EqualError(t, h.dt.RestartDataTransferChannel(ctx, chid), "failed to restart channel, validation error: something went wrong") + require.EqualError(t, h.dt.RestartDataTransferChannel(ctx, chid), datatransfer.ErrRejected.Error()) }, }, "RestartDataTransferChannel: Manager Peer Receive Push Restart fails if validation fails ": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + }, verify: func(t *testing.T, h *harness) { ctx := context.Background() // receive a push request + h.voucherValidator.ExpectSuccessPush() + h.voucherValidator.StubResult(datatransfer.ValidationResult{Accepted: true}) h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) require.Len(t, h.transport.OpenedChannels, 1) require.Len(t, h.network.SentMessages, 0) require.Len(t, h.voucherValidator.ValidationsReceived, 1) // restart the pull request received above - h.voucherValidator.ExpectErrorPush() + h.voucherValidator.ExpectSuccessRevalidation() + h.voucherValidator.StubRevalidationResult(datatransfer.ValidationResult{Accepted: false}) chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} - require.EqualError(t, h.dt.RestartDataTransferChannel(ctx, chid), "failed to restart channel, validation error: something went wrong") + require.EqualError(t, h.dt.RestartDataTransferChannel(ctx, chid), datatransfer.ErrRejected.Error()) }, }, "Fails if channel does not exist": { @@ -640,12 +657,8 @@ type eventVerifier struct { func (e eventVerifier) setup(t *testing.T, dt datatransfer.Manager) { if len(e.expectedEvents) > 0 { received := 0 - max := len(e.expectedEvents) dt.SubscribeToEvents(func(evt datatransfer.Event, state datatransfer.ChannelState) { received++ - if received > max { - t.Fatalf("received too many events: %s", datatransfer.Events[evt.Code]) - } e.events <- evt.Code }) } @@ -662,6 +675,12 @@ func (e eventVerifier) verify(ctx context.Context, t *testing.T) { receivedEvents = append(receivedEvents, event) } } + timer := time.NewTimer(50 * time.Millisecond) + select { + case event := <-e.events: + t.Fatalf("received extra event: %s", datatransfer.Events[event]) + case <-timer.C: + } require.Equal(t, e.expectedEvents, receivedEvents) } } diff --git a/impl/integration_test.go b/impl/integration_test.go index 9c2eee17..bef88a85 100644 --- a/impl/integration_test.go +++ b/impl/integration_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" dss "github.com/ipfs/go-datastore/sync" @@ -182,6 +183,7 @@ func TestRoundTrip(t *testing.T) { dt2.SubscribeToEvents(subscriber) voucher := testutil.FakeDTType{Data: "applesauce"} sv := testutil.NewStubbedValidator() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) var sourceDagService ipldformat.DAGService if data.customSourceStore { @@ -328,6 +330,7 @@ func TestMultipleRoundTripMultipleStores(t *testing.T) { vouchers = append(vouchers, testutil.NewFakeDTType()) } sv := testutil.NewStubbedValidator() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) root, origBytes := testutil.LoadUnixFSFile(ctx, t, gsData.DagService1, loremFile) rootCid := root.(cidlink.Link).Cid @@ -490,6 +493,7 @@ func TestManyReceiversAtOnce(t *testing.T) { vouchers = append(vouchers, testutil.NewFakeDTType()) } sv := testutil.NewStubbedValidator() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) root, origBytes := testutil.LoadUnixFSFile(ctx, t, gsData.DagService1, loremFile) rootCid := root.(cidlink.Link).Cid @@ -556,66 +560,6 @@ func (dc *disconnectCoordinator) onDisconnect() { close(dc.disconnected) } -type restartRevalidator struct { - *testutil.StubbedRevalidator - pullDataSent map[datatransfer.ChannelID][]uint64 - pushDataRcvd map[datatransfer.ChannelID][]uint64 -} - -func newRestartRevalidator() *restartRevalidator { - return &restartRevalidator{ - StubbedRevalidator: testutil.NewStubbedRevalidator(), - pullDataSent: make(map[datatransfer.ChannelID][]uint64), - pushDataRcvd: make(map[datatransfer.ChannelID][]uint64), - } -} - -func (r *restartRevalidator) OnPullDataSent(chid datatransfer.ChannelID, additionalBytesSent uint64) (bool, datatransfer.VoucherResult, error) { - chSent, ok := r.pullDataSent[chid] - if !ok { - chSent = []uint64{} - } - chSent = append(chSent, additionalBytesSent) - r.pullDataSent[chid] = chSent - - return true, nil, nil -} - -func (r *restartRevalidator) pullDataSum(chid datatransfer.ChannelID) uint64 { - pullDataSent, ok := r.pullDataSent[chid] - var total uint64 - if !ok { - return total - } - for _, sent := range pullDataSent { - total += sent - } - return total -} - -func (r *restartRevalidator) OnPushDataReceived(chid datatransfer.ChannelID, additionalBytesReceived uint64) (bool, datatransfer.VoucherResult, error) { - chRcvd, ok := r.pushDataRcvd[chid] - if !ok { - chRcvd = []uint64{} - } - chRcvd = append(chRcvd, additionalBytesReceived) - r.pushDataRcvd[chid] = chRcvd - - return true, nil, nil -} - -func (r *restartRevalidator) pushDataSum(chid datatransfer.ChannelID) uint64 { - pushDataRcvd, ok := r.pushDataRcvd[chid] - var total uint64 - if !ok { - return total - } - for _, rcvd := range pushDataRcvd { - total += rcvd - } - return total -} - // TestAutoRestart tests that if the connection for a push or pull request // goes down, it will automatically restart (given the right config options) func TestAutoRestart(t *testing.T) { @@ -794,6 +738,8 @@ func TestAutoRestart(t *testing.T) { responder.SubscribeToEvents(subscriber) voucher := testutil.FakeDTType{Data: "applesauce"} sv := testutil.NewStubbedValidator() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) + sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) var sourceDagService, destDagService ipldformat.DAGService if tc.isPush { @@ -810,10 +756,6 @@ func TestAutoRestart(t *testing.T) { require.NoError(t, initiator.RegisterVoucherType(&testutil.FakeDTType{}, sv)) require.NoError(t, responder.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - // Register a revalidator that records calls to OnPullDataSent and OnPushDataReceived - srv := newRestartRevalidator() - require.NoError(t, responder.RegisterRevalidator(testutil.NewFakeDTType(), srv)) - // If the test case needs to subscribe to response events, provide // the test case with the responder if tc.registerResponder != nil { @@ -895,12 +837,13 @@ func TestAutoRestart(t *testing.T) { } })() - // Verify that the total amount of data sent / received that was - // reported to the revalidator is correct + chst, err := responder.ChannelState(ctx, chid) + require.NoError(t, err) + // Verify that the total amount of data sent / received was correct if tc.isPush { - require.EqualValues(t, loremFileTransferBytes, srv.pushDataSum(chid)) + require.EqualValues(t, uint64(loremFileTransferBytes), chst.Received()) } else { - require.EqualValues(t, loremFileTransferBytes, srv.pullDataSum(chid)) + require.EqualValues(t, uint64(loremFileTransferBytes), chst.Sent()) } // Verify that the file was transferred to the destination node @@ -988,6 +931,8 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { voucher := testutil.FakeDTType{Data: "applesauce"} sv := testutil.NewStubbedValidator() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) + sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) var sourceDagService, destDagService ipldformat.DAGService if isPush { @@ -1004,10 +949,6 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { require.NoError(t, initiator.RegisterVoucherType(&testutil.FakeDTType{}, sv)) require.NoError(t, responder.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - // Register a revalidator that records calls to OnPullDataSent and OnPushDataReceived - srv := newRestartRevalidator() - require.NoError(t, responder.RegisterRevalidator(testutil.NewFakeDTType(), srv)) - var chid datatransfer.ChannelID if isPush { // Open a push channel @@ -1112,12 +1053,13 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { } })() - // Verify that the total amount of data sent / received that was - // reported to the revalidator is correct + chst, err := responder.ChannelState(ctx, chid) + require.NoError(t, err) + // Verify that the total amount of data sent / received was correct if isPush { - require.EqualValues(t, loremLargeFileTransferBytes, srv.pushDataSum(chid)) + require.EqualValues(t, uint64(loremFileTransferBytes), chst.Received()) } else { - require.EqualValues(t, loremLargeFileTransferBytes, srv.pullDataSum(chid)) + require.EqualValues(t, uint64(loremFileTransferBytes), chst.Sent()) } // Verify that the file was transferred to the destination node @@ -1192,11 +1134,13 @@ func TestRoundTripCancelledRequest(t *testing.T) { var chid datatransfer.ChannelID if data.isPull { - sv.ExpectPausePull() + sv.ExpectSuccessPull() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, LeaveRequestPaused: true}) require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), &voucher, rootCid, gsData.AllSelector) } else { - sv.ExpectPausePush() + sv.ExpectSuccessPush() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, LeaveRequestPaused: true}) require.NoError(t, dt2.RegisterVoucherType(&testutil.FakeDTType{}, sv)) chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, rootCid, gsData.AllSelector) } @@ -1239,33 +1183,62 @@ func TestRoundTripCancelledRequest(t *testing.T) { } type retrievalRevalidator struct { - *testutil.StubbedRevalidator - dataSoFar uint64 - providerPausePoint int - pausePoints []uint64 - finalVoucher datatransfer.VoucherResult - revalVouchers []datatransfer.VoucherResult + *testutil.StubbedValidator + providerPausePoint int + pausePoints []uint64 + leavePausedInitially bool + initialVoucherResult datatransfer.VoucherResult + requiresFinalization bool } -func (r *retrievalRevalidator) OnPullDataSent(chid datatransfer.ChannelID, additionalBytesSent uint64) (bool, datatransfer.VoucherResult, error) { - r.dataSoFar += additionalBytesSent - if r.providerPausePoint < len(r.pausePoints) && - r.dataSoFar >= r.pausePoints[r.providerPausePoint] { - var v datatransfer.VoucherResult = testutil.NewFakeDTType() - if len(r.revalVouchers) > r.providerPausePoint { - v = r.revalVouchers[r.providerPausePoint] - } +func (r *retrievalRevalidator) ValidatePush( + chid datatransfer.ChannelID, + sender peer.ID, + voucher datatransfer.Voucher, + baseCid cid.Cid, + selector ipld.Node) (datatransfer.ValidationResult, error) { + vr := datatransfer.ValidationResult{ + Accepted: true, + RequiresFinalization: r.requiresFinalization, + LeaveRequestPaused: r.leavePausedInitially, + VoucherResult: r.initialVoucherResult, + } + if len(r.pausePoints) > r.providerPausePoint { + vr.DataLimit = r.pausePoints[r.providerPausePoint] r.providerPausePoint++ - return true, v, datatransfer.ErrPause } - return true, nil, nil + r.StubbedValidator.StubResult(vr) + return r.StubbedValidator.ValidatePush(chid, sender, voucher, baseCid, selector) } -func (r *retrievalRevalidator) OnPushDataReceived(chid datatransfer.ChannelID, additionalBytesReceived uint64) (bool, datatransfer.VoucherResult, error) { - return false, nil, nil +func (r *retrievalRevalidator) ValidatePull( + chid datatransfer.ChannelID, + sender peer.ID, + voucher datatransfer.Voucher, + baseCid cid.Cid, + selector ipld.Node) (datatransfer.ValidationResult, error) { + vr := datatransfer.ValidationResult{ + Accepted: true, + RequiresFinalization: r.requiresFinalization, + LeaveRequestPaused: r.leavePausedInitially, + VoucherResult: r.initialVoucherResult, + } + if len(r.pausePoints) > r.providerPausePoint { + vr.DataLimit = r.pausePoints[r.providerPausePoint] + r.providerPausePoint++ + } + r.StubbedValidator.StubResult(vr) + return r.StubbedValidator.ValidatePull(chid, sender, voucher, baseCid, selector) } -func (r *retrievalRevalidator) OnComplete(chid datatransfer.ChannelID) (bool, datatransfer.VoucherResult, error) { - return true, r.finalVoucher, datatransfer.ErrPause + +func (r *retrievalRevalidator) Revalidate(chid datatransfer.ChannelID, channelState datatransfer.ChannelState) (datatransfer.ValidationResult, error) { + vr := datatransfer.ValidationResult{Accepted: true, RequiresFinalization: r.requiresFinalization} + if len(r.pausePoints) > r.providerPausePoint { + vr.DataLimit = r.pausePoints[r.providerPausePoint] + r.providerPausePoint++ + } + r.StubbedValidator.StubRevalidationResult(vr) + return r.StubbedValidator.Revalidate(chid, channelState) } func TestSimulatedRetrievalFlow(t *testing.T) { @@ -1386,17 +1359,19 @@ func TestSimulatedRetrievalFlow(t *testing.T) { } dt2.SubscribeToEvents(clientSubscriber) providerFinished := make(chan struct{}, 1) - providerAccepted := false var providerSubscriber datatransfer.Subscriber = func(event datatransfer.Event, channelState datatransfer.ChannelState) { if event.Code == datatransfer.PauseResponder { - if !providerAccepted { - providerAccepted = true - timer := time.NewTimer(config.unpauseResponderDelay) - go func() { - <-timer.C - _ = dt1.ResumeDataTransferChannel(ctx, chid) - }() - } + timer := time.NewTimer(config.unpauseResponderDelay) + go func() { + <-timer.C + _ = dt1.ResumeDataTransferChannel(ctx, chid) + }() + } + if event.Code == datatransfer.DataLimitExceeded { + dt1.SendVoucherResult(ctx, chid, testutil.NewFakeDTType()) + } + if event.Code == datatransfer.BeginFinalizing { + dt1.SendVoucherResult(ctx, chid, finalVoucherResult) } if event.Code == datatransfer.Error { errChan <- struct{}{} @@ -1407,15 +1382,14 @@ func TestSimulatedRetrievalFlow(t *testing.T) { } dt1.SubscribeToEvents(providerSubscriber) voucher := testutil.FakeDTType{Data: "applesauce"} - sv := testutil.NewStubbedValidator() - sv.ExpectPausePull() - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - srv := &retrievalRevalidator{ - testutil.NewStubbedRevalidator(), 0, 0, config.pausePoints, finalVoucherResult, []datatransfer.VoucherResult{}, + sv := &retrievalRevalidator{ + StubbedValidator: testutil.NewStubbedValidator(), + pausePoints: config.pausePoints, + requiresFinalization: true, + leavePausedInitially: true, } - srv.ExpectSuccessErrResume() - require.NoError(t, dt1.RegisterRevalidator(testutil.NewFakeDTType(), srv)) + require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) require.NoError(t, dt2.RegisterVoucherResultType(testutil.NewFakeDTType())) chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), &voucher, rootCid, gsData.AllSelector) @@ -1434,9 +1408,8 @@ func TestSimulatedRetrievalFlow(t *testing.T) { } } sv.VerifyExpectations(t) - srv.VerifyExpectations(t) gsData.VerifyFileTransferred(t, root, true) - require.Equal(t, srv.providerPausePoint, len(config.pausePoints)) + require.Equal(t, sv.providerPausePoint, len(config.pausePoints)) require.Equal(t, clientPausePoint, len(config.pausePoints)) traces := collectTracing(t).TracesToStrings(3) for _, expectedTrace := range config.expectedTraces { @@ -1521,6 +1494,8 @@ func TestPauseAndResume(t *testing.T) { dt2.SubscribeToEvents(subscriber) voucher := testutil.FakeDTType{Data: "applesauce"} sv := testutil.NewStubbedValidator() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) + sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) var chid datatransfer.ChannelID if isPull { @@ -1938,6 +1913,7 @@ func TestRespondingToPullGraphsyncRequests(t *testing.T) { test: func(t *testing.T, gsData *testutil.GraphsyncTestingData, tp2 datatransfer.Transport, link ipld.Link, id datatransfer.TransferID, gsr *fakeGraphSyncReceiver) { sv := testutil.NewStubbedValidator() sv.ExpectSuccessPull() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) dt1, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) require.NoError(t, err) @@ -2115,6 +2091,7 @@ func TestMultipleMessagesInExtension(t *testing.T) { }) providerFinished := make(chan struct{}, 1) + nextVoucherResult := 0 dt1.SubscribeToEvents(func(event datatransfer.Event, channelState datatransfer.ChannelState) { if event.Code == datatransfer.Error { errChan <- struct{}{} @@ -2122,24 +2099,26 @@ func TestMultipleMessagesInExtension(t *testing.T) { if channelState.Status() == datatransfer.Completed { providerFinished <- struct{}{} } + if event.Code == datatransfer.DataLimitExceeded { + if nextVoucherResult < len(pausePoints) { + dt1.SendVoucherResult(ctx, chid, voucherResults[nextVoucherResult]) + nextVoucherResult++ + } + } + if event.Code == datatransfer.BeginFinalizing { + dt1.SendVoucherResult(ctx, chid, finalVoucherResult) + } }) - sv := testutil.NewStubbedValidator() - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - // Stub in the validator so it returns that exact voucher when calling ValidatePull - // this validator will not pause transfer when accepting a transfer and will start - // sending blocks immediately - sv.StubResult(respVoucher) - - srv := &retrievalRevalidator{ - testutil.NewStubbedRevalidator(), 0, 0, pausePoints, finalVoucherResult, voucherResults, + sv := &retrievalRevalidator{ + StubbedValidator: testutil.NewStubbedValidator(), + pausePoints: pausePoints, + requiresFinalization: true, + initialVoucherResult: respVoucher, } - // The stubbed revalidator will authorize Revalidate and return ErrResume to finisht the transfer - srv.ExpectSuccessErrResume() - require.NoError(t, dt1.RegisterRevalidator(testutil.NewFakeDTType(), srv)) + require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - // Register our response voucher with the client - require.NoError(t, dt2.RegisterVoucherResultType(respVoucher)) + require.NoError(t, dt2.RegisterVoucherResultType(testutil.NewFakeDTType())) voucher := testutil.FakeDTType{Data: "applesauce"} chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), &voucher, rootCid, gsData.AllSelector) @@ -2162,19 +2141,9 @@ func TestMultipleMessagesInExtension(t *testing.T) { } } sv.VerifyExpectations(t) - srv.VerifyExpectations(t) gsData.VerifyFileTransferred(t, root, true) } -// completeRevalidator does not pause when sending the last voucher to confirm the deal is completed -type completeRevalidator struct { - *retrievalRevalidator -} - -func (r *completeRevalidator) OnComplete(chid datatransfer.ChannelID) (bool, datatransfer.VoucherResult, error) { - return true, r.finalVoucher, nil -} - func TestMultipleParallelTransfers(t *testing.T) { // Add more sizes here to trigger more transfers. @@ -2208,25 +2177,13 @@ func TestMultipleParallelTransfers(t *testing.T) { encodedFVR, err := encoding.Encode(finalVoucherResult) require.NoError(t, err) - sv := testutil.NewStubbedValidator() - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - // Stub in the validator so it returns that exact voucher when calling ValidatePull - // this validator will not pause transfer when accepting a transfer and will start - // sending blocks immediately - sv.StubResult(respVoucher) - - // no need for intermediary voucher results - voucherResults := []datatransfer.VoucherResult{} - - pausePoints := []uint64{} - srv := &retrievalRevalidator{ - testutil.NewStubbedRevalidator(), 0, 0, pausePoints, finalVoucherResult, voucherResults, + sv := &retrievalRevalidator{ + StubbedValidator: testutil.NewStubbedValidator(), + initialVoucherResult: respVoucher, } - srv.ExpectSuccessErrResume() - require.NoError(t, dt1.RegisterRevalidator(testutil.NewFakeDTType(), srv)) + require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - // Register our response voucher with the client - require.NoError(t, dt2.RegisterVoucherResultType(respVoucher)) + require.NoError(t, dt2.RegisterVoucherResultType(testutil.NewFakeDTType())) // for each size we create a new random DAG of the given size and try to retrieve it for _, size := range sizes { @@ -2290,6 +2247,9 @@ func TestMultipleParallelTransfers(t *testing.T) { if channelState.Status() == datatransfer.Completed { providerFinished <- struct{}{} } + if event.Code == datatransfer.BeginFinalizing { + dt1.SendVoucherResult(ctx, chid, finalVoucherResult) + } }) root, origBytes := LoadRandomData(ctx, t, gsData.DagService1, size) @@ -2325,7 +2285,6 @@ func TestMultipleParallelTransfers(t *testing.T) { } } sv.VerifyExpectations(t) - srv.VerifyExpectations(t) testutil.VerifyHasFile(gsData.Ctx, t, gsData.DagService2, root, origBytes) }) } diff --git a/impl/receiver.go b/impl/receiver.go index a33cd3a7..210027a0 100644 --- a/impl/receiver.go +++ b/impl/receiver.go @@ -43,7 +43,6 @@ func (r *receiver) receiveRequest(ctx context.Context, initiator peer.ID, incomi )) defer span.End() response, receiveErr := r.manager.OnRequestReceived(chid, incoming) - if receiveErr == datatransfer.ErrResume { chst, err := r.manager.channels.GetByID(ctx, chid) if err != nil { diff --git a/impl/receiving_requests.go b/impl/receiving_requests.go new file mode 100644 index 00000000..a223c4b1 --- /dev/null +++ b/impl/receiving_requests.go @@ -0,0 +1,352 @@ +package impl + +import ( + "context" + + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" + "github.com/libp2p/go-libp2p-core/peer" + "golang.org/x/xerrors" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/message/types" +) + +// this file contains methods for processing incoming request messages + +// receiveNewRequest handles an incoming new request message +func (m *manager) receiveNewRequest(chid datatransfer.ChannelID, incoming datatransfer.Request) (datatransfer.Response, error) { + log.Infof("channel %s: received new channel request from %s", chid, chid.Initiator) + + // process the new message, including validations + result, err := m.acceptRequest(chid, incoming) + + // generate a response message + msg, msgErr := message.ValidationResultResponse(types.NewMessage, incoming.TransferID(), result, err) + if msgErr != nil { + return nil, msgErr + } + + // return the response message and any errors + return msg, m.requestError(result, err, false) +} + +// acceptRequest performs processing (including validation) on a new incoming request +func (m *manager) acceptRequest(chid datatransfer.ChannelID, incoming datatransfer.Request) (datatransfer.ValidationResult, error) { + + // read the voucher and validate the request + stor, err := incoming.Selector() + if err != nil { + return datatransfer.ValidationResult{}, err + } + + voucher, err := m.decodeVoucher(incoming) + if err != nil { + return datatransfer.ValidationResult{}, err + } + + var validatorFunc func(datatransfer.ChannelID, peer.ID, datatransfer.Voucher, cid.Cid, ipld.Node) (datatransfer.ValidationResult, error) + processor, _ := m.validatedTypes.Processor(voucher.Type()) + validator := processor.(datatransfer.RequestValidator) + if incoming.IsPull() { + validatorFunc = validator.ValidatePull + } else { + validatorFunc = validator.ValidatePush + } + + result, err := validatorFunc(chid, chid.Initiator, voucher, incoming.BaseCid(), stor) + + // if an error occurred during validation or the request was not accepted, return + if err != nil || !result.Accepted { + return result, err + } + + // create the channel + var dataSender, dataReceiver peer.ID + if incoming.IsPull() { + dataSender = m.peerID + dataReceiver = chid.Initiator + } else { + dataSender = chid.Initiator + dataReceiver = m.peerID + } + + log.Infow("data-transfer request validated, will create & start tracking channel", "channelID", chid, "payloadCid", incoming.BaseCid()) + _, err = m.channels.CreateNew(m.peerID, incoming.TransferID(), incoming.BaseCid(), stor, voucher, chid.Initiator, dataSender, dataReceiver) + if err != nil { + log.Errorw("failed to create and start tracking channel", "channelID", chid, "err", err) + return result, err + } + + // record that the channel was accepted + log.Debugw("successfully created and started tracking channel", "channelID", chid) + if err := m.channels.Accept(chid); err != nil { + return result, err + } + + // read the channel state + chst, err := m.channels.GetByID(context.TODO(), chid) + if err != nil { + return datatransfer.ValidationResult{}, err + } + + // record validation events + if err := m.recordAcceptedValidationEvents(chst, result); err != nil { + return result, err + } + + // configure the transport + processor, has := m.transportConfigurers.Processor(voucher.Type()) + if has { + transportConfigurer := processor.(datatransfer.TransportConfigurer) + transportConfigurer(chid, voucher, m.transport) + } + m.dataTransferNetwork.Protect(chid.Initiator, chid.String()) + + return result, nil +} + +// receiveRestartRequest handles an incoming restart request message +func (m *manager) receiveRestartRequest(chid datatransfer.ChannelID, incoming datatransfer.Request) (datatransfer.Response, error) { + log.Infof("channel %s: received restart request", chid) + + // process the restart message, including validations + result, err := m.restartRequest(chid, incoming) + + // generate a response message + msg, msgErr := message.ValidationResultResponse(types.RestartMessage, incoming.TransferID(), result, err) + if msgErr != nil { + return nil, msgErr + } + + // return the response message and any errors + return msg, m.requestError(result, err, false) +} + +// restartRequest performs processing (including validation) on a incoming restart request +func (m *manager) restartRequest(chid datatransfer.ChannelID, + incoming datatransfer.Request) (datatransfer.ValidationResult, error) { + + // restart requests are invalid if we the initiator + // (the responder must send a "restart existing channel request") + initiator := chid.Initiator + if m.peerID == initiator { + return datatransfer.ValidationResult{}, xerrors.New("initiator cannot be manager peer for a restart request") + } + + // valide that the request parameters match the original request + // TODO: not sure this is needed -- the request parameters cannot change, + // so perhaps the solution is just to ignore them in the message + if err := m.validateRestartRequest(context.Background(), initiator, chid, incoming); err != nil { + return datatransfer.ValidationResult{}, xerrors.Errorf("restart request for channel %s failed validation: %w", chid, err) + } + + // read the channel state + chst, err := m.channels.GetByID(context.TODO(), chid) + if err != nil { + return datatransfer.ValidationResult{}, err + } + + // perform a revalidation against the last voucher + result, err := m.revalidate(chst) + + // if an error occurred during validation return + if err != nil { + return result, err + } + + // if the request is now rejected, error the channel + if !result.Accepted { + return result, m.recordRejectedValidationEvents(chid, result) + } + + // record the restart events + if err := m.channels.Restart(chid); err != nil { + return result, xerrors.Errorf("failed to restart channel %s: %w", chid, err) + } + + // record validation events + if err := m.recordAcceptedValidationEvents(chst, result); err != nil { + return result, err + } + + // configure the transport + voucher, err := m.decodeVoucher(incoming) + if err != nil { + return result, err + } + processor, has := m.transportConfigurers.Processor(voucher.Type()) + if has { + transportConfigurer := processor.(datatransfer.TransportConfigurer) + transportConfigurer(chid, voucher, m.transport) + } + m.dataTransferNetwork.Protect(initiator, chid.String()) + return result, nil +} + +// processUpdateVoucher handles an incoming request message with an updated voucher +func (m *manager) processUpdateVoucher(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { + + // process the voucher update request, including validations + chst, result, err := m.revalidateRequest(chid, request) + + // generate a response message + messageType := types.VoucherResultMessage + if chst.Status() == datatransfer.Finalizing { + messageType = types.CompleteMessage + } + response, msgErr := message.ValidationResultResponse(messageType, chst.TransferID(), result, err) + if msgErr != nil { + return nil, msgErr + } + + // return the response message and any errors + return response, m.requestError(result, err, true) +} + +// revalidateRequest performs processing (including validation) on a incoming request with an updated voucher +func (m *manager) revalidateRequest(chid datatransfer.ChannelID, + incoming datatransfer.Request) (datatransfer.ChannelState, datatransfer.ValidationResult, error) { + + // decode the voucher and save it on the channel + vouch, err := m.decodeVoucher(incoming) + if err != nil { + return nil, datatransfer.ValidationResult{}, err + } + err = m.channels.NewVoucher(chid, vouch) + if err != nil { + return nil, datatransfer.ValidationResult{}, err + } + + // read the channel state with the saved voucher + chst, err := m.channels.GetByID(context.TODO(), chid) + if err != nil { + return nil, datatransfer.ValidationResult{}, err + } + + // perform revalidation based on this updated voucher + result, err := m.revalidate(chst) + + // if an error occurred during validation return + if err != nil { + return chst, result, err + } + + // if the request is now rejected, error the channel + if !result.Accepted { + return chst, result, m.recordRejectedValidationEvents(chid, result) + } + + // record validation events and return + return chst, result, m.recordAcceptedValidationEvents(chst, result) + +} + +// receiveUpdateRequest handles an incoming request message with an updated voucher +func (m *manager) receiveUpdateRequest(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { + + if request.IsPaused() { + return nil, m.pauseOther(chid) + } + + err := m.resumeOther(chid) + if err != nil { + return nil, err + } + chst, err := m.channels.GetByID(context.TODO(), chid) + if err != nil { + return nil, err + } + if chst.Status() == datatransfer.ResponderPaused || + chst.Status() == datatransfer.ResponderFinalizing { + return nil, datatransfer.ErrPause + } + return nil, nil +} + +// requestError generates an error message for the transport, adding +// ErrPause / ErrResume based off the validation result +// TODO: get away from using ErrPause/ErrResume to indicate pause resume, +// which would remove the need for most of this method +func (m *manager) requestError(result datatransfer.ValidationResult, resultErr error, handleResumes bool) error { + if resultErr != nil { + return resultErr + } + if !result.Accepted { + return datatransfer.ErrRejected + } + if result.LeaveRequestPaused { + return datatransfer.ErrPause + } + if handleResumes { + return datatransfer.ErrResume + } + return nil +} + +// recordRejectedValidationEvents sends changes based on an reject validation to the state machine +func (m *manager) recordRejectedValidationEvents(chid datatransfer.ChannelID, result datatransfer.ValidationResult) error { + if result.VoucherResult != nil { + if err := m.channels.NewVoucherResult(chid, result.VoucherResult); err != nil { + return err + } + } + + return m.channels.Error(chid, datatransfer.ErrRejected) +} + +// recordAcceptedValidationEvents sends changes based on an accepted validation to the state machine +func (m *manager) recordAcceptedValidationEvents(chst datatransfer.ChannelState, result datatransfer.ValidationResult) error { + chid := chst.ChannelID() + + // pause or resume the request as neccesary + if result.LeaveRequestPaused { + if !chst.Status().IsResponderPaused() { + err := m.channels.PauseResponder(chid) + if err != nil { + return err + } + } + } else { + if chst.Status().IsResponderPaused() { + err := m.channels.ResumeResponder(chid) + if err != nil { + return err + } + } + } + + // record the voucher result if present + if result.VoucherResult != nil { + err := m.channels.NewVoucherResult(chid, result.VoucherResult) + if err != nil { + return err + } + } + + // record the change in data limit if different + if result.DataLimit != chst.DataLimit() { + err := m.channels.SetDataLimit(chid, result.DataLimit) + if err != nil { + return err + } + } + + // record the finalization state if different + if result.RequiresFinalization != chst.RequiresFinalization() { + err := m.channels.SetRequiresFinalization(chid, result.RequiresFinalization) + if err != nil { + return err + } + } + return nil +} + +// revalidate looks up the appropriate validator based on the last voucher in a channel +func (m *manager) revalidate(chst datatransfer.ChannelState) (datatransfer.ValidationResult, error) { + processor, _ := m.validatedTypes.Processor(chst.LastVoucher().Type()) + validator := processor.(datatransfer.RequestValidator) + + return validator.Revalidate(chst.ChannelID(), chst) +} diff --git a/impl/responding_test.go b/impl/responding_test.go index 0625135e..3ccac8e1 100644 --- a/impl/responding_test.go +++ b/impl/responding_test.go @@ -28,16 +28,19 @@ func TestDataTransferResponding(t *testing.T) { // create network ctx := context.Background() testCases := map[string]struct { - expectedEvents []datatransfer.EventCode - configureValidator func(sv *testutil.StubbedValidator) - configureRevalidator func(sv *testutil.StubbedRevalidator) - verify func(t *testing.T, h *receiverHarness) + expectedEvents []datatransfer.EventCode + configureValidator func(sv *testutil.StubbedValidator) + verify func(t *testing.T, h *receiverHarness) }{ "new push request validates": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.NewVoucherResult, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + datatransfer.NewVoucherResult, + }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -64,13 +67,34 @@ func TestDataTransferResponding(t *testing.T) { require.False(t, response.IsCancel()) require.False(t, response.IsPaused()) require.True(t, response.IsNew()) - require.True(t, response.IsVoucherResult()) + require.True(t, response.IsValidationResult()) + }, + }, + "new push request rejects": { + configureValidator: func(sv *testutil.StubbedValidator) { + sv.ExpectSuccessPush() + sv.StubResult(datatransfer.ValidationResult{Accepted: false, VoucherResult: testutil.NewFakeDTType()}) + }, + verify: func(t *testing.T, h *receiverHarness) { + h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + require.Len(t, h.network.SentMessages, 1) + responseMessage := h.network.SentMessages[0].Message + require.False(t, responseMessage.IsRequest()) + response, ok := responseMessage.(datatransfer.Response) + require.True(t, ok) + require.False(t, response.Accepted()) + require.Equal(t, response.TransferID(), h.id) + require.False(t, response.IsUpdate()) + require.False(t, response.IsCancel()) + require.False(t, response.IsPaused()) + require.True(t, response.IsNew()) + require.True(t, response.IsValidationResult()) }, }, "new push request errors": { configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectErrorPush() - sv.StubResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -85,13 +109,13 @@ func TestDataTransferResponding(t *testing.T) { require.False(t, response.IsCancel()) require.False(t, response.IsPaused()) require.True(t, response.IsNew()) - require.True(t, response.IsVoucherResult()) + require.True(t, response.IsValidationResult()) }, }, "new push request pauses": { configureValidator: func(sv *testutil.StubbedValidator) { - sv.ExpectPausePush() - sv.StubResult(testutil.NewFakeDTType()) + sv.ExpectSuccessPush() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, LeaveRequestPaused: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -111,15 +135,19 @@ func TestDataTransferResponding(t *testing.T) { require.False(t, response.IsCancel()) require.True(t, response.IsPaused()) require.True(t, response.IsNew()) - require.True(t, response.IsVoucherResult()) + require.True(t, response.IsValidationResult()) require.Len(t, h.transport.PausedChannels, 1) require.Equal(t, channelID(h.id, h.peers), h.transport.PausedChannels[0]) }, }, "new pull request validates": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) }, verify: func(t *testing.T, h *receiverHarness) { response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) @@ -137,7 +165,24 @@ func TestDataTransferResponding(t *testing.T) { require.False(t, response.IsCancel()) require.False(t, response.IsPaused()) require.True(t, response.IsNew()) - require.True(t, response.IsVoucherResult()) + require.True(t, response.IsValidationResult()) + }, + }, + "new pull request rejects": { + configureValidator: func(sv *testutil.StubbedValidator) { + sv.ExpectSuccessPull() + sv.StubResult(datatransfer.ValidationResult{Accepted: false}) + }, + verify: func(t *testing.T, h *receiverHarness) { + response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) + require.EqualError(t, err, datatransfer.ErrRejected.Error()) + require.False(t, response.Accepted()) + require.Equal(t, response.TransferID(), h.id) + require.False(t, response.IsUpdate()) + require.False(t, response.IsCancel()) + require.False(t, response.IsPaused()) + require.True(t, response.IsNew()) + require.True(t, response.IsValidationResult()) }, }, "new pull request errors": { @@ -153,12 +198,13 @@ func TestDataTransferResponding(t *testing.T) { require.False(t, response.IsCancel()) require.False(t, response.IsPaused()) require.True(t, response.IsNew()) - require.True(t, response.IsVoucherResult()) + require.True(t, response.IsValidationResult()) }, }, "new pull request pauses": { configureValidator: func(sv *testutil.StubbedValidator) { - sv.ExpectPausePull() + sv.ExpectSuccessPull() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, LeaveRequestPaused: true}) }, verify: func(t *testing.T, h *receiverHarness) { response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) @@ -170,11 +216,39 @@ func TestDataTransferResponding(t *testing.T) { require.False(t, response.IsCancel()) require.True(t, response.IsPaused()) require.True(t, response.IsNew()) - require.True(t, response.IsVoucherResult()) + require.True(t, response.IsValidationResult()) require.True(t, response.EmptyVoucherResult()) }, }, + "send voucher results from responder succeeds, push request": { + configureValidator: func(sv *testutil.StubbedValidator) { + sv.ExpectSuccessPush() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) + }, + verify: func(t *testing.T, h *receiverHarness) { + h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + newVoucherResult := testutil.NewFakeDTType() + err := h.dt.SendVoucherResult(h.ctx, channelID(h.id, h.peers), newVoucherResult) + require.NoError(t, err) + }, + }, + "send voucher results from responder succeeds, pull request": { + configureValidator: func(sv *testutil.StubbedValidator) { + sv.ExpectSuccessPull() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) + }, + verify: func(t *testing.T, h *receiverHarness) { + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) + newVoucherResult := testutil.NewFakeDTType() + err := h.dt.SendVoucherResult(h.ctx, channelID(h.id, h.peers), newVoucherResult) + require.NoError(t, err) + }, + }, "send vouchers from responder fails, push request": { + configureValidator: func(sv *testutil.StubbedValidator) { + sv.ExpectSuccessPush() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) + }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) newVoucher := testutil.NewFakeDTType() @@ -183,6 +257,10 @@ func TestDataTransferResponding(t *testing.T) { }, }, "send vouchers from responder fails, pull request": { + configureValidator: func(sv *testutil.StubbedValidator) { + sv.ExpectSuccessPull() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) + }, verify: func(t *testing.T, h *receiverHarness) { _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) newVoucher := testutil.NewFakeDTType() @@ -191,15 +269,18 @@ func TestDataTransferResponding(t *testing.T) { }, }, "receive voucher": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.NewVoucherResult, datatransfer.Accept, datatransfer.NewVoucher, datatransfer.ResumeResponder}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + datatransfer.NewVoucherResult, + datatransfer.NewVoucher, + }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(testutil.NewFakeDTType()) - }, - configureRevalidator: func(sv *testutil.StubbedRevalidator) { - sv.ExpectSuccessErrResume() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + sv.ExpectSuccessRevalidation() + sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) }, - verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) @@ -209,13 +290,13 @@ func TestDataTransferResponding(t *testing.T) { "receive pause, unpause": { expectedEvents: []datatransfer.EventCode{ datatransfer.Open, - datatransfer.NewVoucherResult, datatransfer.Accept, + datatransfer.NewVoucherResult, datatransfer.PauseInitiator, datatransfer.ResumeInitiator}, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -228,14 +309,14 @@ func TestDataTransferResponding(t *testing.T) { "receive pause, set pause local, receive unpause": { expectedEvents: []datatransfer.EventCode{ datatransfer.Open, - datatransfer.NewVoucherResult, datatransfer.Accept, + datatransfer.NewVoucherResult, datatransfer.PauseInitiator, datatransfer.PauseResponder, datatransfer.ResumeInitiator}, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -248,10 +329,16 @@ func TestDataTransferResponding(t *testing.T) { }, }, "receive cancel": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.NewVoucherResult, datatransfer.Accept, datatransfer.Cancel, datatransfer.CleanupComplete}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + datatransfer.NewVoucherResult, + datatransfer.Cancel, + datatransfer.CleanupComplete, + }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -264,25 +351,22 @@ func TestDataTransferResponding(t *testing.T) { "validate and revalidate successfully, push": { expectedEvents: []datatransfer.EventCode{ datatransfer.Open, - datatransfer.NewVoucherResult, datatransfer.Accept, + datatransfer.NewVoucherResult, + datatransfer.SetDataLimit, datatransfer.DataReceivedProgress, datatransfer.DataReceived, - datatransfer.NewVoucherResult, - datatransfer.PauseResponder, + datatransfer.DataLimitExceeded, datatransfer.NewVoucher, - datatransfer.NewVoucherResult, datatransfer.ResumeResponder, + datatransfer.NewVoucherResult, + datatransfer.SetDataLimit, }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(testutil.NewFakeDTType()) - }, - configureRevalidator: func(srv *testutil.StubbedRevalidator) { - srv.ExpectPausePushCheck() - srv.StubRevalidationResult(testutil.NewFakeDTType()) - srv.ExpectSuccessErrResume() - srv.StubCheckResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: testutil.NewFakeDTType()}) + sv.ExpectSuccessRevalidation() + sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -291,13 +375,11 @@ func TestDataTransferResponding(t *testing.T) { require.Len(t, h.network.SentMessages, 1) response, ok := h.network.SentMessages[0].Message.(datatransfer.Response) require.True(t, ok) - require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) - require.False(t, response.IsUpdate()) + require.True(t, response.IsUpdate()) require.False(t, response.IsCancel()) require.True(t, response.IsPaused()) - require.True(t, response.IsVoucherResult()) - require.False(t, response.EmptyVoucherResult()) + require.False(t, response.IsValidationResult()) response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) require.EqualError(t, err, datatransfer.ErrResume.Error()) require.True(t, response.Accepted()) @@ -305,64 +387,68 @@ func TestDataTransferResponding(t *testing.T) { require.False(t, response.IsUpdate()) require.False(t, response.IsCancel()) require.False(t, response.IsPaused()) - require.True(t, response.IsVoucherResult()) + require.True(t, response.IsValidationResult()) require.False(t, response.EmptyVoucherResult()) }, }, - "validate and revalidate with err": { + "validate and revalidate with rejection on second voucher": { expectedEvents: []datatransfer.EventCode{ datatransfer.Open, - datatransfer.NewVoucherResult, datatransfer.Accept, + datatransfer.NewVoucherResult, + datatransfer.SetDataLimit, datatransfer.DataReceivedProgress, datatransfer.DataReceived, + datatransfer.DataLimitExceeded, + datatransfer.NewVoucher, datatransfer.NewVoucherResult, + datatransfer.Error, + datatransfer.CleanupComplete, }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(testutil.NewFakeDTType()) - }, - configureRevalidator: func(srv *testutil.StubbedRevalidator) { - srv.ExpectErrorPushCheck() - srv.StubRevalidationResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: testutil.NewFakeDTType()}) + sv.ExpectSuccessRevalidation() + sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: false, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) err := h.transport.EventHandler.OnDataReceived(channelID(h.id, h.peers), cidlink.Link{Cid: testutil.GenerateCids(1)[0]}, 12345, 1, true) - require.Error(t, err) + require.EqualError(t, err, datatransfer.ErrPause.Error()) require.Len(t, h.network.SentMessages, 1) response, ok := h.network.SentMessages[0].Message.(datatransfer.Response) require.True(t, ok) + require.Equal(t, response.TransferID(), h.id) + require.True(t, response.IsUpdate()) + require.False(t, response.IsCancel()) + require.True(t, response.IsPaused()) + require.False(t, response.IsValidationResult()) + response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) + require.EqualError(t, err, datatransfer.ErrRejected.Error()) require.False(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) require.False(t, response.IsCancel()) require.False(t, response.IsPaused()) - require.True(t, response.IsVoucherResult()) + require.True(t, response.IsValidationResult()) require.False(t, response.EmptyVoucherResult()) }, }, - "validate and revalidate with err with second voucher": { + "validate and revalidate with validation error on second voucher": { expectedEvents: []datatransfer.EventCode{ datatransfer.Open, - datatransfer.NewVoucherResult, datatransfer.Accept, + datatransfer.NewVoucherResult, + datatransfer.SetDataLimit, datatransfer.DataReceivedProgress, datatransfer.DataReceived, - datatransfer.NewVoucherResult, - datatransfer.PauseResponder, + datatransfer.DataLimitExceeded, datatransfer.NewVoucher, - datatransfer.NewVoucherResult, }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(testutil.NewFakeDTType()) - }, - configureRevalidator: func(srv *testutil.StubbedRevalidator) { - srv.ExpectPausePushCheck() - srv.StubRevalidationResult(testutil.NewFakeDTType()) - srv.ExpectErrorRevalidation() - srv.StubCheckResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: testutil.NewFakeDTType()}) + sv.ExpectErrorRevalidation() }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -371,46 +457,41 @@ func TestDataTransferResponding(t *testing.T) { require.Len(t, h.network.SentMessages, 1) response, ok := h.network.SentMessages[0].Message.(datatransfer.Response) require.True(t, ok) - require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) - require.False(t, response.IsUpdate()) + require.True(t, response.IsUpdate()) require.False(t, response.IsCancel()) require.True(t, response.IsPaused()) - require.True(t, response.IsVoucherResult()) - require.False(t, response.EmptyVoucherResult()) + require.False(t, response.IsValidationResult()) response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) - require.Error(t, err) + require.EqualError(t, err, "something went wrong") require.False(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) require.False(t, response.IsCancel()) require.False(t, response.IsPaused()) - require.True(t, response.IsVoucherResult()) - require.False(t, response.EmptyVoucherResult()) + require.True(t, response.IsValidationResult()) + require.True(t, response.EmptyVoucherResult()) }, }, "validate and revalidate successfully, pull": { expectedEvents: []datatransfer.EventCode{ datatransfer.Open, - datatransfer.NewVoucherResult, datatransfer.Accept, + datatransfer.NewVoucherResult, + datatransfer.SetDataLimit, datatransfer.DataQueuedProgress, datatransfer.DataQueued, - datatransfer.NewVoucherResult, - datatransfer.PauseResponder, + datatransfer.DataLimitExceeded, datatransfer.NewVoucher, - datatransfer.NewVoucherResult, datatransfer.ResumeResponder, + datatransfer.NewVoucherResult, + datatransfer.SetDataLimit, }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(testutil.NewFakeDTType()) - }, - configureRevalidator: func(srv *testutil.StubbedRevalidator) { - srv.ExpectPausePullCheck() - srv.StubRevalidationResult(testutil.NewFakeDTType()) - srv.ExpectSuccessErrResume() - srv.StubCheckResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: testutil.NewFakeDTType()}) + sv.ExpectSuccessRevalidation() + sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) @@ -422,13 +503,11 @@ func TestDataTransferResponding(t *testing.T) { require.EqualError(t, err, datatransfer.ErrPause.Error()) response, ok := msg.(datatransfer.Response) require.True(t, ok) - require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) - require.False(t, response.IsUpdate()) + require.True(t, response.IsUpdate()) require.False(t, response.IsCancel()) require.True(t, response.IsPaused()) - require.True(t, response.IsVoucherResult()) - require.False(t, response.EmptyVoucherResult()) + require.False(t, response.IsValidationResult()) response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) require.EqualError(t, err, datatransfer.ErrResume.Error()) require.True(t, response.Accepted()) @@ -436,29 +515,28 @@ func TestDataTransferResponding(t *testing.T) { require.False(t, response.IsUpdate()) require.False(t, response.IsCancel()) require.False(t, response.IsPaused()) - require.True(t, response.IsVoucherResult()) + require.True(t, response.IsValidationResult()) require.False(t, response.EmptyVoucherResult()) }, }, "validated, finalize, and complete successfully": { expectedEvents: []datatransfer.EventCode{ datatransfer.Open, - datatransfer.NewVoucherResult, datatransfer.Accept, datatransfer.NewVoucherResult, + datatransfer.SetRequiresFinalization, datatransfer.BeginFinalizing, datatransfer.NewVoucher, datatransfer.ResumeResponder, + datatransfer.NewVoucherResult, + datatransfer.SetRequiresFinalization, datatransfer.CleanupComplete, }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(testutil.NewFakeDTType()) - }, - configureRevalidator: func(srv *testutil.StubbedRevalidator) { - srv.ExpectPauseComplete() - srv.StubRevalidationResult(testutil.NewFakeDTType()) - srv.ExpectSuccessErrResume() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, RequiresFinalization: true, VoucherResult: testutil.NewFakeDTType()}) + sv.ExpectSuccessRevalidation() + sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) @@ -472,29 +550,27 @@ func TestDataTransferResponding(t *testing.T) { require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) require.False(t, response.IsCancel()) + require.True(t, response.IsComplete()) require.True(t, response.IsPaused()) - require.True(t, response.IsVoucherResult()) - require.False(t, response.EmptyVoucherResult()) + require.True(t, response.IsValidationResult()) + require.True(t, response.EmptyVoucherResult()) response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) require.EqualError(t, err, datatransfer.ErrResume.Error()) require.Equal(t, response.TransferID(), h.id) - require.True(t, response.IsVoucherResult()) + require.True(t, response.IsValidationResult()) require.False(t, response.IsPaused()) }, }, "validated, incomplete response": { expectedEvents: []datatransfer.EventCode{ datatransfer.Open, - datatransfer.NewVoucherResult, datatransfer.Accept, datatransfer.Error, datatransfer.CleanupComplete, }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(testutil.NewFakeDTType()) - }, - configureRevalidator: func(srv *testutil.StubbedRevalidator) { + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) }, verify: func(t *testing.T, h *receiverHarness) { _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) @@ -504,10 +580,13 @@ func TestDataTransferResponding(t *testing.T) { }, }, "new push request, customized transport": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.NewVoucherResult, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) }, verify: func(t *testing.T, h *receiverHarness) { err := h.dt.RegisterTransportConfigurer(h.voucher, func(channelID datatransfer.ChannelID, voucher datatransfer.Voucher, transport datatransfer.Transport) { @@ -526,9 +605,13 @@ func TestDataTransferResponding(t *testing.T) { }, }, "new pull request, customized transport": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) }, verify: func(t *testing.T, h *receiverHarness) { err := h.dt.RegisterTransportConfigurer(h.voucher, func(channelID datatransfer.ChannelID, voucher datatransfer.Voucher, transport datatransfer.Transport) { @@ -588,15 +671,9 @@ func TestDataTransferResponding(t *testing.T) { verify.configureValidator(h.sv) } require.NoError(t, h.dt.RegisterVoucherType(h.voucher, h.sv)) - h.srv = testutil.NewStubbedRevalidator() - if verify.configureRevalidator != nil { - verify.configureRevalidator(h.srv) - } - err = h.dt.RegisterRevalidator(updateVoucher, h.srv) require.NoError(t, err) verify.verify(t, h) h.sv.VerifyExpectations(t) - h.srv.VerifyExpectations(t) ev.verify(ctx, t) }) } @@ -606,10 +683,9 @@ func TestDataTransferRestartResponding(t *testing.T) { // create network ctx := context.Background() testCases := map[string]struct { - expectedEvents []datatransfer.EventCode - configureValidator func(sv *testutil.StubbedValidator) - configureRevalidator func(sv *testutil.StubbedRevalidator) - verify func(t *testing.T, h *receiverHarness) + expectedEvents []datatransfer.EventCode + configureValidator func(sv *testutil.StubbedValidator) + verify func(t *testing.T, h *receiverHarness) }{ "receiving a pull restart response": { expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.Restart, datatransfer.ResumeResponder}, @@ -625,12 +701,22 @@ func TestDataTransferRestartResponding(t *testing.T) { }, }, "receiving a push restart request validates and opens a channel for pull": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.NewVoucherResult, datatransfer.Accept, - datatransfer.DataReceivedProgress, datatransfer.DataReceived, datatransfer.DataReceivedProgress, datatransfer.DataReceived, - datatransfer.NewVoucherResult, datatransfer.Restart}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + datatransfer.NewVoucherResult, + datatransfer.DataReceivedProgress, + datatransfer.DataReceived, + datatransfer.DataReceivedProgress, + datatransfer.DataReceived, + datatransfer.Restart, + datatransfer.NewVoucherResult, + }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + sv.ExpectSuccessRevalidation() + sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming push @@ -652,7 +738,7 @@ func TestDataTransferRestartResponding(t *testing.T) { h.baseCid, h.stor) require.NoError(t, err) h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], req) - require.Len(t, h.sv.ValidationsReceived, 2) + require.Len(t, h.sv.RevalidationsReceived, 1) require.Len(t, h.transport.OpenedChannels, 2) require.Len(t, h.network.SentMessages, 0) @@ -673,23 +759,25 @@ func TestDataTransferRestartResponding(t *testing.T) { require.False(t, response.IsCancel()) require.False(t, response.IsPaused()) require.False(t, response.IsNew()) - require.True(t, response.IsVoucherResult()) + require.True(t, response.IsValidationResult()) - // validate the voucher that is validated the second time - vmsg := h.sv.ValidationsReceived[1] - require.Equal(t, h.voucher, vmsg.Voucher) - require.False(t, vmsg.IsPull) - require.Equal(t, h.stor, vmsg.Selector) - require.Equal(t, h.baseCid, vmsg.BaseCid) - require.Equal(t, h.peers[1], vmsg.Other) + vmsg := h.sv.RevalidationsReceived[0] + require.Equal(t, channelID(h.id, h.peers), vmsg.ChannelID) }, }, "receiving a pull restart request validates and sends a success response": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.NewVoucherResult, datatransfer.Accept, - datatransfer.NewVoucherResult, datatransfer.Restart}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + datatransfer.NewVoucherResult, + datatransfer.Restart, + datatransfer.NewVoucherResult, + }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + sv.ExpectSuccessRevalidation() + sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull @@ -704,7 +792,7 @@ func TestDataTransferRestartResponding(t *testing.T) { require.NoError(t, err) response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), restartReq) require.NoError(t, err) - require.Len(t, h.sv.ValidationsReceived, 2) + require.Len(t, h.sv.RevalidationsReceived, 1) require.Len(t, h.transport.OpenedChannels, 0) require.Len(t, h.network.SentMessages, 0) @@ -716,22 +804,21 @@ func TestDataTransferRestartResponding(t *testing.T) { require.False(t, response.IsCancel()) require.False(t, response.IsPaused()) require.False(t, response.IsNew()) - require.True(t, response.IsVoucherResult()) + require.True(t, response.IsValidationResult()) - // validate the voucher that is validated the second time - vmsg := h.sv.ValidationsReceived[1] - require.Equal(t, h.voucher, vmsg.Voucher) - require.True(t, vmsg.IsPull) - require.Equal(t, h.stor, vmsg.Selector) - require.Equal(t, h.baseCid, vmsg.BaseCid) - require.Equal(t, h.peers[1], vmsg.Other) + vmsg := h.sv.RevalidationsReceived[0] + require.Equal(t, channelID(h.id, h.peers), vmsg.ChannelID) }, }, "restart request fails if channel does not exist": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.NewVoucherResult, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + datatransfer.NewVoucherResult, + }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull @@ -751,11 +838,21 @@ func TestDataTransferRestartResponding(t *testing.T) { }, }, "restart request fails if voucher validation fails": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.NewVoucherResult, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + datatransfer.NewVoucherResult, + datatransfer.Error, + datatransfer.CleanupComplete, + }, + configureValidator: func(sv *testutil.StubbedValidator) { + sv.ExpectSuccessPull() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + sv.ExpectSuccessRevalidation() + sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: false}) + }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull - h.sv.ExpectSuccessPull() - h.sv.StubResult(testutil.NewFakeDTType()) _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) require.NoError(t, err) require.Len(t, h.sv.ValidationsReceived, 1) @@ -767,14 +864,18 @@ func TestDataTransferRestartResponding(t *testing.T) { restartReq, err := message.NewRequest(h.id, true, true, h.voucher.Type(), h.voucher, h.baseCid, h.stor) require.NoError(t, err) _, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), restartReq) - require.EqualError(t, err, "failed to validate voucher: something went wrong") + require.EqualError(t, err, datatransfer.ErrRejected.Error()) }, }, "restart request fails if base cid does not match": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.NewVoucherResult, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + datatransfer.NewVoucherResult, + }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull @@ -794,10 +895,14 @@ func TestDataTransferRestartResponding(t *testing.T) { }, }, "restart request fails if voucher type is not decodable": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.NewVoucherResult, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + datatransfer.NewVoucherResult, + }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull @@ -817,10 +922,14 @@ func TestDataTransferRestartResponding(t *testing.T) { }, }, "restart request fails if voucher does not match": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.NewVoucherResult, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + datatransfer.NewVoucherResult, + }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(testutil.NewFakeDTType()) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull @@ -841,7 +950,13 @@ func TestDataTransferRestartResponding(t *testing.T) { }, }, "ReceiveRestartExistingChannelRequest: Reopen Pull Channel": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.DataReceivedProgress, datatransfer.DataReceived, datatransfer.DataReceivedProgress, datatransfer.DataReceived}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.DataReceivedProgress, + datatransfer.DataReceived, + datatransfer.DataReceivedProgress, + datatransfer.DataReceived, + }, configureValidator: func(sv *testutil.StubbedValidator) { }, verify: func(t *testing.T, h *receiverHarness) { @@ -892,7 +1007,9 @@ func TestDataTransferRestartResponding(t *testing.T) { }, }, "ReceiveRestartExistingChannelRequest: Resend Push Request": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + }, configureValidator: func(sv *testutil.StubbedValidator) { }, verify: func(t *testing.T, h *receiverHarness) { @@ -931,8 +1048,13 @@ func TestDataTransferRestartResponding(t *testing.T) { }, }, "ReceiveRestartExistingChannelRequest: errors if peer is not the initiator": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.Accept}, + expectedEvents: []datatransfer.EventCode{ + datatransfer.Open, + datatransfer.Accept, + }, configureValidator: func(sv *testutil.StubbedValidator) { + sv.ExpectSuccessPush() + sv.StubResult(datatransfer.ValidationResult{Accepted: true}) }, verify: func(t *testing.T, h *receiverHarness) { // create an incoming push first @@ -1022,7 +1144,6 @@ type receiverHarness struct { network *testutil.FakeNetwork transport *testutil.FakeTransport sv *testutil.StubbedValidator - srv *testutil.StubbedRevalidator ds datastore.Batching dt datatransfer.Manager stor ipld.Node diff --git a/impl/restart.go b/impl/restart.go index fa1f79ec..536617a8 100644 --- a/impl/restart.go +++ b/impl/restart.go @@ -32,10 +32,15 @@ const ( ) func (m *manager) restartManagerPeerReceivePush(ctx context.Context, channel datatransfer.ChannelState) error { - if err := m.validateRestartVoucher(channel, false); err != nil { + result, err := m.revalidate(channel) + if err != nil { return xerrors.Errorf("failed to restart channel, validation error: %w", err) } + if !result.Accepted { + return datatransfer.ErrRejected + } + // send a libp2p message to the other peer asking to send a "restart push request" req := message.RestartExistingChannelRequest(channel.ChannelID()) @@ -47,10 +52,15 @@ func (m *manager) restartManagerPeerReceivePush(ctx context.Context, channel dat } func (m *manager) restartManagerPeerReceivePull(ctx context.Context, channel datatransfer.ChannelState) error { - if err := m.validateRestartVoucher(channel, true); err != nil { + result, err := m.revalidate(channel) + if err != nil { return xerrors.Errorf("failed to restart channel, validation error: %w", err) } + if !result.Accepted { + return datatransfer.ErrRejected + } + req := message.RestartExistingChannelRequest(channel.ChannelID()) // send a libp2p message to the other peer asking to send a "restart pull request" @@ -61,25 +71,6 @@ func (m *manager) restartManagerPeerReceivePull(ctx context.Context, channel dat return nil } -func (m *manager) validateRestartVoucher(channel datatransfer.ChannelState, isPull bool) error { - // re-validate the original voucher received for safety - chid := channel.ChannelID() - - // recreate the request that would have led to this pull channel being created for validation - req, err := message.NewRequest(chid.ID, false, isPull, channel.Voucher().Type(), channel.Voucher(), - channel.BaseCID(), channel.Selector()) - if err != nil { - return err - } - - // revalidate the voucher by reconstructing the request that would have led to the creation of this channel - if _, _, err := m.validateVoucher(true, chid, channel.OtherPeer(), req, isPull, channel.BaseCID(), channel.Selector()); err != nil { - return err - } - - return nil -} - func (m *manager) openPushRestartChannel(ctx context.Context, channel datatransfer.ChannelState) error { selector := channel.Selector() voucher := channel.Voucher() @@ -173,7 +164,7 @@ func (m *manager) validateRestartRequest(ctx context.Context, otherPeer peer.ID, } // vouchers should match - reqVoucher, err := m.decodeVoucher(req, m.validatedTypes) + reqVoucher, err := m.decodeVoucher(req) if err != nil { return xerrors.Errorf("failed to decode request voucher: %w", err) } diff --git a/impl/restart_integration_test.go b/impl/restart_integration_test.go index 9b06a013..b1f0f4da 100644 --- a/impl/restart_integration_test.go +++ b/impl/restart_integration_test.go @@ -134,6 +134,10 @@ func TestRestartPush(t *testing.T) { // START DATA TRANSFER INSTANCES rh.sv.ExpectSuccessPush() + rh.sv.StubResult(datatransfer.ValidationResult{Accepted: true}) + rh.sv.ExpectSuccessRevalidation() + rh.sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) + testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt1) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt2) @@ -378,6 +382,10 @@ func TestRestartPull(t *testing.T) { // START DATA TRANSFER INSTANCES rh.sv.ExpectSuccessPull() + rh.sv.StubResult(datatransfer.ValidationResult{Accepted: true}) + rh.sv.ExpectSuccessRevalidation() + rh.sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) + testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt1) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt2) diff --git a/impl/utils.go b/impl/utils.go index d02a4442..4c748acc 100644 --- a/impl/utils.go +++ b/impl/utils.go @@ -10,7 +10,6 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/message" - "github.com/filecoin-project/go-data-transfer/v2/registry" ) type statusList []datatransfer.Status @@ -37,33 +36,6 @@ func (m *manager) newRequest(ctx context.Context, selector ipld.Node, isPull boo return message.NewRequest(tid, false, isPull, voucher.Type(), voucher, baseCid, selector) } -func (m *manager) response(isRestart bool, isNew bool, err error, tid datatransfer.TransferID, voucherResult datatransfer.VoucherResult) (datatransfer.Response, error) { - isAccepted := err == nil || err == datatransfer.ErrPause || err == datatransfer.ErrResume - isPaused := err == datatransfer.ErrPause - resultType := datatransfer.EmptyTypeIdentifier - if voucherResult != nil { - resultType = voucherResult.Type() - } - if isRestart { - return message.RestartResponse(tid, isAccepted, isPaused, resultType, voucherResult) - } - - if isNew { - return message.NewResponse(tid, isAccepted, isPaused, resultType, voucherResult) - } - return message.VoucherResultResponse(tid, isAccepted, isPaused, resultType, voucherResult) -} - -func (m *manager) completeResponse(err error, tid datatransfer.TransferID, voucherResult datatransfer.VoucherResult) (datatransfer.Response, error) { - isAccepted := err == nil || err == datatransfer.ErrPause || err == datatransfer.ErrResume - isPaused := err == datatransfer.ErrPause - resultType := datatransfer.EmptyTypeIdentifier - if voucherResult != nil { - resultType = voucherResult.Type() - } - return message.CompleteResponse(tid, isAccepted, isPaused, resultType, voucherResult) -} - func (m *manager) resume(chid datatransfer.ChannelID) error { if chid.Initiator == m.peerID { return m.channels.ResumeInitiator(chid) @@ -126,9 +98,9 @@ func (m *manager) decodeVoucherResult(response datatransfer.Response) (datatrans return encodable.(datatransfer.Registerable), nil } -func (m *manager) decodeVoucher(request datatransfer.Request, registry *registry.Registry) (datatransfer.Voucher, error) { +func (m *manager) decodeVoucher(request datatransfer.Request) (datatransfer.Voucher, error) { vtypStr := datatransfer.TypeIdentifier(request.VoucherType()) - decoder, has := registry.Decoder(vtypStr) + decoder, has := m.validatedTypes.Decoder(vtypStr) if !has { return nil, xerrors.Errorf("unknown voucher type: %s", vtypStr) } diff --git a/manager.go b/manager.go index 9d540abe..61a26e21 100644 --- a/manager.go +++ b/manager.go @@ -8,57 +8,58 @@ import ( "github.com/libp2p/go-libp2p-core/peer" ) +// ValidationResult describes the result of validating a voucher +type ValidationResult struct { + // Accepted indicates whether the request was accepted. If a request is not + // accepted, the request fails. This is true for revalidation as well + Accepted bool + // VoucherResult provides information to the other party about what happened + // with the voucher + VoucherResult + // LeaveRequestPaused indicates whether the request should stay paused + // even if the request was accepted + LeaveRequestPaused bool + // DataLimit specifies how much data this voucher is good for. When the amount + // of data specified is reached (or shortly after), the request will pause + // pending revalidation. 0 indicates no limit. + DataLimit uint64 + // RequiresFinalization indicates at the end of the transfer, the channel should + // be left open for a final settlement + RequiresFinalization bool +} + // RequestValidator is an interface implemented by the client of the // data transfer module to validate requests type RequestValidator interface { // ValidatePush validates a push request received from the peer that will send data + // -- All information about the validation operation is contained in ValidationResult, + // including if it was rejected. Information about why a rejection occurred is embedded + // in the VoucherResult. + // -- error indicates something went wrong with the actual process of trying to validate ValidatePush( - isRestart bool, chid ChannelID, sender peer.ID, voucher Voucher, baseCid cid.Cid, - selector ipld.Node) (VoucherResult, error) + selector ipld.Node) (ValidationResult, error) // ValidatePull validates a pull request received from the peer that will receive data + // -- All information about the validation operation is contained in ValidationResult, + // including if it was rejected. Information about why a rejection occurred should be embedded + // in the VoucherResult. + // -- error indicates something went wrong with the actual process of trying to validate ValidatePull( - isRestart bool, chid ChannelID, receiver peer.ID, voucher Voucher, baseCid cid.Cid, - selector ipld.Node) (VoucherResult, error) -} + selector ipld.Node) (ValidationResult, error) -// Revalidator is a request validator revalidates in progress requests -// by requesting request additional vouchers, and resuming when it receives them -type Revalidator interface { // Revalidate revalidates a request with a new voucher - Revalidate(channelID ChannelID, voucher Voucher) (VoucherResult, error) - // OnPullDataSent is called on the responder side when more bytes are sent - // for a given pull request. The first value indicates whether the request was - // recognized by this revalidator and should be considered 'handled'. If true, - // the remaining two values are interpreted. If 'false' the request is passed on - // to the next revalidators. - // It should return a VoucherResult + ErrPause to - // request revalidation or nil to continue uninterrupted, - // other errors will terminate the request. - OnPullDataSent(chid ChannelID, additionalBytesSent uint64) (bool, VoucherResult, error) - // OnPushDataReceived is called on the responder side when more bytes are received - // for a given push request. The first value indicates whether the request was - // recognized by this revalidator and should be considered 'handled'. If true, - // the remaining two values are interpreted. If 'false' the request is passed on - // to the next revalidators. It should return a VoucherResult + ErrPause to - // request revalidation or nil to continue uninterrupted, - // other errors will terminate the request - OnPushDataReceived(chid ChannelID, additionalBytesReceived uint64) (bool, VoucherResult, error) - // OnComplete is called to make a final request for revalidation -- often for the - // purpose of settlement. The first value indicates whether the request was - // recognized by this revalidator and should be considered 'handled'. If true, - // the remaining two values are interpreted. If 'false' the request is passed on - // to the next revalidators. - // if VoucherResult is non nil, the request will enter a settlement phase awaiting - // a final update - OnComplete(chid ChannelID) (bool, VoucherResult, error) + // -- All information about the validation operation is contained in ValidationResult, + // including if it was rejected. Information about why a rejection occurred should be embedded + // in the VoucherResult. + // -- error indicates something went wrong with the actual process of trying to validate + Revalidate(channelID ChannelID, channel ChannelState) (ValidationResult, error) } // TransportConfigurer provides a mechanism to provide transport specific configuration for a given voucher type @@ -85,13 +86,6 @@ type Manager interface { // or if there is a voucher type registered with an identical identifier RegisterVoucherType(voucherType Voucher, validator RequestValidator) error - // RegisterRevalidator registers a revalidator for the given voucher type - // Note: this is the voucher type used to revalidate. It can share a name - // with the initial validator type and CAN be the same type, or a different type. - // The revalidator can simply be the sampe as the original request validator, - // or a different validator that satisfies the revalidator interface. - RegisterRevalidator(voucherType Voucher, revalidator Revalidator) error - // RegisterVoucherResultType allows deserialization of a voucher result, // so that a listener can read the metadata RegisterVoucherResultType(resultType VoucherResult) error @@ -111,6 +105,9 @@ type Manager interface { // send an intermediate voucher as needed when the receiver sends a request for revalidation SendVoucher(ctx context.Context, chid ChannelID, voucher Voucher) error + // send information from the responder to update the initiator on the state of their voucher + SendVoucherResult(ctx context.Context, chid ChannelID, voucher VoucherResult) error + // close an open channel (effectively a cancel) CloseDataTransferChannel(ctx context.Context, chid ChannelID) error diff --git a/message.go b/message.go index 3fa16b96..7f36b33d 100644 --- a/message.go +++ b/message.go @@ -48,7 +48,7 @@ type Request interface { // Response is a response message for the data transfer protocol type Response interface { Message - IsVoucherResult() bool + IsValidationResult() bool IsComplete() bool Accepted() bool VoucherResultType() TypeIdentifier diff --git a/message/message.go b/message/message.go index 60b183fe..e8acac95 100644 --- a/message/message.go +++ b/message/message.go @@ -8,8 +8,15 @@ var NewRequest = message1_1.NewRequest var RestartExistingChannelRequest = message1_1.RestartExistingChannelRequest var UpdateRequest = message1_1.UpdateRequest var VoucherRequest = message1_1.VoucherRequest + +// DEPRECATED: Use ValidationResultResponse var RestartResponse = message1_1.RestartResponse +var ValidationResultResponse = message1_1.ValidationResultResponse + +// DEPRECATED: Use ValidationResultResponse var NewResponse = message1_1.NewResponse + +// DEPRECATED: Use ValidationResultResponse var VoucherResultResponse = message1_1.VoucherResultResponse var CancelResponse = message1_1.CancelResponse var UpdateResponse = message1_1.UpdateResponse diff --git a/message/message1_1/message_test.go b/message/message1_1/message_test.go index 9a4e0ef4..3e69fa9d 100644 --- a/message/message1_1/message_test.go +++ b/message/message1_1/message_test.go @@ -381,7 +381,7 @@ func TestCompleteResponse(t *testing.T) { assert.False(t, response.IsNew()) assert.False(t, response.IsUpdate()) assert.True(t, response.IsPaused()) - assert.True(t, response.IsVoucherResult()) + assert.True(t, response.IsValidationResult()) assert.True(t, response.EmptyVoucherResult()) assert.True(t, response.IsComplete()) assert.False(t, response.IsRequest()) diff --git a/message/message1_1/transfer_response.go b/message/message1_1/transfer_response.go index 5e91ea2f..105e19a4 100644 --- a/message/message1_1/transfer_response.go +++ b/message/message1_1/transfer_response.go @@ -63,7 +63,7 @@ func (trsp *TransferResponse1_1) IsComplete() bool { return trsp.Type == uint64(types.CompleteMessage) } -func (trsp *TransferResponse1_1) IsVoucherResult() bool { +func (trsp *TransferResponse1_1) IsValidationResult() bool { return trsp.Type == uint64(types.VoucherResultMessage) || trsp.Type == uint64(types.NewMessage) || trsp.Type == uint64(types.CompleteMessage) || trsp.Type == uint64(types.RestartMessage) } diff --git a/message/message1_1/transfer_response_test.go b/message/message1_1/transfer_response_test.go index b6335fb4..02c9cffb 100644 --- a/message/message1_1/transfer_response_test.go +++ b/message/message1_1/transfer_response_test.go @@ -26,7 +26,7 @@ func TestResponseMessageForProtocol(t *testing.T) { require.True(t, ok) require.True(t, resp.IsPaused()) require.Equal(t, voucherResult.Type(), resp.VoucherResultType()) - require.True(t, resp.IsVoucherResult()) + require.True(t, resp.IsValidationResult()) // random protocol out, err = response.MessageForProtocol("RAND") diff --git a/message/message1_1prime/message.go b/message/message1_1prime/message.go index 2cd2b6aa..b0f1471a 100644 --- a/message/message1_1prime/message.go +++ b/message/message1_1prime/message.go @@ -105,6 +105,33 @@ func RestartResponse(id datatransfer.TransferID, accepted bool, isPaused bool, v }, nil } +// ValidationResultResponse response generates a response based on a validation result +// messageType determines what kind of response is created +func ValidationResultResponse( + messageType types.MessageType, + id datatransfer.TransferID, + validationResult datatransfer.ValidationResult, + validationErr error) (datatransfer.Response, error) { + voucherResultType := datatransfer.EmptyTypeIdentifier + if validationResult.VoucherResult != nil { + voucherResultType = validationResult.VoucherResult.Type() + } + vnode, err := encoding.EncodeToNode(validationResult.VoucherResult) + if err != nil { + return nil, xerrors.Errorf("Creating request: %w", err) + } + return &TransferResponse1_1{ + // TODO: when we area able to change the protocol, it would be helpful to record + // Validation errors vs rejections + RequestAccepted: validationErr == nil && validationResult.Accepted, + MessageType: uint64(messageType), + Paused: validationResult.LeaveRequestPaused, + TransferId: uint64(id), + VoucherTypeIdentifier: voucherResultType, + VoucherResultPtr: &vnode, + }, nil +} + // NewResponse builds a new Data Transfer response func NewResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResultType datatransfer.TypeIdentifier, voucherResult encoding.Encodable) (datatransfer.Response, error) { vnode, err := encoding.EncodeToNode(voucherResult) diff --git a/message/message1_1prime/message_test.go b/message/message1_1prime/message_test.go index 9e2ba49c..93e95f27 100644 --- a/message/message1_1prime/message_test.go +++ b/message/message1_1prime/message_test.go @@ -387,7 +387,7 @@ func TestCompleteResponse(t *testing.T) { assert.False(t, response.IsNew()) assert.False(t, response.IsUpdate()) assert.True(t, response.IsPaused()) - assert.True(t, response.IsVoucherResult()) + assert.True(t, response.IsValidationResult()) assert.True(t, response.EmptyVoucherResult()) assert.True(t, response.IsComplete()) assert.False(t, response.IsRequest()) diff --git a/message/message1_1prime/transfer_response.go b/message/message1_1prime/transfer_response.go index 902944fd..32d240d6 100644 --- a/message/message1_1prime/transfer_response.go +++ b/message/message1_1prime/transfer_response.go @@ -59,7 +59,7 @@ func (trsp *TransferResponse1_1) IsComplete() bool { return trsp.MessageType == uint64(types.CompleteMessage) } -func (trsp *TransferResponse1_1) IsVoucherResult() bool { +func (trsp *TransferResponse1_1) IsValidationResult() bool { return trsp.MessageType == uint64(types.VoucherResultMessage) || trsp.MessageType == uint64(types.NewMessage) || trsp.MessageType == uint64(types.CompleteMessage) || trsp.MessageType == uint64(types.RestartMessage) } diff --git a/message/message1_1prime/transfer_response_test.go b/message/message1_1prime/transfer_response_test.go index cffe5122..dbaf2de0 100644 --- a/message/message1_1prime/transfer_response_test.go +++ b/message/message1_1prime/transfer_response_test.go @@ -26,7 +26,7 @@ func TestResponseMessageForProtocol(t *testing.T) { require.True(t, ok) require.True(t, resp.IsPaused()) require.Equal(t, voucherResult.Type(), resp.VoucherResultType()) - require.True(t, resp.IsVoucherResult()) + require.True(t, resp.IsValidationResult()) // random protocol out, err = response.MessageForProtocol("RAND") diff --git a/network/libp2p_impl_test.go b/network/libp2p_impl_test.go index dc71c641..527bfffd 100644 --- a/network/libp2p_impl_test.go +++ b/network/libp2p_impl_test.go @@ -20,6 +20,7 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/message/types" "github.com/filecoin-project/go-data-transfer/v2/network" "github.com/filecoin-project/go-data-transfer/v2/testutil" ) @@ -131,7 +132,7 @@ func TestMessageSendAndReceive(t *testing.T) { accepted := false id := datatransfer.TransferID(rand.Int31()) voucherResult := testutil.NewFakeDTType() - response, err := message.NewResponse(id, accepted, false, voucherResult.Type(), voucherResult) + response, err := message.ValidationResultResponse(types.NewMessage, id, datatransfer.ValidationResult{Accepted: accepted, VoucherResult: voucherResult}, nil) require.NoError(t, err) require.NoError(t, dtnet2.SendMessage(ctx, host1.ID(), response)) diff --git a/statuses.go b/statuses.go index 6a4c89be..32c277fb 100644 --- a/statuses.go +++ b/statuses.go @@ -60,6 +60,18 @@ const ( ChannelNotFoundError ) +func (s Status) IsAccepted() bool { + return s != Requested && s != Cancelled && s != Cancelling && s != Failed && s != Failing && s != ChannelNotFoundError +} + +func (s Status) IsResponderPaused() bool { + return s == ResponderPaused || s == BothPaused || s == Finalizing +} + +func (s Status) InFinalization() bool { + return s == Finalizing || s == Completed || s == Completing +} + // Statuses are human readable names for data transfer states var Statuses = map[Status]string{ // Requested means a data transfer was requested by has not yet been approved diff --git a/testutil/mockchannelstate.go b/testutil/mockchannelstate.go new file mode 100644 index 00000000..a9a86586 --- /dev/null +++ b/testutil/mockchannelstate.go @@ -0,0 +1,170 @@ +package testutil + +import ( + cid "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" + "github.com/libp2p/go-libp2p-core/peer" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" +) + +type MockChannelStateParams struct { + ReceivedCids []cid.Cid + ChannelID datatransfer.ChannelID + Queued uint64 + Sent uint64 + Received uint64 + Complete bool +} + +func NewMockChannelState(params MockChannelStateParams) *MockChannelState { + return &MockChannelState{ + receivedCids: params.ReceivedCids, + chid: params.ChannelID, + queued: params.Queued, + sent: params.Sent, + received: params.Received, + complete: params.Complete, + } +} + +type MockChannelState struct { + receivedCids []cid.Cid + chid datatransfer.ChannelID + queued uint64 + sent uint64 + received uint64 + complete bool +} + +var _ datatransfer.ChannelState = (*MockChannelState)(nil) + +func (m *MockChannelState) Queued() uint64 { + return m.queued +} + +func (m *MockChannelState) SetQueued(queued uint64) { + m.queued = queued +} + +func (m *MockChannelState) Sent() uint64 { + return m.sent +} + +func (m *MockChannelState) SetSent(sent uint64) { + m.sent = sent +} + +func (m *MockChannelState) Received() uint64 { + return m.received +} + +func (m *MockChannelState) SetReceived(received uint64) { + m.received = received +} + +func (m *MockChannelState) ChannelID() datatransfer.ChannelID { + return m.chid +} + +func (m *MockChannelState) SetComplete(complete bool) { + m.complete = complete +} +func (m *MockChannelState) Status() datatransfer.Status { + if m.complete { + return datatransfer.Completed + } + return datatransfer.Ongoing +} + +func (m *MockChannelState) ReceivedCids() []cid.Cid { + return m.receivedCids +} + +func (m *MockChannelState) ReceivedCidsLen() int { + return len(m.receivedCids) +} + +func (m *MockChannelState) ReceivedCidsTotal() int64 { + return (int64)(len(m.receivedCids)) +} + +func (m *MockChannelState) QueuedCidsTotal() int64 { + panic("implement me") +} + +func (m *MockChannelState) SentCidsTotal() int64 { + panic("implement me") +} + +func (m *MockChannelState) TransferID() datatransfer.TransferID { + panic("implement me") +} + +func (m *MockChannelState) BaseCID() cid.Cid { + panic("implement me") +} + +func (m *MockChannelState) Selector() ipld.Node { + panic("implement me") +} + +func (m *MockChannelState) Voucher() datatransfer.Voucher { + panic("implement me") +} + +func (m *MockChannelState) Sender() peer.ID { + panic("implement me") +} + +func (m *MockChannelState) Recipient() peer.ID { + panic("implement me") +} + +func (m *MockChannelState) TotalSize() uint64 { + panic("implement me") +} + +func (m *MockChannelState) IsPull() bool { + panic("implement me") +} + +func (m *MockChannelState) OtherPeer() peer.ID { + panic("implement me") +} + +func (m *MockChannelState) SelfPeer() peer.ID { + panic("implement me") +} + +func (m *MockChannelState) Message() string { + panic("implement me") +} + +func (m *MockChannelState) Vouchers() []datatransfer.Voucher { + panic("implement me") +} + +func (m *MockChannelState) VoucherResults() []datatransfer.VoucherResult { + panic("implement me") +} + +func (m *MockChannelState) LastVoucher() datatransfer.Voucher { + panic("implement me") +} + +func (m *MockChannelState) LastVoucherResult() datatransfer.VoucherResult { + panic("implement me") +} + +func (m *MockChannelState) Stages() *datatransfer.ChannelStages { + panic("implement me") +} + +func (m *MockChannelState) DataLimit() uint64 { + panic("implement me") +} + +func (m *MockChannelState) RequiresFinalization() bool { + panic("implement me") +} diff --git a/testutil/stubbedvalidator.go b/testutil/stubbedvalidator.go index 798d0fad..0e88c7f1 100644 --- a/testutil/stubbedvalidator.go +++ b/testutil/stubbedvalidator.go @@ -19,12 +19,11 @@ func NewStubbedValidator() *StubbedValidator { // ValidatePush returns a stubbed result for a push validation func (sv *StubbedValidator) ValidatePush( - isRestart bool, chid datatransfer.ChannelID, sender peer.ID, voucher datatransfer.Voucher, baseCid cid.Cid, - selector ipld.Node) (datatransfer.VoucherResult, error) { + selector ipld.Node) (datatransfer.ValidationResult, error) { sv.didPush = true sv.ValidationsReceived = append(sv.ValidationsReceived, ReceivedValidation{false, sender, voucher, baseCid, selector}) return sv.result, sv.pushError @@ -32,19 +31,18 @@ func (sv *StubbedValidator) ValidatePush( // ValidatePull returns a stubbed result for a pull validation func (sv *StubbedValidator) ValidatePull( - isRestart bool, chid datatransfer.ChannelID, receiver peer.ID, voucher datatransfer.Voucher, baseCid cid.Cid, - selector ipld.Node) (datatransfer.VoucherResult, error) { + selector ipld.Node) (datatransfer.ValidationResult, error) { sv.didPull = true sv.ValidationsReceived = append(sv.ValidationsReceived, ReceivedValidation{true, receiver, voucher, baseCid, selector}) return sv.result, sv.pullError } // StubResult returns thes given voucher result when a validate call is made -func (sv *StubbedValidator) StubResult(voucherResult datatransfer.VoucherResult) { +func (sv *StubbedValidator) StubResult(voucherResult datatransfer.ValidationResult) { sv.result = voucherResult } @@ -58,11 +56,6 @@ func (sv *StubbedValidator) StubSuccessPush() { sv.pushError = nil } -// StubPausePush sets ValidatePush to pause -func (sv *StubbedValidator) StubPausePush() { - sv.pushError = datatransfer.ErrPause -} - // ExpectErrorPush expects ValidatePush to error func (sv *StubbedValidator) ExpectErrorPush() { sv.expectPush = true @@ -75,12 +68,6 @@ func (sv *StubbedValidator) ExpectSuccessPush() { sv.StubSuccessPush() } -// ExpectPausePush expects ValidatePush to pause -func (sv *StubbedValidator) ExpectPausePush() { - sv.expectPush = true - sv.StubPausePush() -} - // StubErrorPull sets ValidatePull to error func (sv *StubbedValidator) StubErrorPull() { sv.pullError = errors.New("something went wrong") @@ -91,11 +78,6 @@ func (sv *StubbedValidator) StubSuccessPull() { sv.pullError = nil } -// StubPausePull sets ValidatePull to pause -func (sv *StubbedValidator) StubPausePull() { - sv.pullError = datatransfer.ErrPause -} - // ExpectErrorPull expects ValidatePull to error func (sv *StubbedValidator) ExpectErrorPull() { sv.expectPull = true @@ -108,12 +90,6 @@ func (sv *StubbedValidator) ExpectSuccessPull() { sv.StubSuccessPull() } -// ExpectPausePull expects ValidatePull to pause -func (sv *StubbedValidator) ExpectPausePull() { - sv.expectPull = true - sv.StubPausePull() -} - // VerifyExpectations verifies the specified calls were made func (sv *StubbedValidator) VerifyExpectations(t *testing.T) { if sv.expectPush { @@ -122,238 +98,74 @@ func (sv *StubbedValidator) VerifyExpectations(t *testing.T) { if sv.expectPull { require.True(t, sv.didPull) } + if sv.expectRevalidate { + require.True(t, sv.didRevalidate) + } } -// ReceivedValidation records a call to either ValidatePush or ValidatePull -type ReceivedValidation struct { - IsPull bool - Other peer.ID - Voucher datatransfer.Voucher - BaseCid cid.Cid - Selector ipld.Node -} - -// StubbedValidator is a validator that returns predictable results -type StubbedValidator struct { - result datatransfer.VoucherResult - didPush bool - didPull bool - expectPush bool - expectPull bool - pushError error - pullError error - ValidationsReceived []ReceivedValidation -} - -// StubbedRevalidator is a revalidator that returns predictable results -type StubbedRevalidator struct { - revalidationResult datatransfer.VoucherResult - checkResult datatransfer.VoucherResult - didRevalidate bool - didPushCheck bool - didPullCheck bool - didComplete bool - expectRevalidate bool - expectPushCheck bool - expectPullCheck bool - expectComplete bool - revalidationError error - pushCheckError error - pullCheckError error - completeError error -} - -// NewStubbedRevalidator returns a new instance of a stubbed revalidator -func NewStubbedRevalidator() *StubbedRevalidator { - return &StubbedRevalidator{} -} - -// OnPullDataSent returns a stubbed result for checking when pull data is sent -func (srv *StubbedRevalidator) OnPullDataSent(chid datatransfer.ChannelID, additionalBytesSent uint64) (bool, datatransfer.VoucherResult, error) { - srv.didPullCheck = true - return srv.expectPullCheck, srv.revalidationResult, srv.pullCheckError -} - -// OnPushDataReceived returns a stubbed result for checking when push data is received -func (srv *StubbedRevalidator) OnPushDataReceived(chid datatransfer.ChannelID, additionalBytesReceived uint64) (bool, datatransfer.VoucherResult, error) { - srv.didPushCheck = true - return srv.expectPushCheck, srv.revalidationResult, srv.pushCheckError -} - -// OnComplete returns a stubbed result for checking when the requests completes -func (srv *StubbedRevalidator) OnComplete(chid datatransfer.ChannelID) (bool, datatransfer.VoucherResult, error) { - srv.didComplete = true - return srv.expectComplete, srv.revalidationResult, srv.completeError -} - -// Revalidate returns a stubbed result for revalidating a request -func (srv *StubbedRevalidator) Revalidate(chid datatransfer.ChannelID, voucher datatransfer.Voucher) (datatransfer.VoucherResult, error) { - srv.didRevalidate = true - return srv.checkResult, srv.revalidationError +func (sv *StubbedValidator) Revalidate(chid datatransfer.ChannelID, channelState datatransfer.ChannelState) (datatransfer.ValidationResult, error) { + sv.didRevalidate = true + sv.RevalidationsReceived = append(sv.RevalidationsReceived, ReceivedRevalidation{chid, channelState}) + return sv.revalidationResult, sv.revalidationError } // StubRevalidationResult returns the given voucher result when a call is made to Revalidate -func (srv *StubbedRevalidator) StubRevalidationResult(voucherResult datatransfer.VoucherResult) { - srv.revalidationResult = voucherResult -} - -// StubCheckResult returns the given voucher result when a call is made to -// OnPullDataSent, OnPushDataReceived, or OnComplete -func (srv *StubbedRevalidator) StubCheckResult(voucherResult datatransfer.VoucherResult) { - srv.checkResult = voucherResult -} - -// StubErrorPushCheck sets OnPushDataReceived to error -func (srv *StubbedRevalidator) StubErrorPushCheck() { - srv.pushCheckError = errors.New("something went wrong") -} - -// StubSuccessPushCheck sets OnPushDataReceived to succeed -func (srv *StubbedRevalidator) StubSuccessPushCheck() { - srv.pushCheckError = nil -} - -// StubPausePushCheck sets OnPushDataReceived to pause -func (srv *StubbedRevalidator) StubPausePushCheck() { - srv.pushCheckError = datatransfer.ErrPause -} - -// ExpectErrorPushCheck expects OnPushDataReceived to error -func (srv *StubbedRevalidator) ExpectErrorPushCheck() { - srv.expectPushCheck = true - srv.StubErrorPushCheck() -} - -// ExpectSuccessPushCheck expects OnPushDataReceived to succeed -func (srv *StubbedRevalidator) ExpectSuccessPushCheck() { - srv.expectPushCheck = true - srv.StubSuccessPushCheck() -} - -// ExpectPausePushCheck expects OnPushDataReceived to pause -func (srv *StubbedRevalidator) ExpectPausePushCheck() { - srv.expectPushCheck = true - srv.StubPausePushCheck() -} - -// StubErrorPullCheck sets OnPullDataSent to error -func (srv *StubbedRevalidator) StubErrorPullCheck() { - srv.pullCheckError = errors.New("something went wrong") -} - -// StubSuccessPullCheck sets OnPullDataSent to succeed -func (srv *StubbedRevalidator) StubSuccessPullCheck() { - srv.pullCheckError = nil -} - -// StubPausePullCheck sets OnPullDataSent to pause -func (srv *StubbedRevalidator) StubPausePullCheck() { - srv.pullCheckError = datatransfer.ErrPause -} - -// ExpectErrorPullCheck expects OnPullDataSent to error -func (srv *StubbedRevalidator) ExpectErrorPullCheck() { - srv.expectPullCheck = true - srv.StubErrorPullCheck() -} - -// ExpectSuccessPullCheck expects OnPullDataSent to succeed -func (srv *StubbedRevalidator) ExpectSuccessPullCheck() { - srv.expectPullCheck = true - srv.StubSuccessPullCheck() -} - -// ExpectPausePullCheck expects OnPullDataSent to pause -func (srv *StubbedRevalidator) ExpectPausePullCheck() { - srv.expectPullCheck = true - srv.StubPausePullCheck() -} - -// StubErrorComplete sets OnComplete to error -func (srv *StubbedRevalidator) StubErrorComplete() { - srv.completeError = errors.New("something went wrong") -} - -// StubSuccessComplete sets OnComplete to succeed -func (srv *StubbedRevalidator) StubSuccessComplete() { - srv.completeError = nil -} - -// StubPauseComplete sets OnComplete to pause -func (srv *StubbedRevalidator) StubPauseComplete() { - srv.completeError = datatransfer.ErrPause -} - -// ExpectErrorComplete expects OnComplete to error -func (srv *StubbedRevalidator) ExpectErrorComplete() { - srv.expectComplete = true - srv.StubErrorComplete() -} - -// ExpectSuccessComplete expects OnComplete to succeed -func (srv *StubbedRevalidator) ExpectSuccessComplete() { - srv.expectComplete = true - srv.StubSuccessComplete() -} - -// ExpectPauseComplete expects OnComplete to pause -func (srv *StubbedRevalidator) ExpectPauseComplete() { - srv.expectComplete = true - srv.StubPauseComplete() +func (sv *StubbedValidator) StubRevalidationResult(voucherResult datatransfer.ValidationResult) { + sv.revalidationResult = voucherResult } // StubErrorRevalidation sets Revalidate to error -func (srv *StubbedRevalidator) StubErrorRevalidation() { - srv.revalidationError = errors.New("something went wrong") +func (sv *StubbedValidator) StubErrorRevalidation() { + sv.revalidationError = errors.New("something went wrong") } // StubSuccessRevalidation sets Revalidate to succeed -func (srv *StubbedRevalidator) StubSuccessRevalidation() { - srv.revalidationError = nil -} - -// StubPauseRevalidation sets Revalidate to pause -func (srv *StubbedRevalidator) StubPauseRevalidation() { - srv.revalidationError = datatransfer.ErrPause -} - -// ExpectSuccessErrResume configures Revalidate to return an ErrResume -// and expect a Revalidate call. -func (srv *StubbedRevalidator) ExpectSuccessErrResume() { - srv.expectRevalidate = true - srv.revalidationError = datatransfer.ErrResume +func (sv *StubbedValidator) StubSuccessRevalidation() { + sv.revalidationError = nil } // ExpectErrorRevalidation expects Revalidate to error -func (srv *StubbedRevalidator) ExpectErrorRevalidation() { - srv.expectRevalidate = true - srv.StubErrorRevalidation() +func (sv *StubbedValidator) ExpectErrorRevalidation() { + sv.expectRevalidate = true + sv.StubErrorRevalidation() } // ExpectSuccessRevalidation expects Revalidate to succeed -func (srv *StubbedRevalidator) ExpectSuccessRevalidation() { - srv.expectRevalidate = true - srv.StubSuccessRevalidation() +func (sv *StubbedValidator) ExpectSuccessRevalidation() { + sv.expectRevalidate = true + sv.StubSuccessRevalidation() } -// ExpectPauseRevalidation expects Revalidate to pause -func (srv *StubbedRevalidator) ExpectPauseRevalidation() { - srv.expectRevalidate = true - srv.StubPauseRevalidation() +// ReceivedValidation records a call to either ValidatePush or ValidatePull +type ReceivedValidation struct { + IsPull bool + Other peer.ID + Voucher datatransfer.Voucher + BaseCid cid.Cid + Selector ipld.Node } -// VerifyExpectations verifies the specified calls were made -func (srv *StubbedRevalidator) VerifyExpectations(t *testing.T) { - if srv.expectRevalidate { - require.True(t, srv.didRevalidate) - } - if srv.expectPushCheck { - require.True(t, srv.didPushCheck) - } - if srv.expectPullCheck { - require.True(t, srv.didPullCheck) - } - if srv.expectComplete { - require.True(t, srv.didComplete) - } +// ReceivedRevalidation records a call to Revalidate +type ReceivedRevalidation struct { + ChannelID datatransfer.ChannelID + ChannelState datatransfer.ChannelState } + +// StubbedValidator is a validator that returns predictable results +type StubbedValidator struct { + result datatransfer.ValidationResult + revalidationResult datatransfer.ValidationResult + expectRevalidate bool + didRevalidate bool + didPush bool + didPull bool + expectPush bool + expectPull bool + pushError error + pullError error + revalidationError error + ValidationsReceived []ReceivedValidation + RevalidationsReceived []ReceivedRevalidation +} + +var _ datatransfer.RequestValidator = (*StubbedValidator)(nil) diff --git a/transport.go b/transport.go index 6d0b99fa..81187077 100644 --- a/transport.go +++ b/transport.go @@ -27,8 +27,6 @@ type EventsHandler interface { // OnDataQueued is called when data is queued for sending for the given channel ID // return values are: - // message = data transfer message along with data - // err = error // - nil = proceed with sending data // - error = cancel this request // - err == ErrPause - pause this request diff --git a/transport/graphsync/graphsync_test.go b/transport/graphsync/graphsync_test.go index 2cc40689..c2a71468 100644 --- a/transport/graphsync/graphsync_test.go +++ b/transport/graphsync/graphsync_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/ipfs/go-cid" "github.com/ipfs/go-graphsync" "github.com/ipfs/go-graphsync/donotsendfirstblocks" "github.com/ipld/go-ipld-prime" @@ -713,7 +712,7 @@ func TestManager(t *testing.T) { "open channel adds block count to the DoNotSendFirstBlocks extension for v1.2 protocol": { action: func(gsData *harness) { cids := testutil.GenerateCids(2) - channel := &mockChannelState{receivedCids: cids} + channel := testutil.NewMockChannelState(testutil.MockChannelStateParams{ReceivedCids: cids}) stor, _ := gsData.outgoing.Selector() go gsData.outgoingRequestHook() @@ -744,7 +743,7 @@ func TestManager(t *testing.T) { "ChannelsForPeer when request is open": { action: func(gsData *harness) { cids := testutil.GenerateCids(2) - channel := &mockChannelState{receivedCids: cids} + channel := testutil.NewMockChannelState(testutil.MockChannelStateParams{ReceivedCids: cids}) stor, _ := gsData.outgoing.Selector() go gsData.outgoingRequestHook() @@ -774,7 +773,7 @@ func TestManager(t *testing.T) { "open channel cancels an existing request with the same channel ID": { action: func(gsData *harness) { cids := testutil.GenerateCids(2) - channel := &mockChannelState{receivedCids: cids} + channel := testutil.NewMockChannelState(testutil.MockChannelStateParams{ReceivedCids: cids}) stor, _ := gsData.outgoing.Selector() go gsData.outgoingRequestHook() _ = gsData.transport.OpenChannel( @@ -1364,113 +1363,3 @@ func assertHasExtensionMessage(t *testing.T, name graphsync.ExtensionName, exten require.Fail(t, "extension not found") } } - -type mockChannelState struct { - receivedCids []cid.Cid -} - -var _ datatransfer.ChannelState = (*mockChannelState)(nil) - -func (m *mockChannelState) ReceivedCids() []cid.Cid { - return m.receivedCids -} - -func (m *mockChannelState) ReceivedCidsLen() int { - return len(m.receivedCids) -} - -func (m *mockChannelState) ReceivedCidsTotal() int64 { - return (int64)(len(m.receivedCids)) -} - -func (m *mockChannelState) QueuedCidsTotal() int64 { - panic("implement me") -} - -func (m *mockChannelState) SentCidsTotal() int64 { - panic("implement me") -} - -func (m *mockChannelState) Queued() uint64 { - panic("implement me") -} - -func (m *mockChannelState) Sent() uint64 { - panic("implement me") -} - -func (m *mockChannelState) Received() uint64 { - panic("implement me") -} - -func (m *mockChannelState) ChannelID() datatransfer.ChannelID { - panic("implement me") -} - -func (m *mockChannelState) Status() datatransfer.Status { - panic("implement me") -} - -func (m *mockChannelState) TransferID() datatransfer.TransferID { - panic("implement me") -} - -func (m *mockChannelState) BaseCID() cid.Cid { - panic("implement me") -} - -func (m *mockChannelState) Selector() ipld.Node { - panic("implement me") -} - -func (m *mockChannelState) Voucher() datatransfer.Voucher { - panic("implement me") -} - -func (m *mockChannelState) Sender() peer.ID { - panic("implement me") -} - -func (m *mockChannelState) Recipient() peer.ID { - panic("implement me") -} - -func (m *mockChannelState) TotalSize() uint64 { - panic("implement me") -} - -func (m *mockChannelState) IsPull() bool { - panic("implement me") -} - -func (m *mockChannelState) OtherPeer() peer.ID { - panic("implement me") -} - -func (m *mockChannelState) SelfPeer() peer.ID { - panic("implement me") -} - -func (m *mockChannelState) Message() string { - panic("implement me") -} - -func (m *mockChannelState) Vouchers() []datatransfer.Voucher { - panic("implement me") -} - -func (m *mockChannelState) VoucherResults() []datatransfer.VoucherResult { - panic("implement me") -} - -func (m *mockChannelState) LastVoucher() datatransfer.Voucher { - panic("implement me") -} - -func (m *mockChannelState) LastVoucherResult() datatransfer.VoucherResult { - panic("implement me") -} - -func (m *mockChannelState) Stages() *datatransfer.ChannelStages { - panic("implement me") -} diff --git a/types.go b/types.go index f7ddff7f..9201c5aa 100644 --- a/types.go +++ b/types.go @@ -144,6 +144,14 @@ type ChannelState interface { // Queued returns the number of bytes read from the node and queued for sending Queued() uint64 + // DataLimit is the maximum data that can be transferred on this channel before + // revalidation. 0 indicates no limit. + DataLimit() uint64 + + // RequiresFinalization indicates at the end of the transfer, the channel should + // be left open for a final settlement + RequiresFinalization() bool + // Stages returns the timeline of events this data transfer has gone through, // for observability purposes. // From 5253bfe075cd4dd9ec8bbf6b4919011c88cb44cf Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Wed, 11 May 2022 15:33:25 -0700 Subject: [PATCH 03/18] Refactor revalidators v2 (#322) * refactor(validators): remove revalidation move to all revalidation being asynchronous * feat(validators): implied pauses causes datalimit exceeded and requires finalization to leave request paused, regardless of where LeaveRequestPaused is set * refactor(events): reorder events reorder validation events so all get record when transfer finishes * Update impl/impl.go Co-authored-by: Rod Vagg Co-authored-by: Rod Vagg --- impl/impl.go | 103 +++++++++++++++++++++- impl/initiating_test.go | 14 +-- impl/integration_test.go | 61 +++++++------ impl/receiving_requests.go | 137 +++++++++-------------------- impl/responding_test.go | 113 ++++++++++-------------- impl/restart.go | 4 +- impl/restart_integration_test.go | 8 +- manager.go | 31 +++++-- message/message.go | 1 + message/message1_1prime/message.go | 5 +- network/libp2p_impl_test.go | 2 +- testutil/stubbedvalidator.go | 34 +++---- 12 files changed, 288 insertions(+), 225 deletions(-) diff --git a/impl/impl.go b/impl/impl.go index 01b99d72..efd7e878 100644 --- a/impl/impl.go +++ b/impl/impl.go @@ -24,6 +24,7 @@ import ( "github.com/filecoin-project/go-data-transfer/v2/channels" "github.com/filecoin-project/go-data-transfer/v2/encoding" "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/message/types" "github.com/filecoin-project/go-data-transfer/v2/network" "github.com/filecoin-project/go-data-transfer/v2/registry" "github.com/filecoin-project/go-data-transfer/v2/tracing" @@ -334,8 +335,108 @@ func (m *manager) SendVoucherResult(ctx context.Context, channelID datatransfer. return m.channels.NewVoucherResult(channelID, voucherResult) } -func (m *manager) SetDataLimit(ctx context.Context, chid datatransfer.ChannelID, totalData uint64, stopAtEnd bool) { +func (m *manager) UpdateValidationStatus(ctx context.Context, chid datatransfer.ChannelID, result datatransfer.ValidationResult) error { + ctx, _ = m.spansIndex.SpanForChannel(ctx, chid) + ctx, span := otel.Tracer("data-transfer").Start(ctx, "updateValidationStatus", trace.WithAttributes( + attribute.String("channelID", chid.String()), + )) + err := m.updateValidationStatus(ctx, chid, result) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + span.End() + return err +} +// updateValidationStatus is the implementation of the public method, which wraps this private method +// in a trace +func (m *manager) updateValidationStatus(ctx context.Context, chid datatransfer.ChannelID, result datatransfer.ValidationResult) error { + // first check if we are the responder -- only the responder can call UpdateValidationStatus + if chid.Initiator == m.peerID { + err := errors.New("cannot send voucher result for request we initiated") + return err + } + + // dispatch channel events and generate a response message + chst, response, err := m.processValidationUpdate(ctx, chid, result) + + // dispatch transport updates + return m.handleTransportUpdate(ctx, chst, response, result, err) +} + +func (m *manager) processValidationUpdate(ctx context.Context, chid datatransfer.ChannelID, result datatransfer.ValidationResult) (datatransfer.ChannelState, datatransfer.Response, error) { + + // read the channel state + chst, err := m.channels.GetByID(ctx, chid) + if err != nil { + return nil, nil, err + } + + // if the request is now rejected, error the channel + if !result.Accepted { + err = m.recordRejectedValidationEvents(chid, result) + } else { + err = m.recordAcceptedValidationEvents(chst, result) + } + if err != nil { + return nil, nil, err + } + + // generate a response message + messageType := types.VoucherResultMessage + if chst.Status() == datatransfer.Finalizing { + messageType = types.CompleteMessage + } + response, msgErr := message.ValidationResultResponse(messageType, chst.TransferID(), result, err, + result.LeaveRequestPaused(chst)) + if msgErr != nil { + return nil, nil, msgErr + } + + // return the response message and any errors + return chst, response, nil +} + +// handleTransportUpdate updates the transport based on the validation status and the +// response message +// TODO: the ordering here is a bit sensitive, and the transport should +// be refactored to accept multiple operations at once and order these itself +func (m *manager) handleTransportUpdate( + ctx context.Context, + chst datatransfer.ChannelState, + response datatransfer.Message, + result datatransfer.ValidationResult, + resultErr error, +) error { + + pauseRequest := result.LeaveRequestPaused(chst) + // resume channel as needed, sending the response message immediately and returning + if resultErr == nil && result.Accepted && !pauseRequest { + if chst.Status().IsResponderPaused() && !chst.Status().InFinalization() { + return m.transport.(datatransfer.PauseableTransport).ResumeChannel(ctx, response, chst.ChannelID()) + } + } + + // send a response message + if response != nil { + if err := m.dataTransferNetwork.SendMessage(ctx, chst.ChannelID().Initiator, response); err != nil { + return err + } + } + + // close the channel as needed + if resultErr != nil || !result.Accepted { + m.transport.CloseChannel(ctx, chst.ChannelID()) + return resultErr + } + + // pause the channel as needed + if pauseRequest && !chst.Status().IsResponderPaused() && !chst.Status().InFinalization() { + return m.transport.(datatransfer.PauseableTransport).PauseChannel(ctx, chst.ChannelID()) + } + + return nil } // close an open channel (effectively a cancel) diff --git a/impl/initiating_test.go b/impl/initiating_test.go index 34f2d180..0d1274b7 100644 --- a/impl/initiating_test.go +++ b/impl/initiating_test.go @@ -462,7 +462,7 @@ func TestDataTransferRestartInitiating(t *testing.T) { require.Len(t, h.voucherValidator.ValidationsReceived, 1) // restart the push request received above and validate it - h.voucherValidator.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) + h.voucherValidator.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} require.NoError(t, h.dt.RestartDataTransferChannel(ctx, chid)) require.Len(t, h.voucherValidator.RevalidationsReceived, 1) @@ -503,8 +503,8 @@ func TestDataTransferRestartInitiating(t *testing.T) { require.Len(t, h.voucherValidator.ValidationsReceived, 1) // restart the pull request received above - h.voucherValidator.ExpectSuccessRevalidation() - h.voucherValidator.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) + h.voucherValidator.ExpectSuccessValidateRestart() + h.voucherValidator.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pullRequest.TransferID()} require.NoError(t, h.dt.RestartDataTransferChannel(ctx, chid)) require.Len(t, h.transport.OpenedChannels, 0) @@ -544,8 +544,8 @@ func TestDataTransferRestartInitiating(t *testing.T) { require.Len(t, h.voucherValidator.ValidationsReceived, 1) // restart the pull request received above - h.voucherValidator.ExpectSuccessRevalidation() - h.voucherValidator.StubRevalidationResult(datatransfer.ValidationResult{Accepted: false}) + h.voucherValidator.ExpectSuccessValidateRestart() + h.voucherValidator.StubRestartResult(datatransfer.ValidationResult{Accepted: false}) chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pullRequest.TransferID()} require.EqualError(t, h.dt.RestartDataTransferChannel(ctx, chid), datatransfer.ErrRejected.Error()) }, @@ -566,8 +566,8 @@ func TestDataTransferRestartInitiating(t *testing.T) { require.Len(t, h.voucherValidator.ValidationsReceived, 1) // restart the pull request received above - h.voucherValidator.ExpectSuccessRevalidation() - h.voucherValidator.StubRevalidationResult(datatransfer.ValidationResult{Accepted: false}) + h.voucherValidator.ExpectSuccessValidateRestart() + h.voucherValidator.StubRestartResult(datatransfer.ValidationResult{Accepted: false}) chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} require.EqualError(t, h.dt.RestartDataTransferChannel(ctx, chid), datatransfer.ErrRejected.Error()) }, diff --git a/impl/integration_test.go b/impl/integration_test.go index bef88a85..09896c30 100644 --- a/impl/integration_test.go +++ b/impl/integration_test.go @@ -739,7 +739,7 @@ func TestAutoRestart(t *testing.T) { voucher := testutil.FakeDTType{Data: "applesauce"} sv := testutil.NewStubbedValidator() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) - sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) + sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) var sourceDagService, destDagService ipldformat.DAGService if tc.isPush { @@ -932,7 +932,7 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { voucher := testutil.FakeDTType{Data: "applesauce"} sv := testutil.NewStubbedValidator() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) - sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) + sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) var sourceDagService, destDagService ipldformat.DAGService if isPush { @@ -1135,12 +1135,12 @@ func TestRoundTripCancelledRequest(t *testing.T) { var chid datatransfer.ChannelID if data.isPull { sv.ExpectSuccessPull() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, LeaveRequestPaused: true}) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, ForcePause: true}) require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), &voucher, rootCid, gsData.AllSelector) } else { sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, LeaveRequestPaused: true}) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, ForcePause: true}) require.NoError(t, dt2.RegisterVoucherType(&testutil.FakeDTType{}, sv)) chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, rootCid, gsData.AllSelector) } @@ -1200,7 +1200,7 @@ func (r *retrievalRevalidator) ValidatePush( vr := datatransfer.ValidationResult{ Accepted: true, RequiresFinalization: r.requiresFinalization, - LeaveRequestPaused: r.leavePausedInitially, + ForcePause: r.leavePausedInitially, VoucherResult: r.initialVoucherResult, } if len(r.pausePoints) > r.providerPausePoint { @@ -1220,7 +1220,7 @@ func (r *retrievalRevalidator) ValidatePull( vr := datatransfer.ValidationResult{ Accepted: true, RequiresFinalization: r.requiresFinalization, - LeaveRequestPaused: r.leavePausedInitially, + ForcePause: r.leavePausedInitially, VoucherResult: r.initialVoucherResult, } if len(r.pausePoints) > r.providerPausePoint { @@ -1231,16 +1231,14 @@ func (r *retrievalRevalidator) ValidatePull( return r.StubbedValidator.ValidatePull(chid, sender, voucher, baseCid, selector) } -func (r *retrievalRevalidator) Revalidate(chid datatransfer.ChannelID, channelState datatransfer.ChannelState) (datatransfer.ValidationResult, error) { +func (r *retrievalRevalidator) nextStatus() datatransfer.ValidationResult { vr := datatransfer.ValidationResult{Accepted: true, RequiresFinalization: r.requiresFinalization} if len(r.pausePoints) > r.providerPausePoint { vr.DataLimit = r.pausePoints[r.providerPausePoint] r.providerPausePoint++ } - r.StubbedValidator.StubRevalidationResult(vr) - return r.StubbedValidator.Revalidate(chid, channelState) + return vr } - func TestSimulatedRetrievalFlow(t *testing.T) { ctx := context.Background() testCases := map[string]struct { @@ -1295,7 +1293,7 @@ func TestSimulatedRetrievalFlow(t *testing.T) { // responder: send message that we sent all data along with final voucher request "transfer(1)->sendMessage(0)", // responder: receive final voucher and send acceptance message - "transfer(1)->receiveRequest(5)->sendMessage(0)", + "transfer(1)->receiveRequest(5)", }, }, "fast unseal, payment channel not ready": { @@ -1358,6 +1356,13 @@ func TestSimulatedRetrievalFlow(t *testing.T) { } } dt2.SubscribeToEvents(clientSubscriber) + + sv := &retrievalRevalidator{ + StubbedValidator: testutil.NewStubbedValidator(), + pausePoints: config.pausePoints, + requiresFinalization: true, + leavePausedInitially: true, + } providerFinished := make(chan struct{}, 1) var providerSubscriber datatransfer.Subscriber = func(event datatransfer.Event, channelState datatransfer.ChannelState) { if event.Code == datatransfer.PauseResponder { @@ -1367,10 +1372,14 @@ func TestSimulatedRetrievalFlow(t *testing.T) { _ = dt1.ResumeDataTransferChannel(ctx, chid) }() } + if event.Code == datatransfer.NewVoucher && channelState.Queued() > 0 { + dt1.UpdateValidationStatus(ctx, chid, sv.nextStatus()) + } if event.Code == datatransfer.DataLimitExceeded { dt1.SendVoucherResult(ctx, chid, testutil.NewFakeDTType()) } if event.Code == datatransfer.BeginFinalizing { + sv.requiresFinalization = false dt1.SendVoucherResult(ctx, chid, finalVoucherResult) } if event.Code == datatransfer.Error { @@ -1383,12 +1392,6 @@ func TestSimulatedRetrievalFlow(t *testing.T) { dt1.SubscribeToEvents(providerSubscriber) voucher := testutil.FakeDTType{Data: "applesauce"} - sv := &retrievalRevalidator{ - StubbedValidator: testutil.NewStubbedValidator(), - pausePoints: config.pausePoints, - requiresFinalization: true, - leavePausedInitially: true, - } require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) require.NoError(t, dt2.RegisterVoucherResultType(testutil.NewFakeDTType())) @@ -1495,7 +1498,7 @@ func TestPauseAndResume(t *testing.T) { voucher := testutil.FakeDTType{Data: "applesauce"} sv := testutil.NewStubbedValidator() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) - sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) + sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) var chid datatransfer.ChannelID if isPull { @@ -2006,8 +2009,7 @@ func TestMultipleMessagesInExtension(t *testing.T) { gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator, data sender - root, origBytes := LoadRandomData(ctx, t, gsData.DagService1, 256000) - gsData.OrigBytes = origBytes + root := gsData.LoadUnixFSFile(t, false) rootCid := root.(cidlink.Link).Cid tp1 := gsData.SetupGSTransportHost1() tp2 := gsData.SetupGSTransportHost2() @@ -2092,6 +2094,12 @@ func TestMultipleMessagesInExtension(t *testing.T) { providerFinished := make(chan struct{}, 1) nextVoucherResult := 0 + sv := &retrievalRevalidator{ + StubbedValidator: testutil.NewStubbedValidator(), + pausePoints: pausePoints, + requiresFinalization: true, + initialVoucherResult: respVoucher, + } dt1.SubscribeToEvents(func(event datatransfer.Event, channelState datatransfer.ChannelState) { if event.Code == datatransfer.Error { errChan <- struct{}{} @@ -2099,6 +2107,10 @@ func TestMultipleMessagesInExtension(t *testing.T) { if channelState.Status() == datatransfer.Completed { providerFinished <- struct{}{} } + if event.Code == datatransfer.NewVoucher && channelState.Queued() > 0 { + vs := sv.nextStatus() + dt1.UpdateValidationStatus(ctx, chid, vs) + } if event.Code == datatransfer.DataLimitExceeded { if nextVoucherResult < len(pausePoints) { dt1.SendVoucherResult(ctx, chid, voucherResults[nextVoucherResult]) @@ -2106,16 +2118,10 @@ func TestMultipleMessagesInExtension(t *testing.T) { } } if event.Code == datatransfer.BeginFinalizing { + sv.requiresFinalization = false dt1.SendVoucherResult(ctx, chid, finalVoucherResult) } }) - - sv := &retrievalRevalidator{ - StubbedValidator: testutil.NewStubbedValidator(), - pausePoints: pausePoints, - requiresFinalization: true, - initialVoucherResult: respVoucher, - } require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) require.NoError(t, dt2.RegisterVoucherResultType(testutil.NewFakeDTType())) @@ -2241,7 +2247,6 @@ func TestMultipleParallelTransfers(t *testing.T) { return } if event.Code == datatransfer.Error { - fmt.Println(event.Message) errChan <- struct{}{} } if channelState.Status() == datatransfer.Completed { diff --git a/impl/receiving_requests.go b/impl/receiving_requests.go index a223c4b1..cb86afa7 100644 --- a/impl/receiving_requests.go +++ b/impl/receiving_requests.go @@ -23,13 +23,13 @@ func (m *manager) receiveNewRequest(chid datatransfer.ChannelID, incoming datatr result, err := m.acceptRequest(chid, incoming) // generate a response message - msg, msgErr := message.ValidationResultResponse(types.NewMessage, incoming.TransferID(), result, err) + msg, msgErr := message.ValidationResultResponse(types.NewMessage, incoming.TransferID(), result, err, result.ForcePause) if msgErr != nil { return nil, msgErr } // return the response message and any errors - return msg, m.requestError(result, err, false) + return msg, m.requestError(result, err, result.ForcePause) } // acceptRequest performs processing (including validation) on a new incoming request @@ -112,69 +112,70 @@ func (m *manager) receiveRestartRequest(chid datatransfer.ChannelID, incoming da log.Infof("channel %s: received restart request", chid) // process the restart message, including validations - result, err := m.restartRequest(chid, incoming) + stayPaused, result, err := m.restartRequest(chid, incoming) // generate a response message - msg, msgErr := message.ValidationResultResponse(types.RestartMessage, incoming.TransferID(), result, err) + msg, msgErr := message.ValidationResultResponse(types.RestartMessage, incoming.TransferID(), result, err, stayPaused) if msgErr != nil { return nil, msgErr } // return the response message and any errors - return msg, m.requestError(result, err, false) + return msg, m.requestError(result, err, result.ForcePause) } // restartRequest performs processing (including validation) on a incoming restart request func (m *manager) restartRequest(chid datatransfer.ChannelID, - incoming datatransfer.Request) (datatransfer.ValidationResult, error) { + incoming datatransfer.Request) (bool, datatransfer.ValidationResult, error) { // restart requests are invalid if we the initiator // (the responder must send a "restart existing channel request") initiator := chid.Initiator if m.peerID == initiator { - return datatransfer.ValidationResult{}, xerrors.New("initiator cannot be manager peer for a restart request") + return false, datatransfer.ValidationResult{}, xerrors.New("initiator cannot be manager peer for a restart request") } // valide that the request parameters match the original request // TODO: not sure this is needed -- the request parameters cannot change, // so perhaps the solution is just to ignore them in the message if err := m.validateRestartRequest(context.Background(), initiator, chid, incoming); err != nil { - return datatransfer.ValidationResult{}, xerrors.Errorf("restart request for channel %s failed validation: %w", chid, err) + return false, datatransfer.ValidationResult{}, xerrors.Errorf("restart request for channel %s failed validation: %w", chid, err) } // read the channel state chst, err := m.channels.GetByID(context.TODO(), chid) if err != nil { - return datatransfer.ValidationResult{}, err + return false, datatransfer.ValidationResult{}, err } // perform a revalidation against the last voucher - result, err := m.revalidate(chst) + result, err := m.validateRestart(chst) + stayPaused := result.LeaveRequestPaused(chst) // if an error occurred during validation return if err != nil { - return result, err + return stayPaused, result, err } // if the request is now rejected, error the channel if !result.Accepted { - return result, m.recordRejectedValidationEvents(chid, result) + return stayPaused, result, m.recordRejectedValidationEvents(chid, result) } // record the restart events if err := m.channels.Restart(chid); err != nil { - return result, xerrors.Errorf("failed to restart channel %s: %w", chid, err) + return stayPaused, result, xerrors.Errorf("failed to restart channel %s: %w", chid, err) } // record validation events if err := m.recordAcceptedValidationEvents(chst, result); err != nil { - return result, err + return stayPaused, result, err } // configure the transport voucher, err := m.decodeVoucher(incoming) if err != nil { - return result, err + return stayPaused, result, err } processor, has := m.transportConfigurers.Processor(voucher.Type()) if has { @@ -182,65 +183,17 @@ func (m *manager) restartRequest(chid datatransfer.ChannelID, transportConfigurer(chid, voucher, m.transport) } m.dataTransferNetwork.Protect(initiator, chid.String()) - return result, nil + return stayPaused, result, nil } // processUpdateVoucher handles an incoming request message with an updated voucher func (m *manager) processUpdateVoucher(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { - - // process the voucher update request, including validations - chst, result, err := m.revalidateRequest(chid, request) - - // generate a response message - messageType := types.VoucherResultMessage - if chst.Status() == datatransfer.Finalizing { - messageType = types.CompleteMessage - } - response, msgErr := message.ValidationResultResponse(messageType, chst.TransferID(), result, err) - if msgErr != nil { - return nil, msgErr - } - - // return the response message and any errors - return response, m.requestError(result, err, true) -} - -// revalidateRequest performs processing (including validation) on a incoming request with an updated voucher -func (m *manager) revalidateRequest(chid datatransfer.ChannelID, - incoming datatransfer.Request) (datatransfer.ChannelState, datatransfer.ValidationResult, error) { - // decode the voucher and save it on the channel - vouch, err := m.decodeVoucher(incoming) + vouch, err := m.decodeVoucher(request) if err != nil { - return nil, datatransfer.ValidationResult{}, err - } - err = m.channels.NewVoucher(chid, vouch) - if err != nil { - return nil, datatransfer.ValidationResult{}, err - } - - // read the channel state with the saved voucher - chst, err := m.channels.GetByID(context.TODO(), chid) - if err != nil { - return nil, datatransfer.ValidationResult{}, err - } - - // perform revalidation based on this updated voucher - result, err := m.revalidate(chst) - - // if an error occurred during validation return - if err != nil { - return chst, result, err - } - - // if the request is now rejected, error the channel - if !result.Accepted { - return chst, result, m.recordRejectedValidationEvents(chid, result) + return nil, err } - - // record validation events and return - return chst, result, m.recordAcceptedValidationEvents(chst, result) - + return nil, m.channels.NewVoucher(chid, vouch) } // receiveUpdateRequest handles an incoming request message with an updated voucher @@ -269,19 +222,16 @@ func (m *manager) receiveUpdateRequest(chid datatransfer.ChannelID, request data // ErrPause / ErrResume based off the validation result // TODO: get away from using ErrPause/ErrResume to indicate pause resume, // which would remove the need for most of this method -func (m *manager) requestError(result datatransfer.ValidationResult, resultErr error, handleResumes bool) error { +func (m *manager) requestError(result datatransfer.ValidationResult, resultErr error, stayPaused bool) error { if resultErr != nil { return resultErr } if !result.Accepted { return datatransfer.ErrRejected } - if result.LeaveRequestPaused { + if stayPaused { return datatransfer.ErrPause } - if handleResumes { - return datatransfer.ErrResume - } return nil } @@ -300,23 +250,6 @@ func (m *manager) recordRejectedValidationEvents(chid datatransfer.ChannelID, re func (m *manager) recordAcceptedValidationEvents(chst datatransfer.ChannelState, result datatransfer.ValidationResult) error { chid := chst.ChannelID() - // pause or resume the request as neccesary - if result.LeaveRequestPaused { - if !chst.Status().IsResponderPaused() { - err := m.channels.PauseResponder(chid) - if err != nil { - return err - } - } - } else { - if chst.Status().IsResponderPaused() { - err := m.channels.ResumeResponder(chid) - if err != nil { - return err - } - } - } - // record the voucher result if present if result.VoucherResult != nil { err := m.channels.NewVoucherResult(chid, result.VoucherResult) @@ -340,13 +273,31 @@ func (m *manager) recordAcceptedValidationEvents(chst datatransfer.ChannelState, return err } } + + // pause or resume the request as neccesary + if result.LeaveRequestPaused(chst) { + if !chst.Status().IsResponderPaused() { + err := m.channels.PauseResponder(chid) + if err != nil { + return err + } + } + } else { + if chst.Status().IsResponderPaused() { + err := m.channels.ResumeResponder(chid) + if err != nil { + return err + } + } + } + return nil } -// revalidate looks up the appropriate validator based on the last voucher in a channel -func (m *manager) revalidate(chst datatransfer.ChannelState) (datatransfer.ValidationResult, error) { - processor, _ := m.validatedTypes.Processor(chst.LastVoucher().Type()) +// validateRestart looks up the appropriate validator and validates a restart +func (m *manager) validateRestart(chst datatransfer.ChannelState) (datatransfer.ValidationResult, error) { + processor, _ := m.validatedTypes.Processor(chst.Voucher().Type()) validator := processor.(datatransfer.RequestValidator) - return validator.Revalidate(chst.ChannelID(), chst) + return validator.ValidateRestart(chst.ChannelID(), chst) } diff --git a/impl/responding_test.go b/impl/responding_test.go index 3ccac8e1..0fe466f7 100644 --- a/impl/responding_test.go +++ b/impl/responding_test.go @@ -115,7 +115,7 @@ func TestDataTransferResponding(t *testing.T) { "new push request pauses": { configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, LeaveRequestPaused: true, VoucherResult: testutil.NewFakeDTType()}) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, ForcePause: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -204,7 +204,7 @@ func TestDataTransferResponding(t *testing.T) { "new pull request pauses": { configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, LeaveRequestPaused: true}) + sv.StubResult(datatransfer.ValidationResult{Accepted: true, ForcePause: true}) }, verify: func(t *testing.T, h *receiverHarness) { response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) @@ -278,13 +278,11 @@ func TestDataTransferResponding(t *testing.T) { configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) - sv.ExpectSuccessRevalidation() - sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) - require.EqualError(t, err, datatransfer.ErrResume.Error()) + require.NoError(t, err) }, }, "receive pause, unpause": { @@ -358,15 +356,13 @@ func TestDataTransferResponding(t *testing.T) { datatransfer.DataReceived, datatransfer.DataLimitExceeded, datatransfer.NewVoucher, - datatransfer.ResumeResponder, datatransfer.NewVoucherResult, datatransfer.SetDataLimit, + datatransfer.ResumeResponder, }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: testutil.NewFakeDTType()}) - sv.ExpectSuccessRevalidation() - sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -381,7 +377,15 @@ func TestDataTransferResponding(t *testing.T) { require.True(t, response.IsPaused()) require.False(t, response.IsValidationResult()) response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) - require.EqualError(t, err, datatransfer.ErrResume.Error()) + require.NoError(t, err, nil) + require.Nil(t, response) + err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: testutil.NewFakeDTType()}) + require.NoError(t, err) + require.Len(t, h.transport.ResumedChannels, 1) + resCh := h.transport.ResumedChannels[0] + require.Equal(t, resCh.ChannelID, channelID(h.id, h.peers)) + response, ok = resCh.Message.(datatransfer.Response) + require.True(t, ok) require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) @@ -408,8 +412,6 @@ func TestDataTransferResponding(t *testing.T) { configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: testutil.NewFakeDTType()}) - sv.ExpectSuccessRevalidation() - sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: false, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -424,53 +426,23 @@ func TestDataTransferResponding(t *testing.T) { require.True(t, response.IsPaused()) require.False(t, response.IsValidationResult()) response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) - require.EqualError(t, err, datatransfer.ErrRejected.Error()) - require.False(t, response.Accepted()) - require.Equal(t, response.TransferID(), h.id) - require.False(t, response.IsUpdate()) - require.False(t, response.IsCancel()) - require.False(t, response.IsPaused()) - require.True(t, response.IsValidationResult()) - require.False(t, response.EmptyVoucherResult()) - }, - }, - "validate and revalidate with validation error on second voucher": { - expectedEvents: []datatransfer.EventCode{ - datatransfer.Open, - datatransfer.Accept, - datatransfer.NewVoucherResult, - datatransfer.SetDataLimit, - datatransfer.DataReceivedProgress, - datatransfer.DataReceived, - datatransfer.DataLimitExceeded, - datatransfer.NewVoucher, - }, - configureValidator: func(sv *testutil.StubbedValidator) { - sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: testutil.NewFakeDTType()}) - sv.ExpectErrorRevalidation() - }, - verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) - err := h.transport.EventHandler.OnDataReceived(channelID(h.id, h.peers), cidlink.Link{Cid: testutil.GenerateCids(1)[0]}, 12345, 1, true) - require.EqualError(t, err, datatransfer.ErrPause.Error()) - require.Len(t, h.network.SentMessages, 1) - response, ok := h.network.SentMessages[0].Message.(datatransfer.Response) + require.NoError(t, err, nil) + require.Nil(t, response) + err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: false, VoucherResult: testutil.NewFakeDTType()}) + require.NoError(t, err) + require.Len(t, h.transport.ClosedChannels, 1) + require.Equal(t, h.transport.ClosedChannels[0], channelID(h.id, h.peers)) + require.Len(t, h.network.SentMessages, 2) + sentMsg := h.network.SentMessages[1] + response, ok = sentMsg.Message.(datatransfer.Response) require.True(t, ok) - require.Equal(t, response.TransferID(), h.id) - require.True(t, response.IsUpdate()) - require.False(t, response.IsCancel()) - require.True(t, response.IsPaused()) - require.False(t, response.IsValidationResult()) - response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) - require.EqualError(t, err, "something went wrong") require.False(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) require.False(t, response.IsCancel()) require.False(t, response.IsPaused()) require.True(t, response.IsValidationResult()) - require.True(t, response.EmptyVoucherResult()) + require.False(t, response.EmptyVoucherResult()) }, }, "validate and revalidate successfully, pull": { @@ -483,15 +455,13 @@ func TestDataTransferResponding(t *testing.T) { datatransfer.DataQueued, datatransfer.DataLimitExceeded, datatransfer.NewVoucher, - datatransfer.ResumeResponder, datatransfer.NewVoucherResult, datatransfer.SetDataLimit, + datatransfer.ResumeResponder, }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: testutil.NewFakeDTType()}) - sv.ExpectSuccessRevalidation() - sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) @@ -509,7 +479,15 @@ func TestDataTransferResponding(t *testing.T) { require.True(t, response.IsPaused()) require.False(t, response.IsValidationResult()) response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) - require.EqualError(t, err, datatransfer.ErrResume.Error()) + require.NoError(t, err, nil) + require.Nil(t, response) + err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: testutil.NewFakeDTType()}) + require.NoError(t, err) + require.Len(t, h.transport.ResumedChannels, 1) + resCh := h.transport.ResumedChannels[0] + require.Equal(t, resCh.ChannelID, channelID(h.id, h.peers)) + response, ok = resCh.Message.(datatransfer.Response) + require.True(t, ok) require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) @@ -527,16 +505,14 @@ func TestDataTransferResponding(t *testing.T) { datatransfer.SetRequiresFinalization, datatransfer.BeginFinalizing, datatransfer.NewVoucher, - datatransfer.ResumeResponder, datatransfer.NewVoucherResult, datatransfer.SetRequiresFinalization, + datatransfer.ResumeResponder, datatransfer.CleanupComplete, }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() sv.StubResult(datatransfer.ValidationResult{Accepted: true, RequiresFinalization: true, VoucherResult: testutil.NewFakeDTType()}) - sv.ExpectSuccessRevalidation() - sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) @@ -555,7 +531,14 @@ func TestDataTransferResponding(t *testing.T) { require.True(t, response.IsValidationResult()) require.True(t, response.EmptyVoucherResult()) response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) - require.EqualError(t, err, datatransfer.ErrResume.Error()) + require.NoError(t, err, nil) + require.Nil(t, response) + err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + require.NoError(t, err) + require.Len(t, h.network.SentMessages, 2) + sentMsg := h.network.SentMessages[1] + response, ok = sentMsg.Message.(datatransfer.Response) + require.True(t, ok) require.Equal(t, response.TransferID(), h.id) require.True(t, response.IsValidationResult()) require.False(t, response.IsPaused()) @@ -715,8 +698,8 @@ func TestDataTransferRestartResponding(t *testing.T) { configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) - sv.ExpectSuccessRevalidation() - sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + sv.ExpectSuccessValidateRestart() + sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming push @@ -776,8 +759,8 @@ func TestDataTransferRestartResponding(t *testing.T) { configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) - sv.ExpectSuccessRevalidation() - sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + sv.ExpectSuccessValidateRestart() + sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull @@ -848,8 +831,8 @@ func TestDataTransferRestartResponding(t *testing.T) { configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) - sv.ExpectSuccessRevalidation() - sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: false}) + sv.ExpectSuccessValidateRestart() + sv.StubRestartResult(datatransfer.ValidationResult{Accepted: false}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull diff --git a/impl/restart.go b/impl/restart.go index 536617a8..9efe0728 100644 --- a/impl/restart.go +++ b/impl/restart.go @@ -32,7 +32,7 @@ const ( ) func (m *manager) restartManagerPeerReceivePush(ctx context.Context, channel datatransfer.ChannelState) error { - result, err := m.revalidate(channel) + result, err := m.validateRestart(channel) if err != nil { return xerrors.Errorf("failed to restart channel, validation error: %w", err) } @@ -52,7 +52,7 @@ func (m *manager) restartManagerPeerReceivePush(ctx context.Context, channel dat } func (m *manager) restartManagerPeerReceivePull(ctx context.Context, channel datatransfer.ChannelState) error { - result, err := m.revalidate(channel) + result, err := m.validateRestart(channel) if err != nil { return xerrors.Errorf("failed to restart channel, validation error: %w", err) } diff --git a/impl/restart_integration_test.go b/impl/restart_integration_test.go index b1f0f4da..453aac80 100644 --- a/impl/restart_integration_test.go +++ b/impl/restart_integration_test.go @@ -135,8 +135,8 @@ func TestRestartPush(t *testing.T) { // START DATA TRANSFER INSTANCES rh.sv.ExpectSuccessPush() rh.sv.StubResult(datatransfer.ValidationResult{Accepted: true}) - rh.sv.ExpectSuccessRevalidation() - rh.sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) + rh.sv.ExpectSuccessValidateRestart() + rh.sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt1) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt2) @@ -383,8 +383,8 @@ func TestRestartPull(t *testing.T) { // START DATA TRANSFER INSTANCES rh.sv.ExpectSuccessPull() rh.sv.StubResult(datatransfer.ValidationResult{Accepted: true}) - rh.sv.ExpectSuccessRevalidation() - rh.sv.StubRevalidationResult(datatransfer.ValidationResult{Accepted: true}) + rh.sv.ExpectSuccessValidateRestart() + rh.sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt1) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt2) diff --git a/manager.go b/manager.go index 61a26e21..b291dc11 100644 --- a/manager.go +++ b/manager.go @@ -16,9 +16,9 @@ type ValidationResult struct { // VoucherResult provides information to the other party about what happened // with the voucher VoucherResult - // LeaveRequestPaused indicates whether the request should stay paused - // even if the request was accepted - LeaveRequestPaused bool + // ForcePause indicates whether the request should be paused, regardless + // of data limit and finalization status + ForcePause bool // DataLimit specifies how much data this voucher is good for. When the amount // of data specified is reached (or shortly after), the request will pause // pending revalidation. 0 indicates no limit. @@ -28,6 +28,23 @@ type ValidationResult struct { RequiresFinalization bool } +// LeaveRequestPaused indicates whether all conditions are met to resume a request +func (vr ValidationResult) LeaveRequestPaused(chst ChannelState) bool { + if vr.ForcePause { + return true + } + if vr.RequiresFinalization && chst.Status().InFinalization() { + return true + } + var limitFactor uint64 + if chst.IsPull() { + limitFactor = chst.Queued() + } else { + limitFactor = chst.Received() + } + return vr.DataLimit != 0 && limitFactor >= vr.DataLimit +} + // RequestValidator is an interface implemented by the client of the // data transfer module to validate requests type RequestValidator interface { @@ -54,12 +71,12 @@ type RequestValidator interface { baseCid cid.Cid, selector ipld.Node) (ValidationResult, error) - // Revalidate revalidates a request with a new voucher + // ValidateRestart validates restarting a request // -- All information about the validation operation is contained in ValidationResult, // including if it was rejected. Information about why a rejection occurred should be embedded // in the VoucherResult. // -- error indicates something went wrong with the actual process of trying to validate - Revalidate(channelID ChannelID, channel ChannelState) (ValidationResult, error) + ValidateRestart(channelID ChannelID, channel ChannelState) (ValidationResult, error) } // TransportConfigurer provides a mechanism to provide transport specific configuration for a given voucher type @@ -108,6 +125,10 @@ type Manager interface { // send information from the responder to update the initiator on the state of their voucher SendVoucherResult(ctx context.Context, chid ChannelID, voucher VoucherResult) error + // Update the validation status for a given channel, to change data limits, finalization, accepted status, and pause state + // and send new voucher results as + UpdateValidationStatus(ctx context.Context, chid ChannelID, validationResult ValidationResult) error + // close an open channel (effectively a cancel) CloseDataTransferChannel(ctx context.Context, chid ChannelID) error diff --git a/message/message.go b/message/message.go index e8acac95..438e6913 100644 --- a/message/message.go +++ b/message/message.go @@ -11,6 +11,7 @@ var VoucherRequest = message1_1.VoucherRequest // DEPRECATED: Use ValidationResultResponse var RestartResponse = message1_1.RestartResponse + var ValidationResultResponse = message1_1.ValidationResultResponse // DEPRECATED: Use ValidationResultResponse diff --git a/message/message1_1prime/message.go b/message/message1_1prime/message.go index b0f1471a..cdf19843 100644 --- a/message/message1_1prime/message.go +++ b/message/message1_1prime/message.go @@ -111,7 +111,8 @@ func ValidationResultResponse( messageType types.MessageType, id datatransfer.TransferID, validationResult datatransfer.ValidationResult, - validationErr error) (datatransfer.Response, error) { + validationErr error, + paused bool) (datatransfer.Response, error) { voucherResultType := datatransfer.EmptyTypeIdentifier if validationResult.VoucherResult != nil { voucherResultType = validationResult.VoucherResult.Type() @@ -125,7 +126,7 @@ func ValidationResultResponse( // Validation errors vs rejections RequestAccepted: validationErr == nil && validationResult.Accepted, MessageType: uint64(messageType), - Paused: validationResult.LeaveRequestPaused, + Paused: paused, TransferId: uint64(id), VoucherTypeIdentifier: voucherResultType, VoucherResultPtr: &vnode, diff --git a/network/libp2p_impl_test.go b/network/libp2p_impl_test.go index 527bfffd..2747883c 100644 --- a/network/libp2p_impl_test.go +++ b/network/libp2p_impl_test.go @@ -132,7 +132,7 @@ func TestMessageSendAndReceive(t *testing.T) { accepted := false id := datatransfer.TransferID(rand.Int31()) voucherResult := testutil.NewFakeDTType() - response, err := message.ValidationResultResponse(types.NewMessage, id, datatransfer.ValidationResult{Accepted: accepted, VoucherResult: voucherResult}, nil) + response, err := message.ValidationResultResponse(types.NewMessage, id, datatransfer.ValidationResult{Accepted: accepted, VoucherResult: voucherResult}, nil, false) require.NoError(t, err) require.NoError(t, dtnet2.SendMessage(ctx, host1.ID(), response)) diff --git a/testutil/stubbedvalidator.go b/testutil/stubbedvalidator.go index 0e88c7f1..30a8bab7 100644 --- a/testutil/stubbedvalidator.go +++ b/testutil/stubbedvalidator.go @@ -103,37 +103,37 @@ func (sv *StubbedValidator) VerifyExpectations(t *testing.T) { } } -func (sv *StubbedValidator) Revalidate(chid datatransfer.ChannelID, channelState datatransfer.ChannelState) (datatransfer.ValidationResult, error) { +func (sv *StubbedValidator) ValidateRestart(chid datatransfer.ChannelID, channelState datatransfer.ChannelState) (datatransfer.ValidationResult, error) { sv.didRevalidate = true - sv.RevalidationsReceived = append(sv.RevalidationsReceived, ReceivedRevalidation{chid, channelState}) + sv.RevalidationsReceived = append(sv.RevalidationsReceived, ReceivedRestartValidation{chid, channelState}) return sv.revalidationResult, sv.revalidationError } -// StubRevalidationResult returns the given voucher result when a call is made to Revalidate -func (sv *StubbedValidator) StubRevalidationResult(voucherResult datatransfer.ValidationResult) { +// StubRestartResult returns the given voucher result when a call is made to ValidateRestart +func (sv *StubbedValidator) StubRestartResult(voucherResult datatransfer.ValidationResult) { sv.revalidationResult = voucherResult } -// StubErrorRevalidation sets Revalidate to error -func (sv *StubbedValidator) StubErrorRevalidation() { +// StubErrorValidateRestart sets ValidateRestart to error +func (sv *StubbedValidator) StubErrorValidateRestart() { sv.revalidationError = errors.New("something went wrong") } -// StubSuccessRevalidation sets Revalidate to succeed -func (sv *StubbedValidator) StubSuccessRevalidation() { +// StubSuccessValidateRestart sets ValidateRestart to succeed +func (sv *StubbedValidator) StubSuccessValidateRestart() { sv.revalidationError = nil } -// ExpectErrorRevalidation expects Revalidate to error -func (sv *StubbedValidator) ExpectErrorRevalidation() { +// ExpectErrorValidateRestart expects ValidateRestart to error +func (sv *StubbedValidator) ExpectErrorValidateRestart() { sv.expectRevalidate = true - sv.StubErrorRevalidation() + sv.StubErrorValidateRestart() } -// ExpectSuccessRevalidation expects Revalidate to succeed -func (sv *StubbedValidator) ExpectSuccessRevalidation() { +// ExpectSuccessValidateRestart expects ValidateRestart to succeed +func (sv *StubbedValidator) ExpectSuccessValidateRestart() { sv.expectRevalidate = true - sv.StubSuccessRevalidation() + sv.StubSuccessValidateRestart() } // ReceivedValidation records a call to either ValidatePush or ValidatePull @@ -145,8 +145,8 @@ type ReceivedValidation struct { Selector ipld.Node } -// ReceivedRevalidation records a call to Revalidate -type ReceivedRevalidation struct { +// ReceivedRestartValidation records a call to ValidateRestart +type ReceivedRestartValidation struct { ChannelID datatransfer.ChannelID ChannelState datatransfer.ChannelState } @@ -165,7 +165,7 @@ type StubbedValidator struct { pullError error revalidationError error ValidationsReceived []ReceivedValidation - RevalidationsReceived []ReceivedRevalidation + RevalidationsReceived []ReceivedRestartValidation } var _ datatransfer.RequestValidator = (*StubbedValidator)(nil) From 282c7ee4b5a6d10c571163d023db4c7822c01450 Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Tue, 31 May 2022 18:37:58 -0700 Subject: [PATCH 04/18] chore(message): delete old message format code (#330) --- message/message1_1/message.go | 196 ------- message/message1_1/message_test.go | 549 ------------------ message/message1_1/transfer_message.go | 59 -- .../message1_1/transfer_message_cbor_gen.go | 179 ------ message/message1_1/transfer_request.go | 166 ------ .../message1_1/transfer_request_cbor_gen.go | 397 ------------- message/message1_1/transfer_request_test.go | 41 -- message/message1_1/transfer_response.go | 127 ---- .../message1_1/transfer_response_cbor_gen.go | 265 --------- message/message1_1/transfer_response_test.go | 35 -- 10 files changed, 2014 deletions(-) delete mode 100644 message/message1_1/message.go delete mode 100644 message/message1_1/message_test.go delete mode 100644 message/message1_1/transfer_message.go delete mode 100644 message/message1_1/transfer_message_cbor_gen.go delete mode 100644 message/message1_1/transfer_request.go delete mode 100644 message/message1_1/transfer_request_cbor_gen.go delete mode 100644 message/message1_1/transfer_request_test.go delete mode 100644 message/message1_1/transfer_response.go delete mode 100644 message/message1_1/transfer_response_cbor_gen.go delete mode 100644 message/message1_1/transfer_response_test.go diff --git a/message/message1_1/message.go b/message/message1_1/message.go deleted file mode 100644 index c52471f1..00000000 --- a/message/message1_1/message.go +++ /dev/null @@ -1,196 +0,0 @@ -package message1_1 - -import ( - "bytes" - "io" - - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/codec/dagcbor" - "github.com/ipld/go-ipld-prime/datamodel" - cborgen "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/encoding" - "github.com/filecoin-project/go-data-transfer/v2/message/types" -) - -// NewRequest generates a new request for the data transfer protocol -func NewRequest(id datatransfer.TransferID, isRestart bool, isPull bool, vtype datatransfer.TypeIdentifier, voucher encoding.Encodable, baseCid cid.Cid, selector ipld.Node) (datatransfer.Request, error) { - vbytes, err := encoding.Encode(voucher) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) - } - if baseCid == cid.Undef { - return nil, xerrors.Errorf("base CID must be defined") - } - selBytes, err := encoding.Encode(selector) - if err != nil { - return nil, xerrors.Errorf("Error encoding selector") - } - - var typ uint64 - if isRestart { - typ = uint64(types.RestartMessage) - } else { - typ = uint64(types.NewMessage) - } - - return &TransferRequest1_1{ - Type: typ, - Pull: isPull, - Vouch: &cborgen.Deferred{Raw: vbytes}, - Stor: &cborgen.Deferred{Raw: selBytes}, - BCid: &baseCid, - VTyp: vtype, - XferID: uint64(id), - }, nil -} - -// RestartExistingChannelRequest creates a request to ask the other side to restart an existing channel -func RestartExistingChannelRequest(channelId datatransfer.ChannelID) datatransfer.Request { - - return &TransferRequest1_1{Type: uint64(types.RestartExistingChannelRequestMessage), - RestartChannel: channelId} -} - -// CancelRequest request generates a request to cancel an in progress request -func CancelRequest(id datatransfer.TransferID) datatransfer.Request { - return &TransferRequest1_1{ - Type: uint64(types.CancelMessage), - XferID: uint64(id), - } -} - -// UpdateRequest generates a new request update -func UpdateRequest(id datatransfer.TransferID, isPaused bool) datatransfer.Request { - return &TransferRequest1_1{ - Type: uint64(types.UpdateMessage), - Paus: isPaused, - XferID: uint64(id), - } -} - -// VoucherRequest generates a new request for the data transfer protocol -func VoucherRequest(id datatransfer.TransferID, vtype datatransfer.TypeIdentifier, voucher encoding.Encodable) (datatransfer.Request, error) { - vbytes, err := encoding.Encode(voucher) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) - } - return &TransferRequest1_1{ - Type: uint64(types.VoucherMessage), - Vouch: &cborgen.Deferred{Raw: vbytes}, - VTyp: vtype, - XferID: uint64(id), - }, nil -} - -// RestartResponse builds a new Data Transfer response -func RestartResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResultType datatransfer.TypeIdentifier, voucherResult encoding.Encodable) (datatransfer.Response, error) { - vbytes, err := encoding.Encode(voucherResult) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) - } - return &TransferResponse1_1{ - Acpt: accepted, - Type: uint64(types.RestartMessage), - Paus: isPaused, - XferID: uint64(id), - VTyp: voucherResultType, - VRes: &cborgen.Deferred{Raw: vbytes}, - }, nil -} - -// NewResponse builds a new Data Transfer response -func NewResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResultType datatransfer.TypeIdentifier, voucherResult encoding.Encodable) (datatransfer.Response, error) { - vbytes, err := encoding.Encode(voucherResult) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) - } - return &TransferResponse1_1{ - Acpt: accepted, - Type: uint64(types.NewMessage), - Paus: isPaused, - XferID: uint64(id), - VTyp: voucherResultType, - VRes: &cborgen.Deferred{Raw: vbytes}, - }, nil -} - -// VoucherResultResponse builds a new response for a voucher result -func VoucherResultResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResultType datatransfer.TypeIdentifier, voucherResult encoding.Encodable) (datatransfer.Response, error) { - vbytes, err := encoding.Encode(voucherResult) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) - } - return &TransferResponse1_1{ - Acpt: accepted, - Type: uint64(types.VoucherResultMessage), - Paus: isPaused, - XferID: uint64(id), - VTyp: voucherResultType, - VRes: &cborgen.Deferred{Raw: vbytes}, - }, nil -} - -// UpdateResponse returns a new update response -func UpdateResponse(id datatransfer.TransferID, isPaused bool) datatransfer.Response { - return &TransferResponse1_1{ - Type: uint64(types.UpdateMessage), - Paus: isPaused, - XferID: uint64(id), - } -} - -// CancelResponse makes a new cancel response message -func CancelResponse(id datatransfer.TransferID) datatransfer.Response { - return &TransferResponse1_1{ - Type: uint64(types.CancelMessage), - XferID: uint64(id), - } -} - -// CompleteResponse returns a new complete response message -func CompleteResponse(id datatransfer.TransferID, isAccepted bool, isPaused bool, voucherResultType datatransfer.TypeIdentifier, voucherResult encoding.Encodable) (datatransfer.Response, error) { - vbytes, err := encoding.Encode(voucherResult) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) - } - return &TransferResponse1_1{ - Type: uint64(types.CompleteMessage), - Acpt: isAccepted, - Paus: isPaused, - VTyp: voucherResultType, - VRes: &cborgen.Deferred{Raw: vbytes}, - XferID: uint64(id), - }, nil -} - -// FromNet can read a network stream to deserialize a GraphSyncMessage -func FromNet(r io.Reader) (datatransfer.Message, error) { - tresp := TransferMessage1_1{} - err := tresp.UnmarshalCBOR(r) - if err != nil { - return nil, err - } - - if (tresp.IsRequest() && tresp.Request == nil) || (!tresp.IsRequest() && tresp.Response == nil) { - return nil, xerrors.Errorf("invalid/malformed message") - } - - if tresp.IsRequest() { - return tresp.Request, nil - } - return tresp.Response, nil -} - -// FromNet can read a network stream to deserialize a GraphSyncMessage -func FromIPLD(nd datamodel.Node) (datatransfer.Message, error) { - buf := new(bytes.Buffer) - err := dagcbor.Encode(nd, buf) - if err != nil { - return nil, err - } - return FromNet(buf) -} diff --git a/message/message1_1/message_test.go b/message/message1_1/message_test.go deleted file mode 100644 index 3e69fa9d..00000000 --- a/message/message1_1/message_test.go +++ /dev/null @@ -1,549 +0,0 @@ -package message1_1_test - -import ( - "bytes" - "encoding/hex" - "fmt" - "math/rand" - "testing" - - "github.com/ipfs/go-cid" - basicnode "github.com/ipld/go-ipld-prime/node/basic" - "github.com/ipld/go-ipld-prime/traversal/selector/builder" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/message/message1_1" - "github.com/filecoin-project/go-data-transfer/v2/testutil" -) - -func TestNewRequest(t *testing.T) { - baseCid := testutil.GenerateCids(1)[0] - selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() - isPull := true - id := datatransfer.TransferID(rand.Int31()) - voucher := testutil.NewFakeDTType() - request, err := message1_1.NewRequest(id, false, isPull, voucher.Type(), voucher, baseCid, selector) - require.NoError(t, err) - assert.Equal(t, id, request.TransferID()) - assert.False(t, request.IsCancel()) - assert.False(t, request.IsUpdate()) - assert.True(t, request.IsPull()) - assert.True(t, request.IsRequest()) - assert.Equal(t, baseCid.String(), request.BaseCid().String()) - testutil.AssertFakeDTVoucher(t, request, voucher) - receivedSelector, err := request.Selector() - require.NoError(t, err) - require.Equal(t, selector, receivedSelector) - // Sanity check to make sure we can cast to datatransfer.Message - msg, ok := request.(datatransfer.Message) - require.True(t, ok) - - assert.True(t, msg.IsRequest()) - assert.Equal(t, request.TransferID(), msg.TransferID()) - assert.False(t, msg.IsRestart()) - assert.True(t, msg.IsNew()) -} - -func TestRestartRequest(t *testing.T) { - baseCid := testutil.GenerateCids(1)[0] - selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() - isPull := true - id := datatransfer.TransferID(rand.Int31()) - voucher := testutil.NewFakeDTType() - request, err := message1_1.NewRequest(id, true, isPull, voucher.Type(), voucher, baseCid, selector) - require.NoError(t, err) - assert.Equal(t, id, request.TransferID()) - assert.False(t, request.IsCancel()) - assert.False(t, request.IsUpdate()) - assert.True(t, request.IsPull()) - assert.True(t, request.IsRequest()) - assert.Equal(t, baseCid.String(), request.BaseCid().String()) - testutil.AssertFakeDTVoucher(t, request, voucher) - receivedSelector, err := request.Selector() - require.NoError(t, err) - require.Equal(t, selector, receivedSelector) - // Sanity check to make sure we can cast to datatransfer.Message - msg, ok := request.(datatransfer.Message) - require.True(t, ok) - - assert.True(t, msg.IsRequest()) - assert.Equal(t, request.TransferID(), msg.TransferID()) - assert.True(t, msg.IsRestart()) - assert.False(t, msg.IsNew()) -} - -func TestRestartExistingChannelRequest(t *testing.T) { - t.Run("round-trip", func(t *testing.T) { - peers := testutil.GeneratePeers(2) - tid := uint64(1) - chid := datatransfer.ChannelID{Initiator: peers[0], - Responder: peers[1], ID: datatransfer.TransferID(tid)} - req := message1_1.RestartExistingChannelRequest(chid) - - wbuf := new(bytes.Buffer) - require.NoError(t, req.ToNet(wbuf)) - - desMsg, err := message1_1.FromNet(wbuf) - require.NoError(t, err) - req, ok := (desMsg).(datatransfer.Request) - require.True(t, ok) - require.True(t, req.IsRestartExistingChannelRequest()) - achid, err := req.RestartChannelId() - require.NoError(t, err) - require.Equal(t, chid, achid) - }) - t.Run("ipld-prime compat", func(t *testing.T) { - msg, _ := hex.DecodeString("a36449735271f56752657175657374aa6442436964f66454797065076450617573f46450617274f46450756c6cf46453746f72f665566f756368f664565479706066586665724944006e526573746172744368616e6e656c83613161320168526573706f6e7365f6") - desMsg, err := message1_1.FromNet(bytes.NewReader(msg)) - require.NoError(t, err) - req, ok := (desMsg).(datatransfer.Request) - require.True(t, ok) - require.True(t, req.IsRestartExistingChannelRequest()) - achid, err := req.RestartChannelId() - require.NoError(t, err) - tid := uint64(1) - chid := datatransfer.ChannelID{Initiator: peer.ID("1"), - Responder: peer.ID("2"), ID: datatransfer.TransferID(tid)} - require.Equal(t, chid, achid) - }) -} - -func TestTransferRequest_MarshalCBOR(t *testing.T) { - // sanity check MarshalCBOR does its thing w/o error - req, err := NewTestTransferRequest() - require.NoError(t, err) - wbuf := new(bytes.Buffer) - require.NoError(t, req.MarshalCBOR(wbuf)) - assert.Greater(t, wbuf.Len(), 0) -} -func TestTransferRequest_UnmarshalCBOR(t *testing.T) { - t.Run("round-trip", func(t *testing.T) { - req, err := NewTestTransferRequest() - require.NoError(t, err) - wbuf := new(bytes.Buffer) - // use ToNet / message1_1.FromNet - require.NoError(t, req.ToNet(wbuf)) - - desMsg, err := message1_1.FromNet(wbuf) - require.NoError(t, err) - - // Verify round-trip - assert.Equal(t, req.TransferID(), desMsg.TransferID()) - assert.Equal(t, req.IsRequest(), desMsg.IsRequest()) - - desReq := desMsg.(datatransfer.Request) - assert.Equal(t, req.IsPull(), desReq.IsPull()) - assert.Equal(t, req.IsCancel(), desReq.IsCancel()) - assert.Equal(t, req.BaseCid(), desReq.BaseCid()) - testutil.AssertEqualFakeDTVoucher(t, &req, desReq) - testutil.AssertEqualSelector(t, &req, desReq) - }) - t.Run("ipld-prime compat", func(t *testing.T) { - req, err := NewTestTransferRequest() - require.NoError(t, err) - - msg, _ := hex.DecodeString("a36449735271f56752657175657374aa6442436964d82a58230012204bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a6450617274f46450617573f46450756c6cf46453746f72a1612ea064547970650064565479706a46616b6544545479706565566f756368817864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e35665866657249441a4d6582216e526573746172744368616e6e656c8360600068526573706f6e7365f6") - desMsg, err := message1_1.FromNet(bytes.NewReader(msg)) - require.NoError(t, err) - - // Verify round-trip - assert.Equal(t, datatransfer.TransferID(1298498081), desMsg.TransferID()) - assert.Equal(t, req.IsRequest(), desMsg.IsRequest()) - - desReq := desMsg.(datatransfer.Request) - assert.Equal(t, req.IsPull(), desReq.IsPull()) - assert.Equal(t, req.IsCancel(), desReq.IsCancel()) - c, _ := cid.Parse("QmTTA2daxGqo5denp6SwLzzkLJm3fuisYEi9CoWsuHpzfb") - assert.Equal(t, c, desReq.BaseCid()) - testutil.AssertEqualFakeDTVoucher(t, &req, desReq) - testutil.AssertEqualSelector(t, &req, desReq) - }) -} - -func TestResponses(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - voucherResult := testutil.NewFakeDTType() - response, err := message1_1.NewResponse(id, false, true, voucherResult.Type(), voucherResult) // not accepted - require.NoError(t, err) - assert.Equal(t, response.TransferID(), id) - assert.False(t, response.Accepted()) - assert.True(t, response.IsNew()) - assert.False(t, response.IsUpdate()) - assert.True(t, response.IsPaused()) - assert.False(t, response.IsRequest()) - testutil.AssertFakeDTVoucherResult(t, response, voucherResult) - // Sanity check to make sure we can cast to datatransfer.Message - msg, ok := response.(datatransfer.Message) - require.True(t, ok) - - assert.False(t, msg.IsRequest()) - assert.True(t, msg.IsNew()) - assert.False(t, msg.IsUpdate()) - assert.True(t, msg.IsPaused()) - assert.Equal(t, response.TransferID(), msg.TransferID()) -} - -func TestTransferResponse_MarshalCBOR(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - voucherResult := testutil.NewFakeDTType() - response, err := message1_1.NewResponse(id, true, false, voucherResult.Type(), voucherResult) // accepted - require.NoError(t, err) - - // sanity check that we can marshal data - wbuf := new(bytes.Buffer) - require.NoError(t, response.ToNet(wbuf)) - assert.Greater(t, wbuf.Len(), 0) -} - -func TestTransferResponse_UnmarshalCBOR(t *testing.T) { - t.Run("round-trip", func(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - voucherResult := testutil.NewFakeDTType() - response, err := message1_1.NewResponse(id, true, false, voucherResult.Type(), voucherResult) // accepted - require.NoError(t, err) - - wbuf := new(bytes.Buffer) - require.NoError(t, response.ToNet(wbuf)) - - // verify round trip - desMsg, err := message1_1.FromNet(wbuf) - require.NoError(t, err) - assert.False(t, desMsg.IsRequest()) - assert.True(t, desMsg.IsNew()) - assert.False(t, desMsg.IsUpdate()) - assert.False(t, desMsg.IsPaused()) - assert.Equal(t, id, desMsg.TransferID()) - - desResp, ok := desMsg.(datatransfer.Response) - require.True(t, ok) - assert.True(t, desResp.Accepted()) - assert.True(t, desResp.IsNew()) - assert.False(t, desResp.IsUpdate()) - assert.False(t, desMsg.IsPaused()) - testutil.AssertFakeDTVoucherResult(t, desResp, voucherResult) - }) - t.Run("ipld-prime compat", func(t *testing.T) { - voucherResult := testutil.NewFakeDTType() - voucherResult.Data = "\xf5_\xf8\xf1%\b\xb6>\xf2\xbf\xec\xa7Uz\xe9\r\xf61\x1a^\xc1c\x1bJ\x1f\xa8C1\v\xd9ç\x10\xea\xac塽\xd7*п\xe0Iw\x1c\x11\xe7V3\x8b\xd98e\xe6E\xf1\xad웜\x99\xef@\u007f\xbdOƅ\x9ey\x04ŭ}ɽ\x10\xa5\xcc\x16\x97=[(\xec\x1am\xd4=\x9f\x82\xf9\xf1\x8c=\x03A\x8e5" - - msg, _ := hex.DecodeString("a36449735271f46752657175657374f668526573706f6e7365a66441637074f56450617573f46454797065006456526573817864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e3564565479706a46616b65445454797065665866657249441a4d658221") - desMsg, err := message1_1.FromNet(bytes.NewReader(msg)) - require.NoError(t, err) - assert.False(t, desMsg.IsRequest()) - assert.True(t, desMsg.IsNew()) - assert.False(t, desMsg.IsUpdate()) - assert.False(t, desMsg.IsPaused()) - assert.Equal(t, datatransfer.TransferID(1298498081), desMsg.TransferID()) - - desResp, ok := desMsg.(datatransfer.Response) - require.True(t, ok) - assert.True(t, desResp.Accepted()) - assert.True(t, desResp.IsNew()) - assert.False(t, desResp.IsUpdate()) - assert.False(t, desMsg.IsPaused()) - testutil.AssertFakeDTVoucherResult(t, desResp, voucherResult) - }) -} - -func TestRequestCancel(t *testing.T) { - t.Run("round-trip", func(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - req := message1_1.CancelRequest(id) - require.Equal(t, req.TransferID(), id) - require.True(t, req.IsRequest()) - require.True(t, req.IsCancel()) - require.False(t, req.IsUpdate()) - - wbuf := new(bytes.Buffer) - require.NoError(t, req.ToNet(wbuf)) - - deserialized, err := message1_1.FromNet(wbuf) - require.NoError(t, err) - - deserializedRequest, ok := deserialized.(datatransfer.Request) - require.True(t, ok) - require.Equal(t, deserializedRequest.TransferID(), req.TransferID()) - require.Equal(t, deserializedRequest.IsCancel(), req.IsCancel()) - require.Equal(t, deserializedRequest.IsRequest(), req.IsRequest()) - require.Equal(t, deserializedRequest.IsUpdate(), req.IsUpdate()) - }) - t.Run("ipld-prime compat", func(t *testing.T) { - id := datatransfer.TransferID(1298498081) - req := message1_1.CancelRequest(id) - require.Equal(t, req.TransferID(), id) - require.True(t, req.IsRequest()) - require.True(t, req.IsCancel()) - require.False(t, req.IsUpdate()) - - msg, _ := hex.DecodeString("a36449735271f56752657175657374aa6442436964f66450617274f46450617573f46450756c6cf46453746f72f664547970650264565479706065566f756368f6665866657249441a4d6582216e526573746172744368616e6e656c8360600068526573706f6e7365f6") - deserialized, err := message1_1.FromNet(bytes.NewReader(msg)) - require.NoError(t, err) - - deserializedRequest, ok := deserialized.(datatransfer.Request) - require.True(t, ok) - require.Equal(t, deserializedRequest.TransferID(), req.TransferID()) - require.Equal(t, deserializedRequest.IsCancel(), req.IsCancel()) - require.Equal(t, deserializedRequest.IsRequest(), req.IsRequest()) - require.Equal(t, deserializedRequest.IsUpdate(), req.IsUpdate()) - }) -} - -func TestRequestUpdate(t *testing.T) { - t.Run("round-trip", func(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - req := message1_1.UpdateRequest(id, true) - require.Equal(t, req.TransferID(), id) - require.True(t, req.IsRequest()) - require.False(t, req.IsCancel()) - require.True(t, req.IsUpdate()) - require.True(t, req.IsPaused()) - - wbuf := new(bytes.Buffer) - require.NoError(t, req.ToNet(wbuf)) - - deserialized, err := message1_1.FromNet(wbuf) - require.NoError(t, err) - - deserializedRequest, ok := deserialized.(datatransfer.Request) - require.True(t, ok) - require.Equal(t, deserializedRequest.TransferID(), req.TransferID()) - require.Equal(t, deserializedRequest.IsCancel(), req.IsCancel()) - require.Equal(t, deserializedRequest.IsRequest(), req.IsRequest()) - require.Equal(t, deserializedRequest.IsUpdate(), req.IsUpdate()) - require.Equal(t, deserializedRequest.IsPaused(), req.IsPaused()) - }) - t.Run("ipld-prime compat", func(t *testing.T) { - id := datatransfer.TransferID(1298498081) - req := message1_1.UpdateRequest(id, true) - - msg, _ := hex.DecodeString("a36449735271f56752657175657374aa6442436964f66450617274f46450617573f56450756c6cf46453746f72f664547970650164565479706065566f756368f6665866657249441a4d6582216e526573746172744368616e6e656c8360600068526573706f6e7365f6") - deserialized, err := message1_1.FromNet(bytes.NewReader(msg)) - require.NoError(t, err) - - deserializedRequest, ok := deserialized.(datatransfer.Request) - require.True(t, ok) - require.Equal(t, deserializedRequest.TransferID(), req.TransferID()) - require.Equal(t, deserializedRequest.IsCancel(), req.IsCancel()) - require.Equal(t, deserializedRequest.IsRequest(), req.IsRequest()) - require.Equal(t, deserializedRequest.IsUpdate(), req.IsUpdate()) - require.Equal(t, deserializedRequest.IsPaused(), req.IsPaused()) - }) -} - -func TestUpdateResponse(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - response := message1_1.UpdateResponse(id, true) // not accepted - assert.Equal(t, response.TransferID(), id) - assert.False(t, response.Accepted()) - assert.False(t, response.IsNew()) - assert.True(t, response.IsUpdate()) - assert.True(t, response.IsPaused()) - assert.False(t, response.IsRequest()) - - // Sanity check to make sure we can cast to datatransfer.Message - msg, ok := response.(datatransfer.Message) - require.True(t, ok) - - assert.False(t, msg.IsRequest()) - assert.False(t, msg.IsNew()) - assert.True(t, msg.IsUpdate()) - assert.True(t, msg.IsPaused()) - assert.Equal(t, response.TransferID(), msg.TransferID()) -} - -func TestCancelResponse(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - response := message1_1.CancelResponse(id) - assert.Equal(t, response.TransferID(), id) - assert.False(t, response.IsNew()) - assert.False(t, response.IsUpdate()) - assert.True(t, response.IsCancel()) - assert.False(t, response.IsRequest()) - // Sanity check to make sure we can cast to datatransfer.Message - msg, ok := response.(datatransfer.Message) - require.True(t, ok) - - assert.False(t, msg.IsRequest()) - assert.False(t, msg.IsNew()) - assert.False(t, msg.IsUpdate()) - assert.True(t, msg.IsCancel()) - assert.Equal(t, response.TransferID(), msg.TransferID()) -} - -func TestCompleteResponse(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - response, err := message1_1.CompleteResponse(id, true, true, datatransfer.EmptyTypeIdentifier, nil) - require.NoError(t, err) - assert.Equal(t, response.TransferID(), id) - assert.False(t, response.IsNew()) - assert.False(t, response.IsUpdate()) - assert.True(t, response.IsPaused()) - assert.True(t, response.IsValidationResult()) - assert.True(t, response.EmptyVoucherResult()) - assert.True(t, response.IsComplete()) - assert.False(t, response.IsRequest()) - // Sanity check to make sure we can cast to datatransfer.Message - msg, ok := response.(datatransfer.Message) - require.True(t, ok) - - assert.False(t, msg.IsRequest()) - assert.False(t, msg.IsNew()) - assert.False(t, msg.IsUpdate()) - assert.Equal(t, response.TransferID(), msg.TransferID()) -} -func TestToNetFromNetEquivalency(t *testing.T) { - t.Run("round-trip", func(t *testing.T) { - baseCid := testutil.GenerateCids(1)[0] - selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() - isPull := false - id := datatransfer.TransferID(rand.Int31()) - accepted := false - voucher := testutil.NewFakeDTType() - voucherResult := testutil.NewFakeDTType() - request, err := message1_1.NewRequest(id, false, isPull, voucher.Type(), voucher, baseCid, selector) - require.NoError(t, err) - buf := new(bytes.Buffer) - err = request.ToNet(buf) - require.NoError(t, err) - require.Greater(t, buf.Len(), 0) - deserialized, err := message1_1.FromNet(buf) - require.NoError(t, err) - - deserializedRequest, ok := deserialized.(datatransfer.Request) - require.True(t, ok) - - require.Equal(t, deserializedRequest.TransferID(), request.TransferID()) - require.Equal(t, deserializedRequest.IsCancel(), request.IsCancel()) - require.Equal(t, deserializedRequest.IsPull(), request.IsPull()) - require.Equal(t, deserializedRequest.IsRequest(), request.IsRequest()) - require.Equal(t, deserializedRequest.BaseCid(), request.BaseCid()) - testutil.AssertEqualFakeDTVoucher(t, request, deserializedRequest) - testutil.AssertEqualSelector(t, request, deserializedRequest) - - response, err := message1_1.NewResponse(id, accepted, false, voucherResult.Type(), voucherResult) - require.NoError(t, err) - err = response.ToNet(buf) - require.NoError(t, err) - deserialized, err = message1_1.FromNet(buf) - require.NoError(t, err) - - deserializedResponse, ok := deserialized.(datatransfer.Response) - require.True(t, ok) - - require.Equal(t, deserializedResponse.TransferID(), response.TransferID()) - require.Equal(t, deserializedResponse.Accepted(), response.Accepted()) - require.Equal(t, deserializedResponse.IsRequest(), response.IsRequest()) - require.Equal(t, deserializedResponse.IsUpdate(), response.IsUpdate()) - require.Equal(t, deserializedResponse.IsPaused(), response.IsPaused()) - testutil.AssertEqualFakeDTVoucherResult(t, response, deserializedResponse) - - request = message1_1.CancelRequest(id) - err = request.ToNet(buf) - require.NoError(t, err) - deserialized, err = message1_1.FromNet(buf) - require.NoError(t, err) - - deserializedRequest, ok = deserialized.(datatransfer.Request) - require.True(t, ok) - - require.Equal(t, deserializedRequest.TransferID(), request.TransferID()) - require.Equal(t, deserializedRequest.IsCancel(), request.IsCancel()) - require.Equal(t, deserializedRequest.IsRequest(), request.IsRequest()) - }) - t.Run("ipld-prime compat", func(t *testing.T) { - baseCid := testutil.GenerateCids(1)[0] - selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() - isPull := false - id := datatransfer.TransferID(1298498081) - accepted := false - voucher := testutil.NewFakeDTType() - voucherResult := testutil.NewFakeDTType() - request, err := message1_1.NewRequest(id, false, isPull, voucher.Type(), voucher, baseCid, selector) - require.NoError(t, err) - buf := new(bytes.Buffer) - err = request.ToNet(buf) - require.NoError(t, err) - require.Greater(t, buf.Len(), 0) - msg, _ := hex.DecodeString("a36449735271f56752657175657374aa6442436964d82a58230012204bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a6450617274f46450617573f46450756c6cf46453746f72a1612ea064547970650064565479706a46616b6544545479706565566f756368817864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e35665866657249441a4d6582216e526573746172744368616e6e656c8360600068526573706f6e7365f6") - deserialized, err := message1_1.FromNet(bytes.NewReader(msg)) - require.NoError(t, err) - - deserializedRequest, ok := deserialized.(datatransfer.Request) - require.True(t, ok) - - require.Equal(t, deserializedRequest.TransferID(), request.TransferID()) - require.Equal(t, deserializedRequest.IsCancel(), request.IsCancel()) - require.Equal(t, deserializedRequest.IsPull(), request.IsPull()) - require.Equal(t, deserializedRequest.IsRequest(), request.IsRequest()) - c, _ := cid.Parse("QmTTA2daxGqo5denp6SwLzzkLJm3fuisYEi9CoWsuHpzfb") - assert.Equal(t, c, deserializedRequest.BaseCid()) - testutil.AssertEqualFakeDTVoucher(t, request, deserializedRequest) - testutil.AssertEqualSelector(t, request, deserializedRequest) - - response, err := message1_1.NewResponse(id, accepted, false, voucherResult.Type(), voucherResult) - require.NoError(t, err) - err = response.ToNet(buf) - require.NoError(t, err) - msg, _ = hex.DecodeString("a36449735271f46752657175657374f668526573706f6e7365a66441637074f46450617573f464547970650064565265738178644204cb9a1e34c5f08e9b20aa76090e70020bb56c0ca3d3af7296cd1058a5112890fed218488f084d8df9e4835fb54ad045ffd936e3bf7261b0426c51352a097816ed74482bb9084b4a7ed8adc517f3371e0e0434b511625cd1a41792243dccdcfe88094b64565479706a46616b65445454797065665866657249441a4d658221") - deserialized, err = message1_1.FromNet(bytes.NewReader(msg)) - require.NoError(t, err) - - deserializedResponse, ok := deserialized.(datatransfer.Response) - require.True(t, ok) - - require.Equal(t, deserializedResponse.TransferID(), response.TransferID()) - require.Equal(t, deserializedResponse.Accepted(), response.Accepted()) - require.Equal(t, deserializedResponse.IsRequest(), response.IsRequest()) - require.Equal(t, deserializedResponse.IsUpdate(), response.IsUpdate()) - require.Equal(t, deserializedResponse.IsPaused(), response.IsPaused()) - testutil.AssertEqualFakeDTVoucherResult(t, response, deserializedResponse) - - request = message1_1.CancelRequest(id) - err = request.ToNet(buf) - require.NoError(t, err) - msg, _ = hex.DecodeString("a36449735271f56752657175657374aa6442436964f66450617274f46450617573f46450756c6cf46453746f72f664547970650264565479706065566f756368f6665866657249441a4d6582216e526573746172744368616e6e656c8360600068526573706f6e7365f6") - deserialized, err = message1_1.FromNet(bytes.NewReader(msg)) - require.NoError(t, err) - - deserializedRequest, ok = deserialized.(datatransfer.Request) - require.True(t, ok) - - require.Equal(t, deserializedRequest.TransferID(), request.TransferID()) - require.Equal(t, deserializedRequest.IsCancel(), request.IsCancel()) - require.Equal(t, deserializedRequest.IsRequest(), request.IsRequest()) - }) -} - -func TestFromNetMessageValidation(t *testing.T) { - // craft request message with nil request struct - buf := []byte{0x83, 0xf5, 0xf6, 0xf6} - msg, err := message1_1.FromNet(bytes.NewBuffer(buf)) - assert.Error(t, err) - assert.Nil(t, msg) - - // craft response message with nil response struct - buf = []byte{0x83, 0xf4, 0xf6, 0xf6} - msg, err = message1_1.FromNet(bytes.NewBuffer(buf)) - assert.Error(t, err) - assert.Nil(t, msg) -} - -func NewTestTransferRequest() (message1_1.TransferRequest1_1, error) { - bcid := testutil.GenerateCids(1)[0] - selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() - isPull := false - id := datatransfer.TransferID(rand.Int31()) - voucher := testutil.NewFakeDTType() - req, err := message1_1.NewRequest(id, false, isPull, voucher.Type(), voucher, bcid, selector) - if err != nil { - return message1_1.TransferRequest1_1{}, err - } - tr, ok := req.(*message1_1.TransferRequest1_1) - if !ok { - return message1_1.TransferRequest1_1{}, fmt.Errorf("expected *TransferRequest1_1") - } - return *tr, nil -} diff --git a/message/message1_1/transfer_message.go b/message/message1_1/transfer_message.go deleted file mode 100644 index f4e04f32..00000000 --- a/message/message1_1/transfer_message.go +++ /dev/null @@ -1,59 +0,0 @@ -package message1_1 - -import ( - "bytes" - "io" - - "github.com/ipld/go-ipld-prime/codec/dagcbor" - "github.com/ipld/go-ipld-prime/datamodel" - basicnode "github.com/ipld/go-ipld-prime/node/basic" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" -) - -//go:generate cbor-gen-for --map-encoding TransferMessage1_1 - -// transferMessage1_1 is the transfer message for the 1.1 Data Transfer Protocol. -type TransferMessage1_1 struct { - IsRq bool - - Request *TransferRequest1_1 - Response *TransferResponse1_1 -} - -// ========= datatransfer.Message interface - -// IsRequest returns true if this message is a data request -func (tm *TransferMessage1_1) IsRequest() bool { - return tm.IsRq -} - -// TransferID returns the TransferID of this message -func (tm *TransferMessage1_1) TransferID() datatransfer.TransferID { - if tm.IsRequest() { - return tm.Request.TransferID() - } - return tm.Response.TransferID() -} - -// ToNet serializes a transfer message type. It is simply a wrapper for MarshalCBOR, to provide -// symmetry with FromNet -func (tm *TransferMessage1_1) ToIPLD() (datamodel.Node, error) { - buf := new(bytes.Buffer) - err := tm.ToNet(buf) - if err != nil { - return nil, err - } - nb := basicnode.Prototype.Any.NewBuilder() - err = dagcbor.Decode(nb, buf) - if err != nil { - return nil, err - } - return nb.Build(), nil -} - -// ToNet serializes a transfer message type. It is simply a wrapper for MarshalCBOR, to provide -// symmetry with FromNet -func (tm *TransferMessage1_1) ToNet(w io.Writer) error { - return tm.MarshalCBOR(w) -} diff --git a/message/message1_1/transfer_message_cbor_gen.go b/message/message1_1/transfer_message_cbor_gen.go deleted file mode 100644 index b7b25e8e..00000000 --- a/message/message1_1/transfer_message_cbor_gen.go +++ /dev/null @@ -1,179 +0,0 @@ -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -package message1_1 - -import ( - "fmt" - "io" - "sort" - - cid "github.com/ipfs/go-cid" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -var _ = xerrors.Errorf -var _ = cid.Undef -var _ = sort.Sort - -func (t *TransferMessage1_1) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{163}); err != nil { - return err - } - - scratch := make([]byte, 9) - - // t.IsRq (bool) (bool) - if len("IsRq") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"IsRq\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("IsRq"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("IsRq")); err != nil { - return err - } - - if err := cbg.WriteBool(w, t.IsRq); err != nil { - return err - } - - // t.Request (message1_1.TransferRequest1_1) (struct) - if len("Request") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Request\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Request"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Request")); err != nil { - return err - } - - if err := t.Request.MarshalCBOR(w); err != nil { - return err - } - - // t.Response (message1_1.TransferResponse1_1) (struct) - if len("Response") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Response\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Response"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Response")); err != nil { - return err - } - - if err := t.Response.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *TransferMessage1_1) UnmarshalCBOR(r io.Reader) error { - *t = TransferMessage1_1{} - - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) - - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("TransferMessage1_1: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadStringBuf(br, scratch) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.IsRq (bool) (bool) - case "IsRq": - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.IsRq = false - case 21: - t.IsRq = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - // t.Request (message1_1.TransferRequest1_1) (struct) - case "Request": - - { - - b, err := br.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { - return err - } - t.Request = new(TransferRequest1_1) - if err := t.Request.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.Request pointer: %w", err) - } - } - - } - // t.Response (message1_1.TransferResponse1_1) (struct) - case "Response": - - { - - b, err := br.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { - return err - } - t.Response = new(TransferResponse1_1) - if err := t.Response.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.Response pointer: %w", err) - } - } - - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} diff --git a/message/message1_1/transfer_request.go b/message/message1_1/transfer_request.go deleted file mode 100644 index a751ed31..00000000 --- a/message/message1_1/transfer_request.go +++ /dev/null @@ -1,166 +0,0 @@ -package message1_1 - -import ( - "bytes" - "io" - - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/codec/dagcbor" - "github.com/ipld/go-ipld-prime/datamodel" - basicnode "github.com/ipld/go-ipld-prime/node/basic" - "github.com/libp2p/go-libp2p-core/protocol" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/encoding" - "github.com/filecoin-project/go-data-transfer/v2/message/types" -) - -//go:generate cbor-gen-for --map-encoding TransferRequest1_1 - -// TransferRequest1_1 is a struct for the 1.1 Data Transfer Protocol that fulfills the datatransfer.Request interface. -// its members are exported to be used by cbor-gen -type TransferRequest1_1 struct { - BCid *cid.Cid - Type uint64 - Paus bool - Part bool - Pull bool - Stor *cbg.Deferred - Vouch *cbg.Deferred - VTyp datatransfer.TypeIdentifier - XferID uint64 - - RestartChannel datatransfer.ChannelID -} - -func (trq *TransferRequest1_1) MessageForProtocol(targetProtocol protocol.ID) (datatransfer.Message, error) { - switch targetProtocol { - case datatransfer.ProtocolDataTransfer1_2: - return trq, nil - default: - return nil, xerrors.Errorf("protocol not supported") - } -} - -// IsRequest always returns true in this case because this is a transfer request -func (trq *TransferRequest1_1) IsRequest() bool { - return true -} - -func (trq *TransferRequest1_1) IsRestart() bool { - return trq.Type == uint64(types.RestartMessage) -} - -func (trq *TransferRequest1_1) IsRestartExistingChannelRequest() bool { - return trq.Type == uint64(types.RestartExistingChannelRequestMessage) -} - -func (trq *TransferRequest1_1) RestartChannelId() (datatransfer.ChannelID, error) { - if !trq.IsRestartExistingChannelRequest() { - return datatransfer.ChannelID{}, xerrors.New("not a restart request") - } - return trq.RestartChannel, nil -} - -func (trq *TransferRequest1_1) IsNew() bool { - return trq.Type == uint64(types.NewMessage) -} - -func (trq *TransferRequest1_1) IsUpdate() bool { - return trq.Type == uint64(types.UpdateMessage) -} - -func (trq *TransferRequest1_1) IsVoucher() bool { - return trq.Type == uint64(types.VoucherMessage) || trq.Type == uint64(types.NewMessage) -} - -func (trq *TransferRequest1_1) IsPaused() bool { - return trq.Paus -} - -func (trq *TransferRequest1_1) TransferID() datatransfer.TransferID { - return datatransfer.TransferID(trq.XferID) -} - -// ========= datatransfer.Request interface -// IsPull returns true if this is a data pull request -func (trq *TransferRequest1_1) IsPull() bool { - return trq.Pull -} - -// VoucherType returns the Voucher ID -func (trq *TransferRequest1_1) VoucherType() datatransfer.TypeIdentifier { - return trq.VTyp -} - -// Voucher returns the Voucher bytes -func (trq *TransferRequest1_1) Voucher(decoder encoding.Decoder) (encoding.Encodable, error) { - if trq.Vouch == nil { - return nil, xerrors.New("No voucher present to read") - } - return decoder.DecodeFromCbor(trq.Vouch.Raw) -} - -func (trq *TransferRequest1_1) EmptyVoucher() bool { - return trq.VTyp == datatransfer.EmptyTypeIdentifier -} - -// BaseCid returns the Base CID -func (trq *TransferRequest1_1) BaseCid() cid.Cid { - if trq.BCid == nil { - return cid.Undef - } - return *trq.BCid -} - -// Selector returns the message Selector bytes -func (trq *TransferRequest1_1) Selector() (ipld.Node, error) { - if trq.Stor == nil { - return nil, xerrors.New("No selector present to read") - } - builder := basicnode.Prototype.Any.NewBuilder() - reader := bytes.NewReader(trq.Stor.Raw) - err := dagcbor.Decode(builder, reader) - if err != nil { - return nil, xerrors.Errorf("Error decoding selector: %w", err) - } - return builder.Build(), nil -} - -// IsCancel returns true if this is a cancel request -func (trq *TransferRequest1_1) IsCancel() bool { - return trq.Type == uint64(types.CancelMessage) -} - -// IsPartial returns true if this is a partial request -func (trq *TransferRequest1_1) IsPartial() bool { - return trq.Part -} - -func (trq *TransferRequest1_1) ToIPLD() (datamodel.Node, error) { - buf := new(bytes.Buffer) - err := trq.ToNet(buf) - if err != nil { - return nil, err - } - nb := basicnode.Prototype.Any.NewBuilder() - err = dagcbor.Decode(nb, buf) - if err != nil { - return nil, err - } - return nb.Build(), nil -} - -// ToNet serializes a transfer request. It's a wrapper for MarshalCBOR to provide -// symmetry with FromNet -func (trq *TransferRequest1_1) ToNet(w io.Writer) error { - msg := TransferMessage1_1{ - IsRq: true, - Request: trq, - Response: nil, - } - return msg.MarshalCBOR(w) -} diff --git a/message/message1_1/transfer_request_cbor_gen.go b/message/message1_1/transfer_request_cbor_gen.go deleted file mode 100644 index 056fe964..00000000 --- a/message/message1_1/transfer_request_cbor_gen.go +++ /dev/null @@ -1,397 +0,0 @@ -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -package message1_1 - -import ( - "fmt" - "io" - "sort" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - cid "github.com/ipfs/go-cid" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -var _ = xerrors.Errorf -var _ = cid.Undef -var _ = sort.Sort - -func (t *TransferRequest1_1) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{170}); err != nil { - return err - } - - scratch := make([]byte, 9) - - // t.BCid (cid.Cid) (struct) - if len("BCid") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"BCid\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("BCid"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("BCid")); err != nil { - return err - } - - if t.BCid == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCidBuf(scratch, w, *t.BCid); err != nil { - return xerrors.Errorf("failed to write cid field t.BCid: %w", err) - } - } - - // t.Type (uint64) (uint64) - if len("Type") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Type\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Type"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Type")); err != nil { - return err - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Type)); err != nil { - return err - } - - // t.Paus (bool) (bool) - if len("Paus") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Paus\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Paus"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Paus")); err != nil { - return err - } - - if err := cbg.WriteBool(w, t.Paus); err != nil { - return err - } - - // t.Part (bool) (bool) - if len("Part") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Part\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Part"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Part")); err != nil { - return err - } - - if err := cbg.WriteBool(w, t.Part); err != nil { - return err - } - - // t.Pull (bool) (bool) - if len("Pull") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Pull\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Pull"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Pull")); err != nil { - return err - } - - if err := cbg.WriteBool(w, t.Pull); err != nil { - return err - } - - // t.Stor (typegen.Deferred) (struct) - if len("Stor") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Stor\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Stor"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Stor")); err != nil { - return err - } - - if err := t.Stor.MarshalCBOR(w); err != nil { - return err - } - - // t.Vouch (typegen.Deferred) (struct) - if len("Vouch") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Vouch\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouch"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Vouch")); err != nil { - return err - } - - if err := t.Vouch.MarshalCBOR(w); err != nil { - return err - } - - // t.VTyp (datatransfer.TypeIdentifier) (string) - if len("VTyp") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"VTyp\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("VTyp"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("VTyp")); err != nil { - return err - } - - if len(t.VTyp) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.VTyp was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.VTyp))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.VTyp)); err != nil { - return err - } - - // t.XferID (uint64) (uint64) - if len("XferID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"XferID\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("XferID"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("XferID")); err != nil { - return err - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.XferID)); err != nil { - return err - } - - // t.RestartChannel (datatransfer.ChannelID) (struct) - if len("RestartChannel") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"RestartChannel\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("RestartChannel"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("RestartChannel")); err != nil { - return err - } - - if err := t.RestartChannel.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *TransferRequest1_1) UnmarshalCBOR(r io.Reader) error { - *t = TransferRequest1_1{} - - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) - - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("TransferRequest1_1: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadStringBuf(br, scratch) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.BCid (cid.Cid) (struct) - case "BCid": - - { - - b, err := br.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.BCid: %w", err) - } - - t.BCid = &c - } - - } - // t.Type (uint64) (uint64) - case "Type": - - { - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Type = uint64(extra) - - } - // t.Paus (bool) (bool) - case "Paus": - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.Paus = false - case 21: - t.Paus = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - // t.Part (bool) (bool) - case "Part": - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.Part = false - case 21: - t.Part = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - // t.Pull (bool) (bool) - case "Pull": - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.Pull = false - case 21: - t.Pull = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - // t.Stor (typegen.Deferred) (struct) - case "Stor": - - { - - t.Stor = new(cbg.Deferred) - - if err := t.Stor.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("failed to read deferred field: %w", err) - } - } - // t.Vouch (typegen.Deferred) (struct) - case "Vouch": - - { - - t.Vouch = new(cbg.Deferred) - - if err := t.Vouch.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("failed to read deferred field: %w", err) - } - } - // t.VTyp (datatransfer.TypeIdentifier) (string) - case "VTyp": - - { - sval, err := cbg.ReadStringBuf(br, scratch) - if err != nil { - return err - } - - t.VTyp = datatransfer.TypeIdentifier(sval) - } - // t.XferID (uint64) (uint64) - case "XferID": - - { - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.XferID = uint64(extra) - - } - // t.RestartChannel (datatransfer.ChannelID) (struct) - case "RestartChannel": - - { - - if err := t.RestartChannel.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.RestartChannel: %w", err) - } - - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} diff --git a/message/message1_1/transfer_request_test.go b/message/message1_1/transfer_request_test.go deleted file mode 100644 index 7ea148f8..00000000 --- a/message/message1_1/transfer_request_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package message1_1_test - -import ( - "math/rand" - "testing" - - basicnode "github.com/ipld/go-ipld-prime/node/basic" - "github.com/ipld/go-ipld-prime/traversal/selector/builder" - "github.com/stretchr/testify/require" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/message/message1_1" - "github.com/filecoin-project/go-data-transfer/v2/testutil" -) - -func TestRequestMessageForProtocol(t *testing.T) { - baseCid := testutil.GenerateCids(1)[0] - selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() - isPull := true - id := datatransfer.TransferID(rand.Int31()) - voucher := testutil.NewFakeDTType() - - // for the new protocols - request, err := message1_1.NewRequest(id, false, isPull, voucher.Type(), voucher, baseCid, selector) - require.NoError(t, err) - - out12, err := request.MessageForProtocol(datatransfer.ProtocolDataTransfer1_2) - require.NoError(t, err) - require.Equal(t, request, out12) - - req, ok := out12.(datatransfer.Request) - require.True(t, ok) - require.False(t, req.IsRestart()) - require.False(t, req.IsRestartExistingChannelRequest()) - require.Equal(t, baseCid, req.BaseCid()) - require.True(t, req.IsPull()) - n, err := req.Selector() - require.NoError(t, err) - require.Equal(t, selector, n) - require.Equal(t, voucher.Type(), req.VoucherType()) -} diff --git a/message/message1_1/transfer_response.go b/message/message1_1/transfer_response.go deleted file mode 100644 index 105e19a4..00000000 --- a/message/message1_1/transfer_response.go +++ /dev/null @@ -1,127 +0,0 @@ -package message1_1 - -import ( - "bytes" - "io" - - "github.com/ipld/go-ipld-prime/codec/dagcbor" - "github.com/ipld/go-ipld-prime/datamodel" - basicnode "github.com/ipld/go-ipld-prime/node/basic" - "github.com/libp2p/go-libp2p-core/protocol" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/encoding" - "github.com/filecoin-project/go-data-transfer/v2/message/types" -) - -//go:generate cbor-gen-for --map-encoding TransferResponse1_1 - -// TransferResponse1_1 is a private struct that satisfies the datatransfer.Response interface -// It is the response message for the Data Transfer 1.1 and 1.2 Protocol. -type TransferResponse1_1 struct { - Type uint64 - Acpt bool - Paus bool - XferID uint64 - VRes *cbg.Deferred - VTyp datatransfer.TypeIdentifier -} - -func (trsp *TransferResponse1_1) TransferID() datatransfer.TransferID { - return datatransfer.TransferID(trsp.XferID) -} - -// IsRequest always returns false in this case because this is a transfer response -func (trsp *TransferResponse1_1) IsRequest() bool { - return false -} - -// IsNew returns true if this is the first response sent -func (trsp *TransferResponse1_1) IsNew() bool { - return trsp.Type == uint64(types.NewMessage) -} - -// IsUpdate returns true if this response is an update -func (trsp *TransferResponse1_1) IsUpdate() bool { - return trsp.Type == uint64(types.UpdateMessage) -} - -// IsPaused returns true if the responder is paused -func (trsp *TransferResponse1_1) IsPaused() bool { - return trsp.Paus -} - -// IsCancel returns true if the responder has cancelled this response -func (trsp *TransferResponse1_1) IsCancel() bool { - return trsp.Type == uint64(types.CancelMessage) -} - -// IsComplete returns true if the responder has completed this response -func (trsp *TransferResponse1_1) IsComplete() bool { - return trsp.Type == uint64(types.CompleteMessage) -} - -func (trsp *TransferResponse1_1) IsValidationResult() bool { - return trsp.Type == uint64(types.VoucherResultMessage) || trsp.Type == uint64(types.NewMessage) || trsp.Type == uint64(types.CompleteMessage) || - trsp.Type == uint64(types.RestartMessage) -} - -// Accepted returns true if the request is accepted in the response -func (trsp *TransferResponse1_1) Accepted() bool { - return trsp.Acpt -} - -func (trsp *TransferResponse1_1) VoucherResultType() datatransfer.TypeIdentifier { - return trsp.VTyp -} - -func (trsp *TransferResponse1_1) VoucherResult(decoder encoding.Decoder) (encoding.Encodable, error) { - if trsp.VRes == nil { - return nil, xerrors.New("No voucher present to read") - } - return decoder.DecodeFromCbor(trsp.VRes.Raw) -} - -func (trq *TransferResponse1_1) IsRestart() bool { - return trq.Type == uint64(types.RestartMessage) -} - -func (trsp *TransferResponse1_1) EmptyVoucherResult() bool { - return trsp.VTyp == datatransfer.EmptyTypeIdentifier -} - -func (trsp *TransferResponse1_1) MessageForProtocol(targetProtocol protocol.ID) (datatransfer.Message, error) { - switch targetProtocol { - case datatransfer.ProtocolDataTransfer1_2: - return trsp, nil - default: - return nil, xerrors.Errorf("protocol %s not supported", targetProtocol) - } -} - -func (trsp *TransferResponse1_1) ToIPLD() (datamodel.Node, error) { - buf := new(bytes.Buffer) - err := trsp.ToNet(buf) - if err != nil { - return nil, err - } - nb := basicnode.Prototype.Any.NewBuilder() - err = dagcbor.Decode(nb, buf) - if err != nil { - return nil, err - } - return nb.Build(), nil -} - -// ToNet serializes a transfer response. It's a wrapper for MarshalCBOR to provide -// symmetry with FromNet -func (trsp *TransferResponse1_1) ToNet(w io.Writer) error { - msg := TransferMessage1_1{ - IsRq: false, - Request: nil, - Response: trsp, - } - return msg.MarshalCBOR(w) -} diff --git a/message/message1_1/transfer_response_cbor_gen.go b/message/message1_1/transfer_response_cbor_gen.go deleted file mode 100644 index 0a9badc7..00000000 --- a/message/message1_1/transfer_response_cbor_gen.go +++ /dev/null @@ -1,265 +0,0 @@ -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -package message1_1 - -import ( - "fmt" - "io" - "sort" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - cid "github.com/ipfs/go-cid" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -var _ = xerrors.Errorf -var _ = cid.Undef -var _ = sort.Sort - -func (t *TransferResponse1_1) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{166}); err != nil { - return err - } - - scratch := make([]byte, 9) - - // t.Type (uint64) (uint64) - if len("Type") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Type\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Type"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Type")); err != nil { - return err - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Type)); err != nil { - return err - } - - // t.Acpt (bool) (bool) - if len("Acpt") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Acpt\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Acpt"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Acpt")); err != nil { - return err - } - - if err := cbg.WriteBool(w, t.Acpt); err != nil { - return err - } - - // t.Paus (bool) (bool) - if len("Paus") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Paus\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Paus"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Paus")); err != nil { - return err - } - - if err := cbg.WriteBool(w, t.Paus); err != nil { - return err - } - - // t.XferID (uint64) (uint64) - if len("XferID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"XferID\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("XferID"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("XferID")); err != nil { - return err - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.XferID)); err != nil { - return err - } - - // t.VRes (typegen.Deferred) (struct) - if len("VRes") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"VRes\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("VRes"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("VRes")); err != nil { - return err - } - - if err := t.VRes.MarshalCBOR(w); err != nil { - return err - } - - // t.VTyp (datatransfer.TypeIdentifier) (string) - if len("VTyp") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"VTyp\" was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("VTyp"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("VTyp")); err != nil { - return err - } - - if len(t.VTyp) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.VTyp was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.VTyp))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.VTyp)); err != nil { - return err - } - return nil -} - -func (t *TransferResponse1_1) UnmarshalCBOR(r io.Reader) error { - *t = TransferResponse1_1{} - - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) - - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("TransferResponse1_1: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadStringBuf(br, scratch) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Type (uint64) (uint64) - case "Type": - - { - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Type = uint64(extra) - - } - // t.Acpt (bool) (bool) - case "Acpt": - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.Acpt = false - case 21: - t.Acpt = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - // t.Paus (bool) (bool) - case "Paus": - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.Paus = false - case 21: - t.Paus = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - // t.XferID (uint64) (uint64) - case "XferID": - - { - - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.XferID = uint64(extra) - - } - // t.VRes (typegen.Deferred) (struct) - case "VRes": - - { - - t.VRes = new(cbg.Deferred) - - if err := t.VRes.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("failed to read deferred field: %w", err) - } - } - // t.VTyp (datatransfer.TypeIdentifier) (string) - case "VTyp": - - { - sval, err := cbg.ReadStringBuf(br, scratch) - if err != nil { - return err - } - - t.VTyp = datatransfer.TypeIdentifier(sval) - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} diff --git a/message/message1_1/transfer_response_test.go b/message/message1_1/transfer_response_test.go deleted file mode 100644 index 02c9cffb..00000000 --- a/message/message1_1/transfer_response_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package message1_1_test - -import ( - "math/rand" - "testing" - - "github.com/stretchr/testify/require" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/message/message1_1" - "github.com/filecoin-project/go-data-transfer/v2/testutil" -) - -func TestResponseMessageForProtocol(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - voucherResult := testutil.NewFakeDTType() - response, err := message1_1.NewResponse(id, false, true, voucherResult.Type(), voucherResult) // not accepted - require.NoError(t, err) - - // v1.2 protocol - out, err := response.MessageForProtocol(datatransfer.ProtocolDataTransfer1_2) - require.NoError(t, err) - require.Equal(t, response, out) - - resp, ok := (out).(datatransfer.Response) - require.True(t, ok) - require.True(t, resp.IsPaused()) - require.Equal(t, voucherResult.Type(), resp.VoucherResultType()) - require.True(t, resp.IsValidationResult()) - - // random protocol - out, err = response.MessageForProtocol("RAND") - require.Error(t, err) - require.Nil(t, out) -} From 681bfedccef1e086876b366e4c3f2c8b7890f30b Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 3 Jun 2022 10:45:28 +1000 Subject: [PATCH 05/18] feat(ipld): vouchers as plain ipld.Node (#325) * feat(ipld): vouchers as plain ipld.Node * feat: add ValidationResult#Equals() utility * feat(ipld): introduce TypedVoucher tuple type * chore(ipld): ipld.Node -> datamodel.Node * chore: remove RegisterVoucherResultType * fix: minor staticcheck fixes --- README.md | 6 +- benchmarks/benchmark_test.go | 2 +- benchmarks/testinstance/testinstance.go | 3 +- channels/channel_state.go | 79 +++--- channels/channels.go | 42 ++- channels/channels_test.go | 74 ++--- encoding/encoding.go | 171 ------------ encoding/encoding_test.go | 37 --- encoding/testdata/testdata.go | 37 --- encoding/testdata/testdata_cbor_gen.go | 84 ------ go.mod | 4 +- go.sum | 3 +- impl/events.go | 12 +- impl/impl.go | 55 ++-- impl/initiating_test.go | 68 ++--- impl/integration_test.go | 253 ++++++++---------- impl/receiving_requests.go | 50 ++-- impl/responding_test.go | 142 ++++++---- impl/restart.go | 40 +-- impl/restart_integration_test.go | 28 +- impl/utils.go | 33 +-- ipldutils/ipldutils.go | 183 +++++++++++++ manager.go | 40 +-- message.go | 10 +- message/message1_1prime/message.go | 107 ++++---- message/message1_1prime/message_test.go | 99 +++---- message/message1_1prime/schema.go | 29 -- message/message1_1prime/schema.ipldsch | 2 +- message/message1_1prime/transfer_message.go | 29 +- message/message1_1prime/transfer_request.go | 40 ++- .../message1_1prime/transfer_request_test.go | 6 +- message/message1_1prime/transfer_response.go | 23 +- .../message1_1prime/transfer_response_test.go | 6 +- network/libp2p_impl_test.go | 16 +- registry/registry.go | 35 +-- registry/registry_test.go | 20 +- testutil/fakedttype.go | 97 ++++--- testutil/fakedttype_cbor_gen.go | 75 ------ testutil/fakegraphsync.go | 8 +- testutil/faketransport.go | 9 +- testutil/gstestdata.go | 5 +- testutil/message.go | 8 +- testutil/mockchannelstate.go | 14 +- testutil/stubbedvalidator.go | 14 +- testutil/testutil.go | 4 +- transport.go | 3 +- transport/graphsync/graphsync.go | 5 +- types.go | 42 ++- 48 files changed, 938 insertions(+), 1214 deletions(-) delete mode 100644 encoding/encoding.go delete mode 100644 encoding/encoding_test.go delete mode 100644 encoding/testdata/testdata.go delete mode 100644 encoding/testdata/testdata_cbor_gen.go create mode 100644 ipldutils/ipldutils.go delete mode 100644 message/message1_1prime/schema.go delete mode 100644 testutil/fakedttype_cbor_gen.go diff --git a/README.md b/README.md index f2add43b..3ab3f91f 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ func (vl *myValidator) ValidatePush( sender peer.ID, voucher datatransfer.Voucher, baseCid cid.Cid, - selector ipld.Node) error { + selector datamodel.Node) error { v := voucher.(*myVoucher) if v.data == "" || v.data != "validpush" { @@ -99,7 +99,7 @@ func (vl *myValidator) ValidatePull( receiver peer.ID, voucher datatransfer.Voucher, baseCid cid.Cid, - selector ipld.Node) error { + selector datamodel.Node) error { v := voucher.(*myVoucher) if v.data == "" || v.data != "validpull" { @@ -135,7 +135,7 @@ must be sent with the request. Using the trivial examples above: For more detail, please see the [unit tests](https://github.com/filecoin-project/go-data-transfer/blob/master/impl/impl_test.go). ### Open a Push or Pull Request -For a push or pull request, provide a context, a `datatransfer.Voucher`, a host recipient `peer.ID`, a baseCID `cid.CID` and a selector `ipld.Node`. These +For a push or pull request, provide a context, a `datatransfer.Voucher`, a host recipient `peer.ID`, a baseCID `cid.CID` and a selector `datamodel.Node`. These calls return a `datatransfer.ChannelID` and any error: ```go channelID, err := dtm.OpenPullDataChannel(ctx, recipient, voucher, baseCid, selector) diff --git a/benchmarks/benchmark_test.go b/benchmarks/benchmark_test.go index 357f80f1..967a55da 100644 --- a/benchmarks/benchmark_test.go +++ b/benchmarks/benchmark_test.go @@ -105,7 +105,7 @@ func p2pStrestTest(ctx context.Context, b *testing.B, numfiles int, df distFunc, timer := time.NewTimer(30 * time.Second) start := time.Now() for j := 0; j < numfiles; j++ { - _, err := pusher.Manager.OpenPushDataChannel(ctx, receiver.Peer, testutil.NewFakeDTType(), allCids[j], allSelector) + _, err := pusher.Manager.OpenPushDataChannel(ctx, receiver.Peer, testutil.NewTestTypedVoucher(), allCids[j], allSelector) if err != nil { b.Fatalf("received error on request: %s", err.Error()) } diff --git a/benchmarks/testinstance/testinstance.go b/benchmarks/testinstance/testinstance.go index c720f44b..18a8e8de 100644 --- a/benchmarks/testinstance/testinstance.go +++ b/benchmarks/testinstance/testinstance.go @@ -188,8 +188,7 @@ func NewInstance(ctx context.Context, net tn.Network, tempDir string, diskBasedD sv := testutil.NewStubbedValidator() sv.StubSuccessPull() sv.StubSuccessPush() - dt.RegisterVoucherType(testutil.NewFakeDTType(), sv) - dt.RegisterVoucherResultType(testutil.NewFakeDTType()) + dt.RegisterVoucherType(testutil.TestVoucherType, sv) return Instance{ Adapter: dtNet, Peer: p, diff --git a/channels/channel_state.go b/channels/channel_state.go index a24a7feb..31c18d4b 100644 --- a/channels/channel_state.go +++ b/channels/channel_state.go @@ -4,22 +4,19 @@ import ( "bytes" "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/datamodel" basicnode "github.com/ipld/go-ipld-prime/node/basic" peer "github.com/libp2p/go-libp2p-core/peer" datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channels/internal" + ipldutils "github.com/filecoin-project/go-data-transfer/v2/ipldutils" ) // channelState is immutable channel data plus mutable state type channelState struct { ic internal.ChannelState - - // additional voucherResults - voucherResultDecoder DecoderByTypeFunc - voucherDecoder DecoderByTypeFunc } // EmptyChannelState is the zero value for channel state, meaning not present @@ -45,7 +42,7 @@ func (c channelState) BaseCID() cid.Cid { return c.ic.BaseCid } // Selector returns the IPLD selector for this data transfer (represented as // an IPLD node) -func (c channelState) Selector() ipld.Node { +func (c channelState) Selector() datamodel.Node { builder := basicnode.Prototype.Any.NewBuilder() reader := bytes.NewReader(c.ic.Selector.Raw) err := dagcbor.Decode(builder, reader) @@ -56,13 +53,15 @@ func (c channelState) Selector() ipld.Node { } // Voucher returns the voucher for this data transfer -func (c channelState) Voucher() datatransfer.Voucher { +func (c channelState) Voucher() (datatransfer.TypedVoucher, error) { if len(c.ic.Vouchers) == 0 { - return nil + return datatransfer.TypedVoucher{}, nil + } + node, err := ipldutils.DeferredToNode(c.ic.Vouchers[0].Voucher) + if err != nil { + return datatransfer.TypedVoucher{}, err } - decoder, _ := c.voucherDecoder(c.ic.Vouchers[0].Type) - encodable, _ := decoder.DecodeFromCbor(c.ic.Vouchers[0].Voucher.Raw) - return encodable.(datatransfer.Voucher) + return datatransfer.TypedVoucher{Voucher: node, Type: c.ic.Vouchers[0].Type}, nil } // ReceivedCidsTotal returns the number of (non-unique) cids received so far @@ -108,36 +107,46 @@ func (c channelState) Message() string { return c.ic.Message } -func (c channelState) Vouchers() []datatransfer.Voucher { - vouchers := make([]datatransfer.Voucher, 0, len(c.ic.Vouchers)) +func (c channelState) Vouchers() ([]datatransfer.TypedVoucher, error) { + vouchers := make([]datatransfer.TypedVoucher, 0, len(c.ic.Vouchers)) for _, encoded := range c.ic.Vouchers { - decoder, _ := c.voucherDecoder(encoded.Type) - encodable, _ := decoder.DecodeFromCbor(encoded.Voucher.Raw) - vouchers = append(vouchers, encodable.(datatransfer.Voucher)) + node, err := ipldutils.DeferredToNode(encoded.Voucher) + if err != nil { + return nil, err + } + vouchers = append(vouchers, datatransfer.TypedVoucher{Voucher: node, Type: encoded.Type}) } - return vouchers + return vouchers, nil } -func (c channelState) LastVoucher() datatransfer.Voucher { - decoder, _ := c.voucherDecoder(c.ic.Vouchers[len(c.ic.Vouchers)-1].Type) - encodable, _ := decoder.DecodeFromCbor(c.ic.Vouchers[len(c.ic.Vouchers)-1].Voucher.Raw) - return encodable.(datatransfer.Voucher) +func (c channelState) LastVoucher() (datatransfer.TypedVoucher, error) { + ev := c.ic.Vouchers[len(c.ic.Vouchers)-1] + node, err := ipldutils.DeferredToNode(ev.Voucher) + if err != nil { + return datatransfer.TypedVoucher{}, err + } + return datatransfer.TypedVoucher{Voucher: node, Type: ev.Type}, nil } -func (c channelState) LastVoucherResult() datatransfer.VoucherResult { - decoder, _ := c.voucherResultDecoder(c.ic.VoucherResults[len(c.ic.VoucherResults)-1].Type) - encodable, _ := decoder.DecodeFromCbor(c.ic.VoucherResults[len(c.ic.VoucherResults)-1].VoucherResult.Raw) - return encodable.(datatransfer.VoucherResult) +func (c channelState) LastVoucherResult() (datatransfer.TypedVoucher, error) { + evr := c.ic.VoucherResults[len(c.ic.VoucherResults)-1] + node, err := ipldutils.DeferredToNode(evr.VoucherResult) + if err != nil { + return datatransfer.TypedVoucher{}, err + } + return datatransfer.TypedVoucher{Voucher: node, Type: evr.Type}, nil } -func (c channelState) VoucherResults() []datatransfer.VoucherResult { - voucherResults := make([]datatransfer.VoucherResult, 0, len(c.ic.VoucherResults)) +func (c channelState) VoucherResults() ([]datatransfer.TypedVoucher, error) { + voucherResults := make([]datatransfer.TypedVoucher, 0, len(c.ic.VoucherResults)) for _, encoded := range c.ic.VoucherResults { - decoder, _ := c.voucherResultDecoder(encoded.Type) - encodable, _ := decoder.DecodeFromCbor(encoded.VoucherResult.Raw) - voucherResults = append(voucherResults, encodable.(datatransfer.VoucherResult)) + node, err := ipldutils.DeferredToNode(encoded.VoucherResult) + if err != nil { + return nil, err + } + voucherResults = append(voucherResults, datatransfer.TypedVoucher{Voucher: node, Type: encoded.Type}) } - return voucherResults + return voucherResults, nil } func (c channelState) SelfPeer() peer.ID { @@ -174,12 +183,8 @@ func (c channelState) Stages() *datatransfer.ChannelStages { return c.ic.Stages } -func fromInternalChannelState(c internal.ChannelState, voucherDecoder DecoderByTypeFunc, voucherResultDecoder DecoderByTypeFunc) datatransfer.ChannelState { - return channelState{ - ic: c, - voucherResultDecoder: voucherResultDecoder, - voucherDecoder: voucherDecoder, - } +func fromInternalChannelState(c internal.ChannelState) datatransfer.ChannelState { + return channelState{ic: c} } var _ datatransfer.ChannelState = channelState{} diff --git a/channels/channels.go b/channels/channels.go index 0c575f54..4c7f1572 100644 --- a/channels/channels.go +++ b/channels/channels.go @@ -7,7 +7,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" - "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" peer "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -20,11 +20,9 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channels/internal" "github.com/filecoin-project/go-data-transfer/v2/channels/internal/migrations" - "github.com/filecoin-project/go-data-transfer/v2/encoding" + ipldutils "github.com/filecoin-project/go-data-transfer/v2/ipldutils" ) -type DecoderByTypeFunc func(identifier datatransfer.TypeIdentifier) (encoding.Decoder, bool) - type Notifier func(datatransfer.Event, datatransfer.ChannelState) // ErrNotFound is returned when a channel cannot be found with a given channel ID @@ -46,8 +44,6 @@ var ErrWrongType = errors.New("Cannot change type of implementation specific dat // Channels is a thread safe list of channels type Channels struct { notifier Notifier - voucherDecoder DecoderByTypeFunc - voucherResultDecoder DecoderByTypeFunc blockIndexCache *blockIndexCache progressCache *progressCache stateMachines fsm.Group @@ -65,16 +61,10 @@ type ChannelEnvironment interface { // New returns a new thread safe list of channels func New(ds datastore.Batching, notifier Notifier, - voucherDecoder DecoderByTypeFunc, - voucherResultDecoder DecoderByTypeFunc, env ChannelEnvironment, selfPeer peer.ID) (*Channels, error) { - c := &Channels{ - notifier: notifier, - voucherDecoder: voucherDecoder, - voucherResultDecoder: voucherResultDecoder, - } + c := &Channels{notifier: notifier} c.blockIndexCache = newBlockIndexCache() c.progressCache = newProgressCache() channelMigrations, err := migrations.GetChannelStateMigrations(selfPeer) @@ -121,7 +111,7 @@ func (c *Channels) dispatch(eventName fsm.EventName, channel fsm.StateType) { // CreateNew creates a new channel id and channel state and saves to channels. // returns error if the channel exists already. -func (c *Channels) CreateNew(selfPeer peer.ID, tid datatransfer.TransferID, baseCid cid.Cid, selector ipld.Node, voucher datatransfer.Voucher, initiator, dataSender, dataReceiver peer.ID) (datatransfer.ChannelID, error) { +func (c *Channels) CreateNew(selfPeer peer.ID, tid datatransfer.TransferID, baseCid cid.Cid, selector datamodel.Node, voucher datatransfer.TypedVoucher, initiator, dataSender, dataReceiver peer.ID) (datatransfer.ChannelID, error) { var responder peer.ID if dataSender == initiator { responder = dataReceiver @@ -129,11 +119,11 @@ func (c *Channels) CreateNew(selfPeer peer.ID, tid datatransfer.TransferID, base responder = dataSender } chid := datatransfer.ChannelID{Initiator: initiator, Responder: responder, ID: tid} - voucherBytes, err := encoding.Encode(voucher) + initialVoucher, err := ipldutils.NodeToDeferred(voucher.Voucher) if err != nil { return datatransfer.ChannelID{}, err } - selBytes, err := encoding.Encode(selector) + selBytes, err := ipldutils.NodeToBytes(selector) if err != nil { return datatransfer.ChannelID{}, err } @@ -149,10 +139,8 @@ func (c *Channels) CreateNew(selfPeer peer.ID, tid datatransfer.TransferID, base Stages: &datatransfer.ChannelStages{}, Vouchers: []internal.EncodedVoucher{ { - Type: voucher.Type(), - Voucher: &cbg.Deferred{ - Raw: voucherBytes, - }, + Type: voucher.Type, + Voucher: initialVoucher, }, }, Status: datatransfer.Requested, @@ -289,21 +277,21 @@ func (c *Channels) ResumeResponder(chid datatransfer.ChannelID) error { } // NewVoucher records a new voucher for this channel -func (c *Channels) NewVoucher(chid datatransfer.ChannelID, voucher datatransfer.Voucher) error { - voucherBytes, err := encoding.Encode(voucher) +func (c *Channels) NewVoucher(chid datatransfer.ChannelID, voucher datatransfer.TypedVoucher) error { + voucherBytes, err := ipldutils.NodeToBytes(voucher.Voucher) if err != nil { return err } - return c.send(chid, datatransfer.NewVoucher, voucher.Type(), voucherBytes) + return c.send(chid, datatransfer.NewVoucher, voucher.Type, voucherBytes) } // NewVoucherResult records a new voucher result for this channel -func (c *Channels) NewVoucherResult(chid datatransfer.ChannelID, voucherResult datatransfer.VoucherResult) error { - voucherResultBytes, err := encoding.Encode(voucherResult) +func (c *Channels) NewVoucherResult(chid datatransfer.ChannelID, voucherResult datatransfer.TypedVoucher) error { + voucherResultBytes, err := ipldutils.NodeToBytes(voucherResult.Voucher) if err != nil { return err } - return c.send(chid, datatransfer.NewVoucherResult, voucherResult.Type(), voucherResultBytes) + return c.send(chid, datatransfer.NewVoucherResult, voucherResult.Type, voucherResultBytes) } // Complete indicates responder has completed sending/receiving data @@ -485,5 +473,5 @@ func (c *Channels) checkChannelExists(chid datatransfer.ChannelID, code datatran // Convert from the internally used channel state format to the externally exposed ChannelState func (c *Channels) fromInternalChannelState(ch internal.ChannelState) datatransfer.ChannelState { - return fromInternalChannelState(ch, c.voucherDecoder, c.voucherResultDecoder) + return fromInternalChannelState(ch) } diff --git a/channels/channels_test.go b/channels/channels_test.go index 8b739282..505afa34 100644 --- a/channels/channels_test.go +++ b/channels/channels_test.go @@ -16,7 +16,6 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channels" - "github.com/filecoin-project/go-data-transfer/v2/encoding" "github.com/filecoin-project/go-data-transfer/v2/testutil" ) @@ -32,13 +31,13 @@ func TestChannels(t *testing.T) { tid1 := datatransfer.TransferID(0) tid2 := datatransfer.TransferID(1) - fv1 := &testutil.FakeDTType{} - fv2 := &testutil.FakeDTType{} + fv1 := testutil.NewTestTypedVoucher() + fv2 := testutil.NewTestTypedVoucher() cids := testutil.GenerateCids(4) selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() peers := testutil.GeneratePeers(4) - channelList, err := channels.New(ds, notifier, decoderByType, decoderByType, &fakeEnv{}, peers[0]) + channelList, err := channels.New(ds, notifier, &fakeEnv{}, peers[0]) require.NoError(t, err) err = channelList.Start(ctx) @@ -80,7 +79,9 @@ func TestChannels(t *testing.T) { require.NotEqual(t, channels.EmptyChannelState, state) require.Equal(t, cids[0], state.BaseCID()) require.Equal(t, selector, state.Selector()) - require.Equal(t, fv1, state.Voucher()) + voucher, err := state.Voucher() + require.NoError(t, err) + require.True(t, fv1.Equals(voucher)) require.Equal(t, peers[0], state.Sender()) require.Equal(t, peers[1], state.Recipient()) @@ -124,7 +125,7 @@ func TestChannels(t *testing.T) { t.Run("datasent/queued when transfer is already finished", func(t *testing.T) { ds := dss.MutexWrap(datastore.NewMapDatastore()) - channelList, err := channels.New(ds, notifier, decoderByType, decoderByType, &fakeEnv{}, peers[0]) + channelList, err := channels.New(ds, notifier, &fakeEnv{}, peers[0]) require.NoError(t, err) err = channelList.Start(ctx) require.NoError(t, err) @@ -156,7 +157,7 @@ func TestChannels(t *testing.T) { t.Run("updating send/receive values", func(t *testing.T) { ds := dss.MutexWrap(datastore.NewMapDatastore()) - channelList, err := channels.New(ds, notifier, decoderByType, decoderByType, &fakeEnv{}, peers[0]) + channelList, err := channels.New(ds, notifier, &fakeEnv{}, peers[0]) require.NoError(t, err) err = channelList.Start(ctx) require.NoError(t, err) @@ -218,7 +219,7 @@ func TestChannels(t *testing.T) { t.Run("data limit", func(t *testing.T) { ds := dss.MutexWrap(datastore.NewMapDatastore()) - channelList, err := channels.New(ds, notifier, decoderByType, decoderByType, &fakeEnv{}, peers[0]) + channelList, err := channels.New(ds, notifier, &fakeEnv{}, peers[0]) require.NoError(t, err) err = channelList.Start(ctx) require.NoError(t, err) @@ -297,31 +298,53 @@ func TestChannels(t *testing.T) { }) t.Run("new vouchers & voucherResults", func(t *testing.T) { - fv3 := testutil.NewFakeDTType() - fvr1 := testutil.NewFakeDTType() + fv3 := testutil.NewTestTypedVoucher() + fvr1 := testutil.NewTestTypedVoucher() state, err := channelList.GetByID(ctx, datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) require.NoError(t, err) - require.Equal(t, []datatransfer.Voucher{fv1}, state.Vouchers()) - require.Equal(t, fv1, state.Voucher()) - require.Equal(t, fv1, state.LastVoucher()) + vouchers, err := state.Vouchers() + require.NoError(t, err) + require.Len(t, vouchers, 1) + require.True(t, fv1.Equals(vouchers[0])) + voucher, err := state.Voucher() + require.NoError(t, err) + require.True(t, fv1.Equals(voucher)) + voucher, err = state.LastVoucher() + require.NoError(t, err) + require.True(t, fv1.Equals(voucher)) err = channelList.NewVoucher(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, fv3) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.NewVoucher) - require.Equal(t, []datatransfer.Voucher{fv1, fv3}, state.Vouchers()) - require.Equal(t, fv1, state.Voucher()) - require.Equal(t, fv3, state.LastVoucher()) + vouchers, err = state.Vouchers() + require.NoError(t, err) + require.Len(t, vouchers, 2) + require.True(t, fv1.Equals(vouchers[0])) + require.True(t, fv3.Equals(vouchers[1])) + voucher, err = state.Voucher() + require.NoError(t, err) + require.True(t, fv1.Equals(voucher)) + voucher, err = state.LastVoucher() + require.NoError(t, err) + require.True(t, fv3.Equals(voucher)) state, err = channelList.GetByID(ctx, datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) require.NoError(t, err) - require.Equal(t, []datatransfer.VoucherResult{}, state.VoucherResults()) + results, err := state.VoucherResults() + require.NoError(t, err) + require.Equal(t, []datatransfer.TypedVoucher{}, results) err = channelList.NewVoucherResult(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, fvr1) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.NewVoucherResult) - require.Equal(t, []datatransfer.VoucherResult{fvr1}, state.VoucherResults()) - require.Equal(t, fvr1, state.LastVoucherResult()) + voucherResults, err := state.VoucherResults() + require.NoError(t, err) + require.Len(t, voucherResults, 1) + require.True(t, fvr1.Equals(voucherResults[0])) + voucherResult, err := state.LastVoucherResult() + require.NoError(t, err) + require.True(t, fvr1.Equals(voucherResult)) }) t.Run("test finality", func(t *testing.T) { @@ -387,7 +410,7 @@ func TestChannels(t *testing.T) { notifier := func(evt datatransfer.Event, chst datatransfer.ChannelState) { received <- event{evt, chst} } - channelList, err := channels.New(ds, notifier, decoderByType, decoderByType, &fakeEnv{}, peers[0]) + channelList, err := channels.New(ds, notifier, &fakeEnv{}, peers[0]) require.NoError(t, err) err = channelList.Start(ctx) require.NoError(t, err) @@ -469,14 +492,3 @@ func (fe *fakeEnv) ID() peer.ID { func (fe *fakeEnv) CleanupChannel(chid datatransfer.ChannelID) { } - -func decoderByType(identifier datatransfer.TypeIdentifier) (encoding.Decoder, bool) { - if identifier == testutil.NewFakeDTType().Type() { - decoder, err := encoding.NewDecoder(testutil.NewFakeDTType()) - if err != nil { - return nil, false - } - return decoder, true - } - return nil, false -} diff --git a/encoding/encoding.go b/encoding/encoding.go deleted file mode 100644 index dec7abcd..00000000 --- a/encoding/encoding.go +++ /dev/null @@ -1,171 +0,0 @@ -package encoding - -import ( - "bytes" - "reflect" - - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/codec/dagcbor" - "github.com/ipld/go-ipld-prime/datamodel" - "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipld/go-ipld-prime/schema" - cborgen "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" -) - -// Encodable is an object that can be written to CBOR and decoded back -type Encodable interface{} - -// Encode encodes an encodable to CBOR, using the best available path for -// writing to CBOR -func Encode(value Encodable) ([]byte, error) { - if cbgEncodable, ok := value.(cborgen.CBORMarshaler); ok { - buf := new(bytes.Buffer) - err := cbgEncodable.MarshalCBOR(buf) - if err != nil { - return nil, err - } - return buf.Bytes(), nil - } - if ipldEncodable, ok := value.(datamodel.Node); ok { - if tn, ok := ipldEncodable.(schema.TypedNode); ok { - ipldEncodable = tn.Representation() - } - buf := &bytes.Buffer{} - err := dagcbor.Encode(ipldEncodable, buf) - if err != nil { - return nil, err - } - return buf.Bytes(), nil - } - return cbor.DumpObject(value) -} - -func EncodeToNode(encodable Encodable) (datamodel.Node, error) { - byts, err := Encode(encodable) - if err != nil { - return nil, err - } - na := basicnode.Prototype.Any.NewBuilder() - if err := dagcbor.Decode(na, bytes.NewReader(byts)); err != nil { - return nil, err - } - return na.Build(), nil -} - -// Decoder is CBOR decoder for a given encodable type -type Decoder interface { - DecodeFromCbor([]byte) (Encodable, error) - DecodeFromNode(datamodel.Node) (Encodable, error) -} - -// NewDecoder creates a new Decoder that will decode into new instances of the given -// object type. It will use the decoding that is optimal for that type -// It returns error if it's not possible to setup a decoder for this type -func NewDecoder(decodeType Encodable) (Decoder, error) { - // check if type is datamodel.Node, if so, just use style - if ipldDecodable, ok := decodeType.(datamodel.Node); ok { - return &ipldDecoder{ipldDecodable.Prototype()}, nil - } - // check if type is a pointer, as we need that to make new copies - // for cborgen types & regular IPLD types - decodeReflectType := reflect.TypeOf(decodeType) - if decodeReflectType.Kind() != reflect.Ptr { - return nil, xerrors.New("type must be a pointer") - } - // check if type is a cbor-gen type - if _, ok := decodeType.(cborgen.CBORUnmarshaler); ok { - return &cbgDecoder{decodeReflectType}, nil - } - // type does is neither ipld-prime nor cbor-gen, so we need to see if it - // can rountrip with oldschool ipld-format - encoded, err := cbor.DumpObject(decodeType) - if err != nil { - return nil, xerrors.New("Object type did not encode") - } - newDecodable := reflect.New(decodeReflectType.Elem()).Interface() - if err := cbor.DecodeInto(encoded, newDecodable); err != nil { - return nil, xerrors.New("Object type did not decode") - } - return &defaultDecoder{decodeReflectType}, nil -} - -type ipldDecoder struct { - style ipld.NodePrototype -} - -func (decoder *ipldDecoder) DecodeFromCbor(encoded []byte) (Encodable, error) { - builder := decoder.style.NewBuilder() - buf := bytes.NewReader(encoded) - err := dagcbor.Decode(builder, buf) - if err != nil { - return nil, err - } - return builder.Build(), nil -} - -func (decoder *ipldDecoder) DecodeFromNode(node datamodel.Node) (Encodable, error) { - builder := decoder.style.NewBuilder() - if err := builder.AssignNode(node); err != nil { - return nil, err - } - return builder.Build(), nil -} - -type cbgDecoder struct { - cbgType reflect.Type -} - -func (decoder *cbgDecoder) DecodeFromCbor(encoded []byte) (Encodable, error) { - decodedValue := reflect.New(decoder.cbgType.Elem()) - decoded, ok := decodedValue.Interface().(cborgen.CBORUnmarshaler) - if !ok || reflect.ValueOf(decoded).IsNil() { - return nil, xerrors.New("problem instantiating decoded value") - } - buf := bytes.NewReader(encoded) - err := decoded.UnmarshalCBOR(buf) - if err != nil { - return nil, err - } - return decoded, nil -} - -func (decoder *cbgDecoder) DecodeFromNode(node datamodel.Node) (Encodable, error) { - if tn, ok := node.(schema.TypedNode); ok { - node = tn.Representation() - } - buf := &bytes.Buffer{} - if err := dagcbor.Encode(node, buf); err != nil { - return nil, err - } - return decoder.DecodeFromCbor(buf.Bytes()) -} - -type defaultDecoder struct { - ptrType reflect.Type -} - -func (decoder *defaultDecoder) DecodeFromCbor(encoded []byte) (Encodable, error) { - decodedValue := reflect.New(decoder.ptrType.Elem()) - decoded, ok := decodedValue.Interface().(Encodable) - if !ok || reflect.ValueOf(decoded).IsNil() { - return nil, xerrors.New("problem instantiating decoded value") - } - err := cbor.DecodeInto(encoded, decoded) - if err != nil { - return nil, err - } - return decoded, nil -} - -func (decoder *defaultDecoder) DecodeFromNode(node datamodel.Node) (Encodable, error) { - if tn, ok := node.(schema.TypedNode); ok { - node = tn.Representation() - } - buf := &bytes.Buffer{} - if err := dagcbor.Encode(node, buf); err != nil { - return nil, err - } - return decoder.DecodeFromCbor(buf.Bytes()) -} diff --git a/encoding/encoding_test.go b/encoding/encoding_test.go deleted file mode 100644 index 3d751f7a..00000000 --- a/encoding/encoding_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package encoding_test - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/filecoin-project/go-data-transfer/v2/encoding" - "github.com/filecoin-project/go-data-transfer/v2/encoding/testdata" -) - -func TestRoundTrip(t *testing.T) { - testCases := map[string]struct { - val encoding.Encodable - }{ - "can encode/decode IPLD prime types": { - val: testdata.Prime, - }, - "can encode/decode cbor-gen types": { - val: testdata.Cbg, - }, - "can encode/decode old ipld format types": { - val: testdata.Standard, - }, - } - for testCase, data := range testCases { - t.Run(testCase, func(t *testing.T) { - encoded, err := encoding.Encode(data.val) - require.NoError(t, err) - decoder, err := encoding.NewDecoder(data.val) - require.NoError(t, err) - decoded, err := decoder.DecodeFromCbor(encoded) - require.NoError(t, err) - require.Equal(t, data.val, decoded) - }) - } -} diff --git a/encoding/testdata/testdata.go b/encoding/testdata/testdata.go deleted file mode 100644 index 5bed37ba..00000000 --- a/encoding/testdata/testdata.go +++ /dev/null @@ -1,37 +0,0 @@ -package testdata - -import ( - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/ipld/go-ipld-prime/fluent" - basicnode "github.com/ipld/go-ipld-prime/node/basic" -) - -// Prime = an instance of an ipld prime piece of data -var Prime = fluent.MustBuildMap(basicnode.Prototype.Map, 2, func(na fluent.MapAssembler) { - nva := na.AssembleEntry("X") - nva.AssignInt(100) - nva = na.AssembleEntry("Y") - nva.AssignString("appleSauce") -}) - -type standardType struct { - X int - Y string -} - -func init() { - cbor.RegisterCborType(standardType{}) -} - -// Standard = an instance that is neither ipld prime nor cbor -var Standard *standardType = &standardType{X: 100, Y: "appleSauce"} - -//go:generate cbor-gen-for cbgType - -type cbgType struct { - X uint64 - Y string -} - -// Cbg = an instance of a cbor-gen type -var Cbg *cbgType = &cbgType{X: 100, Y: "appleSauce"} diff --git a/encoding/testdata/testdata_cbor_gen.go b/encoding/testdata/testdata_cbor_gen.go deleted file mode 100644 index 67c6c688..00000000 --- a/encoding/testdata/testdata_cbor_gen.go +++ /dev/null @@ -1,84 +0,0 @@ -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -package testdata - -import ( - "fmt" - "io" - - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -var _ = xerrors.Errorf - -func (t *cbgType) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.X (uint64) (uint64) - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.X))); err != nil { - return err - } - - // t.Y (string) (string) - if len(t.Y) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Y was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Y)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Y)); err != nil { - return err - } - return nil -} - -func (t *cbgType) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.X (uint64) (uint64) - - { - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.X = uint64(extra) - - } - // t.Y (string) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Y = string(sval) - } - return nil -} diff --git a/go.mod b/go.mod index de44622d..83c8baf9 100644 --- a/go.mod +++ b/go.mod @@ -20,12 +20,11 @@ require ( github.com/ipfs/go-ipfs-delay v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.1.1 github.com/ipfs/go-ipfs-files v0.0.8 - github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-log/v2 v2.3.0 github.com/ipfs/go-merkledag v0.5.1 github.com/ipfs/go-unixfs v0.3.1 - github.com/ipld/go-ipld-prime v0.16.0 + github.com/ipld/go-ipld-prime v0.16.1-0.20220519105356-1f1151b69dba github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jpillora/backoff v1.0.0 github.com/libp2p/go-libp2p v0.16.0 @@ -71,6 +70,7 @@ require ( github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.2 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect + github.com/ipfs/go-ipld-cbor v0.0.5 // indirect github.com/ipfs/go-ipld-legacy v0.1.0 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect diff --git a/go.sum b/go.sum index e5558906..a4b6b609 100644 --- a/go.sum +++ b/go.sum @@ -483,8 +483,9 @@ github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+4 github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= -github.com/ipld/go-ipld-prime v0.16.0 h1:RS5hhjB/mcpeEPJvfyj0qbOj/QL+/j05heZ0qa97dVo= github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= +github.com/ipld/go-ipld-prime v0.16.1-0.20220519105356-1f1151b69dba h1:1eimQ/EpBUnxyhvSQ9gxzokN9EiDYHCeZ2URkhADIGQ= +github.com/ipld/go-ipld-prime v0.16.1-0.20220519105356-1f1151b69dba/go.mod h1:/bZAYlzT7SJS4UV0al4q67xgKvenm5hKrPCa2wNGN1U= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= diff --git a/impl/events.go b/impl/events.go index ef5c5fb8..3ac2e64c 100644 --- a/impl/events.go +++ b/impl/events.go @@ -38,7 +38,7 @@ func (m *manager) OnChannelOpened(chid datatransfer.ChannelID) error { // back a pause to the transport if the data limit is exceeded func (m *manager) OnDataReceived(chid datatransfer.ChannelID, link ipld.Link, size uint64, index int64, unique bool) error { ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) - ctx, span := otel.Tracer("data-transfer").Start(ctx, "dataReceived", trace.WithAttributes( + _, span := otel.Tracer("data-transfer").Start(ctx, "dataReceived", trace.WithAttributes( attribute.String("channelID", chid.String()), attribute.String("link", link.String()), attribute.Int64("index", index), @@ -69,7 +69,7 @@ func (m *manager) OnDataQueued(chid datatransfer.ChannelID, link ipld.Link, size // machine. ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) - ctx, span := otel.Tracer("data-transfer").Start(ctx, "dataQueued", trace.WithAttributes( + _, span := otel.Tracer("data-transfer").Start(ctx, "dataQueued", trace.WithAttributes( attribute.String("channelID", chid.String()), attribute.String("link", link.String()), attribute.Int64("size", int64(size)), @@ -91,7 +91,7 @@ func (m *manager) OnDataQueued(chid datatransfer.ChannelID, link ipld.Link, size func (m *manager) OnDataSent(chid datatransfer.ChannelID, link ipld.Link, size uint64, index int64, unique bool) error { ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) - ctx, span := otel.Tracer("data-transfer").Start(ctx, "dataSent", trace.WithAttributes( + _, span := otel.Tracer("data-transfer").Start(ctx, "dataSent", trace.WithAttributes( attribute.String("channelID", chid.String()), attribute.String("link", link.String()), attribute.Int64("size", int64(size)), @@ -153,11 +153,11 @@ func (m *manager) OnResponseReceived(chid datatransfer.ChannelID, response datat // is there a voucher response in this message? if !response.EmptyVoucherResult() { // if so decode and save it - vresult, err := m.decodeVoucherResult(response) + vresult, err := response.VoucherResult() if err != nil { return err } - err = m.channels.NewVoucherResult(chid, vresult) + err = m.channels.NewVoucherResult(chid, datatransfer.TypedVoucher{Voucher: vresult, Type: response.VoucherResultType()}) if err != nil { return err } @@ -271,7 +271,7 @@ func (m *manager) OnChannelCompleted(chid datatransfer.ChannelID, completeErr er log.Infow("received OnChannelCompleted, will send completion message to initiator", "chid", chid) // generate and send the final status message - msg, err := message.CompleteResponse(chst.TransferID(), true, chst.RequiresFinalization(), datatransfer.EmptyTypeIdentifier, nil) + msg, err := message.CompleteResponse(chst.TransferID(), true, chst.RequiresFinalization(), nil) if err != nil { return err } diff --git a/impl/impl.go b/impl/impl.go index efd7e878..61f0fe67 100644 --- a/impl/impl.go +++ b/impl/impl.go @@ -10,7 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" - "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/libp2p/go-libp2p-core/peer" "go.opentelemetry.io/otel" @@ -22,7 +22,6 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channelmonitor" "github.com/filecoin-project/go-data-transfer/v2/channels" - "github.com/filecoin-project/go-data-transfer/v2/encoding" "github.com/filecoin-project/go-data-transfer/v2/message" "github.com/filecoin-project/go-data-transfer/v2/message/types" "github.com/filecoin-project/go-data-transfer/v2/network" @@ -36,7 +35,6 @@ var cancelSendTimeout = 30 * time.Second type manager struct { dataTransferNetwork network.DataTransferNetwork validatedTypes *registry.Registry - resultTypes *registry.Registry transportConfigurers *registry.Registry pubSub *pubsub.PubSub readySub *pubsub.PubSub @@ -96,7 +94,6 @@ func NewDataTransfer(ds datastore.Batching, dataTransferNetwork network.DataTran m := &manager{ dataTransferNetwork: dataTransferNetwork, validatedTypes: registry.NewRegistry(), - resultTypes: registry.NewRegistry(), transportConfigurers: registry.NewRegistry(), pubSub: pubsub.New(dispatcher), readySub: pubsub.New(readyDispatcher), @@ -106,7 +103,7 @@ func NewDataTransfer(ds datastore.Batching, dataTransferNetwork network.DataTran spansIndex: tracing.NewSpansIndex(), } - channels, err := channels.New(ds, m.notifier, m.voucherDecoder, m.resultTypes.Decoder, &channelEnvironment{m}, dataTransferNetwork.ID()) + channels, err := channels.New(ds, m.notifier, &channelEnvironment{m}, dataTransferNetwork.ID()) if err != nil { return nil, err } @@ -124,10 +121,6 @@ func NewDataTransfer(ds datastore.Batching, dataTransferNetwork network.DataTran return m, nil } -func (m *manager) voucherDecoder(voucherType datatransfer.TypeIdentifier) (encoding.Decoder, bool) { - return m.validatedTypes.Decoder(voucherType) -} - func (m *manager) notifier(evt datatransfer.Event, chst datatransfer.ChannelState) { err := m.pubSub.Publish(internalEvent{evt, chst}) if err != nil { @@ -173,7 +166,7 @@ func (m *manager) Stop(ctx context.Context) error { // * voucher type does not implement voucher // * there is a voucher type registered with an identical identifier // * voucherType's Kind is not reflect.Ptr -func (m *manager) RegisterVoucherType(voucherType datatransfer.Voucher, validator datatransfer.RequestValidator) error { +func (m *manager) RegisterVoucherType(voucherType datatransfer.TypeIdentifier, validator datatransfer.RequestValidator) error { err := m.validatedTypes.Register(voucherType, validator) if err != nil { return xerrors.Errorf("error registering voucher type: %w", err) @@ -183,7 +176,7 @@ func (m *manager) RegisterVoucherType(voucherType datatransfer.Voucher, validato // OpenPushDataChannel opens a data transfer that will send data to the recipient peer and // transfer parts of the piece that match the selector -func (m *manager) OpenPushDataChannel(ctx context.Context, requestTo peer.ID, voucher datatransfer.Voucher, baseCid cid.Cid, selector ipld.Node) (datatransfer.ChannelID, error) { +func (m *manager) OpenPushDataChannel(ctx context.Context, requestTo peer.ID, voucher datatransfer.TypedVoucher, baseCid cid.Cid, selector datamodel.Node) (datatransfer.ChannelID, error) { log.Infof("open push channel to %s with base cid %s", requestTo, baseCid) req, err := m.newRequest(ctx, selector, false, voucher, baseCid, requestTo) @@ -197,7 +190,7 @@ func (m *manager) OpenPushDataChannel(ctx context.Context, requestTo peer.ID, vo return chid, err } ctx, span := m.spansIndex.SpanForChannel(ctx, chid) - processor, has := m.transportConfigurers.Processor(voucher.Type()) + processor, has := m.transportConfigurers.Processor(voucher.Type) if has { transportConfigurer := processor.(datatransfer.TransportConfigurer) transportConfigurer(chid, voucher, m.transport) @@ -205,7 +198,7 @@ func (m *manager) OpenPushDataChannel(ctx context.Context, requestTo peer.ID, vo m.dataTransferNetwork.Protect(requestTo, chid.String()) monitoredChan := m.channelMonitor.AddPushChannel(chid) if err := m.dataTransferNetwork.SendMessage(ctx, requestTo, req); err != nil { - err = fmt.Errorf("Unable to send request: %w", err) + err = fmt.Errorf("unable to send request: %w", err) span.RecordError(err) span.SetStatus(codes.Error, err.Error()) _ = m.channels.Error(chid, err) @@ -224,7 +217,7 @@ func (m *manager) OpenPushDataChannel(ctx context.Context, requestTo peer.ID, vo // OpenPullDataChannel opens a data transfer that will request data from the sending peer and // transfer parts of the piece that match the selector -func (m *manager) OpenPullDataChannel(ctx context.Context, requestTo peer.ID, voucher datatransfer.Voucher, baseCid cid.Cid, selector ipld.Node) (datatransfer.ChannelID, error) { +func (m *manager) OpenPullDataChannel(ctx context.Context, requestTo peer.ID, voucher datatransfer.TypedVoucher, baseCid cid.Cid, selector datamodel.Node) (datatransfer.ChannelID, error) { log.Infof("open pull channel to %s with base cid %s", requestTo, baseCid) req, err := m.newRequest(ctx, selector, true, voucher, baseCid, requestTo) @@ -238,7 +231,7 @@ func (m *manager) OpenPullDataChannel(ctx context.Context, requestTo peer.ID, vo return chid, err } ctx, span := m.spansIndex.SpanForChannel(ctx, chid) - processor, has := m.transportConfigurers.Processor(voucher.Type()) + processor, has := m.transportConfigurers.Processor(voucher.Type) if has { transportConfigurer := processor.(datatransfer.TransportConfigurer) transportConfigurer(chid, voucher, m.transport) @@ -246,7 +239,7 @@ func (m *manager) OpenPullDataChannel(ctx context.Context, requestTo peer.ID, vo m.dataTransferNetwork.Protect(requestTo, chid.String()) monitoredChan := m.channelMonitor.AddPullChannel(chid) if err := m.transport.OpenChannel(ctx, requestTo, chid, cidlink.Link{Cid: baseCid}, selector, nil, req); err != nil { - err = fmt.Errorf("Unable to send request: %w", err) + err = fmt.Errorf("unable to send request: %w", err) span.RecordError(err) span.SetStatus(codes.Error, err.Error()) _ = m.channels.Error(chid, err) @@ -262,7 +255,7 @@ func (m *manager) OpenPullDataChannel(ctx context.Context, requestTo peer.ID, vo } // SendVoucher sends an intermediate voucher as needed when the receiver sends a request for revalidation -func (m *manager) SendVoucher(ctx context.Context, channelID datatransfer.ChannelID, voucher datatransfer.Voucher) error { +func (m *manager) SendVoucher(ctx context.Context, channelID datatransfer.ChannelID, voucher datatransfer.TypedVoucher) error { chst, err := m.channels.GetByID(ctx, channelID) if err != nil { return err @@ -270,7 +263,6 @@ func (m *manager) SendVoucher(ctx context.Context, channelID datatransfer.Channe ctx, _ = m.spansIndex.SpanForChannel(ctx, channelID) ctx, span := otel.Tracer("data-transfer").Start(ctx, "sendVoucher", trace.WithAttributes( attribute.String("channelID", channelID.String()), - attribute.String("voucherType", string(voucher.Type())), )) defer span.End() if channelID.Initiator != m.peerID { @@ -279,14 +271,14 @@ func (m *manager) SendVoucher(ctx context.Context, channelID datatransfer.Channe span.SetStatus(codes.Error, err.Error()) return err } - updateRequest, err := message.VoucherRequest(channelID.ID, voucher.Type(), voucher) + updateRequest, err := message.VoucherRequest(channelID.ID, &voucher) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) return err } if err := m.dataTransferNetwork.SendMessage(ctx, chst.OtherPeer(), updateRequest); err != nil { - err = fmt.Errorf("Unable to send request: %w", err) + err = fmt.Errorf("unable to send request: %w", err) _ = m.OnRequestDisconnected(channelID, err) span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -295,7 +287,7 @@ func (m *manager) SendVoucher(ctx context.Context, channelID datatransfer.Channe return m.channels.NewVoucher(channelID, voucher) } -func (m *manager) SendVoucherResult(ctx context.Context, channelID datatransfer.ChannelID, voucherResult datatransfer.VoucherResult) error { +func (m *manager) SendVoucherResult(ctx context.Context, channelID datatransfer.ChannelID, voucherResult datatransfer.TypedVoucher) error { chst, err := m.channels.GetByID(ctx, channelID) if err != nil { return err @@ -303,7 +295,6 @@ func (m *manager) SendVoucherResult(ctx context.Context, channelID datatransfer. ctx, _ = m.spansIndex.SpanForChannel(ctx, channelID) ctx, span := otel.Tracer("data-transfer").Start(ctx, "sendVoucherResult", trace.WithAttributes( attribute.String("channelID", channelID.String()), - attribute.String("voucherResultType", string(voucherResult.Type())), )) defer span.End() if channelID.Initiator == m.peerID { @@ -315,9 +306,9 @@ func (m *manager) SendVoucherResult(ctx context.Context, channelID datatransfer. var updateResponse datatransfer.Response if chst.Status().InFinalization() { - updateResponse, err = message.CompleteResponse(channelID.ID, chst.Status().IsAccepted(), chst.Status().IsResponderPaused(), voucherResult.Type(), voucherResult) + updateResponse, err = message.CompleteResponse(channelID.ID, chst.Status().IsAccepted(), chst.Status().IsResponderPaused(), &voucherResult) } else { - updateResponse, err = message.VoucherResultResponse(channelID.ID, chst.Status().IsAccepted(), chst.Status().IsResponderPaused(), voucherResult.Type(), voucherResult) + updateResponse, err = message.VoucherResultResponse(channelID.ID, chst.Status().IsAccepted(), chst.Status().IsResponderPaused(), &voucherResult) } if err != nil { @@ -326,7 +317,7 @@ func (m *manager) SendVoucherResult(ctx context.Context, channelID datatransfer. return err } if err := m.dataTransferNetwork.SendMessage(ctx, chst.OtherPeer(), updateResponse); err != nil { - err = fmt.Errorf("Unable to send request: %w", err) + err = fmt.Errorf("unable to send request: %w", err) _ = m.OnRequestDisconnected(channelID, err) span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -549,7 +540,7 @@ func (m *manager) PauseDataTransferChannel(ctx context.Context, chid datatransfe } if err := m.dataTransferNetwork.SendMessage(ctx, chid.OtherParty(m.peerID), m.pauseMessage(chid)); err != nil { - err = fmt.Errorf("Unable to send pause message: %w", err) + err = fmt.Errorf("unable to send pause message: %w", err) _ = m.OnRequestDisconnected(chid, err) return err } @@ -600,19 +591,9 @@ func (m *manager) InProgressChannels(ctx context.Context) (map[datatransfer.Chan return m.channels.InProgress() } -// RegisterVoucherResultType allows deserialization of a voucher result, -// so that a listener can read the metadata -func (m *manager) RegisterVoucherResultType(resultType datatransfer.VoucherResult) error { - err := m.resultTypes.Register(resultType, nil) - if err != nil { - return xerrors.Errorf("error registering voucher type: %w", err) - } - return nil -} - // RegisterTransportConfigurer registers the given transport configurer to be run on requests with the given voucher // type -func (m *manager) RegisterTransportConfigurer(voucherType datatransfer.Voucher, configurer datatransfer.TransportConfigurer) error { +func (m *manager) RegisterTransportConfigurer(voucherType datatransfer.TypeIdentifier, configurer datatransfer.TransportConfigurer) error { err := m.transportConfigurers.Register(voucherType, configurer) if err != nil { return xerrors.Errorf("error registering transport configurer: %w", err) diff --git a/impl/initiating_test.go b/impl/initiating_test.go index 0d1274b7..8a393834 100644 --- a/impl/initiating_test.go +++ b/impl/initiating_test.go @@ -9,7 +9,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" dss "github.com/ipfs/go-datastore/sync" - "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" @@ -52,7 +52,7 @@ func TestDataTransferInitiating(t *testing.T) { receivedSelector, err := receivedRequest.Selector() require.NoError(t, err) require.Equal(t, receivedSelector, h.stor) - testutil.AssertFakeDTVoucher(t, receivedRequest, h.voucher) + testutil.AssertTestVoucher(t, receivedRequest, h.voucher) }, }, "OpenPullDataTransfer": { @@ -79,7 +79,7 @@ func TestDataTransferInitiating(t *testing.T) { receivedSelector, err := receivedRequest.Selector() require.NoError(t, err) require.Equal(t, receivedSelector, h.stor) - testutil.AssertFakeDTVoucher(t, receivedRequest, h.voucher) + testutil.AssertTestVoucher(t, receivedRequest, h.voucher) }, }, "SendVoucher with no channel open": { @@ -93,7 +93,7 @@ func TestDataTransferInitiating(t *testing.T) { verify: func(t *testing.T, h *harness) { channelID, err := h.dt.OpenPushDataChannel(h.ctx, h.peers[1], h.voucher, h.baseCid, h.stor) require.NoError(t, err) - voucher := testutil.NewFakeDTType() + voucher := testutil.NewTestTypedVoucher() err = h.dt.SendVoucher(ctx, channelID, voucher) require.NoError(t, err) require.Len(t, h.network.SentMessages, 2) @@ -103,7 +103,7 @@ func TestDataTransferInitiating(t *testing.T) { require.True(t, ok) require.True(t, receivedRequest.IsVoucher()) require.False(t, receivedRequest.IsCancel()) - testutil.AssertFakeDTVoucher(t, receivedRequest, voucher) + testutil.AssertTestVoucher(t, receivedRequest, voucher) }, }, "SendVoucher with channel open, pull succeeds": { @@ -111,7 +111,7 @@ func TestDataTransferInitiating(t *testing.T) { verify: func(t *testing.T, h *harness) { channelID, err := h.dt.OpenPullDataChannel(h.ctx, h.peers[1], h.voucher, h.baseCid, h.stor) require.NoError(t, err) - voucher := testutil.NewFakeDTType() + voucher := testutil.NewTestTypedVoucher() err = h.dt.SendVoucher(ctx, channelID, voucher) require.NoError(t, err) require.Len(t, h.transport.OpenedChannels, 1) @@ -122,26 +122,16 @@ func TestDataTransferInitiating(t *testing.T) { require.True(t, ok) require.False(t, receivedRequest.IsCancel()) require.True(t, receivedRequest.IsVoucher()) - testutil.AssertFakeDTVoucher(t, receivedRequest, voucher) + testutil.AssertTestVoucher(t, receivedRequest, voucher) }, }, "reregister voucher type again errors": { verify: func(t *testing.T, h *harness) { - voucher := testutil.NewFakeDTType() sv := testutil.NewStubbedValidator() - err := h.dt.RegisterVoucherType(h.voucher, sv) + err := h.dt.RegisterVoucherType(h.voucher.Type, sv) require.NoError(t, err) - err = h.dt.RegisterVoucherType(voucher, sv) - require.EqualError(t, err, "error registering voucher type: identifier already registered: FakeDTType") - }, - }, - "reregister non pointer errors": { - verify: func(t *testing.T, h *harness) { - sv := testutil.NewStubbedValidator() - err := h.dt.RegisterVoucherType(h.voucher, sv) - require.NoError(t, err) - err = h.dt.RegisterVoucherType(testutil.FakeDTType{}, sv) - require.EqualError(t, err, "error registering voucher type: registering entry type FakeDTType: type must be a pointer") + err = h.dt.RegisterVoucherType(testutil.TestVoucherType, sv) + require.EqualError(t, err, "error registering voucher type: identifier already registered: TestVoucher") }, }, "success response": { @@ -150,7 +140,7 @@ func TestDataTransferInitiating(t *testing.T) { channelID, err := h.dt.OpenPushDataChannel(h.ctx, h.peers[1], h.voucher, h.baseCid, h.stor) require.NoError(t, err) require.NotEmpty(t, channelID) - response, err := message.NewResponse(channelID.ID, true, false, datatransfer.EmptyTypeIdentifier, nil) + response, err := message.NewResponse(channelID.ID, true, false, nil) require.NoError(t, err) err = h.transport.EventHandler.OnResponseReceived(channelID, response) require.NoError(t, err) @@ -162,7 +152,7 @@ func TestDataTransferInitiating(t *testing.T) { channelID, err := h.dt.OpenPushDataChannel(h.ctx, h.peers[1], h.voucher, h.baseCid, h.stor) require.NoError(t, err) require.NotEmpty(t, channelID) - response, err := message.NewResponse(channelID.ID, true, false, h.voucherResult.Type(), h.voucherResult) + response, err := message.NewResponse(channelID.ID, true, false, &h.voucherResult) require.NoError(t, err) err = h.transport.EventHandler.OnResponseReceived(channelID, response) require.NoError(t, err) @@ -174,7 +164,7 @@ func TestDataTransferInitiating(t *testing.T) { channelID, err := h.dt.OpenPushDataChannel(h.ctx, h.peers[1], h.voucher, h.baseCid, h.stor) require.NoError(t, err) require.NotEmpty(t, channelID) - response, err := message.NewResponse(channelID.ID, true, false, datatransfer.EmptyTypeIdentifier, nil) + response, err := message.NewResponse(channelID.ID, true, false, nil) require.NoError(t, err) err = h.transport.EventHandler.OnResponseReceived(channelID, response) require.NoError(t, err) @@ -230,7 +220,7 @@ func TestDataTransferInitiating(t *testing.T) { channelID, err := h.dt.OpenPullDataChannel(h.ctx, h.peers[1], h.voucher, h.baseCid, h.stor) require.NoError(t, err) require.NotEmpty(t, channelID) - response, err := message.NewResponse(channelID.ID, true, false, datatransfer.EmptyTypeIdentifier, nil) + response, err := message.NewResponse(channelID.ID, true, false, nil) require.NoError(t, err) err = h.transport.EventHandler.OnResponseReceived(channelID, response) require.NoError(t, err) @@ -284,7 +274,7 @@ func TestDataTransferInitiating(t *testing.T) { "customizing push transfer": { expectedEvents: []datatransfer.EventCode{datatransfer.Open}, verify: func(t *testing.T, h *harness) { - err := h.dt.RegisterTransportConfigurer(h.voucher, func(channelID datatransfer.ChannelID, voucher datatransfer.Voucher, transport datatransfer.Transport) { + err := h.dt.RegisterTransportConfigurer(h.voucher.Type, func(channelID datatransfer.ChannelID, voucher datatransfer.TypedVoucher, transport datatransfer.Transport) { ft, ok := transport.(*testutil.FakeTransport) if !ok { return @@ -304,7 +294,7 @@ func TestDataTransferInitiating(t *testing.T) { "customizing pull transfer": { expectedEvents: []datatransfer.EventCode{datatransfer.Open}, verify: func(t *testing.T, h *harness) { - err := h.dt.RegisterTransportConfigurer(h.voucher, func(channelID datatransfer.ChannelID, voucher datatransfer.Voucher, transport datatransfer.Transport) { + err := h.dt.RegisterTransportConfigurer(h.voucher.Type, func(channelID datatransfer.ChannelID, voucher datatransfer.TypedVoucher, transport datatransfer.Transport) { ft, ok := transport.(*testutil.FakeTransport) if !ok { return @@ -344,9 +334,8 @@ func TestDataTransferInitiating(t *testing.T) { } ev.setup(t, dt) h.stor = testutil.AllSelector() - h.voucher = testutil.NewFakeDTType() - h.voucherResult = testutil.NewFakeDTType() - err = h.dt.RegisterVoucherResultType(h.voucherResult) + h.voucher = testutil.NewTestTypedVoucher() + h.voucherResult = testutil.NewTestTypedVoucher() require.NoError(t, err) h.baseCid = testutil.GenerateCids(1)[0] verify.verify(t, h) @@ -405,7 +394,7 @@ func TestDataTransferRestartInitiating(t *testing.T) { receivedSelector, err := receivedRequest.Selector() require.NoError(t, err) require.Equal(t, receivedSelector, h.stor) - testutil.AssertFakeDTVoucher(t, receivedRequest, h.voucher) + testutil.AssertTestVoucher(t, receivedRequest, h.voucher) }, }, "RestartDataTransferChannel: Manager Peer Create Push Restart works": { @@ -441,7 +430,7 @@ func TestDataTransferRestartInitiating(t *testing.T) { receivedSelector, err := receivedRequest.Selector() require.NoError(t, err) require.Equal(t, receivedSelector, h.stor) - testutil.AssertFakeDTVoucher(t, receivedRequest, h.voucher) + testutil.AssertTestVoucher(t, receivedRequest, h.voucher) }, }, "RestartDataTransferChannel: Manager Peer Receive Push Restart works ": { @@ -610,17 +599,16 @@ func TestDataTransferRestartInitiating(t *testing.T) { // setup voucher processing h.stor = testutil.AllSelector() - h.voucher = testutil.NewFakeDTType() - require.NoError(t, h.dt.RegisterVoucherType(h.voucher, h.voucherValidator)) - h.voucherResult = testutil.NewFakeDTType() - err = h.dt.RegisterVoucherResultType(h.voucherResult) + h.voucher = testutil.NewTestTypedVoucher() + require.NoError(t, h.dt.RegisterVoucherType(h.voucher.Type, h.voucherValidator)) + h.voucherResult = testutil.NewTestTypedVoucher() require.NoError(t, err) h.baseCid = testutil.GenerateCids(1)[0] h.id = datatransfer.TransferID(rand.Int31()) - h.pushRequest, err = message.NewRequest(h.id, false, false, h.voucher.Type(), h.voucher, h.baseCid, h.stor) + h.pushRequest, err = message.NewRequest(h.id, false, false, &h.voucher, h.baseCid, h.stor) require.NoError(t, err) - h.pullRequest, err = message.NewRequest(h.id, false, true, h.voucher.Type(), h.voucher, h.baseCid, h.stor) + h.pullRequest, err = message.NewRequest(h.id, false, true, &h.voucher, h.baseCid, h.stor) require.NoError(t, err) // run tests steps and verify @@ -639,9 +627,9 @@ type harness struct { ds datastore.Batching dt datatransfer.Manager voucherValidator *testutil.StubbedValidator - stor ipld.Node - voucher *testutil.FakeDTType - voucherResult *testutil.FakeDTType + stor datamodel.Node + voucher datatransfer.TypedVoucher + voucherResult datatransfer.TypedVoucher baseCid cid.Cid id datatransfer.TransferID diff --git a/impl/integration_test.go b/impl/integration_test.go index 09896c30..53c375cc 100644 --- a/impl/integration_test.go +++ b/impl/integration_test.go @@ -27,6 +27,7 @@ import ( "github.com/ipfs/go-unixfs/importer/balanced" ihelper "github.com/ipfs/go-unixfs/importer/helpers" "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" @@ -36,7 +37,6 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channelmonitor" - "github.com/filecoin-project/go-data-transfer/v2/encoding" . "github.com/filecoin-project/go-data-transfer/v2/impl" "github.com/filecoin-project/go-data-transfer/v2/message" "github.com/filecoin-project/go-data-transfer/v2/network" @@ -49,7 +49,8 @@ const loremFile = "lorem.txt" const loremFileTransferBytes = 20439 const loremLargeFile = "lorem_large.txt" -const loremLargeFileTransferBytes = 217452 + +// const loremLargeFileTransferBytes = 217452 // nil means use the default protocols // tests data transfer for the following protocol combinations: @@ -181,7 +182,7 @@ func TestRoundTrip(t *testing.T) { } dt1.SubscribeToEvents(subscriber) dt2.SubscribeToEvents(subscriber) - voucher := testutil.FakeDTType{Data: "applesauce"} + voucher := testutil.NewTestTypedVoucherWith("applesauce") sv := testutil.NewStubbedValidator() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) @@ -191,9 +192,8 @@ func TestRoundTrip(t *testing.T) { bs := bstore.NewBlockstore(namespace.Wrap(ds, datastore.NewKey("blockstore"))) lsys := storeutil.LinkSystemForBlockstore(bs) sourceDagService = merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) - err := dt1.RegisterTransportConfigurer(&testutil.FakeDTType{}, func(channelID datatransfer.ChannelID, testVoucher datatransfer.Voucher, transport datatransfer.Transport) { - fv, ok := testVoucher.(*testutil.FakeDTType) - if ok && fv.Data == voucher.Data { + err := dt1.RegisterTransportConfigurer(testutil.TestVoucherType, func(channelID datatransfer.ChannelID, testVoucher datatransfer.TypedVoucher, transport datatransfer.Transport) { + if testVoucher.Equals(voucher) { gsTransport, ok := transport.(*tp.Transport) if ok { err := gsTransport.UseStore(channelID, lsys) @@ -214,9 +214,8 @@ func TestRoundTrip(t *testing.T) { bs := bstore.NewBlockstore(namespace.Wrap(ds, datastore.NewKey("blockstore"))) lsys := storeutil.LinkSystemForBlockstore(bs) destDagService = merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) - err := dt2.RegisterTransportConfigurer(&testutil.FakeDTType{}, func(channelID datatransfer.ChannelID, testVoucher datatransfer.Voucher, transport datatransfer.Transport) { - fv, ok := testVoucher.(*testutil.FakeDTType) - if ok && fv.Data == voucher.Data { + err := dt2.RegisterTransportConfigurer(testutil.TestVoucherType, func(channelID datatransfer.ChannelID, testVoucher datatransfer.TypedVoucher, transport datatransfer.Transport) { + if testVoucher.Equals(voucher) { gsTransport, ok := transport.(*tp.Transport) if ok { err := gsTransport.UseStore(channelID, lsys) @@ -232,12 +231,12 @@ func TestRoundTrip(t *testing.T) { var chid datatransfer.ChannelID if data.isPull { sv.ExpectSuccessPull() - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), &voucher, rootCid, gsData.AllSelector) + require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) + chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) } else { sv.ExpectSuccessPush() - require.NoError(t, dt2.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, rootCid, gsData.AllSelector) + require.NoError(t, dt2.RegisterVoucherType(testutil.TestVoucherType, sv)) + chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, rootCid, gsData.AllSelector) } require.NoError(t, err) opens := 0 @@ -325,9 +324,9 @@ func TestMultipleRoundTripMultipleStores(t *testing.T) { } dt1.SubscribeToEvents(subscriber) dt2.SubscribeToEvents(subscriber) - vouchers := make([]datatransfer.Voucher, 0, data.requestCount) + vouchers := make([]datatransfer.TypedVoucher, 0, data.requestCount) for i := 0; i < data.requestCount; i++ { - vouchers = append(vouchers, testutil.NewFakeDTType()) + vouchers = append(vouchers, testutil.NewTestTypedVoucher()) } sv := testutil.NewStubbedValidator() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) @@ -347,16 +346,13 @@ func TestMultipleRoundTripMultipleStores(t *testing.T) { linkSystems = append(linkSystems, lsys) } - err = dt2.RegisterTransportConfigurer(&testutil.FakeDTType{}, func(channelID datatransfer.ChannelID, testVoucher datatransfer.Voucher, transport datatransfer.Transport) { - fv, ok := testVoucher.(*testutil.FakeDTType) - if ok { - for i, voucher := range vouchers { - if fv.Data == voucher.(*testutil.FakeDTType).Data { - gsTransport, ok := transport.(*tp.Transport) - if ok { - err := gsTransport.UseStore(channelID, linkSystems[i]) - require.NoError(t, err) - } + err = dt2.RegisterTransportConfigurer(testutil.TestVoucherType, func(channelID datatransfer.ChannelID, testVoucher datatransfer.TypedVoucher, transport datatransfer.Transport) { + for i, voucher := range vouchers { + if testVoucher.Equals(voucher) { + gsTransport, ok := transport.(*tp.Transport) + if ok { + err := gsTransport.UseStore(channelID, linkSystems[i]) + require.NoError(t, err) } } } @@ -365,14 +361,14 @@ func TestMultipleRoundTripMultipleStores(t *testing.T) { if data.isPull { sv.ExpectSuccessPull() - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) + require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) for i := 0; i < data.requestCount; i++ { _, err = dt2.OpenPullDataChannel(ctx, host1.ID(), vouchers[i], rootCid, gsData.AllSelector) require.NoError(t, err) } } else { sv.ExpectSuccessPush() - require.NoError(t, dt2.RegisterVoucherType(&testutil.FakeDTType{}, sv)) + require.NoError(t, dt2.RegisterVoucherType(testutil.TestVoucherType, sv)) for i := 0; i < data.requestCount; i++ { _, err = dt1.OpenPushDataChannel(ctx, host2.ID(), vouchers[i], rootCid, gsData.AllSelector) require.NoError(t, err) @@ -453,10 +449,9 @@ func TestManyReceiversAtOnce(t *testing.T) { err = receiver.Start(gsData.Ctx) require.NoError(t, err) - err = receiver.RegisterTransportConfigurer(&testutil.FakeDTType{}, func(channelID datatransfer.ChannelID, testVoucher datatransfer.Voucher, transport datatransfer.Transport) { - _, isFv := testVoucher.(*testutil.FakeDTType) + err = receiver.RegisterTransportConfigurer(testutil.TestVoucherType, func(channelID datatransfer.ChannelID, testVoucher datatransfer.TypedVoucher, transport datatransfer.Transport) { gsTransport, isGs := transport.(*tp.Transport) - if isFv && isGs { + if isGs { err := gsTransport.UseStore(channelID, altLinkSystem) require.NoError(t, err) } @@ -488,9 +483,9 @@ func TestManyReceiversAtOnce(t *testing.T) { for _, receiver := range receivers { receiver.SubscribeToEvents(subscriber) } - vouchers := make([]datatransfer.Voucher, 0, data.receiverCount) + vouchers := make([]datatransfer.TypedVoucher, 0, data.receiverCount) for i := 0; i < data.receiverCount; i++ { - vouchers = append(vouchers, testutil.NewFakeDTType()) + vouchers = append(vouchers, testutil.NewTestTypedVoucher()) } sv := testutil.NewStubbedValidator() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) @@ -500,7 +495,7 @@ func TestManyReceiversAtOnce(t *testing.T) { if data.isPull { sv.ExpectSuccessPull() - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) + require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) for i, receiver := range receivers { _, err = receiver.OpenPullDataChannel(ctx, host1.ID(), vouchers[i], rootCid, gsData.AllSelector) require.NoError(t, err) @@ -508,7 +503,7 @@ func TestManyReceiversAtOnce(t *testing.T) { } else { sv.ExpectSuccessPush() for i, receiver := range receivers { - require.NoError(t, receiver.RegisterVoucherType(&testutil.FakeDTType{}, sv)) + require.NoError(t, receiver.RegisterVoucherType(testutil.TestVoucherType, sv)) _, err = dt1.OpenPushDataChannel(ctx, hosts[i].ID(), vouchers[i], rootCid, gsData.AllSelector) require.NoError(t, err) } @@ -736,7 +731,7 @@ func TestAutoRestart(t *testing.T) { } initiator.SubscribeToEvents(subscriber) responder.SubscribeToEvents(subscriber) - voucher := testutil.FakeDTType{Data: "applesauce"} + voucher := testutil.NewTestTypedVoucherWith("applesauce") sv := testutil.NewStubbedValidator() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) @@ -753,8 +748,8 @@ func TestAutoRestart(t *testing.T) { root, origBytes := testutil.LoadUnixFSFile(ctx, t, sourceDagService, loremFile) rootCid := root.(cidlink.Link).Cid - require.NoError(t, initiator.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - require.NoError(t, responder.RegisterVoucherType(&testutil.FakeDTType{}, sv)) + require.NoError(t, initiator.RegisterVoucherType(testutil.TestVoucherType, sv)) + require.NoError(t, responder.RegisterVoucherType(testutil.TestVoucherType, sv)) // If the test case needs to subscribe to response events, provide // the test case with the responder @@ -775,10 +770,10 @@ func TestAutoRestart(t *testing.T) { var chid datatransfer.ChannelID if tc.isPush { // Open a push channel - chid, err = initiator.OpenPushDataChannel(ctx, responderHost.ID(), &voucher, rootCid, gsData.AllSelector) + chid, err = initiator.OpenPushDataChannel(ctx, responderHost.ID(), voucher, rootCid, gsData.AllSelector) } else { // Open a pull channel - chid, err = initiator.OpenPullDataChannel(ctx, responderHost.ID(), &voucher, rootCid, gsData.AllSelector) + chid, err = initiator.OpenPullDataChannel(ctx, responderHost.ID(), voucher, rootCid, gsData.AllSelector) } require.NoError(t, err) @@ -929,7 +924,7 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { } dataReceived := onDataReceivedChan(dataReceiver) - voucher := testutil.FakeDTType{Data: "applesauce"} + voucher := testutil.NewTestTypedVoucherWith("applesauce") sv := testutil.NewStubbedValidator() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) @@ -946,16 +941,16 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { root, origBytes := testutil.LoadUnixFSFile(ctx, t, sourceDagService, loremLargeFile) rootCid := root.(cidlink.Link).Cid - require.NoError(t, initiator.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - require.NoError(t, responder.RegisterVoucherType(&testutil.FakeDTType{}, sv)) + require.NoError(t, initiator.RegisterVoucherType(testutil.TestVoucherType, sv)) + require.NoError(t, responder.RegisterVoucherType(testutil.TestVoucherType, sv)) var chid datatransfer.ChannelID if isPush { // Open a push channel - chid, err = initiator.OpenPushDataChannel(ctx, responderHost.ID(), &voucher, rootCid, gsData.AllSelector) + chid, err = initiator.OpenPushDataChannel(ctx, responderHost.ID(), voucher, rootCid, gsData.AllSelector) } else { // Open a pull channel - chid, err = initiator.OpenPullDataChannel(ctx, responderHost.ID(), &voucher, rootCid, gsData.AllSelector) + chid, err = initiator.OpenPullDataChannel(ctx, responderHost.ID(), voucher, rootCid, gsData.AllSelector) } require.NoError(t, err) @@ -987,7 +982,7 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { initiator2GSTspt := gsData.SetupGSTransportHost1() initiator2, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, initiator2GSTspt, restartConf) require.NoError(t, err) - require.NoError(t, initiator2.RegisterVoucherType(&testutil.FakeDTType{}, sv)) + require.NoError(t, initiator2.RegisterVoucherType(testutil.TestVoucherType, sv)) initiator2.SubscribeToEvents(completeSubscriber) testutil.StartAndWaitForReady(ctx, t, initiator2) @@ -1127,7 +1122,7 @@ func TestRoundTripCancelledRequest(t *testing.T) { } dt1.SubscribeToEvents(subscriber) dt2.SubscribeToEvents(subscriber) - voucher := testutil.FakeDTType{Data: "applesauce"} + voucher := testutil.NewTestTypedVoucherWith("applesauce") sv := testutil.NewStubbedValidator() root, _ := testutil.LoadUnixFSFile(ctx, t, gsData.DagService1, loremFile) rootCid := root.(cidlink.Link).Cid @@ -1136,13 +1131,13 @@ func TestRoundTripCancelledRequest(t *testing.T) { if data.isPull { sv.ExpectSuccessPull() sv.StubResult(datatransfer.ValidationResult{Accepted: true, ForcePause: true}) - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), &voucher, rootCid, gsData.AllSelector) + require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) + chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) } else { sv.ExpectSuccessPush() sv.StubResult(datatransfer.ValidationResult{Accepted: true, ForcePause: true}) - require.NoError(t, dt2.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, rootCid, gsData.AllSelector) + require.NoError(t, dt2.RegisterVoucherType(testutil.TestVoucherType, sv)) + chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, rootCid, gsData.AllSelector) } require.NoError(t, err) opens := 0 @@ -1187,21 +1182,23 @@ type retrievalRevalidator struct { providerPausePoint int pausePoints []uint64 leavePausedInitially bool - initialVoucherResult datatransfer.VoucherResult + initialVoucherResult *datatransfer.TypedVoucher requiresFinalization bool } func (r *retrievalRevalidator) ValidatePush( chid datatransfer.ChannelID, sender peer.ID, - voucher datatransfer.Voucher, + voucher datamodel.Node, baseCid cid.Cid, - selector ipld.Node) (datatransfer.ValidationResult, error) { + selector datamodel.Node) (datatransfer.ValidationResult, error) { vr := datatransfer.ValidationResult{ Accepted: true, RequiresFinalization: r.requiresFinalization, ForcePause: r.leavePausedInitially, - VoucherResult: r.initialVoucherResult, + } + if r.initialVoucherResult != nil { + vr.VoucherResult = r.initialVoucherResult } if len(r.pausePoints) > r.providerPausePoint { vr.DataLimit = r.pausePoints[r.providerPausePoint] @@ -1214,14 +1211,16 @@ func (r *retrievalRevalidator) ValidatePush( func (r *retrievalRevalidator) ValidatePull( chid datatransfer.ChannelID, sender peer.ID, - voucher datatransfer.Voucher, + voucher datamodel.Node, baseCid cid.Cid, - selector ipld.Node) (datatransfer.ValidationResult, error) { + selector datamodel.Node) (datatransfer.ValidationResult, error) { vr := datatransfer.ValidationResult{ Accepted: true, RequiresFinalization: r.requiresFinalization, ForcePause: r.leavePausedInitially, - VoucherResult: r.initialVoucherResult, + } + if r.initialVoucherResult != nil { + vr.VoucherResult = r.initialVoucherResult } if len(r.pausePoints) > r.providerPausePoint { vr.DataLimit = r.pausePoints[r.providerPausePoint] @@ -1329,26 +1328,24 @@ func TestSimulatedRetrievalFlow(t *testing.T) { errChan := make(chan struct{}, 2) clientPausePoint := 0 clientFinished := make(chan struct{}, 1) - finalVoucherResult := testutil.NewFakeDTType() - encodedFVR, err := encoding.Encode(finalVoucherResult) + finalVoucherResult := testutil.NewTestTypedVoucher() require.NoError(t, err) var clientSubscriber datatransfer.Subscriber = func(event datatransfer.Event, channelState datatransfer.ChannelState) { if event.Code == datatransfer.Error { errChan <- struct{}{} } if event.Code == datatransfer.NewVoucherResult { - lastVoucherResult := channelState.LastVoucherResult() - encodedLVR, err := encoding.Encode(lastVoucherResult) + lastVoucherResult, err := channelState.LastVoucherResult() require.NoError(t, err) - if bytes.Equal(encodedLVR, encodedFVR) { - _ = dt2.SendVoucher(ctx, chid, testutil.NewFakeDTType()) + if lastVoucherResult.Equals(finalVoucherResult) { + _ = dt2.SendVoucher(ctx, chid, testutil.NewTestTypedVoucher()) } } if event.Code == datatransfer.DataReceived && clientPausePoint < len(config.pausePoints) && channelState.Received() > config.pausePoints[clientPausePoint] { - _ = dt2.SendVoucher(ctx, chid, testutil.NewFakeDTType()) + _ = dt2.SendVoucher(ctx, chid, testutil.NewTestTypedVoucher()) clientPausePoint++ } if channelState.Status() == datatransfer.Completed { @@ -1376,7 +1373,7 @@ func TestSimulatedRetrievalFlow(t *testing.T) { dt1.UpdateValidationStatus(ctx, chid, sv.nextStatus()) } if event.Code == datatransfer.DataLimitExceeded { - dt1.SendVoucherResult(ctx, chid, testutil.NewFakeDTType()) + dt1.SendVoucherResult(ctx, chid, testutil.NewTestTypedVoucher()) } if event.Code == datatransfer.BeginFinalizing { sv.requiresFinalization = false @@ -1390,12 +1387,11 @@ func TestSimulatedRetrievalFlow(t *testing.T) { } } dt1.SubscribeToEvents(providerSubscriber) - voucher := testutil.FakeDTType{Data: "applesauce"} + voucher := testutil.NewTestTypedVoucherWith("applesauce") - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) + require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) - require.NoError(t, dt2.RegisterVoucherResultType(testutil.NewFakeDTType())) - chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), &voucher, rootCid, gsData.AllSelector) + chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) require.NoError(t, err) for providerFinished != nil || clientFinished != nil { @@ -1495,7 +1491,7 @@ func TestPauseAndResume(t *testing.T) { } dt1.SubscribeToEvents(subscriber) dt2.SubscribeToEvents(subscriber) - voucher := testutil.FakeDTType{Data: "applesauce"} + voucher := testutil.NewTestTypedVoucherWith("applesauce") sv := testutil.NewStubbedValidator() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) @@ -1503,12 +1499,12 @@ func TestPauseAndResume(t *testing.T) { var chid datatransfer.ChannelID if isPull { sv.ExpectSuccessPull() - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), &voucher, rootCid, gsData.AllSelector) + require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) + chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) } else { sv.ExpectSuccessPush() - require.NoError(t, dt2.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, rootCid, gsData.AllSelector) + require.NoError(t, dt2.RegisterVoucherType(testutil.TestVoucherType, sv)) + chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, rootCid, gsData.AllSelector) } require.NoError(t, err) opens := 0 @@ -1606,15 +1602,15 @@ func TestUnrecognizedVoucherRoundTrip(t *testing.T) { } dt1.SubscribeToEvents(subscriber) dt2.SubscribeToEvents(subscriber) - voucher := testutil.FakeDTType{Data: "applesauce"} + voucher := testutil.NewTestTypedVoucherWith("applesauce") root, _ := testutil.LoadUnixFSFile(ctx, t, gsData.DagService1, loremFile) rootCid := root.(cidlink.Link).Cid if isPull { - _, err = dt2.OpenPullDataChannel(ctx, host1.ID(), &voucher, rootCid, gsData.AllSelector) + _, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) } else { - _, err = dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, rootCid, gsData.AllSelector) + _, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, rootCid, gsData.AllSelector) } require.NoError(t, err) opens := 0 @@ -1656,8 +1652,8 @@ func TestDataTransferSubscribing(t *testing.T) { dt2, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt2) - require.NoError(t, dt2.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - voucher := testutil.FakeDTType{Data: "applesauce"} + require.NoError(t, dt2.RegisterVoucherType(testutil.TestVoucherType, sv)) + voucher := testutil.NewTestTypedVoucherWith("applesauce") baseCid := testutil.GenerateCids(1)[0] dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) @@ -1677,7 +1673,7 @@ func TestDataTransferSubscribing(t *testing.T) { } unsub1 := dt1.SubscribeToEvents(subscribe1) unsub2 := dt1.SubscribeToEvents(subscribe2) - _, err = dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, baseCid, gsData.AllSelector) + _, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, baseCid, gsData.AllSelector) require.NoError(t, err) select { case <-ctx.Done(): @@ -1706,7 +1702,7 @@ func TestDataTransferSubscribing(t *testing.T) { } unsub3 := dt1.SubscribeToEvents(subscribe3) unsub4 := dt1.SubscribeToEvents(subscribe4) - _, err = dt1.OpenPullDataChannel(ctx, host2.ID(), &voucher, baseCid, gsData.AllSelector) + _, err = dt1.OpenPullDataChannel(ctx, host2.ID(), voucher, baseCid, gsData.AllSelector) require.NoError(t, err) select { case <-ctx.Done(): @@ -1776,7 +1772,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator and data sender host2 := gsData.Host2 // data recipient, makes graphsync request for data - voucher := testutil.NewFakeDTType() + voucher := testutil.NewTestTypedVoucher() link := gsData.LoadUnixFSFile(t, false) // setup receiving peer to just record message coming in @@ -1795,8 +1791,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) - voucherResult := testutil.NewFakeDTType() - err = dt1.RegisterVoucherResultType(voucherResult) + voucherResult := testutil.NewTestTypedVoucher() require.NoError(t, err) t.Run("when request is initiated", func(t *testing.T) { @@ -1811,7 +1806,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { } requestReceived := messageReceived.message.(datatransfer.Request) - response, err := message.NewResponse(requestReceived.TransferID(), true, false, voucherResult.Type(), voucherResult) + response, err := message.NewResponse(requestReceived.TransferID(), true, false, &voucherResult) require.NoError(t, err) nd, err := response.ToIPLD() require.NoError(t, err) @@ -1830,7 +1825,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { }) t.Run("when no request is initiated", func(t *testing.T) { - response, err := message.NewResponse(datatransfer.TransferID(rand.Uint32()), true, false, voucher.Type(), voucher) + response, err := message.NewResponse(datatransfer.TransferID(rand.Uint32()), true, false, &voucher) require.NoError(t, err) nd, err := response.ToIPLD() require.NoError(t, err) @@ -1857,7 +1852,7 @@ func TestResponseHookWhenExtensionNotFound(t *testing.T) { gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator and data sender host2 := gsData.Host2 // data recipient, makes graphsync request for data - voucher := testutil.FakeDTType{Data: "applesauce"} + voucher := testutil.NewTestTypedVoucherWith("applesauce") link := gsData.LoadUnixFSFile(t, false) // setup receiving peer to just record message coming in @@ -1885,7 +1880,7 @@ func TestResponseHookWhenExtensionNotFound(t *testing.T) { } gs1.RegisterIncomingRequestHook(validateHook) - _, err := dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, link.(cidlink.Link).Cid, gsData.AllSelector) + _, err := dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, link.(cidlink.Link).Cid, gsData.AllSelector) require.NoError(t, err) select { @@ -1921,12 +1916,13 @@ func TestRespondingToPullGraphsyncRequests(t *testing.T) { dt1, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) + require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) - voucher := testutil.NewFakeDTType() - request, err := message.NewRequest(id, false, true, voucher.Type(), voucher, testutil.GenerateCids(1)[0], gsData.AllSelector) + voucher := testutil.NewTestTypedVoucher() + request, err := message.NewRequest(id, false, true, &voucher, testutil.GenerateCids(1)[0], gsData.AllSelector) require.NoError(t, err) nd, err := request.ToIPLD() + require.NoError(t, err) gsRequest := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, gsData.AllSelector, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ Name: extension.ExtensionDataTransfer1_1, Data: nd, @@ -1949,9 +1945,9 @@ func TestRespondingToPullGraphsyncRequests(t *testing.T) { dt1, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - voucher := testutil.NewFakeDTType() - dtRequest, err := message.NewRequest(id, false, true, voucher.Type(), voucher, testutil.GenerateCids(1)[0], gsData.AllSelector) + require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) + voucher := testutil.NewTestTypedVoucher() + dtRequest, err := message.NewRequest(id, false, true, &voucher, testutil.GenerateCids(1)[0], gsData.AllSelector) require.NoError(t, err) nd, err := dtRequest.ToIPLD() @@ -2033,24 +2029,20 @@ func TestMultipleMessagesInExtension(t *testing.T) { // In this retrieval flow we expect 2 voucher results: // The first one is sent as a response from the initial request telling the client // the provider has accepted the request and is starting to send blocks - respVoucher := testutil.NewFakeDTType() - encodedRVR, err := encoding.Encode(respVoucher) - require.NoError(t, err) + respVoucher := testutil.NewTestTypedVoucher() // voucher results are sent by the providers to request payment while pausing until a voucher is sent // to revalidate - voucherResults := []datatransfer.VoucherResult{ - &testutil.FakeDTType{Data: "one"}, - &testutil.FakeDTType{Data: "two"}, - &testutil.FakeDTType{Data: "thr"}, - &testutil.FakeDTType{Data: "for"}, - &testutil.FakeDTType{Data: "fiv"}, + voucherResults := []datatransfer.TypedVoucher{ + testutil.NewTestTypedVoucherWith("one"), + testutil.NewTestTypedVoucherWith("two"), + testutil.NewTestTypedVoucherWith("thr"), + testutil.NewTestTypedVoucherWith("for"), + testutil.NewTestTypedVoucherWith("fiv"), } // The final voucher result is sent by the provider to request a last payment voucher - finalVoucherResult := testutil.NewFakeDTType() - encodedFVR, err := encoding.Encode(finalVoucherResult) - require.NoError(t, err) + finalVoucherResult := testutil.NewTestTypedVoucher() dt2.SubscribeToEvents(func(event datatransfer.Event, channelState datatransfer.ChannelState) { if event.Code == datatransfer.Error { @@ -2058,13 +2050,12 @@ func TestMultipleMessagesInExtension(t *testing.T) { } // Here we verify reception of voucherResults by the client if event.Code == datatransfer.NewVoucherResult { - voucherResult := channelState.LastVoucherResult() - encodedVR, err := encoding.Encode(voucherResult) + voucherResult, err := channelState.LastVoucherResult() require.NoError(t, err) // If this voucher result is the response voucher no action is needed // we just know that the provider has accepted the transfer and is sending blocks - if bytes.Equal(encodedVR, encodedRVR) { + if voucherResult.Equals(respVoucher) { // The test will fail if no response voucher is received clientGotResponse <- struct{}{} } @@ -2072,18 +2063,16 @@ func TestMultipleMessagesInExtension(t *testing.T) { // If this voucher is a revalidation request we need to send a new voucher // to revalidate and unpause the transfer if clientPausePoint < 5 { - encodedExpected, err := encoding.Encode(voucherResults[clientPausePoint]) - require.NoError(t, err) - if bytes.Equal(encodedVR, encodedExpected) { - _ = dt2.SendVoucher(ctx, chid, testutil.NewFakeDTType()) + if voucherResult.Equals(voucherResults[clientPausePoint]) { + _ = dt2.SendVoucher(ctx, chid, testutil.NewTestTypedVoucher()) clientPausePoint++ } } // If this voucher result is the final voucher result we need // to send a new voucher to unpause the provider and complete the transfer - if bytes.Equal(encodedVR, encodedFVR) { - _ = dt2.SendVoucher(ctx, chid, testutil.NewFakeDTType()) + if voucherResult.Equals(finalVoucherResult) { + _ = dt2.SendVoucher(ctx, chid, testutil.NewTestTypedVoucher()) } } @@ -2098,7 +2087,7 @@ func TestMultipleMessagesInExtension(t *testing.T) { StubbedValidator: testutil.NewStubbedValidator(), pausePoints: pausePoints, requiresFinalization: true, - initialVoucherResult: respVoucher, + initialVoucherResult: &respVoucher, } dt1.SubscribeToEvents(func(event datatransfer.Event, channelState datatransfer.ChannelState) { if event.Code == datatransfer.Error { @@ -2122,12 +2111,10 @@ func TestMultipleMessagesInExtension(t *testing.T) { dt1.SendVoucherResult(ctx, chid, finalVoucherResult) } }) - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - - require.NoError(t, dt2.RegisterVoucherResultType(testutil.NewFakeDTType())) + require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) - voucher := testutil.FakeDTType{Data: "applesauce"} - chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), &voucher, rootCid, gsData.AllSelector) + voucher := testutil.NewTestTypedVoucherWith("applesauce") + chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) require.NoError(t, err) // Expect the client to receive a response voucher, the provider to complete the transfer and @@ -2174,22 +2161,18 @@ func TestMultipleParallelTransfers(t *testing.T) { // In this retrieval flow we expect 2 voucher results: // The first one is sent as a response from the initial request telling the client // the provider has accepted the request and is starting to send blocks - respVoucher := testutil.NewFakeDTType() - encodedRVR, err := encoding.Encode(respVoucher) + respVoucher := testutil.NewTestTypedVoucher() require.NoError(t, err) // The final voucher result is sent by the provider to let the client know the deal is completed - finalVoucherResult := testutil.NewFakeDTType() - encodedFVR, err := encoding.Encode(finalVoucherResult) + finalVoucherResult := testutil.NewTestTypedVoucher() require.NoError(t, err) sv := &retrievalRevalidator{ StubbedValidator: testutil.NewStubbedValidator(), - initialVoucherResult: respVoucher, + initialVoucherResult: &respVoucher, } - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - - require.NoError(t, dt2.RegisterVoucherResultType(testutil.NewFakeDTType())) + require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) // for each size we create a new random DAG of the given size and try to retrieve it for _, size := range sizes { @@ -2217,21 +2200,21 @@ func TestMultipleParallelTransfers(t *testing.T) { } // Here we verify reception of voucherResults by the client if event.Code == datatransfer.NewVoucherResult { - voucherResult := channelState.LastVoucherResult() - encodedVR, err := encoding.Encode(voucherResult) + voucherResult, err := channelState.LastVoucherResult() + require.NoError(t, err) require.NoError(t, err) // If this voucher result is the response voucher no action is needed // we just know that the provider has accepted the transfer and is sending blocks - if bytes.Equal(encodedVR, encodedRVR) { + if voucherResult.Equals(respVoucher) { // The test will fail if no response voucher is received clientGotResponse <- struct{}{} } // If this voucher result is the final voucher result we need // to send a new voucher to unpause the provider and complete the transfer - if bytes.Equal(encodedVR, encodedFVR) { - _ = dt2.SendVoucher(ctx, chid, testutil.NewFakeDTType()) + if voucherResult.Equals(finalVoucherResult) { + _ = dt2.SendVoucher(ctx, chid, testutil.NewTestTypedVoucher()) } } @@ -2260,7 +2243,7 @@ func TestMultipleParallelTransfers(t *testing.T) { root, origBytes := LoadRandomData(ctx, t, gsData.DagService1, size) rootCid := root.(cidlink.Link).Cid - voucher := testutil.NewFakeDTType() + voucher := testutil.NewTestTypedVoucher() chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) require.NoError(t, err) close(chidReceived) diff --git a/impl/receiving_requests.go b/impl/receiving_requests.go index cb86afa7..4760fffb 100644 --- a/impl/receiving_requests.go +++ b/impl/receiving_requests.go @@ -4,7 +4,7 @@ import ( "context" "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" @@ -41,13 +41,16 @@ func (m *manager) acceptRequest(chid datatransfer.ChannelID, incoming datatransf return datatransfer.ValidationResult{}, err } - voucher, err := m.decodeVoucher(incoming) + voucher, err := incoming.TypedVoucher() if err != nil { return datatransfer.ValidationResult{}, err } + processor, ok := m.validatedTypes.Processor(voucher.Type) + if !ok { + return datatransfer.ValidationResult{}, xerrors.Errorf("unknown voucher type: %s", voucher.Type) + } - var validatorFunc func(datatransfer.ChannelID, peer.ID, datatransfer.Voucher, cid.Cid, ipld.Node) (datatransfer.ValidationResult, error) - processor, _ := m.validatedTypes.Processor(voucher.Type()) + var validatorFunc func(datatransfer.ChannelID, peer.ID, datamodel.Node, cid.Cid, datamodel.Node) (datatransfer.ValidationResult, error) validator := processor.(datatransfer.RequestValidator) if incoming.IsPull() { validatorFunc = validator.ValidatePull @@ -55,7 +58,7 @@ func (m *manager) acceptRequest(chid datatransfer.ChannelID, incoming datatransf validatorFunc = validator.ValidatePush } - result, err := validatorFunc(chid, chid.Initiator, voucher, incoming.BaseCid(), stor) + result, err := validatorFunc(chid, chid.Initiator, voucher.Voucher, incoming.BaseCid(), stor) // if an error occurred during validation or the request was not accepted, return if err != nil || !result.Accepted { @@ -73,7 +76,16 @@ func (m *manager) acceptRequest(chid datatransfer.ChannelID, incoming datatransf } log.Infow("data-transfer request validated, will create & start tracking channel", "channelID", chid, "payloadCid", incoming.BaseCid()) - _, err = m.channels.CreateNew(m.peerID, incoming.TransferID(), incoming.BaseCid(), stor, voucher, chid.Initiator, dataSender, dataReceiver) + _, err = m.channels.CreateNew( + m.peerID, + incoming.TransferID(), + incoming.BaseCid(), + stor, + voucher, + chid.Initiator, + dataSender, + dataReceiver, + ) if err != nil { log.Errorw("failed to create and start tracking channel", "channelID", chid, "err", err) return result, err @@ -97,7 +109,7 @@ func (m *manager) acceptRequest(chid datatransfer.ChannelID, incoming datatransf } // configure the transport - processor, has := m.transportConfigurers.Processor(voucher.Type()) + processor, has := m.transportConfigurers.Processor(voucher.Type) if has { transportConfigurer := processor.(datatransfer.TransportConfigurer) transportConfigurer(chid, voucher, m.transport) @@ -173,14 +185,16 @@ func (m *manager) restartRequest(chid datatransfer.ChannelID, } // configure the transport - voucher, err := m.decodeVoucher(incoming) + voucher, err := incoming.Voucher() if err != nil { return stayPaused, result, err } - processor, has := m.transportConfigurers.Processor(voucher.Type()) + voucherType := incoming.VoucherType() + typedVoucher := datatransfer.TypedVoucher{Voucher: voucher, Type: voucherType} + processor, has := m.transportConfigurers.Processor(voucherType) if has { transportConfigurer := processor.(datatransfer.TransportConfigurer) - transportConfigurer(chid, voucher, m.transport) + transportConfigurer(chid, typedVoucher, m.transport) } m.dataTransferNetwork.Protect(initiator, chid.String()) return stayPaused, result, nil @@ -189,11 +203,11 @@ func (m *manager) restartRequest(chid datatransfer.ChannelID, // processUpdateVoucher handles an incoming request message with an updated voucher func (m *manager) processUpdateVoucher(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { // decode the voucher and save it on the channel - vouch, err := m.decodeVoucher(request) + voucher, err := request.TypedVoucher() if err != nil { return nil, err } - return nil, m.channels.NewVoucher(chid, vouch) + return nil, m.channels.NewVoucher(chid, voucher) } // receiveUpdateRequest handles an incoming request message with an updated voucher @@ -238,7 +252,7 @@ func (m *manager) requestError(result datatransfer.ValidationResult, resultErr e // recordRejectedValidationEvents sends changes based on an reject validation to the state machine func (m *manager) recordRejectedValidationEvents(chid datatransfer.ChannelID, result datatransfer.ValidationResult) error { if result.VoucherResult != nil { - if err := m.channels.NewVoucherResult(chid, result.VoucherResult); err != nil { + if err := m.channels.NewVoucherResult(chid, *result.VoucherResult); err != nil { return err } } @@ -251,8 +265,8 @@ func (m *manager) recordAcceptedValidationEvents(chst datatransfer.ChannelState, chid := chst.ChannelID() // record the voucher result if present - if result.VoucherResult != nil { - err := m.channels.NewVoucherResult(chid, result.VoucherResult) + if result.VoucherResult != nil && result.VoucherResult.Voucher != nil { + err := m.channels.NewVoucherResult(chid, *result.VoucherResult) if err != nil { return err } @@ -296,7 +310,11 @@ func (m *manager) recordAcceptedValidationEvents(chst datatransfer.ChannelState, // validateRestart looks up the appropriate validator and validates a restart func (m *manager) validateRestart(chst datatransfer.ChannelState) (datatransfer.ValidationResult, error) { - processor, _ := m.validatedTypes.Processor(chst.Voucher().Type()) + chv, err := chst.Voucher() + if err != nil { + return datatransfer.ValidationResult{}, err + } + processor, _ := m.validatedTypes.Processor(chv.Type) validator := processor.(datatransfer.RequestValidator) return validator.ValidateRestart(chst.ChannelID(), chst) diff --git a/impl/responding_test.go b/impl/responding_test.go index 0fe466f7..d3adfb5c 100644 --- a/impl/responding_test.go +++ b/impl/responding_test.go @@ -11,6 +11,7 @@ import ( "github.com/ipfs/go-datastore" dss "github.com/ipfs/go-datastore/sync" "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/assert" @@ -40,7 +41,8 @@ func TestDataTransferResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -48,7 +50,7 @@ func TestDataTransferResponding(t *testing.T) { validation := h.sv.ValidationsReceived[0] assert.False(t, validation.IsPull) assert.Equal(t, h.peers[1], validation.Other) - assert.Equal(t, h.voucher, validation.Voucher) + assert.True(t, ipld.DeepEqual(h.voucher.Voucher, validation.Voucher)) assert.Equal(t, h.baseCid, validation.BaseCid) assert.Equal(t, h.stor, validation.Selector) @@ -73,7 +75,8 @@ func TestDataTransferResponding(t *testing.T) { "new push request rejects": { configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: false, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: false, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -94,7 +97,8 @@ func TestDataTransferResponding(t *testing.T) { "new push request errors": { configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectErrorPush() - sv.StubResult(datatransfer.ValidationResult{VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -115,7 +119,8 @@ func TestDataTransferResponding(t *testing.T) { "new push request pauses": { configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, ForcePause: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, ForcePause: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -156,7 +161,7 @@ func TestDataTransferResponding(t *testing.T) { validation := h.sv.ValidationsReceived[0] assert.True(t, validation.IsPull) assert.Equal(t, h.peers[1], validation.Other) - assert.Equal(t, h.voucher, validation.Voucher) + assert.True(t, ipld.DeepEqual(h.voucher.Voucher, validation.Voucher)) assert.Equal(t, h.baseCid, validation.BaseCid) assert.Equal(t, h.stor, validation.Selector) require.True(t, response.Accepted()) @@ -227,7 +232,7 @@ func TestDataTransferResponding(t *testing.T) { }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) - newVoucherResult := testutil.NewFakeDTType() + newVoucherResult := testutil.NewTestTypedVoucher() err := h.dt.SendVoucherResult(h.ctx, channelID(h.id, h.peers), newVoucherResult) require.NoError(t, err) }, @@ -239,7 +244,7 @@ func TestDataTransferResponding(t *testing.T) { }, verify: func(t *testing.T, h *receiverHarness) { _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) - newVoucherResult := testutil.NewFakeDTType() + newVoucherResult := testutil.NewTestTypedVoucher() err := h.dt.SendVoucherResult(h.ctx, channelID(h.id, h.peers), newVoucherResult) require.NoError(t, err) }, @@ -251,7 +256,7 @@ func TestDataTransferResponding(t *testing.T) { }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) - newVoucher := testutil.NewFakeDTType() + newVoucher := testutil.NewTestTypedVoucher() err := h.dt.SendVoucher(h.ctx, channelID(h.id, h.peers), newVoucher) require.EqualError(t, err, "cannot send voucher for request we did not initiate") }, @@ -263,7 +268,7 @@ func TestDataTransferResponding(t *testing.T) { }, verify: func(t *testing.T, h *receiverHarness) { _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) - newVoucher := testutil.NewFakeDTType() + newVoucher := testutil.NewTestTypedVoucher() err := h.dt.SendVoucher(h.ctx, channelID(h.id, h.peers), newVoucher) require.EqualError(t, err, "cannot send voucher for request we did not initiate") }, @@ -277,7 +282,8 @@ func TestDataTransferResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -294,7 +300,8 @@ func TestDataTransferResponding(t *testing.T) { datatransfer.ResumeInitiator}, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -314,7 +321,8 @@ func TestDataTransferResponding(t *testing.T) { datatransfer.ResumeInitiator}, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -336,7 +344,8 @@ func TestDataTransferResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -362,7 +371,8 @@ func TestDataTransferResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -379,7 +389,8 @@ func TestDataTransferResponding(t *testing.T) { response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) require.NoError(t, err, nil) require.Nil(t, response) - err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: &vr}) require.NoError(t, err) require.Len(t, h.transport.ResumedChannels, 1) resCh := h.transport.ResumedChannels[0] @@ -411,7 +422,8 @@ func TestDataTransferResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) @@ -428,7 +440,8 @@ func TestDataTransferResponding(t *testing.T) { response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) require.NoError(t, err, nil) require.Nil(t, response) - err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: false, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: false, VoucherResult: &vr}) require.NoError(t, err) require.Len(t, h.transport.ClosedChannels, 1) require.Equal(t, h.transport.ClosedChannels[0], channelID(h.id, h.peers)) @@ -461,7 +474,8 @@ func TestDataTransferResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) @@ -481,7 +495,8 @@ func TestDataTransferResponding(t *testing.T) { response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) require.NoError(t, err, nil) require.Nil(t, response) - err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: &vr}) require.NoError(t, err) require.Len(t, h.transport.ResumedChannels, 1) resCh := h.transport.ResumedChannels[0] @@ -512,7 +527,8 @@ func TestDataTransferResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, RequiresFinalization: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, RequiresFinalization: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) @@ -533,7 +549,8 @@ func TestDataTransferResponding(t *testing.T) { response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) require.NoError(t, err, nil) require.Nil(t, response) - err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) require.NoError(t, err) require.Len(t, h.network.SentMessages, 2) sentMsg := h.network.SentMessages[1] @@ -572,7 +589,7 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: true}) }, verify: func(t *testing.T, h *receiverHarness) { - err := h.dt.RegisterTransportConfigurer(h.voucher, func(channelID datatransfer.ChannelID, voucher datatransfer.Voucher, transport datatransfer.Transport) { + err := h.dt.RegisterTransportConfigurer(h.voucher.Type, func(channelID datatransfer.ChannelID, voucher datatransfer.TypedVoucher, transport datatransfer.Transport) { ft, ok := transport.(*testutil.FakeTransport) if !ok { return @@ -597,7 +614,7 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: true}) }, verify: func(t *testing.T, h *receiverHarness) { - err := h.dt.RegisterTransportConfigurer(h.voucher, func(channelID datatransfer.ChannelID, voucher datatransfer.Voucher, transport datatransfer.Transport) { + err := h.dt.RegisterTransportConfigurer(h.voucher.Type, func(channelID datatransfer.ChannelID, voucher datatransfer.TypedVoucher, transport datatransfer.Transport) { ft, ok := transport.(*testutil.FakeTransport) if !ok { return @@ -634,26 +651,26 @@ func TestDataTransferResponding(t *testing.T) { } ev.setup(t, dt) h.stor = testutil.AllSelector() - h.voucher = testutil.NewFakeDTType() + h.voucher = testutil.NewTestTypedVoucher() h.baseCid = testutil.GenerateCids(1)[0] h.id = datatransfer.TransferID(rand.Int31()) - h.pullRequest, err = message.NewRequest(h.id, false, true, h.voucher.Type(), h.voucher, h.baseCid, h.stor) + h.pullRequest, err = message.NewRequest(h.id, false, true, &h.voucher, h.baseCid, h.stor) require.NoError(t, err) - h.pushRequest, err = message.NewRequest(h.id, false, false, h.voucher.Type(), h.voucher, h.baseCid, h.stor) + h.pushRequest, err = message.NewRequest(h.id, false, false, &h.voucher, h.baseCid, h.stor) require.NoError(t, err) h.pauseUpdate = message.UpdateRequest(h.id, true) require.NoError(t, err) h.resumeUpdate = message.UpdateRequest(h.id, false) require.NoError(t, err) - updateVoucher := testutil.NewFakeDTType() - h.voucherUpdate, err = message.VoucherRequest(h.id, updateVoucher.Type(), updateVoucher) + updateVoucher := testutil.NewTestTypedVoucher() + h.voucherUpdate, err = message.VoucherRequest(h.id, &updateVoucher) h.cancelUpdate = message.CancelRequest(h.id) require.NoError(t, err) h.sv = testutil.NewStubbedValidator() if verify.configureValidator != nil { verify.configureValidator(h.sv) } - require.NoError(t, h.dt.RegisterVoucherType(h.voucher, h.sv)) + require.NoError(t, h.dt.RegisterVoucherType(h.voucher.Type, h.sv)) require.NoError(t, err) verify.verify(t, h) h.sv.VerifyExpectations(t) @@ -677,7 +694,7 @@ func TestDataTransferRestartResponding(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, channelID) - response, err := message.RestartResponse(channelID.ID, true, false, datatransfer.EmptyTypeIdentifier, nil) + response, err := message.RestartResponse(channelID.ID, true, false, nil) require.NoError(t, err) err = h.transport.EventHandler.OnResponseReceived(channelID, response) require.NoError(t, err) @@ -697,9 +714,11 @@ func TestDataTransferRestartResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) sv.ExpectSuccessValidateRestart() - sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr = testutil.NewTestTypedVoucher() + sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming push @@ -717,8 +736,7 @@ func TestDataTransferRestartResponding(t *testing.T) { require.NoError(t, ev.OnDataReceived(chid, cidlink.Link{Cid: testCids[1]}, 12345, 2, true)) // receive restart push request - req, err := message.NewRequest(h.pushRequest.TransferID(), true, false, h.voucher.Type(), h.voucher, - h.baseCid, h.stor) + req, err := message.NewRequest(h.pushRequest.TransferID(), true, false, &h.voucher, h.baseCid, h.stor) require.NoError(t, err) h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], req) require.Len(t, h.sv.RevalidationsReceived, 1) @@ -758,9 +776,11 @@ func TestDataTransferRestartResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) sv.ExpectSuccessValidateRestart() - sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr = testutil.NewTestTypedVoucher() + sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull @@ -771,7 +791,7 @@ func TestDataTransferRestartResponding(t *testing.T) { require.Len(t, h.network.SentMessages, 0) // receive restart pull request - restartReq, err := message.NewRequest(h.id, true, true, h.voucher.Type(), h.voucher, h.baseCid, h.stor) + restartReq, err := message.NewRequest(h.id, true, true, &h.voucher, h.baseCid, h.stor) require.NoError(t, err) response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), restartReq) require.NoError(t, err) @@ -801,7 +821,8 @@ func TestDataTransferRestartResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull @@ -812,7 +833,7 @@ func TestDataTransferRestartResponding(t *testing.T) { require.Len(t, h.network.SentMessages, 0) // receive restart pull request - restartReq, err := message.NewRequest(h.id, true, true, h.voucher.Type(), h.voucher, h.baseCid, h.stor) + restartReq, err := message.NewRequest(h.id, true, true, &h.voucher, h.baseCid, h.stor) require.NoError(t, err) p := testutil.GeneratePeers(1)[0] chid := datatransfer.ChannelID{ID: h.pullRequest.TransferID(), Initiator: p, Responder: h.peers[0]} @@ -830,7 +851,8 @@ func TestDataTransferRestartResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) sv.ExpectSuccessValidateRestart() sv.StubRestartResult(datatransfer.ValidationResult{Accepted: false}) }, @@ -844,7 +866,7 @@ func TestDataTransferRestartResponding(t *testing.T) { // receive restart pull request h.sv.ExpectErrorPull() - restartReq, err := message.NewRequest(h.id, true, true, h.voucher.Type(), h.voucher, h.baseCid, h.stor) + restartReq, err := message.NewRequest(h.id, true, true, &h.voucher, h.baseCid, h.stor) require.NoError(t, err) _, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), restartReq) require.EqualError(t, err, datatransfer.ErrRejected.Error()) @@ -858,7 +880,8 @@ func TestDataTransferRestartResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull @@ -871,7 +894,7 @@ func TestDataTransferRestartResponding(t *testing.T) { // receive restart pull request randCid := testutil.GenerateCids(1)[0] - restartReq, err := message.NewRequest(h.id, true, true, h.voucher.Type(), h.voucher, randCid, h.stor) + restartReq, err := message.NewRequest(h.id, true, true, &h.voucher, randCid, h.stor) require.NoError(t, err) _, err = h.transport.EventHandler.OnRequestReceived(chid, restartReq) require.EqualError(t, err, fmt.Sprintf("restart request for channel %s failed validation: base cid does not match", chid)) @@ -885,7 +908,8 @@ func TestDataTransferRestartResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull @@ -898,10 +922,10 @@ func TestDataTransferRestartResponding(t *testing.T) { // receive restart pull request - restartReq, err := message.NewRequest(h.id, true, true, "rand", h.voucher, h.baseCid, h.stor) + restartReq, err := message.NewRequest(h.id, true, true, &datatransfer.TypedVoucher{Voucher: h.voucher.Voucher, Type: "rand"}, h.baseCid, h.stor) require.NoError(t, err) _, err = h.transport.EventHandler.OnRequestReceived(chid, restartReq) - require.EqualError(t, err, fmt.Sprintf("restart request for channel %s failed validation: failed to decode request voucher: unknown voucher type: rand", chid)) + require.EqualError(t, err, fmt.Sprintf("restart request for channel %s failed validation: channel and request voucher types do not match", chid)) }, }, "restart request fails if voucher does not match": { @@ -912,7 +936,8 @@ func TestDataTransferRestartResponding(t *testing.T) { }, configureValidator: func(sv *testutil.StubbedValidator) { sv.ExpectSuccessPull() - sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: testutil.NewFakeDTType()}) + vr := testutil.NewTestTypedVoucher() + sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull @@ -924,9 +949,8 @@ func TestDataTransferRestartResponding(t *testing.T) { require.Len(t, h.network.SentMessages, 0) // receive restart pull request - v := testutil.NewFakeDTType() - v.Data = "rand" - restartReq, err := message.NewRequest(h.id, true, true, h.voucher.Type(), v, h.baseCid, h.stor) + v := testutil.NewTestTypedVoucherWith("rand") + restartReq, err := message.NewRequest(h.id, true, true, &v, h.baseCid, h.stor) require.NoError(t, err) _, err = h.transport.EventHandler.OnRequestReceived(chid, restartReq) require.EqualError(t, err, fmt.Sprintf("restart request for channel %s failed validation: channel and request vouchers do not match", chid)) @@ -986,7 +1010,7 @@ func TestDataTransferRestartResponding(t *testing.T) { receivedSelector, err := request.Selector() require.NoError(t, err) require.Equal(t, receivedSelector, h.stor) - testutil.AssertFakeDTVoucher(t, request, h.voucher) + testutil.AssertTestVoucher(t, request, h.voucher) }, }, "ReceiveRestartExistingChannelRequest: Resend Push Request": { @@ -1027,7 +1051,7 @@ func TestDataTransferRestartResponding(t *testing.T) { receivedSelector, err := receivedRequest.Selector() require.NoError(t, err) require.Equal(t, receivedSelector, h.stor) - testutil.AssertFakeDTVoucher(t, receivedRequest, h.voucher) + testutil.AssertTestVoucher(t, receivedRequest, h.voucher) }, }, "ReceiveRestartExistingChannelRequest: errors if peer is not the initiator": { @@ -1093,19 +1117,19 @@ func TestDataTransferRestartResponding(t *testing.T) { } ev.setup(t, dt) h.stor = testutil.AllSelector() - h.voucher = testutil.NewFakeDTType() + h.voucher = testutil.NewTestTypedVoucher() h.baseCid = testutil.GenerateCids(1)[0] h.id = datatransfer.TransferID(rand.Int31()) - h.pullRequest, err = message.NewRequest(h.id, false, true, h.voucher.Type(), h.voucher, h.baseCid, h.stor) + h.pullRequest, err = message.NewRequest(h.id, false, true, &h.voucher, h.baseCid, h.stor) require.NoError(t, err) - h.pushRequest, err = message.NewRequest(h.id, false, false, h.voucher.Type(), h.voucher, h.baseCid, h.stor) + h.pushRequest, err = message.NewRequest(h.id, false, false, &h.voucher, h.baseCid, h.stor) require.NoError(t, err) h.sv = testutil.NewStubbedValidator() if verify.configureValidator != nil { verify.configureValidator(h.sv) } - require.NoError(t, h.dt.RegisterVoucherType(h.voucher, h.sv)) + require.NoError(t, h.dt.RegisterVoucherType(h.voucher.Type, h.sv)) verify.verify(t, h) h.sv.VerifyExpectations(t) @@ -1129,8 +1153,8 @@ type receiverHarness struct { sv *testutil.StubbedValidator ds datastore.Batching dt datatransfer.Manager - stor ipld.Node - voucher *testutil.FakeDTType + stor datamodel.Node + voucher datatransfer.TypedVoucher baseCid cid.Cid } diff --git a/impl/restart.go b/impl/restart.go index 9efe0728..eaee5349 100644 --- a/impl/restart.go +++ b/impl/restart.go @@ -1,16 +1,15 @@ package impl import ( - "bytes" "context" + "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channels" - "github.com/filecoin-project/go-data-transfer/v2/encoding" "github.com/filecoin-project/go-data-transfer/v2/message" ) @@ -73,17 +72,20 @@ func (m *manager) restartManagerPeerReceivePull(ctx context.Context, channel dat func (m *manager) openPushRestartChannel(ctx context.Context, channel datatransfer.ChannelState) error { selector := channel.Selector() - voucher := channel.Voucher() + voucher, err := channel.Voucher() + if err != nil { + return err + } baseCid := channel.BaseCID() requestTo := channel.OtherPeer() chid := channel.ChannelID() - req, err := message.NewRequest(chid.ID, true, false, voucher.Type(), voucher, baseCid, selector) + req, err := message.NewRequest(chid.ID, true, false, &voucher, baseCid, selector) if err != nil { return err } - processor, has := m.transportConfigurers.Processor(voucher.Type()) + processor, has := m.transportConfigurers.Processor(voucher.Type) if has { transportConfigurer := processor.(datatransfer.TransportConfigurer) transportConfigurer(chid, voucher, m.transport) @@ -108,17 +110,20 @@ func (m *manager) openPushRestartChannel(ctx context.Context, channel datatransf func (m *manager) openPullRestartChannel(ctx context.Context, channel datatransfer.ChannelState) error { selector := channel.Selector() - voucher := channel.Voucher() + voucher, err := channel.Voucher() + if err != nil { + return err + } baseCid := channel.BaseCID() requestTo := channel.OtherPeer() chid := channel.ChannelID() - req, err := message.NewRequest(chid.ID, true, true, voucher.Type(), voucher, baseCid, selector) + req, err := message.NewRequest(chid.ID, true, true, &voucher, baseCid, selector) if err != nil { return err } - processor, has := m.transportConfigurers.Processor(voucher.Type()) + processor, has := m.transportConfigurers.Processor(voucher.Type) if has { transportConfigurer := processor.(datatransfer.TransportConfigurer) transportConfigurer(chid, voucher, m.transport) @@ -164,24 +169,19 @@ func (m *manager) validateRestartRequest(ctx context.Context, otherPeer peer.ID, } // vouchers should match - reqVoucher, err := m.decodeVoucher(req) + reqVoucher, err := req.Voucher() if err != nil { - return xerrors.Errorf("failed to decode request voucher: %w", err) + return xerrors.Errorf("failed to fetch request voucher: %w", err) } - if reqVoucher.Type() != channel.Voucher().Type() { - return xerrors.New("channel and request voucher types do not match") - } - - reqBz, err := encoding.Encode(reqVoucher) + channelVoucher, err := channel.Voucher() if err != nil { - return xerrors.New("failed to encode request voucher") + return xerrors.Errorf("failed to fetch channel voucher: %w", err) } - channelBz, err := encoding.Encode(channel.Voucher()) - if err != nil { - return xerrors.New("failed to encode channel voucher") + if req.VoucherType() != channelVoucher.Type { + return xerrors.New("channel and request voucher types do not match") } - if !bytes.Equal(reqBz, channelBz) { + if !ipld.DeepEqual(reqVoucher, channelVoucher.Voucher) { return xerrors.New("channel and request vouchers do not match") } diff --git a/impl/restart_integration_test.go b/impl/restart_integration_test.go index 453aac80..1b4a50ec 100644 --- a/impl/restart_integration_test.go +++ b/impl/restart_integration_test.go @@ -41,8 +41,8 @@ func TestRestartPush(t *testing.T) { "Restart peer create push": { stopAt: 20, openPushF: func(rh *restartHarness) datatransfer.ChannelID { - voucher := testutil.FakeDTType{Data: "applesauce"} - chid, err := rh.dt1.OpenPushDataChannel(rh.testCtx, rh.peer2, &voucher, rh.rootCid, rh.gsData.AllSelector) + voucher := testutil.NewTestTypedVoucherWith("applesauce") + chid, err := rh.dt1.OpenPushDataChannel(rh.testCtx, rh.peer2, voucher, rh.rootCid, rh.gsData.AllSelector) require.NoError(rh.t, err) return chid }, @@ -53,7 +53,7 @@ func TestRestartPush(t *testing.T) { tp1 := rh.gsData.SetupGSTransportHost1() rh.dt1, err = NewDataTransfer(rh.gsData.DtDs1, rh.gsData.DtNet1, tp1) require.NoError(rh.t, err) - require.NoError(rh.t, rh.dt1.RegisterVoucherType(&testutil.FakeDTType{}, rh.sv)) + require.NoError(rh.t, rh.dt1.RegisterVoucherType(testutil.TestVoucherType, rh.sv)) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt1) rh.dt1.SubscribeToEvents(subscriber) require.NoError(rh.t, rh.dt1.RestartDataTransferChannel(rh.testCtx, chId)) @@ -83,8 +83,8 @@ func TestRestartPush(t *testing.T) { "Restart peer receive push": { stopAt: 20, openPushF: func(rh *restartHarness) datatransfer.ChannelID { - voucher := testutil.FakeDTType{Data: "applesauce"} - chid, err := rh.dt1.OpenPushDataChannel(rh.testCtx, rh.peer2, &voucher, rh.rootCid, rh.gsData.AllSelector) + voucher := testutil.NewTestTypedVoucherWith("applesauce") + chid, err := rh.dt1.OpenPushDataChannel(rh.testCtx, rh.peer2, voucher, rh.rootCid, rh.gsData.AllSelector) require.NoError(rh.t, err) return chid }, @@ -95,7 +95,7 @@ func TestRestartPush(t *testing.T) { tp2 := rh.gsData.SetupGSTransportHost2() rh.dt2, err = NewDataTransfer(rh.gsData.DtDs2, rh.gsData.DtNet2, tp2) require.NoError(rh.t, err) - require.NoError(rh.t, rh.dt2.RegisterVoucherType(&testutil.FakeDTType{}, rh.sv)) + require.NoError(rh.t, rh.dt2.RegisterVoucherType(testutil.TestVoucherType, rh.sv)) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt2) rh.dt2.SubscribeToEvents(subscriber) require.NoError(rh.t, rh.dt2.RestartDataTransferChannel(rh.testCtx, chId)) @@ -294,8 +294,8 @@ func TestRestartPull(t *testing.T) { "Restart peer create pull": { stopAt: 40, openPullF: func(rh *restartHarness) datatransfer.ChannelID { - voucher := testutil.FakeDTType{Data: "applesauce"} - chid, err := rh.dt2.OpenPullDataChannel(rh.testCtx, rh.peer1, &voucher, rh.rootCid, rh.gsData.AllSelector) + voucher := testutil.NewTestTypedVoucherWith("applesauce") + chid, err := rh.dt2.OpenPullDataChannel(rh.testCtx, rh.peer1, voucher, rh.rootCid, rh.gsData.AllSelector) require.NoError(rh.t, err) return chid }, @@ -306,7 +306,7 @@ func TestRestartPull(t *testing.T) { tp2 := rh.gsData.SetupGSTransportHost2() rh.dt2, err = NewDataTransfer(rh.gsData.DtDs2, rh.gsData.DtNet2, tp2) require.NoError(rh.t, err) - require.NoError(rh.t, rh.dt2.RegisterVoucherType(&testutil.FakeDTType{}, rh.sv)) + require.NoError(rh.t, rh.dt2.RegisterVoucherType(testutil.TestVoucherType, rh.sv)) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt2) rh.dt2.SubscribeToEvents(subscriber) require.NoError(rh.t, rh.dt2.RestartDataTransferChannel(rh.testCtx, chId)) @@ -333,8 +333,8 @@ func TestRestartPull(t *testing.T) { "Restart peer receive pull": { stopAt: 40, openPullF: func(rh *restartHarness) datatransfer.ChannelID { - voucher := testutil.FakeDTType{Data: "applesauce"} - chid, err := rh.dt2.OpenPullDataChannel(rh.testCtx, rh.peer1, &voucher, rh.rootCid, rh.gsData.AllSelector) + voucher := testutil.NewTestTypedVoucherWith("applesauce") + chid, err := rh.dt2.OpenPullDataChannel(rh.testCtx, rh.peer1, voucher, rh.rootCid, rh.gsData.AllSelector) require.NoError(rh.t, err) return chid }, @@ -345,7 +345,7 @@ func TestRestartPull(t *testing.T) { tp1 := rh.gsData.SetupGSTransportHost1() rh.dt1, err = NewDataTransfer(rh.gsData.DtDs1, rh.gsData.DtNet1, tp1) require.NoError(rh.t, err) - require.NoError(rh.t, rh.dt1.RegisterVoucherType(&testutil.FakeDTType{}, rh.sv)) + require.NoError(rh.t, rh.dt1.RegisterVoucherType(testutil.TestVoucherType, rh.sv)) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt1) rh.dt1.SubscribeToEvents(subscriber) require.NoError(rh.t, rh.dt1.RestartDataTransferChannel(rh.testCtx, chId)) @@ -570,8 +570,8 @@ func newRestartHarness(t *testing.T) *restartHarness { require.NoError(t, err) sv := testutil.NewStubbedValidator() - require.NoError(t, dt1.RegisterVoucherType(&testutil.FakeDTType{}, sv)) - require.NoError(t, dt2.RegisterVoucherType(&testutil.FakeDTType{}, sv)) + require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) + require.NoError(t, dt2.RegisterVoucherType(testutil.TestVoucherType, sv)) sourceDagService := gsData.DagService1 root, origBytes := testutil.LoadUnixFSFile(ctx, t, sourceDagService, largeFile) diff --git a/impl/utils.go b/impl/utils.go index 4c748acc..518b9e0c 100644 --- a/impl/utils.go +++ b/impl/utils.go @@ -4,9 +4,8 @@ import ( "context" "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" "github.com/libp2p/go-libp2p-core/peer" - "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/message" @@ -30,10 +29,10 @@ var resumeTransportStatesResponder = statusList{ } // newRequest encapsulates message creation -func (m *manager) newRequest(ctx context.Context, selector ipld.Node, isPull bool, voucher datatransfer.Voucher, baseCid cid.Cid, to peer.ID) (datatransfer.Request, error) { +func (m *manager) newRequest(ctx context.Context, selector datamodel.Node, isPull bool, voucher datatransfer.TypedVoucher, baseCid cid.Cid, to peer.ID) (datatransfer.Request, error) { // Generate a new transfer ID for the request tid := datatransfer.TransferID(m.transferIDGen.next()) - return message.NewRequest(tid, false, isPull, voucher.Type(), voucher, baseCid, selector) + return message.NewRequest(tid, false, isPull, &voucher, baseCid, selector) } func (m *manager) resume(chid datatransfer.ChannelID) error { @@ -84,29 +83,3 @@ func (m *manager) cancelMessage(chid datatransfer.ChannelID) datatransfer.Messag } return message.CancelResponse(chid.ID) } - -func (m *manager) decodeVoucherResult(response datatransfer.Response) (datatransfer.VoucherResult, error) { - vtypStr := datatransfer.TypeIdentifier(response.VoucherResultType()) - decoder, has := m.resultTypes.Decoder(vtypStr) - if !has { - return nil, xerrors.Errorf("unknown voucher result type: %s", vtypStr) - } - encodable, err := response.VoucherResult(decoder) - if err != nil { - return nil, err - } - return encodable.(datatransfer.Registerable), nil -} - -func (m *manager) decodeVoucher(request datatransfer.Request) (datatransfer.Voucher, error) { - vtypStr := datatransfer.TypeIdentifier(request.VoucherType()) - decoder, has := m.validatedTypes.Decoder(vtypStr) - if !has { - return nil, xerrors.Errorf("unknown voucher type: %s", vtypStr) - } - encodable, err := request.Voucher(decoder) - if err != nil { - return nil, err - } - return encodable.(datatransfer.Registerable), nil -} diff --git a/ipldutils/ipldutils.go b/ipldutils/ipldutils.go new file mode 100644 index 00000000..1b5b0be2 --- /dev/null +++ b/ipldutils/ipldutils.go @@ -0,0 +1,183 @@ +package shared + +import ( + "bytes" + "fmt" + "io" + "reflect" + + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/bindnode" + "github.com/ipld/go-ipld-prime/schema" + cbg "github.com/whyrusleeping/cbor-gen" +) + +type typeWithBindnodeSchema interface { + BindnodeSchema() string +} + +// TODO: remove this I think +type typeWithBindnodePostDecode interface { + BindnodePostDecode() error +} + +// We use the prototype map to store TypedPrototype and Type information +// mapped against Go type names so we only have to run the schema parse once. +// Currently there's not much additional benefit of storing this but there +// may be in the future. +var prototype map[string]schema.TypedPrototype = make(map[string]schema.TypedPrototype) + +var bindnodeOptions = []bindnode.Option{} + +func typeName(ptrValue interface{}) string { + val := reflect.ValueOf(ptrValue).Type() + for val.Kind() == reflect.Ptr { + val = val.Elem() + } + return val.Name() +} + +// lookup of cached TypedPrototype (and therefore Type) for a Go type, if not +// found, initial parse and setup and caching of the TypedPrototype will happen +func prototypeFor(typeName string, ptrType interface{}) (schema.TypedPrototype, error) { + proto, ok := prototype[typeName] + if !ok { + schemaType, err := schemaTypeFor(typeName, ptrType) + if err != nil { + return nil, err + } + if schemaType == nil { + return nil, fmt.Errorf("could not find type [%s] in schema", typeName) + } + proto = bindnode.Prototype(ptrType, schemaType, bindnodeOptions...) + prototype[typeName] = proto + } + return proto, nil +} + +// load the schema for a Go type, which must have a BindnodeSchema() method +// attached to it +func schemaTypeFor(typeName string, ptrType interface{}) (schema.Type, error) { + tws, ok := ptrType.(typeWithBindnodeSchema) + if !ok { + return nil, fmt.Errorf("attempted to perform IPLD mapping on type without BindnodeSchema(): %T", ptrType) + } + schema := tws.BindnodeSchema() + typeSystem, err := ipld.LoadSchemaBytes([]byte(schema)) + if err != nil { + return nil, err + } + schemaType := typeSystem.TypeByName(typeName) + if schemaType == nil { + if !ok { + return nil, fmt.Errorf("schema for [%T] does not contain that named type [%s]", ptrType, typeName) + } + } + return schemaType, nil +} + +// FromReader deserializes DAG-CBOR from a Reader and instantiates the Go type +// that's provided as a pointer via the ptrValue argument. +func FromReader(r io.Reader, ptrValue interface{}) (interface{}, error) { + name := typeName(ptrValue) + proto, err := prototypeFor(name, ptrValue) + if err != nil { + return nil, err + } + node, err := ipld.DecodeStreamingUsingPrototype(r, dagcbor.Decode, proto) + if err != nil { + return nil, err + } + typ := bindnode.Unwrap(node) + if twpd, ok := typ.(typeWithBindnodePostDecode); ok { + // we have some more work to do + if err = twpd.BindnodePostDecode(); err != nil { + return nil, err + } + } + return typ, nil +} + +// FromNode converts an datamodel.Node into an appropriate Go type that's provided as +// a pointer via the ptrValue argument +func FromNode(node datamodel.Node, ptrValue interface{}) (interface{}, error) { + name := typeName(ptrValue) + proto, err := prototypeFor(name, ptrValue) + if err != nil { + return nil, err + } + if tn, ok := node.(schema.TypedNode); ok { + node = tn.Representation() + } + builder := proto.Representation().NewBuilder() + err = builder.AssignNode(node) + if err != nil { + return nil, err + } + typ := bindnode.Unwrap(builder.Build()) + if twpd, ok := typ.(typeWithBindnodePostDecode); ok { + // we have some more work to do + if err = twpd.BindnodePostDecode(); err != nil { + return nil, err + } + } + return typ, nil +} + +// ToNode converts a Go type that's provided as a pointer via the ptrValue +// argument to an datamodel.Node. +func ToNode(ptrValue interface{}) (schema.TypedNode, error) { + name := typeName(ptrValue) + proto, err := prototypeFor(name, ptrValue) + if err != nil { + return nil, err + } + return bindnode.Wrap(ptrValue, proto.Type(), bindnodeOptions...), err +} + +// NodeToWriter is a utility method that serializes an datamodel.Node as DAG-CBOR to +// a Writer +func NodeToWriter(node datamodel.Node, w io.Writer) error { + return ipld.EncodeStreaming(w, node, dagcbor.Encode) +} + +// NodeToBytes is a utility method that serializes an datamodel.Node as DAG-CBOR to +// a []byte +func NodeToBytes(node datamodel.Node) ([]byte, error) { + var buf bytes.Buffer + err := NodeToWriter(node, &buf) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// NodeFromBytes is a utility method that deserializes an untyped datamodel.Node +// from DAG-CBOR format bytes +func NodeFromBytes(b []byte) (datamodel.Node, error) { + return ipld.Decode(b, dagcbor.Decode) +} + +// TypeToWriter is a utility method that serializes a Go type that's provided as a +// pointer via the ptrValue argument as DAG-CBOR to a Writer +func TypeToWriter(ptrValue interface{}, w io.Writer) error { + node, err := ToNode(ptrValue) + if err != nil { + return err + } + return ipld.EncodeStreaming(w, node, dagcbor.Encode) +} + +func NodeToDeferred(node datamodel.Node) (*cbg.Deferred, error) { + byts, err := NodeToBytes(node) + if err != nil { + return nil, err + } + return &cbg.Deferred{Raw: byts}, nil +} + +func DeferredToNode(def *cbg.Deferred) (datamodel.Node, error) { + return NodeFromBytes(def.Raw) +} diff --git a/manager.go b/manager.go index b291dc11..822f76e5 100644 --- a/manager.go +++ b/manager.go @@ -4,7 +4,7 @@ import ( "context" "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" "github.com/libp2p/go-libp2p-core/peer" ) @@ -15,7 +15,7 @@ type ValidationResult struct { Accepted bool // VoucherResult provides information to the other party about what happened // with the voucher - VoucherResult + VoucherResult *TypedVoucher // ForcePause indicates whether the request should be paused, regardless // of data limit and finalization status ForcePause bool @@ -28,6 +28,16 @@ type ValidationResult struct { RequiresFinalization bool } +// Equals checks the deep equality of two ValidationResult values +func (vr ValidationResult) Equals(vr2 ValidationResult) bool { + return vr.Accepted == vr2.Accepted && + vr.ForcePause == vr2.ForcePause && + vr.DataLimit == vr2.DataLimit && + vr.RequiresFinalization == vr2.RequiresFinalization && + (vr.VoucherResult == nil) == (vr2.VoucherResult == nil) && + (vr.VoucherResult == nil || vr.VoucherResult.Equals(*vr2.VoucherResult)) +} + // LeaveRequestPaused indicates whether all conditions are met to resume a request func (vr ValidationResult) LeaveRequestPaused(chst ChannelState) bool { if vr.ForcePause { @@ -56,9 +66,9 @@ type RequestValidator interface { ValidatePush( chid ChannelID, sender peer.ID, - voucher Voucher, + voucher datamodel.Node, baseCid cid.Cid, - selector ipld.Node) (ValidationResult, error) + selector datamodel.Node) (ValidationResult, error) // ValidatePull validates a pull request received from the peer that will receive data // -- All information about the validation operation is contained in ValidationResult, // including if it was rejected. Information about why a rejection occurred should be embedded @@ -67,9 +77,9 @@ type RequestValidator interface { ValidatePull( chid ChannelID, receiver peer.ID, - voucher Voucher, + voucher datamodel.Node, baseCid cid.Cid, - selector ipld.Node) (ValidationResult, error) + selector datamodel.Node) (ValidationResult, error) // ValidateRestart validates restarting a request // -- All information about the validation operation is contained in ValidationResult, @@ -80,7 +90,7 @@ type RequestValidator interface { } // TransportConfigurer provides a mechanism to provide transport specific configuration for a given voucher type -type TransportConfigurer func(chid ChannelID, voucher Voucher, transport Transport) +type TransportConfigurer func(chid ChannelID, voucher TypedVoucher, transport Transport) // ReadyFunc is function that gets called once when the data transfer module is ready type ReadyFunc func(error) @@ -101,29 +111,25 @@ type Manager interface { // RegisterVoucherType registers a validator for the given voucher type // will error if voucher type does not implement voucher // or if there is a voucher type registered with an identical identifier - RegisterVoucherType(voucherType Voucher, validator RequestValidator) error - - // RegisterVoucherResultType allows deserialization of a voucher result, - // so that a listener can read the metadata - RegisterVoucherResultType(resultType VoucherResult) error + RegisterVoucherType(voucherType TypeIdentifier, validator RequestValidator) error // RegisterTransportConfigurer registers the given transport configurer to be run on requests with the given voucher // type - RegisterTransportConfigurer(voucherType Voucher, configurer TransportConfigurer) error + RegisterTransportConfigurer(voucherType TypeIdentifier, configurer TransportConfigurer) error // open a data transfer that will send data to the recipient peer and // transfer parts of the piece that match the selector - OpenPushDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error) + OpenPushDataChannel(ctx context.Context, to peer.ID, voucher TypedVoucher, baseCid cid.Cid, selector datamodel.Node) (ChannelID, error) // open a data transfer that will request data from the sending peer and // transfer parts of the piece that match the selector - OpenPullDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error) + OpenPullDataChannel(ctx context.Context, to peer.ID, voucher TypedVoucher, baseCid cid.Cid, selector datamodel.Node) (ChannelID, error) // send an intermediate voucher as needed when the receiver sends a request for revalidation - SendVoucher(ctx context.Context, chid ChannelID, voucher Voucher) error + SendVoucher(ctx context.Context, chid ChannelID, voucher TypedVoucher) error // send information from the responder to update the initiator on the state of their voucher - SendVoucherResult(ctx context.Context, chid ChannelID, voucher VoucherResult) error + SendVoucherResult(ctx context.Context, chid ChannelID, voucherResult TypedVoucher) error // Update the validation status for a given channel, to change data limits, finalization, accepted status, and pause state // and send new voucher results as diff --git a/message.go b/message.go index 7f36b33d..d54eabb1 100644 --- a/message.go +++ b/message.go @@ -4,11 +4,8 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" "github.com/libp2p/go-libp2p-core/protocol" - - "github.com/filecoin-project/go-data-transfer/v2/encoding" ) var ( @@ -38,9 +35,10 @@ type Request interface { IsPull() bool IsVoucher() bool VoucherType() TypeIdentifier - Voucher(decoder encoding.Decoder) (encoding.Encodable, error) + Voucher() (datamodel.Node, error) + TypedVoucher() (TypedVoucher, error) BaseCid() cid.Cid - Selector() (ipld.Node, error) + Selector() (datamodel.Node, error) IsRestartExistingChannelRequest() bool RestartChannelId() (ChannelID, error) } @@ -52,6 +50,6 @@ type Response interface { IsComplete() bool Accepted() bool VoucherResultType() TypeIdentifier - VoucherResult(decoder encoding.Decoder) (encoding.Encodable, error) + VoucherResult() (datamodel.Node, error) EmptyVoucherResult() bool } diff --git a/message/message1_1prime/message.go b/message/message1_1prime/message.go index e533252d..b740bc01 100644 --- a/message/message1_1prime/message.go +++ b/message/message1_1prime/message.go @@ -5,24 +5,25 @@ import ( "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" - "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" xerrors "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/encoding" + ipldutils "github.com/filecoin-project/go-data-transfer/v2/ipldutils" "github.com/filecoin-project/go-data-transfer/v2/message/types" ) +var emptyTypedVoucher = datatransfer.TypedVoucher{ + Voucher: ipld.Null, + Type: datatransfer.EmptyTypeIdentifier, +} + // NewRequest generates a new request for the data transfer protocol -func NewRequest(id datatransfer.TransferID, isRestart bool, isPull bool, vtype datatransfer.TypeIdentifier, voucher encoding.Encodable, baseCid cid.Cid, selector ipld.Node) (datatransfer.Request, error) { - vnode, err := encoding.EncodeToNode(voucher) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) +func NewRequest(id datatransfer.TransferID, isRestart bool, isPull bool, voucher *datatransfer.TypedVoucher, baseCid cid.Cid, selector datamodel.Node) (datatransfer.Request, error) { + if voucher == nil { + voucher = &emptyTypedVoucher } - if baseCid == cid.Undef { return nil, xerrors.Errorf("base CID must be defined") } @@ -34,13 +35,17 @@ func NewRequest(id datatransfer.TransferID, isRestart bool, isPull bool, vtype d typ = uint64(types.NewMessage) } + if voucher == nil { + voucher = &emptyTypedVoucher + } + return &TransferRequest1_1{ MessageType: typ, Pull: isPull, - VoucherPtr: &vnode, - SelectorPtr: &selector, + VoucherPtr: voucher.Voucher, + SelectorPtr: selector, BaseCidPtr: &baseCid, - VoucherTypeIdentifier: vtype, + VoucherTypeIdentifier: voucher.Type, TransferId: uint64(id), }, nil } @@ -71,32 +76,30 @@ func UpdateRequest(id datatransfer.TransferID, isPaused bool) datatransfer.Reque } // VoucherRequest generates a new request for the data transfer protocol -func VoucherRequest(id datatransfer.TransferID, vtype datatransfer.TypeIdentifier, voucher encoding.Encodable) (datatransfer.Request, error) { - vnode, err := encoding.EncodeToNode(voucher) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) +func VoucherRequest(id datatransfer.TransferID, voucher *datatransfer.TypedVoucher) (datatransfer.Request, error) { + if voucher == nil { + voucher = &emptyTypedVoucher } return &TransferRequest1_1{ MessageType: uint64(types.VoucherMessage), - VoucherPtr: &vnode, - VoucherTypeIdentifier: vtype, + VoucherPtr: voucher.Voucher, + VoucherTypeIdentifier: voucher.Type, TransferId: uint64(id), }, nil } // RestartResponse builds a new Data Transfer response -func RestartResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResultType datatransfer.TypeIdentifier, voucherResult encoding.Encodable) (datatransfer.Response, error) { - vnode, err := encoding.EncodeToNode(voucherResult) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) +func RestartResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResult *datatransfer.TypedVoucher) (datatransfer.Response, error) { + if voucherResult == nil { + voucherResult = &emptyTypedVoucher } return &TransferResponse1_1{ RequestAccepted: accepted, MessageType: uint64(types.RestartMessage), Paused: isPaused, TransferId: uint64(id), - VoucherTypeIdentifier: voucherResultType, - VoucherResultPtr: &vnode, + VoucherResultPtr: voucherResult.Voucher, + VoucherTypeIdentifier: voucherResult.Type, }, nil } @@ -108,13 +111,10 @@ func ValidationResultResponse( validationResult datatransfer.ValidationResult, validationErr error, paused bool) (datatransfer.Response, error) { - voucherResultType := datatransfer.EmptyTypeIdentifier + + voucherResult := &emptyTypedVoucher if validationResult.VoucherResult != nil { - voucherResultType = validationResult.VoucherResult.Type() - } - vnode, err := encoding.EncodeToNode(validationResult.VoucherResult) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) + voucherResult = validationResult.VoucherResult } return &TransferResponse1_1{ // TODO: when we area able to change the protocol, it would be helpful to record @@ -123,40 +123,38 @@ func ValidationResultResponse( MessageType: uint64(messageType), Paused: paused, TransferId: uint64(id), - VoucherTypeIdentifier: voucherResultType, - VoucherResultPtr: &vnode, + VoucherTypeIdentifier: voucherResult.Type, + VoucherResultPtr: voucherResult.Voucher, }, nil } // NewResponse builds a new Data Transfer response -func NewResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResultType datatransfer.TypeIdentifier, voucherResult encoding.Encodable) (datatransfer.Response, error) { - vnode, err := encoding.EncodeToNode(voucherResult) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) +func NewResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResult *datatransfer.TypedVoucher) (datatransfer.Response, error) { + if voucherResult == nil { + voucherResult = &emptyTypedVoucher } return &TransferResponse1_1{ RequestAccepted: accepted, MessageType: uint64(types.NewMessage), Paused: isPaused, TransferId: uint64(id), - VoucherTypeIdentifier: voucherResultType, - VoucherResultPtr: &vnode, + VoucherTypeIdentifier: voucherResult.Type, + VoucherResultPtr: voucherResult.Voucher, }, nil } // VoucherResultResponse builds a new response for a voucher result -func VoucherResultResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResultType datatransfer.TypeIdentifier, voucherResult encoding.Encodable) (datatransfer.Response, error) { - vnode, err := encoding.EncodeToNode(voucherResult) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) +func VoucherResultResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResult *datatransfer.TypedVoucher) (datatransfer.Response, error) { + if voucherResult == nil { + voucherResult = &emptyTypedVoucher } return &TransferResponse1_1{ RequestAccepted: accepted, MessageType: uint64(types.VoucherResultMessage), Paused: isPaused, TransferId: uint64(id), - VoucherTypeIdentifier: voucherResultType, - VoucherResultPtr: &vnode, + VoucherTypeIdentifier: voucherResult.Type, + VoucherResultPtr: voucherResult.Voucher, }, nil } @@ -178,30 +176,27 @@ func CancelResponse(id datatransfer.TransferID) datatransfer.Response { } // CompleteResponse returns a new complete response message -func CompleteResponse(id datatransfer.TransferID, isAccepted bool, isPaused bool, voucherResultType datatransfer.TypeIdentifier, voucherResult encoding.Encodable) (datatransfer.Response, error) { - vnode, err := encoding.EncodeToNode(voucherResult) - if err != nil { - return nil, xerrors.Errorf("Creating request: %w", err) +func CompleteResponse(id datatransfer.TransferID, isAccepted bool, isPaused bool, voucherResult *datatransfer.TypedVoucher) (datatransfer.Response, error) { + if voucherResult == nil { + voucherResult = &emptyTypedVoucher } return &TransferResponse1_1{ MessageType: uint64(types.CompleteMessage), RequestAccepted: isAccepted, Paused: isPaused, - VoucherTypeIdentifier: voucherResultType, - VoucherResultPtr: &vnode, + VoucherTypeIdentifier: voucherResult.Type, + VoucherResultPtr: voucherResult.Voucher, TransferId: uint64(id), }, nil } // FromNet can read a network stream to deserialize a GraphSyncMessage func FromNet(r io.Reader) (datatransfer.Message, error) { - builder := Prototype.TransferMessage.Representation().NewBuilder() - err := dagcbor.Decode(builder, r) + tm, err := ipldutils.FromReader(r, &TransferMessage1_1{}) if err != nil { return nil, err } - node := builder.Build() - tresp := bindnode.Unwrap(node).(*TransferMessage1_1) + tresp := tm.(*TransferMessage1_1) if (tresp.IsRequest && tresp.Request == nil) || (!tresp.IsRequest && tresp.Response == nil) { return nil, xerrors.Errorf("invalid/malformed message") @@ -218,12 +213,12 @@ func FromIPLD(node datamodel.Node) (datatransfer.Message, error) { if tn, ok := node.(schema.TypedNode); ok { // shouldn't need this if from Graphsync node = tn.Representation() } - builder := Prototype.TransferMessage.Representation().NewBuilder() - err := builder.AssignNode(node) + tm, err := ipldutils.FromNode(node, &TransferMessage1_1{}) if err != nil { return nil, err } - tresp := bindnode.Unwrap(builder.Build()).(*TransferMessage1_1) + tresp := tm.(*TransferMessage1_1) + if (tresp.IsRequest && tresp.Request == nil) || (!tresp.IsRequest && tresp.Response == nil) { return nil, xerrors.Errorf("invalid/malformed message") } diff --git a/message/message1_1prime/message_test.go b/message/message1_1prime/message_test.go index 93e95f27..7ef75e53 100644 --- a/message/message1_1prime/message_test.go +++ b/message/message1_1prime/message_test.go @@ -8,16 +8,13 @@ import ( "testing" "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime/codec/dagcbor" basicnode "github.com/ipld/go-ipld-prime/node/basic" - "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/encoding" message1_1 "github.com/filecoin-project/go-data-transfer/v2/message/message1_1prime" "github.com/filecoin-project/go-data-transfer/v2/testutil" ) @@ -27,8 +24,8 @@ func TestNewRequest(t *testing.T) { selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() isPull := true id := datatransfer.TransferID(rand.Int31()) - voucher := testutil.NewFakeDTType() - request, err := message1_1.NewRequest(id, false, isPull, voucher.Type(), voucher, baseCid, selector) + voucher := testutil.NewTestTypedVoucher() + request, err := message1_1.NewRequest(id, false, isPull, &voucher, baseCid, selector) require.NoError(t, err) assert.Equal(t, id, request.TransferID()) assert.False(t, request.IsCancel()) @@ -36,8 +33,7 @@ func TestNewRequest(t *testing.T) { assert.True(t, request.IsPull()) assert.True(t, request.IsRequest()) assert.Equal(t, baseCid.String(), request.BaseCid().String()) - encoding.NewDecoder(request) - testutil.AssertFakeDTVoucher(t, request, voucher) + testutil.AssertTestVoucher(t, request, voucher) receivedSelector, err := request.Selector() require.NoError(t, err) require.Equal(t, selector, receivedSelector) @@ -56,8 +52,8 @@ func TestRestartRequest(t *testing.T) { selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() isPull := true id := datatransfer.TransferID(rand.Int31()) - voucher := testutil.NewFakeDTType() - request, err := message1_1.NewRequest(id, true, isPull, voucher.Type(), voucher, baseCid, selector) + voucher := testutil.NewTestTypedVoucher() + request, err := message1_1.NewRequest(id, true, isPull, &voucher, baseCid, selector) require.NoError(t, err) assert.Equal(t, id, request.TransferID()) assert.False(t, request.IsCancel()) @@ -65,7 +61,7 @@ func TestRestartRequest(t *testing.T) { assert.True(t, request.IsPull()) assert.True(t, request.IsRequest()) assert.Equal(t, baseCid.String(), request.BaseCid().String()) - testutil.AssertFakeDTVoucher(t, request, voucher) + testutil.AssertTestVoucher(t, request, voucher) receivedSelector, err := request.Selector() require.NoError(t, err) require.Equal(t, selector, receivedSelector) @@ -115,19 +111,9 @@ func TestRestartExistingChannelRequest(t *testing.T) { }) } -func TestTransferRequest_MarshalCBOR(t *testing.T) { - // sanity check MarshalCBOR does its thing w/o error - req, err := NewTestTransferRequest() - require.NoError(t, err) - wbuf := new(bytes.Buffer) - node := bindnode.Wrap(&req, message1_1.Prototype.TransferRequest.Type()) - err = dagcbor.Encode(node.Representation(), wbuf) - require.NoError(t, err) - assert.Greater(t, wbuf.Len(), 0) -} func TestTransferRequest_UnmarshalCBOR(t *testing.T) { t.Run("round-trip", func(t *testing.T) { - req, err := NewTestTransferRequest() + req, err := NewTestTransferRequest("test data here") require.NoError(t, err) wbuf := new(bytes.Buffer) // use ToNet / FromNet @@ -144,14 +130,15 @@ func TestTransferRequest_UnmarshalCBOR(t *testing.T) { assert.Equal(t, req.IsPull(), desReq.IsPull()) assert.Equal(t, req.IsCancel(), desReq.IsCancel()) assert.Equal(t, req.BaseCid(), desReq.BaseCid()) - testutil.AssertEqualFakeDTVoucher(t, &req, desReq) + testutil.AssertEqualTestVoucher(t, &req, desReq) testutil.AssertEqualSelector(t, &req, desReq) }) t.Run("cbor-gen compat", func(t *testing.T) { - req, err := NewTestTransferRequest() + vouchByts, _ := hex.DecodeString("f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e35") + req, err := NewTestTransferRequest(string(vouchByts)) require.NoError(t, err) - msg, _ := hex.DecodeString("a36449735271f56752657175657374aa6442436964d82a58230012204bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a6454797065006450617573f46450617274f46450756c6cf46453746f72a1612ea065566f756368817864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e3564565479706a46616b65445454797065665866657249441a4d6582216e526573746172744368616e6e656c8360600068526573706f6e7365f6") + msg, _ := hex.DecodeString("a36449735271f56752657175657374aa6442436964d82a58230012204bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a6454797065006450617573f46450617274f46450756c6cf46453746f72a1612ea065566f756368817864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e3564565479706b54657374566f7563686572665866657249441a4d6582216e526573746172744368616e6e656c8360600068526573706f6e7365f6") desMsg, err := message1_1.FromNet(bytes.NewReader(msg)) require.NoError(t, err) @@ -164,15 +151,15 @@ func TestTransferRequest_UnmarshalCBOR(t *testing.T) { assert.Equal(t, req.IsCancel(), desReq.IsCancel()) c, _ := cid.Parse("QmTTA2daxGqo5denp6SwLzzkLJm3fuisYEi9CoWsuHpzfb") assert.Equal(t, c, desReq.BaseCid()) - testutil.AssertEqualFakeDTVoucher(t, &req, desReq) + testutil.AssertEqualTestVoucher(t, &req, desReq) testutil.AssertEqualSelector(t, &req, desReq) }) } func TestResponses(t *testing.T) { id := datatransfer.TransferID(rand.Int31()) - voucherResult := testutil.NewFakeDTType() - response, err := message1_1.NewResponse(id, false, true, voucherResult.Type(), voucherResult) // not accepted + voucherResult := testutil.NewTestTypedVoucher() + response, err := message1_1.NewResponse(id, false, true, &voucherResult) // not accepted require.NoError(t, err) assert.Equal(t, response.TransferID(), id) assert.False(t, response.Accepted()) @@ -180,7 +167,7 @@ func TestResponses(t *testing.T) { assert.False(t, response.IsUpdate()) assert.True(t, response.IsPaused()) assert.False(t, response.IsRequest()) - testutil.AssertFakeDTVoucherResult(t, response, voucherResult) + testutil.AssertTestVoucherResult(t, response, voucherResult) // Sanity check to make sure we can cast to datatransfer.Message msg, ok := response.(datatransfer.Message) require.True(t, ok) @@ -194,8 +181,8 @@ func TestResponses(t *testing.T) { func TestTransferResponse_MarshalCBOR(t *testing.T) { id := datatransfer.TransferID(rand.Int31()) - voucherResult := testutil.NewFakeDTType() - response, err := message1_1.NewResponse(id, true, false, voucherResult.Type(), voucherResult) // accepted + voucherResult := testutil.NewTestTypedVoucher() + response, err := message1_1.NewResponse(id, true, false, &voucherResult) // accepted require.NoError(t, err) // sanity check that we can marshal data @@ -207,8 +194,8 @@ func TestTransferResponse_MarshalCBOR(t *testing.T) { func TestTransferResponse_UnmarshalCBOR(t *testing.T) { t.Run("round-trip", func(t *testing.T) { id := datatransfer.TransferID(rand.Int31()) - voucherResult := testutil.NewFakeDTType() - response, err := message1_1.NewResponse(id, true, false, voucherResult.Type(), voucherResult) // accepted + voucherResult := testutil.NewTestTypedVoucher() + response, err := message1_1.NewResponse(id, true, false, &voucherResult) // accepted require.NoError(t, err) wbuf := new(bytes.Buffer) @@ -229,13 +216,11 @@ func TestTransferResponse_UnmarshalCBOR(t *testing.T) { assert.True(t, desResp.IsNew()) assert.False(t, desResp.IsUpdate()) assert.False(t, desMsg.IsPaused()) - testutil.AssertFakeDTVoucherResult(t, desResp, voucherResult) + testutil.AssertTestVoucherResult(t, desResp, voucherResult) }) t.Run("cbor-gen compat", func(t *testing.T) { - voucherResult := testutil.NewFakeDTType() - voucherResult.Data = "\xf5_\xf8\xf1%\b\xb6>\xf2\xbf\xec\xa7Uz\xe9\r\xf61\x1a^\xc1c\x1bJ\x1f\xa8C1\v\xd9ç\x10\xea\xac塽\xd7*п\xe0Iw\x1c\x11\xe7V3\x8b\xd98e\xe6E\xf1\xad웜\x99\xef@\u007f\xbdOƅ\x9ey\x04ŭ}ɽ\x10\xa5\xcc\x16\x97=[(\xec\x1am\xd4=\x9f\x82\xf9\xf1\x8c=\x03A\x8e5" - - msg, _ := hex.DecodeString("a36449735271f46752657175657374f668526573706f6e7365a66454797065006441637074f56450617573f4665866657249441a4d6582216456526573817864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e3564565479706a46616b65445454797065") + voucherResult := testutil.NewTestTypedVoucherWith("\xf5_\xf8\xf1%\b\xb6>\xf2\xbf\xec\xa7Uz\xe9\r\xf61\x1a^\xc1c\x1bJ\x1f\xa8C1\v\xd9ç\x10\xea\xac塽\xd7*п\xe0Iw\x1c\x11\xe7V3\x8b\xd98e\xe6E\xf1\xad웜\x99\xef@\u007f\xbdOƅ\x9ey\x04ŭ}ɽ\x10\xa5\xcc\x16\x97=[(\xec\x1am\xd4=\x9f\x82\xf9\xf1\x8c=\x03A\x8e5") + msg, _ := hex.DecodeString("a36449735271f46752657175657374f668526573706f6e7365a66454797065006441637074f56450617573f4665866657249441a4d6582216456526573817864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e3564565479706b54657374566f7563686572") desMsg, err := message1_1.FromNet(bytes.NewReader(msg)) require.NoError(t, err) assert.False(t, desMsg.IsRequest()) @@ -250,7 +235,7 @@ func TestTransferResponse_UnmarshalCBOR(t *testing.T) { assert.True(t, desResp.IsNew()) assert.False(t, desResp.IsUpdate()) assert.False(t, desMsg.IsPaused()) - testutil.AssertFakeDTVoucherResult(t, desResp, voucherResult) + testutil.AssertTestVoucherResult(t, desResp, voucherResult) }) } @@ -381,7 +366,7 @@ func TestCancelResponse(t *testing.T) { func TestCompleteResponse(t *testing.T) { id := datatransfer.TransferID(rand.Int31()) - response, err := message1_1.CompleteResponse(id, true, true, datatransfer.EmptyTypeIdentifier, nil) + response, err := message1_1.CompleteResponse(id, true, true, nil) require.NoError(t, err) assert.Equal(t, response.TransferID(), id) assert.False(t, response.IsNew()) @@ -407,9 +392,9 @@ func TestToNetFromNetEquivalency(t *testing.T) { isPull := false id := datatransfer.TransferID(rand.Int31()) accepted := false - voucher := testutil.NewFakeDTType() - voucherResult := testutil.NewFakeDTType() - request, err := message1_1.NewRequest(id, false, isPull, voucher.Type(), voucher, baseCid, selector) + voucher := testutil.NewTestTypedVoucher() + voucherResult := testutil.NewTestTypedVoucher() + request, err := message1_1.NewRequest(id, false, isPull, &voucher, baseCid, selector) require.NoError(t, err) buf := new(bytes.Buffer) err = request.ToNet(buf) @@ -426,10 +411,10 @@ func TestToNetFromNetEquivalency(t *testing.T) { require.Equal(t, deserializedRequest.IsPull(), request.IsPull()) require.Equal(t, deserializedRequest.IsRequest(), request.IsRequest()) require.Equal(t, deserializedRequest.BaseCid(), request.BaseCid()) - testutil.AssertEqualFakeDTVoucher(t, request, deserializedRequest) + testutil.AssertEqualTestVoucher(t, request, deserializedRequest) testutil.AssertEqualSelector(t, request, deserializedRequest) - response, err := message1_1.NewResponse(id, accepted, false, voucherResult.Type(), voucherResult) + response, err := message1_1.NewResponse(id, accepted, false, &voucherResult) require.NoError(t, err) err = response.ToNet(buf) require.NoError(t, err) @@ -444,7 +429,7 @@ func TestToNetFromNetEquivalency(t *testing.T) { require.Equal(t, deserializedResponse.IsRequest(), response.IsRequest()) require.Equal(t, deserializedResponse.IsUpdate(), response.IsUpdate()) require.Equal(t, deserializedResponse.IsPaused(), response.IsPaused()) - testutil.AssertEqualFakeDTVoucherResult(t, response, deserializedResponse) + testutil.AssertEqualTestVoucherResult(t, response, deserializedResponse) request = message1_1.CancelRequest(id) err = request.ToNet(buf) @@ -465,15 +450,17 @@ func TestToNetFromNetEquivalency(t *testing.T) { isPull := false id := datatransfer.TransferID(1298498081) accepted := false - voucher := testutil.NewFakeDTType() - voucherResult := testutil.NewFakeDTType() - request, err := message1_1.NewRequest(id, false, isPull, voucher.Type(), voucher, baseCid, selector) + vouchByts, _ := hex.DecodeString("f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e35") + voucher := testutil.NewTestTypedVoucherWith(string(vouchByts)) + vouchResultByts, _ := hex.DecodeString("4204cb9a1e34c5f08e9b20aa76090e70020bb56c0ca3d3af7296cd1058a5112890fed218488f084d8df9e4835fb54ad045ffd936e3bf7261b0426c51352a097816ed74482bb9084b4a7ed8adc517f3371e0e0434b511625cd1a41792243dccdcfe88094b") + voucherResult := testutil.NewTestTypedVoucherWith(string(vouchResultByts)) + request, err := message1_1.NewRequest(id, false, isPull, &voucher, baseCid, selector) require.NoError(t, err) buf := new(bytes.Buffer) err = request.ToNet(buf) require.NoError(t, err) require.Greater(t, buf.Len(), 0) - msg, _ := hex.DecodeString("a36449735271f56752657175657374aa6442436964d82a58230012204bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a6454797065006450617573f46450617274f46450756c6cf46453746f72a1612ea065566f756368817864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e3564565479706a46616b65445454797065665866657249441a4d6582216e526573746172744368616e6e656c8360600068526573706f6e7365f6") + msg, _ := hex.DecodeString("a36449735271f56752657175657374aa6442436964d82a58230012204bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a6454797065006450617573f46450617274f46450756c6cf46453746f72a1612ea065566f756368817864f55ff8f12508b63ef2bfeca7557ae90df6311a5ec1631b4a1fa843310bd9c3a710eaace5a1bdd72ad0bfe049771c11e756338bd93865e645f1adec9b9c99ef407fbd4fc6859e7904c5ad7dc9bd10a5cc16973d5b28ec1a6dd43d9f82f9f18c3d03418e3564565479706b54657374566f7563686572665866657249441a4d6582216e526573746172744368616e6e656c8360600068526573706f6e7365f6") deserialized, err := message1_1.FromNet(bytes.NewReader(msg)) require.NoError(t, err) @@ -486,14 +473,14 @@ func TestToNetFromNetEquivalency(t *testing.T) { require.Equal(t, deserializedRequest.IsRequest(), request.IsRequest()) c, _ := cid.Parse("QmTTA2daxGqo5denp6SwLzzkLJm3fuisYEi9CoWsuHpzfb") assert.Equal(t, c, deserializedRequest.BaseCid()) - testutil.AssertEqualFakeDTVoucher(t, request, deserializedRequest) + testutil.AssertEqualTestVoucher(t, request, deserializedRequest) testutil.AssertEqualSelector(t, request, deserializedRequest) - response, err := message1_1.NewResponse(id, accepted, false, voucherResult.Type(), voucherResult) + response, err := message1_1.NewResponse(id, accepted, false, &voucherResult) require.NoError(t, err) err = response.ToNet(buf) require.NoError(t, err) - msg, _ = hex.DecodeString("a36449735271f46752657175657374f668526573706f6e7365a66454797065006441637074f46450617573f4665866657249441a4d65822164565265738178644204cb9a1e34c5f08e9b20aa76090e70020bb56c0ca3d3af7296cd1058a5112890fed218488f084d8df9e4835fb54ad045ffd936e3bf7261b0426c51352a097816ed74482bb9084b4a7ed8adc517f3371e0e0434b511625cd1a41792243dccdcfe88094b64565479706a46616b65445454797065") + msg, _ = hex.DecodeString("a36449735271f46752657175657374f668526573706f6e7365a66454797065006441637074f46450617573f4665866657249441a4d65822164565265738178644204cb9a1e34c5f08e9b20aa76090e70020bb56c0ca3d3af7296cd1058a5112890fed218488f084d8df9e4835fb54ad045ffd936e3bf7261b0426c51352a097816ed74482bb9084b4a7ed8adc517f3371e0e0434b511625cd1a41792243dccdcfe88094b64565479706b54657374566f7563686572") deserialized, err = message1_1.FromNet(bytes.NewReader(msg)) require.NoError(t, err) @@ -505,7 +492,7 @@ func TestToNetFromNetEquivalency(t *testing.T) { require.Equal(t, deserializedResponse.IsRequest(), response.IsRequest()) require.Equal(t, deserializedResponse.IsUpdate(), response.IsUpdate()) require.Equal(t, deserializedResponse.IsPaused(), response.IsPaused()) - testutil.AssertEqualFakeDTVoucherResult(t, response, deserializedResponse) + testutil.AssertEqualTestVoucherResult(t, response, deserializedResponse) request = message1_1.CancelRequest(id) err = request.ToNet(buf) @@ -537,13 +524,13 @@ func TestFromNetMessageValidation(t *testing.T) { assert.Nil(t, msg) } -func NewTestTransferRequest() (message1_1.TransferRequest1_1, error) { +func NewTestTransferRequest(data string) (message1_1.TransferRequest1_1, error) { bcid := testutil.GenerateCids(1)[0] selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() isPull := false id := datatransfer.TransferID(rand.Int31()) - voucher := testutil.NewFakeDTType() - req, err := message1_1.NewRequest(id, false, isPull, voucher.Type(), voucher, bcid, selector) + voucher := testutil.NewTestTypedVoucherWith(data) + req, err := message1_1.NewRequest(id, false, isPull, &voucher, bcid, selector) if err != nil { return message1_1.TransferRequest1_1{}, err } diff --git a/message/message1_1prime/schema.go b/message/message1_1prime/schema.go deleted file mode 100644 index c779b1fc..00000000 --- a/message/message1_1prime/schema.go +++ /dev/null @@ -1,29 +0,0 @@ -package message1_1 - -import ( - _ "embed" - - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/node/bindnode" - "github.com/ipld/go-ipld-prime/schema" -) - -//go:embed schema.ipldsch -var embedSchema []byte - -var Prototype struct { - TransferMessage schema.TypedPrototype - TransferRequest schema.TypedPrototype - TransferResponse schema.TypedPrototype -} - -func init() { - ts, err := ipld.LoadSchemaBytes(embedSchema) - if err != nil { - panic(err) - } - - Prototype.TransferMessage = bindnode.Prototype((*TransferMessage1_1)(nil), ts.TypeByName("TransferMessage")) - Prototype.TransferRequest = bindnode.Prototype((*TransferRequest1_1)(nil), ts.TypeByName("TransferRequest")) - Prototype.TransferResponse = bindnode.Prototype((*TransferResponse1_1)(nil), ts.TypeByName("TransferResponse")) -} diff --git a/message/message1_1prime/schema.ipldsch b/message/message1_1prime/schema.ipldsch index 71413514..d5f9f87a 100644 --- a/message/message1_1prime/schema.ipldsch +++ b/message/message1_1prime/schema.ipldsch @@ -30,7 +30,7 @@ type TransferResponse struct { VoucherTypeIdentifier TypeIdentifier (rename "VTyp") } -type TransferMessage struct { +type TransferMessage1_1 struct { IsRequest Bool (rename "IsRq") Request nullable TransferRequest Response nullable TransferResponse diff --git a/message/message1_1prime/transfer_message.go b/message/message1_1prime/transfer_message.go index fc651220..0944212b 100644 --- a/message/message1_1prime/transfer_message.go +++ b/message/message1_1prime/transfer_message.go @@ -1,16 +1,19 @@ package message1_1 import ( + _ "embed" "io" - "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" - "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" datatransfer "github.com/filecoin-project/go-data-transfer/v2" + ipldutils "github.com/filecoin-project/go-data-transfer/v2/ipldutils" ) +//go:embed schema.ipldsch +var embedSchema []byte + // TransferMessage1_1 is the transfer message for the 1.1 Data Transfer Protocol. type TransferMessage1_1 struct { IsRequest bool @@ -19,6 +22,10 @@ type TransferMessage1_1 struct { Response *TransferResponse1_1 } +func (tm *TransferMessage1_1) BindnodeSchema() string { + return string(embedSchema) +} + // ========= datatransfer.Message interface // TransferID returns the TransferID of this message @@ -29,16 +36,24 @@ func (tm *TransferMessage1_1) TransferID() datatransfer.TransferID { return tm.Response.TransferID() } -func (tm *TransferMessage1_1) toIPLD() schema.TypedNode { - return bindnode.Wrap(&tm, Prototype.TransferMessage.Type()) +func (tm *TransferMessage1_1) toIPLD() (schema.TypedNode, error) { + return ipldutils.ToNode(tm) } -// ToNet serializes a transfer message type. +// ToIPLD converts a transfer message type to an ipld Node func (tm *TransferMessage1_1) ToIPLD() (datamodel.Node, error) { - return tm.toIPLD().Representation(), nil + node, err := tm.toIPLD() + if err != nil { + return nil, err + } + return node.Representation(), nil } // ToNet serializes a transfer message type. func (tm *TransferMessage1_1) ToNet(w io.Writer) error { - return dagcbor.Encode(tm.toIPLD().Representation(), w) + i, err := tm.toIPLD() + if err != nil { + return err + } + return ipldutils.NodeToWriter(i, w) } diff --git a/message/message1_1prime/transfer_request.go b/message/message1_1prime/transfer_request.go index 5d844d92..a01e2452 100644 --- a/message/message1_1prime/transfer_request.go +++ b/message/message1_1prime/transfer_request.go @@ -4,14 +4,13 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" "github.com/libp2p/go-libp2p-core/protocol" xerrors "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/encoding" + ipldutils "github.com/filecoin-project/go-data-transfer/v2/ipldutils" "github.com/filecoin-project/go-data-transfer/v2/message/types" ) @@ -23,8 +22,8 @@ type TransferRequest1_1 struct { Pause bool Partial bool Pull bool - SelectorPtr *datamodel.Node - VoucherPtr *datamodel.Node + SelectorPtr datamodel.Node + VoucherPtr datamodel.Node VoucherTypeIdentifier datatransfer.TypeIdentifier TransferId uint64 RestartChannel datatransfer.ChannelID @@ -91,11 +90,22 @@ func (trq *TransferRequest1_1) VoucherType() datatransfer.TypeIdentifier { } // Voucher returns the Voucher bytes -func (trq *TransferRequest1_1) Voucher(decoder encoding.Decoder) (encoding.Encodable, error) { +func (trq *TransferRequest1_1) Voucher() (datamodel.Node, error) { if trq.VoucherPtr == nil { return nil, xerrors.New("No voucher present to read") } - return decoder.DecodeFromNode(*trq.VoucherPtr) + return trq.VoucherPtr, nil +} + +// TypedVoucher is a convenience method that returns the voucher and its typed +// as a TypedVoucher object +// TODO(rvagg): tests for this +func (trq *TransferRequest1_1) TypedVoucher() (datatransfer.TypedVoucher, error) { + voucher, err := trq.Voucher() + if err != nil { + return datatransfer.TypedVoucher{}, err + } + return datatransfer.TypedVoucher{Voucher: voucher, Type: trq.VoucherType()}, nil } func (trq *TransferRequest1_1) EmptyVoucher() bool { @@ -115,7 +125,7 @@ func (trq *TransferRequest1_1) Selector() (datamodel.Node, error) { if trq.SelectorPtr == nil { return nil, xerrors.New("No selector present to read") } - return *trq.SelectorPtr, nil + return trq.SelectorPtr, nil } // IsCancel returns true if this is a cancel request @@ -128,20 +138,28 @@ func (trq *TransferRequest1_1) IsPartial() bool { return trq.Partial } -func (trsp *TransferRequest1_1) toIPLD() schema.TypedNode { +func (trq *TransferRequest1_1) toIPLD() (schema.TypedNode, error) { msg := TransferMessage1_1{ IsRequest: true, - Request: trsp, + Request: trq, Response: nil, } return msg.toIPLD() } func (trq *TransferRequest1_1) ToIPLD() (datamodel.Node, error) { - return trq.toIPLD().Representation(), nil + msg, err := trq.toIPLD() + if err != nil { + return nil, err + } + return msg.Representation(), nil } // ToNet serializes a transfer request. func (trq *TransferRequest1_1) ToNet(w io.Writer) error { - return dagcbor.Encode(trq.toIPLD().Representation(), w) + i, err := trq.toIPLD() + if err != nil { + return err + } + return ipldutils.NodeToWriter(i, w) } diff --git a/message/message1_1prime/transfer_request_test.go b/message/message1_1prime/transfer_request_test.go index 093653ed..2cff27e3 100644 --- a/message/message1_1prime/transfer_request_test.go +++ b/message/message1_1prime/transfer_request_test.go @@ -18,10 +18,10 @@ func TestRequestMessageForProtocol(t *testing.T) { selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() isPull := true id := datatransfer.TransferID(rand.Int31()) - voucher := testutil.NewFakeDTType() + voucher := testutil.NewTestTypedVoucher() // for the new protocols - request, err := message1_1.NewRequest(id, false, isPull, voucher.Type(), voucher, baseCid, selector) + request, err := message1_1.NewRequest(id, false, isPull, &voucher, baseCid, selector) require.NoError(t, err) out12, err := request.MessageForProtocol(datatransfer.ProtocolDataTransfer1_2) @@ -37,5 +37,5 @@ func TestRequestMessageForProtocol(t *testing.T) { n, err := req.Selector() require.NoError(t, err) require.Equal(t, selector, n) - require.Equal(t, voucher.Type(), req.VoucherType()) + require.Equal(t, testutil.TestVoucherType, req.VoucherType()) } diff --git a/message/message1_1prime/transfer_response.go b/message/message1_1prime/transfer_response.go index 32d240d6..1431ff72 100644 --- a/message/message1_1prime/transfer_response.go +++ b/message/message1_1prime/transfer_response.go @@ -3,14 +3,13 @@ package message1_1 import ( "io" - "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" "github.com/libp2p/go-libp2p-core/protocol" xerrors "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/encoding" + ipldutils "github.com/filecoin-project/go-data-transfer/v2/ipldutils" "github.com/filecoin-project/go-data-transfer/v2/message/types" ) @@ -21,7 +20,7 @@ type TransferResponse1_1 struct { RequestAccepted bool Paused bool TransferId uint64 - VoucherResultPtr *datamodel.Node + VoucherResultPtr datamodel.Node VoucherTypeIdentifier datatransfer.TypeIdentifier } @@ -73,11 +72,11 @@ func (trsp *TransferResponse1_1) VoucherResultType() datatransfer.TypeIdentifier return trsp.VoucherTypeIdentifier } -func (trsp *TransferResponse1_1) VoucherResult(decoder encoding.Decoder) (encoding.Encodable, error) { +func (trsp *TransferResponse1_1) VoucherResult() (datamodel.Node, error) { if trsp.VoucherResultPtr == nil { return nil, xerrors.New("No voucher present to read") } - return decoder.DecodeFromNode(*trsp.VoucherResultPtr) + return trsp.VoucherResultPtr, nil } func (trq *TransferResponse1_1) IsRestart() bool { @@ -97,7 +96,7 @@ func (trsp *TransferResponse1_1) MessageForProtocol(targetProtocol protocol.ID) } } -func (trsp *TransferResponse1_1) toIPLD() schema.TypedNode { +func (trsp *TransferResponse1_1) toIPLD() (schema.TypedNode, error) { msg := TransferMessage1_1{ IsRequest: false, Request: nil, @@ -107,10 +106,18 @@ func (trsp *TransferResponse1_1) toIPLD() schema.TypedNode { } func (trsp *TransferResponse1_1) ToIPLD() (datamodel.Node, error) { - return trsp.toIPLD().Representation(), nil + msg, err := trsp.toIPLD() + if err != nil { + return nil, err + } + return msg.Representation(), nil } // ToNet serializes a transfer response. func (trsp *TransferResponse1_1) ToNet(w io.Writer) error { - return dagcbor.Encode(trsp.toIPLD().Representation(), w) + i, err := trsp.toIPLD() + if err != nil { + return err + } + return ipldutils.NodeToWriter(i, w) } diff --git a/message/message1_1prime/transfer_response_test.go b/message/message1_1prime/transfer_response_test.go index dbaf2de0..9b979371 100644 --- a/message/message1_1prime/transfer_response_test.go +++ b/message/message1_1prime/transfer_response_test.go @@ -13,8 +13,8 @@ import ( func TestResponseMessageForProtocol(t *testing.T) { id := datatransfer.TransferID(rand.Int31()) - voucherResult := testutil.NewFakeDTType() - response, err := message1_1.NewResponse(id, false, true, voucherResult.Type(), voucherResult) // not accepted + voucherResult := testutil.NewTestTypedVoucher() + response, err := message1_1.NewResponse(id, false, true, &voucherResult) // not accepted require.NoError(t, err) // v1.2 protocol @@ -25,7 +25,7 @@ func TestResponseMessageForProtocol(t *testing.T) { resp, ok := (out).(datatransfer.Response) require.True(t, ok) require.True(t, resp.IsPaused()) - require.Equal(t, voucherResult.Type(), resp.VoucherResultType()) + require.Equal(t, testutil.TestVoucherType, resp.VoucherResultType()) require.True(t, resp.IsValidationResult()) // random protocol diff --git a/network/libp2p_impl_test.go b/network/libp2p_impl_test.go index 2747883c..646ba915 100644 --- a/network/libp2p_impl_test.go +++ b/network/libp2p_impl_test.go @@ -102,8 +102,8 @@ func TestMessageSendAndReceive(t *testing.T) { selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() isPull := false id := datatransfer.TransferID(rand.Int31()) - voucher := testutil.NewFakeDTType() - request, err := message.NewRequest(id, false, isPull, voucher.Type(), voucher, baseCid, selector) + voucher := testutil.NewTestTypedVoucher() + request, err := message.NewRequest(id, false, isPull, &voucher, baseCid, selector) require.NoError(t, err) require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) @@ -124,15 +124,15 @@ func TestMessageSendAndReceive(t *testing.T) { assert.Equal(t, request.IsPull(), receivedRequest.IsPull()) assert.Equal(t, request.IsRequest(), receivedRequest.IsRequest()) assert.True(t, receivedRequest.BaseCid().Equals(request.BaseCid())) - testutil.AssertEqualFakeDTVoucher(t, request, receivedRequest) + testutil.AssertEqualTestVoucher(t, request, receivedRequest) testutil.AssertEqualSelector(t, request, receivedRequest) }) t.Run("Send Response", func(t *testing.T) { accepted := false id := datatransfer.TransferID(rand.Int31()) - voucherResult := testutil.NewFakeDTType() - response, err := message.ValidationResultResponse(types.NewMessage, id, datatransfer.ValidationResult{Accepted: accepted, VoucherResult: voucherResult}, nil, false) + voucherResult := testutil.NewTestTypedVoucher() + response, err := message.ValidationResultResponse(types.NewMessage, id, datatransfer.ValidationResult{Accepted: accepted, VoucherResult: &voucherResult}, nil, false) require.NoError(t, err) require.NoError(t, dtnet2.SendMessage(ctx, host1.ID(), response)) @@ -151,7 +151,7 @@ func TestMessageSendAndReceive(t *testing.T) { assert.Equal(t, response.TransferID(), receivedResponse.TransferID()) assert.Equal(t, response.Accepted(), receivedResponse.Accepted()) assert.Equal(t, response.IsRequest(), receivedResponse.IsRequest()) - testutil.AssertEqualFakeDTVoucherResult(t, response, receivedResponse) + testutil.AssertEqualTestVoucherResult(t, response, receivedResponse) }) t.Run("Send Restart Request", func(t *testing.T) { @@ -273,8 +273,8 @@ func TestSendMessageRetry(t *testing.T) { selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() isPull := false id := datatransfer.TransferID(rand.Int31()) - voucher := testutil.NewFakeDTType() - request, err := message.NewRequest(id, false, isPull, voucher.Type(), voucher, baseCid, selector) + voucher := testutil.NewTestTypedVoucher() + request, err := message.NewRequest(id, false, isPull, &voucher, baseCid, selector) require.NoError(t, err) err = dtnet1.SendMessage(ctx, host2.ID(), request) diff --git a/registry/registry.go b/registry/registry.go index 4237cec7..00f3815d 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -6,7 +6,6 @@ import ( "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/encoding" ) // Processor is an interface that processes a certain type of encodable objects @@ -14,11 +13,6 @@ import ( // left to the user of the registry type Processor interface{} -type registryEntry struct { - decoder encoding.Decoder - processor Processor -} - // Registry maintans a register of types of encodable objects and a corresponding // processor for those objects // The encodable types must have a method Type() that specifies and identifier @@ -26,54 +20,41 @@ type registryEntry struct { // on this unique identifier type Registry struct { registryLk sync.RWMutex - entries map[datatransfer.TypeIdentifier]registryEntry + entries map[datatransfer.TypeIdentifier]Processor } // NewRegistry initialzes a new registy func NewRegistry() *Registry { return &Registry{ - entries: make(map[datatransfer.TypeIdentifier]registryEntry), + entries: make(map[datatransfer.TypeIdentifier]Processor), } } // Register registers the given processor for the given entry type -func (r *Registry) Register(entry datatransfer.Registerable, processor Processor) error { - identifier := entry.Type() - decoder, err := encoding.NewDecoder(entry) - if err != nil { - return xerrors.Errorf("registering entry type %s: %w", identifier, err) - } +func (r *Registry) Register(identifier datatransfer.TypeIdentifier, processor Processor) error { r.registryLk.Lock() defer r.registryLk.Unlock() if _, ok := r.entries[identifier]; ok { return xerrors.Errorf("identifier already registered: %s", identifier) } - r.entries[identifier] = registryEntry{decoder, processor} + r.entries[identifier] = processor return nil } -// Decoder gets a decoder for the given identifier -func (r *Registry) Decoder(identifier datatransfer.TypeIdentifier) (encoding.Decoder, bool) { - r.registryLk.RLock() - entry, has := r.entries[identifier] - r.registryLk.RUnlock() - return entry.decoder, has -} - // Processor gets the processing interface for the given identifer func (r *Registry) Processor(identifier datatransfer.TypeIdentifier) (Processor, bool) { r.registryLk.RLock() entry, has := r.entries[identifier] r.registryLk.RUnlock() - return entry.processor, has + return entry, has } // Each iterates through all of the entries in this registry -func (r *Registry) Each(process func(datatransfer.TypeIdentifier, encoding.Decoder, Processor) error) error { +func (r *Registry) Each(process func(datatransfer.TypeIdentifier, Processor) error) error { r.registryLk.RLock() defer r.registryLk.RUnlock() - for identifier, entry := range r.entries { - err := process(identifier, entry.decoder, entry.processor) + for identifier, processor := range r.entries { + err := process(identifier, processor) if err != nil { return err } diff --git a/registry/registry_test.go b/registry/registry_test.go index 63adf363..84fd95c8 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -12,27 +12,15 @@ import ( func TestRegistry(t *testing.T) { r := registry.NewRegistry() t.Run("it registers", func(t *testing.T) { - err := r.Register(&testutil.FakeDTType{}, func() {}) + err := r.Register(testutil.TestVoucherType, func() {}) require.NoError(t, err) }) t.Run("it errors when registred again", func(t *testing.T) { - err := r.Register(&testutil.FakeDTType{}, func() {}) - require.EqualError(t, err, "identifier already registered: FakeDTType") - }) - t.Run("it errors when decoder setup fails", func(t *testing.T) { - err := r.Register(testutil.FakeDTType{}, func() {}) - require.EqualError(t, err, "registering entry type FakeDTType: type must be a pointer") - }) - t.Run("it reads decoders", func(t *testing.T) { - decoder, has := r.Decoder("FakeDTType") - require.True(t, has) - require.NotNil(t, decoder) - decoder, has = r.Decoder("OtherType") - require.False(t, has) - require.Nil(t, decoder) + err := r.Register(testutil.TestVoucherType, func() {}) + require.EqualError(t, err, "identifier already registered: TestVoucher") }) t.Run("it reads processors", func(t *testing.T) { - processor, has := r.Processor("FakeDTType") + processor, has := r.Processor("TestVoucher") require.True(t, has) require.NotNil(t, processor) processor, has = r.Processor("OtherType") diff --git a/testutil/fakedttype.go b/testutil/fakedttype.go index 20cc219e..03db9273 100644 --- a/testutil/fakedttype.go +++ b/testutil/fakedttype.go @@ -3,71 +3,80 @@ package testutil import ( "testing" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/fluent/qp" + basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/stretchr/testify/require" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/encoding" ) -//go:generate cbor-gen-for FakeDTType +const TestVoucherType = datatransfer.TypeIdentifier("TestVoucher") -// FakeDTType simple fake type for using with registries -type FakeDTType struct { - Data string -} - -// Type satisfies registry.Entry -func (ft FakeDTType) Type() datatransfer.TypeIdentifier { - return "FakeDTType" -} - -// AssertFakeDTVoucher asserts that a data transfer requests contains the expected fake data transfer voucher type -func AssertFakeDTVoucher(t *testing.T, request datatransfer.Request, expected *FakeDTType) { - require.Equal(t, datatransfer.TypeIdentifier("FakeDTType"), request.VoucherType()) - fakeDTDecoder, err := encoding.NewDecoder(&FakeDTType{}) +// AssertTestVoucher asserts that a data transfer requests contains the expected fake data transfer voucher type +func AssertTestVoucher(t *testing.T, request datatransfer.Request, expected datatransfer.TypedVoucher) { + require.Equal(t, expected.Type, request.VoucherType()) + voucher, err := request.Voucher() require.NoError(t, err) - decoded, err := request.Voucher(fakeDTDecoder) - require.NoError(t, err) - require.Equal(t, expected, decoded) + require.True(t, ipld.DeepEqual(expected.Voucher, voucher)) } -// AssertEqualFakeDTVoucher asserts that two requests have the same fake data transfer voucher -func AssertEqualFakeDTVoucher(t *testing.T, expectedRequest datatransfer.Request, request datatransfer.Request) { +// AssertEqualTestVoucher asserts that two requests have the same fake data transfer voucher +func AssertEqualTestVoucher(t *testing.T, expectedRequest datatransfer.Request, request datatransfer.Request) { require.Equal(t, expectedRequest.VoucherType(), request.VoucherType()) - fakeDTDecoder, err := encoding.NewDecoder(&FakeDTType{}) - require.NoError(t, err) - expectedDecoded, err := request.Voucher(fakeDTDecoder) + require.Equal(t, TestVoucherType, request.VoucherType()) + expected, err := expectedRequest.Voucher() require.NoError(t, err) - decoded, err := request.Voucher(fakeDTDecoder) + actual, err := request.Voucher() require.NoError(t, err) - require.Equal(t, expectedDecoded, decoded) + require.True(t, ipld.DeepEqual(expected, actual)) } -// AssertFakeDTVoucherResult asserts that a data transfer response contains the expected fake data transfer voucher result type -func AssertFakeDTVoucherResult(t *testing.T, response datatransfer.Response, expected *FakeDTType) { - require.Equal(t, datatransfer.TypeIdentifier("FakeDTType"), response.VoucherResultType()) - fakeDTDecoder, err := encoding.NewDecoder(&FakeDTType{}) - require.NoError(t, err) - decoded, err := response.VoucherResult(fakeDTDecoder) +// AssertTestVoucherResult asserts that a data transfer response contains the expected fake data transfer voucher result type +func AssertTestVoucherResult(t *testing.T, response datatransfer.Response, expected datatransfer.TypedVoucher) { + require.Equal(t, expected.Type, response.VoucherResultType()) + voucherResult, err := response.VoucherResult() require.NoError(t, err) - require.Equal(t, expected, decoded) + require.True(t, ipld.DeepEqual(expected.Voucher, voucherResult)) } -// AssertEqualFakeDTVoucherResult asserts that two responses have the same fake data transfer voucher result -func AssertEqualFakeDTVoucherResult(t *testing.T, expectedResponse datatransfer.Response, response datatransfer.Response) { +// AssertEqualTestVoucherResult asserts that two responses have the same fake data transfer voucher result +func AssertEqualTestVoucherResult(t *testing.T, expectedResponse datatransfer.Response, response datatransfer.Response) { require.Equal(t, expectedResponse.VoucherResultType(), response.VoucherResultType()) - fakeDTDecoder, err := encoding.NewDecoder(&FakeDTType{}) + expectedVoucherResult, err := expectedResponse.VoucherResult() require.NoError(t, err) - expectedDecoded, err := response.VoucherResult(fakeDTDecoder) + actualVoucherResult, err := response.VoucherResult() require.NoError(t, err) - decoded, err := response.VoucherResult(fakeDTDecoder) - require.NoError(t, err) - require.Equal(t, expectedDecoded, decoded) + require.True(t, ipld.DeepEqual(expectedVoucherResult, actualVoucherResult)) +} + +// NewTestVoucher returns a fake voucher with random data +func NewTestVoucher() datamodel.Node { + n, err := qp.BuildList(basicnode.Prototype.Any, 1, func(ma datamodel.ListAssembler) { + qp.ListEntry(ma, qp.String(string(RandomBytes(100)))) + }) + if err != nil { + panic(err) + } + return n } -// NewFakeDTType returns a fake dt type with random data -func NewFakeDTType() *FakeDTType { - return &FakeDTType{Data: string(RandomBytes(100))} +func NewTestTypedVoucher() datatransfer.TypedVoucher { + return datatransfer.TypedVoucher{Voucher: NewTestVoucher(), Type: TestVoucherType} } -var _ datatransfer.Registerable = &FakeDTType{} +// NewTestVoucher returns a fake voucher with random data +func NewTestVoucherWith(data string) datamodel.Node { + n, err := qp.BuildList(basicnode.Prototype.Any, 1, func(ma datamodel.ListAssembler) { + qp.ListEntry(ma, qp.String(data)) + }) + if err != nil { + panic(err) + } + return n +} + +func NewTestTypedVoucherWith(data string) datatransfer.TypedVoucher { + return datatransfer.TypedVoucher{Voucher: NewTestVoucherWith(data), Type: TestVoucherType} +} diff --git a/testutil/fakedttype_cbor_gen.go b/testutil/fakedttype_cbor_gen.go deleted file mode 100644 index d7913605..00000000 --- a/testutil/fakedttype_cbor_gen.go +++ /dev/null @@ -1,75 +0,0 @@ -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -package testutil - -import ( - "fmt" - "io" - "sort" - - cid "github.com/ipfs/go-cid" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -var _ = xerrors.Errorf -var _ = cid.Undef -var _ = sort.Sort - -var lengthBufFakeDTType = []byte{129} - -func (t *FakeDTType) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write(lengthBufFakeDTType); err != nil { - return err - } - - scratch := make([]byte, 9) - - // t.Data (string) (string) - if len(t.Data) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Data was too long") - } - - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Data))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Data)); err != nil { - return err - } - return nil -} - -func (t *FakeDTType) UnmarshalCBOR(r io.Reader) error { - *t = FakeDTType{} - - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) - - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Data (string) (string) - - { - sval, err := cbg.ReadStringBuf(br, scratch) - if err != nil { - return err - } - - t.Data = string(sval) - } - return nil -} diff --git a/testutil/fakegraphsync.go b/testutil/fakegraphsync.go index 42b30873..f0ac8ab0 100644 --- a/testutil/fakegraphsync.go +++ b/testutil/fakegraphsync.go @@ -40,7 +40,7 @@ type ReceivedGraphSyncRequest struct { Ctx context.Context P peer.ID Root ipld.Link - Selector ipld.Node + Selector datamodel.Node Extensions []graphsync.ExtensionData ResponseChan chan graphsync.ResponseProgress ResponseErrChan chan error @@ -193,7 +193,7 @@ func (fgs *FakeGraphSync) AssertDoesNotHavePersistenceOption(t *testing.T, name } // Request initiates a new GraphSync request to the given peer using the given selector spec. -func (fgs *FakeGraphSync) Request(ctx context.Context, p peer.ID, root ipld.Link, selector ipld.Node, extensions ...graphsync.ExtensionData) (<-chan graphsync.ResponseProgress, <-chan error) { +func (fgs *FakeGraphSync) Request(ctx context.Context, p peer.ID, root ipld.Link, selector datamodel.Node, extensions ...graphsync.ExtensionData) (<-chan graphsync.ResponseProgress, <-chan error) { errors := make(chan error) responses := make(chan graphsync.ResponseProgress) fgs.requests <- ReceivedGraphSyncRequest{ctx, p, root, selector, extensions, responses, errors} @@ -387,7 +387,7 @@ func NewFakeBlockData() graphsync.BlockData { type fakeRequest struct { id graphsync.RequestID root cid.Cid - selector ipld.Node + selector datamodel.Node priority graphsync.Priority requestType graphsync.RequestType extensions map[graphsync.ExtensionName]datamodel.Node @@ -404,7 +404,7 @@ func (fr *fakeRequest) Root() cid.Cid { } // Selector returns the byte representation of the selector for this request -func (fr *fakeRequest) Selector() ipld.Node { +func (fr *fakeRequest) Selector() datamodel.Node { return fr.selector } diff --git a/testutil/faketransport.go b/testutil/faketransport.go index 92441364..08faee74 100644 --- a/testutil/faketransport.go +++ b/testutil/faketransport.go @@ -4,6 +4,7 @@ import ( "context" "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" "github.com/libp2p/go-libp2p-core/peer" datatransfer "github.com/filecoin-project/go-data-transfer/v2" @@ -14,7 +15,7 @@ type OpenedChannel struct { DataSender peer.ID ChannelID datatransfer.ChannelID Root ipld.Link - Selector ipld.Node + Selector datamodel.Node Channel datatransfer.ChannelState Message datatransfer.Message } @@ -28,7 +29,7 @@ type ResumedChannel struct { // CustomizedTransfer is just a way to record calls made to transport configurer type CustomizedTransfer struct { ChannelID datatransfer.ChannelID - Voucher datatransfer.Voucher + Voucher datatransfer.TypedVoucher } // FakeTransport is a fake transport with mocked results @@ -57,7 +58,7 @@ func NewFakeTransport() *FakeTransport { // Note: from a data transfer symantic standpoint, it doesn't matter if the // request is push or pull -- OpenChannel is called by the party that is // intending to receive data -func (ft *FakeTransport) OpenChannel(ctx context.Context, dataSender peer.ID, channelID datatransfer.ChannelID, root ipld.Link, stor ipld.Node, channel datatransfer.ChannelState, msg datatransfer.Message) error { +func (ft *FakeTransport) OpenChannel(ctx context.Context, dataSender peer.ID, channelID datatransfer.ChannelID, root ipld.Link, stor datamodel.Node, channel datatransfer.ChannelState, msg datatransfer.Message) error { ft.OpenedChannels = append(ft.OpenedChannels, OpenedChannel{dataSender, channelID, root, stor, channel, msg}) return ft.OpenChannelErr } @@ -95,6 +96,6 @@ func (ft *FakeTransport) CleanupChannel(chid datatransfer.ChannelID) { ft.CleanedUpChannels = append(ft.CleanedUpChannels, chid) } -func (ft *FakeTransport) RecordCustomizedTransfer(chid datatransfer.ChannelID, voucher datatransfer.Voucher) { +func (ft *FakeTransport) RecordCustomizedTransfer(chid datatransfer.ChannelID, voucher datatransfer.TypedVoucher) { ft.CustomizedTransfers = append(ft.CustomizedTransfers, CustomizedTransfer{chid, voucher}) } diff --git a/testutil/gstestdata.go b/testutil/gstestdata.go index ea7c1f69..8f40178c 100644 --- a/testutil/gstestdata.go +++ b/testutil/gstestdata.go @@ -29,6 +29,7 @@ import ( "github.com/ipfs/go-unixfs/importer/balanced" ihelper "github.com/ipfs/go-unixfs/importer/helpers" "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/ipld/go-ipld-prime/traversal/selector" @@ -44,7 +45,7 @@ import ( "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" ) -var allSelector ipld.Node +var allSelector datamodel.Node const loremFile = "lorem.txt" @@ -82,7 +83,7 @@ type GraphsyncTestingData struct { GsNet2 gsnet.GraphSyncNetwork DtNet1 network.DataTransferNetwork DtNet2 network.DataTransferNetwork - AllSelector ipld.Node + AllSelector datamodel.Node OrigBytes []byte TempDir1 string TempDir2 string diff --git a/testutil/message.go b/testutil/message.go index c3745be9..14319cc3 100644 --- a/testutil/message.go +++ b/testutil/message.go @@ -13,18 +13,18 @@ import ( // NewDTRequest makes a new DT Request message func NewDTRequest(t *testing.T, transferID datatransfer.TransferID) datatransfer.Request { - voucher := NewFakeDTType() + voucher := NewTestTypedVoucher() baseCid := GenerateCids(1)[0] selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() - r, err := message.NewRequest(transferID, false, false, voucher.Type(), voucher, baseCid, selector) + r, err := message.NewRequest(transferID, false, false, &voucher, baseCid, selector) require.NoError(t, err) return r } // NewDTResponse makes a new DT Request message func NewDTResponse(t *testing.T, transferID datatransfer.TransferID) datatransfer.Response { - vresult := NewFakeDTType() - r, err := message.NewResponse(transferID, false, false, vresult.Type(), vresult) + vresult := NewTestTypedVoucher() + r, err := message.NewResponse(transferID, false, false, &vresult) require.NoError(t, err) return r } diff --git a/testutil/mockchannelstate.go b/testutil/mockchannelstate.go index a9a86586..a5754bde 100644 --- a/testutil/mockchannelstate.go +++ b/testutil/mockchannelstate.go @@ -2,7 +2,7 @@ package testutil import ( cid "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" "github.com/libp2p/go-libp2p-core/peer" datatransfer "github.com/filecoin-project/go-data-transfer/v2" @@ -105,11 +105,11 @@ func (m *MockChannelState) BaseCID() cid.Cid { panic("implement me") } -func (m *MockChannelState) Selector() ipld.Node { +func (m *MockChannelState) Selector() datamodel.Node { panic("implement me") } -func (m *MockChannelState) Voucher() datatransfer.Voucher { +func (m *MockChannelState) Voucher() (datatransfer.TypedVoucher, error) { panic("implement me") } @@ -141,19 +141,19 @@ func (m *MockChannelState) Message() string { panic("implement me") } -func (m *MockChannelState) Vouchers() []datatransfer.Voucher { +func (m *MockChannelState) Vouchers() ([]datatransfer.TypedVoucher, error) { panic("implement me") } -func (m *MockChannelState) VoucherResults() []datatransfer.VoucherResult { +func (m *MockChannelState) VoucherResults() ([]datatransfer.TypedVoucher, error) { panic("implement me") } -func (m *MockChannelState) LastVoucher() datatransfer.Voucher { +func (m *MockChannelState) LastVoucher() (datatransfer.TypedVoucher, error) { panic("implement me") } -func (m *MockChannelState) LastVoucherResult() datatransfer.VoucherResult { +func (m *MockChannelState) LastVoucherResult() (datatransfer.TypedVoucher, error) { panic("implement me") } diff --git a/testutil/stubbedvalidator.go b/testutil/stubbedvalidator.go index 30a8bab7..1647bdc0 100644 --- a/testutil/stubbedvalidator.go +++ b/testutil/stubbedvalidator.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" @@ -21,9 +21,9 @@ func NewStubbedValidator() *StubbedValidator { func (sv *StubbedValidator) ValidatePush( chid datatransfer.ChannelID, sender peer.ID, - voucher datatransfer.Voucher, + voucher datamodel.Node, baseCid cid.Cid, - selector ipld.Node) (datatransfer.ValidationResult, error) { + selector datamodel.Node) (datatransfer.ValidationResult, error) { sv.didPush = true sv.ValidationsReceived = append(sv.ValidationsReceived, ReceivedValidation{false, sender, voucher, baseCid, selector}) return sv.result, sv.pushError @@ -33,9 +33,9 @@ func (sv *StubbedValidator) ValidatePush( func (sv *StubbedValidator) ValidatePull( chid datatransfer.ChannelID, receiver peer.ID, - voucher datatransfer.Voucher, + voucher datamodel.Node, baseCid cid.Cid, - selector ipld.Node) (datatransfer.ValidationResult, error) { + selector datamodel.Node) (datatransfer.ValidationResult, error) { sv.didPull = true sv.ValidationsReceived = append(sv.ValidationsReceived, ReceivedValidation{true, receiver, voucher, baseCid, selector}) return sv.result, sv.pullError @@ -140,9 +140,9 @@ func (sv *StubbedValidator) ExpectSuccessValidateRestart() { type ReceivedValidation struct { IsPull bool Other peer.ID - Voucher datatransfer.Voucher + Voucher datamodel.Node BaseCid cid.Cid - Selector ipld.Node + Selector datamodel.Node } // ReceivedRestartValidation records a call to ValidateRestart diff --git a/testutil/testutil.go b/testutil/testutil.go index 66600730..cfcde8b5 100644 --- a/testutil/testutil.go +++ b/testutil/testutil.go @@ -9,7 +9,7 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" blocksutil "github.com/ipfs/go-ipfs-blocksutil" - "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" @@ -102,7 +102,7 @@ func AssertEqualSelector(t *testing.T, expectedRequest datatransfer.Request, req } // AllSelector just returns a new instance of a "whole dag selector" -func AllSelector() ipld.Node { +func AllSelector() datamodel.Node { ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) return ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() diff --git a/transport.go b/transport.go index 81187077..ca002a28 100644 --- a/transport.go +++ b/transport.go @@ -4,6 +4,7 @@ import ( "context" ipld "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" peer "github.com/libp2p/go-libp2p-core/peer" ) @@ -98,7 +99,7 @@ type Transport interface { dataSender peer.ID, channelID ChannelID, root ipld.Link, - stor ipld.Node, + stor datamodel.Node, channel ChannelState, msg Message, ) error diff --git a/transport/graphsync/graphsync.go b/transport/graphsync/graphsync.go index 2a78a003..b2e27f14 100644 --- a/transport/graphsync/graphsync.go +++ b/transport/graphsync/graphsync.go @@ -11,6 +11,7 @@ import ( "github.com/ipfs/go-graphsync/donotsendfirstblocks" logging "github.com/ipfs/go-log/v2" ipld "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" peer "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/sync/errgroup" "golang.org/x/xerrors" @@ -111,7 +112,7 @@ func (t *Transport) OpenChannel( dataSender peer.ID, channelID datatransfer.ChannelID, root ipld.Link, - stor ipld.Node, + stor datamodel.Node, channel datatransfer.ChannelState, msg datatransfer.Message, ) error { @@ -937,7 +938,7 @@ func (c *dtChannel) open( chid datatransfer.ChannelID, dataSender peer.ID, root ipld.Link, - stor ipld.Node, + stor datamodel.Node, channel datatransfer.ChannelState, exts []graphsync.ExtensionData, ) (*gsReq, error) { diff --git a/types.go b/types.go index 9201c5aa..b15d4cde 100644 --- a/types.go +++ b/types.go @@ -6,10 +6,9 @@ import ( "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" - - "github.com/filecoin-project/go-data-transfer/v2/encoding" ) //go:generate cbor-gen-for ChannelID ChannelStages ChannelStage Log @@ -21,23 +20,18 @@ type TypeIdentifier string // EmptyTypeIdentifier means there is no voucher present const EmptyTypeIdentifier = TypeIdentifier("") -// Registerable is a type of object in a registry. It must be encodable and must -// have a single method that uniquely identifies its type -type Registerable interface { - encoding.Encodable - // Type is a unique string identifier for this voucher type - Type() TypeIdentifier +// TypedVoucher is a voucher or voucher result in IPLD form and an associated +// type identifier for that voucher or voucher result +type TypedVoucher struct { + Voucher datamodel.Node + Type TypeIdentifier } -// Voucher is used to validate -// a data transfer request against the underlying storage or retrieval deal -// that precipitated it. The only requirement is a voucher can read and write -// from bytes, and has a string identifier type -type Voucher Registerable - -// VoucherResult is used to provide option additional information about a -// voucher being rejected or accepted -type VoucherResult Registerable +// Equals is a utility to compare that two TypedVouchers are the same - both type +// and the voucher's IPLD content +func (tv1 TypedVoucher) Equals(tv2 TypedVoucher) bool { + return tv1.Type == tv2.Type && ipld.DeepEqual(tv1.Voucher, tv2.Voucher) +} // TransferID is an identifier for a data transfer, shared between // request/responder and unique to the requester @@ -74,10 +68,10 @@ type Channel interface { // Selector returns the IPLD selector for this data transfer (represented as // an IPLD node) - Selector() ipld.Node + Selector() datamodel.Node - // Voucher returns the voucher for this data transfer - Voucher() Voucher + // Voucher returns the initial voucher for this data transfer + Voucher() (TypedVoucher, error) // Sender returns the peer id for the node that is sending data Sender() peer.ID @@ -118,16 +112,16 @@ type ChannelState interface { Message() string // Vouchers returns all vouchers sent on this channel - Vouchers() []Voucher + Vouchers() ([]TypedVoucher, error) // VoucherResults are results of vouchers sent on the channel - VoucherResults() []VoucherResult + VoucherResults() ([]TypedVoucher, error) // LastVoucher returns the last voucher sent on the channel - LastVoucher() Voucher + LastVoucher() (TypedVoucher, error) // LastVoucherResult returns the last voucher result sent on the channel - LastVoucherResult() VoucherResult + LastVoucherResult() (TypedVoucher, error) // ReceivedCidsTotal returns the number of (non-unique) cids received so far // on the channel - note that a block can exist in more than one place in the DAG From c833220a2843ae04bdb8a27f418b3f8df326e0a5 Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Wed, 15 Jun 2022 17:40:26 -0700 Subject: [PATCH 06/18] refactor(channelstate): use cborgencompatiblenode (#336) use cborgencomptaiblenode to simply channelstate interface --- channels/channel_state.go | 61 +++++-------------- channels/channels.go | 28 ++------- channels/channels_fsm.go | 9 ++- channels/channels_test.go | 30 +++------ channels/internal/internalchannel.go | 48 ++++++++++++++- channels/internal/internalchannel_cbor_gen.go | 27 ++++---- impl/integration_test.go | 7 +-- impl/receiving_requests.go | 8 +-- impl/restart.go | 15 +---- testutil/mockchannelstate.go | 10 +-- types.go | 10 +-- 11 files changed, 111 insertions(+), 142 deletions(-) diff --git a/channels/channel_state.go b/channels/channel_state.go index 31c18d4b..fec337ba 100644 --- a/channels/channel_state.go +++ b/channels/channel_state.go @@ -1,17 +1,12 @@ package channels import ( - "bytes" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" - basicnode "github.com/ipld/go-ipld-prime/node/basic" peer "github.com/libp2p/go-libp2p-core/peer" datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channels/internal" - ipldutils "github.com/filecoin-project/go-data-transfer/v2/ipldutils" ) // channelState is immutable channel data plus mutable state @@ -43,25 +38,16 @@ func (c channelState) BaseCID() cid.Cid { return c.ic.BaseCid } // Selector returns the IPLD selector for this data transfer (represented as // an IPLD node) func (c channelState) Selector() datamodel.Node { - builder := basicnode.Prototype.Any.NewBuilder() - reader := bytes.NewReader(c.ic.Selector.Raw) - err := dagcbor.Decode(builder, reader) - if err != nil { - log.Error(err) - } - return builder.Build() + return c.ic.Selector.Node } // Voucher returns the voucher for this data transfer -func (c channelState) Voucher() (datatransfer.TypedVoucher, error) { +func (c channelState) Voucher() datatransfer.TypedVoucher { if len(c.ic.Vouchers) == 0 { - return datatransfer.TypedVoucher{}, nil - } - node, err := ipldutils.DeferredToNode(c.ic.Vouchers[0].Voucher) - if err != nil { - return datatransfer.TypedVoucher{}, err + return datatransfer.TypedVoucher{} } - return datatransfer.TypedVoucher{Voucher: node, Type: c.ic.Vouchers[0].Type}, nil + ev := c.ic.Vouchers[0] + return datatransfer.TypedVoucher{Voucher: ev.Voucher.Node, Type: ev.Type} } // ReceivedCidsTotal returns the number of (non-unique) cids received so far @@ -107,46 +93,31 @@ func (c channelState) Message() string { return c.ic.Message } -func (c channelState) Vouchers() ([]datatransfer.TypedVoucher, error) { +func (c channelState) Vouchers() []datatransfer.TypedVoucher { vouchers := make([]datatransfer.TypedVoucher, 0, len(c.ic.Vouchers)) for _, encoded := range c.ic.Vouchers { - node, err := ipldutils.DeferredToNode(encoded.Voucher) - if err != nil { - return nil, err - } - vouchers = append(vouchers, datatransfer.TypedVoucher{Voucher: node, Type: encoded.Type}) + vouchers = append(vouchers, datatransfer.TypedVoucher{Voucher: encoded.Voucher.Node, Type: encoded.Type}) } - return vouchers, nil + return vouchers } -func (c channelState) LastVoucher() (datatransfer.TypedVoucher, error) { +func (c channelState) LastVoucher() datatransfer.TypedVoucher { ev := c.ic.Vouchers[len(c.ic.Vouchers)-1] - node, err := ipldutils.DeferredToNode(ev.Voucher) - if err != nil { - return datatransfer.TypedVoucher{}, err - } - return datatransfer.TypedVoucher{Voucher: node, Type: ev.Type}, nil + + return datatransfer.TypedVoucher{Voucher: ev.Voucher.Node, Type: ev.Type} } -func (c channelState) LastVoucherResult() (datatransfer.TypedVoucher, error) { +func (c channelState) LastVoucherResult() datatransfer.TypedVoucher { evr := c.ic.VoucherResults[len(c.ic.VoucherResults)-1] - node, err := ipldutils.DeferredToNode(evr.VoucherResult) - if err != nil { - return datatransfer.TypedVoucher{}, err - } - return datatransfer.TypedVoucher{Voucher: node, Type: evr.Type}, nil + return datatransfer.TypedVoucher{Voucher: evr.VoucherResult.Node, Type: evr.Type} } -func (c channelState) VoucherResults() ([]datatransfer.TypedVoucher, error) { +func (c channelState) VoucherResults() []datatransfer.TypedVoucher { voucherResults := make([]datatransfer.TypedVoucher, 0, len(c.ic.VoucherResults)) for _, encoded := range c.ic.VoucherResults { - node, err := ipldutils.DeferredToNode(encoded.VoucherResult) - if err != nil { - return nil, err - } - voucherResults = append(voucherResults, datatransfer.TypedVoucher{Voucher: node, Type: encoded.Type}) + voucherResults = append(voucherResults, datatransfer.TypedVoucher{Voucher: encoded.VoucherResult.Node, Type: encoded.Type}) } - return voucherResults, nil + return voucherResults } func (c channelState) SelfPeer() peer.ID { diff --git a/channels/channels.go b/channels/channels.go index 4c7f1572..63a36d9b 100644 --- a/channels/channels.go +++ b/channels/channels.go @@ -9,7 +9,6 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipld/go-ipld-prime/datamodel" peer "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" versioning "github.com/filecoin-project/go-ds-versioning/pkg" @@ -20,7 +19,6 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channels/internal" "github.com/filecoin-project/go-data-transfer/v2/channels/internal/migrations" - ipldutils "github.com/filecoin-project/go-data-transfer/v2/ipldutils" ) type Notifier func(datatransfer.Event, datatransfer.ChannelState) @@ -119,28 +117,20 @@ func (c *Channels) CreateNew(selfPeer peer.ID, tid datatransfer.TransferID, base responder = dataSender } chid := datatransfer.ChannelID{Initiator: initiator, Responder: responder, ID: tid} - initialVoucher, err := ipldutils.NodeToDeferred(voucher.Voucher) - if err != nil { - return datatransfer.ChannelID{}, err - } - selBytes, err := ipldutils.NodeToBytes(selector) - if err != nil { - return datatransfer.ChannelID{}, err - } - err = c.stateMachines.Begin(chid, &internal.ChannelState{ + err := c.stateMachines.Begin(chid, &internal.ChannelState{ SelfPeer: selfPeer, TransferID: tid, Initiator: initiator, Responder: responder, BaseCid: baseCid, - Selector: &cbg.Deferred{Raw: selBytes}, + Selector: internal.CborGenCompatibleNode{Node: selector}, Sender: dataSender, Recipient: dataReceiver, Stages: &datatransfer.ChannelStages{}, Vouchers: []internal.EncodedVoucher{ { Type: voucher.Type, - Voucher: initialVoucher, + Voucher: internal.CborGenCompatibleNode{voucher.Voucher}, }, }, Status: datatransfer.Requested, @@ -278,20 +268,12 @@ func (c *Channels) ResumeResponder(chid datatransfer.ChannelID) error { // NewVoucher records a new voucher for this channel func (c *Channels) NewVoucher(chid datatransfer.ChannelID, voucher datatransfer.TypedVoucher) error { - voucherBytes, err := ipldutils.NodeToBytes(voucher.Voucher) - if err != nil { - return err - } - return c.send(chid, datatransfer.NewVoucher, voucher.Type, voucherBytes) + return c.send(chid, datatransfer.NewVoucher, voucher) } // NewVoucherResult records a new voucher result for this channel func (c *Channels) NewVoucherResult(chid datatransfer.ChannelID, voucherResult datatransfer.TypedVoucher) error { - voucherResultBytes, err := ipldutils.NodeToBytes(voucherResult.Voucher) - if err != nil { - return err - } - return c.send(chid, datatransfer.NewVoucherResult, voucherResult.Type, voucherResultBytes) + return c.send(chid, datatransfer.NewVoucherResult, voucherResult) } // Complete indicates responder has completed sending/receiving data diff --git a/channels/channels_fsm.go b/channels/channels_fsm.go index 12e2cf0b..61a844a5 100644 --- a/channels/channels_fsm.go +++ b/channels/channels_fsm.go @@ -2,7 +2,6 @@ package channels import ( logging "github.com/ipfs/go-log/v2" - cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/go-statemachine/fsm" @@ -148,15 +147,15 @@ var ChannelEvents = fsm.Events{ }), fsm.Event(datatransfer.NewVoucher).FromAny().ToNoChange(). - Action(func(chst *internal.ChannelState, vtype datatransfer.TypeIdentifier, voucherBytes []byte) error { - chst.Vouchers = append(chst.Vouchers, internal.EncodedVoucher{Type: vtype, Voucher: &cbg.Deferred{Raw: voucherBytes}}) + Action(func(chst *internal.ChannelState, voucher datatransfer.TypedVoucher) error { + chst.Vouchers = append(chst.Vouchers, internal.EncodedVoucher{Type: voucher.Type, Voucher: internal.CborGenCompatibleNode{Node: voucher.Voucher}}) chst.AddLog("got new voucher") return nil }), fsm.Event(datatransfer.NewVoucherResult).FromAny().ToNoChange(). - Action(func(chst *internal.ChannelState, vtype datatransfer.TypeIdentifier, voucherResultBytes []byte) error { + Action(func(chst *internal.ChannelState, voucherResult datatransfer.TypedVoucher) error { chst.VoucherResults = append(chst.VoucherResults, - internal.EncodedVoucherResult{Type: vtype, VoucherResult: &cbg.Deferred{Raw: voucherResultBytes}}) + internal.EncodedVoucherResult{Type: voucherResult.Type, VoucherResult: internal.CborGenCompatibleNode{Node: voucherResult.Voucher}}) chst.AddLog("got new voucher result") return nil }), diff --git a/channels/channels_test.go b/channels/channels_test.go index 505afa34..2d4c39c3 100644 --- a/channels/channels_test.go +++ b/channels/channels_test.go @@ -79,8 +79,7 @@ func TestChannels(t *testing.T) { require.NotEqual(t, channels.EmptyChannelState, state) require.Equal(t, cids[0], state.BaseCID()) require.Equal(t, selector, state.Selector()) - voucher, err := state.Voucher() - require.NoError(t, err) + voucher := state.Voucher() require.True(t, fv1.Equals(voucher)) require.Equal(t, peers[0], state.Sender()) require.Equal(t, peers[1], state.Recipient()) @@ -303,47 +302,38 @@ func TestChannels(t *testing.T) { state, err := channelList.GetByID(ctx, datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) require.NoError(t, err) - vouchers, err := state.Vouchers() - require.NoError(t, err) + vouchers := state.Vouchers() require.Len(t, vouchers, 1) require.True(t, fv1.Equals(vouchers[0])) - voucher, err := state.Voucher() - require.NoError(t, err) + voucher := state.Voucher() require.True(t, fv1.Equals(voucher)) - voucher, err = state.LastVoucher() - require.NoError(t, err) + voucher = state.LastVoucher() require.True(t, fv1.Equals(voucher)) err = channelList.NewVoucher(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, fv3) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.NewVoucher) - vouchers, err = state.Vouchers() - require.NoError(t, err) + vouchers = state.Vouchers() require.Len(t, vouchers, 2) require.True(t, fv1.Equals(vouchers[0])) require.True(t, fv3.Equals(vouchers[1])) - voucher, err = state.Voucher() - require.NoError(t, err) + voucher = state.Voucher() require.True(t, fv1.Equals(voucher)) - voucher, err = state.LastVoucher() - require.NoError(t, err) + voucher = state.LastVoucher() require.True(t, fv3.Equals(voucher)) state, err = channelList.GetByID(ctx, datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) require.NoError(t, err) - results, err := state.VoucherResults() - require.NoError(t, err) + results := state.VoucherResults() require.Equal(t, []datatransfer.TypedVoucher{}, results) err = channelList.NewVoucherResult(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, fvr1) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.NewVoucherResult) - voucherResults, err := state.VoucherResults() - require.NoError(t, err) + voucherResults := state.VoucherResults() require.Len(t, voucherResults, 1) require.True(t, fvr1.Equals(voucherResults[0])) - voucherResult, err := state.LastVoucherResult() - require.NoError(t, err) + voucherResult := state.LastVoucherResult() require.True(t, fvr1.Equals(voucherResult)) }) diff --git a/channels/internal/internalchannel.go b/channels/internal/internalchannel.go index 43a8c05d..4a9fabd8 100644 --- a/channels/internal/internalchannel.go +++ b/channels/internal/internalchannel.go @@ -1,15 +1,57 @@ package internal import ( + "bytes" "fmt" + "io" "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" peer "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) +type CborGenCompatibleNode struct { + Node datamodel.Node +} + +func (sn CborGenCompatibleNode) IsNull() bool { + return sn.Node == nil || sn.Node == datamodel.Null +} + +// UnmarshalCBOR is for cbor-gen compatibility +func (sn *CborGenCompatibleNode) UnmarshalCBOR(r io.Reader) error { + // use cbg.Deferred.UnmarshalCBOR to figure out how much to pull + def := cbg.Deferred{} + if err := def.UnmarshalCBOR(r); err != nil { + return err + } + // convert it to a Node + na := basicnode.Prototype.Any.NewBuilder() + if err := dagcbor.Decode(na, bytes.NewReader(def.Raw)); err != nil { + return err + } + sn.Node = na.Build() + return nil +} + +// MarshalCBOR is for cbor-gen compatibility +func (sn *CborGenCompatibleNode) MarshalCBOR(w io.Writer) error { + node := datamodel.Null + if sn != nil && sn.Node != nil { + node = sn.Node + if tn, ok := node.(schema.TypedNode); ok { + node = tn.Representation() + } + } + return dagcbor.Encode(node, w) +} + //go:generate cbor-gen-for --map-encoding ChannelState EncodedVoucher EncodedVoucherResult // EncodedVoucher is how the voucher is stored on disk @@ -17,7 +59,7 @@ type EncodedVoucher struct { // Vouchers identifier for decoding Type datatransfer.TypeIdentifier // used to verify this channel - Voucher *cbg.Deferred + Voucher CborGenCompatibleNode } // EncodedVoucherResult is how the voucher result is stored on disk @@ -25,7 +67,7 @@ type EncodedVoucherResult struct { // Vouchers identifier for decoding Type datatransfer.TypeIdentifier // used to verify this channel - VoucherResult *cbg.Deferred + VoucherResult CborGenCompatibleNode } // ChannelState is the internal representation on disk for the channel fsm @@ -41,7 +83,7 @@ type ChannelState struct { // base CID for the piece being transferred BaseCid cid.Cid // portion of Piece to return, specified by an IPLD selector - Selector *cbg.Deferred + Selector CborGenCompatibleNode // the party that is sending the data (not who initiated the request) Sender peer.ID // the party that is receiving the data (not who initiated the request) diff --git a/channels/internal/internalchannel_cbor_gen.go b/channels/internal/internalchannel_cbor_gen.go index 192ba0cf..58f43c1b 100644 --- a/channels/internal/internalchannel_cbor_gen.go +++ b/channels/internal/internalchannel_cbor_gen.go @@ -130,7 +130,7 @@ func (t *ChannelState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.BaseCid: %w", err) } - // t.Selector (typegen.Deferred) (struct) + // t.Selector (internal.CborGenCompatibleNode) (struct) if len("Selector") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Selector\" was too long") } @@ -555,16 +555,15 @@ func (t *ChannelState) UnmarshalCBOR(r io.Reader) error { t.BaseCid = c } - // t.Selector (typegen.Deferred) (struct) + // t.Selector (internal.CborGenCompatibleNode) (struct) case "Selector": { - t.Selector = new(cbg.Deferred) - if err := t.Selector.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("failed to read deferred field: %w", err) + return xerrors.Errorf("unmarshaling t.Selector: %w", err) } + } // t.Sender (peer.ID) (string) case "Sender": @@ -908,7 +907,7 @@ func (t *EncodedVoucher) MarshalCBOR(w io.Writer) error { return err } - // t.Voucher (typegen.Deferred) (struct) + // t.Voucher (internal.CborGenCompatibleNode) (struct) if len("Voucher") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Voucher\" was too long") } @@ -970,16 +969,15 @@ func (t *EncodedVoucher) UnmarshalCBOR(r io.Reader) error { t.Type = datatransfer.TypeIdentifier(sval) } - // t.Voucher (typegen.Deferred) (struct) + // t.Voucher (internal.CborGenCompatibleNode) (struct) case "Voucher": { - t.Voucher = new(cbg.Deferred) - if err := t.Voucher.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("failed to read deferred field: %w", err) + return xerrors.Errorf("unmarshaling t.Voucher: %w", err) } + } default: @@ -1024,7 +1022,7 @@ func (t *EncodedVoucherResult) MarshalCBOR(w io.Writer) error { return err } - // t.VoucherResult (typegen.Deferred) (struct) + // t.VoucherResult (internal.CborGenCompatibleNode) (struct) if len("VoucherResult") > cbg.MaxLength { return xerrors.Errorf("Value in field \"VoucherResult\" was too long") } @@ -1086,16 +1084,15 @@ func (t *EncodedVoucherResult) UnmarshalCBOR(r io.Reader) error { t.Type = datatransfer.TypeIdentifier(sval) } - // t.VoucherResult (typegen.Deferred) (struct) + // t.VoucherResult (internal.CborGenCompatibleNode) (struct) case "VoucherResult": { - t.VoucherResult = new(cbg.Deferred) - if err := t.VoucherResult.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("failed to read deferred field: %w", err) + return xerrors.Errorf("unmarshaling t.VoucherResult: %w", err) } + } default: diff --git a/impl/integration_test.go b/impl/integration_test.go index 53c375cc..659c59a7 100644 --- a/impl/integration_test.go +++ b/impl/integration_test.go @@ -1335,8 +1335,7 @@ func TestSimulatedRetrievalFlow(t *testing.T) { errChan <- struct{}{} } if event.Code == datatransfer.NewVoucherResult { - lastVoucherResult, err := channelState.LastVoucherResult() - require.NoError(t, err) + lastVoucherResult := channelState.LastVoucherResult() if lastVoucherResult.Equals(finalVoucherResult) { _ = dt2.SendVoucher(ctx, chid, testutil.NewTestTypedVoucher()) } @@ -2050,7 +2049,7 @@ func TestMultipleMessagesInExtension(t *testing.T) { } // Here we verify reception of voucherResults by the client if event.Code == datatransfer.NewVoucherResult { - voucherResult, err := channelState.LastVoucherResult() + voucherResult := channelState.LastVoucherResult() require.NoError(t, err) // If this voucher result is the response voucher no action is needed @@ -2200,7 +2199,7 @@ func TestMultipleParallelTransfers(t *testing.T) { } // Here we verify reception of voucherResults by the client if event.Code == datatransfer.NewVoucherResult { - voucherResult, err := channelState.LastVoucherResult() + voucherResult := channelState.LastVoucherResult() require.NoError(t, err) require.NoError(t, err) diff --git a/impl/receiving_requests.go b/impl/receiving_requests.go index 4760fffb..6974c9fd 100644 --- a/impl/receiving_requests.go +++ b/impl/receiving_requests.go @@ -210,7 +210,7 @@ func (m *manager) processUpdateVoucher(chid datatransfer.ChannelID, request data return nil, m.channels.NewVoucher(chid, voucher) } -// receiveUpdateRequest handles an incoming request message with an updated voucher +// receiveUpdateRequest handles an incoming request message change in pause status func (m *manager) receiveUpdateRequest(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { if request.IsPaused() { @@ -310,10 +310,8 @@ func (m *manager) recordAcceptedValidationEvents(chst datatransfer.ChannelState, // validateRestart looks up the appropriate validator and validates a restart func (m *manager) validateRestart(chst datatransfer.ChannelState) (datatransfer.ValidationResult, error) { - chv, err := chst.Voucher() - if err != nil { - return datatransfer.ValidationResult{}, err - } + chv := chst.Voucher() + processor, _ := m.validatedTypes.Processor(chv.Type) validator := processor.(datatransfer.RequestValidator) diff --git a/impl/restart.go b/impl/restart.go index eaee5349..9fdf526c 100644 --- a/impl/restart.go +++ b/impl/restart.go @@ -72,10 +72,7 @@ func (m *manager) restartManagerPeerReceivePull(ctx context.Context, channel dat func (m *manager) openPushRestartChannel(ctx context.Context, channel datatransfer.ChannelState) error { selector := channel.Selector() - voucher, err := channel.Voucher() - if err != nil { - return err - } + voucher := channel.Voucher() baseCid := channel.BaseCID() requestTo := channel.OtherPeer() chid := channel.ChannelID() @@ -110,10 +107,7 @@ func (m *manager) openPushRestartChannel(ctx context.Context, channel datatransf func (m *manager) openPullRestartChannel(ctx context.Context, channel datatransfer.ChannelState) error { selector := channel.Selector() - voucher, err := channel.Voucher() - if err != nil { - return err - } + voucher := channel.Voucher() baseCid := channel.BaseCID() requestTo := channel.OtherPeer() chid := channel.ChannelID() @@ -173,10 +167,7 @@ func (m *manager) validateRestartRequest(ctx context.Context, otherPeer peer.ID, if err != nil { return xerrors.Errorf("failed to fetch request voucher: %w", err) } - channelVoucher, err := channel.Voucher() - if err != nil { - return xerrors.Errorf("failed to fetch channel voucher: %w", err) - } + channelVoucher := channel.Voucher() if req.VoucherType() != channelVoucher.Type { return xerrors.New("channel and request voucher types do not match") } diff --git a/testutil/mockchannelstate.go b/testutil/mockchannelstate.go index a5754bde..2aa75e30 100644 --- a/testutil/mockchannelstate.go +++ b/testutil/mockchannelstate.go @@ -109,7 +109,7 @@ func (m *MockChannelState) Selector() datamodel.Node { panic("implement me") } -func (m *MockChannelState) Voucher() (datatransfer.TypedVoucher, error) { +func (m *MockChannelState) Voucher() datatransfer.TypedVoucher { panic("implement me") } @@ -141,19 +141,19 @@ func (m *MockChannelState) Message() string { panic("implement me") } -func (m *MockChannelState) Vouchers() ([]datatransfer.TypedVoucher, error) { +func (m *MockChannelState) Vouchers() []datatransfer.TypedVoucher { panic("implement me") } -func (m *MockChannelState) VoucherResults() ([]datatransfer.TypedVoucher, error) { +func (m *MockChannelState) VoucherResults() []datatransfer.TypedVoucher { panic("implement me") } -func (m *MockChannelState) LastVoucher() (datatransfer.TypedVoucher, error) { +func (m *MockChannelState) LastVoucher() datatransfer.TypedVoucher { panic("implement me") } -func (m *MockChannelState) LastVoucherResult() (datatransfer.TypedVoucher, error) { +func (m *MockChannelState) LastVoucherResult() datatransfer.TypedVoucher { panic("implement me") } diff --git a/types.go b/types.go index b15d4cde..1181d21e 100644 --- a/types.go +++ b/types.go @@ -71,7 +71,7 @@ type Channel interface { Selector() datamodel.Node // Voucher returns the initial voucher for this data transfer - Voucher() (TypedVoucher, error) + Voucher() TypedVoucher // Sender returns the peer id for the node that is sending data Sender() peer.ID @@ -112,16 +112,16 @@ type ChannelState interface { Message() string // Vouchers returns all vouchers sent on this channel - Vouchers() ([]TypedVoucher, error) + Vouchers() []TypedVoucher // VoucherResults are results of vouchers sent on the channel - VoucherResults() ([]TypedVoucher, error) + VoucherResults() []TypedVoucher // LastVoucher returns the last voucher sent on the channel - LastVoucher() (TypedVoucher, error) + LastVoucher() TypedVoucher // LastVoucherResult returns the last voucher result sent on the channel - LastVoucherResult() (TypedVoucher, error) + LastVoucherResult() TypedVoucher // ReceivedCidsTotal returns the number of (non-unique) cids received so far // on the channel - note that a block can exist in more than one place in the DAG From 61f0756c816bff36a4a36d9774a13fafe40f2569 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Sat, 25 Jun 2022 17:00:29 +1000 Subject: [PATCH 07/18] feat(ipld): use bindnode/registry (#340) * chore(deps): update libp2p v0.19.4 (#341) * feat(ipld): use bindnode/registry Co-authored-by: Hannah Howard --- benchmarks/benchmark_test.go | 2 +- go.mod | 65 +++-- go.sum | 264 ++++++++++++++----- impl/integration_test.go | 12 +- ipldutils/ipldutils.go | 183 ------------- message.go | 2 +- message/message1_1prime/message.go | 6 +- message/message1_1prime/transfer_message.go | 29 +- message/message1_1prime/transfer_request.go | 19 +- message/message1_1prime/transfer_response.go | 19 +- network/libp2p_impl_test.go | 4 +- testutil/gstestdata.go | 2 +- transport/graphsync/extension/gsextension.go | 5 +- transport/graphsync/graphsync_test.go | 9 +- 14 files changed, 270 insertions(+), 351 deletions(-) delete mode 100644 ipldutils/ipldutils.go diff --git a/benchmarks/benchmark_test.go b/benchmarks/benchmark_test.go index 967a55da..0a170d98 100644 --- a/benchmarks/benchmark_test.go +++ b/benchmarks/benchmark_test.go @@ -64,7 +64,7 @@ func BenchmarkRoundtripSuccess(b *testing.B) { } func p2pStrestTest(ctx context.Context, b *testing.B, numfiles int, df distFunc, tdm *tempDirMaker, diskBasedDatastore bool, limitBandwidth bool) { - mn := mocknet.New(ctx) + mn := mocknet.New() if limitBandwidth { mn.SetLinkDefaults(mocknet.LinkOptions{Latency: 100 * time.Millisecond, Bandwidth: 16 << 20}) } diff --git a/go.mod b/go.mod index 83c8baf9..ea25b21f 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e github.com/ipfs/go-block-format v0.0.3 github.com/ipfs/go-blockservice v0.2.1 - github.com/ipfs/go-cid v0.1.0 + github.com/ipfs/go-cid v0.2.0 github.com/ipfs/go-datastore v0.5.1 github.com/ipfs/go-ds-badger v0.3.0 github.com/ipfs/go-graphsync v0.13.1 @@ -21,14 +21,14 @@ require ( github.com/ipfs/go-ipfs-exchange-offline v0.1.1 github.com/ipfs/go-ipfs-files v0.0.8 github.com/ipfs/go-ipld-format v0.2.0 - github.com/ipfs/go-log/v2 v2.3.0 + github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.5.1 github.com/ipfs/go-unixfs v0.3.1 - github.com/ipld/go-ipld-prime v0.16.1-0.20220519105356-1f1151b69dba + github.com/ipld/go-ipld-prime v0.17.1-0.20220624062450-534ccf82237d github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jpillora/backoff v1.0.0 - github.com/libp2p/go-libp2p v0.16.0 - github.com/libp2p/go-libp2p-core v0.11.0 + github.com/libp2p/go-libp2p v0.19.4 + github.com/libp2p/go-libp2p-core v0.15.1 github.com/stretchr/testify v1.7.0 github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 go.opentelemetry.io/otel v1.3.0 @@ -37,17 +37,17 @@ require ( go.uber.org/atomic v1.9.0 golang.org/x/exp v0.0.0-20210615023648-acb5c1269671 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 + golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f ) require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/Stebalien/go-bitfield v0.0.1 // indirect github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect - github.com/benbjohnson/clock v1.1.0 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/btcsuite/btcd v0.22.0-beta // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/badger v1.6.2 // indirect @@ -62,7 +62,7 @@ require ( github.com/google/gopacket v1.1.19 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/huin/goupnp v1.0.2 // indirect + github.com/huin/goupnp v1.0.3 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.0.0 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect @@ -80,38 +80,34 @@ require ( github.com/ipld/go-codec-dagpb v1.3.1 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/goprocess v0.1.4 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/klauspost/cpuid/v2 v2.0.12 // indirect github.com/koron/go-ssdp v0.0.2 // indirect - github.com/libp2p/go-addr-util v0.1.0 // indirect github.com/libp2p/go-buffer-pool v0.0.2 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-eventbus v0.2.1 // indirect github.com/libp2p/go-libp2p-asn-util v0.1.0 // indirect - github.com/libp2p/go-libp2p-autonat v0.6.0 // indirect github.com/libp2p/go-libp2p-nat v0.1.0 // indirect - github.com/libp2p/go-libp2p-netutil v0.1.0 // indirect - github.com/libp2p/go-libp2p-peerstore v0.4.0 // indirect + github.com/libp2p/go-libp2p-peerstore v0.6.0 // indirect github.com/libp2p/go-libp2p-record v0.1.1 // indirect - github.com/libp2p/go-libp2p-testing v0.5.0 // indirect - github.com/libp2p/go-msgio v0.1.0 // indirect + github.com/libp2p/go-libp2p-testing v0.9.2 // indirect + github.com/libp2p/go-msgio v0.2.0 // indirect github.com/libp2p/go-nat v0.1.0 // indirect - github.com/libp2p/go-netroute v0.1.6 // indirect + github.com/libp2p/go-netroute v0.2.0 // indirect github.com/libp2p/go-openssl v0.0.7 // indirect - github.com/libp2p/go-sockaddr v0.1.1 // indirect - github.com/mattn/go-isatty v0.0.13 // indirect - github.com/miekg/dns v1.1.43 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/miekg/dns v1.1.48 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect - github.com/multiformats/go-base32 v0.0.3 // indirect + github.com/multiformats/go-base32 v0.0.4 // indirect github.com/multiformats/go-base36 v0.1.0 // indirect - github.com/multiformats/go-multiaddr v0.4.0 // indirect + github.com/multiformats/go-multiaddr v0.5.0 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.0.3 // indirect - github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c // indirect + github.com/multiformats/go-multicodec v0.5.0 // indirect github.com/multiformats/go-multihash v0.1.0 // indirect - github.com/multiformats/go-multistream v0.2.2 // indirect + github.com/multiformats/go-multistream v0.3.0 // indirect github.com/multiformats/go-varint v0.0.6 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -123,15 +119,16 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/urfave/cli/v2 v2.0.0 // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect - go.uber.org/multierr v1.7.0 // indirect - go.uber.org/zap v1.19.0 // indirect - golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e // indirect - golang.org/x/mod v0.4.2 // indirect - golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect - golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 // indirect - golang.org/x/tools v0.1.1 // indirect - google.golang.org/grpc v1.40.0 // indirect - google.golang.org/protobuf v1.27.1 // indirect + go.uber.org/goleak v1.1.12 // indirect + go.uber.org/multierr v1.8.0 // indirect + go.uber.org/zap v1.21.0 // indirect + golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2 // indirect + golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect + golang.org/x/tools v0.1.10 // indirect + google.golang.org/grpc v1.45.0 // indirect + google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - lukechampine.com/blake3 v1.1.6 // indirect + lukechampine.com/blake3 v1.1.7 // indirect ) diff --git a/go.sum b/go.sum index a4b6b609..34d3a7c4 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,9 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -110,20 +111,30 @@ github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QH github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.3 h1:ADZftAkglvCiD44c77s5YmMqaP2pzVCFZvBmAlBdAP4= +github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -131,12 +142,17 @@ github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmf github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -163,6 +179,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -170,12 +188,16 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= +github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= @@ -197,11 +219,13 @@ github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVB github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frankban/quicktest v1.14.2 h1:SPb1KFFmM+ybpEjPUhCCkZOM5xlovT5UbrMvWnXyBns= github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= @@ -213,9 +237,11 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.1 h1:DX7uPQ4WgAWfoh+NGGlbJQswnYIVvz0SRlLS3rPZQDA= @@ -226,6 +252,10 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -248,6 +278,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= @@ -277,12 +308,14 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -317,8 +350,9 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -357,8 +391,9 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= +github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= +github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -382,8 +417,9 @@ github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.1.0 h1:YN33LQulcRHjfom/i25yoOZR4Telp1Hr/2RU3d0PnC0= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= +github.com/ipfs/go-cid v0.2.0 h1:01JTiihFq9en9Vz0lc0VDWvZe/uBonGpzo4THP0vcQ0= +github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= @@ -460,8 +496,10 @@ github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBW github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= -github.com/ipfs/go-log/v2 v2.3.0 h1:31Re/cPqFHpsRHgyVwjWADPoF0otB1WrjTy8ZFYwEZU= github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g= +github.com/ipfs/go-log/v2 v2.5.0/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= +github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= github.com/ipfs/go-merkledag v0.5.1 h1:tr17GPP5XtPhvPPiWtu20tSGZiZDuTaJRXBLcr79Umk= github.com/ipfs/go-merkledag v0.5.1/go.mod h1:cLMZXx8J08idkp5+id62iVftUQV+HlYJ3PIhDfZsjA4= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= @@ -484,8 +522,8 @@ github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvB github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= -github.com/ipld/go-ipld-prime v0.16.1-0.20220519105356-1f1151b69dba h1:1eimQ/EpBUnxyhvSQ9gxzokN9EiDYHCeZ2URkhADIGQ= -github.com/ipld/go-ipld-prime v0.16.1-0.20220519105356-1f1151b69dba/go.mod h1:/bZAYlzT7SJS4UV0al4q67xgKvenm5hKrPCa2wNGN1U= +github.com/ipld/go-ipld-prime v0.17.1-0.20220624062450-534ccf82237d h1:aY4pwcHVHonF+edc4gzRr3HA7vAaindLXz7InFIUgiY= +github.com/ipld/go-ipld-prime v0.17.1-0.20220624062450-534ccf82237d/go.mod h1:aYcKm5TIvGfY8P3QBKz/2gKcLxzJ1zDaD+o0bOowhgs= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -515,6 +553,7 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -528,12 +567,15 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -552,7 +594,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= -github.com/libp2p/go-addr-util v0.1.0 h1:acKsntI33w2bTU7tC9a0SaPimJGfSI0bFKC18ChxeVI= github.com/libp2p/go-addr-util v0.1.0/go.mod h1:6I3ZYuFr2O/9D+SoyM0zEw0EF3YkldtTX406BpdQMqw= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= @@ -575,8 +616,9 @@ github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xS github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw= github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= -github.com/libp2p/go-libp2p v0.16.0 h1:aTxzQPllnW+nyC9mY8xaS20BbcrSYMt1HCkjZRHvdGY= github.com/libp2p/go-libp2p v0.16.0/go.mod h1:ump42BsirwAWxKzsCiFnTtN1Yc+DuPu76fyMX364/O4= +github.com/libp2p/go-libp2p v0.19.4 h1:50YL0YwPhWKDd+qbZQDEdnsmVAAkaCQrWUjpdHv4hNA= +github.com/libp2p/go-libp2p v0.19.4/go.mod h1:MIt8y481VDhUe4ErWi1a4bvt/CjjFfOq6kZTothWIXY= github.com/libp2p/go-libp2p-asn-util v0.1.0 h1:rABPCO77SjdbJ/eJ/ynIo8vWICy1VEnL5JAxJbQLo1E= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= @@ -584,16 +626,17 @@ github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQ github.com/libp2p/go-libp2p-autonat v0.2.1/go.mod h1:MWtAhV5Ko1l6QBsHQNSuM6b1sRkXrpk0/LqCr+vCVxI= github.com/libp2p/go-libp2p-autonat v0.2.2/go.mod h1:HsM62HkqZmHR2k1xgX34WuWDzk/nBwNHoeyyT4IWV6A= github.com/libp2p/go-libp2p-autonat v0.4.2/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk= -github.com/libp2p/go-libp2p-autonat v0.6.0 h1:+vbQ1pMzMGjE/RJopiQKK2FRjdCKHPNPrkPm8u+luQU= github.com/libp2p/go-libp2p-autonat v0.6.0/go.mod h1:bFC6kY8jwzNNWoqc8iGE57vsfwyJ/lP4O4DOV1e0B2o= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= -github.com/libp2p/go-libp2p-blankhost v0.2.0 h1:3EsGAi0CBGcZ33GwRuXEYJLLPoVWyXJ1bcJzAJjINkk= github.com/libp2p/go-libp2p-blankhost v0.2.0/go.mod h1:eduNKXGTioTuQAUcZ5epXi9vMl+t4d8ugUBRQ4SqaNQ= +github.com/libp2p/go-libp2p-blankhost v0.3.0 h1:kTnLArltMabZlzY63pgGDA4kkUcLkBFSM98zBssn/IY= +github.com/libp2p/go-libp2p-blankhost v0.3.0/go.mod h1:urPC+7U01nCGgJ3ZsV8jdwTp6Ji9ID0dMTvq+aJ+nZU= github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU= github.com/libp2p/go-libp2p-circuit v0.2.1/go.mod h1:BXPwYDN5A8z4OEY9sOfr2DUQMLQvKt/6oku45YUmjIo= -github.com/libp2p/go-libp2p-circuit v0.4.0 h1:eqQ3sEYkGTtybWgr6JLqJY6QLtPWRErvFjFDfAOO1wc= github.com/libp2p/go-libp2p-circuit v0.4.0/go.mod h1:t/ktoFIUzM6uLQ+o1G6NuBl2ANhBKN9Bc8jRIk31MoA= +github.com/libp2p/go-libp2p-circuit v0.6.0 h1:rw/HlhmUB3OktS/Ygz6+2XABOmHKzZpPUuMNUMosj8w= +github.com/libp2p/go-libp2p-circuit v0.6.0/go.mod h1:kB8hY+zCpMeScyvFrKrGicRdid6vNXbunKE4rXATZ0M= github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco= github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco= github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I= @@ -618,13 +661,15 @@ github.com/libp2p/go-libp2p-core v0.8.5/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJB github.com/libp2p/go-libp2p-core v0.8.6/go.mod h1:dgHr0l0hIKfWpGpqAMbpo19pen9wJfdCGv51mTmdpmM= github.com/libp2p/go-libp2p-core v0.9.0/go.mod h1:ESsbz31oC3C1AvMJoGx26RTuCkNhmkSRCqZ0kQtJ2/8= github.com/libp2p/go-libp2p-core v0.10.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= -github.com/libp2p/go-libp2p-core v0.11.0 h1:75jAgdA+IChNa+/mZXogfmrGkgwxkVvxmIC7pV+F6sI= github.com/libp2p/go-libp2p-core v0.11.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= +github.com/libp2p/go-libp2p-core v0.12.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= +github.com/libp2p/go-libp2p-core v0.14.0/go.mod h1:tLasfcVdTXnixsLB0QYaT1syJOhsbrhG7q6pGrHtBg8= +github.com/libp2p/go-libp2p-core v0.15.1 h1:0RY+Mi/ARK9DgG1g9xVQLb8dDaaU8tCePMtGALEfBnM= +github.com/libp2p/go-libp2p-core v0.15.1/go.mod h1:agSaboYM4hzB1cWekgVReqV5M4g5M+2eNNejV+1EEhs= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-discovery v0.6.0 h1:1XdPmhMJr8Tmj/yUfkJMIi8mgwWrLUsCB3bMxdT+DSo= github.com/libp2p/go-libp2p-discovery v0.6.0/go.mod h1:/u1voHt0tKIe5oIA1RHBKQLVCWPna2dXmPNHc2zR9S8= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= @@ -633,8 +678,8 @@ github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiY github.com/libp2p/go-libp2p-mplex v0.2.2/go.mod h1:74S9eum0tVQdAfFiKxAyKzNdSuLqw5oadDq7+L/FELo= github.com/libp2p/go-libp2p-mplex v0.2.3/go.mod h1:CK3p2+9qH9x+7ER/gWWDYJ3QW5ZxWDkm+dVvjfuG3ek= github.com/libp2p/go-libp2p-mplex v0.4.0/go.mod h1:yCyWJE2sc6TBTnFpjvLuEJgTSw/u+MamvzILKdX7asw= -github.com/libp2p/go-libp2p-mplex v0.4.1 h1:/pyhkP1nLwjG3OM+VuaNJkQT/Pqq73WzB3aDN3Fx1sc= github.com/libp2p/go-libp2p-mplex v0.4.1/go.mod h1:cmy+3GfqfM1PceHTLL7zQzAAYaryDu6iPSC+CIb094g= +github.com/libp2p/go-libp2p-mplex v0.5.0/go.mod h1:eLImPJLkj3iG5t5lq68w3Vm5NAQ5BcKwrrb2VmOYb3M= github.com/libp2p/go-libp2p-nat v0.0.5/go.mod h1:1qubaE5bTZMJE+E/uu2URroMbzdubFz1ChgiN79yKPE= github.com/libp2p/go-libp2p-nat v0.0.6/go.mod h1:iV59LVhB3IkFvS6S6sauVTSOrNEANnINbI/fkaLimiw= github.com/libp2p/go-libp2p-nat v0.1.0 h1:vigUi2MEN+fwghe5ijpScxtbbDz+L/6y8XwlzYOJgSY= @@ -642,8 +687,9 @@ github.com/libp2p/go-libp2p-nat v0.1.0/go.mod h1:DQzAG+QbDYjN1/C3B6vXucLtz3u9rEo github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ= github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= github.com/libp2p/go-libp2p-noise v0.2.0/go.mod h1:IEbYhBBzGyvdLBoxxULL/SGbJARhUeqlO8lVSREYu2Q= -github.com/libp2p/go-libp2p-noise v0.3.0 h1:NCVH7evhVt9njbTQshzT7N1S3Q6fjj9M11FCgfH5+cA= github.com/libp2p/go-libp2p-noise v0.3.0/go.mod h1:JNjHbociDJKHD64KTkzGnzqJ0FEV5gHJa6AB00kbCNQ= +github.com/libp2p/go-libp2p-noise v0.4.0 h1:khcMsGhHNdGqKE5LDLrnHwZvdGVMsrnD4GTkTWkwmLU= +github.com/libp2p/go-libp2p-noise v0.4.0/go.mod h1:BzzY5pyzCYSyJbQy9oD8z5oP2idsafjt4/X42h9DjZU= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= @@ -652,17 +698,22 @@ github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRj github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= -github.com/libp2p/go-libp2p-peerstore v0.4.0 h1:DOhRJLnM9Dc9lIXi3rPDZBf789LXy1BrzwIs7Tj0cKA= github.com/libp2p/go-libp2p-peerstore v0.4.0/go.mod h1:rDJUFyzEWPpXpEwywkcTYYzDHlwza8riYMaUzaN6hX0= +github.com/libp2p/go-libp2p-peerstore v0.6.0 h1:HJminhQSGISBIRb93N6WK3t6Fa8OOTnHd/VBjL4mY5A= +github.com/libp2p/go-libp2p-peerstore v0.6.0/go.mod h1:DGEmKdXrcYpK9Jha3sS7MhqYdInxJy84bIPtSu65bKc= github.com/libp2p/go-libp2p-pnet v0.2.0 h1:J6htxttBipJujEjz1y0a5+eYoiPcFHhSYHH6na5f0/k= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA= github.com/libp2p/go-libp2p-quic-transport v0.13.0/go.mod h1:39/ZWJ1TW/jx1iFkKzzUg00W6tDJh73FC0xYudjr7Hc= -github.com/libp2p/go-libp2p-quic-transport v0.15.0 h1:DR0mP6kcieowikBprWkcNtbquRKOPWb5dLZ4ahDZujk= github.com/libp2p/go-libp2p-quic-transport v0.15.0/go.mod h1:wv4uGwjcqe8Mhjj7N/Ic0aKjA+/10UnMlSzLO0yRpYQ= +github.com/libp2p/go-libp2p-quic-transport v0.16.0/go.mod h1:1BXjVMzr+w7EkPfiHkKnwsWjPjtfaNT0q8RS3tGDvEQ= +github.com/libp2p/go-libp2p-quic-transport v0.17.0 h1:yFh4Gf5MlToAYLuw/dRvuzYd1EnE2pX3Lq1N6KDiWRQ= +github.com/libp2p/go-libp2p-quic-transport v0.17.0/go.mod h1:x4pw61P3/GRCcSLypcQJE/Q2+E9f4X+5aRcZLXf20LM= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.1.1 h1:ZJK2bHXYUBqObHX+rHLSNrM3M8fmJUlUHrodDPPATmY= github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= +github.com/libp2p/go-libp2p-resource-manager v0.2.1 h1:/0yqQQ4oT+3fEhUGGP2PhuIhdv10+pu5jLhvFNfUx/w= +github.com/libp2p/go-libp2p-resource-manager v0.2.1/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= @@ -673,8 +724,10 @@ github.com/libp2p/go-libp2p-swarm v0.2.3/go.mod h1:P2VO/EpxRyDxtChXz/VPVXyTnszHv github.com/libp2p/go-libp2p-swarm v0.2.8/go.mod h1:JQKMGSth4SMqonruY0a8yjlPVIkb0mdNSwckW7OYziM= github.com/libp2p/go-libp2p-swarm v0.3.0/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= github.com/libp2p/go-libp2p-swarm v0.5.0/go.mod h1:sU9i6BoHE0Ve5SKz3y9WfKrh8dUat6JknzUehFx8xW4= -github.com/libp2p/go-libp2p-swarm v0.8.0 h1:nRHNRhi86L7jhka02N4MoV+PSFFPoJFkHNQwCTFxNhw= github.com/libp2p/go-libp2p-swarm v0.8.0/go.mod h1:sOMp6dPuqco0r0GHTzfVheVBh6UEL0L1lXUZ5ot2Fvc= +github.com/libp2p/go-libp2p-swarm v0.10.0/go.mod h1:71ceMcV6Rg/0rIQ97rsZWMzto1l9LnNquef+efcRbmA= +github.com/libp2p/go-libp2p-swarm v0.10.2 h1:UaXf+CTq6Ns1N2V1EgqJ9Q3xaRsiN7ImVlDMpirMAWw= +github.com/libp2p/go-libp2p-swarm v0.10.2/go.mod h1:Pdkq0QU5a+qu+oyqIV3bknMsnzk9lnNyKvB9acJ5aZs= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -684,19 +737,25 @@ github.com/libp2p/go-libp2p-testing v0.1.2-0.20200422005655-8775583591d8/go.mod github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehtsBXMrXnCfJIgDti5g= github.com/libp2p/go-libp2p-testing v0.4.0/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= github.com/libp2p/go-libp2p-testing v0.4.2/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= -github.com/libp2p/go-libp2p-testing v0.5.0 h1:bTjC29TTQ/ODq0ld3+0KLq3irdA5cAH3OMbRi0/QsvE= github.com/libp2p/go-libp2p-testing v0.5.0/go.mod h1:QBk8fqIL1XNcno/l3/hhaIEn4aLRijpYOR+zVjjlh+A= +github.com/libp2p/go-libp2p-testing v0.7.0/go.mod h1:OLbdn9DbgdMwv00v+tlp1l3oe2Cl+FAjoWIA2pa0X6E= +github.com/libp2p/go-libp2p-testing v0.9.0/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= +github.com/libp2p/go-libp2p-testing v0.9.2 h1:dCpODRtRaDZKF8HXT9qqqgON+OMEB423Knrgeod8j84= +github.com/libp2p/go-libp2p-testing v0.9.2/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= github.com/libp2p/go-libp2p-tls v0.3.0/go.mod h1:fwF5X6PWGxm6IDRwF3V8AVCCj/hOd5oFlg+wo2FxJDY= -github.com/libp2p/go-libp2p-tls v0.3.1 h1:lsE2zYte+rZCEOHF72J1Fg3XK3dGQyKvI6i5ehJfEp0= github.com/libp2p/go-libp2p-tls v0.3.1/go.mod h1:fwF5X6PWGxm6IDRwF3V8AVCCj/hOd5oFlg+wo2FxJDY= +github.com/libp2p/go-libp2p-tls v0.4.1 h1:1ByJUbyoMXvYXDoW6lLsMxqMViQNXmt+CfQqlnCpY+M= +github.com/libp2p/go-libp2p-tls v0.4.1/go.mod h1:EKCixHEysLNDlLUoKxv+3f/Lp90O2EXNjTr0UQDnrIw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= github.com/libp2p/go-libp2p-transport-upgrader v0.4.2/go.mod h1:NR8ne1VwfreD5VIWIU62Agt/J18ekORFU/j1i2y8zvk= github.com/libp2p/go-libp2p-transport-upgrader v0.4.3/go.mod h1:bpkldbOWXMrXhpZbSV1mQxTrefOg2Fi+k1ClDSA4ppw= -github.com/libp2p/go-libp2p-transport-upgrader v0.5.0 h1:7SDl3O2+AYOgfE40Mis83ClpfGNkNA6m4FwhbOHs+iI= github.com/libp2p/go-libp2p-transport-upgrader v0.5.0/go.mod h1:Rc+XODlB3yce7dvFV4q/RmyJGsFcCZRkeZMu/Zdg0mo= +github.com/libp2p/go-libp2p-transport-upgrader v0.7.0/go.mod h1:GIR2aTRp1J5yjVlkUoFqMkdobfob6RnAwYg/RZPhrzg= +github.com/libp2p/go-libp2p-transport-upgrader v0.7.1 h1:MSMe+tUfxpC9GArTz7a4G5zQKQgGh00Vio87d3j3xIg= +github.com/libp2p/go-libp2p-transport-upgrader v0.7.1/go.mod h1:GIR2aTRp1J5yjVlkUoFqMkdobfob6RnAwYg/RZPhrzg= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= github.com/libp2p/go-libp2p-yamux v0.2.2/go.mod h1:lIohaR0pT6mOt0AZ0L2dFze9hds9Req3OfS+B+dv4qw= github.com/libp2p/go-libp2p-yamux v0.2.5/go.mod h1:Zpgj6arbyQrmZ3wxSZxfBmbdnWtbZ48OpsfmQVTErwA= @@ -705,24 +764,27 @@ github.com/libp2p/go-libp2p-yamux v0.2.8/go.mod h1:/t6tDqeuZf0INZMTgd0WxIRbtK2Ez github.com/libp2p/go-libp2p-yamux v0.4.0/go.mod h1:+DWDjtFMzoAwYLVkNZftoucn7PelNoy5nm3tZ3/Zw30= github.com/libp2p/go-libp2p-yamux v0.5.0/go.mod h1:AyR8k5EzyM2QN9Bbdg6X1SkVVuqLwTGf0L4DFq9g6po= github.com/libp2p/go-libp2p-yamux v0.5.4/go.mod h1:tfrXbyaTqqSU654GTvK3ocnSZL3BuHoeTSqhcel1wsE= -github.com/libp2p/go-libp2p-yamux v0.6.0 h1:TKayW983n92JhCGdCo7ej7eEb+DQ0VYfKNOxlN/1kNQ= github.com/libp2p/go-libp2p-yamux v0.6.0/go.mod h1:MRhd6mAYnFRnSISp4M8i0ClV/j+mWHo2mYLifWGw33k= +github.com/libp2p/go-libp2p-yamux v0.8.0/go.mod h1:yTkPgN2ib8FHyU1ZcVD7aelzyAqXXwEPbyx+aSKm9h8= +github.com/libp2p/go-libp2p-yamux v0.8.1/go.mod h1:rUozF8Jah2dL9LLGyBaBeTQeARdwhefMCTQVQt6QobE= +github.com/libp2p/go-libp2p-yamux v0.9.1 h1:oplewiRix8s45SOrI30rCPZG5mM087YZp+VYhXAh4+c= +github.com/libp2p/go-libp2p-yamux v0.9.1/go.mod h1:wRc6wvyxQINFcKe7daL4BeQ02Iyp+wxyC8WCNfngBrA= github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= -github.com/libp2p/go-maddr-filter v0.1.0 h1:4ACqZKw8AqiuJfwFGq1CYDFugfXTOos+qQ3DETkhtCE= github.com/libp2p/go-maddr-filter v0.1.0/go.mod h1:VzZhTXkMucEGGEOSKddrwGiOv0tUhgnKqNEmIAz/bPU= github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= github.com/libp2p/go-mplex v0.1.1/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= github.com/libp2p/go-mplex v0.1.2/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= github.com/libp2p/go-mplex v0.2.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= -github.com/libp2p/go-mplex v0.3.0 h1:U1T+vmCYJaEoDJPV1aq31N56hS+lJgb397GsylNSgrU= github.com/libp2p/go-mplex v0.3.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= +github.com/libp2p/go-mplex v0.4.0/go.mod h1:y26Lx+wNVtMYMaPu300Cbot5LkEZ4tJaNYeHeT9dh6E= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= -github.com/libp2p/go-msgio v0.1.0 h1:8Q7g/528ivAlfXTFWvWhVjTE8XG8sDTkRUKPYh9+5Q8= github.com/libp2p/go-msgio v0.1.0/go.mod h1:eNlv2vy9V2X/kNldcZ+SShFE++o2Yjxwx6RAYsmgJnE= +github.com/libp2p/go-msgio v0.2.0 h1:W6shmB+FeynDrUVl2dgFQvzfBZcXiyqY4VmpQLu9FqU= +github.com/libp2p/go-msgio v0.2.0/go.mod h1:dBVM1gW3Jk9XqHkU4eKdGvVHdLa51hoGfll6jMJMSlY= github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= @@ -730,8 +792,9 @@ github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.5/go.mod h1:V1SR3AaECRkEQCoFFzYwVYWvYIEtlxx89+O3qcpCl4A= -github.com/libp2p/go-netroute v0.1.6 h1:ruPJStbYyXVYGQ81uzEDzuvbYRLKRrLvTYd33yomC38= github.com/libp2p/go-netroute v0.1.6/go.mod h1:AqhkMh0VuWmfgtxKPp3Oc1LdU5QSWS7wl0QLhSZqXxQ= +github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4nWRE= +github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= @@ -749,23 +812,26 @@ github.com/libp2p/go-reuseport-transport v0.1.0 h1:C3PHeHjmnz8m6f0uydObj02tMEoi7 github.com/libp2p/go-reuseport-transport v0.1.0/go.mod h1:vev0C0uMkzriDY59yFHD9v+ujJvYmDQVLowvAjEOmfw= github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/libp2p/go-sockaddr v0.1.0/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= -github.com/libp2p/go-sockaddr v0.1.1 h1:yD80l2ZOdGksnOyHrhxDdTDFrf7Oy+v3FMVArIRgZxQ= github.com/libp2p/go-sockaddr v0.1.1/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= -github.com/libp2p/go-stream-muxer-multistream v0.3.0 h1:TqnSHPJEIqDEO7h1wZZ0p3DXdvDSiLHQidKKUGZtiOY= github.com/libp2p/go-stream-muxer-multistream v0.3.0/go.mod h1:yDh8abSIzmZtqtOt64gFJUXEryejzNb0lisTt+fAMJA= +github.com/libp2p/go-stream-muxer-multistream v0.4.0 h1:HsM/9OdtqnIzjVXcxTXjmqKrj3gJ8kacaOJwJS1ipaY= +github.com/libp2p/go-stream-muxer-multistream v0.4.0/go.mod h1:nb+dGViZleRP4XcyHuZSVrJCBl55nRBOMmiSL/dyziw= github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY= github.com/libp2p/go-tcp-transport v0.2.0/go.mod h1:vX2U0CnWimU4h0SGSEsg++AzvBcroCGYw28kh94oLe0= github.com/libp2p/go-tcp-transport v0.2.3/go.mod h1:9dvr03yqrPyYGIEN6Dy5UvdJZjyPFvl1S/igQ5QD1SU= -github.com/libp2p/go-tcp-transport v0.4.0 h1:VDyg4j6en3OuXf90gfDQh5Sy9KowO9udnd0OU8PP6zg= github.com/libp2p/go-tcp-transport v0.4.0/go.mod h1:0y52Rwrn4076xdJYu/51/qJIdxz+EWDAOG2S45sV3VI= +github.com/libp2p/go-tcp-transport v0.5.0/go.mod h1:UPPL0DIjQqiWRwVAb+CEQlaAG0rp/mCqJfIhFcLHc4Y= +github.com/libp2p/go-tcp-transport v0.5.1 h1:edOOs688VLZAozWC7Kj5/6HHXKNwi9M6wgRmmLa8M6Q= +github.com/libp2p/go-tcp-transport v0.5.1/go.mod h1:UPPL0DIjQqiWRwVAb+CEQlaAG0rp/mCqJfIhFcLHc4Y= github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM= github.com/libp2p/go-ws-transport v0.3.0/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= github.com/libp2p/go-ws-transport v0.4.0/go.mod h1:EcIEKqf/7GDjth6ksuS/6p7R49V4CBY6/E7R/iyhYUA= -github.com/libp2p/go-ws-transport v0.5.0 h1:cO6x4P0v6PfxbKnxmf5cY2Ny4OPDGYkUqNvZzp/zdlo= github.com/libp2p/go-ws-transport v0.5.0/go.mod h1:I2juo1dNTbl8BKSBYo98XY85kU2xds1iamArLvl8kNg= +github.com/libp2p/go-ws-transport v0.6.0 h1:326XBL6Q+5CQ2KtjXz32+eGu02W/Kz2+Fm4SpXdr0q4= +github.com/libp2p/go-ws-transport v0.6.0/go.mod h1:dXqtI9e2JV9FtF1NOtWVZSKXh5zXvnuwPXfj8GPBbYU= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= @@ -775,15 +841,22 @@ github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/h github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI= github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux/v2 v2.2.0/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZjqROGxzPpPQ= -github.com/libp2p/go-yamux/v2 v2.3.0 h1:luRV68GS1vqqr6EFUjtu1kr51d+IbW0gSowu8emYWAI= github.com/libp2p/go-yamux/v2 v2.3.0/go.mod h1:iTU+lOIn/2h0AgKcL49clNTwfEw+WSfDYrXe05EyKIs= +github.com/libp2p/go-yamux/v3 v3.0.1/go.mod h1:s2LsDhHbh+RfCsQoICSYt58U2f8ijtPANFD8BmE74Bo= +github.com/libp2p/go-yamux/v3 v3.0.2/go.mod h1:s2LsDhHbh+RfCsQoICSYt58U2f8ijtPANFD8BmE74Bo= +github.com/libp2p/go-yamux/v3 v3.1.1/go.mod h1:jeLEQgLXqE2YqX1ilAClIfCMDY+0uXQUKmmb/qp0gT4= +github.com/libp2p/go-yamux/v3 v3.1.2 h1:lNEy28MBk1HavUAlzKgShp+F6mn/ea1nDYWftZhFW9Q= +github.com/libp2p/go-yamux/v3 v3.1.2/go.mod h1:jeLEQgLXqE2YqX1ilAClIfCMDY+0uXQUKmmb/qp0gT4= github.com/libp2p/zeroconf/v2 v2.1.1/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= github.com/lucas-clemente/quic-go v0.23.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0= -github.com/lucas-clemente/quic-go v0.24.0 h1:ToR7SIIEdrgOhgVTHvPgdVRJfgVy+N0wQAagH7L4d5g= github.com/lucas-clemente/quic-go v0.24.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0= +github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg= +github.com/lucas-clemente/quic-go v0.27.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= +github.com/lucas-clemente/quic-go v0.27.1 h1:sOw+4kFSVrdWOYmUjufQ9GBVPqZ+tu+jMtXxXNmRJyk= +github.com/lucas-clemente/quic-go v0.27.1/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -793,10 +866,15 @@ github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2o github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= -github.com/marten-seemann/qtls-go1-16 v0.1.4 h1:xbHbOGGhrenVtII6Co8akhLEdrawwB2iHl5yhJRpnco= github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= -github.com/marten-seemann/qtls-go1-17 v0.1.0 h1:P9ggrs5xtwiqXv/FHNwntmuLMNq3KaSIG93AtAZ48xk= +github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ= +github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8= +github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWChPSI4+hgW7jc= +github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= +github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI= +github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y= +github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -804,8 +882,9 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -815,8 +894,9 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= +github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= @@ -843,14 +923,16 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base32 v0.0.4 h1:+qMh4a2f37b4xTNs6mqitDinryCI+tfO2dRVMN9mjSE= +github.com/multiformats/go-base32 v0.0.4/go.mod h1:jNLFzjPZtp3aIARHbJRZIaPuspdH0J6q39uUM5pnABM= github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= @@ -864,8 +946,10 @@ github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= -github.com/multiformats/go-multiaddr v0.4.0 h1:hL/K4ZJhJ5PTw3nwylq9lGU5yArzcAroZmex1ghSEkQ= github.com/multiformats/go-multiaddr v0.4.0/go.mod h1:YcpyLH8ZPudLxQlemYBPhSm0/oCXAT8Z4mzFpyoPyRc= +github.com/multiformats/go-multiaddr v0.4.1/go.mod h1:3afI9HfVW8csiF8UZqtpYRiDyew8pRX7qLIGHu9FLuM= +github.com/multiformats/go-multiaddr v0.5.0 h1:i/JuOoVg4szYQ4YEzDGtb2h0o8M7CG/Yq6cGlcjWZpM= +github.com/multiformats/go-multiaddr v0.5.0/go.mod h1:3KAxNkUqLTJ20AAwN4XVX4kZar+bR+gh4zgbfr3SNug= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= @@ -887,8 +971,10 @@ github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77 github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c h1:VyANTtZ0wsx0IAZnCZhfMmAmfUyzJq/5JQi2hHOtKS0= github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.5.0 h1:EgU6cBe/D7WRwQb1KmnBvU7lrcFGMggZVTPtOW9dDHs= +github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -902,8 +988,9 @@ github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0Erjhoj github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38= github.com/multiformats/go-multistream v0.2.1/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= -github.com/multiformats/go-multistream v0.2.2 h1:TCYu1BHTDr1F/Qm75qwYISQdzGcRdC21nFgQW7l7GBo= github.com/multiformats/go-multistream v0.2.2/go.mod h1:UIcnm7Zuo8HKG+HkWgfQsGL+/MIEhyTqbODbIUwSXKs= +github.com/multiformats/go-multistream v0.3.0 h1:yX1v4IWseLPmr0rmnDo148wWJbNx40JxBZGmQb5fUP4= +github.com/multiformats/go-multistream v0.3.0/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -933,8 +1020,9 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -943,6 +1031,8 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -956,6 +1046,8 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= @@ -984,8 +1076,9 @@ github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeD github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1001,8 +1094,10 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.33.0 h1:rHgav/0a6+uYgGdNt3jwz8FNSesO/Hsang3O0T9A5SE= +github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1013,6 +1108,10 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y= +github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0= +github.com/raulk/go-watchdog v1.2.0 h1:konN75pw2BMmZ+AfuAm5rtFsWcJpKF3m02rKituuXNo= +github.com/raulk/go-watchdog v1.2.0/go.mod h1:lzSbAl5sh4rtI8tYHU01BWIDzgzqaQLj6RcA1i4mlqI= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -1057,6 +1156,7 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= @@ -1102,14 +1202,16 @@ github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cb github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/warpfork/go-testmark v0.3.0 h1:Q81c4u7hT+BR5kNfNQhEF0VT2pmL7+Kk0wD+ORYl7iA= github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= +github.com/warpfork/go-testmark v0.10.0 h1:E86YlUMYfwIacEsQGlnTvjk1IgYkyTGjPhF0RnwTCmw= +github.com/warpfork/go-testmark v0.10.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= @@ -1139,6 +1241,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= @@ -1173,23 +1276,29 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU= go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= @@ -1222,8 +1331,10 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e h1:VvfwVmMH40bpMeizC9/K7ipM5Qjucuu16RWfneFPyhQ= golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1249,7 +1360,6 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -1261,8 +1371,9 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1315,8 +1426,14 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2 h1:6mzvA99KwZxbOrxww4EvWVQUnN1+xEu9tafK5ZxkYeA= +golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1325,6 +1442,7 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1338,6 +1456,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1380,6 +1499,7 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1415,10 +1535,15 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1486,13 +1611,17 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -1580,8 +1709,9 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1594,8 +1724,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1638,8 +1769,9 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c= lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= +lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/impl/integration_test.go b/impl/integration_test.go index 659c59a7..af447baa 100644 --- a/impl/integration_test.go +++ b/impl/integration_test.go @@ -1807,8 +1807,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { response, err := message.NewResponse(requestReceived.TransferID(), true, false, &voucherResult) require.NoError(t, err) - nd, err := response.ToIPLD() - require.NoError(t, err) + nd := response.ToIPLD() request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, gsData.AllSelector, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ Name: extension.ExtensionDataTransfer1_1, Data: nd, @@ -1826,8 +1825,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { t.Run("when no request is initiated", func(t *testing.T) { response, err := message.NewResponse(datatransfer.TransferID(rand.Uint32()), true, false, &voucher) require.NoError(t, err) - nd, err := response.ToIPLD() - require.NoError(t, err) + nd := response.ToIPLD() request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, gsData.AllSelector, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ Name: extension.ExtensionDataTransfer1_1, Data: nd, @@ -1920,8 +1918,7 @@ func TestRespondingToPullGraphsyncRequests(t *testing.T) { voucher := testutil.NewTestTypedVoucher() request, err := message.NewRequest(id, false, true, &voucher, testutil.GenerateCids(1)[0], gsData.AllSelector) require.NoError(t, err) - nd, err := request.ToIPLD() - require.NoError(t, err) + nd := request.ToIPLD() gsRequest := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, gsData.AllSelector, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ Name: extension.ExtensionDataTransfer1_1, Data: nd, @@ -1949,8 +1946,7 @@ func TestRespondingToPullGraphsyncRequests(t *testing.T) { dtRequest, err := message.NewRequest(id, false, true, &voucher, testutil.GenerateCids(1)[0], gsData.AllSelector) require.NoError(t, err) - nd, err := dtRequest.ToIPLD() - require.NoError(t, err) + nd := dtRequest.ToIPLD() request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, gsData.AllSelector, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ Name: extension.ExtensionDataTransfer1_1, Data: nd, diff --git a/ipldutils/ipldutils.go b/ipldutils/ipldutils.go deleted file mode 100644 index 1b5b0be2..00000000 --- a/ipldutils/ipldutils.go +++ /dev/null @@ -1,183 +0,0 @@ -package shared - -import ( - "bytes" - "fmt" - "io" - "reflect" - - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/codec/dagcbor" - "github.com/ipld/go-ipld-prime/datamodel" - "github.com/ipld/go-ipld-prime/node/bindnode" - "github.com/ipld/go-ipld-prime/schema" - cbg "github.com/whyrusleeping/cbor-gen" -) - -type typeWithBindnodeSchema interface { - BindnodeSchema() string -} - -// TODO: remove this I think -type typeWithBindnodePostDecode interface { - BindnodePostDecode() error -} - -// We use the prototype map to store TypedPrototype and Type information -// mapped against Go type names so we only have to run the schema parse once. -// Currently there's not much additional benefit of storing this but there -// may be in the future. -var prototype map[string]schema.TypedPrototype = make(map[string]schema.TypedPrototype) - -var bindnodeOptions = []bindnode.Option{} - -func typeName(ptrValue interface{}) string { - val := reflect.ValueOf(ptrValue).Type() - for val.Kind() == reflect.Ptr { - val = val.Elem() - } - return val.Name() -} - -// lookup of cached TypedPrototype (and therefore Type) for a Go type, if not -// found, initial parse and setup and caching of the TypedPrototype will happen -func prototypeFor(typeName string, ptrType interface{}) (schema.TypedPrototype, error) { - proto, ok := prototype[typeName] - if !ok { - schemaType, err := schemaTypeFor(typeName, ptrType) - if err != nil { - return nil, err - } - if schemaType == nil { - return nil, fmt.Errorf("could not find type [%s] in schema", typeName) - } - proto = bindnode.Prototype(ptrType, schemaType, bindnodeOptions...) - prototype[typeName] = proto - } - return proto, nil -} - -// load the schema for a Go type, which must have a BindnodeSchema() method -// attached to it -func schemaTypeFor(typeName string, ptrType interface{}) (schema.Type, error) { - tws, ok := ptrType.(typeWithBindnodeSchema) - if !ok { - return nil, fmt.Errorf("attempted to perform IPLD mapping on type without BindnodeSchema(): %T", ptrType) - } - schema := tws.BindnodeSchema() - typeSystem, err := ipld.LoadSchemaBytes([]byte(schema)) - if err != nil { - return nil, err - } - schemaType := typeSystem.TypeByName(typeName) - if schemaType == nil { - if !ok { - return nil, fmt.Errorf("schema for [%T] does not contain that named type [%s]", ptrType, typeName) - } - } - return schemaType, nil -} - -// FromReader deserializes DAG-CBOR from a Reader and instantiates the Go type -// that's provided as a pointer via the ptrValue argument. -func FromReader(r io.Reader, ptrValue interface{}) (interface{}, error) { - name := typeName(ptrValue) - proto, err := prototypeFor(name, ptrValue) - if err != nil { - return nil, err - } - node, err := ipld.DecodeStreamingUsingPrototype(r, dagcbor.Decode, proto) - if err != nil { - return nil, err - } - typ := bindnode.Unwrap(node) - if twpd, ok := typ.(typeWithBindnodePostDecode); ok { - // we have some more work to do - if err = twpd.BindnodePostDecode(); err != nil { - return nil, err - } - } - return typ, nil -} - -// FromNode converts an datamodel.Node into an appropriate Go type that's provided as -// a pointer via the ptrValue argument -func FromNode(node datamodel.Node, ptrValue interface{}) (interface{}, error) { - name := typeName(ptrValue) - proto, err := prototypeFor(name, ptrValue) - if err != nil { - return nil, err - } - if tn, ok := node.(schema.TypedNode); ok { - node = tn.Representation() - } - builder := proto.Representation().NewBuilder() - err = builder.AssignNode(node) - if err != nil { - return nil, err - } - typ := bindnode.Unwrap(builder.Build()) - if twpd, ok := typ.(typeWithBindnodePostDecode); ok { - // we have some more work to do - if err = twpd.BindnodePostDecode(); err != nil { - return nil, err - } - } - return typ, nil -} - -// ToNode converts a Go type that's provided as a pointer via the ptrValue -// argument to an datamodel.Node. -func ToNode(ptrValue interface{}) (schema.TypedNode, error) { - name := typeName(ptrValue) - proto, err := prototypeFor(name, ptrValue) - if err != nil { - return nil, err - } - return bindnode.Wrap(ptrValue, proto.Type(), bindnodeOptions...), err -} - -// NodeToWriter is a utility method that serializes an datamodel.Node as DAG-CBOR to -// a Writer -func NodeToWriter(node datamodel.Node, w io.Writer) error { - return ipld.EncodeStreaming(w, node, dagcbor.Encode) -} - -// NodeToBytes is a utility method that serializes an datamodel.Node as DAG-CBOR to -// a []byte -func NodeToBytes(node datamodel.Node) ([]byte, error) { - var buf bytes.Buffer - err := NodeToWriter(node, &buf) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// NodeFromBytes is a utility method that deserializes an untyped datamodel.Node -// from DAG-CBOR format bytes -func NodeFromBytes(b []byte) (datamodel.Node, error) { - return ipld.Decode(b, dagcbor.Decode) -} - -// TypeToWriter is a utility method that serializes a Go type that's provided as a -// pointer via the ptrValue argument as DAG-CBOR to a Writer -func TypeToWriter(ptrValue interface{}, w io.Writer) error { - node, err := ToNode(ptrValue) - if err != nil { - return err - } - return ipld.EncodeStreaming(w, node, dagcbor.Encode) -} - -func NodeToDeferred(node datamodel.Node) (*cbg.Deferred, error) { - byts, err := NodeToBytes(node) - if err != nil { - return nil, err - } - return &cbg.Deferred{Raw: byts}, nil -} - -func DeferredToNode(def *cbg.Deferred) (datamodel.Node, error) { - return NodeFromBytes(def.Raw) -} diff --git a/message.go b/message.go index d54eabb1..93ea17d9 100644 --- a/message.go +++ b/message.go @@ -25,7 +25,7 @@ type Message interface { IsCancel() bool TransferID() TransferID ToNet(w io.Writer) error - ToIPLD() (datamodel.Node, error) + ToIPLD() datamodel.Node MessageForProtocol(targetProtocol protocol.ID) (newMsg Message, err error) } diff --git a/message/message1_1prime/message.go b/message/message1_1prime/message.go index b740bc01..4d852978 100644 --- a/message/message1_1prime/message.go +++ b/message/message1_1prime/message.go @@ -5,12 +5,12 @@ import ( "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" xerrors "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - ipldutils "github.com/filecoin-project/go-data-transfer/v2/ipldutils" "github.com/filecoin-project/go-data-transfer/v2/message/types" ) @@ -192,7 +192,7 @@ func CompleteResponse(id datatransfer.TransferID, isAccepted bool, isPaused bool // FromNet can read a network stream to deserialize a GraphSyncMessage func FromNet(r io.Reader) (datatransfer.Message, error) { - tm, err := ipldutils.FromReader(r, &TransferMessage1_1{}) + tm, err := bindnodeRegistry.TypeFromReader(r, &TransferMessage1_1{}, dagcbor.Decode) if err != nil { return nil, err } @@ -213,7 +213,7 @@ func FromIPLD(node datamodel.Node) (datatransfer.Message, error) { if tn, ok := node.(schema.TypedNode); ok { // shouldn't need this if from Graphsync node = tn.Representation() } - tm, err := ipldutils.FromNode(node, &TransferMessage1_1{}) + tm, err := bindnodeRegistry.TypeFromNode(node, &TransferMessage1_1{}) if err != nil { return nil, err } diff --git a/message/message1_1prime/transfer_message.go b/message/message1_1prime/transfer_message.go index 0944212b..5f1f6286 100644 --- a/message/message1_1prime/transfer_message.go +++ b/message/message1_1prime/transfer_message.go @@ -4,13 +4,16 @@ import ( _ "embed" "io" + "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" + bindnoderegistry "github.com/ipld/go-ipld-prime/node/bindnode/registry" "github.com/ipld/go-ipld-prime/schema" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - ipldutils "github.com/filecoin-project/go-data-transfer/v2/ipldutils" ) +var bindnodeRegistry = bindnoderegistry.NewRegistry() + //go:embed schema.ipldsch var embedSchema []byte @@ -22,10 +25,6 @@ type TransferMessage1_1 struct { Response *TransferResponse1_1 } -func (tm *TransferMessage1_1) BindnodeSchema() string { - return string(embedSchema) -} - // ========= datatransfer.Message interface // TransferID returns the TransferID of this message @@ -36,24 +35,22 @@ func (tm *TransferMessage1_1) TransferID() datatransfer.TransferID { return tm.Response.TransferID() } -func (tm *TransferMessage1_1) toIPLD() (schema.TypedNode, error) { - return ipldutils.ToNode(tm) +func (tm *TransferMessage1_1) toIPLD() schema.TypedNode { + return bindnodeRegistry.TypeToNode(tm) } // ToIPLD converts a transfer message type to an ipld Node func (tm *TransferMessage1_1) ToIPLD() (datamodel.Node, error) { - node, err := tm.toIPLD() - if err != nil { - return nil, err - } - return node.Representation(), nil + return tm.toIPLD().Representation(), nil } // ToNet serializes a transfer message type. func (tm *TransferMessage1_1) ToNet(w io.Writer) error { - i, err := tm.toIPLD() - if err != nil { - return err + return bindnodeRegistry.TypeToWriter(tm.toIPLD(), w, dagcbor.Encode) +} + +func init() { + if err := bindnodeRegistry.RegisterType((*TransferMessage1_1)(nil), string(embedSchema), "TransferMessage1_1"); err != nil { + panic(err.Error()) } - return ipldutils.NodeToWriter(i, w) } diff --git a/message/message1_1prime/transfer_request.go b/message/message1_1prime/transfer_request.go index a01e2452..3600329f 100644 --- a/message/message1_1prime/transfer_request.go +++ b/message/message1_1prime/transfer_request.go @@ -4,13 +4,14 @@ import ( "io" "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" "github.com/libp2p/go-libp2p-core/protocol" xerrors "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - ipldutils "github.com/filecoin-project/go-data-transfer/v2/ipldutils" "github.com/filecoin-project/go-data-transfer/v2/message/types" ) @@ -138,7 +139,7 @@ func (trq *TransferRequest1_1) IsPartial() bool { return trq.Partial } -func (trq *TransferRequest1_1) toIPLD() (schema.TypedNode, error) { +func (trq *TransferRequest1_1) toIPLD() schema.TypedNode { msg := TransferMessage1_1{ IsRequest: true, Request: trq, @@ -147,19 +148,11 @@ func (trq *TransferRequest1_1) toIPLD() (schema.TypedNode, error) { return msg.toIPLD() } -func (trq *TransferRequest1_1) ToIPLD() (datamodel.Node, error) { - msg, err := trq.toIPLD() - if err != nil { - return nil, err - } - return msg.Representation(), nil +func (trq *TransferRequest1_1) ToIPLD() datamodel.Node { + return trq.toIPLD().Representation() } // ToNet serializes a transfer request. func (trq *TransferRequest1_1) ToNet(w io.Writer) error { - i, err := trq.toIPLD() - if err != nil { - return err - } - return ipldutils.NodeToWriter(i, w) + return ipld.EncodeStreaming(w, trq.toIPLD(), dagcbor.Encode) } diff --git a/message/message1_1prime/transfer_response.go b/message/message1_1prime/transfer_response.go index 1431ff72..69c19f23 100644 --- a/message/message1_1prime/transfer_response.go +++ b/message/message1_1prime/transfer_response.go @@ -3,13 +3,14 @@ package message1_1 import ( "io" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" "github.com/libp2p/go-libp2p-core/protocol" xerrors "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - ipldutils "github.com/filecoin-project/go-data-transfer/v2/ipldutils" "github.com/filecoin-project/go-data-transfer/v2/message/types" ) @@ -96,7 +97,7 @@ func (trsp *TransferResponse1_1) MessageForProtocol(targetProtocol protocol.ID) } } -func (trsp *TransferResponse1_1) toIPLD() (schema.TypedNode, error) { +func (trsp *TransferResponse1_1) toIPLD() schema.TypedNode { msg := TransferMessage1_1{ IsRequest: false, Request: nil, @@ -105,19 +106,11 @@ func (trsp *TransferResponse1_1) toIPLD() (schema.TypedNode, error) { return msg.toIPLD() } -func (trsp *TransferResponse1_1) ToIPLD() (datamodel.Node, error) { - msg, err := trsp.toIPLD() - if err != nil { - return nil, err - } - return msg.Representation(), nil +func (trsp *TransferResponse1_1) ToIPLD() datamodel.Node { + return trsp.toIPLD().Representation() } // ToNet serializes a transfer response. func (trsp *TransferResponse1_1) ToNet(w io.Writer) error { - i, err := trsp.toIPLD() - if err != nil { - return err - } - return ipldutils.NodeToWriter(i, w) + return ipld.EncodeStreaming(w, trsp.toIPLD(), dagcbor.Encode) } diff --git a/network/libp2p_impl_test.go b/network/libp2p_impl_test.go index 646ba915..89489155 100644 --- a/network/libp2p_impl_test.go +++ b/network/libp2p_impl_test.go @@ -76,7 +76,7 @@ func TestMessageSendAndReceive(t *testing.T) { ctx := context.Background() ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - mn := mocknet.New(ctx) + mn := mocknet.New() host1, err := mn.GenPeer() require.NoError(t, err) @@ -231,7 +231,7 @@ func TestSendMessageRetry(t *testing.T) { ctx := context.Background() ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - mn := mocknet.New(ctx) + mn := mocknet.New() host1, err := mn.GenPeer() require.NoError(t, err) diff --git a/testutil/gstestdata.go b/testutil/gstestdata.go index 8f40178c..3fa22989 100644 --- a/testutil/gstestdata.go +++ b/testutil/gstestdata.go @@ -115,7 +115,7 @@ func NewGraphsyncTestingData(ctx context.Context, t *testing.T, host1Protocols [ // setup an IPLD loader/storer for blockstore 2 gsData.LinkSystem2 = storeutil.LinkSystemForBlockstore(gsData.Bs2) - gsData.Mn = mocknet.New(ctx) + gsData.Mn = mocknet.New() // setup network var err error diff --git a/transport/graphsync/extension/gsextension.go b/transport/graphsync/extension/gsextension.go index a8c34c35..532cb398 100644 --- a/transport/graphsync/extension/gsextension.go +++ b/transport/graphsync/extension/gsextension.go @@ -39,10 +39,7 @@ func ToExtensionData(msg datatransfer.Message, supportedExtensions []graphsync.E if err != nil { continue } - nd, err := versionedMsg.ToIPLD() - if err != nil { - return nil, err - } + nd := versionedMsg.ToIPLD() exts = append(exts, graphsync.ExtensionData{ Name: supportedExtension, Data: nd, diff --git a/transport/graphsync/graphsync_test.go b/transport/graphsync/graphsync_test.go index c2a71468..c5bb4a07 100644 --- a/transport/graphsync/graphsync_test.go +++ b/transport/graphsync/graphsync_test.go @@ -1287,8 +1287,7 @@ func (dtc *dtConfig) extensions(t *testing.T, transferID datatransfer.TransferID } else { msg = testutil.NewDTRequest(t, transferID) } - nd, err := msg.ToIPLD() - require.NoError(t, err) + nd := msg.ToIPLD() extensions[extName] = nd } } @@ -1335,8 +1334,7 @@ func assertDecodesToMessage(t *testing.T, data datamodel.Node, expected datatran } func assertHasOutgoingMessage(t *testing.T, extensions []graphsync.ExtensionData, expected datatransfer.Message) { - nd, err := expected.ToIPLD() - require.NoError(t, err) + nd := expected.ToIPLD() found := false for _, e := range extensions { if e.Name == extension.ExtensionDataTransfer1_1 { @@ -1350,8 +1348,7 @@ func assertHasOutgoingMessage(t *testing.T, extensions []graphsync.ExtensionData } func assertHasExtensionMessage(t *testing.T, name graphsync.ExtensionName, extensions []graphsync.ExtensionData, expected datatransfer.Message) { - nd, err := expected.ToIPLD() - require.NoError(t, err) + nd := expected.ToIPLD() found := false for _, e := range extensions { if e.Name == name { From 0ef707366f15e48368425fe06bc10589e1b533d3 Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Wed, 15 Jun 2022 20:21:54 -0700 Subject: [PATCH 08/18] Refactor transport part 1 (#324) * refactor(transport): move libp2p message layer into transport * refactor(graphsync): cleanup channel state tracking * refactor(dtchannel): extract dtchannel to package * test(impl): get remaining tests passing * refactor(transport): much simpler interface * style(lint): fix imports * refactor(transport): remote unneccesary methods * fix(rebase): fix errors after rebase --- benchmarks/testinstance/testinstance.go | 4 +- channelmonitor/channelmonitor.go | 30 +- channelmonitor/channelmonitor_test.go | 38 +- channels/channels.go | 30 +- channels/channels_fsm.go | 1 - channels/channels_test.go | 32 +- go.mod | 2 +- go.sum | 4 +- impl/environment.go | 10 +- impl/events.go | 26 +- impl/impl.go | 213 +-- impl/initiating_test.go | 143 +- impl/receiving_requests.go | 18 +- impl/responding_test.go | 240 +--- impl/restart.go | 97 +- impl/utils.go | 1 + {impl => itest}/integration_test.go | 64 +- {impl => itest}/restart_integration_test.go | 20 +- testutil/faketransport.go | 93 +- testutil/gstestdata.go | 4 +- testutil/testutil.go | 7 + transport.go | 85 +- transport/graphsync/dtchannel/dtchannel.go | 375 ++++++ transport/graphsync/executor/executor.go | 109 ++ transport/graphsync/executor/executor_test.go | 140 ++ transport/graphsync/graphsync.go | 1155 ++--------------- transport/graphsync/graphsync_test.go | 4 +- transport/graphsync/hooks.go | 449 +++++++ {impl => transport/graphsync}/receiver.go | 121 +- transport/graphsync/utils.go | 118 ++ 30 files changed, 1803 insertions(+), 1830 deletions(-) rename {impl => itest}/integration_test.go (97%) rename {impl => itest}/restart_integration_test.go (97%) create mode 100644 transport/graphsync/dtchannel/dtchannel.go create mode 100644 transport/graphsync/executor/executor.go create mode 100644 transport/graphsync/executor/executor_test.go create mode 100644 transport/graphsync/hooks.go rename {impl => transport/graphsync}/receiver.go (53%) create mode 100644 transport/graphsync/utils.go diff --git a/benchmarks/testinstance/testinstance.go b/benchmarks/testinstance/testinstance.go index 18a8e8de..24ce4691 100644 --- a/benchmarks/testinstance/testinstance.go +++ b/benchmarks/testinstance/testinstance.go @@ -164,8 +164,8 @@ func NewInstance(ctx context.Context, net tn.Network, tempDir string, diskBasedD linkSystem := storeutil.LinkSystemForBlockstore(bstore) gs := gsimpl.New(ctx, gsNet, linkSystem, gsimpl.RejectAllRequestsByDefault()) - transport := gstransport.NewTransport(p, gs) - dt, err := dtimpl.NewDataTransfer(namespace.Wrap(dstore, datastore.NewKey("/data-transfers/transfers")), dtNet, transport) + transport := gstransport.NewTransport(p, gs, dtNet) + dt, err := dtimpl.NewDataTransfer(namespace.Wrap(dstore, datastore.NewKey("/data-transfers/transfers")), p, transport) if err != nil { return Instance{}, err } diff --git a/channelmonitor/channelmonitor.go b/channelmonitor/channelmonitor.go index 6a5863e7..2eb0b5be 100644 --- a/channelmonitor/channelmonitor.go +++ b/channelmonitor/channelmonitor.go @@ -21,7 +21,6 @@ type monitorAPI interface { SubscribeToEvents(subscriber datatransfer.Subscriber) datatransfer.Unsubscribe RestartDataTransferChannel(ctx context.Context, chid datatransfer.ChannelID) error CloseDataTransferChannelWithError(ctx context.Context, chid datatransfer.ChannelID, cherr error) error - ConnectTo(context.Context, peer.ID) error PeerID() peer.ID } @@ -84,18 +83,8 @@ func checkConfig(cfg *Config) { } } -// AddPushChannel adds a push channel to the channel monitor -func (m *Monitor) AddPushChannel(chid datatransfer.ChannelID) *monitoredChannel { - return m.addChannel(chid, true) -} - -// AddPullChannel adds a pull channel to the channel monitor -func (m *Monitor) AddPullChannel(chid datatransfer.ChannelID) *monitoredChannel { - return m.addChannel(chid, false) -} - -// addChannel adds a channel to the channel monitor -func (m *Monitor) addChannel(chid datatransfer.ChannelID, isPush bool) *monitoredChannel { +// AddChannel adds a channel to the channel monitor +func (m *Monitor) AddChannel(chid datatransfer.ChannelID, isPull bool) *monitoredChannel { if !m.enabled() { return nil } @@ -106,7 +95,7 @@ func (m *Monitor) addChannel(chid datatransfer.ChannelID, isPush bool) *monitore // Check if there is already a monitor for this channel if _, ok := m.channels[chid]; ok { tp := "push" - if !isPush { + if isPull { tp = "pull" } log.Warnf("ignoring add %s channel %s: %s channel with that id already exists", @@ -454,22 +443,11 @@ func (mc *monitoredChannel) doRestartChannel() error { } func (mc *monitoredChannel) sendRestartMessage(restartCount int) error { - // Establish a connection to the peer, in case the connection went down. - // Note that at the networking layer there is logic to retry if a network - // connection cannot be established, so this may take some time. p := mc.chid.OtherParty(mc.mgr.PeerID()) - log.Debugf("%s: re-establishing connection to %s", mc.chid, p) - start := time.Now() - err := mc.mgr.ConnectTo(mc.ctx, p) - if err != nil { - return xerrors.Errorf("%s: failed to reconnect to peer %s after %s: %w", - mc.chid, p, time.Since(start), err) - } - log.Debugf("%s: re-established connection to %s in %s", mc.chid, p, time.Since(start)) // Send a restart message for the channel log.Debugf("%s: sending restart message to %s (%d consecutive restarts)", mc.chid, p, restartCount) - err = mc.mgr.RestartDataTransferChannel(mc.ctx, mc.chid) + err := mc.mgr.RestartDataTransferChannel(mc.ctx, mc.chid) if err != nil { return xerrors.Errorf("%s: failed to send restart message to %s: %w", mc.chid, p, err) } diff --git a/channelmonitor/channelmonitor_test.go b/channelmonitor/channelmonitor_test.go index af91142e..aa0acc32 100644 --- a/channelmonitor/channelmonitor_test.go +++ b/channelmonitor/channelmonitor_test.go @@ -29,9 +29,6 @@ func TestChannelMonitorAutoRestart(t *testing.T) { } testCases := []testCase{{ name: "attempt restart", - }, { - name: "fail to reconnect to peer", - errReconnect: true, }, { name: "fail to send restart message", errSendRestartMsg: true, @@ -41,7 +38,7 @@ func TestChannelMonitorAutoRestart(t *testing.T) { for _, tc := range testCases { t.Run(name+": "+tc.name, func(t *testing.T) { ch := testutil.NewMockChannelState(testutil.MockChannelStateParams{ChannelID: ch1}) - mockAPI := newMockMonitorAPI(ch, tc.errReconnect, tc.errSendRestartMsg) + mockAPI := newMockMonitorAPI(ch, tc.errSendRestartMsg) triggerErrorEvent := func() { if isPush { @@ -59,9 +56,9 @@ func TestChannelMonitorAutoRestart(t *testing.T) { var mch *monitoredChannel if isPush { - mch = m.AddPushChannel(ch1) + mch = m.AddChannel(ch1, false) } else { - mch = m.AddPullChannel(ch1) + mch = m.AddChannel(ch1, true) } // Simulate the responder sending Accept @@ -115,7 +112,7 @@ func TestChannelMonitorMaxConsecutiveRestarts(t *testing.T) { runTest := func(name string, isPush bool) { t.Run(name, func(t *testing.T) { ch := testutil.NewMockChannelState(testutil.MockChannelStateParams{ChannelID: ch1}) - mockAPI := newMockMonitorAPI(ch, false, false) + mockAPI := newMockMonitorAPI(ch, false) triggerErrorEvent := func() { if isPush { @@ -134,12 +131,12 @@ func TestChannelMonitorMaxConsecutiveRestarts(t *testing.T) { var mch *monitoredChannel if isPush { - mch = m.AddPushChannel(ch1) + mch = m.AddChannel(ch1, false) mockAPI.dataQueued(10) mockAPI.dataSent(5) } else { - mch = m.AddPullChannel(ch1) + mch = m.AddChannel(ch1, true) mockAPI.dataReceived(5) } @@ -198,7 +195,7 @@ func TestChannelMonitorQueuedRestart(t *testing.T) { runTest := func(name string, isPush bool) { t.Run(name, func(t *testing.T) { ch := testutil.NewMockChannelState(testutil.MockChannelStateParams{ChannelID: ch1}) - mockAPI := newMockMonitorAPI(ch, false, false) + mockAPI := newMockMonitorAPI(ch, false) triggerErrorEvent := func() { if isPush { @@ -216,12 +213,12 @@ func TestChannelMonitorQueuedRestart(t *testing.T) { }) if isPush { - m.AddPushChannel(ch1) + m.AddChannel(ch1, false) mockAPI.dataQueued(10) mockAPI.dataSent(5) } else { - m.AddPullChannel(ch1) + m.AddChannel(ch1, true) mockAPI.dataReceived(5) } @@ -285,7 +282,7 @@ func TestChannelMonitorTimeouts(t *testing.T) { for _, tc := range testCases { t.Run(name+": "+tc.name, func(t *testing.T) { ch := testutil.NewMockChannelState(testutil.MockChannelStateParams{ChannelID: ch1}) - mockAPI := newMockMonitorAPI(ch, false, false) + mockAPI := newMockMonitorAPI(ch, false) verifyClosedAndShutdown := func(chCtx context.Context, timeout time.Duration) { mockAPI.verifyChannelClosed(t, true) @@ -310,10 +307,10 @@ func TestChannelMonitorTimeouts(t *testing.T) { var chCtx context.Context if isPush { - mch := m.AddPushChannel(ch1) + mch := m.AddChannel(ch1, false) chCtx = mch.ctx } else { - mch := m.AddPullChannel(ch1) + mch := m.AddChannel(ch1, true) chCtx = mch.ctx } @@ -370,7 +367,6 @@ func verifyChannelShutdown(t *testing.T, shutdownCtx context.Context) { type mockMonitorAPI struct { ch *testutil.MockChannelState - connectErrors bool restartErrors bool restartMessages chan struct{} closeErr chan error @@ -379,10 +375,9 @@ type mockMonitorAPI struct { subscribers map[int]datatransfer.Subscriber } -func newMockMonitorAPI(ch *testutil.MockChannelState, errOnReconnect, errOnRestart bool) *mockMonitorAPI { +func newMockMonitorAPI(ch *testutil.MockChannelState, errOnRestart bool) *mockMonitorAPI { return &mockMonitorAPI{ ch: ch, - connectErrors: errOnReconnect, restartErrors: errOnRestart, restartMessages: make(chan struct{}, 100), closeErr: make(chan error, 1), @@ -414,13 +409,6 @@ func (m *mockMonitorAPI) fireEvent(e datatransfer.Event, state datatransfer.Chan } } -func (m *mockMonitorAPI) ConnectTo(ctx context.Context, id peer.ID) error { - if m.connectErrors { - return xerrors.Errorf("connect err") - } - return nil -} - func (m *mockMonitorAPI) PeerID() peer.ID { return "p" } diff --git a/channels/channels.go b/channels/channels.go index 63a36d9b..1d289aa2 100644 --- a/channels/channels.go +++ b/channels/channels.go @@ -23,19 +23,6 @@ import ( type Notifier func(datatransfer.Event, datatransfer.ChannelState) -// ErrNotFound is returned when a channel cannot be found with a given channel ID -type ErrNotFound struct { - ChannelID datatransfer.ChannelID -} - -func (e *ErrNotFound) Error() string { - return "No channel for channel ID " + e.ChannelID.String() -} - -func NewErrNotFound(chid datatransfer.ChannelID) error { - return &ErrNotFound{ChannelID: chid} -} - // ErrWrongType is returned when a caller attempts to change the type of implementation data after setting it var ErrWrongType = errors.New("Cannot change type of implementation specific data after setting it") @@ -50,8 +37,6 @@ type Channels struct { // ChannelEnvironment -- just a proxy for DTNetwork for now type ChannelEnvironment interface { - Protect(id peer.ID, tag string) - Unprotect(id peer.ID, tag string) bool ID() peer.ID CleanupChannel(chid datatransfer.ChannelID) } @@ -109,7 +94,7 @@ func (c *Channels) dispatch(eventName fsm.EventName, channel fsm.StateType) { // CreateNew creates a new channel id and channel state and saves to channels. // returns error if the channel exists already. -func (c *Channels) CreateNew(selfPeer peer.ID, tid datatransfer.TransferID, baseCid cid.Cid, selector datamodel.Node, voucher datatransfer.TypedVoucher, initiator, dataSender, dataReceiver peer.ID) (datatransfer.ChannelID, error) { +func (c *Channels) CreateNew(selfPeer peer.ID, tid datatransfer.TransferID, baseCid cid.Cid, selector datamodel.Node, voucher datatransfer.TypedVoucher, initiator, dataSender, dataReceiver peer.ID) (datatransfer.ChannelID, datatransfer.Channel, error) { var responder peer.ID if dataSender == initiator { responder = dataReceiver @@ -117,7 +102,7 @@ func (c *Channels) CreateNew(selfPeer peer.ID, tid datatransfer.TransferID, base responder = dataSender } chid := datatransfer.ChannelID{Initiator: initiator, Responder: responder, ID: tid} - err := c.stateMachines.Begin(chid, &internal.ChannelState{ + channel := &internal.ChannelState{ SelfPeer: selfPeer, TransferID: tid, Initiator: initiator, @@ -134,13 +119,14 @@ func (c *Channels) CreateNew(selfPeer peer.ID, tid datatransfer.TransferID, base }, }, Status: datatransfer.Requested, - }) + } + err := c.stateMachines.Begin(chid, channel) if err != nil { log.Errorw("failed to create new tracking channel for data-transfer", "channelID", chid, "err", err) - return datatransfer.ChannelID{}, err + return datatransfer.ChannelID{}, nil, err } log.Debugw("created tracking channel for data-transfer, emitting channel Open event", "channelID", chid) - return chid, c.stateMachines.Send(chid, datatransfer.Open) + return chid, c.fromInternalChannelState(*channel), c.stateMachines.Send(chid, datatransfer.Open) } // InProgress returns a list of in progress channels @@ -164,7 +150,7 @@ func (c *Channels) GetByID(ctx context.Context, chid datatransfer.ChannelID) (da var internalChannel internal.ChannelState err := c.stateMachines.GetSync(ctx, chid, &internalChannel) if err != nil { - return nil, NewErrNotFound(chid) + return nil, datatransfer.ErrChannelNotFound } return c.fromInternalChannelState(internalChannel), nil } @@ -448,7 +434,7 @@ func (c *Channels) checkChannelExists(chid datatransfer.ChannelID, code datatran } if !has { return xerrors.Errorf("cannot send FSM event %s to data-transfer channel %s: %w", - datatransfer.Events[code], chid, NewErrNotFound(chid)) + datatransfer.Events[code], chid, datatransfer.ErrChannelNotFound) } return nil } diff --git a/channels/channels_fsm.go b/channels/channels_fsm.go index 61a844a5..4a095054 100644 --- a/channels/channels_fsm.go +++ b/channels/channels_fsm.go @@ -277,7 +277,6 @@ func cleanupConnection(ctx fsm.Context, env ChannelEnvironment, channel internal otherParty = channel.Responder } env.CleanupChannel(datatransfer.ChannelID{ID: channel.TransferID, Initiator: channel.Initiator, Responder: channel.Responder}) - env.Unprotect(otherParty, datatransfer.ChannelID{ID: channel.TransferID, Initiator: channel.Initiator, Responder: channel.Responder}.String()) return ctx.Trigger(datatransfer.CleanupComplete) } diff --git a/channels/channels_test.go b/channels/channels_test.go index 2d4c39c3..23c16ef8 100644 --- a/channels/channels_test.go +++ b/channels/channels_test.go @@ -43,19 +43,19 @@ func TestChannels(t *testing.T) { err = channelList.Start(ctx) require.NoError(t, err) t.Run("adding channels", func(t *testing.T) { - chid, err := channelList.CreateNew(peers[0], tid1, cids[0], selector, fv1, peers[0], peers[0], peers[1]) + chid, _, err := channelList.CreateNew(peers[0], tid1, cids[0], selector, fv1, peers[0], peers[0], peers[1]) require.NoError(t, err) require.Equal(t, peers[0], chid.Initiator) require.Equal(t, tid1, chid.ID) // cannot add twice for same channel id - _, err = channelList.CreateNew(peers[0], tid1, cids[1], selector, fv2, peers[0], peers[1], peers[0]) + _, _, err = channelList.CreateNew(peers[0], tid1, cids[1], selector, fv2, peers[0], peers[1], peers[0]) require.Error(t, err) state := checkEvent(ctx, t, received, datatransfer.Open) require.Equal(t, datatransfer.Requested, state.Status()) // can add for different id - chid, err = channelList.CreateNew(peers[2], tid2, cids[1], selector, fv2, peers[3], peers[2], peers[3]) + chid, _, err = channelList.CreateNew(peers[2], tid2, cids[1], selector, fv2, peers[3], peers[2], peers[3]) require.NoError(t, err) require.Equal(t, peers[3], chid.Initiator) require.Equal(t, tid2, chid.ID) @@ -87,7 +87,7 @@ func TestChannels(t *testing.T) { // empty if channel does not exist state, err = channelList.GetByID(ctx, datatransfer.ChannelID{Initiator: peers[1], Responder: peers[1], ID: tid1}) require.Equal(t, nil, state) - require.True(t, xerrors.As(err, new(*channels.ErrNotFound))) + require.True(t, errors.Is(err, datatransfer.ErrChannelNotFound)) // works for other channel as well state, err = channelList.GetByID(ctx, datatransfer.ChannelID{Initiator: peers[3], Responder: peers[2], ID: tid2}) @@ -107,7 +107,7 @@ func TestChannels(t *testing.T) { require.Equal(t, state.Status(), datatransfer.Ongoing) err = channelList.Accept(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}) - require.True(t, xerrors.As(err, new(*channels.ErrNotFound))) + require.True(t, errors.Is(err, datatransfer.ErrChannelNotFound)) }) t.Run("transfer queued", func(t *testing.T) { @@ -129,7 +129,7 @@ func TestChannels(t *testing.T) { err = channelList.Start(ctx) require.NoError(t, err) - chid, err := channelList.CreateNew(peers[0], tid1, cids[0], selector, fv1, peers[0], peers[0], peers[1]) + chid, _, err := channelList.CreateNew(peers[0], tid1, cids[0], selector, fv1, peers[0], peers[0], peers[1]) require.NoError(t, err) checkEvent(ctx, t, received, datatransfer.Open) require.NoError(t, channelList.Accept(chid)) @@ -161,7 +161,7 @@ func TestChannels(t *testing.T) { err = channelList.Start(ctx) require.NoError(t, err) - _, err = channelList.CreateNew(peers[0], tid1, cids[0], selector, fv1, peers[0], peers[0], peers[1]) + _, _, err = channelList.CreateNew(peers[0], tid1, cids[0], selector, fv1, peers[0], peers[0], peers[1]) require.NoError(t, err) state := checkEvent(ctx, t, received, datatransfer.Open) require.Equal(t, datatransfer.Requested, state.Status()) @@ -191,9 +191,9 @@ func TestChannels(t *testing.T) { // errors if channel does not exist err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[1], 200, 2, true) - require.True(t, xerrors.As(err, new(*channels.ErrNotFound))) + require.True(t, errors.Is(err, datatransfer.ErrChannelNotFound)) err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[1], 200, 2, true) - require.True(t, xerrors.As(err, new(*channels.ErrNotFound))) + require.True(t, errors.Is(err, datatransfer.ErrChannelNotFound)) err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 50, 2, true) require.NoError(t, err) @@ -223,7 +223,7 @@ func TestChannels(t *testing.T) { err = channelList.Start(ctx) require.NoError(t, err) - _, err = channelList.CreateNew(peers[0], tid1, cids[0], selector, fv1, peers[1], peers[0], peers[1]) + _, _, err = channelList.CreateNew(peers[0], tid1, cids[0], selector, fv1, peers[1], peers[0], peers[1]) require.NoError(t, err) state := checkEvent(ctx, t, received, datatransfer.Open) @@ -361,7 +361,7 @@ func TestChannels(t *testing.T) { state = checkEvent(ctx, t, received, datatransfer.CleanupComplete) require.Equal(t, datatransfer.Failed, state.Status()) - chid, err := channelList.CreateNew(peers[0], tid2, cids[1], selector, fv2, peers[2], peers[1], peers[2]) + chid, _, err := channelList.CreateNew(peers[0], tid2, cids[1], selector, fv2, peers[2], peers[1], peers[2]) require.NoError(t, err) require.Equal(t, peers[2], chid.Initiator) require.Equal(t, tid2, chid.ID) @@ -378,7 +378,7 @@ func TestChannels(t *testing.T) { t.Run("test self peer and other peer", func(t *testing.T) { // sender is self peer - chid, err := channelList.CreateNew(peers[1], tid1, cids[0], selector, fv1, peers[1], peers[1], peers[2]) + chid, _, err := channelList.CreateNew(peers[1], tid1, cids[0], selector, fv1, peers[1], peers[1], peers[2]) require.NoError(t, err) ch, err := channelList.GetByID(context.Background(), chid) require.NoError(t, err) @@ -386,7 +386,7 @@ func TestChannels(t *testing.T) { require.Equal(t, peers[2], ch.OtherPeer()) // recipient is self peer - chid, err = channelList.CreateNew(peers[2], datatransfer.TransferID(1001), cids[0], selector, fv1, peers[1], peers[2], peers[1]) + chid, _, err = channelList.CreateNew(peers[2], datatransfer.TransferID(1001), cids[0], selector, fv1, peers[1], peers[2], peers[1]) require.NoError(t, err) ch, err = channelList.GetByID(context.Background(), chid) require.NoError(t, err) @@ -405,7 +405,7 @@ func TestChannels(t *testing.T) { err = channelList.Start(ctx) require.NoError(t, err) - chid, err := channelList.CreateNew(peers[3], tid1, cids[0], selector, fv1, peers[3], peers[0], peers[3]) + chid, _, err := channelList.CreateNew(peers[3], tid1, cids[0], selector, fv1, peers[3], peers[0], peers[3]) require.NoError(t, err) state := checkEvent(ctx, t, received, datatransfer.Open) require.Equal(t, datatransfer.Requested, state.Status()) @@ -420,7 +420,7 @@ func TestChannels(t *testing.T) { t.Run("test self peer and other peer", func(t *testing.T) { peers := testutil.GeneratePeers(3) // sender is self peer - chid, err := channelList.CreateNew(peers[1], tid1, cids[0], selector, fv1, peers[1], peers[1], peers[2]) + chid, _, err := channelList.CreateNew(peers[1], tid1, cids[0], selector, fv1, peers[1], peers[1], peers[2]) require.NoError(t, err) ch, err := channelList.GetByID(context.Background(), chid) require.NoError(t, err) @@ -428,7 +428,7 @@ func TestChannels(t *testing.T) { require.Equal(t, peers[2], ch.OtherPeer()) // recipient is self peer - chid, err = channelList.CreateNew(peers[2], datatransfer.TransferID(1001), cids[0], selector, fv1, peers[1], peers[2], peers[1]) + chid, _, err = channelList.CreateNew(peers[2], datatransfer.TransferID(1001), cids[0], selector, fv1, peers[1], peers[2], peers[1]) require.NoError(t, err) ch, err = channelList.GetByID(context.Background(), chid) require.NoError(t, err) diff --git a/go.mod b/go.mod index ea25b21f..972e065e 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-cid v0.2.0 github.com/ipfs/go-datastore v0.5.1 github.com/ipfs/go-ds-badger v0.3.0 - github.com/ipfs/go-graphsync v0.13.1 + github.com/ipfs/go-graphsync v0.13.2-0.20220531040852-fa5a9f2d7a86 github.com/ipfs/go-ipfs-blockstore v1.1.2 github.com/ipfs/go-ipfs-blocksutil v0.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 diff --git a/go.sum b/go.sum index 34d3a7c4..d34f268a 100644 --- a/go.sum +++ b/go.sum @@ -441,8 +441,8 @@ github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIyk github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= -github.com/ipfs/go-graphsync v0.13.1 h1:lWiP/WLycoPUYyj3IDEi1GJNP30kFuYOvimcfeuZyQs= -github.com/ipfs/go-graphsync v0.13.1/go.mod h1:y8e8G6CmZeL9Srvx1l15CtGiRdf3h5JdQuqPz/iYL0A= +github.com/ipfs/go-graphsync v0.13.2-0.20220531040852-fa5a9f2d7a86 h1:PVLY+D9dz9SwQADbEaxLF5Kc+xOVP+SltDw3GvSdHmk= +github.com/ipfs/go-graphsync v0.13.2-0.20220531040852-fa5a9f2d7a86/go.mod h1:y8e8G6CmZeL9Srvx1l15CtGiRdf3h5JdQuqPz/iYL0A= github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE= github.com/ipfs/go-ipfs-blockstore v1.1.2 h1:WCXoZcMYnvOTmlpX+RSSnhVN0uCmbWTeepTGX5lgiXw= github.com/ipfs/go-ipfs-blockstore v1.1.2/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY= diff --git a/impl/environment.go b/impl/environment.go index 2e774f12..4753344f 100644 --- a/impl/environment.go +++ b/impl/environment.go @@ -10,16 +10,8 @@ type channelEnvironment struct { m *manager } -func (ce *channelEnvironment) Protect(id peer.ID, tag string) { - ce.m.dataTransferNetwork.Protect(id, tag) -} - -func (ce *channelEnvironment) Unprotect(id peer.ID, tag string) bool { - return ce.m.dataTransferNetwork.Unprotect(id, tag) -} - func (ce *channelEnvironment) ID() peer.ID { - return ce.m.dataTransferNetwork.ID() + return ce.m.peerID } func (ce *channelEnvironment) CleanupChannel(chid datatransfer.ChannelID) { diff --git a/impl/events.go b/impl/events.go index 3ac2e64c..e81e0fee 100644 --- a/impl/events.go +++ b/impl/events.go @@ -2,6 +2,7 @@ package impl import ( "context" + "fmt" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" @@ -11,6 +12,7 @@ import ( "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channels" "github.com/filecoin-project/go-data-transfer/v2/message" ) @@ -51,7 +53,7 @@ func (m *manager) OnDataReceived(chid datatransfer.ChannelID, link ipld.Link, si if err == datatransfer.ErrPause { msg := message.UpdateResponse(chid.ID, true) ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) - if err := m.dataTransferNetwork.SendMessage(ctx, chid.Initiator, msg); err != nil { + if err := m.transport.SendMessage(ctx, chid, msg); err != nil { return err } } @@ -277,7 +279,7 @@ func (m *manager) OnChannelCompleted(chid datatransfer.ChannelID, completeErr er } log.Infow("sending completion message to initiator", "chid", chid) ctx, _ := m.spansIndex.SpanForChannel(context.Background(), chid) - if err := m.dataTransferNetwork.SendMessage(ctx, chid.Initiator, msg); err != nil { + if err := m.transport.SendMessage(ctx, chid, msg); err != nil { err := xerrors.Errorf("channel %s: failed to send completion message to initiator: %w", chid, err) log.Warnw("failed to send completion message to initiator", "chid", chid, "err", err) return m.OnRequestDisconnected(chid, err) @@ -299,3 +301,23 @@ func (m *manager) OnContextAugment(chid datatransfer.ChannelID) func(context.Con return ctx } } + +func (m *manager) OnRestartExistingChannelRequestReceived(chid datatransfer.ChannelID) error { + ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) + // validate channel exists -> in non-terminal state and that the sender matches + channel, err := m.channels.GetByID(context.TODO(), chid) + if err != nil || channel == nil { + // nothing to do here, we wont handle the request + return err + } + + // channel should NOT be terminated + if channels.IsChannelTerminated(channel.Status()) { + return fmt.Errorf("cannot restart channel %s: channel already terminated", chid) + } + + if err := m.openRestartChannel(ctx, channel); err != nil { + return fmt.Errorf("failed to open restart channel %s: %s", chid, err) + } + return nil +} diff --git a/impl/impl.go b/impl/impl.go index 61f0fe67..16a6f462 100644 --- a/impl/impl.go +++ b/impl/impl.go @@ -11,7 +11,6 @@ import ( "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" "github.com/ipld/go-ipld-prime/datamodel" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/libp2p/go-libp2p-core/peer" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -24,7 +23,6 @@ import ( "github.com/filecoin-project/go-data-transfer/v2/channels" "github.com/filecoin-project/go-data-transfer/v2/message" "github.com/filecoin-project/go-data-transfer/v2/message/types" - "github.com/filecoin-project/go-data-transfer/v2/network" "github.com/filecoin-project/go-data-transfer/v2/registry" "github.com/filecoin-project/go-data-transfer/v2/tracing" ) @@ -33,7 +31,6 @@ var log = logging.Logger("dt-impl") var cancelSendTimeout = 30 * time.Second type manager struct { - dataTransferNetwork network.DataTransferNetwork validatedTypes *registry.Registry transportConfigurers *registry.Registry pubSub *pubsub.PubSub @@ -90,20 +87,19 @@ func ChannelRestartConfig(cfg channelmonitor.Config) DataTransferOption { } // NewDataTransfer initializes a new instance of a data transfer manager -func NewDataTransfer(ds datastore.Batching, dataTransferNetwork network.DataTransferNetwork, transport datatransfer.Transport, options ...DataTransferOption) (datatransfer.Manager, error) { +func NewDataTransfer(ds datastore.Batching, peerID peer.ID, transport datatransfer.Transport, options ...DataTransferOption) (datatransfer.Manager, error) { m := &manager{ - dataTransferNetwork: dataTransferNetwork, validatedTypes: registry.NewRegistry(), transportConfigurers: registry.NewRegistry(), pubSub: pubsub.New(dispatcher), readySub: pubsub.New(readyDispatcher), - peerID: dataTransferNetwork.ID(), + peerID: peerID, transport: transport, transferIDGen: newTimeCounter(), spansIndex: tracing.NewSpansIndex(), } - channels, err := channels.New(ds, m.notifier, &channelEnvironment{m}, dataTransferNetwork.ID()) + channels, err := channels.New(ds, m.notifier, &channelEnvironment{m}, peerID) if err != nil { return nil, err } @@ -143,8 +139,6 @@ func (m *manager) Start(ctx context.Context) error { } }() - dtReceiver := &receiver{m} - m.dataTransferNetwork.SetDelegate(dtReceiver) return m.transport.SetEventHandler(m) } @@ -184,21 +178,26 @@ func (m *manager) OpenPushDataChannel(ctx context.Context, requestTo peer.ID, vo return datatransfer.ChannelID{}, err } - chid, err := m.channels.CreateNew(m.peerID, req.TransferID(), baseCid, selector, voucher, + chid, channel, err := m.channels.CreateNew(m.peerID, req.TransferID(), baseCid, selector, voucher, m.peerID, m.peerID, requestTo) // initiator = us, sender = us, receiver = them if err != nil { return chid, err } + return chid, m.openChannel(ctx, channel, req) +} + +func (m *manager) openChannel(ctx context.Context, channel datatransfer.Channel, request datatransfer.Request) error { + chid := channel.ChannelID() + voucher := channel.Voucher() ctx, span := m.spansIndex.SpanForChannel(ctx, chid) processor, has := m.transportConfigurers.Processor(voucher.Type) if has { transportConfigurer := processor.(datatransfer.TransportConfigurer) transportConfigurer(chid, voucher, m.transport) } - m.dataTransferNetwork.Protect(requestTo, chid.String()) - monitoredChan := m.channelMonitor.AddPushChannel(chid) - if err := m.dataTransferNetwork.SendMessage(ctx, requestTo, req); err != nil { - err = fmt.Errorf("unable to send request: %w", err) + monitoredChan := m.channelMonitor.AddChannel(chid, channel.IsPull()) + if err := m.transport.OpenChannel(ctx, channel, request); err != nil { + err = fmt.Errorf("Unable to send request: %w", err) span.RecordError(err) span.SetStatus(codes.Error, err.Error()) _ = m.channels.Error(chid, err) @@ -209,10 +208,10 @@ func (m *manager) OpenPushDataChannel(ctx context.Context, requestTo peer.ID, vo monitoredChan.Shutdown() } - return chid, err + return err } - return chid, nil + return nil } // OpenPullDataChannel opens a data transfer that will request data from the sending peer and @@ -225,38 +224,20 @@ func (m *manager) OpenPullDataChannel(ctx context.Context, requestTo peer.ID, vo return datatransfer.ChannelID{}, err } // initiator = us, sender = them, receiver = us - chid, err := m.channels.CreateNew(m.peerID, req.TransferID(), baseCid, selector, voucher, + chid, channel, err := m.channels.CreateNew(m.peerID, req.TransferID(), baseCid, selector, voucher, m.peerID, requestTo, m.peerID) if err != nil { return chid, err } - ctx, span := m.spansIndex.SpanForChannel(ctx, chid) - processor, has := m.transportConfigurers.Processor(voucher.Type) - if has { - transportConfigurer := processor.(datatransfer.TransportConfigurer) - transportConfigurer(chid, voucher, m.transport) - } - m.dataTransferNetwork.Protect(requestTo, chid.String()) - monitoredChan := m.channelMonitor.AddPullChannel(chid) - if err := m.transport.OpenChannel(ctx, requestTo, chid, cidlink.Link{Cid: baseCid}, selector, nil, req); err != nil { - err = fmt.Errorf("unable to send request: %w", err) - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - _ = m.channels.Error(chid, err) - - // If pull channel monitoring is enabled, shutdown the monitor as it - // wasn't possible to start the data transfer - if monitoredChan != nil { - monitoredChan.Shutdown() - } - return chid, err - } - return chid, nil + return chid, m.openChannel(ctx, channel, req) } // SendVoucher sends an intermediate voucher as needed when the receiver sends a request for revalidation func (m *manager) SendVoucher(ctx context.Context, channelID datatransfer.ChannelID, voucher datatransfer.TypedVoucher) error { - chst, err := m.channels.GetByID(ctx, channelID) + has, err := m.channels.HasChannel(channelID) + if !has { + return datatransfer.ErrChannelNotFound + } if err != nil { return err } @@ -277,8 +258,8 @@ func (m *manager) SendVoucher(ctx context.Context, channelID datatransfer.Channe span.SetStatus(codes.Error, err.Error()) return err } - if err := m.dataTransferNetwork.SendMessage(ctx, chst.OtherPeer(), updateRequest); err != nil { - err = fmt.Errorf("unable to send request: %w", err) + if err := m.transport.SendMessage(ctx, channelID, updateRequest); err != nil { + err = fmt.Errorf("Unable to send request: %w", err) _ = m.OnRequestDisconnected(channelID, err) span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -316,8 +297,8 @@ func (m *manager) SendVoucherResult(ctx context.Context, channelID datatransfer. span.SetStatus(codes.Error, err.Error()) return err } - if err := m.dataTransferNetwork.SendMessage(ctx, chst.OtherPeer(), updateResponse); err != nil { - err = fmt.Errorf("unable to send request: %w", err) + if err := m.transport.SendMessage(ctx, channelID, updateResponse); err != nil { + err = fmt.Errorf("Unable to send request: %w", err) _ = m.OnRequestDisconnected(channelID, err) span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -343,6 +324,7 @@ func (m *manager) UpdateValidationStatus(ctx context.Context, chid datatransfer. // updateValidationStatus is the implementation of the public method, which wraps this private method // in a trace func (m *manager) updateValidationStatus(ctx context.Context, chid datatransfer.ChannelID, result datatransfer.ValidationResult) error { + // first check if we are the responder -- only the responder can call UpdateValidationStatus if chid.Initiator == m.peerID { err := errors.New("cannot send voucher result for request we initiated") @@ -353,13 +335,17 @@ func (m *manager) updateValidationStatus(ctx context.Context, chid datatransfer. chst, response, err := m.processValidationUpdate(ctx, chid, result) // dispatch transport updates - return m.handleTransportUpdate(ctx, chst, response, result, err) + return m.transport.UpdateChannel(ctx, chid, datatransfer.ChannelUpdate{ + Paused: result.LeaveRequestPaused(chst), + Closed: err != nil || !result.Accepted, + SendMessage: response, + }) } func (m *manager) processValidationUpdate(ctx context.Context, chid datatransfer.ChannelID, result datatransfer.ValidationResult) (datatransfer.ChannelState, datatransfer.Response, error) { // read the channel state - chst, err := m.channels.GetByID(ctx, chid) + chst, err := m.channels.GetByID(context.TODO(), chid) if err != nil { return nil, nil, err } @@ -389,47 +375,6 @@ func (m *manager) processValidationUpdate(ctx context.Context, chid datatransfer return chst, response, nil } -// handleTransportUpdate updates the transport based on the validation status and the -// response message -// TODO: the ordering here is a bit sensitive, and the transport should -// be refactored to accept multiple operations at once and order these itself -func (m *manager) handleTransportUpdate( - ctx context.Context, - chst datatransfer.ChannelState, - response datatransfer.Message, - result datatransfer.ValidationResult, - resultErr error, -) error { - - pauseRequest := result.LeaveRequestPaused(chst) - // resume channel as needed, sending the response message immediately and returning - if resultErr == nil && result.Accepted && !pauseRequest { - if chst.Status().IsResponderPaused() && !chst.Status().InFinalization() { - return m.transport.(datatransfer.PauseableTransport).ResumeChannel(ctx, response, chst.ChannelID()) - } - } - - // send a response message - if response != nil { - if err := m.dataTransferNetwork.SendMessage(ctx, chst.ChannelID().Initiator, response); err != nil { - return err - } - } - - // close the channel as needed - if resultErr != nil || !result.Accepted { - m.transport.CloseChannel(ctx, chst.ChannelID()) - return resultErr - } - - // pause the channel as needed - if pauseRequest && !chst.Status().IsResponderPaused() && !chst.Status().InFinalization() { - return m.transport.(datatransfer.PauseableTransport).PauseChannel(ctx, chst.ChannelID()) - } - - return nil -} - // close an open channel (effectively a cancel) func (m *manager) CloseDataTransferChannel(ctx context.Context, chid datatransfer.ChannelID) error { log.Infof("close channel %s", chid) @@ -444,7 +389,10 @@ func (m *manager) CloseDataTransferChannel(ctx context.Context, chid datatransfe )) defer span.End() // Close the channel on the local transport - err = m.transport.CloseChannel(ctx, chid) + err = m.transport.UpdateChannel(ctx, chid, datatransfer.ChannelUpdate{ + Paused: chst.Status().IsResponderPaused(), + Closed: true, + }) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) @@ -456,7 +404,7 @@ func (m *manager) CloseDataTransferChannel(ctx context.Context, chid datatransfe sctx, cancel := context.WithTimeout(context.Background(), cancelSendTimeout) defer cancel() log.Infof("%s: sending cancel channel to %s for channel %s", m.peerID, chst.OtherPeer(), chid) - err = m.dataTransferNetwork.SendMessage(sctx, chst.OtherPeer(), m.cancelMessage(chid)) + err = m.transport.SendMessage(sctx, chid, m.cancelMessage(chid)) if err != nil { err = fmt.Errorf("unable to send cancel message for channel %s to peer %s: %w", chid, m.peerID, err) @@ -474,12 +422,6 @@ func (m *manager) CloseDataTransferChannel(ctx context.Context, chid datatransfe return nil } -// ConnectTo opens a connection to a peer on the data-transfer protocol, -// retrying if necessary -func (m *manager) ConnectTo(ctx context.Context, p peer.ID) error { - return m.dataTransferNetwork.ConnectWithRetry(ctx, p) -} - // close an open channel and fire an error event func (m *manager) CloseDataTransferChannelWithError(ctx context.Context, chid datatransfer.ChannelID, cherr error) error { log.Infof("close channel %s with error %s", chid, cherr) @@ -494,24 +436,21 @@ func (m *manager) CloseDataTransferChannelWithError(ctx context.Context, chid da )) defer span.End() - // Cancel the channel on the local transport - err = m.transport.CloseChannel(ctx, chid) - if err != nil { - log.Warnf("unable to close channel %s: %s", chid, err) - } - - // Try to send a cancel message to the remote peer. It's quite likely - // we aren't able to send the message to the peer because the channel - // is already in an error state, which is probably because of connection - // issues, so if we cant send the message just log a warning. + // Close transfport and try to send a cancel message to the remote peer. + // It's quite likely we aren't able to send the message to the peer because + // the channel is already in an error state, which is probably because of + // connection issues, so if we cant send the message just log a warning. log.Infof("%s: sending cancel channel to %s for channel %s", m.peerID, chst.OtherPeer(), chid) - err = m.dataTransferNetwork.SendMessage(ctx, chst.OtherPeer(), m.cancelMessage(chid)) + err = m.transport.UpdateChannel(ctx, chid, datatransfer.ChannelUpdate{ + Paused: chst.Status().IsResponderPaused(), + Closed: true, + SendMessage: m.cancelMessage(chid), + }) if err != nil { // Just log a warning here because it's important that we fire the // error event with the original error so that it doesn't get masked // by subsequent errors. - log.Warnf("unable to send cancel message for channel %s to peer %s: %w", - chid, m.peerID, err) + log.Warnf("unable to close channel %s: %s", chid, err) } // Fire an error event @@ -527,24 +466,20 @@ func (m *manager) CloseDataTransferChannelWithError(ctx context.Context, chid da func (m *manager) PauseDataTransferChannel(ctx context.Context, chid datatransfer.ChannelID) error { log.Infof("pause channel %s", chid) - pausable, ok := m.transport.(datatransfer.PauseableTransport) - if !ok { + if !m.transport.Capabilities().Pausable { return datatransfer.ErrUnsupported } ctx, _ = m.spansIndex.SpanForChannel(ctx, chid) - err := pausable.PauseChannel(ctx, chid) + err := m.transport.UpdateChannel(ctx, chid, datatransfer.ChannelUpdate{ + Paused: true, + SendMessage: m.pauseMessage(chid), + }) if err != nil { log.Warnf("Error attempting to pause at transport level: %s", err.Error()) } - if err := m.dataTransferNetwork.SendMessage(ctx, chid.OtherParty(m.peerID), m.pauseMessage(chid)); err != nil { - err = fmt.Errorf("unable to send pause message: %w", err) - _ = m.OnRequestDisconnected(chid, err) - return err - } - return m.pause(chid) } @@ -552,14 +487,16 @@ func (m *manager) PauseDataTransferChannel(ctx context.Context, chid datatransfe func (m *manager) ResumeDataTransferChannel(ctx context.Context, chid datatransfer.ChannelID) error { log.Infof("resume channel %s", chid) - pausable, ok := m.transport.(datatransfer.PauseableTransport) - if !ok { + if !m.transport.Capabilities().Pausable { return datatransfer.ErrUnsupported } ctx, _ = m.spansIndex.SpanForChannel(ctx, chid) - err := pausable.ResumeChannel(ctx, m.resumeMessage(chid), chid) + err := m.transport.UpdateChannel(ctx, chid, datatransfer.ChannelUpdate{ + Paused: false, + SendMessage: m.resumeMessage(chid), + }) if err != nil { log.Warnf("Error attempting to resume at transport level: %s", err.Error()) } @@ -627,40 +564,10 @@ func (m *manager) RestartDataTransferChannel(ctx context.Context, chid datatrans )) defer span.End() // initiate restart - chType := m.channelDataTransferType(channel) - switch chType { - case ManagerPeerReceivePush: - return m.restartManagerPeerReceivePush(ctx, channel) - case ManagerPeerReceivePull: - return m.restartManagerPeerReceivePull(ctx, channel) - case ManagerPeerCreatePull: - return m.openPullRestartChannel(ctx, channel) - case ManagerPeerCreatePush: - return m.openPushRestartChannel(ctx, channel) - } - - return nil -} - -func (m *manager) channelDataTransferType(channel datatransfer.ChannelState) ChannelDataTransferType { - initiator := channel.ChannelID().Initiator - if channel.IsPull() { - // we created a pull channel - if initiator == m.peerID { - return ManagerPeerCreatePull - } - - // we received a pull channel - return ManagerPeerReceivePull - } - - // we created a push channel - if initiator == m.peerID { - return ManagerPeerCreatePush + if chid.Initiator == m.peerID { + return m.openRestartChannel(ctx, channel) } - - // we received a push channel - return ManagerPeerReceivePush + return m.restartManagerPeerReceive(ctx, channel) } func (m *manager) PeerID() peer.ID { diff --git a/impl/initiating_test.go b/impl/initiating_test.go index 8a393834..04113267 100644 --- a/impl/initiating_test.go +++ b/impl/initiating_test.go @@ -13,10 +13,8 @@ import ( cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" - "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/channels" . "github.com/filecoin-project/go-data-transfer/v2/impl" "github.com/filecoin-project/go-data-transfer/v2/message" "github.com/filecoin-project/go-data-transfer/v2/testutil" @@ -37,13 +35,13 @@ func TestDataTransferInitiating(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, channelID) require.Equal(t, channelID.Initiator, h.peers[0]) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 1) - messageReceived := h.network.SentMessages[0] - require.Equal(t, messageReceived.PeerID, h.peers[1]) - received := messageReceived.Message - require.True(t, received.IsRequest()) - receivedRequest, ok := received.(datatransfer.Request) + require.Len(t, h.transport.OpenedChannels, 1) + openChannel := h.transport.OpenedChannels[0] + require.Equal(t, openChannel.Channel.ChannelID(), channelID) + require.Equal(t, openChannel.Channel.Sender(), h.peers[0]) + require.Equal(t, openChannel.Channel.BaseCID(), h.baseCid) + require.Equal(t, openChannel.Channel.Selector(), h.stor) + receivedRequest, ok := openChannel.Message.(datatransfer.Request) require.True(t, ok) require.Equal(t, receivedRequest.TransferID(), channelID.ID) require.Equal(t, receivedRequest.BaseCid(), h.baseCid) @@ -62,13 +60,12 @@ func TestDataTransferInitiating(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, channelID) require.Equal(t, channelID.Initiator, h.peers[0]) - require.Len(t, h.network.SentMessages, 0) require.Len(t, h.transport.OpenedChannels, 1) openChannel := h.transport.OpenedChannels[0] - require.Equal(t, openChannel.ChannelID, channelID) - require.Equal(t, openChannel.DataSender, h.peers[1]) - require.Equal(t, openChannel.Root, cidlink.Link{Cid: h.baseCid}) - require.Equal(t, openChannel.Selector, h.stor) + require.Equal(t, openChannel.Channel.ChannelID(), channelID) + require.Equal(t, openChannel.Channel.Sender(), h.peers[1]) + require.Equal(t, openChannel.Channel.BaseCID(), h.baseCid) + require.Equal(t, openChannel.Channel.Selector(), h.stor) require.True(t, openChannel.Message.IsRequest()) receivedRequest, ok := openChannel.Message.(datatransfer.Request) require.True(t, ok) @@ -85,7 +82,7 @@ func TestDataTransferInitiating(t *testing.T) { "SendVoucher with no channel open": { verify: func(t *testing.T, h *harness) { err := h.dt.SendVoucher(h.ctx, datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: 999999}, h.voucher) - require.True(t, xerrors.As(err, new(*channels.ErrNotFound))) + require.EqualError(t, err, datatransfer.ErrChannelNotFound.Error()) }, }, "SendVoucher with channel open, push succeeds": { @@ -96,8 +93,9 @@ func TestDataTransferInitiating(t *testing.T) { voucher := testutil.NewTestTypedVoucher() err = h.dt.SendVoucher(ctx, channelID, voucher) require.NoError(t, err) - require.Len(t, h.network.SentMessages, 2) - received := h.network.SentMessages[1].Message + require.Len(t, h.transport.OpenedChannels, 1) + require.Len(t, h.transport.MessagesSent, 1) + received := h.transport.MessagesSent[0].Message require.True(t, received.IsRequest()) receivedRequest, ok := received.(datatransfer.Request) require.True(t, ok) @@ -115,8 +113,8 @@ func TestDataTransferInitiating(t *testing.T) { err = h.dt.SendVoucher(ctx, channelID, voucher) require.NoError(t, err) require.Len(t, h.transport.OpenedChannels, 1) - require.Len(t, h.network.SentMessages, 1) - received := h.network.SentMessages[0].Message + require.Len(t, h.transport.MessagesSent, 1) + received := h.transport.MessagesSent[0].Message require.True(t, received.IsRequest()) receivedRequest, ok := received.(datatransfer.Request) require.True(t, ok) @@ -172,8 +170,9 @@ func TestDataTransferInitiating(t *testing.T) { require.NoError(t, err) require.Len(t, h.transport.PausedChannels, 1) require.Equal(t, h.transport.PausedChannels[0], channelID) - require.Len(t, h.network.SentMessages, 2) - pauseMessage := h.network.SentMessages[1].Message + require.Len(t, h.transport.OpenedChannels, 1) + require.Len(t, h.transport.MessagesSent, 1) + pauseMessage := h.transport.MessagesSent[0].Message require.True(t, pauseMessage.IsUpdate()) require.True(t, pauseMessage.IsPaused()) require.True(t, pauseMessage.IsRequest()) @@ -183,8 +182,9 @@ func TestDataTransferInitiating(t *testing.T) { require.NoError(t, err) require.Len(t, h.transport.ResumedChannels, 1) resumedChannel := h.transport.ResumedChannels[0] - require.Equal(t, resumedChannel.ChannelID, channelID) - resumeMessage := resumedChannel.Message + require.Equal(t, resumedChannel, channelID) + require.Len(t, h.transport.MessagesSent, 2) + resumeMessage := h.transport.MessagesSent[1].Message require.True(t, resumeMessage.IsUpdate()) require.False(t, resumeMessage.IsPaused()) require.True(t, resumeMessage.IsRequest()) @@ -204,9 +204,9 @@ func TestDataTransferInitiating(t *testing.T) { require.Equal(t, h.transport.ClosedChannels[0], channelID) require.Eventually(t, func() bool { - return len(h.network.SentMessages) == 2 + return len(h.transport.MessagesSent) == 1 }, 5*time.Second, 200*time.Millisecond) - cancelMessage := h.network.SentMessages[1].Message + cancelMessage := h.transport.MessagesSent[0].Message require.False(t, cancelMessage.IsUpdate()) require.False(t, cancelMessage.IsPaused()) require.True(t, cancelMessage.IsRequest()) @@ -227,9 +227,9 @@ func TestDataTransferInitiating(t *testing.T) { err = h.dt.PauseDataTransferChannel(h.ctx, channelID) require.NoError(t, err) require.Len(t, h.transport.PausedChannels, 1) - require.Equal(t, h.transport.PausedChannels[0], channelID) - require.Len(t, h.network.SentMessages, 1) - pauseMessage := h.network.SentMessages[0].Message + require.Len(t, h.transport.OpenedChannels, 1) + require.Len(t, h.transport.MessagesSent, 1) + pauseMessage := h.transport.MessagesSent[0].Message require.True(t, pauseMessage.IsUpdate()) require.True(t, pauseMessage.IsPaused()) require.True(t, pauseMessage.IsRequest()) @@ -239,8 +239,9 @@ func TestDataTransferInitiating(t *testing.T) { require.NoError(t, err) require.Len(t, h.transport.ResumedChannels, 1) resumedChannel := h.transport.ResumedChannels[0] - require.Equal(t, resumedChannel.ChannelID, channelID) - resumeMessage := resumedChannel.Message + require.Equal(t, resumedChannel, channelID) + require.Len(t, h.transport.MessagesSent, 2) + resumeMessage := h.transport.MessagesSent[1].Message require.True(t, resumeMessage.IsUpdate()) require.False(t, resumeMessage.IsPaused()) require.True(t, resumeMessage.IsRequest()) @@ -260,10 +261,10 @@ func TestDataTransferInitiating(t *testing.T) { require.Equal(t, h.transport.ClosedChannels[0], channelID) require.Eventually(t, func() bool { - return len(h.network.SentMessages) == 1 + return len(h.transport.MessagesSent) == 1 }, 5*time.Second, 200*time.Millisecond) - cancelMessage := h.network.SentMessages[0].Message + cancelMessage := h.transport.MessagesSent[0].Message require.False(t, cancelMessage.IsUpdate()) require.False(t, cancelMessage.IsPaused()) require.True(t, cancelMessage.IsRequest()) @@ -321,10 +322,9 @@ func TestDataTransferInitiating(t *testing.T) { defer cancel() h.ctx = ctx h.peers = testutil.GeneratePeers(2) - h.network = testutil.NewFakeNetwork(h.peers[0]) h.transport = testutil.NewFakeTransport() h.ds = dss.MutexWrap(datastore.NewMapDatastore()) - dt, err := NewDataTransfer(h.ds, h.network, h.transport, verify.options...) + dt, err := NewDataTransfer(h.ds, h.peers[0], h.transport, verify.options...) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt) h.dt = dt @@ -359,7 +359,6 @@ func TestDataTransferRestartInitiating(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, channelID) require.Len(t, h.transport.OpenedChannels, 1) - require.Len(t, h.network.SentMessages, 0) // some cids should already be received testCids := testutil.GenerateCids(2) @@ -371,17 +370,16 @@ func TestDataTransferRestartInitiating(t *testing.T) { // restart that pull channel err = h.dt.RestartDataTransferChannel(ctx, channelID) require.NoError(t, err) - require.Len(t, h.transport.OpenedChannels, 2) - require.Len(t, h.network.SentMessages, 0) + require.Len(t, h.transport.RestartedChannels, 1) - openChannel := h.transport.OpenedChannels[1] - require.Equal(t, openChannel.ChannelID, channelID) - require.Equal(t, openChannel.DataSender, h.peers[1]) - require.Equal(t, openChannel.Root, cidlink.Link{Cid: h.baseCid}) - require.Equal(t, openChannel.Selector, h.stor) - require.True(t, openChannel.Message.IsRequest()) + restartedChannel := h.transport.RestartedChannels[0] + require.Equal(t, restartedChannel.Channel.ChannelID(), channelID) + require.Equal(t, restartedChannel.Channel.Sender(), h.peers[1]) + require.Equal(t, restartedChannel.Channel.BaseCID(), h.baseCid) + require.Equal(t, restartedChannel.Channel.Selector(), h.stor) + require.True(t, restartedChannel.Message.IsRequest()) - receivedRequest, ok := openChannel.Message.(datatransfer.Request) + receivedRequest := restartedChannel.Message require.True(t, ok) require.Equal(t, receivedRequest.TransferID(), channelID.ID) require.Equal(t, receivedRequest.BaseCid(), h.baseCid) @@ -404,22 +402,17 @@ func TestDataTransferRestartInitiating(t *testing.T) { channelID, err := h.dt.OpenPushDataChannel(h.ctx, h.peers[1], h.voucher, h.baseCid, h.stor) require.NoError(t, err) require.NotEmpty(t, channelID) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 1) + require.Len(t, h.transport.OpenedChannels, 1) // restart that push channel err = h.dt.RestartDataTransferChannel(ctx, channelID) require.NoError(t, err) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 2) + require.Len(t, h.transport.RestartedChannels, 1) // assert restart request is well formed - messageReceived := h.network.SentMessages[1] - require.Equal(t, messageReceived.PeerID, h.peers[1]) - received := messageReceived.Message - require.True(t, received.IsRequest()) - receivedRequest, ok := received.(datatransfer.Request) - require.True(t, ok) + restartedChannel := h.transport.RestartedChannels[0] + require.Equal(t, restartedChannel.Channel.ChannelID(), channelID) + receivedRequest := restartedChannel.Message require.Equal(t, receivedRequest.TransferID(), channelID.ID) require.Equal(t, receivedRequest.BaseCid(), h.baseCid) require.False(t, receivedRequest.IsCancel()) @@ -445,26 +438,23 @@ func TestDataTransferRestartInitiating(t *testing.T) { h.voucherValidator.StubResult(datatransfer.ValidationResult{Accepted: true}) // receive a push request - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) - require.Len(t, h.transport.OpenedChannels, 1) - require.Len(t, h.network.SentMessages, 0) + chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} + h.transport.EventHandler.OnRequestReceived(chid, h.pushRequest) require.Len(t, h.voucherValidator.ValidationsReceived, 1) // restart the push request received above and validate it h.voucherValidator.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) - chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} require.NoError(t, h.dt.RestartDataTransferChannel(ctx, chid)) require.Len(t, h.voucherValidator.RevalidationsReceived, 1) - require.Len(t, h.transport.OpenedChannels, 1) - require.Len(t, h.network.SentMessages, 1) + require.Len(t, h.transport.MessagesSent, 1) // assert validation on restart vmsg := h.voucherValidator.RevalidationsReceived[0] require.Equal(t, channelID(h.id, h.peers), vmsg.ChannelID) // assert req was sent correctly - req := h.network.SentMessages[0] - require.Equal(t, req.PeerID, h.peers[1]) + req := h.transport.MessagesSent[0] + require.Equal(t, chid, req.ChannelID) received := req.Message require.True(t, received.IsRequest()) receivedRequest, ok := received.(datatransfer.Request) @@ -486,27 +476,24 @@ func TestDataTransferRestartInitiating(t *testing.T) { h.voucherValidator.ExpectSuccessPull() h.voucherValidator.StubResult(datatransfer.ValidationResult{Accepted: true}) - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pullRequest) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 1) + chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} + h.transport.EventHandler.OnRequestReceived(chid, h.pullRequest) require.Len(t, h.voucherValidator.ValidationsReceived, 1) // restart the pull request received above h.voucherValidator.ExpectSuccessValidateRestart() h.voucherValidator.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) - chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pullRequest.TransferID()} require.NoError(t, h.dt.RestartDataTransferChannel(ctx, chid)) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 2) require.Len(t, h.voucherValidator.RevalidationsReceived, 1) + require.Len(t, h.transport.MessagesSent, 1) // assert validation on restart vmsg := h.voucherValidator.RevalidationsReceived[0] require.Equal(t, channelID(h.id, h.peers), vmsg.ChannelID) // assert req was sent correctly - req := h.network.SentMessages[1] - require.Equal(t, req.PeerID, h.peers[1]) + req := h.transport.MessagesSent[0] + require.Equal(t, chid, req.ChannelID) received := req.Message require.True(t, received.IsRequest()) receivedRequest, ok := received.(datatransfer.Request) @@ -527,15 +514,13 @@ func TestDataTransferRestartInitiating(t *testing.T) { // receive a pull request h.voucherValidator.ExpectSuccessPull() h.voucherValidator.StubResult(datatransfer.ValidationResult{Accepted: true}) - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pullRequest) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 1) + chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} + h.transport.EventHandler.OnRequestReceived(chid, h.pullRequest) require.Len(t, h.voucherValidator.ValidationsReceived, 1) // restart the pull request received above h.voucherValidator.ExpectSuccessValidateRestart() h.voucherValidator.StubRestartResult(datatransfer.ValidationResult{Accepted: false}) - chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pullRequest.TransferID()} require.EqualError(t, h.dt.RestartDataTransferChannel(ctx, chid), datatransfer.ErrRejected.Error()) }, }, @@ -549,15 +534,13 @@ func TestDataTransferRestartInitiating(t *testing.T) { // receive a push request h.voucherValidator.ExpectSuccessPush() h.voucherValidator.StubResult(datatransfer.ValidationResult{Accepted: true}) - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) - require.Len(t, h.transport.OpenedChannels, 1) - require.Len(t, h.network.SentMessages, 0) + chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} + h.transport.EventHandler.OnRequestReceived(chid, h.pushRequest) require.Len(t, h.voucherValidator.ValidationsReceived, 1) // restart the pull request received above h.voucherValidator.ExpectSuccessValidateRestart() h.voucherValidator.StubRestartResult(datatransfer.ValidationResult{Accepted: false}) - chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} require.EqualError(t, h.dt.RestartDataTransferChannel(ctx, chid), datatransfer.ErrRejected.Error()) }, }, @@ -579,13 +562,12 @@ func TestDataTransferRestartInitiating(t *testing.T) { // create the harness h.ctx = ctx h.peers = testutil.GeneratePeers(2) - h.network = testutil.NewFakeNetwork(h.peers[0]) h.transport = testutil.NewFakeTransport() h.ds = dss.MutexWrap(datastore.NewMapDatastore()) h.voucherValidator = testutil.NewStubbedValidator() // setup data transfer`` - dt, err := NewDataTransfer(h.ds, h.network, h.transport) + dt, err := NewDataTransfer(h.ds, h.peers[0], h.transport) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt) h.dt = dt @@ -622,7 +604,6 @@ func TestDataTransferRestartInitiating(t *testing.T) { type harness struct { ctx context.Context peers []peer.ID - network *testutil.FakeNetwork transport *testutil.FakeTransport ds datastore.Batching dt datatransfer.Manager diff --git a/impl/receiving_requests.go b/impl/receiving_requests.go index 6974c9fd..7665a930 100644 --- a/impl/receiving_requests.go +++ b/impl/receiving_requests.go @@ -76,7 +76,7 @@ func (m *manager) acceptRequest(chid datatransfer.ChannelID, incoming datatransf } log.Infow("data-transfer request validated, will create & start tracking channel", "channelID", chid, "payloadCid", incoming.BaseCid()) - _, err = m.channels.CreateNew( + _, _, err = m.channels.CreateNew( m.peerID, incoming.TransferID(), incoming.BaseCid(), @@ -114,7 +114,6 @@ func (m *manager) acceptRequest(chid datatransfer.ChannelID, incoming datatransf transportConfigurer := processor.(datatransfer.TransportConfigurer) transportConfigurer(chid, voucher, m.transport) } - m.dataTransferNetwork.Protect(chid.Initiator, chid.String()) return result, nil } @@ -147,19 +146,19 @@ func (m *manager) restartRequest(chid datatransfer.ChannelID, return false, datatransfer.ValidationResult{}, xerrors.New("initiator cannot be manager peer for a restart request") } - // valide that the request parameters match the original request - // TODO: not sure this is needed -- the request parameters cannot change, - // so perhaps the solution is just to ignore them in the message - if err := m.validateRestartRequest(context.Background(), initiator, chid, incoming); err != nil { - return false, datatransfer.ValidationResult{}, xerrors.Errorf("restart request for channel %s failed validation: %w", chid, err) - } - // read the channel state chst, err := m.channels.GetByID(context.TODO(), chid) if err != nil { return false, datatransfer.ValidationResult{}, err } + // valide that the request parameters match the original request + // TODO: not sure this is needed -- the request parameters cannot change, + // so perhaps the solution is just to ignore them in the message + if err := m.validateRestartRequest(context.Background(), initiator, chst, incoming); err != nil { + return false, datatransfer.ValidationResult{}, xerrors.Errorf("restart request for channel %s failed validation: %w", chid, err) + } + // perform a revalidation against the last voucher result, err := m.validateRestart(chst) stayPaused := result.LeaveRequestPaused(chst) @@ -196,7 +195,6 @@ func (m *manager) restartRequest(chid datatransfer.ChannelID, transportConfigurer := processor.(datatransfer.TransportConfigurer) transportConfigurer(chid, typedVoucher, m.transport) } - m.dataTransferNetwork.Protect(initiator, chid.String()) return stayPaused, result, nil } diff --git a/impl/responding_test.go b/impl/responding_test.go index d3adfb5c..a7ca4e3e 100644 --- a/impl/responding_test.go +++ b/impl/responding_test.go @@ -19,7 +19,6 @@ import ( "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/channels" . "github.com/filecoin-project/go-data-transfer/v2/impl" "github.com/filecoin-project/go-data-transfer/v2/message" "github.com/filecoin-project/go-data-transfer/v2/testutil" @@ -45,7 +44,8 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) + require.NoError(t, err) require.Len(t, h.sv.ValidationsReceived, 1) validation := h.sv.ValidationsReceived[0] assert.False(t, validation.IsPull) @@ -53,16 +53,6 @@ func TestDataTransferResponding(t *testing.T) { assert.True(t, ipld.DeepEqual(h.voucher.Voucher, validation.Voucher)) assert.Equal(t, h.baseCid, validation.BaseCid) assert.Equal(t, h.stor, validation.Selector) - - require.Len(t, h.transport.OpenedChannels, 1) - openChannel := h.transport.OpenedChannels[0] - require.Equal(t, openChannel.ChannelID, channelID(h.id, h.peers)) - require.Equal(t, openChannel.DataSender, h.peers[1]) - require.Equal(t, openChannel.Root, cidlink.Link{Cid: h.baseCid}) - require.Equal(t, openChannel.Selector, h.stor) - require.False(t, openChannel.Message.IsRequest()) - response, ok := openChannel.Message.(datatransfer.Response) - require.True(t, ok) require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) @@ -79,12 +69,8 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: false, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) - require.Len(t, h.network.SentMessages, 1) - responseMessage := h.network.SentMessages[0].Message - require.False(t, responseMessage.IsRequest()) - response, ok := responseMessage.(datatransfer.Response) - require.True(t, ok) + response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) + require.EqualError(t, err, datatransfer.ErrRejected.Error()) require.False(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) @@ -101,12 +87,8 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) - require.Len(t, h.network.SentMessages, 1) - responseMessage := h.network.SentMessages[0].Message - require.False(t, responseMessage.IsRequest()) - response, ok := responseMessage.(datatransfer.Response) - require.True(t, ok) + response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) + require.Error(t, err) require.False(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) @@ -123,17 +105,8 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: true, ForcePause: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) - - require.Len(t, h.transport.OpenedChannels, 1) - openChannel := h.transport.OpenedChannels[0] - require.Equal(t, openChannel.ChannelID, channelID(h.id, h.peers)) - require.Equal(t, openChannel.DataSender, h.peers[1]) - require.Equal(t, openChannel.Root, cidlink.Link{Cid: h.baseCid}) - require.Equal(t, openChannel.Selector, h.stor) - require.False(t, openChannel.Message.IsRequest()) - response, ok := openChannel.Message.(datatransfer.Response) - require.True(t, ok) + response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) + require.EqualError(t, err, datatransfer.ErrPause.Error()) require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) @@ -141,8 +114,6 @@ func TestDataTransferResponding(t *testing.T) { require.True(t, response.IsPaused()) require.True(t, response.IsNew()) require.True(t, response.IsValidationResult()) - require.Len(t, h.transport.PausedChannels, 1) - require.Equal(t, channelID(h.id, h.peers), h.transport.PausedChannels[0]) }, }, "new pull request validates": { @@ -214,7 +185,6 @@ func TestDataTransferResponding(t *testing.T) { verify: func(t *testing.T, h *receiverHarness) { response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) require.EqualError(t, err, datatransfer.ErrPause.Error()) - require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) @@ -231,7 +201,7 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: true}) }, verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) newVoucherResult := testutil.NewTestTypedVoucher() err := h.dt.SendVoucherResult(h.ctx, channelID(h.id, h.peers), newVoucherResult) require.NoError(t, err) @@ -255,7 +225,7 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: true}) }, verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) newVoucher := testutil.NewTestTypedVoucher() err := h.dt.SendVoucher(h.ctx, channelID(h.id, h.peers), newVoucher) require.EqualError(t, err, "cannot send voucher for request we did not initiate") @@ -286,7 +256,7 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) require.NoError(t, err) }, @@ -304,7 +274,7 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pauseUpdate) require.NoError(t, err) _, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.resumeUpdate) @@ -325,7 +295,7 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pauseUpdate) require.NoError(t, err) err = h.dt.PauseDataTransferChannel(h.ctx, channelID(h.id, h.peers)) @@ -348,7 +318,7 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.cancelUpdate) require.NoError(t, err) require.Len(t, h.transport.CleanedUpChannels, 1) @@ -375,11 +345,11 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) err := h.transport.EventHandler.OnDataReceived(channelID(h.id, h.peers), cidlink.Link{Cid: testutil.GenerateCids(1)[0]}, 12345, 1, true) require.EqualError(t, err, datatransfer.ErrPause.Error()) - require.Len(t, h.network.SentMessages, 1) - response, ok := h.network.SentMessages[0].Message.(datatransfer.Response) + require.Len(t, h.transport.MessagesSent, 1) + response, ok := h.transport.MessagesSent[0].Message.(datatransfer.Response) require.True(t, ok) require.Equal(t, response.TransferID(), h.id) require.True(t, response.IsUpdate()) @@ -393,9 +363,9 @@ func TestDataTransferResponding(t *testing.T) { err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: &vr}) require.NoError(t, err) require.Len(t, h.transport.ResumedChannels, 1) - resCh := h.transport.ResumedChannels[0] - require.Equal(t, resCh.ChannelID, channelID(h.id, h.peers)) - response, ok = resCh.Message.(datatransfer.Response) + require.Equal(t, h.transport.ResumedChannels[0], channelID(h.id, h.peers)) + require.Len(t, h.transport.MessagesSent, 2) + response, ok = h.transport.MessagesSent[1].Message.(datatransfer.Response) require.True(t, ok) require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) @@ -426,11 +396,11 @@ func TestDataTransferResponding(t *testing.T) { sv.StubResult(datatransfer.ValidationResult{Accepted: true, DataLimit: 1000, VoucherResult: &vr}) }, verify: func(t *testing.T, h *receiverHarness) { - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) err := h.transport.EventHandler.OnDataReceived(channelID(h.id, h.peers), cidlink.Link{Cid: testutil.GenerateCids(1)[0]}, 12345, 1, true) require.EqualError(t, err, datatransfer.ErrPause.Error()) - require.Len(t, h.network.SentMessages, 1) - response, ok := h.network.SentMessages[0].Message.(datatransfer.Response) + require.Len(t, h.transport.MessagesSent, 1) + response, ok := h.transport.MessagesSent[0].Message.(datatransfer.Response) require.True(t, ok) require.Equal(t, response.TransferID(), h.id) require.True(t, response.IsUpdate()) @@ -445,9 +415,8 @@ func TestDataTransferResponding(t *testing.T) { require.NoError(t, err) require.Len(t, h.transport.ClosedChannels, 1) require.Equal(t, h.transport.ClosedChannels[0], channelID(h.id, h.peers)) - require.Len(t, h.network.SentMessages, 2) - sentMsg := h.network.SentMessages[1] - response, ok = sentMsg.Message.(datatransfer.Response) + require.Len(t, h.transport.MessagesSent, 2) + response, ok = h.transport.MessagesSent[1].Message.(datatransfer.Response) require.True(t, ok) require.False(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) @@ -499,9 +468,9 @@ func TestDataTransferResponding(t *testing.T) { err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: &vr}) require.NoError(t, err) require.Len(t, h.transport.ResumedChannels, 1) - resCh := h.transport.ResumedChannels[0] - require.Equal(t, resCh.ChannelID, channelID(h.id, h.peers)) - response, ok = resCh.Message.(datatransfer.Response) + require.Equal(t, h.transport.ResumedChannels[0], channelID(h.id, h.peers)) + require.Len(t, h.transport.MessagesSent, 1) + response, ok = h.transport.MessagesSent[0].Message.(datatransfer.Response) require.True(t, ok) require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) @@ -535,8 +504,8 @@ func TestDataTransferResponding(t *testing.T) { require.NoError(t, err) err = h.transport.EventHandler.OnChannelCompleted(channelID(h.id, h.peers), nil) require.NoError(t, err) - require.Len(t, h.network.SentMessages, 1) - response, ok := h.network.SentMessages[0].Message.(datatransfer.Response) + require.Len(t, h.transport.MessagesSent, 1) + response, ok := h.transport.MessagesSent[0].Message.(datatransfer.Response) require.True(t, ok) require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) @@ -552,9 +521,8 @@ func TestDataTransferResponding(t *testing.T) { vr := testutil.NewTestTypedVoucher() err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, VoucherResult: &vr}) require.NoError(t, err) - require.Len(t, h.network.SentMessages, 2) - sentMsg := h.network.SentMessages[1] - response, ok = sentMsg.Message.(datatransfer.Response) + require.Len(t, h.transport.MessagesSent, 2) + response, ok = h.transport.MessagesSent[1].Message.(datatransfer.Response) require.True(t, ok) require.Equal(t, response.TransferID(), h.id) require.True(t, response.IsValidationResult()) @@ -597,7 +565,7 @@ func TestDataTransferResponding(t *testing.T) { ft.RecordCustomizedTransfer(channelID, voucher) }) require.NoError(t, err) - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) require.Len(t, h.transport.CustomizedTransfers, 1) customizedTransfer := h.transport.CustomizedTransfers[0] require.Equal(t, channelID(h.id, h.peers), customizedTransfer.ChannelID) @@ -638,10 +606,9 @@ func TestDataTransferResponding(t *testing.T) { defer cancel() h.ctx = ctx h.peers = testutil.GeneratePeers(2) - h.network = testutil.NewFakeNetwork(h.peers[0]) h.transport = testutil.NewFakeTransport() h.ds = dss.MutexWrap(datastore.NewMapDatastore()) - dt, err := NewDataTransfer(h.ds, h.network, h.transport) + dt, err := NewDataTransfer(h.ds, h.peers[0], h.transport) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt) h.dt = dt @@ -722,10 +689,8 @@ func TestDataTransferRestartResponding(t *testing.T) { }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming push - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) require.Len(t, h.sv.ValidationsReceived, 1) - require.Len(t, h.transport.OpenedChannels, 1) - require.Len(t, h.network.SentMessages, 0) // some cids are received chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} @@ -738,21 +703,9 @@ func TestDataTransferRestartResponding(t *testing.T) { // receive restart push request req, err := message.NewRequest(h.pushRequest.TransferID(), true, false, &h.voucher, h.baseCid, h.stor) require.NoError(t, err) - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], req) + response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), req) + require.NoError(t, err) require.Len(t, h.sv.RevalidationsReceived, 1) - require.Len(t, h.transport.OpenedChannels, 2) - require.Len(t, h.network.SentMessages, 0) - - // validate channel that is opened second time - openChannel := h.transport.OpenedChannels[1] - require.Equal(t, openChannel.ChannelID, channelID(h.id, h.peers)) - require.Equal(t, openChannel.DataSender, h.peers[1]) - require.Equal(t, openChannel.Root, cidlink.Link{Cid: h.baseCid}) - require.Equal(t, openChannel.Selector, h.stor) - // assert do not send cids are sent - require.False(t, openChannel.Message.IsRequest()) - response, ok := openChannel.Message.(datatransfer.Response) - require.True(t, ok) require.True(t, response.IsRestart()) require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) @@ -784,11 +737,8 @@ func TestDataTransferRestartResponding(t *testing.T) { }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull - _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) - require.NoError(t, err) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) require.Len(t, h.sv.ValidationsReceived, 1) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 0) // receive restart pull request restartReq, err := message.NewRequest(h.id, true, true, &h.voucher, h.baseCid, h.stor) @@ -796,8 +746,6 @@ func TestDataTransferRestartResponding(t *testing.T) { response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), restartReq) require.NoError(t, err) require.Len(t, h.sv.RevalidationsReceived, 1) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 0) // validate response require.True(t, response.IsRestart()) @@ -826,11 +774,8 @@ func TestDataTransferRestartResponding(t *testing.T) { }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull - _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) - require.NoError(t, err) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) require.Len(t, h.sv.ValidationsReceived, 1) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 0) // receive restart pull request restartReq, err := message.NewRequest(h.id, true, true, &h.voucher, h.baseCid, h.stor) @@ -838,7 +783,7 @@ func TestDataTransferRestartResponding(t *testing.T) { p := testutil.GeneratePeers(1)[0] chid := datatransfer.ChannelID{ID: h.pullRequest.TransferID(), Initiator: p, Responder: h.peers[0]} _, err = h.transport.EventHandler.OnRequestReceived(chid, restartReq) - require.True(t, xerrors.As(err, new(*channels.ErrNotFound))) + require.EqualError(t, err, datatransfer.ErrChannelNotFound.Error()) }, }, "restart request fails if voucher validation fails": { @@ -858,11 +803,8 @@ func TestDataTransferRestartResponding(t *testing.T) { }, verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull - _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) - require.NoError(t, err) + _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) require.Len(t, h.sv.ValidationsReceived, 1) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 0) // receive restart pull request h.sv.ExpectErrorPull() @@ -886,11 +828,8 @@ func TestDataTransferRestartResponding(t *testing.T) { verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull chid := channelID(h.id, h.peers) - _, err := h.transport.EventHandler.OnRequestReceived(chid, h.pullRequest) - require.NoError(t, err) + _, _ = h.transport.EventHandler.OnRequestReceived(chid, h.pullRequest) require.Len(t, h.sv.ValidationsReceived, 1) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 0) // receive restart pull request randCid := testutil.GenerateCids(1)[0] @@ -914,11 +853,8 @@ func TestDataTransferRestartResponding(t *testing.T) { verify: func(t *testing.T, h *receiverHarness) { // receive an incoming pull chid := channelID(h.id, h.peers) - _, err := h.transport.EventHandler.OnRequestReceived(chid, h.pullRequest) - require.NoError(t, err) + _, _ = h.transport.EventHandler.OnRequestReceived(chid, h.pullRequest) require.Len(t, h.sv.ValidationsReceived, 1) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 0) // receive restart pull request @@ -945,8 +881,6 @@ func TestDataTransferRestartResponding(t *testing.T) { _, err := h.transport.EventHandler.OnRequestReceived(chid, h.pullRequest) require.NoError(t, err) require.Len(t, h.sv.ValidationsReceived, 1) - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 0) // receive restart pull request v := testutil.NewTestTypedVoucherWith("rand") @@ -980,25 +914,22 @@ func TestDataTransferRestartResponding(t *testing.T) { require.NoError(t, ev.OnDataReceived(channelID, cidlink.Link{Cid: testCids[1]}, 12345, 2, true)) // send a request to restart the same pull channel - restartReq := message.RestartExistingChannelRequest(channelID) - h.network.Delegate.ReceiveRestartExistingChannelRequest(ctx, h.peers[1], restartReq) + err = h.transport.EventHandler.OnRestartExistingChannelRequestReceived(channelID) + require.NoError(t, err) - require.Len(t, h.transport.OpenedChannels, 2) - require.Len(t, h.network.SentMessages, 0) + require.Len(t, h.transport.OpenedChannels, 1) + require.Len(t, h.transport.RestartedChannels, 1) // assert correct channel was created in response to this - require.Len(t, h.transport.OpenedChannels, 2) - openChannel := h.transport.OpenedChannels[1] - require.Equal(t, openChannel.ChannelID, channelID) - require.Equal(t, openChannel.DataSender, h.peers[1]) - require.Equal(t, openChannel.Root, cidlink.Link{Cid: h.baseCid}) - require.Equal(t, openChannel.Selector, h.stor) - require.True(t, openChannel.Message.IsRequest()) - require.EqualValues(t, len(testCids), openChannel.Channel.ReceivedCidsTotal()) + restartedChannel := h.transport.RestartedChannels[0] + require.Equal(t, restartedChannel.Channel.ChannelID(), channelID) + require.Equal(t, restartedChannel.Channel.Sender(), h.peers[1]) + require.Equal(t, restartedChannel.Channel.BaseCID(), h.baseCid) + require.Equal(t, restartedChannel.Channel.Selector(), h.stor) + require.EqualValues(t, len(testCids), restartedChannel.Channel.ReceivedCidsTotal()) // assert a restart request is in the channel - request, ok := openChannel.Message.(datatransfer.Request) - require.True(t, ok) + request := restartedChannel.Message require.True(t, request.IsRestart()) require.Equal(t, request.TransferID(), channelID.ID) require.Equal(t, request.BaseCid(), h.baseCid) @@ -1026,20 +957,15 @@ func TestDataTransferRestartResponding(t *testing.T) { require.NotEmpty(t, channelID) // send a request to restart the same push request - restartReq := message.RestartExistingChannelRequest(channelID) - h.network.Delegate.ReceiveRestartExistingChannelRequest(ctx, h.peers[1], restartReq) + err = h.transport.EventHandler.OnRestartExistingChannelRequestReceived(channelID) + require.NoError(t, err) - // assert correct message was sent in response to this - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 2) + require.Len(t, h.transport.OpenedChannels, 1) + require.Len(t, h.transport.RestartedChannels, 1) // assert restart request is well formed - messageReceived := h.network.SentMessages[1] - require.Equal(t, messageReceived.PeerID, h.peers[1]) - received := messageReceived.Message - require.True(t, received.IsRequest()) - receivedRequest, ok := received.(datatransfer.Request) - require.True(t, ok) + restartedChannel := h.transport.RestartedChannels[0] + receivedRequest := restartedChannel.Message require.Equal(t, receivedRequest.TransferID(), channelID.ID) require.Equal(t, receivedRequest.BaseCid(), h.baseCid) require.False(t, receivedRequest.IsCancel()) @@ -1054,48 +980,6 @@ func TestDataTransferRestartResponding(t *testing.T) { testutil.AssertTestVoucher(t, receivedRequest, h.voucher) }, }, - "ReceiveRestartExistingChannelRequest: errors if peer is not the initiator": { - expectedEvents: []datatransfer.EventCode{ - datatransfer.Open, - datatransfer.Accept, - }, - configureValidator: func(sv *testutil.StubbedValidator) { - sv.ExpectSuccessPush() - sv.StubResult(datatransfer.ValidationResult{Accepted: true}) - }, - verify: func(t *testing.T, h *receiverHarness) { - // create an incoming push first - h.network.Delegate.ReceiveRequest(h.ctx, h.peers[1], h.pushRequest) - require.Len(t, h.sv.ValidationsReceived, 1) - - // restart req does not anything as we are not the initiator - chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} - restartReq := message.RestartExistingChannelRequest(chid) - h.network.Delegate.ReceiveRestartExistingChannelRequest(ctx, h.peers[1], restartReq) - - require.Len(t, h.transport.OpenedChannels, 1) - require.Len(t, h.network.SentMessages, 0) - }, - }, - "ReceiveRestartExistingChannelRequest: errors if sending peer is not the counter-party on the channel": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open}, - configureValidator: func(sv *testutil.StubbedValidator) { - }, - verify: func(t *testing.T, h *receiverHarness) { - // create an outgoing push request first - p := testutil.GeneratePeers(1)[0] - channelID, err := h.dt.OpenPushDataChannel(h.ctx, p, h.voucher, h.baseCid, h.stor) - require.NoError(t, err) - require.NotEmpty(t, channelID) - - // sending peer is not the counter-party on the channel - restartReq := message.RestartExistingChannelRequest(channelID) - h.network.Delegate.ReceiveRestartExistingChannelRequest(ctx, h.peers[1], restartReq) - - require.Len(t, h.transport.OpenedChannels, 0) - require.Len(t, h.network.SentMessages, 1) - }, - }, } for testCase, verify := range testCases { t.Run(testCase, func(t *testing.T) { @@ -1104,10 +988,9 @@ func TestDataTransferRestartResponding(t *testing.T) { defer cancel() h.ctx = ctx h.peers = testutil.GeneratePeers(2) - h.network = testutil.NewFakeNetwork(h.peers[0]) h.transport = testutil.NewFakeTransport() h.ds = dss.MutexWrap(datastore.NewMapDatastore()) - dt, err := NewDataTransfer(h.ds, h.network, h.transport) + dt, err := NewDataTransfer(h.ds, h.peers[0], h.transport) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt) h.dt = dt @@ -1148,7 +1031,6 @@ type receiverHarness struct { cancelUpdate datatransfer.Request ctx context.Context peers []peer.ID - network *testutil.FakeNetwork transport *testutil.FakeTransport sv *testutil.StubbedValidator ds datastore.Batching diff --git a/impl/restart.go b/impl/restart.go index 9fdf526c..9381aba4 100644 --- a/impl/restart.go +++ b/impl/restart.go @@ -4,7 +4,6 @@ import ( "context" "github.com/ipld/go-ipld-prime" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" @@ -13,44 +12,12 @@ import ( "github.com/filecoin-project/go-data-transfer/v2/message" ) -// ChannelDataTransferType identifies the type of a data transfer channel for the purposes of a restart -type ChannelDataTransferType int +func (m *manager) restartManagerPeerReceive(ctx context.Context, channel datatransfer.ChannelState) error { -const ( - // ManagerPeerCreatePull is the type of a channel wherein the manager peer created a Pull Data Transfer - ManagerPeerCreatePull ChannelDataTransferType = iota - - // ManagerPeerCreatePush is the type of a channel wherein the manager peer created a Push Data Transfer - ManagerPeerCreatePush - - // ManagerPeerReceivePull is the type of a channel wherein the manager peer received a Pull Data Transfer Request - ManagerPeerReceivePull - - // ManagerPeerReceivePush is the type of a channel wherein the manager peer received a Push Data Transfer Request - ManagerPeerReceivePush -) - -func (m *manager) restartManagerPeerReceivePush(ctx context.Context, channel datatransfer.ChannelState) error { - result, err := m.validateRestart(channel) - if err != nil { - return xerrors.Errorf("failed to restart channel, validation error: %w", err) + if !m.transport.Capabilities().Restartable { + return datatransfer.ErrUnsupported } - if !result.Accepted { - return datatransfer.ErrRejected - } - - // send a libp2p message to the other peer asking to send a "restart push request" - req := message.RestartExistingChannelRequest(channel.ChannelID()) - - if err := m.dataTransferNetwork.SendMessage(ctx, channel.OtherPeer(), req); err != nil { - return xerrors.Errorf("unable to send restart request: %w", err) - } - - return nil -} - -func (m *manager) restartManagerPeerReceivePull(ctx context.Context, channel datatransfer.ChannelState) error { result, err := m.validateRestart(channel) if err != nil { return xerrors.Errorf("failed to restart channel, validation error: %w", err) @@ -60,24 +27,26 @@ func (m *manager) restartManagerPeerReceivePull(ctx context.Context, channel dat return datatransfer.ErrRejected } + // send a libp2p message to the other peer asking to send a "restart push request" req := message.RestartExistingChannelRequest(channel.ChannelID()) - // send a libp2p message to the other peer asking to send a "restart pull request" - if err := m.dataTransferNetwork.SendMessage(ctx, channel.OtherPeer(), req); err != nil { + if err := m.transport.SendMessage(ctx, channel.ChannelID(), req); err != nil { return xerrors.Errorf("unable to send restart request: %w", err) } - return nil } -func (m *manager) openPushRestartChannel(ctx context.Context, channel datatransfer.ChannelState) error { +func (m *manager) openRestartChannel(ctx context.Context, channel datatransfer.ChannelState) error { selector := channel.Selector() voucher := channel.Voucher() baseCid := channel.BaseCID() requestTo := channel.OtherPeer() chid := channel.ChannelID() - req, err := message.NewRequest(chid.ID, true, false, &voucher, baseCid, selector) + if !m.transport.Capabilities().Restartable { + return datatransfer.ErrUnsupported + } + req, err := message.NewRequest(chid.ID, true, channel.IsPull(), &voucher, baseCid, selector) if err != nil { return err } @@ -87,47 +56,12 @@ func (m *manager) openPushRestartChannel(ctx context.Context, channel datatransf transportConfigurer := processor.(datatransfer.TransportConfigurer) transportConfigurer(chid, voucher, m.transport) } - m.dataTransferNetwork.Protect(requestTo, chid.String()) // Monitor the state of the connection for the channel - monitoredChan := m.channelMonitor.AddPushChannel(chid) + monitoredChan := m.channelMonitor.AddChannel(chid, channel.IsPull()) log.Infof("sending push restart channel to %s for channel %s", requestTo, chid) - if err := m.dataTransferNetwork.SendMessage(ctx, requestTo, req); err != nil { - // If push channel monitoring is enabled, shutdown the monitor as it - // wasn't possible to start the data transfer - if monitoredChan != nil { - monitoredChan.Shutdown() - } - - return xerrors.Errorf("Unable to send restart request: %w", err) - } - - return nil -} - -func (m *manager) openPullRestartChannel(ctx context.Context, channel datatransfer.ChannelState) error { - selector := channel.Selector() - voucher := channel.Voucher() - baseCid := channel.BaseCID() - requestTo := channel.OtherPeer() - chid := channel.ChannelID() - - req, err := message.NewRequest(chid.ID, true, true, &voucher, baseCid, selector) + err = m.transport.RestartChannel(ctx, channel, req) if err != nil { - return err - } - - processor, has := m.transportConfigurers.Processor(voucher.Type) - if has { - transportConfigurer := processor.(datatransfer.TransportConfigurer) - transportConfigurer(chid, voucher, m.transport) - } - m.dataTransferNetwork.Protect(requestTo, chid.String()) - - // Monitor the state of the connection for the channel - monitoredChan := m.channelMonitor.AddPullChannel(chid) - log.Infof("sending open channel to %s to restart channel %s", requestTo, chid) - if err := m.transport.OpenChannel(ctx, requestTo, chid, cidlink.Link{Cid: baseCid}, selector, channel, req); err != nil { // If pull channel monitoring is enabled, shutdown the monitor as it // wasn't possible to start the data transfer if monitoredChan != nil { @@ -140,12 +74,7 @@ func (m *manager) openPullRestartChannel(ctx context.Context, channel datatransf return nil } -func (m *manager) validateRestartRequest(ctx context.Context, otherPeer peer.ID, chid datatransfer.ChannelID, req datatransfer.Request) error { - // channel should exist - channel, err := m.channels.GetByID(ctx, chid) - if err != nil { - return err - } +func (m *manager) validateRestartRequest(ctx context.Context, otherPeer peer.ID, channel datatransfer.ChannelState, req datatransfer.Request) error { // channel is not terminated if channels.IsChannelTerminated(channel.Status()) { diff --git a/impl/utils.go b/impl/utils.go index 518b9e0c..1a2b1351 100644 --- a/impl/utils.go +++ b/impl/utils.go @@ -83,3 +83,4 @@ func (m *manager) cancelMessage(chid datatransfer.ChannelID) datatransfer.Messag } return message.CancelResponse(chid.ID) } + diff --git a/impl/integration_test.go b/itest/integration_test.go similarity index 97% rename from impl/integration_test.go rename to itest/integration_test.go index af447baa..fab58ea6 100644 --- a/impl/integration_test.go +++ b/itest/integration_test.go @@ -1,4 +1,4 @@ -package impl_test +package itest import ( "bytes" @@ -145,10 +145,10 @@ func TestRoundTrip(t *testing.T) { tp1 := gsData.SetupGSTransportHost1() tp2 := gsData.SetupGSTransportHost2() - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) - dt2, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) + dt2, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt2) @@ -301,10 +301,10 @@ func TestMultipleRoundTripMultipleStores(t *testing.T) { tp1 := gsData.SetupGSTransportHost1() tp2 := gsData.SetupGSTransportHost2() - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) - dt2, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) + dt2, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt2) @@ -418,7 +418,7 @@ func TestManyReceiversAtOnce(t *testing.T) { host1 := gsData.Host1 // initiator, data sender tp1 := gsData.SetupGSTransportHost1() - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) @@ -440,11 +440,11 @@ func TestManyReceiversAtOnce(t *testing.T) { destDagService := merkledag.NewDAGService(blockservice.New(altBs, offline.Exchange(altBs))) gs := gsimpl.New(gsData.Ctx, gsnet, lsys) - gsTransport := tp.NewTransport(host.ID(), gs) + gsTransport := tp.NewTransport(host.ID(), gs, dtnet) dtDs := namespace.Wrap(ds, datastore.NewKey("datatransfer")) - receiver, err := NewDataTransfer(dtDs, dtnet, gsTransport) + receiver, err := NewDataTransfer(dtDs, host.ID(), gsTransport) require.NoError(t, err) err = receiver.Start(gsData.Ctx) require.NoError(t, err) @@ -708,12 +708,12 @@ func TestAutoRestart(t *testing.T) { MaxConsecutiveRestarts: 10, CompleteTimeout: 100 * time.Millisecond, }) - initiator, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, initiatorGSTspt, restartConf) + initiator, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), initiatorGSTspt, restartConf) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, initiator) defer initiator.Stop(ctx) - responder, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, responderGSTspt) + responder, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), responderGSTspt) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, responder) defer responder.Stop(ctx) @@ -882,12 +882,12 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { MaxConsecutiveRestarts: 10, CompleteTimeout: 100 * time.Millisecond, }) - initiator, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, initiatorGSTspt, restartConf) + initiator, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), initiatorGSTspt, restartConf) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, initiator) defer initiator.Stop(ctx) - responder, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, responderGSTspt) + responder, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), responderGSTspt) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, responder) defer responder.Stop(ctx) @@ -980,7 +980,7 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { // 2. Create a new initiator initiator2GSTspt := gsData.SetupGSTransportHost1() - initiator2, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, initiator2GSTspt, restartConf) + initiator2, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), initiator2GSTspt, restartConf) require.NoError(t, err) require.NoError(t, initiator2.RegisterVoucherType(testutil.TestVoucherType, sv)) initiator2.SubscribeToEvents(completeSubscriber) @@ -1091,10 +1091,10 @@ func TestRoundTripCancelledRequest(t *testing.T) { tp1 := gsData.SetupGSTransportHost1() tp2 := gsData.SetupGSTransportHost2() - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) - dt2, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) + dt2, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt2) @@ -1318,10 +1318,10 @@ func TestSimulatedRetrievalFlow(t *testing.T) { tp1 := gsData.SetupGSTransportHost1() tp2 := gsData.SetupGSTransportHost2() - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) - dt2, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) + dt2, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt2) var chid datatransfer.ChannelID @@ -1437,10 +1437,10 @@ func TestPauseAndResume(t *testing.T) { tp1 := gsData.SetupGSTransportHost1() tp2 := gsData.SetupGSTransportHost2() - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) - dt2, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) + dt2, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt2) finished := make(chan struct{}, 2) @@ -1578,10 +1578,10 @@ func TestUnrecognizedVoucherRoundTrip(t *testing.T) { tp1 := gsData.SetupGSTransportHost1() tp2 := gsData.SetupGSTransportHost2() - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) - dt2, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) + dt2, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt2) @@ -1648,14 +1648,14 @@ func TestDataTransferSubscribing(t *testing.T) { sv := testutil.NewStubbedValidator() sv.StubErrorPull() sv.StubErrorPush() - dt2, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) + dt2, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt2) require.NoError(t, dt2.RegisterVoucherType(testutil.TestVoucherType, sv)) voucher := testutil.NewTestTypedVoucherWith("applesauce") baseCid := testutil.GenerateCids(1)[0] - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) subscribe1Calls := make(chan struct{}, 1) @@ -1787,7 +1787,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { gsData.GsNet2.SetDelegate(gsr) tp1 := gsData.SetupGSTransportHost1() - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) voucherResult := testutil.NewTestTypedVoucher() @@ -1865,8 +1865,8 @@ func TestResponseHookWhenExtensionNotFound(t *testing.T) { gsData.GsNet2.SetDelegate(gsr) gs1 := gsData.SetupGraphsyncHost1() - tp1 := tp.NewTransport(host1.ID(), gs1) - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + tp1 := tp.NewTransport(host1.ID(), gs1, gsData.DtNet1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) t.Run("when it's not our extension, does not error and does not validate", func(t *testing.T) { @@ -1910,7 +1910,7 @@ func TestRespondingToPullGraphsyncRequests(t *testing.T) { sv.ExpectSuccessPull() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) - dt1, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) + dt1, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) @@ -1938,7 +1938,7 @@ func TestRespondingToPullGraphsyncRequests(t *testing.T) { test: func(t *testing.T, gsData *testutil.GraphsyncTestingData, tp2 datatransfer.Transport, link ipld.Link, id datatransfer.TransferID, gsr *fakeGraphSyncReceiver) { sv := testutil.NewStubbedValidator() sv.ExpectErrorPull() - dt1, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) + dt1, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) @@ -2005,11 +2005,11 @@ func TestMultipleMessagesInExtension(t *testing.T) { tp1 := gsData.SetupGSTransportHost1() tp2 := gsData.SetupGSTransportHost2() - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) - dt2, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) + dt2, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt2) @@ -2145,11 +2145,11 @@ func TestMultipleParallelTransfers(t *testing.T) { tp1 := gsData.SetupGSTransportHost1() tp2 := gsData.SetupGSTransportHost2() - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) - dt2, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) + dt2, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt2) diff --git a/impl/restart_integration_test.go b/itest/restart_integration_test.go similarity index 97% rename from impl/restart_integration_test.go rename to itest/restart_integration_test.go index 1b4a50ec..d63fd728 100644 --- a/impl/restart_integration_test.go +++ b/itest/restart_integration_test.go @@ -1,4 +1,4 @@ -package impl_test +package itest import ( "context" @@ -51,7 +51,7 @@ func TestRestartPush(t *testing.T) { require.NoError(t, rh.dt1.Stop(rh.testCtx)) time.Sleep(100 * time.Millisecond) tp1 := rh.gsData.SetupGSTransportHost1() - rh.dt1, err = NewDataTransfer(rh.gsData.DtDs1, rh.gsData.DtNet1, tp1) + rh.dt1, err = NewDataTransfer(rh.gsData.DtDs1, rh.gsData.Host1.ID(), tp1) require.NoError(rh.t, err) require.NoError(rh.t, rh.dt1.RegisterVoucherType(testutil.TestVoucherType, rh.sv)) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt1) @@ -93,7 +93,7 @@ func TestRestartPush(t *testing.T) { require.NoError(t, rh.dt2.Stop(rh.testCtx)) time.Sleep(100 * time.Millisecond) tp2 := rh.gsData.SetupGSTransportHost2() - rh.dt2, err = NewDataTransfer(rh.gsData.DtDs2, rh.gsData.DtNet2, tp2) + rh.dt2, err = NewDataTransfer(rh.gsData.DtDs2, rh.gsData.Host2.ID(), tp2) require.NoError(rh.t, err) require.NoError(rh.t, rh.dt2.RegisterVoucherType(testutil.TestVoucherType, rh.sv)) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt2) @@ -108,7 +108,8 @@ func TestRestartPush(t *testing.T) { // initiator: abort GS response "transfer(0)->response(0)->abortRequest(0)", // initiator: receive restart request and send restart channel message - "transfer(0)->receiveRequest(0)->sendMessage(0)", + "transfer(0)->receiveRequest(0)", + "transfer(0)->sendMessage(1)", // initiator: receive second GS request in response to restart channel message // and execute GS response "transfer(0)->response(1)->executeTask(0)", @@ -304,7 +305,7 @@ func TestRestartPull(t *testing.T) { require.NoError(t, rh.dt2.Stop(rh.testCtx)) time.Sleep(100 * time.Millisecond) tp2 := rh.gsData.SetupGSTransportHost2() - rh.dt2, err = NewDataTransfer(rh.gsData.DtDs2, rh.gsData.DtNet2, tp2) + rh.dt2, err = NewDataTransfer(rh.gsData.DtDs2, rh.gsData.Host2.ID(), tp2) require.NoError(rh.t, err) require.NoError(rh.t, rh.dt2.RegisterVoucherType(testutil.TestVoucherType, rh.sv)) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt2) @@ -343,7 +344,7 @@ func TestRestartPull(t *testing.T) { require.NoError(t, rh.dt1.Stop(rh.testCtx)) time.Sleep(100 * time.Millisecond) tp1 := rh.gsData.SetupGSTransportHost1() - rh.dt1, err = NewDataTransfer(rh.gsData.DtDs1, rh.gsData.DtNet1, tp1) + rh.dt1, err = NewDataTransfer(rh.gsData.DtDs1, rh.gsData.Host1.ID(), tp1) require.NoError(rh.t, err) require.NoError(rh.t, rh.dt1.RegisterVoucherType(testutil.TestVoucherType, rh.sv)) testutil.StartAndWaitForReady(rh.testCtx, t, rh.dt1) @@ -356,7 +357,8 @@ func TestRestartPull(t *testing.T) { // initiator: initial outgoing gs request terminates "transfer(0)->request(0)->terminateRequest(0)", // initiator: respond to restart request and send second GS request - "transfer(0)->receiveRequest(0)->request(0)", + "transfer(0)->receiveRequest(0)", + "transfer(0)->request(1)->executeTask(0)", // initiator: receive completion message from responder that they sent all the data "transfer(0)->receiveResponse(0)", // responder: receive GS request and execute response @@ -563,10 +565,10 @@ func newRestartHarness(t *testing.T) *restartHarness { tp1 := gsData.SetupGSTransportHost1() tp2 := gsData.SetupGSTransportHost2() - dt1, err := NewDataTransfer(gsData.DtDs1, gsData.DtNet1, tp1) + dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) - dt2, err := NewDataTransfer(gsData.DtDs2, gsData.DtNet2, tp2) + dt2, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) sv := testutil.NewStubbedValidator() diff --git a/testutil/faketransport.go b/testutil/faketransport.go index 08faee74..6bd822ac 100644 --- a/testutil/faketransport.go +++ b/testutil/faketransport.go @@ -3,25 +3,23 @@ package testutil import ( "context" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/datamodel" - "github.com/libp2p/go-libp2p-core/peer" - datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) // OpenedChannel records a call to open a channel type OpenedChannel struct { - DataSender peer.ID - ChannelID datatransfer.ChannelID - Root ipld.Link - Selector datamodel.Node - Channel datatransfer.ChannelState - Message datatransfer.Message + Channel datatransfer.Channel + Message datatransfer.Request +} + +// RestartedChannel records a call to restart a channel +type RestartedChannel struct { + Channel datatransfer.ChannelState + Message datatransfer.Request } -// ResumedChannel records a call to resume a channel -type ResumedChannel struct { +// Records a message sent +type MessageSent struct { ChannelID datatransfer.ChannelID Message datatransfer.Message } @@ -32,16 +30,19 @@ type CustomizedTransfer struct { Voucher datatransfer.TypedVoucher } +var _ datatransfer.Transport = &FakeTransport{} + // FakeTransport is a fake transport with mocked results type FakeTransport struct { OpenedChannels []OpenedChannel OpenChannelErr error + RestartedChannels []RestartedChannel + RestartChannelErr error ClosedChannels []datatransfer.ChannelID - CloseChannelErr error PausedChannels []datatransfer.ChannelID - PauseChannelErr error - ResumedChannels []ResumedChannel - ResumeChannelErr error + ResumedChannels []datatransfer.ChannelID + MessagesSent []MessageSent + UpdateError error CleanedUpChannels []datatransfer.ChannelID CustomizedTransfers []CustomizedTransfer EventHandler datatransfer.EventsHandler @@ -53,20 +54,55 @@ func NewFakeTransport() *FakeTransport { return &FakeTransport{} } +// Capabilities tells datatransfer what kinds of capabilities this transport supports +func (ft *FakeTransport) Capabilities() datatransfer.TransportCapabilities { + return datatransfer.TransportCapabilities{ + Restartable: true, + Pausable: true, + } +} + // OpenChannel initiates an outgoing request for the other peer to send data // to us on this channel // Note: from a data transfer symantic standpoint, it doesn't matter if the // request is push or pull -- OpenChannel is called by the party that is // intending to receive data -func (ft *FakeTransport) OpenChannel(ctx context.Context, dataSender peer.ID, channelID datatransfer.ChannelID, root ipld.Link, stor datamodel.Node, channel datatransfer.ChannelState, msg datatransfer.Message) error { - ft.OpenedChannels = append(ft.OpenedChannels, OpenedChannel{dataSender, channelID, root, stor, channel, msg}) +func (ft *FakeTransport) OpenChannel(ctx context.Context, channel datatransfer.Channel, msg datatransfer.Request) error { + ft.OpenedChannels = append(ft.OpenedChannels, OpenedChannel{channel, msg}) return ft.OpenChannelErr } -// CloseChannel closes the given channel -func (ft *FakeTransport) CloseChannel(ctx context.Context, chid datatransfer.ChannelID) error { - ft.ClosedChannels = append(ft.ClosedChannels, chid) - return ft.CloseChannelErr +// RestartChannel restarts a channel +func (ft *FakeTransport) RestartChannel(ctx context.Context, channelState datatransfer.ChannelState, msg datatransfer.Request) error { + ft.RestartedChannels = append(ft.RestartedChannels, RestartedChannel{channelState, msg}) + return ft.RestartChannelErr +} + +// WithChannel takes actions on a channel +func (ft *FakeTransport) UpdateChannel(ctx context.Context, chid datatransfer.ChannelID, update datatransfer.ChannelUpdate) error { + + if update.SendMessage != nil { + ft.MessagesSent = append(ft.MessagesSent, MessageSent{chid, update.SendMessage}) + } + + if update.Closed { + ft.ClosedChannels = append(ft.ClosedChannels, chid) + return ft.UpdateError + } + + if !update.Paused { + ft.ResumedChannels = append(ft.ResumedChannels, chid) + } else { + ft.PausedChannels = append(ft.PausedChannels, chid) + } + + return ft.UpdateError +} + +// SendMessage sends a data transfer message over the channel to the other peer +func (ft *FakeTransport) SendMessage(ctx context.Context, chid datatransfer.ChannelID, msg datatransfer.Message) error { + ft.MessagesSent = append(ft.MessagesSent, MessageSent{chid, msg}) + return ft.UpdateError } // SetEventHandler sets the handler for events on channels @@ -75,22 +111,11 @@ func (ft *FakeTransport) SetEventHandler(events datatransfer.EventsHandler) erro return ft.SetEventHandlerErr } +// Shutdown close this transport func (ft *FakeTransport) Shutdown(ctx context.Context) error { return nil } -// PauseChannel paused the given channel ID -func (ft *FakeTransport) PauseChannel(ctx context.Context, chid datatransfer.ChannelID) error { - ft.PausedChannels = append(ft.PausedChannels, chid) - return ft.PauseChannelErr -} - -// ResumeChannel resumes the given channel -func (ft *FakeTransport) ResumeChannel(ctx context.Context, msg datatransfer.Message, chid datatransfer.ChannelID) error { - ft.ResumedChannels = append(ft.ResumedChannels, ResumedChannel{chid, msg}) - return ft.ResumeChannelErr -} - // CleanupChannel cleans up the given channel func (ft *FakeTransport) CleanupChannel(chid datatransfer.ChannelID) { ft.CleanedUpChannels = append(ft.CleanedUpChannels, chid) diff --git a/testutil/gstestdata.go b/testutil/gstestdata.go index 3fa22989..f9d71869 100644 --- a/testutil/gstestdata.go +++ b/testutil/gstestdata.go @@ -181,7 +181,7 @@ func (gsData *GraphsyncTestingData) SetupGSTransportHost1(opts ...gstransport.Op opts = append(opts, gstransport.SupportedExtensions(supportedExtensions)) } - return gstransport.NewTransport(gsData.Host1.ID(), gs, opts...) + return gstransport.NewTransport(gsData.Host1.ID(), gs, gsData.DtNet1, opts...) } // SetupGraphsyncHost2 sets up a new, real graphsync instance on top of the second host @@ -206,7 +206,7 @@ func (gsData *GraphsyncTestingData) SetupGSTransportHost2(opts ...gstransport.Op } opts = append(opts, gstransport.SupportedExtensions(supportedExtensions)) } - return gstransport.NewTransport(gsData.Host2.ID(), gs, opts...) + return gstransport.NewTransport(gsData.Host2.ID(), gs, gsData.DtNet2, opts...) } // LoadUnixFSFile loads a fixtures file we can test dag transfer with diff --git a/testutil/testutil.go b/testutil/testutil.go index cfcde8b5..dec67a72 100644 --- a/testutil/testutil.go +++ b/testutil/testutil.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "math/rand" "testing" blocks "github.com/ipfs/go-block-format" @@ -122,3 +123,9 @@ func StartAndWaitForReady(ctx context.Context, t *testing.T, manager datatransfe require.NoError(t, err) } } + +// GenerateChannelID generates a new data transfer channel id for use in tests +func GenerateChannelID() datatransfer.ChannelID { + p := GeneratePeers(2) + return datatransfer.ChannelID{Initiator: p[0], Responder: p[1], ID: datatransfer.TransferID(rand.Int31())} +} diff --git a/transport.go b/transport.go index ca002a28..dfadad56 100644 --- a/transport.go +++ b/transport.go @@ -4,12 +4,13 @@ import ( "context" ipld "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/datamodel" - peer "github.com/libp2p/go-libp2p-core/peer" ) -// EventsHandler are semantic data transfer events that happen as a result of graphsync hooks +// EventsHandler are semantic data transfer events that happen as a result of transport events type EventsHandler interface { + // ChannelState queries for the current channel state + ChannelState(ctx context.Context, chid ChannelID) (ChannelState, error) + // OnChannelOpened is called when we send a request for data to the other // peer on the given channel ID // return values are: @@ -72,58 +73,74 @@ type EventsHandler interface { // OnContextAugment allows the transport to attach data transfer tracing information // to its local context, in order to create a hierarchical trace OnContextAugment(chid ChannelID) func(context.Context) context.Context + + OnRestartExistingChannelRequestReceived(chid ChannelID) error } /* Transport defines the interface for a transport layer for data transfer. Where the data transfer manager will coordinate setting up push and -pull requests, validation, etc, the transport layer is responsible for moving +pull requests, persistence, validation, etc, the transport layer is responsible for moving data back and forth, and may be medium specific. For example, some transports may have the ability to pause and resume requests, while others may not. -Some may support individual data events, while others may only support message +Some may dispatch data update events, while others may only support message events. Some transport layers may opt to use the actual data transfer network protocols directly while others may be able to encode messages in their own data protocol. Transport is the minimum interface that must be satisfied to serve as a datatransfer -transport layer. Transports must be able to open (open is always called by the receiving peer) -and close channels, and set at an event handler */ +transport layer. Transports must be able to open and close channels, set at an event handler, +and send messages. Beyond that, additional commands may or may not be supported. +Whether a command is supported can be determined ahead by calling Capabilities(). +*/ type Transport interface { - // OpenChannel initiates an outgoing request for the other peer to send data - // to us on this channel - // Note: from a data transfer symantic standpoint, it doesn't matter if the - // request is push or pull -- OpenChannel is called by the party that is - // intending to receive data + // Capabilities tells datatransfer what kinds of capabilities this transport supports + Capabilities() TransportCapabilities + // OpenChannel opens a channel on a given transport to move data back and forth. + // OpenChannel MUST ALWAYS called by the initiator. OpenChannel( ctx context.Context, - dataSender peer.ID, - channelID ChannelID, - root ipld.Link, - stor datamodel.Node, - channel ChannelState, - msg Message, + channel Channel, + req Request, ) error - // CloseChannel closes the given channel - CloseChannel(ctx context.Context, chid ChannelID) error + // UpdateChannel sends one or more updates the transport channel at once, + // such as pausing/resuming, closing the transfer, or sending additional + // messages over the channel. Grouping the commands allows the transport + // the ability to plan how to execute these updates based on the capabilities + // and API of the underlying transport protocol and library + UpdateChannel(ctx context.Context, chid ChannelID, update ChannelUpdate) error // SetEventHandler sets the handler for events on channels SetEventHandler(events EventsHandler) error - // CleanupChannel is called on the otherside of a cancel - removes any associated - // data for the channel + // CleanupChannel removes any associated data on a closed channel CleanupChannel(chid ChannelID) + // SendMessage sends a data transfer message over the channel to the other peer + SendMessage(ctx context.Context, chid ChannelID, msg Message) error + // Shutdown unregisters the current EventHandler and ends all active data transfers Shutdown(ctx context.Context) error + + // Optional Methods: Some channels may not support these + + // Restart restarts a channel on the initiator side + // RestartChannel MUST ALWAYS called by the initiator + RestartChannel(ctx context.Context, channel ChannelState, req Request) error } -// PauseableTransport is a transport that can also pause and resume channels -type PauseableTransport interface { - Transport - // PauseChannel paused the given channel ID - PauseChannel(ctx context.Context, - chid ChannelID, - ) error - // ResumeChannel resumes the given channel - ResumeChannel(ctx context.Context, - msg Message, - chid ChannelID, - ) error +// TransportCapabilities describes additional capabilities supported by ChannelActions +type TransportCapabilities struct { + // Restarable indicates ChannelActions will support RestartActions + Restartable bool + // Pausable indicates ChannelActions will support PauseActions + Pausable bool +} + +// ChannelUpdate describes updates to a channel - changing it's paused status, closing the transfer, +// and additional messages to send +type ChannelUpdate struct { + // Paused sets the paused status of the channel. If pause/resumes are not supported, this is a no op + Paused bool + // Closed sets whether the channel is closed + Closed bool + // SendMessage sends an additional message + SendMessage Message } diff --git a/transport/graphsync/dtchannel/dtchannel.go b/transport/graphsync/dtchannel/dtchannel.go new file mode 100644 index 00000000..e0a003be --- /dev/null +++ b/transport/graphsync/dtchannel/dtchannel.go @@ -0,0 +1,375 @@ +package dtchannel + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/ipfs/go-graphsync" + "github.com/ipfs/go-graphsync/donotsendfirstblocks" + logging "github.com/ipfs/go-log/v2" + ipld "github.com/ipld/go-ipld-prime" + peer "github.com/libp2p/go-libp2p-core/peer" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/executor" +) + +const maxGSCancelWait = time.Second + +var log = logging.Logger("dt_graphsync") + +// state is the state graphsync data transfer channel +type state uint64 + +// +const ( + channelClosed state = iota + channelOpen + channelPaused +) + +// Info needed to keep track of a data transfer channel +type Channel struct { + channelID datatransfer.ChannelID + gs graphsync.GraphExchange + + lk sync.RWMutex + state state + requestID *graphsync.RequestID + completed chan struct{} + requesterCancelled bool + pendingExtensions []graphsync.ExtensionData + + storeLk sync.RWMutex + storeRegistered bool + + receivedCidsTotal int64 +} + +func NewChannel(channelID datatransfer.ChannelID, gs graphsync.GraphExchange) *Channel { + return &Channel{ + channelID: channelID, + gs: gs, + } +} + +// Open a graphsync request for data to the remote peer +func (c *Channel) Open( + ctx context.Context, + requestID graphsync.RequestID, + dataSender peer.ID, + root ipld.Link, + stor ipld.Node, + exts []graphsync.ExtensionData, +) (*executor.Executor, error) { + c.lk.Lock() + defer c.lk.Unlock() + + // If there is an existing graphsync request for this channelID + if c.requestID != nil { + // Cancel the existing graphsync request + completed := c.completed + errch := c.cancel(ctx) + + // Wait for the complete callback to be called + c.lk.Unlock() + err := waitForCompleteHook(ctx, completed) + c.lk.Lock() + if err != nil { + return nil, fmt.Errorf("%s: waiting for cancelled graphsync request to complete: %w", c.channelID, err) + } + + // Wait for the cancel request method to complete + select { + case err = <-errch: + case <-ctx.Done(): + err = fmt.Errorf("timed out waiting for graphsync request to be cancelled") + } + if err != nil { + return nil, fmt.Errorf("%s: restarting graphsync request: %w", c.channelID, err) + } + } + + // add do not send cids ext as needed + if c.receivedCidsTotal > 0 { + data := donotsendfirstblocks.EncodeDoNotSendFirstBlocks(c.receivedCidsTotal) + exts = append(exts, graphsync.ExtensionData{ + Name: graphsync.ExtensionsDoNotSendFirstBlocks, + Data: data, + }) + } + + // Set up a completed channel that will be closed when the request + // completes (or is cancelled) + completed := make(chan struct{}) + var onCompleteOnce sync.Once + onComplete := func() { + // Ensure the channel is only closed once + onCompleteOnce.Do(func() { + c.MarkTransferComplete() + log.Infow("closing the completion ch for data-transfer channel", "chid", c.channelID) + close(completed) + }) + } + c.completed = completed + + // Open a new graphsync request + msg := fmt.Sprintf("Opening graphsync request to %s for root %s", dataSender, root) + if c.receivedCidsTotal > 0 { + msg += fmt.Sprintf(" with %d Blocks already received", c.receivedCidsTotal) + } + log.Info(msg) + c.requestID = &requestID + ctx = context.WithValue(ctx, graphsync.RequestIDContextKey{}, *c.requestID) + responseChan, errChan := c.gs.Request(ctx, dataSender, root, stor, exts...) + c.state = channelOpen + // Save a mapping from the graphsync key to the channel ID so that + // subsequent graphsync callbacks are associated with this channel + + e := executor.NewExecutor(c.channelID, responseChan, errChan, onComplete) + return e, nil +} + +func waitForCompleteHook(ctx context.Context, completed chan struct{}) error { + // Wait for the cancel to propagate through to graphsync, and for + // the graphsync request to complete + select { + case <-completed: + return nil + case <-time.After(maxGSCancelWait): + // Fail-safe: give up waiting after a certain amount of time + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// gsReqOpened is called when graphsync makes a request to the remote peer to ask for data +func (c *Channel) GsReqOpened(sender peer.ID, requestID graphsync.RequestID, hookActions graphsync.OutgoingRequestHookActions) { + // Tell graphsync to store the received blocks in the registered store + if c.hasStore() { + hookActions.UsePersistenceOption("data-transfer-" + c.channelID.String()) + } + log.Infow("outgoing graphsync request", "peer", sender, "graphsync request id", requestID, "data transfer channel id", c.channelID) +} + +// gsDataRequestRcvd is called when the transport receives an incoming request +// for data. +func (c *Channel) GsDataRequestRcvd(sender peer.ID, requestID graphsync.RequestID, pauseRequest bool, hookActions graphsync.IncomingRequestHookActions) { + c.lk.Lock() + defer c.lk.Unlock() + log.Debugf("%s: received request for data, req_id=%d", c.channelID, requestID) + // If the requester had previously cancelled their request, send any + // message that was queued since the cancel + if c.requesterCancelled { + c.requesterCancelled = false + + extensions := c.pendingExtensions + c.pendingExtensions = nil + for _, ext := range extensions { + hookActions.SendExtensionData(ext) + } + } + + // Tell graphsync to load blocks from the registered store + if c.hasStore() { + hookActions.UsePersistenceOption("data-transfer-" + c.channelID.String()) + } + + // Save a mapping from the graphsync key to the channel ID so that + // subsequent graphsync callbacks are associated with this channel + c.requestID = &requestID + log.Infow("incoming graphsync request", "peer", sender, "graphsync request id", requestID, "data transfer channel id", c.channelID) + + if pauseRequest { + c.state = channelPaused + return + } + c.state = channelOpen +} + +func (c *Channel) MarkPaused() { + c.lk.Lock() + defer c.lk.Unlock() + c.state = channelPaused +} + +func (c *Channel) Paused() bool { + c.lk.RLock() + defer c.lk.RUnlock() + return c.state == channelPaused +} + +func (c *Channel) Pause(ctx context.Context) error { + c.lk.Lock() + defer c.lk.Unlock() + + // Check if the channel was already cancelled + if c.requestID == nil { + log.Debugf("%s: channel was cancelled so not pausing channel", c.channelID) + return nil + } + + if c.state != channelOpen { + log.Debugf("%s: channel is not open so not pausing channel", c.channelID) + return nil + } + + c.state = channelPaused + + // If the requester cancelled, bail out + if c.requesterCancelled { + log.Debugf("%s: requester has cancelled so not pausing response for now", c.channelID) + return nil + } + + // Pause the response + log.Debugf("%s: pausing response", c.channelID) + return c.gs.Pause(ctx, *c.requestID) +} + +func (c *Channel) Resume(ctx context.Context, extensions []graphsync.ExtensionData) error { + c.lk.Lock() + defer c.lk.Unlock() + + // Check if the channel was already cancelled + if c.requestID == nil { + log.Debugf("%s: channel was cancelled so not resuming channel", c.channelID) + return nil + } + if c.state != channelPaused { + log.Debugf("%s: channel is not paused so not resuming channel", c.channelID) + return nil + } + + c.state = channelOpen + + // If the requester cancelled, bail out + if c.requesterCancelled { + // If there was an associated message, we still want to send it to the + // remote peer. We're not sending any message now, so instead queue up + // the message to be sent next time the peer makes a request to us. + c.pendingExtensions = append(c.pendingExtensions, extensions...) + + log.Debugf("%s: requester has cancelled so not unpausing for now", c.channelID) + return nil + } + + log.Debugf("%s: unpausing response", c.channelID) + return c.gs.Unpause(ctx, *c.requestID, extensions...) +} + +func (c *Channel) MarkTransferComplete() { + c.lk.Lock() + defer c.lk.Unlock() + c.state = channelClosed +} + +// Called when the responder gets a cancel message from the requester +func (c *Channel) OnRequesterCancelled() { + c.lk.Lock() + defer c.lk.Unlock() + + c.requesterCancelled = true +} + +func (c *Channel) hasStore() bool { + c.storeLk.RLock() + defer c.storeLk.RUnlock() + + return c.storeRegistered +} + +// Use the given loader and storer to get / put blocks for the data-transfer. +// Note that each data-transfer channel uses a separate blockstore. +func (c *Channel) UseStore(lsys ipld.LinkSystem) error { + c.storeLk.Lock() + defer c.storeLk.Unlock() + + // Register the channel's store with graphsync + err := c.gs.RegisterPersistenceOption("data-transfer-"+c.channelID.String(), lsys) + if err != nil { + return err + } + + c.storeRegistered = true + + return nil +} + +func (c *Channel) UpdateReceivedCidsIfGreater(nextIdx int64) { + c.lk.Lock() + defer c.lk.Unlock() + if c.receivedCidsTotal < nextIdx { + c.receivedCidsTotal = nextIdx + } +} + +func (c *Channel) Cleanup() { + c.lk.Lock() + defer c.lk.Unlock() + + log.Debugf("%s: cleaning up channel", c.channelID) + + if c.hasStore() { + // Unregister the channel's store from graphsync + opt := "data-transfer-" + c.channelID.String() + err := c.gs.UnregisterPersistenceOption(opt) + if err != nil { + log.Errorf("failed to unregister persistence option %s: %s", opt, err) + } + } + +} + +func (c *Channel) Close(ctx context.Context) error { + // Cancel the graphsync request + c.lk.Lock() + errch := c.cancel(ctx) + c.lk.Unlock() + + // Wait for the cancel message to complete + select { + case err := <-errch: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +// cancel the graphsync request. +// Note: must be called under the lock. +func (c *Channel) cancel(ctx context.Context) chan error { + errch := make(chan error, 1) + + // Check that the request has not already been cancelled + if c.requesterCancelled || c.state == channelClosed { + errch <- nil + return errch + } + + // Clear the graphsync key to indicate that the request has been cancelled + requestID := c.requestID + c.requestID = nil + c.state = channelClosed + go func() { + log.Debugf("%s: cancelling request", c.channelID) + err := c.gs.Cancel(ctx, *requestID) + + // Ignore "request not found" errors + if err != nil && !errors.Is(graphsync.RequestNotFoundErr{}, err) { + errch <- fmt.Errorf("cancelling graphsync request for channel %s: %w", c.channelID, err) + } else { + errch <- nil + } + }() + + return errch +} + +func (c *Channel) IsCurrentRequest(requestID graphsync.RequestID) bool { + return c.requestID != nil && *c.requestID == requestID +} diff --git a/transport/graphsync/executor/executor.go b/transport/graphsync/executor/executor.go new file mode 100644 index 00000000..0bc23e4f --- /dev/null +++ b/transport/graphsync/executor/executor.go @@ -0,0 +1,109 @@ +package executor + +import ( + "fmt" + + "github.com/ipfs/go-graphsync" + logging "github.com/ipfs/go-log/v2" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" +) + +var log = logging.Logger("dt_graphsync") + +// EventsHandler are the data transfer events that can be dispatched by the execetor +type EventsHandler interface { + OnRequestCancelled(datatransfer.ChannelID, error) error + OnChannelCompleted(datatransfer.ChannelID, error) error +} + +// Executor handles consuming channels on an outgoing GraphSync request +type Executor struct { + channelID datatransfer.ChannelID + responseChan <-chan graphsync.ResponseProgress + errChan <-chan error + onComplete func() +} + +// NewExecutor sets up a new executor to consume a graphsync request +func NewExecutor( + channelID datatransfer.ChannelID, + responseChan <-chan graphsync.ResponseProgress, + errChan <-chan error, + onComplete func()) *Executor { + return &Executor{channelID, responseChan, errChan, onComplete} +} + +// Start initiates consumption of a graphsync request +func (e *Executor) Start(events EventsHandler, + completedRequestListener func(channelID datatransfer.ChannelID)) { + go e.executeRequest(events, completedRequestListener) +} + +// Read from the graphsync response and error channels until they are closed, +// and return the last error on the error channel +func (e *Executor) consumeResponses() error { + var lastError error + for range e.responseChan { + } + log.Infof("channel %s: finished consuming graphsync response channel", e.channelID) + + for err := range e.errChan { + lastError = err + } + log.Infof("channel %s: finished consuming graphsync error channel", e.channelID) + + return lastError +} + +// Read from the graphsync response and error channels until they are closed +// or there is an error, then call the channel completed callback +func (e *Executor) executeRequest( + events EventsHandler, + completedRequestListener func(channelID datatransfer.ChannelID)) { + // Make sure to call the onComplete callback before returning + defer func() { + log.Infow("gs request complete for channel", "chid", e.channelID) + e.onComplete() + }() + + // Consume the response and error channels for the graphsync request + lastError := e.consumeResponses() + + // Request cancelled by client + if _, ok := lastError.(graphsync.RequestClientCancelledErr); ok { + terr := fmt.Errorf("graphsync request cancelled") + log.Warnf("channel %s: %s", e.channelID, terr) + if err := events.OnRequestCancelled(e.channelID, terr); err != nil { + log.Error(err) + } + return + } + + // Request cancelled by responder + if _, ok := lastError.(graphsync.RequestCancelledErr); ok { + log.Infof("channel %s: graphsync request cancelled by responder", e.channelID) + // TODO Should we do anything for RequestCancelledErr ? + return + } + + if lastError != nil { + log.Warnf("channel %s: graphsync error: %s", e.channelID, lastError) + } + + log.Debugf("channel %s: finished executing graphsync request", e.channelID) + + var completeErr error + if lastError != nil { + completeErr = fmt.Errorf("channel %s: graphsync request failed to complete: %w", e.channelID, lastError) + } + + // Used by the tests to listen for when a request completes + if completedRequestListener != nil { + completedRequestListener(e.channelID) + } + err := events.OnChannelCompleted(e.channelID, completeErr) + if err != nil { + log.Errorf("channel %s: processing OnChannelCompleted: %s", e.channelID, err) + } +} diff --git a/transport/graphsync/executor/executor_test.go b/transport/graphsync/executor/executor_test.go new file mode 100644 index 00000000..9e5523c1 --- /dev/null +++ b/transport/graphsync/executor/executor_test.go @@ -0,0 +1,140 @@ +package executor_test + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + + "github.com/ipfs/go-graphsync" + "github.com/stretchr/testify/require" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/testutil" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/executor" +) + +func TestExecutor(t *testing.T) { + ctx := context.Background() + chid := testutil.GenerateChannelID() + testCases := map[string]struct { + responseProgresses []graphsync.ResponseProgress + responseErrors []error + hasCompletedRequestHandler bool + expectedEventRecord fakeEvents + }{ + "simple no errors, no listener": { + expectedEventRecord: fakeEvents{ + completedChannel: chid, + completedError: nil, + }, + }, + "simple with error, no listener": { + responseErrors: []error{errors.New("something went wrong")}, + expectedEventRecord: fakeEvents{ + completedChannel: chid, + completedError: fmt.Errorf("channel %s: graphsync request failed to complete: %w", chid, errors.New("something went wrong")), + }, + }, + "client cancelled request error, no listener": { + responseErrors: []error{graphsync.RequestClientCancelledErr{}}, + expectedEventRecord: fakeEvents{ + cancelledChannel: chid, + cancelledErr: errors.New("graphsync request cancelled"), + }, + }, + // no events called here + "cancelled request error, no listener": { + responseErrors: []error{graphsync.RequestCancelledErr{}}, + }, + "has completed request handler": { + expectedEventRecord: fakeEvents{ + completedChannel: chid, + completedError: nil, + }, + hasCompletedRequestHandler: true, + }, + } + for testCase, data := range testCases { + t.Run(testCase, func(t *testing.T) { + responseChan := make(chan graphsync.ResponseProgress) + errChan := make(chan error) + events := &fakeEvents{} + fcrl := &fakeCompletedRequestListener{} + + completed := make(chan struct{}) + var onCompleteOnce sync.Once + + onComplete := func() { + onCompleteOnce.Do(func() { + close(completed) + }) + } + e := executor.NewExecutor(chid, responseChan, errChan, onComplete) + if data.hasCompletedRequestHandler { + e.Start(events, fcrl.complete) + } else { + e.Start(events, nil) + } + + for _, progress := range data.responseProgresses { + select { + case <-ctx.Done(): + t.Fatal("unable to queue all responses") + case responseChan <- progress: + } + } + close(responseChan) + + for _, err := range data.responseErrors { + select { + case <-ctx.Done(): + t.Fatal("unable to queue all errors") + case errChan <- err: + } + } + close(errChan) + + select { + case <-ctx.Done(): + t.Fatal("did not complete request") + case <-completed: + } + + require.Equal(t, data.expectedEventRecord, *events) + if data.hasCompletedRequestHandler { + require.Equal(t, chid, fcrl.calledChannel) + } else { + require.NotEqual(t, chid, fcrl.calledChannel) + } + }) + } +} + +type fakeEvents struct { + completedChannel datatransfer.ChannelID + completedError error + cancelledChannel datatransfer.ChannelID + cancelledErr error +} + +func (fe *fakeEvents) OnChannelCompleted(chid datatransfer.ChannelID, err error) error { + fe.completedChannel = chid + fe.completedError = err + return nil +} + +func (fe *fakeEvents) OnRequestCancelled(chid datatransfer.ChannelID, err error) error { + fe.cancelledChannel = chid + fe.cancelledErr = err + return nil +} + +type fakeCompletedRequestListener struct { + calledChannel datatransfer.ChannelID +} + +func (fcrl *fakeCompletedRequestListener) complete(channelID datatransfer.ChannelID) { + fcrl.calledChannel = channelID +} diff --git a/transport/graphsync/graphsync.go b/transport/graphsync/graphsync.go index b2e27f14..058a2aec 100644 --- a/transport/graphsync/graphsync.go +++ b/transport/graphsync/graphsync.go @@ -2,21 +2,21 @@ package graphsync import ( "context" - "errors" - "fmt" "sync" "time" "github.com/ipfs/go-graphsync" - "github.com/ipfs/go-graphsync/donotsendfirstblocks" logging "github.com/ipfs/go-log/v2" ipld "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" peer "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/sync/errgroup" "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/network" + dtchannel "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/dtchannel" "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" ) @@ -26,7 +26,6 @@ var log = logging.Logger("dt_graphsync") // before opening a new one. // This constant defines the maximum time to wait for the request to be // cancelled. -const maxGSCancelWait = time.Second var defaultSupportedExtensions = []graphsync.ExtensionName{ extension.ExtensionDataTransfer1_1, @@ -71,6 +70,7 @@ func RegisterCompletedResponseListener(l func(channelID datatransfer.ChannelID)) type Transport struct { events datatransfer.EventsHandler gs graphsync.GraphExchange + dtNet network.DataTransferNetwork peerID peer.ID supportedExtensions []graphsync.ExtensionName @@ -80,7 +80,7 @@ type Transport struct { // Map from data transfer channel ID to information about that channel dtChannelsLk sync.RWMutex - dtChannels map[datatransfer.ChannelID]*dtChannel + dtChannels map[datatransfer.ChannelID]*dtchannel.Channel // Used in graphsync callbacks to map from graphsync request to the // associated data-transfer channel ID. @@ -88,12 +88,13 @@ type Transport struct { } // NewTransport makes a new hooks manager with the given hook events interface -func NewTransport(peerID peer.ID, gs graphsync.GraphExchange, options ...Option) *Transport { +func NewTransport(peerID peer.ID, gs graphsync.GraphExchange, dtNet network.DataTransferNetwork, options ...Option) *Transport { t := &Transport{ gs: gs, + dtNet: dtNet, peerID: peerID, supportedExtensions: defaultSupportedExtensions, - dtChannels: make(map[datatransfer.ChannelID]*dtChannel), + dtChannels: make(map[datatransfer.ChannelID]*dtchannel.Channel), requestIDToChannelID: newRequestIDToChannelIDMap(), } for _, option := range options { @@ -102,18 +103,67 @@ func NewTransport(peerID peer.ID, gs graphsync.GraphExchange, options ...Option) return t } -// OpenChannel initiates an outgoing request for the other peer to send data -// to us on this channel -// Note: from a data transfer symantic standpoint, it doesn't matter if the -// request is push or pull -- OpenChannel is called by the party that is -// intending to receive data +func (t *Transport) Capabilities() datatransfer.TransportCapabilities { + return datatransfer.TransportCapabilities{ + Pausable: true, + Restartable: true, + } +} + +// OpenChannel opens a channel on a given transport to move data back and forth. +// OpenChannel MUST ALWAYS called by the initiator. func (t *Transport) OpenChannel( + ctx context.Context, + channel datatransfer.Channel, + req datatransfer.Request) error { + t.dtNet.Protect(channel.OtherPeer(), channel.ChannelID().String()) + if channel.IsPull() { + return t.openRequest(ctx, + channel.Sender(), + channel.ChannelID(), + cidlink.Link{Cid: channel.BaseCID()}, + channel.Selector(), + req) + } + return t.dtNet.SendMessage(ctx, channel.OtherPeer(), req) +} + +// RestartChannel restarts a channel on the initiator side +// RestartChannel MUST ALWAYS called by the initiator +func (t *Transport) RestartChannel( + ctx context.Context, + channelState datatransfer.ChannelState, + req datatransfer.Request) error { + log.Debugf("%s: re-establishing connection to %s", channelState.ChannelID(), channelState.OtherPeer()) + start := time.Now() + err := t.dtNet.ConnectWithRetry(ctx, channelState.OtherPeer()) + if err != nil { + return xerrors.Errorf("%s: failed to reconnect to peer %s after %s: %w", + channelState.ChannelID(), channelState.OtherPeer(), time.Since(start), err) + } + log.Debugf("%s: re-established connection to %s in %s", channelState.ChannelID(), channelState.OtherPeer(), time.Since(start)) + + t.dtNet.Protect(channelState.OtherPeer(), channelState.ChannelID().String()) + + ch := t.trackDTChannel(channelState.ChannelID()) + ch.UpdateReceivedCidsIfGreater(channelState.ReceivedCidsTotal()) + if channelState.IsPull() { + return t.openRequest(ctx, + channelState.Sender(), + channelState.ChannelID(), + cidlink.Link{Cid: channelState.BaseCID()}, + channelState.Selector(), + req) + } + return t.dtNet.SendMessage(ctx, channelState.OtherPeer(), req) +} + +func (t *Transport) openRequest( ctx context.Context, dataSender peer.ID, channelID datatransfer.ChannelID, root ipld.Link, stor datamodel.Node, - channel datatransfer.ChannelState, msg datatransfer.Message, ) error { if t.events == nil { @@ -124,151 +174,72 @@ func (t *Transport) OpenChannel( if err != nil { return err } - // If this is a restart request, the client can indicate the blocks that - // it has already received, so that the provider knows not to resend - // those blocks - restartExts, err := t.getRestartExtension(ctx, dataSender, channel) - if err != nil { - return err - } - exts = append(exts, restartExts...) // Start tracking the data-transfer channel ch := t.trackDTChannel(channelID) + requestID := graphsync.NewRequestID() + t.requestIDToChannelID.set(requestID, false, channelID) + // Open a graphsync request to the remote peer - req, err := ch.open(ctx, channelID, dataSender, root, stor, channel, exts) + execetor, err := ch.Open(ctx, requestID, dataSender, root, stor, exts) + if err != nil { return err } - // Process incoming data - go t.executeGsRequest(req) - + execetor.Start(t.events, t.completedRequestListener) return nil } -// Get the extension data for sending a Restart message, depending on the -// protocol version of the peer -func (t *Transport) getRestartExtension(ctx context.Context, p peer.ID, channel datatransfer.ChannelState) ([]graphsync.ExtensionData, error) { - if channel == nil { - return nil, nil - } - return getDoNotSendFirstBlocksExtension(channel) -} - -// Skip the first N blocks because they were already received -func getDoNotSendFirstBlocksExtension(channel datatransfer.ChannelState) ([]graphsync.ExtensionData, error) { - skipBlockCount := channel.ReceivedCidsTotal() - data := donotsendfirstblocks.EncodeDoNotSendFirstBlocks(skipBlockCount) - return []graphsync.ExtensionData{{ - Name: graphsync.ExtensionsDoNotSendFirstBlocks, - Data: data, - }}, nil -} - -// Read from the graphsync response and error channels until they are closed, -// and return the last error on the error channel -func (t *Transport) consumeResponses(req *gsReq) error { - var lastError error - for range req.responseChan { - } - log.Infof("channel %s: finished consuming graphsync response channel", req.channelID) - - for err := range req.errChan { - lastError = err - } - log.Infof("channel %s: finished consuming graphsync error channel", req.channelID) - - return lastError -} - -// Read from the graphsync response and error channels until they are closed -// or there is an error, then call the channel completed callback -func (t *Transport) executeGsRequest(req *gsReq) { - // Make sure to call the onComplete callback before returning - defer func() { - log.Infow("gs request complete for channel", "chid", req.channelID) - req.onComplete() - }() - - // Consume the response and error channels for the graphsync request - lastError := t.consumeResponses(req) +// UpdateChannel sends one or more updates the transport channel at once, +// such as pausing/resuming, closing the transfer, or sending additional +// messages over the channel. Grouping the commands allows the transport +// the ability to plan how to execute these updates +func (t *Transport) UpdateChannel(ctx context.Context, chid datatransfer.ChannelID, update datatransfer.ChannelUpdate) error { - // Request cancelled by client - if _, ok := lastError.(graphsync.RequestClientCancelledErr); ok { - terr := xerrors.Errorf("graphsync request cancelled") - log.Warnf("channel %s: %s", req.channelID, terr) - if err := t.events.OnRequestCancelled(req.channelID, terr); err != nil { - log.Error(err) + ch, err := t.getDTChannel(chid) + if err != nil { + if update.SendMessage != nil && !update.Closed { + return t.dtNet.SendMessage(ctx, t.otherPeer(chid), update.SendMessage) } - return - } - - // Request cancelled by responder - if _, ok := lastError.(graphsync.RequestCancelledErr); ok { - log.Infof("channel %s: graphsync request cancelled by responder", req.channelID) - // TODO Should we do anything for RequestCancelledErr ? - return + return err } - if lastError != nil { - log.Warnf("channel %s: graphsync error: %s", req.channelID, lastError) - } + if !update.Paused && ch.Paused() { - log.Debugf("channel %s: finished executing graphsync request", req.channelID) + var extensions []graphsync.ExtensionData + if update.SendMessage != nil { + var err error + extensions, err = extension.ToExtensionData(update.SendMessage, t.supportedExtensions) + if err != nil { + return err + } + } - var completeErr error - if lastError != nil { - completeErr = xerrors.Errorf("channel %s: graphsync request failed to complete: %w", req.channelID, lastError) + return ch.Resume(ctx, extensions) } - // Used by the tests to listen for when a request completes - if t.completedRequestListener != nil { - t.completedRequestListener(req.channelID) + if update.SendMessage != nil { + if err := t.dtNet.SendMessage(ctx, t.otherPeer(chid), update.SendMessage); err != nil { + return err + } } - err := t.events.OnChannelCompleted(req.channelID, completeErr) - if err != nil { - log.Errorf("channel %s: processing OnChannelCompleted: %s", req.channelID, err) + if update.Closed { + return ch.Close(ctx) } -} -// PauseChannel pauses the given data-transfer channel -func (t *Transport) PauseChannel(ctx context.Context, chid datatransfer.ChannelID) error { - ch, err := t.getDTChannel(chid) - if err != nil { - return err + if update.Paused && !ch.Paused() { + return ch.Pause(ctx) } - return ch.pause(ctx) -} -// ResumeChannel resumes the given data-transfer channel and sends the message -// if there is one -func (t *Transport) ResumeChannel( - ctx context.Context, - msg datatransfer.Message, - chid datatransfer.ChannelID, -) error { - ch, err := t.getDTChannel(chid) - if err != nil { - return err - } - return ch.resume(ctx, msg) + return nil } -// CloseChannel closes the given data-transfer channel -func (t *Transport) CloseChannel(ctx context.Context, chid datatransfer.ChannelID) error { - ch, err := t.getDTChannel(chid) - if err != nil { - return err - } - - err = ch.close(ctx) - if err != nil { - return xerrors.Errorf("closing channel: %w", err) - } - return nil +// SendMessage sends a data transfer message over the channel to the other peer +func (t *Transport) SendMessage(ctx context.Context, chid datatransfer.ChannelID, msg datatransfer.Message) error { + return t.dtNet.SendMessage(ctx, t.otherPeer(chid), msg) } // CleanupChannel is called on the otherside of a cancel - removes any associated @@ -284,10 +255,15 @@ func (t *Transport) CleanupChannel(chid datatransfer.ChannelID) { t.dtChannelsLk.Unlock() + // Clean up mapping from gs key to channel ID + t.requestIDToChannelID.deleteRefs(chid) + // Clean up the channel if ok { - ch.cleanup() + ch.Cleanup() } + + t.dtNet.Unprotect(t.otherPeer(chid), chid.String()) } // SetEventHandler sets the handler for events on channels @@ -309,6 +285,8 @@ func (t *Transport) SetEventHandler(events datatransfer.EventsHandler) error { t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterRequestorCancelledListener(t.gsRequestorCancelledListener)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterNetworkErrorListener(t.gsNetworkSendErrorListener)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterReceiverNetworkErrorListener(t.gsNetworkReceiveErrorListener)) + + t.dtNet.SetDelegate(&receiver{t}) return nil } @@ -325,7 +303,7 @@ func (t *Transport) Shutdown(ctx context.Context) error { for _, ch := range t.dtChannels { ch := ch eg.Go(func() error { - return ch.shutdown(ctx) + return ch.Close(ctx) }) } @@ -339,7 +317,7 @@ func (t *Transport) Shutdown(ctx context.Context) error { // UseStore tells the graphsync transport to use the given loader and storer for this channelID func (t *Transport) UseStore(channelID datatransfer.ChannelID, lsys ipld.LinkSystem) error { ch := t.trackDTChannel(channelID) - return ch.useStore(lsys) + return ch.UseStore(lsys) } // ChannelGraphsyncRequests describes any graphsync request IDs associated with a given channel @@ -387,7 +365,7 @@ func (t *Transport) ChannelsForPeer(p peer.ID) ChannelsForPeer { channelGraphsyncRequests := collection[chid] // finally, determine if the request key matches the current GraphSync key we're tracking for // this channel, indicating it's the current graphsync request - if t.dtChannels[chid] != nil && t.dtChannels[chid].requestID != nil && (*t.dtChannels[chid].requestID) == requestID { + if t.dtChannels[chid] != nil && t.dtChannels[chid].IsCurrentRequest(requestID) { channelGraphsyncRequests.Current = requestID } else { // otherwise this id was a previous graphsync request on a channel that was restarted @@ -403,911 +381,4 @@ func (t *Transport) ChannelsForPeer(p peer.ID) ChannelsForPeer { } } -// gsOutgoingRequestHook is called when a graphsync request is made -func (t *Transport) gsOutgoingRequestHook(p peer.ID, request graphsync.RequestData, hookActions graphsync.OutgoingRequestHookActions) { - message, _ := extension.GetTransferData(request, t.supportedExtensions) - - // extension not found; probably not our request. - if message == nil { - return - } - - // A graphsync request is made when either - // - The local node opens a data-transfer pull channel, so the local node - // sends a graphsync request to ask the remote peer for the data - // - The remote peer opened a data-transfer push channel, and in response - // the local node sends a graphsync request to ask for the data - var initiator peer.ID - var responder peer.ID - if message.IsRequest() { - // This is a pull request so the data-transfer initiator is the local node - initiator = t.peerID - responder = p - } else { - // This is a push response so the data-transfer initiator is the remote - // peer: They opened the push channel, we respond by sending a - // graphsync request for the data - initiator = p - responder = t.peerID - } - chid := datatransfer.ChannelID{Initiator: initiator, Responder: responder, ID: message.TransferID()} - - // A data transfer channel was opened - err := t.events.OnChannelOpened(chid) - if err != nil { - // There was an error opening the channel, bail out - log.Errorf("processing OnChannelOpened for %s: %s", chid, err) - t.CleanupChannel(chid) - return - } - - // Start tracking the channel if we're not already - ch := t.trackDTChannel(chid) - - // Signal that the channel has been opened - ch.gsReqOpened(request.ID(), hookActions) -} - -// gsIncomingBlockHook is called when a block is received -func (t *Transport) gsIncomingBlockHook(p peer.ID, response graphsync.ResponseData, block graphsync.BlockData, hookActions graphsync.IncomingBlockHookActions) { - chid, ok := t.requestIDToChannelID.load(response.RequestID()) - if !ok { - return - } - - err := t.events.OnDataReceived(chid, block.Link(), block.BlockSize(), block.Index(), block.BlockSizeOnWire() != 0) - if err != nil && err != datatransfer.ErrPause { - hookActions.TerminateWithError(err) - return - } - - if err == datatransfer.ErrPause { - hookActions.PauseRequest() - } -} - -func (t *Transport) gsBlockSentHook(p peer.ID, request graphsync.RequestData, block graphsync.BlockData) { - // When a data transfer is restarted, the requester sends a list of CIDs - // that it already has. Graphsync calls the sent hook for all blocks even - // if they are in the list (meaning, they aren't actually sent over the - // wire). So here we check if the block was actually sent - // over the wire before firing the data sent event. - if block.BlockSizeOnWire() == 0 { - return - } - - chid, ok := t.requestIDToChannelID.load(request.ID()) - if !ok { - return - } - - if err := t.events.OnDataSent(chid, block.Link(), block.BlockSize(), block.Index(), block.BlockSizeOnWire() != 0); err != nil { - log.Errorf("failed to process data sent: %+v", err) - } -} - -func (t *Transport) gsOutgoingBlockHook(p peer.ID, request graphsync.RequestData, block graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { - // When a data transfer is restarted, the requester sends a list of CIDs - // that it already has. Graphsync calls the outgoing block hook for all - // blocks even if they are in the list (meaning, they aren't actually going - // to be sent over the wire). So here we check if the block is actually - // going to be sent over the wire before firing the data queued event. - if block.BlockSizeOnWire() == 0 { - return - } - - chid, ok := t.requestIDToChannelID.load(request.ID()) - if !ok { - return - } - - // OnDataQueued is called when a block is queued to be sent to the remote - // peer. It can return ErrPause to pause the response (eg if payment is - // required) and it can return a message that will be sent with the block - // (eg to ask for payment). - msg, err := t.events.OnDataQueued(chid, block.Link(), block.BlockSize(), block.Index(), block.BlockSizeOnWire() != 0) - if err != nil && err != datatransfer.ErrPause { - hookActions.TerminateWithError(err) - return - } - - if err == datatransfer.ErrPause { - hookActions.PauseResponse() - } - - if msg != nil { - // gsOutgoingBlockHook uses a unique extension name so it can be attached with data from a different hook - // outgoingBlkExtensions also includes the default extension name so it remains compatible with all data-transfer protocol versions out there - extensions, err := extension.ToExtensionData(msg, outgoingBlkExtensions) - if err != nil { - hookActions.TerminateWithError(err) - return - } - for _, extension := range extensions { - hookActions.SendExtensionData(extension) - } - } -} - -// gsReqQueuedHook is called when graphsync enqueues an incoming request for data -func (t *Transport) gsReqQueuedHook(p peer.ID, request graphsync.RequestData, hookActions graphsync.RequestQueuedHookActions) { - msg, err := extension.GetTransferData(request, t.supportedExtensions) - if err != nil { - log.Errorf("failed GetTransferData, req=%+v, err=%s", request, err) - } - // extension not found; probably not our request. - if msg == nil { - return - } - - var chid datatransfer.ChannelID - if msg.IsRequest() { - // when a data transfer request comes in on graphsync, the remote peer - // initiated a pull - chid = datatransfer.ChannelID{ID: msg.TransferID(), Initiator: p, Responder: t.peerID} - dtRequest := msg.(datatransfer.Request) - if dtRequest.IsNew() { - log.Infof("%s, pull request queued, req_id=%d", chid, request.ID()) - t.events.OnTransferQueued(chid) - } else { - log.Infof("%s, pull restart request queued, req_id=%d", chid, request.ID()) - } - } else { - // when a data transfer response comes in on graphsync, this node - // initiated a push, and the remote peer responded with a request - // for data - chid = datatransfer.ChannelID{ID: msg.TransferID(), Initiator: t.peerID, Responder: p} - response := msg.(datatransfer.Response) - if response.IsNew() { - log.Infof("%s, GS pull request queued in response to our push, req_id=%d", chid, request.ID()) - t.events.OnTransferQueued(chid) - } else { - log.Infof("%s, GS pull request queued in response to our restart push, req_id=%d", chid, request.ID()) - } - } - augmentContext := t.events.OnContextAugment(chid) - if augmentContext != nil { - hookActions.AugmentContext(augmentContext) - } -} - -// gsReqRecdHook is called when graphsync receives an incoming request for data -func (t *Transport) gsReqRecdHook(p peer.ID, request graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { - // if this is a push request the sender is us. - msg, err := extension.GetTransferData(request, t.supportedExtensions) - if err != nil { - hookActions.TerminateWithError(err) - return - } - - // extension not found; probably not our request. - if msg == nil { - return - } - - // An incoming graphsync request for data is received when either - // - The remote peer opened a data-transfer pull channel, so the local node - // receives a graphsync request for the data - // - The local node opened a data-transfer push channel, and in response - // the remote peer sent a graphsync request for the data, and now the - // local node receives that request for data - var chid datatransfer.ChannelID - var responseMessage datatransfer.Message - var ch *dtChannel - if msg.IsRequest() { - // when a data transfer request comes in on graphsync, the remote peer - // initiated a pull - chid = datatransfer.ChannelID{ID: msg.TransferID(), Initiator: p, Responder: t.peerID} - - log.Debugf("%s: received request for data (pull), req_id=%d", chid, request.ID()) - - // Lock the channel for the duration of this method - ch = t.trackDTChannel(chid) - ch.lk.Lock() - defer ch.lk.Unlock() - - request := msg.(datatransfer.Request) - responseMessage, err = t.events.OnRequestReceived(chid, request) - } else { - // when a data transfer response comes in on graphsync, this node - // initiated a push, and the remote peer responded with a request - // for data - chid = datatransfer.ChannelID{ID: msg.TransferID(), Initiator: t.peerID, Responder: p} - - log.Debugf("%s: received request for data (push), req_id=%d", chid, request.ID()) - - // Lock the channel for the duration of this method - ch = t.trackDTChannel(chid) - ch.lk.Lock() - defer ch.lk.Unlock() - - response := msg.(datatransfer.Response) - err = t.events.OnResponseReceived(chid, response) - } - - // If we need to send a response, add the response message as an extension - if responseMessage != nil { - // gsReqRecdHook uses a unique extension name so it can be attached with data from a different hook - // incomingReqExtensions also includes default extension name so it remains compatible with previous data-transfer - // protocol versions out there. - extensions, extensionErr := extension.ToExtensionData(responseMessage, incomingReqExtensions) - if extensionErr != nil { - hookActions.TerminateWithError(err) - return - } - for _, extension := range extensions { - hookActions.SendExtensionData(extension) - } - } - - if err != nil && err != datatransfer.ErrPause { - hookActions.TerminateWithError(err) - return - } - - // Check if the callback indicated that the channel should be paused - // immediately (eg because data is still being unsealed) - paused := false - if err == datatransfer.ErrPause { - log.Debugf("%s: pausing graphsync response", chid) - - paused = true - hookActions.PauseResponse() - } - - // If this is a restart request, and the data transfer still hasn't got - // out of the paused state (eg because we're still unsealing), start this - // graphsync response in the paused state. - if ch.isOpen && !ch.xferStarted && !paused { - log.Debugf("%s: pausing graphsync response after restart", chid) - - paused = true - hookActions.PauseResponse() - } - - // If the transfer is not paused, record that the transfer has started - if !paused { - ch.xferStarted = true - } - - ch.gsDataRequestRcvd(request.ID(), hookActions) - - hookActions.ValidateRequest() -} - -// gsCompletedResponseListener is a graphsync.OnCompletedResponseListener. We use it learn when the data transfer is complete -// for the side that is responding to a graphsync request -func (t *Transport) gsCompletedResponseListener(p peer.ID, request graphsync.RequestData, status graphsync.ResponseStatusCode) { - chid, ok := t.requestIDToChannelID.load(request.ID()) - if !ok { - return - } - - if status == graphsync.RequestCancelled { - return - } - - var completeErr error - if status != graphsync.RequestCompletedFull { - statusStr := gsResponseStatusCodeString(status) - completeErr = xerrors.Errorf("graphsync response to peer %s did not complete: response status code %s", p, statusStr) - } - - // Used by the tests to listen for when a response completes - if t.completedResponseListener != nil { - t.completedResponseListener(chid) - } - - err := t.events.OnChannelCompleted(chid, completeErr) - if err != nil { - log.Error(err) - } -} - -// Remove this map once this PR lands: https://github.com/ipfs/go-graphsync/pull/148 -var gsResponseStatusCodes = map[graphsync.ResponseStatusCode]string{ - graphsync.RequestAcknowledged: "RequestAcknowledged", - graphsync.AdditionalPeers: "AdditionalPeers", - graphsync.NotEnoughGas: "NotEnoughGas", - graphsync.OtherProtocol: "OtherProtocol", - graphsync.PartialResponse: "PartialResponse", - graphsync.RequestPaused: "RequestPaused", - graphsync.RequestCompletedFull: "RequestCompletedFull", - graphsync.RequestCompletedPartial: "RequestCompletedPartial", - graphsync.RequestRejected: "RequestRejected", - graphsync.RequestFailedBusy: "RequestFailedBusy", - graphsync.RequestFailedUnknown: "RequestFailedUnknown", - graphsync.RequestFailedLegal: "RequestFailedLegal", - graphsync.RequestFailedContentNotFound: "RequestFailedContentNotFound", - graphsync.RequestCancelled: "RequestCancelled", -} - -func gsResponseStatusCodeString(code graphsync.ResponseStatusCode) string { - str, ok := gsResponseStatusCodes[code] - if ok { - return str - } - return gsResponseStatusCodes[graphsync.RequestFailedUnknown] -} - -func (t *Transport) gsRequestUpdatedHook(p peer.ID, request graphsync.RequestData, update graphsync.RequestData, hookActions graphsync.RequestUpdatedHookActions) { - chid, ok := t.requestIDToChannelID.load(request.ID()) - if !ok { - return - } - - responseMessage, err := t.processExtension(chid, update, p, t.supportedExtensions) - - if responseMessage != nil { - extensions, extensionErr := extension.ToExtensionData(responseMessage, t.supportedExtensions) - if extensionErr != nil { - hookActions.TerminateWithError(err) - return - } - for _, extension := range extensions { - hookActions.SendExtensionData(extension) - } - } - - if err != nil && err != datatransfer.ErrPause { - hookActions.TerminateWithError(err) - } - -} - -// gsIncomingResponseHook is a graphsync.OnIncomingResponseHook. We use it to pass on responses -func (t *Transport) gsIncomingResponseHook(p peer.ID, response graphsync.ResponseData, hookActions graphsync.IncomingResponseHookActions) { - chid, ok := t.requestIDToChannelID.load(response.RequestID()) - if !ok { - return - } - - responseMessage, err := t.processExtension(chid, response, p, incomingReqExtensions) - - if responseMessage != nil { - extensions, extensionErr := extension.ToExtensionData(responseMessage, t.supportedExtensions) - if extensionErr != nil { - hookActions.TerminateWithError(err) - return - } - for _, extension := range extensions { - hookActions.UpdateRequestWithExtensions(extension) - } - } - - if err != nil { - hookActions.TerminateWithError(err) - } - - // In a case where the transfer sends blocks immediately this extension may contain both a - // response message and a revalidation request so we trigger OnResponseReceived again for this - // specific extension name - _, err = t.processExtension(chid, response, p, []graphsync.ExtensionName{extension.ExtensionOutgoingBlock1_1}) - - if err != nil { - hookActions.TerminateWithError(err) - } -} - -func (t *Transport) processExtension(chid datatransfer.ChannelID, gsMsg extension.GsExtended, p peer.ID, exts []graphsync.ExtensionName) (datatransfer.Message, error) { - - // if this is a push request the sender is us. - msg, err := extension.GetTransferData(gsMsg, exts) - if err != nil { - return nil, err - } - - // extension not found; probably not our request. - if msg == nil { - return nil, nil - } - - if msg.IsRequest() { - - // only accept request message updates when original message was also request - if (chid != datatransfer.ChannelID{ID: msg.TransferID(), Initiator: p, Responder: t.peerID}) { - return nil, errors.New("received request on response channel") - } - dtRequest := msg.(datatransfer.Request) - return t.events.OnRequestReceived(chid, dtRequest) - } - - // only accept response message updates when original message was also response - if (chid != datatransfer.ChannelID{ID: msg.TransferID(), Initiator: t.peerID, Responder: p}) { - return nil, errors.New("received response on request channel") - } - - dtResponse := msg.(datatransfer.Response) - return nil, t.events.OnResponseReceived(chid, dtResponse) -} - -func (t *Transport) gsRequestorCancelledListener(p peer.ID, request graphsync.RequestData) { - chid, ok := t.requestIDToChannelID.load(request.ID()) - if !ok { - return - } - - ch, err := t.getDTChannel(chid) - if err != nil { - if !xerrors.Is(datatransfer.ErrChannelNotFound, err) { - log.Errorf("requestor cancelled: getting channel %s: %s", chid, err) - } - return - } - - log.Debugf("%s: requester cancelled data-transfer", chid) - ch.onRequesterCancelled() -} - -// Called when there is a graphsync error sending data -func (t *Transport) gsNetworkSendErrorListener(p peer.ID, request graphsync.RequestData, gserr error) { - // Fire an error if the graphsync request was made by this node or the remote peer - chid, ok := t.requestIDToChannelID.load(request.ID()) - if !ok { - return - } - - err := t.events.OnSendDataError(chid, gserr) - if err != nil { - log.Errorf("failed to fire transport send error %s: %s", gserr, err) - } -} - -// Called when there is a graphsync error receiving data -func (t *Transport) gsNetworkReceiveErrorListener(p peer.ID, gserr error) { - // Fire a receive data error on all ongoing graphsync transfers with that - // peer - t.requestIDToChannelID.forEach(func(k graphsync.RequestID, sending bool, chid datatransfer.ChannelID) { - if chid.Initiator != p && chid.Responder != p { - return - } - - err := t.events.OnReceiveDataError(chid, gserr) - if err != nil { - log.Errorf("failed to fire transport receive error %s: %s", gserr, err) - } - }) -} - -func (t *Transport) newDTChannel(chid datatransfer.ChannelID) *dtChannel { - return &dtChannel{ - t: t, - channelID: chid, - opened: make(chan graphsync.RequestID, 1), - } -} - -func (t *Transport) trackDTChannel(chid datatransfer.ChannelID) *dtChannel { - t.dtChannelsLk.Lock() - defer t.dtChannelsLk.Unlock() - - ch, ok := t.dtChannels[chid] - if !ok { - ch = t.newDTChannel(chid) - t.dtChannels[chid] = ch - } - - return ch -} - -func (t *Transport) getDTChannel(chid datatransfer.ChannelID) (*dtChannel, error) { - if t.events == nil { - return nil, datatransfer.ErrHandlerNotSet - } - - t.dtChannelsLk.RLock() - defer t.dtChannelsLk.RUnlock() - - ch, ok := t.dtChannels[chid] - if !ok { - return nil, xerrors.Errorf("channel %s: %w", chid, datatransfer.ErrChannelNotFound) - } - return ch, nil -} - -// Info needed to keep track of a data transfer channel -type dtChannel struct { - channelID datatransfer.ChannelID - t *Transport - - lk sync.RWMutex - isOpen bool - requestID *graphsync.RequestID - completed chan struct{} - requesterCancelled bool - xferStarted bool - pendingExtensions []graphsync.ExtensionData - - opened chan graphsync.RequestID - - storeLk sync.RWMutex - storeRegistered bool -} - -// Info needed to monitor an ongoing graphsync request -type gsReq struct { - channelID datatransfer.ChannelID - responseChan <-chan graphsync.ResponseProgress - errChan <-chan error - onComplete func() -} - -// Open a graphsync request for data to the remote peer -func (c *dtChannel) open( - ctx context.Context, - chid datatransfer.ChannelID, - dataSender peer.ID, - root ipld.Link, - stor datamodel.Node, - channel datatransfer.ChannelState, - exts []graphsync.ExtensionData, -) (*gsReq, error) { - c.lk.Lock() - defer c.lk.Unlock() - - // If there is an existing graphsync request for this channelID - if c.requestID != nil { - // Cancel the existing graphsync request - completed := c.completed - errch := c.cancel(ctx) - - // Wait for the complete callback to be called - err := waitForCompleteHook(ctx, completed) - if err != nil { - return nil, xerrors.Errorf("%s: waiting for cancelled graphsync request to complete: %w", chid, err) - } - - // Wait for the cancel request method to complete - select { - case err = <-errch: - case <-ctx.Done(): - err = xerrors.Errorf("timed out waiting for graphsync request to be cancelled") - } - if err != nil { - return nil, xerrors.Errorf("%s: restarting graphsync request: %w", chid, err) - } - } - - // Set up a completed channel that will be closed when the request - // completes (or is cancelled) - completed := make(chan struct{}) - var onCompleteOnce sync.Once - onComplete := func() { - // Ensure the channel is only closed once - onCompleteOnce.Do(func() { - log.Infow("closing the completion ch for data-transfer channel", "chid", chid) - close(completed) - }) - } - c.completed = completed - - // Open a new graphsync request - msg := fmt.Sprintf("Opening graphsync request to %s for root %s", dataSender, root) - if channel != nil { - msg += fmt.Sprintf(" with %d Blocks already received", channel.ReceivedCidsTotal()) - } - log.Info(msg) - responseChan, errChan := c.t.gs.Request(ctx, dataSender, root, stor, exts...) - - // Wait for graphsync "request opened" callback - select { - case <-ctx.Done(): - return nil, ctx.Err() - case requestID := <-c.opened: - // Mark the channel as open and save the Graphsync request key - c.isOpen = true - c.requestID = &requestID - } - - return &gsReq{ - channelID: chid, - responseChan: responseChan, - errChan: errChan, - onComplete: onComplete, - }, nil -} - -func waitForCompleteHook(ctx context.Context, completed chan struct{}) error { - // Wait for the cancel to propagate through to graphsync, and for - // the graphsync request to complete - select { - case <-completed: - return nil - case <-time.After(maxGSCancelWait): - // Fail-safe: give up waiting after a certain amount of time - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - -// gsReqOpened is called when graphsync makes a request to the remote peer to ask for data -func (c *dtChannel) gsReqOpened(requestID graphsync.RequestID, hookActions graphsync.OutgoingRequestHookActions) { - // Tell graphsync to store the received blocks in the registered store - if c.hasStore() { - hookActions.UsePersistenceOption("data-transfer-" + c.channelID.String()) - } - log.Infow("outgoing graphsync request", "peer", c.channelID.OtherParty(c.t.peerID), "graphsync request id", requestID, "data transfer channel id", c.channelID) - // Save a mapping from the graphsync key to the channel ID so that - // subsequent graphsync callbacks are associated with this channel - c.t.requestIDToChannelID.set(requestID, false, c.channelID) - - c.opened <- requestID -} - -// gsDataRequestRcvd is called when the transport receives an incoming request -// for data. -// Note: Must be called under the lock. -func (c *dtChannel) gsDataRequestRcvd(requestID graphsync.RequestID, hookActions graphsync.IncomingRequestHookActions) { - log.Debugf("%s: received request for data, req_id=%d", c.channelID, requestID) - - // If the requester had previously cancelled their request, send any - // message that was queued since the cancel - if c.requesterCancelled { - c.requesterCancelled = false - - extensions := c.pendingExtensions - c.pendingExtensions = nil - for _, ext := range extensions { - hookActions.SendExtensionData(ext) - } - } - - // Tell graphsync to load blocks from the registered store - if c.hasStore() { - hookActions.UsePersistenceOption("data-transfer-" + c.channelID.String()) - } - - // Save a mapping from the graphsync key to the channel ID so that - // subsequent graphsync callbacks are associated with this channel - c.requestID = &requestID - log.Infow("incoming graphsync request", "peer", c.channelID.OtherParty(c.t.peerID), "graphsync request id", requestID, "data transfer channel id", c.channelID) - c.t.requestIDToChannelID.set(requestID, true, c.channelID) - - c.isOpen = true -} - -func (c *dtChannel) pause(ctx context.Context) error { - c.lk.Lock() - defer c.lk.Unlock() - - // Check if the channel was already cancelled - if c.requestID == nil { - log.Debugf("%s: channel was cancelled so not pausing channel", c.channelID) - return nil - } - - // If the requester cancelled, bail out - if c.requesterCancelled { - log.Debugf("%s: requester has cancelled so not pausing response", c.channelID) - return nil - } - - // Pause the response - log.Debugf("%s: pausing response", c.channelID) - return c.t.gs.Pause(ctx, *c.requestID) -} - -func (c *dtChannel) resume(ctx context.Context, msg datatransfer.Message) error { - c.lk.Lock() - defer c.lk.Unlock() - - // Check if the channel was already cancelled - if c.requestID == nil { - log.Debugf("%s: channel was cancelled so not resuming channel", c.channelID) - return nil - } - - var extensions []graphsync.ExtensionData - if msg != nil { - var err error - extensions, err = extension.ToExtensionData(msg, c.t.supportedExtensions) - if err != nil { - return err - } - } - - // If the requester cancelled, bail out - if c.requesterCancelled { - // If there was an associated message, we still want to send it to the - // remote peer. We're not sending any message now, so instead queue up - // the message to be sent next time the peer makes a request to us. - c.pendingExtensions = append(c.pendingExtensions, extensions...) - - log.Debugf("%s: requester has cancelled so not unpausing response", c.channelID) - return nil - } - - // Record that the transfer has started - c.xferStarted = true - - log.Debugf("%s: unpausing response", c.channelID) - return c.t.gs.Unpause(ctx, *c.requestID, extensions...) -} - -func (c *dtChannel) close(ctx context.Context) error { - var errch chan error - c.lk.Lock() - { - // Check if the channel was already cancelled - if c.requestID != nil { - errch = c.cancel(ctx) - } - } - c.lk.Unlock() - - // Wait for the cancel message to complete - select { - case err := <-errch: - return err - case <-ctx.Done(): - return ctx.Err() - } -} - -// Called when the responder gets a cancel message from the requester -func (c *dtChannel) onRequesterCancelled() { - c.lk.Lock() - defer c.lk.Unlock() - - c.requesterCancelled = true -} - -func (c *dtChannel) hasStore() bool { - c.storeLk.RLock() - defer c.storeLk.RUnlock() - - return c.storeRegistered -} - -// Use the given loader and storer to get / put blocks for the data-transfer. -// Note that each data-transfer channel uses a separate blockstore. -func (c *dtChannel) useStore(lsys ipld.LinkSystem) error { - c.storeLk.Lock() - defer c.storeLk.Unlock() - - // Register the channel's store with graphsync - err := c.t.gs.RegisterPersistenceOption("data-transfer-"+c.channelID.String(), lsys) - if err != nil { - return err - } - - c.storeRegistered = true - - return nil -} - -func (c *dtChannel) cleanup() { - c.lk.Lock() - defer c.lk.Unlock() - - log.Debugf("%s: cleaning up channel", c.channelID) - - if c.hasStore() { - // Unregister the channel's store from graphsync - opt := "data-transfer-" + c.channelID.String() - err := c.t.gs.UnregisterPersistenceOption(opt) - if err != nil { - log.Errorf("failed to unregister persistence option %s: %s", opt, err) - } - } - - // Clean up mapping from gs key to channel ID - c.t.requestIDToChannelID.deleteRefs(c.channelID) -} - -func (c *dtChannel) shutdown(ctx context.Context) error { - // Cancel the graphsync request - c.lk.Lock() - errch := c.cancel(ctx) - c.lk.Unlock() - - // Wait for the cancel message to complete - select { - case err := <-errch: - return err - case <-ctx.Done(): - return ctx.Err() - } -} - -// Cancel the graphsync request. -// Note: must be called under the lock. -func (c *dtChannel) cancel(ctx context.Context) chan error { - errch := make(chan error, 1) - - // Check that the request has not already been cancelled - if c.requesterCancelled || c.requestID == nil { - errch <- nil - return errch - } - - // Clear the graphsync key to indicate that the request has been cancelled - requestID := c.requestID - c.requestID = nil - - go func() { - log.Debugf("%s: cancelling request", c.channelID) - err := c.t.gs.Cancel(ctx, *requestID) - - // Ignore "request not found" errors - if err != nil && !xerrors.Is(graphsync.RequestNotFoundErr{}, err) { - errch <- xerrors.Errorf("cancelling graphsync request for channel %s: %w", c.channelID, err) - } else { - errch <- nil - } - }() - - return errch -} - -type channelInfo struct { - sending bool - channelID datatransfer.ChannelID -} - -// Used in graphsync callbacks to map from graphsync request to the -// associated data-transfer channel ID. -type requestIDToChannelIDMap struct { - lk sync.RWMutex - m map[graphsync.RequestID]channelInfo -} - -func newRequestIDToChannelIDMap() *requestIDToChannelIDMap { - return &requestIDToChannelIDMap{ - m: make(map[graphsync.RequestID]channelInfo), - } -} - -// get the value for a key -func (m *requestIDToChannelIDMap) load(key graphsync.RequestID) (datatransfer.ChannelID, bool) { - m.lk.RLock() - defer m.lk.RUnlock() - - val, ok := m.m[key] - return val.channelID, ok -} - -// get the value if any of the keys exists in the map -func (m *requestIDToChannelIDMap) any(ks ...graphsync.RequestID) (datatransfer.ChannelID, bool) { - m.lk.RLock() - defer m.lk.RUnlock() - - for _, k := range ks { - val, ok := m.m[k] - if ok { - return val.channelID, ok - } - } - return datatransfer.ChannelID{}, false -} - -// set the value for a key -func (m *requestIDToChannelIDMap) set(key graphsync.RequestID, sending bool, chid datatransfer.ChannelID) { - m.lk.Lock() - defer m.lk.Unlock() - - m.m[key] = channelInfo{sending, chid} -} - -// call f for each key / value in the map -func (m *requestIDToChannelIDMap) forEach(f func(k graphsync.RequestID, isSending bool, chid datatransfer.ChannelID)) { - m.lk.RLock() - defer m.lk.RUnlock() - - for k, ch := range m.m { - f(k, ch.sending, ch.channelID) - } -} - -// delete any keys that reference this value -func (m *requestIDToChannelIDMap) deleteRefs(id datatransfer.ChannelID) { - m.lk.Lock() - defer m.lk.Unlock() - - for k, ch := range m.m { - if ch.channelID == id { - delete(m.m, k) - } - } -} +var _ datatransfer.Transport = (*Transport)(nil) diff --git a/transport/graphsync/graphsync_test.go b/transport/graphsync/graphsync_test.go index c5bb4a07..5eccc205 100644 --- a/transport/graphsync/graphsync_test.go +++ b/transport/graphsync/graphsync_test.go @@ -1,5 +1,6 @@ package graphsync_test +/* import ( "context" "errors" @@ -7,7 +8,6 @@ import ( "math/rand" "testing" "time" - "github.com/ipfs/go-graphsync" "github.com/ipfs/go-graphsync/donotsendfirstblocks" "github.com/ipld/go-ipld-prime" @@ -17,7 +17,6 @@ import ( peer "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/message" "github.com/filecoin-project/go-data-transfer/v2/testutil" @@ -1360,3 +1359,4 @@ func assertHasExtensionMessage(t *testing.T, name graphsync.ExtensionName, exten require.Fail(t, "extension not found") } } +*/ diff --git a/transport/graphsync/hooks.go b/transport/graphsync/hooks.go new file mode 100644 index 00000000..021fea4e --- /dev/null +++ b/transport/graphsync/hooks.go @@ -0,0 +1,449 @@ +package graphsync + +import ( + "errors" + + "github.com/ipfs/go-graphsync" + peer "github.com/libp2p/go-libp2p-core/peer" + "golang.org/x/xerrors" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" +) + +// gsOutgoingRequestHook is called when a graphsync request is made +func (t *Transport) gsOutgoingRequestHook(p peer.ID, request graphsync.RequestData, hookActions graphsync.OutgoingRequestHookActions) { + message, _ := extension.GetTransferData(request, t.supportedExtensions) + + // extension not found; probably not our request. + if message == nil { + return + } + + // A graphsync request is made when either + // - The local node opens a data-transfer pull channel, so the local node + // sends a graphsync request to ask the remote peer for the data + // - The remote peer opened a data-transfer push channel, and in response + // the local node sends a graphsync request to ask for the data + var initiator peer.ID + var responder peer.ID + if message.IsRequest() { + // This is a pull request so the data-transfer initiator is the local node + initiator = t.peerID + responder = p + } else { + // This is a push response so the data-transfer initiator is the remote + // peer: They opened the push channel, we respond by sending a + // graphsync request for the data + initiator = p + responder = t.peerID + } + chid := datatransfer.ChannelID{Initiator: initiator, Responder: responder, ID: message.TransferID()} + + // A data transfer channel was opened + err := t.events.OnChannelOpened(chid) + if err != nil { + // There was an error opening the channel, bail out + log.Errorf("processing OnChannelOpened for %s: %s", chid, err) + t.CleanupChannel(chid) + return + } + + // Start tracking the channel if we're not already + ch := t.trackDTChannel(chid) + + // Signal that the channel has been opened + ch.GsReqOpened(p, request.ID(), hookActions) +} + +// gsIncomingBlockHook is called when a block is received +func (t *Transport) gsIncomingBlockHook(p peer.ID, response graphsync.ResponseData, block graphsync.BlockData, hookActions graphsync.IncomingBlockHookActions) { + chid, ok := t.requestIDToChannelID.load(response.RequestID()) + if !ok { + return + } + + ch, err := t.getDTChannel(chid) + if err != nil { + hookActions.TerminateWithError(err) + return + } + + ch.UpdateReceivedCidsIfGreater(block.Index()) + + err = t.events.OnDataReceived(chid, block.Link(), block.BlockSize(), block.Index(), block.BlockSizeOnWire() != 0) + if err != nil && err != datatransfer.ErrPause { + hookActions.TerminateWithError(err) + return + } + + if err == datatransfer.ErrPause { + ch.MarkPaused() + hookActions.PauseRequest() + } +} + +func (t *Transport) gsBlockSentHook(p peer.ID, request graphsync.RequestData, block graphsync.BlockData) { + // When a data transfer is restarted, the requester sends a list of CIDs + // that it already has. Graphsync calls the sent hook for all blocks even + // if they are in the list (meaning, they aren't actually sent over the + // wire). So here we check if the block was actually sent + // over the wire before firing the data sent event. + if block.BlockSizeOnWire() == 0 { + return + } + + chid, ok := t.requestIDToChannelID.load(request.ID()) + if !ok { + return + } + + if err := t.events.OnDataSent(chid, block.Link(), block.BlockSize(), block.Index(), block.BlockSizeOnWire() != 0); err != nil { + log.Errorf("failed to process data sent: %+v", err) + } +} + +func (t *Transport) gsOutgoingBlockHook(p peer.ID, request graphsync.RequestData, block graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { + // When a data transfer is restarted, the requester sends a list of CIDs + // that it already has. Graphsync calls the outgoing block hook for all + // blocks even if they are in the list (meaning, they aren't actually going + // to be sent over the wire). So here we check if the block is actually + // going to be sent over the wire before firing the data queued event. + if block.BlockSizeOnWire() == 0 { + return + } + + chid, ok := t.requestIDToChannelID.load(request.ID()) + if !ok { + return + } + + ch, err := t.getDTChannel(chid) + if err != nil { + hookActions.TerminateWithError(err) + return + } + + // OnDataQueued is called when a block is queued to be sent to the remote + // peer. It can return ErrPause to pause the response (eg if payment is + // required) and it can return a message that will be sent with the block + // (eg to ask for payment). + msg, err := t.events.OnDataQueued(chid, block.Link(), block.BlockSize(), block.Index(), block.BlockSizeOnWire() != 0) + if err != nil && err != datatransfer.ErrPause { + hookActions.TerminateWithError(err) + return + } + + if err == datatransfer.ErrPause { + ch.MarkPaused() + hookActions.PauseResponse() + } + + if msg != nil { + // gsOutgoingBlockHook uses a unique extension name so it can be attached with data from a different hook + // outgoingBlkExtensions also includes the default extension name so it remains compatible with all data-transfer protocol versions out there + extensions, err := extension.ToExtensionData(msg, outgoingBlkExtensions) + if err != nil { + hookActions.TerminateWithError(err) + return + } + for _, extension := range extensions { + hookActions.SendExtensionData(extension) + } + } +} + +// gsReqQueuedHook is called when graphsync enqueues an incoming request for data +func (t *Transport) gsReqQueuedHook(p peer.ID, request graphsync.RequestData, hookActions graphsync.RequestQueuedHookActions) { + msg, err := extension.GetTransferData(request, t.supportedExtensions) + if err != nil { + log.Errorf("failed GetTransferData, req=%+v, err=%s", request, err) + } + // extension not found; probably not our request. + if msg == nil { + return + } + + var chid datatransfer.ChannelID + if msg.IsRequest() { + // when a data transfer request comes in on graphsync, the remote peer + // initiated a pull + chid = datatransfer.ChannelID{ID: msg.TransferID(), Initiator: p, Responder: t.peerID} + dtRequest := msg.(datatransfer.Request) + if dtRequest.IsNew() { + log.Infof("%s, pull request queued, req_id=%d", chid, request.ID()) + t.events.OnTransferQueued(chid) + } else { + log.Infof("%s, pull restart request queued, req_id=%d", chid, request.ID()) + } + } else { + // when a data transfer response comes in on graphsync, this node + // initiated a push, and the remote peer responded with a request + // for data + chid = datatransfer.ChannelID{ID: msg.TransferID(), Initiator: t.peerID, Responder: p} + response := msg.(datatransfer.Response) + if response.IsNew() { + log.Infof("%s, GS pull request queued in response to our push, req_id=%d", chid, request.ID()) + t.events.OnTransferQueued(chid) + } else { + log.Infof("%s, GS pull request queued in response to our restart push, req_id=%d", chid, request.ID()) + } + } + augmentContext := t.events.OnContextAugment(chid) + if augmentContext != nil { + hookActions.AugmentContext(augmentContext) + } +} + +// gsReqRecdHook is called when graphsync receives an incoming request for data +func (t *Transport) gsReqRecdHook(p peer.ID, request graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + // if this is a push request the sender is us. + msg, err := extension.GetTransferData(request, t.supportedExtensions) + if err != nil { + hookActions.TerminateWithError(err) + return + } + + // extension not found; probably not our request. + if msg == nil { + return + } + + // An incoming graphsync request for data is received when either + // - The remote peer opened a data-transfer pull channel, so the local node + // receives a graphsync request for the data + // - The local node opened a data-transfer push channel, and in response + // the remote peer sent a graphsync request for the data, and now the + // local node receives that request for data + var chid datatransfer.ChannelID + var responseMessage datatransfer.Message + if msg.IsRequest() { + // when a data transfer request comes in on graphsync, the remote peer + // initiated a pull + chid = datatransfer.ChannelID{ID: msg.TransferID(), Initiator: p, Responder: t.peerID} + + log.Debugf("%s: received request for data (pull), req_id=%d", chid, request.ID()) + + request := msg.(datatransfer.Request) + responseMessage, err = t.events.OnRequestReceived(chid, request) + } else { + // when a data transfer response comes in on graphsync, this node + // initiated a push, and the remote peer responded with a request + // for data + chid = datatransfer.ChannelID{ID: msg.TransferID(), Initiator: t.peerID, Responder: p} + + log.Debugf("%s: received request for data (push), req_id=%d", chid, request.ID()) + + response := msg.(datatransfer.Response) + err = t.events.OnResponseReceived(chid, response) + } + + // If we need to send a response, add the response message as an extension + if responseMessage != nil { + // gsReqRecdHook uses a unique extension name so it can be attached with data from a different hook + // incomingReqExtensions also includes default extension name so it remains compatible with previous data-transfer + // protocol versions out there. + extensions, extensionErr := extension.ToExtensionData(responseMessage, incomingReqExtensions) + if extensionErr != nil { + hookActions.TerminateWithError(err) + return + } + for _, extension := range extensions { + hookActions.SendExtensionData(extension) + } + } + + if err != nil && err != datatransfer.ErrPause { + hookActions.TerminateWithError(err) + return + } + + // Check if the callback indicated that the channel should be paused + // immediately (eg because data is still being unsealed) + paused := false + if err == datatransfer.ErrPause { + log.Debugf("%s: pausing graphsync response", chid) + + paused = true + hookActions.PauseResponse() + } + ch := t.trackDTChannel(chid) + t.requestIDToChannelID.set(request.ID(), true, chid) + + ch.GsDataRequestRcvd(p, request.ID(), paused, hookActions) + + hookActions.ValidateRequest() +} + +// gsCompletedResponseListener is a graphsync.OnCompletedResponseListener. We use it learn when the data transfer is complete +// for the side that is responding to a graphsync request +func (t *Transport) gsCompletedResponseListener(p peer.ID, request graphsync.RequestData, status graphsync.ResponseStatusCode) { + chid, ok := t.requestIDToChannelID.load(request.ID()) + if !ok { + return + } + + if status == graphsync.RequestCancelled { + return + } + + ch, err := t.getDTChannel(chid) + if err != nil { + return + } + ch.MarkTransferComplete() + + var completeErr error + if status != graphsync.RequestCompletedFull { + completeErr = xerrors.Errorf("graphsync response to peer %s did not complete: response status code %s", p, status.String()) + } + + // Used by the tests to listen for when a response completes + if t.completedResponseListener != nil { + t.completedResponseListener(chid) + } + + err = t.events.OnChannelCompleted(chid, completeErr) + if err != nil { + log.Error(err) + } + +} + +func (t *Transport) gsRequestUpdatedHook(p peer.ID, request graphsync.RequestData, update graphsync.RequestData, hookActions graphsync.RequestUpdatedHookActions) { + chid, ok := t.requestIDToChannelID.load(request.ID()) + if !ok { + return + } + + responseMessage, err := t.processExtension(chid, update, p, t.supportedExtensions) + + if responseMessage != nil { + extensions, extensionErr := extension.ToExtensionData(responseMessage, t.supportedExtensions) + if extensionErr != nil { + hookActions.TerminateWithError(err) + return + } + for _, extension := range extensions { + hookActions.SendExtensionData(extension) + } + } + + if err != nil && err != datatransfer.ErrPause { + hookActions.TerminateWithError(err) + } + +} + +// gsIncomingResponseHook is a graphsync.OnIncomingResponseHook. We use it to pass on responses +func (t *Transport) gsIncomingResponseHook(p peer.ID, response graphsync.ResponseData, hookActions graphsync.IncomingResponseHookActions) { + chid, ok := t.requestIDToChannelID.load(response.RequestID()) + if !ok { + return + } + responseMessage, err := t.processExtension(chid, response, p, incomingReqExtensions) + + if responseMessage != nil { + extensions, extensionErr := extension.ToExtensionData(responseMessage, t.supportedExtensions) + if extensionErr != nil { + hookActions.TerminateWithError(err) + return + } + for _, extension := range extensions { + hookActions.UpdateRequestWithExtensions(extension) + } + } + + if err != nil { + hookActions.TerminateWithError(err) + } + + // In a case where the transfer sends blocks immediately this extension may contain both a + // response message and a revalidation request so we trigger OnResponseReceived again for this + // specific extension name + _, err = t.processExtension(chid, response, p, []graphsync.ExtensionName{extension.ExtensionOutgoingBlock1_1}) + + if err != nil { + hookActions.TerminateWithError(err) + } +} + +func (t *Transport) processExtension(chid datatransfer.ChannelID, gsMsg extension.GsExtended, p peer.ID, exts []graphsync.ExtensionName) (datatransfer.Message, error) { + + // if this is a push request the sender is us. + msg, err := extension.GetTransferData(gsMsg, exts) + if err != nil { + return nil, err + } + + // extension not found; probably not our request. + if msg == nil { + return nil, nil + } + + if msg.IsRequest() { + + // only accept request message updates when original message was also request + if (chid != datatransfer.ChannelID{ID: msg.TransferID(), Initiator: p, Responder: t.peerID}) { + return nil, errors.New("received request on response channel") + } + dtRequest := msg.(datatransfer.Request) + return t.events.OnRequestReceived(chid, dtRequest) + } + + // only accept response message updates when original message was also response + if (chid != datatransfer.ChannelID{ID: msg.TransferID(), Initiator: t.peerID, Responder: p}) { + return nil, errors.New("received response on request channel") + } + + dtResponse := msg.(datatransfer.Response) + return nil, t.events.OnResponseReceived(chid, dtResponse) +} + +func (t *Transport) gsRequestorCancelledListener(p peer.ID, request graphsync.RequestData) { + chid, ok := t.requestIDToChannelID.load(request.ID()) + if !ok { + return + } + + ch, err := t.getDTChannel(chid) + if err != nil { + if !xerrors.Is(datatransfer.ErrChannelNotFound, err) { + log.Errorf("requestor cancelled: getting channel %s: %s", chid, err) + } + return + } + + log.Debugf("%s: requester cancelled data-transfer", chid) + ch.OnRequesterCancelled() +} + +// Called when there is a graphsync error sending data +func (t *Transport) gsNetworkSendErrorListener(p peer.ID, request graphsync.RequestData, gserr error) { + // Fire an error if the graphsync request was made by this node or the remote peer + chid, ok := t.requestIDToChannelID.load(request.ID()) + if !ok { + return + } + + err := t.events.OnSendDataError(chid, gserr) + if err != nil { + log.Errorf("failed to fire transport send error %s: %s", gserr, err) + } +} + +// Called when there is a graphsync error receiving data +func (t *Transport) gsNetworkReceiveErrorListener(p peer.ID, gserr error) { + // Fire a receive data error on all ongoing graphsync transfers with that + // peer + t.requestIDToChannelID.forEach(func(k graphsync.RequestID, sending bool, chid datatransfer.ChannelID) { + if chid.Initiator != p && chid.Responder != p { + return + } + + err := t.events.OnReceiveDataError(chid, gserr) + if err != nil { + log.Errorf("failed to fire transport receive error %s: %s", gserr, err) + } + }) +} diff --git a/impl/receiver.go b/transport/graphsync/receiver.go similarity index 53% rename from impl/receiver.go rename to transport/graphsync/receiver.go index 210027a0..585a8859 100644 --- a/impl/receiver.go +++ b/transport/graphsync/receiver.go @@ -1,8 +1,9 @@ -package impl +package graphsync import ( "context" + "github.com/ipfs/go-graphsync" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/libp2p/go-libp2p-core/peer" "go.opentelemetry.io/otel" @@ -10,11 +11,11 @@ import ( "go.opentelemetry.io/otel/trace" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/channels" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" ) type receiver struct { - manager *manager + transport *Transport } // ReceiveRequest takes an incoming data transfer request, validates the voucher and @@ -30,9 +31,9 @@ func (r *receiver) ReceiveRequest( } func (r *receiver) receiveRequest(ctx context.Context, initiator peer.ID, incoming datatransfer.Request) error { - chid := datatransfer.ChannelID{Initiator: initiator, Responder: r.manager.peerID, ID: incoming.TransferID()} - ctx, _ = r.manager.spansIndex.SpanForChannel(ctx, chid) - ctx, span := otel.Tracer("data-transfer").Start(ctx, "receiveRequest", trace.WithAttributes( + chid := datatransfer.ChannelID{Initiator: initiator, Responder: r.transport.peerID, ID: incoming.TransferID()} + ctx = r.transport.events.OnContextAugment(chid)(ctx) + ctx, span := otel.Tracer("gs-data-transfer").Start(ctx, "receiveRequest", trace.WithAttributes( attribute.String("channelID", chid.String()), attribute.String("baseCid", incoming.BaseCid().String()), attribute.Bool("isNew", incoming.IsNew()), @@ -42,46 +43,59 @@ func (r *receiver) receiveRequest(ctx context.Context, initiator peer.ID, incomi attribute.Bool("isPaused", incoming.IsPaused()), )) defer span.End() - response, receiveErr := r.manager.OnRequestReceived(chid, incoming) - if receiveErr == datatransfer.ErrResume { - chst, err := r.manager.channels.GetByID(ctx, chid) - if err != nil { - return err + response, receiveErr := r.transport.events.OnRequestReceived(chid, incoming) + ch, err := r.transport.getDTChannel(chid) + initiateGraphsyncRequest := (response != nil) && (response.IsNew() || response.IsRestart()) && response.Accepted() && !incoming.IsPull() + if err != nil { + if !initiateGraphsyncRequest { + if response != nil { + return r.transport.dtNet.SendMessage(ctx, initiator, response) + } + return receiveErr } - if resumeTransportStatesResponder.Contains(chst.Status()) { - return r.manager.transport.(datatransfer.PauseableTransport).ResumeChannel(ctx, response, chid) + ch = r.transport.trackDTChannel(chid) + } + + if receiveErr == datatransfer.ErrResume && ch.Paused() { + + var extensions []graphsync.ExtensionData + if response != nil { + var err error + extensions, err = extension.ToExtensionData(response, r.transport.supportedExtensions) + if err != nil { + return err + } } - receiveErr = nil + + return ch.Resume(ctx, extensions) } if response != nil { - if (response.IsNew() || response.IsRestart()) && response.Accepted() && !incoming.IsPull() { - var channel datatransfer.ChannelState + if initiateGraphsyncRequest { + stor, _ := incoming.Selector() if response.IsRestart() { - var err error - channel, err = r.manager.channels.GetByID(ctx, chid) + channel, err := r.transport.events.ChannelState(ctx, chid) if err != nil { return err } + ch.UpdateReceivedCidsIfGreater(channel.ReceivedCidsTotal()) } - - stor, _ := incoming.Selector() - if err := r.manager.transport.OpenChannel(ctx, initiator, chid, cidlink.Link{Cid: incoming.BaseCid()}, stor, channel, response); err != nil { + if err := r.transport.openRequest(ctx, initiator, chid, cidlink.Link{Cid: incoming.BaseCid()}, stor, response); err != nil { return err } } else { - if err := r.manager.dataTransferNetwork.SendMessage(ctx, initiator, response); err != nil { + if err := r.transport.dtNet.SendMessage(ctx, initiator, response); err != nil { return err } } } if receiveErr == datatransfer.ErrPause { - return r.manager.transport.(datatransfer.PauseableTransport).PauseChannel(ctx, chid) + return ch.Pause(ctx) } if receiveErr != nil { - _ = r.manager.transport.CloseChannel(ctx, chid) + _ = ch.Close(ctx) return receiveErr } @@ -103,9 +117,9 @@ func (r *receiver) receiveResponse( ctx context.Context, sender peer.ID, incoming datatransfer.Response) error { - chid := datatransfer.ChannelID{Initiator: r.manager.peerID, Responder: sender, ID: incoming.TransferID()} - ctx, _ = r.manager.spansIndex.SpanForChannel(ctx, chid) - ctx, span := otel.Tracer("data-transfer").Start(ctx, "receiveResponse", trace.WithAttributes( + chid := datatransfer.ChannelID{Initiator: r.transport.peerID, Responder: sender, ID: incoming.TransferID()} + ctx = r.transport.events.OnContextAugment(chid)(ctx) + ctx, span := otel.Tracer("gs-data-transfer").Start(ctx, "receiveResponse", trace.WithAttributes( attribute.String("channelID", chid.String()), attribute.Bool("accepted", incoming.Accepted()), attribute.Bool("isComplete", incoming.IsComplete()), @@ -116,16 +130,20 @@ func (r *receiver) receiveResponse( attribute.Bool("isPaused", incoming.IsPaused()), )) defer span.End() - err := r.manager.OnResponseReceived(chid, incoming) - if err == datatransfer.ErrPause { - return r.manager.transport.(datatransfer.PauseableTransport).PauseChannel(ctx, chid) - } + receiveErr := r.transport.events.OnResponseReceived(chid, incoming) + ch, err := r.transport.getDTChannel(chid) if err != nil { + return err + } + if receiveErr == datatransfer.ErrPause { + return ch.Pause(ctx) + } + if receiveErr != nil { log.Warnf("closing channel %s after getting error processing response from %s: %s", chid, sender, err) - _ = r.manager.transport.CloseChannel(ctx, chid) - return err + _ = ch.Close(ctx) + return receiveErr } return nil } @@ -144,48 +162,27 @@ func (r *receiver) ReceiveRestartExistingChannelRequest(ctx context.Context, return } - ctx, _ = r.manager.spansIndex.SpanForChannel(ctx, ch) - ctx, span := otel.Tracer("data-transfer").Start(ctx, "receiveRequest", trace.WithAttributes( + ctx = r.transport.events.OnContextAugment(ch)(ctx) + ctx, span := otel.Tracer("gs-data-transfer").Start(ctx, "receiveRequest", trace.WithAttributes( attribute.String("channelID", ch.String()), )) defer span.End() log.Infof("channel %s: received restart existing channel request from %s", ch, sender) - // validate channel exists -> in non-terminal state and that the sender matches - channel, err := r.manager.channels.GetByID(ctx, ch) - if err != nil || channel == nil { - // nothing to do here, we wont handle the request - return - } - // initiator should be me - if channel.ChannelID().Initiator != r.manager.peerID { + if ch.Initiator != r.transport.peerID { log.Errorf("cannot restart channel %s: channel initiator is not the manager peer", ch) return } - // other peer should be the counter party on the channel - if channel.OtherPeer() != sender { + if ch.Responder != sender { log.Errorf("cannot restart channel %s: channel counterparty is not the sender peer", ch) return } - // channel should NOT be terminated - if channels.IsChannelTerminated(channel.Status()) { - log.Errorf("cannot restart channel %s: channel already terminated", ch) - return - } - - switch r.manager.channelDataTransferType(channel) { - case ManagerPeerCreatePush: - if err := r.manager.openPushRestartChannel(ctx, channel); err != nil { - log.Errorf("failed to open push restart channel %s: %s", ch, err) - } - case ManagerPeerCreatePull: - if err := r.manager.openPullRestartChannel(ctx, channel); err != nil { - log.Errorf("failed to open pull restart channel %s: %s", ch, err) - } - default: - log.Error("peer is not the creator of the channel") + err = r.transport.events.OnRestartExistingChannelRequestReceived(ch) + if err != nil { + log.Errorf(err.Error()) } + return } diff --git a/transport/graphsync/utils.go b/transport/graphsync/utils.go new file mode 100644 index 00000000..71f8568c --- /dev/null +++ b/transport/graphsync/utils.go @@ -0,0 +1,118 @@ +package graphsync + +import ( + "sync" + + "github.com/ipfs/go-graphsync" + peer "github.com/libp2p/go-libp2p-core/peer" + "golang.org/x/xerrors" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/dtchannel" +) + +func (t *Transport) trackDTChannel(chid datatransfer.ChannelID) *dtchannel.Channel { + t.dtChannelsLk.Lock() + defer t.dtChannelsLk.Unlock() + + ch, ok := t.dtChannels[chid] + if !ok { + ch = dtchannel.NewChannel(chid, t.gs) + t.dtChannels[chid] = ch + } + + return ch +} + +func (t *Transport) getDTChannel(chid datatransfer.ChannelID) (*dtchannel.Channel, error) { + if t.events == nil { + return nil, datatransfer.ErrHandlerNotSet + } + + t.dtChannelsLk.RLock() + defer t.dtChannelsLk.RUnlock() + + ch, ok := t.dtChannels[chid] + if !ok { + return nil, xerrors.Errorf("channel %s: %w", chid, datatransfer.ErrChannelNotFound) + } + return ch, nil +} + +func (t *Transport) otherPeer(chid datatransfer.ChannelID) peer.ID { + if chid.Initiator == t.peerID { + return chid.Responder + } + return chid.Initiator +} + +type channelInfo struct { + sending bool + channelID datatransfer.ChannelID +} + +// Used in graphsync callbacks to map from graphsync request to the +// associated data-transfer channel ID. +type requestIDToChannelIDMap struct { + lk sync.RWMutex + m map[graphsync.RequestID]channelInfo +} + +func newRequestIDToChannelIDMap() *requestIDToChannelIDMap { + return &requestIDToChannelIDMap{ + m: make(map[graphsync.RequestID]channelInfo), + } +} + +// get the value for a key +func (m *requestIDToChannelIDMap) load(key graphsync.RequestID) (datatransfer.ChannelID, bool) { + m.lk.RLock() + defer m.lk.RUnlock() + + val, ok := m.m[key] + return val.channelID, ok +} + +// get the value if any of the keys exists in the map +func (m *requestIDToChannelIDMap) any(ks ...graphsync.RequestID) (datatransfer.ChannelID, bool) { + m.lk.RLock() + defer m.lk.RUnlock() + + for _, k := range ks { + val, ok := m.m[k] + if ok { + return val.channelID, ok + } + } + return datatransfer.ChannelID{}, false +} + +// set the value for a key +func (m *requestIDToChannelIDMap) set(key graphsync.RequestID, sending bool, chid datatransfer.ChannelID) { + m.lk.Lock() + defer m.lk.Unlock() + + m.m[key] = channelInfo{sending, chid} +} + +// call f for each key / value in the map +func (m *requestIDToChannelIDMap) forEach(f func(k graphsync.RequestID, isSending bool, chid datatransfer.ChannelID)) { + m.lk.RLock() + defer m.lk.RUnlock() + + for k, ch := range m.m { + f(k, ch.sending, ch.channelID) + } +} + +// delete any keys that reference this value +func (m *requestIDToChannelIDMap) deleteRefs(id datatransfer.ChannelID) { + m.lk.Lock() + defer m.lk.Unlock() + + for k, ch := range m.m { + if ch.channelID == id { + delete(m.m, k) + } + } +} From fa4e28a17fd9d95cc5d091c11bc7498235148d1b Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Wed, 15 Jun 2022 20:28:06 -0700 Subject: [PATCH 09/18] Feat/refactor transport protocol update (#337) * refactor(network): move network to subdirectory move the network to a subdirectory, also cleanup usage of all selector, move gstestdata to the itest directory * feat(network): introduce wrapped protocol introduces a new wrapped protocol that encodes the transport id so that we can support multiple transports using the network protocol * style(lint): fix imports * fix(rebase): fix issues after rebase --- benchmarks/benchmark_test.go | 10 +- benchmarks/testinstance/testinstance.go | 2 +- benchmarks/testnet/interface.go | 2 +- benchmarks/testnet/peernet.go | 2 +- impl/initiating_test.go | 5 +- impl/responding_test.go | 5 +- {testutil => itest}/fixtures/lorem.txt | 0 {testutil => itest}/fixtures/lorem_large.txt | 0 {testutil => itest}/gstestdata.go | 23 +-- itest/integration_test.go | 146 +++++++++--------- itest/restart_integration_test.go | 19 +-- message.go | 45 +++++- message/message.go | 1 + message/message1_1prime/message.go | 25 +-- message/message1_1prime/message_test.go | 65 ++++++++ message/message1_1prime/schema.ipldsch | 6 + message/message1_1prime/transfer_message.go | 16 ++ message/message1_1prime/transfer_request.go | 38 ++++- .../message1_1prime/transfer_request_test.go | 20 ++- message/message1_1prime/transfer_response.go | 39 ++++- .../message1_1prime/transfer_response_test.go | 21 ++- testutil/fakegraphsync.go | 3 +- testutil/faketransport.go | 5 + testutil/testnet.go | 71 --------- testutil/testutil.go | 11 -- transport.go | 10 ++ transport/graphsync/extension/gsextension.go | 11 +- transport/graphsync/graphsync.go | 20 ++- transport/graphsync/receiver.go | 4 +- .../helpers/network}/interface.go | 25 ++- .../helpers/network}/libp2p_impl.go | 60 +++++-- .../helpers/network}/libp2p_impl_test.go | 18 +-- 32 files changed, 458 insertions(+), 270 deletions(-) rename {testutil => itest}/fixtures/lorem.txt (100%) rename {testutil => itest}/fixtures/lorem_large.txt (100%) rename {testutil => itest}/gstestdata.go (93%) delete mode 100644 testutil/testnet.go rename {network => transport/helpers/network}/interface.go (63%) rename {network => transport/helpers/network}/libp2p_impl.go (84%) rename {network => transport/helpers/network}/libp2p_impl_test.go (93%) diff --git a/benchmarks/benchmark_test.go b/benchmarks/benchmark_test.go index 0a170d98..5216f534 100644 --- a/benchmarks/benchmark_test.go +++ b/benchmarks/benchmark_test.go @@ -24,9 +24,7 @@ import ( "github.com/ipfs/go-merkledag" "github.com/ipfs/go-unixfs/importer/balanced" ihelper "github.com/ipfs/go-unixfs/importer/helpers" - basicnode "github.com/ipld/go-ipld-prime/node/basic" - ipldselector "github.com/ipld/go-ipld-prime/traversal/selector" - "github.com/ipld/go-ipld-prime/traversal/selector/builder" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" @@ -77,10 +75,6 @@ func p2pStrestTest(ctx context.Context, b *testing.B, numfiles int, df distFunc, thisCids := df(ctx, b, instances[:1]) allCids = append(allCids, thisCids...) } - ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) - - allSelector := ssb.ExploreRecursive(ipldselector.RecursionLimitNone(), - ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() runtime.GC() b.ResetTimer() @@ -105,7 +99,7 @@ func p2pStrestTest(ctx context.Context, b *testing.B, numfiles int, df distFunc, timer := time.NewTimer(30 * time.Second) start := time.Now() for j := 0; j < numfiles; j++ { - _, err := pusher.Manager.OpenPushDataChannel(ctx, receiver.Peer, testutil.NewTestTypedVoucher(), allCids[j], allSelector) + _, err := pusher.Manager.OpenPushDataChannel(ctx, receiver.Peer, testutil.NewTestTypedVoucher(), allCids[j], selectorparse.CommonSelector_ExploreAllRecursively) if err != nil { b.Fatalf("received error on request: %s", err.Error()) } diff --git a/benchmarks/testinstance/testinstance.go b/benchmarks/testinstance/testinstance.go index 24ce4691..bfc72747 100644 --- a/benchmarks/testinstance/testinstance.go +++ b/benchmarks/testinstance/testinstance.go @@ -21,9 +21,9 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" tn "github.com/filecoin-project/go-data-transfer/v2/benchmarks/testnet" dtimpl "github.com/filecoin-project/go-data-transfer/v2/impl" - dtnet "github.com/filecoin-project/go-data-transfer/v2/network" "github.com/filecoin-project/go-data-transfer/v2/testutil" gstransport "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" + dtnet "github.com/filecoin-project/go-data-transfer/v2/transport/helpers/network" ) // TempDirGenerator is any interface that can generate temporary directories diff --git a/benchmarks/testnet/interface.go b/benchmarks/testnet/interface.go index dc5cf55c..72858184 100644 --- a/benchmarks/testnet/interface.go +++ b/benchmarks/testnet/interface.go @@ -4,7 +4,7 @@ import ( gsnet "github.com/ipfs/go-graphsync/network" "github.com/libp2p/go-libp2p-core/peer" - dtnet "github.com/filecoin-project/go-data-transfer/v2/network" + dtnet "github.com/filecoin-project/go-data-transfer/v2/transport/helpers/network" ) // Network is an interface for generating graphsync network interfaces diff --git a/benchmarks/testnet/peernet.go b/benchmarks/testnet/peernet.go index c6ca1778..ba83ff09 100644 --- a/benchmarks/testnet/peernet.go +++ b/benchmarks/testnet/peernet.go @@ -7,7 +7,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" mockpeernet "github.com/libp2p/go-libp2p/p2p/net/mock" - dtnet "github.com/filecoin-project/go-data-transfer/v2/network" + dtnet "github.com/filecoin-project/go-data-transfer/v2/transport/helpers/network" ) type peernet struct { diff --git a/impl/initiating_test.go b/impl/initiating_test.go index 04113267..f6c4f0ec 100644 --- a/impl/initiating_test.go +++ b/impl/initiating_test.go @@ -11,6 +11,7 @@ import ( dss "github.com/ipfs/go-datastore/sync" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" @@ -333,7 +334,7 @@ func TestDataTransferInitiating(t *testing.T) { events: make(chan datatransfer.EventCode, len(verify.expectedEvents)), } ev.setup(t, dt) - h.stor = testutil.AllSelector() + h.stor = selectorparse.CommonSelector_ExploreAllRecursively h.voucher = testutil.NewTestTypedVoucher() h.voucherResult = testutil.NewTestTypedVoucher() require.NoError(t, err) @@ -580,7 +581,7 @@ func TestDataTransferRestartInitiating(t *testing.T) { ev.setup(t, dt) // setup voucher processing - h.stor = testutil.AllSelector() + h.stor = selectorparse.CommonSelector_ExploreAllRecursively h.voucher = testutil.NewTestTypedVoucher() require.NoError(t, h.dt.RegisterVoucherType(h.voucher.Type, h.voucherValidator)) h.voucherResult = testutil.NewTestTypedVoucher() diff --git a/impl/responding_test.go b/impl/responding_test.go index a7ca4e3e..1537a485 100644 --- a/impl/responding_test.go +++ b/impl/responding_test.go @@ -13,6 +13,7 @@ import ( "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -617,7 +618,7 @@ func TestDataTransferResponding(t *testing.T) { events: make(chan datatransfer.EventCode, len(verify.expectedEvents)), } ev.setup(t, dt) - h.stor = testutil.AllSelector() + h.stor = selectorparse.CommonSelector_ExploreAllRecursively h.voucher = testutil.NewTestTypedVoucher() h.baseCid = testutil.GenerateCids(1)[0] h.id = datatransfer.TransferID(rand.Int31()) @@ -999,7 +1000,7 @@ func TestDataTransferRestartResponding(t *testing.T) { events: make(chan datatransfer.EventCode, len(verify.expectedEvents)), } ev.setup(t, dt) - h.stor = testutil.AllSelector() + h.stor = selectorparse.CommonSelector_ExploreAllRecursively h.voucher = testutil.NewTestTypedVoucher() h.baseCid = testutil.GenerateCids(1)[0] h.id = datatransfer.TransferID(rand.Int31()) diff --git a/testutil/fixtures/lorem.txt b/itest/fixtures/lorem.txt similarity index 100% rename from testutil/fixtures/lorem.txt rename to itest/fixtures/lorem.txt diff --git a/testutil/fixtures/lorem_large.txt b/itest/fixtures/lorem_large.txt similarity index 100% rename from testutil/fixtures/lorem_large.txt rename to itest/fixtures/lorem_large.txt diff --git a/testutil/gstestdata.go b/itest/gstestdata.go similarity index 93% rename from testutil/gstestdata.go rename to itest/gstestdata.go index f9d71869..02b2f7dc 100644 --- a/testutil/gstestdata.go +++ b/itest/gstestdata.go @@ -1,4 +1,4 @@ -package testutil +package itest import ( "bytes" @@ -29,37 +29,30 @@ import ( "github.com/ipfs/go-unixfs/importer/balanced" ihelper "github.com/ipfs/go-unixfs/importer/helpers" "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" - basicnode "github.com/ipld/go-ipld-prime/node/basic" - "github.com/ipld/go-ipld-prime/traversal/selector" - "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/protocol" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/network" gstransport "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" + "github.com/filecoin-project/go-data-transfer/v2/transport/helpers/network" ) -var allSelector datamodel.Node - const loremFile = "lorem.txt" +const loremFileTransferBytes = 20439 -func init() { - ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) - allSelector = ssb.ExploreRecursive(selector.RecursionLimitNone(), - ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() -} +const loremLargeFile = "lorem_large.txt" +const loremLargeFileTransferBytes = 217452 const unixfsChunkSize uint64 = 1 << 10 const unixfsLinksPerLevel = 1024 var extsForProtocol = map[protocol.ID]graphsync.ExtensionName{ - datatransfer.ProtocolDataTransfer1_2: extension.ExtensionDataTransfer1_1, + network.ProtocolDataTransfer1_2: extension.ExtensionDataTransfer1_1, + network.ProtocolFilDataTransfer1_2: extension.ExtensionDataTransfer1_1, } // GraphsyncTestingData is a test harness for testing data transfer on top of @@ -83,7 +76,6 @@ type GraphsyncTestingData struct { GsNet2 gsnet.GraphSyncNetwork DtNet1 network.DataTransferNetwork DtNet2 network.DataTransferNetwork - AllSelector datamodel.Node OrigBytes []byte TempDir1 string TempDir2 string @@ -152,7 +144,6 @@ func NewGraphsyncTestingData(ctx context.Context, t *testing.T, host1Protocols [ require.NoError(t, err) gsData.TempDir2 = tempdir // create a selector for the whole UnixFS dag - gsData.AllSelector = allSelector gsData.host1Protocols = host1Protocols gsData.host2Protocols = host2Protocols return gsData diff --git a/itest/integration_test.go b/itest/integration_test.go index fab58ea6..7be07a71 100644 --- a/itest/integration_test.go +++ b/itest/integration_test.go @@ -29,6 +29,7 @@ import ( "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" @@ -39,19 +40,12 @@ import ( "github.com/filecoin-project/go-data-transfer/v2/channelmonitor" . "github.com/filecoin-project/go-data-transfer/v2/impl" "github.com/filecoin-project/go-data-transfer/v2/message" - "github.com/filecoin-project/go-data-transfer/v2/network" "github.com/filecoin-project/go-data-transfer/v2/testutil" tp "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" + "github.com/filecoin-project/go-data-transfer/v2/transport/helpers/network" ) -const loremFile = "lorem.txt" -const loremFileTransferBytes = 20439 - -const loremLargeFile = "lorem_large.txt" - -// const loremLargeFileTransferBytes = 217452 - // nil means use the default protocols // tests data transfer for the following protocol combinations: // default protocol -> default protocols @@ -61,7 +55,10 @@ var protocolsForTest = map[string]struct { host1Protocols []protocol.ID host2Protocols []protocol.ID }{ - "(v1.2 -> v1.2)": {nil, nil}, + "(wrapped v1.2 -> wrapped v1.2)": {nil, nil}, + "(v1.2 -> wrapped v1.2)": {[]protocol.ID{network.ProtocolFilDataTransfer1_2}, nil}, + "(wrapped v1.2 -> v1.2)": {nil, []protocol.ID{network.ProtocolFilDataTransfer1_2}}, + "(v1.2 -> v1.2)": {[]protocol.ID{network.ProtocolFilDataTransfer1_2}, []protocol.ID{network.ProtocolFilDataTransfer1_2}}, } // tests data transfer for the protocol combinations that support restart messages @@ -69,7 +66,10 @@ var protocolsForRestartTest = map[string]struct { host1Protocols []protocol.ID host2Protocols []protocol.ID }{ - "(v1.2 -> v1.2)": {nil, nil}, + "(wrapped v1.2 -> wrapped v1.2)": {nil, nil}, + "(v1.2 -> wrapped v1.2)": {[]protocol.ID{network.ProtocolFilDataTransfer1_2}, nil}, + "(wrapped v1.2 -> v1.2)": {nil, []protocol.ID{network.ProtocolFilDataTransfer1_2}}, + "(v1.2 -> v1.2)": {[]protocol.ID{network.ProtocolFilDataTransfer1_2}, []protocol.ID{network.ProtocolFilDataTransfer1_2}}, } func TestRoundTrip(t *testing.T) { @@ -138,7 +138,7 @@ func TestRoundTrip(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - gsData := testutil.NewGraphsyncTestingData(ctx, t, ps.host1Protocols, ps.host2Protocols) + gsData := NewGraphsyncTestingData(ctx, t, ps.host1Protocols, ps.host2Protocols) host1 := gsData.Host1 // initiator, data sender host2 := gsData.Host2 // data recipient @@ -205,7 +205,7 @@ func TestRoundTrip(t *testing.T) { } else { sourceDagService = gsData.DagService1 } - root, origBytes := testutil.LoadUnixFSFile(ctx, t, sourceDagService, loremFile) + root, origBytes := LoadUnixFSFile(ctx, t, sourceDagService, loremFile) rootCid := root.(cidlink.Link).Cid var destDagService ipldformat.DAGService @@ -232,11 +232,11 @@ func TestRoundTrip(t *testing.T) { if data.isPull { sv.ExpectSuccessPull() require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) - chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) } else { sv.ExpectSuccessPush() require.NoError(t, dt2.RegisterVoucherType(testutil.TestVoucherType, sv)) - chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) } require.NoError(t, err) opens := 0 @@ -260,7 +260,7 @@ func TestRoundTrip(t *testing.T) { } } require.Equal(t, sentIncrements, receivedIncrements) - testutil.VerifyHasFile(ctx, t, destDagService, root, origBytes) + VerifyHasFile(ctx, t, destDagService, root, origBytes) if data.isPull { assert.Equal(t, chid.Initiator, host2.ID()) } else { @@ -294,7 +294,7 @@ func TestMultipleRoundTripMultipleStores(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator, data sender host2 := gsData.Host2 // data recipient @@ -331,7 +331,7 @@ func TestMultipleRoundTripMultipleStores(t *testing.T) { sv := testutil.NewStubbedValidator() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) - root, origBytes := testutil.LoadUnixFSFile(ctx, t, gsData.DagService1, loremFile) + root, origBytes := LoadUnixFSFile(ctx, t, gsData.DagService1, loremFile) rootCid := root.(cidlink.Link).Cid destDagServices := make([]ipldformat.DAGService, 0, data.requestCount) @@ -363,14 +363,14 @@ func TestMultipleRoundTripMultipleStores(t *testing.T) { sv.ExpectSuccessPull() require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) for i := 0; i < data.requestCount; i++ { - _, err = dt2.OpenPullDataChannel(ctx, host1.ID(), vouchers[i], rootCid, gsData.AllSelector) + _, err = dt2.OpenPullDataChannel(ctx, host1.ID(), vouchers[i], rootCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) } } else { sv.ExpectSuccessPush() require.NoError(t, dt2.RegisterVoucherType(testutil.TestVoucherType, sv)) for i := 0; i < data.requestCount; i++ { - _, err = dt1.OpenPushDataChannel(ctx, host2.ID(), vouchers[i], rootCid, gsData.AllSelector) + _, err = dt1.OpenPushDataChannel(ctx, host2.ID(), vouchers[i], rootCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) } } @@ -389,7 +389,7 @@ func TestMultipleRoundTripMultipleStores(t *testing.T) { } } for _, destDagService := range destDagServices { - testutil.VerifyHasFile(ctx, t, destDagService, root, origBytes) + VerifyHasFile(ctx, t, destDagService, root, origBytes) } }) } @@ -414,7 +414,7 @@ func TestManyReceiversAtOnce(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator, data sender tp1 := gsData.SetupGSTransportHost1() @@ -490,21 +490,21 @@ func TestManyReceiversAtOnce(t *testing.T) { sv := testutil.NewStubbedValidator() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) - root, origBytes := testutil.LoadUnixFSFile(ctx, t, gsData.DagService1, loremFile) + root, origBytes := LoadUnixFSFile(ctx, t, gsData.DagService1, loremFile) rootCid := root.(cidlink.Link).Cid if data.isPull { sv.ExpectSuccessPull() require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) for i, receiver := range receivers { - _, err = receiver.OpenPullDataChannel(ctx, host1.ID(), vouchers[i], rootCid, gsData.AllSelector) + _, err = receiver.OpenPullDataChannel(ctx, host1.ID(), vouchers[i], rootCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) } } else { sv.ExpectSuccessPush() for i, receiver := range receivers { require.NoError(t, receiver.RegisterVoucherType(testutil.TestVoucherType, sv)) - _, err = dt1.OpenPushDataChannel(ctx, hosts[i].ID(), vouchers[i], rootCid, gsData.AllSelector) + _, err = dt1.OpenPushDataChannel(ctx, hosts[i].ID(), vouchers[i], rootCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) } } @@ -523,7 +523,7 @@ func TestManyReceiversAtOnce(t *testing.T) { } } for _, destDagService := range destDagServices { - testutil.VerifyHasFile(ctx, t, destDagService, root, origBytes) + VerifyHasFile(ctx, t, destDagService, root, origBytes) } }) } @@ -692,7 +692,7 @@ func TestAutoRestart(t *testing.T) { // The retry config for the network layer: make 5 attempts, backing off by 1s each time netRetry := network.RetryParameters(time.Second, time.Second, 5, 1) - gsData := testutil.NewGraphsyncTestingData(ctx, t, ps.host1Protocols, ps.host2Protocols) + gsData := NewGraphsyncTestingData(ctx, t, ps.host1Protocols, ps.host2Protocols) gsData.DtNet1 = network.NewFromLibp2pHost(gsData.Host1, netRetry) initiatorHost := gsData.Host1 // initiator, data sender responderHost := gsData.Host2 // data recipient @@ -745,7 +745,7 @@ func TestAutoRestart(t *testing.T) { destDagService = gsData.DagService1 } - root, origBytes := testutil.LoadUnixFSFile(ctx, t, sourceDagService, loremFile) + root, origBytes := LoadUnixFSFile(ctx, t, sourceDagService, loremFile) rootCid := root.(cidlink.Link).Cid require.NoError(t, initiator.RegisterVoucherType(testutil.TestVoucherType, sv)) @@ -770,10 +770,10 @@ func TestAutoRestart(t *testing.T) { var chid datatransfer.ChannelID if tc.isPush { // Open a push channel - chid, err = initiator.OpenPushDataChannel(ctx, responderHost.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = initiator.OpenPushDataChannel(ctx, responderHost.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) } else { // Open a pull channel - chid, err = initiator.OpenPullDataChannel(ctx, responderHost.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = initiator.OpenPullDataChannel(ctx, responderHost.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) } require.NoError(t, err) @@ -842,7 +842,7 @@ func TestAutoRestart(t *testing.T) { } // Verify that the file was transferred to the destination node - testutil.VerifyHasFile(ctx, t, destDagService, root, origBytes) + VerifyHasFile(ctx, t, destDagService, root, origBytes) }) } } @@ -866,7 +866,7 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { // The retry config for the network layer: make 5 attempts, backing off by 1s each time netRetry := network.RetryParameters(time.Second, time.Second, 5, 1) - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) gsData.DtNet1 = network.NewFromLibp2pHost(gsData.Host1, netRetry) initiatorHost := gsData.Host1 // initiator, data sender responderHost := gsData.Host2 // data recipient @@ -938,7 +938,7 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { destDagService = gsData.DagService1 } - root, origBytes := testutil.LoadUnixFSFile(ctx, t, sourceDagService, loremLargeFile) + root, origBytes := LoadUnixFSFile(ctx, t, sourceDagService, loremLargeFile) rootCid := root.(cidlink.Link).Cid require.NoError(t, initiator.RegisterVoucherType(testutil.TestVoucherType, sv)) @@ -947,10 +947,10 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { var chid datatransfer.ChannelID if isPush { // Open a push channel - chid, err = initiator.OpenPushDataChannel(ctx, responderHost.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = initiator.OpenPushDataChannel(ctx, responderHost.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) } else { // Open a pull channel - chid, err = initiator.OpenPullDataChannel(ctx, responderHost.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = initiator.OpenPullDataChannel(ctx, responderHost.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) } require.NoError(t, err) @@ -1058,7 +1058,7 @@ func TestAutoRestartAfterBouncingInitiator(t *testing.T) { } // Verify that the file was transferred to the destination node - testutil.VerifyHasFile(ctx, t, destDagService, root, origBytes) + VerifyHasFile(ctx, t, destDagService, root, origBytes) } t.Run("push", func(t *testing.T) { @@ -1084,7 +1084,7 @@ func TestRoundTripCancelledRequest(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator, data sender host2 := gsData.Host2 @@ -1124,7 +1124,7 @@ func TestRoundTripCancelledRequest(t *testing.T) { dt2.SubscribeToEvents(subscriber) voucher := testutil.NewTestTypedVoucherWith("applesauce") sv := testutil.NewStubbedValidator() - root, _ := testutil.LoadUnixFSFile(ctx, t, gsData.DagService1, loremFile) + root, _ := LoadUnixFSFile(ctx, t, gsData.DagService1, loremFile) rootCid := root.(cidlink.Link).Cid var chid datatransfer.ChannelID @@ -1132,12 +1132,12 @@ func TestRoundTripCancelledRequest(t *testing.T) { sv.ExpectSuccessPull() sv.StubResult(datatransfer.ValidationResult{Accepted: true, ForcePause: true}) require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) - chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) } else { sv.ExpectSuccessPush() sv.StubResult(datatransfer.ValidationResult{Accepted: true, ForcePause: true}) require.NoError(t, dt2.RegisterVoucherType(testutil.TestVoucherType, sv)) - chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) } require.NoError(t, err) opens := 0 @@ -1310,7 +1310,7 @@ func TestSimulatedRetrievalFlow(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 4*time.Second) defer cancel() - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator, data sender root := gsData.LoadUnixFSFile(t, false) @@ -1390,7 +1390,7 @@ func TestSimulatedRetrievalFlow(t *testing.T) { require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) - chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) for providerFinished != nil || clientFinished != nil { @@ -1428,7 +1428,7 @@ func TestPauseAndResume(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator, data sender host2 := gsData.Host2 // data recipient @@ -1499,11 +1499,11 @@ func TestPauseAndResume(t *testing.T) { if isPull { sv.ExpectSuccessPull() require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) - chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) } else { sv.ExpectSuccessPush() require.NoError(t, dt2.RegisterVoucherType(testutil.TestVoucherType, sv)) - chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) } require.NoError(t, err) opens := 0 @@ -1571,7 +1571,7 @@ func TestUnrecognizedVoucherRoundTrip(t *testing.T) { // ctx, cancel := context.WithTimeout(ctx, 5*time.Second) // defer cancel() - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator, data sender host2 := gsData.Host2 // data recipient @@ -1603,13 +1603,13 @@ func TestUnrecognizedVoucherRoundTrip(t *testing.T) { dt2.SubscribeToEvents(subscriber) voucher := testutil.NewTestTypedVoucherWith("applesauce") - root, _ := testutil.LoadUnixFSFile(ctx, t, gsData.DagService1, loremFile) + root, _ := LoadUnixFSFile(ctx, t, gsData.DagService1, loremFile) rootCid := root.(cidlink.Link).Cid if isPull { - _, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) + _, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) } else { - _, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, rootCid, gsData.AllSelector) + _, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) } require.NoError(t, err) opens := 0 @@ -1640,7 +1640,7 @@ func TestDataTransferSubscribing(t *testing.T) { ctx := context.Background() ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) host2 := gsData.Host2 tp1 := gsData.SetupGSTransportHost1() @@ -1672,7 +1672,7 @@ func TestDataTransferSubscribing(t *testing.T) { } unsub1 := dt1.SubscribeToEvents(subscribe1) unsub2 := dt1.SubscribeToEvents(subscribe2) - _, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, baseCid, gsData.AllSelector) + _, err = dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, baseCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) select { case <-ctx.Done(): @@ -1701,7 +1701,7 @@ func TestDataTransferSubscribing(t *testing.T) { } unsub3 := dt1.SubscribeToEvents(subscribe3) unsub4 := dt1.SubscribeToEvents(subscribe4) - _, err = dt1.OpenPullDataChannel(ctx, host2.ID(), voucher, baseCid, gsData.AllSelector) + _, err = dt1.OpenPullDataChannel(ctx, host2.ID(), voucher, baseCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) select { case <-ctx.Done(): @@ -1768,7 +1768,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { ctx := context.Background() ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator and data sender host2 := gsData.Host2 // data recipient, makes graphsync request for data voucher := testutil.NewTestTypedVoucher() @@ -1779,7 +1779,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { r := &receiver{ messageReceived: make(chan receivedMessage), } - dtnet2.SetDelegate(r) + dtnet2.SetDelegate("graphsync", r) gsr := &fakeGraphSyncReceiver{ receivedMessages: make(chan receivedGraphSyncMessage), @@ -1794,7 +1794,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { require.NoError(t, err) t.Run("when request is initiated", func(t *testing.T) { - _, err := dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, link.(cidlink.Link).Cid, gsData.AllSelector) + _, err := dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, link.(cidlink.Link).Cid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) var messageReceived receivedMessage @@ -1808,7 +1808,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { response, err := message.NewResponse(requestReceived.TransferID(), true, false, &voucherResult) require.NoError(t, err) nd := response.ToIPLD() - request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, gsData.AllSelector, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ + request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, selectorparse.CommonSelector_ExploreAllRecursively, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ Name: extension.ExtensionDataTransfer1_1, Data: nd, }) @@ -1826,7 +1826,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { response, err := message.NewResponse(datatransfer.TransferID(rand.Uint32()), true, false, &voucher) require.NoError(t, err) nd := response.ToIPLD() - request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, gsData.AllSelector, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ + request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, selectorparse.CommonSelector_ExploreAllRecursively, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ Name: extension.ExtensionDataTransfer1_1, Data: nd, }) @@ -1846,7 +1846,7 @@ func TestResponseHookWhenExtensionNotFound(t *testing.T) { ctx := context.Background() ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator and data sender host2 := gsData.Host2 // data recipient, makes graphsync request for data voucher := testutil.NewTestTypedVoucherWith("applesauce") @@ -1857,7 +1857,7 @@ func TestResponseHookWhenExtensionNotFound(t *testing.T) { r := &receiver{ messageReceived: make(chan receivedMessage), } - dtnet2.SetDelegate(r) + dtnet2.SetDelegate("graphsync", r) gsr := &fakeGraphSyncReceiver{ receivedMessages: make(chan receivedGraphSyncMessage), @@ -1877,7 +1877,7 @@ func TestResponseHookWhenExtensionNotFound(t *testing.T) { } gs1.RegisterIncomingRequestHook(validateHook) - _, err := dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, link.(cidlink.Link).Cid, gsData.AllSelector) + _, err := dt1.OpenPushDataChannel(ctx, host2.ID(), voucher, link.(cidlink.Link).Cid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) select { @@ -1886,7 +1886,7 @@ func TestResponseHookWhenExtensionNotFound(t *testing.T) { case <-r.messageReceived: } - request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, gsData.AllSelector, graphsync.Priority(rand.Int31())) + request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, selectorparse.CommonSelector_ExploreAllRecursively, graphsync.Priority(rand.Int31())) builder := gsmsg.NewBuilder() builder.AddRequest(request) gsmessage, err := builder.Build() @@ -1902,10 +1902,10 @@ func TestRespondingToPullGraphsyncRequests(t *testing.T) { //create network ctx := context.Background() testCases := map[string]struct { - test func(*testing.T, *testutil.GraphsyncTestingData, datatransfer.Transport, ipld.Link, datatransfer.TransferID, *fakeGraphSyncReceiver) + test func(*testing.T, *GraphsyncTestingData, datatransfer.Transport, ipld.Link, datatransfer.TransferID, *fakeGraphSyncReceiver) }{ "When a pull request is initiated and validated": { - test: func(t *testing.T, gsData *testutil.GraphsyncTestingData, tp2 datatransfer.Transport, link ipld.Link, id datatransfer.TransferID, gsr *fakeGraphSyncReceiver) { + test: func(t *testing.T, gsData *GraphsyncTestingData, tp2 datatransfer.Transport, link ipld.Link, id datatransfer.TransferID, gsr *fakeGraphSyncReceiver) { sv := testutil.NewStubbedValidator() sv.ExpectSuccessPull() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) @@ -1916,10 +1916,10 @@ func TestRespondingToPullGraphsyncRequests(t *testing.T) { require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) voucher := testutil.NewTestTypedVoucher() - request, err := message.NewRequest(id, false, true, &voucher, testutil.GenerateCids(1)[0], gsData.AllSelector) + request, err := message.NewRequest(id, false, true, &voucher, testutil.GenerateCids(1)[0], selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) nd := request.ToIPLD() - gsRequest := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, gsData.AllSelector, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ + gsRequest := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, selectorparse.CommonSelector_ExploreAllRecursively, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ Name: extension.ExtensionDataTransfer1_1, Data: nd, }) @@ -1935,7 +1935,7 @@ func TestRespondingToPullGraphsyncRequests(t *testing.T) { }, }, "When request is initiated, but fails validation": { - test: func(t *testing.T, gsData *testutil.GraphsyncTestingData, tp2 datatransfer.Transport, link ipld.Link, id datatransfer.TransferID, gsr *fakeGraphSyncReceiver) { + test: func(t *testing.T, gsData *GraphsyncTestingData, tp2 datatransfer.Transport, link ipld.Link, id datatransfer.TransferID, gsr *fakeGraphSyncReceiver) { sv := testutil.NewStubbedValidator() sv.ExpectErrorPull() dt1, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) @@ -1943,11 +1943,11 @@ func TestRespondingToPullGraphsyncRequests(t *testing.T) { testutil.StartAndWaitForReady(ctx, t, dt1) require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) voucher := testutil.NewTestTypedVoucher() - dtRequest, err := message.NewRequest(id, false, true, &voucher, testutil.GenerateCids(1)[0], gsData.AllSelector) + dtRequest, err := message.NewRequest(id, false, true, &voucher, testutil.GenerateCids(1)[0], selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) nd := dtRequest.ToIPLD() - request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, gsData.AllSelector, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ + request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, selectorparse.CommonSelector_ExploreAllRecursively, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ Name: extension.ExtensionDataTransfer1_1, Data: nd, }) @@ -1970,7 +1970,7 @@ func TestRespondingToPullGraphsyncRequests(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) // setup receiving peer to just record message coming in gsr := &fakeGraphSyncReceiver{ @@ -1997,7 +1997,7 @@ func TestMultipleMessagesInExtension(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) defer cancel() - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator, data sender root := gsData.LoadUnixFSFile(t, false) @@ -2109,7 +2109,7 @@ func TestMultipleMessagesInExtension(t *testing.T) { require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) voucher := testutil.NewTestTypedVoucherWith("applesauce") - chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) // Expect the client to receive a response voucher, the provider to complete the transfer and @@ -2139,7 +2139,7 @@ func TestMultipleParallelTransfers(t *testing.T) { ctx := context.Background() - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator, data sender tp1 := gsData.SetupGSTransportHost1() @@ -2239,7 +2239,7 @@ func TestMultipleParallelTransfers(t *testing.T) { rootCid := root.(cidlink.Link).Cid voucher := testutil.NewTestTypedVoucher() - chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, gsData.AllSelector) + chid, err = dt2.OpenPullDataChannel(ctx, host1.ID(), voucher, rootCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(t, err) close(chidReceived) // Expect the client to receive a response voucher, the provider to complete the transfer and @@ -2268,7 +2268,7 @@ func TestMultipleParallelTransfers(t *testing.T) { } } sv.VerifyExpectations(t) - testutil.VerifyHasFile(gsData.Ctx, t, gsData.DagService2, root, origBytes) + VerifyHasFile(gsData.Ctx, t, gsData.DagService2, root, origBytes) }) } } diff --git a/itest/restart_integration_test.go b/itest/restart_integration_test.go index d63fd728..68e02263 100644 --- a/itest/restart_integration_test.go +++ b/itest/restart_integration_test.go @@ -10,6 +10,7 @@ import ( ipldformat "github.com/ipfs/go-ipld-format" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" "go.uber.org/atomic" @@ -42,7 +43,7 @@ func TestRestartPush(t *testing.T) { stopAt: 20, openPushF: func(rh *restartHarness) datatransfer.ChannelID { voucher := testutil.NewTestTypedVoucherWith("applesauce") - chid, err := rh.dt1.OpenPushDataChannel(rh.testCtx, rh.peer2, voucher, rh.rootCid, rh.gsData.AllSelector) + chid, err := rh.dt1.OpenPushDataChannel(rh.testCtx, rh.peer2, voucher, rh.rootCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(rh.t, err) return chid }, @@ -84,7 +85,7 @@ func TestRestartPush(t *testing.T) { stopAt: 20, openPushF: func(rh *restartHarness) datatransfer.ChannelID { voucher := testutil.NewTestTypedVoucherWith("applesauce") - chid, err := rh.dt1.OpenPushDataChannel(rh.testCtx, rh.peer2, voucher, rh.rootCid, rh.gsData.AllSelector) + chid, err := rh.dt1.OpenPushDataChannel(rh.testCtx, rh.peer2, voucher, rh.rootCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(rh.t, err) return chid }, @@ -259,7 +260,7 @@ func TestRestartPush(t *testing.T) { // verify all cids are present on the receiver - testutil.VerifyHasFile(rh.testCtx, t, rh.destDagService, rh.root, rh.origBytes) + VerifyHasFile(rh.testCtx, t, rh.destDagService, rh.root, rh.origBytes) rh.sv.VerifyExpectations(t) // we should ONLY see two opens and two completes @@ -296,7 +297,7 @@ func TestRestartPull(t *testing.T) { stopAt: 40, openPullF: func(rh *restartHarness) datatransfer.ChannelID { voucher := testutil.NewTestTypedVoucherWith("applesauce") - chid, err := rh.dt2.OpenPullDataChannel(rh.testCtx, rh.peer1, voucher, rh.rootCid, rh.gsData.AllSelector) + chid, err := rh.dt2.OpenPullDataChannel(rh.testCtx, rh.peer1, voucher, rh.rootCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(rh.t, err) return chid }, @@ -335,7 +336,7 @@ func TestRestartPull(t *testing.T) { stopAt: 40, openPullF: func(rh *restartHarness) datatransfer.ChannelID { voucher := testutil.NewTestTypedVoucherWith("applesauce") - chid, err := rh.dt2.OpenPullDataChannel(rh.testCtx, rh.peer1, voucher, rh.rootCid, rh.gsData.AllSelector) + chid, err := rh.dt2.OpenPullDataChannel(rh.testCtx, rh.peer1, voucher, rh.rootCid, selectorparse.CommonSelector_ExploreAllRecursively) require.NoError(rh.t, err) return chid }, @@ -502,7 +503,7 @@ func TestRestartPull(t *testing.T) { _, _, err = waitF(10*time.Second, 2) require.NoError(t, err) - testutil.VerifyHasFile(rh.testCtx, t, rh.destDagService, rh.root, rh.origBytes) + VerifyHasFile(rh.testCtx, t, rh.destDagService, rh.root, rh.origBytes) rh.sv.VerifyExpectations(t) // we should ONLY see two opens and two completes @@ -536,7 +537,7 @@ type restartHarness struct { peer1 peer.ID peer2 peer.ID - gsData *testutil.GraphsyncTestingData + gsData *GraphsyncTestingData dt1 datatransfer.Manager dt2 datatransfer.Manager sv *testutil.StubbedValidator @@ -553,7 +554,7 @@ func newRestartHarness(t *testing.T) *restartHarness { ctx, cancel := context.WithTimeout(ctx, 120*time.Second) // Setup host - gsData := testutil.NewGraphsyncTestingData(ctx, t, nil, nil) + gsData := NewGraphsyncTestingData(ctx, t, nil, nil) host1 := gsData.Host1 // initiator, data sender host2 := gsData.Host2 // data recipient peer1 := host1.ID() @@ -576,7 +577,7 @@ func newRestartHarness(t *testing.T) *restartHarness { require.NoError(t, dt2.RegisterVoucherType(testutil.TestVoucherType, sv)) sourceDagService := gsData.DagService1 - root, origBytes := testutil.LoadUnixFSFile(ctx, t, sourceDagService, largeFile) + root, origBytes := LoadUnixFSFile(ctx, t, sourceDagService, largeFile) rootCid := root.(cidlink.Link).Cid destDagService := gsData.DagService2 diff --git a/message.go b/message.go index 93ea17d9..9eed49fe 100644 --- a/message.go +++ b/message.go @@ -1,17 +1,51 @@ package datatransfer import ( + "errors" + "fmt" "io" + "strconv" + "strings" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/datamodel" - "github.com/libp2p/go-libp2p-core/protocol" ) +type MessageVersion struct { + Major uint64 + Minor uint64 + Patch uint64 +} + +func (mv MessageVersion) String() string { + return fmt.Sprintf("%d.%d.%d", mv.Major, mv.Minor, mv.Patch) +} + +// MessageVersionFromString parses a string into a message version +func MessageVersionFromString(versionString string) (MessageVersion, error) { + versions := strings.Split(versionString, ".") + if len(versions) != 3 { + return MessageVersion{}, errors.New("not a version string") + } + major, err := strconv.ParseUint(versions[0], 10, 0) + if err != nil { + return MessageVersion{}, errors.New("unable to parse major version") + } + minor, err := strconv.ParseUint(versions[1], 10, 0) + if err != nil { + return MessageVersion{}, errors.New("unable to parse major version") + } + patch, err := strconv.ParseUint(versions[2], 10, 0) + if err != nil { + return MessageVersion{}, errors.New("unable to parse major version") + } + return MessageVersion{Major: major, Minor: minor, Patch: patch}, nil +} + var ( - // ProtocolDataTransfer1_2 is the protocol identifier for the latest - // version of data-transfer (supports do-not-send-first-blocks extension) - ProtocolDataTransfer1_2 protocol.ID = "/fil/datatransfer/1.2.0" + // DataTransfer1_2 is the identifier for the current + // supported version of data-transfer + DataTransfer1_2 MessageVersion = MessageVersion{1, 2, 0} ) // Message is a message for the data transfer protocol @@ -26,7 +60,8 @@ type Message interface { TransferID() TransferID ToNet(w io.Writer) error ToIPLD() datamodel.Node - MessageForProtocol(targetProtocol protocol.ID) (newMsg Message, err error) + MessageForVersion(targetProtocol MessageVersion) (newMsg Message, err error) + WrappedForTransport(transportID TransportID) Message } // Request is a response message for the data transfer protocol diff --git a/message/message.go b/message/message.go index 438e6913..639046a4 100644 --- a/message/message.go +++ b/message/message.go @@ -23,5 +23,6 @@ var CancelResponse = message1_1.CancelResponse var UpdateResponse = message1_1.UpdateResponse var FromNet = message1_1.FromNet var FromIPLD = message1_1.FromIPLD +var FromNetWrapped = message1_1.FromNetWrapped var CompleteResponse = message1_1.CompleteResponse var CancelRequest = message1_1.CancelRequest diff --git a/message/message1_1prime/message.go b/message/message1_1prime/message.go index 4d852978..7690dbe6 100644 --- a/message/message1_1prime/message.go +++ b/message/message1_1prime/message.go @@ -198,6 +198,10 @@ func FromNet(r io.Reader) (datatransfer.Message, error) { } tresp := tm.(*TransferMessage1_1) + return fromMessage(tresp) +} + +func fromMessage(tresp *TransferMessage1_1) (datatransfer.Message, error) { if (tresp.IsRequest && tresp.Request == nil) || (!tresp.IsRequest && tresp.Response == nil) { return nil, xerrors.Errorf("invalid/malformed message") } @@ -208,6 +212,17 @@ func FromNet(r io.Reader) (datatransfer.Message, error) { return tresp.Response, nil } +// FromNetWrraped can read a network stream to deserialize a message + transport ID +func FromNetWrapped(r io.Reader) (datatransfer.TransportID, datatransfer.Message, error) { + tm, err := bindnodeRegistry.TypeFromReader(r, &WrappedTransferMessage1_1{}, dagcbor.Decode) + if err != nil { + return "", nil, err + } + wtresp := tm.(*WrappedTransferMessage1_1) + msg, err := fromMessage(&wtresp.Message) + return datatransfer.TransportID(wtresp.TransportID), msg, err +} + // FromNet can read a network stream to deserialize a GraphSyncMessage func FromIPLD(node datamodel.Node) (datatransfer.Message, error) { if tn, ok := node.(schema.TypedNode); ok { // shouldn't need this if from Graphsync @@ -218,13 +233,5 @@ func FromIPLD(node datamodel.Node) (datatransfer.Message, error) { return nil, err } tresp := tm.(*TransferMessage1_1) - - if (tresp.IsRequest && tresp.Request == nil) || (!tresp.IsRequest && tresp.Response == nil) { - return nil, xerrors.Errorf("invalid/malformed message") - } - - if tresp.IsRequest { - return tresp.Request, nil - } - return tresp.Response, nil + return fromMessage(tresp) } diff --git a/message/message1_1prime/message_test.go b/message/message1_1prime/message_test.go index 7ef75e53..bc1cd840 100644 --- a/message/message1_1prime/message_test.go +++ b/message/message1_1prime/message_test.go @@ -504,6 +504,71 @@ func TestToNetFromNetEquivalency(t *testing.T) { deserializedRequest, ok = deserialized.(datatransfer.Request) require.True(t, ok) + require.Equal(t, deserializedRequest.TransferID(), request.TransferID()) + require.Equal(t, deserializedRequest.IsCancel(), request.IsCancel()) + require.Equal(t, deserializedRequest.IsRequest(), request.IsRequest()) + }) + t.Run("round-trip with wrapping", func(t *testing.T) { + transportID := datatransfer.TransportID("applesauce") + baseCid := testutil.GenerateCids(1)[0] + selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() + isPull := false + id := datatransfer.TransferID(rand.Int31()) + accepted := false + voucher := testutil.NewTestTypedVoucher() + voucherResult := testutil.NewTestTypedVoucher() + request, err := message1_1.NewRequest(id, false, isPull, &voucher, baseCid, selector) + require.NoError(t, err) + wrequest := request.WrappedForTransport(transportID) + buf := new(bytes.Buffer) + err = wrequest.ToNet(buf) + require.NoError(t, err) + require.Greater(t, buf.Len(), 0) + receivedTransportID, deserialized, err := message1_1.FromNetWrapped(buf) + require.NoError(t, err) + + require.Equal(t, transportID, receivedTransportID) + deserializedRequest, ok := deserialized.(datatransfer.Request) + require.True(t, ok) + + require.Equal(t, deserializedRequest.TransferID(), request.TransferID()) + require.Equal(t, deserializedRequest.IsCancel(), request.IsCancel()) + require.Equal(t, deserializedRequest.IsPull(), request.IsPull()) + require.Equal(t, deserializedRequest.IsRequest(), request.IsRequest()) + require.Equal(t, deserializedRequest.BaseCid(), request.BaseCid()) + testutil.AssertEqualTestVoucher(t, request, deserializedRequest) + testutil.AssertEqualSelector(t, request, deserializedRequest) + + response, err := message1_1.NewResponse(id, accepted, false, &voucherResult) + require.NoError(t, err) + wresponse := response.WrappedForTransport(transportID) + err = wresponse.ToNet(buf) + require.NoError(t, err) + receivedTransportID, deserialized, err = message1_1.FromNetWrapped(buf) + require.NoError(t, err) + require.Equal(t, transportID, receivedTransportID) + + deserializedResponse, ok := deserialized.(datatransfer.Response) + require.True(t, ok) + + require.Equal(t, deserializedResponse.TransferID(), response.TransferID()) + require.Equal(t, deserializedResponse.Accepted(), response.Accepted()) + require.Equal(t, deserializedResponse.IsRequest(), response.IsRequest()) + require.Equal(t, deserializedResponse.IsUpdate(), response.IsUpdate()) + require.Equal(t, deserializedResponse.IsPaused(), response.IsPaused()) + testutil.AssertEqualTestVoucherResult(t, response, deserializedResponse) + + request = message1_1.CancelRequest(id) + wrequest = request.WrappedForTransport(transportID) + err = wrequest.ToNet(buf) + require.NoError(t, err) + receivedTransportID, deserialized, err = message1_1.FromNetWrapped(buf) + require.NoError(t, err) + require.Equal(t, transportID, receivedTransportID) + + deserializedRequest, ok = deserialized.(datatransfer.Request) + require.True(t, ok) + require.Equal(t, deserializedRequest.TransferID(), request.TransferID()) require.Equal(t, deserializedRequest.IsCancel(), request.IsCancel()) require.Equal(t, deserializedRequest.IsRequest(), request.IsRequest()) diff --git a/message/message1_1prime/schema.ipldsch b/message/message1_1prime/schema.ipldsch index d5f9f87a..06ed2071 100644 --- a/message/message1_1prime/schema.ipldsch +++ b/message/message1_1prime/schema.ipldsch @@ -1,6 +1,7 @@ type PeerID string # peer.ID, really should be bytes (this is non-utf8) but is string for backward compat type TransferID int type TypeIdentifier string +type TransportID string type ChannelID struct { Initiator PeerID @@ -35,3 +36,8 @@ type TransferMessage1_1 struct { Request nullable TransferRequest Response nullable TransferResponse } + +type WrappedTransferMessage1_1 struct { + TransportID TransportID (rename "ID") + Message TransferMessage1_1 (rename "Msg") +} \ No newline at end of file diff --git a/message/message1_1prime/transfer_message.go b/message/message1_1prime/transfer_message.go index 5f1f6286..4740fd8e 100644 --- a/message/message1_1prime/transfer_message.go +++ b/message/message1_1prime/transfer_message.go @@ -53,4 +53,20 @@ func init() { if err := bindnodeRegistry.RegisterType((*TransferMessage1_1)(nil), string(embedSchema), "TransferMessage1_1"); err != nil { panic(err.Error()) } + if err := bindnodeRegistry.RegisterType((*WrappedTransferMessage1_1)(nil), string(embedSchema), "WrappedTransferMessage1_1"); err != nil { + panic(err.Error()) + } +} + +type WrappedTransferMessage1_1 struct { + TransportID string + Message TransferMessage1_1 +} + +func (wtm *WrappedTransferMessage1_1) BindnodeSchema() string { + return string(embedSchema) +} + +func (wtm *WrappedTransferMessage1_1) toIPLD() schema.TypedNode { + return bindnodeRegistry.TypeToNode(wtm) } diff --git a/message/message1_1prime/transfer_request.go b/message/message1_1prime/transfer_request.go index 3600329f..175e5992 100644 --- a/message/message1_1prime/transfer_request.go +++ b/message/message1_1prime/transfer_request.go @@ -8,7 +8,6 @@ import ( "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" - "github.com/libp2p/go-libp2p-core/protocol" xerrors "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" @@ -30,15 +29,19 @@ type TransferRequest1_1 struct { RestartChannel datatransfer.ChannelID } -func (trq *TransferRequest1_1) MessageForProtocol(targetProtocol protocol.ID) (datatransfer.Message, error) { - switch targetProtocol { - case datatransfer.ProtocolDataTransfer1_2: +func (trq *TransferRequest1_1) MessageForVersion(version datatransfer.MessageVersion) (datatransfer.Message, error) { + switch version { + case datatransfer.DataTransfer1_2: return trq, nil default: return nil, xerrors.Errorf("protocol not supported") } } +func (trq *TransferRequest1_1) WrappedForTransport(transportID datatransfer.TransportID) datatransfer.Message { + return &WrappedTransferRequest1_1{trq, string(transportID)} +} + // IsRequest always returns true in this case because this is a transfer request func (trq *TransferRequest1_1) IsRequest() bool { return true @@ -156,3 +159,30 @@ func (trq *TransferRequest1_1) ToIPLD() datamodel.Node { func (trq *TransferRequest1_1) ToNet(w io.Writer) error { return ipld.EncodeStreaming(w, trq.toIPLD(), dagcbor.Encode) } + +// WrappedTransferRequest1_1 is used to serialize a request along with a +// transport id +type WrappedTransferRequest1_1 struct { + *TransferRequest1_1 + TransportID string +} + +func (trsp *WrappedTransferRequest1_1) toIPLD() schema.TypedNode { + msg := WrappedTransferMessage1_1{ + TransportID: trsp.TransportID, + Message: TransferMessage1_1{ + IsRequest: true, + Request: trsp.TransferRequest1_1, + Response: nil, + }, + } + return msg.toIPLD() +} + +func (trq *WrappedTransferRequest1_1) ToIPLD() datamodel.Node { + return trq.toIPLD().Representation() +} + +func (trq *WrappedTransferRequest1_1) ToNet(w io.Writer) error { + return ipld.EncodeStreaming(w, trq.toIPLD(), dagcbor.Encode) +} diff --git a/message/message1_1prime/transfer_request_test.go b/message/message1_1prime/transfer_request_test.go index 2cff27e3..88a98942 100644 --- a/message/message1_1prime/transfer_request_test.go +++ b/message/message1_1prime/transfer_request_test.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/go-data-transfer/v2/testutil" ) -func TestRequestMessageForProtocol(t *testing.T) { +func TestRequestMessageForVersion(t *testing.T) { baseCid := testutil.GenerateCids(1)[0] selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() isPull := true @@ -24,7 +24,8 @@ func TestRequestMessageForProtocol(t *testing.T) { request, err := message1_1.NewRequest(id, false, isPull, &voucher, baseCid, selector) require.NoError(t, err) - out12, err := request.MessageForProtocol(datatransfer.ProtocolDataTransfer1_2) + // v1.2 new protocol + out12, err := request.MessageForVersion(datatransfer.DataTransfer1_2) require.NoError(t, err) require.Equal(t, request, out12) @@ -38,4 +39,19 @@ func TestRequestMessageForProtocol(t *testing.T) { require.NoError(t, err) require.Equal(t, selector, n) require.Equal(t, testutil.TestVoucherType, req.VoucherType()) + + wrappedOut12 := out12.WrappedForTransport(datatransfer.LegacyTransportID) + require.Equal(t, &message1_1.WrappedTransferRequest1_1{ + TransferRequest1_1: request.(*message1_1.TransferRequest1_1), + TransportID: string(datatransfer.LegacyTransportID), + }, wrappedOut12) + + // random protocol should fail + _, err = request.MessageForVersion(datatransfer.MessageVersion{ + Major: rand.Uint64(), + Minor: rand.Uint64(), + Patch: rand.Uint64(), + }) + require.Error(t, err) + } diff --git a/message/message1_1prime/transfer_response.go b/message/message1_1prime/transfer_response.go index 69c19f23..77cf36a1 100644 --- a/message/message1_1prime/transfer_response.go +++ b/message/message1_1prime/transfer_response.go @@ -7,7 +7,6 @@ import ( "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/schema" - "github.com/libp2p/go-libp2p-core/protocol" xerrors "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" @@ -88,15 +87,18 @@ func (trsp *TransferResponse1_1) EmptyVoucherResult() bool { return trsp.VoucherTypeIdentifier == datatransfer.EmptyTypeIdentifier } -func (trsp *TransferResponse1_1) MessageForProtocol(targetProtocol protocol.ID) (datatransfer.Message, error) { - switch targetProtocol { - case datatransfer.ProtocolDataTransfer1_2: +func (trsp *TransferResponse1_1) MessageForVersion(version datatransfer.MessageVersion) (datatransfer.Message, error) { + switch version { + case datatransfer.DataTransfer1_2: return trsp, nil default: - return nil, xerrors.Errorf("protocol %s not supported", targetProtocol) + return nil, xerrors.Errorf("protocol %s not supported", version) } } +func (trsp *TransferResponse1_1) WrappedForTransport(transportID datatransfer.TransportID) datatransfer.Message { + return &WrappedTransferResponse1_1{trsp, string(transportID)} +} func (trsp *TransferResponse1_1) toIPLD() schema.TypedNode { msg := TransferMessage1_1{ IsRequest: false, @@ -114,3 +116,30 @@ func (trsp *TransferResponse1_1) ToIPLD() datamodel.Node { func (trsp *TransferResponse1_1) ToNet(w io.Writer) error { return ipld.EncodeStreaming(w, trsp.toIPLD(), dagcbor.Encode) } + +// WrappedTransferResponse1_1 is used to serialize a response along with a +// transport id +type WrappedTransferResponse1_1 struct { + *TransferResponse1_1 + TransportID string +} + +func (trsp *WrappedTransferResponse1_1) toIPLD() schema.TypedNode { + msg := WrappedTransferMessage1_1{ + TransportID: trsp.TransportID, + Message: TransferMessage1_1{ + IsRequest: false, + Request: nil, + Response: trsp.TransferResponse1_1, + }, + } + return msg.toIPLD() +} + +func (trsp *WrappedTransferResponse1_1) ToIPLD() datamodel.Node { + return trsp.toIPLD().Representation() +} + +func (trsp *WrappedTransferResponse1_1) ToNet(w io.Writer) error { + return ipld.EncodeStreaming(w, trsp.toIPLD(), dagcbor.Encode) +} diff --git a/message/message1_1prime/transfer_response_test.go b/message/message1_1prime/transfer_response_test.go index 9b979371..f29773bf 100644 --- a/message/message1_1prime/transfer_response_test.go +++ b/message/message1_1prime/transfer_response_test.go @@ -11,14 +11,14 @@ import ( "github.com/filecoin-project/go-data-transfer/v2/testutil" ) -func TestResponseMessageForProtocol(t *testing.T) { +func TestResponseMessageForVersion(t *testing.T) { id := datatransfer.TransferID(rand.Int31()) voucherResult := testutil.NewTestTypedVoucher() response, err := message1_1.NewResponse(id, false, true, &voucherResult) // not accepted require.NoError(t, err) - // v1.2 protocol - out, err := response.MessageForProtocol(datatransfer.ProtocolDataTransfer1_2) + // v1.2 new protocol + out, err := response.MessageForVersion(datatransfer.DataTransfer1_2) require.NoError(t, err) require.Equal(t, response, out) @@ -28,8 +28,17 @@ func TestResponseMessageForProtocol(t *testing.T) { require.Equal(t, testutil.TestVoucherType, resp.VoucherResultType()) require.True(t, resp.IsValidationResult()) - // random protocol - out, err = response.MessageForProtocol("RAND") + wrappedOut := out.WrappedForTransport(datatransfer.LegacyTransportID) + require.Equal(t, &message1_1.WrappedTransferResponse1_1{ + TransferResponse1_1: response.(*message1_1.TransferResponse1_1), + TransportID: string(datatransfer.LegacyTransportID), + }, wrappedOut) + + // random protocol should fail + _, err = response.MessageForVersion(datatransfer.MessageVersion{ + Major: rand.Uint64(), + Minor: rand.Uint64(), + Patch: rand.Uint64(), + }) require.Error(t, err) - require.Nil(t, out) } diff --git a/testutil/fakegraphsync.go b/testutil/fakegraphsync.go index f0ac8ab0..758a3f40 100644 --- a/testutil/fakegraphsync.go +++ b/testutil/fakegraphsync.go @@ -13,6 +13,7 @@ import ( "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/traversal" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" @@ -430,7 +431,7 @@ func NewFakeRequest(id graphsync.RequestID, extensions map[graphsync.ExtensionNa return &fakeRequest{ id: id, root: GenerateCids(1)[0], - selector: allSelector, + selector: selectorparse.CommonSelector_ExploreAllRecursively, priority: graphsync.Priority(rand.Int()), extensions: extensions, requestType: graphsync.RequestTypeNew, diff --git a/testutil/faketransport.go b/testutil/faketransport.go index 6bd822ac..f5843ee4 100644 --- a/testutil/faketransport.go +++ b/testutil/faketransport.go @@ -54,6 +54,11 @@ func NewFakeTransport() *FakeTransport { return &FakeTransport{} } +// ID is a unique identifier for this transport +func (ft *FakeTransport) ID() datatransfer.TransportID { + return "fake" +} + // Capabilities tells datatransfer what kinds of capabilities this transport supports func (ft *FakeTransport) Capabilities() datatransfer.TransportCapabilities { return datatransfer.TransportCapabilities{ diff --git a/testutil/testnet.go b/testutil/testnet.go deleted file mode 100644 index 88d30a07..00000000 --- a/testutil/testnet.go +++ /dev/null @@ -1,71 +0,0 @@ -package testutil - -import ( - "context" - - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/protocol" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/network" -) - -// FakeSentMessage is a recording of a message sent on the FakeNetwork -type FakeSentMessage struct { - PeerID peer.ID - Message datatransfer.Message -} - -// FakeNetwork is a network that satisfies the DataTransferNetwork interface but -// does not actually do anything -type FakeNetwork struct { - PeerID peer.ID - SentMessages []FakeSentMessage - Delegate network.Receiver -} - -// NewFakeNetwork returns a new fake data transfer network instance -func NewFakeNetwork(id peer.ID) *FakeNetwork { - return &FakeNetwork{PeerID: id} -} - -var _ network.DataTransferNetwork = (*FakeNetwork)(nil) - -// SendMessage sends a GraphSync message to a peer. -func (fn *FakeNetwork) SendMessage(ctx context.Context, p peer.ID, m datatransfer.Message) error { - fn.SentMessages = append(fn.SentMessages, FakeSentMessage{p, m}) - return nil -} - -// SetDelegate registers the Reciver to handle messages received from the -// network. -func (fn *FakeNetwork) SetDelegate(receiver network.Receiver) { - fn.Delegate = receiver -} - -// ConnectTo establishes a connection to the given peer -func (fn *FakeNetwork) ConnectTo(_ context.Context, _ peer.ID) error { - panic("not implemented") -} - -func (fn *FakeNetwork) ConnectWithRetry(ctx context.Context, p peer.ID) error { - panic("implement me") -} - -// ID returns a stubbed id for host of this network -func (fn *FakeNetwork) ID() peer.ID { - return fn.PeerID -} - -// Protect does nothing on the fake network -func (fn *FakeNetwork) Protect(id peer.ID, tag string) { -} - -// Unprotect does nothing on the fake network -func (fn *FakeNetwork) Unprotect(id peer.ID, tag string) bool { - return false -} - -func (fn *FakeNetwork) Protocol(ctx context.Context, id peer.ID) (protocol.ID, error) { - return datatransfer.ProtocolDataTransfer1_2, nil -} diff --git a/testutil/testutil.go b/testutil/testutil.go index dec67a72..0c143969 100644 --- a/testutil/testutil.go +++ b/testutil/testutil.go @@ -10,10 +10,6 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" blocksutil "github.com/ipfs/go-ipfs-blocksutil" - "github.com/ipld/go-ipld-prime/datamodel" - basicnode "github.com/ipld/go-ipld-prime/node/basic" - "github.com/ipld/go-ipld-prime/traversal/selector" - "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/jbenet/go-random" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" @@ -102,13 +98,6 @@ func AssertEqualSelector(t *testing.T, expectedRequest datatransfer.Request, req require.Equal(t, expectedSelector, selector) } -// AllSelector just returns a new instance of a "whole dag selector" -func AllSelector() datamodel.Node { - ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) - return ssb.ExploreRecursive(selector.RecursionLimitNone(), - ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() -} - // StartAndWaitForReady is a utility function to start a module and verify it reaches the ready state func StartAndWaitForReady(ctx context.Context, t *testing.T, manager datatransfer.Manager) { ready := make(chan error, 1) diff --git a/transport.go b/transport.go index dfadad56..ecaed685 100644 --- a/transport.go +++ b/transport.go @@ -6,6 +6,13 @@ import ( ipld "github.com/ipld/go-ipld-prime" ) +// TransportID identifies a unique transport +type TransportID string + +// LegacyTransportID is the only transport for the fil/data-transfer protocol -- +// i.e. graphsync +const LegacyTransportID TransportID = "graphsync" + // EventsHandler are semantic data transfer events that happen as a result of transport events type EventsHandler interface { // ChannelState queries for the current channel state @@ -94,6 +101,9 @@ and send messages. Beyond that, additional commands may or may not be supported. Whether a command is supported can be determined ahead by calling Capabilities(). */ type Transport interface { + // ID is a unique identifier for this transport + ID() TransportID + // Capabilities tells datatransfer what kinds of capabilities this transport supports Capabilities() TransportCapabilities // OpenChannel opens a channel on a given transport to move data back and forth. diff --git a/transport/graphsync/extension/gsextension.go b/transport/graphsync/extension/gsextension.go index 532cb398..f161ee12 100644 --- a/transport/graphsync/extension/gsextension.go +++ b/transport/graphsync/extension/gsextension.go @@ -5,7 +5,6 @@ import ( "github.com/ipfs/go-graphsync" "github.com/ipld/go-ipld-prime/datamodel" - "github.com/libp2p/go-libp2p-core/protocol" datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/message" @@ -21,10 +20,10 @@ const ( ) // ProtocolMap maps graphsync extensions to their libp2p protocols -var ProtocolMap = map[graphsync.ExtensionName]protocol.ID{ - ExtensionIncomingRequest1_1: datatransfer.ProtocolDataTransfer1_2, - ExtensionOutgoingBlock1_1: datatransfer.ProtocolDataTransfer1_2, - ExtensionDataTransfer1_1: datatransfer.ProtocolDataTransfer1_2, +var ProtocolMap = map[graphsync.ExtensionName]datatransfer.MessageVersion{ + ExtensionIncomingRequest1_1: datatransfer.DataTransfer1_2, + ExtensionOutgoingBlock1_1: datatransfer.DataTransfer1_2, + ExtensionDataTransfer1_1: datatransfer.DataTransfer1_2, } // ToExtensionData converts a message to a graphsync extension @@ -35,7 +34,7 @@ func ToExtensionData(msg datatransfer.Message, supportedExtensions []graphsync.E if !ok { return nil, errors.New("unsupported protocol") } - versionedMsg, err := msg.MessageForProtocol(protoID) + versionedMsg, err := msg.MessageForVersion(protoID) if err != nil { continue } diff --git a/transport/graphsync/graphsync.go b/transport/graphsync/graphsync.go index 058a2aec..06dcd327 100644 --- a/transport/graphsync/graphsync.go +++ b/transport/graphsync/graphsync.go @@ -15,13 +15,15 @@ import ( "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/network" dtchannel "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/dtchannel" "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" + "github.com/filecoin-project/go-data-transfer/v2/transport/helpers/network" ) var log = logging.Logger("dt_graphsync") +var transportID datatransfer.TransportID = "graphsync" + // When restarting a data transfer, we cancel the existing graphsync request // before opening a new one. // This constant defines the maximum time to wait for the request to be @@ -103,6 +105,10 @@ func NewTransport(peerID peer.ID, gs graphsync.GraphExchange, dtNet network.Data return t } +func (t *Transport) ID() datatransfer.TransportID { + return transportID +} + func (t *Transport) Capabilities() datatransfer.TransportCapabilities { return datatransfer.TransportCapabilities{ Pausable: true, @@ -125,7 +131,7 @@ func (t *Transport) OpenChannel( channel.Selector(), req) } - return t.dtNet.SendMessage(ctx, channel.OtherPeer(), req) + return t.dtNet.SendMessage(ctx, channel.OtherPeer(), transportID, req) } // RestartChannel restarts a channel on the initiator side @@ -155,7 +161,7 @@ func (t *Transport) RestartChannel( channelState.Selector(), req) } - return t.dtNet.SendMessage(ctx, channelState.OtherPeer(), req) + return t.dtNet.SendMessage(ctx, channelState.OtherPeer(), transportID, req) } func (t *Transport) openRequest( @@ -201,7 +207,7 @@ func (t *Transport) UpdateChannel(ctx context.Context, chid datatransfer.Channel ch, err := t.getDTChannel(chid) if err != nil { if update.SendMessage != nil && !update.Closed { - return t.dtNet.SendMessage(ctx, t.otherPeer(chid), update.SendMessage) + return t.dtNet.SendMessage(ctx, t.otherPeer(chid), transportID, update.SendMessage) } return err } @@ -221,7 +227,7 @@ func (t *Transport) UpdateChannel(ctx context.Context, chid datatransfer.Channel } if update.SendMessage != nil { - if err := t.dtNet.SendMessage(ctx, t.otherPeer(chid), update.SendMessage); err != nil { + if err := t.dtNet.SendMessage(ctx, t.otherPeer(chid), transportID, update.SendMessage); err != nil { return err } } @@ -239,7 +245,7 @@ func (t *Transport) UpdateChannel(ctx context.Context, chid datatransfer.Channel // SendMessage sends a data transfer message over the channel to the other peer func (t *Transport) SendMessage(ctx context.Context, chid datatransfer.ChannelID, msg datatransfer.Message) error { - return t.dtNet.SendMessage(ctx, t.otherPeer(chid), msg) + return t.dtNet.SendMessage(ctx, t.otherPeer(chid), transportID, msg) } // CleanupChannel is called on the otherside of a cancel - removes any associated @@ -286,7 +292,7 @@ func (t *Transport) SetEventHandler(events datatransfer.EventsHandler) error { t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterNetworkErrorListener(t.gsNetworkSendErrorListener)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterReceiverNetworkErrorListener(t.gsNetworkReceiveErrorListener)) - t.dtNet.SetDelegate(&receiver{t}) + t.dtNet.SetDelegate(transportID, &receiver{t}) return nil } diff --git a/transport/graphsync/receiver.go b/transport/graphsync/receiver.go index 585a8859..ba9b1639 100644 --- a/transport/graphsync/receiver.go +++ b/transport/graphsync/receiver.go @@ -49,7 +49,7 @@ func (r *receiver) receiveRequest(ctx context.Context, initiator peer.ID, incomi if err != nil { if !initiateGraphsyncRequest { if response != nil { - return r.transport.dtNet.SendMessage(ctx, initiator, response) + return r.transport.dtNet.SendMessage(ctx, initiator, transportID, response) } return receiveErr } @@ -84,7 +84,7 @@ func (r *receiver) receiveRequest(ctx context.Context, initiator peer.ID, incomi return err } } else { - if err := r.transport.dtNet.SendMessage(ctx, initiator, response); err != nil { + if err := r.transport.dtNet.SendMessage(ctx, initiator, transportID, response); err != nil { return err } } diff --git a/network/interface.go b/transport/helpers/network/interface.go similarity index 63% rename from network/interface.go rename to transport/helpers/network/interface.go index 965a337d..2ed90303 100644 --- a/network/interface.go +++ b/transport/helpers/network/interface.go @@ -2,6 +2,8 @@ package network import ( "context" + "errors" + "strings" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" @@ -9,6 +11,24 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) +const ( + // ProtocolFilDataTransfer1_2 is the legacy filecoin data transfer protocol + // that assumes a graphsync transport + ProtocolFilDataTransfer1_2 protocol.ID = "/fil/datatransfer/1.2.0" + // ProtocolDataTransfer1_2 is the protocol identifier for current data transfer + // protocol which wraps transport information in the protocol + ProtocolDataTransfer1_2 protocol.ID = "/datatransfer/1.2.0" +) + +// MessageVersion extracts the message version from the full protocol +func MessageVersion(protocol protocol.ID) (datatransfer.MessageVersion, error) { + protocolParts := strings.Split(string(protocol), "/") + if len(protocolParts) == 0 { + return datatransfer.MessageVersion{}, errors.New("no protocol to parse") + } + return datatransfer.MessageVersionFromString(protocolParts[len(protocolParts)-1]) +} + // DataTransferNetwork provides network connectivity for GraphSync. type DataTransferNetwork interface { Protect(id peer.ID, tag string) @@ -18,11 +38,12 @@ type DataTransferNetwork interface { SendMessage( context.Context, peer.ID, + datatransfer.TransportID, datatransfer.Message) error // SetDelegate registers the Reciver to handle messages received from the // network. - SetDelegate(Receiver) + SetDelegate(datatransfer.TransportID, Receiver) // ConnectTo establishes a connection to the given peer ConnectTo(context.Context, peer.ID) error @@ -53,6 +74,4 @@ type Receiver interface { incoming datatransfer.Response) ReceiveRestartExistingChannelRequest(ctx context.Context, sender peer.ID, incoming datatransfer.Request) - - ReceiveError(error) } diff --git a/network/libp2p_impl.go b/transport/helpers/network/libp2p_impl.go similarity index 84% rename from network/libp2p_impl.go rename to transport/helpers/network/libp2p_impl.go index 1408ab2e..412e9349 100644 --- a/network/libp2p_impl.go +++ b/transport/helpers/network/libp2p_impl.go @@ -43,7 +43,8 @@ const defaultMaxAttemptDuration = 5 * time.Minute const defaultBackoffFactor = 5 var defaultDataTransferProtocols = []protocol.ID{ - datatransfer.ProtocolDataTransfer1_2, + ProtocolDataTransfer1_2, + ProtocolFilDataTransfer1_2, } // Option is an option for configuring the libp2p storage market network @@ -85,6 +86,7 @@ func NewFromLibp2pHost(host host.Host, options ...Option) DataTransferNetwork { minAttemptDuration: defaultMinAttemptDuration, maxAttemptDuration: defaultMaxAttemptDuration, backoffFactor: defaultBackoffFactor, + receivers: make(map[datatransfer.TransportID]Receiver), } dataTransferNetwork.setDataTransferProtocols(defaultDataTransferProtocols) @@ -100,7 +102,7 @@ func NewFromLibp2pHost(host host.Host, options ...Option) DataTransferNetwork { type libp2pDataTransferNetwork struct { host host.Host // inbound messages from the network are forwarded to the receiver - receiver Receiver + receivers map[datatransfer.TransportID]Receiver openStreamTimeout time.Duration sendMessageTimeout time.Duration @@ -159,6 +161,7 @@ func (impl *libp2pDataTransferNetwork) openStream(ctx context.Context, id peer.I func (dtnet *libp2pDataTransferNetwork) SendMessage( ctx context.Context, p peer.ID, + transportID datatransfer.TransportID, outgoing datatransfer.Message) error { ctx, span := otel.Tracer("data-transfer").Start(ctx, "sendMessage", trace.WithAttributes( @@ -180,7 +183,15 @@ func (dtnet *libp2pDataTransferNetwork) SendMessage( return err } - outgoing, err = outgoing.MessageForProtocol(s.Protocol()) + messageVersion, err := MessageVersion(s.Protocol()) + if err != nil { + err = xerrors.Errorf("failed to determine message version for protocol: %w", err) + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return err + } + + outgoing, err = outgoing.MessageForVersion(messageVersion) if err != nil { err = xerrors.Errorf("failed to convert message for protocol: %w", err) span.RecordError(err) @@ -188,7 +199,7 @@ func (dtnet *libp2pDataTransferNetwork) SendMessage( return err } - if err = dtnet.msgToStream(ctx, s, outgoing); err != nil { + if err = dtnet.msgToStream(ctx, s, transportID, outgoing); err != nil { if err2 := s.Reset(); err2 != nil { log.Error(err) span.RecordError(err2) @@ -203,8 +214,8 @@ func (dtnet *libp2pDataTransferNetwork) SendMessage( return s.Close() } -func (dtnet *libp2pDataTransferNetwork) SetDelegate(r Receiver) { - dtnet.receiver = r +func (dtnet *libp2pDataTransferNetwork) SetDelegate(transportID datatransfer.TransportID, r Receiver) { + dtnet.receivers[transportID] = r for _, p := range dtnet.dtProtocols { dtnet.host.SetStreamHandler(p, dtnet.handleNewStream) } @@ -234,29 +245,42 @@ func (dtnet *libp2pDataTransferNetwork) ConnectWithRetry(ctx context.Context, p func (dtnet *libp2pDataTransferNetwork) handleNewStream(s network.Stream) { defer s.Close() // nolint: errcheck,gosec - if dtnet.receiver == nil { + if len(dtnet.receivers) == 0 { s.Reset() // nolint: errcheck,gosec return } p := s.Conn().RemotePeer() for { + var transportID datatransfer.TransportID var received datatransfer.Message var err error switch s.Protocol() { - case datatransfer.ProtocolDataTransfer1_2: + case ProtocolFilDataTransfer1_2: + if dtnet.receivers[datatransfer.LegacyTransportID] == nil { + s.Reset() // nolint: errcheck,gosec + return + } + transportID = datatransfer.LegacyTransportID received, err = message.FromNet(s) + case ProtocolDataTransfer1_2: + transportID, received, err = message.FromNetWrapped(s) } if err != nil { if err != io.EOF && err != io.ErrUnexpectedEOF { s.Reset() // nolint: errcheck,gosec - go dtnet.receiver.ReceiveError(err) - log.Debugf("net handleNewStream from %s error: %s", p, err) + log.Errorf("net handleNewStream from %s error: %s", p, err) } return } + // if we have no transport handler, reset the stream + if dtnet.receivers[transportID] == nil { + s.Reset() // nolint: errcheck,gosec + return + } + ctx := context.Background() log.Debugf("net handleNewStream from %s", p) @@ -264,15 +288,15 @@ func (dtnet *libp2pDataTransferNetwork) handleNewStream(s network.Stream) { receivedRequest, ok := received.(datatransfer.Request) if ok { if receivedRequest.IsRestartExistingChannelRequest() { - dtnet.receiver.ReceiveRestartExistingChannelRequest(ctx, p, receivedRequest) + dtnet.receivers[transportID].ReceiveRestartExistingChannelRequest(ctx, p, receivedRequest) } else { - dtnet.receiver.ReceiveRequest(ctx, p, receivedRequest) + dtnet.receivers[transportID].ReceiveRequest(ctx, p, receivedRequest) } } } else { receivedResponse, ok := received.(datatransfer.Response) if ok { - dtnet.receiver.ReceiveResponse(ctx, p, receivedResponse) + dtnet.receivers[transportID].ReceiveResponse(ctx, p, receivedResponse) } } } @@ -290,7 +314,7 @@ func (dtnet *libp2pDataTransferNetwork) Unprotect(id peer.ID, tag string) bool { return dtnet.host.ConnManager().Unprotect(id, tag) } -func (dtnet *libp2pDataTransferNetwork) msgToStream(ctx context.Context, s network.Stream, msg datatransfer.Message) error { +func (dtnet *libp2pDataTransferNetwork) msgToStream(ctx context.Context, s network.Stream, transportID datatransfer.TransportID, msg datatransfer.Message) error { if msg.IsRequest() { log.Debugf("Outgoing request message for transfer ID: %d", msg.TransferID()) } @@ -309,7 +333,12 @@ func (dtnet *libp2pDataTransferNetwork) msgToStream(ctx context.Context, s netwo }() switch s.Protocol() { - case datatransfer.ProtocolDataTransfer1_2: + case ProtocolFilDataTransfer1_2: + if transportID != datatransfer.LegacyTransportID { + return fmt.Errorf("cannot send messages for transports other than graphsync on legacy protocol") + } + case ProtocolDataTransfer1_2: + msg = msg.WrappedForTransport(transportID) default: return fmt.Errorf("unrecognized protocol on remote: %s", s.Protocol()) } @@ -318,7 +347,6 @@ func (dtnet *libp2pDataTransferNetwork) msgToStream(ctx context.Context, s netwo log.Debugf("error: %s", err) return err } - return nil } diff --git a/network/libp2p_impl_test.go b/transport/helpers/network/libp2p_impl_test.go similarity index 93% rename from network/libp2p_impl_test.go rename to transport/helpers/network/libp2p_impl_test.go index 89489155..ceffc8af 100644 --- a/network/libp2p_impl_test.go +++ b/transport/helpers/network/libp2p_impl_test.go @@ -21,8 +21,8 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/message" "github.com/filecoin-project/go-data-transfer/v2/message/types" - "github.com/filecoin-project/go-data-transfer/v2/network" "github.com/filecoin-project/go-data-transfer/v2/testutil" + "github.com/filecoin-project/go-data-transfer/v2/transport/helpers/network" ) // Receiver is an interface for receiving messages from the DataTransferNetwork. @@ -91,8 +91,8 @@ func TestMessageSendAndReceive(t *testing.T) { messageReceived: make(chan struct{}), connectedPeers: make(chan peer.ID, 2), } - dtnet1.SetDelegate(r) - dtnet2.SetDelegate(r) + dtnet1.SetDelegate("graphsync", r) + dtnet2.SetDelegate("graphsync", r) err = dtnet1.ConnectTo(ctx, host2.ID()) require.NoError(t, err) @@ -105,7 +105,7 @@ func TestMessageSendAndReceive(t *testing.T) { voucher := testutil.NewTestTypedVoucher() request, err := message.NewRequest(id, false, isPull, &voucher, baseCid, selector) require.NoError(t, err) - require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) + require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), "graphsync", request)) select { case <-ctx.Done(): @@ -134,7 +134,7 @@ func TestMessageSendAndReceive(t *testing.T) { voucherResult := testutil.NewTestTypedVoucher() response, err := message.ValidationResultResponse(types.NewMessage, id, datatransfer.ValidationResult{Accepted: accepted, VoucherResult: &voucherResult}, nil, false) require.NoError(t, err) - require.NoError(t, dtnet2.SendMessage(ctx, host1.ID(), response)) + require.NoError(t, dtnet2.SendMessage(ctx, host1.ID(), "graphsync", response)) select { case <-ctx.Done(): @@ -161,7 +161,7 @@ func TestMessageSendAndReceive(t *testing.T) { Responder: peers[1], ID: id} request := message.RestartExistingChannelRequest(chId) - require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) + require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), "graphsync", request)) select { case <-ctx.Done(): @@ -263,8 +263,8 @@ func TestSendMessageRetry(t *testing.T) { messageReceived: make(chan struct{}), connectedPeers: make(chan peer.ID, 2), } - dtnet1.SetDelegate(r) - dtnet2.SetDelegate(r) + dtnet1.SetDelegate("graphsync", r) + dtnet2.SetDelegate("graphsync", r) err = dtnet1.ConnectTo(ctx, host2.ID()) require.NoError(t, err) @@ -277,7 +277,7 @@ func TestSendMessageRetry(t *testing.T) { request, err := message.NewRequest(id, false, isPull, &voucher, baseCid, selector) require.NoError(t, err) - err = dtnet1.SendMessage(ctx, host2.ID(), request) + err = dtnet1.SendMessage(ctx, host2.ID(), "graphsync", request) if !tcase.expSuccess { require.Error(t, err) return From 0b556bf2f7dca10f6708ca93fa2fc92a773df3e1 Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Wed, 15 Jun 2022 20:30:18 -0700 Subject: [PATCH 10/18] Feat/refactor transport protocol update part 2 (#338) * feat(network): transport versioning and detection Support multiple transports on the libp2p protocol, via different protocol naming, and using libp2p to do protocol negotiation * Update transport/helpers/network/libp2p_impl.go Co-authored-by: Rod Vagg * Update transport/helpers/network/libp2p_impl.go Co-authored-by: Rod Vagg * fix(network): add versions check for legacy transport Co-authored-by: Rod Vagg --- impl/utils.go | 1 - itest/integration_test.go | 4 +- message.go | 30 ++- message/message1_1prime/message.go | 27 ++- message/message1_1prime/message_test.go | 22 +- message/message1_1prime/schema.ipldsch | 7 + message/message1_1prime/transfer_message.go | 5 +- message/message1_1prime/transfer_request.go | 32 ++- .../message1_1prime/transfer_request_test.go | 10 +- message/message1_1prime/transfer_response.go | 27 ++- .../message1_1prime/transfer_response_test.go | 10 +- testutil/faketransport.go | 5 + transport.go | 7 + transport/graphsync/extension/gsextension.go | 2 +- transport/graphsync/graphsync.go | 9 +- transport/helpers/network/interface.go | 18 +- transport/helpers/network/libp2p_impl.go | 228 +++++++++++------- transport/helpers/network/libp2p_impl_test.go | 8 +- 18 files changed, 305 insertions(+), 147 deletions(-) diff --git a/impl/utils.go b/impl/utils.go index 1a2b1351..518b9e0c 100644 --- a/impl/utils.go +++ b/impl/utils.go @@ -83,4 +83,3 @@ func (m *manager) cancelMessage(chid datatransfer.ChannelID) datatransfer.Messag } return message.CancelResponse(chid.ID) } - diff --git a/itest/integration_test.go b/itest/integration_test.go index 7be07a71..f13e69e4 100644 --- a/itest/integration_test.go +++ b/itest/integration_test.go @@ -1779,7 +1779,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { r := &receiver{ messageReceived: make(chan receivedMessage), } - dtnet2.SetDelegate("graphsync", r) + dtnet2.SetDelegate(datatransfer.LegacyTransportID, []datatransfer.Version{datatransfer.LegacyTransportVersion}, r) gsr := &fakeGraphSyncReceiver{ receivedMessages: make(chan receivedGraphSyncMessage), @@ -1857,7 +1857,7 @@ func TestResponseHookWhenExtensionNotFound(t *testing.T) { r := &receiver{ messageReceived: make(chan receivedMessage), } - dtnet2.SetDelegate("graphsync", r) + dtnet2.SetDelegate(datatransfer.LegacyTransportID, []datatransfer.Version{datatransfer.LegacyTransportVersion}, r) gsr := &fakeGraphSyncReceiver{ receivedMessages: make(chan receivedGraphSyncMessage), diff --git a/message.go b/message.go index 9eed49fe..b44b43e2 100644 --- a/message.go +++ b/message.go @@ -11,41 +11,41 @@ import ( "github.com/ipld/go-ipld-prime/datamodel" ) -type MessageVersion struct { +type Version struct { Major uint64 Minor uint64 Patch uint64 } -func (mv MessageVersion) String() string { +func (mv Version) String() string { return fmt.Sprintf("%d.%d.%d", mv.Major, mv.Minor, mv.Patch) } // MessageVersionFromString parses a string into a message version -func MessageVersionFromString(versionString string) (MessageVersion, error) { +func MessageVersionFromString(versionString string) (Version, error) { versions := strings.Split(versionString, ".") if len(versions) != 3 { - return MessageVersion{}, errors.New("not a version string") + return Version{}, errors.New("not a version string") } major, err := strconv.ParseUint(versions[0], 10, 0) if err != nil { - return MessageVersion{}, errors.New("unable to parse major version") + return Version{}, errors.New("unable to parse major version") } minor, err := strconv.ParseUint(versions[1], 10, 0) if err != nil { - return MessageVersion{}, errors.New("unable to parse major version") + return Version{}, errors.New("unable to parse major version") } patch, err := strconv.ParseUint(versions[2], 10, 0) if err != nil { - return MessageVersion{}, errors.New("unable to parse major version") + return Version{}, errors.New("unable to parse major version") } - return MessageVersion{Major: major, Minor: minor, Patch: patch}, nil + return Version{Major: major, Minor: minor, Patch: patch}, nil } var ( // DataTransfer1_2 is the identifier for the current // supported version of data-transfer - DataTransfer1_2 MessageVersion = MessageVersion{1, 2, 0} + DataTransfer1_2 Version = Version{1, 2, 0} ) // Message is a message for the data transfer protocol @@ -60,8 +60,16 @@ type Message interface { TransferID() TransferID ToNet(w io.Writer) error ToIPLD() datamodel.Node - MessageForVersion(targetProtocol MessageVersion) (newMsg Message, err error) - WrappedForTransport(transportID TransportID) Message + MessageForVersion(targetProtocol Version) (newMsg Message, err error) + Version() Version + WrappedForTransport(transportID TransportID, transportVersion Version) TransportedMessage +} + +// TransportedMessage is a message that can also report how it was transported +type TransportedMessage interface { + Message + TransportID() TransportID + TransportVersion() Version } // Request is a response message for the data transfer protocol diff --git a/message/message1_1prime/message.go b/message/message1_1prime/message.go index 7690dbe6..641686bf 100644 --- a/message/message1_1prime/message.go +++ b/message/message1_1prime/message.go @@ -212,15 +212,34 @@ func fromMessage(tresp *TransferMessage1_1) (datatransfer.Message, error) { return tresp.Response, nil } +func fromWrappedMessage(wtresp *WrappedTransferMessage1_1) (datatransfer.TransportedMessage, error) { + tresp := wtresp.Message + if (tresp.IsRequest && tresp.Request == nil) || (!tresp.IsRequest && tresp.Response == nil) { + return nil, xerrors.Errorf("invalid/malformed message") + } + + if tresp.IsRequest { + return &WrappedTransferRequest1_1{ + tresp.Request, + wtresp.TransportVersion, + wtresp.TransportID, + }, nil + } + return &WrappedTransferResponse1_1{ + tresp.Response, + wtresp.TransportID, + wtresp.TransportVersion, + }, nil +} + // FromNetWrraped can read a network stream to deserialize a message + transport ID -func FromNetWrapped(r io.Reader) (datatransfer.TransportID, datatransfer.Message, error) { +func FromNetWrapped(r io.Reader) (datatransfer.TransportedMessage, error) { tm, err := bindnodeRegistry.TypeFromReader(r, &WrappedTransferMessage1_1{}, dagcbor.Decode) if err != nil { - return "", nil, err + return nil, err } wtresp := tm.(*WrappedTransferMessage1_1) - msg, err := fromMessage(&wtresp.Message) - return datatransfer.TransportID(wtresp.TransportID), msg, err + return fromWrappedMessage(wtresp) } // FromNet can read a network stream to deserialize a GraphSyncMessage diff --git a/message/message1_1prime/message_test.go b/message/message1_1prime/message_test.go index bc1cd840..33050deb 100644 --- a/message/message1_1prime/message_test.go +++ b/message/message1_1prime/message_test.go @@ -510,6 +510,7 @@ func TestToNetFromNetEquivalency(t *testing.T) { }) t.Run("round-trip with wrapping", func(t *testing.T) { transportID := datatransfer.TransportID("applesauce") + transportVersion := datatransfer.Version{Major: 1, Minor: 5, Patch: 0} baseCid := testutil.GenerateCids(1)[0] selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() isPull := false @@ -519,15 +520,16 @@ func TestToNetFromNetEquivalency(t *testing.T) { voucherResult := testutil.NewTestTypedVoucher() request, err := message1_1.NewRequest(id, false, isPull, &voucher, baseCid, selector) require.NoError(t, err) - wrequest := request.WrappedForTransport(transportID) + wrequest := request.WrappedForTransport(transportID, transportVersion) buf := new(bytes.Buffer) err = wrequest.ToNet(buf) require.NoError(t, err) require.Greater(t, buf.Len(), 0) - receivedTransportID, deserialized, err := message1_1.FromNetWrapped(buf) + deserialized, err := message1_1.FromNetWrapped(buf) require.NoError(t, err) - require.Equal(t, transportID, receivedTransportID) + require.Equal(t, transportID, deserialized.TransportID()) + require.Equal(t, transportVersion, deserialized.TransportVersion()) deserializedRequest, ok := deserialized.(datatransfer.Request) require.True(t, ok) @@ -541,12 +543,13 @@ func TestToNetFromNetEquivalency(t *testing.T) { response, err := message1_1.NewResponse(id, accepted, false, &voucherResult) require.NoError(t, err) - wresponse := response.WrappedForTransport(transportID) + wresponse := response.WrappedForTransport(transportID, transportVersion) err = wresponse.ToNet(buf) require.NoError(t, err) - receivedTransportID, deserialized, err = message1_1.FromNetWrapped(buf) + deserialized, err = message1_1.FromNetWrapped(buf) require.NoError(t, err) - require.Equal(t, transportID, receivedTransportID) + require.Equal(t, transportID, deserialized.TransportID()) + require.Equal(t, transportVersion, deserialized.TransportVersion()) deserializedResponse, ok := deserialized.(datatransfer.Response) require.True(t, ok) @@ -559,12 +562,13 @@ func TestToNetFromNetEquivalency(t *testing.T) { testutil.AssertEqualTestVoucherResult(t, response, deserializedResponse) request = message1_1.CancelRequest(id) - wrequest = request.WrappedForTransport(transportID) + wrequest = request.WrappedForTransport(transportID, transportVersion) err = wrequest.ToNet(buf) require.NoError(t, err) - receivedTransportID, deserialized, err = message1_1.FromNetWrapped(buf) + deserialized, err = message1_1.FromNetWrapped(buf) require.NoError(t, err) - require.Equal(t, transportID, receivedTransportID) + require.Equal(t, transportID, deserialized.TransportID()) + require.Equal(t, transportVersion, deserialized.TransportVersion()) deserializedRequest, ok = deserialized.(datatransfer.Request) require.True(t, ok) diff --git a/message/message1_1prime/schema.ipldsch b/message/message1_1prime/schema.ipldsch index 06ed2071..32683663 100644 --- a/message/message1_1prime/schema.ipldsch +++ b/message/message1_1prime/schema.ipldsch @@ -37,7 +37,14 @@ type TransferMessage1_1 struct { Response nullable TransferResponse } +type Version struct { + Major Int + Minor Int + Patch Int +} representation tuple + type WrappedTransferMessage1_1 struct { TransportID TransportID (rename "ID") + TransportVersion Version (rename "TV") Message TransferMessage1_1 (rename "Msg") } \ No newline at end of file diff --git a/message/message1_1prime/transfer_message.go b/message/message1_1prime/transfer_message.go index 4740fd8e..0f28e52d 100644 --- a/message/message1_1prime/transfer_message.go +++ b/message/message1_1prime/transfer_message.go @@ -59,8 +59,9 @@ func init() { } type WrappedTransferMessage1_1 struct { - TransportID string - Message TransferMessage1_1 + TransportID string + TransportVersion datatransfer.Version + Message TransferMessage1_1 } func (wtm *WrappedTransferMessage1_1) BindnodeSchema() string { diff --git a/message/message1_1prime/transfer_request.go b/message/message1_1prime/transfer_request.go index 175e5992..007d651d 100644 --- a/message/message1_1prime/transfer_request.go +++ b/message/message1_1prime/transfer_request.go @@ -29,7 +29,7 @@ type TransferRequest1_1 struct { RestartChannel datatransfer.ChannelID } -func (trq *TransferRequest1_1) MessageForVersion(version datatransfer.MessageVersion) (datatransfer.Message, error) { +func (trq *TransferRequest1_1) MessageForVersion(version datatransfer.Version) (datatransfer.Message, error) { switch version { case datatransfer.DataTransfer1_2: return trq, nil @@ -38,8 +38,16 @@ func (trq *TransferRequest1_1) MessageForVersion(version datatransfer.MessageVer } } -func (trq *TransferRequest1_1) WrappedForTransport(transportID datatransfer.TransportID) datatransfer.Message { - return &WrappedTransferRequest1_1{trq, string(transportID)} +func (trq *TransferRequest1_1) Version() datatransfer.Version { + return datatransfer.DataTransfer1_2 +} + +func (trq *TransferRequest1_1) WrappedForTransport(transportID datatransfer.TransportID, transportVersion datatransfer.Version) datatransfer.TransportedMessage { + return &WrappedTransferRequest1_1{ + TransferRequest1_1: trq, + transportID: string(transportID), + transportVersion: transportVersion, + } } // IsRequest always returns true in this case because this is a transfer request @@ -164,15 +172,25 @@ func (trq *TransferRequest1_1) ToNet(w io.Writer) error { // transport id type WrappedTransferRequest1_1 struct { *TransferRequest1_1 - TransportID string + transportVersion datatransfer.Version + transportID string +} + +func (trq *WrappedTransferRequest1_1) TransportID() datatransfer.TransportID { + return datatransfer.TransportID(trq.transportID) +} + +func (trq *WrappedTransferRequest1_1) TransportVersion() datatransfer.Version { + return trq.transportVersion } -func (trsp *WrappedTransferRequest1_1) toIPLD() schema.TypedNode { +func (trq *WrappedTransferRequest1_1) toIPLD() schema.TypedNode { msg := WrappedTransferMessage1_1{ - TransportID: trsp.TransportID, + TransportID: trq.transportID, + TransportVersion: trq.transportVersion, Message: TransferMessage1_1{ IsRequest: true, - Request: trsp.TransferRequest1_1, + Request: trq.TransferRequest1_1, Response: nil, }, } diff --git a/message/message1_1prime/transfer_request_test.go b/message/message1_1prime/transfer_request_test.go index 88a98942..410917b8 100644 --- a/message/message1_1prime/transfer_request_test.go +++ b/message/message1_1prime/transfer_request_test.go @@ -40,14 +40,12 @@ func TestRequestMessageForVersion(t *testing.T) { require.Equal(t, selector, n) require.Equal(t, testutil.TestVoucherType, req.VoucherType()) - wrappedOut12 := out12.WrappedForTransport(datatransfer.LegacyTransportID) - require.Equal(t, &message1_1.WrappedTransferRequest1_1{ - TransferRequest1_1: request.(*message1_1.TransferRequest1_1), - TransportID: string(datatransfer.LegacyTransportID), - }, wrappedOut12) + wrappedOut12 := out12.WrappedForTransport(datatransfer.LegacyTransportID, datatransfer.LegacyTransportVersion) + require.Equal(t, datatransfer.LegacyTransportID, wrappedOut12.TransportID()) + require.Equal(t, datatransfer.LegacyTransportVersion, wrappedOut12.TransportVersion()) // random protocol should fail - _, err = request.MessageForVersion(datatransfer.MessageVersion{ + _, err = request.MessageForVersion(datatransfer.Version{ Major: rand.Uint64(), Minor: rand.Uint64(), Patch: rand.Uint64(), diff --git a/message/message1_1prime/transfer_response.go b/message/message1_1prime/transfer_response.go index 77cf36a1..3e3c41f5 100644 --- a/message/message1_1prime/transfer_response.go +++ b/message/message1_1prime/transfer_response.go @@ -87,7 +87,7 @@ func (trsp *TransferResponse1_1) EmptyVoucherResult() bool { return trsp.VoucherTypeIdentifier == datatransfer.EmptyTypeIdentifier } -func (trsp *TransferResponse1_1) MessageForVersion(version datatransfer.MessageVersion) (datatransfer.Message, error) { +func (trsp *TransferResponse1_1) MessageForVersion(version datatransfer.Version) (datatransfer.Message, error) { switch version { case datatransfer.DataTransfer1_2: return trsp, nil @@ -96,8 +96,16 @@ func (trsp *TransferResponse1_1) MessageForVersion(version datatransfer.MessageV } } -func (trsp *TransferResponse1_1) WrappedForTransport(transportID datatransfer.TransportID) datatransfer.Message { - return &WrappedTransferResponse1_1{trsp, string(transportID)} +func (trsp *TransferResponse1_1) Version() datatransfer.Version { + return datatransfer.DataTransfer1_2 +} + +func (trsp *TransferResponse1_1) WrappedForTransport(transportID datatransfer.TransportID, transportVersion datatransfer.Version) datatransfer.TransportedMessage { + return &WrappedTransferResponse1_1{ + TransferResponse1_1: trsp, + transportID: string(transportID), + transportVersion: transportVersion, + } } func (trsp *TransferResponse1_1) toIPLD() schema.TypedNode { msg := TransferMessage1_1{ @@ -121,12 +129,21 @@ func (trsp *TransferResponse1_1) ToNet(w io.Writer) error { // transport id type WrappedTransferResponse1_1 struct { *TransferResponse1_1 - TransportID string + transportID string + transportVersion datatransfer.Version +} + +func (trsp *WrappedTransferResponse1_1) TransportID() datatransfer.TransportID { + return datatransfer.TransportID(trsp.transportID) +} +func (trsp *WrappedTransferResponse1_1) TransportVersion() datatransfer.Version { + return trsp.transportVersion } func (trsp *WrappedTransferResponse1_1) toIPLD() schema.TypedNode { msg := WrappedTransferMessage1_1{ - TransportID: trsp.TransportID, + TransportID: trsp.transportID, + TransportVersion: trsp.transportVersion, Message: TransferMessage1_1{ IsRequest: false, Request: nil, diff --git a/message/message1_1prime/transfer_response_test.go b/message/message1_1prime/transfer_response_test.go index f29773bf..71fcf703 100644 --- a/message/message1_1prime/transfer_response_test.go +++ b/message/message1_1prime/transfer_response_test.go @@ -28,14 +28,12 @@ func TestResponseMessageForVersion(t *testing.T) { require.Equal(t, testutil.TestVoucherType, resp.VoucherResultType()) require.True(t, resp.IsValidationResult()) - wrappedOut := out.WrappedForTransport(datatransfer.LegacyTransportID) - require.Equal(t, &message1_1.WrappedTransferResponse1_1{ - TransferResponse1_1: response.(*message1_1.TransferResponse1_1), - TransportID: string(datatransfer.LegacyTransportID), - }, wrappedOut) + wrappedOut := out.WrappedForTransport(datatransfer.LegacyTransportID, datatransfer.LegacyTransportVersion) + require.Equal(t, datatransfer.LegacyTransportID, wrappedOut.TransportID()) + require.Equal(t, datatransfer.LegacyTransportVersion, wrappedOut.TransportVersion()) // random protocol should fail - _, err = response.MessageForVersion(datatransfer.MessageVersion{ + _, err = response.MessageForVersion(datatransfer.Version{ Major: rand.Uint64(), Minor: rand.Uint64(), Patch: rand.Uint64(), diff --git a/testutil/faketransport.go b/testutil/faketransport.go index f5843ee4..7e0d20c8 100644 --- a/testutil/faketransport.go +++ b/testutil/faketransport.go @@ -59,6 +59,11 @@ func (ft *FakeTransport) ID() datatransfer.TransportID { return "fake" } +// Versions indicates what versions of this transport are supported +func (ft *FakeTransport) Versions() []datatransfer.Version { + return []datatransfer.Version{{Major: 1, Minor: 1, Patch: 0}} +} + // Capabilities tells datatransfer what kinds of capabilities this transport supports func (ft *FakeTransport) Capabilities() datatransfer.TransportCapabilities { return datatransfer.TransportCapabilities{ diff --git a/transport.go b/transport.go index ecaed685..546c2574 100644 --- a/transport.go +++ b/transport.go @@ -13,6 +13,10 @@ type TransportID string // i.e. graphsync const LegacyTransportID TransportID = "graphsync" +// LegacyTransportVersion is the only transport version for the fil/data-transfer protocol -- +// i.e. graphsync 1.0.0 +var LegacyTransportVersion Version = Version{1, 0, 0} + // EventsHandler are semantic data transfer events that happen as a result of transport events type EventsHandler interface { // ChannelState queries for the current channel state @@ -104,6 +108,9 @@ type Transport interface { // ID is a unique identifier for this transport ID() TransportID + // Versions indicates what versions of this transport are supported + Versions() []Version + // Capabilities tells datatransfer what kinds of capabilities this transport supports Capabilities() TransportCapabilities // OpenChannel opens a channel on a given transport to move data back and forth. diff --git a/transport/graphsync/extension/gsextension.go b/transport/graphsync/extension/gsextension.go index f161ee12..28aeb667 100644 --- a/transport/graphsync/extension/gsextension.go +++ b/transport/graphsync/extension/gsextension.go @@ -20,7 +20,7 @@ const ( ) // ProtocolMap maps graphsync extensions to their libp2p protocols -var ProtocolMap = map[graphsync.ExtensionName]datatransfer.MessageVersion{ +var ProtocolMap = map[graphsync.ExtensionName]datatransfer.Version{ ExtensionIncomingRequest1_1: datatransfer.DataTransfer1_2, ExtensionOutgoingBlock1_1: datatransfer.DataTransfer1_2, ExtensionDataTransfer1_1: datatransfer.DataTransfer1_2, diff --git a/transport/graphsync/graphsync.go b/transport/graphsync/graphsync.go index 06dcd327..07fe307e 100644 --- a/transport/graphsync/graphsync.go +++ b/transport/graphsync/graphsync.go @@ -23,6 +23,7 @@ import ( var log = logging.Logger("dt_graphsync") var transportID datatransfer.TransportID = "graphsync" +var supportedVersions = []datatransfer.Version{{Major: 1, Minor: 0, Patch: 0}} // When restarting a data transfer, we cancel the existing graphsync request // before opening a new one. @@ -109,6 +110,10 @@ func (t *Transport) ID() datatransfer.TransportID { return transportID } +func (t *Transport) Versions() []datatransfer.Version { + return supportedVersions +} + func (t *Transport) Capabilities() datatransfer.TransportCapabilities { return datatransfer.TransportCapabilities{ Pausable: true, @@ -142,7 +147,7 @@ func (t *Transport) RestartChannel( req datatransfer.Request) error { log.Debugf("%s: re-establishing connection to %s", channelState.ChannelID(), channelState.OtherPeer()) start := time.Now() - err := t.dtNet.ConnectWithRetry(ctx, channelState.OtherPeer()) + err := t.dtNet.ConnectWithRetry(ctx, channelState.OtherPeer(), transportID) if err != nil { return xerrors.Errorf("%s: failed to reconnect to peer %s after %s: %w", channelState.ChannelID(), channelState.OtherPeer(), time.Since(start), err) @@ -292,7 +297,7 @@ func (t *Transport) SetEventHandler(events datatransfer.EventsHandler) error { t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterNetworkErrorListener(t.gsNetworkSendErrorListener)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterReceiverNetworkErrorListener(t.gsNetworkReceiveErrorListener)) - t.dtNet.SetDelegate(transportID, &receiver{t}) + t.dtNet.SetDelegate(transportID, supportedVersions, &receiver{t}) return nil } diff --git a/transport/helpers/network/interface.go b/transport/helpers/network/interface.go index 2ed90303..022a1a37 100644 --- a/transport/helpers/network/interface.go +++ b/transport/helpers/network/interface.go @@ -20,11 +20,19 @@ const ( ProtocolDataTransfer1_2 protocol.ID = "/datatransfer/1.2.0" ) +// ProtocolDescription describes how you are connected to a given +// peer on a given transport, if at all +type ProtocolDescription struct { + IsLegacy bool + MessageVersion datatransfer.Version + TransportVersion datatransfer.Version +} + // MessageVersion extracts the message version from the full protocol -func MessageVersion(protocol protocol.ID) (datatransfer.MessageVersion, error) { +func MessageVersion(protocol protocol.ID) (datatransfer.Version, error) { protocolParts := strings.Split(string(protocol), "/") if len(protocolParts) == 0 { - return datatransfer.MessageVersion{}, errors.New("no protocol to parse") + return datatransfer.Version{}, errors.New("no protocol to parse") } return datatransfer.MessageVersionFromString(protocolParts[len(protocolParts)-1]) } @@ -43,7 +51,7 @@ type DataTransferNetwork interface { // SetDelegate registers the Reciver to handle messages received from the // network. - SetDelegate(datatransfer.TransportID, Receiver) + SetDelegate(datatransfer.TransportID, []datatransfer.Version, Receiver) // ConnectTo establishes a connection to the given peer ConnectTo(context.Context, peer.ID) error @@ -51,14 +59,14 @@ type DataTransferNetwork interface { // ConnectWithRetry establishes a connection to the given peer, retrying if // necessary, and opens a stream on the data-transfer protocol to verify // the peer will accept messages on the protocol - ConnectWithRetry(ctx context.Context, p peer.ID) error + ConnectWithRetry(ctx context.Context, p peer.ID, transportID datatransfer.TransportID) error // ID returns the peer id of this libp2p host ID() peer.ID // Protocol returns the protocol version of the peer, connecting to // the peer if necessary - Protocol(context.Context, peer.ID) (protocol.ID, error) + Protocol(context.Context, peer.ID, datatransfer.TransportID) (ProtocolDescription, error) } // Receiver is an interface for receiving messages from the GraphSyncNetwork. diff --git a/transport/helpers/network/libp2p_impl.go b/transport/helpers/network/libp2p_impl.go index 412e9349..d4fe0edb 100644 --- a/transport/helpers/network/libp2p_impl.go +++ b/transport/helpers/network/libp2p_impl.go @@ -2,8 +2,10 @@ package network import ( "context" + "errors" "fmt" "io" + "strings" "time" logging "github.com/ipfs/go-log/v2" @@ -16,7 +18,6 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" - "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/message" @@ -47,31 +48,35 @@ var defaultDataTransferProtocols = []protocol.ID{ ProtocolFilDataTransfer1_2, } +func isLegacyProtocol(protocol protocol.ID) bool { + return protocol == ProtocolFilDataTransfer1_2 +} + // Option is an option for configuring the libp2p storage market network type Option func(*libp2pDataTransferNetwork) // DataTransferProtocols OVERWRITES the default libp2p protocols we use for data transfer with the given protocols. func DataTransferProtocols(protocols []protocol.ID) Option { - return func(impl *libp2pDataTransferNetwork) { - impl.setDataTransferProtocols(protocols) + return func(dtnet *libp2pDataTransferNetwork) { + dtnet.setDataTransferProtocols(protocols) } } // SendMessageParameters changes the default parameters around sending messages func SendMessageParameters(openStreamTimeout time.Duration, sendMessageTimeout time.Duration) Option { - return func(impl *libp2pDataTransferNetwork) { - impl.sendMessageTimeout = sendMessageTimeout - impl.openStreamTimeout = openStreamTimeout + return func(dtnet *libp2pDataTransferNetwork) { + dtnet.sendMessageTimeout = sendMessageTimeout + dtnet.openStreamTimeout = openStreamTimeout } } // RetryParameters changes the default parameters around connection reopening func RetryParameters(minDuration time.Duration, maxDuration time.Duration, attempts float64, backoffFactor float64) Option { - return func(impl *libp2pDataTransferNetwork) { - impl.maxStreamOpenAttempts = attempts - impl.minAttemptDuration = minDuration - impl.maxAttemptDuration = maxDuration - impl.backoffFactor = backoffFactor + return func(dtnet *libp2pDataTransferNetwork) { + dtnet.maxStreamOpenAttempts = attempts + dtnet.minAttemptDuration = minDuration + dtnet.maxAttemptDuration = maxDuration + dtnet.backoffFactor = backoffFactor } } @@ -86,7 +91,8 @@ func NewFromLibp2pHost(host host.Host, options ...Option) DataTransferNetwork { minAttemptDuration: defaultMinAttemptDuration, maxAttemptDuration: defaultMaxAttemptDuration, backoffFactor: defaultBackoffFactor, - receivers: make(map[datatransfer.TransportID]Receiver), + receivers: make(map[protocol.ID]receiverData), + transportProtocols: make(map[datatransfer.TransportID]transportProtocols), } dataTransferNetwork.setDataTransferProtocols(defaultDataTransferProtocols) @@ -97,44 +103,54 @@ func NewFromLibp2pHost(host host.Host, options ...Option) DataTransferNetwork { return &dataTransferNetwork } +type transportProtocols struct { + protocols []protocol.ID + protocolStrings []string +} + +type receiverData struct { + ProtocolDescription + transportID datatransfer.TransportID + receiver Receiver +} + // libp2pDataTransferNetwork transforms the libp2p host interface, which sends and receives // NetMessage objects, into the data transfer network interface. type libp2pDataTransferNetwork struct { host host.Host // inbound messages from the network are forwarded to the receiver - receivers map[datatransfer.TransportID]Receiver - + receivers map[protocol.ID]receiverData + transportProtocols map[datatransfer.TransportID]transportProtocols openStreamTimeout time.Duration sendMessageTimeout time.Duration maxStreamOpenAttempts float64 minAttemptDuration time.Duration maxAttemptDuration time.Duration dtProtocols []protocol.ID - dtProtocolStrings []string backoffFactor float64 } -func (impl *libp2pDataTransferNetwork) openStream(ctx context.Context, id peer.ID, protocols ...protocol.ID) (network.Stream, error) { +func (dtnet *libp2pDataTransferNetwork) openStream(ctx context.Context, id peer.ID, protocols ...protocol.ID) (network.Stream, error) { b := &backoff.Backoff{ - Min: impl.minAttemptDuration, - Max: impl.maxAttemptDuration, - Factor: impl.backoffFactor, + Min: dtnet.minAttemptDuration, + Max: dtnet.maxAttemptDuration, + Factor: dtnet.backoffFactor, Jitter: true, } start := time.Now() for { - tctx, cancel := context.WithTimeout(ctx, impl.openStreamTimeout) + tctx, cancel := context.WithTimeout(ctx, dtnet.openStreamTimeout) defer cancel() // will use the first among the given protocols that the remote peer supports at := time.Now() - s, err := impl.host.NewStream(tctx, id, protocols...) + s, err := dtnet.host.NewStream(tctx, id, protocols...) if err == nil { nAttempts := b.Attempt() + 1 if b.Attempt() > 0 { log.Debugf("opened stream to %s on attempt %g of %g after %s", - id, nAttempts, impl.maxStreamOpenAttempts, time.Since(start)) + id, nAttempts, dtnet.maxStreamOpenAttempts, time.Since(start)) } return s, err @@ -142,13 +158,13 @@ func (impl *libp2pDataTransferNetwork) openStream(ctx context.Context, id peer.I // b.Attempt() starts from zero nAttempts := b.Attempt() + 1 - if nAttempts >= impl.maxStreamOpenAttempts { - return nil, xerrors.Errorf("exhausted %g attempts but failed to open stream to %s, err: %w", impl.maxStreamOpenAttempts, id, err) + if nAttempts >= dtnet.maxStreamOpenAttempts { + return nil, fmt.Errorf("exhausted %g attempts but failed to open stream to %s, err: %w", dtnet.maxStreamOpenAttempts, id, err) } d := b.Duration() log.Warnf("failed to open stream to %s on attempt %g of %g after %s, waiting %s to try again, err: %s", - id, nAttempts, impl.maxStreamOpenAttempts, time.Since(at), d, err) + id, nAttempts, dtnet.maxStreamOpenAttempts, time.Since(at), d, err) select { case <-ctx.Done(): @@ -176,30 +192,36 @@ func (dtnet *libp2pDataTransferNetwork) SendMessage( )) defer span.End() - s, err := dtnet.openStream(ctx, p, dtnet.dtProtocols...) + + transportProtocols, ok := dtnet.transportProtocols[transportID] + if !ok { + return datatransfer.ErrUnsupported + } + s, err := dtnet.openStream(ctx, p, transportProtocols.protocols...) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) return err } - messageVersion, err := MessageVersion(s.Protocol()) - if err != nil { - err = xerrors.Errorf("failed to determine message version for protocol: %w", err) + receiverData, ok := dtnet.receivers[s.Protocol()] + if !ok { + // this shouldn't happen, but let's be careful just in case to avoid a panic + err := errors.New("no receiver set") span.RecordError(err) span.SetStatus(codes.Error, err.Error()) return err } - outgoing, err = outgoing.MessageForVersion(messageVersion) + outgoing, err = outgoing.MessageForVersion(receiverData.MessageVersion) if err != nil { - err = xerrors.Errorf("failed to convert message for protocol: %w", err) + err = fmt.Errorf("failed to convert message for protocol: %w", err) span.RecordError(err) span.SetStatus(codes.Error, err.Error()) return err } - if err = dtnet.msgToStream(ctx, s, transportID, outgoing); err != nil { + if err = dtnet.msgToStream(ctx, s, outgoing, receiverData); err != nil { if err2 := s.Reset(); err2 != nil { log.Error(err) span.RecordError(err2) @@ -214,9 +236,55 @@ func (dtnet *libp2pDataTransferNetwork) SendMessage( return s.Close() } -func (dtnet *libp2pDataTransferNetwork) SetDelegate(transportID datatransfer.TransportID, r Receiver) { - dtnet.receivers[transportID] = r - for _, p := range dtnet.dtProtocols { +func (dtnet *libp2pDataTransferNetwork) SetDelegate(transportID datatransfer.TransportID, versions []datatransfer.Version, r Receiver) { + transportProtocols := transportProtocols{} + for _, dtProtocol := range dtnet.dtProtocols { + messageVersion, _ := MessageVersion(dtProtocol) + if isLegacyProtocol(dtProtocol) { + if transportID == datatransfer.LegacyTransportID { + supportsLegacyVersion := false + for _, version := range versions { + if version == datatransfer.LegacyTransportVersion { + supportsLegacyVersion = true + break + } + } + if !supportsLegacyVersion { + continue + } + dtnet.receivers[dtProtocol] = receiverData{ + ProtocolDescription: ProtocolDescription{ + IsLegacy: true, + TransportVersion: datatransfer.LegacyTransportVersion, + MessageVersion: messageVersion, + }, + transportID: transportID, + receiver: r, + } + transportProtocols.protocols = append(transportProtocols.protocols, dtProtocol) + transportProtocols.protocolStrings = append(transportProtocols.protocolStrings, string(dtProtocol)) + } + } else { + for _, version := range versions { + joinedProtocol := strings.Join([]string{string(dtProtocol), string(transportID), version.String()}, "/") + dtnet.receivers[protocol.ID(joinedProtocol)] = receiverData{ + ProtocolDescription: ProtocolDescription{ + IsLegacy: false, + TransportVersion: version, + MessageVersion: messageVersion, + }, + transportID: transportID, + receiver: r, + } + transportProtocols.protocols = append(transportProtocols.protocols, protocol.ID(joinedProtocol)) + transportProtocols.protocolStrings = append(transportProtocols.protocolStrings, joinedProtocol) + } + } + } + + dtnet.transportProtocols[transportID] = transportProtocols + + for _, p := range transportProtocols.protocols { dtnet.host.SetStreamHandler(p, dtnet.handleNewStream) } } @@ -228,10 +296,14 @@ func (dtnet *libp2pDataTransferNetwork) ConnectTo(ctx context.Context, p peer.ID // ConnectWithRetry establishes a connection to the given peer, retrying if // necessary, and opens a stream on the data-transfer protocol to verify // the peer will accept messages on the protocol -func (dtnet *libp2pDataTransferNetwork) ConnectWithRetry(ctx context.Context, p peer.ID) error { +func (dtnet *libp2pDataTransferNetwork) ConnectWithRetry(ctx context.Context, p peer.ID, transportID datatransfer.TransportID) error { + transportProtocols, ok := dtnet.transportProtocols[transportID] + if !ok { + return datatransfer.ErrUnsupported + } // Open a stream over the data-transfer protocol, to make sure that the // peer is listening on the protocol - s, err := dtnet.openStream(ctx, p, dtnet.dtProtocols...) + s, err := dtnet.openStream(ctx, p, transportProtocols.protocols...) if err != nil { return err } @@ -250,21 +322,20 @@ func (dtnet *libp2pDataTransferNetwork) handleNewStream(s network.Stream) { return } + receiverData, ok := dtnet.receivers[s.Protocol()] + if !ok { + s.Reset() // nolint: errcheck,gosec + return + } p := s.Conn().RemotePeer() + // if we have no transport handler, reset the stream for { - var transportID datatransfer.TransportID var received datatransfer.Message var err error - switch s.Protocol() { - case ProtocolFilDataTransfer1_2: - if dtnet.receivers[datatransfer.LegacyTransportID] == nil { - s.Reset() // nolint: errcheck,gosec - return - } - transportID = datatransfer.LegacyTransportID + if receiverData.IsLegacy { received, err = message.FromNet(s) - case ProtocolDataTransfer1_2: - transportID, received, err = message.FromNetWrapped(s) + } else { + received, err = message.FromNetWrapped(s) } if err != nil { @@ -275,12 +346,6 @@ func (dtnet *libp2pDataTransferNetwork) handleNewStream(s network.Stream) { return } - // if we have no transport handler, reset the stream - if dtnet.receivers[transportID] == nil { - s.Reset() // nolint: errcheck,gosec - return - } - ctx := context.Background() log.Debugf("net handleNewStream from %s", p) @@ -288,15 +353,15 @@ func (dtnet *libp2pDataTransferNetwork) handleNewStream(s network.Stream) { receivedRequest, ok := received.(datatransfer.Request) if ok { if receivedRequest.IsRestartExistingChannelRequest() { - dtnet.receivers[transportID].ReceiveRestartExistingChannelRequest(ctx, p, receivedRequest) + receiverData.receiver.ReceiveRestartExistingChannelRequest(ctx, p, receivedRequest) } else { - dtnet.receivers[transportID].ReceiveRequest(ctx, p, receivedRequest) + receiverData.receiver.ReceiveRequest(ctx, p, receivedRequest) } } } else { receivedResponse, ok := received.(datatransfer.Response) if ok { - dtnet.receivers[transportID].ReceiveResponse(ctx, p, receivedResponse) + receiverData.receiver.ReceiveResponse(ctx, p, receivedResponse) } } } @@ -314,7 +379,7 @@ func (dtnet *libp2pDataTransferNetwork) Unprotect(id peer.ID, tag string) bool { return dtnet.host.ConnManager().Unprotect(id, tag) } -func (dtnet *libp2pDataTransferNetwork) msgToStream(ctx context.Context, s network.Stream, transportID datatransfer.TransportID, msg datatransfer.Message) error { +func (dtnet *libp2pDataTransferNetwork) msgToStream(ctx context.Context, s network.Stream, msg datatransfer.Message, receiverData receiverData) error { if msg.IsRequest() { log.Debugf("Outgoing request message for transfer ID: %d", msg.TransferID()) } @@ -332,15 +397,8 @@ func (dtnet *libp2pDataTransferNetwork) msgToStream(ctx context.Context, s netwo } }() - switch s.Protocol() { - case ProtocolFilDataTransfer1_2: - if transportID != datatransfer.LegacyTransportID { - return fmt.Errorf("cannot send messages for transports other than graphsync on legacy protocol") - } - case ProtocolDataTransfer1_2: - msg = msg.WrappedForTransport(transportID) - default: - return fmt.Errorf("unrecognized protocol on remote: %s", s.Protocol()) + if !receiverData.IsLegacy { + msg = msg.WrappedForTransport(receiverData.transportID, receiverData.TransportVersion) } if err := msg.ToNet(s); err != nil { @@ -350,35 +408,41 @@ func (dtnet *libp2pDataTransferNetwork) msgToStream(ctx context.Context, s netwo return nil } -func (impl *libp2pDataTransferNetwork) Protocol(ctx context.Context, id peer.ID) (protocol.ID, error) { +func (dtnet *libp2pDataTransferNetwork) Protocol(ctx context.Context, id peer.ID, transportID datatransfer.TransportID) (ProtocolDescription, error) { + transportProtocols, ok := dtnet.transportProtocols[transportID] + if !ok { + return ProtocolDescription{}, datatransfer.ErrUnsupported + } + // Check the cache for the peer's protocol version - firstProto, err := impl.host.Peerstore().FirstSupportedProtocol(id, impl.dtProtocolStrings...) + firstProto, err := dtnet.host.Peerstore().FirstSupportedProtocol(id, transportProtocols.protocolStrings...) if err != nil { - return "", err + return ProtocolDescription{}, err } if firstProto != "" { - return protocol.ID(firstProto), nil + receiverData, ok := dtnet.receivers[protocol.ID(firstProto)] + if !ok { + return ProtocolDescription{}, err + } + return receiverData.ProtocolDescription, nil } // The peer's protocol version is not in the cache, so connect to the peer. // Note that when the stream is opened, the peer's protocol will be added // to the cache. - s, err := impl.openStream(ctx, id, impl.dtProtocols...) + s, err := dtnet.openStream(ctx, id, dtnet.dtProtocols...) if err != nil { - return "", err + return ProtocolDescription{}, err } _ = s.Close() - - return s.Protocol(), nil + receiverData, ok := dtnet.receivers[s.Protocol()] + if !ok { + return ProtocolDescription{}, err + } + return receiverData.ProtocolDescription, nil } -func (impl *libp2pDataTransferNetwork) setDataTransferProtocols(protocols []protocol.ID) { - impl.dtProtocols = append([]protocol.ID{}, protocols...) - - // Keep a string version of the protocols for performance reasons - impl.dtProtocolStrings = make([]string, 0, len(impl.dtProtocols)) - for _, proto := range impl.dtProtocols { - impl.dtProtocolStrings = append(impl.dtProtocolStrings, string(proto)) - } +func (dtnet *libp2pDataTransferNetwork) setDataTransferProtocols(protocols []protocol.ID) { + dtnet.dtProtocols = append([]protocol.ID{}, protocols...) } diff --git a/transport/helpers/network/libp2p_impl_test.go b/transport/helpers/network/libp2p_impl_test.go index ceffc8af..76b59ef6 100644 --- a/transport/helpers/network/libp2p_impl_test.go +++ b/transport/helpers/network/libp2p_impl_test.go @@ -91,8 +91,8 @@ func TestMessageSendAndReceive(t *testing.T) { messageReceived: make(chan struct{}), connectedPeers: make(chan peer.ID, 2), } - dtnet1.SetDelegate("graphsync", r) - dtnet2.SetDelegate("graphsync", r) + dtnet1.SetDelegate("graphsync", []datatransfer.Version{datatransfer.LegacyTransportVersion}, r) + dtnet2.SetDelegate("graphsync", []datatransfer.Version{datatransfer.LegacyTransportVersion}, r) err = dtnet1.ConnectTo(ctx, host2.ID()) require.NoError(t, err) @@ -263,8 +263,8 @@ func TestSendMessageRetry(t *testing.T) { messageReceived: make(chan struct{}), connectedPeers: make(chan peer.ID, 2), } - dtnet1.SetDelegate("graphsync", r) - dtnet2.SetDelegate("graphsync", r) + dtnet1.SetDelegate("graphsync", []datatransfer.Version{datatransfer.LegacyTransportVersion}, r) + dtnet2.SetDelegate("graphsync", []datatransfer.Version{datatransfer.LegacyTransportVersion}, r) err = dtnet1.ConnectTo(ctx, host2.ID()) require.NoError(t, err) From 2f7bb84c0e07da20c8a3fdd4484e8ea199c8f3a4 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 16 Jun 2022 00:35:34 -0700 Subject: [PATCH 11/18] feat(channels): update channel interfaces for new events --- channels/caches.go | 133 --- channels/channel_state.go | 24 +- channels/channels.go | 131 +-- channels/channels_fsm.go | 19 +- channels/channels_test.go | 72 +- channels/internal/internalchannel.go | 6 +- channels/internal/internalchannel_cbor_gen.go | 144 +-- channels/internal/migrations/migrations.go | 99 +- .../migrations/migrations_cbor_gen.go | 876 ++++++++++++++++++ go.mod | 4 +- go.sum | 2 + testutil/fakegraphsync.go | 65 +- testutil/faketransport.go | 40 +- testutil/mockchannelstate.go | 52 +- transport.go | 164 ++-- types.go | 18 +- 16 files changed, 1273 insertions(+), 576 deletions(-) delete mode 100644 channels/caches.go create mode 100644 channels/internal/migrations/migrations_cbor_gen.go diff --git a/channels/caches.go b/channels/caches.go deleted file mode 100644 index 2e2d75a9..00000000 --- a/channels/caches.go +++ /dev/null @@ -1,133 +0,0 @@ -package channels - -import ( - "sync" - "sync/atomic" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" -) - -type readIndexFn func(datatransfer.ChannelID) (int64, error) - -type cacheKey struct { - evt datatransfer.EventCode - chid datatransfer.ChannelID -} - -type blockIndexCache struct { - lk sync.RWMutex - values map[cacheKey]*int64 -} - -func newBlockIndexCache() *blockIndexCache { - return &blockIndexCache{ - values: make(map[cacheKey]*int64), - } -} - -func (bic *blockIndexCache) getValue(evt datatransfer.EventCode, chid datatransfer.ChannelID, readFromOriginal readIndexFn) (*int64, error) { - idxKey := cacheKey{evt, chid} - bic.lk.RLock() - value := bic.values[idxKey] - bic.lk.RUnlock() - if value != nil { - return value, nil - } - bic.lk.Lock() - defer bic.lk.Unlock() - value = bic.values[idxKey] - if value != nil { - return value, nil - } - newValue, err := readFromOriginal(chid) - if err != nil { - return nil, err - } - bic.values[idxKey] = &newValue - return &newValue, nil -} - -func (bic *blockIndexCache) updateIfGreater(evt datatransfer.EventCode, chid datatransfer.ChannelID, newIndex int64, readFromOriginal readIndexFn) (bool, error) { - value, err := bic.getValue(evt, chid, readFromOriginal) - if err != nil { - return false, err - } - for { - currentIndex := atomic.LoadInt64(value) - if newIndex <= currentIndex { - return false, nil - } - if atomic.CompareAndSwapInt64(value, currentIndex, newIndex) { - return true, nil - } - } -} - -type progressState struct { - dataLimit uint64 - progress *uint64 -} - -type readProgressFn func(datatransfer.ChannelID) (dataLimit uint64, progress uint64, err error) - -type progressCache struct { - lk sync.RWMutex - values map[datatransfer.ChannelID]progressState -} - -func newProgressCache() *progressCache { - return &progressCache{ - values: make(map[datatransfer.ChannelID]progressState), - } -} - -func (pc *progressCache) getValue(chid datatransfer.ChannelID, readProgress readProgressFn) (progressState, error) { - pc.lk.RLock() - value, ok := pc.values[chid] - pc.lk.RUnlock() - if ok { - return value, nil - } - pc.lk.Lock() - defer pc.lk.Unlock() - value, ok = pc.values[chid] - if ok { - return value, nil - } - dataLimit, progress, err := readProgress(chid) - if err != nil { - return progressState{}, err - } - newValue := progressState{ - dataLimit: dataLimit, - progress: &progress, - } - pc.values[chid] = newValue - return newValue, nil -} - -func (pc *progressCache) progress(chid datatransfer.ChannelID, additionalData uint64, readFromOriginal readProgressFn) (bool, error) { - state, err := pc.getValue(chid, readFromOriginal) - if err != nil { - return false, err - } - total := atomic.AddUint64(state.progress, additionalData) - return state.dataLimit != 0 && total >= state.dataLimit, nil -} - -func (pc *progressCache) setDataLimit(chid datatransfer.ChannelID, newLimit uint64) { - pc.lk.RLock() - value, ok := pc.values[chid] - pc.lk.RUnlock() - if !ok { - return - } - pc.lk.Lock() - defer pc.lk.Unlock() - value, ok = pc.values[chid] - if !ok { - return - } - value.dataLimit = newLimit - pc.values[chid] = value -} diff --git a/channels/channel_state.go b/channels/channel_state.go index fec337ba..93ce3388 100644 --- a/channels/channel_state.go +++ b/channels/channel_state.go @@ -50,22 +50,22 @@ func (c channelState) Voucher() datatransfer.TypedVoucher { return datatransfer.TypedVoucher{Voucher: ev.Voucher.Node, Type: ev.Type} } -// ReceivedCidsTotal returns the number of (non-unique) cids received so far -// on the channel - note that a block can exist in more than one place in the DAG -func (c channelState) ReceivedCidsTotal() int64 { - return c.ic.ReceivedBlocksTotal +// ReceivedIndex returns the index, a transport specific identifier for "where" +// we are in receiving data for a transfer +func (c channelState) ReceivedIndex() datamodel.Node { + return c.ic.ReceivedIndex.Node } -// QueuedCidsTotal returns the number of (non-unique) cids queued so far -// on the channel - note that a block can exist in more than one place in the DAG -func (c channelState) QueuedCidsTotal() int64 { - return c.ic.QueuedBlocksTotal +// QueuedIndex returns the index, a transport specific identifier for "where" +// we are in queing data for a transfer +func (c channelState) QueuedIndex() datamodel.Node { + return c.ic.QueuedIndex.Node } -// SentCidsTotal returns the number of (non-unique) cids sent so far -// on the channel - note that a block can exist in more than one place in the DAG -func (c channelState) SentCidsTotal() int64 { - return c.ic.SentBlocksTotal +// SentIndex returns the index, a transport specific identifier for "where" +// we are in sending data for a transfer +func (c channelState) SentIndex() datamodel.Node { + return c.ic.SentIndex.Node } // Sender returns the peer id for the node that is sending data diff --git a/channels/channels.go b/channels/channels.go index 1d289aa2..43176c58 100644 --- a/channels/channels.go +++ b/channels/channels.go @@ -29,8 +29,6 @@ var ErrWrongType = errors.New("Cannot change type of implementation specific dat // Channels is a thread safe list of channels type Channels struct { notifier Notifier - blockIndexCache *blockIndexCache - progressCache *progressCache stateMachines fsm.Group migrateStateMachines func(context.Context) error } @@ -48,8 +46,6 @@ func New(ds datastore.Batching, selfPeer peer.ID) (*Channels, error) { c := &Channels{notifier: notifier} - c.blockIndexCache = newBlockIndexCache() - c.progressCache = newProgressCache() channelMigrations, err := migrations.GetChannelStateMigrations(selfPeer) if err != nil { return nil, err @@ -173,63 +169,29 @@ func (c *Channels) Restart(chid datatransfer.ChannelID) error { return c.send(chid, datatransfer.Restart) } +// CompleteCleanupOnRestart tells a channel to restart func (c *Channels) CompleteCleanupOnRestart(chid datatransfer.ChannelID) error { return c.send(chid, datatransfer.CompleteCleanupOnRestart) } -func (c *Channels) getQueuedIndex(chid datatransfer.ChannelID) (int64, error) { - chst, err := c.GetByID(context.TODO(), chid) - if err != nil { - return 0, err - } - return chst.QueuedCidsTotal(), nil -} - -func (c *Channels) getReceivedIndex(chid datatransfer.ChannelID) (int64, error) { - chst, err := c.GetByID(context.TODO(), chid) - if err != nil { - return 0, err - } - return chst.ReceivedCidsTotal(), nil -} - -func (c *Channels) getSentIndex(chid datatransfer.ChannelID) (int64, error) { - chst, err := c.GetByID(context.TODO(), chid) - if err != nil { - return 0, err - } - return chst.SentCidsTotal(), nil +// DataSent records data being sent +func (c *Channels) DataSent(chid datatransfer.ChannelID, delta uint64, index datamodel.Node) error { + return c.fireProgressEvent(chid, datatransfer.DataSent, datatransfer.DataSentProgress, delta, index) } -func (c *Channels) getQueuedProgress(chid datatransfer.ChannelID) (uint64, uint64, error) { - chst, err := c.GetByID(context.TODO(), chid) - if err != nil { - return 0, 0, err - } - dataLimit := chst.DataLimit() - return dataLimit, chst.Queued(), nil -} - -func (c *Channels) getReceivedProgress(chid datatransfer.ChannelID) (uint64, uint64, error) { - chst, err := c.GetByID(context.TODO(), chid) - if err != nil { - return 0, 0, err - } - dataLimit := chst.DataLimit() - return dataLimit, chst.Received(), nil -} - -func (c *Channels) DataSent(chid datatransfer.ChannelID, k cid.Cid, delta uint64, index int64, unique bool) error { - return c.fireProgressEvent(chid, datatransfer.DataSent, datatransfer.DataSentProgress, delta, index, unique, c.getSentIndex, nil) +// DataQueued records data being queued +func (c *Channels) DataQueued(chid datatransfer.ChannelID, delta uint64, index datamodel.Node) error { + return c.fireProgressEvent(chid, datatransfer.DataQueued, datatransfer.DataQueuedProgress, delta, index) } -func (c *Channels) DataQueued(chid datatransfer.ChannelID, k cid.Cid, delta uint64, index int64, unique bool) error { - return c.fireProgressEvent(chid, datatransfer.DataQueued, datatransfer.DataQueuedProgress, delta, index, unique, c.getQueuedIndex, c.getQueuedProgress) +// DataReceived records data being received +func (c *Channels) DataReceived(chid datatransfer.ChannelID, delta uint64, index datamodel.Node) error { + return c.fireProgressEvent(chid, datatransfer.DataReceived, datatransfer.DataReceivedProgress, delta, index) } -// Returns true if this is the first time the block has been received -func (c *Channels) DataReceived(chid datatransfer.ChannelID, k cid.Cid, delta uint64, index int64, unique bool) error { - return c.fireProgressEvent(chid, datatransfer.DataReceived, datatransfer.DataReceivedProgress, delta, index, unique, c.getReceivedIndex, c.getReceivedProgress) +// DataLimitExceeded records a data limit exceeded event +func (c *Channels) DataLimitExceeded(chid datatransfer.ChannelID) error { + return c.send(chid, datatransfer.DataLimitExceeded) } // PauseInitiator pauses the initator of this channel @@ -331,7 +293,6 @@ func (c *Channels) ReceiveDataError(chid datatransfer.ChannelID, err error) erro // SetDataLimit means a data limit has been set on this channel func (c *Channels) SetDataLimit(chid datatransfer.ChannelID, dataLimit uint64) error { - c.progressCache.setDataLimit(chid, dataLimit) return c.send(chid, datatransfer.SetDataLimit, dataLimit) } @@ -347,75 +308,19 @@ func (c *Channels) HasChannel(chid datatransfer.ChannelID) (bool, error) { // fireProgressEvent fires // - an event for queuing / sending / receiving blocks -// - a corresponding "progress" event if the block has not been seen before -// - a DataLimitExceeded event if the progress goes past the data limit -// For example, if a block is being sent for the first time, the method will -// fire both DataSent AND DataSentProgress. -// If a block is resent, the method will fire DataSent but not DataSentProgress. -// If a block is sent for the first time, and more data has been sent than the data limit, -// the method will fire DataSent AND DataProgress AND DataLimitExceeded AND it will return -// datatransfer.ErrPause as the error -func (c *Channels) fireProgressEvent(chid datatransfer.ChannelID, evt datatransfer.EventCode, progressEvt datatransfer.EventCode, delta uint64, index int64, unique bool, readFromOriginal readIndexFn, readProgress readProgressFn) error { +// - a corresponding "progress" event +func (c *Channels) fireProgressEvent(chid datatransfer.ChannelID, evt datatransfer.EventCode, progressEvt datatransfer.EventCode, delta uint64, index datamodel.Node) error { if err := c.checkChannelExists(chid, evt); err != nil { return err } - pause, progress, err := c.checkEvents(chid, evt, delta, index, unique, readFromOriginal, readProgress) - - if err != nil { + // Fire the progress event + if err := c.stateMachines.Send(chid, progressEvt, delta); err != nil { return err } - // Fire the progress event if there is progress - if progress { - if err := c.stateMachines.Send(chid, progressEvt, delta); err != nil { - return err - } - } - // Fire the regular event - if err := c.stateMachines.Send(chid, evt, index); err != nil { - return err - } - - // fire the pause event if we past our data limit - if pause { - // pause. Data limits only exist on the responder, so we always pause the responder - if err := c.stateMachines.Send(chid, datatransfer.DataLimitExceeded); err != nil { - return err - } - // return a pause error so the transfer knows to pause - return datatransfer.ErrPause - } - return nil -} - -func (c *Channels) checkEvents(chid datatransfer.ChannelID, evt datatransfer.EventCode, delta uint64, index int64, unique bool, readFromOriginal readIndexFn, readProgress readProgressFn) (pause bool, progress bool, err error) { - - // if this is not a unique block, no data progress is made, return - if !unique { - return - } - - // check if data progress is made - progress, err = c.blockIndexCache.updateIfGreater(evt, chid, index, readFromOriginal) - if err != nil { - return false, false, err - } - - // if no data progress, return - if !progress { - return - } - - // if we don't check data limits on this function, return - if readProgress == nil { - return - } - - // check if we're past our data limit - pause, err = c.progressCache.progress(chid, delta, readProgress) - return + return c.stateMachines.Send(chid, evt, index) } func (c *Channels) send(chid datatransfer.ChannelID, code datatransfer.EventCode, args ...interface{}) error { diff --git a/channels/channels_fsm.go b/channels/channels_fsm.go index 4a095054..ddbc5418 100644 --- a/channels/channels_fsm.go +++ b/channels/channels_fsm.go @@ -2,6 +2,7 @@ package channels import ( logging "github.com/ipfs/go-log/v2" + "github.com/ipld/go-ipld-prime/datamodel" "github.com/filecoin-project/go-statemachine/fsm" @@ -60,10 +61,8 @@ var ChannelEvents = fsm.Events{ }), fsm.Event(datatransfer.DataReceived).FromAny().ToNoChange(). - Action(func(chst *internal.ChannelState, rcvdBlocksTotal int64) error { - if rcvdBlocksTotal > chst.ReceivedBlocksTotal { - chst.ReceivedBlocksTotal = rcvdBlocksTotal - } + Action(func(chst *internal.ChannelState, receivedIndex datamodel.Node) error { + chst.ReceivedIndex = internal.CborGenCompatibleNode{Node: receivedIndex} chst.AddLog("") return nil }), @@ -77,10 +76,8 @@ var ChannelEvents = fsm.Events{ fsm.Event(datatransfer.DataSent). FromMany(transferringStates...).ToNoChange(). From(datatransfer.TransferFinished).ToNoChange(). - Action(func(chst *internal.ChannelState, sentBlocksTotal int64) error { - if sentBlocksTotal > chst.SentBlocksTotal { - chst.SentBlocksTotal = sentBlocksTotal - } + Action(func(chst *internal.ChannelState, sentIndex datamodel.Node) error { + chst.SentIndex = internal.CborGenCompatibleNode{Node: sentIndex} chst.AddLog("") return nil }), @@ -95,10 +92,8 @@ var ChannelEvents = fsm.Events{ fsm.Event(datatransfer.DataQueued). FromMany(transferringStates...).ToNoChange(). From(datatransfer.TransferFinished).ToNoChange(). - Action(func(chst *internal.ChannelState, queuedBlocksTotal int64) error { - if queuedBlocksTotal > chst.QueuedBlocksTotal { - chst.QueuedBlocksTotal = queuedBlocksTotal - } + Action(func(chst *internal.ChannelState, queuedIndex datamodel.Node) error { + chst.QueuedIndex = internal.CborGenCompatibleNode{Node: queuedIndex} chst.AddLog("") return nil }), diff --git a/channels/channels_test.go b/channels/channels_test.go index 23c16ef8..442b56c7 100644 --- a/channels/channels_test.go +++ b/channels/channels_test.go @@ -141,13 +141,13 @@ func TestChannels(t *testing.T) { require.Equal(t, datatransfer.TransferFinished, state.Status()) // send a data-sent event and ensure it's a no-op - err = channelList.DataSent(chid, cids[1], 1, 1, true) + err = channelList.DataSent(chid, 1, basicnode.NewInt(1)) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.DataSent) require.Equal(t, datatransfer.TransferFinished, state.Status()) // send a data-queued event and ensure it's a no-op. - err = channelList.DataQueued(chid, cids[1], 1, 1, true) + err = channelList.DataQueued(chid, 1, basicnode.NewInt(1)) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.DataQueued) require.Equal(t, datatransfer.TransferFinished, state.Status()) @@ -168,51 +168,32 @@ func TestChannels(t *testing.T) { require.Equal(t, uint64(0), state.Received()) require.Equal(t, uint64(0), state.Sent()) - err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[0], 50, 1, true) + err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, 50, basicnode.NewInt(1)) require.NoError(t, err) _ = checkEvent(ctx, t, received, datatransfer.DataReceivedProgress) state = checkEvent(ctx, t, received, datatransfer.DataReceived) require.Equal(t, uint64(50), state.Received()) require.Equal(t, uint64(0), state.Sent()) - err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 100, 1, true) + err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, 100, basicnode.NewInt(1)) require.NoError(t, err) _ = checkEvent(ctx, t, received, datatransfer.DataSentProgress) state = checkEvent(ctx, t, received, datatransfer.DataSent) require.Equal(t, uint64(50), state.Received()) require.Equal(t, uint64(100), state.Sent()) - // send block again has no effect - err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 100, 1, true) - require.NoError(t, err) - state = checkEvent(ctx, t, received, datatransfer.DataSent) - require.Equal(t, uint64(50), state.Received()) - require.Equal(t, uint64(100), state.Sent()) - // errors if channel does not exist - err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[1], 200, 2, true) + err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, 200, basicnode.NewInt(2)) require.True(t, errors.Is(err, datatransfer.ErrChannelNotFound)) - err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[1], 200, 2, true) + err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, 200, basicnode.NewInt(2)) require.True(t, errors.Is(err, datatransfer.ErrChannelNotFound)) - err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 50, 2, true) + err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, 50, basicnode.NewInt(2)) require.NoError(t, err) _ = checkEvent(ctx, t, received, datatransfer.DataReceivedProgress) state = checkEvent(ctx, t, received, datatransfer.DataReceived) require.Equal(t, uint64(100), state.Received()) require.Equal(t, uint64(100), state.Sent()) - - err = channelList.DataSent(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[1], 25, 2, false) - require.NoError(t, err) - state = checkEvent(ctx, t, received, datatransfer.DataSent) - require.Equal(t, uint64(100), state.Received()) - require.Equal(t, uint64(100), state.Sent()) - - err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, cids[0], 50, 3, false) - require.NoError(t, err) - state = checkEvent(ctx, t, received, datatransfer.DataReceived) - require.Equal(t, uint64(100), state.Received()) - require.Equal(t, uint64(100), state.Sent()) }) t.Run("data limit", func(t *testing.T) { @@ -227,47 +208,34 @@ func TestChannels(t *testing.T) { require.NoError(t, err) state := checkEvent(ctx, t, received, datatransfer.Open) - err = channelList.DataQueued(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[0], 300, 1, true) - require.NoError(t, err) - _ = checkEvent(ctx, t, received, datatransfer.DataQueuedProgress) - state = checkEvent(ctx, t, received, datatransfer.DataQueued) - require.Equal(t, uint64(300), state.Queued()) - err = channelList.SetDataLimit(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, 400) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.SetDataLimit) require.Equal(t, state.DataLimit(), uint64(400)) - // send block again has no effect - err = channelList.DataQueued(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[0], 300, 1, true) + err = channelList.DataLimitExceeded(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}) require.NoError(t, err) - state = checkEvent(ctx, t, received, datatransfer.DataQueued) - require.Equal(t, uint64(300), state.Queued()) - - err = channelList.DataQueued(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[1], 200, 2, true) - require.EqualError(t, err, datatransfer.ErrPause.Error()) - _ = checkEvent(ctx, t, received, datatransfer.DataQueuedProgress) - _ = checkEvent(ctx, t, received, datatransfer.DataQueued) state = checkEvent(ctx, t, received, datatransfer.DataLimitExceeded) - require.Equal(t, uint64(500), state.Queued()) + require.Equal(t, datatransfer.ResponderPaused, state.Status()) err = channelList.SetDataLimit(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, 700) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.SetDataLimit) require.Equal(t, state.DataLimit(), uint64(700)) - err = channelList.DataQueued(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[2], 150, 3, true) - require.NoError(t, err) - _ = checkEvent(ctx, t, received, datatransfer.DataQueuedProgress) - state = checkEvent(ctx, t, received, datatransfer.DataQueued) - require.Equal(t, uint64(650), state.Queued()) + err = channelList.ResumeResponder(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}) + state = checkEvent(ctx, t, received, datatransfer.ResumeResponder) + require.Equal(t, datatransfer.Ongoing, state.Status()) + + err = channelList.PauseInitiator(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}) + state = checkEvent(ctx, t, received, datatransfer.PauseInitiator) + require.Equal(t, datatransfer.InitiatorPaused, state.Status()) - err = channelList.DataQueued(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, cids[3], 200, 4, true) - require.EqualError(t, err, datatransfer.ErrPause.Error()) - _ = checkEvent(ctx, t, received, datatransfer.DataQueuedProgress) - _ = checkEvent(ctx, t, received, datatransfer.DataQueued) + err = channelList.DataLimitExceeded(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}) + require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.DataLimitExceeded) - require.Equal(t, uint64(850), state.Queued()) + require.Equal(t, datatransfer.BothPaused, state.Status()) + }) t.Run("pause/resume", func(t *testing.T) { diff --git a/channels/internal/internalchannel.go b/channels/internal/internalchannel.go index 4a9fabd8..0b1e2bee 100644 --- a/channels/internal/internalchannel.go +++ b/channels/internal/internalchannel.go @@ -104,13 +104,13 @@ type ChannelState struct { VoucherResults []EncodedVoucherResult // Number of blocks that have been received, including blocks that are // present in more than one place in the DAG - ReceivedBlocksTotal int64 + ReceivedIndex CborGenCompatibleNode // Number of blocks that have been queued, including blocks that are // present in more than one place in the DAG - QueuedBlocksTotal int64 + QueuedIndex CborGenCompatibleNode // Number of blocks that have been sent, including blocks that are // present in more than one place in the DAG - SentBlocksTotal int64 + SentIndex CborGenCompatibleNode // DataLimit is the maximum data that can be transferred on this channel before // revalidation. 0 indicates no limit. DataLimit uint64 diff --git a/channels/internal/internalchannel_cbor_gen.go b/channels/internal/internalchannel_cbor_gen.go index 58f43c1b..3726f8ca 100644 --- a/channels/internal/internalchannel_cbor_gen.go +++ b/channels/internal/internalchannel_cbor_gen.go @@ -345,70 +345,52 @@ func (t *ChannelState) MarshalCBOR(w io.Writer) error { } } - // t.ReceivedBlocksTotal (int64) (int64) - if len("ReceivedBlocksTotal") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"ReceivedBlocksTotal\" was too long") + // t.ReceivedIndex (internal.CborGenCompatibleNode) (struct) + if len("ReceivedIndex") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"ReceivedIndex\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ReceivedBlocksTotal"))); err != nil { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ReceivedIndex"))); err != nil { return err } - if _, err := io.WriteString(w, string("ReceivedBlocksTotal")); err != nil { + if _, err := io.WriteString(w, string("ReceivedIndex")); err != nil { return err } - if t.ReceivedBlocksTotal >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.ReceivedBlocksTotal)); err != nil { - return err - } - } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.ReceivedBlocksTotal-1)); err != nil { - return err - } + if err := t.ReceivedIndex.MarshalCBOR(w); err != nil { + return err } - // t.QueuedBlocksTotal (int64) (int64) - if len("QueuedBlocksTotal") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"QueuedBlocksTotal\" was too long") + // t.QueuedIndex (internal.CborGenCompatibleNode) (struct) + if len("QueuedIndex") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"QueuedIndex\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("QueuedBlocksTotal"))); err != nil { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("QueuedIndex"))); err != nil { return err } - if _, err := io.WriteString(w, string("QueuedBlocksTotal")); err != nil { + if _, err := io.WriteString(w, string("QueuedIndex")); err != nil { return err } - if t.QueuedBlocksTotal >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.QueuedBlocksTotal)); err != nil { - return err - } - } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.QueuedBlocksTotal-1)); err != nil { - return err - } + if err := t.QueuedIndex.MarshalCBOR(w); err != nil { + return err } - // t.SentBlocksTotal (int64) (int64) - if len("SentBlocksTotal") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"SentBlocksTotal\" was too long") + // t.SentIndex (internal.CborGenCompatibleNode) (struct) + if len("SentIndex") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"SentIndex\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SentBlocksTotal"))); err != nil { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SentIndex"))); err != nil { return err } - if _, err := io.WriteString(w, string("SentBlocksTotal")); err != nil { + if _, err := io.WriteString(w, string("SentIndex")); err != nil { return err } - if t.SentBlocksTotal >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SentBlocksTotal)); err != nil { - return err - } - } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.SentBlocksTotal-1)); err != nil { - return err - } + if err := t.SentIndex.MarshalCBOR(w); err != nil { + return err } // t.DataLimit (uint64) (uint64) @@ -733,83 +715,35 @@ func (t *ChannelState) UnmarshalCBOR(r io.Reader) error { t.VoucherResults[i] = v } - // t.ReceivedBlocksTotal (int64) (int64) - case "ReceivedBlocksTotal": + // t.ReceivedIndex (internal.CborGenCompatibleNode) (struct) + case "ReceivedIndex": + { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) + + if err := t.ReceivedIndex.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.ReceivedIndex: %w", err) } - t.ReceivedBlocksTotal = int64(extraI) } - // t.QueuedBlocksTotal (int64) (int64) - case "QueuedBlocksTotal": + // t.QueuedIndex (internal.CborGenCompatibleNode) (struct) + case "QueuedIndex": + { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) + + if err := t.QueuedIndex.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.QueuedIndex: %w", err) } - t.QueuedBlocksTotal = int64(extraI) } - // t.SentBlocksTotal (int64) (int64) - case "SentBlocksTotal": + // t.SentIndex (internal.CborGenCompatibleNode) (struct) + case "SentIndex": + { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) + + if err := t.SentIndex.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.SentIndex: %w", err) } - t.SentBlocksTotal = int64(extraI) } // t.DataLimit (uint64) (uint64) case "DataLimit": diff --git a/channels/internal/migrations/migrations.go b/channels/internal/migrations/migrations.go index b6a1ed6a..a1f263f2 100644 --- a/channels/internal/migrations/migrations.go +++ b/channels/internal/migrations/migrations.go @@ -1,13 +1,110 @@ package migrations import ( + "github.com/ipfs/go-cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" peer "github.com/libp2p/go-libp2p-core/peer" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channels/internal" versioning "github.com/filecoin-project/go-ds-versioning/pkg" "github.com/filecoin-project/go-ds-versioning/pkg/versioned" ) +//go:generate cbor-gen-for --map-encoding ChannelStateV2 + +// ChannelStateV2 is the internal representation on disk for the channel fsm, version 2 +type ChannelStateV2 struct { + // PeerId of the manager peer + SelfPeer peer.ID + // an identifier for this channel shared by request and responder, set by requester through protocol + TransferID datatransfer.TransferID + // Initiator is the person who intiated this datatransfer request + Initiator peer.ID + // Responder is the person who is responding to this datatransfer request + Responder peer.ID + // base CID for the piece being transferred + BaseCid cid.Cid + // portion of Piece to return, specified by an IPLD selector + Selector internal.CborGenCompatibleNode + // the party that is sending the data (not who initiated the request) + Sender peer.ID + // the party that is receiving the data (not who initiated the request) + Recipient peer.ID + // expected amount of data to be transferred + TotalSize uint64 + // current status of this deal + Status datatransfer.Status + // total bytes read from this node and queued for sending (0 if receiver) + Queued uint64 + // total bytes sent from this node (0 if receiver) + Sent uint64 + // total bytes received by this node (0 if sender) + Received uint64 + // more informative status on a channel + Message string + Vouchers []internal.EncodedVoucher + VoucherResults []internal.EncodedVoucherResult + // Number of blocks that have been received, including blocks that are + // present in more than one place in the DAG + ReceivedBlocksTotal int64 + // Number of blocks that have been queued, including blocks that are + // present in more than one place in the DAG + QueuedBlocksTotal int64 + // Number of blocks that have been sent, including blocks that are + // present in more than one place in the DAG + SentBlocksTotal int64 + // DataLimit is the maximum data that can be transferred on this channel before + // revalidation. 0 indicates no limit. + DataLimit uint64 + // RequiresFinalization indicates at the end of the transfer, the channel should + // be left open for a final settlement + RequiresFinalization bool + // Stages traces the execution fo a data transfer. + // + // EXPERIMENTAL; subject to change. + Stages *datatransfer.ChannelStages +} + +func NoOpChannelState0To2(oldChannelState *ChannelStateV2) (*ChannelStateV2, error) { + return oldChannelState, nil +} + +func MigrateChannelState2To3(oldChannelState *ChannelStateV2) (*internal.ChannelState, error) { + receivedIndex := basicnode.NewInt(oldChannelState.ReceivedBlocksTotal) + sentIndex := basicnode.NewInt(oldChannelState.SentBlocksTotal) + queuedIndex := basicnode.NewInt(oldChannelState.QueuedBlocksTotal) + + return &internal.ChannelState{ + SelfPeer: oldChannelState.SelfPeer, + TransferID: oldChannelState.TransferID, + Initiator: oldChannelState.Initiator, + Responder: oldChannelState.Responder, + BaseCid: oldChannelState.BaseCid, + Selector: oldChannelState.Selector, + Sender: oldChannelState.Sender, + Recipient: oldChannelState.Recipient, + TotalSize: oldChannelState.TotalSize, + Status: oldChannelState.Status, + Queued: oldChannelState.Queued, + Sent: oldChannelState.Sent, + Received: oldChannelState.Received, + Message: oldChannelState.Message, + Vouchers: oldChannelState.Vouchers, + VoucherResults: oldChannelState.VoucherResults, + ReceivedIndex: internal.CborGenCompatibleNode{Node: receivedIndex}, + SentIndex: internal.CborGenCompatibleNode{Node: sentIndex}, + QueuedIndex: internal.CborGenCompatibleNode{Node: queuedIndex}, + DataLimit: oldChannelState.DataLimit, + RequiresFinalization: oldChannelState.RequiresFinalization, + Stages: oldChannelState.Stages, + }, nil +} + // GetChannelStateMigrations returns a migration list for the channel states func GetChannelStateMigrations(selfPeer peer.ID) (versioning.VersionedMigrationList, error) { - return versioned.BuilderList{}.Build() + return versioned.BuilderList{ + versioned.NewVersionedBuilder(NoOpChannelState0To2, "2"), + versioned.NewVersionedBuilder(MigrateChannelState2To3, "3").OldVersion("2"), + }.Build() } diff --git a/channels/internal/migrations/migrations_cbor_gen.go b/channels/internal/migrations/migrations_cbor_gen.go new file mode 100644 index 00000000..c4ca74fd --- /dev/null +++ b/channels/internal/migrations/migrations_cbor_gen.go @@ -0,0 +1,876 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package migrations + +import ( + "fmt" + "io" + "sort" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + internal "github.com/filecoin-project/go-data-transfer/v2/channels/internal" + cid "github.com/ipfs/go-cid" + peer "github.com/libp2p/go-libp2p-core/peer" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = sort.Sort + +func (t *ChannelStateV2) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{182}); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.SelfPeer (peer.ID) (string) + if len("SelfPeer") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"SelfPeer\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SelfPeer"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("SelfPeer")); err != nil { + return err + } + + if len(t.SelfPeer) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.SelfPeer was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.SelfPeer))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.SelfPeer)); err != nil { + return err + } + + // t.TransferID (datatransfer.TransferID) (uint64) + if len("TransferID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"TransferID\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("TransferID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("TransferID")); err != nil { + return err + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.TransferID)); err != nil { + return err + } + + // t.Initiator (peer.ID) (string) + if len("Initiator") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Initiator\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Initiator"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Initiator")); err != nil { + return err + } + + if len(t.Initiator) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Initiator was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Initiator))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Initiator)); err != nil { + return err + } + + // t.Responder (peer.ID) (string) + if len("Responder") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Responder\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Responder"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Responder")); err != nil { + return err + } + + if len(t.Responder) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Responder was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Responder))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Responder)); err != nil { + return err + } + + // t.BaseCid (cid.Cid) (struct) + if len("BaseCid") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"BaseCid\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("BaseCid"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("BaseCid")); err != nil { + return err + } + + if err := cbg.WriteCidBuf(scratch, w, t.BaseCid); err != nil { + return xerrors.Errorf("failed to write cid field t.BaseCid: %w", err) + } + + // t.Selector (internal.CborGenCompatibleNode) (struct) + if len("Selector") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Selector\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Selector"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Selector")); err != nil { + return err + } + + if err := t.Selector.MarshalCBOR(w); err != nil { + return err + } + + // t.Sender (peer.ID) (string) + if len("Sender") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Sender\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Sender"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Sender")); err != nil { + return err + } + + if len(t.Sender) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Sender was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Sender))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Sender)); err != nil { + return err + } + + // t.Recipient (peer.ID) (string) + if len("Recipient") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Recipient\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Recipient"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Recipient")); err != nil { + return err + } + + if len(t.Recipient) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Recipient was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Recipient))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Recipient)); err != nil { + return err + } + + // t.TotalSize (uint64) (uint64) + if len("TotalSize") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"TotalSize\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("TotalSize"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("TotalSize")); err != nil { + return err + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.TotalSize)); err != nil { + return err + } + + // t.Status (datatransfer.Status) (uint64) + if len("Status") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Status\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Status"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Status")); err != nil { + return err + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Status)); err != nil { + return err + } + + // t.Queued (uint64) (uint64) + if len("Queued") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Queued\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Queued"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Queued")); err != nil { + return err + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Queued)); err != nil { + return err + } + + // t.Sent (uint64) (uint64) + if len("Sent") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Sent\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Sent"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Sent")); err != nil { + return err + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Sent)); err != nil { + return err + } + + // t.Received (uint64) (uint64) + if len("Received") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Received\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Received"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Received")); err != nil { + return err + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Received)); err != nil { + return err + } + + // t.Message (string) (string) + if len("Message") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Message\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Message"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Message")); err != nil { + return err + } + + if len(t.Message) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Message was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Message))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Message)); err != nil { + return err + } + + // t.Vouchers ([]internal.EncodedVoucher) (slice) + if len("Vouchers") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Vouchers\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouchers"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Vouchers")); err != nil { + return err + } + + if len(t.Vouchers) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Vouchers was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Vouchers))); err != nil { + return err + } + for _, v := range t.Vouchers { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + + // t.VoucherResults ([]internal.EncodedVoucherResult) (slice) + if len("VoucherResults") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"VoucherResults\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("VoucherResults"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("VoucherResults")); err != nil { + return err + } + + if len(t.VoucherResults) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.VoucherResults was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.VoucherResults))); err != nil { + return err + } + for _, v := range t.VoucherResults { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + + // t.ReceivedBlocksTotal (int64) (int64) + if len("ReceivedBlocksTotal") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"ReceivedBlocksTotal\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ReceivedBlocksTotal"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("ReceivedBlocksTotal")); err != nil { + return err + } + + if t.ReceivedBlocksTotal >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.ReceivedBlocksTotal)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.ReceivedBlocksTotal-1)); err != nil { + return err + } + } + + // t.QueuedBlocksTotal (int64) (int64) + if len("QueuedBlocksTotal") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"QueuedBlocksTotal\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("QueuedBlocksTotal"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("QueuedBlocksTotal")); err != nil { + return err + } + + if t.QueuedBlocksTotal >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.QueuedBlocksTotal)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.QueuedBlocksTotal-1)); err != nil { + return err + } + } + + // t.SentBlocksTotal (int64) (int64) + if len("SentBlocksTotal") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"SentBlocksTotal\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SentBlocksTotal"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("SentBlocksTotal")); err != nil { + return err + } + + if t.SentBlocksTotal >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SentBlocksTotal)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.SentBlocksTotal-1)); err != nil { + return err + } + } + + // t.DataLimit (uint64) (uint64) + if len("DataLimit") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DataLimit\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DataLimit"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("DataLimit")); err != nil { + return err + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DataLimit)); err != nil { + return err + } + + // t.RequiresFinalization (bool) (bool) + if len("RequiresFinalization") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"RequiresFinalization\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("RequiresFinalization"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("RequiresFinalization")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.RequiresFinalization); err != nil { + return err + } + + // t.Stages (datatransfer.ChannelStages) (struct) + if len("Stages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Stages\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Stages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Stages")); err != nil { + return err + } + + if err := t.Stages.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *ChannelStateV2) UnmarshalCBOR(r io.Reader) error { + *t = ChannelStateV2{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ChannelStateV2: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.SelfPeer (peer.ID) (string) + case "SelfPeer": + + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } + + t.SelfPeer = peer.ID(sval) + } + // t.TransferID (datatransfer.TransferID) (uint64) + case "TransferID": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.TransferID = datatransfer.TransferID(extra) + + } + // t.Initiator (peer.ID) (string) + case "Initiator": + + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } + + t.Initiator = peer.ID(sval) + } + // t.Responder (peer.ID) (string) + case "Responder": + + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } + + t.Responder = peer.ID(sval) + } + // t.BaseCid (cid.Cid) (struct) + case "BaseCid": + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.BaseCid: %w", err) + } + + t.BaseCid = c + + } + // t.Selector (internal.CborGenCompatibleNode) (struct) + case "Selector": + + { + + if err := t.Selector.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.Selector: %w", err) + } + + } + // t.Sender (peer.ID) (string) + case "Sender": + + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } + + t.Sender = peer.ID(sval) + } + // t.Recipient (peer.ID) (string) + case "Recipient": + + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } + + t.Recipient = peer.ID(sval) + } + // t.TotalSize (uint64) (uint64) + case "TotalSize": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.TotalSize = uint64(extra) + + } + // t.Status (datatransfer.Status) (uint64) + case "Status": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Status = datatransfer.Status(extra) + + } + // t.Queued (uint64) (uint64) + case "Queued": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Queued = uint64(extra) + + } + // t.Sent (uint64) (uint64) + case "Sent": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Sent = uint64(extra) + + } + // t.Received (uint64) (uint64) + case "Received": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Received = uint64(extra) + + } + // t.Message (string) (string) + case "Message": + + { + sval, err := cbg.ReadStringBuf(br, scratch) + if err != nil { + return err + } + + t.Message = string(sval) + } + // t.Vouchers ([]internal.EncodedVoucher) (slice) + case "Vouchers": + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.Vouchers: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Vouchers = make([]internal.EncodedVoucher, extra) + } + + for i := 0; i < int(extra); i++ { + + var v internal.EncodedVoucher + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Vouchers[i] = v + } + + // t.VoucherResults ([]internal.EncodedVoucherResult) (slice) + case "VoucherResults": + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.VoucherResults: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.VoucherResults = make([]internal.EncodedVoucherResult, extra) + } + + for i := 0; i < int(extra); i++ { + + var v internal.EncodedVoucherResult + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.VoucherResults[i] = v + } + + // t.ReceivedBlocksTotal (int64) (int64) + case "ReceivedBlocksTotal": + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.ReceivedBlocksTotal = int64(extraI) + } + // t.QueuedBlocksTotal (int64) (int64) + case "QueuedBlocksTotal": + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.QueuedBlocksTotal = int64(extraI) + } + // t.SentBlocksTotal (int64) (int64) + case "SentBlocksTotal": + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.SentBlocksTotal = int64(extraI) + } + // t.DataLimit (uint64) (uint64) + case "DataLimit": + + { + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DataLimit = uint64(extra) + + } + // t.RequiresFinalization (bool) (bool) + case "RequiresFinalization": + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.RequiresFinalization = false + case 21: + t.RequiresFinalization = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + // t.Stages (datatransfer.ChannelStages) (struct) + case "Stages": + + { + + b, err := br.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := br.UnreadByte(); err != nil { + return err + } + t.Stages = new(datatransfer.ChannelStages) + if err := t.Stages.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.Stages pointer: %w", err) + } + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/go.mod b/go.mod index 972e065e..6fd46cee 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/filecoin-project/go-data-transfer/v2 -go 1.17 +go 1.18 require ( github.com/bep/debounce v1.2.0 @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-cid v0.2.0 github.com/ipfs/go-datastore v0.5.1 github.com/ipfs/go-ds-badger v0.3.0 - github.com/ipfs/go-graphsync v0.13.2-0.20220531040852-fa5a9f2d7a86 + github.com/ipfs/go-graphsync v0.13.2-0.20220613213258-f3c765731a53 github.com/ipfs/go-ipfs-blockstore v1.1.2 github.com/ipfs/go-ipfs-blocksutil v0.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 diff --git a/go.sum b/go.sum index d34f268a..845ab292 100644 --- a/go.sum +++ b/go.sum @@ -443,6 +443,8 @@ github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1 github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-graphsync v0.13.2-0.20220531040852-fa5a9f2d7a86 h1:PVLY+D9dz9SwQADbEaxLF5Kc+xOVP+SltDw3GvSdHmk= github.com/ipfs/go-graphsync v0.13.2-0.20220531040852-fa5a9f2d7a86/go.mod h1:y8e8G6CmZeL9Srvx1l15CtGiRdf3h5JdQuqPz/iYL0A= +github.com/ipfs/go-graphsync v0.13.2-0.20220613213258-f3c765731a53 h1:TxhigBtTsopDclUsakL9ET5sE6zQOhrkwR0R0AV9Lbk= +github.com/ipfs/go-graphsync v0.13.2-0.20220613213258-f3c765731a53/go.mod h1:y8e8G6CmZeL9Srvx1l15CtGiRdf3h5JdQuqPz/iYL0A= github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE= github.com/ipfs/go-ipfs-blockstore v1.1.2 h1:WCXoZcMYnvOTmlpX+RSSnhVN0uCmbWTeepTGX5lgiXw= github.com/ipfs/go-ipfs-blockstore v1.1.2/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY= diff --git a/testutil/fakegraphsync.go b/testutil/fakegraphsync.go index 758a3f40..844100cc 100644 --- a/testutil/fakegraphsync.go +++ b/testutil/fakegraphsync.go @@ -74,26 +74,26 @@ func (update Update) DTMessage(t *testing.T) datatransfer.Message { // FakeGraphSync implements a GraphExchange but does nothing type FakeGraphSync struct { - requests chan ReceivedGraphSyncRequest // records calls to fakeGraphSync.Request - pauses chan graphsync.RequestID - resumes chan Resume - cancels chan graphsync.RequestID - updates chan Update - persistenceOptionsLk sync.RWMutex - persistenceOptions map[string]ipld.LinkSystem - leaveRequestsOpen bool - OutgoingRequestHook graphsync.OnOutgoingRequestHook - IncomingBlockHook graphsync.OnIncomingBlockHook - OutgoingBlockHook graphsync.OnOutgoingBlockHook - IncomingRequestQueuedHook graphsync.OnIncomingRequestQueuedHook - IncomingRequestHook graphsync.OnIncomingRequestHook - CompletedResponseListener graphsync.OnResponseCompletedListener - RequestUpdatedHook graphsync.OnRequestUpdatedHook - IncomingResponseHook graphsync.OnIncomingResponseHook - RequestorCancelledListener graphsync.OnRequestorCancelledListener - BlockSentListener graphsync.OnBlockSentListener - NetworkErrorListener graphsync.OnNetworkErrorListener - ReceiverNetworkErrorListener graphsync.OnReceiverNetworkErrorListener + requests chan ReceivedGraphSyncRequest // records calls to fakeGraphSync.Request + pauses chan graphsync.RequestID + resumes chan Resume + cancels chan graphsync.RequestID + updates chan Update + persistenceOptionsLk sync.RWMutex + persistenceOptions map[string]ipld.LinkSystem + leaveRequestsOpen bool + OutgoingRequestHook graphsync.OnOutgoingRequestHook + IncomingBlockHook graphsync.OnIncomingBlockHook + OutgoingBlockHook graphsync.OnOutgoingBlockHook + IncomingRequestProcessingListener graphsync.OnRequestProcessingListener + IncomingRequestHook graphsync.OnIncomingRequestHook + CompletedResponseListener graphsync.OnResponseCompletedListener + RequestUpdatedHook graphsync.OnRequestUpdatedHook + IncomingResponseHook graphsync.OnIncomingResponseHook + RequestorCancelledListener graphsync.OnRequestorCancelledListener + BlockSentListener graphsync.OnBlockSentListener + NetworkErrorListener graphsync.OnNetworkErrorListener + ReceiverNetworkErrorListener graphsync.OnReceiverNetworkErrorListener } // NewFakeGraphSync returns a new fake graphsync implementation @@ -233,11 +233,11 @@ func (fgs *FakeGraphSync) RegisterIncomingRequestHook(hook graphsync.OnIncomingR } } -// RegisterIncomingRequestQueuedHook adds a hook that runs when an incoming GS request is queued. -func (fgs *FakeGraphSync) RegisterIncomingRequestQueuedHook(hook graphsync.OnIncomingRequestQueuedHook) graphsync.UnregisterHookFunc { - fgs.IncomingRequestQueuedHook = hook +// RegisterIncomingRequestProcessingListener adds a hook that runs when an incoming GS request begins processing +func (fgs *FakeGraphSync) RegisterIncomingRequestProcessingListener(hook graphsync.OnRequestProcessingListener) graphsync.UnregisterHookFunc { + fgs.IncomingRequestProcessingListener = hook return func() { - fgs.IncomingRequestQueuedHook = nil + fgs.IncomingRequestProcessingListener = nil } } @@ -342,7 +342,7 @@ func (fgs *FakeGraphSync) Stats() graphsync.Stats { return graphsync.Stats{} } -func (fgs *FakeGraphSync) RegisterOutgoingRequestProcessingListener(graphsync.OnOutgoingRequestProcessingListener) graphsync.UnregisterHookFunc { +func (fgs *FakeGraphSync) RegisterOutgoingRequestProcessingListener(graphsync.OnRequestProcessingListener) graphsync.UnregisterHookFunc { // TODO: just a stub for now, hopefully nobody needs this return func() {} } @@ -533,6 +533,7 @@ type FakeIncomingRequestHookActions struct { Validated bool SentExtensions []graphsync.ExtensionData Paused bool + CtxAugFuncs []func(context.Context) context.Context } func (fa *FakeIncomingRequestHookActions) SendExtensionData(ext graphsync.ExtensionData) { @@ -558,6 +559,10 @@ func (fa *FakeIncomingRequestHookActions) PauseResponse() { fa.Paused = true } +func (fa *FakeIncomingRequestHookActions) AugmentContext(ctxAugFunc func(reqCtx context.Context) context.Context) { + fa.CtxAugFuncs = append(fa.CtxAugFuncs, ctxAugFunc) +} + var _ graphsync.IncomingRequestHookActions = &FakeIncomingRequestHookActions{} type FakeRequestUpdatedActions struct { @@ -594,13 +599,3 @@ func (fa *FakeIncomingResponseHookActions) UpdateRequestWithExtensions(extension } var _ graphsync.IncomingResponseHookActions = &FakeIncomingResponseHookActions{} - -type FakeRequestQueuedHookActions struct { - ctxAugFuncs []func(context.Context) context.Context -} - -func (fa *FakeRequestQueuedHookActions) AugmentContext(ctxAugFunc func(reqCtx context.Context) context.Context) { - fa.ctxAugFuncs = append(fa.ctxAugFuncs, ctxAugFunc) -} - -var _ graphsync.RequestQueuedHookActions = &FakeRequestQueuedHookActions{} diff --git a/testutil/faketransport.go b/testutil/faketransport.go index 7e0d20c8..519691cc 100644 --- a/testutil/faketransport.go +++ b/testutil/faketransport.go @@ -18,12 +18,18 @@ type RestartedChannel struct { Message datatransfer.Request } -// Records a message sent +// MessageSent records a message sent type MessageSent struct { ChannelID datatransfer.ChannelID Message datatransfer.Message } +// DataLimitSet records setting a data limit +type DataLimitSet struct { + ChannelID datatransfer.ChannelID + DataLimit uint64 +} + // CustomizedTransfer is just a way to record calls made to transport configurer type CustomizedTransfer struct { ChannelID datatransfer.ChannelID @@ -41,6 +47,7 @@ type FakeTransport struct { ClosedChannels []datatransfer.ChannelID PausedChannels []datatransfer.ChannelID ResumedChannels []datatransfer.ChannelID + DataLimitsSet []DataLimitSet MessagesSent []MessageSent UpdateError error CleanedUpChannels []datatransfer.ChannelID @@ -89,29 +96,34 @@ func (ft *FakeTransport) RestartChannel(ctx context.Context, channelState datatr } // WithChannel takes actions on a channel -func (ft *FakeTransport) UpdateChannel(ctx context.Context, chid datatransfer.ChannelID, update datatransfer.ChannelUpdate) error { +func (ft *FakeTransport) SendChannelCommand(ctx context.Context, chid datatransfer.ChannelID, command datatransfer.ChannelCommand) error { - if update.SendMessage != nil { - ft.MessagesSent = append(ft.MessagesSent, MessageSent{chid, update.SendMessage}) + update := command.ChannelUpdate() + message, sendMessage := update.MessageToSend() + if sendMessage { + ft.MessagesSent = append(ft.MessagesSent, MessageSent{chid, message}) } - if update.Closed { + closed, hasClosed := update.Closed() + if hasClosed && closed { ft.ClosedChannels = append(ft.ClosedChannels, chid) return ft.UpdateError } - if !update.Paused { - ft.ResumedChannels = append(ft.ResumedChannels, chid) - } else { - ft.PausedChannels = append(ft.PausedChannels, chid) + paused, hasPaused := update.Paused() + if hasPaused { + if !paused { + ft.ResumedChannels = append(ft.ResumedChannels, chid) + } else { + ft.PausedChannels = append(ft.PausedChannels, chid) + } } - return ft.UpdateError -} + dataLimit, hasDataLimit := update.DataLimit() + if hasDataLimit { + ft.DataLimitsSet = append(ft.DataLimitsSet, DataLimitSet{chid, dataLimit}) + } -// SendMessage sends a data transfer message over the channel to the other peer -func (ft *FakeTransport) SendMessage(ctx context.Context, chid datatransfer.ChannelID, msg datatransfer.Message) error { - ft.MessagesSent = append(ft.MessagesSent, MessageSent{chid, msg}) return ft.UpdateError } diff --git a/testutil/mockchannelstate.go b/testutil/mockchannelstate.go index 2aa75e30..cd3be68b 100644 --- a/testutil/mockchannelstate.go +++ b/testutil/mockchannelstate.go @@ -9,32 +9,32 @@ import ( ) type MockChannelStateParams struct { - ReceivedCids []cid.Cid - ChannelID datatransfer.ChannelID - Queued uint64 - Sent uint64 - Received uint64 - Complete bool + ReceivedIndex datamodel.Node + ChannelID datatransfer.ChannelID + Queued uint64 + Sent uint64 + Received uint64 + Complete bool } func NewMockChannelState(params MockChannelStateParams) *MockChannelState { return &MockChannelState{ - receivedCids: params.ReceivedCids, - chid: params.ChannelID, - queued: params.Queued, - sent: params.Sent, - received: params.Received, - complete: params.Complete, + receivedIndex: params.ReceivedIndex, + chid: params.ChannelID, + queued: params.Queued, + sent: params.Sent, + received: params.Received, + complete: params.Complete, } } type MockChannelState struct { - receivedCids []cid.Cid - chid datatransfer.ChannelID - queued uint64 - sent uint64 - received uint64 - complete bool + receivedIndex datamodel.Node + chid datatransfer.ChannelID + queued uint64 + sent uint64 + received uint64 + complete bool } var _ datatransfer.ChannelState = (*MockChannelState)(nil) @@ -77,23 +77,15 @@ func (m *MockChannelState) Status() datatransfer.Status { return datatransfer.Ongoing } -func (m *MockChannelState) ReceivedCids() []cid.Cid { - return m.receivedCids +func (m *MockChannelState) ReceivedIndex() datamodel.Node { + return m.receivedIndex } -func (m *MockChannelState) ReceivedCidsLen() int { - return len(m.receivedCids) -} - -func (m *MockChannelState) ReceivedCidsTotal() int64 { - return (int64)(len(m.receivedCids)) -} - -func (m *MockChannelState) QueuedCidsTotal() int64 { +func (m *MockChannelState) QueuedIndex() datamodel.Node { panic("implement me") } -func (m *MockChannelState) SentCidsTotal() int64 { +func (m *MockChannelState) SentIndex() datamodel.Node { panic("implement me") } diff --git a/transport.go b/transport.go index 546c2574..bc434e75 100644 --- a/transport.go +++ b/transport.go @@ -3,7 +3,7 @@ package datatransfer import ( "context" - ipld "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" ) // TransportID identifies a unique transport @@ -22,70 +22,61 @@ type EventsHandler interface { // ChannelState queries for the current channel state ChannelState(ctx context.Context, chid ChannelID) (ChannelState, error) - // OnChannelOpened is called when we send a request for data to the other - // peer on the given channel ID - // return values are: - // - error = ignore incoming data for this channel - OnChannelOpened(chid ChannelID) error + // OnChannelOpened is called at the point the transport begins processing the + // request (prior to that it may simply be queued) -- only applies to initiator + OnChannelOpened(chid ChannelID) + + // OnTransferInitiated is called at the point the transport actually begins sending/receiving data + OnTransferInitiated(chid ChannelID) + + // OnRequestReceived is called when we receive a new request for the given channel ID + // return values is a channel command to take action based on this response + OnRequestReceived(chid ChannelID, msg Request) (ChannelCommand, error) + // OnResponseReceived is called when we receive a response to a request - // - nil = continue receiving data - // - error = cancel this request - OnResponseReceived(chid ChannelID, msg Response) error + OnResponseReceived(chid ChannelID, msg Response) + // OnDataReceive is called when we receive data for the given channel ID - // return values are: - // - nil = proceed with sending data - // - error = cancel this request - // - err == ErrPause - pause this request - OnDataReceived(chid ChannelID, link ipld.Link, size uint64, index int64, unique bool) error + // index is a transport dependent of serializing "here's where I am in this transport" + OnDataReceived(chid ChannelID, size uint64, index datamodel.Node) // OnDataQueued is called when data is queued for sending for the given channel ID - // return values are: - // - nil = proceed with sending data - // - error = cancel this request - // - err == ErrPause - pause this request - OnDataQueued(chid ChannelID, link ipld.Link, size uint64, index int64, unique bool) (Message, error) + // index is a transport dependent of serializing "here's where I am in this transport" + OnDataQueued(chid ChannelID, size uint64, index datamodel.Node) // OnDataSent is called when we send data for the given channel ID - OnDataSent(chid ChannelID, link ipld.Link, size uint64, index int64, unique bool) error - - // OnTransferQueued is called when a new data transfer request is queued in the transport layer. - OnTransferQueued(chid ChannelID) - - // OnRequestReceived is called when we receive a new request to send data - // for the given channel ID - // return values are: - // message = data transfer message along with reply - // err = error - // - nil = proceed with sending data - // - error = cancel this request - // - err == ErrPause - pause this request (only for new requests) - // - err == ErrResume - resume this request (only for update requests) - OnRequestReceived(chid ChannelID, msg Request) (Response, error) - // OnChannelCompleted is called when we finish transferring data for the given channel ID - // Error returns are logged but otherwise have no effect - OnChannelCompleted(chid ChannelID, err error) error - - // OnRequestCancelled is called when a request we opened (with the given channel Id) to + // index is a transport dependent of serializing "here's where I am in this transport" + OnDataSent(chid ChannelID, size uint64, index datamodel.Node) + + // OnDataLimitReached is called when a channel hits a previously set data limit + OnDataLimitReached(chid ChannelID) + + // OnTransferCompleted is called when we finish transferring data for the given channel ID + OnTransferCompleted(chid ChannelID, err error) + + // OnTransferCannceled is called when a request we opened (with the given channel Id) to // receive data is cancelled by us. - // Error returns are logged but otherwise have no effect - OnRequestCancelled(chid ChannelID, err error) error + OnTransferCancelled(chid ChannelID, err error) + + // OnMessageSendError is called when a network error occurs while sending a message + OnMessageSendError(chid ChannelID, err error) // OnRequestDisconnected is called when a network error occurs trying to send a request - OnRequestDisconnected(chid ChannelID, err error) error + OnRequestDisconnected(chid ChannelID, err error) // OnSendDataError is called when a network error occurs sending data // at the transport layer - OnSendDataError(chid ChannelID, err error) error + OnSendDataError(chid ChannelID, err error) // OnReceiveDataError is called when a network error occurs receiving data // at the transport layer - OnReceiveDataError(chid ChannelID, err error) error + OnReceiveDataError(chid ChannelID, err error) // OnContextAugment allows the transport to attach data transfer tracing information // to its local context, in order to create a hierarchical trace OnContextAugment(chid ChannelID) func(context.Context) context.Context - OnRestartExistingChannelRequestReceived(chid ChannelID) error + OnRestartExistingChannelRequestReceived(chid ChannelID) } /* @@ -126,13 +117,11 @@ type Transport interface { // messages over the channel. Grouping the commands allows the transport // the ability to plan how to execute these updates based on the capabilities // and API of the underlying transport protocol and library - UpdateChannel(ctx context.Context, chid ChannelID, update ChannelUpdate) error + SendChannelCommand(ctx context.Context, chid ChannelID, command ChannelCommand) error // SetEventHandler sets the handler for events on channels SetEventHandler(events EventsHandler) error // CleanupChannel removes any associated data on a closed channel CleanupChannel(chid ChannelID) - // SendMessage sends a data transfer message over the channel to the other peer - SendMessage(ctx context.Context, chid ChannelID, msg Message) error // Shutdown unregisters the current EventHandler and ends all active data transfers Shutdown(ctx context.Context) error @@ -153,11 +142,76 @@ type TransportCapabilities struct { // ChannelUpdate describes updates to a channel - changing it's paused status, closing the transfer, // and additional messages to send -type ChannelUpdate struct { - // Paused sets the paused status of the channel. If pause/resumes are not supported, this is a no op - Paused bool - // Closed sets whether the channel is closed - Closed bool - // SendMessage sends an additional message - SendMessage Message +type ChannelUpdate interface { + Paused() (bool, bool) + Closed() (bool, bool) + MessageToSend() (Message, bool) + DataLimit() (uint64, bool) +} + +type channelUpdate struct { + paused bool + hasPaused bool + closed bool + hasClosed bool + messageToSend Message + hasMessageToSend bool + dataLimit uint64 + hasDataLimit bool +} + +func (cu channelUpdate) Paused() (bool, bool) { + return cu.paused, cu.hasPaused +} + +func (cu channelUpdate) Closed() (bool, bool) { + return cu.closed, cu.hasClosed +} + +func (cu channelUpdate) MessageToSend() (Message, bool) { + return cu.messageToSend, cu.hasMessageToSend +} + +func (cu channelUpdate) DataLimit() (uint64, bool) { + return cu.dataLimit, cu.hasDataLimit +} + +type ChannelCommand struct { + cu channelUpdate +} + +func NewCommand() ChannelCommand { + return ChannelCommand{} +} + +func (cc ChannelCommand) SetPaused(paused bool) ChannelCommand { + cu := cc.cu + cu.paused = paused + cu.hasPaused = true + return ChannelCommand{cu} +} + +func (cc ChannelCommand) SetClosed(closed bool) ChannelCommand { + cu := cc.cu + cu.closed = true + cu.hasClosed = true + return ChannelCommand{cu} +} + +func (cc ChannelCommand) SendMessage(msg Message) ChannelCommand { + cu := cc.cu + cu.messageToSend = msg + cu.hasMessageToSend = true + return ChannelCommand{cu} +} + +func (cc ChannelCommand) SetDataLimit(datalimit uint64) ChannelCommand { + cu := cc.cu + cu.dataLimit = datalimit + cu.hasDataLimit = true + return ChannelCommand{cu} +} + +func (cc ChannelCommand) ChannelUpdate() ChannelUpdate { + return cc.cu } diff --git a/types.go b/types.go index 1181d21e..5723a9fd 100644 --- a/types.go +++ b/types.go @@ -123,17 +123,17 @@ type ChannelState interface { // LastVoucherResult returns the last voucher result sent on the channel LastVoucherResult() TypedVoucher - // ReceivedCidsTotal returns the number of (non-unique) cids received so far - // on the channel - note that a block can exist in more than one place in the DAG - ReceivedCidsTotal() int64 + // ReceivedIndex returns the index, a transport specific identifier for "where" + // we are in receiving data for a transfer + ReceivedIndex() datamodel.Node - // QueuedCidsTotal returns the number of (non-unique) cids queued so far - // on the channel - note that a block can exist in more than one place in the DAG - QueuedCidsTotal() int64 + // QueuedIndex returns the index, a transport specific identifier for "where" + // we are in queing data for a transfer + QueuedIndex() datamodel.Node - // SentCidsTotal returns the number of (non-unique) cids sent so far - // on the channel - note that a block can exist in more than one place in the DAG - SentCidsTotal() int64 + // SentIndex returns the index, a transport specific identifier for "where" + // we are in sending data for a transfer + SentIndex() datamodel.Node // Queued returns the number of bytes read from the node and queued for sending Queued() uint64 From a671bb6bb5c6eb5e833e411318ccdf7328ceaa86 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 30 Jun 2022 01:33:33 -0700 Subject: [PATCH 12/18] refactor(transport): refactor events interface refactors the transport events interface - moves data limit tracking into the transport - makes index tracking opaque work inside the transport - refactors event flow -- we now have an initiated event + queued & awaiting acceptance states - leans into async communication between transport and data transfer - adds a bunch of transport tests - refactor pausing to support more correct resumption in state machine --- benchmarks/testinstance/testinstance.go | 2 +- channels/channel_state.go | 19 + channels/channels.go | 9 +- channels/channels_fsm.go | 141 +- channels/channels_test.go | 62 +- channels/internal/internalchannel.go | 4 + channels/internal/internalchannel_cbor_gen.go | 70 +- errors.go | 8 - events.go | 14 +- go.mod | 3 +- go.sum | 32 +- impl/events.go | 240 +-- impl/impl.go | 162 +- impl/initiating_test.go | 44 +- impl/receiving_requests.go | 119 +- impl/responding_test.go | 98 +- impl/utils.go | 7 + itest/gstestdata.go | 18 +- itest/integration_test.go | 40 +- manager.go | 3 + message/message1_1prime/message.go | 24 +- message/message1_1prime/message_test.go | 21 +- .../message1_1prime/transfer_response_test.go | 3 +- statuses.go | 81 +- testutil/faketransport.go | 45 +- testutil/message.go | 30 - testutil/mockchannelstate.go | 150 +- transport.go | 242 ++- transport/graphsync/dtchannel/dtchannel.go | 182 ++- transport/graphsync/executor/executor.go | 16 +- transport/graphsync/executor/executor_test.go | 23 +- transport/graphsync/graphsync.go | 75 +- transport/graphsync/graphsync_test.go | 1362 ----------------- transport/graphsync/hooks.go | 249 +-- transport/graphsync/initiating_test.go | 1324 ++++++++++++++++ transport/graphsync/receiver.go | 70 +- transport/graphsync/responding_test.go | 412 +++++ transport/graphsync/testharness/events.go | 61 + .../graphsync/testharness}/fakegraphsync.go | 192 ++- transport/graphsync/testharness/harness.go | 284 ++++ transport/graphsync/testharness/testnet.go | 106 ++ transport/helpers/network/libp2p_impl_test.go | 3 +- types.go | 12 + 43 files changed, 3501 insertions(+), 2561 deletions(-) delete mode 100644 testutil/message.go delete mode 100644 transport/graphsync/graphsync_test.go create mode 100644 transport/graphsync/initiating_test.go create mode 100644 transport/graphsync/responding_test.go create mode 100644 transport/graphsync/testharness/events.go rename {testutil => transport/graphsync/testharness}/fakegraphsync.go (79%) create mode 100644 transport/graphsync/testharness/harness.go create mode 100644 transport/graphsync/testharness/testnet.go diff --git a/benchmarks/testinstance/testinstance.go b/benchmarks/testinstance/testinstance.go index bfc72747..685cd264 100644 --- a/benchmarks/testinstance/testinstance.go +++ b/benchmarks/testinstance/testinstance.go @@ -164,7 +164,7 @@ func NewInstance(ctx context.Context, net tn.Network, tempDir string, diskBasedD linkSystem := storeutil.LinkSystemForBlockstore(bstore) gs := gsimpl.New(ctx, gsNet, linkSystem, gsimpl.RejectAllRequestsByDefault()) - transport := gstransport.NewTransport(p, gs, dtNet) + transport := gstransport.NewTransport(gs, dtNet) dt, err := dtimpl.NewDataTransfer(namespace.Wrap(dstore, datastore.NewKey("/data-transfers/transfers")), p, transport) if err != nil { return Instance{}, err diff --git a/channels/channel_state.go b/channels/channel_state.go index 93ce3388..efabd74d 100644 --- a/channels/channel_state.go +++ b/channels/channel_state.go @@ -139,6 +139,25 @@ func (c channelState) RequiresFinalization() bool { return c.ic.RequiresFinalization } +func (c channelState) InitiatorPaused() bool { + return c.ic.InitiatorPaused +} + +func (c channelState) ResponderPaused() bool { + return c.ic.ResponderPaused || c.ic.Status == datatransfer.Finalizing +} + +func (c channelState) BothPaused() bool { + return c.InitiatorPaused() && c.ResponderPaused() +} + +func (c channelState) SelfPaused() bool { + if c.ic.SelfPeer == c.ic.Initiator { + return c.InitiatorPaused() + } + return c.ResponderPaused() +} + // Stages returns the current ChannelStages object, or an empty object. // It is unsafe for the caller to modify the return value, and changes may not // be persisted. It should be treated as immutable. diff --git a/channels/channels.go b/channels/channels.go index 43176c58..f2139c59 100644 --- a/channels/channels.go +++ b/channels/channels.go @@ -160,8 +160,8 @@ func (c *Channels) ChannelOpened(chid datatransfer.ChannelID) error { return c.send(chid, datatransfer.Opened) } -func (c *Channels) TransferRequestQueued(chid datatransfer.ChannelID) error { - return c.send(chid, datatransfer.TransferRequestQueued) +func (c *Channels) TransferInitiated(chid datatransfer.ChannelID) error { + return c.send(chid, datatransfer.TransferInitiated) } // Restart marks a data transfer as restarted @@ -291,6 +291,11 @@ func (c *Channels) ReceiveDataError(chid datatransfer.ChannelID, err error) erro return c.send(chid, datatransfer.ReceiveDataError, err) } +// SendMessageError indicates an error sending a message to the transport layer +func (c *Channels) SendMessageError(chid datatransfer.ChannelID, err error) error { + return c.send(chid, datatransfer.SendMessageError, err) +} + // SetDataLimit means a data limit has been set on this channel func (c *Channels) SetDataLimit(chid datatransfer.ChannelID, dataLimit uint64) error { return c.send(chid, datatransfer.SetDataLimit, dataLimit) diff --git a/channels/channels_fsm.go b/channels/channels_fsm.go index ddbc5418..51dfc6d5 100644 --- a/channels/channels_fsm.go +++ b/channels/channels_fsm.go @@ -12,16 +12,6 @@ import ( var log = logging.Logger("data-transfer") -var transferringStates = []fsm.StateKey{ - datatransfer.Requested, - datatransfer.Ongoing, - datatransfer.InitiatorPaused, - datatransfer.ResponderPaused, - datatransfer.BothPaused, - datatransfer.ResponderCompleted, - datatransfer.ResponderFinalizing, -} - // ChannelEvents describe the events taht can var ChannelEvents = fsm.Events{ // Open a channel @@ -29,23 +19,32 @@ var ChannelEvents = fsm.Events{ chst.AddLog("") return nil }), + // Remote peer has accepted the Open channel request - fsm.Event(datatransfer.Accept).From(datatransfer.Requested).To(datatransfer.Ongoing).Action(func(chst *internal.ChannelState) error { - chst.AddLog("") - return nil - }), + fsm.Event(datatransfer.Accept). + From(datatransfer.Requested).To(datatransfer.Queued). + From(datatransfer.AwaitingAcceptance).To(datatransfer.Ongoing). + Action(func(chst *internal.ChannelState) error { + chst.AddLog("") + return nil + }), - fsm.Event(datatransfer.TransferRequestQueued).FromAny().ToJustRecord().Action(func(chst *internal.ChannelState) error { - chst.Message = "" - chst.AddLog("") - return nil - }), + // The transport has indicated it's begun sending/receiving data + fsm.Event(datatransfer.TransferInitiated). + From(datatransfer.Requested).To(datatransfer.AwaitingAcceptance). + From(datatransfer.Queued).To(datatransfer.Ongoing). + From(datatransfer.Ongoing).ToJustRecord(). + Action(func(chst *internal.ChannelState) error { + chst.AddLog("") + return nil + }), fsm.Event(datatransfer.Restart).FromAny().ToJustRecord().Action(func(chst *internal.ChannelState) error { chst.Message = "" chst.AddLog("") return nil }), + fsm.Event(datatransfer.Cancel).FromAny().To(datatransfer.Cancelling).Action(func(chst *internal.ChannelState) error { chst.AddLog("") return nil @@ -60,81 +59,90 @@ var ChannelEvents = fsm.Events{ return nil }), - fsm.Event(datatransfer.DataReceived).FromAny().ToNoChange(). + fsm.Event(datatransfer.DataReceived).FromMany(datatransfer.TransferringStates.AsFSMStates()...).ToNoChange(). Action(func(chst *internal.ChannelState, receivedIndex datamodel.Node) error { chst.ReceivedIndex = internal.CborGenCompatibleNode{Node: receivedIndex} chst.AddLog("") return nil }), - fsm.Event(datatransfer.DataReceivedProgress).FromMany(transferringStates...).ToNoChange(). + fsm.Event(datatransfer.DataReceivedProgress).FromMany(datatransfer.TransferringStates.AsFSMStates()...).ToNoChange(). Action(func(chst *internal.ChannelState, delta uint64) error { chst.Received += delta chst.AddLog("received data") return nil }), - fsm.Event(datatransfer.DataSent). - FromMany(transferringStates...).ToNoChange(). - From(datatransfer.TransferFinished).ToNoChange(). + fsm.Event(datatransfer.DataSent).FromMany(datatransfer.TransferringStates.AsFSMStates()...).ToNoChange(). Action(func(chst *internal.ChannelState, sentIndex datamodel.Node) error { chst.SentIndex = internal.CborGenCompatibleNode{Node: sentIndex} chst.AddLog("") return nil }), - fsm.Event(datatransfer.DataSentProgress).FromMany(transferringStates...).ToNoChange(). + fsm.Event(datatransfer.DataSentProgress).FromMany(datatransfer.TransferringStates.AsFSMStates()...).ToNoChange(). Action(func(chst *internal.ChannelState, delta uint64) error { chst.Sent += delta chst.AddLog("sending data") return nil }), - fsm.Event(datatransfer.DataQueued). - FromMany(transferringStates...).ToNoChange(). - From(datatransfer.TransferFinished).ToNoChange(). + fsm.Event(datatransfer.DataQueued).FromMany(datatransfer.TransferringStates.AsFSMStates()...).ToNoChange(). Action(func(chst *internal.ChannelState, queuedIndex datamodel.Node) error { chst.QueuedIndex = internal.CborGenCompatibleNode{Node: queuedIndex} chst.AddLog("") return nil }), - fsm.Event(datatransfer.DataQueuedProgress).FromMany(transferringStates...).ToNoChange(). + fsm.Event(datatransfer.DataQueuedProgress).FromMany(datatransfer.TransferringStates.AsFSMStates()...).ToNoChange(). Action(func(chst *internal.ChannelState, delta uint64) error { chst.Queued += delta chst.AddLog("") return nil }), + fsm.Event(datatransfer.SetDataLimit).FromAny().ToJustRecord(). Action(func(chst *internal.ChannelState, dataLimit uint64) error { chst.DataLimit = dataLimit chst.AddLog("") return nil }), + fsm.Event(datatransfer.SetRequiresFinalization).FromAny().ToJustRecord(). Action(func(chst *internal.ChannelState, RequiresFinalization bool) error { chst.RequiresFinalization = RequiresFinalization chst.AddLog("") return nil }), + fsm.Event(datatransfer.Disconnected).FromAny().ToNoChange().Action(func(chst *internal.ChannelState, err error) error { chst.Message = err.Error() chst.AddLog("data transfer disconnected: %s", chst.Message) return nil }), + fsm.Event(datatransfer.SendDataError).FromAny().ToNoChange().Action(func(chst *internal.ChannelState, err error) error { chst.Message = err.Error() chst.AddLog("data transfer send error: %s", chst.Message) return nil }), + fsm.Event(datatransfer.ReceiveDataError).FromAny().ToNoChange().Action(func(chst *internal.ChannelState, err error) error { chst.Message = err.Error() chst.AddLog("data transfer receive error: %s", chst.Message) return nil }), + + fsm.Event(datatransfer.SendMessageError).FromAny().ToNoChange().Action(func(chst *internal.ChannelState, err error) error { + chst.Message = err.Error() + chst.AddLog("data transfer errored sending message: %s", chst.Message) + return nil + }), + fsm.Event(datatransfer.RequestCancelled).FromAny().ToNoChange().Action(func(chst *internal.ChannelState, err error) error { chst.Message = err.Error() chst.AddLog("data transfer request cancelled: %s", chst.Message) return nil }), + fsm.Event(datatransfer.Error).FromAny().To(datatransfer.Failing).Action(func(chst *internal.ChannelState, err error) error { chst.Message = err.Error() chst.AddLog("data transfer erred: %s", chst.Message) @@ -147,6 +155,7 @@ var ChannelEvents = fsm.Events{ chst.AddLog("got new voucher") return nil }), + fsm.Event(datatransfer.NewVoucherResult).FromAny().ToNoChange(). Action(func(chst *internal.ChannelState, voucherResult datatransfer.TypedVoucher) error { chst.VoucherResults = append(chst.VoucherResults, @@ -155,46 +164,54 @@ var ChannelEvents = fsm.Events{ return nil }), + // TODO: There are four states from which the request can be "paused": request, queued, awaiting acceptance + // and ongoing. There four states of being + // paused (no pause, initiator pause, responder pause, both paused). Until the state machine software + // supports orthogonal regions (https://en.wikipedia.org/wiki/UML_state_machine#Orthogonal_regions) + // we end up with a cartesian product of states and as you can see, fairly complicated state transfers. + // Previously, we had dealt with this by moving directly to the Ongoing state upon return from pause but this + // seems less than ideal. We need some kind of support for pausing being an independent aspect of state + // Possibly we should just remove whether a state is paused from the state entirely. fsm.Event(datatransfer.PauseInitiator). - FromMany(datatransfer.Requested, datatransfer.Ongoing).To(datatransfer.InitiatorPaused). - From(datatransfer.ResponderPaused).To(datatransfer.BothPaused). - FromAny().ToJustRecord().Action(func(chst *internal.ChannelState) error { - chst.AddLog("") - return nil - }), + FromMany(datatransfer.Ongoing, datatransfer.Requested, datatransfer.Queued, datatransfer.AwaitingAcceptance).ToJustRecord(). + Action(func(chst *internal.ChannelState) error { + chst.InitiatorPaused = true + chst.AddLog("") + return nil + }), fsm.Event(datatransfer.PauseResponder). - FromMany(datatransfer.Requested, datatransfer.Ongoing).To(datatransfer.ResponderPaused). - From(datatransfer.InitiatorPaused).To(datatransfer.BothPaused). - FromAny().ToJustRecord().Action(func(chst *internal.ChannelState) error { - chst.AddLog("") - return nil - }), + FromMany(datatransfer.Ongoing, datatransfer.Requested, datatransfer.Queued, datatransfer.AwaitingAcceptance, datatransfer.TransferFinished).ToJustRecord(). + Action(func(chst *internal.ChannelState) error { + chst.ResponderPaused = true + chst.AddLog("") + return nil + }), fsm.Event(datatransfer.DataLimitExceeded). - FromMany(datatransfer.Requested, datatransfer.Ongoing).To(datatransfer.ResponderPaused). - From(datatransfer.InitiatorPaused).To(datatransfer.BothPaused). - FromAny().ToJustRecord().Action(func(chst *internal.ChannelState) error { - chst.AddLog("") - return nil - }), + FromMany(datatransfer.Ongoing, datatransfer.Requested, datatransfer.Queued, datatransfer.AwaitingAcceptance, datatransfer.ResponderCompleted, datatransfer.ResponderFinalizing).ToJustRecord(). + Action(func(chst *internal.ChannelState) error { + chst.ResponderPaused = true + chst.AddLog("") + return nil + }), fsm.Event(datatransfer.ResumeInitiator). - From(datatransfer.InitiatorPaused).To(datatransfer.Ongoing). - From(datatransfer.BothPaused).To(datatransfer.ResponderPaused). - FromAny().ToJustRecord().Action(func(chst *internal.ChannelState) error { - chst.AddLog("") - return nil - }), + FromMany(datatransfer.Ongoing, datatransfer.Requested, datatransfer.Queued, datatransfer.AwaitingAcceptance, datatransfer.ResponderCompleted, datatransfer.ResponderFinalizing).ToJustRecord(). + Action(func(chst *internal.ChannelState) error { + chst.InitiatorPaused = false + chst.AddLog("") + return nil + }), fsm.Event(datatransfer.ResumeResponder). - From(datatransfer.ResponderPaused).To(datatransfer.Ongoing). - From(datatransfer.BothPaused).To(datatransfer.InitiatorPaused). + FromMany(datatransfer.Ongoing, datatransfer.Requested, datatransfer.Queued, datatransfer.AwaitingAcceptance, datatransfer.TransferFinished).ToJustRecord(). From(datatransfer.Finalizing).To(datatransfer.Completing). - FromAny().ToJustRecord().Action(func(chst *internal.ChannelState) error { - chst.AddLog("") - return nil - }), + Action(func(chst *internal.ChannelState) error { + chst.ResponderPaused = false + chst.AddLog("") + return nil + }), // The transfer has finished on the local node - all data was sent / received fsm.Event(datatransfer.FinishTransfer). @@ -202,10 +219,10 @@ var ChannelEvents = fsm.Events{ FromMany(datatransfer.Failing, datatransfer.Cancelling).ToJustRecord(). From(datatransfer.ResponderCompleted).To(datatransfer.Completing). From(datatransfer.ResponderFinalizing).To(datatransfer.ResponderFinalizingTransferFinished). - // If we are in the requested state, it means the other party simply never responded to our + // If we are in the AwaitingAcceptance state, it means the other party simply never responded to our // our data transfer, or we never actually contacted them. In any case, it's safe to skip // the finalization process and complete the transfer - From(datatransfer.Requested).To(datatransfer.Completing). + From(datatransfer.AwaitingAcceptance).To(datatransfer.Completing). Action(func(chst *internal.ChannelState) error { chst.AddLog("") return nil @@ -224,9 +241,7 @@ var ChannelEvents = fsm.Events{ fsm.Event(datatransfer.ResponderCompletes). FromAny().To(datatransfer.ResponderCompleted). FromMany(datatransfer.Failing, datatransfer.Cancelling).ToJustRecord(). - From(datatransfer.ResponderPaused).To(datatransfer.ResponderFinalizing). From(datatransfer.TransferFinished).To(datatransfer.Completing). - From(datatransfer.ResponderFinalizing).To(datatransfer.ResponderCompleted). From(datatransfer.ResponderFinalizingTransferFinished).To(datatransfer.Completing).Action(func(chst *internal.ChannelState) error { chst.AddLog("") return nil diff --git a/channels/channels_test.go b/channels/channels_test.go index 442b56c7..b6863cc8 100644 --- a/channels/channels_test.go +++ b/channels/channels_test.go @@ -104,55 +104,23 @@ func TestChannels(t *testing.T) { err = channelList.Accept(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.Accept) - require.Equal(t, state.Status(), datatransfer.Ongoing) + require.Equal(t, state.Status(), datatransfer.Queued) err = channelList.Accept(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}) require.True(t, errors.Is(err, datatransfer.ErrChannelNotFound)) }) - t.Run("transfer queued", func(t *testing.T) { + t.Run("transfer initiated", func(t *testing.T) { state, err := channelList.GetByID(ctx, datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) require.NoError(t, err) - require.Equal(t, state.Status(), datatransfer.Ongoing) + require.Equal(t, state.Status(), datatransfer.Queued) - err = channelList.TransferRequestQueued(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) + err = channelList.TransferInitiated(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) require.NoError(t, err) - state = checkEvent(ctx, t, received, datatransfer.TransferRequestQueued) + state = checkEvent(ctx, t, received, datatransfer.TransferInitiated) require.Equal(t, state.Status(), datatransfer.Ongoing) }) - t.Run("datasent/queued when transfer is already finished", func(t *testing.T) { - ds := dss.MutexWrap(datastore.NewMapDatastore()) - - channelList, err := channels.New(ds, notifier, &fakeEnv{}, peers[0]) - require.NoError(t, err) - err = channelList.Start(ctx) - require.NoError(t, err) - - chid, _, err := channelList.CreateNew(peers[0], tid1, cids[0], selector, fv1, peers[0], peers[0], peers[1]) - require.NoError(t, err) - checkEvent(ctx, t, received, datatransfer.Open) - require.NoError(t, channelList.Accept(chid)) - checkEvent(ctx, t, received, datatransfer.Accept) - - // move the channel to `TransferFinished` state. - require.NoError(t, channelList.FinishTransfer(chid)) - state := checkEvent(ctx, t, received, datatransfer.FinishTransfer) - require.Equal(t, datatransfer.TransferFinished, state.Status()) - - // send a data-sent event and ensure it's a no-op - err = channelList.DataSent(chid, 1, basicnode.NewInt(1)) - require.NoError(t, err) - state = checkEvent(ctx, t, received, datatransfer.DataSent) - require.Equal(t, datatransfer.TransferFinished, state.Status()) - - // send a data-queued event and ensure it's a no-op. - err = channelList.DataQueued(chid, 1, basicnode.NewInt(1)) - require.NoError(t, err) - state = checkEvent(ctx, t, received, datatransfer.DataQueued) - require.Equal(t, datatransfer.TransferFinished, state.Status()) - }) - t.Run("updating send/receive values", func(t *testing.T) { ds := dss.MutexWrap(datastore.NewMapDatastore()) @@ -168,6 +136,10 @@ func TestChannels(t *testing.T) { require.Equal(t, uint64(0), state.Received()) require.Equal(t, uint64(0), state.Sent()) + err = channelList.TransferInitiated(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) + require.NoError(t, err) + _ = checkEvent(ctx, t, received, datatransfer.TransferInitiated) + err = channelList.DataReceived(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}, 50, basicnode.NewInt(1)) require.NoError(t, err) _ = checkEvent(ctx, t, received, datatransfer.DataReceivedProgress) @@ -216,7 +188,7 @@ func TestChannels(t *testing.T) { err = channelList.DataLimitExceeded(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.DataLimitExceeded) - require.Equal(t, datatransfer.ResponderPaused, state.Status()) + require.True(t, state.ResponderPaused()) err = channelList.SetDataLimit(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}, 700) require.NoError(t, err) @@ -225,16 +197,16 @@ func TestChannels(t *testing.T) { err = channelList.ResumeResponder(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}) state = checkEvent(ctx, t, received, datatransfer.ResumeResponder) - require.Equal(t, datatransfer.Ongoing, state.Status()) + require.False(t, state.ResponderPaused()) err = channelList.PauseInitiator(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}) state = checkEvent(ctx, t, received, datatransfer.PauseInitiator) - require.Equal(t, datatransfer.InitiatorPaused, state.Status()) + require.True(t, state.InitiatorPaused()) err = channelList.DataLimitExceeded(datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: tid1}) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.DataLimitExceeded) - require.Equal(t, datatransfer.BothPaused, state.Status()) + require.True(t, state.BothPaused()) }) @@ -246,17 +218,19 @@ func TestChannels(t *testing.T) { err = channelList.PauseInitiator(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.PauseInitiator) - require.Equal(t, datatransfer.InitiatorPaused, state.Status()) + require.True(t, state.InitiatorPaused()) + require.False(t, state.BothPaused()) err = channelList.PauseResponder(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.PauseResponder) - require.Equal(t, datatransfer.BothPaused, state.Status()) + require.True(t, state.BothPaused()) err = channelList.ResumeInitiator(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) require.NoError(t, err) state = checkEvent(ctx, t, received, datatransfer.ResumeInitiator) - require.Equal(t, datatransfer.ResponderPaused, state.Status()) + require.True(t, state.ResponderPaused()) + require.False(t, state.BothPaused()) err = channelList.ResumeResponder(datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: tid1}) require.NoError(t, err) diff --git a/channels/internal/internalchannel.go b/channels/internal/internalchannel.go index 0b1e2bee..4209edaa 100644 --- a/channels/internal/internalchannel.go +++ b/channels/internal/internalchannel.go @@ -117,6 +117,10 @@ type ChannelState struct { // RequiresFinalization indicates at the end of the transfer, the channel should // be left open for a final settlement RequiresFinalization bool + // ResponderPaused indicates whether the responder is in a paused state + ResponderPaused bool + // InitiatorPaused indicates whether the initiator is in a paused state + InitiatorPaused bool // Stages traces the execution fo a data transfer. // // EXPERIMENTAL; subject to change. diff --git a/channels/internal/internalchannel_cbor_gen.go b/channels/internal/internalchannel_cbor_gen.go index 3726f8ca..c7b1e958 100644 --- a/channels/internal/internalchannel_cbor_gen.go +++ b/channels/internal/internalchannel_cbor_gen.go @@ -23,7 +23,7 @@ func (t *ChannelState) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{182}); err != nil { + if _, err := w.Write([]byte{184, 24}); err != nil { return err } @@ -425,6 +425,38 @@ func (t *ChannelState) MarshalCBOR(w io.Writer) error { return err } + // t.ResponderPaused (bool) (bool) + if len("ResponderPaused") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"ResponderPaused\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ResponderPaused"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("ResponderPaused")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.ResponderPaused); err != nil { + return err + } + + // t.InitiatorPaused (bool) (bool) + if len("InitiatorPaused") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"InitiatorPaused\" was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("InitiatorPaused"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("InitiatorPaused")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.InitiatorPaused); err != nil { + return err + } + // t.Stages (datatransfer.ChannelStages) (struct) if len("Stages") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Stages\" was too long") @@ -778,6 +810,42 @@ func (t *ChannelState) UnmarshalCBOR(r io.Reader) error { default: return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) } + // t.ResponderPaused (bool) (bool) + case "ResponderPaused": + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.ResponderPaused = false + case 21: + t.ResponderPaused = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + // t.InitiatorPaused (bool) (bool) + case "InitiatorPaused": + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.InitiatorPaused = false + case 21: + t.InitiatorPaused = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } // t.Stages (datatransfer.ChannelStages) (struct) case "Stages": diff --git a/errors.go b/errors.go index 0e9903f6..592444c0 100644 --- a/errors.go +++ b/errors.go @@ -17,14 +17,6 @@ const ErrHandlerNotSet = errorType("event handler has not been set") // ErrChannelNotFound means the channel this command was issued for does not exist const ErrChannelNotFound = errorType("channel not found") -// ErrPause is a special error that the DataReceived / DataSent hooks can -// use to pause the channel -const ErrPause = errorType("pause channel") - -// ErrResume is a special error that the RequestReceived / ResponseReceived hooks can -// use to resume the channel -const ErrResume = errorType("resume channel") - // ErrRejected indicates a request was not accepted const ErrRejected = errorType("response rejected") diff --git a/events.go b/events.go index 9ccd47c9..1cc42be8 100644 --- a/events.go +++ b/events.go @@ -61,7 +61,7 @@ const ( // initiator BeginFinalizing - // Disconnected emits when we are not able to connect to the other party + // DEPRECATED in favor of SendMessageError Disconnected // Complete is emitted when a data transfer is complete @@ -91,7 +91,7 @@ const ( // data has been received. DataReceivedProgress - // Deprecated in favour of RequestCancelled + // DEPRECATED in favour of RequestCancelled RequestTimedOut // SendDataError indicates that the transport layer had an error trying @@ -102,7 +102,7 @@ const ( // receiving data from the remote peer ReceiveDataError - // TransferRequestQueued indicates that a new data transfer request has been queued in the transport layer + // DEPRECATED in favor of TransferInitiated TransferRequestQueued // RequestCancelled indicates that a transport layer request was cancelled by the request opener @@ -123,6 +123,12 @@ const ( // pausing the responder, but is distinct from PauseResponder to indicate why the pause // happened DataLimitExceeded + + // TransferInitiated indicates the transport has begun transferring data + TransferInitiated + + // SendMessageError indicates an error sending a data transfer message + SendMessageError ) // Events are human readable names for data transfer events @@ -161,6 +167,8 @@ var Events = map[EventCode]string{ SetDataLimit: "SetDataLimit", SetRequiresFinalization: "SetRequiresFinalization", DataLimitExceeded: "DataLimitExceeded", + TransferInitiated: "TransferInitiated", + SendMessageError: "SendMessageError", } // Event is a struct containing information about a data transfer event diff --git a/go.mod b/go.mod index 6fd46cee..988d4813 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-cid v0.2.0 github.com/ipfs/go-datastore v0.5.1 github.com/ipfs/go-ds-badger v0.3.0 - github.com/ipfs/go-graphsync v0.13.2-0.20220613213258-f3c765731a53 + github.com/ipfs/go-graphsync v0.13.3-0.20220625074430-a95496cf1534 github.com/ipfs/go-ipfs-blockstore v1.1.2 github.com/ipfs/go-ipfs-blocksutil v0.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 @@ -119,7 +119,6 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/urfave/cli/v2 v2.0.0 // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect - go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.21.0 // indirect golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect diff --git a/go.sum b/go.sum index 845ab292..494c158a 100644 --- a/go.sum +++ b/go.sum @@ -126,7 +126,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= @@ -148,7 +147,6 @@ github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= @@ -196,7 +194,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -391,7 +388,6 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= @@ -441,10 +437,8 @@ github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIyk github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= -github.com/ipfs/go-graphsync v0.13.2-0.20220531040852-fa5a9f2d7a86 h1:PVLY+D9dz9SwQADbEaxLF5Kc+xOVP+SltDw3GvSdHmk= -github.com/ipfs/go-graphsync v0.13.2-0.20220531040852-fa5a9f2d7a86/go.mod h1:y8e8G6CmZeL9Srvx1l15CtGiRdf3h5JdQuqPz/iYL0A= -github.com/ipfs/go-graphsync v0.13.2-0.20220613213258-f3c765731a53 h1:TxhigBtTsopDclUsakL9ET5sE6zQOhrkwR0R0AV9Lbk= -github.com/ipfs/go-graphsync v0.13.2-0.20220613213258-f3c765731a53/go.mod h1:y8e8G6CmZeL9Srvx1l15CtGiRdf3h5JdQuqPz/iYL0A= +github.com/ipfs/go-graphsync v0.13.3-0.20220625074430-a95496cf1534 h1:sn7viAPyx3qZVhfRpXhW23mPtzl9rjJKtJ/HM/HsyZg= +github.com/ipfs/go-graphsync v0.13.3-0.20220625074430-a95496cf1534/go.mod h1:RKAui2+/HmlIVnuAXJIn0jltvOAXkl7wz3SYysmYnPI= github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE= github.com/ipfs/go-ipfs-blockstore v1.1.2 h1:WCXoZcMYnvOTmlpX+RSSnhVN0uCmbWTeepTGX5lgiXw= github.com/ipfs/go-ipfs-blockstore v1.1.2/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY= @@ -618,7 +612,6 @@ github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xS github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw= github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= -github.com/libp2p/go-libp2p v0.16.0/go.mod h1:ump42BsirwAWxKzsCiFnTtN1Yc+DuPu76fyMX364/O4= github.com/libp2p/go-libp2p v0.19.4 h1:50YL0YwPhWKDd+qbZQDEdnsmVAAkaCQrWUjpdHv4hNA= github.com/libp2p/go-libp2p v0.19.4/go.mod h1:MIt8y481VDhUe4ErWi1a4bvt/CjjFfOq6kZTothWIXY= github.com/libp2p/go-libp2p-asn-util v0.1.0 h1:rABPCO77SjdbJ/eJ/ynIo8vWICy1VEnL5JAxJbQLo1E= @@ -628,7 +621,6 @@ github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQ github.com/libp2p/go-libp2p-autonat v0.2.1/go.mod h1:MWtAhV5Ko1l6QBsHQNSuM6b1sRkXrpk0/LqCr+vCVxI= github.com/libp2p/go-libp2p-autonat v0.2.2/go.mod h1:HsM62HkqZmHR2k1xgX34WuWDzk/nBwNHoeyyT4IWV6A= github.com/libp2p/go-libp2p-autonat v0.4.2/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk= -github.com/libp2p/go-libp2p-autonat v0.6.0/go.mod h1:bFC6kY8jwzNNWoqc8iGE57vsfwyJ/lP4O4DOV1e0B2o= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= github.com/libp2p/go-libp2p-blankhost v0.2.0/go.mod h1:eduNKXGTioTuQAUcZ5epXi9vMl+t4d8ugUBRQ4SqaNQ= @@ -661,7 +653,6 @@ github.com/libp2p/go-libp2p-core v0.8.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJB github.com/libp2p/go-libp2p-core v0.8.2/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.5/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.6/go.mod h1:dgHr0l0hIKfWpGpqAMbpo19pen9wJfdCGv51mTmdpmM= -github.com/libp2p/go-libp2p-core v0.9.0/go.mod h1:ESsbz31oC3C1AvMJoGx26RTuCkNhmkSRCqZ0kQtJ2/8= github.com/libp2p/go-libp2p-core v0.10.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= github.com/libp2p/go-libp2p-core v0.11.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= github.com/libp2p/go-libp2p-core v0.12.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= @@ -672,7 +663,6 @@ github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxn github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-discovery v0.6.0/go.mod h1:/u1voHt0tKIe5oIA1RHBKQLVCWPna2dXmPNHc2zR9S8= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= @@ -689,7 +679,6 @@ github.com/libp2p/go-libp2p-nat v0.1.0/go.mod h1:DQzAG+QbDYjN1/C3B6vXucLtz3u9rEo github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ= github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= github.com/libp2p/go-libp2p-noise v0.2.0/go.mod h1:IEbYhBBzGyvdLBoxxULL/SGbJARhUeqlO8lVSREYu2Q= -github.com/libp2p/go-libp2p-noise v0.3.0/go.mod h1:JNjHbociDJKHD64KTkzGnzqJ0FEV5gHJa6AB00kbCNQ= github.com/libp2p/go-libp2p-noise v0.4.0 h1:khcMsGhHNdGqKE5LDLrnHwZvdGVMsrnD4GTkTWkwmLU= github.com/libp2p/go-libp2p-noise v0.4.0/go.mod h1:BzzY5pyzCYSyJbQy9oD8z5oP2idsafjt4/X42h9DjZU= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= @@ -707,7 +696,6 @@ github.com/libp2p/go-libp2p-pnet v0.2.0 h1:J6htxttBipJujEjz1y0a5+eYoiPcFHhSYHH6n github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA= github.com/libp2p/go-libp2p-quic-transport v0.13.0/go.mod h1:39/ZWJ1TW/jx1iFkKzzUg00W6tDJh73FC0xYudjr7Hc= -github.com/libp2p/go-libp2p-quic-transport v0.15.0/go.mod h1:wv4uGwjcqe8Mhjj7N/Ic0aKjA+/10UnMlSzLO0yRpYQ= github.com/libp2p/go-libp2p-quic-transport v0.16.0/go.mod h1:1BXjVMzr+w7EkPfiHkKnwsWjPjtfaNT0q8RS3tGDvEQ= github.com/libp2p/go-libp2p-quic-transport v0.17.0 h1:yFh4Gf5MlToAYLuw/dRvuzYd1EnE2pX3Lq1N6KDiWRQ= github.com/libp2p/go-libp2p-quic-transport v0.17.0/go.mod h1:x4pw61P3/GRCcSLypcQJE/Q2+E9f4X+5aRcZLXf20LM= @@ -738,7 +726,6 @@ github.com/libp2p/go-libp2p-testing v0.1.1/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eq github.com/libp2p/go-libp2p-testing v0.1.2-0.20200422005655-8775583591d8/go.mod h1:Qy8sAncLKpwXtS2dSnDOP8ktexIAHKu+J+pnZOFZLTc= github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehtsBXMrXnCfJIgDti5g= github.com/libp2p/go-libp2p-testing v0.4.0/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= -github.com/libp2p/go-libp2p-testing v0.4.2/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= github.com/libp2p/go-libp2p-testing v0.5.0/go.mod h1:QBk8fqIL1XNcno/l3/hhaIEn4aLRijpYOR+zVjjlh+A= github.com/libp2p/go-libp2p-testing v0.7.0/go.mod h1:OLbdn9DbgdMwv00v+tlp1l3oe2Cl+FAjoWIA2pa0X6E= github.com/libp2p/go-libp2p-testing v0.9.0/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= @@ -746,14 +733,12 @@ github.com/libp2p/go-libp2p-testing v0.9.2 h1:dCpODRtRaDZKF8HXT9qqqgON+OMEB423Kn github.com/libp2p/go-libp2p-testing v0.9.2/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= github.com/libp2p/go-libp2p-tls v0.3.0/go.mod h1:fwF5X6PWGxm6IDRwF3V8AVCCj/hOd5oFlg+wo2FxJDY= -github.com/libp2p/go-libp2p-tls v0.3.1/go.mod h1:fwF5X6PWGxm6IDRwF3V8AVCCj/hOd5oFlg+wo2FxJDY= github.com/libp2p/go-libp2p-tls v0.4.1 h1:1ByJUbyoMXvYXDoW6lLsMxqMViQNXmt+CfQqlnCpY+M= github.com/libp2p/go-libp2p-tls v0.4.1/go.mod h1:EKCixHEysLNDlLUoKxv+3f/Lp90O2EXNjTr0UQDnrIw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= github.com/libp2p/go-libp2p-transport-upgrader v0.4.2/go.mod h1:NR8ne1VwfreD5VIWIU62Agt/J18ekORFU/j1i2y8zvk= -github.com/libp2p/go-libp2p-transport-upgrader v0.4.3/go.mod h1:bpkldbOWXMrXhpZbSV1mQxTrefOg2Fi+k1ClDSA4ppw= github.com/libp2p/go-libp2p-transport-upgrader v0.5.0/go.mod h1:Rc+XODlB3yce7dvFV4q/RmyJGsFcCZRkeZMu/Zdg0mo= github.com/libp2p/go-libp2p-transport-upgrader v0.7.0/go.mod h1:GIR2aTRp1J5yjVlkUoFqMkdobfob6RnAwYg/RZPhrzg= github.com/libp2p/go-libp2p-transport-upgrader v0.7.1 h1:MSMe+tUfxpC9GArTz7a4G5zQKQgGh00Vio87d3j3xIg= @@ -766,7 +751,6 @@ github.com/libp2p/go-libp2p-yamux v0.2.8/go.mod h1:/t6tDqeuZf0INZMTgd0WxIRbtK2Ez github.com/libp2p/go-libp2p-yamux v0.4.0/go.mod h1:+DWDjtFMzoAwYLVkNZftoucn7PelNoy5nm3tZ3/Zw30= github.com/libp2p/go-libp2p-yamux v0.5.0/go.mod h1:AyR8k5EzyM2QN9Bbdg6X1SkVVuqLwTGf0L4DFq9g6po= github.com/libp2p/go-libp2p-yamux v0.5.4/go.mod h1:tfrXbyaTqqSU654GTvK3ocnSZL3BuHoeTSqhcel1wsE= -github.com/libp2p/go-libp2p-yamux v0.6.0/go.mod h1:MRhd6mAYnFRnSISp4M8i0ClV/j+mWHo2mYLifWGw33k= github.com/libp2p/go-libp2p-yamux v0.8.0/go.mod h1:yTkPgN2ib8FHyU1ZcVD7aelzyAqXXwEPbyx+aSKm9h8= github.com/libp2p/go-libp2p-yamux v0.8.1/go.mod h1:rUozF8Jah2dL9LLGyBaBeTQeARdwhefMCTQVQt6QobE= github.com/libp2p/go-libp2p-yamux v0.9.1 h1:oplewiRix8s45SOrI30rCPZG5mM087YZp+VYhXAh4+c= @@ -784,7 +768,6 @@ github.com/libp2p/go-mplex v0.4.0/go.mod h1:y26Lx+wNVtMYMaPu300Cbot5LkEZ4tJaNYeH github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= -github.com/libp2p/go-msgio v0.1.0/go.mod h1:eNlv2vy9V2X/kNldcZ+SShFE++o2Yjxwx6RAYsmgJnE= github.com/libp2p/go-msgio v0.2.0 h1:W6shmB+FeynDrUVl2dgFQvzfBZcXiyqY4VmpQLu9FqU= github.com/libp2p/go-msgio v0.2.0/go.mod h1:dBVM1gW3Jk9XqHkU4eKdGvVHdLa51hoGfll6jMJMSlY= github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= @@ -831,7 +814,6 @@ github.com/libp2p/go-tcp-transport v0.5.1/go.mod h1:UPPL0DIjQqiWRwVAb+CEQlaAG0rp github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM= github.com/libp2p/go-ws-transport v0.3.0/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= github.com/libp2p/go-ws-transport v0.4.0/go.mod h1:EcIEKqf/7GDjth6ksuS/6p7R49V4CBY6/E7R/iyhYUA= -github.com/libp2p/go-ws-transport v0.5.0/go.mod h1:I2juo1dNTbl8BKSBYo98XY85kU2xds1iamArLvl8kNg= github.com/libp2p/go-ws-transport v0.6.0 h1:326XBL6Q+5CQ2KtjXz32+eGu02W/Kz2+Fm4SpXdr0q4= github.com/libp2p/go-ws-transport v0.6.0/go.mod h1:dXqtI9e2JV9FtF1NOtWVZSKXh5zXvnuwPXfj8GPBbYU= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= @@ -843,7 +825,6 @@ github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/h github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI= github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux/v2 v2.2.0/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZjqROGxzPpPQ= -github.com/libp2p/go-yamux/v2 v2.3.0/go.mod h1:iTU+lOIn/2h0AgKcL49clNTwfEw+WSfDYrXe05EyKIs= github.com/libp2p/go-yamux/v3 v3.0.1/go.mod h1:s2LsDhHbh+RfCsQoICSYt58U2f8ijtPANFD8BmE74Bo= github.com/libp2p/go-yamux/v3 v3.0.2/go.mod h1:s2LsDhHbh+RfCsQoICSYt58U2f8ijtPANFD8BmE74Bo= github.com/libp2p/go-yamux/v3 v3.1.1/go.mod h1:jeLEQgLXqE2YqX1ilAClIfCMDY+0uXQUKmmb/qp0gT4= @@ -854,7 +835,6 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= github.com/lucas-clemente/quic-go v0.23.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0= -github.com/lucas-clemente/quic-go v0.24.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0= github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg= github.com/lucas-clemente/quic-go v0.27.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= github.com/lucas-clemente/quic-go v0.27.1 h1:sOw+4kFSVrdWOYmUjufQ9GBVPqZ+tu+jMtXxXNmRJyk= @@ -1096,7 +1076,6 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.33.0 h1:rHgav/0a6+uYgGdNt3jwz8FNSesO/Hsang3O0T9A5SE= github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= @@ -1122,7 +1101,6 @@ github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBO github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1205,7 +1183,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= @@ -1297,7 +1274,6 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= @@ -1333,7 +1309,6 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -1429,7 +1404,6 @@ golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -1537,7 +1511,6 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1711,7 +1684,6 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/impl/events.go b/impl/events.go index e81e0fee..a7c680fe 100644 --- a/impl/events.go +++ b/impl/events.go @@ -2,144 +2,88 @@ package impl import ( "context" + "errors" "fmt" - "github.com/ipld/go-ipld-prime" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - "golang.org/x/xerrors" - datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channels" "github.com/filecoin-project/go-data-transfer/v2/message" + "golang.org/x/xerrors" ) // OnChannelOpened is called when we send a request for data to the other // peer on the given channel ID -func (m *manager) OnChannelOpened(chid datatransfer.ChannelID) error { - log.Infof("channel %s: opened", chid) - - // Check if the channel is being tracked - has, err := m.channels.HasChannel(chid) +func (m *manager) OnTransportEvent(chid datatransfer.ChannelID, evt datatransfer.TransportEvent) { + ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) + err := m.processTransferEvent(ctx, chid, evt) if err != nil { - return err - } - if !has { - return datatransfer.ErrChannelNotFound + log.Infof("error on channel: %s, closing channel", err) + err := m.closeChannelWithError(ctx, chid, err) + if err != nil { + log.Errorf("error closing channel: %s", err) + } } - - // Fire an event - return m.channels.ChannelOpened(chid) } -// OnDataReceived is called when the transport layer reports that it has -// received some data from the sender. -// It fires an event on the channel, updating the sum of received data and reports -// back a pause to the transport if the data limit is exceeded -func (m *manager) OnDataReceived(chid datatransfer.ChannelID, link ipld.Link, size uint64, index int64, unique bool) error { - ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) - _, span := otel.Tracer("data-transfer").Start(ctx, "dataReceived", trace.WithAttributes( - attribute.String("channelID", chid.String()), - attribute.String("link", link.String()), - attribute.Int64("index", index), - attribute.Int64("size", int64(size)), - )) - defer span.End() - - err := m.channels.DataReceived(chid, link.(cidlink.Link).Cid, size, index, unique) - // if this channel is now paused, send the pause message - if err == datatransfer.ErrPause { +func (m *manager) processTransferEvent(ctx context.Context, chid datatransfer.ChannelID, transportEvent datatransfer.TransportEvent) error { + switch evt := transportEvent.(type) { + case datatransfer.TransportOpenedChannel: + return m.channels.ChannelOpened(chid) + case datatransfer.TransportInitiatedTransfer: + return m.channels.TransferInitiated(chid) + case datatransfer.TransportReceivedData: + return m.channels.DataReceived(chid, evt.Size, evt.Index) + case datatransfer.TransportSentData: + return m.channels.DataSent(chid, evt.Size, evt.Index) + case datatransfer.TransportQueuedData: + return m.channels.DataQueued(chid, evt.Size, evt.Index) + case datatransfer.TransportReachedDataLimit: + if err := m.channels.DataLimitExceeded(chid); err != nil { + return err + } msg := message.UpdateResponse(chid.ID, true) - ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) - if err := m.transport.SendMessage(ctx, chid, msg); err != nil { + return m.transport.SendMessage(ctx, chid, msg) + /*case datatransfer.TransportReceivedVoucherRequest: + voucher, err := evt.Request.TypedVoucher() + if err != nil { return err } - } - - return err -} - -// OnDataQueued is called when the transport layer reports that it has queued -// up some data to be sent to the requester. -// It fires an event on the channel, updating the sum of queued data and reports -// back a pause to the transport if the data limit is exceeded -func (m *manager) OnDataQueued(chid datatransfer.ChannelID, link ipld.Link, size uint64, index int64, unique bool) (datatransfer.Message, error) { - // The transport layer reports that some data has been queued up to be sent - // to the requester, so fire a DataQueued event on the channels state - // machine. - - ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) - _, span := otel.Tracer("data-transfer").Start(ctx, "dataQueued", trace.WithAttributes( - attribute.String("channelID", chid.String()), - attribute.String("link", link.String()), - attribute.Int64("size", int64(size)), - )) - defer span.End() - - var msg datatransfer.Message - err := m.channels.DataQueued(chid, link.(cidlink.Link).Cid, size, index, unique) - // if this channel is now paused, send the pause message - if err == datatransfer.ErrPause { - msg = message.UpdateResponse(chid.ID, true) - } - - return msg, err -} - -// OnDataSent is called when the transport layer reports that it has finished -// sending data to the requester. -func (m *manager) OnDataSent(chid datatransfer.ChannelID, link ipld.Link, size uint64, index int64, unique bool) error { - - ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) - _, span := otel.Tracer("data-transfer").Start(ctx, "dataSent", trace.WithAttributes( - attribute.String("channelID", chid.String()), - attribute.String("link", link.String()), - attribute.Int64("size", int64(size)), - )) - defer span.End() - - return m.channels.DataSent(chid, link.(cidlink.Link).Cid, size, index, unique) -} - -// OnRequestReceived is called when a Request message is received from the initiator -// on the responder -func (m *manager) OnRequestReceived(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { - - // if request is restart request, process as restart - if request.IsRestart() { - return m.receiveRestartRequest(chid, request) - } - - // if request is new request, process as new - if request.IsNew() { - return m.receiveNewRequest(chid, request) - } - - // if request is cancel request, process as cancel - if request.IsCancel() { + return m.channels.NewVoucher(chid, voucher) + case datatransfer.TransportReceivedUpdateRequest: + if evt.Request.IsPaused() { + return m.pauseOther(chid) + } + return m.resumeOther(chid) + case datatransfer.TransportReceivedCancelRequest: log.Infof("channel %s: received cancel request, cleaning up channel", chid) - - m.transport.CleanupChannel(chid) - return nil, m.channels.Cancel(chid) - } - - // if request contains a new voucher, process updated voucher - if request.IsVoucher() { - return m.processUpdateVoucher(chid, request) + return m.channels.Cancel(chid) + case datatransfer.TransportReceivedResponse: + return m.receiveResponse(chid, evt.Response)*/ + case datatransfer.TransportTransferCancelled: + log.Warnf("channel %+v was cancelled: %s", chid, evt.ErrorMessage) + return m.channels.RequestCancelled(chid, errors.New(evt.ErrorMessage)) + + case datatransfer.TransportErrorSendingData: + log.Debugf("channel %+v had transport send error: %s", chid, evt.ErrorMessage) + return m.channels.SendDataError(chid, errors.New(evt.ErrorMessage)) + case datatransfer.TransportErrorReceivingData: + log.Debugf("channel %+v had transport receive error: %s", chid, evt.ErrorMessage) + return m.channels.ReceiveDataError(chid, errors.New(evt.ErrorMessage)) + case datatransfer.TransportCompletedTransfer: + return m.channelCompleted(chid, evt.Success, evt.ErrorMessage) + case datatransfer.TransportReceivedRestartExistingChannelRequest: + return m.restartExistingChannelRequestReceived(chid) + case datatransfer.TransportErrorSendingMessage: + return m.channels.SendMessageError(chid, errors.New(evt.ErrorMessage)) + case datatransfer.TransportPaused: + return m.pause(chid) + case datatransfer.TransportResumed: + return m.resume(chid) } - - // otherwise process as an "update" message (i.e. a pause or resume) - return m.receiveUpdateRequest(chid, request) -} - -// OnTransferQueued is called when the transport layer receives a request but has not yet processed it -func (m *manager) OnTransferQueued(chid datatransfer.ChannelID) { - m.channels.TransferRequestQueued(chid) + return nil } -// OnRequestReceived is called when a Response message is received from the responder +// OnResponseReceived is called when a Response message is received from the responder // on the initiator func (m *manager) OnResponseReceived(chid datatransfer.ChannelID, response datatransfer.Response) error { @@ -217,34 +161,19 @@ func (m *manager) OnResponseReceived(chid datatransfer.ChannelID, response datat return m.resumeOther(chid) } -// OnRequestCancelled is called when a transport reports a channel is cancelled -func (m *manager) OnRequestCancelled(chid datatransfer.ChannelID, err error) error { - log.Warnf("channel %+v was cancelled: %s", chid, err) - return m.channels.RequestCancelled(chid, err) -} - -// OnRequestCancelled is called when a transport reports a channel disconnected -func (m *manager) OnRequestDisconnected(chid datatransfer.ChannelID, err error) error { - log.Warnf("channel %+v has stalled or disconnected: %s", chid, err) - return m.channels.Disconnected(chid, err) -} - -// OnSendDataError is called when a transport has a network error sending data -func (m *manager) OnSendDataError(chid datatransfer.ChannelID, err error) error { - log.Debugf("channel %+v had transport send error: %s", chid, err) - return m.channels.SendDataError(chid, err) -} - -// OnReceiveDataError is called when a transport has a network error receiving data -func (m *manager) OnReceiveDataError(chid datatransfer.ChannelID, err error) error { - log.Debugf("channel %+v had transport receive error: %s", chid, err) - return m.channels.ReceiveDataError(chid, err) +// OnContextAugment provides an oppurtunity for transports to have data transfer add data to their context (i.e. +// to tie into tracing, etc) +func (m *manager) OnContextAugment(chid datatransfer.ChannelID) func(context.Context) context.Context { + return func(ctx context.Context) context.Context { + ctx, _ = m.spansIndex.SpanForChannel(ctx, chid) + return ctx + } } -// OnChannelCompleted is called +// channelCompleted is called // - by the requester when all data for a transfer has been received // - by the responder when all data for a transfer has been sent -func (m *manager) OnChannelCompleted(chid datatransfer.ChannelID, completeErr error) error { +func (m *manager) channelCompleted(chid datatransfer.ChannelID, success bool, errorMessage string) error { // read the channel state chst, err := m.channels.GetByID(context.TODO(), chid) @@ -253,10 +182,10 @@ func (m *manager) OnChannelCompleted(chid datatransfer.ChannelID, completeErr er } // If the transferred errored on completion - if completeErr != nil { - // send an error, but only if we haven't already errored for some reason - if chst.Status() != datatransfer.Failing && chst.Status() != datatransfer.Failed { - err := xerrors.Errorf("data transfer channel %s failed to transfer data: %w", chid, completeErr) + if !success { + // send an error, but only if we haven't already errored/finished transfer already for some reason + if !chst.Status().TransferComplete() { + err := fmt.Errorf("data transfer channel %s failed to transfer data: %s", chid, errorMessage) log.Warnf(err.Error()) return m.channels.Error(chid, err) } @@ -273,16 +202,13 @@ func (m *manager) OnChannelCompleted(chid datatransfer.ChannelID, completeErr er log.Infow("received OnChannelCompleted, will send completion message to initiator", "chid", chid) // generate and send the final status message - msg, err := message.CompleteResponse(chst.TransferID(), true, chst.RequiresFinalization(), nil) - if err != nil { - return err - } + msg := message.CompleteResponse(chst.TransferID(), true, chst.RequiresFinalization(), nil) log.Infow("sending completion message to initiator", "chid", chid) ctx, _ := m.spansIndex.SpanForChannel(context.Background(), chid) if err := m.transport.SendMessage(ctx, chid, msg); err != nil { err := xerrors.Errorf("channel %s: failed to send completion message to initiator: %w", chid, err) log.Warnw("failed to send completion message to initiator", "chid", chid, "err", err) - return m.OnRequestDisconnected(chid, err) + return m.channels.SendMessageError(chid, err) } log.Infow("successfully sent completion message to initiator", "chid", chid) @@ -293,16 +219,7 @@ func (m *manager) OnChannelCompleted(chid datatransfer.ChannelID, completeErr er return m.channels.Complete(chid) } -// OnContextAugment provides an oppurtunity for transports to have data transfer add data to their context (i.e. -// to tie into tracing, etc) -func (m *manager) OnContextAugment(chid datatransfer.ChannelID) func(context.Context) context.Context { - return func(ctx context.Context) context.Context { - ctx, _ = m.spansIndex.SpanForChannel(ctx, chid) - return ctx - } -} - -func (m *manager) OnRestartExistingChannelRequestReceived(chid datatransfer.ChannelID) error { +func (m *manager) restartExistingChannelRequestReceived(chid datatransfer.ChannelID) error { ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) // validate channel exists -> in non-terminal state and that the sender matches channel, err := m.channels.GetByID(context.TODO(), chid) @@ -319,5 +236,6 @@ func (m *manager) OnRestartExistingChannelRequestReceived(chid datatransfer.Chan if err := m.openRestartChannel(ctx, channel); err != nil { return fmt.Errorf("failed to open restart channel %s: %s", chid, err) } + return nil } diff --git a/impl/impl.go b/impl/impl.go index 16a6f462..156869b6 100644 --- a/impl/impl.go +++ b/impl/impl.go @@ -252,15 +252,10 @@ func (m *manager) SendVoucher(ctx context.Context, channelID datatransfer.Channe span.SetStatus(codes.Error, err.Error()) return err } - updateRequest, err := message.VoucherRequest(channelID.ID, &voucher) - if err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - return err - } + updateRequest := message.VoucherRequest(channelID.ID, &voucher) if err := m.transport.SendMessage(ctx, channelID, updateRequest); err != nil { err = fmt.Errorf("Unable to send request: %w", err) - _ = m.OnRequestDisconnected(channelID, err) + _ = m.channels.SendMessageError(channelID, err) span.RecordError(err) span.SetStatus(codes.Error, err.Error()) return err @@ -287,19 +282,14 @@ func (m *manager) SendVoucherResult(ctx context.Context, channelID datatransfer. var updateResponse datatransfer.Response if chst.Status().InFinalization() { - updateResponse, err = message.CompleteResponse(channelID.ID, chst.Status().IsAccepted(), chst.Status().IsResponderPaused(), &voucherResult) + updateResponse = message.CompleteResponse(channelID.ID, chst.Status().IsAccepted(), chst.ResponderPaused(), &voucherResult) } else { - updateResponse, err = message.VoucherResultResponse(channelID.ID, chst.Status().IsAccepted(), chst.Status().IsResponderPaused(), &voucherResult) + updateResponse = message.VoucherResultResponse(channelID.ID, chst.Status().IsAccepted(), chst.ResponderPaused(), &voucherResult) } - if err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - return err - } if err := m.transport.SendMessage(ctx, channelID, updateResponse); err != nil { err = fmt.Errorf("Unable to send request: %w", err) - _ = m.OnRequestDisconnected(channelID, err) + _ = m.channels.SendMessageError(channelID, err) span.RecordError(err) span.SetStatus(codes.Error, err.Error()) return err @@ -331,33 +321,16 @@ func (m *manager) updateValidationStatus(ctx context.Context, chid datatransfer. return err } - // dispatch channel events and generate a response message - chst, response, err := m.processValidationUpdate(ctx, chid, result) - - // dispatch transport updates - return m.transport.UpdateChannel(ctx, chid, datatransfer.ChannelUpdate{ - Paused: result.LeaveRequestPaused(chst), - Closed: err != nil || !result.Accepted, - SendMessage: response, - }) -} - -func (m *manager) processValidationUpdate(ctx context.Context, chid datatransfer.ChannelID, result datatransfer.ValidationResult) (datatransfer.ChannelState, datatransfer.Response, error) { - // read the channel state chst, err := m.channels.GetByID(context.TODO(), chid) if err != nil { - return nil, nil, err + return err } - // if the request is now rejected, error the channel - if !result.Accepted { - err = m.recordRejectedValidationEvents(chid, result) - } else { - err = m.recordAcceptedValidationEvents(chst, result) - } + // dispatch channel events and generate a response message + err = m.processValidationUpdate(ctx, chst, result) if err != nil { - return nil, nil, err + return err } // generate a response message @@ -365,100 +338,105 @@ func (m *manager) processValidationUpdate(ctx context.Context, chid datatransfer if chst.Status() == datatransfer.Finalizing { messageType = types.CompleteMessage } - response, msgErr := message.ValidationResultResponse(messageType, chst.TransferID(), result, err, + response := message.ValidationResultResponse(messageType, chid.ID, result, err, result.LeaveRequestPaused(chst)) - if msgErr != nil { - return nil, nil, msgErr + + // dispatch transport updates + return m.transport.ChannelUpdated(ctx, chid, response) +} + +func (m *manager) processValidationUpdate(ctx context.Context, chst datatransfer.ChannelState, result datatransfer.ValidationResult) error { + // if the request is now rejected, error the channel + if !result.Accepted { + return m.recordRejectedValidationEvents(chst.ChannelID(), result) } + return m.recordAcceptedValidationEvents(chst, result) - // return the response message and any errors - return chst, response, nil } // close an open channel (effectively a cancel) func (m *manager) CloseDataTransferChannel(ctx context.Context, chid datatransfer.ChannelID) error { log.Infof("close channel %s", chid) - chst, err := m.channels.GetByID(ctx, chid) - if err != nil { - return err - } ctx, _ = m.spansIndex.SpanForChannel(ctx, chid) ctx, span := otel.Tracer("data-transfer").Start(ctx, "closeChannel", trace.WithAttributes( attribute.String("channelID", chid.String()), )) defer span.End() - // Close the channel on the local transport - err = m.transport.UpdateChannel(ctx, chid, datatransfer.ChannelUpdate{ - Paused: chst.Status().IsResponderPaused(), - Closed: true, - }) + + err := m.closeChannel(ctx, chid) if err != nil { span.RecordError(err) span.SetStatus(codes.Error, err.Error()) log.Warnf("unable to close channel %s: %s", chid, err) } + return err +} + +func (m *manager) closeChannel(ctx context.Context, chid datatransfer.ChannelID) error { + // Fire a cancel event + err := m.channels.Cancel(chid) + if err != nil { + return xerrors.Errorf("unable to send cancel to channel FSM: %w", err) + } + + // Close the channel on the local transport + err = m.transport.ChannelUpdated(ctx, chid, nil) // Send a cancel message to the remote peer async go func() { sctx, cancel := context.WithTimeout(context.Background(), cancelSendTimeout) defer cancel() - log.Infof("%s: sending cancel channel to %s for channel %s", m.peerID, chst.OtherPeer(), chid) + log.Infof("%s: sending cancel channel to %s for channel %s", m.peerID, m.otherPeer(chid), chid) err = m.transport.SendMessage(sctx, chid, m.cancelMessage(chid)) if err != nil { err = fmt.Errorf("unable to send cancel message for channel %s to peer %s: %w", chid, m.peerID, err) - _ = m.OnRequestDisconnected(chid, err) log.Warn(err) } }() - // Fire a cancel event - fsmerr := m.channels.Cancel(chid) - if fsmerr != nil { - return xerrors.Errorf("unable to send cancel to channel FSM: %w", fsmerr) - } - - return nil + return err } // close an open channel and fire an error event func (m *manager) CloseDataTransferChannelWithError(ctx context.Context, chid datatransfer.ChannelID, cherr error) error { log.Infof("close channel %s with error %s", chid, cherr) - chst, err := m.channels.GetByID(ctx, chid) - if err != nil { - return err - } ctx, _ = m.spansIndex.SpanForChannel(ctx, chid) ctx, span := otel.Tracer("data-transfer").Start(ctx, "closeChannel", trace.WithAttributes( attribute.String("channelID", chid.String()), )) defer span.End() + err := m.closeChannelWithError(ctx, chid, cherr) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + log.Warnf("unable to close channel %s: %s", chid, err) + } + return err +} + +func (m *manager) closeChannelWithError(ctx context.Context, chid datatransfer.ChannelID, cherr error) error { + + // Fire an error event + if err := m.channels.Error(chid, cherr); err != nil { + return xerrors.Errorf("unable to send error %s to channel FSM: %w", cherr, err) + } + // Close transfport and try to send a cancel message to the remote peer. // It's quite likely we aren't able to send the message to the peer because // the channel is already in an error state, which is probably because of // connection issues, so if we cant send the message just log a warning. - log.Infof("%s: sending cancel channel to %s for channel %s", m.peerID, chst.OtherPeer(), chid) - err = m.transport.UpdateChannel(ctx, chid, datatransfer.ChannelUpdate{ - Paused: chst.Status().IsResponderPaused(), - Closed: true, - SendMessage: m.cancelMessage(chid), - }) - if err != nil { + log.Infof("%s: sending cancel channel to %s for channel %s", m.peerID, m.otherPeer(chid), chid) + + if err := m.transport.ChannelUpdated(ctx, chid, m.cancelMessage(chid)); err != nil { // Just log a warning here because it's important that we fire the // error event with the original error so that it doesn't get masked // by subsequent errors. log.Warnf("unable to close channel %s: %s", chid, err) } - - // Fire an error event - err = m.channels.Error(chid, cherr) - if err != nil { - return xerrors.Errorf("unable to send error %s to channel FSM: %w", cherr, err) - } - return nil } @@ -472,15 +450,16 @@ func (m *manager) PauseDataTransferChannel(ctx context.Context, chid datatransfe ctx, _ = m.spansIndex.SpanForChannel(ctx, chid) - err := m.transport.UpdateChannel(ctx, chid, datatransfer.ChannelUpdate{ - Paused: true, - SendMessage: m.pauseMessage(chid), - }) - if err != nil { - log.Warnf("Error attempting to pause at transport level: %s", err.Error()) + // fire the pause + if err := m.pause(chid); err != nil { + return err } - return m.pause(chid) + // update transport + if err := m.transport.ChannelUpdated(ctx, chid, m.pauseMessage(chid)); err != nil { + log.Warnf("Error attempting to pause at transport level: %s", err.Error()) + } + return nil } // resume a running data transfer channel @@ -493,15 +472,16 @@ func (m *manager) ResumeDataTransferChannel(ctx context.Context, chid datatransf ctx, _ = m.spansIndex.SpanForChannel(ctx, chid) - err := m.transport.UpdateChannel(ctx, chid, datatransfer.ChannelUpdate{ - Paused: false, - SendMessage: m.resumeMessage(chid), - }) - if err != nil { - log.Warnf("Error attempting to resume at transport level: %s", err.Error()) + // fire the resume + if err := m.resume(chid); err != nil { + return err } - return m.resume(chid) + // update transport + if err := m.transport.ChannelUpdated(ctx, chid, m.resumeMessage(chid)); err != nil { + log.Warnf("Error attempting to resume at transport level: %s", err.Error()) + } + return nil } // get channel state diff --git a/impl/initiating_test.go b/impl/initiating_test.go index f6c4f0ec..a8da316e 100644 --- a/impl/initiating_test.go +++ b/impl/initiating_test.go @@ -10,7 +10,7 @@ import ( "github.com/ipfs/go-datastore" dss "github.com/ipfs/go-datastore/sync" "github.com/ipld/go-ipld-prime/datamodel" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" @@ -139,8 +139,7 @@ func TestDataTransferInitiating(t *testing.T) { channelID, err := h.dt.OpenPushDataChannel(h.ctx, h.peers[1], h.voucher, h.baseCid, h.stor) require.NoError(t, err) require.NotEmpty(t, channelID) - response, err := message.NewResponse(channelID.ID, true, false, nil) - require.NoError(t, err) + response := message.NewResponse(channelID.ID, true, false, nil) err = h.transport.EventHandler.OnResponseReceived(channelID, response) require.NoError(t, err) }, @@ -151,8 +150,7 @@ func TestDataTransferInitiating(t *testing.T) { channelID, err := h.dt.OpenPushDataChannel(h.ctx, h.peers[1], h.voucher, h.baseCid, h.stor) require.NoError(t, err) require.NotEmpty(t, channelID) - response, err := message.NewResponse(channelID.ID, true, false, &h.voucherResult) - require.NoError(t, err) + response := message.NewResponse(channelID.ID, true, false, &h.voucherResult) err = h.transport.EventHandler.OnResponseReceived(channelID, response) require.NoError(t, err) }, @@ -163,14 +161,13 @@ func TestDataTransferInitiating(t *testing.T) { channelID, err := h.dt.OpenPushDataChannel(h.ctx, h.peers[1], h.voucher, h.baseCid, h.stor) require.NoError(t, err) require.NotEmpty(t, channelID) - response, err := message.NewResponse(channelID.ID, true, false, nil) - require.NoError(t, err) + response := message.NewResponse(channelID.ID, true, false, nil) err = h.transport.EventHandler.OnResponseReceived(channelID, response) require.NoError(t, err) err = h.dt.PauseDataTransferChannel(h.ctx, channelID) require.NoError(t, err) - require.Len(t, h.transport.PausedChannels, 1) - require.Equal(t, h.transport.PausedChannels[0], channelID) + require.Len(t, h.transport.ChannelsUpdated, 1) + require.Equal(t, h.transport.ChannelsUpdated[0], channelID) require.Len(t, h.transport.OpenedChannels, 1) require.Len(t, h.transport.MessagesSent, 1) pauseMessage := h.transport.MessagesSent[0].Message @@ -181,8 +178,8 @@ func TestDataTransferInitiating(t *testing.T) { require.Equal(t, pauseMessage.TransferID(), channelID.ID) err = h.dt.ResumeDataTransferChannel(h.ctx, channelID) require.NoError(t, err) - require.Len(t, h.transport.ResumedChannels, 1) - resumedChannel := h.transport.ResumedChannels[0] + require.Len(t, h.transport.ChannelsUpdated, 2) + resumedChannel := h.transport.ChannelsUpdated[1] require.Equal(t, resumedChannel, channelID) require.Len(t, h.transport.MessagesSent, 2) resumeMessage := h.transport.MessagesSent[1].Message @@ -201,8 +198,8 @@ func TestDataTransferInitiating(t *testing.T) { require.NotEmpty(t, channelID) err = h.dt.CloseDataTransferChannel(h.ctx, channelID) require.NoError(t, err) - require.Len(t, h.transport.ClosedChannels, 1) - require.Equal(t, h.transport.ClosedChannels[0], channelID) + require.Len(t, h.transport.ChannelsUpdated, 1) + require.Equal(t, h.transport.ChannelsUpdated[0], channelID) require.Eventually(t, func() bool { return len(h.transport.MessagesSent) == 1 @@ -221,13 +218,12 @@ func TestDataTransferInitiating(t *testing.T) { channelID, err := h.dt.OpenPullDataChannel(h.ctx, h.peers[1], h.voucher, h.baseCid, h.stor) require.NoError(t, err) require.NotEmpty(t, channelID) - response, err := message.NewResponse(channelID.ID, true, false, nil) - require.NoError(t, err) + response := message.NewResponse(channelID.ID, true, false, nil) err = h.transport.EventHandler.OnResponseReceived(channelID, response) require.NoError(t, err) err = h.dt.PauseDataTransferChannel(h.ctx, channelID) require.NoError(t, err) - require.Len(t, h.transport.PausedChannels, 1) + require.Len(t, h.transport.ChannelsUpdated, 1) require.Len(t, h.transport.OpenedChannels, 1) require.Len(t, h.transport.MessagesSent, 1) pauseMessage := h.transport.MessagesSent[0].Message @@ -238,8 +234,8 @@ func TestDataTransferInitiating(t *testing.T) { require.Equal(t, pauseMessage.TransferID(), channelID.ID) err = h.dt.ResumeDataTransferChannel(h.ctx, channelID) require.NoError(t, err) - require.Len(t, h.transport.ResumedChannels, 1) - resumedChannel := h.transport.ResumedChannels[0] + require.Len(t, h.transport.ChannelsUpdated, 2) + resumedChannel := h.transport.ChannelsUpdated[1] require.Equal(t, resumedChannel, channelID) require.Len(t, h.transport.MessagesSent, 2) resumeMessage := h.transport.MessagesSent[1].Message @@ -258,8 +254,8 @@ func TestDataTransferInitiating(t *testing.T) { require.NotEmpty(t, channelID) err = h.dt.CloseDataTransferChannel(h.ctx, channelID) require.NoError(t, err) - require.Len(t, h.transport.ClosedChannels, 1) - require.Equal(t, h.transport.ClosedChannels[0], channelID) + require.Len(t, h.transport.ChannelsUpdated, 1) + require.Equal(t, h.transport.ChannelsUpdated[0], channelID) require.Eventually(t, func() bool { return len(h.transport.MessagesSent) == 1 @@ -353,7 +349,7 @@ func TestDataTransferRestartInitiating(t *testing.T) { verify func(t *testing.T, h *harness) }{ "RestartDataTransferChannel: Manager Peer Create Pull Restart works": { - expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.DataReceivedProgress, datatransfer.DataReceived, datatransfer.DataReceivedProgress, datatransfer.DataReceived}, + expectedEvents: []datatransfer.EventCode{datatransfer.Open, datatransfer.TransferInitiated, datatransfer.DataReceivedProgress, datatransfer.DataReceived, datatransfer.DataReceivedProgress, datatransfer.DataReceived}, verify: func(t *testing.T, h *harness) { // open a pull channel channelID, err := h.dt.OpenPullDataChannel(h.ctx, h.peers[1], h.voucher, h.baseCid, h.stor) @@ -362,11 +358,11 @@ func TestDataTransferRestartInitiating(t *testing.T) { require.Len(t, h.transport.OpenedChannels, 1) // some cids should already be received - testCids := testutil.GenerateCids(2) ev, ok := h.dt.(datatransfer.EventsHandler) require.True(t, ok) - require.NoError(t, ev.OnDataReceived(channelID, cidlink.Link{Cid: testCids[0]}, 12345, 1, true)) - require.NoError(t, ev.OnDataReceived(channelID, cidlink.Link{Cid: testCids[1]}, 12345, 2, true)) + ev.OnTransportEvent(channelID, datatransfer.TransportInitiatedTransfer{}) + ev.OnTransportEvent(channelID, datatransfer.TransportReceivedData{Size: 12345, Index: basicnode.NewInt(1)}) + ev.OnTransportEvent(channelID, datatransfer.TransportReceivedData{Size: 12345, Index: basicnode.NewInt(2)}) // restart that pull channel err = h.dt.RestartDataTransferChannel(ctx, channelID) diff --git a/impl/receiving_requests.go b/impl/receiving_requests.go index 7665a930..825ed3f2 100644 --- a/impl/receiving_requests.go +++ b/impl/receiving_requests.go @@ -13,7 +13,31 @@ import ( "github.com/filecoin-project/go-data-transfer/v2/message/types" ) -// this file contains methods for processing incoming request messages +func (m *manager) OnRequestReceived(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { + + // if request is restart request, process as restart + if request.IsRestart() { + return m.receiveRestartRequest(chid, request) + } + + // if request is new request, process as new + if request.IsNew() { + return m.receiveNewRequest(chid, request) + } + + // if request is cancel request, process as cancel + if request.IsCancel() { + log.Infof("channel %s: received cancel request, cleaning up channel", chid) + + return nil, m.channels.Cancel(chid) + } + // if request contains a new voucher, process updated voucher + if request.IsVoucher() { + return m.processUpdateVoucher(chid, request) + } + // otherwise process as an "update" message (i.e. a pause or resume) + return m.receiveUpdateRequest(chid, request) +} // receiveNewRequest handles an incoming new request message func (m *manager) receiveNewRequest(chid datatransfer.ChannelID, incoming datatransfer.Request) (datatransfer.Response, error) { @@ -23,13 +47,13 @@ func (m *manager) receiveNewRequest(chid datatransfer.ChannelID, incoming datatr result, err := m.acceptRequest(chid, incoming) // generate a response message - msg, msgErr := message.ValidationResultResponse(types.NewMessage, incoming.TransferID(), result, err, result.ForcePause) - if msgErr != nil { - return nil, msgErr - } + msg := message.ValidationResultResponse(types.NewMessage, incoming.TransferID(), result, err, result.ForcePause) - // return the response message and any errors - return msg, m.requestError(result, err, result.ForcePause) + // return the channel update + if err == nil && !result.Accepted { + err = datatransfer.ErrRejected + } + return msg, err } // acceptRequest performs processing (including validation) on a new incoming request @@ -126,13 +150,13 @@ func (m *manager) receiveRestartRequest(chid datatransfer.ChannelID, incoming da stayPaused, result, err := m.restartRequest(chid, incoming) // generate a response message - msg, msgErr := message.ValidationResultResponse(types.RestartMessage, incoming.TransferID(), result, err, stayPaused) - if msgErr != nil { - return nil, msgErr - } + msg := message.ValidationResultResponse(types.RestartMessage, incoming.TransferID(), result, err, stayPaused) // return the response message and any errors - return msg, m.requestError(result, err, result.ForcePause) + if err == nil && !result.Accepted { + err = datatransfer.ErrRejected + } + return msg, err } // restartRequest performs processing (including validation) on a incoming restart request @@ -198,55 +222,6 @@ func (m *manager) restartRequest(chid datatransfer.ChannelID, return stayPaused, result, nil } -// processUpdateVoucher handles an incoming request message with an updated voucher -func (m *manager) processUpdateVoucher(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { - // decode the voucher and save it on the channel - voucher, err := request.TypedVoucher() - if err != nil { - return nil, err - } - return nil, m.channels.NewVoucher(chid, voucher) -} - -// receiveUpdateRequest handles an incoming request message change in pause status -func (m *manager) receiveUpdateRequest(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { - - if request.IsPaused() { - return nil, m.pauseOther(chid) - } - - err := m.resumeOther(chid) - if err != nil { - return nil, err - } - chst, err := m.channels.GetByID(context.TODO(), chid) - if err != nil { - return nil, err - } - if chst.Status() == datatransfer.ResponderPaused || - chst.Status() == datatransfer.ResponderFinalizing { - return nil, datatransfer.ErrPause - } - return nil, nil -} - -// requestError generates an error message for the transport, adding -// ErrPause / ErrResume based off the validation result -// TODO: get away from using ErrPause/ErrResume to indicate pause resume, -// which would remove the need for most of this method -func (m *manager) requestError(result datatransfer.ValidationResult, resultErr error, stayPaused bool) error { - if resultErr != nil { - return resultErr - } - if !result.Accepted { - return datatransfer.ErrRejected - } - if stayPaused { - return datatransfer.ErrPause - } - return nil -} - // recordRejectedValidationEvents sends changes based on an reject validation to the state machine func (m *manager) recordRejectedValidationEvents(chid datatransfer.ChannelID, result datatransfer.ValidationResult) error { if result.VoucherResult != nil { @@ -288,14 +263,14 @@ func (m *manager) recordAcceptedValidationEvents(chst datatransfer.ChannelState, // pause or resume the request as neccesary if result.LeaveRequestPaused(chst) { - if !chst.Status().IsResponderPaused() { + if !chst.ResponderPaused() { err := m.channels.PauseResponder(chid) if err != nil { return err } } } else { - if chst.Status().IsResponderPaused() { + if chst.ResponderPaused() { err := m.channels.ResumeResponder(chid) if err != nil { return err @@ -306,6 +281,24 @@ func (m *manager) recordAcceptedValidationEvents(chst datatransfer.ChannelState, return nil } +// processUpdateVoucher handles an incoming request message with an updated voucher +func (m *manager) processUpdateVoucher(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { + // decode the voucher and save it on the channel + voucher, err := request.TypedVoucher() + if err != nil { + return nil, err + } + return nil, m.channels.NewVoucher(chid, voucher) +} + +// receiveUpdateRequest handles an incoming request message change in pause status +func (m *manager) receiveUpdateRequest(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { + if request.IsPaused() { + return nil, m.pauseOther(chid) + } + return nil, m.resumeOther(chid) +} + // validateRestart looks up the appropriate validator and validates a restart func (m *manager) validateRestart(chst datatransfer.ChannelState) (datatransfer.ValidationResult, error) { chv := chst.Voucher() diff --git a/impl/responding_test.go b/impl/responding_test.go index 1537a485..92abd7e1 100644 --- a/impl/responding_test.go +++ b/impl/responding_test.go @@ -12,12 +12,11 @@ import ( dss "github.com/ipfs/go-datastore/sync" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" . "github.com/filecoin-project/go-data-transfer/v2/impl" @@ -107,7 +106,7 @@ func TestDataTransferResponding(t *testing.T) { }, verify: func(t *testing.T, h *receiverHarness) { response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) - require.EqualError(t, err, datatransfer.ErrPause.Error()) + require.NoError(t, err) require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) @@ -185,7 +184,7 @@ func TestDataTransferResponding(t *testing.T) { }, verify: func(t *testing.T, h *receiverHarness) { response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) - require.EqualError(t, err, datatransfer.ErrPause.Error()) + require.NoError(t, err) require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) require.False(t, response.IsUpdate()) @@ -302,7 +301,7 @@ func TestDataTransferResponding(t *testing.T) { err = h.dt.PauseDataTransferChannel(h.ctx, channelID(h.id, h.peers)) require.NoError(t, err) _, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.resumeUpdate) - require.EqualError(t, err, datatransfer.ErrPause.Error()) + require.NoError(t, err) }, }, "receive cancel": { @@ -322,8 +321,6 @@ func TestDataTransferResponding(t *testing.T) { _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.cancelUpdate) require.NoError(t, err) - require.Len(t, h.transport.CleanedUpChannels, 1) - require.Equal(t, channelID(h.id, h.peers), h.transport.CleanedUpChannels[0]) }, }, "validate and revalidate successfully, push": { @@ -332,6 +329,7 @@ func TestDataTransferResponding(t *testing.T) { datatransfer.Accept, datatransfer.NewVoucherResult, datatransfer.SetDataLimit, + datatransfer.TransferInitiated, datatransfer.DataReceivedProgress, datatransfer.DataReceived, datatransfer.DataLimitExceeded, @@ -347,8 +345,9 @@ func TestDataTransferResponding(t *testing.T) { }, verify: func(t *testing.T, h *receiverHarness) { _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) - err := h.transport.EventHandler.OnDataReceived(channelID(h.id, h.peers), cidlink.Link{Cid: testutil.GenerateCids(1)[0]}, 12345, 1, true) - require.EqualError(t, err, datatransfer.ErrPause.Error()) + h.transport.EventHandler.OnTransportEvent(channelID(h.id, h.peers), datatransfer.TransportInitiatedTransfer{}) + h.transport.EventHandler.OnTransportEvent(channelID(h.id, h.peers), datatransfer.TransportReceivedData{Size: 12345, Index: basicnode.NewInt(1)}) + h.transport.EventHandler.OnTransportEvent(channelID(h.id, h.peers), datatransfer.TransportReachedDataLimit{}) require.Len(t, h.transport.MessagesSent, 1) response, ok := h.transport.MessagesSent[0].Message.(datatransfer.Response) require.True(t, ok) @@ -357,14 +356,14 @@ func TestDataTransferResponding(t *testing.T) { require.False(t, response.IsCancel()) require.True(t, response.IsPaused()) require.False(t, response.IsValidationResult()) - response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) - require.NoError(t, err, nil) + response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) + require.NoError(t, err) require.Nil(t, response) vr := testutil.NewTestTypedVoucher() err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: &vr}) require.NoError(t, err) - require.Len(t, h.transport.ResumedChannels, 1) - require.Equal(t, h.transport.ResumedChannels[0], channelID(h.id, h.peers)) + require.Len(t, h.transport.ChannelsUpdated, 1) + require.Equal(t, h.transport.ChannelsUpdated[0], channelID(h.id, h.peers)) require.Len(t, h.transport.MessagesSent, 2) response, ok = h.transport.MessagesSent[1].Message.(datatransfer.Response) require.True(t, ok) @@ -383,6 +382,7 @@ func TestDataTransferResponding(t *testing.T) { datatransfer.Accept, datatransfer.NewVoucherResult, datatransfer.SetDataLimit, + datatransfer.TransferInitiated, datatransfer.DataReceivedProgress, datatransfer.DataReceived, datatransfer.DataLimitExceeded, @@ -398,8 +398,9 @@ func TestDataTransferResponding(t *testing.T) { }, verify: func(t *testing.T, h *receiverHarness) { _, _ = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pushRequest) - err := h.transport.EventHandler.OnDataReceived(channelID(h.id, h.peers), cidlink.Link{Cid: testutil.GenerateCids(1)[0]}, 12345, 1, true) - require.EqualError(t, err, datatransfer.ErrPause.Error()) + h.transport.EventHandler.OnTransportEvent(channelID(h.id, h.peers), datatransfer.TransportInitiatedTransfer{}) + h.transport.EventHandler.OnTransportEvent(channelID(h.id, h.peers), datatransfer.TransportReceivedData{Size: 12345, Index: basicnode.NewInt(1)}) + h.transport.EventHandler.OnTransportEvent(channelID(h.id, h.peers), datatransfer.TransportReachedDataLimit{}) require.Len(t, h.transport.MessagesSent, 1) response, ok := h.transport.MessagesSent[0].Message.(datatransfer.Response) require.True(t, ok) @@ -408,14 +409,14 @@ func TestDataTransferResponding(t *testing.T) { require.False(t, response.IsCancel()) require.True(t, response.IsPaused()) require.False(t, response.IsValidationResult()) - response, err = h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) - require.NoError(t, err, nil) + response, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.voucherUpdate) + require.NoError(t, err) require.Nil(t, response) vr := testutil.NewTestTypedVoucher() err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: false, VoucherResult: &vr}) require.NoError(t, err) - require.Len(t, h.transport.ClosedChannels, 1) - require.Equal(t, h.transport.ClosedChannels[0], channelID(h.id, h.peers)) + require.Len(t, h.transport.ChannelsUpdated, 1) + require.Equal(t, h.transport.ChannelsUpdated[0], channelID(h.id, h.peers)) require.Len(t, h.transport.MessagesSent, 2) response, ok = h.transport.MessagesSent[1].Message.(datatransfer.Response) require.True(t, ok) @@ -434,6 +435,7 @@ func TestDataTransferResponding(t *testing.T) { datatransfer.Accept, datatransfer.NewVoucherResult, datatransfer.SetDataLimit, + datatransfer.TransferInitiated, datatransfer.DataQueuedProgress, datatransfer.DataQueued, datatransfer.DataLimitExceeded, @@ -450,12 +452,11 @@ func TestDataTransferResponding(t *testing.T) { verify: func(t *testing.T, h *receiverHarness) { _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) require.NoError(t, err) - msg, err := h.transport.EventHandler.OnDataQueued( - channelID(h.id, h.peers), - cidlink.Link{Cid: testutil.GenerateCids(1)[0]}, - 12345, 1, true) - require.EqualError(t, err, datatransfer.ErrPause.Error()) - response, ok := msg.(datatransfer.Response) + h.transport.EventHandler.OnTransportEvent(channelID(h.id, h.peers), datatransfer.TransportInitiatedTransfer{}) + h.transport.EventHandler.OnTransportEvent(channelID(h.id, h.peers), datatransfer.TransportQueuedData{Size: 12345, Index: basicnode.NewInt(1)}) + h.transport.EventHandler.OnTransportEvent(channelID(h.id, h.peers), datatransfer.TransportReachedDataLimit{}) + require.Len(t, h.transport.MessagesSent, 1) + response, ok := h.transport.MessagesSent[0].Message.(datatransfer.Response) require.True(t, ok) require.Equal(t, response.TransferID(), h.id) require.True(t, response.IsUpdate()) @@ -468,10 +469,10 @@ func TestDataTransferResponding(t *testing.T) { vr := testutil.NewTestTypedVoucher() err = h.dt.UpdateValidationStatus(ctx, channelID(h.id, h.peers), datatransfer.ValidationResult{Accepted: true, DataLimit: 50000, VoucherResult: &vr}) require.NoError(t, err) - require.Len(t, h.transport.ResumedChannels, 1) - require.Equal(t, h.transport.ResumedChannels[0], channelID(h.id, h.peers)) - require.Len(t, h.transport.MessagesSent, 1) - response, ok = h.transport.MessagesSent[0].Message.(datatransfer.Response) + require.Len(t, h.transport.ChannelsUpdated, 1) + require.Equal(t, h.transport.ChannelsUpdated[0], channelID(h.id, h.peers)) + require.Len(t, h.transport.MessagesSent, 2) + response, ok = h.transport.MessagesSent[1].Message.(datatransfer.Response) require.True(t, ok) require.True(t, response.Accepted()) require.Equal(t, response.TransferID(), h.id) @@ -488,6 +489,7 @@ func TestDataTransferResponding(t *testing.T) { datatransfer.Accept, datatransfer.NewVoucherResult, datatransfer.SetRequiresFinalization, + datatransfer.TransferInitiated, datatransfer.BeginFinalizing, datatransfer.NewVoucher, datatransfer.NewVoucherResult, @@ -503,8 +505,8 @@ func TestDataTransferResponding(t *testing.T) { verify: func(t *testing.T, h *receiverHarness) { _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) require.NoError(t, err) - err = h.transport.EventHandler.OnChannelCompleted(channelID(h.id, h.peers), nil) - require.NoError(t, err) + h.transport.EventHandler.OnTransportEvent(channelID(h.id, h.peers), datatransfer.TransportInitiatedTransfer{}) + h.transport.EventHandler.OnTransportEvent(channelID(h.id, h.peers), datatransfer.TransportCompletedTransfer{Success: true}) require.Len(t, h.transport.MessagesSent, 1) response, ok := h.transport.MessagesSent[0].Message.(datatransfer.Response) require.True(t, ok) @@ -544,8 +546,7 @@ func TestDataTransferResponding(t *testing.T) { verify: func(t *testing.T, h *receiverHarness) { _, err := h.transport.EventHandler.OnRequestReceived(channelID(h.id, h.peers), h.pullRequest) require.NoError(t, err) - err = h.transport.EventHandler.OnChannelCompleted(channelID(h.id, h.peers), xerrors.Errorf("err")) - require.NoError(t, err) + h.transport.EventHandler.OnTransportEvent(channelID(h.id, h.peers), datatransfer.TransportCompletedTransfer{Success: false, ErrorMessage: "something went wrong"}) }, }, "new push request, customized transport": { @@ -631,7 +632,7 @@ func TestDataTransferResponding(t *testing.T) { h.resumeUpdate = message.UpdateRequest(h.id, false) require.NoError(t, err) updateVoucher := testutil.NewTestTypedVoucher() - h.voucherUpdate, err = message.VoucherRequest(h.id, &updateVoucher) + h.voucherUpdate = message.VoucherRequest(h.id, &updateVoucher) h.cancelUpdate = message.CancelRequest(h.id) require.NoError(t, err) h.sv = testutil.NewStubbedValidator() @@ -662,8 +663,7 @@ func TestDataTransferRestartResponding(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, channelID) - response, err := message.RestartResponse(channelID.ID, true, false, nil) - require.NoError(t, err) + response := message.RestartResponse(channelID.ID, true, false, nil) err = h.transport.EventHandler.OnResponseReceived(channelID, response) require.NoError(t, err) }, @@ -673,6 +673,7 @@ func TestDataTransferRestartResponding(t *testing.T) { datatransfer.Open, datatransfer.Accept, datatransfer.NewVoucherResult, + datatransfer.TransferInitiated, datatransfer.DataReceivedProgress, datatransfer.DataReceived, datatransfer.DataReceivedProgress, @@ -695,11 +696,9 @@ func TestDataTransferRestartResponding(t *testing.T) { // some cids are received chid := datatransfer.ChannelID{Initiator: h.peers[1], Responder: h.peers[0], ID: h.pushRequest.TransferID()} - testCids := testutil.GenerateCids(2) - ev, ok := h.dt.(datatransfer.EventsHandler) - require.True(t, ok) - require.NoError(t, ev.OnDataReceived(chid, cidlink.Link{Cid: testCids[0]}, 12345, 1, true)) - require.NoError(t, ev.OnDataReceived(chid, cidlink.Link{Cid: testCids[1]}, 12345, 2, true)) + h.transport.EventHandler.OnTransportEvent(chid, datatransfer.TransportInitiatedTransfer{}) + h.transport.EventHandler.OnTransportEvent(chid, datatransfer.TransportReceivedData{Size: 12345, Index: basicnode.NewInt(1)}) + h.transport.EventHandler.OnTransportEvent(chid, datatransfer.TransportReceivedData{Size: 12345, Index: basicnode.NewInt(2)}) // receive restart push request req, err := message.NewRequest(h.pushRequest.TransferID(), true, false, &h.voucher, h.baseCid, h.stor) @@ -894,6 +893,7 @@ func TestDataTransferRestartResponding(t *testing.T) { "ReceiveRestartExistingChannelRequest: Reopen Pull Channel": { expectedEvents: []datatransfer.EventCode{ datatransfer.Open, + datatransfer.TransferInitiated, datatransfer.DataReceivedProgress, datatransfer.DataReceived, datatransfer.DataReceivedProgress, @@ -908,15 +908,13 @@ func TestDataTransferRestartResponding(t *testing.T) { require.NotEmpty(t, channelID) // some cids should already be received - testCids := testutil.GenerateCids(2) - ev, ok := h.dt.(datatransfer.EventsHandler) - require.True(t, ok) - require.NoError(t, ev.OnDataReceived(channelID, cidlink.Link{Cid: testCids[0]}, 12345, 1, true)) - require.NoError(t, ev.OnDataReceived(channelID, cidlink.Link{Cid: testCids[1]}, 12345, 2, true)) + // some cids are received + h.transport.EventHandler.OnTransportEvent(channelID, datatransfer.TransportInitiatedTransfer{}) + h.transport.EventHandler.OnTransportEvent(channelID, datatransfer.TransportReceivedData{Size: 12345, Index: basicnode.NewInt(1)}) + h.transport.EventHandler.OnTransportEvent(channelID, datatransfer.TransportReceivedData{Size: 12345, Index: basicnode.NewInt(2)}) // send a request to restart the same pull channel - err = h.transport.EventHandler.OnRestartExistingChannelRequestReceived(channelID) - require.NoError(t, err) + h.transport.EventHandler.OnTransportEvent(channelID, datatransfer.TransportReceivedRestartExistingChannelRequest{}) require.Len(t, h.transport.OpenedChannels, 1) require.Len(t, h.transport.RestartedChannels, 1) @@ -927,7 +925,7 @@ func TestDataTransferRestartResponding(t *testing.T) { require.Equal(t, restartedChannel.Channel.Sender(), h.peers[1]) require.Equal(t, restartedChannel.Channel.BaseCID(), h.baseCid) require.Equal(t, restartedChannel.Channel.Selector(), h.stor) - require.EqualValues(t, len(testCids), restartedChannel.Channel.ReceivedCidsTotal()) + require.EqualValues(t, basicnode.NewInt(2), restartedChannel.Channel.ReceivedIndex()) // assert a restart request is in the channel request := restartedChannel.Message @@ -958,7 +956,7 @@ func TestDataTransferRestartResponding(t *testing.T) { require.NotEmpty(t, channelID) // send a request to restart the same push request - err = h.transport.EventHandler.OnRestartExistingChannelRequestReceived(channelID) + h.transport.EventHandler.OnTransportEvent(channelID, datatransfer.TransportReceivedRestartExistingChannelRequest{}) require.NoError(t, err) require.Len(t, h.transport.OpenedChannels, 1) diff --git a/impl/utils.go b/impl/utils.go index 518b9e0c..4a74d838 100644 --- a/impl/utils.go +++ b/impl/utils.go @@ -35,6 +35,13 @@ func (m *manager) newRequest(ctx context.Context, selector datamodel.Node, isPul return message.NewRequest(tid, false, isPull, &voucher, baseCid, selector) } +func (m *manager) otherPeer(chid datatransfer.ChannelID) peer.ID { + if chid.Initiator == m.peerID { + return chid.Responder + } + return chid.Initiator +} + func (m *manager) resume(chid datatransfer.ChannelID) error { if chid.Initiator == m.peerID { return m.channels.ResumeInitiator(chid) diff --git a/itest/gstestdata.go b/itest/gstestdata.go index 02b2f7dc..2b1b808a 100644 --- a/itest/gstestdata.go +++ b/itest/gstestdata.go @@ -76,6 +76,8 @@ type GraphsyncTestingData struct { GsNet2 gsnet.GraphSyncNetwork DtNet1 network.DataTransferNetwork DtNet2 network.DataTransferNetwork + Gs1 graphsync.GraphExchange + Gs2 graphsync.GraphExchange OrigBytes []byte TempDir1 string TempDir2 string @@ -151,13 +153,17 @@ func NewGraphsyncTestingData(ctx context.Context, t *testing.T, host1Protocols [ // SetupGraphsyncHost1 sets up a new, real graphsync instance on top of the first host func (gsData *GraphsyncTestingData) SetupGraphsyncHost1() graphsync.GraphExchange { + if gsData.Gs1 != nil { + return gsData.Gs1 + } // setup graphsync if gsData.gs1Cancel != nil { gsData.gs1Cancel() } gsCtx, gsCancel := context.WithCancel(gsData.Ctx) gsData.gs1Cancel = gsCancel - return gsimpl.New(gsCtx, gsData.GsNet1, gsData.LinkSystem1) + gsData.Gs1 = gsimpl.New(gsCtx, gsData.GsNet1, gsData.LinkSystem1) + return gsData.Gs1 } // SetupGSTransportHost1 sets up a new grapshync transport over real graphsync on the first host @@ -172,18 +178,22 @@ func (gsData *GraphsyncTestingData) SetupGSTransportHost1(opts ...gstransport.Op opts = append(opts, gstransport.SupportedExtensions(supportedExtensions)) } - return gstransport.NewTransport(gsData.Host1.ID(), gs, gsData.DtNet1, opts...) + return gstransport.NewTransport(gs, gsData.DtNet1, opts...) } // SetupGraphsyncHost2 sets up a new, real graphsync instance on top of the second host func (gsData *GraphsyncTestingData) SetupGraphsyncHost2() graphsync.GraphExchange { + if gsData.Gs2 != nil { + return gsData.Gs2 + } // setup graphsync if gsData.gs2Cancel != nil { gsData.gs2Cancel() } gsCtx, gsCancel := context.WithCancel(gsData.Ctx) gsData.gs2Cancel = gsCancel - return gsimpl.New(gsCtx, gsData.GsNet2, gsData.LinkSystem2) + gsData.Gs2 = gsimpl.New(gsCtx, gsData.GsNet2, gsData.LinkSystem2) + return gsData.Gs2 } // SetupGSTransportHost2 sets up a new grapshync transport over real graphsync on the second host @@ -197,7 +207,7 @@ func (gsData *GraphsyncTestingData) SetupGSTransportHost2(opts ...gstransport.Op } opts = append(opts, gstransport.SupportedExtensions(supportedExtensions)) } - return gstransport.NewTransport(gsData.Host2.ID(), gs, gsData.DtNet2, opts...) + return gstransport.NewTransport(gs, gsData.DtNet2, opts...) } // LoadUnixFSFile loads a fixtures file we can test dag transfer with diff --git a/itest/integration_test.go b/itest/integration_test.go index f13e69e4..a98a67d8 100644 --- a/itest/integration_test.go +++ b/itest/integration_test.go @@ -440,7 +440,7 @@ func TestManyReceiversAtOnce(t *testing.T) { destDagService := merkledag.NewDAGService(blockservice.New(altBs, offline.Exchange(altBs))) gs := gsimpl.New(gsData.Ctx, gsnet, lsys) - gsTransport := tp.NewTransport(host.ID(), gs, dtnet) + gsTransport := tp.NewTransport(gs, dtnet) dtDs := namespace.Wrap(ds, datastore.NewKey("datatransfer")) @@ -1443,6 +1443,7 @@ func TestPauseAndResume(t *testing.T) { dt2, err := NewDataTransfer(gsData.DtDs2, gsData.Host2.ID(), tp2) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt2) + finished := make(chan struct{}, 2) errChan := make(chan struct{}, 2) opened := make(chan struct{}, 2) @@ -1494,8 +1495,27 @@ func TestPauseAndResume(t *testing.T) { sv := testutil.NewStubbedValidator() sv.StubResult(datatransfer.ValidationResult{Accepted: true}) sv.StubRestartResult(datatransfer.ValidationResult{Accepted: true}) - var chid datatransfer.ChannelID + + gsData.Gs1.RegisterOutgoingBlockHook(func(p peer.ID, r graphsync.RequestData, block graphsync.BlockData, ha graphsync.OutgoingBlockHookActions) { + if block.Index() == 5 && block.BlockSizeOnWire() > 0 { + require.NoError(t, dt1.PauseDataTransferChannel(ctx, chid)) + go func() { + time.Sleep(100 * time.Millisecond) + require.NoError(t, dt1.ResumeDataTransferChannel(ctx, chid)) + }() + } + }) + gsData.Gs2.RegisterIncomingBlockHook(func(p peer.ID, r graphsync.ResponseData, block graphsync.BlockData, ha graphsync.IncomingBlockHookActions) { + if block.Index() == 5 { + require.NoError(t, dt2.PauseDataTransferChannel(ctx, chid)) + go func() { + time.Sleep(50 * time.Millisecond) + require.NoError(t, dt2.ResumeDataTransferChannel(ctx, chid)) + }() + } + }) + if isPull { sv.ExpectSuccessPull() require.NoError(t, dt1.RegisterVoucherType(testutil.TestVoucherType, sv)) @@ -1533,18 +1553,8 @@ func TestPauseAndResume(t *testing.T) { resumeResponders++ case sentIncrement := <-sent: sentIncrements = append(sentIncrements, sentIncrement) - if len(sentIncrements) == 5 { - require.NoError(t, dt1.PauseDataTransferChannel(ctx, chid)) - time.Sleep(100 * time.Millisecond) - require.NoError(t, dt1.ResumeDataTransferChannel(ctx, chid)) - } case receivedIncrement := <-received: receivedIncrements = append(receivedIncrements, receivedIncrement) - if len(receivedIncrements) == 10 { - require.NoError(t, dt2.PauseDataTransferChannel(ctx, chid)) - time.Sleep(100 * time.Millisecond) - require.NoError(t, dt2.ResumeDataTransferChannel(ctx, chid)) - } case <-errChan: t.Fatal("received error on data transfer") } @@ -1805,7 +1815,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { } requestReceived := messageReceived.message.(datatransfer.Request) - response, err := message.NewResponse(requestReceived.TransferID(), true, false, &voucherResult) + response := message.NewResponse(requestReceived.TransferID(), true, false, &voucherResult) require.NoError(t, err) nd := response.ToIPLD() request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, selectorparse.CommonSelector_ExploreAllRecursively, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ @@ -1823,7 +1833,7 @@ func TestRespondingToPushGraphsyncRequests(t *testing.T) { }) t.Run("when no request is initiated", func(t *testing.T) { - response, err := message.NewResponse(datatransfer.TransferID(rand.Uint32()), true, false, &voucher) + response := message.NewResponse(datatransfer.TransferID(rand.Uint32()), true, false, &voucher) require.NoError(t, err) nd := response.ToIPLD() request := gsmsg.NewRequest(graphsync.NewRequestID(), link.(cidlink.Link).Cid, selectorparse.CommonSelector_ExploreAllRecursively, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ @@ -1865,7 +1875,7 @@ func TestResponseHookWhenExtensionNotFound(t *testing.T) { gsData.GsNet2.SetDelegate(gsr) gs1 := gsData.SetupGraphsyncHost1() - tp1 := tp.NewTransport(host1.ID(), gs1, gsData.DtNet1) + tp1 := tp.NewTransport(gs1, gsData.DtNet1) dt1, err := NewDataTransfer(gsData.DtDs1, gsData.Host1.ID(), tp1) require.NoError(t, err) testutil.StartAndWaitForReady(ctx, t, dt1) diff --git a/manager.go b/manager.go index 822f76e5..b7a6f217 100644 --- a/manager.go +++ b/manager.go @@ -40,6 +40,9 @@ func (vr ValidationResult) Equals(vr2 ValidationResult) bool { // LeaveRequestPaused indicates whether all conditions are met to resume a request func (vr ValidationResult) LeaveRequestPaused(chst ChannelState) bool { + if chst == nil { + return false + } if vr.ForcePause { return true } diff --git a/message/message1_1prime/message.go b/message/message1_1prime/message.go index 641686bf..a6fccd71 100644 --- a/message/message1_1prime/message.go +++ b/message/message1_1prime/message.go @@ -76,7 +76,7 @@ func UpdateRequest(id datatransfer.TransferID, isPaused bool) datatransfer.Reque } // VoucherRequest generates a new request for the data transfer protocol -func VoucherRequest(id datatransfer.TransferID, voucher *datatransfer.TypedVoucher) (datatransfer.Request, error) { +func VoucherRequest(id datatransfer.TransferID, voucher *datatransfer.TypedVoucher) datatransfer.Request { if voucher == nil { voucher = &emptyTypedVoucher } @@ -85,11 +85,11 @@ func VoucherRequest(id datatransfer.TransferID, voucher *datatransfer.TypedVouch VoucherPtr: voucher.Voucher, VoucherTypeIdentifier: voucher.Type, TransferId: uint64(id), - }, nil + } } // RestartResponse builds a new Data Transfer response -func RestartResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResult *datatransfer.TypedVoucher) (datatransfer.Response, error) { +func RestartResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResult *datatransfer.TypedVoucher) datatransfer.Response { if voucherResult == nil { voucherResult = &emptyTypedVoucher } @@ -100,7 +100,7 @@ func RestartResponse(id datatransfer.TransferID, accepted bool, isPaused bool, v TransferId: uint64(id), VoucherResultPtr: voucherResult.Voucher, VoucherTypeIdentifier: voucherResult.Type, - }, nil + } } // ValidationResultResponse response generates a response based on a validation result @@ -110,7 +110,7 @@ func ValidationResultResponse( id datatransfer.TransferID, validationResult datatransfer.ValidationResult, validationErr error, - paused bool) (datatransfer.Response, error) { + paused bool) datatransfer.Response { voucherResult := &emptyTypedVoucher if validationResult.VoucherResult != nil { @@ -125,11 +125,11 @@ func ValidationResultResponse( TransferId: uint64(id), VoucherTypeIdentifier: voucherResult.Type, VoucherResultPtr: voucherResult.Voucher, - }, nil + } } // NewResponse builds a new Data Transfer response -func NewResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResult *datatransfer.TypedVoucher) (datatransfer.Response, error) { +func NewResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResult *datatransfer.TypedVoucher) datatransfer.Response { if voucherResult == nil { voucherResult = &emptyTypedVoucher } @@ -140,11 +140,11 @@ func NewResponse(id datatransfer.TransferID, accepted bool, isPaused bool, vouch TransferId: uint64(id), VoucherTypeIdentifier: voucherResult.Type, VoucherResultPtr: voucherResult.Voucher, - }, nil + } } // VoucherResultResponse builds a new response for a voucher result -func VoucherResultResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResult *datatransfer.TypedVoucher) (datatransfer.Response, error) { +func VoucherResultResponse(id datatransfer.TransferID, accepted bool, isPaused bool, voucherResult *datatransfer.TypedVoucher) datatransfer.Response { if voucherResult == nil { voucherResult = &emptyTypedVoucher } @@ -155,7 +155,7 @@ func VoucherResultResponse(id datatransfer.TransferID, accepted bool, isPaused b TransferId: uint64(id), VoucherTypeIdentifier: voucherResult.Type, VoucherResultPtr: voucherResult.Voucher, - }, nil + } } // UpdateResponse returns a new update response @@ -176,7 +176,7 @@ func CancelResponse(id datatransfer.TransferID) datatransfer.Response { } // CompleteResponse returns a new complete response message -func CompleteResponse(id datatransfer.TransferID, isAccepted bool, isPaused bool, voucherResult *datatransfer.TypedVoucher) (datatransfer.Response, error) { +func CompleteResponse(id datatransfer.TransferID, isAccepted bool, isPaused bool, voucherResult *datatransfer.TypedVoucher) datatransfer.Response { if voucherResult == nil { voucherResult = &emptyTypedVoucher } @@ -187,7 +187,7 @@ func CompleteResponse(id datatransfer.TransferID, isAccepted bool, isPaused bool VoucherTypeIdentifier: voucherResult.Type, VoucherResultPtr: voucherResult.Voucher, TransferId: uint64(id), - }, nil + } } // FromNet can read a network stream to deserialize a GraphSyncMessage diff --git a/message/message1_1prime/message_test.go b/message/message1_1prime/message_test.go index 33050deb..b84b39c0 100644 --- a/message/message1_1prime/message_test.go +++ b/message/message1_1prime/message_test.go @@ -159,8 +159,7 @@ func TestTransferRequest_UnmarshalCBOR(t *testing.T) { func TestResponses(t *testing.T) { id := datatransfer.TransferID(rand.Int31()) voucherResult := testutil.NewTestTypedVoucher() - response, err := message1_1.NewResponse(id, false, true, &voucherResult) // not accepted - require.NoError(t, err) + response := message1_1.NewResponse(id, false, true, &voucherResult) // not accepted assert.Equal(t, response.TransferID(), id) assert.False(t, response.Accepted()) assert.True(t, response.IsNew()) @@ -182,8 +181,7 @@ func TestResponses(t *testing.T) { func TestTransferResponse_MarshalCBOR(t *testing.T) { id := datatransfer.TransferID(rand.Int31()) voucherResult := testutil.NewTestTypedVoucher() - response, err := message1_1.NewResponse(id, true, false, &voucherResult) // accepted - require.NoError(t, err) + response := message1_1.NewResponse(id, true, false, &voucherResult) // accepted // sanity check that we can marshal data wbuf := new(bytes.Buffer) @@ -195,8 +193,7 @@ func TestTransferResponse_UnmarshalCBOR(t *testing.T) { t.Run("round-trip", func(t *testing.T) { id := datatransfer.TransferID(rand.Int31()) voucherResult := testutil.NewTestTypedVoucher() - response, err := message1_1.NewResponse(id, true, false, &voucherResult) // accepted - require.NoError(t, err) + response := message1_1.NewResponse(id, true, false, &voucherResult) // accepted wbuf := new(bytes.Buffer) require.NoError(t, response.ToNet(wbuf)) @@ -366,8 +363,7 @@ func TestCancelResponse(t *testing.T) { func TestCompleteResponse(t *testing.T) { id := datatransfer.TransferID(rand.Int31()) - response, err := message1_1.CompleteResponse(id, true, true, nil) - require.NoError(t, err) + response := message1_1.CompleteResponse(id, true, true, nil) assert.Equal(t, response.TransferID(), id) assert.False(t, response.IsNew()) assert.False(t, response.IsUpdate()) @@ -414,8 +410,7 @@ func TestToNetFromNetEquivalency(t *testing.T) { testutil.AssertEqualTestVoucher(t, request, deserializedRequest) testutil.AssertEqualSelector(t, request, deserializedRequest) - response, err := message1_1.NewResponse(id, accepted, false, &voucherResult) - require.NoError(t, err) + response := message1_1.NewResponse(id, accepted, false, &voucherResult) err = response.ToNet(buf) require.NoError(t, err) deserialized, err = message1_1.FromNet(buf) @@ -476,8 +471,7 @@ func TestToNetFromNetEquivalency(t *testing.T) { testutil.AssertEqualTestVoucher(t, request, deserializedRequest) testutil.AssertEqualSelector(t, request, deserializedRequest) - response, err := message1_1.NewResponse(id, accepted, false, &voucherResult) - require.NoError(t, err) + response := message1_1.NewResponse(id, accepted, false, &voucherResult) err = response.ToNet(buf) require.NoError(t, err) msg, _ = hex.DecodeString("a36449735271f46752657175657374f668526573706f6e7365a66454797065006441637074f46450617573f4665866657249441a4d65822164565265738178644204cb9a1e34c5f08e9b20aa76090e70020bb56c0ca3d3af7296cd1058a5112890fed218488f084d8df9e4835fb54ad045ffd936e3bf7261b0426c51352a097816ed74482bb9084b4a7ed8adc517f3371e0e0434b511625cd1a41792243dccdcfe88094b64565479706b54657374566f7563686572") @@ -541,8 +535,7 @@ func TestToNetFromNetEquivalency(t *testing.T) { testutil.AssertEqualTestVoucher(t, request, deserializedRequest) testutil.AssertEqualSelector(t, request, deserializedRequest) - response, err := message1_1.NewResponse(id, accepted, false, &voucherResult) - require.NoError(t, err) + response := message1_1.NewResponse(id, accepted, false, &voucherResult) wresponse := response.WrappedForTransport(transportID, transportVersion) err = wresponse.ToNet(buf) require.NoError(t, err) diff --git a/message/message1_1prime/transfer_response_test.go b/message/message1_1prime/transfer_response_test.go index 71fcf703..85741882 100644 --- a/message/message1_1prime/transfer_response_test.go +++ b/message/message1_1prime/transfer_response_test.go @@ -14,8 +14,7 @@ import ( func TestResponseMessageForVersion(t *testing.T) { id := datatransfer.TransferID(rand.Int31()) voucherResult := testutil.NewTestTypedVoucher() - response, err := message1_1.NewResponse(id, false, true, &voucherResult) // not accepted - require.NoError(t, err) + response := message1_1.NewResponse(id, false, true, &voucherResult) // not accepted // v1.2 new protocol out, err := response.MessageForVersion(datatransfer.DataTransfer1_2) diff --git a/statuses.go b/statuses.go index 32c277fb..6e915373 100644 --- a/statuses.go +++ b/statuses.go @@ -1,5 +1,7 @@ package datatransfer +import "github.com/filecoin-project/go-statemachine/fsm" + // Status is the status of transfer for a given channel type Status uint64 @@ -40,13 +42,13 @@ const ( // Cancelled means the data transfer ended prematurely Cancelled - // InitiatorPaused means the data sender has paused the channel (only the sender can unpause this) + // DEPRECATED: Use InitiatorPaused() method on ChannelState InitiatorPaused - // ResponderPaused means the data receiver has paused the channel (only the receiver can unpause this) + // DEPRECATED: Use ResponderPaused() method on ChannelState ResponderPaused - // BothPaused means both sender and receiver have paused the channel seperately (both must unpause) + // DEPRECATED: Use BothPaused() method on ChannelState BothPaused // ResponderFinalizing is a unique state where the responder is awaiting a final voucher @@ -58,18 +60,79 @@ const ( // ChannelNotFoundError means the searched for data transfer does not exist ChannelNotFoundError + + // Queued indicates a data transfer request has been accepted, but is not actively transfering yet + Queued + + // AwaitingAcceptance indicates a transfer request is actively being processed by the transport + // even if the remote has not yet responded that it's accepted the transfer + AwaitingAcceptance ) -func (s Status) IsAccepted() bool { - return s != Requested && s != Cancelled && s != Cancelling && s != Failed && s != Failing && s != ChannelNotFoundError +type statusList []Status + +func (sl statusList) Contains(s Status) bool { + for _, ts := range sl { + if ts == s { + return true + } + } + return false } -func (s Status) IsResponderPaused() bool { - return s == ResponderPaused || s == BothPaused || s == Finalizing +func (sl statusList) AsFSMStates() []fsm.StateKey { + sk := make([]fsm.StateKey, 0, len(sl)) + for _, s := range sl { + sk = append(sk, s) + } + return sk +} + +var NotAcceptedStates = statusList{ + Requested, + AwaitingAcceptance, + Cancelled, + Cancelling, + Failed, + Failing, + ChannelNotFoundError} + +func (s Status) IsAccepted() bool { + return !NotAcceptedStates.Contains(s) } +var FinalizationStatuses = statusList{Finalizing, Completed, Completing} + func (s Status) InFinalization() bool { - return s == Finalizing || s == Completed || s == Completing + return FinalizationStatuses.Contains(s) +} + +var TransferCompleteStates = statusList{ + TransferFinished, + ResponderFinalizingTransferFinished, + Finalizing, + Completed, + Completing, + Failing, + Failed, + Cancelling, + Cancelled, + ChannelNotFoundError, +} + +func (s Status) TransferComplete() bool { + return TransferCompleteStates.Contains(s) +} + +var TransferringStates = statusList{ + Ongoing, + ResponderCompleted, + ResponderFinalizing, + AwaitingAcceptance, +} + +func (s Status) Transferring() bool { + return TransferringStates.Contains(s) } // Statuses are human readable names for data transfer states @@ -92,4 +155,6 @@ var Statuses = map[Status]string{ ResponderFinalizing: "ResponderFinalizing", ResponderFinalizingTransferFinished: "ResponderFinalizingTransferFinished", ChannelNotFoundError: "ChannelNotFoundError", + Queued: "Queued", + AwaitingAcceptance: "AwaitingAcceptance", } diff --git a/testutil/faketransport.go b/testutil/faketransport.go index 519691cc..58726448 100644 --- a/testutil/faketransport.go +++ b/testutil/faketransport.go @@ -24,12 +24,6 @@ type MessageSent struct { Message datatransfer.Message } -// DataLimitSet records setting a data limit -type DataLimitSet struct { - ChannelID datatransfer.ChannelID - DataLimit uint64 -} - // CustomizedTransfer is just a way to record calls made to transport configurer type CustomizedTransfer struct { ChannelID datatransfer.ChannelID @@ -44,12 +38,9 @@ type FakeTransport struct { OpenChannelErr error RestartedChannels []RestartedChannel RestartChannelErr error - ClosedChannels []datatransfer.ChannelID - PausedChannels []datatransfer.ChannelID - ResumedChannels []datatransfer.ChannelID - DataLimitsSet []DataLimitSet MessagesSent []MessageSent UpdateError error + ChannelsUpdated []datatransfer.ChannelID CleanedUpChannels []datatransfer.ChannelID CustomizedTransfers []CustomizedTransfer EventHandler datatransfer.EventsHandler @@ -96,34 +87,18 @@ func (ft *FakeTransport) RestartChannel(ctx context.Context, channelState datatr } // WithChannel takes actions on a channel -func (ft *FakeTransport) SendChannelCommand(ctx context.Context, chid datatransfer.ChannelID, command datatransfer.ChannelCommand) error { +func (ft *FakeTransport) ChannelUpdated(ctx context.Context, chid datatransfer.ChannelID, msg datatransfer.Message) error { - update := command.ChannelUpdate() - message, sendMessage := update.MessageToSend() - if sendMessage { - ft.MessagesSent = append(ft.MessagesSent, MessageSent{chid, message}) - } - - closed, hasClosed := update.Closed() - if hasClosed && closed { - ft.ClosedChannels = append(ft.ClosedChannels, chid) - return ft.UpdateError - } - - paused, hasPaused := update.Paused() - if hasPaused { - if !paused { - ft.ResumedChannels = append(ft.ResumedChannels, chid) - } else { - ft.PausedChannels = append(ft.PausedChannels, chid) - } - } - - dataLimit, hasDataLimit := update.DataLimit() - if hasDataLimit { - ft.DataLimitsSet = append(ft.DataLimitsSet, DataLimitSet{chid, dataLimit}) + if msg != nil { + ft.MessagesSent = append(ft.MessagesSent, MessageSent{chid, msg}) } + ft.ChannelsUpdated = append(ft.ChannelsUpdated, chid) + return nil +} +// SendMessage sends a data transfer message over the channel to the other peer +func (ft *FakeTransport) SendMessage(ctx context.Context, chid datatransfer.ChannelID, msg datatransfer.Message) error { + ft.MessagesSent = append(ft.MessagesSent, MessageSent{chid, msg}) return ft.UpdateError } diff --git a/testutil/message.go b/testutil/message.go deleted file mode 100644 index 14319cc3..00000000 --- a/testutil/message.go +++ /dev/null @@ -1,30 +0,0 @@ -package testutil - -import ( - "testing" - - basicnode "github.com/ipld/go-ipld-prime/node/basic" - "github.com/ipld/go-ipld-prime/traversal/selector/builder" - "github.com/stretchr/testify/require" - - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/message" -) - -// NewDTRequest makes a new DT Request message -func NewDTRequest(t *testing.T, transferID datatransfer.TransferID) datatransfer.Request { - voucher := NewTestTypedVoucher() - baseCid := GenerateCids(1)[0] - selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() - r, err := message.NewRequest(transferID, false, false, &voucher, baseCid, selector) - require.NoError(t, err) - return r -} - -// NewDTResponse makes a new DT Request message -func NewDTResponse(t *testing.T, transferID datatransfer.TransferID) datatransfer.Response { - vresult := NewTestTypedVoucher() - r, err := message.NewResponse(transferID, false, false, &vresult) - require.NoError(t, err) - return r -} diff --git a/testutil/mockchannelstate.go b/testutil/mockchannelstate.go index cd3be68b..92fc5f06 100644 --- a/testutil/mockchannelstate.go +++ b/testutil/mockchannelstate.go @@ -2,6 +2,7 @@ package testutil import ( cid "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" "github.com/libp2p/go-libp2p-core/peer" @@ -9,32 +10,60 @@ import ( ) type MockChannelStateParams struct { - ReceivedIndex datamodel.Node - ChannelID datatransfer.ChannelID - Queued uint64 - Sent uint64 - Received uint64 - Complete bool + ReceivedIndex datamodel.Node + SentIndex datamodel.Node + QueuedIndex datamodel.Node + ChannelID datatransfer.ChannelID + Queued uint64 + Sent uint64 + Received uint64 + Complete bool + BaseCID cid.Cid + Selector ipld.Node + Voucher datatransfer.TypedVoucher + IsPull bool + Self peer.ID + DataLimit uint64 + InitiatorPaused bool + ResponderPaused bool } func NewMockChannelState(params MockChannelStateParams) *MockChannelState { return &MockChannelState{ - receivedIndex: params.ReceivedIndex, - chid: params.ChannelID, - queued: params.Queued, - sent: params.Sent, - received: params.Received, - complete: params.Complete, + receivedIndex: params.ReceivedIndex, + sentIndex: params.SentIndex, + queuedIndex: params.QueuedIndex, + dataLimit: params.DataLimit, + chid: params.ChannelID, + queued: params.Queued, + sent: params.Sent, + received: params.Received, + complete: params.Complete, + isPull: params.IsPull, + self: params.Self, + baseCID: params.BaseCID, + initiatorPaused: params.InitiatorPaused, + responderPaused: params.ResponderPaused, } } type MockChannelState struct { - receivedIndex datamodel.Node - chid datatransfer.ChannelID - queued uint64 - sent uint64 - received uint64 - complete bool + receivedIndex datamodel.Node + sentIndex datamodel.Node + queuedIndex datamodel.Node + dataLimit uint64 + chid datatransfer.ChannelID + queued uint64 + sent uint64 + received uint64 + complete bool + isPull bool + baseCID cid.Cid + selector ipld.Node + voucher datatransfer.TypedVoucher + self peer.ID + initiatorPaused bool + responderPaused bool } var _ datatransfer.ChannelState = (*MockChannelState)(nil) @@ -77,40 +106,67 @@ func (m *MockChannelState) Status() datatransfer.Status { return datatransfer.Ongoing } +func (m *MockChannelState) SetReceivedIndex(receivedIndex datamodel.Node) { + m.receivedIndex = receivedIndex +} + func (m *MockChannelState) ReceivedIndex() datamodel.Node { + if m.receivedIndex == nil { + return datamodel.Null + } return m.receivedIndex } func (m *MockChannelState) QueuedIndex() datamodel.Node { - panic("implement me") + if m.queuedIndex == nil { + return datamodel.Null + } + return m.queuedIndex +} + +func (m *MockChannelState) SetQueuedIndex(queuedIndex datamodel.Node) { + m.queuedIndex = queuedIndex } func (m *MockChannelState) SentIndex() datamodel.Node { - panic("implement me") + if m.sentIndex == nil { + return datamodel.Null + } + return m.sentIndex +} + +func (m *MockChannelState) SetSentIndex(sentIndex datamodel.Node) { + m.sentIndex = sentIndex } func (m *MockChannelState) TransferID() datatransfer.TransferID { - panic("implement me") + return m.chid.ID } func (m *MockChannelState) BaseCID() cid.Cid { - panic("implement me") + return m.baseCID } func (m *MockChannelState) Selector() datamodel.Node { - panic("implement me") + return m.selector } func (m *MockChannelState) Voucher() datatransfer.TypedVoucher { - panic("implement me") + return m.voucher } func (m *MockChannelState) Sender() peer.ID { - panic("implement me") + if m.isPull { + return m.chid.Responder + } + return m.chid.Initiator } func (m *MockChannelState) Recipient() peer.ID { - panic("implement me") + if m.isPull { + return m.chid.Initiator + } + return m.chid.Responder } func (m *MockChannelState) TotalSize() uint64 { @@ -118,15 +174,18 @@ func (m *MockChannelState) TotalSize() uint64 { } func (m *MockChannelState) IsPull() bool { - panic("implement me") + return m.isPull } func (m *MockChannelState) OtherPeer() peer.ID { - panic("implement me") + if m.self == m.chid.Initiator { + return m.chid.Responder + } + return m.chid.Initiator } func (m *MockChannelState) SelfPeer() peer.ID { - panic("implement me") + return m.self } func (m *MockChannelState) Message() string { @@ -153,10 +212,41 @@ func (m *MockChannelState) Stages() *datatransfer.ChannelStages { panic("implement me") } +func (m *MockChannelState) SetDataLimit(dataLimit uint64) { + m.dataLimit = dataLimit +} + func (m *MockChannelState) DataLimit() uint64 { - panic("implement me") + return m.dataLimit } func (m *MockChannelState) RequiresFinalization() bool { panic("implement me") } + +func (m *MockChannelState) SetResponderPaused(responderPaused bool) { + m.responderPaused = responderPaused +} + +func (m *MockChannelState) ResponderPaused() bool { + return m.responderPaused +} + +func (m *MockChannelState) SetInitiatorPaused(initiatorPaused bool) { + m.initiatorPaused = initiatorPaused +} + +func (m *MockChannelState) InitiatorPaused() bool { + return m.initiatorPaused +} + +func (m *MockChannelState) BothPaused() bool { + return m.initiatorPaused && m.responderPaused +} + +func (m *MockChannelState) SelfPaused() bool { + if m.self == m.chid.Initiator { + return m.initiatorPaused + } + return m.responderPaused +} diff --git a/transport.go b/transport.go index bc434e75..13b860ee 100644 --- a/transport.go +++ b/transport.go @@ -17,66 +17,118 @@ const LegacyTransportID TransportID = "graphsync" // i.e. graphsync 1.0.0 var LegacyTransportVersion Version = Version{1, 0, 0} -// EventsHandler are semantic data transfer events that happen as a result of transport events -type EventsHandler interface { - // ChannelState queries for the current channel state - ChannelState(ctx context.Context, chid ChannelID) (ChannelState, error) +type TransportEvent interface { + transportEvent() +} - // OnChannelOpened is called at the point the transport begins processing the - // request (prior to that it may simply be queued) -- only applies to initiator - OnChannelOpened(chid ChannelID) +// TransportOpenedChannel occurs when the transport begins processing the +// request (prior to that it may simply be queued) -- only applies to initiator +type TransportOpenedChannel struct{} - // OnTransferInitiated is called at the point the transport actually begins sending/receiving data - OnTransferInitiated(chid ChannelID) +// TransportInitiatedTransfer occurs when the transport actually begins sending/receiving data +type TransportInitiatedTransfer struct{} - // OnRequestReceived is called when we receive a new request for the given channel ID - // return values is a channel command to take action based on this response - OnRequestReceived(chid ChannelID, msg Request) (ChannelCommand, error) +// TransportReceivedData occurs when we receive data for the given channel ID +// index is a transport dependent of serializing "here's where I am in this transport" +type TransportReceivedData struct { + Size uint64 + Index datamodel.Node +} - // OnResponseReceived is called when we receive a response to a request - OnResponseReceived(chid ChannelID, msg Response) +// TransportSentData occurs when we send data for the given channel ID +// index is a transport dependent of serializing "here's where I am in this transport" +type TransportSentData struct { + Size uint64 + Index datamodel.Node +} - // OnDataReceive is called when we receive data for the given channel ID - // index is a transport dependent of serializing "here's where I am in this transport" - OnDataReceived(chid ChannelID, size uint64, index datamodel.Node) +// TransportQueuedData occurs when data is queued for sending for the given channel ID +// index is a transport dependent of serializing "here's where I am in this transport" +type TransportQueuedData struct { + Size uint64 + Index datamodel.Node +} - // OnDataQueued is called when data is queued for sending for the given channel ID - // index is a transport dependent of serializing "here's where I am in this transport" - OnDataQueued(chid ChannelID, size uint64, index datamodel.Node) +// TransportReachedDataLimit occurs when a channel hits a previously set data limit +type TransportReachedDataLimit struct{} - // OnDataSent is called when we send data for the given channel ID - // index is a transport dependent of serializing "here's where I am in this transport" - OnDataSent(chid ChannelID, size uint64, index datamodel.Node) +/* +type TransportReceivedVoucherRequest struct { + Request Request +} - // OnDataLimitReached is called when a channel hits a previously set data limit - OnDataLimitReached(chid ChannelID) +type TransportReceivedUpdateRequest struct { + Request Request +} - // OnTransferCompleted is called when we finish transferring data for the given channel ID - OnTransferCompleted(chid ChannelID, err error) +type TransportReceivedCancelRequest struct { + Request Request +} - // OnTransferCannceled is called when a request we opened (with the given channel Id) to - // receive data is cancelled by us. - OnTransferCancelled(chid ChannelID, err error) +// TransportReceivedResponse occurs when we receive a response to a request +type TransportReceivedResponse struct { + Response Response +} +*/ - // OnMessageSendError is called when a network error occurs while sending a message - OnMessageSendError(chid ChannelID, err error) +// TransportTransferCancelled occurs when a request we opened (with the given channel Id) to +// receive data is cancelled by us. +type TransportTransferCancelled struct { + ErrorMessage string +} - // OnRequestDisconnected is called when a network error occurs trying to send a request - OnRequestDisconnected(chid ChannelID, err error) +// TransportErrorSendingData occurs when a network error occurs trying to send a request +type TransportErrorSendingData struct { + ErrorMessage string +} + +// TransportErrorReceivingData occurs when a network error occurs receiving data +// at the transport layer +type TransportErrorReceivingData struct { + ErrorMessage string +} + +// TransportCompletedTransfer occurs when we finish transferring data for the given channel ID +type TransportCompletedTransfer struct { + Success bool + ErrorMessage string +} + +type TransportReceivedRestartExistingChannelRequest struct{} + +// TransportErrorSendingMessage occurs when a network error occurs trying to send a request +type TransportErrorSendingMessage struct { + ErrorMessage string +} + +type TransportPaused struct{} - // OnSendDataError is called when a network error occurs sending data - // at the transport layer - OnSendDataError(chid ChannelID, err error) +type TransportResumed struct{} + +// EventsHandler are semantic data transfer events that happen as a result of transport events +type EventsHandler interface { + // ChannelState queries for the current channel state + ChannelState(ctx context.Context, chid ChannelID) (ChannelState, error) - // OnReceiveDataError is called when a network error occurs receiving data - // at the transport layer - OnReceiveDataError(chid ChannelID, err error) + // OnRequestReceived occurs when we receive a request for the given channel ID + // return values are a message to send an error if the transport should be closed + OnRequestReceived(chid ChannelID, msg Request) (Response, error) + + // OnRequestReceived occurs when we receive a response to a request + OnResponseReceived(chid ChannelID, msg Response) error + + // OnTransportEvent is dispatched when an event occurs on the transport + // It MAY be dispatched asynchronously by the transport to the time the + // event occurs + // However, the other handler functions may ONLY be called on the same channel + // after all events are dispatched. In other words, the transport MUST allow + // the handler to process all events before calling the other functions which + // have a synchronous return + OnTransportEvent(chid ChannelID, event TransportEvent) // OnContextAugment allows the transport to attach data transfer tracing information // to its local context, in order to create a hierarchical trace OnContextAugment(chid ChannelID) func(context.Context) context.Context - - OnRestartExistingChannelRequestReceived(chid ChannelID) } /* @@ -112,16 +164,16 @@ type Transport interface { req Request, ) error - // UpdateChannel sends one or more updates the transport channel at once, - // such as pausing/resuming, closing the transfer, or sending additional - // messages over the channel. Grouping the commands allows the transport - // the ability to plan how to execute these updates based on the capabilities - // and API of the underlying transport protocol and library - SendChannelCommand(ctx context.Context, chid ChannelID, command ChannelCommand) error + // ChannelUpdated notifies the transport that state of the channel has been updated, + // along with an optional message to send over the transport to tell + // the other peer about the update + ChannelUpdated(ctx context.Context, chid ChannelID, message Message) error // SetEventHandler sets the handler for events on channels SetEventHandler(events EventsHandler) error // CleanupChannel removes any associated data on a closed channel CleanupChannel(chid ChannelID) + // SendMessage sends a data transfer message over the channel to the other peer + SendMessage(ctx context.Context, chid ChannelID, msg Message) error // Shutdown unregisters the current EventHandler and ends all active data transfers Shutdown(ctx context.Context) error @@ -140,78 +192,22 @@ type TransportCapabilities struct { Pausable bool } -// ChannelUpdate describes updates to a channel - changing it's paused status, closing the transfer, -// and additional messages to send -type ChannelUpdate interface { - Paused() (bool, bool) - Closed() (bool, bool) - MessageToSend() (Message, bool) - DataLimit() (uint64, bool) -} - -type channelUpdate struct { - paused bool - hasPaused bool - closed bool - hasClosed bool - messageToSend Message - hasMessageToSend bool - dataLimit uint64 - hasDataLimit bool -} - -func (cu channelUpdate) Paused() (bool, bool) { - return cu.paused, cu.hasPaused -} - -func (cu channelUpdate) Closed() (bool, bool) { - return cu.closed, cu.hasClosed -} - -func (cu channelUpdate) MessageToSend() (Message, bool) { - return cu.messageToSend, cu.hasMessageToSend -} - -func (cu channelUpdate) DataLimit() (uint64, bool) { - return cu.dataLimit, cu.hasDataLimit -} - -type ChannelCommand struct { - cu channelUpdate -} - -func NewCommand() ChannelCommand { - return ChannelCommand{} -} - -func (cc ChannelCommand) SetPaused(paused bool) ChannelCommand { - cu := cc.cu - cu.paused = paused - cu.hasPaused = true - return ChannelCommand{cu} -} - -func (cc ChannelCommand) SetClosed(closed bool) ChannelCommand { - cu := cc.cu - cu.closed = true - cu.hasClosed = true - return ChannelCommand{cu} -} - -func (cc ChannelCommand) SendMessage(msg Message) ChannelCommand { - cu := cc.cu - cu.messageToSend = msg - cu.hasMessageToSend = true - return ChannelCommand{cu} -} - -func (cc ChannelCommand) SetDataLimit(datalimit uint64) ChannelCommand { - cu := cc.cu - cu.dataLimit = datalimit - cu.hasDataLimit = true - return ChannelCommand{cu} -} - -func (cc ChannelCommand) ChannelUpdate() ChannelUpdate { - return cc.cu -} +func (TransportOpenedChannel) transportEvent() {} +func (TransportInitiatedTransfer) transportEvent() {} +func (TransportReceivedData) transportEvent() {} +func (TransportSentData) transportEvent() {} +func (TransportQueuedData) transportEvent() {} +func (TransportReachedDataLimit) transportEvent() {} + +/*func (TransportReceivedVoucherRequest) transportEvent() {} +func (TransportReceivedUpdateRequest) transportEvent() {} +func (TransportReceivedCancelRequest) transportEvent() {} +func (TransportReceivedResponse) transportEvent() {}*/ +func (TransportTransferCancelled) transportEvent() {} +func (TransportErrorSendingData) transportEvent() {} +func (TransportErrorReceivingData) transportEvent() {} +func (TransportCompletedTransfer) transportEvent() {} +func (TransportReceivedRestartExistingChannelRequest) transportEvent() {} +func (TransportErrorSendingMessage) transportEvent() {} +func (TransportPaused) transportEvent() {} +func (TransportResumed) transportEvent() {} diff --git a/transport/graphsync/dtchannel/dtchannel.go b/transport/graphsync/dtchannel/dtchannel.go index e0a003be..81a9944f 100644 --- a/transport/graphsync/dtchannel/dtchannel.go +++ b/transport/graphsync/dtchannel/dtchannel.go @@ -33,6 +33,7 @@ const ( // Info needed to keep track of a data transfer channel type Channel struct { + isSender bool channelID datatransfer.ChannelID gs graphsync.GraphExchange @@ -46,7 +47,11 @@ type Channel struct { storeLk sync.RWMutex storeRegistered bool - receivedCidsTotal int64 + receivedIndex int64 + sentIndex int64 + queuedIndex int64 + dataLimit uint64 + progress uint64 } func NewChannel(channelID datatransfer.ChannelID, gs graphsync.GraphExchange) *Channel { @@ -94,8 +99,8 @@ func (c *Channel) Open( } // add do not send cids ext as needed - if c.receivedCidsTotal > 0 { - data := donotsendfirstblocks.EncodeDoNotSendFirstBlocks(c.receivedCidsTotal) + if c.receivedIndex > 0 { + data := donotsendfirstblocks.EncodeDoNotSendFirstBlocks(c.receivedIndex) exts = append(exts, graphsync.ExtensionData{ Name: graphsync.ExtensionsDoNotSendFirstBlocks, Data: data, @@ -118,8 +123,8 @@ func (c *Channel) Open( // Open a new graphsync request msg := fmt.Sprintf("Opening graphsync request to %s for root %s", dataSender, root) - if c.receivedCidsTotal > 0 { - msg += fmt.Sprintf(" with %d Blocks already received", c.receivedCidsTotal) + if c.receivedIndex > 0 { + msg += fmt.Sprintf(" with %d Blocks already received", c.receivedIndex) } log.Info(msg) c.requestID = &requestID @@ -158,7 +163,7 @@ func (c *Channel) GsReqOpened(sender peer.ID, requestID graphsync.RequestID, hoo // gsDataRequestRcvd is called when the transport receives an incoming request // for data. -func (c *Channel) GsDataRequestRcvd(sender peer.ID, requestID graphsync.RequestID, pauseRequest bool, hookActions graphsync.IncomingRequestHookActions) { +func (c *Channel) GsDataRequestRcvd(sender peer.ID, requestID graphsync.RequestID, chst datatransfer.ChannelState, hookActions graphsync.IncomingRequestHookActions) { c.lk.Lock() defer c.lk.Unlock() log.Debugf("%s: received request for data, req_id=%d", c.channelID, requestID) @@ -184,11 +189,26 @@ func (c *Channel) GsDataRequestRcvd(sender peer.ID, requestID graphsync.RequestI c.requestID = &requestID log.Infow("incoming graphsync request", "peer", sender, "graphsync request id", requestID, "data transfer channel id", c.channelID) - if pauseRequest { + c.state = channelOpen + + err := c.updateFromChannelState(chst) + if err != nil { + hookActions.TerminateWithError(err) + return + } + + action := c.actionFromChannelState(chst) + switch action { + case Pause: c.state = channelPaused + hookActions.PauseResponse() + case Close: + c.state = channelClosed + hookActions.TerminateWithError(datatransfer.ErrRejected) return + default: } - c.state = channelOpen + hookActions.ValidateRequest() } func (c *Channel) MarkPaused() { @@ -262,6 +282,113 @@ func (c *Channel) Resume(ctx context.Context, extensions []graphsync.ExtensionDa return c.gs.Unpause(ctx, *c.requestID, extensions...) } +type Action string + +const ( + NoAction Action = "" + Close Action = "close" + Pause Action = "pause" + Resume Action = "resume" +) + +// UpdateFromChannelState updates internal graphsync channel state form a datatransfer +// channel state +func (c *Channel) UpdateFromChannelState(chst datatransfer.ChannelState) error { + c.lk.Lock() + defer c.lk.Unlock() + return c.updateFromChannelState(chst) +} + +func (c *Channel) updateFromChannelState(chst datatransfer.ChannelState) error { + // read the sent value + sentNode := chst.SentIndex() + if !sentNode.IsNull() { + sentIndex, err := sentNode.AsInt() + if err != nil { + return err + } + if sentIndex > c.sentIndex { + c.sentIndex = sentIndex + } + } + + // read the received + receivedNode := chst.ReceivedIndex() + if !receivedNode.IsNull() { + receivedIndex, err := receivedNode.AsInt() + if err != nil { + return err + } + if receivedIndex > c.receivedIndex { + c.receivedIndex = receivedIndex + } + } + + // read the queued + queuedNode := chst.QueuedIndex() + if !queuedNode.IsNull() { + queuedIndex, err := queuedNode.AsInt() + if err != nil { + return err + } + if queuedIndex > c.queuedIndex { + c.queuedIndex = queuedIndex + } + } + + // set progress + var progress uint64 + if chst.Sender() == chst.SelfPeer() { + progress = chst.Queued() + } else { + progress = chst.Received() + } + if progress > c.progress { + c.progress = progress + } + + // set data limit + c.dataLimit = chst.DataLimit() + return nil +} + +// ActionFromChannelState comparse internal graphsync channel state with the data transfer +// state and determines what if any action should be taken on graphsync +func (c *Channel) ActionFromChannelState(chst datatransfer.ChannelState) Action { + c.lk.Lock() + defer c.lk.Unlock() + return c.actionFromChannelState(chst) +} + +func (c *Channel) actionFromChannelState(chst datatransfer.ChannelState) Action { + // if the state is closed, and we haven't closed, we need to close + if !c.requesterCancelled && c.state != channelClosed && chst.Status().TransferComplete() { + return Close + } + + // if the state is running, and we're paused, we need to pause + if c.requestID != nil && c.state == channelPaused && !chst.SelfPaused() { + return Resume + } + + // if the state is paused, and the transfer is running, we need to resume + if c.requestID != nil && c.state == channelOpen && chst.SelfPaused() { + return Pause + } + + return NoAction +} + +func (c *Channel) ReconcileChannelState(chst datatransfer.ChannelState) (Action, error) { + c.lk.Lock() + defer c.lk.Unlock() + err := c.updateFromChannelState(chst) + if err != nil { + return NoAction, err + } + return c.actionFromChannelState(chst), nil +} + func (c *Channel) MarkTransferComplete() { c.lk.Lock() defer c.lk.Unlock() @@ -300,12 +427,45 @@ func (c *Channel) UseStore(lsys ipld.LinkSystem) error { return nil } -func (c *Channel) UpdateReceivedCidsIfGreater(nextIdx int64) { +func (c *Channel) UpdateReceivedIndexIfGreater(nextIdx int64) bool { c.lk.Lock() defer c.lk.Unlock() - if c.receivedCidsTotal < nextIdx { - c.receivedCidsTotal = nextIdx + if c.receivedIndex < nextIdx { + c.receivedIndex = nextIdx + return true + } + return false +} + +func (c *Channel) UpdateQueuedIndexIfGreater(nextIdx int64) bool { + c.lk.Lock() + defer c.lk.Unlock() + if c.queuedIndex < nextIdx { + c.queuedIndex = nextIdx + return true + } + return false +} + +func (c *Channel) UpdateSentIndexIfGreater(nextIdx int64) bool { + c.lk.Lock() + defer c.lk.Unlock() + if c.sentIndex < nextIdx { + c.sentIndex = nextIdx + return true + } + return false +} + +func (c *Channel) UpdateProgress(additionalData uint64) bool { + c.lk.Lock() + defer c.lk.Unlock() + c.progress += additionalData + reachedLimit := c.dataLimit != 0 && c.progress >= c.dataLimit + if reachedLimit { + c.state = channelPaused } + return reachedLimit } func (c *Channel) Cleanup() { diff --git a/transport/graphsync/executor/executor.go b/transport/graphsync/executor/executor.go index 0bc23e4f..b0a8b4c0 100644 --- a/transport/graphsync/executor/executor.go +++ b/transport/graphsync/executor/executor.go @@ -13,8 +13,7 @@ var log = logging.Logger("dt_graphsync") // EventsHandler are the data transfer events that can be dispatched by the execetor type EventsHandler interface { - OnRequestCancelled(datatransfer.ChannelID, error) error - OnChannelCompleted(datatransfer.ChannelID, error) error + OnTransportEvent(datatransfer.ChannelID, datatransfer.TransportEvent) } // Executor handles consuming channels on an outgoing GraphSync request @@ -74,9 +73,7 @@ func (e *Executor) executeRequest( if _, ok := lastError.(graphsync.RequestClientCancelledErr); ok { terr := fmt.Errorf("graphsync request cancelled") log.Warnf("channel %s: %s", e.channelID, terr) - if err := events.OnRequestCancelled(e.channelID, terr); err != nil { - log.Error(err) - } + events.OnTransportEvent(e.channelID, datatransfer.TransportTransferCancelled{ErrorMessage: terr.Error()}) return } @@ -102,8 +99,11 @@ func (e *Executor) executeRequest( if completedRequestListener != nil { completedRequestListener(e.channelID) } - err := events.OnChannelCompleted(e.channelID, completeErr) - if err != nil { - log.Errorf("channel %s: processing OnChannelCompleted: %s", e.channelID, err) + + if completeErr == nil { + events.OnTransportEvent(e.channelID, datatransfer.TransportCompletedTransfer{Success: true}) + } else { + events.OnTransportEvent(e.channelID, datatransfer.TransportCompletedTransfer{Success: false, ErrorMessage: completeErr.Error()}) } + } diff --git a/transport/graphsync/executor/executor_test.go b/transport/graphsync/executor/executor_test.go index 9e5523c1..089a151d 100644 --- a/transport/graphsync/executor/executor_test.go +++ b/transport/graphsync/executor/executor_test.go @@ -34,7 +34,7 @@ func TestExecutor(t *testing.T) { responseErrors: []error{errors.New("something went wrong")}, expectedEventRecord: fakeEvents{ completedChannel: chid, - completedError: fmt.Errorf("channel %s: graphsync request failed to complete: %w", chid, errors.New("something went wrong")), + completedError: fmt.Errorf("channel %s: graphsync request failed to complete: %s", chid, errors.New("something went wrong")), }, }, "client cancelled request error, no listener": { @@ -119,16 +119,17 @@ type fakeEvents struct { cancelledErr error } -func (fe *fakeEvents) OnChannelCompleted(chid datatransfer.ChannelID, err error) error { - fe.completedChannel = chid - fe.completedError = err - return nil -} - -func (fe *fakeEvents) OnRequestCancelled(chid datatransfer.ChannelID, err error) error { - fe.cancelledChannel = chid - fe.cancelledErr = err - return nil +func (fe *fakeEvents) OnTransportEvent(chid datatransfer.ChannelID, transportEvent datatransfer.TransportEvent) { + switch evt := transportEvent.(type) { + case datatransfer.TransportCompletedTransfer: + fe.completedChannel = chid + if !evt.Success { + fe.completedError = errors.New(evt.ErrorMessage) + } + case datatransfer.TransportTransferCancelled: + fe.cancelledChannel = chid + fe.cancelledErr = errors.New(evt.ErrorMessage) + } } type fakeCompletedRequestListener struct { diff --git a/transport/graphsync/graphsync.go b/transport/graphsync/graphsync.go index 07fe307e..28a9e8b1 100644 --- a/transport/graphsync/graphsync.go +++ b/transport/graphsync/graphsync.go @@ -36,12 +36,10 @@ var defaultSupportedExtensions = []graphsync.ExtensionName{ var incomingReqExtensions = []graphsync.ExtensionName{ extension.ExtensionIncomingRequest1_1, - extension.ExtensionDataTransfer1_1, } var outgoingBlkExtensions = []graphsync.ExtensionName{ extension.ExtensionOutgoingBlock1_1, - extension.ExtensionDataTransfer1_1, } // Option is an option for setting up the graphsync transport @@ -91,11 +89,11 @@ type Transport struct { } // NewTransport makes a new hooks manager with the given hook events interface -func NewTransport(peerID peer.ID, gs graphsync.GraphExchange, dtNet network.DataTransferNetwork, options ...Option) *Transport { +func NewTransport(gs graphsync.GraphExchange, dtNet network.DataTransferNetwork, options ...Option) *Transport { t := &Transport{ gs: gs, dtNet: dtNet, - peerID: peerID, + peerID: dtNet.ID(), supportedExtensions: defaultSupportedExtensions, dtChannels: make(map[datatransfer.ChannelID]*dtchannel.Channel), requestIDToChannelID: newRequestIDToChannelIDMap(), @@ -128,6 +126,7 @@ func (t *Transport) OpenChannel( channel datatransfer.Channel, req datatransfer.Request) error { t.dtNet.Protect(channel.OtherPeer(), channel.ChannelID().String()) + t.trackDTChannel(channel.ChannelID()) if channel.IsPull() { return t.openRequest(ctx, channel.Sender(), @@ -157,8 +156,13 @@ func (t *Transport) RestartChannel( t.dtNet.Protect(channelState.OtherPeer(), channelState.ChannelID().String()) ch := t.trackDTChannel(channelState.ChannelID()) - ch.UpdateReceivedCidsIfGreater(channelState.ReceivedCidsTotal()) + err = ch.UpdateFromChannelState(channelState) + if err != nil { + return err + } + if channelState.IsPull() { + return t.openRequest(ctx, channelState.Sender(), channelState.ChannelID(), @@ -203,49 +207,59 @@ func (t *Transport) openRequest( return nil } -// UpdateChannel sends one or more updates the transport channel at once, -// such as pausing/resuming, closing the transfer, or sending additional -// messages over the channel. Grouping the commands allows the transport -// the ability to plan how to execute these updates -func (t *Transport) UpdateChannel(ctx context.Context, chid datatransfer.ChannelID, update datatransfer.ChannelUpdate) error { - +func (t *Transport) reconcileChannelStates(ctx context.Context, chid datatransfer.ChannelID) (*dtchannel.Channel, dtchannel.Action, error) { + chst, err := t.events.ChannelState(ctx, chid) + if err != nil { + return nil, dtchannel.NoAction, err + } ch, err := t.getDTChannel(chid) if err != nil { - if update.SendMessage != nil && !update.Closed { - return t.dtNet.SendMessage(ctx, t.otherPeer(chid), transportID, update.SendMessage) + return nil, dtchannel.NoAction, err + } + action, err := ch.ReconcileChannelState(chst) + return ch, action, err +} + +// ChannelUpdated notifies the transport that state of the channel has been updated, +// along with an optional message to send over the transport to tell +// the other peer about the update +func (t *Transport) ChannelUpdated(ctx context.Context, chid datatransfer.ChannelID, message datatransfer.Message) error { + ch, action, err := t.reconcileChannelStates(ctx, chid) + if err != nil { + if message != nil { + return t.dtNet.SendMessage(ctx, t.otherPeer(chid), transportID, message) } return err } + return t.processAction(ctx, chid, ch, action, message) +} - if !update.Paused && ch.Paused() { - +func (t *Transport) processAction(ctx context.Context, chid datatransfer.ChannelID, ch *dtchannel.Channel, action dtchannel.Action, message datatransfer.Message) error { + if action == dtchannel.Resume { var extensions []graphsync.ExtensionData - if update.SendMessage != nil { + if message != nil { var err error - extensions, err = extension.ToExtensionData(update.SendMessage, t.supportedExtensions) + extensions, err = extension.ToExtensionData(message, t.supportedExtensions) if err != nil { return err } } - return ch.Resume(ctx, extensions) } - if update.SendMessage != nil { - if err := t.dtNet.SendMessage(ctx, t.otherPeer(chid), transportID, update.SendMessage); err != nil { + if message != nil { + if err := t.dtNet.SendMessage(ctx, t.otherPeer(chid), transportID, message); err != nil { return err } } - - if update.Closed { + switch action { + case dtchannel.Close: return ch.Close(ctx) - } - - if update.Paused && !ch.Paused() { + case dtchannel.Pause: return ch.Pause(ctx) + default: + return nil } - - return nil } // SendMessage sends a data transfer message over the channel to the other peer @@ -284,15 +298,16 @@ func (t *Transport) SetEventHandler(events datatransfer.EventsHandler) error { } t.events = events - t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterIncomingRequestQueuedHook(t.gsReqQueuedHook)) + t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterIncomingRequestProcessingListener(t.gsRequestProcessingListener)) + t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterOutgoingRequestProcessingListener(t.gsRequestProcessingListener)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterIncomingRequestHook(t.gsReqRecdHook)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterCompletedResponseListener(t.gsCompletedResponseListener)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterIncomingBlockHook(t.gsIncomingBlockHook)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterOutgoingBlockHook(t.gsOutgoingBlockHook)) - t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterBlockSentListener(t.gsBlockSentHook)) + t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterBlockSentListener(t.gsBlockSentListener)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterOutgoingRequestHook(t.gsOutgoingRequestHook)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterIncomingResponseHook(t.gsIncomingResponseHook)) - t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterRequestUpdatedHook(t.gsRequestUpdatedHook)) + //t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterRequestUpdatedHook(t.gsRequestUpdatedHook)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterRequestorCancelledListener(t.gsRequestorCancelledListener)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterNetworkErrorListener(t.gsNetworkSendErrorListener)) t.unregisterFuncs = append(t.unregisterFuncs, t.gs.RegisterReceiverNetworkErrorListener(t.gsNetworkReceiveErrorListener)) diff --git a/transport/graphsync/graphsync_test.go b/transport/graphsync/graphsync_test.go deleted file mode 100644 index 5eccc205..00000000 --- a/transport/graphsync/graphsync_test.go +++ /dev/null @@ -1,1362 +0,0 @@ -package graphsync_test - -/* -import ( - "context" - "errors" - "io" - "math/rand" - "testing" - "time" - "github.com/ipfs/go-graphsync" - "github.com/ipfs/go-graphsync/donotsendfirstblocks" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/datamodel" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/node/basicnode" - peer "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/protocol" - "github.com/stretchr/testify/require" - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/message" - "github.com/filecoin-project/go-data-transfer/v2/testutil" - . "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" - "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" -) - -func TestManager(t *testing.T) { - testCases := map[string]struct { - requestConfig gsRequestConfig - responseConfig gsResponseConfig - updatedConfig gsRequestConfig - events fakeEvents - action func(gsData *harness) - check func(t *testing.T, events *fakeEvents, gsData *harness) - protocol protocol.ID - }{ - "gs outgoing request with recognized dt pull channel will record incoming blocks": { - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingBlockHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) - require.True(t, events.OnDataReceivedCalled) - require.NoError(t, gsData.incomingBlockHookActions.TerminationError) - }, - }, - "gs outgoing request with recognized dt push channel will record incoming blocks": { - requestConfig: gsRequestConfig{ - dtIsResponse: true, - }, - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingBlockHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - require.True(t, events.OnDataReceivedCalled) - require.NoError(t, gsData.incomingBlockHookActions.TerminationError) - }, - }, - "non-data-transfer gs request will not record incoming blocks and send updates": { - requestConfig: gsRequestConfig{ - dtExtensionMissing: true, - }, - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingBlockHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{}) - require.False(t, events.OnDataReceivedCalled) - require.NoError(t, gsData.incomingBlockHookActions.TerminationError) - }, - }, - "gs request unrecognized opened channel will not record incoming blocks": { - events: fakeEvents{ - OnChannelOpenedError: errors.New("Not recognized"), - }, - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingBlockHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) - require.False(t, events.OnDataReceivedCalled) - require.NoError(t, gsData.incomingBlockHookActions.TerminationError) - }, - }, - "gs incoming block with data receive error will halt request": { - events: fakeEvents{ - OnDataReceivedError: errors.New("something went wrong"), - }, - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingBlockHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) - require.True(t, events.OnDataReceivedCalled) - require.Error(t, gsData.incomingBlockHookActions.TerminationError) - }, - }, - "outgoing gs request with recognized dt request can receive gs response": { - responseConfig: gsResponseConfig{ - dtIsResponse: true, - }, - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingResponseHOok() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) - require.Equal(t, 0, events.OnRequestReceivedCallCount) - require.Equal(t, 1, events.OnResponseReceivedCallCount) - require.NoError(t, gsData.incomingResponseHookActions.TerminationError) - }, - }, - "outgoing gs request with recognized dt request cannot receive gs response with dt request": { - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingResponseHOok() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) - require.Equal(t, 0, events.OnRequestReceivedCallCount) - require.Equal(t, 0, events.OnResponseReceivedCallCount) - require.Error(t, gsData.incomingResponseHookActions.TerminationError) - }, - }, - "outgoing gs request with recognized dt response can receive gs response": { - requestConfig: gsRequestConfig{ - dtIsResponse: true, - }, - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingResponseHOok() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.Equal(t, 0, events.OnResponseReceivedCallCount) - require.NoError(t, gsData.incomingResponseHookActions.TerminationError) - }, - }, - "outgoing gs request with recognized dt response cannot receive gs response with dt response": { - requestConfig: gsRequestConfig{ - dtIsResponse: true, - }, - responseConfig: gsResponseConfig{ - dtIsResponse: true, - }, - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingResponseHOok() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - require.Equal(t, 0, events.OnResponseReceivedCallCount) - require.Equal(t, 0, events.OnRequestReceivedCallCount) - require.Error(t, gsData.incomingResponseHookActions.TerminationError) - }, - }, - "outgoing gs request with recognized dt request will error with malformed update": { - responseConfig: gsResponseConfig{ - dtExtensionMalformed: true, - }, - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingResponseHOok() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) - require.Equal(t, 0, events.OnRequestReceivedCallCount) - require.Equal(t, 0, events.OnResponseReceivedCallCount) - require.Error(t, gsData.incomingResponseHookActions.TerminationError) - }, - }, - "outgoing gs request with recognized dt request will ignore non-data-transfer update": { - responseConfig: gsResponseConfig{ - dtExtensionMissing: true, - }, - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingResponseHOok() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) - require.Equal(t, 0, events.OnRequestReceivedCallCount) - require.Equal(t, 0, events.OnResponseReceivedCallCount) - require.NoError(t, gsData.incomingResponseHookActions.TerminationError) - }, - }, - "outgoing gs request with recognized dt response can send message on update": { - events: fakeEvents{ - RequestReceivedResponse: testutil.NewDTResponse(t, datatransfer.TransferID(rand.Uint32())), - }, - requestConfig: gsRequestConfig{ - dtIsResponse: true, - }, - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingResponseHOok() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.Equal(t, 0, events.OnResponseReceivedCallCount) - require.NoError(t, gsData.incomingResponseHookActions.TerminationError) - assertHasOutgoingMessage(t, gsData.incomingResponseHookActions.SentExtensions, - events.RequestReceivedResponse) - }, - }, - "outgoing gs request with recognized dt response err will error": { - requestConfig: gsRequestConfig{ - dtIsResponse: true, - }, - events: fakeEvents{ - OnRequestReceivedErrors: []error{errors.New("something went wrong")}, - }, - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.incomingResponseHOok() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 0, events.OnResponseReceivedCallCount) - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.Error(t, gsData.incomingResponseHookActions.TerminationError) - }, - }, - "incoming gs request with recognized dt request will validate gs request & send dt response": { - action: func(gsData *harness) { - gsData.incomingRequestHook() - }, - events: fakeEvents{ - RequestReceivedResponse: testutil.NewDTResponse(t, datatransfer.TransferID(rand.Uint32())), - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.Equal(t, 0, events.OnResponseReceivedCallCount) - require.Equal(t, events.RequestReceivedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - dtRequestData, _ := gsData.request.Extension(extension.ExtensionDataTransfer1_1) - assertDecodesToMessage(t, dtRequestData, events.RequestReceivedRequest) - require.True(t, gsData.incomingRequestHookActions.Validated) - assertHasExtensionMessage(t, extension.ExtensionDataTransfer1_1, gsData.incomingRequestHookActions.SentExtensions, events.RequestReceivedResponse) - require.NoError(t, gsData.incomingRequestHookActions.TerminationError) - - channelsForPeer := gsData.transport.ChannelsForPeer(gsData.other) - require.Equal(t, channelsForPeer, ChannelsForPeer{ - SendingChannels: map[datatransfer.ChannelID]ChannelGraphsyncRequests{ - events.RequestReceivedChannelID: { - Current: gsData.request.ID(), - }, - }, - ReceivingChannels: map[datatransfer.ChannelID]ChannelGraphsyncRequests{}, - }) - }, - }, - "incoming gs request with recognized dt response will validate gs request": { - requestConfig: gsRequestConfig{ - dtIsResponse: true, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 0, events.OnRequestReceivedCallCount) - require.Equal(t, 1, events.OnResponseReceivedCallCount) - require.Equal(t, events.ResponseReceivedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) - dtResponseData, _ := gsData.request.Extension(extension.ExtensionDataTransfer1_1) - assertDecodesToMessage(t, dtResponseData, events.ResponseReceivedResponse) - require.True(t, gsData.incomingRequestHookActions.Validated) - require.NoError(t, gsData.incomingRequestHookActions.TerminationError) - }, - }, - "malformed data transfer extension on incoming request will terminate": { - requestConfig: gsRequestConfig{ - dtExtensionMalformed: true, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 0, events.OnRequestReceivedCallCount) - require.False(t, gsData.incomingRequestHookActions.Validated) - require.Error(t, gsData.incomingRequestHookActions.TerminationError) - }, - }, - "unrecognized incoming dt request will terminate but send response": { - events: fakeEvents{ - RequestReceivedResponse: testutil.NewDTResponse(t, datatransfer.TransferID(rand.Uint32())), - OnRequestReceivedErrors: []error{errors.New("something went wrong")}, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.Equal(t, 0, events.OnResponseReceivedCallCount) - require.Equal(t, events.RequestReceivedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - dtRequestData, _ := gsData.request.Extension(extension.ExtensionDataTransfer1_1) - assertDecodesToMessage(t, dtRequestData, events.RequestReceivedRequest) - require.False(t, gsData.incomingRequestHookActions.Validated) - assertHasExtensionMessage(t, extension.ExtensionIncomingRequest1_1, gsData.incomingRequestHookActions.SentExtensions, events.RequestReceivedResponse) - require.Error(t, gsData.incomingRequestHookActions.TerminationError) - }, - }, - "incoming gs request with recognized dt request will record outgoing blocks": { - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.outgoingBlockHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.True(t, events.OnDataQueuedCalled) - require.NoError(t, gsData.outgoingBlockHookActions.TerminationError) - }, - }, - - "incoming gs request with recognized dt response will record outgoing blocks": { - requestConfig: gsRequestConfig{ - dtIsResponse: true, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.outgoingBlockHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnResponseReceivedCallCount) - require.True(t, events.OnDataQueuedCalled) - require.NoError(t, gsData.outgoingBlockHookActions.TerminationError) - }, - }, - "non-data-transfer request will not record outgoing blocks": { - requestConfig: gsRequestConfig{ - dtExtensionMissing: true, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.outgoingBlockHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 0, events.OnRequestReceivedCallCount) - require.False(t, events.OnDataQueuedCalled) - }, - }, - "outgoing data queued error will terminate request": { - events: fakeEvents{ - OnDataQueuedError: errors.New("something went wrong"), - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.outgoingBlockHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.True(t, events.OnDataQueuedCalled) - require.Error(t, gsData.outgoingBlockHookActions.TerminationError) - }, - }, - "outgoing data queued error == pause will pause request": { - events: fakeEvents{ - OnDataQueuedError: datatransfer.ErrPause, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.outgoingBlockHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.True(t, events.OnDataQueuedCalled) - require.True(t, gsData.outgoingBlockHookActions.Paused) - require.NoError(t, gsData.outgoingBlockHookActions.TerminationError) - }, - }, - "incoming gs request with recognized dt request will send updates": { - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.outgoingBlockHook() - }, - events: fakeEvents{ - OnDataQueuedMessage: testutil.NewDTResponse(t, datatransfer.TransferID(rand.Uint32())), - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.True(t, events.OnDataQueuedCalled) - require.NoError(t, gsData.outgoingBlockHookActions.TerminationError) - assertHasExtensionMessage(t, extension.ExtensionOutgoingBlock1_1, gsData.outgoingBlockHookActions.SentExtensions, - events.OnDataQueuedMessage) - }, - }, - "incoming gs request with recognized dt request can receive update": { - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.requestUpdatedHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 2, events.OnRequestReceivedCallCount) - require.NoError(t, gsData.requestUpdatedHookActions.TerminationError) - }, - }, - "incoming gs request with recognized dt request cannot receive update with dt response": { - updatedConfig: gsRequestConfig{ - dtIsResponse: true, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.requestUpdatedHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.Equal(t, 0, events.OnResponseReceivedCallCount) - require.Error(t, gsData.requestUpdatedHookActions.TerminationError) - }, - }, - "incoming gs request with recognized dt response can receive update": { - requestConfig: gsRequestConfig{ - dtIsResponse: true, - }, - updatedConfig: gsRequestConfig{ - dtIsResponse: true, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.requestUpdatedHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 2, events.OnResponseReceivedCallCount) - require.NoError(t, gsData.requestUpdatedHookActions.TerminationError) - }, - }, - "incoming gs request with recognized dt response cannot receive update with dt request": { - requestConfig: gsRequestConfig{ - dtIsResponse: true, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.requestUpdatedHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnResponseReceivedCallCount) - require.Equal(t, 0, events.OnRequestReceivedCallCount) - require.Error(t, gsData.requestUpdatedHookActions.TerminationError) - }, - }, - "incoming gs request with recognized dt request will error with malformed update": { - updatedConfig: gsRequestConfig{ - dtExtensionMalformed: true, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.requestUpdatedHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.Error(t, gsData.requestUpdatedHookActions.TerminationError) - }, - }, - "incoming gs request with recognized dt request will ignore non-data-transfer update": { - updatedConfig: gsRequestConfig{ - dtExtensionMissing: true, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.requestUpdatedHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.NoError(t, gsData.requestUpdatedHookActions.TerminationError) - }, - }, - "incoming gs request with recognized dt request can send message on update": { - events: fakeEvents{ - RequestReceivedResponse: testutil.NewDTResponse(t, datatransfer.TransferID(rand.Uint32())), - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.requestUpdatedHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 2, events.OnRequestReceivedCallCount) - require.NoError(t, gsData.requestUpdatedHookActions.TerminationError) - assertHasOutgoingMessage(t, gsData.requestUpdatedHookActions.SentExtensions, - events.RequestReceivedResponse) - }, - }, - "recognized incoming request will record successful request completion": { - responseConfig: gsResponseConfig{ - status: graphsync.RequestCompletedFull, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.responseCompletedListener() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.True(t, events.OnChannelCompletedCalled) - require.True(t, events.ChannelCompletedSuccess) - }, - }, - - "recognized incoming request will record unsuccessful request completion": { - responseConfig: gsResponseConfig{ - status: graphsync.RequestCompletedPartial, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.responseCompletedListener() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.True(t, events.OnChannelCompletedCalled) - require.False(t, events.ChannelCompletedSuccess) - }, - }, - "recognized incoming request will not record request cancellation": { - responseConfig: gsResponseConfig{ - status: graphsync.RequestCancelled, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.responseCompletedListener() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.False(t, events.OnChannelCompletedCalled) - }, - }, - "non-data-transfer request will not record request completed": { - requestConfig: gsRequestConfig{ - dtExtensionMissing: true, - }, - responseConfig: gsResponseConfig{ - status: graphsync.RequestCompletedPartial, - }, - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.responseCompletedListener() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 0, events.OnRequestReceivedCallCount) - require.False(t, events.OnChannelCompletedCalled) - }, - }, - "recognized incoming request can be closed": { - action: func(gsData *harness) { - gsData.incomingRequestHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - err := gsData.transport.CloseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - require.NoError(t, err) - require.Equal(t, 1, events.OnRequestReceivedCallCount) - gsData.fgs.AssertCancelReceived(gsData.ctx, t) - }, - }, - "unrecognized request cannot be closed": { - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - err := gsData.transport.CloseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - require.Error(t, err) - }, - }, - "recognized incoming request that requestor cancelled will not close via graphsync": { - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.requestorCancelledListener() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - err := gsData.transport.CloseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - require.NoError(t, err) - require.Equal(t, 1, events.OnRequestReceivedCallCount) - gsData.fgs.AssertNoCancelReceived(t) - }, - }, - "recognized incoming request can be paused": { - action: func(gsData *harness) { - gsData.incomingRequestHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - err := gsData.transport.PauseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - require.NoError(t, err) - require.Equal(t, 1, events.OnRequestReceivedCallCount) - gsData.fgs.AssertPauseReceived(gsData.ctx, t) - }, - }, - "unrecognized request cannot be paused": { - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - err := gsData.transport.PauseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - require.Error(t, err) - }, - }, - "recognized incoming request that requestor cancelled will not pause via graphsync": { - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.requestorCancelledListener() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - err := gsData.transport.PauseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - require.NoError(t, err) - require.Equal(t, 1, events.OnRequestReceivedCallCount) - gsData.fgs.AssertNoPauseReceived(t) - }, - }, - - "incoming request can be queued": { - action: func(gsData *harness) { - gsData.incomingRequestQueuedHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.True(t, events.TransferQueuedCalled) - require.Equal(t, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}, - events.TransferQueuedChannelID) - }, - }, - - "incoming request with dtResponse can be queued": { - requestConfig: gsRequestConfig{ - dtIsResponse: true, - }, - responseConfig: gsResponseConfig{ - dtIsResponse: true, - }, - action: func(gsData *harness) { - gsData.incomingRequestQueuedHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.True(t, events.TransferQueuedCalled) - require.Equal(t, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, - events.TransferQueuedChannelID) - }, - }, - - "recognized incoming request can be resumed": { - action: func(gsData *harness) { - gsData.incomingRequestHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - err := gsData.transport.ResumeChannel(gsData.ctx, - gsData.incoming, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}, - ) - require.NoError(t, err) - require.Equal(t, 1, events.OnRequestReceivedCallCount) - gsData.fgs.AssertResumeReceived(gsData.ctx, t) - }, - }, - - "unrecognized request cannot be resumed": { - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - err := gsData.transport.ResumeChannel(gsData.ctx, - gsData.incoming, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}, - ) - require.Error(t, err) - }, - }, - "recognized incoming request that requestor cancelled will not resume via graphsync but will resume otherwise": { - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.requestorCancelledListener() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - err := gsData.transport.ResumeChannel(gsData.ctx, - gsData.incoming, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}, - ) - require.NoError(t, err) - require.Equal(t, 1, events.OnRequestReceivedCallCount) - gsData.fgs.AssertNoResumeReceived(t) - gsData.incomingRequestHook() - assertHasOutgoingMessage(t, gsData.incomingRequestHookActions.SentExtensions, gsData.incoming) - }, - }, - "recognized incoming request will record network send error": { - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.networkErrorListener(errors.New("something went wrong")) - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.True(t, events.OnSendDataErrorCalled) - }, - }, - "recognized outgoing request will record network send error": { - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.networkErrorListener(errors.New("something went wrong")) - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.True(t, events.OnSendDataErrorCalled) - }, - }, - "recognized incoming request will record network receive error": { - action: func(gsData *harness) { - gsData.incomingRequestHook() - gsData.receiverNetworkErrorListener(errors.New("something went wrong")) - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.Equal(t, 1, events.OnRequestReceivedCallCount) - require.True(t, events.OnReceiveDataErrorCalled) - }, - }, - "recognized outgoing request will record network receive error": { - action: func(gsData *harness) { - gsData.outgoingRequestHook() - gsData.receiverNetworkErrorListener(errors.New("something went wrong")) - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - require.True(t, events.OnReceiveDataErrorCalled) - }, - }, - "open channel adds block count to the DoNotSendFirstBlocks extension for v1.2 protocol": { - action: func(gsData *harness) { - cids := testutil.GenerateCids(2) - channel := testutil.NewMockChannelState(testutil.MockChannelStateParams{ReceivedCids: cids}) - stor, _ := gsData.outgoing.Selector() - - go gsData.outgoingRequestHook() - _ = gsData.transport.OpenChannel( - gsData.ctx, - gsData.other, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, - cidlink.Link{Cid: gsData.outgoing.BaseCid()}, - stor, - channel, - gsData.outgoing) - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - requestReceived := gsData.fgs.AssertRequestReceived(gsData.ctx, t) - - ext := requestReceived.Extensions - require.Len(t, ext, 2) - doNotSend := ext[1] - - name := doNotSend.Name - require.Equal(t, graphsync.ExtensionsDoNotSendFirstBlocks, name) - data := doNotSend.Data - blockCount, err := donotsendfirstblocks.DecodeDoNotSendFirstBlocks(data) - require.NoError(t, err) - require.EqualValues(t, blockCount, 2) - }, - }, - "ChannelsForPeer when request is open": { - action: func(gsData *harness) { - cids := testutil.GenerateCids(2) - channel := testutil.NewMockChannelState(testutil.MockChannelStateParams{ReceivedCids: cids}) - stor, _ := gsData.outgoing.Selector() - - go gsData.outgoingRequestHook() - _ = gsData.transport.OpenChannel( - gsData.ctx, - gsData.other, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, - cidlink.Link{Cid: gsData.outgoing.BaseCid()}, - stor, - channel, - gsData.outgoing) - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - gsData.fgs.AssertRequestReceived(gsData.ctx, t) - - channelsForPeer := gsData.transport.ChannelsForPeer(gsData.other) - require.Equal(t, channelsForPeer, ChannelsForPeer{ - ReceivingChannels: map[datatransfer.ChannelID]ChannelGraphsyncRequests{ - events.ChannelOpenedChannelID: { - Current: gsData.request.ID(), - }, - }, - SendingChannels: map[datatransfer.ChannelID]ChannelGraphsyncRequests{}, - }) - }, - }, - "open channel cancels an existing request with the same channel ID": { - action: func(gsData *harness) { - cids := testutil.GenerateCids(2) - channel := testutil.NewMockChannelState(testutil.MockChannelStateParams{ReceivedCids: cids}) - stor, _ := gsData.outgoing.Selector() - go gsData.outgoingRequestHook() - _ = gsData.transport.OpenChannel( - gsData.ctx, - gsData.other, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, - cidlink.Link{Cid: gsData.outgoing.BaseCid()}, - stor, - channel, - gsData.outgoing) - - go gsData.altOutgoingRequestHook() - _ = gsData.transport.OpenChannel( - gsData.ctx, - gsData.other, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, - cidlink.Link{Cid: gsData.outgoing.BaseCid()}, - stor, - channel, - gsData.outgoing) - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - gsData.fgs.AssertRequestReceived(gsData.ctx, t) - gsData.fgs.AssertRequestReceived(gsData.ctx, t) - - ctxt, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - gsData.fgs.AssertCancelReceived(ctxt, t) - - channelsForPeer := gsData.transport.ChannelsForPeer(gsData.other) - require.Equal(t, channelsForPeer, ChannelsForPeer{ - ReceivingChannels: map[datatransfer.ChannelID]ChannelGraphsyncRequests{ - events.ChannelOpenedChannelID: { - Current: gsData.altRequest.ID(), - Previous: []graphsync.RequestID{gsData.request.ID()}, - }, - }, - SendingChannels: map[datatransfer.ChannelID]ChannelGraphsyncRequests{}, - }) - }, - }, - "OnChannelCompleted called when outgoing request completes successfully": { - action: func(gsData *harness) { - gsData.fgs.LeaveRequestsOpen() - stor, _ := gsData.outgoing.Selector() - - go gsData.outgoingRequestHook() - _ = gsData.transport.OpenChannel( - gsData.ctx, - gsData.other, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, - cidlink.Link{Cid: gsData.outgoing.BaseCid()}, - stor, - nil, - gsData.outgoing) - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - requestReceived := gsData.fgs.AssertRequestReceived(gsData.ctx, t) - close(requestReceived.ResponseChan) - close(requestReceived.ResponseErrChan) - - require.Eventually(t, func() bool { - return events.OnChannelCompletedCalled == true - }, 2*time.Second, 100*time.Millisecond) - require.True(t, events.ChannelCompletedSuccess) - }, - }, - "OnChannelCompleted called when outgoing request completes with error": { - action: func(gsData *harness) { - gsData.fgs.LeaveRequestsOpen() - stor, _ := gsData.outgoing.Selector() - - go gsData.outgoingRequestHook() - _ = gsData.transport.OpenChannel( - gsData.ctx, - gsData.other, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, - cidlink.Link{Cid: gsData.outgoing.BaseCid()}, - stor, - nil, - gsData.outgoing) - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - requestReceived := gsData.fgs.AssertRequestReceived(gsData.ctx, t) - close(requestReceived.ResponseChan) - requestReceived.ResponseErrChan <- graphsync.RequestFailedUnknownErr{} - close(requestReceived.ResponseErrChan) - - require.Eventually(t, func() bool { - return events.OnChannelCompletedCalled == true - }, 2*time.Second, 100*time.Millisecond) - require.False(t, events.ChannelCompletedSuccess) - }, - }, - "OnChannelComplete when outgoing request cancelled by caller": { - action: func(gsData *harness) { - gsData.fgs.LeaveRequestsOpen() - stor, _ := gsData.outgoing.Selector() - - go gsData.outgoingRequestHook() - _ = gsData.transport.OpenChannel( - gsData.ctx, - gsData.other, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, - cidlink.Link{Cid: gsData.outgoing.BaseCid()}, - stor, - nil, - gsData.outgoing) - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - requestReceived := gsData.fgs.AssertRequestReceived(gsData.ctx, t) - extensions := make(map[graphsync.ExtensionName]datamodel.Node) - for _, ext := range requestReceived.Extensions { - extensions[ext.Name] = ext.Data - } - request := testutil.NewFakeRequest(graphsync.NewRequestID(), extensions) - gsData.fgs.OutgoingRequestHook(gsData.other, request, gsData.outgoingRequestHookActions) - _ = gsData.transport.CloseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) - ctxt, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - gsData.fgs.AssertCancelReceived(ctxt, t) - }, - }, - "request times out if we get request context cancelled error": { - action: func(gsData *harness) { - gsData.fgs.LeaveRequestsOpen() - stor, _ := gsData.outgoing.Selector() - - go gsData.outgoingRequestHook() - _ = gsData.transport.OpenChannel( - gsData.ctx, - gsData.other, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, - cidlink.Link{Cid: gsData.outgoing.BaseCid()}, - stor, - nil, - gsData.outgoing) - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - requestReceived := gsData.fgs.AssertRequestReceived(gsData.ctx, t) - close(requestReceived.ResponseChan) - requestReceived.ResponseErrChan <- graphsync.RequestClientCancelledErr{} - close(requestReceived.ResponseErrChan) - - require.Eventually(t, func() bool { - return events.OnRequestCancelledCalled == true - }, 2*time.Second, 100*time.Millisecond) - require.Equal(t, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, events.OnRequestCancelledChannelId) - }, - }, - "request cancelled out if transport shuts down": { - action: func(gsData *harness) { - gsData.fgs.LeaveRequestsOpen() - stor, _ := gsData.outgoing.Selector() - - go gsData.outgoingRequestHook() - _ = gsData.transport.OpenChannel( - gsData.ctx, - gsData.other, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, - cidlink.Link{Cid: gsData.outgoing.BaseCid()}, - stor, - nil, - gsData.outgoing) - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - gsData.fgs.AssertRequestReceived(gsData.ctx, t) - - gsData.transport.Shutdown(gsData.ctx) - - ctxt, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - gsData.fgs.AssertCancelReceived(ctxt, t) - - require.Nil(t, gsData.fgs.IncomingRequestHook) - require.Nil(t, gsData.fgs.CompletedResponseListener) - require.Nil(t, gsData.fgs.IncomingBlockHook) - require.Nil(t, gsData.fgs.OutgoingBlockHook) - require.Nil(t, gsData.fgs.BlockSentListener) - require.Nil(t, gsData.fgs.OutgoingRequestHook) - require.Nil(t, gsData.fgs.IncomingResponseHook) - require.Nil(t, gsData.fgs.RequestUpdatedHook) - require.Nil(t, gsData.fgs.RequestorCancelledListener) - require.Nil(t, gsData.fgs.NetworkErrorListener) - }, - }, - "request pause works even if called when request is still pending": { - action: func(gsData *harness) { - gsData.fgs.LeaveRequestsOpen() - stor, _ := gsData.outgoing.Selector() - - go gsData.outgoingRequestHook() - _ = gsData.transport.OpenChannel( - gsData.ctx, - gsData.other, - datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, - cidlink.Link{Cid: gsData.outgoing.BaseCid()}, - stor, - nil, - gsData.outgoing) - - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - requestReceived := gsData.fgs.AssertRequestReceived(gsData.ctx, t) - assertHasOutgoingMessage(t, requestReceived.Extensions, gsData.outgoing) - completed := make(chan struct{}) - go func() { - err := gsData.transport.PauseChannel(context.Background(), datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) - require.NoError(t, err) - close(completed) - }() - time.Sleep(100 * time.Millisecond) - extensions := make(map[graphsync.ExtensionName]datamodel.Node) - for _, ext := range requestReceived.Extensions { - extensions[ext.Name] = ext.Data - } - request := testutil.NewFakeRequest(graphsync.NewRequestID(), extensions) - gsData.fgs.OutgoingRequestHook(gsData.other, request, gsData.outgoingRequestHookActions) - select { - case <-gsData.ctx.Done(): - t.Fatal("never paused channel") - case <-completed: - } - }, - }, - "UseStore can change store used for outgoing requests": { - action: func(gsData *harness) { - lsys := cidlink.DefaultLinkSystem() - lsys.StorageReadOpener = func(ipld.LinkContext, ipld.Link) (io.Reader, error) { - return nil, nil - } - lsys.StorageWriteOpener = func(ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) { - return nil, nil, nil - } - _ = gsData.transport.UseStore(datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, lsys) - gsData.outgoingRequestHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - expectedChannel := "data-transfer-" + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}.String() - gsData.fgs.AssertHasPersistenceOption(t, expectedChannel) - require.Equal(t, expectedChannel, gsData.outgoingRequestHookActions.PersistenceOption) - gsData.transport.CleanupChannel(datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) - gsData.fgs.AssertDoesNotHavePersistenceOption(t, expectedChannel) - }, - }, - "UseStore can change store used for incoming requests": { - action: func(gsData *harness) { - lsys := cidlink.DefaultLinkSystem() - lsys.StorageReadOpener = func(ipld.LinkContext, ipld.Link) (io.Reader, error) { - return nil, nil - } - lsys.StorageWriteOpener = func(ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) { - return nil, nil, nil - } - _ = gsData.transport.UseStore(datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}, lsys) - gsData.incomingRequestHook() - }, - check: func(t *testing.T, events *fakeEvents, gsData *harness) { - expectedChannel := "data-transfer-" + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}.String() - gsData.fgs.AssertHasPersistenceOption(t, expectedChannel) - require.Equal(t, expectedChannel, gsData.incomingRequestHookActions.PersistenceOption) - gsData.transport.CleanupChannel(datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) - gsData.fgs.AssertDoesNotHavePersistenceOption(t, expectedChannel) - }, - }, - } - - ctx := context.Background() - for testCase, data := range testCases { - t.Run(testCase, func(t *testing.T) { - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - peers := testutil.GeneratePeers(2) - transferID := datatransfer.TransferID(rand.Uint32()) - requestID := graphsync.NewRequestID() - request := data.requestConfig.makeRequest(t, transferID, requestID) - altRequest := data.requestConfig.makeRequest(t, transferID, graphsync.NewRequestID()) - response := data.responseConfig.makeResponse(t, transferID, requestID) - updatedRequest := data.updatedConfig.makeRequest(t, transferID, requestID) - block := testutil.NewFakeBlockData() - fgs := testutil.NewFakeGraphSync() - outgoing := testutil.NewDTRequest(t, transferID) - incoming := testutil.NewDTResponse(t, transferID) - transport := NewTransport(peers[0], fgs) - gsData := &harness{ - ctx: ctx, - outgoing: outgoing, - incoming: incoming, - transport: transport, - fgs: fgs, - self: peers[0], - transferID: transferID, - other: peers[1], - altRequest: altRequest, - request: request, - response: response, - updatedRequest: updatedRequest, - block: block, - outgoingRequestHookActions: &testutil.FakeOutgoingRequestHookActions{}, - outgoingBlockHookActions: &testutil.FakeOutgoingBlockHookActions{}, - incomingBlockHookActions: &testutil.FakeIncomingBlockHookActions{}, - incomingRequestHookActions: &testutil.FakeIncomingRequestHookActions{}, - requestUpdatedHookActions: &testutil.FakeRequestUpdatedActions{}, - incomingResponseHookActions: &testutil.FakeIncomingResponseHookActions{}, - requestQueuedHookActions: &testutil.FakeRequestQueuedHookActions{}, - } - require.NoError(t, transport.SetEventHandler(&data.events)) - if data.action != nil { - data.action(gsData) - } - data.check(t, &data.events, gsData) - }) - } -} - -type fakeEvents struct { - ChannelOpenedChannelID datatransfer.ChannelID - RequestReceivedChannelID datatransfer.ChannelID - ResponseReceivedChannelID datatransfer.ChannelID - OnChannelOpenedError error - OnDataReceivedCalled bool - OnDataReceivedError error - OnDataSentCalled bool - OnRequestReceivedCallCount int - OnRequestReceivedErrors []error - OnResponseReceivedCallCount int - OnResponseReceivedErrors []error - OnChannelCompletedCalled bool - OnChannelCompletedErr error - OnDataQueuedCalled bool - OnDataQueuedMessage datatransfer.Message - OnDataQueuedError error - - OnRequestCancelledCalled bool - OnRequestCancelledChannelId datatransfer.ChannelID - OnSendDataErrorCalled bool - OnSendDataErrorChannelID datatransfer.ChannelID - OnReceiveDataErrorCalled bool - OnReceiveDataErrorChannelID datatransfer.ChannelID - OnContextAugmentFunc func(context.Context) context.Context - TransferQueuedCalled bool - TransferQueuedChannelID datatransfer.ChannelID - - ChannelCompletedSuccess bool - RequestReceivedRequest datatransfer.Request - RequestReceivedResponse datatransfer.Response - ResponseReceivedResponse datatransfer.Response -} - -func (fe *fakeEvents) OnDataQueued(chid datatransfer.ChannelID, link ipld.Link, size uint64, index int64, unique bool) (datatransfer.Message, error) { - fe.OnDataQueuedCalled = true - - return fe.OnDataQueuedMessage, fe.OnDataQueuedError -} - -func (fe *fakeEvents) OnRequestCancelled(chid datatransfer.ChannelID, err error) error { - fe.OnRequestCancelledCalled = true - fe.OnRequestCancelledChannelId = chid - - return nil -} - -func (fe *fakeEvents) OnTransferQueued(chid datatransfer.ChannelID) { - fe.TransferQueuedCalled = true - fe.TransferQueuedChannelID = chid -} - -func (fe *fakeEvents) OnRequestDisconnected(chid datatransfer.ChannelID, err error) error { - return nil -} - -func (fe *fakeEvents) OnSendDataError(chid datatransfer.ChannelID, err error) error { - fe.OnSendDataErrorCalled = true - fe.OnSendDataErrorChannelID = chid - return nil -} - -func (fe *fakeEvents) OnReceiveDataError(chid datatransfer.ChannelID, err error) error { - fe.OnReceiveDataErrorCalled = true - fe.OnReceiveDataErrorChannelID = chid - return nil -} - -func (fe *fakeEvents) OnChannelOpened(chid datatransfer.ChannelID) error { - fe.ChannelOpenedChannelID = chid - return fe.OnChannelOpenedError -} - -func (fe *fakeEvents) OnDataReceived(chid datatransfer.ChannelID, link ipld.Link, size uint64, index int64, unique bool) error { - fe.OnDataReceivedCalled = true - return fe.OnDataReceivedError -} - -func (fe *fakeEvents) OnDataSent(chid datatransfer.ChannelID, link ipld.Link, size uint64, index int64, unique bool) error { - fe.OnDataSentCalled = true - return nil -} - -func (fe *fakeEvents) OnRequestReceived(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { - fe.OnRequestReceivedCallCount++ - fe.RequestReceivedChannelID = chid - fe.RequestReceivedRequest = request - var err error - if len(fe.OnRequestReceivedErrors) > 0 { - err, fe.OnRequestReceivedErrors = fe.OnRequestReceivedErrors[0], fe.OnRequestReceivedErrors[1:] - } - return fe.RequestReceivedResponse, err -} - -func (fe *fakeEvents) OnResponseReceived(chid datatransfer.ChannelID, response datatransfer.Response) error { - fe.OnResponseReceivedCallCount++ - fe.ResponseReceivedResponse = response - fe.ResponseReceivedChannelID = chid - var err error - if len(fe.OnResponseReceivedErrors) > 0 { - err, fe.OnResponseReceivedErrors = fe.OnResponseReceivedErrors[0], fe.OnResponseReceivedErrors[1:] - } - return err -} - -func (fe *fakeEvents) OnChannelCompleted(chid datatransfer.ChannelID, completeErr error) error { - fe.OnChannelCompletedCalled = true - fe.ChannelCompletedSuccess = completeErr == nil - return fe.OnChannelCompletedErr -} - -func (fe *fakeEvents) OnContextAugment(chid datatransfer.ChannelID) func(context.Context) context.Context { - return fe.OnContextAugmentFunc -} - -type harness struct { - outgoing datatransfer.Request - incoming datatransfer.Response - ctx context.Context - transport *Transport - fgs *testutil.FakeGraphSync - transferID datatransfer.TransferID - self peer.ID - other peer.ID - block graphsync.BlockData - request graphsync.RequestData - altRequest graphsync.RequestData - response graphsync.ResponseData - updatedRequest graphsync.RequestData - outgoingRequestHookActions *testutil.FakeOutgoingRequestHookActions - incomingBlockHookActions *testutil.FakeIncomingBlockHookActions - outgoingBlockHookActions *testutil.FakeOutgoingBlockHookActions - incomingRequestHookActions *testutil.FakeIncomingRequestHookActions - requestUpdatedHookActions *testutil.FakeRequestUpdatedActions - incomingResponseHookActions *testutil.FakeIncomingResponseHookActions - requestQueuedHookActions *testutil.FakeRequestQueuedHookActions -} - -func (ha *harness) outgoingRequestHook() { - ha.fgs.OutgoingRequestHook(ha.other, ha.request, ha.outgoingRequestHookActions) -} - -func (ha *harness) altOutgoingRequestHook() { - ha.fgs.OutgoingRequestHook(ha.other, ha.altRequest, ha.outgoingRequestHookActions) -} - -func (ha *harness) incomingBlockHook() { - ha.fgs.IncomingBlockHook(ha.other, ha.response, ha.block, ha.incomingBlockHookActions) -} -func (ha *harness) outgoingBlockHook() { - ha.fgs.OutgoingBlockHook(ha.other, ha.request, ha.block, ha.outgoingBlockHookActions) -} - -func (ha *harness) incomingRequestHook() { - ha.fgs.IncomingRequestHook(ha.other, ha.request, ha.incomingRequestHookActions) -} - -func (ha *harness) incomingRequestQueuedHook() { - ha.fgs.IncomingRequestQueuedHook(ha.other, ha.request, ha.requestQueuedHookActions) -} - -func (ha *harness) requestUpdatedHook() { - ha.fgs.RequestUpdatedHook(ha.other, ha.request, ha.updatedRequest, ha.requestUpdatedHookActions) -} -func (ha *harness) incomingResponseHOok() { - ha.fgs.IncomingResponseHook(ha.other, ha.response, ha.incomingResponseHookActions) -} -func (ha *harness) responseCompletedListener() { - ha.fgs.CompletedResponseListener(ha.other, ha.request, ha.response.Status()) -} -func (ha *harness) requestorCancelledListener() { - ha.fgs.RequestorCancelledListener(ha.other, ha.request) -} -func (ha *harness) networkErrorListener(err error) { - ha.fgs.NetworkErrorListener(ha.other, ha.request, err) -} -func (ha *harness) receiverNetworkErrorListener(err error) { - ha.fgs.ReceiverNetworkErrorListener(ha.other, err) -} - -type dtConfig struct { - dtExtensionMissing bool - dtIsResponse bool - dtExtensionMalformed bool -} - -func (dtc *dtConfig) extensions(t *testing.T, transferID datatransfer.TransferID, extName graphsync.ExtensionName) map[graphsync.ExtensionName]datamodel.Node { - extensions := make(map[graphsync.ExtensionName]datamodel.Node) - if !dtc.dtExtensionMissing { - if dtc.dtExtensionMalformed { - extensions[extName] = basicnode.NewInt(10) - } else { - var msg datatransfer.Message - if dtc.dtIsResponse { - msg = testutil.NewDTResponse(t, transferID) - } else { - msg = testutil.NewDTRequest(t, transferID) - } - nd := msg.ToIPLD() - extensions[extName] = nd - } - } - return extensions -} - -type gsRequestConfig struct { - dtExtensionMissing bool - dtIsResponse bool - dtExtensionMalformed bool -} - -func (grc *gsRequestConfig) makeRequest(t *testing.T, transferID datatransfer.TransferID, requestID graphsync.RequestID) graphsync.RequestData { - dtConfig := dtConfig{ - dtExtensionMissing: grc.dtExtensionMissing, - dtIsResponse: grc.dtIsResponse, - dtExtensionMalformed: grc.dtExtensionMalformed, - } - extensions := dtConfig.extensions(t, transferID, extension.ExtensionDataTransfer1_1) - return testutil.NewFakeRequest(requestID, extensions) -} - -type gsResponseConfig struct { - dtExtensionMissing bool - dtIsResponse bool - dtExtensionMalformed bool - status graphsync.ResponseStatusCode -} - -func (grc *gsResponseConfig) makeResponse(t *testing.T, transferID datatransfer.TransferID, requestID graphsync.RequestID) graphsync.ResponseData { - dtConfig := dtConfig{ - dtExtensionMissing: grc.dtExtensionMissing, - dtIsResponse: grc.dtIsResponse, - dtExtensionMalformed: grc.dtExtensionMalformed, - } - extensions := dtConfig.extensions(t, transferID, extension.ExtensionDataTransfer1_1) - return testutil.NewFakeResponse(requestID, extensions, grc.status) -} - -func assertDecodesToMessage(t *testing.T, data datamodel.Node, expected datatransfer.Message) { - actual, err := message.FromIPLD(data) - require.NoError(t, err) - require.Equal(t, expected, actual) -} - -func assertHasOutgoingMessage(t *testing.T, extensions []graphsync.ExtensionData, expected datatransfer.Message) { - nd := expected.ToIPLD() - found := false - for _, e := range extensions { - if e.Name == extension.ExtensionDataTransfer1_1 { - require.True(t, ipld.DeepEqual(nd, e.Data), "data matches") - found = true - } - } - if !found { - require.Fail(t, "extension not found") - } -} - -func assertHasExtensionMessage(t *testing.T, name graphsync.ExtensionName, extensions []graphsync.ExtensionData, expected datatransfer.Message) { - nd := expected.ToIPLD() - found := false - for _, e := range extensions { - if e.Name == name { - require.True(t, ipld.DeepEqual(nd, e.Data), "data matches") - found = true - } - } - if !found { - require.Fail(t, "extension not found") - } -} -*/ diff --git a/transport/graphsync/hooks.go b/transport/graphsync/hooks.go index 021fea4e..8c0ae35e 100644 --- a/transport/graphsync/hooks.go +++ b/transport/graphsync/hooks.go @@ -1,9 +1,12 @@ package graphsync import ( + "context" "errors" + "fmt" "github.com/ipfs/go-graphsync" + basicnode "github.com/ipld/go-ipld-prime/node/basic" peer "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" @@ -13,35 +16,15 @@ import ( // gsOutgoingRequestHook is called when a graphsync request is made func (t *Transport) gsOutgoingRequestHook(p peer.ID, request graphsync.RequestData, hookActions graphsync.OutgoingRequestHookActions) { - message, _ := extension.GetTransferData(request, t.supportedExtensions) - // extension not found; probably not our request. - if message == nil { + chid, ok := t.requestIDToChannelID.load(request.ID()) + if !ok { return } - // A graphsync request is made when either - // - The local node opens a data-transfer pull channel, so the local node - // sends a graphsync request to ask the remote peer for the data - // - The remote peer opened a data-transfer push channel, and in response - // the local node sends a graphsync request to ask for the data - var initiator peer.ID - var responder peer.ID - if message.IsRequest() { - // This is a pull request so the data-transfer initiator is the local node - initiator = t.peerID - responder = p - } else { - // This is a push response so the data-transfer initiator is the remote - // peer: They opened the push channel, we respond by sending a - // graphsync request for the data - initiator = p - responder = t.peerID - } - chid := datatransfer.ChannelID{Initiator: initiator, Responder: responder, ID: message.TransferID()} + // Start tracking the channel if we're not already + ch, err := t.getDTChannel(chid) - // A data transfer channel was opened - err := t.events.OnChannelOpened(chid) if err != nil { // There was an error opening the channel, bail out log.Errorf("processing OnChannelOpened for %s: %s", chid, err) @@ -49,8 +32,8 @@ func (t *Transport) gsOutgoingRequestHook(p peer.ID, request graphsync.RequestDa return } - // Start tracking the channel if we're not already - ch := t.trackDTChannel(chid) + // A data transfer channel was opened + t.events.OnTransportEvent(chid, datatransfer.TransportOpenedChannel{}) // Signal that the channel has been opened ch.GsReqOpened(p, request.ID(), hookActions) @@ -69,49 +52,36 @@ func (t *Transport) gsIncomingBlockHook(p peer.ID, response graphsync.ResponseDa return } - ch.UpdateReceivedCidsIfGreater(block.Index()) + if ch.UpdateReceivedIndexIfGreater(block.Index()) && block.BlockSizeOnWire() != 0 { - err = t.events.OnDataReceived(chid, block.Link(), block.BlockSize(), block.Index(), block.BlockSizeOnWire() != 0) - if err != nil && err != datatransfer.ErrPause { - hookActions.TerminateWithError(err) - return - } + t.events.OnTransportEvent(chid, datatransfer.TransportReceivedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) - if err == datatransfer.ErrPause { - ch.MarkPaused() - hookActions.PauseRequest() + if ch.UpdateProgress(block.BlockSizeOnWire()) { + t.events.OnTransportEvent(chid, datatransfer.TransportReachedDataLimit{}) + hookActions.PauseRequest() + } } + } -func (t *Transport) gsBlockSentHook(p peer.ID, request graphsync.RequestData, block graphsync.BlockData) { - // When a data transfer is restarted, the requester sends a list of CIDs - // that it already has. Graphsync calls the sent hook for all blocks even - // if they are in the list (meaning, they aren't actually sent over the - // wire). So here we check if the block was actually sent - // over the wire before firing the data sent event. - if block.BlockSizeOnWire() == 0 { +func (t *Transport) gsBlockSentListener(p peer.ID, request graphsync.RequestData, block graphsync.BlockData) { + chid, ok := t.requestIDToChannelID.load(request.ID()) + if !ok { return } - chid, ok := t.requestIDToChannelID.load(request.ID()) - if !ok { + ch, err := t.getDTChannel(chid) + if err != nil { + log.Errorf("sent hook error: %s, for channel %s", err, chid) return } - if err := t.events.OnDataSent(chid, block.Link(), block.BlockSize(), block.Index(), block.BlockSizeOnWire() != 0); err != nil { - log.Errorf("failed to process data sent: %+v", err) + if ch.UpdateSentIndexIfGreater(block.Index()) && block.BlockSizeOnWire() != 0 { + t.events.OnTransportEvent(chid, datatransfer.TransportSentData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) } } func (t *Transport) gsOutgoingBlockHook(p peer.ID, request graphsync.RequestData, block graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { - // When a data transfer is restarted, the requester sends a list of CIDs - // that it already has. Graphsync calls the outgoing block hook for all - // blocks even if they are in the list (meaning, they aren't actually going - // to be sent over the wire). So here we check if the block is actually - // going to be sent over the wire before firing the data queued event. - if block.BlockSizeOnWire() == 0 { - return - } chid, ok := t.requestIDToChannelID.load(request.ID()) if !ok { @@ -124,75 +94,25 @@ func (t *Transport) gsOutgoingBlockHook(p peer.ID, request graphsync.RequestData return } - // OnDataQueued is called when a block is queued to be sent to the remote - // peer. It can return ErrPause to pause the response (eg if payment is - // required) and it can return a message that will be sent with the block - // (eg to ask for payment). - msg, err := t.events.OnDataQueued(chid, block.Link(), block.BlockSize(), block.Index(), block.BlockSizeOnWire() != 0) - if err != nil && err != datatransfer.ErrPause { - hookActions.TerminateWithError(err) - return - } - - if err == datatransfer.ErrPause { - ch.MarkPaused() - hookActions.PauseResponse() - } + if ch.UpdateQueuedIndexIfGreater(block.Index()) && block.BlockSizeOnWire() != 0 { + t.events.OnTransportEvent(chid, datatransfer.TransportQueuedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) - if msg != nil { - // gsOutgoingBlockHook uses a unique extension name so it can be attached with data from a different hook - // outgoingBlkExtensions also includes the default extension name so it remains compatible with all data-transfer protocol versions out there - extensions, err := extension.ToExtensionData(msg, outgoingBlkExtensions) - if err != nil { - hookActions.TerminateWithError(err) - return - } - for _, extension := range extensions { - hookActions.SendExtensionData(extension) + if ch.UpdateProgress(block.BlockSizeOnWire()) { + t.events.OnTransportEvent(chid, datatransfer.TransportReachedDataLimit{}) + hookActions.PauseResponse() } } } // gsReqQueuedHook is called when graphsync enqueues an incoming request for data -func (t *Transport) gsReqQueuedHook(p peer.ID, request graphsync.RequestData, hookActions graphsync.RequestQueuedHookActions) { - msg, err := extension.GetTransferData(request, t.supportedExtensions) - if err != nil { - log.Errorf("failed GetTransferData, req=%+v, err=%s", request, err) - } - // extension not found; probably not our request. - if msg == nil { +func (t *Transport) gsRequestProcessingListener(p peer.ID, request graphsync.RequestData, requestCount int) { + + chid, ok := t.requestIDToChannelID.load(request.ID()) + if !ok { return } - var chid datatransfer.ChannelID - if msg.IsRequest() { - // when a data transfer request comes in on graphsync, the remote peer - // initiated a pull - chid = datatransfer.ChannelID{ID: msg.TransferID(), Initiator: p, Responder: t.peerID} - dtRequest := msg.(datatransfer.Request) - if dtRequest.IsNew() { - log.Infof("%s, pull request queued, req_id=%d", chid, request.ID()) - t.events.OnTransferQueued(chid) - } else { - log.Infof("%s, pull restart request queued, req_id=%d", chid, request.ID()) - } - } else { - // when a data transfer response comes in on graphsync, this node - // initiated a push, and the remote peer responded with a request - // for data - chid = datatransfer.ChannelID{ID: msg.TransferID(), Initiator: t.peerID, Responder: p} - response := msg.(datatransfer.Response) - if response.IsNew() { - log.Infof("%s, GS pull request queued in response to our push, req_id=%d", chid, request.ID()) - t.events.OnTransferQueued(chid) - } else { - log.Infof("%s, GS pull request queued in response to our restart push, req_id=%d", chid, request.ID()) - } - } - augmentContext := t.events.OnContextAugment(chid) - if augmentContext != nil { - hookActions.AugmentContext(augmentContext) - } + t.events.OnTransportEvent(chid, datatransfer.TransportInitiatedTransfer{}) } // gsReqRecdHook is called when graphsync receives an incoming request for data @@ -225,7 +145,21 @@ func (t *Transport) gsReqRecdHook(p peer.ID, request graphsync.RequestData, hook log.Debugf("%s: received request for data (pull), req_id=%d", chid, request.ID()) request := msg.(datatransfer.Request) + + // graphsync never receives dt push requests as new graphsync requests -- is so, we should error + isNewOrRestart := (request.IsNew() || request.IsRestart()) + if isNewOrRestart && !request.IsPull() { + hookActions.TerminateWithError(datatransfer.ErrUnsupported) + return + } + responseMessage, err = t.events.OnRequestReceived(chid, request) + + // if we're going to accept this new/restart request, protect connection + if isNewOrRestart && err == nil { + t.dtNet.Protect(p, chid.String()) + } + } else { // when a data transfer response comes in on graphsync, this node // initiated a push, and the remote peer responded with a request @@ -253,26 +187,21 @@ func (t *Transport) gsReqRecdHook(p peer.ID, request graphsync.RequestData, hook } } - if err != nil && err != datatransfer.ErrPause { + if err != nil { hookActions.TerminateWithError(err) return } - // Check if the callback indicated that the channel should be paused - // immediately (eg because data is still being unsealed) - paused := false - if err == datatransfer.ErrPause { - log.Debugf("%s: pausing graphsync response", chid) + hookActions.AugmentContext(t.events.OnContextAugment(chid)) - paused = true - hookActions.PauseResponse() + chst, err := t.events.ChannelState(context.TODO(), chid) + if err != nil { + hookActions.TerminateWithError(err) } - ch := t.trackDTChannel(chid) - t.requestIDToChannelID.set(request.ID(), true, chid) - ch.GsDataRequestRcvd(p, request.ID(), paused, hookActions) - - hookActions.ValidateRequest() + t.requestIDToChannelID.set(request.ID(), true, chid) + ch := t.trackDTChannel(chid) + ch.GsDataRequestRcvd(p, request.ID(), chst, hookActions) } // gsCompletedResponseListener is a graphsync.OnCompletedResponseListener. We use it learn when the data transfer is complete @@ -293,9 +222,11 @@ func (t *Transport) gsCompletedResponseListener(p peer.ID, request graphsync.Req } ch.MarkTransferComplete() - var completeErr error - if status != graphsync.RequestCompletedFull { - completeErr = xerrors.Errorf("graphsync response to peer %s did not complete: response status code %s", p, status.String()) + var completeEvent datatransfer.TransportCompletedTransfer + if status == graphsync.RequestCompletedFull { + completeEvent.Success = true + } else { + completeEvent.ErrorMessage = fmt.Sprintf("graphsync response to peer %s did not complete: response status code %s", p, status.String()) } // Used by the tests to listen for when a response completes @@ -303,35 +234,7 @@ func (t *Transport) gsCompletedResponseListener(p peer.ID, request graphsync.Req t.completedResponseListener(chid) } - err = t.events.OnChannelCompleted(chid, completeErr) - if err != nil { - log.Error(err) - } - -} - -func (t *Transport) gsRequestUpdatedHook(p peer.ID, request graphsync.RequestData, update graphsync.RequestData, hookActions graphsync.RequestUpdatedHookActions) { - chid, ok := t.requestIDToChannelID.load(request.ID()) - if !ok { - return - } - - responseMessage, err := t.processExtension(chid, update, p, t.supportedExtensions) - - if responseMessage != nil { - extensions, extensionErr := extension.ToExtensionData(responseMessage, t.supportedExtensions) - if extensionErr != nil { - hookActions.TerminateWithError(err) - return - } - for _, extension := range extensions { - hookActions.SendExtensionData(extension) - } - } - - if err != nil && err != datatransfer.ErrPause { - hookActions.TerminateWithError(err) - } + t.events.OnTransportEvent(chid, completeEvent) } @@ -344,14 +247,7 @@ func (t *Transport) gsIncomingResponseHook(p peer.ID, response graphsync.Respons responseMessage, err := t.processExtension(chid, response, p, incomingReqExtensions) if responseMessage != nil { - extensions, extensionErr := extension.ToExtensionData(responseMessage, t.supportedExtensions) - if extensionErr != nil { - hookActions.TerminateWithError(err) - return - } - for _, extension := range extensions { - hookActions.UpdateRequestWithExtensions(extension) - } + t.dtNet.SendMessage(context.TODO(), p, transportID, responseMessage) } if err != nil { @@ -361,7 +257,11 @@ func (t *Transport) gsIncomingResponseHook(p peer.ID, response graphsync.Respons // In a case where the transfer sends blocks immediately this extension may contain both a // response message and a revalidation request so we trigger OnResponseReceived again for this // specific extension name - _, err = t.processExtension(chid, response, p, []graphsync.ExtensionName{extension.ExtensionOutgoingBlock1_1}) + responseMessage, err = t.processExtension(chid, response, p, outgoingBlkExtensions) + + if responseMessage != nil { + t.dtNet.SendMessage(context.TODO(), p, transportID, responseMessage) + } if err != nil { hookActions.TerminateWithError(err) @@ -375,14 +275,12 @@ func (t *Transport) processExtension(chid datatransfer.ChannelID, gsMsg extensio if err != nil { return nil, err } - // extension not found; probably not our request. if msg == nil { return nil, nil } if msg.IsRequest() { - // only accept request message updates when original message was also request if (chid != datatransfer.ChannelID{ID: msg.TransferID(), Initiator: p, Responder: t.peerID}) { return nil, errors.New("received request on response channel") @@ -397,6 +295,7 @@ func (t *Transport) processExtension(chid datatransfer.ChannelID, gsMsg extensio } dtResponse := msg.(datatransfer.Response) + return nil, t.events.OnResponseReceived(chid, dtResponse) } @@ -426,10 +325,7 @@ func (t *Transport) gsNetworkSendErrorListener(p peer.ID, request graphsync.Requ return } - err := t.events.OnSendDataError(chid, gserr) - if err != nil { - log.Errorf("failed to fire transport send error %s: %s", gserr, err) - } + t.events.OnTransportEvent(chid, datatransfer.TransportErrorSendingData{ErrorMessage: gserr.Error()}) } // Called when there is a graphsync error receiving data @@ -441,9 +337,6 @@ func (t *Transport) gsNetworkReceiveErrorListener(p peer.ID, gserr error) { return } - err := t.events.OnReceiveDataError(chid, gserr) - if err != nil { - log.Errorf("failed to fire transport receive error %s: %s", gserr, err) - } + t.events.OnTransportEvent(chid, datatransfer.TransportErrorReceivingData{ErrorMessage: gserr.Error()}) }) } diff --git a/transport/graphsync/initiating_test.go b/transport/graphsync/initiating_test.go new file mode 100644 index 00000000..952ccab6 --- /dev/null +++ b/transport/graphsync/initiating_test.go @@ -0,0 +1,1324 @@ +package graphsync_test + +import ( + "context" + "fmt" + "testing" + "time" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/testharness" + "github.com/ipfs/go-graphsync" + "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/stretchr/testify/require" +) + +func TestInitiatingPullRequestSuccessFlow(t *testing.T) { + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + th := testharness.SetupHarness(ctx, testharness.PullRequest()) + var receivedRequest testharness.ReceivedGraphSyncRequest + var request graphsync.RequestData + t.Run("opens successfully", func(t *testing.T) { + err := th.Transport.OpenChannel(th.Ctx, th.Channel, th.NewRequest(t)) + require.NoError(t, err) + require.Len(t, th.DtNet.ProtectedPeers, 1) + require.Equal(t, th.DtNet.ProtectedPeers[0], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + require.Len(t, th.Fgs.ReceivedRequests, 1) + receivedRequest = th.Fgs.ReceivedRequests[0] + request = receivedRequest.ToRequestData(t) + msg, err := extension.GetTransferData(request, []graphsync.ExtensionName{ + extension.ExtensionDataTransfer1_1, + }) + require.NoError(t, err) + require.Equal(t, th.NewRequest(t), msg) + }) + t.Run("configures persistence", func(t *testing.T) { + th.Transport.UseStore(th.Channel.ChannelID(), cidlink.DefaultLinkSystem()) + th.Fgs.AssertHasPersistenceOption(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String())) + }) + t.Run("receives outgoing request hook", func(t *testing.T) { + th.OutgoingRequestHook(request) + require.Equal(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String()), th.OutgoingRequestHookActions.PersistenceOption) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportOpenedChannel{}) + }) + t.Run("receives outgoing processing listener", func(t *testing.T) { + th.OutgoingRequestProcessingListener(request) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportInitiatedTransfer{}) + }) + dtResponse := th.Response() + response := receivedRequest.Response(t, dtResponse, nil, graphsync.PartialResponse) + t.Run("receives response", func(t *testing.T) { + th.IncomingResponseHook(response) + require.Equal(t, th.Events.ReceivedResponse, dtResponse) + }) + + t.Run("received block", func(t *testing.T) { + block := testharness.NewFakeBlockData(12345, 1, true) + th.IncomingBlockHook(response, block) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReceivedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + block = testharness.NewFakeBlockData(12345, 2, true) + th.IncomingBlockHook(response, block) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReceivedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + // block not on wire has no effect + block = testharness.NewFakeBlockData(12345, 3, false) + th.IncomingBlockHook(response, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReceivedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + // block with lower index has no effect + block = testharness.NewFakeBlockData(67890, 1, true) + th.IncomingBlockHook(response, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReceivedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + }) + + t.Run("receive pause", func(t *testing.T) { + dtPauseResponse := th.UpdateResponse(true) + pauseResponse := receivedRequest.Response(t, nil, dtPauseResponse, graphsync.RequestPaused) + th.IncomingResponseHook(pauseResponse) + require.Equal(t, th.Events.ReceivedResponse, dtPauseResponse) + }) + + t.Run("send update", func(t *testing.T) { + vRequest := th.VoucherRequest() + th.Transport.SendMessage(ctx, th.Channel.ChannelID(), vRequest) + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: vRequest}) + }) + + t.Run("receive resume", func(t *testing.T) { + dtResumeResponse := th.UpdateResponse(false) + pauseResponse := receivedRequest.Response(t, nil, dtResumeResponse, graphsync.PartialResponse) + th.IncomingResponseHook(pauseResponse) + require.Equal(t, th.Events.ReceivedResponse, dtResumeResponse) + }) + + t.Run("pause", func(t *testing.T) { + th.Channel.SetInitiatorPaused(true) + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateRequest(true)) + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.UpdateRequest(true)}) + require.Len(t, th.Fgs.Pauses, 1) + require.Equal(t, th.Fgs.Pauses[0], request.ID()) + }) + t.Run("pause again", func(t *testing.T) { + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateRequest(true)) + // should send message again + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.UpdateRequest(true)}) + // should not pause again + require.Len(t, th.Fgs.Pauses, 1) + }) + t.Run("resume", func(t *testing.T) { + th.Channel.SetInitiatorPaused(false) + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateRequest(false)) + require.Len(t, th.Fgs.Resumes, 1) + resume := th.Fgs.Resumes[0] + require.Equal(t, request.ID(), resume.RequestID) + msg := resume.DTMessage(t) + require.Equal(t, msg, th.UpdateRequest(false)) + }) + t.Run("resume again", func(t *testing.T) { + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateRequest(false)) + // should send message again + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.UpdateRequest(true)}) + // should not resume again + require.Len(t, th.Fgs.Resumes, 1) + }) + + t.Run("restart request", func(t *testing.T) { + restartIndex := int64(5) + th.Channel.SetReceivedIndex(basicnode.NewInt(restartIndex)) + err := th.Transport.RestartChannel(ctx, th.Channel, th.RestartRequest(t)) + require.NoError(t, err) + require.Len(t, th.DtNet.ProtectedPeers, 2) + require.Equal(t, th.DtNet.ProtectedPeers[1], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + require.Len(t, th.DtNet.ConnectWithRetryAttempts, 1) + require.Equal(t, th.DtNet.ConnectWithRetryAttempts[0], testharness.ConnectWithRetryAttempt{th.Channel.OtherPeer(), "graphsync"}) + require.Len(t, th.Fgs.Cancels, 1) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportTransferCancelled{ErrorMessage: "graphsync request cancelled"}) + require.Equal(t, request.ID(), th.Fgs.Cancels[0]) + require.Len(t, th.Fgs.ReceivedRequests, 2) + receivedRequest = th.Fgs.ReceivedRequests[1] + request = receivedRequest.ToRequestData(t) + msg, err := extension.GetTransferData(request, []graphsync.ExtensionName{ + extension.ExtensionDataTransfer1_1, + }) + require.NoError(t, err) + require.Equal(t, th.RestartRequest(t), msg) + nd, has := request.Extension(graphsync.ExtensionsDoNotSendFirstBlocks) + require.True(t, has) + val, err := nd.AsInt() + require.NoError(t, err) + require.Equal(t, restartIndex, val) + }) + + t.Run("complete request", func(t *testing.T) { + close(receivedRequest.ResponseChan) + close(receivedRequest.ResponseErrChan) + select { + case <-th.CompletedRequests: + case <-ctx.Done(): + t.Fatalf("did not complete request") + } + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportCompletedTransfer{Success: true}) + }) + + t.Run("cleanup request", func(t *testing.T) { + th.Transport.CleanupChannel(th.Channel.ChannelID()) + require.Len(t, th.DtNet.UnprotectedPeers, 1) + require.Equal(t, th.DtNet.UnprotectedPeers[0], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + }) +} + +type ctxKey struct{} + +func TestInitiatingPushRequestSuccessFlow(t *testing.T) { + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + th := testharness.SetupHarness(ctx) + t.Run("opens successfully", func(t *testing.T) { + err := th.Transport.OpenChannel(th.Ctx, th.Channel, th.NewRequest(t)) + require.NoError(t, err) + require.Len(t, th.DtNet.ProtectedPeers, 1) + require.Equal(t, th.DtNet.ProtectedPeers[0], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.NewRequest(t)}) + }) + t.Run("configures persistence", func(t *testing.T) { + th.Transport.UseStore(th.Channel.ChannelID(), cidlink.DefaultLinkSystem()) + th.Fgs.AssertHasPersistenceOption(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String())) + }) + dtResponse := th.Response() + requestID := graphsync.NewRequestID() + request := testharness.NewFakeRequest(requestID, map[graphsync.ExtensionName]datamodel.Node{extension.ExtensionDataTransfer1_1: dtResponse.ToIPLD()}, graphsync.RequestTypeNew) + //response := receivedRequest.Response(t, dtResponse, nil, graphsync.PartialResponse) + t.Run("receives incoming request hook", func(t *testing.T) { + th.Events.ReturnedOnContextAugmentFunc = func(ctx context.Context) context.Context { + return context.WithValue(ctx, ctxKey{}, "applesauce") + } + th.IncomingRequestHook(request) + require.Equal(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String()), th.IncomingRequestHookActions.PersistenceOption) + require.True(t, th.IncomingRequestHookActions.Validated) + require.False(t, th.IncomingBlockHookActions.Paused) + require.NoError(t, th.IncomingRequestHookActions.TerminationError) + th.IncomingRequestHookActions.AssertAugmentedContextKey(t, ctxKey{}, "applesauce") + require.Equal(t, th.Events.ReceivedResponse, dtResponse) + }) + + t.Run("receives incoming processing listener", func(t *testing.T) { + th.IncomingRequestProcessingListener(request) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportInitiatedTransfer{}) + }) + + t.Run("queued block", func(t *testing.T) { + block := testharness.NewFakeBlockData(12345, 1, true) + th.OutgoingBlockHook(request, block) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportQueuedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + block = testharness.NewFakeBlockData(12345, 2, true) + th.OutgoingBlockHook(request, block) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportQueuedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + // block not on wire has no effect + block = testharness.NewFakeBlockData(12345, 3, false) + th.OutgoingBlockHook(request, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportQueuedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + // block with lower index has no effect + block = testharness.NewFakeBlockData(67890, 1, true) + th.OutgoingBlockHook(request, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportQueuedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + }) + + t.Run("sent block", func(t *testing.T) { + block := testharness.NewFakeBlockData(12345, 1, true) + th.BlockSentListener(request, block) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportSentData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + block = testharness.NewFakeBlockData(12345, 2, true) + th.BlockSentListener(request, block) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportSentData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + // block not on wire has no effect + block = testharness.NewFakeBlockData(12345, 3, false) + th.BlockSentListener(request, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportSentData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + // block with lower index has no effect + block = testharness.NewFakeBlockData(67890, 1, true) + th.BlockSentListener(request, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportSentData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + }) + + t.Run("receive pause", func(t *testing.T) { + th.RequestorCancelledListener(request) + dtPauseResponse := th.UpdateResponse(true) + th.DtNet.Delegates[0].Receiver.ReceiveResponse(ctx, th.Channel.OtherPeer(), dtPauseResponse) + require.Equal(t, th.Events.ReceivedResponse, dtPauseResponse) + }) + + t.Run("send update", func(t *testing.T) { + vRequest := th.VoucherRequest() + th.Transport.SendMessage(ctx, th.Channel.ChannelID(), vRequest) + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: vRequest}) + }) + + t.Run("receive resume", func(t *testing.T) { + dtResumeResponse := th.UpdateResponse(false) + request = testharness.NewFakeRequest(requestID, map[graphsync.ExtensionName]datamodel.Node{extension.ExtensionDataTransfer1_1: dtResumeResponse.ToIPLD()}, graphsync.RequestTypeNew) + // reset hook behavior + th.IncomingRequestHookActions = &testharness.FakeIncomingRequestHookActions{} + th.IncomingRequestHook(request) + require.Equal(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String()), th.IncomingRequestHookActions.PersistenceOption) + require.True(t, th.IncomingRequestHookActions.Validated) + require.False(t, th.IncomingBlockHookActions.Paused) + require.NoError(t, th.IncomingRequestHookActions.TerminationError) + th.IncomingRequestHookActions.AssertAugmentedContextKey(t, ctxKey{}, "applesauce") + require.Equal(t, th.Events.ReceivedResponse, dtResumeResponse) + }) + + t.Run("pause", func(t *testing.T) { + th.Channel.SetInitiatorPaused(true) + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateRequest(true)) + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.UpdateRequest(true)}) + require.Len(t, th.Fgs.Pauses, 1) + require.Equal(t, th.Fgs.Pauses[0], request.ID()) + }) + t.Run("pause again", func(t *testing.T) { + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateRequest(true)) + // should send message again + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.UpdateRequest(true)}) + // should not pause again + require.Len(t, th.Fgs.Pauses, 1) + }) + t.Run("resume", func(t *testing.T) { + th.Channel.SetInitiatorPaused(false) + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateRequest(false)) + require.Len(t, th.Fgs.Resumes, 1) + resume := th.Fgs.Resumes[0] + require.Equal(t, request.ID(), resume.RequestID) + msg := resume.DTMessage(t) + require.Equal(t, msg, th.UpdateRequest(false)) + }) + t.Run("resume again", func(t *testing.T) { + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateRequest(false)) + // should send message again + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.UpdateRequest(true)}) + // should not resume again + require.Len(t, th.Fgs.Resumes, 1) + }) + + t.Run("restart request", func(t *testing.T) { + err := th.Transport.RestartChannel(ctx, th.Channel, th.RestartRequest(t)) + require.NoError(t, err) + require.Len(t, th.DtNet.ProtectedPeers, 2) + require.Equal(t, th.DtNet.ProtectedPeers[1], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + require.Len(t, th.DtNet.ConnectWithRetryAttempts, 1) + require.Equal(t, th.DtNet.ConnectWithRetryAttempts[0], testharness.ConnectWithRetryAttempt{th.Channel.OtherPeer(), "graphsync"}) + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.NewRequest(t)}) + }) + + t.Run("complete request", func(t *testing.T) { + th.ResponseCompletedListener(request, graphsync.RequestCompletedFull) + select { + case <-th.CompletedResponses: + case <-ctx.Done(): + t.Fatalf("did not complete request") + } + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportCompletedTransfer{Success: true}) + }) + + t.Run("cleanup request", func(t *testing.T) { + th.Transport.CleanupChannel(th.Channel.ChannelID()) + require.Len(t, th.DtNet.UnprotectedPeers, 1) + require.Equal(t, th.DtNet.UnprotectedPeers[0], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + }) +} + +/* "gs outgoing request with recognized dt push channel will record incoming blocks": { + requestConfig: gsRequestConfig{ + dtIsResponse: true, + }, + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.incomingBlockHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + require.True(t, events.OnDataReceivedCalled) + require.NoError(t, gsData.incomingBlockHookActions.TerminationError) + }, + }, + "non-data-transfer gs request will not record incoming blocks and send updates": { + requestConfig: gsRequestConfig{ + dtExtensionMissing: true, + }, + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.incomingBlockHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{}) + require.False(t, events.OnDataReceivedCalled) + require.NoError(t, gsData.incomingBlockHookActions.TerminationError) + }, + }, + "gs request unrecognized opened channel will not record incoming blocks": { + events: fakeEvents{ + OnChannelOpenedError: errors.New("Not recognized"), + }, + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.incomingBlockHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) + require.False(t, events.OnDataReceivedCalled) + require.NoError(t, gsData.incomingBlockHookActions.TerminationError) + }, + }, + "gs incoming block with data receive error will halt request": { + events: fakeEvents{ + OnDataReceivedError: errors.New("something went wrong"), + }, + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.incomingBlockHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) + require.True(t, events.OnDataReceivedCalled) + require.Error(t, gsData.incomingBlockHookActions.TerminationError) + }, + }, + "outgoing gs request with recognized dt request can receive gs response": { + responseConfig: gsResponseConfig{ + dtIsResponse: true, + }, + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.incomingResponseHOok() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) + require.Equal(t, 0, events.OnRequestReceivedCallCount) + require.Equal(t, 1, events.OnResponseReceivedCallCount) + require.NoError(t, gsData.incomingResponseHookActions.TerminationError) + }, + }, + "outgoing gs request with recognized dt request cannot receive gs response with dt request": { + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.incomingResponseHOok() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) + require.Equal(t, 0, events.OnRequestReceivedCallCount) + require.Equal(t, 0, events.OnResponseReceivedCallCount) + require.Error(t, gsData.incomingResponseHookActions.TerminationError) + }, + }, + "outgoing gs request with recognized dt response can receive gs response": { + requestConfig: gsRequestConfig{ + dtIsResponse: true, + }, + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.incomingResponseHOok() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.Equal(t, 0, events.OnResponseReceivedCallCount) + require.NoError(t, gsData.incomingResponseHookActions.TerminationError) + }, + }, + "outgoing gs request with recognized dt response cannot receive gs response with dt response": { + requestConfig: gsRequestConfig{ + dtIsResponse: true, + }, + responseConfig: gsResponseConfig{ + dtIsResponse: true, + }, + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.incomingResponseHOok() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + require.Equal(t, 0, events.OnResponseReceivedCallCount) + require.Equal(t, 0, events.OnRequestReceivedCallCount) + require.Error(t, gsData.incomingResponseHookActions.TerminationError) + }, + }, + "outgoing gs request with recognized dt request will error with malformed update": { + responseConfig: gsResponseConfig{ + dtExtensionMalformed: true, + }, + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.incomingResponseHOok() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) + require.Equal(t, 0, events.OnRequestReceivedCallCount) + require.Equal(t, 0, events.OnResponseReceivedCallCount) + require.Error(t, gsData.incomingResponseHookActions.TerminationError) + }, + }, + "outgoing gs request with recognized dt request will ignore non-data-transfer update": { + responseConfig: gsResponseConfig{ + dtExtensionMissing: true, + }, + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.incomingResponseHOok() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) + require.Equal(t, 0, events.OnRequestReceivedCallCount) + require.Equal(t, 0, events.OnResponseReceivedCallCount) + require.NoError(t, gsData.incomingResponseHookActions.TerminationError) + }, + }, + "outgoing gs request with recognized dt response can send message on update": { + events: fakeEvents{ + RequestReceivedResponse: testutil.NewDTResponse(t, datatransfer.TransferID(rand.Uint32())), + }, + requestConfig: gsRequestConfig{ + dtIsResponse: true, + }, + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.incomingResponseHOok() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, events.ChannelOpenedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.Equal(t, 0, events.OnResponseReceivedCallCount) + require.NoError(t, gsData.incomingResponseHookActions.TerminationError) + assertHasOutgoingMessage(t, gsData.incomingResponseHookActions.SentExtensions, + events.RequestReceivedResponse) + }, + }, + "outgoing gs request with recognized dt response err will error": { + requestConfig: gsRequestConfig{ + dtIsResponse: true, + }, + events: fakeEvents{ + OnRequestReceivedErrors: []error{errors.New("something went wrong")}, + }, + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.incomingResponseHOok() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 0, events.OnResponseReceivedCallCount) + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.Error(t, gsData.incomingResponseHookActions.TerminationError) + }, + }, + "incoming gs request with recognized dt request will validate gs request & send dt response": { + action: func(gsData *harness) { + gsData.incomingRequestHook() + }, + events: fakeEvents{ + RequestReceivedResponse: testutil.NewDTResponse(t, datatransfer.TransferID(rand.Uint32())), + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.Equal(t, 0, events.OnResponseReceivedCallCount) + require.Equal(t, events.RequestReceivedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + dtRequestData, _ := gsData.request.Extension(extension.ExtensionDataTransfer1_1) + assertDecodesToMessage(t, dtRequestData, events.RequestReceivedRequest) + require.True(t, gsData.incomingRequestHookActions.Validated) + assertHasExtensionMessage(t, extension.ExtensionDataTransfer1_1, gsData.incomingRequestHookActions.SentExtensions, events.RequestReceivedResponse) + require.NoError(t, gsData.incomingRequestHookActions.TerminationError) + + channelsForPeer := gsData.transport.ChannelsForPeer(gsData.other) + require.Equal(t, channelsForPeer, ChannelsForPeer{ + SendingChannels: map[datatransfer.ChannelID]ChannelGraphsyncRequests{ + events.RequestReceivedChannelID: { + Current: gsData.request.ID(), + }, + }, + ReceivingChannels: map[datatransfer.ChannelID]ChannelGraphsyncRequests{}, + }) + }, + }, + "incoming gs request with recognized dt response will validate gs request": { + requestConfig: gsRequestConfig{ + dtIsResponse: true, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 0, events.OnRequestReceivedCallCount) + require.Equal(t, 1, events.OnResponseReceivedCallCount) + require.Equal(t, events.ResponseReceivedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) + dtResponseData, _ := gsData.request.Extension(extension.ExtensionDataTransfer1_1) + assertDecodesToMessage(t, dtResponseData, events.ResponseReceivedResponse) + require.True(t, gsData.incomingRequestHookActions.Validated) + require.NoError(t, gsData.incomingRequestHookActions.TerminationError) + }, + }, + "malformed data transfer extension on incoming request will terminate": { + requestConfig: gsRequestConfig{ + dtExtensionMalformed: true, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 0, events.OnRequestReceivedCallCount) + require.False(t, gsData.incomingRequestHookActions.Validated) + require.Error(t, gsData.incomingRequestHookActions.TerminationError) + }, + }, + "unrecognized incoming dt request will terminate but send response": { + events: fakeEvents{ + RequestReceivedResponse: testutil.NewDTResponse(t, datatransfer.TransferID(rand.Uint32())), + OnRequestReceivedErrors: []error{errors.New("something went wrong")}, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.Equal(t, 0, events.OnResponseReceivedCallCount) + require.Equal(t, events.RequestReceivedChannelID, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + dtRequestData, _ := gsData.request.Extension(extension.ExtensionDataTransfer1_1) + assertDecodesToMessage(t, dtRequestData, events.RequestReceivedRequest) + require.False(t, gsData.incomingRequestHookActions.Validated) + assertHasExtensionMessage(t, extension.ExtensionIncomingRequest1_1, gsData.incomingRequestHookActions.SentExtensions, events.RequestReceivedResponse) + require.Error(t, gsData.incomingRequestHookActions.TerminationError) + }, + }, + "incoming gs request with recognized dt request will record outgoing blocks": { + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.outgoingBlockHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.True(t, events.OnDataQueuedCalled) + require.NoError(t, gsData.outgoingBlockHookActions.TerminationError) + }, + }, + + "incoming gs request with recognized dt response will record outgoing blocks": { + requestConfig: gsRequestConfig{ + dtIsResponse: true, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.outgoingBlockHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnResponseReceivedCallCount) + require.True(t, events.OnDataQueuedCalled) + require.NoError(t, gsData.outgoingBlockHookActions.TerminationError) + }, + }, + "non-data-transfer request will not record outgoing blocks": { + requestConfig: gsRequestConfig{ + dtExtensionMissing: true, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.outgoingBlockHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 0, events.OnRequestReceivedCallCount) + require.False(t, events.OnDataQueuedCalled) + }, + }, + "outgoing data queued error will terminate request": { + events: fakeEvents{ + OnDataQueuedError: errors.New("something went wrong"), + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.outgoingBlockHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.True(t, events.OnDataQueuedCalled) + require.Error(t, gsData.outgoingBlockHookActions.TerminationError) + }, + }, + "outgoing data queued error == pause will pause request": { + events: fakeEvents{ + OnDataQueuedError: datatransfer.ErrPause, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.outgoingBlockHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.True(t, events.OnDataQueuedCalled) + require.True(t, gsData.outgoingBlockHookActions.Paused) + require.NoError(t, gsData.outgoingBlockHookActions.TerminationError) + }, + }, + "incoming gs request with recognized dt request will send updates": { + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.outgoingBlockHook() + }, + events: fakeEvents{ + OnDataQueuedMessage: testutil.NewDTResponse(t, datatransfer.TransferID(rand.Uint32())), + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.True(t, events.OnDataQueuedCalled) + require.NoError(t, gsData.outgoingBlockHookActions.TerminationError) + assertHasExtensionMessage(t, extension.ExtensionOutgoingBlock1_1, gsData.outgoingBlockHookActions.SentExtensions, + events.OnDataQueuedMessage) + }, + }, + "incoming gs request with recognized dt request can receive update": { + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.requestUpdatedHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 2, events.OnRequestReceivedCallCount) + require.NoError(t, gsData.requestUpdatedHookActions.TerminationError) + }, + }, + "incoming gs request with recognized dt request cannot receive update with dt response": { + updatedConfig: gsRequestConfig{ + dtIsResponse: true, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.requestUpdatedHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.Equal(t, 0, events.OnResponseReceivedCallCount) + require.Error(t, gsData.requestUpdatedHookActions.TerminationError) + }, + }, + "incoming gs request with recognized dt response can receive update": { + requestConfig: gsRequestConfig{ + dtIsResponse: true, + }, + updatedConfig: gsRequestConfig{ + dtIsResponse: true, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.requestUpdatedHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 2, events.OnResponseReceivedCallCount) + require.NoError(t, gsData.requestUpdatedHookActions.TerminationError) + }, + }, + "incoming gs request with recognized dt response cannot receive update with dt request": { + requestConfig: gsRequestConfig{ + dtIsResponse: true, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.requestUpdatedHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnResponseReceivedCallCount) + require.Equal(t, 0, events.OnRequestReceivedCallCount) + require.Error(t, gsData.requestUpdatedHookActions.TerminationError) + }, + }, + "incoming gs request with recognized dt request will error with malformed update": { + updatedConfig: gsRequestConfig{ + dtExtensionMalformed: true, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.requestUpdatedHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.Error(t, gsData.requestUpdatedHookActions.TerminationError) + }, + }, + "incoming gs request with recognized dt request will ignore non-data-transfer update": { + updatedConfig: gsRequestConfig{ + dtExtensionMissing: true, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.requestUpdatedHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.NoError(t, gsData.requestUpdatedHookActions.TerminationError) + }, + }, + "incoming gs request with recognized dt request can send message on update": { + events: fakeEvents{ + RequestReceivedResponse: testutil.NewDTResponse(t, datatransfer.TransferID(rand.Uint32())), + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.requestUpdatedHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 2, events.OnRequestReceivedCallCount) + require.NoError(t, gsData.requestUpdatedHookActions.TerminationError) + assertHasOutgoingMessage(t, gsData.requestUpdatedHookActions.SentExtensions, + events.RequestReceivedResponse) + }, + }, + "recognized incoming request will record successful request completion": { + responseConfig: gsResponseConfig{ + status: graphsync.RequestCompletedFull, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.responseCompletedListener() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.True(t, events.OnChannelCompletedCalled) + require.True(t, events.ChannelCompletedSuccess) + }, + }, + + "recognized incoming request will record unsuccessful request completion": { + responseConfig: gsResponseConfig{ + status: graphsync.RequestCompletedPartial, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.responseCompletedListener() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.True(t, events.OnChannelCompletedCalled) + require.False(t, events.ChannelCompletedSuccess) + }, + }, + "recognized incoming request will not record request cancellation": { + responseConfig: gsResponseConfig{ + status: graphsync.RequestCancelled, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.responseCompletedListener() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.False(t, events.OnChannelCompletedCalled) + }, + }, + "non-data-transfer request will not record request completed": { + requestConfig: gsRequestConfig{ + dtExtensionMissing: true, + }, + responseConfig: gsResponseConfig{ + status: graphsync.RequestCompletedPartial, + }, + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.responseCompletedListener() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 0, events.OnRequestReceivedCallCount) + require.False(t, events.OnChannelCompletedCalled) + }, + }, + "recognized incoming request can be closed": { + action: func(gsData *harness) { + gsData.incomingRequestHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + err := gsData.transport.CloseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + require.NoError(t, err) + require.Equal(t, 1, events.OnRequestReceivedCallCount) + gsData.fgs.AssertCancelReceived(gsData.ctx, t) + }, + }, + "unrecognized request cannot be closed": { + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + err := gsData.transport.CloseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + require.Error(t, err) + }, + }, + "recognized incoming request that requestor cancelled will not close via graphsync": { + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.requestorCancelledListener() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + err := gsData.transport.CloseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + require.NoError(t, err) + require.Equal(t, 1, events.OnRequestReceivedCallCount) + gsData.fgs.AssertNoCancelReceived(t) + }, + }, + "recognized incoming request can be paused": { + action: func(gsData *harness) { + gsData.incomingRequestHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + err := gsData.transport.PauseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + require.NoError(t, err) + require.Equal(t, 1, events.OnRequestReceivedCallCount) + gsData.fgs.AssertPauseReceived(gsData.ctx, t) + }, + }, + "unrecognized request cannot be paused": { + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + err := gsData.transport.PauseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + require.Error(t, err) + }, + }, + "recognized incoming request that requestor cancelled will not pause via graphsync": { + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.requestorCancelledListener() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + err := gsData.transport.PauseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + require.NoError(t, err) + require.Equal(t, 1, events.OnRequestReceivedCallCount) + gsData.fgs.AssertNoPauseReceived(t) + }, + }, + + "incoming request can be queued": { + action: func(gsData *harness) { + gsData.incomingRequestQueuedHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.True(t, events.TransferQueuedCalled) + require.Equal(t, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}, + events.TransferQueuedChannelID) + }, + }, + + "incoming request with dtResponse can be queued": { + requestConfig: gsRequestConfig{ + dtIsResponse: true, + }, + responseConfig: gsResponseConfig{ + dtIsResponse: true, + }, + action: func(gsData *harness) { + gsData.incomingRequestQueuedHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.True(t, events.TransferQueuedCalled) + require.Equal(t, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, + events.TransferQueuedChannelID) + }, + }, + + "recognized incoming request can be resumed": { + action: func(gsData *harness) { + gsData.incomingRequestHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + err := gsData.transport.ResumeChannel(gsData.ctx, + gsData.incoming, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}, + ) + require.NoError(t, err) + require.Equal(t, 1, events.OnRequestReceivedCallCount) + gsData.fgs.AssertResumeReceived(gsData.ctx, t) + }, + }, + + "unrecognized request cannot be resumed": { + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + err := gsData.transport.ResumeChannel(gsData.ctx, + gsData.incoming, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}, + ) + require.Error(t, err) + }, + }, + "recognized incoming request that requestor cancelled will not resume via graphsync but will resume otherwise": { + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.requestorCancelledListener() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + err := gsData.transport.ResumeChannel(gsData.ctx, + gsData.incoming, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}, + ) + require.NoError(t, err) + require.Equal(t, 1, events.OnRequestReceivedCallCount) + gsData.fgs.AssertNoResumeReceived(t) + gsData.incomingRequestHook() + assertHasOutgoingMessage(t, gsData.incomingRequestHookActions.SentExtensions, gsData.incoming) + }, + }, + "recognized incoming request will record network send error": { + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.networkErrorListener(errors.New("something went wrong")) + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.True(t, events.OnSendDataErrorCalled) + }, + }, + "recognized outgoing request will record network send error": { + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.networkErrorListener(errors.New("something went wrong")) + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.True(t, events.OnSendDataErrorCalled) + }, + }, + "recognized incoming request will record network receive error": { + action: func(gsData *harness) { + gsData.incomingRequestHook() + gsData.receiverNetworkErrorListener(errors.New("something went wrong")) + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.Equal(t, 1, events.OnRequestReceivedCallCount) + require.True(t, events.OnReceiveDataErrorCalled) + }, + }, + "recognized outgoing request will record network receive error": { + action: func(gsData *harness) { + gsData.outgoingRequestHook() + gsData.receiverNetworkErrorListener(errors.New("something went wrong")) + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + require.True(t, events.OnReceiveDataErrorCalled) + }, + }, + "open channel adds block count to the DoNotSendFirstBlocks extension for v1.2 protocol": { + action: func(gsData *harness) { + cids := testutil.GenerateCids(2) + channel := testutil.NewMockChannelState(testutil.MockChannelStateParams{ReceivedCids: cids}) + stor, _ := gsData.outgoing.Selector() + + go gsData.outgoingRequestHook() + _ = gsData.transport.OpenChannel( + gsData.ctx, + gsData.other, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, + cidlink.Link{Cid: gsData.outgoing.BaseCid()}, + stor, + channel, + gsData.outgoing) + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + requestReceived := gsData.fgs.AssertRequestReceived(gsData.ctx, t) + + ext := requestReceived.Extensions + require.Len(t, ext, 2) + doNotSend := ext[1] + + name := doNotSend.Name + require.Equal(t, graphsync.ExtensionsDoNotSendFirstBlocks, name) + data := doNotSend.Data + blockCount, err := donotsendfirstblocks.DecodeDoNotSendFirstBlocks(data) + require.NoError(t, err) + require.EqualValues(t, blockCount, 2) + }, + }, + "ChannelsForPeer when request is open": { + action: func(gsData *harness) { + cids := testutil.GenerateCids(2) + channel := testutil.NewMockChannelState(testutil.MockChannelStateParams{ReceivedCids: cids}) + stor, _ := gsData.outgoing.Selector() + + go gsData.outgoingRequestHook() + _ = gsData.transport.OpenChannel( + gsData.ctx, + gsData.other, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, + cidlink.Link{Cid: gsData.outgoing.BaseCid()}, + stor, + channel, + gsData.outgoing) + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + gsData.fgs.AssertRequestReceived(gsData.ctx, t) + + channelsForPeer := gsData.transport.ChannelsForPeer(gsData.other) + require.Equal(t, channelsForPeer, ChannelsForPeer{ + ReceivingChannels: map[datatransfer.ChannelID]ChannelGraphsyncRequests{ + events.ChannelOpenedChannelID: { + Current: gsData.request.ID(), + }, + }, + SendingChannels: map[datatransfer.ChannelID]ChannelGraphsyncRequests{}, + }) + }, + }, + "open channel cancels an existing request with the same channel ID": { + action: func(gsData *harness) { + cids := testutil.GenerateCids(2) + channel := testutil.NewMockChannelState(testutil.MockChannelStateParams{ReceivedCids: cids}) + stor, _ := gsData.outgoing.Selector() + go gsData.outgoingRequestHook() + _ = gsData.transport.OpenChannel( + gsData.ctx, + gsData.other, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, + cidlink.Link{Cid: gsData.outgoing.BaseCid()}, + stor, + channel, + gsData.outgoing) + + go gsData.altOutgoingRequestHook() + _ = gsData.transport.OpenChannel( + gsData.ctx, + gsData.other, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, + cidlink.Link{Cid: gsData.outgoing.BaseCid()}, + stor, + channel, + gsData.outgoing) + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + gsData.fgs.AssertRequestReceived(gsData.ctx, t) + gsData.fgs.AssertRequestReceived(gsData.ctx, t) + + ctxt, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + gsData.fgs.AssertCancelReceived(ctxt, t) + + channelsForPeer := gsData.transport.ChannelsForPeer(gsData.other) + require.Equal(t, channelsForPeer, ChannelsForPeer{ + ReceivingChannels: map[datatransfer.ChannelID]ChannelGraphsyncRequests{ + events.ChannelOpenedChannelID: { + Current: gsData.altRequest.ID(), + Previous: []graphsync.RequestID{gsData.request.ID()}, + }, + }, + SendingChannels: map[datatransfer.ChannelID]ChannelGraphsyncRequests{}, + }) + }, + }, + "OnChannelCompleted called when outgoing request completes successfully": { + action: func(gsData *harness) { + gsData.fgs.LeaveRequestsOpen() + stor, _ := gsData.outgoing.Selector() + + go gsData.outgoingRequestHook() + _ = gsData.transport.OpenChannel( + gsData.ctx, + gsData.other, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, + cidlink.Link{Cid: gsData.outgoing.BaseCid()}, + stor, + nil, + gsData.outgoing) + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + requestReceived := gsData.fgs.AssertRequestReceived(gsData.ctx, t) + close(requestReceived.ResponseChan) + close(requestReceived.ResponseErrChan) + + require.Eventually(t, func() bool { + return events.OnChannelCompletedCalled == true + }, 2*time.Second, 100*time.Millisecond) + require.True(t, events.ChannelCompletedSuccess) + }, + }, + "OnChannelCompleted called when outgoing request completes with error": { + action: func(gsData *harness) { + gsData.fgs.LeaveRequestsOpen() + stor, _ := gsData.outgoing.Selector() + + go gsData.outgoingRequestHook() + _ = gsData.transport.OpenChannel( + gsData.ctx, + gsData.other, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, + cidlink.Link{Cid: gsData.outgoing.BaseCid()}, + stor, + nil, + gsData.outgoing) + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + requestReceived := gsData.fgs.AssertRequestReceived(gsData.ctx, t) + close(requestReceived.ResponseChan) + requestReceived.ResponseErrChan <- graphsync.RequestFailedUnknownErr{} + close(requestReceived.ResponseErrChan) + + require.Eventually(t, func() bool { + return events.OnChannelCompletedCalled == true + }, 2*time.Second, 100*time.Millisecond) + require.False(t, events.ChannelCompletedSuccess) + }, + }, + "OnChannelComplete when outgoing request cancelled by caller": { + action: func(gsData *harness) { + gsData.fgs.LeaveRequestsOpen() + stor, _ := gsData.outgoing.Selector() + + go gsData.outgoingRequestHook() + _ = gsData.transport.OpenChannel( + gsData.ctx, + gsData.other, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, + cidlink.Link{Cid: gsData.outgoing.BaseCid()}, + stor, + nil, + gsData.outgoing) + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + requestReceived := gsData.fgs.AssertRequestReceived(gsData.ctx, t) + extensions := make(map[graphsync.ExtensionName]datamodel.Node) + for _, ext := range requestReceived.Extensions { + extensions[ext.Name] = ext.Data + } + request := testutil.NewFakeRequest(graphsync.NewRequestID(), extensions) + gsData.fgs.OutgoingRequestHook(gsData.other, request, gsData.outgoingRequestHookActions) + _ = gsData.transport.CloseChannel(gsData.ctx, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) + ctxt, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + gsData.fgs.AssertCancelReceived(ctxt, t) + }, + }, + "request times out if we get request context cancelled error": { + action: func(gsData *harness) { + gsData.fgs.LeaveRequestsOpen() + stor, _ := gsData.outgoing.Selector() + + go gsData.outgoingRequestHook() + _ = gsData.transport.OpenChannel( + gsData.ctx, + gsData.other, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, + cidlink.Link{Cid: gsData.outgoing.BaseCid()}, + stor, + nil, + gsData.outgoing) + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + requestReceived := gsData.fgs.AssertRequestReceived(gsData.ctx, t) + close(requestReceived.ResponseChan) + requestReceived.ResponseErrChan <- graphsync.RequestClientCancelledErr{} + close(requestReceived.ResponseErrChan) + + require.Eventually(t, func() bool { + return events.OnRequestCancelledCalled == true + }, 2*time.Second, 100*time.Millisecond) + require.Equal(t, datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, events.OnRequestCancelledChannelId) + }, + }, + "request cancelled out if transport shuts down": { + action: func(gsData *harness) { + gsData.fgs.LeaveRequestsOpen() + stor, _ := gsData.outgoing.Selector() + + go gsData.outgoingRequestHook() + _ = gsData.transport.OpenChannel( + gsData.ctx, + gsData.other, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, + cidlink.Link{Cid: gsData.outgoing.BaseCid()}, + stor, + nil, + gsData.outgoing) + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + gsData.fgs.AssertRequestReceived(gsData.ctx, t) + + gsData.transport.Shutdown(gsData.ctx) + + ctxt, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + gsData.fgs.AssertCancelReceived(ctxt, t) + + require.Nil(t, gsData.fgs.IncomingRequestHook) + require.Nil(t, gsData.fgs.CompletedResponseListener) + require.Nil(t, gsData.fgs.IncomingBlockHook) + require.Nil(t, gsData.fgs.OutgoingBlockHook) + require.Nil(t, gsData.fgs.BlockSentListener) + require.Nil(t, gsData.fgs.OutgoingRequestHook) + require.Nil(t, gsData.fgs.IncomingResponseHook) + require.Nil(t, gsData.fgs.RequestUpdatedHook) + require.Nil(t, gsData.fgs.RequestorCancelledListener) + require.Nil(t, gsData.fgs.NetworkErrorListener) + }, + }, + "request pause works even if called when request is still pending": { + action: func(gsData *harness) { + gsData.fgs.LeaveRequestsOpen() + stor, _ := gsData.outgoing.Selector() + + go gsData.outgoingRequestHook() + _ = gsData.transport.OpenChannel( + gsData.ctx, + gsData.other, + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, + cidlink.Link{Cid: gsData.outgoing.BaseCid()}, + stor, + nil, + gsData.outgoing) + + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + requestReceived := gsData.fgs.AssertRequestReceived(gsData.ctx, t) + assertHasOutgoingMessage(t, requestReceived.Extensions, gsData.outgoing) + completed := make(chan struct{}) + go func() { + err := gsData.transport.PauseChannel(context.Background(), datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) + require.NoError(t, err) + close(completed) + }() + time.Sleep(100 * time.Millisecond) + extensions := make(map[graphsync.ExtensionName]datamodel.Node) + for _, ext := range requestReceived.Extensions { + extensions[ext.Name] = ext.Data + } + request := testutil.NewFakeRequest(graphsync.NewRequestID(), extensions) + gsData.fgs.OutgoingRequestHook(gsData.other, request, gsData.outgoingRequestHookActions) + select { + case <-gsData.ctx.Done(): + t.Fatal("never paused channel") + case <-completed: + } + }, + }, + "UseStore can change store used for outgoing requests": { + action: func(gsData *harness) { + lsys := cidlink.DefaultLinkSystem() + lsys.StorageReadOpener = func(ipld.LinkContext, ipld.Link) (io.Reader, error) { + return nil, nil + } + lsys.StorageWriteOpener = func(ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) { + return nil, nil, nil + } + _ = gsData.transport.UseStore(datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}, lsys) + gsData.outgoingRequestHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + expectedChannel := "data-transfer-" + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}.String() + gsData.fgs.AssertHasPersistenceOption(t, expectedChannel) + require.Equal(t, expectedChannel, gsData.outgoingRequestHookActions.PersistenceOption) + gsData.transport.CleanupChannel(datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.other, Initiator: gsData.self}) + gsData.fgs.AssertDoesNotHavePersistenceOption(t, expectedChannel) + }, + }, + "UseStore can change store used for incoming requests": { + action: func(gsData *harness) { + lsys := cidlink.DefaultLinkSystem() + lsys.StorageReadOpener = func(ipld.LinkContext, ipld.Link) (io.Reader, error) { + return nil, nil + } + lsys.StorageWriteOpener = func(ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) { + return nil, nil, nil + } + _ = gsData.transport.UseStore(datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}, lsys) + gsData.incomingRequestHook() + }, + check: func(t *testing.T, events *fakeEvents, gsData *harness) { + expectedChannel := "data-transfer-" + datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}.String() + gsData.fgs.AssertHasPersistenceOption(t, expectedChannel) + require.Equal(t, expectedChannel, gsData.incomingRequestHookActions.PersistenceOption) + gsData.transport.CleanupChannel(datatransfer.ChannelID{ID: gsData.transferID, Responder: gsData.self, Initiator: gsData.other}) + gsData.fgs.AssertDoesNotHavePersistenceOption(t, expectedChannel) + }, + },*/ diff --git a/transport/graphsync/receiver.go b/transport/graphsync/receiver.go index ba9b1639..f6fe6038 100644 --- a/transport/graphsync/receiver.go +++ b/transport/graphsync/receiver.go @@ -3,7 +3,6 @@ package graphsync import ( "context" - "github.com/ipfs/go-graphsync" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/libp2p/go-libp2p-core/peer" "go.opentelemetry.io/otel" @@ -11,7 +10,6 @@ import ( "go.opentelemetry.io/otel/trace" datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" ) type receiver struct { @@ -43,9 +41,14 @@ func (r *receiver) receiveRequest(ctx context.Context, initiator peer.ID, incomi attribute.Bool("isPaused", incoming.IsPaused()), )) defer span.End() + isNewOrRestart := incoming.IsNew() || incoming.IsRestart() + // a graphsync pull request MUST come in via graphsync + if isNewOrRestart && incoming.IsPull() { + return datatransfer.ErrUnsupported + } response, receiveErr := r.transport.events.OnRequestReceived(chid, incoming) + initiateGraphsyncRequest := isNewOrRestart && response != nil && receiveErr == nil ch, err := r.transport.getDTChannel(chid) - initiateGraphsyncRequest := (response != nil) && (response.IsNew() || response.IsRestart()) && response.Accepted() && !incoming.IsPull() if err != nil { if !initiateGraphsyncRequest { if response != nil { @@ -56,50 +59,39 @@ func (r *receiver) receiveRequest(ctx context.Context, initiator peer.ID, incomi ch = r.transport.trackDTChannel(chid) } - if receiveErr == datatransfer.ErrResume && ch.Paused() { - - var extensions []graphsync.ExtensionData + if receiveErr != nil { if response != nil { - var err error - extensions, err = extension.ToExtensionData(response, r.transport.supportedExtensions) - if err != nil { + if err := r.transport.dtNet.SendMessage(ctx, initiator, transportID, response); err != nil { return err } + _ = ch.Close(ctx) + return receiveErr } - - return ch.Resume(ctx, extensions) } - if response != nil { - if initiateGraphsyncRequest { - stor, _ := incoming.Selector() - if response.IsRestart() { - channel, err := r.transport.events.ChannelState(ctx, chid) - if err != nil { - return err - } - ch.UpdateReceivedCidsIfGreater(channel.ReceivedCidsTotal()) - } - if err := r.transport.openRequest(ctx, initiator, chid, cidlink.Link{Cid: incoming.BaseCid()}, stor, response); err != nil { - return err - } - } else { - if err := r.transport.dtNet.SendMessage(ctx, initiator, transportID, response); err != nil { - return err - } - } + if isNewOrRestart { + r.transport.dtNet.Protect(initiator, chid.String()) + } + chst, err := r.transport.events.ChannelState(ctx, chid) + if err != nil { + return err } - if receiveErr == datatransfer.ErrPause { - return ch.Pause(ctx) + err = ch.UpdateFromChannelState(chst) + if err != nil { + return err } - if receiveErr != nil { - _ = ch.Close(ctx) - return receiveErr + if initiateGraphsyncRequest { + stor, _ := incoming.Selector() + if err := r.transport.openRequest(ctx, initiator, chid, cidlink.Link{Cid: incoming.BaseCid()}, stor, response); err != nil { + return err + } + response = nil } - return nil + action := ch.ActionFromChannelState(chst) + return r.transport.processAction(ctx, chid, ch, action, response) } // ReceiveResponse handles responses to our Push or Pull data transfer request. @@ -135,9 +127,6 @@ func (r *receiver) receiveResponse( if err != nil { return err } - if receiveErr == datatransfer.ErrPause { - return ch.Pause(ctx) - } if receiveErr != nil { log.Warnf("closing channel %s after getting error processing response from %s: %s", chid, sender, err) @@ -180,9 +169,6 @@ func (r *receiver) ReceiveRestartExistingChannelRequest(ctx context.Context, return } - err = r.transport.events.OnRestartExistingChannelRequestReceived(ch) - if err != nil { - log.Errorf(err.Error()) - } + r.transport.events.OnTransportEvent(ch, datatransfer.TransportReceivedRestartExistingChannelRequest{}) return } diff --git a/transport/graphsync/responding_test.go b/transport/graphsync/responding_test.go new file mode 100644 index 00000000..07d70184 --- /dev/null +++ b/transport/graphsync/responding_test.go @@ -0,0 +1,412 @@ +package graphsync_test + +import ( + "context" + "fmt" + "testing" + "time" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/testharness" + "github.com/ipfs/go-graphsync" + "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/stretchr/testify/require" +) + +func TestRespondingPullSuccessFlow(t *testing.T) { + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + th := testharness.SetupHarness(ctx, testharness.PullRequest(), testharness.Responder()) + + // this actually happens in the request received event handler itself in a real life case, but here we just run it before + t.Run("configures persistence", func(t *testing.T) { + th.Transport.UseStore(th.Channel.ChannelID(), cidlink.DefaultLinkSystem()) + th.Fgs.AssertHasPersistenceOption(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String())) + }) + requestID := graphsync.NewRequestID() + dtRequest := th.NewRequest(t) + request := testharness.NewFakeRequest(requestID, map[graphsync.ExtensionName]datamodel.Node{extension.ExtensionDataTransfer1_1: dtRequest.ToIPLD()}, graphsync.RequestTypeNew) + + // this the actual start of request processing + t.Run("received and responds successfully", func(t *testing.T) { + dtResponse := th.Response() + th.Events.ReturnedRequestReceivedResponse = dtResponse + th.Channel.SetResponderPaused(true) + th.Channel.SetDataLimit(10000) + th.Events.ReturnedOnContextAugmentFunc = func(ctx context.Context) context.Context { + return context.WithValue(ctx, ctxKey{}, "applesauce") + } + th.IncomingRequestHook(request) + require.Equal(t, dtRequest, th.Events.ReceivedRequest) + require.Len(t, th.DtNet.ProtectedPeers, 1) + require.Equal(t, th.DtNet.ProtectedPeers[0], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + require.Equal(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String()), th.IncomingRequestHookActions.PersistenceOption) + require.True(t, th.IncomingRequestHookActions.Validated) + require.True(t, th.IncomingRequestHookActions.Paused) + require.NoError(t, th.IncomingRequestHookActions.TerminationError) + sentResponse := th.IncomingRequestHookActions.DTMessage(t) + require.Equal(t, dtResponse, sentResponse) + th.IncomingRequestHookActions.AssertAugmentedContextKey(t, ctxKey{}, "applesauce") + }) + + t.Run("receives incoming processing listener", func(t *testing.T) { + th.IncomingRequestProcessingListener(request) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportInitiatedTransfer{}) + }) + + t.Run("unpause request", func(t *testing.T) { + th.Channel.SetResponderPaused(false) + dtValidationResponse := th.ValidationResultResponse(false) + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), dtValidationResponse) + require.Len(t, th.Fgs.Resumes, 1) + require.Equal(t, dtValidationResponse, th.Fgs.Resumes[0].DTMessage(t)) + }) + + t.Run("queued block / data limits", func(t *testing.T) { + // consume first block + block := testharness.NewFakeBlockData(8000, 1, true) + th.OutgoingBlockHook(request, block) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportQueuedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + + // consume second block -- should hit data limit + block = testharness.NewFakeBlockData(3000, 2, true) + th.OutgoingBlockHook(request, block) + require.True(t, th.OutgoingBlockHookActions.Paused) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportQueuedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReachedDataLimit{}) + + // reset data limit + th.Channel.SetResponderPaused(false) + th.Channel.SetDataLimit(20000) + dtValidationResponse := th.ValidationResultResponse(false) + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), dtValidationResponse) + require.Len(t, th.Fgs.Resumes, 2) + require.Equal(t, dtValidationResponse, th.Fgs.Resumes[1].DTMessage(t)) + + // block not on wire has no effect + block = testharness.NewFakeBlockData(12345, 3, false) + th.OutgoingBlockHook(request, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportQueuedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + // block with lower index has no effect + block = testharness.NewFakeBlockData(67890, 1, true) + th.OutgoingBlockHook(request, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportQueuedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + + // consume third block + block = testharness.NewFakeBlockData(5000, 4, true) + th.OutgoingBlockHook(request, block) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportQueuedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + + // consume fourth block should hit data limit again + block = testharness.NewFakeBlockData(5000, 5, true) + th.OutgoingBlockHook(request, block) + require.True(t, th.OutgoingBlockHookActions.Paused) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportQueuedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReachedDataLimit{}) + + }) + + t.Run("sent block", func(t *testing.T) { + block := testharness.NewFakeBlockData(12345, 1, true) + th.BlockSentListener(request, block) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportSentData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + block = testharness.NewFakeBlockData(12345, 2, true) + th.BlockSentListener(request, block) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportSentData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + // block not on wire has no effect + block = testharness.NewFakeBlockData(12345, 3, false) + th.BlockSentListener(request, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportSentData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + // block with lower index has no effect + block = testharness.NewFakeBlockData(67890, 1, true) + th.BlockSentListener(request, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportSentData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + }) + + t.Run("receive pause", func(t *testing.T) { + th.RequestorCancelledListener(request) + dtPauseRequest := th.UpdateRequest(true) + th.Events.ReturnedRequestReceivedResponse = nil + th.DtNet.Delegates[0].Receiver.ReceiveRequest(ctx, th.Channel.OtherPeer(), dtPauseRequest) + require.Equal(t, th.Events.ReceivedRequest, dtPauseRequest) + }) + + t.Run("receive resume", func(t *testing.T) { + dtResumeRequest := th.UpdateRequest(false) + request = testharness.NewFakeRequest(requestID, map[graphsync.ExtensionName]datamodel.Node{extension.ExtensionDataTransfer1_1: dtResumeRequest.ToIPLD()}, graphsync.RequestTypeNew) + // reset hook behavior + th.IncomingRequestHookActions = &testharness.FakeIncomingRequestHookActions{} + th.IncomingRequestHook(request) + // only protect on new and restart requests + require.Len(t, th.DtNet.ProtectedPeers, 1) + require.Equal(t, th.DtNet.ProtectedPeers[0], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + require.Equal(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String()), th.IncomingRequestHookActions.PersistenceOption) + require.True(t, th.IncomingRequestHookActions.Validated) + require.False(t, th.IncomingBlockHookActions.Paused) + require.NoError(t, th.IncomingRequestHookActions.TerminationError) + th.IncomingRequestHookActions.AssertAugmentedContextKey(t, ctxKey{}, "applesauce") + require.Equal(t, th.Events.ReceivedRequest, dtResumeRequest) + }) + + t.Run("pause", func(t *testing.T) { + th.Channel.SetResponderPaused(true) + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateResponse(true)) + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.UpdateResponse(true)}) + require.Len(t, th.Fgs.Pauses, 1) + require.Equal(t, th.Fgs.Pauses[0], request.ID()) + }) + + t.Run("pause again", func(t *testing.T) { + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateResponse(true)) + // should send message again + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.UpdateResponse(true)}) + // should not pause again + require.Len(t, th.Fgs.Pauses, 1) + }) + + t.Run("resume", func(t *testing.T) { + th.Channel.SetResponderPaused(false) + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateResponse(true)) + require.Len(t, th.Fgs.Resumes, 3) + resume := th.Fgs.Resumes[2] + require.Equal(t, request.ID(), resume.RequestID) + msg := resume.DTMessage(t) + require.Equal(t, msg, th.UpdateResponse(false)) + }) + t.Run("resume again", func(t *testing.T) { + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateResponse(false)) + // should send message again + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.UpdateResponse(true)}) + // should not resume again + require.Len(t, th.Fgs.Resumes, 3) + }) + + t.Run("restart request", func(t *testing.T) { + dtRestartRequest := th.RestartRequest(t) + request := testharness.NewFakeRequest(requestID, map[graphsync.ExtensionName]datamodel.Node{extension.ExtensionDataTransfer1_1: dtRestartRequest.ToIPLD()}, graphsync.RequestTypeNew) + th.IncomingRequestHook(request) + // protect again for a restart + require.Len(t, th.DtNet.ProtectedPeers, 2) + require.Equal(t, th.DtNet.ProtectedPeers[1], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + require.Equal(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String()), th.IncomingRequestHookActions.PersistenceOption) + require.True(t, th.IncomingRequestHookActions.Validated) + require.False(t, th.IncomingRequestHookActions.Paused) + require.NoError(t, th.IncomingRequestHookActions.TerminationError) + th.IncomingRequestHookActions.AssertAugmentedContextKey(t, ctxKey{}, "applesauce") + }) + + t.Run("complete request", func(t *testing.T) { + th.ResponseCompletedListener(request, graphsync.RequestCompletedFull) + select { + case <-th.CompletedResponses: + case <-ctx.Done(): + t.Fatalf("did not complete request") + } + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportCompletedTransfer{Success: true}) + }) + + t.Run("cleanup request", func(t *testing.T) { + th.Transport.CleanupChannel(th.Channel.ChannelID()) + require.Len(t, th.DtNet.UnprotectedPeers, 1) + require.Equal(t, th.DtNet.UnprotectedPeers[0], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + }) +} + +func TestRespondingPushSuccessFlow(t *testing.T) { + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + th := testharness.SetupHarness(ctx, testharness.Responder()) + var receivedRequest testharness.ReceivedGraphSyncRequest + var request graphsync.RequestData + + contextAugmentedCalls := []struct{}{} + th.Events.ReturnedOnContextAugmentFunc = func(ctx context.Context) context.Context { + contextAugmentedCalls = append(contextAugmentedCalls, struct{}{}) + return ctx + } + t.Run("configures persistence", func(t *testing.T) { + th.Transport.UseStore(th.Channel.ChannelID(), cidlink.DefaultLinkSystem()) + th.Fgs.AssertHasPersistenceOption(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String())) + }) + t.Run("receive new request", func(t *testing.T) { + dtResponse := th.Response() + th.Events.ReturnedRequestReceivedResponse = dtResponse + th.Channel.SetResponderPaused(true) + th.Channel.SetDataLimit(10000) + + th.DtNet.Delegates[0].Receiver.ReceiveRequest(ctx, th.Channel.OtherPeer(), th.NewRequest(t)) + require.Equal(t, th.NewRequest(t), th.Events.ReceivedRequest) + require.Len(t, th.DtNet.ProtectedPeers, 1) + require.Equal(t, th.DtNet.ProtectedPeers[0], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + require.Len(t, th.Fgs.ReceivedRequests, 1) + receivedRequest = th.Fgs.ReceivedRequests[0] + request = receivedRequest.ToRequestData(t) + msg, err := extension.GetTransferData(request, []graphsync.ExtensionName{ + extension.ExtensionDataTransfer1_1, + }) + require.NoError(t, err) + require.Equal(t, dtResponse, msg) + require.Len(t, th.Fgs.Pauses, 1) + require.Equal(t, request.ID(), th.Fgs.Pauses[0]) + require.Len(t, contextAugmentedCalls, 1) + }) + + t.Run("unpause request", func(t *testing.T) { + th.Channel.SetResponderPaused(false) + dtValidationResponse := th.ValidationResultResponse(false) + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), dtValidationResponse) + require.Len(t, th.Fgs.Resumes, 1) + require.Equal(t, dtValidationResponse, th.Fgs.Resumes[0].DTMessage(t)) + }) + + t.Run("receives outgoing request hook", func(t *testing.T) { + th.OutgoingRequestHook(request) + require.Equal(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String()), th.OutgoingRequestHookActions.PersistenceOption) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportOpenedChannel{}) + }) + + t.Run("receives outgoing processing listener", func(t *testing.T) { + th.OutgoingRequestProcessingListener(request) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportInitiatedTransfer{}) + }) + response := receivedRequest.Response(t, nil, nil, graphsync.PartialResponse) + t.Run("received block / data limits", func(t *testing.T) { + th.IncomingResponseHook(response) + // consume first block + block := testharness.NewFakeBlockData(8000, 1, true) + th.IncomingBlockHook(response, block) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReceivedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + + // consume second block -- should hit data limit + block = testharness.NewFakeBlockData(3000, 2, true) + th.IncomingBlockHook(response, block) + require.True(t, th.IncomingBlockHookActions.Paused) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReceivedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReachedDataLimit{}) + + // reset data limit + th.Channel.SetResponderPaused(false) + th.Channel.SetDataLimit(20000) + dtValidationResponse := th.ValidationResultResponse(false) + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), dtValidationResponse) + require.Len(t, th.Fgs.Resumes, 2) + require.Equal(t, dtValidationResponse, th.Fgs.Resumes[1].DTMessage(t)) + + // block not on wire has no effect + block = testharness.NewFakeBlockData(12345, 3, false) + th.IncomingBlockHook(response, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReceivedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + // block with lower index has no effect + block = testharness.NewFakeBlockData(67890, 1, true) + th.OutgoingBlockHook(request, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReceivedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + + // consume third block + block = testharness.NewFakeBlockData(5000, 4, true) + th.IncomingBlockHook(response, block) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReceivedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + + // consume fourth block should hit data limit again + block = testharness.NewFakeBlockData(5000, 5, true) + th.IncomingBlockHook(response, block) + require.True(t, th.IncomingBlockHookActions.Paused) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReceivedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReachedDataLimit{}) + + }) + + t.Run("receive pause", func(t *testing.T) { + dtPauseRequest := th.UpdateRequest(true) + pauseResponse := receivedRequest.Response(t, nil, dtPauseRequest, graphsync.RequestPaused) + th.IncomingResponseHook(pauseResponse) + th.Events.ReturnedRequestReceivedResponse = nil + require.Equal(t, th.Events.ReceivedRequest, dtPauseRequest) + }) + + t.Run("receive resume", func(t *testing.T) { + dtResumeRequest := th.UpdateRequest(false) + pauseResponse := receivedRequest.Response(t, nil, dtResumeRequest, graphsync.PartialResponse) + th.IncomingResponseHook(pauseResponse) + require.Equal(t, th.Events.ReceivedRequest, dtResumeRequest) + }) + + t.Run("pause", func(t *testing.T) { + th.Channel.SetResponderPaused(true) + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateResponse(true)) + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.UpdateResponse(true)}) + require.Len(t, th.Fgs.Pauses, 1) + require.Equal(t, th.Fgs.Pauses[0], request.ID()) + }) + t.Run("pause again", func(t *testing.T) { + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateResponse(true)) + // should send message again + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.UpdateResponse(true)}) + // should not pause again + require.Len(t, th.Fgs.Pauses, 1) + }) + t.Run("resume", func(t *testing.T) { + th.Channel.SetResponderPaused(false) + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateResponse(false)) + require.Len(t, th.Fgs.Resumes, 3) + resume := th.Fgs.Resumes[2] + require.Equal(t, request.ID(), resume.RequestID) + msg := resume.DTMessage(t) + require.Equal(t, msg, th.UpdateResponse(false)) + }) + t.Run("resume again", func(t *testing.T) { + th.Transport.ChannelUpdated(ctx, th.Channel.ChannelID(), th.UpdateResponse(false)) + // should send message again + th.DtNet.AssertSentMessage(t, testharness.FakeSentMessage{PeerID: th.Channel.OtherPeer(), TransportID: "graphsync", Message: th.UpdateResponse(false)}) + // should not resume again + require.Len(t, th.Fgs.Resumes, 3) + }) + + t.Run("restart request", func(t *testing.T) { + restartIndex := int64(5) + th.Channel.SetReceivedIndex(basicnode.NewInt(restartIndex)) + dtResponse := th.RestartResponse(false) + th.Events.ReturnedRequestReceivedResponse = dtResponse + th.DtNet.Delegates[0].Receiver.ReceiveRequest(ctx, th.Channel.OtherPeer(), th.NewRequest(t)) + require.Len(t, th.DtNet.ProtectedPeers, 2) + require.Equal(t, th.DtNet.ProtectedPeers[1], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + require.Len(t, th.Fgs.Cancels, 1) + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportTransferCancelled{ErrorMessage: "graphsync request cancelled"}) + require.Equal(t, request.ID(), th.Fgs.Cancels[0]) + require.Len(t, th.Fgs.ReceivedRequests, 2) + receivedRequest = th.Fgs.ReceivedRequests[1] + request = receivedRequest.ToRequestData(t) + msg, err := extension.GetTransferData(request, []graphsync.ExtensionName{ + extension.ExtensionDataTransfer1_1, + }) + require.NoError(t, err) + require.Equal(t, dtResponse, msg) + nd, has := request.Extension(graphsync.ExtensionsDoNotSendFirstBlocks) + require.True(t, has) + val, err := nd.AsInt() + require.NoError(t, err) + require.Equal(t, restartIndex, val) + require.Len(t, contextAugmentedCalls, 2) + }) + + t.Run("complete request", func(t *testing.T) { + close(receivedRequest.ResponseChan) + close(receivedRequest.ResponseErrChan) + select { + case <-th.CompletedRequests: + case <-ctx.Done(): + t.Fatalf("did not complete request") + } + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportCompletedTransfer{Success: true}) + }) + + t.Run("cleanup request", func(t *testing.T) { + th.Transport.CleanupChannel(th.Channel.ChannelID()) + require.Len(t, th.DtNet.UnprotectedPeers, 1) + require.Equal(t, th.DtNet.UnprotectedPeers[0], testharness.TaggedPeer{th.Channel.OtherPeer(), th.Channel.ChannelID().String()}) + }) +} diff --git a/transport/graphsync/testharness/events.go b/transport/graphsync/testharness/events.go new file mode 100644 index 00000000..17f4fd3d --- /dev/null +++ b/transport/graphsync/testharness/events.go @@ -0,0 +1,61 @@ +package testharness + +import ( + "context" + "testing" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/stretchr/testify/require" +) + +type ReceivedTransportEvent struct { + ChannelID datatransfer.ChannelID + TransportEvent datatransfer.TransportEvent +} + +type FakeEvents struct { + // function return value parameters + ReturnedRequestReceivedResponse datatransfer.Response + ReturnedRequestReceivedError error + ReturnedResponseReceivedError error + ReturnedChannelState datatransfer.ChannelState + ReturnedOnContextAugmentFunc func(context.Context) context.Context + + // recording of actions + OnRequestReceivedCalled bool + ReceivedRequest datatransfer.Request + OnResponseReceivedCalled bool + ReceivedResponse datatransfer.Response + ReceivedTransportEvents []ReceivedTransportEvent +} + +func (fe *FakeEvents) OnTransportEvent(chid datatransfer.ChannelID, evt datatransfer.TransportEvent) { + fe.ReceivedTransportEvents = append(fe.ReceivedTransportEvents, ReceivedTransportEvent{chid, evt}) +} + +func (fe *FakeEvents) AssertTransportEvent(t *testing.T, chid datatransfer.ChannelID, evt datatransfer.TransportEvent) { + require.Contains(t, fe.ReceivedTransportEvents, ReceivedTransportEvent{chid, evt}) +} + +func (fe *FakeEvents) RefuteTransportEvent(t *testing.T, chid datatransfer.ChannelID, evt datatransfer.TransportEvent) { + require.NotContains(t, fe.ReceivedTransportEvents, ReceivedTransportEvent{chid, evt}) +} +func (fe *FakeEvents) OnRequestReceived(chid datatransfer.ChannelID, request datatransfer.Request) (datatransfer.Response, error) { + fe.OnRequestReceivedCalled = true + fe.ReceivedRequest = request + return fe.ReturnedRequestReceivedResponse, fe.ReturnedRequestReceivedError +} + +func (fe *FakeEvents) OnResponseReceived(chid datatransfer.ChannelID, response datatransfer.Response) error { + fe.OnResponseReceivedCalled = true + fe.ReceivedResponse = response + return fe.ReturnedResponseReceivedError +} + +func (fe *FakeEvents) OnContextAugment(chid datatransfer.ChannelID) func(context.Context) context.Context { + return fe.ReturnedOnContextAugmentFunc +} + +func (fe *FakeEvents) ChannelState(ctx context.Context, chid datatransfer.ChannelID) (datatransfer.ChannelState, error) { + return fe.ReturnedChannelState, nil +} diff --git a/testutil/fakegraphsync.go b/transport/graphsync/testharness/fakegraphsync.go similarity index 79% rename from testutil/fakegraphsync.go rename to transport/graphsync/testharness/fakegraphsync.go index 844100cc..7d6be3ec 100644 --- a/testutil/fakegraphsync.go +++ b/transport/graphsync/testharness/fakegraphsync.go @@ -1,4 +1,4 @@ -package testutil +package testharness import ( "context" @@ -19,13 +19,14 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/testutil" "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" ) -func matchDtMessage(t *testing.T, extensions []graphsync.ExtensionData) datatransfer.Message { +func matchDtMessage(t *testing.T, extensions []graphsync.ExtensionData, extName graphsync.ExtensionName) datatransfer.Message { var matchedExtension *graphsync.ExtensionData for _, ext := range extensions { - if ext.Name == extension.ExtensionDataTransfer1_1 { + if ext.Name == extName { matchedExtension = &ext break } @@ -47,9 +48,37 @@ type ReceivedGraphSyncRequest struct { ResponseErrChan chan error } +func (gsRequest ReceivedGraphSyncRequest) ToRequestData(t *testing.T) graphsync.RequestData { + extensions := make(map[graphsync.ExtensionName]datamodel.Node) + for _, extension := range gsRequest.Extensions { + extensions[extension.Name] = extension.Data + } + requestID, ok := gsRequest.requestID() + require.True(t, ok) + return NewFakeRequest(requestID, extensions, graphsync.RequestTypeNew) +} + +func (gsRequest ReceivedGraphSyncRequest) Response(t *testing.T, incomingRequestMsg datatransfer.Message, blockMessage datatransfer.Message, code graphsync.ResponseStatusCode) graphsync.ResponseData { + extensions := make(map[graphsync.ExtensionName]datamodel.Node) + if incomingRequestMsg != nil { + extensions[extension.ExtensionIncomingRequest1_1] = incomingRequestMsg.ToIPLD() + } + if blockMessage != nil { + extensions[extension.ExtensionOutgoingBlock1_1] = blockMessage.ToIPLD() + } + requestID, ok := gsRequest.requestID() + require.True(t, ok) + return NewFakeResponse(requestID, extensions, code) +} + +func (gsRequest ReceivedGraphSyncRequest) requestID() (graphsync.RequestID, bool) { + request, ok := gsRequest.Ctx.Value(graphsync.RequestIDContextKey{}).(graphsync.RequestID) + return request, ok +} + // DTMessage returns the data transfer message among the graphsync extensions sent with this request func (gsRequest ReceivedGraphSyncRequest) DTMessage(t *testing.T) datatransfer.Message { - return matchDtMessage(t, gsRequest.Extensions) + return matchDtMessage(t, gsRequest.Extensions, extension.ExtensionDataTransfer1_1) } type Resume struct { @@ -59,7 +88,7 @@ type Resume struct { // DTMessage returns the data transfer message among the graphsync extensions sent with this request func (resume Resume) DTMessage(t *testing.T) datatransfer.Message { - return matchDtMessage(t, resume.Extensions) + return matchDtMessage(t, resume.Extensions, extension.ExtensionDataTransfer1_1) } type Update struct { @@ -69,16 +98,16 @@ type Update struct { // DTMessage returns the data transfer message among the graphsync extensions sent with this request func (update Update) DTMessage(t *testing.T) datatransfer.Message { - return matchDtMessage(t, update.Extensions) + return matchDtMessage(t, update.Extensions, extension.ExtensionDataTransfer1_1) } // FakeGraphSync implements a GraphExchange but does nothing type FakeGraphSync struct { - requests chan ReceivedGraphSyncRequest // records calls to fakeGraphSync.Request - pauses chan graphsync.RequestID - resumes chan Resume - cancels chan graphsync.RequestID - updates chan Update + ReceivedRequests []ReceivedGraphSyncRequest // records calls to fakeGraphSync.Request + Pauses []graphsync.RequestID + Resumes []Resume + Cancels []graphsync.RequestID + Updates []Update persistenceOptionsLk sync.RWMutex persistenceOptions map[string]ipld.LinkSystem leaveRequestsOpen bool @@ -86,6 +115,7 @@ type FakeGraphSync struct { IncomingBlockHook graphsync.OnIncomingBlockHook OutgoingBlockHook graphsync.OnOutgoingBlockHook IncomingRequestProcessingListener graphsync.OnRequestProcessingListener + OutgoingRequestProcessingListener graphsync.OnRequestProcessingListener IncomingRequestHook graphsync.OnIncomingRequestHook CompletedResponseListener graphsync.OnResponseCompletedListener RequestUpdatedHook graphsync.OnRequestUpdatedHook @@ -99,11 +129,6 @@ type FakeGraphSync struct { // NewFakeGraphSync returns a new fake graphsync implementation func NewFakeGraphSync() *FakeGraphSync { return &FakeGraphSync{ - requests: make(chan ReceivedGraphSyncRequest, 2), - pauses: make(chan graphsync.RequestID, 1), - resumes: make(chan Resume, 1), - cancels: make(chan graphsync.RequestID, 1), - updates: make(chan Update, 1), persistenceOptions: make(map[string]ipld.LinkSystem), } } @@ -112,70 +137,6 @@ func (fgs *FakeGraphSync) LeaveRequestsOpen() { fgs.leaveRequestsOpen = true } -// AssertNoRequestReceived asserts that no requests should ahve been received by this graphsync implementation -func (fgs *FakeGraphSync) AssertNoRequestReceived(t *testing.T) { - require.Empty(t, fgs.requests, "should not receive request") -} - -// AssertRequestReceived asserts a request should be received before the context closes (and returns said request) -func (fgs *FakeGraphSync) AssertRequestReceived(ctx context.Context, t *testing.T) ReceivedGraphSyncRequest { - var requestReceived ReceivedGraphSyncRequest - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case requestReceived = <-fgs.requests: - } - return requestReceived -} - -// AssertNoPauseReceived asserts that no pause requests should ahve been received by this graphsync implementation -func (fgs *FakeGraphSync) AssertNoPauseReceived(t *testing.T) { - require.Empty(t, fgs.pauses, "should not receive pause request") -} - -// AssertPauseReceived asserts a pause request should be received before the context closes (and returns said request) -func (fgs *FakeGraphSync) AssertPauseReceived(ctx context.Context, t *testing.T) graphsync.RequestID { - var pauseReceived graphsync.RequestID - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case pauseReceived = <-fgs.pauses: - } - return pauseReceived -} - -// AssertNoResumeReceived asserts that no resume requests should ahve been received by this graphsync implementation -func (fgs *FakeGraphSync) AssertNoResumeReceived(t *testing.T) { - require.Empty(t, fgs.resumes, "should not receive resume request") -} - -// AssertResumeReceived asserts a resume request should be received before the context closes (and returns said request) -func (fgs *FakeGraphSync) AssertResumeReceived(ctx context.Context, t *testing.T) Resume { - var resumeReceived Resume - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case resumeReceived = <-fgs.resumes: - } - return resumeReceived -} - -// AssertNoCancelReceived asserts that no requests were cancelled by thiss graphsync implementation -func (fgs *FakeGraphSync) AssertNoCancelReceived(t *testing.T) { - require.Empty(t, fgs.cancels, "should not cancel request") -} - -// AssertCancelReceived asserts a requests was cancelled before the context closes (and returns said request id) -func (fgs *FakeGraphSync) AssertCancelReceived(ctx context.Context, t *testing.T) graphsync.RequestID { - var cancelReceived graphsync.RequestID - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case cancelReceived = <-fgs.cancels: - } - return cancelReceived -} - // AssertHasPersistenceOption verifies that a persistence option was registered func (fgs *FakeGraphSync) AssertHasPersistenceOption(t *testing.T, name string) ipld.LinkSystem { fgs.persistenceOptionsLk.RLock() @@ -195,9 +156,9 @@ func (fgs *FakeGraphSync) AssertDoesNotHavePersistenceOption(t *testing.T, name // Request initiates a new GraphSync request to the given peer using the given selector spec. func (fgs *FakeGraphSync) Request(ctx context.Context, p peer.ID, root ipld.Link, selector datamodel.Node, extensions ...graphsync.ExtensionData) (<-chan graphsync.ResponseProgress, <-chan error) { - errors := make(chan error) - responses := make(chan graphsync.ResponseProgress) - fgs.requests <- ReceivedGraphSyncRequest{ctx, p, root, selector, extensions, responses, errors} + errors := make(chan error, 1) + responses := make(chan graphsync.ResponseProgress, 1) + fgs.ReceivedRequests = append(fgs.ReceivedRequests, ReceivedGraphSyncRequest{ctx, p, root, selector, extensions, responses, errors}) if !fgs.leaveRequestsOpen { close(responses) close(errors) @@ -291,18 +252,28 @@ func (fgs *FakeGraphSync) RegisterCompletedResponseListener(listener graphsync.O // Unpause unpauses a request that was paused in a block hook based on request ID func (fgs *FakeGraphSync) Unpause(ctx context.Context, requestID graphsync.RequestID, extensions ...graphsync.ExtensionData) error { - fgs.resumes <- Resume{requestID, extensions} + fgs.Resumes = append(fgs.Resumes, Resume{requestID, extensions}) return nil } // Pause pauses a request based on request ID func (fgs *FakeGraphSync) Pause(ctx context.Context, requestID graphsync.RequestID) error { - fgs.pauses <- requestID + fgs.Pauses = append(fgs.Pauses, requestID) return nil } func (fgs *FakeGraphSync) Cancel(ctx context.Context, requestID graphsync.RequestID) error { - fgs.cancels <- requestID + if fgs.leaveRequestsOpen { + for _, rr := range fgs.ReceivedRequests { + existingRequestID, has := rr.requestID() + if has && requestID.String() == existingRequestID.String() { + close(rr.ResponseChan) + rr.ResponseErrChan <- graphsync.RequestClientCancelledErr{} + close(rr.ResponseErrChan) + } + } + } + fgs.Cancels = append(fgs.Cancels, requestID) return nil } @@ -342,22 +313,25 @@ func (fgs *FakeGraphSync) Stats() graphsync.Stats { return graphsync.Stats{} } -func (fgs *FakeGraphSync) RegisterOutgoingRequestProcessingListener(graphsync.OnRequestProcessingListener) graphsync.UnregisterHookFunc { - // TODO: just a stub for now, hopefully nobody needs this - return func() {} +func (fgs *FakeGraphSync) RegisterOutgoingRequestProcessingListener(listener graphsync.OnRequestProcessingListener) graphsync.UnregisterHookFunc { + fgs.OutgoingRequestProcessingListener = listener + return func() { + fgs.OutgoingRequestProcessingListener = nil + } } func (fgs *FakeGraphSync) SendUpdate(ctx context.Context, id graphsync.RequestID, extensions ...graphsync.ExtensionData) error { - fgs.updates <- Update{RequestID: id, Extensions: extensions} + fgs.Updates = append(fgs.Updates, Update{RequestID: id, Extensions: extensions}) return nil } var _ graphsync.GraphExchange = &FakeGraphSync{} type fakeBlkData struct { - link ipld.Link - size uint64 - index int64 + link ipld.Link + size uint64 + onWire bool + index int64 } func (fbd fakeBlkData) Link() ipld.Link { @@ -369,7 +343,10 @@ func (fbd fakeBlkData) BlockSize() uint64 { } func (fbd fakeBlkData) BlockSizeOnWire() uint64 { - return fbd.size + if fbd.onWire { + return fbd.size + } + return 0 } func (fbd fakeBlkData) Index() int64 { @@ -377,11 +354,12 @@ func (fbd fakeBlkData) Index() int64 { } // NewFakeBlockData returns a fake block that matches the block data interface -func NewFakeBlockData() graphsync.BlockData { +func NewFakeBlockData(size uint64, index int64, onWire bool) graphsync.BlockData { return &fakeBlkData{ - link: cidlink.Link{Cid: GenerateCids(1)[0]}, - size: rand.Uint64(), - index: int64(rand.Uint32()), + link: cidlink.Link{Cid: testutil.GenerateCids(1)[0]}, + size: size, + index: index, + onWire: onWire, } } @@ -427,14 +405,14 @@ func (fr *fakeRequest) Type() graphsync.RequestType { } // NewFakeRequest returns a fake request that matches the request data interface -func NewFakeRequest(id graphsync.RequestID, extensions map[graphsync.ExtensionName]datamodel.Node) graphsync.RequestData { +func NewFakeRequest(id graphsync.RequestID, extensions map[graphsync.ExtensionName]datamodel.Node, requestType graphsync.RequestType) graphsync.RequestData { return &fakeRequest{ id: id, - root: GenerateCids(1)[0], + root: testutil.GenerateCids(1)[0], selector: selectorparse.CommonSelector_ExploreAllRecursively, priority: graphsync.Priority(rand.Int()), extensions: extensions, - requestType: graphsync.RequestTypeNew, + requestType: requestType, } } @@ -563,6 +541,18 @@ func (fa *FakeIncomingRequestHookActions) AugmentContext(ctxAugFunc func(reqCtx fa.CtxAugFuncs = append(fa.CtxAugFuncs, ctxAugFunc) } +func (fa *FakeIncomingRequestHookActions) AssertAugmentedContextKey(t *testing.T, key interface{}, value interface{}) { + ctx := context.Background() + for _, f := range fa.CtxAugFuncs { + ctx = f(ctx) + } + require.Equal(t, value, ctx.Value(key)) +} + +func (fa *FakeIncomingRequestHookActions) DTMessage(t *testing.T) datatransfer.Message { + return matchDtMessage(t, fa.SentExtensions, extension.ExtensionIncomingRequest1_1) +} + var _ graphsync.IncomingRequestHookActions = &FakeIncomingRequestHookActions{} type FakeRequestUpdatedActions struct { diff --git a/transport/graphsync/testharness/harness.go b/transport/graphsync/testharness/harness.go new file mode 100644 index 00000000..b68e9499 --- /dev/null +++ b/transport/graphsync/testharness/harness.go @@ -0,0 +1,284 @@ +package testharness + +import ( + "context" + "math/rand" + "testing" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/message/types" + "github.com/filecoin-project/go-data-transfer/v2/testutil" + dtgs "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" + "github.com/ipfs/go-graphsync" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" + "github.com/stretchr/testify/require" +) + +type harnessConfig struct { + isPull bool + isResponder bool + makeEvents func(gsData *GsTestHarness) *FakeEvents + makeNetwork func(gsData *GsTestHarness) *FakeNetwork + transportOptions []dtgs.Option +} + +type Option func(*harnessConfig) + +func PullRequest() Option { + return func(hc *harnessConfig) { + hc.isPull = true + } +} + +func Responder() Option { + return func(hc *harnessConfig) { + hc.isResponder = true + } +} + +func Events(makeEvents func(gsData *GsTestHarness) *FakeEvents) Option { + return func(hc *harnessConfig) { + hc.makeEvents = makeEvents + } +} + +func Network(makeNetwork func(gsData *GsTestHarness) *FakeNetwork) Option { + return func(hc *harnessConfig) { + hc.makeNetwork = makeNetwork + } +} + +func TransportOptions(options []dtgs.Option) Option { + return func(hc *harnessConfig) { + hc.transportOptions = options + } +} + +func SetupHarness(ctx context.Context, options ...Option) *GsTestHarness { + hc := &harnessConfig{} + for _, option := range options { + option(hc) + } + peers := testutil.GeneratePeers(2) + transferID := datatransfer.TransferID(rand.Uint32()) + fgs := NewFakeGraphSync() + fgs.LeaveRequestsOpen() + voucher := testutil.NewTestTypedVoucher() + baseCid := testutil.GenerateCids(1)[0] + selector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() + chid := datatransfer.ChannelID{Initiator: peers[0], Responder: peers[1], ID: transferID} + if hc.isResponder { + chid = datatransfer.ChannelID{Initiator: peers[1], Responder: peers[0], ID: transferID} + } + channel := testutil.NewMockChannelState(testutil.MockChannelStateParams{ + BaseCID: baseCid, + Voucher: voucher, + Selector: selector, + IsPull: hc.isPull, + Self: peers[0], + ChannelID: chid, + }) + gsData := &GsTestHarness{ + Ctx: ctx, + Fgs: fgs, + Channel: channel, + CompletedRequests: make(chan datatransfer.ChannelID, 16), + CompletedResponses: make(chan datatransfer.ChannelID, 16), + OutgoingRequestHookActions: &FakeOutgoingRequestHookActions{}, + OutgoingBlockHookActions: &FakeOutgoingBlockHookActions{}, + IncomingBlockHookActions: &FakeIncomingBlockHookActions{}, + IncomingRequestHookActions: &FakeIncomingRequestHookActions{}, + RequestUpdateHookActions: &FakeRequestUpdatedActions{}, + IncomingResponseHookActions: &FakeIncomingResponseHookActions{}, + } + if hc.makeEvents != nil { + gsData.Events = hc.makeEvents(gsData) + } else { + gsData.Events = &FakeEvents{ + ReturnedChannelState: channel, + } + } + if hc.makeNetwork != nil { + gsData.DtNet = hc.makeNetwork(gsData) + } else { + gsData.DtNet = NewFakeNetwork(peers[0]) + } + gsData.Transport = dtgs.NewTransport(gsData.Fgs, gsData.DtNet, + append(hc.transportOptions, + dtgs.RegisterCompletedRequestListener(gsData.completedRequestListener), + dtgs.RegisterCompletedResponseListener(gsData.completedResponseListener))...) + gsData.Transport.SetEventHandler(gsData.Events) + return gsData +} + +type GsTestHarness struct { + Ctx context.Context + Fgs *FakeGraphSync + Channel *testutil.MockChannelState + RequestID graphsync.RequestID + AltRequestID graphsync.RequestID + Events *FakeEvents + DtNet *FakeNetwork + OutgoingRequestHookActions *FakeOutgoingRequestHookActions + IncomingBlockHookActions *FakeIncomingBlockHookActions + OutgoingBlockHookActions *FakeOutgoingBlockHookActions + IncomingRequestHookActions *FakeIncomingRequestHookActions + RequestUpdateHookActions *FakeRequestUpdatedActions + IncomingResponseHookActions *FakeIncomingResponseHookActions + Transport *dtgs.Transport + CompletedRequests chan datatransfer.ChannelID + CompletedResponses chan datatransfer.ChannelID +} + +func (th *GsTestHarness) completedRequestListener(chid datatransfer.ChannelID) { + th.CompletedRequests <- chid +} +func (th *GsTestHarness) completedResponseListener(chid datatransfer.ChannelID) { + th.CompletedResponses <- chid +} + +func (th *GsTestHarness) NewRequest(t *testing.T) datatransfer.Request { + vouch := th.Channel.Voucher() + message, err := message.NewRequest(th.Channel.TransferID(), false, th.Channel.IsPull(), &vouch, th.Channel.BaseCID(), th.Channel.Selector()) + require.NoError(t, err) + return message +} + +func (th *GsTestHarness) RestartRequest(t *testing.T) datatransfer.Request { + vouch := th.Channel.Voucher() + message, err := message.NewRequest(th.Channel.TransferID(), true, th.Channel.IsPull(), &vouch, th.Channel.BaseCID(), th.Channel.Selector()) + require.NoError(t, err) + return message +} + +func (th *GsTestHarness) VoucherRequest() datatransfer.Request { + newVouch := testutil.NewTestTypedVoucher() + return message.VoucherRequest(th.Channel.TransferID(), &newVouch) +} + +func (th *GsTestHarness) UpdateRequest(pause bool) datatransfer.Request { + return message.UpdateRequest(th.Channel.TransferID(), pause) +} + +func (th *GsTestHarness) Response() datatransfer.Response { + voucherResult := testutil.NewTestTypedVoucher() + return message.NewResponse(th.Channel.TransferID(), true, false, &voucherResult) +} + +func (th *GsTestHarness) ValidationResultResponse(pause bool) datatransfer.Response { + voucherResult := testutil.NewTestTypedVoucher() + return message.ValidationResultResponse(types.VoucherResultMessage, th.Channel.TransferID(), datatransfer.ValidationResult{VoucherResult: &voucherResult, Accepted: true}, nil, pause) +} + +func (th *GsTestHarness) RestartResponse(pause bool) datatransfer.Response { + voucherResult := testutil.NewTestTypedVoucher() + return message.ValidationResultResponse(types.RestartMessage, th.Channel.TransferID(), datatransfer.ValidationResult{VoucherResult: &voucherResult, Accepted: true}, nil, pause) +} + +func (th *GsTestHarness) UpdateResponse(paused bool) datatransfer.Response { + return message.UpdateResponse(th.Channel.TransferID(), true) +} + +func (th *GsTestHarness) OutgoingRequestHook(request graphsync.RequestData) { + th.Fgs.OutgoingRequestHook(th.Channel.OtherPeer(), request, th.OutgoingRequestHookActions) +} + +func (th *GsTestHarness) OutgoingRequestProcessingListener(request graphsync.RequestData) { + th.Fgs.OutgoingRequestProcessingListener(th.Channel.OtherPeer(), request, 0) +} + +func (th *GsTestHarness) IncomingBlockHook(response graphsync.ResponseData, block graphsync.BlockData) { + th.Fgs.IncomingBlockHook(th.Channel.OtherPeer(), response, block, th.IncomingBlockHookActions) +} + +func (th *GsTestHarness) OutgoingBlockHook(request graphsync.RequestData, block graphsync.BlockData) { + th.Fgs.OutgoingBlockHook(th.Channel.OtherPeer(), request, block, th.OutgoingBlockHookActions) +} + +func (th *GsTestHarness) IncomingRequestHook(request graphsync.RequestData) { + th.Fgs.IncomingRequestHook(th.Channel.OtherPeer(), request, th.IncomingRequestHookActions) +} + +func (th *GsTestHarness) IncomingRequestProcessingListener(request graphsync.RequestData) { + th.Fgs.IncomingRequestProcessingListener(th.Channel.OtherPeer(), request, 1) +} + +func (th *GsTestHarness) IncomingResponseHook(response graphsync.ResponseData) { + th.Fgs.IncomingResponseHook(th.Channel.OtherPeer(), response, th.IncomingResponseHookActions) +} + +func (th *GsTestHarness) ResponseCompletedListener(request graphsync.RequestData, code graphsync.ResponseStatusCode) { + th.Fgs.CompletedResponseListener(th.Channel.OtherPeer(), request, code) +} + +func (th *GsTestHarness) RequestorCancelledListener(request graphsync.RequestData) { + th.Fgs.RequestorCancelledListener(th.Channel.OtherPeer(), request) +} + +/* +func (ha *GsTestHarness) networkErrorListener(err error) { + ha.Fgs.NetworkErrorListener(ha.other, ha.request, err) +} +func (ha *GsTestHarness) receiverNetworkErrorListener(err error) { + ha.Fgs.ReceiverNetworkErrorListener(ha.other, err) +} +*/ + +func (th *GsTestHarness) BlockSentListener(request graphsync.RequestData, block graphsync.BlockData) { + th.Fgs.BlockSentListener(th.Channel.OtherPeer(), request, block) +} + +func (ha *GsTestHarness) makeRequest(requestID graphsync.RequestID, messageNode datamodel.Node, requestType graphsync.RequestType) graphsync.RequestData { + extensions := make(map[graphsync.ExtensionName]datamodel.Node) + if messageNode != nil { + extensions[extension.ExtensionDataTransfer1_1] = messageNode + } + return NewFakeRequest(requestID, extensions, requestType) +} + +func (ha *GsTestHarness) makeResponse(requestID graphsync.RequestID, messageNode datamodel.Node, responseCode graphsync.ResponseStatusCode) graphsync.ResponseData { + extensions := make(map[graphsync.ExtensionName]datamodel.Node) + if messageNode != nil { + extensions[extension.ExtensionDataTransfer1_1] = messageNode + } + return NewFakeResponse(requestID, extensions, responseCode) +} + +func assertDecodesToMessage(t *testing.T, data datamodel.Node, expected datatransfer.Message) { + actual, err := message.FromIPLD(data) + require.NoError(t, err) + require.Equal(t, expected, actual) +} + +func assertHasOutgoingMessage(t *testing.T, extensions []graphsync.ExtensionData, expected datatransfer.Message) { + nd := expected.ToIPLD() + found := false + for _, e := range extensions { + if e.Name == extension.ExtensionDataTransfer1_1 { + require.True(t, ipld.DeepEqual(nd, e.Data), "data matches") + found = true + } + } + if !found { + require.Fail(t, "extension not found") + } +} + +func assertHasExtensionMessage(t *testing.T, name graphsync.ExtensionName, extensions []graphsync.ExtensionData, expected datatransfer.Message) { + nd := expected.ToIPLD() + found := false + for _, e := range extensions { + if e.Name == name { + require.True(t, ipld.DeepEqual(nd, e.Data), "data matches") + found = true + } + } + if !found { + require.Fail(t, "extension not found") + } +} diff --git a/transport/graphsync/testharness/testnet.go b/transport/graphsync/testharness/testnet.go new file mode 100644 index 00000000..14ab118b --- /dev/null +++ b/transport/graphsync/testharness/testnet.go @@ -0,0 +1,106 @@ +package testharness + +import ( + "context" + "testing" + + "github.com/libp2p/go-libp2p-core/peer" + "github.com/stretchr/testify/require" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/transport/helpers/network" +) + +// FakeSentMessage is a recording of a message sent on the FakeNetwork +type FakeSentMessage struct { + PeerID peer.ID + TransportID datatransfer.TransportID + Message datatransfer.Message +} + +type FakeDelegates struct { + TransportID datatransfer.TransportID + Versions []datatransfer.Version + Receiver network.Receiver +} + +type ConnectWithRetryAttempt struct { + PeerID peer.ID + TransportID datatransfer.TransportID +} + +type TaggedPeer struct { + PeerID peer.ID + Tag string +} + +// FakeNetwork is a network that satisfies the DataTransferNetwork interface but +// does not actually do anything +type FakeNetwork struct { + SentMessages []FakeSentMessage + Delegates []FakeDelegates + ConnectWithRetryAttempts []ConnectWithRetryAttempt + ProtectedPeers []TaggedPeer + UnprotectedPeers []TaggedPeer + + ReturnedPeerDescription network.ProtocolDescription + ReturnedPeerID peer.ID + ReturnedSendMessageError error + ReturnedConnectWithRetryError error +} + +// NewFakeNetwork returns a new fake data transfer network instance +func NewFakeNetwork(id peer.ID) *FakeNetwork { + return &FakeNetwork{ReturnedPeerID: id} +} + +var _ network.DataTransferNetwork = (*FakeNetwork)(nil) + +// SendMessage sends a GraphSync message to a peer. +func (fn *FakeNetwork) SendMessage(ctx context.Context, + p peer.ID, + t datatransfer.TransportID, + m datatransfer.Message) error { + fn.SentMessages = append(fn.SentMessages, FakeSentMessage{p, t, m}) + return fn.ReturnedSendMessageError +} + +// SetDelegate registers the Reciver to handle messages received from the +// network. +func (fn *FakeNetwork) SetDelegate(t datatransfer.TransportID, v []datatransfer.Version, r network.Receiver) { + fn.Delegates = append(fn.Delegates, FakeDelegates{t, v, r}) +} + +// ConnectTo establishes a connection to the given peer +func (fn *FakeNetwork) ConnectTo(_ context.Context, _ peer.ID) error { + return nil +} + +func (fn *FakeNetwork) ConnectWithRetry(ctx context.Context, p peer.ID, transportID datatransfer.TransportID) error { + fn.ConnectWithRetryAttempts = append(fn.ConnectWithRetryAttempts, ConnectWithRetryAttempt{p, transportID}) + return fn.ReturnedConnectWithRetryError +} + +// ID returns a stubbed id for host of this network +func (fn *FakeNetwork) ID() peer.ID { + return fn.ReturnedPeerID +} + +// Protect does nothing on the fake network +func (fn *FakeNetwork) Protect(id peer.ID, tag string) { + fn.ProtectedPeers = append(fn.ProtectedPeers, TaggedPeer{id, tag}) +} + +// Unprotect does nothing on the fake network +func (fn *FakeNetwork) Unprotect(id peer.ID, tag string) bool { + fn.UnprotectedPeers = append(fn.UnprotectedPeers, TaggedPeer{id, tag}) + return false +} + +func (fn *FakeNetwork) Protocol(ctx context.Context, id peer.ID, transportID datatransfer.TransportID) (network.ProtocolDescription, error) { + return fn.ReturnedPeerDescription, nil +} + +func (fn *FakeNetwork) AssertSentMessage(t *testing.T, sentMessage FakeSentMessage) { + require.Contains(t, fn.SentMessages, sentMessage) +} diff --git a/transport/helpers/network/libp2p_impl_test.go b/transport/helpers/network/libp2p_impl_test.go index 76b59ef6..1b689004 100644 --- a/transport/helpers/network/libp2p_impl_test.go +++ b/transport/helpers/network/libp2p_impl_test.go @@ -132,8 +132,7 @@ func TestMessageSendAndReceive(t *testing.T) { accepted := false id := datatransfer.TransferID(rand.Int31()) voucherResult := testutil.NewTestTypedVoucher() - response, err := message.ValidationResultResponse(types.NewMessage, id, datatransfer.ValidationResult{Accepted: accepted, VoucherResult: &voucherResult}, nil, false) - require.NoError(t, err) + response := message.ValidationResultResponse(types.NewMessage, id, datatransfer.ValidationResult{Accepted: accepted, VoucherResult: &voucherResult}, nil, false) require.NoError(t, dtnet2.SendMessage(ctx, host1.ID(), "graphsync", response)) select { diff --git a/types.go b/types.go index 5723a9fd..e97c8038 100644 --- a/types.go +++ b/types.go @@ -146,6 +146,18 @@ type ChannelState interface { // be left open for a final settlement RequiresFinalization() bool + // InitiatorPaused indicates whether the initiator of this channel is in a paused state + InitiatorPaused() bool + + // ResponderPaused indicates whether the responder of this channel is in a paused state + ResponderPaused() bool + + // BothPaused indicates both sides of the transfer have paused the transfer + BothPaused() bool + + // SelfPaused indicates whether the local peer for this channel is in a paused state + SelfPaused() bool + // Stages returns the timeline of events this data transfer has gone through, // for observability purposes. // From c0ede22e197a885d9e90ea8eb9eaf337b4f37067 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 30 Jun 2022 01:38:10 -0700 Subject: [PATCH 13/18] style(lint): fix imports --- channels/internal/migrations/migrations.go | 5 +++-- impl/events.go | 3 ++- transport/graphsync/initiating_test.go | 7 ++++--- transport/graphsync/responding_test.go | 7 ++++--- transport/graphsync/testharness/events.go | 3 ++- transport/graphsync/testharness/harness.go | 13 +++++++------ 6 files changed, 22 insertions(+), 16 deletions(-) diff --git a/channels/internal/migrations/migrations.go b/channels/internal/migrations/migrations.go index a1f263f2..7e0a16f6 100644 --- a/channels/internal/migrations/migrations.go +++ b/channels/internal/migrations/migrations.go @@ -5,10 +5,11 @@ import ( basicnode "github.com/ipld/go-ipld-prime/node/basic" peer "github.com/libp2p/go-libp2p-core/peer" - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/channels/internal" versioning "github.com/filecoin-project/go-ds-versioning/pkg" "github.com/filecoin-project/go-ds-versioning/pkg/versioned" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/channels/internal" ) //go:generate cbor-gen-for --map-encoding ChannelStateV2 diff --git a/impl/events.go b/impl/events.go index a7c680fe..9cdc131a 100644 --- a/impl/events.go +++ b/impl/events.go @@ -5,10 +5,11 @@ import ( "errors" "fmt" + "golang.org/x/xerrors" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channels" "github.com/filecoin-project/go-data-transfer/v2/message" - "golang.org/x/xerrors" ) // OnChannelOpened is called when we send a request for data to the other diff --git a/transport/graphsync/initiating_test.go b/transport/graphsync/initiating_test.go index 952ccab6..ff956f27 100644 --- a/transport/graphsync/initiating_test.go +++ b/transport/graphsync/initiating_test.go @@ -6,14 +6,15 @@ import ( "testing" "time" - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" - "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/testharness" "github.com/ipfs/go-graphsync" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/stretchr/testify/require" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/testharness" ) func TestInitiatingPullRequestSuccessFlow(t *testing.T) { diff --git a/transport/graphsync/responding_test.go b/transport/graphsync/responding_test.go index 07d70184..93343574 100644 --- a/transport/graphsync/responding_test.go +++ b/transport/graphsync/responding_test.go @@ -6,14 +6,15 @@ import ( "testing" "time" - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" - "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/testharness" "github.com/ipfs/go-graphsync" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/stretchr/testify/require" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/testharness" ) func TestRespondingPullSuccessFlow(t *testing.T) { diff --git a/transport/graphsync/testharness/events.go b/transport/graphsync/testharness/events.go index 17f4fd3d..dd37635a 100644 --- a/transport/graphsync/testharness/events.go +++ b/transport/graphsync/testharness/events.go @@ -4,8 +4,9 @@ import ( "context" "testing" - datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/stretchr/testify/require" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" ) type ReceivedTransportEvent struct { diff --git a/transport/graphsync/testharness/harness.go b/transport/graphsync/testharness/harness.go index b68e9499..c545ad8b 100644 --- a/transport/graphsync/testharness/harness.go +++ b/transport/graphsync/testharness/harness.go @@ -5,18 +5,19 @@ import ( "math/rand" "testing" - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/message" - "github.com/filecoin-project/go-data-transfer/v2/message/types" - "github.com/filecoin-project/go-data-transfer/v2/testutil" - dtgs "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" - "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" "github.com/ipfs/go-graphsync" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/stretchr/testify/require" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/message/types" + "github.com/filecoin-project/go-data-transfer/v2/testutil" + dtgs "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" ) type harnessConfig struct { From 2ac4a2b51e50c344c9f6fdc008710898f030eefe Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 30 Jun 2022 02:53:21 -0700 Subject: [PATCH 14/18] style(mod): tidy --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 988d4813..608fa39f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/filecoin-project/go-data-transfer/v2 -go 1.18 +go 1.17 require ( github.com/bep/debounce v1.2.0 diff --git a/go.sum b/go.sum index 494c158a..7da44a75 100644 --- a/go.sum +++ b/go.sum @@ -147,6 +147,7 @@ github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= @@ -1101,6 +1102,7 @@ github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBO github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1183,6 +1185,7 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg= github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= From 1188d126ebe5d3c284640490f30b53710763df9b Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Mon, 18 Jul 2022 15:29:57 -0700 Subject: [PATCH 15/18] Update statuses.go Co-authored-by: Rod Vagg --- statuses.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/statuses.go b/statuses.go index 6e915373..3b263de0 100644 --- a/statuses.go +++ b/statuses.go @@ -65,7 +65,9 @@ const ( Queued // AwaitingAcceptance indicates a transfer request is actively being processed by the transport - // even if the remote has not yet responded that it's accepted the transfer + // even if the remote has not yet responded that it's accepted the transfer. Such a state can + // occur, for example, in a requestor-initiated transfer that starts processing prior to receiving + // acceptance from the server. AwaitingAcceptance ) From 69fa13655c62f98f5847fd1b76b1071955ab18f2 Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Tue, 19 Jul 2022 10:59:31 -0700 Subject: [PATCH 16/18] Update impl/events.go Co-authored-by: Rod Vagg --- impl/events.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/impl/events.go b/impl/events.go index 9cdc131a..c648c23e 100644 --- a/impl/events.go +++ b/impl/events.go @@ -12,8 +12,7 @@ import ( "github.com/filecoin-project/go-data-transfer/v2/message" ) -// OnChannelOpened is called when we send a request for data to the other -// peer on the given channel ID +// OnTransportEvent is dispatched when an event occurs on the transport func (m *manager) OnTransportEvent(chid datatransfer.ChannelID, evt datatransfer.TransportEvent) { ctx, _ := m.spansIndex.SpanForChannel(context.TODO(), chid) err := m.processTransferEvent(ctx, chid, evt) From f106e6eda2835df824e5b7cdcaf64bd81a6f86db Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Tue, 19 Jul 2022 15:57:03 -0700 Subject: [PATCH 17/18] test(graphsync): add additional tests add additional tests, remove commented out code, improve migrations --- channels/channels.go | 2 +- channels/channels_test.go | 154 ++++++++++ channels/internal/migrations/migrations.go | 10 +- impl/events.go | 16 - transport.go | 54 ++-- transport/graphsync/exceptions_test.go | 286 ++++++++++++++++++ transport/graphsync/graphsync.go | 4 +- transport/graphsync/hooks.go | 12 +- transport/graphsync/initiating_test.go | 2 +- transport/graphsync/receiver.go | 12 +- transport/graphsync/testharness/events.go | 12 + .../graphsync/testharness/fakegraphsync.go | 20 +- 12 files changed, 519 insertions(+), 65 deletions(-) create mode 100644 transport/graphsync/exceptions_test.go diff --git a/channels/channels.go b/channels/channels.go index f2139c59..bc00c420 100644 --- a/channels/channels.go +++ b/channels/channels.go @@ -58,7 +58,7 @@ func New(ds datastore.Batching, StateEntryFuncs: ChannelStateEntryFuncs, Notifier: c.dispatch, FinalityStates: ChannelFinalityStates, - }, channelMigrations, versioning.VersionKey("2")) + }, channelMigrations, versioning.VersionKey("3")) if err != nil { return nil, err } diff --git a/channels/channels_test.go b/channels/channels_test.go index b6863cc8..c62311a2 100644 --- a/channels/channels_test.go +++ b/channels/channels_test.go @@ -1,11 +1,15 @@ package channels_test import ( + "bytes" "context" "errors" + "math/rand" "testing" "time" + versionedds "github.com/filecoin-project/go-ds-versioning/pkg/datastore" + "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" dss "github.com/ipfs/go-datastore/sync" basicnode "github.com/ipld/go-ipld-prime/node/basic" @@ -16,7 +20,10 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channels" + "github.com/filecoin-project/go-data-transfer/v2/channels/internal" + "github.com/filecoin-project/go-data-transfer/v2/channels/internal/migrations" "github.com/filecoin-project/go-data-transfer/v2/testutil" + versioning "github.com/filecoin-project/go-ds-versioning/pkg" ) func TestChannels(t *testing.T) { @@ -392,6 +399,153 @@ func TestIsChannelCleaningUp(t *testing.T) { require.False(t, channels.IsChannelCleaningUp(datatransfer.Cancelled)) } +func TestMigrations(t *testing.T) { + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + ds := dss.MutexWrap(datastore.NewMapDatastore()) + received := make(chan event) + notifier := func(evt datatransfer.Event, chst datatransfer.ChannelState) { + received <- event{evt, chst} + } + numChannels := 5 + transferIDs := make([]datatransfer.TransferID, numChannels) + initiators := make([]peer.ID, numChannels) + responders := make([]peer.ID, numChannels) + baseCids := make([]cid.Cid, numChannels) + + totalSizes := make([]uint64, numChannels) + sents := make([]uint64, numChannels) + receiveds := make([]uint64, numChannels) + + messages := make([]string, numChannels) + vouchers := make([]datatransfer.TypedVoucher, numChannels) + voucherResults := make([]datatransfer.TypedVoucher, numChannels) + sentIndex := make([]int64, numChannels) + receivedIndex := make([]int64, numChannels) + queuedIndex := make([]int64, numChannels) + allSelector := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any).Matcher().Node() + selfPeer := testutil.GeneratePeers(1)[0] + + list, err := migrations.GetChannelStateMigrations(selfPeer) + require.NoError(t, err) + vds, up := versionedds.NewVersionedDatastore(ds, list, versioning.VersionKey("2")) + require.NoError(t, up(ctx)) + + initialStatuses := []datatransfer.Status{ + datatransfer.Requested, + datatransfer.InitiatorPaused, + datatransfer.ResponderPaused, + datatransfer.BothPaused, + datatransfer.Ongoing, + } + for i := 0; i < numChannels; i++ { + transferIDs[i] = datatransfer.TransferID(rand.Uint64()) + initiators[i] = testutil.GeneratePeers(1)[0] + responders[i] = testutil.GeneratePeers(1)[0] + baseCids[i] = testutil.GenerateCids(1)[0] + totalSizes[i] = rand.Uint64() + sents[i] = rand.Uint64() + receiveds[i] = rand.Uint64() + messages[i] = string(testutil.RandomBytes(20)) + vouchers[i] = testutil.NewTestTypedVoucher() + voucherResults[i] = testutil.NewTestTypedVoucher() + sentIndex[i] = rand.Int63() + receivedIndex[i] = rand.Int63() + queuedIndex[i] = rand.Int63() + channel := migrations.ChannelStateV2{ + TransferID: transferIDs[i], + Initiator: initiators[i], + Responder: responders[i], + BaseCid: baseCids[i], + Selector: internal.CborGenCompatibleNode{ + Node: allSelector, + }, + Sender: initiators[i], + Recipient: responders[i], + TotalSize: totalSizes[i], + Status: initialStatuses[i], + Sent: sents[i], + Received: receiveds[i], + Message: messages[i], + Vouchers: []internal.EncodedVoucher{ + { + Type: vouchers[i].Type, + Voucher: internal.CborGenCompatibleNode{ + Node: vouchers[i].Voucher, + }, + }, + }, + VoucherResults: []internal.EncodedVoucherResult{ + { + Type: voucherResults[i].Type, + VoucherResult: internal.CborGenCompatibleNode{ + Node: voucherResults[i].Voucher, + }, + }, + }, + SentBlocksTotal: sentIndex[i], + ReceivedBlocksTotal: receivedIndex[i], + QueuedBlocksTotal: queuedIndex[i], + SelfPeer: selfPeer, + } + buf := new(bytes.Buffer) + err = channel.MarshalCBOR(buf) + require.NoError(t, err) + err = vds.Put(ctx, datastore.NewKey(datatransfer.ChannelID{ + Initiator: initiators[i], + Responder: responders[i], + ID: transferIDs[i], + }.String()), buf.Bytes()) + require.NoError(t, err) + } + + channelList, err := channels.New(ds, notifier, &fakeEnv{}, selfPeer) + require.NoError(t, err) + err = channelList.Start(ctx) + require.NoError(t, err) + + expectedStatuses := []datatransfer.Status{ + datatransfer.Requested, + datatransfer.Ongoing, + datatransfer.Ongoing, + datatransfer.Ongoing, + datatransfer.Ongoing, + } + + expectedInitiatorPaused := []bool{false, true, false, true, false} + expectedResponderPaused := []bool{false, false, true, true, false} + for i := 0; i < numChannels; i++ { + + channel, err := channelList.GetByID(ctx, datatransfer.ChannelID{ + Initiator: initiators[i], + Responder: responders[i], + ID: transferIDs[i], + }) + require.NoError(t, err) + require.Equal(t, selfPeer, channel.SelfPeer()) + require.Equal(t, transferIDs[i], channel.TransferID()) + require.Equal(t, baseCids[i], channel.BaseCID()) + require.Equal(t, allSelector, channel.Selector()) + require.Equal(t, initiators[i], channel.Sender()) + require.Equal(t, responders[i], channel.Recipient()) + require.Equal(t, totalSizes[i], channel.TotalSize()) + require.Equal(t, sents[i], channel.Sent()) + require.Equal(t, receiveds[i], channel.Received()) + require.Equal(t, messages[i], channel.Message()) + require.Equal(t, vouchers[i], channel.LastVoucher()) + require.Equal(t, voucherResults[i], channel.LastVoucherResult()) + require.Equal(t, expectedStatuses[i], channel.Status()) + require.Equal(t, expectedInitiatorPaused[i], channel.InitiatorPaused()) + require.Equal(t, expectedResponderPaused[i], channel.ResponderPaused()) + require.Equal(t, basicnode.NewInt(sentIndex[i]), channel.SentIndex()) + require.Equal(t, basicnode.NewInt(receivedIndex[i]), channel.ReceivedIndex()) + require.Equal(t, basicnode.NewInt(queuedIndex[i]), channel.QueuedIndex()) + + } +} + type event struct { event datatransfer.Event state datatransfer.ChannelState diff --git a/channels/internal/migrations/migrations.go b/channels/internal/migrations/migrations.go index 7e0a16f6..210dfc00 100644 --- a/channels/internal/migrations/migrations.go +++ b/channels/internal/migrations/migrations.go @@ -76,6 +76,12 @@ func MigrateChannelState2To3(oldChannelState *ChannelStateV2) (*internal.Channel sentIndex := basicnode.NewInt(oldChannelState.SentBlocksTotal) queuedIndex := basicnode.NewInt(oldChannelState.QueuedBlocksTotal) + responderPaused := oldChannelState.Status == datatransfer.ResponderPaused || oldChannelState.Status == datatransfer.BothPaused + initiatorPaused := oldChannelState.Status == datatransfer.InitiatorPaused || oldChannelState.Status == datatransfer.BothPaused + newStatus := oldChannelState.Status + if newStatus == datatransfer.ResponderPaused || newStatus == datatransfer.InitiatorPaused || newStatus == datatransfer.BothPaused { + newStatus = datatransfer.Ongoing + } return &internal.ChannelState{ SelfPeer: oldChannelState.SelfPeer, TransferID: oldChannelState.TransferID, @@ -86,7 +92,7 @@ func MigrateChannelState2To3(oldChannelState *ChannelStateV2) (*internal.Channel Sender: oldChannelState.Sender, Recipient: oldChannelState.Recipient, TotalSize: oldChannelState.TotalSize, - Status: oldChannelState.Status, + Status: newStatus, Queued: oldChannelState.Queued, Sent: oldChannelState.Sent, Received: oldChannelState.Received, @@ -98,6 +104,8 @@ func MigrateChannelState2To3(oldChannelState *ChannelStateV2) (*internal.Channel QueuedIndex: internal.CborGenCompatibleNode{Node: queuedIndex}, DataLimit: oldChannelState.DataLimit, RequiresFinalization: oldChannelState.RequiresFinalization, + InitiatorPaused: initiatorPaused, + ResponderPaused: responderPaused, Stages: oldChannelState.Stages, }, nil } diff --git a/impl/events.go b/impl/events.go index c648c23e..af2085f1 100644 --- a/impl/events.go +++ b/impl/events.go @@ -43,22 +43,6 @@ func (m *manager) processTransferEvent(ctx context.Context, chid datatransfer.Ch } msg := message.UpdateResponse(chid.ID, true) return m.transport.SendMessage(ctx, chid, msg) - /*case datatransfer.TransportReceivedVoucherRequest: - voucher, err := evt.Request.TypedVoucher() - if err != nil { - return err - } - return m.channels.NewVoucher(chid, voucher) - case datatransfer.TransportReceivedUpdateRequest: - if evt.Request.IsPaused() { - return m.pauseOther(chid) - } - return m.resumeOther(chid) - case datatransfer.TransportReceivedCancelRequest: - log.Infof("channel %s: received cancel request, cleaning up channel", chid) - return m.channels.Cancel(chid) - case datatransfer.TransportReceivedResponse: - return m.receiveResponse(chid, evt.Response)*/ case datatransfer.TransportTransferCancelled: log.Warnf("channel %+v was cancelled: %s", chid, evt.ErrorMessage) return m.channels.RequestCancelled(chid, errors.New(evt.ErrorMessage)) diff --git a/transport.go b/transport.go index 13b860ee..91aebdfc 100644 --- a/transport.go +++ b/transport.go @@ -52,25 +52,6 @@ type TransportQueuedData struct { // TransportReachedDataLimit occurs when a channel hits a previously set data limit type TransportReachedDataLimit struct{} -/* -type TransportReceivedVoucherRequest struct { - Request Request -} - -type TransportReceivedUpdateRequest struct { - Request Request -} - -type TransportReceivedCancelRequest struct { - Request Request -} - -// TransportReceivedResponse occurs when we receive a response to a request -type TransportReceivedResponse struct { - Response Response -} -*/ - // TransportTransferCancelled occurs when a request we opened (with the given channel Id) to // receive data is cancelled by us. type TransportTransferCancelled struct { @@ -110,13 +91,6 @@ type EventsHandler interface { // ChannelState queries for the current channel state ChannelState(ctx context.Context, chid ChannelID) (ChannelState, error) - // OnRequestReceived occurs when we receive a request for the given channel ID - // return values are a message to send an error if the transport should be closed - OnRequestReceived(chid ChannelID, msg Request) (Response, error) - - // OnRequestReceived occurs when we receive a response to a request - OnResponseReceived(chid ChannelID, msg Response) error - // OnTransportEvent is dispatched when an event occurs on the transport // It MAY be dispatched asynchronously by the transport to the time the // event occurs @@ -126,6 +100,17 @@ type EventsHandler interface { // have a synchronous return OnTransportEvent(chid ChannelID, event TransportEvent) + // OnRequestReceived occurs when we receive a request for the given channel ID + // return values are a message to send an error if the transport should be closed + // TODO: in a future improvement, a received request should become a + // just TransportEvent, and should be handled asynchronously + OnRequestReceived(chid ChannelID, msg Request) (Response, error) + + // OnRequestReceived occurs when we receive a response to a request + // TODO: in a future improvement, a received response should become a + // just TransportEvent, and should be handled asynchronously + OnResponseReceived(chid ChannelID, msg Response) error + // OnContextAugment allows the transport to attach data transfer tracing information // to its local context, in order to create a hierarchical trace OnContextAugment(chid ChannelID) func(context.Context) context.Context @@ -192,17 +177,12 @@ type TransportCapabilities struct { Pausable bool } -func (TransportOpenedChannel) transportEvent() {} -func (TransportInitiatedTransfer) transportEvent() {} -func (TransportReceivedData) transportEvent() {} -func (TransportSentData) transportEvent() {} -func (TransportQueuedData) transportEvent() {} -func (TransportReachedDataLimit) transportEvent() {} - -/*func (TransportReceivedVoucherRequest) transportEvent() {} -func (TransportReceivedUpdateRequest) transportEvent() {} -func (TransportReceivedCancelRequest) transportEvent() {} -func (TransportReceivedResponse) transportEvent() {}*/ +func (TransportOpenedChannel) transportEvent() {} +func (TransportInitiatedTransfer) transportEvent() {} +func (TransportReceivedData) transportEvent() {} +func (TransportSentData) transportEvent() {} +func (TransportQueuedData) transportEvent() {} +func (TransportReachedDataLimit) transportEvent() {} func (TransportTransferCancelled) transportEvent() {} func (TransportErrorSendingData) transportEvent() {} func (TransportErrorReceivingData) transportEvent() {} diff --git a/transport/graphsync/exceptions_test.go b/transport/graphsync/exceptions_test.go new file mode 100644 index 00000000..1e70c74f --- /dev/null +++ b/transport/graphsync/exceptions_test.go @@ -0,0 +1,286 @@ +package graphsync_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/testutil" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/testharness" + "github.com/ipfs/go-graphsync" + "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/stretchr/testify/require" + "golang.org/x/exp/rand" +) + +func TestTransferExceptions(t *testing.T) { + ctx := context.Background() + testCases := []struct { + name string + parameters []testharness.Option + test func(t *testing.T, th *testharness.GsTestHarness) + }{ + { + name: "error executing pull graphsync request", + parameters: []testharness.Option{testharness.PullRequest()}, + test: func(t *testing.T, th *testharness.GsTestHarness) { + th.Transport.OpenChannel(th.Ctx, th.Channel, th.NewRequest(t)) + receivedRequest := th.Fgs.ReceivedRequests[0] + close(receivedRequest.ResponseChan) + receivedRequest.ResponseErrChan <- errors.New("something went wrong") + close(receivedRequest.ResponseErrChan) + select { + case <-th.CompletedRequests: + case <-ctx.Done(): + t.Fatalf("did not complete request") + } + th.Events.AssertTransportEventEventually(t, th.Channel.ChannelID(), datatransfer.TransportCompletedTransfer{Success: false, ErrorMessage: fmt.Sprintf("channel %s: graphsync request failed to complete: something went wrong", th.Channel.ChannelID())}) + }, + }, + { + name: "unrecognized outgoing pull request", + parameters: []testharness.Option{testharness.PullRequest()}, + test: func(t *testing.T, th *testharness.GsTestHarness) { + // open a channel + th.Transport.OpenChannel(th.Ctx, th.Channel, th.NewRequest(t)) + // configure a store + th.Transport.UseStore(th.Channel.ChannelID(), cidlink.DefaultLinkSystem()) + // setup a seperate request with different request ID but with contained message + otherRequest := testharness.NewFakeRequest(graphsync.NewRequestID(), map[graphsync.ExtensionName]datamodel.Node{ + extension.ExtensionDataTransfer1_1: th.NewRequest(t).ToIPLD(), + }, graphsync.RequestTypeNew) + // run outgoing request hook on this request + th.OutgoingRequestHook(otherRequest) + // no channel opened + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportOpenedChannel{}) + // no store configuration + require.Empty(t, th.OutgoingRequestHookActions.PersistenceOption) + // run outgoing request processing listener + th.OutgoingRequestProcessingListener(otherRequest) + // no transfer initiated event + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportInitiatedTransfer{}) + dtResponse := th.Response() + // create a response with the wrong request ID + otherResponse := testharness.NewFakeResponse(otherRequest.ID(), map[graphsync.ExtensionName]datamodel.Node{ + extension.ExtensionIncomingRequest1_1: dtResponse.ToIPLD(), + }, graphsync.PartialResponse) + // run incoming response hook + th.IncomingResponseHook(otherResponse) + // no response received + require.Nil(t, th.Events.ReceivedResponse) + // run blook hook + block := testharness.NewFakeBlockData(12345, 1, true) + th.IncomingBlockHook(otherResponse, block) + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportReceivedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + }, + }, + { + name: "error cancelling on restart request", + parameters: []testharness.Option{testharness.PullRequest()}, + test: func(t *testing.T, th *testharness.GsTestHarness) { + // open a channel + _ = th.Transport.OpenChannel(th.Ctx, th.Channel, th.NewRequest(t)) + th.Fgs.ReturnedCancelError = errors.New("something went wrong") + err := th.Transport.RestartChannel(th.Ctx, th.Channel, th.RestartRequest(t)) + require.EqualError(t, err, fmt.Sprintf("%s: restarting graphsync request: cancelling graphsync request for channel %s: %s", th.Channel.ChannelID(), th.Channel.ChannelID(), "something went wrong")) + }, + }, + { + name: "error reconnecting during restart", + parameters: []testharness.Option{testharness.PullRequest()}, + test: func(t *testing.T, th *testharness.GsTestHarness) { + // open a channel + _ = th.Transport.OpenChannel(th.Ctx, th.Channel, th.NewRequest(t)) + expectedErr := errors.New("something went wrong") + th.DtNet.ReturnedConnectWithRetryError = expectedErr + err := th.Transport.RestartChannel(th.Ctx, th.Channel, th.RestartRequest(t)) + require.ErrorIs(t, err, expectedErr) + }, + }, + { + name: "unrecognized incoming graphsync request dt response", + test: func(t *testing.T, th *testharness.GsTestHarness) { + dtResponse := th.Response() + requestID := graphsync.NewRequestID() + request := testharness.NewFakeRequest(requestID, map[graphsync.ExtensionName]datamodel.Node{extension.ExtensionDataTransfer1_1: dtResponse.ToIPLD()}, graphsync.RequestTypeNew) + th.IncomingRequestHook(request) + require.False(t, th.IncomingRequestHookActions.Validated) + require.Error(t, th.IncomingRequestHookActions.TerminationError) + require.Equal(t, th.Events.ReceivedResponse, dtResponse) + }, + }, + { + name: "incoming graphsync request w/ dt response gets OnResponseReceived error", + test: func(t *testing.T, th *testharness.GsTestHarness) { + _ = th.Transport.OpenChannel(th.Ctx, th.Channel, th.NewRequest(t)) + dtResponse := th.Response() + requestID := graphsync.NewRequestID() + request := testharness.NewFakeRequest(requestID, map[graphsync.ExtensionName]datamodel.Node{extension.ExtensionDataTransfer1_1: dtResponse.ToIPLD()}, graphsync.RequestTypeNew) + th.Events.ReturnedResponseReceivedError = errors.New("something went wrong") + th.IncomingRequestHook(request) + require.False(t, th.IncomingRequestHookActions.Validated) + require.EqualError(t, th.IncomingRequestHookActions.TerminationError, "something went wrong") + require.Equal(t, th.Events.ReceivedResponse, dtResponse) + }, + }, + { + name: "pull request cancelled", + parameters: []testharness.Option{testharness.PullRequest()}, + test: func(t *testing.T, th *testharness.GsTestHarness) { + _ = th.Transport.OpenChannel(th.Ctx, th.Channel, th.NewRequest(t)) + require.Len(t, th.Fgs.ReceivedRequests, 1) + receivedRequest := th.Fgs.ReceivedRequests[0] + close(receivedRequest.ResponseChan) + receivedRequest.ResponseErrChan <- graphsync.RequestClientCancelledErr{} + close(receivedRequest.ResponseErrChan) + th.Events.AssertTransportEventEventually(t, th.Channel.ChannelID(), datatransfer.TransportTransferCancelled{ + ErrorMessage: "graphsync request cancelled", + }) + }, + }, + { + name: "error opening sending push message", + test: func(t *testing.T, th *testharness.GsTestHarness) { + th.DtNet.ReturnedSendMessageError = errors.New("something went wrong") + err := th.Transport.OpenChannel(th.Ctx, th.Channel, th.NewRequest(t)) + require.EqualError(t, err, "something went wrong") + }, + }, + { + name: "unrecognized incoming graphsync push request", + test: func(t *testing.T, th *testharness.GsTestHarness) { + // open a channel + th.Transport.OpenChannel(th.Ctx, th.Channel, th.NewRequest(t)) + // configure a store + th.Transport.UseStore(th.Channel.ChannelID(), cidlink.DefaultLinkSystem()) + voucherResult := testutil.NewTestTypedVoucher() + otherRequest := testharness.NewFakeRequest(graphsync.NewRequestID(), map[graphsync.ExtensionName]datamodel.Node{ + extension.ExtensionDataTransfer1_1: message.NewResponse(datatransfer.TransferID(rand.Uint64()), true, false, &voucherResult).ToIPLD(), + }, graphsync.RequestTypeNew) + // run incoming request hook on new request + th.IncomingRequestHook(otherRequest) + // should error + require.Error(t, th.IncomingRequestHookActions.TerminationError) + // run incoming request processing listener + th.IncomingRequestProcessingListener(otherRequest) + // no transfer initiated event + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportInitiatedTransfer{}) + // run block queued hook + block := testharness.NewFakeBlockData(12345, 1, true) + th.OutgoingBlockHook(otherRequest, block) + // no block queued event + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportQueuedData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + // run block sent hook + th.BlockSentListener(otherRequest, block) + // no block sent event + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportSentData{Size: block.BlockSize(), Index: basicnode.NewInt(block.Index())}) + // run complete listener + th.ResponseCompletedListener(otherRequest, graphsync.RequestCompletedFull) + // no complete event + th.Events.RefuteTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportCompletedTransfer{Success: true}) + }, + }, + { + name: "channel update on unrecognized channel", + test: func(t *testing.T, th *testharness.GsTestHarness) { + err := th.Transport.ChannelUpdated(th.Ctx, th.Channel.ChannelID(), th.NewRequest(t)) + require.Error(t, err) + }, + }, + { + name: "incoming request errors in OnRequestReceived", + parameters: []testharness.Option{testharness.PullRequest(), testharness.Responder()}, + test: func(t *testing.T, th *testharness.GsTestHarness) { + th.Transport.UseStore(th.Channel.ChannelID(), cidlink.DefaultLinkSystem()) + th.Fgs.AssertHasPersistenceOption(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String())) + requestID := graphsync.NewRequestID() + dtRequest := th.NewRequest(t) + request := testharness.NewFakeRequest(requestID, map[graphsync.ExtensionName]datamodel.Node{extension.ExtensionDataTransfer1_1: dtRequest.ToIPLD()}, graphsync.RequestTypeNew) + voucherResult := testutil.NewTestTypedVoucher() + dtResponse := message.NewResponse(th.Channel.TransferID(), false, false, &voucherResult) + th.Events.ReturnedRequestReceivedResponse = dtResponse + th.Events.ReturnedRequestReceivedError = errors.New("something went wrong") + th.Events.ReturnedOnContextAugmentFunc = func(ctx context.Context) context.Context { + return context.WithValue(ctx, ctxKey{}, "applesauce") + } + th.IncomingRequestHook(request) + require.Equal(t, dtRequest, th.Events.ReceivedRequest) + require.Empty(t, th.DtNet.ProtectedPeers) + require.Empty(t, th.IncomingRequestHookActions.PersistenceOption) + require.False(t, th.IncomingRequestHookActions.Validated) + require.False(t, th.IncomingRequestHookActions.Paused) + require.EqualError(t, th.IncomingRequestHookActions.TerminationError, "something went wrong") + sentResponse := th.IncomingRequestHookActions.DTMessage(t) + require.Equal(t, dtResponse, sentResponse) + th.IncomingRequestHookActions.RefuteAugmentedContextKey(t, ctxKey{}) + }, + }, + { + name: "incoming gs request with contained push request errors", + parameters: []testharness.Option{testharness.Responder()}, + test: func(t *testing.T, th *testharness.GsTestHarness) { + requestID := graphsync.NewRequestID() + dtRequest := th.NewRequest(t) + request := testharness.NewFakeRequest(requestID, map[graphsync.ExtensionName]datamodel.Node{extension.ExtensionDataTransfer1_1: dtRequest.ToIPLD()}, graphsync.RequestTypeNew) + dtResponse := th.Response() + th.Events.ReturnedRequestReceivedResponse = dtResponse + th.IncomingRequestHook(request) + require.EqualError(t, th.IncomingRequestHookActions.TerminationError, datatransfer.ErrUnsupported.Error()) + }, + }, + { + name: "incoming requests completes with error code for graphsync", + parameters: []testharness.Option{testharness.PullRequest(), testharness.Responder()}, + test: func(t *testing.T, th *testharness.GsTestHarness) { + requestID := graphsync.NewRequestID() + dtRequest := th.NewRequest(t) + request := testharness.NewFakeRequest(requestID, map[graphsync.ExtensionName]datamodel.Node{extension.ExtensionDataTransfer1_1: dtRequest.ToIPLD()}, graphsync.RequestTypeNew) + dtResponse := th.Response() + th.Events.ReturnedRequestReceivedResponse = dtResponse + th.IncomingRequestHook(request) + + th.ResponseCompletedListener(request, graphsync.RequestFailedUnknown) + select { + case <-th.CompletedResponses: + case <-ctx.Done(): + t.Fatalf("did not complete request") + } + th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportCompletedTransfer{Success: false, ErrorMessage: fmt.Sprintf("graphsync response to peer %s did not complete: response status code %s", th.Channel.Recipient(), graphsync.RequestFailedUnknown.String())}) + + }, + }, + { + name: "incoming push request message errors in OnRequestReceived", + parameters: []testharness.Option{testharness.Responder()}, + test: func(t *testing.T, th *testharness.GsTestHarness) { + th.Transport.UseStore(th.Channel.ChannelID(), cidlink.DefaultLinkSystem()) + th.Fgs.AssertHasPersistenceOption(t, fmt.Sprintf("data-transfer-%s", th.Channel.ChannelID().String())) + voucherResult := testutil.NewTestTypedVoucher() + dtResponse := message.NewResponse(th.Channel.TransferID(), false, false, &voucherResult) + th.Events.ReturnedRequestReceivedResponse = dtResponse + th.Events.ReturnedRequestReceivedError = errors.New("something went wrong") + th.DtNet.Delegates[0].Receiver.ReceiveRequest(ctx, th.Channel.OtherPeer(), th.NewRequest(t)) + require.Equal(t, th.NewRequest(t), th.Events.ReceivedRequest) + require.Empty(t, th.DtNet.ProtectedPeers) + require.Empty(t, th.Fgs.ReceivedRequests) + require.Len(t, th.DtNet.SentMessages, 1) + require.Equal(t, testharness.FakeSentMessage{Message: dtResponse, TransportID: "graphsync", PeerID: th.Channel.OtherPeer()}, th.DtNet.SentMessages[0]) + }, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + th := testharness.SetupHarness(ctx, testCase.parameters...) + testCase.test(t, th) + }) + } +} diff --git a/transport/graphsync/graphsync.go b/transport/graphsync/graphsync.go index 28a9e8b1..41d1e085 100644 --- a/transport/graphsync/graphsync.go +++ b/transport/graphsync/graphsync.go @@ -227,7 +227,9 @@ func (t *Transport) ChannelUpdated(ctx context.Context, chid datatransfer.Channe ch, action, err := t.reconcileChannelStates(ctx, chid) if err != nil { if message != nil { - return t.dtNet.SendMessage(ctx, t.otherPeer(chid), transportID, message) + if sendErr := t.dtNet.SendMessage(ctx, t.otherPeer(chid), transportID, message); sendErr != nil { + return sendErr + } } return err } diff --git a/transport/graphsync/hooks.go b/transport/graphsync/hooks.go index 8c0ae35e..1e90301c 100644 --- a/transport/graphsync/hooks.go +++ b/transport/graphsync/hooks.go @@ -11,6 +11,7 @@ import ( "golang.org/x/xerrors" datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/dtchannel" "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" ) @@ -199,8 +200,17 @@ func (t *Transport) gsReqRecdHook(p peer.ID, request graphsync.RequestData, hook hookActions.TerminateWithError(err) } + var ch *dtchannel.Channel + if msg.IsRequest() { + ch = t.trackDTChannel(chid) + } else { + ch, err = t.getDTChannel(chid) + if err != nil { + hookActions.TerminateWithError(err) + return + } + } t.requestIDToChannelID.set(request.ID(), true, chid) - ch := t.trackDTChannel(chid) ch.GsDataRequestRcvd(p, request.ID(), chst, hookActions) } diff --git a/transport/graphsync/initiating_test.go b/transport/graphsync/initiating_test.go index ff956f27..8b0d1e53 100644 --- a/transport/graphsync/initiating_test.go +++ b/transport/graphsync/initiating_test.go @@ -161,7 +161,7 @@ func TestInitiatingPullRequestSuccessFlow(t *testing.T) { case <-ctx.Done(): t.Fatalf("did not complete request") } - th.Events.AssertTransportEvent(t, th.Channel.ChannelID(), datatransfer.TransportCompletedTransfer{Success: true}) + th.Events.AssertTransportEventEventually(t, th.Channel.ChannelID(), datatransfer.TransportCompletedTransfer{Success: true}) }) t.Run("cleanup request", func(t *testing.T) { diff --git a/transport/graphsync/receiver.go b/transport/graphsync/receiver.go index f6fe6038..2391fe1f 100644 --- a/transport/graphsync/receiver.go +++ b/transport/graphsync/receiver.go @@ -30,7 +30,10 @@ func (r *receiver) ReceiveRequest( func (r *receiver) receiveRequest(ctx context.Context, initiator peer.ID, incoming datatransfer.Request) error { chid := datatransfer.ChannelID{Initiator: initiator, Responder: r.transport.peerID, ID: incoming.TransferID()} - ctx = r.transport.events.OnContextAugment(chid)(ctx) + ctxAugment := r.transport.events.OnContextAugment(chid) + if ctxAugment != nil { + ctx = ctxAugment(ctx) + } ctx, span := otel.Tracer("gs-data-transfer").Start(ctx, "receiveRequest", trace.WithAttributes( attribute.String("channelID", chid.String()), attribute.String("baseCid", incoming.BaseCid().String()), @@ -50,9 +53,12 @@ func (r *receiver) receiveRequest(ctx context.Context, initiator peer.ID, incomi initiateGraphsyncRequest := isNewOrRestart && response != nil && receiveErr == nil ch, err := r.transport.getDTChannel(chid) if err != nil { - if !initiateGraphsyncRequest { + if !initiateGraphsyncRequest || receiveErr != nil { if response != nil { - return r.transport.dtNet.SendMessage(ctx, initiator, transportID, response) + if sendErr := r.transport.dtNet.SendMessage(ctx, initiator, transportID, response); sendErr != nil { + return sendErr + } + return receiveErr } return receiveErr } diff --git a/transport/graphsync/testharness/events.go b/transport/graphsync/testharness/events.go index dd37635a..398a3ba8 100644 --- a/transport/graphsync/testharness/events.go +++ b/transport/graphsync/testharness/events.go @@ -3,6 +3,7 @@ package testharness import ( "context" "testing" + "time" "github.com/stretchr/testify/require" @@ -38,6 +39,17 @@ func (fe *FakeEvents) AssertTransportEvent(t *testing.T, chid datatransfer.Chann require.Contains(t, fe.ReceivedTransportEvents, ReceivedTransportEvent{chid, evt}) } +func (fe *FakeEvents) AssertTransportEventEventually(t *testing.T, chid datatransfer.ChannelID, evt datatransfer.TransportEvent) { + require.Eventually(t, func() bool { + for _, receivedEvent := range fe.ReceivedTransportEvents { + if (receivedEvent == ReceivedTransportEvent{chid, evt}) { + return true + } + } + return false + }, time.Second, time.Millisecond) +} + func (fe *FakeEvents) RefuteTransportEvent(t *testing.T, chid datatransfer.ChannelID, evt datatransfer.TransportEvent) { require.NotContains(t, fe.ReceivedTransportEvents, ReceivedTransportEvent{chid, evt}) } diff --git a/transport/graphsync/testharness/fakegraphsync.go b/transport/graphsync/testharness/fakegraphsync.go index 7d6be3ec..5f31b454 100644 --- a/transport/graphsync/testharness/fakegraphsync.go +++ b/transport/graphsync/testharness/fakegraphsync.go @@ -124,6 +124,10 @@ type FakeGraphSync struct { BlockSentListener graphsync.OnBlockSentListener NetworkErrorListener graphsync.OnNetworkErrorListener ReceiverNetworkErrorListener graphsync.OnReceiverNetworkErrorListener + ReturnedCancelError error + ReturnedPauseError error + ReturnedResumeError error + ReturnedSendUpdateError error } // NewFakeGraphSync returns a new fake graphsync implementation @@ -253,13 +257,13 @@ func (fgs *FakeGraphSync) RegisterCompletedResponseListener(listener graphsync.O // Unpause unpauses a request that was paused in a block hook based on request ID func (fgs *FakeGraphSync) Unpause(ctx context.Context, requestID graphsync.RequestID, extensions ...graphsync.ExtensionData) error { fgs.Resumes = append(fgs.Resumes, Resume{requestID, extensions}) - return nil + return fgs.ReturnedResumeError } // Pause pauses a request based on request ID func (fgs *FakeGraphSync) Pause(ctx context.Context, requestID graphsync.RequestID) error { fgs.Pauses = append(fgs.Pauses, requestID) - return nil + return fgs.ReturnedPauseError } func (fgs *FakeGraphSync) Cancel(ctx context.Context, requestID graphsync.RequestID) error { @@ -274,7 +278,7 @@ func (fgs *FakeGraphSync) Cancel(ctx context.Context, requestID graphsync.Reques } } fgs.Cancels = append(fgs.Cancels, requestID) - return nil + return fgs.ReturnedCancelError } // RegisterRequestorCancelledListener adds a listener on the responder for requests cancelled by the requestor @@ -322,7 +326,7 @@ func (fgs *FakeGraphSync) RegisterOutgoingRequestProcessingListener(listener gra func (fgs *FakeGraphSync) SendUpdate(ctx context.Context, id graphsync.RequestID, extensions ...graphsync.ExtensionData) error { fgs.Updates = append(fgs.Updates, Update{RequestID: id, Extensions: extensions}) - return nil + return fgs.ReturnedSendUpdateError } var _ graphsync.GraphExchange = &FakeGraphSync{} @@ -549,6 +553,14 @@ func (fa *FakeIncomingRequestHookActions) AssertAugmentedContextKey(t *testing.T require.Equal(t, value, ctx.Value(key)) } +func (fa *FakeIncomingRequestHookActions) RefuteAugmentedContextKey(t *testing.T, key interface{}) { + ctx := context.Background() + for _, f := range fa.CtxAugFuncs { + ctx = f(ctx) + } + require.Nil(t, ctx.Value(key)) +} + func (fa *FakeIncomingRequestHookActions) DTMessage(t *testing.T) datatransfer.Message { return matchDtMessage(t, fa.SentExtensions, extension.ExtensionIncomingRequest1_1) } From 9a3438a0a4f566f67a34be9a401e621260ddf572 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 21 Jul 2022 12:30:36 -0700 Subject: [PATCH 18/18] style(imports): fix imports --- channels/channels_test.go | 5 +++-- transport/graphsync/exceptions_test.go | 11 ++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/channels/channels_test.go b/channels/channels_test.go index c62311a2..3e26af1d 100644 --- a/channels/channels_test.go +++ b/channels/channels_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - versionedds "github.com/filecoin-project/go-ds-versioning/pkg/datastore" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" dss "github.com/ipfs/go-datastore/sync" @@ -18,12 +17,14 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/xerrors" + versioning "github.com/filecoin-project/go-ds-versioning/pkg" + versionedds "github.com/filecoin-project/go-ds-versioning/pkg/datastore" + datatransfer "github.com/filecoin-project/go-data-transfer/v2" "github.com/filecoin-project/go-data-transfer/v2/channels" "github.com/filecoin-project/go-data-transfer/v2/channels/internal" "github.com/filecoin-project/go-data-transfer/v2/channels/internal/migrations" "github.com/filecoin-project/go-data-transfer/v2/testutil" - versioning "github.com/filecoin-project/go-ds-versioning/pkg" ) func TestChannels(t *testing.T) { diff --git a/transport/graphsync/exceptions_test.go b/transport/graphsync/exceptions_test.go index 1e70c74f..c4845721 100644 --- a/transport/graphsync/exceptions_test.go +++ b/transport/graphsync/exceptions_test.go @@ -7,17 +7,18 @@ import ( "testing" "time" - datatransfer "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/go-data-transfer/v2/message" - "github.com/filecoin-project/go-data-transfer/v2/testutil" - "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" - "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/testharness" "github.com/ipfs/go-graphsync" "github.com/ipld/go-ipld-prime/datamodel" cidlink "github.com/ipld/go-ipld-prime/linking/cid" basicnode "github.com/ipld/go-ipld-prime/node/basic" "github.com/stretchr/testify/require" "golang.org/x/exp/rand" + + datatransfer "github.com/filecoin-project/go-data-transfer/v2" + "github.com/filecoin-project/go-data-transfer/v2/message" + "github.com/filecoin-project/go-data-transfer/v2/testutil" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension" + "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/testharness" ) func TestTransferExceptions(t *testing.T) {