Web3bot/sync corylocal

.github/workflows/automerge.yml

@@ -0,0 +1,51 @@
+# File managed by web3-bot. DO NOT EDIT.
+# See https://github.com/protocol/.github/ for details.
+
+# Automatically merge pull requests opened by web3-bot, as soon as (and only if) all tests pass.
+# This reduces the friction associated with updating with our workflows.
+
+on: [ pull_request ]
+name: Automerge
+
+jobs:
+  automerge-check:
+    if: github.event.pull_request.user.login == 'web3-bot'
+    runs-on: ubuntu-latest
+    outputs:
+      status: ${{ steps.should-automerge.outputs.status }}
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Check if we should automerge
+      id: should-automerge
+      run: |
+        for commit in $(git rev-list --first-parent origin/${{ github.event.pull_request.base.ref }}..${{ github.event.pull_request.head.sha }}); do
+          committer=$(git show --format=$'%ce' -s $commit)
+          echo "Committer: $committer"
+          if [[ "$committer" != "web3-bot@users.noreply.github.com" ]]; then
+            echo "Commit $commit wasn't committed by web3-bot, but by $committer."
+            echo "::set-output name=status::false"
+            exit
+          fi
+        done
+        echo "::set-output name=status::true"
+  automerge:
+    needs: automerge-check
+    runs-on: ubuntu-latest
+    if: ${{ needs.automerge-check.outputs.status == 'true' }}
+    steps:
+    - name: Wait on tests
+      uses: lewagon/wait-on-check-action@bafe56a6863672c681c3cf671f5e10b20abf2eaa # v0.2
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        wait-interval: 10
+        running-workflow-name: 'automerge' # the name of this job
+    - name: Merge PR
+      uses: pascalgn/automerge-action@741c311a47881be9625932b0a0de1b0937aab1ae # v0.13.1
+      env:
+        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        MERGE_LABELS: ""
+        MERGE_METHOD: "squash"
+        MERGE_DELETE_BRANCH: true

.github/workflows/go-check.yml

@@ -0,0 +1,50 @@
+# File managed by web3-bot. DO NOT EDIT.
+# See https://github.com/protocol/.github/ for details.
+
+on: [push, pull_request]
+name: Go Checks
+
+jobs:
+  unit:
+    runs-on: ubuntu-latest
+    name: All
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - uses: actions/setup-go@v2
+        with:
+          go-version: "1.16.x"
+      - name: Install staticcheck
+        run: go install honnef.co/go/tools/cmd/staticcheck@434f5f3816b358fe468fa83dcba62d794e7fe04b # 2021.1 (v0.2.0)
+      - name: Check that go.mod is tidy
+        uses: protocol/multiple-go-modules@v1.0
+        with:
+          run: |
+            go mod tidy
+            if [[ -n $(git ls-files --other --exclude-standard --directory -- go.sum) ]]; then
+              echo "go.sum was added by go mod tidy"
+              exit 1
+            fi
+            git diff --exit-code -- go.sum go.mod
+      - name: gofmt
+        if: ${{ success() || failure() }} # run this step even if the previous one failed
+        run: |
+          out=$(gofmt -s -l .)
+          if [[ -n "$out" ]]; then
+            echo $out | awk '{print "::error file=" $0 ",line=0,col=0::File is not gofmt-ed."}'
+            exit 1
+          fi
+      - name: go vet
+        if: ${{ success() || failure() }} # run this step even if the previous one failed
+        uses: protocol/multiple-go-modules@v1.0
+        with:
+          run: go vet ./...
+      - name: staticcheck
+        if: ${{ success() || failure() }} # run this step even if the previous one failed
+        uses: protocol/multiple-go-modules@v1.0
+        with:
+          run: |
+            set -o pipefail
+            staticcheck ./... | sed -e 's@\(.*\)\.go@./\1.go@g'
+

.github/workflows/go-test.yml

@@ -0,0 +1,47 @@
+# File managed by web3-bot. DO NOT EDIT.
+# See https://github.com/protocol/.github/ for details.
+
+on: [push, pull_request]
+name: Go Test
+
+jobs:
+  unit:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ "ubuntu", "windows", "macos" ]
+        go: [ "1.15.x", "1.16.x" ]
+    runs-on: ${{ matrix.os }}-latest
+    name: ${{ matrix.os}} (go ${{ matrix.go }})
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Go information
+        run: |
+          go version
+          go env
+      - name: Run tests
+        uses: protocol/multiple-go-modules@v1.0
+        with:
+          run: go test -v -coverprofile coverage.txt ./...
+      - name: Run tests (32 bit)
+        if: ${{ matrix.os != 'macos' }} # can't run 32 bit tests on OSX.
+        uses: protocol/multiple-go-modules@v1.0
+        env:
+          GOARCH: 386
+        with:
+          run: go test -v ./...
+      - name: Run tests with race detector
+        if: ${{ matrix.os == 'ubuntu' }} # speed things up. Windows and OSX VMs are slow
+        uses: protocol/multiple-go-modules@v1.0
+        with:
+          run: go test -v -race ./...
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27 # v1.5.0
+        with:
+          file: coverage.txt
+          env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }}

benchmarks_test.go

@@ -19,9 +19,9 @@ import (
 
 	bitswap "github.com/ipfs/go-bitswap"
 	bssession "github.com/ipfs/go-bitswap/internal/session"
+	bsnet "github.com/ipfs/go-bitswap/network"
 	testinstance "github.com/ipfs/go-bitswap/testinstance"
 	tn "github.com/ipfs/go-bitswap/testnet"
-	bsnet "github.com/ipfs/go-bitswap/network"
 	cid "github.com/ipfs/go-cid"
 	delay "github.com/ipfs/go-ipfs-delay"
 	mockrouting "github.com/ipfs/go-ipfs-routing/mock"
@@ -53,55 +53,55 @@ type bench struct {
 var benches = []bench{
 	// Fetch from two seed nodes that both have all 100 blocks
 	// - request one at a time, in series
-	bench{"3Nodes-AllToAll-OneAtATime", 3, 100, allToAll, oneAtATime},
+	{"3Nodes-AllToAll-OneAtATime", 3, 100, allToAll, oneAtATime},
 	// - request all 100 with a single GetBlocks() call
-	bench{"3Nodes-AllToAll-BigBatch", 3, 100, allToAll, batchFetchAll},
+	{"3Nodes-AllToAll-BigBatch", 3, 100, allToAll, batchFetchAll},
 
 	// Fetch from two seed nodes, one at a time, where:
 	// - node A has blocks 0 - 74
 	// - node B has blocks 25 - 99
-	bench{"3Nodes-Overlap1-OneAtATime", 3, 100, overlap1, oneAtATime},
+	{"3Nodes-Overlap1-OneAtATime", 3, 100, overlap1, oneAtATime},
 
 	// Fetch from two seed nodes, where:
 	// - node A has even blocks
 	// - node B has odd blocks
 	// - both nodes have every third block
 
 	// - request one at a time, in series
-	bench{"3Nodes-Overlap3-OneAtATime", 3, 100, overlap2, oneAtATime},
+	{"3Nodes-Overlap3-OneAtATime", 3, 100, overlap2, oneAtATime},
 	// - request 10 at a time, in series
-	bench{"3Nodes-Overlap3-BatchBy10", 3, 100, overlap2, batchFetchBy10},
+	{"3Nodes-Overlap3-BatchBy10", 3, 100, overlap2, batchFetchBy10},
 	// - request all 100 in parallel as individual GetBlock() calls
-	bench{"3Nodes-Overlap3-AllConcurrent", 3, 100, overlap2, fetchAllConcurrent},
+	{"3Nodes-Overlap3-AllConcurrent", 3, 100, overlap2, fetchAllConcurrent},
 	// - request all 100 with a single GetBlocks() call
-	bench{"3Nodes-Overlap3-BigBatch", 3, 100, overlap2, batchFetchAll},
+	{"3Nodes-Overlap3-BigBatch", 3, 100, overlap2, batchFetchAll},
 	// - request 1, then 10, then 89 blocks (similar to how IPFS would fetch a file)
-	bench{"3Nodes-Overlap3-UnixfsFetch", 3, 100, overlap2, unixfsFileFetch},
+	{"3Nodes-Overlap3-UnixfsFetch", 3, 100, overlap2, unixfsFileFetch},
 
 	// Fetch from nine seed nodes, all nodes have all blocks
 	// - request one at a time, in series
-	bench{"10Nodes-AllToAll-OneAtATime", 10, 100, allToAll, oneAtATime},
+	{"10Nodes-AllToAll-OneAtATime", 10, 100, allToAll, oneAtATime},
 	// - request 10 at a time, in series
-	bench{"10Nodes-AllToAll-BatchFetchBy10", 10, 100, allToAll, batchFetchBy10},
+	{"10Nodes-AllToAll-BatchFetchBy10", 10, 100, allToAll, batchFetchBy10},
 	// - request all 100 with a single GetBlocks() call
-	bench{"10Nodes-AllToAll-BigBatch", 10, 100, allToAll, batchFetchAll},
+	{"10Nodes-AllToAll-BigBatch", 10, 100, allToAll, batchFetchAll},
 	// - request all 100 in parallel as individual GetBlock() calls
-	bench{"10Nodes-AllToAll-AllConcurrent", 10, 100, allToAll, fetchAllConcurrent},
+	{"10Nodes-AllToAll-AllConcurrent", 10, 100, allToAll, fetchAllConcurrent},
 	// - request 1, then 10, then 89 blocks (similar to how IPFS would fetch a file)
-	bench{"10Nodes-AllToAll-UnixfsFetch", 10, 100, allToAll, unixfsFileFetch},
+	{"10Nodes-AllToAll-UnixfsFetch", 10, 100, allToAll, unixfsFileFetch},
 	// - follow a typical IPFS request pattern for 1000 blocks
-	bench{"10Nodes-AllToAll-UnixfsFetchLarge", 10, 1000, allToAll, unixfsFileFetchLarge},
+	{"10Nodes-AllToAll-UnixfsFetchLarge", 10, 1000, allToAll, unixfsFileFetchLarge},
 
 	// Fetch from nine seed nodes, blocks are distributed randomly across all nodes (no dups)
 	// - request one at a time, in series
-	bench{"10Nodes-OnePeerPerBlock-OneAtATime", 10, 100, onePeerPerBlock, oneAtATime},
+	{"10Nodes-OnePeerPerBlock-OneAtATime", 10, 100, onePeerPerBlock, oneAtATime},
 	// - request all 100 with a single GetBlocks() call
-	bench{"10Nodes-OnePeerPerBlock-BigBatch", 10, 100, onePeerPerBlock, batchFetchAll},
+	{"10Nodes-OnePeerPerBlock-BigBatch", 10, 100, onePeerPerBlock, batchFetchAll},
 	// - request 1, then 10, then 89 blocks (similar to how IPFS would fetch a file)
-	bench{"10Nodes-OnePeerPerBlock-UnixfsFetch", 10, 100, onePeerPerBlock, unixfsFileFetch},
+	{"10Nodes-OnePeerPerBlock-UnixfsFetch", 10, 100, onePeerPerBlock, unixfsFileFetch},
 
 	// Fetch from 199 seed nodes, all nodes have all blocks, fetch all 20 blocks with a single GetBlocks() call
-	bench{"200Nodes-AllToAll-BigBatch", 200, 20, allToAll, batchFetchAll},
+	{"200Nodes-AllToAll-BigBatch", 200, 20, allToAll, batchFetchAll},
 }
 
 func BenchmarkFixedDelay(b *testing.B) {
@@ -127,9 +127,9 @@ type mixedBench struct {
 }
 
 var mixedBenches = []mixedBench{
-	mixedBench{bench{"3Nodes-Overlap3-OneAtATime", 3, 10, overlap2, oneAtATime}, 1, 2},
-	mixedBench{bench{"3Nodes-AllToAll-OneAtATime", 3, 10, allToAll, oneAtATime}, 1, 2},
-	mixedBench{bench{"3Nodes-Overlap3-AllConcurrent", 3, 10, overlap2, fetchAllConcurrent}, 1, 2},
+	{bench{"3Nodes-Overlap3-OneAtATime", 3, 10, overlap2, oneAtATime}, 1, 2},
+	{bench{"3Nodes-AllToAll-OneAtATime", 3, 10, allToAll, oneAtATime}, 1, 2},
+	{bench{"3Nodes-Overlap3-AllConcurrent", 3, 10, overlap2, fetchAllConcurrent}, 1, 2},
 	// mixedBench{bench{"3Nodes-Overlap3-UnixfsFetch", 3, 100, overlap2, unixfsFileFetch}, 1, 2},
 }
 

bitswap.go

@@ -118,6 +118,12 @@ func WithScoreLedger(scoreLedger deciface.ScoreLedger) Option {
 	}
 }
 
+func SetSimulateDontHavesOnTimeout(send bool) Option {
+	return func(bs *Bitswap) {
+		bs.simulateDontHavesOnTimeout = send
+	}
+}
+
 // New initializes a BitSwap instance that communicates over the provided
 // BitSwapNetwork. This function registers the returned instance as the network
 // delegate. Runs until context is cancelled or bitswap.Close is called.
@@ -149,9 +155,12 @@ func New(parent context.Context, network bsnet.BitSwapNetwork,
 	// has an old version of Bitswap that doesn't support DONT_HAVE messages,
 	// or when no response is received within a timeout.
 	var sm *bssm.SessionManager
+	var bs *Bitswap
 	onDontHaveTimeout := func(p peer.ID, dontHaves []cid.Cid) {
 		// Simulate a message arriving with DONT_HAVEs
-		sm.ReceiveFrom(ctx, p, nil, nil, dontHaves)
+		if bs.simulateDontHavesOnTimeout {
+			sm.ReceiveFrom(ctx, p, nil, nil, dontHaves)
+		}
 	}
 	peerQueueFactory := func(ctx context.Context, p peer.ID) bspm.PeerQueue {
 		return bsmq.New(ctx, p, network, onDontHaveTimeout)
@@ -182,25 +191,26 @@ func New(parent context.Context, network bsnet.BitSwapNetwork,
 	notif := notifications.New()
 	sm = bssm.New(ctx, sessionFactory, sim, sessionPeerManagerFactory, bpm, pm, notif, network.Self())
 
-	bs := &Bitswap{
-		blockstore:       bstore,
-		network:          network,
-		process:          px,
-		newBlocks:        make(chan cid.Cid, HasBlockBufferSize),
-		provideKeys:      make(chan cid.Cid, provideKeysBufferSize),
-		pm:               pm,
-		pqm:              pqm,
-		sm:                      sm,
-		sim:                     sim,
-		notif:                   notif,
-		counters:                new(counters),
-		dupMetric:               dupHist,
-		allMetric:               allHist,
-		sentHistogram:           sentHistogram,
-		provideEnabled:          true,
-		provSearchDelay:         defaultProvSearchDelay,
-		rebroadcastDelay:        delay.Fixed(time.Minute),
-		engineBstoreWorkerCount: defaulEngineBlockstoreWorkerCount,
+	bs = &Bitswap{
+		blockstore:                 bstore,
+		network:                    network,
+		process:                    px,
+		newBlocks:                  make(chan cid.Cid, HasBlockBufferSize),
+		provideKeys:                make(chan cid.Cid, provideKeysBufferSize),
+		pm:                         pm,
+		pqm:                        pqm,
+		sm:                         sm,
+		sim:                        sim,
+		notif:                      notif,
+		counters:                   new(counters),
+		dupMetric:                  dupHist,
+		allMetric:                  allHist,
+		sentHistogram:              sentHistogram,
+		provideEnabled:             true,
+		provSearchDelay:            defaultProvSearchDelay,
+		rebroadcastDelay:           delay.Fixed(time.Minute),
+		engineBstoreWorkerCount:    defaulEngineBlockstoreWorkerCount,
+		simulateDontHavesOnTimeout: true,
 	}
 
 	// apply functional options before starting and running bitswap
@@ -293,6 +303,9 @@ type Bitswap struct {
 
 	// the score ledger used by the decision engine
 	engineScoreLedger deciface.ScoreLedger
+
+	// whether we should actually simulate dont haves on request timeout
+	simulateDontHavesOnTimeout bool
 }
 
 type counters struct {

bitswap_with_sessions_test.go

@@ -74,7 +74,7 @@ func TestSessionBetweenPeers(t *testing.T) {
 	defer cancel()
 
 	vnet := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(time.Millisecond))
-	ig := testinstance.NewTestInstanceGenerator(vnet, nil, nil)
+	ig := testinstance.NewTestInstanceGenerator(vnet, nil, []bitswap.Option{bitswap.SetSimulateDontHavesOnTimeout(false)})
 	defer ig.Close()
 	bgen := blocksutil.NewBlockGenerator()
 

go.mod

@@ -1,6 +1,7 @@
 module github.com/ipfs/go-bitswap
 
 require (
+	github.com/benbjohnson/clock v1.1.0
 	github.com/cskr/pubsub v1.0.2
 	github.com/gogo/protobuf v1.3.1
 	github.com/google/uuid v1.1.2
@@ -30,4 +31,4 @@ require (
 	go.uber.org/zap v1.16.0
 )
 
-go 1.12
+go 1.15

go.sum

@@ -7,6 +7,8 @@ github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETF
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
 github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
 github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=