Merge pull request #663 from keep-network/resubscriptions-dance

Subscription dance: backoff and logging

Refs #680

Depends on keep-network/keep-common#62

The referenced `keep-common` PR solves three problems and two of them are important for ECDSA client.

Currently, the subscription mechanism used by ECDSA client does not implement a backoff. Each failed subscription is retried 5 seconds after it failed. For many individual subscriptions ECDSA client is opening, a massive retry with no backoff can be interpreted as misbehavior by a third party Ethereum provider and further attempts can be completely blocked. Some operators already experienced it with Alchemy.

The second problem is that all the logging regarding resubscriptions is done on the warning level. With the lack of backoff mentioned in the previous point, this can produce a deadly mixture - keep client could be trying to reconnect for a long time, Ethereum client could be rejecting those attempts interpreting them as DoS/misbehavior, and in this scenario, operator would receive only warnings with no single error. 

Both those points are addressed here by bumping up the dependency on `keep-common` and regenerating contract bindings. Also, the code in `ethereum.go` is getting cleaned up a little to reflect the fact all `Watch*` attempts are getting retried until they succeed or until unsubscribe happens.

For example, let's say that the client was able to establish all subscriptions, they were working fine for two minutes, failed, and were established again successfully. Dropped WS connection is fine assuming it's not happening too often. Two minutes is certainly alarming, so the client will now alarm with an error:

> 2021-01-07T18:29:20.522+0100	ERROR	keep-contract-BondedECDSAKeepFactory	subscription to event BondedECDSAKeepCreated had to be retried [2m0.006063728s] since the last attempt; please inspect Ethereum client connectivity

Now, let's say that the retry will fail. Again, the client will alarm with an error:

> 2021-01-07T18:31:21.532+0100	ERROR	keep-contract-BondedECDSAKeep	subscription to event KeepClosed failed: [dial tcp 127.0.0.1:8546: connect: connection refused]; resubscription attempt will be performed

Last but not least, we now have a backoff. The delay is increased twice until it reaches the maximum backoff time which is set to 2 minutes.

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/gogo/protobuf v1.3.1
 	github.com/google/gofuzz v1.1.0
 	github.com/ipfs/go-log v1.0.4
-	github.com/keep-network/keep-common v1.3.0
+	github.com/keep-network/keep-common v1.3.1-0.20210127121021-c6adefdc082b
 	github.com/keep-network/keep-core v1.3.2-0.20201229154408-59ac640ed0cb
 	github.com/keep-network/tbtc v1.1.1-0.20201117095624-38508bdb562e
 	github.com/pkg/errors v0.9.1

go.sum

@@ -332,8 +332,9 @@ github.com/keep-network/cli v1.20.0/go.mod h1:nzsst4JjU+rGE8Q5J839fYxectxWHpLhxK
 github.com/keep-network/go-libp2p-bootstrap v0.0.0-20200423153828-ed815bc50aec h1:2pXAsi4OUUjZKr5ds5UOF2IxXN+jVW0WetVO+czkf+A=
 github.com/keep-network/go-libp2p-bootstrap v0.0.0-20200423153828-ed815bc50aec/go.mod h1:xR8jf3/VJAjh3nWu5tFe8Yxnt2HvWsqZHfGef1P5oDk=
 github.com/keep-network/keep-common v1.2.1-0.20201116151638-8af057907255/go.mod h1:emxogTbBdey7M3jOzfxZOdfn139kN2mI2b2wA6AHKKo=
-github.com/keep-network/keep-common v1.3.0 h1:4aWCznDfzk0kZS+bzlRxYqq5ffvuFlA/DvD+viptHfQ=
 github.com/keep-network/keep-common v1.3.0/go.mod h1:emxogTbBdey7M3jOzfxZOdfn139kN2mI2b2wA6AHKKo=
+github.com/keep-network/keep-common v1.3.1-0.20210127121021-c6adefdc082b h1:xWeLAJYE1jbzkkDFkNTKVjhAhslfH5bHaxWjM9Ba5R8=
+github.com/keep-network/keep-common v1.3.1-0.20210127121021-c6adefdc082b/go.mod h1:emxogTbBdey7M3jOzfxZOdfn139kN2mI2b2wA6AHKKo=
 github.com/keep-network/keep-core v1.3.2-0.20201229154408-59ac640ed0cb h1:pDhLagUiOYWOn72UiG3jSDVDXZ4dYlsfRa0ToF/ru18=
 github.com/keep-network/keep-core v1.3.2-0.20201229154408-59ac640ed0cb/go.mod h1:4KezOJWc//c5lbtAE1ob7qYE/Gs5+a1QTjGBzzv2tu8=
 github.com/keep-network/tbtc v1.1.1-0.20201117095624-38508bdb562e h1:go3irX4olJUZOi59gaBX7iYIFFbJEV5/zbfCG4tOPaQ=

pkg/chain/ethereum/ethereum.go

@@ -74,7 +74,7 @@ func (ec *EthereumChain) RegisterAsMemberCandidate(application common.Address) e
 func (ec *EthereumChain) OnBondedECDSAKeepCreated(
 	handler func(event *eth.BondedECDSAKeepCreatedEvent),
 ) subscription.EventSubscription {
-	subscription, err := ec.bondedECDSAKeepFactoryContract.WatchBondedECDSAKeepCreated(
+	subscription := ec.bondedECDSAKeepFactoryContract.WatchBondedECDSAKeepCreated(
 		func(
 			KeepAddress common.Address,
 			Members []common.Address,
@@ -89,16 +89,10 @@ func (ec *EthereumChain) OnBondedECDSAKeepCreated(
 				HonestThreshold: HonestThreshold.Uint64(),
 			})
 		},
-		func(err error) error {
-			return fmt.Errorf("watch keep created failed: [%v]", err)
-		},
 		nil,
 		nil,
 		nil,
 	)
-	if err != nil {
-		logger.Errorf("could not watch BondedECDSAKeepCreated event: [%v]", err)
-	}
 
 	return subscription
 }
@@ -116,10 +110,7 @@ func (ec *EthereumChain) OnKeepClosed(
 		func(blockNumber uint64) {
 			handler(&eth.KeepClosedEvent{BlockNumber: blockNumber})
 		},
-		func(err error) error {
-			return fmt.Errorf("keep closed callback failed: [%v]", err)
-		},
-	)
+	), nil
 }
 
 // OnKeepTerminated installs a callback that is invoked on-chain when keep
@@ -136,10 +127,7 @@ func (ec *EthereumChain) OnKeepTerminated(
 		func(blockNumber uint64) {
 			handler(&eth.KeepTerminatedEvent{BlockNumber: blockNumber})
 		},
-		func(err error) error {
-			return fmt.Errorf("keep terminated callback failed: [%v]", err)
-		},
-	)
+	), nil
 }
 
 // OnPublicKeyPublished installs a callback that is invoked when an on-chain
@@ -162,10 +150,7 @@ func (ec *EthereumChain) OnPublicKeyPublished(
 				PublicKey: PublicKey,
 			})
 		},
-		func(err error) error {
-			return fmt.Errorf("keep created callback failed: [%v]", err)
-		},
-	)
+	), nil
 }
 
 // OnConflictingPublicKeySubmitted installs a callback that is invoked when an
@@ -190,11 +175,8 @@ func (ec *EthereumChain) OnConflictingPublicKeySubmitted(
 				ConflictingPublicKey: ConflictingPublicKey,
 			})
 		},
-		func(err error) error {
-			return fmt.Errorf("keep created callback failed: [%v]", err)
-		},
 		nil,
-	)
+	), nil
 }
 
 // OnSignatureRequested installs a callback that is invoked on-chain
@@ -218,11 +200,8 @@ func (ec *EthereumChain) OnSignatureRequested(
 				BlockNumber: blockNumber,
 			})
 		},
-		func(err error) error {
-			return fmt.Errorf("keep signature requested callback failed: [%v]", err)
-		},
 		nil,
-	)
+	), nil
 }
 
 // SubmitKeepPublicKey submits a public key to a keep contract deployed under