Use sorted hash map for nonce accounting

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/allegro/bigcache v1.2.1 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/cespare/cp v1.1.1 // indirect
+	github.com/elliotchance/orderedmap v1.5.0 // indirect
 	github.com/ethereum/go-ethereum v1.10.26
 	github.com/fatih/color v1.12.0 // indirect
 	github.com/golang/glog v0.0.0-20210429001901-424d2337a529

go.sum

@@ -347,6 +347,8 @@ github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7j
 github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
+github.com/elliotchance/orderedmap v1.5.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=

pm/recipient.go

@@ -4,6 +4,7 @@ import (
 	"crypto/hmac"
 	"crypto/rand"
 	"crypto/sha256"
+	"github.com/elliotchance/orderedmap"
 	"math/big"
 	"sync"
 
@@ -21,6 +22,9 @@ var errInsufficientSenderReserve = errors.New("insufficient sender reserve")
 // maxWinProb = 2^256 - 1
 var maxWinProb = new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(1))
 
+// max number of sender nonces for a given recipient random hash
+var maxSenderNonces = 50
+
 var paramsExpirationBlock = big.NewInt(10)
 var paramsExpiryBuffer = int64(1)
 
@@ -85,10 +89,7 @@ type recipient struct {
 	secret       [32]byte
 	maxfacevalue *big.Int
 
-	senderNonces map[string]*struct {
-		nonce           uint32
-		expirationBlock *big.Int
-	}
+	senderNonces     map[string]*orderedmap.OrderedMap
 	senderNoncesLock sync.Mutex
 
 	cfg TicketParamsConfig
@@ -123,12 +124,9 @@ func NewRecipientWithSecret(addr ethcommon.Address, broker Broker, val Validator
 		addr:         addr,
 		secret:       secret,
 		maxfacevalue: big.NewInt(0),
-		senderNonces: make(map[string]*struct {
-			nonce           uint32
-			expirationBlock *big.Int
-		}),
-		cfg:  cfg,
-		quit: make(chan struct{}),
+		senderNonces: make(map[string]*orderedmap.OrderedMap),
+		cfg:          cfg,
+		quit:         make(chan struct{}),
 	}
 }
 
@@ -363,16 +361,21 @@ func (r *recipient) updateSenderNonce(rand *big.Int, ticket *Ticket) error {
 	defer r.senderNoncesLock.Unlock()
 
 	randStr := rand.String()
-	sn, ok := r.senderNonces[randStr]
-	if ok && ticket.SenderNonce <= sn.nonce {
-		return errors.Errorf("invalid ticket senderNonce sender=%v nonce=%v highest=%v", ticket.Sender.Hex(), ticket.SenderNonce, sn.nonce)
+	nonceToExpBlock, randKeySeen := r.senderNonces[randStr]
+	if randKeySeen {
+		_, nonceSeen := nonceToExpBlock.Get(ticket.SenderNonce)
+		if nonceSeen {
+			return errors.Errorf("invalid ticket senderNonce: already seen sender=%v nonce=%v", ticket.Sender.Hex(), ticket.SenderNonce)
+		}
+	} else {
+		r.senderNonces[randStr] = orderedmap.NewOrderedMap()
+	}
+	// add new nonce
+	r.senderNonces[randStr].Set(ticket.SenderNonce, ticket.ParamsExpirationBlock)
+	// delete first seen nonce, if ordered map exceeds max size
+	if r.senderNonces[randStr].Len() > maxSenderNonces {
+		r.senderNonces[randStr].Delete(r.senderNonces[randStr].Front().Key)
 	}
-
-	r.senderNonces[randStr] = &struct {
-		nonce           uint32
-		expirationBlock *big.Int
-	}{ticket.SenderNonce, ticket.ParamsExpirationBlock}
-
 	return nil
 }
 
@@ -393,9 +396,15 @@ func (r *recipient) senderNoncesCleanupLoop() {
 			glog.Error(err)
 		case latestL1Block := <-sink:
 			r.senderNoncesLock.Lock()
-			for recipientRand, sn := range r.senderNonces {
-				if sn.expirationBlock.Cmp(latestL1Block) <= 0 {
-					delete(r.senderNonces, recipientRand)
+			for recipientRand, nonceMap := range r.senderNonces {
+				for _, key := range nonceMap.Keys() {
+					expirationBlock, _ := nonceMap.Get(key)
+					if expirationBlock.(*big.Int).Cmp(latestL1Block) <= 0 {
+						nonceMap.Delete(key)
+						if nonceMap.Len() == 0 {
+							delete(r.senderNonces, recipientRand)
+						}
+					}
 				}
 			}
 			r.senderNoncesLock.Unlock()

pm/recipient_test.go

@@ -3,6 +3,7 @@ package pm
 import (
 	"crypto/hmac"
 	"crypto/sha256"
+	"github.com/elliotchance/orderedmap"
 	"math/big"
 	"strings"
 	"sync"
@@ -224,8 +225,8 @@ func TestReceiveTicket_ValidNonWinningTicket(t *testing.T) {
 	recipientRand := genRecipientRand(sender, secret, params)
 	senderNonce := r.(*recipient).senderNonces[recipientRand.String()]
 
-	if senderNonce.nonce != newSenderNonce {
-		t.Errorf("expected senderNonce to be %d, got %d", newSenderNonce, senderNonce.nonce)
+	if senderNonce.Front().Key != newSenderNonce {
+		t.Errorf("expected senderNonce to be %d, got %d", newSenderNonce, senderNonce.Front().Key)
 	}
 }
 
@@ -257,8 +258,8 @@ func TestReceiveTicket_ValidWinningTicket(t *testing.T) {
 	recipientRand := genRecipientRand(sender, secret, params)
 	senderNonce := r.(*recipient).senderNonces[recipientRand.String()]
 
-	if senderNonce.nonce != newSenderNonce {
-		t.Errorf("expected senderNonce to be %d, got %d", newSenderNonce, senderNonce.nonce)
+	if senderNonce.Front().Key != newSenderNonce {
+		t.Errorf("expected senderNonce to be %d, got %d", newSenderNonce, senderNonce.Front().Key)
 	}
 }
 
@@ -308,6 +309,7 @@ func TestReceiveTicket_InvalidSenderNonce(t *testing.T) {
 }
 
 func TestReceiveTicket_ValidNonWinningTicket_Concurrent(t *testing.T) {
+	assert := assert.New(t)
 	sender, b, v, gm, sm, tm, cfg, sig := newRecipientFixtureOrFatal(t)
 	r := newRecipientOrFatal(t, RandAddress(), b, v, gm, sm, tm, cfg)
 	params, err := r.TicketParams(sender, big.NewRat(1, 1))
@@ -330,12 +332,33 @@ func TestReceiveTicket_ValidNonWinningTicket_Concurrent(t *testing.T) {
 			}
 		}(uint32(i))
 	}
-
 	wg.Wait()
+	assert.Zero(errCount)
+}
 
-	if errCount == 0 {
-		t.Error("expected more than zero senderNonce errors for concurrent ticket receipt")
+func TestReceiveTicket_NonceMapFill(t *testing.T) {
+	assert := assert.New(t)
+	sender, b, v, gm, sm, tm, cfg, sig := newRecipientFixtureOrFatal(t)
+	r := newRecipientOrFatal(t, RandAddress(), b, v, gm, sm, tm, cfg)
+	params, err := r.TicketParams(sender, big.NewRat(1, 1))
+	require.Nil(t, err)
+	// fill nonce map to capacity
+	for i := 0; i < maxSenderNonces; i++ {
+		ticket := newTicket(sender, params, uint32(i))
+		_, _, err := r.ReceiveTicket(ticket, sig, params.Seed)
+		assert.NoError(err)
 	}
+	// ensure can't use existing nonce
+	_, _, err = r.ReceiveTicket(newTicket(sender, params, uint32(0)), sig, params.Seed)
+	assert.Error(err)
+
+	// adding new element above capacity - it should exclude nonce 0
+	_, _, err = r.ReceiveTicket(newTicket(sender, params, uint32(maxSenderNonces)), sig, params.Seed)
+
+	// check that we can use nonce 0 now
+	_, _, err = r.ReceiveTicket(newTicket(sender, params, uint32(0)), sig, params.Seed)
+	assert.NoError(err)
+
 }
 
 func TestReceiveTicket_ValidTicket_Expired(t *testing.T) {
@@ -653,30 +676,23 @@ func TestSenderNoncesCleanupLoop(t *testing.T) {
 	tm := &stubTimeManager{}
 
 	r := &recipient{
-		tm:   tm,
-		quit: make(chan struct{}),
-		senderNonces: make(map[string]*struct {
-			nonce           uint32
-			expirationBlock *big.Int
-		}),
+		tm:           tm,
+		quit:         make(chan struct{}),
+		senderNonces: make(map[string]*orderedmap.OrderedMap),
 	}
 
 	// add some senderNonces
 	rand0 := "blastoise"
 	rand1 := "charizard"
 	rand2 := "raichu"
-	r.senderNonces[rand0] = &struct {
-		nonce           uint32
-		expirationBlock *big.Int
-	}{1, big.NewInt(3)}
-	r.senderNonces[rand1] = &struct {
-		nonce           uint32
-		expirationBlock *big.Int
-	}{1, big.NewInt(2)}
-	r.senderNonces[rand2] = &struct {
-		nonce           uint32
-		expirationBlock *big.Int
-	}{1, big.NewInt(1)}
+	r.senderNonces[rand0] = orderedmap.NewOrderedMap()
+	r.senderNonces[rand0].Set(1, big.NewInt(3))
+
+	r.senderNonces[rand1] = orderedmap.NewOrderedMap()
+	r.senderNonces[rand1].Set(1, big.NewInt(2))
+
+	r.senderNonces[rand2] = orderedmap.NewOrderedMap()
+	r.senderNonces[rand2].Set(1, big.NewInt(1))
 
 	go r.senderNoncesCleanupLoop()
 	time.Sleep(20 * time.Millisecond)