From 64d664f0233d6144796f390637a6857dc8fa790c Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Fri, 24 Nov 2023 15:08:15 +0800
Subject: [PATCH 01/19] adr: Un-Ordered Account(support concurrent transactions
 inclusion)

ref: #13009
---
 .../architecture/adr-069-unordered-account.md | 105 ++++++++++++++++++
 1 file changed, 105 insertions(+)
 create mode 100644 docs/architecture/adr-069-unordered-account.md

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
new file mode 100644
index 000000000000..49a05eaabcb9
--- /dev/null
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -0,0 +1,105 @@
+# ADR 069: Un-Ordered Account
+
+## Changelog
+
+* Nov 24, 2023: Initial Draft
+
+## Abstract
+
+Support gaps in account nonce values to support un-ordered(concurrent) transaction inclusion.
+
+## Context
+
+Right now the transactions from a particular sender must be included in strict order because of the nonce value requirement, which makes it tricky to send many transactions concurrently, for example, when a previous pending transaction fails or timeouts, all the following transactions are all blocked. Relayer and exchanges are some typical examples.
+
+The main purpose of nonce value is to protect against replay attack, but for that purpose we only need to make sure the nonce is unique, so we can relax the orderness requirement without lose the uniqueness, in that way we can improve the user experience of concurrent transaction sending.
+
+## Decision
+
+Change the nonce logic to allow gaps to exist, which can be filled by other transactions later, or never filled at all, the prototype implementation use a bitmap to record these gap values.
+
+It's debatable how we should configure the user accounts, for example, should we change the default behavior directly or let user to turn this feature on explicitly, or should we allow user to set different gap capacity for different accounts.
+
+```golang
+const MaxGap = 1024
+
+  type Account struct {
+    ...
+    SequenceNumber int64
++   Gaps *IntSet
+  }
+
+// CheckNonce checks if the input nonce is valid, if yes, modify internal state.
+func(acc *Account) CheckNonce(nonce int64) error {
+  switch {
+    case nonce == acct.SequenceNumber:
+    	return errors.New("nonce is occupied")
+    case nonce >= acct.SequenceNumber + 1:
+      gaps := nonce - acct.SequenceNumber - 1
+      if gaps > MaxGap {
+        return errors.New("max gap is exceeded")
+      }
+    	for i := 0; i < gaps; i++ {
+      	acct.Gaps.Add(i + acct.SequenceNumber + 1)
+    	}
+    	acct.SequenceNumber = nonce
+    case nonce < acct.SequenceNumber:
+    	if !acct.Gaps.Contains(nonce) {
+      	return errors.New("nonce is occupied")
+    	}
+    	acct.Gaps.Remove(nonce)
+  }
+  return nil
+}
+```
+
+Prototype implementation of `IntSet`:
+
+```golang
+type IntSet struct {
+  capacity int
+  bitmap roaringbitmap.BitMap
+}
+
+func NewIntSet(capacity int) *IntSet {
+  return IntSet{
+    capacity: capacity,
+    bitmap: *roaringbitmap.New(),
+  }
+}
+
+func (is *IntSet) Add(n int) {
+  if is.bitmap.Length() >= is.capacity {
+    // pop the minimal one
+    is.Remove(is.bitmap.Minimal())
+  }
+  
+  is.bitmap.Add(n)
+}
+
+func (is *IntSet) Remove(n int) {
+  is.bitmap.Remove(n)
+}
+
+func (is *IntSet) Contains(n int) bool {
+  return is.bitmap.Contains(n)
+}
+```
+
+## Status
+
+Proposed.
+
+## Consequences
+
+### Positive
+
+* Only optional fields are added to `Account`, migration is easy.
+
+### Negative
+
+- Limited runtime overhead.
+
+## References
+
+* https://github.com/cosmos/cosmos-sdk/issues/13009

From 998f092e1d91c5b9d68be20e40efb2ee2a7752f5 Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Fri, 24 Nov 2023 15:11:30 +0800
Subject: [PATCH 02/19] add positive

---
 docs/architecture/adr-069-unordered-account.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 49a05eaabcb9..ad59d491a5c4 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -94,6 +94,7 @@ Proposed.
 
 ### Positive
 
+* Support concurrent transaction inclusion.
 * Only optional fields are added to `Account`, migration is easy.
 
 ### Negative

From 6613f1a481356c35dbb5075f459c815fc12daf0c Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Tue, 28 Nov 2023 09:42:41 +0800
Subject: [PATCH 03/19] update adr

---
 .../architecture/adr-069-unordered-account.md | 124 ++++++++++++------
 1 file changed, 82 insertions(+), 42 deletions(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index ad59d491a5c4..8e12b22d8c31 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -4,58 +4,87 @@
 
 * Nov 24, 2023: Initial Draft
 
+## Status
+
+Proposed
+
 ## Abstract
 
-Support gaps in account nonce values to support un-ordered(concurrent) transaction inclusion.
+Add an extra nonce "lane" to support un-ordered(concurrent) transaction incluion.
 
 ## Context
 
-Right now the transactions from a particular sender must be included in strict order because of the nonce value requirement, which makes it tricky to send many transactions concurrently, for example, when a previous pending transaction fails or timeouts, all the following transactions are all blocked. Relayer and exchanges are some typical examples.
-
-The main purpose of nonce value is to protect against replay attack, but for that purpose we only need to make sure the nonce is unique, so we can relax the orderness requirement without lose the uniqueness, in that way we can improve the user experience of concurrent transaction sending.
+Right now the nonce value (account sequence number) prevents replay-attack and make sure the transactions from the same sender are included into blocks and executed in order. But it makes it tricky to send many transactions concurrently in a reliable way. IBC relayer and crypto exchanges would be typical examples of such use cases.
 
 ## Decision
 
-Change the nonce logic to allow gaps to exist, which can be filled by other transactions later, or never filled at all, the prototype implementation use a bitmap to record these gap values.
+Add an extra nonce lane to support optional un-ordered transaction inclusion, the default lane provides ordered semantic, the same as current behavior, the new one provides the un-ordered semantic. The transaction can choose which lane to use.
+
+One of the design goals is to keep minimal overhead and breakage to the existing users who don't use the new feature.
 
-It's debatable how we should configure the user accounts, for example, should we change the default behavior directly or let user to turn this feature on explicitly, or should we allow user to set different gap capacity for different accounts.
+### Transaction Format
+
+It don't change the transaction format itself, but re-use the high bit of the exisitng 64bits nonce value to identify the lane, `0` being the default ordered lane, `1` being the new unordered lane.
+
+### Account State
+
+The new nonce lane needs to add some optional fields to the account state:
 
 ```golang
-const MaxGap = 1024
+type UnOrderedNonce struct {
+  Sequence uint64
+  Gaps IntSet
+}
 
-  type Account struct {
-    ...
-    SequenceNumber int64
-+   Gaps *IntSet
-  }
+type Account struct {
+  // default to `nil`, only initialized when the new feature is first used.
+  unorderedNonce *UnOrderedNonce
+}
+```
+
+The un-ordered nonce state includes a normal sequence value plus the gap values in recent history, the gap set has a maximum capacity to limit the resource usage, when the capacity is reached, the oldest gap value is simply dropped, which means the pending transaction with that value as nonce will not be accepted anymore.
+
+### Expiration
+
+It would be good to expire the gap nonces after certain timeout is reached, to mitigate the risk that a middleman  intercept an old transaction re-execute it in a long future, which might cause unexpected result.
+
+### Prototype Implementation
 
-// CheckNonce checks if the input nonce is valid, if yes, modify internal state.
-func(acc *Account) CheckNonce(nonce int64) error {
+The prototype implementation use a roaring bitmap to record these gap values.
+
+```golang
+// MaxGap is the maximum gaps a new nonce value can introduce
+const MaxGap = 1024
+
+// CheckNonce checks if the nonce in tx is valid, if yes, also update the internal state.
+func(u *UnOrderedNonce) CheckNonce(nonce int64) error {
   switch {
-    case nonce == acct.SequenceNumber:
+    case nonce == u.Sequence:
+      // special case, the current sequence number must have been occupied
     	return errors.New("nonce is occupied")
-    case nonce >= acct.SequenceNumber + 1:
-      gaps := nonce - acct.SequenceNumber - 1
+    case nonce >= u.Sequence + 1:
+      // the number of gaps introduced by this nonce value, could be zero if it happens to be `u.Sequence + 1`
+      gaps := nonce - u.Sequence - 1
       if gaps > MaxGap {
         return errors.New("max gap is exceeded")
       }
-    	for i := 0; i < gaps; i++ {
-      	acct.Gaps.Add(i + acct.SequenceNumber + 1)
-    	}
-    	acct.SequenceNumber = nonce
-    case nonce < acct.SequenceNumber:
-    	if !acct.Gaps.Contains(nonce) {
-      	return errors.New("nonce is occupied")
+      // record the gaps into the bitmap
+      for i := 0; i < gaps; i++ {
+        acct.Gaps.Add(i + u.Sequence + 1)
+      }
+      // record the latest nonce
+    	u.Sequence = nonce
+    case nonce < u.Sequence:
+      // try to use a gap value
+    	if !u.Gaps.Contains(nonce) {
+        return errors.New("nonce is occupied")
     	}
-    	acct.Gaps.Remove(nonce)
+    	u.Gaps.Remove(nonce)
   }
   return nil
 }
-```
 
-Prototype implementation of `IntSet`:
-
-```golang
+// IntSet is a set of integers with a capacity, when capacity reached, drop the smallest value
 type IntSet struct {
   capacity int
   bitmap roaringbitmap.BitMap
@@ -68,38 +97,49 @@ func NewIntSet(capacity int) *IntSet {
   }
 }
 
-func (is *IntSet) Add(n int) {
-  if is.bitmap.Length() >= is.capacity {
-    // pop the minimal one
-    is.Remove(is.bitmap.Minimal())
+func (is *IntSet) Add(n uint64) {
+  if is.bitmap.GetCardinality() >= is.capacity {
+    // drop the smallest one
+    is.bitmap.Remove(is.bitmap.Minimal())
   }
-  
+
   is.bitmap.Add(n)
 }
 
-func (is *IntSet) Remove(n int) {
+// AddRange adds the integers in [rangeStart, rangeEnd) to the bitmap.
+func (is *IntSet) AddRange(start, end uint64) {
+  n := end - start
+  if is.bitmap.GetCardinality() + n > is.capacity {
+    // drop the smallest ones
+    toDrop := is.bitmap.GetCardinality() + n - is.capacity
+    for i:= 0; i<toDrop; i++ {
+      is.bitmap.Remove(is.bitmap.Minimal())
+    }
+  }
+
+  is.bitmap.AddRange(start, end)
+}
+
+func (is *IntSet) Remove(n uint64) {
   is.bitmap.Remove(n)
 }
 
-func (is *IntSet) Contains(n int) bool {
+func (is *IntSet) Contains(n uint64) bool {
   return is.bitmap.Contains(n)
 }
 ```
 
-## Status
-
-Proposed.
-
 ## Consequences
 
 ### Positive
 
 * Support concurrent transaction inclusion.
-* Only optional fields are added to `Account`, migration is easy.
+* Only optional fields are added to account state, no state migration is needed.
+* Don't need to change transaction format.
 
 ### Negative
 
-- Limited runtime overhead.
+- Some runtime overhead when the new feature is used.
 
 ## References
 

From 345987cf97887b6b63a222c104d10a815dd50d04 Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Tue, 28 Nov 2023 09:44:40 +0800
Subject: [PATCH 04/19] update title

---
 docs/architecture/adr-069-unordered-account.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 8e12b22d8c31..357699b88621 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -1,4 +1,4 @@
-# ADR 069: Un-Ordered Account
+# ADR 069: Un-Ordered Nonce Lane
 
 ## Changelog
 

From 50db7fe208e628d5c3a1182ed84d903c135f0ee2 Mon Sep 17 00:00:00 2001
From: yihuang <huang@crypto.com>
Date: Tue, 28 Nov 2023 09:45:47 +0800
Subject: [PATCH 05/19] Update docs/architecture/adr-069-unordered-account.md

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
---
 docs/architecture/adr-069-unordered-account.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 357699b88621..8323e7ab20e2 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -10,7 +10,7 @@ Proposed
 
 ## Abstract
 
-Add an extra nonce "lane" to support un-ordered(concurrent) transaction incluion.
+Add an extra nonce "lane" to support un-ordered(concurrent) transaction inclusion.
 
 ## Context
 

From 0ec3426dd41db2d41e79741c098362ccd0313e1b Mon Sep 17 00:00:00 2001
From: yihuang <huang@crypto.com>
Date: Tue, 28 Nov 2023 09:45:56 +0800
Subject: [PATCH 06/19] Update docs/architecture/adr-069-unordered-account.md

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
---
 docs/architecture/adr-069-unordered-account.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 8323e7ab20e2..1f55d82dac1e 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -24,7 +24,7 @@ One of the design goals is to keep minimal overhead and breakage to the existing
 
 ### Transaction Format
 
-It don't change the transaction format itself, but re-use the high bit of the exisitng 64bits nonce value to identify the lane, `0` being the default ordered lane, `1` being the new unordered lane.
+It doesn't change the transaction format itself, but re-use the high bit of the exisitng 64bits nonce value to identify the lane, `0` being the default ordered lane, `1` being the new unordered lane.
 
 ### Account State
 

From ebef016818b4be2035c2dba1f422d8e806cf469c Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Tue, 28 Nov 2023 09:47:31 +0800
Subject: [PATCH 07/19] fix tab

---
 docs/architecture/adr-069-unordered-account.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 1f55d82dac1e..4dcbd023fe59 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -61,7 +61,7 @@ func(u *UnOrderedNonce) CheckNonce(nonce int64) error {
   switch {
     case nonce == u.Sequence:
       // special case, the current sequence number must have been occupied
-    	return errors.New("nonce is occupied")
+      return errors.New("nonce is occupied")
     case nonce >= u.Sequence + 1:
       // the number of gaps introduced by this nonce value, could be zero if it happens to be `u.Sequence + 1`
       gaps := nonce - u.Sequence - 1
@@ -73,13 +73,13 @@ func(u *UnOrderedNonce) CheckNonce(nonce int64) error {
         acct.Gaps.Add(i + u.Sequence + 1)
       }
       // record the latest nonce
-    	u.Sequence = nonce
+      u.Sequence = nonce
     case nonce < u.Sequence:
       // try to use a gap value
-    	if !u.Gaps.Contains(nonce) {
+      if !u.Gaps.Contains(nonce) {
         return errors.New("nonce is occupied")
-    	}
-    	u.Gaps.Remove(nonce)
+      }
+      u.Gaps.Remove(nonce)
   }
   return nil
 }
@@ -91,7 +91,7 @@ type IntSet struct {
 }
 
 func NewIntSet(capacity int) *IntSet {
-  return IntSet{
+  return &IntSet{
     capacity: capacity,
     bitmap: *roaringbitmap.New(),
   }

From d95720a620e4642c2d82ae557b838c601d5f17c7 Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Tue, 28 Nov 2023 09:49:16 +0800
Subject: [PATCH 08/19] consequences

---
 docs/architecture/adr-069-unordered-account.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 4dcbd023fe59..7418bd77174d 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -135,7 +135,8 @@ func (is *IntSet) Contains(n uint64) bool {
 
 * Support concurrent transaction inclusion.
 * Only optional fields are added to account state, no state migration is needed.
-* Don't need to change transaction format.
+* No runtime overhead when the new feature is not used.
+* No need to change transaction format.
 
 ### Negative
 

From aad93e25da5972ab781adc18e723b03dda85a97e Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Tue, 28 Nov 2023 10:06:53 +0800
Subject: [PATCH 09/19] add expiration logic

---
 .../architecture/adr-069-unordered-account.md | 54 +++++++++++++------
 1 file changed, 39 insertions(+), 15 deletions(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 7418bd77174d..88e018aa5c81 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -33,6 +33,7 @@ The new nonce lane needs to add some optional fields to the account state:
 ```golang
 type UnOrderedNonce struct {
   Sequence uint64
+  Timestamp uint64  // the block time when the Sequence is updated
   Gaps IntSet
 }
 
@@ -46,40 +47,63 @@ The un-ordered nonce state includes a normal sequence value plus the gap values
 
 ### Expiration
 
-It would be good to expire the gap nonces after certain timeout is reached, to mitigate the risk that a middleman  intercept an old transaction re-execute it in a long future, which might cause unexpected result.
+It would be good to expire the gap nonces after certain timeout is reached, to mitigate the risk that a middleman intercept an old transaction and re-execute it in a longer future, which might cause unexpected result.
 
 ### Prototype Implementation
 
 The prototype implementation use a roaring bitmap to record these gap values.
 
 ```golang
-// MaxGap is the maximum gaps a new nonce value can introduce
-const MaxGap = 1024
+const (
+  // GapsCapacity is the capacity of the set of gap values.
+  GapsCapacity = 1024
+  // MaxGap is the maximum gaps a new nonce value can introduce
+  MaxGap = 1024
+  // GapsExpirationDuration is the duration in seconds for the gaps to expire
+  GapsExpirationDuration = 60 * 60 * 24
+)
+
+// getGaps returns the gap set, or create a new one if it's expired
+func(u *UnOrderedNonce) getGaps(timestamp uint64) *IntSet {
+  if timestamp > u.Timestamp + GapsExpirationDuration {
+    return NewIntSet(GapsCapacity)
+  }
+
+  return &u.Gaps
+}
 
 // CheckNonce checks if the nonce in tx is valid, if yes, also update the internal state.
-func(u *UnOrderedNonce) CheckNonce(nonce int64) error {
+func(u *UnOrderedNonce) CheckNonce(nonce uint64, timestamp uint64) error {
   switch {
     case nonce == u.Sequence:
       // special case, the current sequence number must have been occupied
       return errors.New("nonce is occupied")
+
     case nonce >= u.Sequence + 1:
       // the number of gaps introduced by this nonce value, could be zero if it happens to be `u.Sequence + 1`
       gaps := nonce - u.Sequence - 1
       if gaps > MaxGap {
         return errors.New("max gap is exceeded")
       }
+
+      gapSet := acct.getGaps(timestamp)
       // record the gaps into the bitmap
-      for i := 0; i < gaps; i++ {
-        acct.Gaps.Add(i + u.Sequence + 1)
-      }
-      // record the latest nonce
+      gapSet.AddRange(u.Sequence + 1, u.Sequence + gaps + 1)
+
+      // update the latest nonce
+      u.Gaps = *gapSet
       u.Sequence = nonce
-    case nonce < u.Sequence:
-      // try to use a gap value
-      if !u.Gaps.Contains(nonce) {
-        return errors.New("nonce is occupied")
+      u.Timestamp = timestamp
+
+    default:
+      // `nonce < u.Sequence`, the tx try to use a historical nonce
+      gapSet := acct.getGaps(timestamp)
+      if !gapSet.Contains(nonce) {
+        return errors.New("nonce is occupied or expired")
       }
-      u.Gaps.Remove(nonce)
+
+      gapSet.Remove(nonce)
+      u.Gaps = *gapSet
   }
   return nil
 }
@@ -110,9 +134,9 @@ func (is *IntSet) Add(n uint64) {
 func (is *IntSet) AddRange(start, end uint64) {
   n := end - start
   if is.bitmap.GetCardinality() + n > is.capacity {
-    // drop the smallest ones
+    // drop the smallest ones until the capacity is not exceeded
     toDrop := is.bitmap.GetCardinality() + n - is.capacity
-    for i:= 0; i<toDrop; i++ {
+    for i := uint64(0); i < toDrop; i++ {
       is.bitmap.Remove(is.bitmap.Minimal())
     }
   }

From d62abcd94809fdc4f2fc52ffa2e7a20170b81145 Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Tue, 28 Nov 2023 10:15:36 +0800
Subject: [PATCH 10/19] add high level account logic

---
 .../architecture/adr-069-unordered-account.md | 35 +++++++++++++++----
 1 file changed, 29 insertions(+), 6 deletions(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 88e018aa5c81..2708d7228427 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -53,6 +53,29 @@ It would be good to expire the gap nonces after certain timeout is reached, to m
 
 The prototype implementation use a roaring bitmap to record these gap values.
 
+```golang
+const (
+  MSBCheckMask = 1 << 63
+  MSBClearMask = ^(1 << 63)
+)
+
+// CheckNonce switches to un-ordered lane if the MSB of the nonce is set.
+func (acct *Account) CheckNonce(nonce uint64, blockTime uint64) error {
+  if nonce & MSBCheckMask != 0 {
+    nonce &= MSBClearMask
+
+    if acct.unorderedNonce == nil {
+      acct.unorderedNonce = NewUnOrderedNonce()
+    }
+    return acct.unorderedNonce.CheckNonce(nonce, blockTime)
+  }
+
+  // current nonce logic
+}
+```
+
+
+
 ```golang
 const (
   // GapsCapacity is the capacity of the set of gap values.
@@ -64,8 +87,8 @@ const (
 )
 
 // getGaps returns the gap set, or create a new one if it's expired
-func(u *UnOrderedNonce) getGaps(timestamp uint64) *IntSet {
-  if timestamp > u.Timestamp + GapsExpirationDuration {
+func(u *UnOrderedNonce) getGaps(blockTime uint64) *IntSet {
+  if blockTime > u.Timestamp + GapsExpirationDuration {
     return NewIntSet(GapsCapacity)
   }
 
@@ -73,7 +96,7 @@ func(u *UnOrderedNonce) getGaps(timestamp uint64) *IntSet {
 }
 
 // CheckNonce checks if the nonce in tx is valid, if yes, also update the internal state.
-func(u *UnOrderedNonce) CheckNonce(nonce uint64, timestamp uint64) error {
+func(u *UnOrderedNonce) CheckNonce(nonce uint64, blockTime uint64) error {
   switch {
     case nonce == u.Sequence:
       // special case, the current sequence number must have been occupied
@@ -86,18 +109,18 @@ func(u *UnOrderedNonce) CheckNonce(nonce uint64, timestamp uint64) error {
         return errors.New("max gap is exceeded")
       }
 
-      gapSet := acct.getGaps(timestamp)
+      gapSet := acct.getGaps(blockTime)
       // record the gaps into the bitmap
       gapSet.AddRange(u.Sequence + 1, u.Sequence + gaps + 1)
 
       // update the latest nonce
       u.Gaps = *gapSet
       u.Sequence = nonce
-      u.Timestamp = timestamp
+      u.Timestamp = blockTime
 
     default:
       // `nonce < u.Sequence`, the tx try to use a historical nonce
-      gapSet := acct.getGaps(timestamp)
+      gapSet := acct.getGaps(blockTime)
       if !gapSet.Contains(nonce) {
         return errors.New("nonce is occupied or expired")
       }

From 8ef01585b93fb09b4d37b591e98a68a29b2b5b6d Mon Sep 17 00:00:00 2001
From: yihuang <huang@crypto.com>
Date: Tue, 28 Nov 2023 10:29:33 +0800
Subject: [PATCH 11/19] Apply suggestions from code review

---
 docs/architecture/adr-069-unordered-account.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 2708d7228427..6ac9022d7447 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -18,13 +18,13 @@ Right now the nonce value (account sequence number) prevents replay-attack and m
 
 ## Decision
 
-Add an extra nonce lane to support optional un-ordered transaction inclusion, the default lane provides ordered semantic, the same as current behavior, the new one provides the un-ordered semantic. The transaction can choose which lane to use.
+Add an extra nonce lane to support optional un-ordered transaction inclusion, the default lane provides ordered semantic the same as before, the new one will provide un-ordered semantic. The transaction can choose which lane to use.
 
 One of the design goals is to keep minimal overhead and breakage to the existing users who don't use the new feature.
 
 ### Transaction Format
 
-It doesn't change the transaction format itself, but re-use the high bit of the exisitng 64bits nonce value to identify the lane, `0` being the default ordered lane, `1` being the new unordered lane.
+It doesn't change the transaction format itself, but re-use the MSB(most significant bit) of the existing 64bits nonce value to identify the lane, `0` being the default ordered lane, `1` being the new un-ordered lane.
 
 ### Account State
 
@@ -43,7 +43,7 @@ type Account struct {
 }
 ```
 
-The un-ordered nonce state includes a normal sequence value plus the gap values in recent history, the gap set has a maximum capacity to limit the resource usage, when the capacity is reached, the oldest gap value is simply dropped, which means the pending transaction with that value as nonce will not be accepted anymore.
+The un-ordered nonce state includes a normal sequence value plus the set of unused(gap) values in recent history, these recorded gap values can be reused by future transactions, after used they are removed from the set and can't be used again, the gap set has a maximum capacity to limit the resource usage, when the capacity is reached, the oldest gap value is removed, which also makes the pending transaction using that value as nonce will not be accepted anymore.
 
 ### Expiration
 

From 8ab25370326701dcf91aef04d756f97c319e22c1 Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Thu, 30 Nov 2023 16:51:02 +0800
Subject: [PATCH 12/19] add field to BaseAccount

---
 .../architecture/adr-069-unordered-account.md | 87 ++++++++++---------
 1 file changed, 44 insertions(+), 43 deletions(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 6ac9022d7447..d01ab20cf5ac 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -10,28 +10,41 @@ Proposed
 
 ## Abstract
 
-Add an extra nonce "lane" to support un-ordered(concurrent) transaction inclusion.
+We propose to add an extra nonce "lane" to support un-ordered (concurrent) transaction inclusion.
 
 ## Context
 
-Right now the nonce value (account sequence number) prevents replay-attack and make sure the transactions from the same sender are included into blocks and executed in order. But it makes it tricky to send many transactions concurrently in a reliable way. IBC relayer and crypto exchanges would be typical examples of such use cases.
+As of today, the nonce value (account sequence number) prevents replay-attack and ensures the transactions from the same sender are included into blocks and executed in sequential order. However it makes it tricky to send many transactions concurrently in a reliable way. IBC relayer and crypto exchanges would be typical examples of such use cases.
 
 ## Decision
 
-Add an extra nonce lane to support optional un-ordered transaction inclusion, the default lane provides ordered semantic the same as before, the new one will provide un-ordered semantic. The transaction can choose which lane to use.
+Add an extra un-ordered nonce lane to support un-ordered transaction inclusion, it works at the same time as the default ordered lane, the transaction builder can choose either lane to use.
 
-One of the design goals is to keep minimal overhead and breakage to the existing users who don't use the new feature.
+The un-ordered nonce lane accepts nonce values that is greater than the current sequence number plus 1, which effectively creates "gaps" of nonce values, those gap values are tracked in the account state, and can be filled by future transactions, thus allows transactions to be executed in a un-ordered way.
+
+It also tracks the last block time when the latest nonce is updated, and expire the gaps certain timeout reached, to mitigate the risk that a middleman intercept an old transaction and re-execute it in a longer future, which might cause unexpected result.
+
+The design introducs almost zero overhead for users who don't use the new feature.
 
 ### Transaction Format
 
-It doesn't change the transaction format itself, but re-use the MSB(most significant bit) of the existing 64bits nonce value to identify the lane, `0` being the default ordered lane, `1` being the new un-ordered lane.
+Add a boolean field `unordered` to the `BaseAccount` message, when it's set to `true`, the transaction will use the un-ordered lane, otherwise the default ordered lane.
+
+```protobuf
+message BaseAccount {
+  ...
+  uint64 account_number = 3;
+  uint64 sequence       = 4;
+  boolean unordered     = 5;
+}
+```
 
 ### Account State
 
-The new nonce lane needs to add some optional fields to the account state:
+Add some optional fields to the account state:
 
 ```golang
-type UnOrderedNonce struct {
+type UnorderedNonceManager struct {
   Sequence uint64
   Timestamp uint64  // the block time when the Sequence is updated
   Gaps IntSet
@@ -39,42 +52,31 @@ type UnOrderedNonce struct {
 
 type Account struct {
   // default to `nil`, only initialized when the new feature is first used.
-  unorderedNonce *UnOrderedNonce
+  unorderedNonceManager *UnorderedNonceManager
 }
 ```
 
 The un-ordered nonce state includes a normal sequence value plus the set of unused(gap) values in recent history, these recorded gap values can be reused by future transactions, after used they are removed from the set and can't be used again, the gap set has a maximum capacity to limit the resource usage, when the capacity is reached, the oldest gap value is removed, which also makes the pending transaction using that value as nonce will not be accepted anymore.
 
-### Expiration
-
-It would be good to expire the gap nonces after certain timeout is reached, to mitigate the risk that a middleman intercept an old transaction and re-execute it in a longer future, which might cause unexpected result.
+### Nonce Validation Logic
 
-### Prototype Implementation
-
-The prototype implementation use a roaring bitmap to record these gap values.
+The prototype implementation use a roaring bitmap to record these gap values, where the set bits represents the the gaps.
 
 ```golang
-const (
-  MSBCheckMask = 1 << 63
-  MSBClearMask = ^(1 << 63)
-)
-
 // CheckNonce switches to un-ordered lane if the MSB of the nonce is set.
-func (acct *Account) CheckNonce(nonce uint64, blockTime uint64) error {
-  if nonce & MSBCheckMask != 0 {
-    nonce &= MSBClearMask
-
-    if acct.unorderedNonce == nil {
-      acct.unorderedNonce = NewUnOrderedNonce()
+func (acct *Account) CheckNonce(nonce uint64, unordered bool, blockTime uint64) error {
+  if unordered {
+    if acct.unorderedNonceManager == nil {
+      acct.unorderedNonceManager = NewUnorderedNonceManager()
     }
-    return acct.unorderedNonce.CheckNonce(nonce, blockTime)
+    return acct.unorderedNonceManager.CheckNonce(nonce, blockTime)
   }
 
-  // current nonce logic
+  // current ordered nonce logic
 }
 ```
 
-
+### UnorderedNonceManager
 
 ```golang
 const (
@@ -87,46 +89,46 @@ const (
 )
 
 // getGaps returns the gap set, or create a new one if it's expired
-func(u *UnOrderedNonce) getGaps(blockTime uint64) *IntSet {
-  if blockTime > u.Timestamp + GapsExpirationDuration {
+func(unm *UnorderedNonceManager) getGaps(blockTime uint64) *IntSet {
+  if blockTime > unm.Timestamp + GapsExpirationDuration {
     return NewIntSet(GapsCapacity)
   }
 
-  return &u.Gaps
+  return &unm.Gaps
 }
 
 // CheckNonce checks if the nonce in tx is valid, if yes, also update the internal state.
-func(u *UnOrderedNonce) CheckNonce(nonce uint64, blockTime uint64) error {
+func(unm *UnorderedNonceManager) CheckNonce(nonce uint64, blockTime uint64) error {
   switch {
-    case nonce == u.Sequence:
+    case nonce == unm.Sequence:
       // special case, the current sequence number must have been occupied
       return errors.New("nonce is occupied")
 
-    case nonce >= u.Sequence + 1:
-      // the number of gaps introduced by this nonce value, could be zero if it happens to be `u.Sequence + 1`
-      gaps := nonce - u.Sequence - 1
+    case nonce > unm.Sequence:
+      // the number of gaps introduced by this nonce value, could be zero if it happens to be `unm.Sequence + 1`
+      gaps := nonce - unm.Sequence - 1
       if gaps > MaxGap {
         return errors.New("max gap is exceeded")
       }
 
       gapSet := acct.getGaps(blockTime)
       // record the gaps into the bitmap
-      gapSet.AddRange(u.Sequence + 1, u.Sequence + gaps + 1)
+      gapSet.AddRange(unm.Sequence + 1, unm.Sequence + gaps + 1)
 
       // update the latest nonce
-      u.Gaps = *gapSet
-      u.Sequence = nonce
-      u.Timestamp = blockTime
+      unm.Gaps = *gapSet
+      unm.Sequence = nonce
+      unm.Timestamp = blockTime
 
     default:
-      // `nonce < u.Sequence`, the tx try to use a historical nonce
+      // `nonce < unm.Sequence`, the tx try to use a historical nonce
       gapSet := acct.getGaps(blockTime)
       if !gapSet.Contains(nonce) {
         return errors.New("nonce is occupied or expired")
       }
 
       gapSet.Remove(nonce)
-      u.Gaps = *gapSet
+      unm.Gaps = *gapSet
   }
   return nil
 }
@@ -183,7 +185,6 @@ func (is *IntSet) Contains(n uint64) bool {
 * Support concurrent transaction inclusion.
 * Only optional fields are added to account state, no state migration is needed.
 * No runtime overhead when the new feature is not used.
-* No need to change transaction format.
 
 ### Negative
 

From 2c32a75e8ccdfc6d9ac434bb481e3d77987e6535 Mon Sep 17 00:00:00 2001
From: yihuang <huang@crypto.com>
Date: Thu, 30 Nov 2023 16:57:49 +0800
Subject: [PATCH 13/19] Update docs/architecture/adr-069-unordered-account.md

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
---
 docs/architecture/adr-069-unordered-account.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index d01ab20cf5ac..0a1bcb655c00 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -111,7 +111,7 @@ func(unm *UnorderedNonceManager) CheckNonce(nonce uint64, blockTime uint64) erro
         return errors.New("max gap is exceeded")
       }
 
-      gapSet := acct.getGaps(blockTime)
+      gapSet := unm.getGaps(blockTime)
       // record the gaps into the bitmap
       gapSet.AddRange(unm.Sequence + 1, unm.Sequence + gaps + 1)
 

From a2df33e78e5c1ab8224e20e3fb866d02118dbd61 Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Mon, 4 Dec 2023 16:08:16 +0800
Subject: [PATCH 14/19] rewrite to use tx hash dictionary

---
 .../architecture/adr-069-unordered-account.md | 182 +++++-------------
 1 file changed, 53 insertions(+), 129 deletions(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 0a1bcb655c00..a88667d3f293 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -1,8 +1,8 @@
-# ADR 069: Un-Ordered Nonce Lane
+# ADR 070: Un-Ordered Transaction Inclusion
 
 ## Changelog
 
-* Nov 24, 2023: Initial Draft
+* Dec 4, 2023: Initial Draft
 
 ## Status
 
@@ -10,185 +10,109 @@ Proposed
 
 ## Abstract
 
-We propose to add an extra nonce "lane" to support un-ordered (concurrent) transaction inclusion.
+We propose a way to do replay-attack protection without enforcing the order of transactions, it don't use nonce at all. In this way we can support un-ordered transaction inclusion.
 
 ## Context
 
-As of today, the nonce value (account sequence number) prevents replay-attack and ensures the transactions from the same sender are included into blocks and executed in sequential order. However it makes it tricky to send many transactions concurrently in a reliable way. IBC relayer and crypto exchanges would be typical examples of such use cases.
+As of today, the nonce value (account sequence number) prevents replay-attack and ensures the transactions from the same sender are included into blocks and executed in sequential order. However it makes it tricky to send many transactions from the same sender concurrently in a reliable way. IBC relayer and crypto exchanges are typical examples of such use cases.
 
 ## Decision
 
-Add an extra un-ordered nonce lane to support un-ordered transaction inclusion, it works at the same time as the default ordered lane, the transaction builder can choose either lane to use.
+We propose to add a boolean field `unordered` to transaction body to mark "un-ordered" transactions.
 
-The un-ordered nonce lane accepts nonce values that is greater than the current sequence number plus 1, which effectively creates "gaps" of nonce values, those gap values are tracked in the account state, and can be filled by future transactions, thus allows transactions to be executed in a un-ordered way.
+Un-ordered transactions will bypass the nonce rules and follow the rules described below instead, in contrary, the default transactions are not impacted by this proposal, they'll follow the nonce rules the same as before.
 
-It also tracks the last block time when the latest nonce is updated, and expire the gaps certain timeout reached, to mitigate the risk that a middleman intercept an old transaction and re-execute it in a longer future, which might cause unexpected result.
+When an un-ordered transaction are included into block, the transaction hash is recorded in a dictionary, new transactions are checked against this dictionary for duplicates, and to prevent the dictionary grow indefinitly, the transaction must specify `timeout_height` for expiration, so it's safe to removed it from the dictionary after it's expired.
 
-The design introducs almost zero overhead for users who don't use the new feature.
+The dictionary can be simply implemented as an in-memory golang map, a preliminary analysis shows that the memory consumption won't be too big, for example `32M = 32 * 1024 * 1024` can support 1024 blocks where each block contains 1024 unordered transactions. For safty, we should limit the range of `timeout_height` to prevent very long expiration, and limit the size of the dictionary.
 
 ### Transaction Format
 
-Add a boolean field `unordered` to the `BaseAccount` message, when it's set to `true`, the transaction will use the un-ordered lane, otherwise the default ordered lane.
-
 ```protobuf
-message BaseAccount {
+message TxBody {
   ...
-  uint64 account_number = 3;
-  uint64 sequence       = 4;
-  boolean unordered     = 5;
-}
-```
-
-### Account State
-
-Add some optional fields to the account state:
-
-```golang
-type UnorderedNonceManager struct {
-  Sequence uint64
-  Timestamp uint64  // the block time when the Sequence is updated
-  Gaps IntSet
-}
 
-type Account struct {
-  // default to `nil`, only initialized when the new feature is first used.
-  unorderedNonceManager *UnorderedNonceManager
+  boolean unordered = 4; 
 }
 ```
 
-The un-ordered nonce state includes a normal sequence value plus the set of unused(gap) values in recent history, these recorded gap values can be reused by future transactions, after used they are removed from the set and can't be used again, the gap set has a maximum capacity to limit the resource usage, when the capacity is reached, the oldest gap value is removed, which also makes the pending transaction using that value as nonce will not be accepted anymore.
-
-### Nonce Validation Logic
+### Ante Handlers
 
-The prototype implementation use a roaring bitmap to record these gap values, where the set bits represents the the gaps.
+Bypass the nonce decorator for un-ordered transactions.
 
 ```golang
-// CheckNonce switches to un-ordered lane if the MSB of the nonce is set.
-func (acct *Account) CheckNonce(nonce uint64, unordered bool, blockTime uint64) error {
-  if unordered {
-    if acct.unorderedNonceManager == nil {
-      acct.unorderedNonceManager = NewUnorderedNonceManager()
-    }
-    return acct.unorderedNonceManager.CheckNonce(nonce, blockTime)
+func (isd IncrementSequenceDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool, next sdk.AnteHandler) (sdk.Context, error) {
+  if tx.UnOrdered() {
+    return next(ctx, tx, simulate)
   }
-
-  // current ordered nonce logic
+  
+  // the previous logic
 }
 ```
 
-### UnorderedNonceManager
+A decorator for the new logic.
 
 ```golang
+type TxHash [32]byte
+
 const (
-  // GapsCapacity is the capacity of the set of gap values.
-  GapsCapacity = 1024
-  // MaxGap is the maximum gaps a new nonce value can introduce
-  MaxGap = 1024
-  // GapsExpirationDuration is the duration in seconds for the gaps to expire
-  GapsExpirationDuration = 60 * 60 * 24
-)
+  // MaxNumberOfTxHash * 32 = 128M max memory usage
+  MaxNumberOfTxHash = 1024 * 1024 * 4
 
-// getGaps returns the gap set, or create a new one if it's expired
-func(unm *UnorderedNonceManager) getGaps(blockTime uint64) *IntSet {
-  if blockTime > unm.Timestamp + GapsExpirationDuration {
-    return NewIntSet(GapsCapacity)
-  }
+  // MaxUnOrderedTTL defines the maximum ttl an un-order tx can set
+  MaxUnOrderedTTL = 1024
+)
 
-  return &unm.Gaps
+type DedupTxDecorator struct {
+  hashes map[TxHash]struct{}
 }
 
-// CheckNonce checks if the nonce in tx is valid, if yes, also update the internal state.
-func(unm *UnorderedNonceManager) CheckNonce(nonce uint64, blockTime uint64) error {
-  switch {
-    case nonce == unm.Sequence:
-      // special case, the current sequence number must have been occupied
-      return errors.New("nonce is occupied")
-
-    case nonce > unm.Sequence:
-      // the number of gaps introduced by this nonce value, could be zero if it happens to be `unm.Sequence + 1`
-      gaps := nonce - unm.Sequence - 1
-      if gaps > MaxGap {
-        return errors.New("max gap is exceeded")
-      }
-
-      gapSet := unm.getGaps(blockTime)
-      // record the gaps into the bitmap
-      gapSet.AddRange(unm.Sequence + 1, unm.Sequence + gaps + 1)
-
-      // update the latest nonce
-      unm.Gaps = *gapSet
-      unm.Sequence = nonce
-      unm.Timestamp = blockTime
-
-    default:
-      // `nonce < unm.Sequence`, the tx try to use a historical nonce
-      gapSet := acct.getGaps(blockTime)
-      if !gapSet.Contains(nonce) {
-        return errors.New("nonce is occupied or expired")
-      }
-
-      gapSet.Remove(nonce)
-      unm.Gaps = *gapSet
+func (dtd *DedupTxDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool, next sdk.AnteHandler) (sdk.Context, error) {
+  // only apply to un-ordered transactions
+  if !tx.UnOrdered() {
+    return next(ctx, tx, simulate)
   }
-  return nil
-}
-
-// IntSet is a set of integers with a capacity, when capacity reached, drop the smallest value
-type IntSet struct {
-  capacity int
-  bitmap roaringbitmap.BitMap
-}
 
-func NewIntSet(capacity int) *IntSet {
-  return &IntSet{
-    capacity: capacity,
-    bitmap: *roaringbitmap.New(),
+  if tx.TimeoutHeight() == 0 {
+    return nil, errorsmod.Wrap(sdkerrors.ErrLogic, "unordered tx must set timeout-height")
   }
-}
 
-func (is *IntSet) Add(n uint64) {
-  if is.bitmap.GetCardinality() >= is.capacity {
-    // drop the smallest one
-    is.bitmap.Remove(is.bitmap.Minimal())
+  if tx.TimeoutHeight() > ctx.BlockHeight() + MaxUnOrderedTTL {
+    return nil, errorsmod.Wrapf(sdkerrors.ErrLogic, "unordered tx ttl exceeds %d", MaxUnOrderedTTL)
   }
 
-  is.bitmap.Add(n)
-}
-
-// AddRange adds the integers in [rangeStart, rangeEnd) to the bitmap.
-func (is *IntSet) AddRange(start, end uint64) {
-  n := end - start
-  if is.bitmap.GetCardinality() + n > is.capacity {
-    // drop the smallest ones until the capacity is not exceeded
-    toDrop := is.bitmap.GetCardinality() + n - is.capacity
-    for i := uint64(0); i < toDrop; i++ {
-      is.bitmap.Remove(is.bitmap.Minimal())
+  if !ctx.IsCheckTx() {
+    // a new tx included in the block, add the hash to the dictionary
+    if len(dtd.hashes) >= MaxNumberOfTxHash {
+      return nil, errorsmod.Wrap(sdkerrors.ErrLogic, "dedup map is full")
+    }
+    dtd.hashes[tx.Hash()] = struct{}
+  } else {
+    // check for duplicates
+    if _, ok := dtd.hashes[tx.Hash()]; ok {
+      return nil, errorsmod.Wrap(sdkerrors.ErrLogic, "tx is dupliated")
     }
   }
 
-  is.bitmap.AddRange(start, end)
+  return next(ctx, tx, simulate)
 }
+```
 
-func (is *IntSet) Remove(n uint64) {
-  is.bitmap.Remove(n)
-}
+### Start Up
 
-func (is *IntSet) Contains(n uint64) bool {
-  return is.bitmap.Contains(n)
-}
-```
+On start up, the node needs to re-fill the tx hash dictionary by scanning `MaxUnOrderedTTL` number of historical blocks for un-ordered transactions.
+
+An alternative design is to store the tx hash dictionary in kv store, then no need to warm up on start up.
 
 ## Consequences
 
 ### Positive
 
-* Support concurrent transaction inclusion.
-* Only optional fields are added to account state, no state migration is needed.
-* No runtime overhead when the new feature is not used.
+* Support un-ordered and concurrent transaction inclusion.
 
 ### Negative
 
-- Some runtime overhead when the new feature is used.
+- Start up overhead to scan historical blocks.
 
 ## References
 

From 2eafdbbf84764845352409aaa83b2bfdeb3b62b7 Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Mon, 4 Dec 2023 17:10:09 +0800
Subject: [PATCH 15/19] add expiration at endblocker

---
 .../architecture/adr-069-unordered-account.md | 72 +++++++++++++++++--
 1 file changed, 65 insertions(+), 7 deletions(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index a88667d3f293..c64d33dc6328 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -20,7 +20,7 @@ As of today, the nonce value (account sequence number) prevents replay-attack an
 
 We propose to add a boolean field `unordered` to transaction body to mark "un-ordered" transactions.
 
-Un-ordered transactions will bypass the nonce rules and follow the rules described below instead, in contrary, the default transactions are not impacted by this proposal, they'll follow the nonce rules the same as before.
+Un-ordered transactions will bypass the nonce rules and follow the rules described below instead, in contrary, the default ordered transactions are not impacted by this proposal, they'll follow the nonce rules the same as before.
 
 When an un-ordered transaction are included into block, the transaction hash is recorded in a dictionary, new transactions are checked against this dictionary for duplicates, and to prevent the dictionary grow indefinitly, the transaction must specify `timeout_height` for expiration, so it's safe to removed it from the dictionary after it's expired.
 
@@ -36,6 +36,60 @@ message TxBody {
 }
 ```
 
+### `DedupTxHashManager`
+
+```golang
+// can reduce frequency we check the expiration.
+const ExpireCheckInterval = 1
+
+// DedupTxHashManager contains the tx hash dictionary for duplicates checking,
+// and expire them when block number progresses.
+type DedupTxHashManager struct {
+  // tx hash -> expire block number
+  // for duplicates checking and expiration
+  hashes map[TxHash]uint64
+}
+
+func (dtm *DedupTxHashManager) Contains(hash TxHash) (ok bool) {
+  dtm.mutex.RLock()
+  defer dtm.mutex.RUnlock()
+
+  _, ok = dtm.hashes[hash]
+  return
+}
+
+func (dtm *DedupTxHashManager) Size() int {
+  dtm.mutex.RLock()
+  defer dtm.mutex.RUnlock()
+
+  return len(dtm).hashes
+}
+
+func (dtm *DedupTxHashManager) Add(hash TxHash, expire uint64) (ok bool) {
+  dtm.mutex.Lock()
+  defer dtm.mutex.Unlock()
+
+  dtm.hashes[hash] = expire
+  return
+}
+
+// EndBlock remove expired tx hashes, need to wire in abci cycles.
+func (dtm *DedupTxHashManager) EndBlock(ctx sdk.Context) {
+  if ctx.BlockNumber() % ExpireCheckInterval != 0 {
+    return
+  }
+
+  dtm.mutex.Lock()
+  defer dtm.mutex.Unlock()
+
+  for k, expire := range dtm.hashes {
+    if ctx.BlockNumber() > expire {
+      delete(dtm.hashes, k)
+    }
+  }
+}
+```
+
 ### Ante Handlers
 
 Bypass the nonce decorator for un-ordered transactions.
@@ -64,7 +118,7 @@ const (
 )
 
 type DedupTxDecorator struct {
-  hashes map[TxHash]struct{}
+  m *DedupTxHashManager
 }
 
 func (dtd *DedupTxDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool, next sdk.AnteHandler) (sdk.Context, error) {
@@ -83,14 +137,14 @@ func (dtd *DedupTxDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate boo
 
   if !ctx.IsCheckTx() {
     // a new tx included in the block, add the hash to the dictionary
-    if len(dtd.hashes) >= MaxNumberOfTxHash {
+    if dtd.m.Size() >= MaxNumberOfTxHash {
       return nil, errorsmod.Wrap(sdkerrors.ErrLogic, "dedup map is full")
     }
-    dtd.hashes[tx.Hash()] = struct{}
+    dtd.m.Add(tx.Hash(), tx.TimeoutHeight())
   } else {
     // check for duplicates
-    if _, ok := dtd.hashes[tx.Hash()]; ok {
-      return nil, errorsmod.Wrap(sdkerrors.ErrLogic, "tx is dupliated")
+    if dtd.m.Contains(tx.Hash()) {
+      return nil, errorsmod.Wrap(sdkerrors.ErrLogic, "tx is duplicated")
     }
   }
 
@@ -98,9 +152,13 @@ func (dtd *DedupTxDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate boo
 }
 ```
 
+### EndBlocker
+
+Wire up the `EndBlock` method of `DedupTxHashManager` into the application's abci life cycle.
+
 ### Start Up
 
-On start up, the node needs to re-fill the tx hash dictionary by scanning `MaxUnOrderedTTL` number of historical blocks for un-ordered transactions.
+On start up, the node needs to re-fill the tx hash dictionary of `DedupTxHashManager` by scanning `MaxUnOrderedTTL` number of historical blocks for un-ordered transactions.
 
 An alternative design is to store the tx hash dictionary in kv store, then no need to warm up on start up.
 

From 174cb3b80fdf4f9b8a8820836bda9a030b6a3691 Mon Sep 17 00:00:00 2001
From: yihuang <huang@crypto.com>
Date: Tue, 5 Dec 2023 15:20:18 +0800
Subject: [PATCH 16/19] Update docs/architecture/adr-069-unordered-account.md

Co-authored-by: Aleksandr Bezobchuk <alexanderbez@users.noreply.github.com>
---
 docs/architecture/adr-069-unordered-account.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index c64d33dc6328..4bff667f3fc7 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -22,7 +22,7 @@ We propose to add a boolean field `unordered` to transaction body to mark "un-or
 
 Un-ordered transactions will bypass the nonce rules and follow the rules described below instead, in contrary, the default ordered transactions are not impacted by this proposal, they'll follow the nonce rules the same as before.
 
-When an un-ordered transaction are included into block, the transaction hash is recorded in a dictionary, new transactions are checked against this dictionary for duplicates, and to prevent the dictionary grow indefinitly, the transaction must specify `timeout_height` for expiration, so it's safe to removed it from the dictionary after it's expired.
+When an un-ordered transaction is included into a block, the transaction hash is recorded in a dictionary. New transactions are checked against this dictionary for duplicates, and to prevent the dictionary grow indefinitely, the transaction must specify `timeout_height` for expiration, so it's safe to removed it from the dictionary after it's expired.
 
 The dictionary can be simply implemented as an in-memory golang map, a preliminary analysis shows that the memory consumption won't be too big, for example `32M = 32 * 1024 * 1024` can support 1024 blocks where each block contains 1024 unordered transactions. For safty, we should limit the range of `timeout_height` to prevent very long expiration, and limit the size of the dictionary.
 

From 1dc13edfb1b5bdc31675e15b434f5f2d85eed3b1 Mon Sep 17 00:00:00 2001
From: yihuang <huang@crypto.com>
Date: Tue, 5 Dec 2023 15:20:43 +0800
Subject: [PATCH 17/19] Update docs/architecture/adr-069-unordered-account.md

Co-authored-by: Aleksandr Bezobchuk <alexanderbez@users.noreply.github.com>
---
 docs/architecture/adr-069-unordered-account.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 4bff667f3fc7..ec9d5647dff5 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -10,7 +10,7 @@ Proposed
 
 ## Abstract
 
-We propose a way to do replay-attack protection without enforcing the order of transactions, it don't use nonce at all. In this way we can support un-ordered transaction inclusion.
+We propose a way to do replay-attack protection without enforcing the order of transactions, without requiring the use of nonces. In this way, we can support un-ordered transaction inclusion.
 
 ## Context
 

From 944760c2fae44ba8022ae25ca7700abe0a5aeff9 Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Tue, 5 Dec 2023 16:34:47 +0800
Subject: [PATCH 18/19] purge in background

---
 .../architecture/adr-069-unordered-account.md | 109 ++++++++++++++----
 1 file changed, 86 insertions(+), 23 deletions(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index ec9d5647dff5..2eeb6e98754a 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -32,22 +32,39 @@ The dictionary can be simply implemented as an in-memory golang map, a prelimina
 message TxBody {
   ...
 
-  boolean unordered = 4; 
+  bool unordered = 4; 
 }
 ```
 
 ### `DedupTxHashManager`
 
 ```golang
-// can reduce frequency we check the expiration.
-const ExpireCheckInterval = 1
+const PurgeLoopSleepMS = 500
 
 // DedupTxHashManager contains the tx hash dictionary for duplicates checking,
 // and expire them when block number progresses.
 type DedupTxHashManager struct {
+  mutex sync.RWMutex
   // tx hash -> expire block number
   // for duplicates checking and expiration
   hashes map[TxHash]uint64
+  // channel to receive latest block numbers
+  blockCh chan uint64
+}
+
+func NewDedupTxHashManager() *DedupTxHashManager {
+  m := &DedupTxHashManager{
+    hashes: make(map[TxHash]uint64),
+    blockCh: make(ch *uint64, 16),
+  }
+  go m.purgeLoop()
+  return m
+}
+
+func (dtm *DedupTxHashManager) Close() error {
+  close(dtm.blockCh)
+  dtm.blockCh = nil
+  return nil
 }
 
 func (dtm *DedupTxHashManager) Contains(hash TxHash) (ok bool) {
@@ -62,7 +79,7 @@ func (dtm *DedupTxHashManager) Size() int {
   dtm.mutex.RLock()
   defer dtm.mutex.RUnlock()
 
-  return len(dtm).hashes
+  return len(dtm.hashes)
 }
 
 func (dtm *DedupTxHashManager) Add(hash TxHash, expire uint64) (ok bool) {
@@ -73,21 +90,73 @@ func (dtm *DedupTxHashManager) Add(hash TxHash, expire uint64) (ok bool) {
   return
 }
 
-// EndBlock remove expired tx hashes, need to wire in abci cycles.
+// EndBlock send the latest block number to the background purge loop
 func (dtm *DedupTxHashManager) EndBlock(ctx sdk.Context) {
-  if ctx.BlockNumber() % ExpireCheckInterval != 0 {
-    return
+  n := ctx.BlockNumber()
+  dtm.blockCh <- &n
+}
+
+// purgeLoop removes expired tx hashes at background
+func (dtm *DedupTxHashManager) purgeLoop() error {
+  for {
+    blocks := channelBatchRecv(dtm.blockCh)
+    if len(blocks) == 0 {
+      // channel closed
+      break
+    }
+    
+    latest := *blocks[len(blocks)-1]
+    hashes := dtm.expired(latest)
+    if len(hashes) > 0 {
+      dtm.purge(hashes)
+    }
+    
+    // avoid burning cpu in catching up phase
+    time.Sleep(PurgeLoopSleepMS * time.Millisecond)
+  }
+}
+
+// expired find out expired tx hashes based on latest block number
+func (dtm *DedupTxHashManager) expired(block uint64) []TxHash {
+  dtm.mutex.RLock()
+  defer dtm.mutex.RUnlock()
+
+  var result []TxHash
+  for h, expire := range dtm.hashes {
+    if block > expire {
+      result = append(result, h)
+    }
   }
+  return result
+}
 
+func (dtm *DedupTxHashManager) purge(hashes []TxHash) {
   dtm.mutex.Lock()
   defer dtm.mutex.Unlock()
 
-  for k, expire := range dtm.hashes {
-    if ctx.BlockNumber() > expire {
-      delete(dtm.hashes, k)
-    }
+  for _, hash := range hashes {
+    delete(dtm.hashes, hash)
   }
 }
+
+// channelBatchRecv try to exhaust the channel buffer when it's not empty,
+// and block when it's empty.
+func channelBatchRecv[T any](ch <-chan *T) []*T {
+	item := <-ch  // block if channel is empty
+	if item == nil {
+		// channel is closed
+		return nil
+	}
+
+	remaining := len(ch)
+	result := make([]*T, 0, remaining+1)
+	result = append(result, item)
+	for i := 0; i < remaining; i++ {
+		result = append(result, <-ch)
+	}
+
+	return result
+}
 ```
 
 ### Ante Handlers
@@ -99,7 +168,7 @@ func (isd IncrementSequenceDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, sim
   if tx.UnOrdered() {
     return next(ctx, tx, simulate)
   }
-  
+
   // the previous logic
 }
 ```
@@ -110,9 +179,6 @@ A decorator for the new logic.
 type TxHash [32]byte
 
 const (
-  // MaxNumberOfTxHash * 32 = 128M max memory usage
-  MaxNumberOfTxHash = 1024 * 1024 * 4
-
   // MaxUnOrderedTTL defines the maximum ttl an un-order tx can set
   MaxUnOrderedTTL = 1024
 )
@@ -135,17 +201,14 @@ func (dtd *DedupTxDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate boo
     return nil, errorsmod.Wrapf(sdkerrors.ErrLogic, "unordered tx ttl exceeds %d", MaxUnOrderedTTL)
   }
 
+  // check for duplicates
+  if dtd.m.Contains(tx.Hash()) {
+    return nil, errorsmod.Wrap(sdkerrors.ErrLogic, "tx is duplicated")
+  }
+
   if !ctx.IsCheckTx() {
     // a new tx included in the block, add the hash to the dictionary
-    if dtd.m.Size() >= MaxNumberOfTxHash {
-      return nil, errorsmod.Wrap(sdkerrors.ErrLogic, "dedup map is full")
-    }
     dtd.m.Add(tx.Hash(), tx.TimeoutHeight())
-  } else {
-    // check for duplicates
-    if dtd.m.Contains(tx.Hash()) {
-      return nil, errorsmod.Wrap(sdkerrors.ErrLogic, "tx is duplicated")
-    }
   }
 
   return next(ctx, tx, simulate)

From 3aa46f37bdfda966706de2ec74b8b652a5eaeed1 Mon Sep 17 00:00:00 2001
From: HuangYi <huang@crypto.com>
Date: Tue, 5 Dec 2023 16:46:39 +0800
Subject: [PATCH 19/19] OnNewBlock

---
 docs/architecture/adr-069-unordered-account.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/architecture/adr-069-unordered-account.md b/docs/architecture/adr-069-unordered-account.md
index 2eeb6e98754a..37945db7712a 100644
--- a/docs/architecture/adr-069-unordered-account.md
+++ b/docs/architecture/adr-069-unordered-account.md
@@ -90,10 +90,10 @@ func (dtm *DedupTxHashManager) Add(hash TxHash, expire uint64) (ok bool) {
   return
 }
 
-// EndBlock send the latest block number to the background purge loop
-func (dtm *DedupTxHashManager) EndBlock(ctx sdk.Context) {
-  n := ctx.BlockNumber()
-  dtm.blockCh <- &n
+// OnNewBlock send the latest block number to the background purge loop,
+// it should be called in abci commit event.
+func (dtm *DedupTxHashManager) OnNewBlock(blockNumber uint64) {
+  dtm.blockCh <- &blockNumber
 }
 
 // purgeLoop removes expired tx hashes at background
@@ -215,13 +215,13 @@ func (dtd *DedupTxDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate boo
 }
 ```
 
-### EndBlocker
+### `OnNewBlock`
 
-Wire up the `EndBlock` method of `DedupTxHashManager` into the application's abci life cycle.
+Wire the `OnNewBlock` method of `DedupTxHashManager` into the baseapp's abci commit event.
 
 ### Start Up
 
-On start up, the node needs to re-fill the tx hash dictionary of `DedupTxHashManager` by scanning `MaxUnOrderedTTL` number of historical blocks for un-ordered transactions.
+On start up, the node needs to re-fill the tx hash dictionary of `DedupTxHashManager` by scanning `MaxUnOrderedTTL` number of historical blocks for existing un-expired un-ordered transactions.
 
 An alternative design is to store the tx hash dictionary in kv store, then no need to warm up on start up.