E2E Testing fixes and cleanup (#1765)

# Goal
The goal of this PR is the final misc pieces of getting the e2e tests
less flaky. Does this fix all of it? No. Most? Yes.

Closes #1731 

# Discussion
- Correct batch error asserting
- Ids used as "bad" should use close to the max
- Use Immortal Eras due to issues with speed and
[AncientBirthBlock](https://substrate.stackexchange.com/questions/10411/fast-block-production-causes-mortality-checking-failing-with-ancientbirthblock)
- Cleanup before funding
- Move assertAddNewKey
- Add new assertHasMessage

e2e/capacity/transactions.test.ts

@@ -38,7 +38,9 @@ import {
   generatePaginatedUpsertSignaturePayloadV2,
   generatePaginatedDeleteSignaturePayloadV2,
   getCapacity,
-  getTestHandle
+  getTestHandle,
+  assertHasMessage,
+  assertAddNewKey
 } from "../scaffolding/helpers";
 import { FeeDetails } from "@polkadot/types/interfaces";
 import { ipfsCid } from "../messages/ipfs";
@@ -60,16 +62,6 @@ describe("Capacity Transactions", function () {
         assert.notEqual(schemaId, undefined, "setup should populate schemaId");
       });
 
-      async function assertAddNewKey(capacityKeys: KeyringPair, addKeyPayload: AddKeyData, newControlKeypair: KeyringPair) {
-        const addKeyPayloadCodec: Codec = ExtrinsicHelper.api.registry.createType("PalletMsaAddKeyData", addKeyPayload);
-        const ownerSig: Sr25519Signature = signPayloadSr25519(capacityKeys, addKeyPayloadCodec);
-        const newSig: Sr25519Signature = signPayloadSr25519(newControlKeypair, addKeyPayloadCodec);
-        const addPublicKeyOp = ExtrinsicHelper.addPublicKeyToMsa(capacityKeys, ownerSig, newSig, addKeyPayload);
-        const { eventMap } = await addPublicKeyOp.signAndSend();
-        assertEvent(eventMap, "system.ExtrinsicSuccess");
-        assertEvent(eventMap, "msa.PublicKeyAdded");
-      }
-
       function getCapacityFee(chainEvents: EventMap): bigint {
         if (chainEvents["capacity.CapacityWithdrawn"] &&
           ExtrinsicHelper.api.events.capacity.CapacityWithdrawn.is(chainEvents["capacity.CapacityWithdrawn"])) {
@@ -118,6 +110,7 @@ describe("Capacity Transactions", function () {
         before(async function () {
           capacityKeys = createKeys("CapacityKeys");
           capacityProvider = await createMsaAndProvider(fundingSource, capacityKeys, "CapacityProvider", FUNDS_AMOUNT);
+          // Stake enough for all transactions
           await assert.doesNotReject(stakeToProvider(fundingSource, capacityKeys, capacityProvider, stakedForMsa));
         })
 
@@ -186,11 +179,11 @@ describe("Capacity Transactions", function () {
           assertEvent(eventMap, "capacity.CapacityWithdrawn");
           assertEvent(eventMap, "msa.DelegationGranted");
 
-          let fee = getCapacityFee(eventMap);
+          const fee = getCapacityFee(eventMap);
           // assuming no other txns charged against capacity (b/c of async tests), this should be the maximum amount left.
           const maximumExpectedRemaining = stakedForMsa / TokenPerCapacity - fee
 
-          let remaining = capacityStaked.remainingCapacity.toBigInt();
+          const remaining = capacityStaked.remainingCapacity.toBigInt();
           assert(remaining <= maximumExpectedRemaining, `expected ${remaining} to be <= ${maximumExpectedRemaining}`);
           assert.equal(capacityStaked.totalTokensStaked.toBigInt(), stakedForMsa);
           assert.equal(capacityStaked.totalCapacityIssued.toBigInt(), stakedForMsa / TokenPerCapacity);
@@ -209,6 +202,7 @@ describe("Capacity Transactions", function () {
 
         beforeEach(async function () {
           starting_block = (await ExtrinsicHelper.apiPromise.rpc.chain.getHeader()).number.toNumber();
+          // Stake each time so that we always have enough capacity to do the call
           await assert.doesNotReject(stakeToProvider(fundingSource, capacityKeys, capacityProvider, amountStaked));
         });
 
@@ -240,8 +234,7 @@ describe("Capacity Transactions", function () {
               page_size: 999
             }
           );
-          const response: MessageResponse = get.content[get.content.length - 1];
-          assert.equal(response.payload, "0xdeadbeef", "payload should be 0xdeadbeef");
+          assertHasMessage(get, x => x.payload.isSome && x.payload.toString() === "0xdeadbeef");
         });
       });
 
@@ -254,8 +247,10 @@ describe("Capacity Transactions", function () {
         before(async function () {
           capacityKeys = createKeys("CapacityKeys");
           capacityProvider = await createMsaAndProvider(fundingSource, capacityKeys, "CapacityProvider", FUNDS_AMOUNT);
-        })
+        });
+
         beforeEach(async function () {
+          // Stake each time so that we always have enough capacity to do the call
           await assert.doesNotReject(stakeToProvider(fundingSource, capacityKeys, capacityProvider, amountStaked));
         });
 
@@ -480,7 +475,8 @@ describe("Capacity Transactions", function () {
         before(async function () {
           capacityKeys = createKeys("CapacityKeys");
           capacityProvider = await createMsaAndProvider(fundingSource, capacityKeys, "CapacityProvider", FUNDS_AMOUNT);
-        })
+        });
+
         it("successfully pays with Capacity for eligible transaction - claimHandle", async function () {
           await assert.doesNotReject(stakeToProvider(fundingSource, capacityKeys, capacityProvider, amountStaked));
 

e2e/messages/addIPFSMessage.test.ts

@@ -5,7 +5,7 @@ import { base32 } from 'multiformats/bases/base32';
 import { CID } from 'multiformats/cid'
 import { PARQUET_BROADCAST } from "../schemas/fixtures/parquetBroadcastSchemaType";
 import assert from "assert";
-import { createAndFundKeypair } from "../scaffolding/helpers";
+import { assertHasMessage, createAndFundKeypair } from "../scaffolding/helpers";
 import { ExtrinsicHelper } from "../scaffolding/extrinsicHelpers";
 import { u16 } from "@polkadot/types";
 import { MessageResponse } from "@frequency-chain/api-augment/interfaces";
@@ -102,9 +102,10 @@ describe("Add Offchain Message", function () {
 
     it("should successfully retrieve added message and returned CID should have Base32 encoding", async function () {
         const f = await ExtrinsicHelper.apiPromise.rpc.messages.getBySchemaId(schemaId, { from_block: starting_block, from_index: 0, to_block: starting_block + 999, page_size: 999 });
-        const response: MessageResponse = f.content[f.content.length - 1];
-        const cid = Buffer.from(response.cid.unwrap()).toString();
-        assert.equal(cid, ipfs_cid_32, 'returned CID should match base32-encoded CID');
+        assertHasMessage(f, x => {
+          const cid = x.cid.isSome && Buffer.from(x.cid.unwrap()).toString();
+          return cid === ipfs_cid_32;
+        });
     })
 
     describe("Add OnChain Message and successfully retrieve it", function () {
@@ -124,8 +125,7 @@ describe("Add Offchain Message", function () {
                     page_size: 999
                     }
                 );
-            const response: MessageResponse = get.content[get.content.length - 1];
-            assert.equal(response.payload, "0xdeadbeef", "payload should be 0xdeadbeef");
+            assertHasMessage(get, x => x.payload.isSome && x.payload.toString() === "0xdeadbeef");
         });
     });
 });

e2e/miscellaneous/utilityBatch.test.ts

@@ -1,7 +1,7 @@
 import assert from "assert";
 import { KeyringPair } from "@polkadot/keyring/types";
 import { ExtrinsicHelper } from "../scaffolding/extrinsicHelpers";
-import { createAndFundKeypair, getNonce } from "../scaffolding/helpers";
+import { DOLLARS, createAndFundKeypair } from "../scaffolding/helpers";
 import { ApiTypes, SubmittableExtrinsic } from "@polkadot/api/types";
 import { getFundingSource } from "../scaffolding/funding";
 
@@ -11,10 +11,9 @@ describe("Utility Batch Filtering", function () {
 
     const fundingSource = getFundingSource("misc-util-batch");
 
-    before(async function () {
-        let nonce = await getNonce(fundingSource);
-        sender = await createAndFundKeypair(fundingSource, 50_000_000n, "utility-sender", nonce++);
-        recipient = await createAndFundKeypair(fundingSource, 50_000_000n, "utility-recipient", nonce++);
+    beforeEach(async function () {
+      sender = await createAndFundKeypair(fundingSource, 5n * DOLLARS, 'utility-sender');
+      recipient = await createAndFundKeypair(fundingSource, 5n * DOLLARS, 'utility-recipient');
     });
 
     it("should successfully execute ✅ batch with allowed calls", async function () {
@@ -44,11 +43,10 @@ describe("Utility Batch Filtering", function () {
         const batchAll = ExtrinsicHelper.executeUtilityBatchAll(sender, badBatch);
         try {
             await batchAll.fundAndSend(fundingSource);
+            assert.fail("batchAll should have caused an error");
         } catch (err) {
-            error = err;
-            assert.notEqual(error, undefined, " batchAll should return an error");
+            assert.notEqual(err, undefined, " batchAll should return an error");
         }
-        assert.notEqual(error, undefined, " batchAll should return an error");
     });
 
     it("should fail to execute ❌ batch with disallowed calls", async function () {
@@ -88,44 +86,38 @@ describe("Utility Batch Filtering", function () {
         const badBatch: SubmittableExtrinsic<ApiTypes>[] = [];
         badBatch.push(ExtrinsicHelper.api.tx.msa.retireMsa())
         const batch = ExtrinsicHelper.executeUtilityBatchAll(sender, badBatch);
-        let error: any;
         try {
             await batch.fundAndSend(fundingSource);
+            assert.fail("batch should have caused an error");
         } catch (err) {
-            error = err;
-            assert.notEqual(error, undefined, "should return an error");
+            assert.notEqual(err, undefined, "should return an error");
         }
-        assert.notEqual(error, undefined, "should return an error");
     });
 
     it("should fail to execute ❌ batch with `Pays::Yes` `create_provider`call blocked by Frequency", async function () {
         // bad batch: with frequency related Pays::Yes call
         const badBatch: SubmittableExtrinsic<ApiTypes>[] = [];
         badBatch.push(ExtrinsicHelper.api.tx.msa.createProvider("I am a ba(tch)d provider"))
         const batch = ExtrinsicHelper.executeUtilityBatchAll(sender, badBatch);
-        let error: any;
         try {
             await batch.fundAndSend(fundingSource);
+            assert.fail("batch should have caused an error");
         } catch (err) {
-            error = err;
-            assert.notEqual(error, undefined, "should return an error");
+            assert.notEqual(err, undefined, "should return an error");
         }
-        assert.notEqual(error, undefined, "should return an error");
     });
 
     it("should fail to execute ❌ batch with `Pays::Yes` `create_schema` call blocked by Frequency", async function () {
         // bad batch: with frequency related Pays::Yes call
         const badBatch: SubmittableExtrinsic<ApiTypes>[] = [];
         badBatch.push(ExtrinsicHelper.api.tx.msa.createProvider("I am a ba(tch)d provider"))
         const batch = ExtrinsicHelper.executeUtilityBatchAll(sender, badBatch);
-        let error: any;
         try {
             await batch.fundAndSend(fundingSource);
+            assert.fail("batch should have caused an error");
         } catch (err) {
-            error = err;
-            assert.notEqual(error, undefined, "should return an error");
+            assert.notEqual(err, undefined, "should return an error");
         }
-        assert.notEqual(error, undefined, "should return an error");
     });
 
     it("should fail to execute ❌ batch with nested batch", async function () {
@@ -136,13 +128,11 @@ describe("Utility Batch Filtering", function () {
         innerBatch.push(ExtrinsicHelper.api.tx.system.remark("Hello From Batch"))
         nestedBatch.push(ExtrinsicHelper.api.tx.utility.batch(innerBatch))
         const batch = ExtrinsicHelper.executeUtilityBatchAll(sender, nestedBatch);
-        let error: any;
         try {
             await batch.fundAndSend(fundingSource);
+            assert.fail("batch should have caused an error");
         } catch (err) {
-            error = err;
-            assert.notEqual(error, undefined, "should return an error");
+            assert.notEqual(err, undefined, "should return an error");
         }
-        assert.notEqual(error, undefined, "should return an error");
     });
 });

e2e/msa/msaKeyManagement.test.ts

@@ -7,6 +7,8 @@ import { u64 } from "@polkadot/types";
 import { Codec } from "@polkadot/types/types";
 import { getFundingSource } from "../scaffolding/funding";
 
+const maxU64 = 18_446_744_073_709_551_615n;
+
 describe("MSA Key management", function () {
     const fundingSource = getFundingSource("msa-key-management");
 
@@ -75,7 +77,7 @@ describe("MSA Key management", function () {
         it("should fail to add public key if origin does not own MSA (NotMsaOwner)", async function () {
             const newPayload = await generateAddKeyPayload({
                 ...defaultPayload,
-                msaId: new u64(ExtrinsicHelper.api.registry, 999), // If we create more than 999 MSAs in our test suites, this will fail
+                msaId: new u64(ExtrinsicHelper.api.registry, maxU64),
             });
             addKeyData = ExtrinsicHelper.api.registry.createType("PalletMsaAddKeyData", newPayload);
             ownerSig = signPayloadSr25519(keys, addKeyData);
@@ -155,7 +157,7 @@ describe("MSA Key management", function () {
             assert.notEqual(event, undefined, 'should have added public key');
 
             // Cleanup
-            await assert.doesNotReject(ExtrinsicHelper.deletePublicKey(keys, thirdKey.publicKey).signAndSend());
+            await assert.doesNotReject(ExtrinsicHelper.deletePublicKey(keys, thirdKey.publicKey).signAndSend('current'));
         });
     });
 

e2e/scaffolding/extrinsicHelpers.ts

@@ -130,7 +130,8 @@ export class Extrinsic<N = unknown, T extends ISubmittableResult = ISubmittableR
 
         try {
             const op = this.extrinsic();
-            return await firstValueFrom(op.signAndSend(this.keys, { nonce }).pipe(
+            // Era is 0 for tests due to issues with BirthBlock
+            return await firstValueFrom(op.signAndSend(this.keys, { nonce, era: 0 }).pipe(
                 tap((result) => {
                     // If we learn a transaction has an error status (this does NOT include RPC errors)
                     // Then throw an error
@@ -151,15 +152,17 @@ export class Extrinsic<N = unknown, T extends ISubmittableResult = ISubmittableR
 
     public async sudoSignAndSend() {
         const nonce = await autoNonce.auto(this.keys);
-        return await firstValueFrom(this.api.tx.sudo.sudo(this.extrinsic()).signAndSend(this.keys, { nonce }).pipe(
+        // Era is 0 for tests due to issues with BirthBlock
+        return await firstValueFrom(this.api.tx.sudo.sudo(this.extrinsic()).signAndSend(this.keys, { nonce, era: 0 }).pipe(
             filter(({ status }) => status.isInBlock || status.isFinalized),
             this.parseResult(this.event),
         ))
     }
 
     public async payWithCapacity(inputNonce?: AutoNonce) {
         const nonce = await autoNonce.auto(this.keys, inputNonce);
-        return await firstValueFrom(this.api.tx.frequencyTxPayment.payWithCapacity(this.extrinsic()).signAndSend(this.keys, { nonce }).pipe(
+        // Era is 0 for tests due to issues with BirthBlock
+        return await firstValueFrom(this.api.tx.frequencyTxPayment.payWithCapacity(this.extrinsic()).signAndSend(this.keys, { nonce, era: 0 }).pipe(
             filter(({ status }) => status.isInBlock || status.isFinalized),
             this.parseResult(this.event),
         ))

e2e/scaffolding/helpers.ts

@@ -1,7 +1,7 @@
 import { Keyring } from "@polkadot/api";
 import { KeyringPair } from "@polkadot/keyring/types";
 import { u16, u32, u64, Option } from "@polkadot/types";
-import type { PalletCapacityCapacityDetails } from "@polkadot/types/lookup";
+import type { FrameSystemAccountInfo, PalletCapacityCapacityDetails } from "@polkadot/types/lookup";
 import { Codec } from "@polkadot/types/types";
 import { u8aToHex, u8aWrapBytes } from "@polkadot/util";
 import { mnemonicGenerate } from '@polkadot/util-crypto';
@@ -14,7 +14,7 @@ import {
   ItemizedSignaturePayload, ItemizedSignaturePayloadV2, PaginatedDeleteSignaturePayload,
   PaginatedDeleteSignaturePayloadV2, PaginatedUpsertSignaturePayload, PaginatedUpsertSignaturePayloadV2
 } from "./extrinsicHelpers";
-import { HandleResponse, MessageSourceId, PageHash } from "@frequency-chain/api-augment/interfaces";
+import { BlockPaginationResponseMessage, HandleResponse, MessageResponse, MessageSourceId, PageHash } from "@frequency-chain/api-augment/interfaces";
 import assert from "assert";
 import { AVRO_GRAPH_CHANGE } from "../schemas/fixtures/avroGraphChangeSchemaType";
 import { PARQUET_BROADCAST } from "../schemas/fixtures/parquetBroadcastSchemaType";
@@ -165,16 +165,24 @@ export function createKeys(name: string = 'first pair'): KeyringPair {
   return keypair;
 }
 
+function canDrainAccount(info: FrameSystemAccountInfo): Boolean {
+  return !info.isEmpty
+    && info.data.free.toNumber() > 1_500_000 // ~Cost to do the transfer
+    && info.data.reserved.toNumber() < 1
+    && info.data.frozen.toNumber() < 1;
+}
+
 export async function drainKeys(keyPairs: KeyringPair[], dest: string) {
   try {
-    await Promise.allSettled(keyPairs.map(async (keypair) => {
-      const info = await ExtrinsicHelper.getAccountInfo(keypair.address);
-        if (!info.isEmpty && info.data.free.toNumber() > 0) {
-          await ExtrinsicHelper.emptyAccount(keypair, dest).signAndSend();
-        }
-    }));
+    await Promise.all(
+      keyPairs.map(async (keypair) => {
+        const info = await ExtrinsicHelper.getAccountInfo(keypair.address);
+        // Only drain keys that can be
+        if (canDrainAccount(info)) await ExtrinsicHelper.emptyAccount(keypair, dest).signAndSend();
+      })
+    );
   } catch (e) {
-    console.log("Error draining accounts: ", e);
+    console.log('Error draining accounts: ', e);
   }
 }
 
@@ -210,17 +218,17 @@ export function log(...args: any[]) {
   }
 }
 
-export async function createProviderKeysAndId(source: KeyringPair): Promise<[KeyringPair, u64]> {
-  const providerKeys = await createAndFundKeypair(source);
+export async function createProviderKeysAndId(source: KeyringPair, amount?: bigint): Promise<[KeyringPair, u64]> {
+  const providerKeys = await createAndFundKeypair(source, amount);
   await ExtrinsicHelper.createMsa(providerKeys).fundAndSend(source);
   const createProviderOp = ExtrinsicHelper.createProvider(providerKeys, "PrivateProvider");
   const { target: providerEvent } = await createProviderOp.fundAndSend(source);
   const providerId = providerEvent?.data.providerId || new u64(ExtrinsicHelper.api.registry, 0);
   return [providerKeys, providerId];
 }
 
-export async function createDelegator(source: KeyringPair): Promise<[KeyringPair, u64]> {
-  let keys = await createAndFundKeypair(source);
+export async function createDelegator(source: KeyringPair, amount?: bigint): Promise<[KeyringPair, u64]> {
+  let keys = await createAndFundKeypair(source, amount);
   const createMsa = ExtrinsicHelper.createMsa(keys);
   const { target: msaCreatedEvent } = await createMsa.fundAndSend(source);
   const delegatorMsaId = msaCreatedEvent?.data.msaId || new u64(ExtrinsicHelper.api.registry, 0);
@@ -440,3 +448,27 @@ export function assertEvent(events: EventMap, eventName: string) {
 export function assertExtrinsicSuccess(eventMap: EventMap) {
   assert.notEqual(eventMap["system.ExtrinsicSuccess"], undefined);
 }
+
+export function assertHasMessage(response: BlockPaginationResponseMessage, testFn: (x: MessageResponse) => Boolean) {
+  const messages = response.content;
+  assert(messages.length > 0, "Expected some messages, but found none.");
+
+  const found = messages.find(testFn);
+
+  if (found) {
+    assert.notEqual(found, undefined);
+  } else {
+    const allPayloads = messages.map(x => x.payload.toString());
+    assert.fail(`Unable to find message in response (length: ${messages.length}, Payloads: ${allPayloads.join(", ")})`);
+  }
+}
+
+export async function assertAddNewKey(capacityKeys: KeyringPair, addKeyPayload: AddKeyData, newControlKeypair: KeyringPair) {
+  const addKeyPayloadCodec: Codec = ExtrinsicHelper.api.registry.createType("PalletMsaAddKeyData", addKeyPayload);
+  const ownerSig: Sr25519Signature = signPayloadSr25519(capacityKeys, addKeyPayloadCodec);
+  const newSig: Sr25519Signature = signPayloadSr25519(newControlKeypair, addKeyPayloadCodec);
+  const addPublicKeyOp = ExtrinsicHelper.addPublicKeyToMsa(capacityKeys, ownerSig, newSig, addKeyPayload);
+  const { eventMap } = await addPublicKeyOp.signAndSend();
+  assertEvent(eventMap, "system.ExtrinsicSuccess");
+  assertEvent(eventMap, "msa.PublicKeyAdded");
+}