From 8f206d35af04a1fb30a045cc66f6826256778c62 Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Thu, 31 Oct 2024 18:37:14 +0800 Subject: [PATCH 01/12] cache polish --- packages/eth-providers/src/base-provider.ts | 191 +++++--------------- packages/eth-providers/src/utils/utils.ts | 7 + 2 files changed, 53 insertions(+), 145 deletions(-) diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index 9f409613..58e6e66f 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -14,28 +14,22 @@ import { } from '@ethersproject/abstract-provider'; import { AcalaEvmTX, checkSignatureType, parseTransaction } from '@acala-network/eth-transactions'; import { AccessList, accessListify } from 'ethers/lib/utils'; -import { AccountId, H160, Header } from '@polkadot/types/interfaces'; import { ApiPromise } from '@polkadot/api'; import { AsyncAction } from 'rxjs/internal/scheduler/AsyncAction'; import { AsyncScheduler } from 'rxjs/internal/scheduler/AsyncScheduler'; import { BigNumber, BigNumberish, Wallet } from 'ethers'; import { Deferrable, defineReadOnly, resolveProperties } from '@ethersproject/properties'; -import { EvmAccountInfo, EvmContractInfo } from '@acala-network/types/interfaces'; import { Formatter } from '@ethersproject/providers'; -import { FrameSystemAccountInfo } from '@polkadot/types/lookup'; +import { Header } from '@polkadot/types/interfaces'; import { ISubmittableResult } from '@polkadot/types/types'; import { Logger } from '@ethersproject/logger'; +import { ModuleEvmModuleAccountInfo } from '@polkadot/types/lookup'; import { Network } from '@ethersproject/networks'; import { Observable, ReplaySubject, Subscription, firstValueFrom, throwError } from 'rxjs'; -import { Option, decorateStorage, unwrapStorageType } from '@polkadot/types'; -import { Storage } from '@polkadot/types/metadata/decorate/types'; import { SubmittableExtrinsic } from '@polkadot/api/types'; -import { VersionedRegistry } from '@polkadot/api/base/types'; -import { createHeaderExtended } from '@polkadot/api-derive'; import { filter, first, timeout } from 'rxjs/operators'; import { getAddress } from '@ethersproject/address'; import { hexDataLength, hexValue, hexZeroPad, hexlify, isHexString, joinSignature } from '@ethersproject/bytes'; -import { isNull, u8aToHex, u8aToU8a } from '@polkadot/util'; import BN from 'bn.js'; import LRUCache from 'lru-cache'; @@ -44,7 +38,6 @@ import { BLOCK_GAS_LIMIT, BLOCK_STORAGE_LIMIT, CACHE_SIZE_WARNING, - DUMMY_ADDRESS, DUMMY_BLOCK_NONCE, DUMMY_LOGS_BLOOM, EMPTY_HEX_STRING, @@ -78,6 +71,7 @@ import { filterLogByTopics, getAllReceiptsAtBlock, getHealthResult, + getTimestamp, getTransactionRequest, hexlifyRpcResult, isEvmExtrinsic, @@ -309,8 +303,7 @@ export abstract class BaseProvider extends AbstractProvider { readonly localMode: boolean; readonly verbose: boolean; readonly maxBlockCacheSize: number; - readonly storages: WeakMap, Storage> = new WeakMap(); - readonly storageCache: LRUCache; + readonly queryCache: LRUCache; readonly blockCache: BlockCache; readonly finalizedBlockHashes: MaxSizeSet; @@ -347,7 +340,7 @@ export abstract class BaseProvider extends AbstractProvider { this.localMode = localMode; this.verbose = verbose; this.maxBlockCacheSize = maxBlockCacheSize; - this.storageCache = new LRUCache({ max: storageCacheSize }); + this.queryCache = new LRUCache({ max: storageCacheSize }); this.blockCache = new BlockCache(this.maxBlockCacheSize); this.finalizedBlockHashes = new MaxSizeSet(this.maxBlockCacheSize); @@ -523,67 +516,6 @@ export abstract class BaseProvider extends AbstractProvider { defineReadOnly(this, '_api', api); }; - queryStorage = async ( - module: `${string}.${string}`, - args: any[], - _blockTag?: BlockTag | Promise | Eip1898BlockTag - ): Promise => { - const blockTag = await this._ensureSafeModeBlockTagFinalization(await parseBlockTag(_blockTag)); - const blockHash = await this._getBlockHash(blockTag); - - const registry = await this.api.getBlockRegistry(u8aToU8a(blockHash)); - - if (!this.storages.get(registry)) { - const storage = decorateStorage( - registry.registry, - registry.metadata.asLatest, - registry.metadata.version, - ); - this.storages.set(registry, storage); - } - - const storage = this.storages.get(registry)!; - - const [section, method] = module.split('.'); - - const entry = storage[section][method]; - const key = entry(...args); - - const outputType = unwrapStorageType( - registry.registry, - entry.meta.type, - entry.meta.modifier.isOptional, - ); - - const cacheKey = `${module}-${blockHash}-${args.join(',')}`; - const cached = this.storageCache.get(cacheKey); - - let input: Uint8Array | null = null; - - if (cached) { - input = cached; - } else { - const value: any = await this.api.rpc.state.getStorage(key, blockHash); - - const isEmpty = isNull(value); - - // we convert to Uint8Array since it maps to the raw encoding, all - // data will be correctly encoded (incl. numbers, excl. :code) - input = isEmpty - ? null - : u8aToU8a(entry.meta.modifier.isOptional ? value.toU8a() : value.isSome ? value.unwrap().toU8a() : null); - - this.storageCache.set(cacheKey, input); - } - - const result = registry.registry.createTypeUnsafe(outputType, [input], { - blockHash, - isPedantic: !entry.meta.modifier.isOptional, - }); - - return result as any as T; - }; - get api(): ApiPromise { return this._api ?? logger.throwError('the api needs to be set', Logger.errors.UNKNOWN_ERROR); } @@ -672,19 +604,15 @@ export abstract class BaseProvider extends AbstractProvider { const blockHash = header.hash.toHex(); const blockNumber = header.number.toNumber(); - const [block, validators, now, receiptsFromSubql] = await Promise.all([ + const [block, headerExtended, timestamp, receiptsFromSubql] = await Promise.all([ this.api.rpc.chain.getBlock(blockHash), - this.api.query.session ? this.queryStorage('session.validators', [], blockHash) : ([] as any), - this.queryStorage('timestamp.now', [], blockHash), + this.api.derive.chain.getHeader(blockHash), + getTimestamp(this.api, blockHash), this.subql?.getAllReceiptsAtBlock(blockHash), ]); - const headerExtended = createHeaderExtended(header.registry, header, validators); - // blockscout need `toLowerCase` - const author = headerExtended.author - ? (await this.getEvmAddress(headerExtended.author.toString())).toLowerCase() - : DUMMY_ADDRESS; + const author = (await this.getEvmAddress(headerExtended.author.toString())).toLowerCase(); let receipts: TransactionReceipt[]; if (receiptsFromSubql?.length) { @@ -709,7 +637,7 @@ export abstract class BaseProvider extends AbstractProvider { number: blockNumber, stateRoot: headerExtended.stateRoot.toHex(), transactionsRoot: headerExtended.extrinsicsRoot.toHex(), - timestamp: Math.floor(now.toNumber() / 1000), + timestamp: Math.floor(timestamp / 1000), nonce: DUMMY_BLOCK_NONCE, mixHash: ZERO_BLOCK_HASH, difficulty: ZERO, @@ -750,11 +678,8 @@ export abstract class BaseProvider extends AbstractProvider { const substrateAddress = await this.getSubstrateAddress(address, blockHash); - const accountInfo = await this.queryStorage( - 'system.account', - [substrateAddress], - blockHash - ); + const apiAt = await this.api.at(blockHash); + const accountInfo = await apiAt.query.system.account(substrateAddress); return nativeToEthDecimal(accountInfo.data.free.toBigInt()); }; @@ -779,9 +704,7 @@ export abstract class BaseProvider extends AbstractProvider { } const accountInfo = await this.queryAccountInfo(addressOrName, blockTag); - const minedNonce = accountInfo.isNone - ? 0 - : accountInfo.unwrap().nonce.toNumber(); + const minedNonce = accountInfo?.nonce?.toNumber?.() ?? 0; return minedNonce + pendingNonce; }; @@ -791,23 +714,14 @@ export abstract class BaseProvider extends AbstractProvider { _blockTag?: BlockTag | Promise | Eip1898BlockTag ): Promise => { const blockTag = await this._ensureSafeModeBlockTagFinalization(await parseBlockTag(_blockTag)); + const blockHash = await this._getBlockHash(blockTag); - const [address, blockHash] = await Promise.all([ - addressOrName, - this._getBlockHash(blockTag), - ]); - - const contractInfo = await this.queryContractInfo(address, blockHash); - - if (contractInfo.isNone) { - return '0x'; - } - - const codeHash = contractInfo.unwrap().codeHash; - - const api = blockHash ? await this.api.at(blockHash) : this.api; + const accountInfo = await this.queryAccountInfo(addressOrName, blockHash); + const contractInfo = accountInfo?.contractInfo.unwrapOr(null); + if (!contractInfo) { return '0x'; } - const code = await api.query.evm.codes(codeHash); + const apiAt = await this.api.at(blockHash); + const code = await apiAt.query.evm.codes(contractInfo.codeHash); return code.toHex(); }; @@ -904,7 +818,8 @@ export abstract class BaseProvider extends AbstractProvider { Promise.resolve(position).then(hexValue), ]); - const code = await this.queryStorage('evm.accountStorages', [address, hexZeroPad(resolvedPosition, 32)], blockHash); + const apiAt = await this.api.at(blockHash); + const code = await apiAt.query.evm.accountStorages(address, hexZeroPad(resolvedPosition, 32)); return code.toHex(); }; @@ -1185,20 +1100,20 @@ export abstract class BaseProvider extends AbstractProvider { }; }; - getSubstrateAddress = async (addressOrName: string, blockTag?: BlockTag): Promise => { - const [address, blockHash] = await Promise.all([ - addressOrName, - this._getBlockHash(blockTag), - ]); - - const substrateAccount = await this.queryStorage>('evmAccounts.accounts', [address], blockHash); + getSubstrateAddress = async (address: string, blockTag?: BlockTag): Promise => { + const blockHash = await this._getBlockHash(blockTag); + const apiAt = await this.api.at(blockHash); + const substrateAccount = await apiAt.query.evmAccounts.accounts(address); - return substrateAccount.isEmpty ? computeDefaultSubstrateAddress(address) : substrateAccount.toString(); + return substrateAccount.isEmpty + ? computeDefaultSubstrateAddress(address) + : substrateAccount.toString(); }; getEvmAddress = async (substrateAddress: string, blockTag?: BlockTag): Promise => { const blockHash = await this._getBlockHash(blockTag); - const evmAddress = await this.queryStorage>('evmAccounts.evmAddresses', [substrateAddress], blockHash); + const apiAt = await this.api.at(blockHash); + const evmAddress = await apiAt.query.evmAccounts.evmAddresses(substrateAddress); return getAddress(evmAddress.isEmpty ? computeDefaultEvmAddress(substrateAddress) : evmAddress.toString()); }; @@ -1206,7 +1121,7 @@ export abstract class BaseProvider extends AbstractProvider { queryAccountInfo = async ( addressOrName: string | Promise, _blockTag?: BlockTag | Promise | Eip1898BlockTag - ): Promise> => { + ): Promise => { const blockTag = await this._ensureSafeModeBlockTagFinalization(await parseBlockTag(_blockTag)); const [address, blockHash] = await Promise.all([ @@ -1214,22 +1129,10 @@ export abstract class BaseProvider extends AbstractProvider { this._getBlockHash(blockTag), ]); - const accountInfo = await this.queryStorage>('evm.accounts', [address], blockHash); + const apiAt = await this.api.at(blockHash); + const accountInfo = await apiAt.query.evm.accounts(address); - return accountInfo; - }; - - queryContractInfo = async ( - addressOrName: string | Promise, - blockTag?: BlockTag | Promise - ): Promise> => { - const accountInfo = await this.queryAccountInfo(addressOrName, blockTag); - - if (accountInfo.isNone) { - return this.api.createType>('Option', null); - } - - return accountInfo.unwrap().contractInfo; + return accountInfo.unwrapOr(null); }; _getSubstrateGasParams = (ethTx: Partial): { @@ -1287,7 +1190,7 @@ export abstract class BaseProvider extends AbstractProvider { storageLimit: storageLimit.toBigInt(), tip: 0n, }; - } catch (error) { + } catch { // v2 v2 = true; @@ -1471,7 +1374,8 @@ export abstract class BaseProvider extends AbstractProvider { result.blockNumber = startBlock; result.blockHash = startBlockHash; - result.timestamp = Math.floor((await this.queryStorage('timestamp.now', [], result.blockHash)).toNumber() / 1000); + const timestamp = await getTimestamp(this.api, result.blockHash); + result.timestamp = Math.floor(timestamp / 1000); result.wait = async (confirms?: number, timeoutMs?: number) => { if (confirms === null || confirms === undefined) { @@ -1561,26 +1465,23 @@ export abstract class BaseProvider extends AbstractProvider { } const isFinalized = blockNumber.lte(await this.finalizedBlockNumber); - const cacheKey = `blockHash-${blockNumber.toString()}`; + const cacheKey = `blockHash-${blockNumber.toHexString()}`; if (isFinalized) { - const cached = this.storageCache.get(cacheKey); + const cached = this.queryCache.get(cacheKey); if (cached) { - return u8aToHex(cached); + return cached; } } - const _blockHash = await this.api.rpc.chain.getBlockHash(blockNumber.toBigInt()); - - if (_blockHash.isEmpty) { - //@ts-ignore - return logger.throwError('header not found', PROVIDER_ERRORS.HEADER_NOT_FOUND, { blockNumber }); - } - - const blockHash = _blockHash.toHex(); + // TODO: test header not found should throw + const blockHash = (await this.api.rpc.chain.getBlockHash(blockNumber.toBigInt())).toHex(); + // if (_blockHash.isEmpty) { + // return logger.throwError('header not found', Logger.errors.CALL_EXCEPTION, { blockNumber }); + // } if (isFinalized) { - this.storageCache.set(cacheKey, _blockHash.toU8a()); + this.queryCache.set(cacheKey, blockHash); } return blockHash; diff --git a/packages/eth-providers/src/utils/utils.ts b/packages/eth-providers/src/utils/utils.ts index a26d43ff..4d4384b1 100644 --- a/packages/eth-providers/src/utils/utils.ts +++ b/packages/eth-providers/src/utils/utils.ts @@ -1,4 +1,5 @@ import { AnyFunction } from '@polkadot/types/types'; +import { ApiPromise } from '@polkadot/api'; import { BigNumber, BigNumberish } from '@ethersproject/bignumber'; import { Extrinsic } from '@polkadot/types/interfaces'; import { FrameSystemEventRecord } from '@polkadot/types/lookup'; @@ -411,3 +412,9 @@ export const toBN = (bigNumberis: BigNumberish = 0): BN => { // eslint-disable-next-line @typescript-eslint/no-explicit-any return new BN(bigNumberis as any); }; + +export const getTimestamp = async (api: ApiPromise, blockHash: string): Promise => { + const apiAt = await api.at(blockHash); + const timestamp = await apiAt.query.timestamp.now(); + return timestamp.toNumber(); +}; From a46f904e2f5531c33b2aff3d4189b8970fdc7f4a Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Thu, 31 Oct 2024 18:59:30 +0800 Subject: [PATCH 02/12] fix --- packages/eth-providers/src/base-provider.ts | 10 +++++----- packages/eth-rpc-adapter/src/wrapped-provider.ts | 4 +--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index 58e6e66f..d1c811cc 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -703,8 +703,8 @@ export abstract class BaseProvider extends AbstractProvider { ).length; } - const accountInfo = await this.queryAccountInfo(addressOrName, blockTag); - const minedNonce = accountInfo?.nonce?.toNumber?.() ?? 0; + const evmAccountInfo = await this.queryEvmAccountInfo(addressOrName, blockTag); + const minedNonce = evmAccountInfo?.nonce?.toNumber?.() ?? 0; return minedNonce + pendingNonce; }; @@ -716,8 +716,8 @@ export abstract class BaseProvider extends AbstractProvider { const blockTag = await this._ensureSafeModeBlockTagFinalization(await parseBlockTag(_blockTag)); const blockHash = await this._getBlockHash(blockTag); - const accountInfo = await this.queryAccountInfo(addressOrName, blockHash); - const contractInfo = accountInfo?.contractInfo.unwrapOr(null); + const evmAccountInfo = await this.queryEvmAccountInfo(addressOrName, blockHash); + const contractInfo = evmAccountInfo?.contractInfo.unwrapOr(null); if (!contractInfo) { return '0x'; } const apiAt = await this.api.at(blockHash); @@ -1118,7 +1118,7 @@ export abstract class BaseProvider extends AbstractProvider { return getAddress(evmAddress.isEmpty ? computeDefaultEvmAddress(substrateAddress) : evmAddress.toString()); }; - queryAccountInfo = async ( + queryEvmAccountInfo = async ( addressOrName: string | Promise, _blockTag?: BlockTag | Promise | Eip1898BlockTag ): Promise => { diff --git a/packages/eth-rpc-adapter/src/wrapped-provider.ts b/packages/eth-rpc-adapter/src/wrapped-provider.ts index 7c1793b0..a128f73e 100644 --- a/packages/eth-rpc-adapter/src/wrapped-provider.ts +++ b/packages/eth-rpc-adapter/src/wrapped-provider.ts @@ -7,7 +7,6 @@ const TRACE_METHODS = [ '_onNewHead', '_onNewFinalizedHead', '_notifySubscribers', - 'queryStorage', 'getNetwork', 'getBlockNumber', 'getBlockData', @@ -26,8 +25,7 @@ const TRACE_METHODS = [ 'estimateResources', 'getSubstrateAddress', 'getEvmAddress', - 'queryAccountInfo', - 'queryContractInfo', + 'queryEvmAccountInfo', 'prepareTransaction', 'sendRawTransaction', 'sendTransaction', From 5102af2e7b7f9b174d54baa77c60bcee9581e6bb Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Fri, 1 Nov 2024 14:40:49 +0800 Subject: [PATCH 03/12] update --- packages/eth-providers/src/base-provider.ts | 40 +++++++++++++------ .../eth_getBlockByNumber.test.ts | 5 +++ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index d1c811cc..704cab43 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -14,6 +14,7 @@ import { } from '@ethersproject/abstract-provider'; import { AcalaEvmTX, checkSignatureType, parseTransaction } from '@acala-network/eth-transactions'; import { AccessList, accessListify } from 'ethers/lib/utils'; +import { ApiDecoration, SubmittableExtrinsic } from '@polkadot/api/types'; import { ApiPromise } from '@polkadot/api'; import { AsyncAction } from 'rxjs/internal/scheduler/AsyncAction'; import { AsyncScheduler } from 'rxjs/internal/scheduler/AsyncScheduler'; @@ -26,7 +27,6 @@ import { Logger } from '@ethersproject/logger'; import { ModuleEvmModuleAccountInfo } from '@polkadot/types/lookup'; import { Network } from '@ethersproject/networks'; import { Observable, ReplaySubject, Subscription, firstValueFrom, throwError } from 'rxjs'; -import { SubmittableExtrinsic } from '@polkadot/api/types'; import { filter, first, timeout } from 'rxjs/operators'; import { getAddress } from '@ethersproject/address'; import { hexDataLength, hexValue, hexZeroPad, hexlify, isHexString, joinSignature } from '@ethersproject/bytes'; @@ -304,6 +304,7 @@ export abstract class BaseProvider extends AbstractProvider { readonly verbose: boolean; readonly maxBlockCacheSize: number; readonly queryCache: LRUCache; + readonly apiCache: LRUCache>; readonly blockCache: BlockCache; readonly finalizedBlockHashes: MaxSizeSet; @@ -341,6 +342,7 @@ export abstract class BaseProvider extends AbstractProvider { this.verbose = verbose; this.maxBlockCacheSize = maxBlockCacheSize; this.queryCache = new LRUCache({ max: storageCacheSize }); + this.apiCache = new LRUCache({ max: storageCacheSize }); this.blockCache = new BlockCache(this.maxBlockCacheSize); this.finalizedBlockHashes = new MaxSizeSet(this.maxBlockCacheSize); @@ -571,6 +573,16 @@ export abstract class BaseProvider extends AbstractProvider { await this.api.disconnect(); }; + getApiAt = async (blockHash: string): Promise> => { + const cached = this.apiCache.get(blockHash); + if (cached) return cached; + + const apiAt = await this.getApiAt(blockHash); + this.apiCache.set(blockHash, apiAt); // cache key is blockhash, so no need to check for finalization + + return apiAt; + }; + getNetwork = async (): Promise => { await this.isReady(); @@ -678,7 +690,7 @@ export abstract class BaseProvider extends AbstractProvider { const substrateAddress = await this.getSubstrateAddress(address, blockHash); - const apiAt = await this.api.at(blockHash); + const apiAt = await this.getApiAt(blockHash); const accountInfo = await apiAt.query.system.account(substrateAddress); return nativeToEthDecimal(accountInfo.data.free.toBigInt()); @@ -720,7 +732,7 @@ export abstract class BaseProvider extends AbstractProvider { const contractInfo = evmAccountInfo?.contractInfo.unwrapOr(null); if (!contractInfo) { return '0x'; } - const apiAt = await this.api.at(blockHash); + const apiAt = await this.getApiAt(blockHash); const code = await apiAt.query.evm.codes(contractInfo.codeHash); return code.toHex(); @@ -818,7 +830,7 @@ export abstract class BaseProvider extends AbstractProvider { Promise.resolve(position).then(hexValue), ]); - const apiAt = await this.api.at(blockHash); + const apiAt = await this.getApiAt(blockHash); const code = await apiAt.query.evm.accountStorages(address, hexZeroPad(resolvedPosition, 32)); return code.toHex(); @@ -1102,7 +1114,7 @@ export abstract class BaseProvider extends AbstractProvider { getSubstrateAddress = async (address: string, blockTag?: BlockTag): Promise => { const blockHash = await this._getBlockHash(blockTag); - const apiAt = await this.api.at(blockHash); + const apiAt = await this.getApiAt(blockHash); const substrateAccount = await apiAt.query.evmAccounts.accounts(address); return substrateAccount.isEmpty @@ -1112,7 +1124,7 @@ export abstract class BaseProvider extends AbstractProvider { getEvmAddress = async (substrateAddress: string, blockTag?: BlockTag): Promise => { const blockHash = await this._getBlockHash(blockTag); - const apiAt = await this.api.at(blockHash); + const apiAt = await this.getApiAt(blockHash); const evmAddress = await apiAt.query.evmAccounts.evmAddresses(substrateAddress); return getAddress(evmAddress.isEmpty ? computeDefaultEvmAddress(substrateAddress) : evmAddress.toString()); @@ -1129,7 +1141,7 @@ export abstract class BaseProvider extends AbstractProvider { this._getBlockHash(blockTag), ]); - const apiAt = await this.api.at(blockHash); + const apiAt = await this.getApiAt(blockHash); const accountInfo = await apiAt.query.evm.accounts(address); return accountInfo.unwrapOr(null); @@ -1474,11 +1486,12 @@ export abstract class BaseProvider extends AbstractProvider { } } - // TODO: test header not found should throw - const blockHash = (await this.api.rpc.chain.getBlockHash(blockNumber.toBigInt())).toHex(); - // if (_blockHash.isEmpty) { - // return logger.throwError('header not found', Logger.errors.CALL_EXCEPTION, { blockNumber }); - // } + const _blockHash = await this.api.rpc.chain.getBlockHash(blockNumber.toBigInt()); + if (_blockHash.isEmpty) { + //@ts-ignore + return logger.throwError('header not found', PROVIDER_ERRORS.HEADER_NOT_FOUND, { blockNumber }); + } + const blockHash = _blockHash.toHex(); if (isFinalized) { this.queryCache.set(cacheKey, blockHash); @@ -1501,6 +1514,9 @@ export abstract class BaseProvider extends AbstractProvider { const blockNumber = _blockNumber ?? (await this._getBlockNumber(blockHash)); const canonicalHash = await this.api.rpc.chain.getBlockHash(blockNumber); + if (canonicalHash.isEmpty) { + return logger.throwError('header not found', Logger.errors.CALL_EXCEPTION, { blockNumber }); + } return canonicalHash.toString() === blockHash; }; diff --git a/packages/eth-rpc-adapter/src/__tests__/endpoint-tests/eth_getBlockByNumber.test.ts b/packages/eth-rpc-adapter/src/__tests__/endpoint-tests/eth_getBlockByNumber.test.ts index daabb701..2d649bc9 100644 --- a/packages/eth-rpc-adapter/src/__tests__/endpoint-tests/eth_getBlockByNumber.test.ts +++ b/packages/eth-rpc-adapter/src/__tests__/endpoint-tests/eth_getBlockByNumber.test.ts @@ -25,4 +25,9 @@ describe('eth_getBlockByNumber', () => { value: '0x0', }); }); + + it('returns null when block not found', async () => { + const res = (await eth_getBlockByNumber([0x12345678, false])).data.result; + expect(res).toBeNull(); + }); }); From 56bbc57dc24fcc716b36c11fedf6bce2461e889e Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Fri, 1 Nov 2024 14:45:42 +0800 Subject: [PATCH 04/12] fix --- packages/eth-providers/src/base-provider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index 704cab43..d8972683 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -577,7 +577,7 @@ export abstract class BaseProvider extends AbstractProvider { const cached = this.apiCache.get(blockHash); if (cached) return cached; - const apiAt = await this.getApiAt(blockHash); + const apiAt = await this.api.at(blockHash); this.apiCache.set(blockHash, apiAt); // cache key is blockhash, so no need to check for finalization return apiAt; From d459d81b6814ee64767a6464873f99ad3263d500 Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Fri, 1 Nov 2024 23:39:47 +0800 Subject: [PATCH 05/12] polish --- packages/eth-providers/src/base-provider.ts | 62 +++++++++++++++------ 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index d8972683..00998872 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -578,7 +578,10 @@ export abstract class BaseProvider extends AbstractProvider { if (cached) return cached; const apiAt = await this.api.at(blockHash); - this.apiCache.set(blockHash, apiAt); // cache key is blockhash, so no need to check for finalization + + // do we need to check for finalization here? + // ApiAt is only a decoration and the actuall call is through api, so should be fine? + this.apiCache.set(blockHash, apiAt); return apiAt; }; @@ -616,6 +619,10 @@ export abstract class BaseProvider extends AbstractProvider { const blockHash = header.hash.toHex(); const blockNumber = header.number.toNumber(); + const cacheKey = `block-${blockHash}`; + const cached = this.queryCache.get(cacheKey); + if (cached) return cached; + const [block, headerExtended, timestamp, receiptsFromSubql] = await Promise.all([ this.api.rpc.chain.getBlock(blockHash), this.api.derive.chain.getHeader(blockHash), @@ -624,7 +631,7 @@ export abstract class BaseProvider extends AbstractProvider { ]); // blockscout need `toLowerCase` - const author = (await this.getEvmAddress(headerExtended.author.toString())).toLowerCase(); + const author = (await this.getEvmAddress(headerExtended.author.toString(), blockHash)).toLowerCase(); let receipts: TransactionReceipt[]; if (receiptsFromSubql?.length) { @@ -643,7 +650,7 @@ export abstract class BaseProvider extends AbstractProvider { const gasUsed = receipts.reduce((totalGas, tx) => totalGas.add(tx.gasUsed), BIGNUMBER_ZERO); - return { + const blockData: BlockData = { hash: blockHash, parentHash: headerExtended.parentHash.toHex(), number: blockNumber, @@ -667,6 +674,13 @@ export abstract class BaseProvider extends AbstractProvider { transactions, }; + + const isFinalized = blockNumber <= await this.finalizedBlockNumber; + if (isFinalized) { + this.queryCache.set(cacheKey, blockData); + } + + return blockData; }; getBlock = async (_blockHashOrBlockTag: BlockTag | string | Promise): Promise => @@ -1476,14 +1490,11 @@ export abstract class BaseProvider extends AbstractProvider { return logger.throwArgumentError('block number should be less than u32', 'blockNumber', blockNumber); } - const isFinalized = blockNumber.lte(await this.finalizedBlockNumber); const cacheKey = `blockHash-${blockNumber.toHexString()}`; - if (isFinalized) { - const cached = this.queryCache.get(cacheKey); - if (cached) { - return cached; - } + const cached = this.queryCache.get(cacheKey); + if (cached) { + return cached; } const _blockHash = await this.api.rpc.chain.getBlockHash(blockNumber.toBigInt()); @@ -1493,6 +1504,8 @@ export abstract class BaseProvider extends AbstractProvider { } const blockHash = _blockHash.toHex(); + // no need to check for canonicality here since this hash is just queries from rpc.chain.getBlockHash + const isFinalized = blockNumber.lte(await this.finalizedBlockNumber); if (isFinalized) { this.queryCache.set(cacheKey, blockHash); } @@ -1555,18 +1568,35 @@ export abstract class BaseProvider extends AbstractProvider { }; _getBlockHeader = async (blockTag?: BlockTag | Promise): Promise
=> { - const blockHash = await this._getBlockHash(await blockTag); + const [blockHash, blockNumber] = await Promise.all([ + this._getBlockHash(await blockTag), + this._getBlockNumber(await blockTag), + ]); + + const cacheKey = `header-${blockNumber}`; + const cached = this.queryCache.get
(cacheKey); + if (cached) { + return cached; + } try { - const header = await this.api.rpc.chain.getHeader(blockHash); + const [header, isCanonical] = await Promise.all([ + this.api.rpc.chain.getHeader(blockHash), + this._isBlockCanonical(blockHash, blockNumber), + ]); + + const isFinalized = ( + await this.finalizedBlockNumber >= blockNumber && + isCanonical + ); + + if (isFinalized) { + this.queryCache.set(cacheKey, header); + } return header; } catch (error) { - if ( - typeof error === 'object' && - typeof (error as any).message === 'string' && - (error as any).message.match(/Unable to retrieve header and parent from supplied hash/gi) - ) { + if ((error as any)?.message?.match?.(/Unable to retrieve header and parent from supplied hash/gi)) { //@ts-ignore return logger.throwError('header not found', PROVIDER_ERRORS.HEADER_NOT_FOUND, { blockHash }); } From 8b329fc6cd6ad7c1eeb56a0688ba946f413a14df Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Sat, 2 Nov 2024 00:07:39 +0800 Subject: [PATCH 06/12] fix --- packages/eth-providers/src/base-provider.ts | 25 +++++++-------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index 00998872..7e620eb5 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -342,7 +342,7 @@ export abstract class BaseProvider extends AbstractProvider { this.verbose = verbose; this.maxBlockCacheSize = maxBlockCacheSize; this.queryCache = new LRUCache({ max: storageCacheSize }); - this.apiCache = new LRUCache({ max: storageCacheSize }); + this.apiCache = new LRUCache({ max: 100 }); this.blockCache = new BlockCache(this.maxBlockCacheSize); this.finalizedBlockHashes = new MaxSizeSet(this.maxBlockCacheSize); @@ -580,7 +580,7 @@ export abstract class BaseProvider extends AbstractProvider { const apiAt = await this.api.at(blockHash); // do we need to check for finalization here? - // ApiAt is only a decoration and the actuall call is through api, so should be fine? + // ApiAt is only a decoration and the actuall result is from rpc call, so should be fine? this.apiCache.set(blockHash, apiAt); return apiAt; @@ -1504,7 +1504,7 @@ export abstract class BaseProvider extends AbstractProvider { } const blockHash = _blockHash.toHex(); - // no need to check for canonicality here since this hash is just queries from rpc.chain.getBlockHash + // no need to check for canonicality here since this blockHash is just queried from rpc const isFinalized = blockNumber.lte(await this.finalizedBlockNumber); if (isFinalized) { this.queryCache.set(cacheKey, blockHash); @@ -1568,28 +1568,19 @@ export abstract class BaseProvider extends AbstractProvider { }; _getBlockHeader = async (blockTag?: BlockTag | Promise): Promise
=> { - const [blockHash, blockNumber] = await Promise.all([ - this._getBlockHash(await blockTag), - this._getBlockNumber(await blockTag), - ]); + const blockHash = await this._getBlockHash(await blockTag); - const cacheKey = `header-${blockNumber}`; + const cacheKey = `header-${blockHash}`; const cached = this.queryCache.get
(cacheKey); if (cached) { return cached; } try { - const [header, isCanonical] = await Promise.all([ - this.api.rpc.chain.getHeader(blockHash), - this._isBlockCanonical(blockHash, blockNumber), - ]); - - const isFinalized = ( - await this.finalizedBlockNumber >= blockNumber && - isCanonical - ); + const header = await this.api.rpc.chain.getHeader(blockHash); + // no need to check for canonicality here since this header is just queried from rpc + const isFinalized = header.number.toNumber() <= await this.finalizedBlockNumber; if (isFinalized) { this.queryCache.set(cacheKey, header); } From adc7a0f83d2da429fee3c06afc0574d47377524c Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Sat, 2 Nov 2024 00:38:17 +0800 Subject: [PATCH 07/12] fix --- packages/eth-providers/src/base-provider.ts | 114 ++++++++++-------- .../eth_getBlockByNumber.test.ts | 11 +- .../src/__tests__/utils/eth-rpc-apis.ts | 1 + 3 files changed, 71 insertions(+), 55 deletions(-) diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index 7e620eb5..020aa453 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -619,67 +619,75 @@ export abstract class BaseProvider extends AbstractProvider { const blockHash = header.hash.toHex(); const blockNumber = header.number.toNumber(); + let blockDataFull: BlockData; + const cacheKey = `block-${blockHash}`; const cached = this.queryCache.get(cacheKey); - if (cached) return cached; - - const [block, headerExtended, timestamp, receiptsFromSubql] = await Promise.all([ - this.api.rpc.chain.getBlock(blockHash), - this.api.derive.chain.getHeader(blockHash), - getTimestamp(this.api, blockHash), - this.subql?.getAllReceiptsAtBlock(blockHash), - ]); + if (cached) { + blockDataFull = cached; + } else { + const [block, headerExtended, timestamp, receiptsFromSubql] = await Promise.all([ + this.api.rpc.chain.getBlock(blockHash), + this.api.derive.chain.getHeader(blockHash), + getTimestamp(this.api, blockHash), + this.subql?.getAllReceiptsAtBlock(blockHash), + ]); - // blockscout need `toLowerCase` - const author = (await this.getEvmAddress(headerExtended.author.toString(), blockHash)).toLowerCase(); + // blockscout need `toLowerCase` + const author = (await this.getEvmAddress(headerExtended.author.toString(), blockHash)).toLowerCase(); - let receipts: TransactionReceipt[]; - if (receiptsFromSubql?.length) { - receipts = receiptsFromSubql.map(subqlReceiptAdapter); - } else { - /* ---------- - if nothing is returned from subql, either no tx exists in this block, - or the block not finalized. So we still need to ask block cache. - ---------- */ - receipts = this.blockCache.getAllReceiptsAtBlock(blockHash); - } + let receipts: TransactionReceipt[]; + if (receiptsFromSubql?.length) { + receipts = receiptsFromSubql.map(subqlReceiptAdapter); + } else { + /* ---------- + if nothing is returned from subql, either no tx exists in this block, + or the block not finalized. So we still need to ask block cache. + ---------- */ + receipts = this.blockCache.getAllReceiptsAtBlock(blockHash); + } - const transactions = full - ? receipts.map(tx => receiptToTransaction(tx, block)) - : receipts.map(tx => tx.transactionHash as `0x${string}`); - - const gasUsed = receipts.reduce((totalGas, tx) => totalGas.add(tx.gasUsed), BIGNUMBER_ZERO); - - const blockData: BlockData = { - hash: blockHash, - parentHash: headerExtended.parentHash.toHex(), - number: blockNumber, - stateRoot: headerExtended.stateRoot.toHex(), - transactionsRoot: headerExtended.extrinsicsRoot.toHex(), - timestamp: Math.floor(timestamp / 1000), - nonce: DUMMY_BLOCK_NONCE, - mixHash: ZERO_BLOCK_HASH, - difficulty: ZERO, - totalDifficulty: ZERO, - gasLimit: BigNumber.from(BLOCK_GAS_LIMIT), - gasUsed, - - miner: author, - extraData: EMPTY_HEX_STRING, - sha3Uncles: EMTPY_UNCLE_HASH, - receiptsRoot: headerExtended.extrinsicsRoot.toHex(), - logsBloom: DUMMY_LOGS_BLOOM, // TODO: ??? - size: block.encodedLength, - uncles: EMTPY_UNCLES, - - transactions, - }; + const transactions = receipts.map(tx => receiptToTransaction(tx, block)); + const gasUsed = receipts.reduce((totalGas, tx) => totalGas.add(tx.gasUsed), BIGNUMBER_ZERO); + + const blockDataFull: BlockData = { + hash: blockHash, + parentHash: headerExtended.parentHash.toHex(), + number: blockNumber, + stateRoot: headerExtended.stateRoot.toHex(), + transactionsRoot: headerExtended.extrinsicsRoot.toHex(), + timestamp: Math.floor(timestamp / 1000), + nonce: DUMMY_BLOCK_NONCE, + mixHash: ZERO_BLOCK_HASH, + difficulty: ZERO, + totalDifficulty: ZERO, + gasLimit: BigNumber.from(BLOCK_GAS_LIMIT), + gasUsed, + + miner: author, + extraData: EMPTY_HEX_STRING, + sha3Uncles: EMTPY_UNCLE_HASH, + receiptsRoot: headerExtended.extrinsicsRoot.toHex(), + logsBloom: DUMMY_LOGS_BLOOM, // TODO: ??? + size: block.encodedLength, + uncles: EMTPY_UNCLES, + + transactions, + }; - const isFinalized = blockNumber <= await this.finalizedBlockNumber; - if (isFinalized) { - this.queryCache.set(cacheKey, blockData); + const isFinalized = blockNumber <= await this.finalizedBlockNumber; + if (isFinalized) { + this.queryCache.set(cacheKey, blockDataFull); + } } + const blockData = full + ? blockDataFull + : { + ...blockDataFull, + transactions: blockDataFull.transactions.map(tx => (tx as TX).hash as `0x${string}`), + }; + return blockData; }; diff --git a/packages/eth-rpc-adapter/src/__tests__/endpoint-tests/eth_getBlockByNumber.test.ts b/packages/eth-rpc-adapter/src/__tests__/endpoint-tests/eth_getBlockByNumber.test.ts index 2d649bc9..87268db3 100644 --- a/packages/eth-rpc-adapter/src/__tests__/endpoint-tests/eth_getBlockByNumber.test.ts +++ b/packages/eth-rpc-adapter/src/__tests__/endpoint-tests/eth_getBlockByNumber.test.ts @@ -1,10 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { deployErc20, eth_getBlockByNumber, testSetup } from '../utils'; +import { deployErc20, eth_getBlockByHash, eth_getBlockByNumber, testSetup } from '../utils'; const { wallet, provider } = testSetup; -describe('eth_getBlockByNumber', () => { +describe('eth_getBlockByNumber and eth_getBlockByHash', () => { it('get correct block info', async () => { const token = await deployErc20(wallet); const txHash = token.deployTransaction.hash; @@ -24,6 +24,13 @@ describe('eth_getBlockByNumber', () => { to: null, value: '0x0', }); + + // getblockbyhash should return the same result + const resHash = (await eth_getBlockByHash([resFull.hash, false])).data.result; + const resHashFull = (await eth_getBlockByHash([resFull.hash, true])).data.result; + + expect(resHash).to.deep.equal(res); + expect(resHashFull).to.deep.equal(resFull); }); it('returns null when block not found', async () => { diff --git a/packages/eth-rpc-adapter/src/__tests__/utils/eth-rpc-apis.ts b/packages/eth-rpc-adapter/src/__tests__/utils/eth-rpc-apis.ts index dd89153b..0bb8f1e7 100644 --- a/packages/eth-rpc-adapter/src/__tests__/utils/eth-rpc-apis.ts +++ b/packages/eth-rpc-adapter/src/__tests__/utils/eth-rpc-apis.ts @@ -21,6 +21,7 @@ export const rpcGet = export const eth_call = rpcGet('eth_call'); export const eth_blockNumber = rpcGet('eth_blockNumber'); export const eth_getBlockByNumber = rpcGet('eth_getBlockByNumber'); +export const eth_getBlockByHash = rpcGet('eth_getBlockByHash'); export const eth_getTransactionReceipt = rpcGet('eth_getTransactionReceipt'); export const eth_getLogs = rpcGet<{ data: { result: LogHexified[]; error?: JsonRpcError } }>('eth_getLogs'); export const eth_getTransactionByHash = rpcGet('eth_getTransactionByHash'); From f2dec75eb4eec1f0cb4eaf9366929acb0be947ca Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Sat, 2 Nov 2024 00:47:40 +0800 Subject: [PATCH 08/12] fix --- packages/eth-providers/src/base-provider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index 020aa453..9aa7a201 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -650,7 +650,7 @@ export abstract class BaseProvider extends AbstractProvider { const transactions = receipts.map(tx => receiptToTransaction(tx, block)); const gasUsed = receipts.reduce((totalGas, tx) => totalGas.add(tx.gasUsed), BIGNUMBER_ZERO); - const blockDataFull: BlockData = { + blockDataFull = { hash: blockHash, parentHash: headerExtended.parentHash.toHex(), number: blockNumber, From d7fb75a0a8b9832ac96c12f2e7a9e144e764d1ba Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Sun, 3 Nov 2024 17:48:22 +0800 Subject: [PATCH 09/12] polish --- packages/eth-providers/src/base-provider.ts | 11 ++++++++--- packages/eth-providers/src/utils/parseBlock.ts | 2 +- packages/eth-providers/src/utils/utils.ts | 6 ------ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index 9aa7a201..499424cb 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -71,7 +71,6 @@ import { filterLogByTopics, getAllReceiptsAtBlock, getHealthResult, - getTimestamp, getTransactionRequest, hexlifyRpcResult, isEvmExtrinsic, @@ -609,6 +608,12 @@ export abstract class BaseProvider extends AbstractProvider { : this.bestBlockNumber ); + getTimestamp = async (blockHash: string): Promise => { + const apiAt = await this.getApiAt(blockHash); + const timestamp = await apiAt.query.timestamp.now(); + return timestamp.toNumber(); + }; + getBlockData = async (_blockTag: BlockTag | Promise, full?: boolean): Promise => { const blockTag = await this._ensureSafeModeBlockTagFinalization(_blockTag); const header = await this._getBlockHeader(blockTag); @@ -629,7 +634,7 @@ export abstract class BaseProvider extends AbstractProvider { const [block, headerExtended, timestamp, receiptsFromSubql] = await Promise.all([ this.api.rpc.chain.getBlock(blockHash), this.api.derive.chain.getHeader(blockHash), - getTimestamp(this.api, blockHash), + this.getTimestamp(blockHash), this.subql?.getAllReceiptsAtBlock(blockHash), ]); @@ -1408,7 +1413,7 @@ export abstract class BaseProvider extends AbstractProvider { result.blockNumber = startBlock; result.blockHash = startBlockHash; - const timestamp = await getTimestamp(this.api, result.blockHash); + const timestamp = await this.getTimestamp(result.blockHash); result.timestamp = Math.floor(timestamp / 1000); result.wait = async (confirms?: number, timeoutMs?: number) => { diff --git a/packages/eth-providers/src/utils/parseBlock.ts b/packages/eth-providers/src/utils/parseBlock.ts index d46ef9bb..d26e8962 100644 --- a/packages/eth-providers/src/utils/parseBlock.ts +++ b/packages/eth-providers/src/utils/parseBlock.ts @@ -37,7 +37,7 @@ export const getAllReceiptsAtBlock = async ( blockHash: string, targetTxHash?: string ): Promise => { - const apiAtTargetBlock = await api.at(blockHash); + const apiAtTargetBlock = await api.at(blockHash); // TODO: integrate cache for this const [block, blockEvents] = await Promise.all([ api.rpc.chain.getBlock(blockHash), diff --git a/packages/eth-providers/src/utils/utils.ts b/packages/eth-providers/src/utils/utils.ts index 4d4384b1..51d2d5ee 100644 --- a/packages/eth-providers/src/utils/utils.ts +++ b/packages/eth-providers/src/utils/utils.ts @@ -412,9 +412,3 @@ export const toBN = (bigNumberis: BigNumberish = 0): BN => { // eslint-disable-next-line @typescript-eslint/no-explicit-any return new BN(bigNumberis as any); }; - -export const getTimestamp = async (api: ApiPromise, blockHash: string): Promise => { - const apiAt = await api.at(blockHash); - const timestamp = await apiAt.query.timestamp.now(); - return timestamp.toNumber(); -}; From 95b49d38e7490ef930e6429e3e3aab4b70d77151 Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Mon, 4 Nov 2024 14:43:15 +0800 Subject: [PATCH 10/12] encapsulate api cache to a class --- packages/eth-providers/src/base-provider.ts | 36 ++++++------------- .../eth-providers/src/utils/ApiAtCache.ts | 31 ++++++++++++++++ .../eth-providers/src/utils/parseBlock.ts | 17 ++++++--- 3 files changed, 54 insertions(+), 30 deletions(-) create mode 100644 packages/eth-providers/src/utils/ApiAtCache.ts diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index 499424cb..9d7f63ff 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -14,7 +14,6 @@ import { } from '@ethersproject/abstract-provider'; import { AcalaEvmTX, checkSignatureType, parseTransaction } from '@acala-network/eth-transactions'; import { AccessList, accessListify } from 'ethers/lib/utils'; -import { ApiDecoration, SubmittableExtrinsic } from '@polkadot/api/types'; import { ApiPromise } from '@polkadot/api'; import { AsyncAction } from 'rxjs/internal/scheduler/AsyncAction'; import { AsyncScheduler } from 'rxjs/internal/scheduler/AsyncScheduler'; @@ -27,6 +26,7 @@ import { Logger } from '@ethersproject/logger'; import { ModuleEvmModuleAccountInfo } from '@polkadot/types/lookup'; import { Network } from '@ethersproject/networks'; import { Observable, ReplaySubject, Subscription, firstValueFrom, throwError } from 'rxjs'; +import { SubmittableExtrinsic } from '@polkadot/api/types'; import { filter, first, timeout } from 'rxjs/operators'; import { getAddress } from '@ethersproject/address'; import { hexDataLength, hexValue, hexZeroPad, hexlify, isHexString, joinSignature } from '@ethersproject/bytes'; @@ -91,6 +91,7 @@ import { BlockCache, CacheInspect } from './utils/BlockCache'; import { MaxSizeSet } from './utils/MaxSizeSet'; import { SubqlProvider } from './utils/subqlProvider'; import { _Metadata } from './utils/gqlTypes'; +import { apiCache } from './utils/ApiAtCache'; export interface HeadsInfo { internalState: { @@ -303,7 +304,6 @@ export abstract class BaseProvider extends AbstractProvider { readonly verbose: boolean; readonly maxBlockCacheSize: number; readonly queryCache: LRUCache; - readonly apiCache: LRUCache>; readonly blockCache: BlockCache; readonly finalizedBlockHashes: MaxSizeSet; @@ -341,7 +341,6 @@ export abstract class BaseProvider extends AbstractProvider { this.verbose = verbose; this.maxBlockCacheSize = maxBlockCacheSize; this.queryCache = new LRUCache({ max: storageCacheSize }); - this.apiCache = new LRUCache({ max: 100 }); this.blockCache = new BlockCache(this.maxBlockCacheSize); this.finalizedBlockHashes = new MaxSizeSet(this.maxBlockCacheSize); @@ -572,19 +571,6 @@ export abstract class BaseProvider extends AbstractProvider { await this.api.disconnect(); }; - getApiAt = async (blockHash: string): Promise> => { - const cached = this.apiCache.get(blockHash); - if (cached) return cached; - - const apiAt = await this.api.at(blockHash); - - // do we need to check for finalization here? - // ApiAt is only a decoration and the actuall result is from rpc call, so should be fine? - this.apiCache.set(blockHash, apiAt); - - return apiAt; - }; - getNetwork = async (): Promise => { await this.isReady(); @@ -609,7 +595,7 @@ export abstract class BaseProvider extends AbstractProvider { ); getTimestamp = async (blockHash: string): Promise => { - const apiAt = await this.getApiAt(blockHash); + const apiAt = await apiCache.getApiAt(this.api, blockHash); const timestamp = await apiAt.query.timestamp.now(); return timestamp.toNumber(); }; @@ -717,7 +703,7 @@ export abstract class BaseProvider extends AbstractProvider { const substrateAddress = await this.getSubstrateAddress(address, blockHash); - const apiAt = await this.getApiAt(blockHash); + const apiAt = await apiCache.getApiAt(this.api, blockHash); const accountInfo = await apiAt.query.system.account(substrateAddress); return nativeToEthDecimal(accountInfo.data.free.toBigInt()); @@ -759,7 +745,7 @@ export abstract class BaseProvider extends AbstractProvider { const contractInfo = evmAccountInfo?.contractInfo.unwrapOr(null); if (!contractInfo) { return '0x'; } - const apiAt = await this.getApiAt(blockHash); + const apiAt = await apiCache.getApiAt(this.api, blockHash); const code = await apiAt.query.evm.codes(contractInfo.codeHash); return code.toHex(); @@ -796,7 +782,7 @@ export abstract class BaseProvider extends AbstractProvider { callRequest: SubstrateEvmCallRequest, at?: string, ): Promise => { - const api = at ? await this.api.at(at) : this.api; + const api = at ? await apiCache.getApiAt(this.api, at) : this.api; // call evm rpc when `state_call` is not supported yet if (!api.call.evmRuntimeRPCApi) { @@ -857,7 +843,7 @@ export abstract class BaseProvider extends AbstractProvider { Promise.resolve(position).then(hexValue), ]); - const apiAt = await this.getApiAt(blockHash); + const apiAt = await apiCache.getApiAt(this.api, blockHash); const code = await apiAt.query.evm.accountStorages(address, hexZeroPad(resolvedPosition, 32)); return code.toHex(); @@ -961,7 +947,7 @@ export abstract class BaseProvider extends AbstractProvider { extrinsic: SubmittableExtrinsic<'promise', ISubmittableResult>, at?: string, ) => { - const apiAt = await this.api.at(at ?? await this.bestBlockHash); + const apiAt = await apiCache.getApiAt(this.api, at ?? await this.bestBlockHash); const u8a = extrinsic.toU8a(); const lenIncreaseAfterSignature = 100; // approximate length increase after signature @@ -1141,7 +1127,7 @@ export abstract class BaseProvider extends AbstractProvider { getSubstrateAddress = async (address: string, blockTag?: BlockTag): Promise => { const blockHash = await this._getBlockHash(blockTag); - const apiAt = await this.getApiAt(blockHash); + const apiAt = await apiCache.getApiAt(this.api, blockHash); const substrateAccount = await apiAt.query.evmAccounts.accounts(address); return substrateAccount.isEmpty @@ -1151,7 +1137,7 @@ export abstract class BaseProvider extends AbstractProvider { getEvmAddress = async (substrateAddress: string, blockTag?: BlockTag): Promise => { const blockHash = await this._getBlockHash(blockTag); - const apiAt = await this.getApiAt(blockHash); + const apiAt = await apiCache.getApiAt(this.api, blockHash); const evmAddress = await apiAt.query.evmAccounts.evmAddresses(substrateAddress); return getAddress(evmAddress.isEmpty ? computeDefaultEvmAddress(substrateAddress) : evmAddress.toString()); @@ -1168,7 +1154,7 @@ export abstract class BaseProvider extends AbstractProvider { this._getBlockHash(blockTag), ]); - const apiAt = await this.getApiAt(blockHash); + const apiAt = await apiCache.getApiAt(this.api, blockHash); const accountInfo = await apiAt.query.evm.accounts(address); return accountInfo.unwrapOr(null); diff --git a/packages/eth-providers/src/utils/ApiAtCache.ts b/packages/eth-providers/src/utils/ApiAtCache.ts new file mode 100644 index 00000000..b4e251ff --- /dev/null +++ b/packages/eth-providers/src/utils/ApiAtCache.ts @@ -0,0 +1,31 @@ +import { ApiDecoration } from '@polkadot/api/types'; +import { ApiPromise } from '@polkadot/api'; +import LRUCache from 'lru-cache'; + +class ApiAtCache { + #apiAtCache: LRUCache>; + + constructor(maxCacheSize: number = 100) { + this.#apiAtCache = new LRUCache>({ + max: maxCacheSize, + }); + } + + getApiAt = async ( + api: ApiPromise, + blockHash: string + ): Promise> => { + const cached = this.#apiAtCache.get(blockHash); + if (cached) return cached; + + const apiAt = await api.at(blockHash); + + // do we need to check for finalization here? + // ApiAt is only a decoration and the actuall result is from rpc call, so should be fine? + this.#apiAtCache.set(blockHash, apiAt); + + return apiAt; + }; +} + +export const apiCache = new ApiAtCache(100); diff --git a/packages/eth-providers/src/utils/parseBlock.ts b/packages/eth-providers/src/utils/parseBlock.ts index d26e8962..2973a336 100644 --- a/packages/eth-providers/src/utils/parseBlock.ts +++ b/packages/eth-providers/src/utils/parseBlock.ts @@ -15,6 +15,7 @@ import { GenericExtrinsic } from '@polkadot/types'; import { TransactionReceipt } from '@ethersproject/abstract-provider'; import { BIGNUMBER_ZERO, ONE_HUNDRED_GWEI } from '../consts'; +import { apiCache } from './ApiAtCache'; import { findEvmEvent, formatter, @@ -37,26 +38,32 @@ export const getAllReceiptsAtBlock = async ( blockHash: string, targetTxHash?: string ): Promise => { - const apiAtTargetBlock = await api.at(blockHash); // TODO: integrate cache for this + const apiAt = await apiCache.getApiAt(api, blockHash); const [block, blockEvents] = await Promise.all([ api.rpc.chain.getBlock(blockHash), - apiAtTargetBlock.query.system.events(), + apiAt.query.system.events(), ]); - return parseReceiptsFromBlockData(api, block, blockEvents, targetTxHash); + return await parseReceiptsFromBlockData(api, block, blockEvents, targetTxHash, true); }; export const parseReceiptsFromBlockData = async ( api: ApiPromise, block: SignedBlock, blockEvents: FrameSystemEventRecord[], - targetTxHash?: string + targetTxHash?: string, + // this method is also used by subql, so disable cacheing by default to avoid potential compatibilty issues + useCache: boolean = false, ): Promise => { const { header } = block.block; const blockNumber = header.number.toNumber(); const blockHash = header.hash.toHex(); - const _apiAtParentBlock = api.at(header.parentHash); // don't wait here in case not being used + + // don't wait here in case not being used + const _apiAtParentBlock = useCache + ? apiCache.getApiAt(api, header.parentHash.toHex()) + : api.at(header.parentHash); const succeededEvmExtrinsics = block.block.extrinsics .map((extrinsic, idx) => { From 3404c82f4f728cc1115c49f1d0627a5a790f3c70 Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Tue, 5 Nov 2024 14:20:36 +0800 Subject: [PATCH 11/12] add cache tests --- .../src/__tests__/evm-rpc-provider.test.ts | 41 ------- .../src/__tests__/provider-cache.test.ts | 115 ++++++++++++++++++ .../eth-providers/src/utils/ApiAtCache.ts | 8 +- 3 files changed, 119 insertions(+), 45 deletions(-) create mode 100644 packages/eth-providers/src/__tests__/provider-cache.test.ts diff --git a/packages/eth-providers/src/__tests__/evm-rpc-provider.test.ts b/packages/eth-providers/src/__tests__/evm-rpc-provider.test.ts index d63f7203..abc655fe 100644 --- a/packages/eth-providers/src/__tests__/evm-rpc-provider.test.ts +++ b/packages/eth-providers/src/__tests__/evm-rpc-provider.test.ts @@ -55,47 +55,6 @@ describe('getReceiptAtBlock', async () => { }); }); -// TODO: maybe setup a subway to test -describe.skip('all cache', async () => { - const provider = EvmRpcProvider.from(ACALA_NODE_URL); - await provider.isReady(); - - afterAll(async () => await provider.disconnect()); - - it('getBlockHeader at latest block => header cache', async () => { - const { time: time1, res: header1 } = await runWithTiming(() => provider._getBlockHeader('latest'), 1); - const { time: time2, res: header2 } = await runWithTiming(() => provider._getBlockHeader('latest'), 1); - - // latest header should already be cached at the start - console.log('latest header:', { time1, time2 }); - expect(time1).to.be.lt(10); - expect(time2).to.be.lt(10); - expect(header1.toJSON()).to.deep.equal(header2.toJSON()); - }); - - it('getBlockHeader at random block => header cache', async () => { - const { time: time1, res: header1 } = await runWithTiming(() => provider._getBlockHeader(1234567), 1); - const { time: time2, res: header2 } = await runWithTiming(() => provider._getBlockHeader(1234567), 1); - - // second time should be 100x faster with cache, in poor network 800ms => 0.5ms - console.log('getBlockHeader:', { time1, time2 }); - expect(time2).to.be.lt(time1 / 20); // conservative multiplier - expect(time2).to.be.lt(10); // no async call - expect(header1.toJSON()).to.deep.equal(header2.toJSON()); - }); - - it('getBlockData at random block => header cache + storage cache + receipt cache', async () => { - const { time: time1, res: blockData1 } = await runWithTiming(() => provider.getBlockData(1234321), 1); - const { time: time2, res: blockData2 } = await runWithTiming(() => provider.getBlockData(1234321), 1); - - // second time should be 100x faster with cache, usually 1500ms => 3ms - console.log('getBlockData: ', { time1, time2 }); - expect(time2).to.be.lt(time1 / 20); // conservative multiplier - expect(time2).to.be.lt(30); // no async call - expect(blockData1).to.deep.equal(blockData2); - }); -}); - describe.concurrent('rpc test', async () => { const provider = EvmRpcProvider.from(endpoint); diff --git a/packages/eth-providers/src/__tests__/provider-cache.test.ts b/packages/eth-providers/src/__tests__/provider-cache.test.ts new file mode 100644 index 00000000..d33ac99e --- /dev/null +++ b/packages/eth-providers/src/__tests__/provider-cache.test.ts @@ -0,0 +1,115 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import dotenv from 'dotenv'; + +import { EvmRpcProvider } from '../rpc-provider'; +import { apiCache } from '../utils/ApiAtCache'; +import { runWithTiming } from '../utils'; + +dotenv.config(); + +const ACALA_NODE_URL = 'wss://acala-rpc.dwellir.com'; + +describe.only('provider cache', async () => { + let provider: EvmRpcProvider; + let provider2: EvmRpcProvider; + + beforeAll(async () => { + provider = EvmRpcProvider.from(ACALA_NODE_URL); + provider2 = EvmRpcProvider.from(ACALA_NODE_URL); // provider 2 to query some info without affecting cache + await provider.isReady(); + await provider2.isReady(); + }); + + afterAll(async () => await Promise.all([ + provider.disconnect(), + provider2.disconnect(), + ])); + + it('get apiAt', async() => { + const curBlock = await provider.getBlockNumber(); + const randomBlock = curBlock - Math.floor(Math.random() * 100000); + const blockHash = await provider2._getBlockHash(randomBlock); + + const { time: time1, res: apiAt1 } = await runWithTiming(() => apiCache.getApiAt(provider.api, blockHash), 1); + const { time: time2, res: apiAt2 } = await runWithTiming(() => apiCache.getApiAt(provider.api, blockHash), 1); + + expect(time1).to.be.gt(0, 'first get apiAt failed!'); + expect(time2).to.be.gt(0, 'second get apiAt failed!'); + console.log('get random apiAt:', { time1, time2 }); + + expect(time2).to.be.lt(time1 / 20); // conservative multiplier + expect(time2).to.be.lt(50); // no async call so should be almost instant + expect(apiAt1).to.equal(apiAt2); // should be the same instance + }); + + it('get block hash', async() => { + const curBlock = await provider.getBlockNumber(); + const randomBlock = curBlock - Math.floor(Math.random() * 100000); + + const { time: time1, res: hash1 } = await runWithTiming(() => provider._getBlockHash(randomBlock), 1); + const { time: time2, res: hash2 } = await runWithTiming(() => provider._getBlockHash(randomBlock), 1); + + expect(time1).to.be.gt(0, 'first get block hash failed!'); + expect(time2).to.be.gt(0, 'second get block hash failed!'); + console.log('get random block hash:', { time1, time2 }); + + expect(time2).to.be.lt(time1 / 20); // conservative multiplier + expect(time2).to.be.lt(50); // no async call so should be almost instant + expect(hash1).to.deep.equal(hash2); + }); + + it('get block', async() => { + const curBlock = await provider.getBlockNumber(); + const randomBlock = curBlock - Math.floor(Math.random() * 100000); + const blockHash = await provider2._getBlockHash(randomBlock); + + const { time: time1, res: blockNumber1 } = await runWithTiming(() => provider._getBlockNumber(blockHash), 1); + const { time: time2, res: blockNumber2 } = await runWithTiming(() => provider._getBlockNumber(blockHash), 1); + + expect(time1).to.be.gt(0, 'first get block number failed!'); + expect(time2).to.be.gt(0, 'second get block number failed!'); + console.log('get random block number:', { time1, time2 }); + + expect(time2).to.be.lt(time1 / 20); // conservative multiplier + expect(time2).to.be.lt(50); // no async call so should be almost instant + expect(blockNumber1).to.deep.equal(blockNumber2); + }); + + it('get block header', async() => { + const curBlock = await provider.getBlockNumber(); + const randomBlock = curBlock - Math.floor(Math.random() * 100000); + + const { time: time1, res: header1 } = await runWithTiming(() => provider._getBlockHeader(randomBlock), 1); + const { time: time2, res: header2 } = await runWithTiming(() => provider._getBlockHeader(randomBlock), 1); + + expect(time1).to.be.gt(0, 'first get header failed!'); + expect(time2).to.be.gt(0, 'second get header failed!'); + console.log('get random header:', { time1, time2 }); + + expect(time2).to.be.lt(time1 / 20); // conservative multiplier + expect(time2).to.be.lt(50); // no async call so should be almost instant + expect(header1.toJSON()).to.deep.equal(header2.toJSON()); + }); + + it('get block data', async () => { + const curBlock = await provider.getBlockNumber(); + const randomBlock = curBlock - Math.floor(Math.random() * 100000); + + const { time: time1, res: blockData1 } = await runWithTiming(() => provider.getBlockData(randomBlock), 1); + const { time: time2, res: blockData2 } = await runWithTiming(() => provider.getBlockData(randomBlock), 1); + const { time: time3, res: blockData3 } = await runWithTiming(() => provider.getBlockData(randomBlock, true), 1); + + expect(time1).to.be.gt(0, 'first get blockData failed!'); + expect(time2).to.be.gt(0, 'second get blockData failed!'); + expect(time3).to.be.gt(0, 'third get blockData failed!'); + console.log('get random blockData:', { time1, time2, time3 }); + + expect(time2).to.be.lt(time1 / 20); // conservative multiplier + expect(time2).to.be.lt(50); // no async call so should be almost instant + expect(time3).to.be.lt(time1 / 20); // conservative multiplier + expect(time3).to.be.lt(50); // no async call so should be almost instant + expect(blockData1).to.deep.equal(blockData2); + expect(blockData3.hash).to.deep.equal(blockData2.hash); + }); +}); + diff --git a/packages/eth-providers/src/utils/ApiAtCache.ts b/packages/eth-providers/src/utils/ApiAtCache.ts index b4e251ff..a9cc2438 100644 --- a/packages/eth-providers/src/utils/ApiAtCache.ts +++ b/packages/eth-providers/src/utils/ApiAtCache.ts @@ -3,10 +3,10 @@ import { ApiPromise } from '@polkadot/api'; import LRUCache from 'lru-cache'; class ApiAtCache { - #apiAtCache: LRUCache>; + #cache: LRUCache>; constructor(maxCacheSize: number = 100) { - this.#apiAtCache = new LRUCache>({ + this.#cache = new LRUCache>({ max: maxCacheSize, }); } @@ -15,14 +15,14 @@ class ApiAtCache { api: ApiPromise, blockHash: string ): Promise> => { - const cached = this.#apiAtCache.get(blockHash); + const cached = this.#cache.get(blockHash); if (cached) return cached; const apiAt = await api.at(blockHash); // do we need to check for finalization here? // ApiAt is only a decoration and the actuall result is from rpc call, so should be fine? - this.#apiAtCache.set(blockHash, apiAt); + this.#cache.set(blockHash, apiAt); return apiAt; }; From b0e18c8751d213bf01c16b8b616698b98a9589ea Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Tue, 5 Nov 2024 14:21:34 +0800 Subject: [PATCH 12/12] fix --- packages/eth-providers/src/__tests__/provider-cache.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/eth-providers/src/__tests__/provider-cache.test.ts b/packages/eth-providers/src/__tests__/provider-cache.test.ts index d33ac99e..3f0852f1 100644 --- a/packages/eth-providers/src/__tests__/provider-cache.test.ts +++ b/packages/eth-providers/src/__tests__/provider-cache.test.ts @@ -9,7 +9,7 @@ dotenv.config(); const ACALA_NODE_URL = 'wss://acala-rpc.dwellir.com'; -describe.only('provider cache', async () => { +describe.concurrent('provider cache', async () => { let provider: EvmRpcProvider; let provider2: EvmRpcProvider;