move Subscription functions (#4895)

* move Subscription functions

* move subscriptions to web3_eth

* test signed commit

* implement eth subscribe methods in api

* refactor

* fix

* add unit tests

* add callback. fix types. add subscribe and unsubscribe tests

* integration tests

* fix integration geth test

* fix geth background run

* fix

* test

* test

* fix tests

* fix

* add from block for logs event

* fix

* fix

* resolve conflicts

* fix types and callbacks

* fix unit test

* remove unnecessary isomorphic-ws

* add http.api and ws.api

Co-authored-by: Junaid <86780488+jdevcs@users.noreply.github.com>

* hot fix

* fix

Co-authored-by: Junaid <86780488+jdevcs@users.noreply.github.com>

.github/Dockerfile

@@ -1,5 +1,11 @@
 # syntax=docker/dockerfile:1
 FROM ethereum/client-go
-RUN echo '{"config":{"chainId":15,"homesteadBlock":0,"eip150Block":0,"eip155Block":0,"eip158Block":0,"byzantiumBlock":0,"constantinopleBlock":0,"petersburgBlock":0,"istanbulBlock":0,"berlinBlock":0,"londonBlock":0},"alloc":{"0xdc6bad79dab7ea733098f66f6c6f9dd008da3258":{"balance":"100000000000000000000"},"0x962f9a9c2a6c092474d24def35eccb3d9363265e":{"balance":"100000000000000000000"}},"coinbase":"0x0000000000000000000000000000000000000000","difficulty":"0x20000","gasLimit":"0x2fefd8","nonce":"0x0000000000000042","mixhash":"0x0000000000000000000000000000000000000000000000000000000000000000","parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000","timestamp":"0x00"}' > /genesis.json
-RUN geth init  --datadir data /genesis.json
-EXPOSE 8545
+RUN echo '{"config":{"chainId":4224,"homesteadBlock":1,"eip150Block":2,"eip155Block":3,"eip158Block":3,"byzantiumBlock":4},"alloc":{"0x0000000000000000000000000000000000000000":{"balance":"10000000000000000000000"},"0xdc6bad79dab7ea733098f66f6c6f9dd008da3258":{"balance":"100000000000000000000"},"0x962f9a9c2a6c092474d24def35eccb3d9363265e":{"balance":"100000000000000000000"}},"coinbase":"0x0000000000000000000000000000000000000000","difficulty":"0x1","gasLimit":"0x2fefd8","nonce":"0x0","mixhash":"0x0000000000000000000000000000000000000000000000000000000000000000","parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000","timestamp":"0x00","number":"0x0","gasUsed":"0x0"}' > /genesis.json
+
+RUN echo '123456' > /pass
+RUN echo '4c3758228f536f7a210f8936182fb5b728046970b8e3215d0b5cb4c4faae8a4e' > /k1
+
+RUN geth --datadir data account import k1 --password /pass
+
+RUN geth --datadir data init /genesis.json
+

.github/workflows/build.yml

@@ -63,7 +63,7 @@ jobs:
                 run: yarn ganache-background && yarn test:integration
 
             -   if: (env.TEST_CMD == 'e2e_geth')
-                run: yarn geth:build && yarn geth && yarn test:integration
+                run: yarn geth:build && yarn geth-background && yarn test:integration
 
             -   if: (env.TEST_CMD == 'e2e_browsers')
                 run: yarn ganache-background && yarn test:integration:browsers

karma.config.js

@@ -1,20 +1,25 @@
 const path = require('path');
+const webpack = require('webpack');
 const os = require('os');
 const { lstatSync, readdirSync } = require('fs');
-// get listing of packages in the mono repo
 const basePath = path.resolve(__dirname, 'packages');
 const packages = readdirSync(basePath).filter(name => {
+	if (name === 'web3-providers-ipc') {
+		return false;
+	}
 	return lstatSync(path.join(basePath, name)).isDirectory();
 });
+
 const listOfTests = packages.map(packageName =>
-	path.join('packages', packageName, 'test', 'integration', '**', '*.ts'),
+	path.join('packages', packageName, 'test', 'integration', '**', '*.test.ts'),
 );
 
+const outputPath = path.join(os.tmpdir(), '_karma_webpack_');
+
 const webpackConfig = {
 	mode: 'development',
 	output: {
-		filename: '[name].js',
-		path: path.join(os.tmpdir(), '_karma_webpack_'),
+		path: outputPath,
 	},
 	stats: {
 		modules: false,
@@ -25,10 +30,16 @@ const webpackConfig = {
 			{
 				test: /\.ts?$/,
 				use: 'ts-loader',
-				exclude: /node_modules/,
+				exclude: [/node_modules/, /unit/],
 			},
 		],
 	},
+	externals: {
+		fs: 'commonjs fs',
+		path: 'commonjs path',
+		net: 'commonjs net',
+	},
+
 	resolve: {
 		extensions: ['.ts', '.js'],
 		modules: [
@@ -39,7 +50,13 @@ const webpackConfig = {
 		],
 		fallback: {
 			crypto: require.resolve('crypto-browserify'),
-			stream: require.resolve('stream-browserify'),
+			stream: 'readable-stream',
+			assert: require.resolve('assert'),
+			// jest: require.resolve('jest'),
+		},
+		alias: {
+			'isomorphic-ws': path.join(__dirname, 'tools', 'isomorphic-ws'),
+			jest: path.join(__dirname, 'node_modules', 'jest'),
 		},
 	},
 	watch: false,
@@ -57,17 +74,30 @@ const webpackConfig = {
 			},
 		},
 	},
-	plugins: [],
+	plugins: [
+		new webpack.ProvidePlugin({
+			Buffer: ['buffer', 'Buffer'],
+		}),
+		new webpack.ProvidePlugin({
+			process: 'process/browser',
+		}),
+	],
 };
 module.exports = function (config) {
 	config.set({
+		frameworks: ['webpack', 'jasmine', 'browserify'],
 		plugins: [
+			'karma-browserify',
 			'karma-webpack',
 			'karma-jasmine',
 			'karma-chrome-launcher',
 			'karma-firefox-launcher',
 		],
-		browsers: ['ChromeHeadless', 'FirefoxHeadless'],
+		browsers: [
+			// 'Chrome',
+			'ChromeHeadless',
+			// 'FirefoxHeadless',
+		],
 		// base path that will be used to resolve all patterns (eg. files, exclude)
 		basePath: '',
 		colors: true,
@@ -76,22 +106,16 @@ module.exports = function (config) {
 		singleRun: true,
 		port: 9876,
 		concurrency: 10,
-		// frameworks to use
-		// available frameworks: https://npmjs.org/browse/keyword/karma-adapter
-		frameworks: ['webpack', 'jasmine'],
-		// list of files / patterns to load in the browser
-		// Here I'm including all of the the Jest tests which are all under the __tests__ directory.
-		// You may need to tweak this patter to find your test files/
 		files: listOfTests,
-		// preprocess matching files before serving them to the browser
-		// available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor
 		preprocessors: {
-			// Use webpack to bundle our tests files
 			...listOfTests.reduce(
-				(res, packagePath) => ({ ...res, [packagePath]: ['webpack'] }),
+				(res, packagePath) => ({ ...res, [packagePath]: ['webpack', 'browserify'] }),
 				{},
 			),
 		},
+		browserify: {
+			basedir: outputPath,
+		},
 		webpack: webpackConfig,
 	});
 };

package.json

@@ -37,7 +37,8 @@
 		"ganache": "npx ganache -m 'smart guide what forget tired jungle always expire rescue boring glue champion' --account=\"0x4c3758228f536f7a210f8936182fb5b728046970b8e3215d0b5cb4c4faae8a4e,100000000000000000000\" --account=\"0x34aeb1f338c17e6b440c189655c89fcef148893a24a7f15c0cb666d9cf5eacb3,100000000000000000000\"",
 		"ganache-background": "yarn ganache > /dev/null & client=$! npx wait-port 8545",
 		"geth:build": "docker build ./.github -t gethnode -f .github/Dockerfile",
-		"geth": "docker run -d -p 8545:8545 gethnode --datadir data --networkid 15 --http.addr 0.0.0.0 --http.port 8545 --http",
+		"geth": "docker run -p 8545:8545 gethnode --mine --miner.noverify --datadir data --networkid 4224 --nodiscover --nat \"any\" --ws --ws.addr 0.0.0.0 --ws.port 8545 --http --http.addr 0.0.0.0 --http.port 8545 --allow-insecure-unlock --http.api personal,web3,eth,admin,debug,miner,txpool,net --ws.api personal,web3,eth,admin,debug,miner,txpool,net --unlock 0 --password /pass",
+		"geth-background": "docker run -d -p 8545:8545 gethnode --mine --miner.noverify --datadir data --networkid 4224 --nodiscover --nat \"any\" --ws --ws.addr 0.0.0.0 --ws.port 8545 --http --http.addr 0.0.0.0 --http.port 8545 --allow-insecure-unlock --http.api personal,web3,eth,admin,debug,miner,txpool,net --ws.api personal,web3,eth,admin,debug,miner,txpool,net --unlock 0 --password /pass",
 		"lint": "lerna run lint --stream --parallel",
 		"lint:fix": "lerna run lint:fix --stream --parallel",
 		"format": "lerna run format --stream --parallel",
@@ -50,25 +51,31 @@
 	},
 	"devDependencies": {
 		"@types/node": "*",
+		"assert": "^2.0.0",
+		"browserify": "^17.0.0",
 		"buffer": "^6.0.3",
+		"bufferutil": "^4.0.6",
 		"crypto-browserify": "^3.12.0",
 		"eslint": "^8.3.0",
+		"https-browserify": "^1.0.0",
 		"husky": "^7.0.4",
 		"jasmine": "^4.0.2",
 		"jest": "^27.3.1",
 		"karma": "^6.3.17",
+		"karma-browserify": "^8.1.0",
 		"karma-chrome-launcher": "^3.1.1",
 		"karma-firefox-launcher": "^2.1.2",
 		"karma-jasmine": "^4.0.1",
 		"karma-webpack": "^5.0.0",
 		"lerna": "^4.0.0",
 		"lint-staged": "12.1.2",
 		"prettier": "^2.4.1",
-		"stream-browserify": "^3.0.0",
 		"ts-jest": "^27.0.7",
 		"ts-loader": "^9.2.8",
 		"ts-node": "^10.4.0",
 		"typescript": "^4.5.2",
+		"utf-8-validate": "^5.0.9",
+		"watchify": "^4.0.0",
 		"webpack": "^5.70.0"
 	},
 	"packageManager": "yarn@1.22.15"

packages/web3-common/src/eth_execution_api.ts

@@ -291,7 +291,7 @@ export type EthExecutionAPI = {
 			| ['logs', { address?: HexString; topics?: HexString[] }]
 	) => HexString;
 	eth_unsubscribe: (subscriptionId: HexString) => HexString;
-
+	eth_clearSubscriptions: (keepSyncing?: boolean) => void;
 	// Non-supported by execution-apis specs
 	eth_getCompilers: () => string[];
 	eth_compileSolidity: (code: string) => CompileResult;

packages/web3-core/src/web3_context.ts

@@ -83,7 +83,6 @@ export class Web3Context<
 		providerOrContext: SupportedProviders<API> | Web3ContextInitOptions<API, RegisteredSubs>,
 	) {
 		super();
-
 		if (
 			typeof providerOrContext === 'string' ||
 			isSupportedProvider(providerOrContext as SupportedProviders<API>)

packages/web3-core/src/web3_subscription_manager.ts

@@ -3,6 +3,13 @@ import { isSupportSubscriptions } from './utils';
 import { Web3RequestManager, Web3RequestManagerEvent } from './web3_request_manager';
 import { Web3SubscriptionConstructor } from './web3_subscriptions';
 
+type ShouldUnsubscribeCondition = ({
+	id: sub,
+}: {
+	id: string;
+	sub: unknown;
+}) => boolean | undefined;
+
 export class Web3SubscriptionManager<
 	API extends Web3APISpec,
 	RegisteredSubs extends { [key: string]: Web3SubscriptionConstructor<API> },
@@ -33,8 +40,7 @@ export class Web3SubscriptionManager<
 			throw new ProviderError('Provider not available');
 		}
 
-		const Klass = this.registeredSubscriptions[name];
-
+		const Klass: RegisteredSubs[T] = this.registeredSubscriptions[name];
 		if (!Klass) {
 			throw new SubscriptionError('Invalid subscription type');
 		}
@@ -78,16 +84,17 @@ export class Web3SubscriptionManager<
 		if (!this._subscriptions.has(sub.id)) {
 			throw new SubscriptionError(`Subscription with id "${sub.id}" does not exists`);
 		}
-
+		const { id } = sub;
 		await sub.unsubscribe();
-		this._subscriptions.delete(sub.id);
+		this._subscriptions.delete(id);
 	}
 
-	public async unsubscribe() {
+	public async unsubscribe(condition?: ShouldUnsubscribeCondition) {
 		const result = [];
-
-		for (const [, sub] of this._subscriptions.entries()) {
-			result.push(this.removeSubscription(sub));
+		for (const [id, sub] of this.subscriptions.entries()) {
+			if (!condition || (typeof condition === 'function' && condition({ id, sub }))) {
+				result.push(this.removeSubscription(sub));
+			}
 		}
 
 		return Promise.all(result);

packages/web3-core/src/web3_subscriptions.ts

@@ -1,7 +1,6 @@
 // eslint-disable-next-line max-classes-per-file
 import {
 	BlockOutput,
-	SyncOutput,
 	Web3BaseProvider,
 	Web3EventEmitter,
 	Web3EventMap,
@@ -16,10 +15,6 @@ import {
 import { HexString } from 'web3-utils';
 import { Web3RequestManager } from './web3_request_manager';
 
-type CommonSubscriptionEvents = {
-	error: Error;
-	connected: number;
-};
 export abstract class Web3Subscription<
 	EventMap extends Web3EventMap,
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -48,19 +43,21 @@ export abstract class Web3Subscription<
 	}
 
 	public async subscribe() {
-		const result = await this._requestManager.send({
+		this._id = await this._requestManager.send({
 			method: 'eth_subscribe',
 			params: this._buildSubscriptionParams(),
 		});
-		this._id = result;
 
 		const messageListener = (
-			_: Error | null,
+			err: Error | null,
 			data?: JsonRpcSubscriptionResult | JsonRpcNotification<Log>,
 		) => {
 			if (data && jsonRpc.isResponseWithNotification(data)) {
 				this._processSubscriptionResult(data?.params.result);
 			}
+			if (err) {
+				this._processSubscriptionError(err);
+			}
 		};
 
 		(this._requestManager.provider as Web3BaseProvider).on<Log>('message', messageListener);
@@ -95,6 +92,11 @@ export abstract class Web3Subscription<
 		// Do nothing - This should be overridden in subclass.
 	}
 
+	// eslint-disable-next-line class-methods-use-this, @typescript-eslint/no-unused-vars
+	protected _processSubscriptionError(_err: Error) {
+		// Do nothing - This should be overridden in subclass.
+	}
+
 	// eslint-disable-next-line class-methods-use-this
 	protected _buildSubscriptionParams(): Web3APIParams<API, 'eth_subscribe'> {
 		// This should be overridden in the subclass
@@ -112,39 +114,3 @@ export type Web3SubscriptionConstructor<
 	args: any,
 	options: { requestManager: Web3RequestManager<API> },
 ) => SubscriptionType;
-
-// TODO: This class to be moved `web3-eth` package.
-export class LogsSubscription extends Web3Subscription<
-	CommonSubscriptionEvents & {
-		data: { fromBlock: number; address: HexString | HexString[]; topics: (HexString | null)[] };
-		changed: {
-			fromBlock: number;
-			address: HexString | HexString[];
-			topics: (HexString | null)[];
-			removed: true;
-		};
-	},
-	{ address?: HexString; topics?: HexString[] }
-> {}
-
-// TODO: This class to be moved `web3-eth` package.
-export class PendingTransactionsSubscription extends Web3Subscription<
-	CommonSubscriptionEvents & {
-		data: HexString;
-	}
-> {}
-
-// TODO: This class to be moved `web3-eth` package.
-export class NewBlockHeadersSubscription extends Web3Subscription<
-	CommonSubscriptionEvents & {
-		data: BlockOutput;
-	}
-> {}
-
-// TODO: This class to be moved `web3-eth` package.
-export class SyncingSubscription extends Web3Subscription<
-	CommonSubscriptionEvents & {
-		data: SyncOutput;
-		changed: boolean;
-	}
-> {}

packages/web3-core/test/unit/fixtures/example_subscription.ts

@@ -1,4 +1,4 @@
-import { Web3Subscription } from '../../../src/web3_subscriptions';
+import { Web3Subscription } from '../../../src';
 
 export class ExampleSubscription extends Web3Subscription<
 	{ data: string },

packages/web3-core/test/unit/web3_subscription.test.ts

@@ -5,6 +5,7 @@ describe('Web3Subscription', () => {
 	let sub: ExampleSubscription;
 
 	beforeEach(() => {
+		// const web3 = new Web3('http://localhost:8545');
 		requestManager = {
 			send: jest.fn(),
 			on: jest.fn(),

packages/web3-eth/package.json

@@ -22,7 +22,7 @@
 		"test:ci": "jest --coverage=true --coverage-reporters=json --verbose",
 		"test:watch": "npm test -- --watch",
 		"test:unit": "jest --config=./test/unit/jest.config.js",
-		"test:integration": "jest --config=./test/integration/jest.config.js --passWithNoTests"
+		"test:integration": "jest --config=./test/integration/jest.config.js --forceExit"
 	},
 	"devDependencies": {
 		"@types/jest": "^27.0.3",
@@ -45,8 +45,9 @@
 		"web3-common": "1.0.0-alpha.0",
 		"web3-core": "4.0.0-alpha.0",
 		"web3-eth-accounts": "4.0.0-alpha.0",
+		"web3-net": "4.0.0-alpha.0",
+		"web3-providers-ws": "4.0.0-alpha.0",
 		"web3-utils": "4.0.0-alpha.1",
-		"web3-validator": "^0.1.0-alpha.0",
-		"web3-net": "4.0.0-alpha.0"
+		"web3-validator": "^0.1.0-alpha.0"
 	}
 }