diff --git a/.prettierignore b/.prettierignore index f1afd7481b5..961dd96231b 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,4 +1,5 @@ **/build +**/dist **/.github **/*.env **/node_modules @@ -16,6 +17,7 @@ packages/chain-events/.eslintrc.js **/contractTypes **/factories **/eth/types +**/eth/artifacts **/.eslintrc.js **/ios/App/App diff --git a/packages/chain-events/Procfile b/packages/chain-events/Procfile index b64bf8a8628..79a89848f55 100644 --- a/packages/chain-events/Procfile +++ b/packages/chain-events/Procfile @@ -1,6 +1,6 @@ -web: node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/app/Server.js -subscriber1: CHAIN_SUBSCRIBER_INDEX=0 node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script -subscriber2: CHAIN_SUBSCRIBER_INDEX=1 node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script -subscriber3: CHAIN_SUBSCRIBER_INDEX=2 node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script -consumer: node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainEventsConsumer/chainEventsConsumer.js run-as-script +web: node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/app/Server.js +subscriber1: CHAIN_SUBSCRIBER_INDEX=0 node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script +subscriber2: CHAIN_SUBSCRIBER_INDEX=1 node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script +subscriber3: CHAIN_SUBSCRIBER_INDEX=2 node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script +consumer: node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainEventsConsumer/chainEventsConsumer.js run-as-script release: cd packages/chain-events && npx sequelize-cli db:migrate --config services/database/sequelize.json diff --git a/packages/chain-events/package.json b/packages/chain-events/package.json index 9a0970a6eb1..9e7bd9f0c33 100644 --- a/packages/chain-events/package.json +++ b/packages/chain-events/package.json @@ -29,9 +29,9 @@ "batch-poll": "ts-node -T ./scripts/batchPoller.ts", "preyalcpublish": "yarn build", "emit-events": "ts-node --project tsconfig.json ./scripts/emitChainEvents.ts", - "start-app": "ts-node --project tsconfig.services.json services/app/Server.ts", - "start-consumer": "ts-node --project tsconfig.services.json services/ChainEventsConsumer/chainEventsConsumer.ts run-as-script", - "start-subscriber": "ts-node --project tsconfig.services.json services/ChainSubscriber/chainSubscriber.ts run-as-script", + "start-app": "ts-node-dev --max-old-space-size=4096 --respawn --transpile-only --project tsconfig.services.json services/app/Server.ts", + "start-consumer": "ts-node-dev --max-old-space-size=4096 --respawn --transpile-only --project tsconfig.services.json services/ChainEventsConsumer/chainEventsConsumer.ts run-as-script", + "start-subscriber": "ts-node-dev --max-old-space-size=4096 --respawn --transpile-only --project tsconfig.services.json services/ChainSubscriber/chainSubscriber.ts run-as-script", "start-all": "concurrently -p '{name}' -c yellow,blue,magenta -n app,consumer,subscriber 'yarn start-app' 'yarn start-consumer' 'yarn start-subscriber'", "create-db": "npx sequelize db:create", "migrate-db": "npx sequelize db:migrate", @@ -80,6 +80,7 @@ "pg-format": "^1.0.4", "rollbar": "^2.25.2", "sleep-promise": "^8.0.1", + "ts-node-dev": "^2.0.0", "typescript-logging": "^0.6.4", "underscore": "^1.10.2", "web3": "^1.3.1", diff --git a/packages/chain-events/scripts/migrateChainEntities.ts b/packages/chain-events/scripts/migrateChainEntities.ts index fb7623b5a1e..78477a4891b 100644 --- a/packages/chain-events/scripts/migrateChainEntities.ts +++ b/packages/chain-events/scripts/migrateChainEntities.ts @@ -28,6 +28,9 @@ import type { BrokerConfig } from 'rascal'; import { RABBITMQ_URI } from '../../commonwealth/server/config'; import { constructSubstrateUrl } from '../../commonwealth/shared/substrate'; import { CHAIN_EVENT_SERVICE_SECRET, CW_SERVER_URL } from '../services/config'; +import NotificationsHandler from '../services/ChainEventsConsumer/ChainEventHandlers/notification'; +import models from '../services/database/database'; +import { EventKind } from '../src/chains/substrate/types'; const log = factory.getLogger(formatFilename(__filename)); @@ -95,6 +98,14 @@ async function migrateChainEntity( rmqController, chain ); + + const excludedNotificationEvents = [EventKind.DemocracyTabled]; + const notificationsHandler = new NotificationsHandler( + models, + rmqController, + excludedNotificationEvents + ); + let fetcher: IStorageFetcher; const range: IDisconnectedRange = { startBlock: 0 }; if (chainInstance.base === ChainBase.Substrate) { @@ -153,7 +164,8 @@ async function migrateChainEntity( try { // eslint-disable-next-line no-await-in-loop const dbEvent = await migrationHandler.handle(event); - await entityArchivalHandler.handle(event, dbEvent); + const ceEvent = await entityArchivalHandler.handle(event, dbEvent); + await notificationsHandler.handle(event, ceEvent); } catch (e) { log.error(`Event handle failure: ${e.message}`); } diff --git a/packages/chain-events/scripts/publishCustomRabbitMQMessage.ts b/packages/chain-events/scripts/publishCustomRabbitMQMessage.ts new file mode 100644 index 00000000000..9a9bd46334b --- /dev/null +++ b/packages/chain-events/scripts/publishCustomRabbitMQMessage.ts @@ -0,0 +1,83 @@ +import type { IEventData } from '../src/chains/aave/types'; +import type { CWEvent } from '../src'; +import { SupportedNetwork } from '../src'; +import { publishRmqMsg } from 'common-common/src/rabbitmq/util'; +import { RABBITMQ_API_URI } from '../services/config'; +import { RascalExchanges, RascalRoutingKeys } from 'common-common/src/rabbitmq'; +import models from 'chain-events/services/database/database'; + +async function main() { + const ceData = { + id: 10, + kind: 'proposal-created', + values: ['0'], + targets: ['0xE710CEd57456D3A16152c32835B5FB4E72D9eA5b'], + endBlock: 16203604, + executor: '0x64c7d40c07EFAbec2AafdC243bF59eaF2195c6dc', + ipfsHash: + '0x3876d28a014bc20432dcc3549ba95710446b98431d84c7f84fde6abe1baf527f', + proposer: '0xb55a948763e0d386b6dEfcD8070a522216AE42b1', + strategy: '0x90Dfd35F4a0BB2d30CDf66508085e33C353475D9', + calldatas: [ + '0x00000000000000000000000092d6c1e31e14520e676a687f0a93788b716beff5000000000000000000000000a8541f948411b3f95d9e89e8d339a56a9ed3d00b000000000000000000000000000000000000000000002fa54641bae8aaa00000', + ], + signatures: ['transfer(address,address,uint256)'], + startBlock: 16177324, + }; + const chainEvent: CWEvent = { + blockNumber: 16170754, + data: ceData, + network: SupportedNetwork.Aave, + chain: 'dydx', + }; + + const publishJson = await publishRmqMsg( + RABBITMQ_API_URI, + RascalExchanges.ChainEvents, + RascalRoutingKeys.ChainEvents, + chainEvent + ); + + console.log(publishJson); +} + +async function clear() { + try { + let eventsDeleted = 0, + entitiesDeleted = 0; + await models.sequelize.transaction(async (t) => { + const entityId = ( + await models.ChainEntity.findOne({ + where: { chain: 'dydx', type_id: '10' }, + transaction: t, + }) + )?.id; + + if (entityId) { + eventsDeleted = await models.ChainEvent.destroy({ + where: { entity_id: entityId }, + transaction: t, + }); + + entitiesDeleted = await models.ChainEntity.destroy({ + where: { id: entityId }, + transaction: t, + }); + } else { + console.log('Entity does not exist.'); + } + }); + + console.log( + `Events deleted: ${eventsDeleted}\nEntities deleted: ${entitiesDeleted}` + ); + } catch (e) { + console.log('Failed to clear - reverted.'); + console.error(e); + } + + process.exit(1); +} + +if (process.argv[2] === 'clear') clear(); +else main(); diff --git a/packages/chain-events/scripts/testSequelizeQuery.ts b/packages/chain-events/scripts/testSequelizeQuery.ts deleted file mode 100644 index 46015304ac4..00000000000 --- a/packages/chain-events/scripts/testSequelizeQuery.ts +++ /dev/null @@ -1,21 +0,0 @@ -import models from '../services/database/database'; - -async function testSequelizeQuery() { - const eventTypes = ( - await models.ChainEventType.findAll({ - attributes: ['id'], - where: { chain: 'edgeware' }, - }) - ).map((x) => x.id); - - const dbResult = await models.ChainEvent.max('block_number', { - where: { - chain_event_type_id: eventTypes, - }, - }); - - console.log('Database Result:', dbResult); - process.exit(0); -} - -testSequelizeQuery(); diff --git a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/entityArchival.ts b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/entityArchival.ts index 39989846aa9..8a0253a9038 100644 --- a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/entityArchival.ts +++ b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/entityArchival.ts @@ -9,6 +9,7 @@ import { RascalPublications } from 'common-common/src/rabbitmq/types'; import type { DB } from '../../database/database'; +import type { ChainEventInstance } from 'chain-events/services/database/models/chain_event'; import type { CWEvent, IChainEntityKind, @@ -40,7 +41,10 @@ export default class extends IEventHandler { * `dbEvent` is the database entry corresponding to the `event`. */ // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types - public async handle(event: CWEvent, dbEvent) { + public async handle( + event: CWEvent, + dbEvent: ChainEventInstance + ) { // eslint-disable-next-line @typescript-eslint/no-shadow const log = factory.getLogger( addPrefix(__filename, [event.network, event.chain]) @@ -173,5 +177,7 @@ export default class extends IEventHandler { break; } } + + return dbEvent; } } diff --git a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/migration.ts b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/migration.ts index ce476149235..bb70619778b 100644 --- a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/migration.ts +++ b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/migration.ts @@ -2,26 +2,22 @@ * Processes events during migration, upgrading from simple notifications to entities. */ import type { WhereOptions } from 'sequelize'; -import type { - RabbitMQController, - RmqCENotificationCUD, - RmqCETypeCUD, -} from 'common-common/src/rabbitmq'; -import { RascalPublications } from 'common-common/src/rabbitmq'; +import type { RabbitMQController } from 'common-common/src/rabbitmq'; import { factory, formatFilename } from 'common-common/src/logging'; import type { CWEvent } from '../../../src'; import { - IEventHandler, + EntityEventKind, eventToEntity, getUniqueEntityKey, - EntityEventKind, + IEventHandler, } from '../../../src'; import type { DB } from '../../database/database'; import type { ChainEventAttributes, ChainEventInstance, } from '../../database/models/chain_event'; + const log = factory.getLogger(formatFilename(__filename)); export default class extends IEventHandler { @@ -47,48 +43,18 @@ export default class extends IEventHandler { fieldValue: string, eventType: EntityEventKind ) => { - const [dbEventType, created] = - await this._models.ChainEventType.findOrCreate({ - where: { - id: `${chain}-${event.data.kind.toString()}`, - chain, - event_network: event.network, - event_name: event.data.kind.toString(), - }, - }); - log.trace( - `${created ? 'created' : 'found'} chain event type: ${dbEventType.id}` - ); - - if (created) { - const publishData: RmqCETypeCUD.RmqMsgType = { - chainEventTypeId: dbEventType.id, - cud: 'create', - }; - - await this._rmqController.safePublish( - publishData, - dbEventType.id, - RascalPublications.ChainEventTypeCUDMain, - { - sequelize: this._models.sequelize, - model: this._models.ChainEventType, - } - ); - } - const queryFieldName = `event_data.${fieldName}`; const queryArgs: WhereOptions = eventType === EntityEventKind.Vote ? { - chain_event_type_id: dbEventType.id, [queryFieldName]: fieldValue, // votes will be unique by data rather than by type event_data: event.data as any, + chain, } : { - chain_event_type_id: dbEventType.id, [queryFieldName]: fieldValue, + chain, }; const existingEvent = await this._models.ChainEvent.findOne({ where: queryArgs, @@ -101,31 +67,12 @@ export default class extends IEventHandler { } log.info('No existing event found, creating new event in db!'); - const dbEvent = await this._models.ChainEvent.create({ - chain_event_type_id: dbEventType.id, + return await this._models.ChainEvent.create({ block_number: event.blockNumber, event_data: event.data, + network: event.network, + chain, }); - - const formattedEvent: ChainEventAttributes = dbEvent.toJSON(); - formattedEvent.ChainEventType = dbEventType.toJSON(); - - const publishData: RmqCENotificationCUD.RmqMsgType = { - ChainEvent: formattedEvent, - event, - cud: 'create', - }; - - await this._rmqController.safePublish( - publishData, - dbEvent.id, - RascalPublications.ChainEventNotificationsCUDMain, - { - sequelize: this._models.sequelize, - model: this._models.ChainEvent, - } - ); - return dbEvent; }; const entity = eventToEntity(event.network, event.data.kind); diff --git a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/notification.ts b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/notification.ts index 9d243bacc9a..b62a059c6c1 100644 --- a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/notification.ts +++ b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/notification.ts @@ -3,11 +3,15 @@ import type { RmqCENotificationCUD } from 'common-common/src/rabbitmq/types'; import { RascalPublications } from 'common-common/src/rabbitmq/types'; import { addPrefix, factory } from '../../../src/logging'; -import type { ChainEventAttributes } from '../../database/models/chain_event'; +import type { ChainEventInstance } from '../../database/models/chain_event'; import type { DB } from '../../database/database'; import type { CWEvent, IChainEventKind } from 'chain-events/src'; -import { IEventHandler } from 'chain-events/src'; +import { + EntityEventKind, + eventToEntity, + IEventHandler, +} from 'chain-events/src'; export default class extends IEventHandler { public readonly name = 'Notification Producer'; @@ -21,7 +25,7 @@ export default class extends IEventHandler { } // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types - public async handle(event: CWEvent, dbEvent) { + public async handle(event: CWEvent, dbEvent: ChainEventInstance) { const log = factory.getLogger( addPrefix(__filename, [event.network, event.chain]) ); @@ -36,25 +40,24 @@ export default class extends IEventHandler { return dbEvent; } - let dbEventType; - try { - dbEventType = await dbEvent.getChainEventType(); - if (!dbEventType) { - log.error(`Failed to fetch event type! Ignoring.`); - return; - } - } catch (e) { - log.error( - `Failed to get chain-event type for event: ${JSON.stringify(event)}` - ); + if (!dbEvent.entity_id) { + log.info(`No related entity, skipping!`); return dbEvent; } - const formattedEvent: ChainEventAttributes = dbEvent.toJSON(); - formattedEvent.ChainEventType = dbEventType.toJSON(); + const [, eventEntityKind] = eventToEntity(event.network, event.data.kind); + if ( + eventEntityKind != EntityEventKind.Create && + eventEntityKind != EntityEventKind.Complete + ) { + log.trace( + `Event does not mark the creation or completion of an entity. Skipping event!` + ); + return dbEvent; + } const publishData: RmqCENotificationCUD.RmqMsgType = { - ChainEvent: formattedEvent, + ChainEvent: dbEvent.toJSON(), event, cud: 'create', }; diff --git a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/storage.ts b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/storage.ts index 530fd26f631..e1cd27f04e7 100644 --- a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/storage.ts +++ b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/storage.ts @@ -2,10 +2,7 @@ * Generic handler that stores the event in the database. */ import { addPrefix, factory } from 'common-common/src/logging'; -import type { - RabbitMQController, - RmqCETypeCUD, -} from 'common-common/src/rabbitmq'; +import type { RabbitMQController } from 'common-common/src/rabbitmq'; import { RascalPublications } from 'common-common/src/rabbitmq'; import NodeCache from 'node-cache'; import hash from 'object-hash'; @@ -82,51 +79,11 @@ export default class extends IEventHandler { return; } - // locate event type and add event (and event type if needed) to database - const [dbEventType, created] = - await this._models.ChainEventType.findOrCreate({ - where: { - id: `${chain}-${event.data.kind.toString()}`, - chain, - event_network: event.network, - event_name: event.data.kind.toString(), - }, - }); - - if (created) { - const publishData: RmqCETypeCUD.RmqMsgType = { - chainEventTypeId: dbEventType.id, - cud: 'create', - }; - - await this._rmqController.safePublish( - publishData, - dbEventType.id, - RascalPublications.ChainEventTypeCUDMain, - { - sequelize: this._models.sequelize, - model: this._models.ChainEventType, - } - ); - - log.info(`STORAGE HANDLER MESSAGE PUBLISHED`); - } - - if (!dbEventType) { - log.error(`unknown event type: ${event.data.kind}`); - return; - } else { - if (created) { - log.info(`Created new ChainEventType: ${dbEventType.id}`); - } else { - log.trace(`found chain event type: ${dbEventType.id}`); - } - } - const eventData = { - chain_event_type_id: dbEventType.id, block_number: event.blockNumber, event_data: event.data, + network: event.network, + chain, }; // duplicate event check @@ -137,8 +94,6 @@ export default class extends IEventHandler { if (!cachedEvent) { const dbEvent = await this._models.ChainEvent.create(eventData); - // populate chainEventType, so we don't need to re-populate it in subsequence handlers - dbEvent.ChainEventType = dbEventType; // no need to save the entire event data since the key is the hash of the data this.eventCache.set(eventKey, true); diff --git a/packages/chain-events/services/ChainEventsConsumer/chainEventsConsumer.ts b/packages/chain-events/services/ChainEventsConsumer/chainEventsConsumer.ts index 67b3178723a..9fd3927769f 100644 --- a/packages/chain-events/services/ChainEventsConsumer/chainEventsConsumer.ts +++ b/packages/chain-events/services/ChainEventsConsumer/chainEventsConsumer.ts @@ -78,8 +78,8 @@ export async function setupChainEventConsumer(): Promise { // WARNING: due to dbEvent in each handler ORDER OF HANDLERS MATTERS! const allChainEventHandlers = [ storageHandler, - notificationsHandler, entityArchivalHandler, + notificationsHandler, ]; // setup Chain diff --git a/packages/chain-events/services/ChainEventsConsumer/republishMessages.ts b/packages/chain-events/services/ChainEventsConsumer/republishMessages.ts deleted file mode 100644 index 172f687e0a6..00000000000 --- a/packages/chain-events/services/ChainEventsConsumer/republishMessages.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type { - RabbitMQController, - RmqCETypeCUD, -} from 'common-common/src/rabbitmq'; -import { - RascalPublications, - RepublishFailedMessages, -} from 'common-common/src/rabbitmq'; -import * as Sequelize from 'sequelize'; - -import type { DB } from '../database/database'; - -/** - * A worker that periodically republishes data from the database if it's queued value is between -1 and 5. A queued - * value of -1 - * - */ -export class RepublishMessages extends RepublishFailedMessages { - constructor(_rmqController: RabbitMQController, _models: DB) { - super(_rmqController, _models, 180000); - } - - protected async job(): Promise { - const result = await this._models.ChainEventType.findAll({ - where: { - queued: { - [Sequelize.Op.between]: [-1, 5], - }, - }, - }); - - // TODO - // if (result.length > 100) {} - - for (const eventType of result) { - const publishData: RmqCETypeCUD.RmqMsgType = { - chainEventTypeId: eventType.id, - cud: 'create', - }; - - await this._rmqController.safePublish( - publishData, - eventType.id, - RascalPublications.ChainEventTypeCUDMain, - { - sequelize: this._models.sequelize, - model: this._models.ChainEventType, - } - ); - } - } -} diff --git a/packages/chain-events/services/ChainSubscriber/util.ts b/packages/chain-events/services/ChainSubscriber/util.ts index cfc40a44d25..e398aed4ffb 100644 --- a/packages/chain-events/services/ChainSubscriber/util.ts +++ b/packages/chain-events/services/ChainSubscriber/util.ts @@ -8,6 +8,7 @@ import type { SubstrateEvents } from '../../src'; import { createListener, ErcLoggingHandler, + getChainEventNetwork, LoggingHandler, SupportedNetwork, } from '../../src'; @@ -218,15 +219,15 @@ async function setupNewListeners( ) { for (const chain of newChains) { let network: SupportedNetwork; - if (chain.base === ChainBase.Substrate) - network = SupportedNetwork.Substrate; - else if (chain.base === ChainBase.CosmosSDK) - network = SupportedNetwork.Cosmos; - else if (chain.network === ChainNetwork.Compound) - network = SupportedNetwork.Compound; - else if (chain.network === ChainNetwork.Aave) - network = SupportedNetwork.Aave; - + try { + network = getChainEventNetwork(chain.network, chain.base); + } catch (e) { + log.error( + `Unknown chain base: ${chain.base} \tand network: ${chain.network}`, + e + ); + continue; + } try { log.info(`Starting listener for: ${chain.id}`); listenerInstances[chain.id] = await createListener(chain.id, network, { @@ -379,21 +380,8 @@ export function getListenerNames( async function discoverReconnectRange(this: DB, chain: string) { let latestBlock; try { - const eventTypes = ( - await this.ChainEventType.findAll({ - where: { chain }, - }) - ).map((x) => x.id); - - if (eventTypes.length === 0) { - log.info(`[${chain}]: No event types exist in the database`); - return { startBlock: null }; - } - latestBlock = await this.ChainEvent.max('block_number', { - where: { - chain_event_type_id: eventTypes, - }, + where: { chain }, }); if (latestBlock) { diff --git a/packages/chain-events/services/app/routes/entities.ts b/packages/chain-events/services/app/routes/entities.ts index 8bc45eb5e26..7debfb30255 100644 --- a/packages/chain-events/services/app/routes/entities.ts +++ b/packages/chain-events/services/app/routes/entities.ts @@ -1,5 +1,5 @@ import type { Response, NextFunction, Request } from 'express'; -import { AppError } from 'common-common/src/errors'; +import { AppError, ServerError } from 'common-common/src/errors'; import type { DB } from '../../database/database'; @@ -22,7 +22,6 @@ const entities: any = async ( { model: models.ChainEvent, order: [[models.ChainEvent, 'id', 'asc']], - include: [models.ChainEventType], }, ], order: [['created_at', 'DESC']], @@ -42,11 +41,17 @@ const entities: any = async ( if (req.query.completed) { entityFindOptions.where.completed = true; } - const entities = await models.ChainEntity.findAll(entityFindOptions); - return res.json({ - status: 'Success', - result: entities.map((e) => e.toJSON()), - }); + + try { + const fetchedEntities = await models.ChainEntity.findAll(entityFindOptions); + return res.json({ + status: 'Success', + result: fetchedEntities.map((e) => e.toJSON()), + }); + } catch (err) { + console.error(err); + return next(new ServerError(`Failed to fetch entities from DB`, err)); + } }; export default entities; diff --git a/packages/chain-events/services/app/routes/eventActivity.ts b/packages/chain-events/services/app/routes/eventActivity.ts index 14d2ede2298..34e56fabd3c 100644 --- a/packages/chain-events/services/app/routes/eventActivity.ts +++ b/packages/chain-events/services/app/routes/eventActivity.ts @@ -1,6 +1,5 @@ import type { NextFunction, Request, Response } from 'express'; -import { AppError } from 'common-common/src/errors'; -import { QueryTypes } from 'sequelize'; +import { AppError, ServerError } from 'common-common/src/errors'; import type { DB } from '../../database/database'; @@ -18,26 +17,21 @@ const eventActivity: any = async ( return next(new AppError(Errors.NeedLimit)); } - const events = await models.sequelize.query( - ` - SELECT ce.id, - ce.chain_event_type_id, - ce.block_number, - ce.event_data, - ce.created_at, - ce.updated_at, - ce.entity_id, - cet.chain, - cet.event_network - FROM "ChainEvents" ce - JOIN "ChainEventTypes" cet ON ce.chain_event_type_id = cet.id - ORDER BY ce.created_at DESC - LIMIT ?; - `, - { replacements: [req.query.limit], raw: true, type: QueryTypes.SELECT } - ); - - return res.json({ status: 'Success', result: events }); + try { + // we can order by id since the resulting order is almost exactly the same as when ordered by created_at + // but ordering by id is much faster due to primary key index + const events = await models.ChainEvent.findAll({ + order: [['id', 'DESC']], + limit: req.query.limit, + }); + return res.json({ + status: 'Success', + result: events.map((e) => e.toJSON()), + }); + } catch (e) { + console.error(e); + return next(new ServerError(`Failed to fetch events from DB`, e)); + } }; export default eventActivity; diff --git a/packages/chain-events/services/database/database.ts b/packages/chain-events/services/database/database.ts index a7ca48f25d0..0c3b3339c34 100644 --- a/packages/chain-events/services/database/database.ts +++ b/packages/chain-events/services/database/database.ts @@ -6,8 +6,6 @@ import type { ChainEntityModelStatic } from './models/chain_entity'; import ChainEntityFactory from './models/chain_entity'; import type { ChainEventModelStatic } from './models/chain_event'; import ChainEventFactory from './models/chain_event'; -import type { ChainEventTypeModelStatic } from './models/chain_event_type'; -import ChainEventTypeFactory from './models/chain_event_type'; import { factory, formatFilename } from 'common-common/src/logging'; const log = factory.getLogger(formatFilename(__filename)); @@ -15,7 +13,6 @@ const log = factory.getLogger(formatFilename(__filename)); export type Models = { ChainEntity: ChainEntityModelStatic; ChainEvent: ChainEventModelStatic; - ChainEventType: ChainEventTypeModelStatic; }; export interface DB extends Models { @@ -52,7 +49,6 @@ export const sequelize = new Sequelize(DATABASE_URI, { const models: Models = { ChainEntity: ChainEntityFactory(sequelize, DataTypes), ChainEvent: ChainEventFactory(sequelize, DataTypes), - ChainEventType: ChainEventTypeFactory(sequelize, DataTypes), }; const db: DB = { diff --git a/packages/chain-events/services/database/migrations/20221206100637-move-network-to-event.js b/packages/chain-events/services/database/migrations/20221206100637-move-network-to-event.js new file mode 100644 index 00000000000..428270b2058 --- /dev/null +++ b/packages/chain-events/services/database/migrations/20221206100637-move-network-to-event.js @@ -0,0 +1,41 @@ +'use strict'; + +module.exports = { + up: async (queryInterface, Sequelize) => { + await queryInterface.sequelize.transaction(async (t) => { + await queryInterface.addColumn( + 'ChainEvents', + 'network', + { + type: Sequelize.STRING, + allowNull: true, + }, + { transaction: t } + ); + + await queryInterface.sequelize.query( + ` + UPDATE "ChainEvents" CE + SET network = CET.event_network + FROM "ChainEventTypes" CET + WHERE CE.chain_event_type_id = CET.id; + `, + { transaction: t } + ); + + await queryInterface.changeColumn( + 'ChainEvents', + 'network', + { + type: Sequelize.STRING, + allowNull: false, + }, + { transaction: t } + ); + }); + }, + + down: async (queryInterface, Sequelize) => { + await queryInterface.removeColumn('ChainEvents', 'network'); + }, +}; diff --git a/packages/chain-events/services/database/migrations/20230401044247-redo-cet-migrations.js b/packages/chain-events/services/database/migrations/20230401044247-redo-cet-migrations.js new file mode 100644 index 00000000000..59a3b1e23c5 --- /dev/null +++ b/packages/chain-events/services/database/migrations/20230401044247-redo-cet-migrations.js @@ -0,0 +1,40 @@ +'use strict'; + +module.exports = { + up: async (queryInterface, Sequelize) => { + await queryInterface.sequelize.transaction(async (t) => { + await queryInterface.addColumn( + 'ChainEvents', + 'chain', + { + type: Sequelize.STRING, + allowNull: true, + }, + { transaction: t } + ); + await queryInterface.sequelize.query( + ` + UPDATE "ChainEvents" CE + SET chain = CET.chain + FROM "ChainEventTypes" CET + WHERE CE.chain_event_type_id = CET.id; + `, + { transaction: t } + ); + + await queryInterface.changeColumn( + 'ChainEvents', + 'chain', + { + type: Sequelize.STRING, + allowNull: false, + }, + { transaction: t } + ); + }); + }, + + down: async (queryInterface, Sequelize) => { + await queryInterface.removeColumn('ChainEvents', 'chain'); + }, +}; diff --git a/packages/chain-events/services/database/migrations/20230401044247-remove-chain-event-types.js b/packages/chain-events/services/database/migrations/20230401044247-remove-chain-event-types.js new file mode 100644 index 00000000000..0d8fa60bba0 --- /dev/null +++ b/packages/chain-events/services/database/migrations/20230401044247-remove-chain-event-types.js @@ -0,0 +1,45 @@ +'use strict'; + +module.exports = { + up: async (queryInterface, Sequelize) => { + await queryInterface.sequelize.transaction(async (t) => { + await queryInterface.removeColumn('ChainEvents', 'chain_event_type_id', { + transaction: t, + }); + await queryInterface.dropTable('ChainEventTypes', { transaction: t }); + }); + }, + + down: async (queryInterface, Sequelize) => { + await queryInterface.sequelize.transaction(async (t) => { + await queryInterface.createTable( + 'ChainEventTypes', + { + // id = chain-event_name (event_name is value of string enum) + id: { type: Sequelize.STRING, primaryKey: true }, + chain: { type: Sequelize.STRING, allowNull: false }, + // should never be null, but added here for migration purposes + event_network: { type: Sequelize.STRING, allowNull: true }, + event_name: { type: Sequelize.STRING, allowNull: false }, + queued: { + type: Sequelize.SMALLINT, + allowNull: false, + defaultValue: 0, + }, + }, + { transaction: t } + ); + + await queryInterface.addColumn( + 'ChainEvents', + 'chain_event_type_id', + { + type: Sequelize.STRING, + allowNull: true, + references: { model: 'ChainEventTypes', key: 'id' }, + }, + { transaction: t } + ); + }); + }, +}; diff --git a/packages/chain-events/services/database/models/chain_entity.ts b/packages/chain-events/services/database/models/chain_entity.ts index 24dd00f45ed..6cd71a80b56 100644 --- a/packages/chain-events/services/database/models/chain_entity.ts +++ b/packages/chain-events/services/database/models/chain_entity.ts @@ -1,7 +1,7 @@ import type * as Sequelize from 'sequelize'; import type { DataTypes } from 'sequelize'; -import type { ChainEventAttributes } from './chain_event'; +import type { ChainEventAttributes, ChainEventInstance } from './chain_event'; import type { ModelStatic, ModelInstance } from './types'; export type ChainEntityAttributes = { @@ -18,7 +18,9 @@ export type ChainEntityAttributes = { ChainEvents?: ChainEventAttributes[]; }; -export type ChainEntityInstance = ModelInstance; +export type ChainEntityInstance = ModelInstance & { + getChainEvents: Sequelize.HasManyGetAssociationsMixin; +}; export type ChainEntityModelStatic = ModelStatic; diff --git a/packages/chain-events/services/database/models/chain_event.ts b/packages/chain-events/services/database/models/chain_event.ts index 7ba1827c13e..dd69cd9745b 100644 --- a/packages/chain-events/services/database/models/chain_event.ts +++ b/packages/chain-events/services/database/models/chain_event.ts @@ -3,27 +3,27 @@ import type { DataTypes } from 'sequelize'; import type { ModelStatic, ModelInstance } from './types'; import type { - ChainEventTypeAttributes, - ChainEventTypeInstance, -} from './chain_event_type'; -import type { ChainEntityAttributes } from './chain_entity'; + ChainEntityAttributes, + ChainEntityInstance, +} from './chain_entity'; +import type { SupportedNetwork } from '../../../src'; export type ChainEventAttributes = { id: number; - chain_event_type_id: string; block_number: number; event_data: any; queued: number; entity_id?: number; + network: SupportedNetwork; + chain: string; created_at?: Date; updated_at?: Date; - ChainEventType?: ChainEventTypeAttributes; ChainEntity?: ChainEntityAttributes; }; export type ChainEventInstance = ModelInstance & { - getChainEventType: Sequelize.HasOneGetAssociationMixin; + getChainEntity: Sequelize.HasOneGetAssociationMixin; }; export type ChainEventModelStatic = ModelStatic; @@ -36,13 +36,14 @@ export default ( 'ChainEvent', { id: { type: dataTypes.INTEGER, primaryKey: true, autoIncrement: true }, - chain_event_type_id: { type: dataTypes.STRING, allowNull: false }, block_number: { type: dataTypes.INTEGER, allowNull: false }, entity_id: { type: dataTypes.INTEGER, allowNull: true }, event_data: { type: dataTypes.JSONB, allowNull: false }, created_at: { type: dataTypes.DATE, allowNull: false }, updated_at: { type: dataTypes.DATE, allowNull: false }, queued: { type: dataTypes.SMALLINT, allowNull: false, defaultValue: 0 }, + chain: { type: dataTypes.STRING, allowNull: false }, + network: { type: dataTypes.STRING, allowNull: false }, }, { tableName: 'ChainEvents', @@ -59,11 +60,6 @@ export default ( ); ChainEvent.associate = (models) => { - // master event type - models.ChainEvent.belongsTo(models.ChainEventType, { - foreignKey: 'chain_event_type_id', - targetKey: 'id', - }); models.ChainEvent.belongsTo(models.ChainEntity, { foreignKey: 'entity_id', targetKey: 'id', diff --git a/packages/chain-events/services/database/models/chain_event_type.ts b/packages/chain-events/services/database/models/chain_event_type.ts deleted file mode 100644 index 940fcccc448..00000000000 --- a/packages/chain-events/services/database/models/chain_event_type.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type * as Sequelize from 'sequelize'; -import type { DataTypes } from 'sequelize'; - -import type { ChainEventAttributes } from './chain_event'; -import type { ModelStatic, ModelInstance } from './types'; - -export type ChainEventTypeAttributes = { - id: string; - chain: string; - event_network: string; - event_name: string; - queued: number; - ChainEvents?: ChainEventAttributes[]; -}; - -export type ChainEventTypeInstance = ModelInstance; - -export type ChainEventTypeModelStatic = ModelStatic; - -export default ( - sequelize: Sequelize.Sequelize, - dataTypes: typeof DataTypes -): ChainEventTypeModelStatic => { - const ChainEventType = sequelize.define( - 'ChainEventType', - { - // id = chain-event_name (event_name is value of string enum) - id: { type: dataTypes.STRING, primaryKey: true }, - chain: { type: dataTypes.STRING, allowNull: false }, - // should never be null, but added here for migration purposes - event_network: { type: dataTypes.STRING, allowNull: true }, - event_name: { type: dataTypes.STRING, allowNull: false }, - queued: { type: dataTypes.SMALLINT, allowNull: false, defaultValue: 0 }, - }, - { - tableName: 'ChainEventTypes', - timestamps: false, - underscored: true, - indexes: [{ fields: ['id'] }, { fields: ['chain', 'event_name'] }], - } - ); - - ChainEventType.associate = (models) => { - // many emitted events of this type - models.ChainEventType.hasMany(models.ChainEvent, { as: 'events' }); - }; - - return ChainEventType; -}; diff --git a/packages/chain-events/src/interfaces.ts b/packages/chain-events/src/interfaces.ts index 4a06943a83b..fac21c0e542 100644 --- a/packages/chain-events/src/interfaces.ts +++ b/packages/chain-events/src/interfaces.ts @@ -19,6 +19,7 @@ import type { Api as CompoundApi } from './chains/compound/types'; import type { Api as CommonwealthApi } from './chains/commonwealth/types'; import type { Api as AaveApi } from './chains/aave/types'; import type { Listener } from './Listener'; +import { ChainBase, ChainNetwork } from 'common-common/src/types'; // add other events here as union types export type IChainEntityKind = @@ -205,6 +206,27 @@ export interface IEventTitle { export type TitlerFilter = (kind: IChainEventKind) => IEventTitle; +/** + * This function takes the network and base attribute of our current chains model and returns the relevant chain-event + * network. The chain-event network is an instance of SupportedNetwork enum. This is NOT the same as the network on + * the commonwealth chains model. This function is useful for determining which SupportedNetwork listener a + * Commonwealth 'chain' should use. Throws if the given chainNetwork and chainBase don't match a chain-event network + * i.e. SupportedNetwork. + * @param chainNetwork The network attribute of the Commonwealth chain model + * @param chainBase The base attribute of the Commonwealth chain model + */ +export function getChainEventNetwork( + chainNetwork: string, + chainBase: string +): SupportedNetwork { + if (chainBase === ChainBase.Substrate) return SupportedNetwork.Substrate; + else if (chainBase === ChainBase.CosmosSDK) return SupportedNetwork.Cosmos; + else if (chainNetwork === ChainNetwork.Compound) + return SupportedNetwork.Compound; + else if (chainNetwork === ChainNetwork.Aave) return SupportedNetwork.Aave; + else throw new Error('No matching SupportedNetwork'); +} + /** * Returns the key of the value that is unique to the entities chain and type i.e. the key whose associated value * becomes the type_id of the chain-entity. The combination of chain, type, and type_id must/will always be unique. diff --git a/packages/chain-events/test/integration/service-tests/chainEventsConsumer.test.ts b/packages/chain-events/test/integration/service-tests/chainEventsConsumer.test.ts index 7f7b9819425..56731b0510d 100644 --- a/packages/chain-events/test/integration/service-tests/chainEventsConsumer.test.ts +++ b/packages/chain-events/test/integration/service-tests/chainEventsConsumer.test.ts @@ -4,7 +4,6 @@ import { RascalQueues, RascalRoutingKeys, RmqCENotificationCUD, - RmqCETypeCUD, RmqEntityCUD, } from 'common-common/src/rabbitmq'; import type { ServiceConsumer } from 'common-common/src/serviceConsumer'; @@ -185,21 +184,6 @@ describe('Tests for the ChainEventsConsumer service', () => { expect(dbResult).to.not.be.null; - // check whether the new chain-event-type was pushed to the appropriate queue - const message = await getRmqMessage( - RABBITMQ_API_URI, - RascalQueues.ChainEventTypeCUDMain, - false - ); - expect(message).to.have.property('length'); - expect(message.length).to.equal(1); - expect(JSON.parse(message[0].payload)).to.deep.equal({ - chainEventTypeId: 'random-chain-transfer', - cud: 'create', - }); - expect(RmqCETypeCUD.isValidMsgFormat(JSON.parse(message[0].payload))).to.be - .true; - await models.ChainEvent.destroy({ where: { block_number: chainEvent.blockNumber, @@ -212,10 +196,6 @@ describe('Tests for the ChainEventsConsumer service', () => { }, }, }); - - await models.ChainEventType.destroy({ - where: { id: `${chain}-${ceData.kind}` }, - }); }); it('Should push new chain-events to the chain-event notifications queue', async () => { diff --git a/packages/common-common/src/rabbitmq/rabbitMQConfig.ts b/packages/common-common/src/rabbitmq/rabbitMQConfig.ts index 3e4dfdc4c70..7c8e94501e5 100644 --- a/packages/common-common/src/rabbitmq/rabbitMQConfig.ts +++ b/packages/common-common/src/rabbitmq/rabbitMQConfig.ts @@ -117,10 +117,6 @@ export function getRabbitMQConfig(rabbitmq_uri: string): Rascal.BrokerConfig { 'x-message-ttl': 600000, }, }, - [RascalQueues.ChainEventTypeCUDMain]: { - ...queueConfig, - options: queueOptions, - }, [RascalQueues.SnapshotProposalNotifications]: { ...queueConfig, options: { @@ -161,12 +157,6 @@ export function getRabbitMQConfig(rabbitmq_uri: string): Rascal.BrokerConfig { destinationType: 'queue', bindingKey: RascalBindings.ChainEventNotifications, }, - [RascalBindings.ChainEventType]: { - source: RascalExchanges.CUD, - destination: RascalQueues.ChainEventTypeCUDMain, - destinationType: 'queue', - bindingKey: RascalRoutingKeys.ChainEventTypeCUD, - }, [RascalBindings.SnapshotProposalNotifications]: { source: RascalExchanges.Notifications, destination: RascalQueues.SnapshotProposalNotifications, @@ -207,11 +197,6 @@ export function getRabbitMQConfig(rabbitmq_uri: string): Rascal.BrokerConfig { routingKey: RascalRoutingKeys.ChainEventNotifications, ...publicationConfig, }, - [RascalPublications.ChainEventTypeCUDMain]: { - exchange: RascalExchanges.CUD, - routingKey: RascalRoutingKeys.ChainEventTypeCUD, - ...publicationConfig, - }, [RascalPublications.SnapshotProposalNotifications]: { exchange: RascalExchanges.Notifications, routingKey: RascalRoutingKeys.SnapshotProposalNotifications, @@ -240,10 +225,6 @@ export function getRabbitMQConfig(rabbitmq_uri: string): Rascal.BrokerConfig { queue: RascalQueues.ChainEventNotifications, ...subscriptionConfig, }, - [RascalSubscriptions.ChainEventTypeCUDMain]: { - queue: RascalQueues.ChainEventTypeCUDMain, - ...subscriptionConfig, - }, [RascalSubscriptions.SnapshotProposalNotifications]: { queue: RascalQueues.SnapshotProposalNotifications, ...subscriptionConfig, diff --git a/packages/common-common/src/rabbitmq/rabbitMQController.ts b/packages/common-common/src/rabbitmq/rabbitMQController.ts index b4be0f3f28e..e3a8fc99ff5 100644 --- a/packages/common-common/src/rabbitmq/rabbitMQController.ts +++ b/packages/common-common/src/rabbitmq/rabbitMQController.ts @@ -9,15 +9,13 @@ import type Rollbar from 'rollbar'; import type { Sequelize } from 'sequelize'; import type { ChainEntityModelStatic } from 'chain-events/services/database/models/chain_entity'; import type { ChainEventModelStatic } from 'chain-events/services/database/models/chain_event'; -import type { ChainEventTypeModelStatic } from 'chain-events/services/database/models/chain_event_type'; import { factory, formatFilename } from 'common-common/src/logging'; const log = factory.getLogger(formatFilename(__filename)); export type SafeRmqPublishSupported = | ChainEntityModelStatic - | ChainEventModelStatic - | ChainEventTypeModelStatic; + | ChainEventModelStatic; export class RabbitMQControllerError extends Error { constructor(msg: string) { diff --git a/packages/common-common/src/rabbitmq/types/chainEventNotificationsCUD.ts b/packages/common-common/src/rabbitmq/types/chainEventNotificationsCUD.ts index 90792696438..a8e5359ce2c 100644 --- a/packages/common-common/src/rabbitmq/types/chainEventNotificationsCUD.ts +++ b/packages/common-common/src/rabbitmq/types/chainEventNotificationsCUD.ts @@ -23,19 +23,16 @@ export const RmqCENotificationCUD: RmqMsgNamespace = { - getInvalidFormatError(chainEventType: any): RmqMsgFormatError { - return new RmqMsgFormatError( - `The following chain-event-type is improperly formatted: ${JSON.stringify( - chainEventType - )}` - ); - }, - - isValidMsgFormat(data: any): data is IRmqMsgCreateCETypeCUD { - return !!( - data.chainEventTypeId && - typeof data.chainEventTypeId === 'string' && - data.cud === 'create' - ); - }, - - checkMsgFormat(data: any): void { - const valid = this.isValidMsgFormat(data); - if (!valid) { - console.log( - `The following chain-event-type is improperly formatted: ${JSON.stringify( - data - )}` - ); - throw this.getInvalidFormatError(data); - } - }, -}; - -export namespace RmqCETypeCUD { - export type RmqMsgType = IRmqMsgCreateCETypeCUD; -} diff --git a/packages/common-common/src/rabbitmq/types/index.ts b/packages/common-common/src/rabbitmq/types/index.ts index f9ce9e90b6c..573bf664049 100644 --- a/packages/common-common/src/rabbitmq/types/index.ts +++ b/packages/common-common/src/rabbitmq/types/index.ts @@ -1,10 +1,8 @@ import type { RmqEntityCUD } from './chainEntityCUD'; import type { RmqCENotificationCUD } from './chainEventNotificationsCUD'; -import type { RmqCETypeCUD } from './chainEventTypeCUD'; export * from './chainEntityCUD'; export * from './chainEventNotificationsCUD'; -export * from './chainEventTypeCUD'; export * from './chainEvents'; export * from './chainEventNotification'; @@ -30,7 +28,6 @@ export class RmqMsgFormatError extends Error { export type TRmqMessages = | RmqEntityCUD.RmqMsgType | RmqCENotificationCUD.RmqMsgType - | RmqCETypeCUD.RmqMsgType | RmqCWEvent.RmqMsgType | RmqCENotification.RmqMsgType | RmqSnapshotEvent.RmqMsgType @@ -47,7 +44,6 @@ export enum RascalPublications { ChainEntityCUDMain = 'ChainEntityCUDMainPublication', ChainEventNotificationsCUDMain = 'ChainEventNotificationsCUDMainPublication', ChainEventNotifications = 'ChainEventNotificationsPublication', - ChainEventTypeCUDMain = 'ChainEventTypeCUDMainPublication', SnapshotProposalNotifications = 'SnapshotProposalNotificationsPublication', SnapshotListener = 'SnapshotListenerPublication', } @@ -57,7 +53,6 @@ export enum RascalSubscriptions { ChainEntityCUDMain = 'ChainEntityCUDMainSubscription', ChainEventNotificationsCUDMain = 'ChainEventNotificationsCUDSubscription', ChainEventNotifications = 'ChainEventNotificationsSubscription', - ChainEventTypeCUDMain = 'ChainEventTypeCUDMainSubscription', SnapshotProposalNotifications = 'SnapshotProposalNotificationsSubscription', SnapshotListener = 'SnapshotListenerSubscription', } @@ -75,7 +70,6 @@ export enum RascalQueues { ChainEntityCUDMain = 'ChainEntityCUDMainQueue', ChainEventNotificationsCUDMain = 'ChainEventNotificationsCUDMainQueue', ChainEventNotifications = 'ChainEventNotificationsQueue', - ChainEventTypeCUDMain = 'ChainEventTypeCUDMainQueue', DeadLetter = 'DeadLetterQueue', SnapshotProposalNotifications = 'SnapshotProposalNotificationsQueue', SnapshotListener = 'SnapshotListenerQueue', @@ -86,7 +80,6 @@ export enum RascalBindings { ChainEntityCUDMain = 'ChainEntityCUDMainBinding', ChainEventNotificationsCUD = 'ChainEventNotificationsCUDBinding', ChainEventNotifications = 'ChainEventNotificationsBinding', - ChainEventType = 'ChainEventTypeBinding', SnapshotProposalNotifications = 'SnapshotProposalNotificationsBinding', SnapshotListener = 'SnapshotListenerBinding', DeadLetter = 'DeadLetterBinding', @@ -97,7 +90,6 @@ export enum RascalRoutingKeys { ChainEntityCUD = 'ChainEntityCUD', ChainEventNotificationsCUD = 'ChainEventNotificationsCUD', ChainEventNotifications = 'ChainEventNotifications', - ChainEventTypeCUD = 'ChainEventTypeCUD', SnapshotProposalNotifications = 'SnapshotProposalNotifications', SnapshotListener = 'SnapshotListener', DeadLetter = 'DeadLetter', diff --git a/packages/commonwealth/client/scripts/controllers/server/chain_entities.ts b/packages/commonwealth/client/scripts/controllers/server/chain_entities.ts index c3b802b0fe0..7159917c3af 100644 --- a/packages/commonwealth/client/scripts/controllers/server/chain_entities.ts +++ b/packages/commonwealth/client/scripts/controllers/server/chain_entities.ts @@ -13,10 +13,11 @@ import { import { SubstrateTypes } from 'chain-events/src/types'; import type { ProposalType } from 'common-common/src/types'; import { ChainBase, ChainNetwork } from 'common-common/src/types'; -import { getBaseUrl, getFetch } from 'helpers/getUrl'; +import getFetch from 'helpers/getFetch'; import type { ChainInfo } from 'models'; -import { ChainEntity, ChainEvent, ChainEventType } from 'models'; +import { ChainEntity, ChainEvent } from 'models'; import { proposalSlugToChainEntityType } from '../../identifiers'; +import app from 'state'; export function chainToEventNetwork(c: ChainInfo): SupportedNetwork { if (c.base === ChainBase.Substrate) return SupportedNetwork.Substrate; @@ -95,8 +96,8 @@ class ChainEntityController { // load the chain-entity objects const [entities, entityMetas] = await Promise.all([ - getFetch(getBaseUrl() + '/entities', options), - getFetch(getBaseUrl() + '/getEntityMeta', options), + getFetch(`${app.serverUrl()}/ce/entities`, options), + getFetch(`${app.serverUrl()}/getEntityMeta`, options), ]); const data = []; @@ -127,7 +128,9 @@ class ChainEntityController { } public async getRawEntities(chain: string): Promise { - const entities = await getFetch(getBaseUrl() + '/entities', { chain }); + const entities = await getFetch(`${app.serverUrl()}/ce/entities`, { + chain, + }); const data = []; if (Array.isArray(entities)) { for (const entityJSON of entities) { @@ -171,20 +174,9 @@ class ChainEntityController { // eslint-disable-next-line no-continue if (!eventEntity) continue; const [entityKind] = eventEntity; - // create event type - const eventType = new ChainEventType( - `${chain}-${cwEvent.data.kind.toString()}`, - chain, - network, - cwEvent.data.kind.toString() - ); // create event - const event = new ChainEvent( - cwEvent.blockNumber, - cwEvent.data, - eventType - ); + const event = new ChainEvent(cwEvent.blockNumber, cwEvent.data); // create entity const fieldName = getUniqueEntityKey(network, entityKind); diff --git a/packages/commonwealth/client/scripts/controllers/server/notifications.ts b/packages/commonwealth/client/scripts/controllers/server/notifications.ts index bf70ed35a1c..643117e2497 100644 --- a/packages/commonwealth/client/scripts/controllers/server/notifications.ts +++ b/packages/commonwealth/client/scripts/controllers/server/notifications.ts @@ -1,7 +1,7 @@ /* eslint-disable no-restricted-syntax */ import $ from 'jquery'; import m from 'mithril'; -import { ChainEventType, Notification, NotificationSubscription } from 'models'; +import { Notification, NotificationSubscription } from 'models'; import { modelFromServer } from 'models/NotificationSubscription'; import app from 'state'; @@ -42,6 +42,9 @@ interface NotifOptions { class NotificationsController { private _discussionStore: NotificationStore = new NotificationStore(); private _chainEventStore: NotificationStore = new NotificationStore(); + // these are the chains that chain-events has active listeners for (used to detemine what chains are shown on the + // notification settings page + private _chainEventSubscribedChainIds: string[] = []; private _maxChainEventNotificationId: number = Number.POSITIVE_INFINITY; private _maxDiscussionNotificationId: number = Number.POSITIVE_INFINITY; @@ -49,6 +52,10 @@ class NotificationsController { private _numPages = 0; private _numUnread = 0; + public get chainEventSubscribedChainIds(): string[] { + return this._chainEventSubscribedChainIds; + } + public get numPages(): number { return this._numPages; } @@ -83,7 +90,6 @@ class NotificationsController { if (subscription) { return this.enableSubscriptions([subscription]); } else { - // TODO: Change to POST /subscription return post( '/createSubscription', { @@ -92,7 +98,7 @@ class NotificationsController { is_active: true, }, (result) => { - const newSubscription = NotificationSubscription.fromJSON(result); + const newSubscription = modelFromServer(result); if (newSubscription.category === 'chain-event') app.socket.chainEventsNs.addChainEventSubscriptions([ newSubscription, @@ -335,25 +341,13 @@ class NotificationsController { for (const subscriptionJSON of subscriptions) { const subscription = NotificationSubscription.fromJSON(subscriptionJSON); - // save the chainEventType for the subscription if the subscription type is chain-event - let chainEventType = null; - if (subscriptionJSON.ChainEventType) { - chainEventType = ChainEventType.fromJSON( - subscriptionJSON.ChainEventType - ); - } - // save the notification read + notification instances if any for (const notificationsReadJSON of subscriptionJSON.NotificationsReads) { const data = { is_read: notificationsReadJSON.is_read, ...notificationsReadJSON.Notification, }; - const notification = Notification.fromJSON( - data, - subscription, - chainEventType - ); + const notification = Notification.fromJSON(data, subscription); if (subscription.category === 'chain-event') { if (!this._chainEventStore.getById(notification.id)) @@ -388,11 +382,18 @@ class NotificationsController { }); } + public getSubscribedChains() { + return post('/getSubscribedChains', {}, (result) => { + this._chainEventSubscribedChainIds = result.map((x) => x.id); + }); + } + public async refresh() { return Promise.all([ this.getDiscussionNotifications(), this.getChainEventNotifications(), this.getSubscriptions(), + this.getSubscribedChains(), ]); } } diff --git a/packages/commonwealth/client/scripts/controllers/server/socket/chainEventsNs.ts b/packages/commonwealth/client/scripts/controllers/server/socket/chainEventsNs.ts index f7229cac6d1..691dc587528 100644 --- a/packages/commonwealth/client/scripts/controllers/server/socket/chainEventsNs.ts +++ b/packages/commonwealth/client/scripts/controllers/server/socket/chainEventsNs.ts @@ -25,12 +25,12 @@ export class ChainEventsNamespace { public addChainEventSubscriptions(subs: NotificationSubscription[]) { if (this._isConnected) { - const eventTypes = subs.filter((x) => !!x.chainEventTypeId); const roomsToJoin = []; - for (const eventType of eventTypes) { - if (!this.subscriptionRoomsJoined.has(eventType)) { - roomsToJoin.push(eventType); - this.subscriptionRoomsJoined.add(eventType); + for (const sub of subs) { + const chain = sub?.Chain?.id || sub.Chain; + if (!this.subscriptionRoomsJoined.has(chain)) { + roomsToJoin.push(chain); + this.subscriptionRoomsJoined.add(chain); } } if (roomsToJoin.length > 0) { @@ -44,12 +44,12 @@ export class ChainEventsNamespace { public deleteChainEventSubscriptions(subs: NotificationSubscription[]) { if (this._isConnected) { - const eventTypes = subs.filter((x) => !!x.chainEventTypeId); const roomsToLeave = []; - for (const eventType of eventTypes) { - if (this.subscriptionRoomsJoined.has(eventType)) { - roomsToLeave.push(eventType); - this.subscriptionRoomsJoined.delete(eventType); + for (const sub of subs) { + const chain = sub?.Chain?.id || sub.Chain; + if (this.subscriptionRoomsJoined.has(chain)) { + roomsToLeave.push(chain); + this.subscriptionRoomsJoined.delete(chain); } } @@ -64,8 +64,7 @@ export class ChainEventsNamespace { private onChainEvent(notification: ChainEventNotification) { const subscription = app.user.notifications.subscriptions.find( - (sub) => - sub.chainEventTypeId === notification.ChainEvent.ChainEventType.id + (sub) => sub.getChain === notification.ChainEvent.chain ); if (!subscription) { // will theoretically never happen as subscriptions are added/removed on Socket.io as they happen locally diff --git a/packages/commonwealth/client/scripts/helpers/getFetch.ts b/packages/commonwealth/client/scripts/helpers/getFetch.ts new file mode 100644 index 00000000000..7c8a6ede464 --- /dev/null +++ b/packages/commonwealth/client/scripts/helpers/getFetch.ts @@ -0,0 +1,23 @@ +export default async function getFetch( + url: string, + queryParams?: { [key: string]: any } +) { + let queryUrl; + if (queryParams) queryUrl = url + '?' + new URLSearchParams(queryParams); + try { + const response = await fetch(queryUrl || url, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + }, + }); + if (response.ok) { + const { result } = await response.json(); + return result; + } else { + console.error(`Fetch to ${queryUrl} failed, `, response); + } + } catch (e) { + console.error(e); + } +} diff --git a/packages/commonwealth/client/scripts/helpers/getUrl.ts b/packages/commonwealth/client/scripts/helpers/getUrl.ts deleted file mode 100644 index 39e5b095332..00000000000 --- a/packages/commonwealth/client/scripts/helpers/getUrl.ts +++ /dev/null @@ -1,48 +0,0 @@ -import app from 'state'; - -export enum ServiceUrls { - chainEvents = 'chain-events', -} - -export function getBaseUrl(service?: ServiceUrls) { - if (location.hostname === 'localhost' || location.hostname === '127.0.0.1') { - switch (service) { - case ServiceUrls.chainEvents: - return 'http://localhost:8081/api'; - default: - return 'http://localhost:8080' + app.serverUrl(); - } - } else { - switch (service) { - case ServiceUrls.chainEvents: - return 'https://chain-events.herokuapp.com/api'; - default: - // e.g. https://commonwealth.im/api - return window.location.origin + app.serverUrl(); - } - } -} - -export async function getFetch( - url: string, - queryParams?: { [key: string]: any } -) { - let queryUrl; - if (queryParams) queryUrl = url + '?' + new URLSearchParams(queryParams); - try { - const response = await fetch(queryUrl || url, { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - }, - }); - if (response.ok) { - const { result } = await response.json(); - return result; - } else { - console.error(`Fetch to ${queryUrl} failed, `, response); - } - } catch (e) { - console.error(e); - } -} diff --git a/packages/commonwealth/client/scripts/models/ChainEvent.ts b/packages/commonwealth/client/scripts/models/ChainEvent.ts index 19eb7efe118..9d21e1b942d 100644 --- a/packages/commonwealth/client/scripts/models/ChainEvent.ts +++ b/packages/commonwealth/client/scripts/models/ChainEvent.ts @@ -1,30 +1,32 @@ -import type { IChainEventData } from 'chain-events/src'; import _ from 'underscore'; -import ChainEventType from './ChainEventType'; +import type { IChainEventData, SupportedNetwork } from 'chain-events/src'; class ChainEvent { public readonly id?: number; public readonly blockNumber: number; public readonly data: IChainEventData; - public readonly type: ChainEventType; + public readonly chain: string; + public readonly network: SupportedNetwork; public eq(e: ChainEvent) { return e.data.kind === this.data.kind && _.isEqual(this.data, e.data); } - constructor(blockNumber, data, type, id?) { + constructor(blockNumber, data, id?, chain?, network?) { this.id = id; this.blockNumber = blockNumber; this.data = data; - this.type = type; + this.chain = chain; + this.network = network; } - public static fromJSON(json, chainEventType?: ChainEventType) { + public static fromJSON(json) { return new ChainEvent( json.blockNumber || json.block_number, json.data || json.event_data, - chainEventType || ChainEventType.fromJSON(json.ChainEventType), - json.id + json.id, + json.chain, + json.network ); } } diff --git a/packages/commonwealth/client/scripts/models/ChainEventType.ts b/packages/commonwealth/client/scripts/models/ChainEventType.ts index 21a29bbf5a2..58bb68cc5ca 100644 --- a/packages/commonwealth/client/scripts/models/ChainEventType.ts +++ b/packages/commonwealth/client/scripts/models/ChainEventType.ts @@ -22,5 +22,3 @@ class ChainEventType { ); } } - -export default ChainEventType; diff --git a/packages/commonwealth/client/scripts/models/DashboardActivityNotification.ts b/packages/commonwealth/client/scripts/models/DashboardActivityNotification.ts index 65a272f6535..f697d178838 100644 --- a/packages/commonwealth/client/scripts/models/DashboardActivityNotification.ts +++ b/packages/commonwealth/client/scripts/models/DashboardActivityNotification.ts @@ -1,5 +1,4 @@ import type { IChainEventData, SupportedNetwork } from 'chain-events/src'; -import type { ChainEventType } from 'models'; import moment from 'moment'; class DashboardActivityNotification { @@ -18,7 +17,6 @@ class DashboardActivityNotification { } // Chain Event Notifications - public readonly typeId?: ChainEventType; public readonly blockNumber?: number; public readonly eventData?: IChainEventData; public readonly chainEventId?: number; @@ -36,7 +34,6 @@ class DashboardActivityNotification { likeCount, commentCount, isRead, - typeId, blockNumber, eventData, id, @@ -53,7 +50,6 @@ class DashboardActivityNotification { likeCount?: number; commentCount?: number; isRead?: boolean; - typeId?: ChainEventType; blockNumber?: number; eventData?: IChainEventData; id?: number; @@ -70,7 +66,6 @@ class DashboardActivityNotification { this.likeCount = likeCount; this.commentCount = commentCount; this._isRead = isRead; - this.typeId = typeId; this.blockNumber = blockNumber; this.eventData = eventData; this.chainEventId = id; @@ -90,12 +85,11 @@ class DashboardActivityNotification { likeCount: json.like_count, commentCount: json.comment_count, isRead: json.is_read, - typeId: json.chain_event_type_id, blockNumber: json.block_number, eventData: json.event_data, id: json.id, updatedAt: json.updated_at, - eventNetwork: json.event_network, + eventNetwork: json.network || json.event_network, chain: json.chain, iconUrl: json.icon_url, }); diff --git a/packages/commonwealth/client/scripts/models/Notification.ts b/packages/commonwealth/client/scripts/models/Notification.ts index 38963c5ab25..6d51262f449 100644 --- a/packages/commonwealth/client/scripts/models/Notification.ts +++ b/packages/commonwealth/client/scripts/models/Notification.ts @@ -1,4 +1,3 @@ -import type { ChainEventType } from 'models/index'; import moment from 'moment'; import ChainEvent from './ChainEvent'; import type NotificationSubscription from './NotificationSubscription'; @@ -32,20 +31,14 @@ class Notification { } } - public static fromJSON( - json, - subscription: NotificationSubscription, - chainEventType?: ChainEventType - ) { + public static fromJSON(json, subscription: NotificationSubscription) { return new Notification( json.id, json.notification_data, json.is_read, json.created_at, subscription, - json?.ChainEvent - ? ChainEvent.fromJSON(json.ChainEvent, chainEventType) - : undefined + json?.ChainEvent ? ChainEvent.fromJSON(json.ChainEvent) : undefined ); } } diff --git a/packages/commonwealth/client/scripts/models/NotificationSubscription.ts b/packages/commonwealth/client/scripts/models/NotificationSubscription.ts index 54429e8711e..60f35e21379 100644 --- a/packages/commonwealth/client/scripts/models/NotificationSubscription.ts +++ b/packages/commonwealth/client/scripts/models/NotificationSubscription.ts @@ -15,7 +15,7 @@ class NotificationSubscription { public readonly Thread: ThreadT; public readonly id?: number; - public readonly chainEventTypeId?: any; + public readonly chainEntityId?: any; private _immediateEmail: boolean; public get immediateEmail() { @@ -35,6 +35,11 @@ class NotificationSubscription { return this._isActive; } + // TODO: should resolve Chain vs chain + public get getChain() { + return this.Chain.id || this.Chain; + } + public enable() { this._isActive = true; } @@ -51,7 +56,6 @@ class NotificationSubscription { createdAt, immediateEmail, Chain?, - ChainEventTypeId?, comment?: CommentT, thread?: ThreadT ) { @@ -62,7 +66,6 @@ class NotificationSubscription { this.createdAt = moment(createdAt); this._immediateEmail = immediateEmail; this.Chain = Chain; - this.chainEventTypeId = ChainEventTypeId; this.Comment = comment; this.Thread = thread; } @@ -76,7 +79,6 @@ class NotificationSubscription { json.created_at, json.immediate_email, json.chain_id, - json.chain_event_type_id, json.Comment || json.offchain_comment_id, json.Thread || json.offchain_thread_id ); @@ -92,7 +94,6 @@ export const modelFromServer = (subscription: SubscriptionInstance) => { created_at, immediate_email, Chain, - chain_event_type_id, Comment, Thread, } = subscription; @@ -125,7 +126,6 @@ export const modelFromServer = (subscription: SubscriptionInstance) => { created_at, immediate_email, Chain, - chain_event_type_id, modeledComment, modeledThread ); diff --git a/packages/commonwealth/client/scripts/models/index.ts b/packages/commonwealth/client/scripts/models/index.ts index 6890b48d1c3..0c4665e80d1 100644 --- a/packages/commonwealth/client/scripts/models/index.ts +++ b/packages/commonwealth/client/scripts/models/index.ts @@ -31,7 +31,6 @@ export { default as SearchQuery } from './SearchQuery'; export { default as SearchResult } from './SearchResult'; export { default as SocialAccount } from './SocialAccount'; export { default as StorageModule } from './StorageModule'; -export { default as ChainEventType } from './ChainEventType'; export { default as ChainEvent } from './ChainEvent'; export { default as ChainEntity } from './ChainEntity'; export { default as StarredCommunity } from './StarredCommunity'; diff --git a/packages/commonwealth/client/scripts/views/components/notification_row.ts b/packages/commonwealth/client/scripts/views/components/notification_row.ts index f5bc14b46e6..7177c1ae3ae 100644 --- a/packages/commonwealth/client/scripts/views/components/notification_row.ts +++ b/packages/commonwealth/client/scripts/views/components/notification_row.ts @@ -326,12 +326,12 @@ const NotificationRow: m.Component< if (!notification.chainEvent) { throw new Error('chain event notification does not have expected data'); } - const chainId = notification.chainEvent.type.chain; + const chainId = notification.chainEvent.chain; // construct compatible CW event from DB by inserting network from type const chainEvent: CWEvent = { blockNumber: notification.chainEvent.blockNumber, - network: notification.chainEvent.type.eventNetwork, + network: notification.chainEvent.network, data: notification.chainEvent.data, }; const chainName = app.config.chains.getById(chainId)?.name; diff --git a/packages/commonwealth/client/scripts/views/pages/notification_settings/index.tsx b/packages/commonwealth/client/scripts/views/pages/notification_settings/index.tsx index 5bfc589a539..0e4a64a101f 100644 --- a/packages/commonwealth/client/scripts/views/pages/notification_settings/index.tsx +++ b/packages/commonwealth/client/scripts/views/pages/notification_settings/index.tsx @@ -28,6 +28,7 @@ import { CWPopoverMenu } from '../../components/component_kit/cw_popover/cw_popo import { CWTextInput } from '../../components/component_kit/cw_text_input'; import { CWCard } from '../../components/component_kit/cw_card'; import { CWSpinner } from '../../components/component_kit/cw_spinner'; +import { NotificationCategories } from '../../../../../../common-common/src/types'; const emailIntervalFrequencyMap = { never: 'Never', @@ -53,7 +54,29 @@ class NotificationSettingsPage extends ClassComponent { return ; } - const bundledSubs = bundleSubs(app.user.notifications.subscriptions); + // bundled discussion subscriptions + const bundledSubs = bundleSubs( + app.user.notifications.subscriptions.filter( + (x) => x.category !== 'chain-event' + ) + ); + // bundled chain-event subscriptions + const chainEventSubs = bundleSubs( + app.user.notifications.subscriptions.filter( + (x) => x.category === 'chain-event' + ) + ); + + const subscribedChainIds = + app.user.notifications.chainEventSubscribedChainIds; + + // chains/communities the user has addresses for but does not have existing subscriptions for + const relevantSubscribedChains = app.user.addresses + .map((x) => x.chain) + .filter( + (x) => subscribedChainIds.includes(x.id) && !chainEventSubs[x.id] + ); + const currentFrequency = app.user.emailInterval; return ( @@ -68,6 +91,133 @@ class NotificationSettingsPage extends ClassComponent { Notification settings for all new threads, comments, mentions, likes, and chain events in the following communities. + + Chain Events + +
+ + Community + + + Email + + + In-App + +
+ {relevantSubscribedChains.map((chain) => { + return ( +
+
+
+
+ + + {chain.name} + +
+
+ { + app.user.notifications + .enableImmediateEmails([]) + .then(() => { + m.redraw(); + }); + }} + /> + { + app.user.notifications + .subscribe(NotificationCategories.ChainEvent, chain.id) + .then(() => { + m.redraw(); + }); + }} + /> +
+
+ ); + })} + {Object.entries(chainEventSubs).map(([chainName, subs]) => { + const chainInfo = app.config.chains.getById(chainName); + const hasSomeEmailSubs = subs.some((s) => s.immediateEmail); + const hasSomeInAppSubs = subs.some((s) => s.isActive); + return ( +
+
+
+
+ + + {chainInfo?.name} + +
+
+ { + hasSomeEmailSubs + ? app.user.notifications + .disableImmediateEmails(subs) + .then(() => { + m.redraw(); + }) + : app.user.notifications + .enableImmediateEmails(subs) + .then(() => { + m.redraw(); + }); + }} + /> + s.isActive)} + onchange={() => { + hasSomeInAppSubs + ? app.user.notifications + .disableSubscriptions(subs) + .then(() => { + m.redraw(); + }) + : app.user.notifications + .enableSubscriptions(subs) + .then(() => { + m.redraw(); + }); + }} + /> +
+
+ ); + })} + + Discussion +