Revert "Remove Chain Event Types #3 (#3044)" (#3163)

* Revert "Remove Chain Event Types #3 (#3044)"

This reverts commit 68ba0c5.

* revert migration

* add back the migrations that did run successfully

* txn

* fix

---------

Co-authored-by: Timothee Legros <timothee@ualberta.ca>

.prettierignore

@@ -1,5 +1,4 @@
 **/build
-**/dist
 **/.github
 **/*.env
 **/node_modules
@@ -17,7 +16,6 @@ packages/chain-events/.eslintrc.js
 **/contractTypes
 **/factories
 **/eth/types
-**/eth/artifacts
 **/.eslintrc.js
 **/ios/App/App
 

packages/chain-events/Procfile

@@ -1,6 +1,6 @@
-web: node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/app/Server.js
-subscriber1: CHAIN_SUBSCRIBER_INDEX=0 node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script
-subscriber2: CHAIN_SUBSCRIBER_INDEX=1 node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script
-subscriber3: CHAIN_SUBSCRIBER_INDEX=2 node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script
-consumer: node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainEventsConsumer/chainEventsConsumer.js run-as-script
+web: node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/app/Server.js
+subscriber1: CHAIN_SUBSCRIBER_INDEX=0 node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script
+subscriber2: CHAIN_SUBSCRIBER_INDEX=1 node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script
+subscriber3: CHAIN_SUBSCRIBER_INDEX=2 node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script
+consumer: node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainEventsConsumer/chainEventsConsumer.js run-as-script
 release: cd packages/chain-events && npx sequelize-cli db:migrate --config services/database/sequelize.json

packages/chain-events/package.json

@@ -29,9 +29,9 @@
     "batch-poll": "ts-node -T ./scripts/batchPoller.ts",
     "preyalcpublish": "yarn build",
     "emit-events": "ts-node --project tsconfig.json ./scripts/emitChainEvents.ts",
-    "start-app": "ts-node-dev --max-old-space-size=4096 --respawn --transpile-only --project tsconfig.services.json services/app/Server.ts",
-    "start-consumer": "ts-node-dev --max-old-space-size=4096 --respawn --transpile-only --project tsconfig.services.json services/ChainEventsConsumer/chainEventsConsumer.ts run-as-script",
-    "start-subscriber": "ts-node-dev --max-old-space-size=4096 --respawn --transpile-only --project tsconfig.services.json services/ChainSubscriber/chainSubscriber.ts run-as-script",
+    "start-app": "ts-node --project tsconfig.services.json services/app/Server.ts",
+    "start-consumer": "ts-node --project tsconfig.services.json services/ChainEventsConsumer/chainEventsConsumer.ts run-as-script",
+    "start-subscriber": "ts-node --project tsconfig.services.json services/ChainSubscriber/chainSubscriber.ts run-as-script",
     "start-all": "concurrently -p '{name}' -c yellow,blue,magenta -n app,consumer,subscriber 'yarn start-app' 'yarn start-consumer' 'yarn start-subscriber'",
     "create-db": "npx sequelize db:create",
     "migrate-db": "npx sequelize db:migrate",
@@ -80,7 +80,6 @@
     "pg-format": "^1.0.4",
     "rollbar": "^2.25.2",
     "sleep-promise": "^8.0.1",
-    "ts-node-dev": "^2.0.0",
     "typescript-logging": "^0.6.4",
     "underscore": "^1.10.2",
     "web3": "^1.3.1",

packages/chain-events/scripts/migrateChainEntities.ts

@@ -28,9 +28,6 @@ import type { BrokerConfig } from 'rascal';
 import { RABBITMQ_URI } from '../../commonwealth/server/config';
 import { constructSubstrateUrl } from '../../commonwealth/shared/substrate';
 import { CHAIN_EVENT_SERVICE_SECRET, CW_SERVER_URL } from '../services/config';
-import NotificationsHandler from '../services/ChainEventsConsumer/ChainEventHandlers/notification';
-import models from '../services/database/database';
-import { EventKind } from '../src/chains/substrate/types';
 
 const log = factory.getLogger(formatFilename(__filename));
 
@@ -98,14 +95,6 @@ async function migrateChainEntity(
       rmqController,
       chain
     );
-
-    const excludedNotificationEvents = [EventKind.DemocracyTabled];
-    const notificationsHandler = new NotificationsHandler(
-      models,
-      rmqController,
-      excludedNotificationEvents
-    );
-
     let fetcher: IStorageFetcher<any>;
     const range: IDisconnectedRange = { startBlock: 0 };
     if (chainInstance.base === ChainBase.Substrate) {
@@ -164,8 +153,7 @@ async function migrateChainEntity(
       try {
         // eslint-disable-next-line no-await-in-loop
         const dbEvent = await migrationHandler.handle(event);
-        const ceEvent = await entityArchivalHandler.handle(event, dbEvent);
-        await notificationsHandler.handle(event, ceEvent);
+        await entityArchivalHandler.handle(event, dbEvent);
       } catch (e) {
         log.error(`Event handle failure: ${e.message}`);
       }

packages/chain-events/scripts/testSequelizeQuery.ts

@@ -0,0 +1,21 @@
+import models from '../services/database/database';
+
+async function testSequelizeQuery() {
+  const eventTypes = (
+    await models.ChainEventType.findAll({
+      attributes: ['id'],
+      where: { chain: 'edgeware' },
+    })
+  ).map((x) => x.id);
+
+  const dbResult = await models.ChainEvent.max('block_number', {
+    where: {
+      chain_event_type_id: eventTypes,
+    },
+  });
+
+  console.log('Database Result:', dbResult);
+  process.exit(0);
+}
+
+testSequelizeQuery();

packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/entityArchival.ts

@@ -9,7 +9,6 @@ import { RascalPublications } from 'common-common/src/rabbitmq/types';
 
 import type { DB } from '../../database/database';
 
-import type { ChainEventInstance } from 'chain-events/services/database/models/chain_event';
 import type {
   CWEvent,
   IChainEntityKind,
@@ -41,10 +40,7 @@ export default class extends IEventHandler {
    * `dbEvent` is the database entry corresponding to the `event`.
    */
   // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
-  public async handle(
-    event: CWEvent<IChainEventData>,
-    dbEvent: ChainEventInstance
-  ) {
+  public async handle(event: CWEvent<IChainEventData>, dbEvent) {
     // eslint-disable-next-line @typescript-eslint/no-shadow
     const log = factory.getLogger(
       addPrefix(__filename, [event.network, event.chain])
@@ -177,7 +173,5 @@ export default class extends IEventHandler {
         break;
       }
     }
-
-    return dbEvent;
   }
 }

packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/migration.ts

@@ -2,22 +2,26 @@
  * Processes events during migration, upgrading from simple notifications to entities.
  */
 import type { WhereOptions } from 'sequelize';
-import type { RabbitMQController } from 'common-common/src/rabbitmq';
+import type {
+  RabbitMQController,
+  RmqCENotificationCUD,
+  RmqCETypeCUD,
+} from 'common-common/src/rabbitmq';
+import { RascalPublications } from 'common-common/src/rabbitmq';
 import { factory, formatFilename } from 'common-common/src/logging';
 
 import type { CWEvent } from '../../../src';
 import {
-  EntityEventKind,
+  IEventHandler,
   eventToEntity,
   getUniqueEntityKey,
-  IEventHandler,
+  EntityEventKind,
 } from '../../../src';
 import type { DB } from '../../database/database';
 import type {
   ChainEventAttributes,
   ChainEventInstance,
 } from '../../database/models/chain_event';
-
 const log = factory.getLogger(formatFilename(__filename));
 
 export default class extends IEventHandler<ChainEventInstance> {
@@ -43,18 +47,48 @@ export default class extends IEventHandler<ChainEventInstance> {
       fieldValue: string,
       eventType: EntityEventKind
     ) => {
+      const [dbEventType, created] =
+        await this._models.ChainEventType.findOrCreate({
+          where: {
+            id: `${chain}-${event.data.kind.toString()}`,
+            chain,
+            event_network: event.network,
+            event_name: event.data.kind.toString(),
+          },
+        });
+      log.trace(
+        `${created ? 'created' : 'found'} chain event type: ${dbEventType.id}`
+      );
+
+      if (created) {
+        const publishData: RmqCETypeCUD.RmqMsgType = {
+          chainEventTypeId: dbEventType.id,
+          cud: 'create',
+        };
+
+        await this._rmqController.safePublish(
+          publishData,
+          dbEventType.id,
+          RascalPublications.ChainEventTypeCUDMain,
+          {
+            sequelize: this._models.sequelize,
+            model: this._models.ChainEventType,
+          }
+        );
+      }
+
       const queryFieldName = `event_data.${fieldName}`;
       const queryArgs: WhereOptions<ChainEventAttributes> =
         eventType === EntityEventKind.Vote
           ? {
+              chain_event_type_id: dbEventType.id,
               [queryFieldName]: fieldValue,
               // votes will be unique by data rather than by type
               event_data: event.data as any,
-              chain,
             }
           : {
+              chain_event_type_id: dbEventType.id,
               [queryFieldName]: fieldValue,
-              chain,
             };
       const existingEvent = await this._models.ChainEvent.findOne({
         where: queryArgs,
@@ -67,12 +101,31 @@ export default class extends IEventHandler<ChainEventInstance> {
       }
 
       log.info('No existing event found, creating new event in db!');
-      return await this._models.ChainEvent.create({
+      const dbEvent = await this._models.ChainEvent.create({
+        chain_event_type_id: dbEventType.id,
         block_number: event.blockNumber,
         event_data: event.data,
-        network: event.network,
-        chain,
       });
+
+      const formattedEvent: ChainEventAttributes = dbEvent.toJSON();
+      formattedEvent.ChainEventType = dbEventType.toJSON();
+
+      const publishData: RmqCENotificationCUD.RmqMsgType = {
+        ChainEvent: formattedEvent,
+        event,
+        cud: 'create',
+      };
+
+      await this._rmqController.safePublish(
+        publishData,
+        dbEvent.id,
+        RascalPublications.ChainEventNotificationsCUDMain,
+        {
+          sequelize: this._models.sequelize,
+          model: this._models.ChainEvent,
+        }
+      );
+      return dbEvent;
     };
 
     const entity = eventToEntity(event.network, event.data.kind);

packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/notification.ts

@@ -3,15 +3,11 @@ import type { RmqCENotificationCUD } from 'common-common/src/rabbitmq/types';
 import { RascalPublications } from 'common-common/src/rabbitmq/types';
 
 import { addPrefix, factory } from '../../../src/logging';
-import type { ChainEventInstance } from '../../database/models/chain_event';
+import type { ChainEventAttributes } from '../../database/models/chain_event';
 import type { DB } from '../../database/database';
 
 import type { CWEvent, IChainEventKind } from 'chain-events/src';
-import {
-  EntityEventKind,
-  eventToEntity,
-  IEventHandler,
-} from 'chain-events/src';
+import { IEventHandler } from 'chain-events/src';
 
 export default class extends IEventHandler {
   public readonly name = 'Notification Producer';
@@ -25,7 +21,7 @@ export default class extends IEventHandler {
   }
 
   // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
-  public async handle(event: CWEvent, dbEvent: ChainEventInstance) {
+  public async handle(event: CWEvent, dbEvent) {
     const log = factory.getLogger(
       addPrefix(__filename, [event.network, event.chain])
     );
@@ -40,24 +36,25 @@ export default class extends IEventHandler {
       return dbEvent;
     }
 
-    if (!dbEvent.entity_id) {
-      log.info(`No related entity, skipping!`);
-      return dbEvent;
-    }
-
-    const [, eventEntityKind] = eventToEntity(event.network, event.data.kind);
-    if (
-      eventEntityKind != EntityEventKind.Create &&
-      eventEntityKind != EntityEventKind.Complete
-    ) {
-      log.trace(
-        `Event does not mark the creation or completion of an entity. Skipping event!`
+    let dbEventType;
+    try {
+      dbEventType = await dbEvent.getChainEventType();
+      if (!dbEventType) {
+        log.error(`Failed to fetch event type! Ignoring.`);
+        return;
+      }
+    } catch (e) {
+      log.error(
+        `Failed to get chain-event type for event: ${JSON.stringify(event)}`
       );
       return dbEvent;
     }
 
+    const formattedEvent: ChainEventAttributes = dbEvent.toJSON();
+    formattedEvent.ChainEventType = dbEventType.toJSON();
+
     const publishData: RmqCENotificationCUD.RmqMsgType = {
-      ChainEvent: dbEvent.toJSON(),
+      ChainEvent: formattedEvent,
       event,
       cud: 'create',
     };