Chain Event Types Removal Finale (#3309)

* from event-types client side * move chain + network to event -> working chain activity dashboard * switch ce commands to ts-node-dev * chain + network added to event in ChainEventConsumer * save * filter out non entity creation/completion events for notifications * remove EventTypeCUD RabbitMQ types + config + pub/sub calls * remove ChainEventType model * remove chain-event-type migration * CET model removal fixes * CW remove CET migration * CW remove CET model * CET removal migrations + model update * migration fixes * migration fixes * chain subscriptions + chain-event notification settings + subscribedChains * add full chain to createSubscription route result + fix UI * turn off sourceMap, use proper tsconfig when building ce, and fixed scraper imports * add type check to CI + fix type errors * add type check to CI + fix type errors + remove format in CE repo * set allowJs: false in tsconfig so sourceMap files don't interrupt ts-node * RabbitMQ merge fixes * Make CI lint only the changes in the branch rather than everything * fix tsconfig + run prettier * merge fixes * merge fixes * discoverReconnectRange fix * consumer erc20 notification handler bug fix * load-env-var fix * Procfile fix * master merge fixes * fix chain-event notifications * fix chain-event email notifications * format * format * CE css fix * remove subscriptions count for chain-events * fix getSubscribedChains call * notification settings page fixes + remove unnecessary logs * format * delete script * migration script fix * prettier format * publishCustomRabbitMQMessage.ts improvements * prettier format * more script improvements for testing/QA * add dump file to gitignore * 8 min local * 15 seconds local * add fkey from NR to Sub * format * eslint + format + .prettierignore update * Ensure CET Removal Backwards Compatibility. (#2765) * Use CE proxy for events too. * Ensure backwards compatibility for deployment + fix link to chain. * Prettier fixes. * Remove log. * fixed type errors * fix lint * fix lint * format * small logging fix * migration log improvement * ce procfile fix * add missing index to notif table * Subscriptions pkey + not null * fix type * test improved CE queries * Procfile fix * heroku test * heroku test 2 * cleaup * redo migration * redo migration CE + network migration fix * fix order of CE migrations * logging for debugging 503 * error handling CE - 503 debugging * event route - 503 debugging * optimize event route db query * prettier --------- Co-authored-by: Jake Naviasky <jake@commonwealth.im> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Muon Shot <120686579+CowMuon@users.noreply.github.com>
hicommonwealth · Apr 2, 2023 · 6632d14 · 6632d14
1 parent e4dc43c
commit 6632d14
Show file tree

Hide file tree

Showing 80 changed files with 1,141 additions and 1,110 deletions.
diff --git a/.prettierignore b/.prettierignore
@@ -1,4 +1,5 @@
 **/build
+**/dist
 **/.github
 **/*.env
 **/node_modules
@@ -16,6 +17,7 @@ packages/chain-events/.eslintrc.js
 **/contractTypes
 **/factories
 **/eth/types
+**/eth/artifacts
 **/.eslintrc.js
 **/ios/App/App
 
diff --git a/packages/chain-events/Procfile b/packages/chain-events/Procfile
@@ -1,6 +1,6 @@
-web: node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/app/Server.js
-subscriber1: CHAIN_SUBSCRIBER_INDEX=0 node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script
-subscriber2: CHAIN_SUBSCRIBER_INDEX=1 node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script
-subscriber3: CHAIN_SUBSCRIBER_INDEX=2 node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script
-consumer: node --max_old_space_size=$(../../../../scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainEventsConsumer/chainEventsConsumer.js run-as-script
+web: node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/app/Server.js
+subscriber1: CHAIN_SUBSCRIBER_INDEX=0 node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script
+subscriber2: CHAIN_SUBSCRIBER_INDEX=1 node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script
+subscriber3: CHAIN_SUBSCRIBER_INDEX=2 node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainSubscriber/chainSubscriber.js run-as-script
+consumer: node --max_old_space_size=$(./scripts/get-max-old-space-size.sh) packages/chain-events/build/chain-events/services/ChainEventsConsumer/chainEventsConsumer.js run-as-script
 release: cd packages/chain-events && npx sequelize-cli db:migrate --config services/database/sequelize.json
diff --git a/packages/chain-events/package.json b/packages/chain-events/package.json
@@ -29,9 +29,9 @@
     "batch-poll": "ts-node -T ./scripts/batchPoller.ts",
     "preyalcpublish": "yarn build",
     "emit-events": "ts-node --project tsconfig.json ./scripts/emitChainEvents.ts",
-    "start-app": "ts-node --project tsconfig.services.json services/app/Server.ts",
-    "start-consumer": "ts-node --project tsconfig.services.json services/ChainEventsConsumer/chainEventsConsumer.ts run-as-script",
-    "start-subscriber": "ts-node --project tsconfig.services.json services/ChainSubscriber/chainSubscriber.ts run-as-script",
+    "start-app": "ts-node-dev --max-old-space-size=4096 --respawn --transpile-only --project tsconfig.services.json services/app/Server.ts",
+    "start-consumer": "ts-node-dev --max-old-space-size=4096 --respawn --transpile-only --project tsconfig.services.json services/ChainEventsConsumer/chainEventsConsumer.ts run-as-script",
+    "start-subscriber": "ts-node-dev --max-old-space-size=4096 --respawn --transpile-only --project tsconfig.services.json services/ChainSubscriber/chainSubscriber.ts run-as-script",
     "start-all": "concurrently -p '{name}' -c yellow,blue,magenta -n app,consumer,subscriber 'yarn start-app' 'yarn start-consumer' 'yarn start-subscriber'",
     "create-db": "npx sequelize db:create",
     "migrate-db": "npx sequelize db:migrate",
@@ -80,6 +80,7 @@
     "pg-format": "^1.0.4",
     "rollbar": "^2.25.2",
     "sleep-promise": "^8.0.1",
+    "ts-node-dev": "^2.0.0",
     "typescript-logging": "^0.6.4",
     "underscore": "^1.10.2",
     "web3": "^1.3.1",

diff --git a/packages/chain-events/scripts/migrateChainEntities.ts b/packages/chain-events/scripts/migrateChainEntities.ts
@@ -28,6 +28,9 @@ import type { BrokerConfig } from 'rascal';
 import { RABBITMQ_URI } from '../../commonwealth/server/config';
 import { constructSubstrateUrl } from '../../commonwealth/shared/substrate';
 import { CHAIN_EVENT_SERVICE_SECRET, CW_SERVER_URL } from '../services/config';
+import NotificationsHandler from '../services/ChainEventsConsumer/ChainEventHandlers/notification';
+import models from '../services/database/database';
+import { EventKind } from '../src/chains/substrate/types';
 
 const log = factory.getLogger(formatFilename(__filename));
 
@@ -95,6 +98,14 @@ async function migrateChainEntity(
       rmqController,
       chain
     );
+
+    const excludedNotificationEvents = [EventKind.DemocracyTabled];
+    const notificationsHandler = new NotificationsHandler(
+      models,
+      rmqController,
+      excludedNotificationEvents
+    );
+
     let fetcher: IStorageFetcher<any>;
     const range: IDisconnectedRange = { startBlock: 0 };
     if (chainInstance.base === ChainBase.Substrate) {
@@ -153,7 +164,8 @@ async function migrateChainEntity(
       try {
         // eslint-disable-next-line no-await-in-loop
         const dbEvent = await migrationHandler.handle(event);
-        await entityArchivalHandler.handle(event, dbEvent);
+        const ceEvent = await entityArchivalHandler.handle(event, dbEvent);
+        await notificationsHandler.handle(event, ceEvent);
       } catch (e) {
         log.error(`Event handle failure: ${e.message}`);
       }

diff --git a/packages/chain-events/scripts/publishCustomRabbitMQMessage.ts b/packages/chain-events/scripts/publishCustomRabbitMQMessage.ts
@@ -0,0 +1,83 @@
+import type { IEventData } from '../src/chains/aave/types';
+import type { CWEvent } from '../src';
+import { SupportedNetwork } from '../src';
+import { publishRmqMsg } from 'common-common/src/rabbitmq/util';
+import { RABBITMQ_API_URI } from '../services/config';
+import { RascalExchanges, RascalRoutingKeys } from 'common-common/src/rabbitmq';
+import models from 'chain-events/services/database/database';
+
+async function main() {
+  const ceData = {
+    id: 10,
+    kind: 'proposal-created',
+    values: ['0'],
+    targets: ['0xE710CEd57456D3A16152c32835B5FB4E72D9eA5b'],
+    endBlock: 16203604,
+    executor: '0x64c7d40c07EFAbec2AafdC243bF59eaF2195c6dc',
+    ipfsHash:
+      '0x3876d28a014bc20432dcc3549ba95710446b98431d84c7f84fde6abe1baf527f',
+    proposer: '0xb55a948763e0d386b6dEfcD8070a522216AE42b1',
+    strategy: '0x90Dfd35F4a0BB2d30CDf66508085e33C353475D9',
+    calldatas: [
+      '0x00000000000000000000000092d6c1e31e14520e676a687f0a93788b716beff5000000000000000000000000a8541f948411b3f95d9e89e8d339a56a9ed3d00b000000000000000000000000000000000000000000002fa54641bae8aaa00000',
+    ],
+    signatures: ['transfer(address,address,uint256)'],
+    startBlock: 16177324,
+  };
+  const chainEvent: CWEvent<IEventData> = {
+    blockNumber: 16170754,
+    data: <any>ceData,
+    network: SupportedNetwork.Aave,
+    chain: 'dydx',
+  };
+
+  const publishJson = await publishRmqMsg(
+    RABBITMQ_API_URI,
+    RascalExchanges.ChainEvents,
+    RascalRoutingKeys.ChainEvents,
+    chainEvent
+  );
+
+  console.log(publishJson);
+}
+
+async function clear() {
+  try {
+    let eventsDeleted = 0,
+      entitiesDeleted = 0;
+    await models.sequelize.transaction(async (t) => {
+      const entityId = (
+        await models.ChainEntity.findOne({
+          where: { chain: 'dydx', type_id: '10' },
+          transaction: t,
+        })
+      )?.id;
+
+      if (entityId) {
+        eventsDeleted = await models.ChainEvent.destroy({
+          where: { entity_id: entityId },
+          transaction: t,
+        });
+
+        entitiesDeleted = await models.ChainEntity.destroy({
+          where: { id: entityId },
+          transaction: t,
+        });
+      } else {
+        console.log('Entity does not exist.');
+      }
+    });
+
+    console.log(
+      `Events deleted: ${eventsDeleted}\nEntities deleted: ${entitiesDeleted}`
+    );
+  } catch (e) {
+    console.log('Failed to clear - reverted.');
+    console.error(e);
+  }
+
+  process.exit(1);
+}
+
+if (process.argv[2] === 'clear') clear();
+else main();
diff --git a/packages/chain-events/scripts/testSequelizeQuery.ts b/packages/chain-events/scripts/testSequelizeQuery.ts
diff --git a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/entityArchival.ts b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/entityArchival.ts
@@ -9,6 +9,7 @@ import { RascalPublications } from 'common-common/src/rabbitmq/types';
 
 import type { DB } from '../../database/database';
 
+import type { ChainEventInstance } from 'chain-events/services/database/models/chain_event';
 import type {
   CWEvent,
   IChainEntityKind,
@@ -40,7 +41,10 @@ export default class extends IEventHandler {
    * `dbEvent` is the database entry corresponding to the `event`.
    */
   // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
-  public async handle(event: CWEvent<IChainEventData>, dbEvent) {
+  public async handle(
+    event: CWEvent<IChainEventData>,
+    dbEvent: ChainEventInstance
+  ) {
     // eslint-disable-next-line @typescript-eslint/no-shadow
     const log = factory.getLogger(
       addPrefix(__filename, [event.network, event.chain])
@@ -173,5 +177,7 @@ export default class extends IEventHandler {
         break;
       }
     }
+
+    return dbEvent;
   }
 }
diff --git a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/migration.ts b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/migration.ts
@@ -2,26 +2,22 @@
  * Processes events during migration, upgrading from simple notifications to entities.
  */
 import type { WhereOptions } from 'sequelize';
-import type {
-  RabbitMQController,
-  RmqCENotificationCUD,
-  RmqCETypeCUD,
-} from 'common-common/src/rabbitmq';
-import { RascalPublications } from 'common-common/src/rabbitmq';
+import type { RabbitMQController } from 'common-common/src/rabbitmq';
 import { factory, formatFilename } from 'common-common/src/logging';
 
 import type { CWEvent } from '../../../src';
 import {
-  IEventHandler,
+  EntityEventKind,
   eventToEntity,
   getUniqueEntityKey,
-  EntityEventKind,
+  IEventHandler,
 } from '../../../src';
 import type { DB } from '../../database/database';
 import type {
   ChainEventAttributes,
   ChainEventInstance,
 } from '../../database/models/chain_event';
+
 const log = factory.getLogger(formatFilename(__filename));
 
 export default class extends IEventHandler<ChainEventInstance> {
@@ -47,48 +43,18 @@ export default class extends IEventHandler<ChainEventInstance> {
       fieldValue: string,
       eventType: EntityEventKind
     ) => {
-      const [dbEventType, created] =
-        await this._models.ChainEventType.findOrCreate({
-          where: {
-            id: `${chain}-${event.data.kind.toString()}`,
-            chain,
-            event_network: event.network,
-            event_name: event.data.kind.toString(),
-          },
-        });
-      log.trace(
-        `${created ? 'created' : 'found'} chain event type: ${dbEventType.id}`
-      );
-
-      if (created) {
-        const publishData: RmqCETypeCUD.RmqMsgType = {
-          chainEventTypeId: dbEventType.id,
-          cud: 'create',
-        };
-
-        await this._rmqController.safePublish(
-          publishData,
-          dbEventType.id,
-          RascalPublications.ChainEventTypeCUDMain,
-          {
-            sequelize: this._models.sequelize,
-            model: this._models.ChainEventType,
-          }
-        );
-      }
-
       const queryFieldName = `event_data.${fieldName}`;
       const queryArgs: WhereOptions<ChainEventAttributes> =
         eventType === EntityEventKind.Vote
           ? {
-              chain_event_type_id: dbEventType.id,
               [queryFieldName]: fieldValue,
               // votes will be unique by data rather than by type
               event_data: event.data as any,
+              chain,
             }
           : {
-              chain_event_type_id: dbEventType.id,
               [queryFieldName]: fieldValue,
+              chain,
             };
       const existingEvent = await this._models.ChainEvent.findOne({
         where: queryArgs,
@@ -101,31 +67,12 @@ export default class extends IEventHandler<ChainEventInstance> {
       }
 
       log.info('No existing event found, creating new event in db!');
-      const dbEvent = await this._models.ChainEvent.create({
-        chain_event_type_id: dbEventType.id,
+      return await this._models.ChainEvent.create({
         block_number: event.blockNumber,
         event_data: event.data,
+        network: event.network,
+        chain,
       });
-
-      const formattedEvent: ChainEventAttributes = dbEvent.toJSON();
-      formattedEvent.ChainEventType = dbEventType.toJSON();
-
-      const publishData: RmqCENotificationCUD.RmqMsgType = {
-        ChainEvent: formattedEvent,
-        event,
-        cud: 'create',
-      };
-
-      await this._rmqController.safePublish(
-        publishData,
-        dbEvent.id,
-        RascalPublications.ChainEventNotificationsCUDMain,
-        {
-          sequelize: this._models.sequelize,
-          model: this._models.ChainEvent,
-        }
-      );
-      return dbEvent;
     };
 
     const entity = eventToEntity(event.network, event.data.kind);

diff --git a/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/notification.ts b/packages/chain-events/services/ChainEventsConsumer/ChainEventHandlers/notification.ts
@@ -3,11 +3,15 @@ import type { RmqCENotificationCUD } from 'common-common/src/rabbitmq/types';
 import { RascalPublications } from 'common-common/src/rabbitmq/types';
 
 import { addPrefix, factory } from '../../../src/logging';
-import type { ChainEventAttributes } from '../../database/models/chain_event';
+import type { ChainEventInstance } from '../../database/models/chain_event';
 import type { DB } from '../../database/database';
 
 import type { CWEvent, IChainEventKind } from 'chain-events/src';
-import { IEventHandler } from 'chain-events/src';
+import {
+  EntityEventKind,
+  eventToEntity,
+  IEventHandler,
+} from 'chain-events/src';
 
 export default class extends IEventHandler {
   public readonly name = 'Notification Producer';
@@ -21,7 +25,7 @@ export default class extends IEventHandler {
   }
 
   // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
-  public async handle(event: CWEvent, dbEvent) {
+  public async handle(event: CWEvent, dbEvent: ChainEventInstance) {
     const log = factory.getLogger(
       addPrefix(__filename, [event.network, event.chain])
     );
@@ -36,25 +40,24 @@ export default class extends IEventHandler {
       return dbEvent;
     }
 
-    let dbEventType;
-    try {
-      dbEventType = await dbEvent.getChainEventType();
-      if (!dbEventType) {
-        log.error(`Failed to fetch event type! Ignoring.`);
-        return;
-      }
-    } catch (e) {
-      log.error(
-        `Failed to get chain-event type for event: ${JSON.stringify(event)}`
-      );
+    if (!dbEvent.entity_id) {
+      log.info(`No related entity, skipping!`);
       return dbEvent;
     }
 
-    const formattedEvent: ChainEventAttributes = dbEvent.toJSON();
-    formattedEvent.ChainEventType = dbEventType.toJSON();
+    const [, eventEntityKind] = eventToEntity(event.network, event.data.kind);
+    if (
+      eventEntityKind != EntityEventKind.Create &&
+      eventEntityKind != EntityEventKind.Complete
+    ) {
+      log.trace(
+        `Event does not mark the creation or completion of an entity. Skipping event!`
+      );
+      return dbEvent;
+    }
 
     const publishData: RmqCENotificationCUD.RmqMsgType = {
-      ChainEvent: formattedEvent,
+      ChainEvent: dbEvent.toJSON(),
       event,
       cud: 'create',
     };