Merge pull request #33 from NethermindEth/feature/var-charts-fixes

VaR charts fixes
NethermindEth · Apr 30, 2024 · d807bdf · d807bdf
2 parents ab47ca3 + 3f2acae
commit d807bdf
Show file tree

Hide file tree

Showing 26 changed files with 895 additions and 305 deletions.
diff --git a/app/backend/.env.dev b/app/backend/.env.dev
@@ -42,3 +42,7 @@ STARKNET_MODULE_ENABLED=true
 STARKNET_START_BLOCK=615000
 STARKNET_MAX_BLOCK_RANGE=50
 STARKNET_POLL_INTERVAL_MS=60000
+
+VAR_MODULE_ENABLED=true
+VAR_POLL_INTERVAL_MS=60000
+VAR_BACKFILL_PERIOD_DAYS=7
diff --git a/app/backend/.env.local b/app/backend/.env.local
@@ -41,3 +41,7 @@ STARKNET_MODULE_ENABLED=true
 STARKNET_START_BLOCK=610000
 STARKNET_MAX_BLOCK_RANGE=50
 STARKNET_POLL_INTERVAL_MS=60000
+
+VAR_MODULE_ENABLED=true
+VAR_POLL_INTERVAL_MS=60000
+VAR_BACKFILL_PERIOD_DAYS=7
diff --git a/app/backend/package.json b/app/backend/package.json
@@ -3,7 +3,7 @@
   "version": "0.0.1",
   "main": "index.ts",
   "scripts": {
-    "start": "node src/index.ts",
+    "start": "ts-node src/index.ts",
     "dev": "nodemon src/index.ts",
     "build": "tsc && tsc-alias",
     "test": "mocha --timeout 10000",

diff --git a/app/backend/src/Application.ts b/app/backend/src/Application.ts
@@ -14,6 +14,7 @@ import { createPriceUpdaterModule } from "@/core/modules/pricing/PriceUpdaterMod
 import { createProxyModule } from "@/core/modules/proxy/ProxyModule";
 import { createStarknetBlockModule } from "@/core/modules/indexers/l2/starknet/StarknetBlockModule";
 import StarknetClient from "@/core/clients/blockchain/starknet/StarknetClient";
+import { createVarUpdaterModule } from "@/core/modules/var/VarUpdaterModule";
 
 export class Application {
   constructor(config: Config) {
@@ -67,6 +68,7 @@ export class Application {
           polygonZkEvmClient,
         ),
         createStarknetBlockModule(config, logger, database, starknetClient),
+        createVarUpdaterModule(config, logger, database),
       ];
 
       for (const module of modules) {

diff --git a/app/backend/src/api/Api.ts b/app/backend/src/api/Api.ts
@@ -20,6 +20,7 @@ import BlockValueRepository from "@/database/repositories/BlockValueRepository";
 import cors from "cors";
 import { authenticateApiKey } from "@/api/middleware/auth";
 import { Config } from "@/config";
+import VarRepository from "@/database/repositories/VarRepository";
 
 export class Api {
   private readonly app: express.Application;
@@ -54,6 +55,7 @@ export class Api {
     );
     const syncStatusController = new SyncStatusController(
       new SyncStatusRepository(this.database.getKnex()),
+      new VarRepository(this.database.getKnex(), this.config.varModule),
       this.logger,
     );
     const pricingController = new PricingController(

diff --git a/app/backend/src/api/controllers/SyncStatusController.ts b/app/backend/src/api/controllers/SyncStatusController.ts
@@ -10,13 +10,20 @@ import {
   sendSuccessResponse,
 } from "@/api/utils/responseUtils";
 import { chainTableMapping } from "@/database/repositories/BlockValueRepository";
+import { VarRepository } from "@/database/repositories/VarRepository";
 
 export class SyncStatusController {
-  private syncStatusRepository: SyncStatusRepository;
-  private logger: Logger;
-
-  constructor(syncStatusRepository: SyncStatusRepository, logger: Logger) {
+  private readonly syncStatusRepository: SyncStatusRepository;
+  private readonly varRepository: VarRepository;
+  private readonly logger: Logger;
+
+  constructor(
+    syncStatusRepository: SyncStatusRepository,
+    varRepository: VarRepository,
+    logger: Logger,
+  ) {
     this.syncStatusRepository = syncStatusRepository;
+    this.varRepository = varRepository;
     this.logger = logger;
   }
 
@@ -96,9 +103,8 @@ export class SyncStatusController {
       const params = this.extractParams(req, res);
       if (!params) return;
 
-      const result = await this.syncStatusRepository.getVarAverage(
+      const result = await this.varRepository.getVarAverage(
         params.chainId,
-        undefined,
         params.from,
         params.to,
         params.precision,

diff --git a/app/backend/src/config/Config.ts b/app/backend/src/config/Config.ts
@@ -8,6 +8,7 @@ export interface Config {
   readonly pricingModule: PricingModuleConfig;
   readonly ethereumMonitorModule: EthereumMonitorConfig;
   readonly optimismModule: OptimismModuleConfig;
+  readonly varModule: VarModuleConfig;
   readonly polygonZkEvmModule: PolygonZkEvmModuleConfig;
   readonly starknetModule: StarknetModuleConfig;
 }
@@ -79,3 +80,9 @@ export interface StarknetModuleConfig {
   readonly maxBlockRange: number;
   readonly pollIntervalMs: number;
 }
+
+export interface VarModuleConfig {
+  readonly enabled: boolean;
+  readonly backfillPeriodDays: number;
+  readonly pollIntervalMs: number;
+}
diff --git a/app/backend/src/config/config.local.ts b/app/backend/src/config/config.local.ts
@@ -8,6 +8,7 @@ import {
   PolygonZkEvmModuleConfig,
   PricingModuleConfig,
   StarknetModuleConfig,
+  VarModuleConfig,
 } from "./Config";
 import { Env } from "@/tools/Env";
 import { LogLevel } from "@/tools/Logger";
@@ -82,6 +83,12 @@ export function getLocalConfig(env: Env): Config {
     pollIntervalMs: env.integer("STARKNET_POLL_INTERVAL_MS", 60000),
   };
 
+  const varModuleConfig: VarModuleConfig = {
+    enabled: env.boolean("VAR_MODULE_ENABLED", true),
+    pollIntervalMs: env.integer("VAR_POLL_INTERVAL_MS", 60000),
+    backfillPeriodDays: env.integer("VAR_BACKFILL_PERIOD_DAYS", 7),
+  };
+
   return {
     database: databaseConfig,
     api: apiConfig,
@@ -91,5 +98,6 @@ export function getLocalConfig(env: Env): Config {
     optimismModule: optimismModuleConfig,
     polygonZkEvmModule: polygonZkEvmModuleConfig,
     starknetModule: starknetModuleConfig,
+    varModule: varModuleConfig,
   };
 }
diff --git a/app/backend/src/config/config.test.ts b/app/backend/src/config/config.test.ts
@@ -8,6 +8,7 @@ import {
   PolygonZkEvmModuleConfig,
   PricingModuleConfig,
   StarknetModuleConfig,
+  VarModuleConfig,
 } from "./Config";
 import { Env } from "@/tools/Env";
 import { LogLevel } from "@/tools/Logger";
@@ -97,6 +98,12 @@ export function getTestConfig(env: Env): Config {
     pollIntervalMs: env.integer("STARKNET_POLL_INTERVAL_MS", 60000),
   };
 
+  const varModuleConfig: VarModuleConfig = {
+    enabled: env.boolean("VAR_MODULE_ENABLED", true),
+    pollIntervalMs: env.integer("VAR_POLL_INTERVAL_MS", 60000),
+    backfillPeriodDays: env.integer("VAR_BACKFILL_PERIOD_DAYS", 7),
+  };
+
   return {
     database: databaseConfig,
     api: apiConfig,
@@ -106,5 +113,6 @@ export function getTestConfig(env: Env): Config {
     optimismModule: optimismModuleConfig,
     polygonZkEvmModule: polygonZkEvmModuleConfig,
     starknetModule: starknetModuleConfig,
+    varModule: varModuleConfig,
   };
 }
diff --git a/app/backend/src/core/controllers/appraiser/types.ts b/app/backend/src/core/controllers/appraiser/types.ts
@@ -25,17 +25,6 @@ export function getValue(
     : { byContract: {}, byType: {} };
 }
 
-// export function getVar(block: BlockValueRecord | undefined | null): ValueMapping {
-//   return block
-//     ? {byContract: block.var_by_contract, byType: block.var_by_type}
-//     : {byContract: {}, byType: {}};
-// }
-
-// export function setVar(block: BlockValueRecord, value: ValueMapping) {
-//   block.var_by_type = value.byType ?? {};
-//   block.var_by_contract = value.byContract ?? {};
-// }
-
 export function mergeValues(
   mappings: ValueMapping[],
   ignoreZero: boolean = false,
@@ -88,3 +77,10 @@ export function mergeValues(
 
   return result;
 }
+
+export function mergeBlockValues(
+  blocks: BlockValueRecord[],
+  ignoreZero: boolean = false,
+) {
+  return mergeValues(blocks.map(getValue), ignoreZero);
+}
diff --git a/app/backend/src/core/controllers/indexers/l1/L1LogMonitorController.ts b/app/backend/src/core/controllers/indexers/l1/L1LogMonitorController.ts
@@ -57,6 +57,8 @@ class L1LogMonitorController {
       (currentHeight - fromBlock + 1) / this.schedulerConfig.maxBlockLogRange,
     );
 
+    if (numberOfBatches <= 0) return;
+
     this.logger.info(
       `Fetching Ethereum contract logs from ${fromBlock} to ${currentHeight} (${numberOfBatches} batches)`,
     );

diff --git a/app/backend/src/core/controllers/indexers/shared/BlockIndexerController.ts b/app/backend/src/core/controllers/indexers/shared/BlockIndexerController.ts
@@ -52,6 +52,8 @@ abstract class BlockIndexerController {
       (currentHeight - fromBlock + 1) / this.indexerConfig.maxBlockRange,
     );
 
+    if (numberOfBatches <= 0) return;
+
     this.logger.info(
       `[chainId ${this.chainId}] Fetching blocks from ${fromBlock} to ${currentHeight} (${numberOfBatches} batches)`,
     );