DRAFT: Add filters for Events and Extrinsics.

.github/workflows/dev.yaml

@@ -18,7 +18,7 @@ jobs:
       repository: stats-api
       file: ./api/Dockerfile
     secrets: inherit
-    
+
   build_and_deploy_backend_cerestats_crawler:
     uses: Cerebellum-Network/reusable-workflows/.github/workflows/deploy-to-ecr.yaml@master
     with:
@@ -32,7 +32,7 @@ jobs:
     uses: Cerebellum-Network/reusable-workflows/.github/workflows/build-and-upload-static.yaml@1.0.0
     with:
       runs-on: '["self-hosted", "cere-network-large-workers"]'
-      build_container: 'node:14-buster-slim'
+      build_container: 'node:16-buster-slim'
       deploy_container: 'ubuntu:20.04'
       install_packages_command: 'cp -rf frontend/frontend.config-cere-dev.js frontend/frontend.config.js; yarn install'
       build_command: 'yarn workspace frontend generate'

.github/workflows/prod.yaml

@@ -55,7 +55,7 @@ jobs:
     uses: Cerebellum-Network/reusable-workflows/.github/workflows/build-and-upload-static.yaml@1.0.0
     with:
       runs-on: '["self-hosted", "cere-network-large-workers"]'
-      build_container: 'node:14-buster-slim'
+      build_container: 'node:16-buster-slim'
       deploy_container: 'ubuntu:20.04'
       install_packages_command: 'cp -rf frontend/frontend.config-cere-prd.js frontend/frontend.config.js; yarn install'
       build_command: 'yarn workspace frontend generate'

.github/workflows/stage.yaml

@@ -35,7 +35,7 @@ jobs:
     uses: Cerebellum-Network/reusable-workflows/.github/workflows/build-and-upload-static.yaml@master
     with:
       runs-on: '["self-hosted", "cere-network-large-workers"]'
-      build_container: 'node:14-buster-slim'
+      build_container: 'node:16-buster-slim'
       deploy_container: 'ubuntu:20.04'
       install_packages_command: 'cp -rf frontend/frontend.config-cere-dev.js frontend/frontend.config.js; yarn install'
       build_command: 'yarn workspace frontend generate'

README.md

@@ -34,9 +34,9 @@ apt install docker-ce
 sudo curl -L "https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
 chmod +x /usr/local/bin/docker-compose
 ```
-# node v14
+# node v16
 ```bash
-curl -sL https://deb.nodesource.com/setup_14.x -o nodesource_setup.sh
+curl -fsSL https://deb.nodesource.com/setup_16.x -o nodesource_setup.sh
 bash nodesource_setup.sh
 apt install nodejs
 ```

api/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:fermium
+FROM node:galium
 
 WORKDIR /usr/app/api
 
@@ -9,6 +9,6 @@ COPY ./db /usr/app/db
 
 EXPOSE 8000
 
-RUN yarn 
+RUN yarn
 
 CMD ["yarn", "start"]

backend/crawlers/blockHarvester.js

@@ -9,6 +9,7 @@ const {
   processExtrinsics,
   processEvents,
   processLogs,
+  processMetadata,
   getDisplayName,
   wait,
   logHarvestError,
@@ -101,6 +102,31 @@ const harvestBlock = async (api, client, blockNumber) => {
     // ToDo fix with CBI-1451
     const isElection = false;
 
+    // Runtime upgrade
+    const runtimeUpgrade = blockEvents.find(
+      ({ event: { section, method } }) => section === 'system' && method === 'CodeUpdated',
+    );
+
+    if (runtimeUpgrade) {
+      const { specName } = runtimeVersion.toJSON();
+      const { specVersion } = runtimeVersion;
+
+      // TODO: enable again
+      // see: https://github.com/polkadot-js/api/issues/4596
+      // const metadata = await api.rpc.state.getMetadata(blockHash);
+
+      await processMetadata(
+        client,
+        api,
+        blockNumber,
+        blockHash.toString(),
+        specName.toString(),
+        specVersion.toNumber(),
+        timestamp,
+        loggerOptions,
+      );
+    }
+
     // Store block extrinsics (async)
     processExtrinsics(
       api,

backend/crawlers/blockListener.js

@@ -7,6 +7,7 @@ const {
   isNodeSynced,
   wait,
   shortHash,
+  processMetadata,
   processExtrinsics,
   processEvents,
   processLogs,
@@ -199,6 +200,24 @@ const crawler = async () => {
           ),
         ]);
 
+        // Process Metadata
+        try {
+          const { specName } = runtimeVersion.toJSON();
+          const { specVersion } = runtimeVersion;
+          await processMetadata(
+            client,
+            api,
+            blockNumber,
+            blockHash.toString(),
+            specName.toString(),
+            specVersion.toNumber(),
+            timestamp,
+            loggerOptions,
+          );
+        } catch (e) {
+          console.log('Failed to store Metatdata', e);
+        }
+
         // Update finalized blocks
         await updateFinalized(client, finalizedBlock, loggerOptions);
 

backend/docker/crawler/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:fermium
+FROM node:galium
 
 WORKDIR /usr/app/crawler
 
@@ -7,6 +7,6 @@ COPY ./yarn.lock /usr/app
 COPY ./backend /usr/app/crawler
 COPY ./db /usr/app/db
 
-RUN yarn 
+RUN yarn
 
 CMD ["yarn", "start"]

backend/lib/utils.js

@@ -238,6 +238,63 @@ module.exports = {
       logger.error(loggerOptions, `Error adding extrinsic ${blockNumber}-${index}: ${JSON.stringify(error)}`);
     }
   },
+  processMetadata: async (
+    client,
+    api,
+    blockNumber,
+    blockHash,
+    specName,
+    specVersion,
+    timestamp,
+    loggerOptions,
+  ) => {
+    let metadata;
+    try {
+      metadata = await api.rpc.state.getMetadata(blockHash);
+      logger.debug(loggerOptions, `Got runtime metadata at ${blockHash}!`);
+    } catch (error) {
+      logger.error(
+        loggerOptions,
+        `Error fetching runtime metadata at ${blockHash}: ${JSON.stringify(
+          error,
+        )}`,
+      );
+    }
+
+    const data = [
+      blockNumber,
+      specName,
+      specVersion,
+      Object.keys(metadata.toJSON().metadata)[0],
+      metadata.magicNumber,
+      metadata.toJSON().metadata,
+      timestamp,
+    ];
+    const query = `
+    INSERT INTO runtime (
+      block_number,
+      spec_name,
+      spec_version,
+      metadata_version,
+      metadata_magic_number,
+      metadata,
+      timestamp
+    ) VALUES (
+      $1,
+      $2,
+      $3,
+      $4,
+      $5,
+      $6,
+      $7
+    )
+    ON CONFLICT (spec_version)
+    DO UPDATE SET
+      block_number = EXCLUDED.block_number
+    WHERE EXCLUDED.block_number < runtime.block_number;`;
+
+    await module.exports.dbParamQuery(client, query, data, loggerOptions);
+  },
   processEvents: async (
     client, blockNumber, blockEvents, timestamp, loggerOptions,
   ) => {

db/migrations/20231009110035-create-runtime-table.js

@@ -0,0 +1,53 @@
+'use strict';
+
+var dbm;
+var type;
+var seed;
+var fs = require('fs');
+var path = require('path');
+var Promise;
+
+/**
+  * We receive the dbmigrate dependency from dbmigrate initially.
+  * This enables us to not have to rely on NODE_PATH.
+  */
+exports.setup = function(options, seedLink) {
+  dbm = options.dbmigrate;
+  type = dbm.dataType;
+  seed = seedLink;
+  Promise = options.Promise;
+};
+
+exports.up = function(db) {
+  var filePath = path.join(__dirname, 'sqls', '20231009110035-create-runtime-table-up.sql');
+  return new Promise( function( resolve, reject ) {
+    fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
+      if (err) return reject(err);
+      console.log('received data: ' + data);
+
+      resolve(data);
+    });
+  })
+  .then(function(data) {
+    return db.runSql(data);
+  });
+};
+
+exports.down = function(db) {
+  var filePath = path.join(__dirname, 'sqls', '20231009110035-create-runtime-table-down.sql');
+  return new Promise( function( resolve, reject ) {
+    fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
+      if (err) return reject(err);
+      console.log('received data: ' + data);
+
+      resolve(data);
+    });
+  })
+  .then(function(data) {
+    return db.runSql(data);
+  });
+};
+
+exports._meta = {
+  "version": 1
+};

db/migrations/sqls/20231009110035-create-runtime-table-down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS runtime;

db/migrations/sqls/20231009110035-create-runtime-table-up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS runtime (
+  block_number BIGINT NOT NULL,
+  spec_name TEXT NOT NULL,
+  spec_version INT NOT NULL,
+  metadata_version TEXT NOT NULL,
+  metadata_magic_number INT NOT NULL,
+  metadata JSON NOT NULL,
+  timestamp BIGINT NOT NULL,
+  PRIMARY KEY ( spec_version )
+);

frontend/locales/en.json

@@ -200,7 +200,12 @@
     "extrinsics": {
       "title": "Extrinsics",
       "head_title": "Cere Stats - Extrinsics",
-      "head_content": "{networkName} Extrinsics"
+      "head_content": "{networkName} Extrinsics",
+      "id": "ID",
+      "timestamp": "Date",
+      "hash": "Hash",
+      "extrinsic": "Extrinsic",
+      "success": "Status"
     },
     "transfers": {
       "title": "Transfers",