Merge pull request #583 from godwokenrises/merge-1.9-rc

chore: merge 1.9.0 into main

README.md

@@ -86,7 +86,7 @@ pg_url=<database url, e.g. "postgres://username:password@localhost:5432/dbname">
 Update blocks / transactions / logs info in database by update command, include start block and end block.
 
 ```bash
-./target/release/gw-web3-indexer update <optional start block, default to 0> <optional end block, default to local tip>
+./target/release/gw-web3-indexer update <optional start block, default to 0> <optional end block, default to local tip> <optional cpu cores to use for update, default to half of local cores>
 ```
 
 ### Start API server

crates/indexer/Cargo.toml

@@ -36,3 +36,4 @@ dotenv = "0.15.0"
 rayon = "1.5.3"
 futures = "0.3.21"
 itertools = "0.10.3"
+num_cpus = "1.0"

crates/indexer/src/cpu_count.rs

@@ -0,0 +1,7 @@
+lazy_static::lazy_static! {
+  pub static ref CPU_COUNT: Option<usize> = {
+      std::env::args()
+      .nth(4)
+      .map(|num| num.parse::<usize>().unwrap())
+  };
+}

crates/indexer/src/insert_l2_block.rs

@@ -14,13 +14,16 @@ use sqlx::{
 use sqlx::{Postgres, QueryBuilder};
 
 use crate::{
+    cpu_count::CPU_COUNT,
     pool::POOL_FOR_UPDATE,
     types::{Block, Log, Transaction, TransactionWithLogs},
 };
 
 use itertools::Itertools;
 use rayon::prelude::*;
 
+extern crate num_cpus;
+
 const INSERT_LOGS_BATCH_SIZE: usize = 5000;
 
 pub struct DbBlock<'a> {
@@ -374,7 +377,7 @@ pub async fn update_web3_txs_and_logs(
     .collect::<Result<Vec<_>, sqlx::Error>>()?;
 
     if logs_len != 0 {
-        let mut logs_querys = logs_slice
+        let logs_querys = logs_slice
         .into_par_iter()
         .map(|db_logs| {
             let mut logs_query_builder: QueryBuilder<Postgres> = QueryBuilder::new(
@@ -393,15 +396,36 @@ pub async fn update_web3_txs_and_logs(
             logs_query_builder
         }).collect::<Vec<_>>();
 
-        logs_querys
-            .par_iter_mut()
-            .map(|query_builder| {
-                let query = query_builder.build();
-                smol::block_on(query.execute(&*POOL_FOR_UPDATE)).map_err(|err| anyhow!(err))
-            })
-            .collect::<Vec<_>>()
+        let logs_slice_size: usize = if let Some(cpu_num) = *CPU_COUNT {
+            cpu_num
+        } else {
+            let cpu_count = num_cpus::get();
+            let size = cpu_count / 2;
+            if size > 0 {
+                size
+            } else {
+                1
+            }
+        };
+
+        let logs_query_slice = logs_querys
+            .into_iter()
+            .chunks(logs_slice_size)
             .into_iter()
-            .collect::<Result<Vec<_>>>()?;
+            .map(|chunk| chunk.collect())
+            .collect::<Vec<Vec<_>>>();
+
+        for mut query_builder_vec in logs_query_slice {
+            query_builder_vec
+                .par_iter_mut()
+                .map(|query_builder| {
+                    let query = query_builder.build();
+                    smol::block_on(query.execute(&*POOL_FOR_UPDATE)).map_err(|err| anyhow!(err))
+                })
+                .collect::<Vec<_>>()
+                .into_iter()
+                .collect::<Result<Vec<_>>>()?;
+        }
     }
 
     Ok((txs_len, logs_len))

crates/indexer/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod config;
+pub mod cpu_count;
 pub mod helper;
 pub mod indexer;
 pub mod insert_l2_block;

package.json

@@ -30,6 +30,6 @@
     "eslint": "^8.5.0",
     "prettier": "^2.2.1"
   },
-  "version": "1.9.0-rc1",
+  "version": "1.9.0",
   "author": "hupeng <bitrocks.hu@gmail.com>"
 }

packages/api-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@godwoken-web3/api-server",
-  "version": "1.9.0-rc1",
+  "version": "1.9.0",
   "private": true,
   "scripts": {
     "start": "concurrently \"tsc -w\" \"nodemon ./bin/cluster\"",
@@ -32,8 +32,8 @@
   "dependencies": {
     "@ckb-lumos/base": "0.18.0-rc6",
     "@ckb-lumos/toolkit": "0.18.0-rc6",
+    "@godwoken-web3/godwoken": "1.9.0",
     "@ethersproject/bignumber": "^5.7.0",
-    "@godwoken-web3/godwoken": "1.9.0-rc1",
     "@newrelic/native-metrics": "^7.0.1",
     "@opentelemetry/api": "^1.3.0",
     "@opentelemetry/exporter-jaeger": "^1.8.0",

packages/api-server/src/db/query.ts

@@ -88,17 +88,15 @@ export class Query {
     const blockData = await this.knex<DBBlock>("blocks")
       .select("number")
       .orderBy("number", "desc")
-      .first()
-      .cache();
+      .first();
 
     return toBigIntOpt(blockData?.number);
   }
 
   async getTipBlock(): Promise<Block | undefined> {
     const block = await this.knex<DBBlock>("blocks")
       .orderBy("number", "desc")
-      .first()
-      .cache();
+      .first();
 
     if (!block) {
       return undefined;

packages/godwoken/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@godwoken-web3/godwoken",
-  "version": "1.9.0-rc1",
+  "version": "1.9.0",
   "private": true,
   "main": "lib/index.js",
   "scripts": {