Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Multithread compute_cells_and_proofs #5805

Merged
merged 2 commits into from
May 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions beacon_node/beacon_chain/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ authors = ["Paul Hauner <paul@paulhauner.com>", "Age Manning <Age@AgeManning.com
edition = { workspace = true }
autotests = false # using a single test binary compiles faster

[[bench]]
name = "benches"
harness = false

[features]
default = ["participation_metrics"]
write_ssz_files = [] # Writes debugging .ssz files to /tmp during block processing.
Expand All @@ -17,6 +21,7 @@ test_backfill = []
maplit = { workspace = true }
environment = { workspace = true }
serde_json = { workspace = true }
criterion = { workspace = true }

[dependencies]
bitvec = { workspace = true }
Expand Down
66 changes: 66 additions & 0 deletions beacon_node/beacon_chain/benches/benches.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#![allow(deprecated)]
jimmygchen marked this conversation as resolved.
Show resolved Hide resolved

use std::sync::Arc;

use bls::Signature;
use criterion::Criterion;
use criterion::{black_box, criterion_group, criterion_main, Benchmark};
use eth2_network_config::TRUSTED_SETUP_BYTES;
use kzg::{Kzg, KzgCommitment, TrustedSetup};
use types::{
beacon_block_body::KzgCommitments, BeaconBlock, BeaconBlockDeneb, Blob, BlobsList, ChainSpec,
DataColumnSidecar, EmptyBlock, EthSpec, MainnetEthSpec, SignedBeaconBlock,
};

fn create_test_block_and_blobs<E: EthSpec>(
num_of_blobs: usize,
spec: &ChainSpec,
) -> (SignedBeaconBlock<E>, BlobsList<E>) {
let mut block = BeaconBlock::Deneb(BeaconBlockDeneb::empty(spec));
let mut body = block.body_mut();
let blob_kzg_commitments = body.blob_kzg_commitments_mut().unwrap();
*blob_kzg_commitments =
KzgCommitments::<E>::new(vec![KzgCommitment::empty_for_testing(); num_of_blobs]).unwrap();

let signed_block = SignedBeaconBlock::from_block(block, Signature::empty());

let blobs = (0..num_of_blobs)
.map(|_| Blob::<E>::default())
.collect::<Vec<_>>()
.into();

(signed_block, blobs)
}

fn all_benches(c: &mut Criterion) {
type E = MainnetEthSpec;

let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP_BYTES)
.map_err(|e| format!("Unable to read trusted setup file: {}", e))
.expect("should have trusted setup");
let kzg = Arc::new(Kzg::new_from_trusted_setup(trusted_setup).expect("should create kzg"));

for blob_count in [1, 2, 3, 6] {
let kzg = kzg.clone();
let spec = E::default_spec();
let (signed_block, blob_sidecars) = create_test_block_and_blobs::<E>(blob_count, &spec);

let column_sidecars =
DataColumnSidecar::build_sidecars(&blob_sidecars, &signed_block, &kzg.clone()).unwrap();

c.bench(
&format!("reconstruct_{}", blob_count),
Benchmark::new("kzg/reconstruct", move |b| {
b.iter(|| {
black_box(DataColumnSidecar::reconstruct(
&kzg,
&column_sidecars.iter().as_slice()[0..column_sidecars.len() / 2],
))
})
}),
);
}
}

criterion_group!(benches, all_benches,);
criterion_main!(benches);
61 changes: 35 additions & 26 deletions consensus/types/src/data_column_sidecar.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ use kzg::{KzgCommitment, KzgProof};
use merkle_proof::verify_merkle_proof;
#[cfg(test)]
use mockall_double::double;
use rayon::prelude::*;
use safe_arith::ArithError;
use serde::{Deserialize, Serialize};
use ssz::Encode;
Expand Down Expand Up @@ -120,10 +121,15 @@ impl<E: EthSpec> DataColumnSidecar<E> {
vec![Vec::with_capacity(E::max_blobs_per_block()); E::number_of_columns()];

// NOTE: assumes blob sidecars are ordered by index
for blob in blobs {
let blob = KzgBlob::from_bytes(blob).map_err(KzgError::from)?;
let (blob_cells, blob_cell_proofs) = kzg.compute_cells_and_proofs(&blob)?;
let blob_cells_and_proofs_vec = blobs
.into_par_iter()
.map(|blob| {
let blob = KzgBlob::from_bytes(blob).map_err(KzgError::from)?;
kzg.compute_cells_and_proofs(&blob)
})
.collect::<Result<Vec<_>, KzgError>>()?;

for (blob_cells, blob_cell_proofs) in blob_cells_and_proofs_vec {
// we iterate over each column, and we construct the column from "top to bottom",
// pushing on the cell and the corresponding proof at each column index. we do this for
// each blob (i.e. the outer loop).
Expand Down Expand Up @@ -197,31 +203,34 @@ impl<E: EthSpec> DataColumnSidecar<E> {
))?;
let num_of_blobs = first_data_column.kzg_commitments.len();

for row_index in 0..num_of_blobs {
let mut cells: Vec<KzgCell> = vec![];
let mut cell_ids: Vec<u64> = vec![];
for data_column in data_columns {
let cell =
data_column
.column
.get(row_index)
.ok_or(KzgError::InconsistentArrayLength(format!(
let blob_cells_and_proofs_vec = (0..num_of_blobs)
.into_par_iter()
.map(|row_index| {
let mut cells: Vec<KzgCell> = vec![];
let mut cell_ids: Vec<u64> = vec![];
for data_column in data_columns {
let cell = data_column.column.get(row_index).ok_or(
KzgError::InconsistentArrayLength(format!(
"Missing data column at index {row_index}"
)))?;

cells.push(ssz_cell_to_crypto_cell::<E>(cell)?);
cell_ids.push(data_column.index);
}
// recover_all_cells does not expect sorted
let all_cells = kzg.recover_all_cells(&cell_ids, &cells)?;
let blob = kzg.cells_to_blob(&all_cells)?;

// Note: This function computes all cells and proofs. According to Justin this is okay,
// computing a partial set may be more expensive and requires code paths that don't exist.
// Computing the blobs cells is technically unnecessary but very cheap. It's done here again
// for simplicity.
let (blob_cells, blob_cell_proofs) = kzg.compute_cells_and_proofs(&blob)?;
)),
)?;

cells.push(ssz_cell_to_crypto_cell::<E>(cell)?);
cell_ids.push(data_column.index);
}
// recover_all_cells does not expect sorted
let all_cells = kzg.recover_all_cells(&cell_ids, &cells)?;
let blob = kzg.cells_to_blob(&all_cells)?;

// Note: This function computes all cells and proofs. According to Justin this is okay,
// computing a partial set may be more expensive and requires code paths that don't exist.
// Computing the blobs cells is technically unnecessary but very cheap. It's done here again
// for simplicity.
kzg.compute_cells_and_proofs(&blob)
})
.collect::<Result<Vec<_>, KzgError>>()?;

for (blob_cells, blob_cell_proofs) in blob_cells_and_proofs_vec {
// we iterate over each column, and we construct the column from "top to bottom",
// pushing on the cell and the corresponding proof at each column index. we do this for
// each blob (i.e. the outer loop).
Expand Down
3 changes: 2 additions & 1 deletion crypto/kzg/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ impl Kzg {
trusted_setup: KzgSettings::load_trusted_setup(
&trusted_setup.g1_points(),
&trusted_setup.g2_points(),
// Enable precomputed table for 8 bits
// Enable precomputed table for 8 bits, with 96MB of memory overhead per process
// Ref: https://notes.ethereum.org/@jtraglia/windowed_multiplications
8,
)?,
})
Expand Down
Loading