Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add data column kzg verification and update c-kzg. #5701

Merged
merged 7 commits into from
May 9, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Add data column kzg verification and update c-kzg.
  • Loading branch information
jimmygchen committed May 3, 2024
commit 5650f7c25b7d66d2725d58eadeea6d8c519bc06e
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 11 additions & 13 deletions beacon_node/beacon_chain/src/data_column_verification.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
use crate::block_verification::{process_block_slash_info, BlockSlashInfo};
use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
use crate::kzg_utils::validate_data_column;
use crate::{metrics, BeaconChain, BeaconChainError, BeaconChainTypes};
use derivative::Derivative;
use kzg::{Error as KzgError, Kzg};
use ssz_derive::{Decode, Encode};
use std::iter;
use std::sync::Arc;
use types::data_column_sidecar::{ColumnIndex, DataColumnIdentifier};
use types::{
Expand Down Expand Up @@ -180,15 +182,10 @@ impl<E: EthSpec> KzgVerifiedDataColumn<E> {
/// Returns an error if the kzg verification check fails.
pub fn verify_kzg_for_data_column<E: EthSpec>(
data_column: Arc<DataColumnSidecar<E>>,
_kzg: &Kzg,
kzg: &Kzg,
) -> Result<KzgVerifiedDataColumn<E>, KzgError> {
// TODO(das): validate data column
// validate_blob::<E>(
// kzg,
// &data_column.blob,
// data_column.kzg_commitment,
// data_column.kzg_proof,
// )?;
let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES);
validate_data_column(kzg, iter::once(&data_column))?;
Ok(KzgVerifiedDataColumn { data_column })
}

Expand All @@ -198,13 +195,14 @@ pub fn verify_kzg_for_data_column<E: EthSpec>(
/// Note: This function should be preferred over calling `verify_kzg_for_data_column`
/// in a loop since this function kzg verifies a list of data columns more efficiently.
pub fn verify_kzg_for_data_column_list<'a, E: EthSpec, I>(
_data_column_iter: I,
_kzg: &'a Kzg,
data_column_iter: I,
kzg: &'a Kzg,
) -> Result<(), KzgError>
where
I: Iterator<Item = &'a Arc<DataColumnSidecar<E>>>,
I: Iterator<Item = &'a Arc<DataColumnSidecar<E>>> + Clone,
{
// TODO(das): implement kzg verification
let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_BATCH_TIMES);
validate_data_column(kzg, data_column_iter)?;
Ok(())
}

Expand Down
56 changes: 54 additions & 2 deletions beacon_node/beacon_chain/src/kzg_utils.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
use kzg::{Blob as KzgBlob, Error as KzgError, Kzg};
use types::{Blob, EthSpec, Hash256, KzgCommitment, KzgProof};
use kzg::{Blob as KzgBlob, Bytes48, Cell as KzgCell, Error as KzgError, Kzg};
use std::sync::Arc;
use types::data_column_sidecar::Cell;
use types::{Blob, DataColumnSidecar, EthSpec, Hash256, KzgCommitment, KzgProof};

/// Converts a blob ssz List object to an array to be used with the kzg
/// crypto library.
fn ssz_blob_to_crypto_blob<E: EthSpec>(blob: &Blob<E>) -> Result<KzgBlob, KzgError> {
KzgBlob::from_bytes(blob.as_ref()).map_err(Into::into)
}

/// Converts a cell ssz List object to an array to be used with the kzg
/// crypto library.
fn ssz_cell_to_crypto_cell<E: EthSpec>(cell: &Cell<E>) -> Result<KzgCell, KzgError> {
KzgCell::from_bytes(cell.as_ref()).map_err(Into::into)
}

/// Validate a single blob-commitment-proof triplet from a `BlobSidecar`.
pub fn validate_blob<E: EthSpec>(
kzg: &Kzg,
Expand All @@ -19,6 +27,50 @@ pub fn validate_blob<E: EthSpec>(
kzg.verify_blob_kzg_proof(&kzg_blob, kzg_commitment, kzg_proof)
}

/// Validate a batch of `DataColumnSidecar`.
pub fn validate_data_column<'a, E: EthSpec, I>(
kzg: &Kzg,
data_column_iter: I,
) -> Result<(), KzgError>
where
I: Iterator<Item = &'a Arc<DataColumnSidecar<E>>> + Clone,
{
let cells = data_column_iter
.clone()
.flat_map(|data_column| data_column.column.iter().map(ssz_cell_to_crypto_cell::<E>))
.collect::<Result<Vec<_>, KzgError>>()?;

let proofs = data_column_iter
.clone()
.flat_map(|data_column| {
data_column
.kzg_proofs
.iter()
.map(|&proof| Bytes48::from(proof))
})
.collect::<Vec<_>>();

let coordinates = data_column_iter
.clone()
.flat_map(|data_column| {
let col_index = data_column.index;
(0..data_column.column.len()).map(move |row| (row as u64, col_index))
})
.collect::<Vec<(u64, u64)>>();

let commitments = data_column_iter
.clone()
.flat_map(|data_column| {
data_column
.kzg_commitments
.iter()
.map(|&commitment| Bytes48::from(commitment))
})
.collect::<Vec<_>>();

kzg.verify_cell_proof_batch(&cells, &proofs, &coordinates, &commitments)
}

/// Validate a batch of blob-commitment-proof triplets from multiple `BlobSidecars`.
pub fn validate_blobs<E: EthSpec>(
kzg: &Kzg,
Expand Down
4 changes: 4 additions & 0 deletions beacon_node/beacon_chain/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1145,6 +1145,10 @@ lazy_static! {
try_create_histogram("kzg_verification_single_seconds", "Runtime of single kzg verification");
pub static ref KZG_VERIFICATION_BATCH_TIMES: Result<Histogram> =
try_create_histogram("kzg_verification_batch_seconds", "Runtime of batched kzg verification");
pub static ref KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES: Result<Histogram> =
try_create_histogram("kzg_verification_data_column_single_seconds", "Runtime of single data column kzg verification");
pub static ref KZG_VERIFICATION_DATA_COLUMN_BATCH_TIMES: Result<Histogram> =
try_create_histogram("kzg_verification_data_column_batch_seconds", "Runtime of batched data column kzg verification");

pub static ref BLOCK_PRODUCTION_BLOBS_VERIFICATION_TIMES: Result<Histogram> = try_create_histogram(
"beacon_block_production_blobs_verification_seconds",
Expand Down
2 changes: 1 addition & 1 deletion beacon_node/beacon_processor/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1015,7 +1015,7 @@ impl<E: EthSpec> BeaconProcessor<E> {
self.spawn_worker(item, idle_tx);
} else if let Some(item) = rpc_blob_queue.pop() {
self.spawn_worker(item, idle_tx);
// TODO(das): decide proper priorization for sampling columns
// TODO(das): decide proper prioritization for sampling columns
} else if let Some(item) = rpc_verify_data_column_queue.pop() {
self.spawn_worker(item, idle_tx);
} else if let Some(item) = sampling_result_queue.pop() {
Expand Down
44 changes: 24 additions & 20 deletions crypto/kzg/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ pub use c_kzg::{
Blob, Bytes32, Bytes48, KzgSettings, BYTES_PER_BLOB, BYTES_PER_COMMITMENT,
BYTES_PER_FIELD_ELEMENT, BYTES_PER_PROOF, FIELD_ELEMENTS_PER_BLOB,
};
use c_kzg::{Cell, CELLS_PER_BLOB};
pub use c_kzg::{Cell, CELLS_PER_EXT_BLOB};
use mockall::automock;

#[derive(Debug)]
Expand Down Expand Up @@ -151,8 +151,14 @@ impl Kzg {
pub fn compute_cells_and_proofs(
&self,
blob: &Blob,
) -> Result<(Box<[Cell; CELLS_PER_BLOB]>, Box<[KzgProof; CELLS_PER_BLOB]>), Error> {
let (cells, proofs) = c_kzg::Cell::compute_cells_and_proofs(blob, &self.trusted_setup)
) -> Result<
(
Box<[Cell; CELLS_PER_EXT_BLOB]>,
Box<[KzgProof; CELLS_PER_EXT_BLOB]>,
),
Error,
> {
let (cells, proofs) = c_kzg::Cell::compute_cells_and_kzg_proofs(blob, &self.trusted_setup)
.map_err(Into::<Error>::into)?;
let proofs = Box::new(proofs.map(|proof| KzgProof::from(proof.to_bytes().into_inner())));
Ok((cells, proofs))
Expand All @@ -166,25 +172,17 @@ impl Kzg {
pub fn verify_cell_proof_batch(
&self,
cells: &[Cell],
kzg_proofs: &[KzgProof],
kzg_proofs: &[Bytes48],
coordinates: &[(u64, u64)],
kzg_commitments: &[KzgCommitment],
kzg_commitments: &[Bytes48],
) -> Result<(), Error> {
let commitments_bytes: Vec<Bytes48> = kzg_commitments
.iter()
.map(|comm| Bytes48::from(*comm))
.collect();
let proofs_bytes: Vec<Bytes48> = kzg_proofs
.iter()
.map(|proof| Bytes48::from(*proof))
.collect();
let (rows, columns): (Vec<u64>, Vec<u64>) = coordinates.iter().cloned().unzip();
if !c_kzg::KzgProof::verify_cell_proof_batch(
&commitments_bytes,
if !c_kzg::KzgProof::verify_cell_kzg_proof_batch(
kzg_commitments,
&rows,
&columns,
cells,
&proofs_bytes,
kzg_proofs,
&self.trusted_setup,
)? {
Err(Error::KzgVerificationFailed)
Expand All @@ -196,18 +194,24 @@ impl Kzg {

pub mod mock {
use crate::{Error, KzgProof};
use c_kzg::{Blob, Cell, CELLS_PER_BLOB};
use c_kzg::{Blob, Cell, CELLS_PER_EXT_BLOB};

pub const MOCK_KZG_BYTES_PER_CELL: usize = 2048;

#[allow(clippy::type_complexity)]
pub fn compute_cells_and_proofs(
_blob: &Blob,
) -> Result<(Box<[Cell; CELLS_PER_BLOB]>, Box<[KzgProof; CELLS_PER_BLOB]>), Error> {
) -> Result<
(
Box<[Cell; CELLS_PER_EXT_BLOB]>,
Box<[KzgProof; CELLS_PER_EXT_BLOB]>,
),
Error,
> {
let empty_cell = Cell::new([0; MOCK_KZG_BYTES_PER_CELL]);
Ok((
Box::new([empty_cell; CELLS_PER_BLOB]),
Box::new([KzgProof::empty(); CELLS_PER_BLOB]),
Box::new([empty_cell; CELLS_PER_EXT_BLOB]),
Box::new([KzgProof::empty(); CELLS_PER_EXT_BLOB]),
))
}
}
Expand Down
Loading