From df3919fb54c69b8d783925ec1774d13b51ead670 Mon Sep 17 00:00:00 2001 From: Samuel Laferriere Date: Mon, 13 Jan 2025 17:52:49 -0500 Subject: [PATCH 1/3] docs: add documentation for disperser v2 grpc api and related functions/structs --- api/proto/common/common.proto | 14 +++++++++- api/proto/disperser/v2/disperser_v2.proto | 33 ++++++++++++++++++----- encoding/encoding.go | 3 +++ encoding/kzg/prover/prover.go | 3 +++ encoding/rs/utils.go | 13 ++++++++- 5 files changed, 57 insertions(+), 9 deletions(-) diff --git a/api/proto/common/common.proto b/api/proto/common/common.proto index a75327ed7c..07390c6922 100644 --- a/api/proto/common/common.proto +++ b/api/proto/common/common.proto @@ -2,6 +2,9 @@ syntax = "proto3"; package common; option go_package = "github.com/Layr-Labs/eigenda/api/grpc/common"; +// G1Commitment represents the serialized coordinates of a G1 KZG commitment. +// We use gnark-crypto so adopt its serialization, which is big-endian. See: +// https://github.com/Consensys/gnark-crypto/blob/779e884dabb38b92e677f4891286637a3d2e5734/ecc/bn254/fp/element.go#L862 message G1Commitment { // The X coordinate of the KZG commitment. This is the raw byte representation of the field element. bytes x = 1; @@ -10,11 +13,20 @@ message G1Commitment { } // BlobCommitment represents commitment of a specific blob, containing its -// KZG commitment, degree proof, the actual degree, and data length in number of symbols. +// KZG commitment, degree proof, the actual degree, and data length in number of symbols (field elements). +// It deserializes into https://github.com/Layr-Labs/eigenda/blob/ce89dab18d2f8f55004002e17dd3a18529277845/encoding/data.go#L27 +// +// See https://github.com/Layr-Labs/eigenda/blob/master/docs/spec/attestation/encoding.md#validation-via-kzg +// to understand how this commitment is used to validate the blob. message BlobCommitment { + // Concatenation of the x and y coordinates of `common.G1Commitment`. bytes commitment = 1; + // Serialization of the G2Commitment to the blob length. bytes length_commitment = 2; + // Serialization of the G2Affine element representing the proof of the blob length. bytes length_proof = 3; + // The length of the blob in symbols (field elements). + // TODO: is this length always a power of 2? Are there any other characteristics that we should list? etc. uint32 length = 4; } diff --git a/api/proto/disperser/v2/disperser_v2.proto b/api/proto/disperser/v2/disperser_v2.proto index 151c7523fe..e037274f9f 100644 --- a/api/proto/disperser/v2/disperser_v2.proto +++ b/api/proto/disperser/v2/disperser_v2.proto @@ -18,21 +18,40 @@ service Disperser { rpc GetBlobStatus(BlobStatusRequest) returns (BlobStatusReply) {} // GetBlobCommitment is a utility method that calculates commitment for a blob payload. + // It is provided to help clients who are trying to construct a DisperseBlobRequest.blob_header + // and don't have the ability to calculate the commitment themselves (expensive operation which requires SRS points). + // + // For an example usage, see how our disperser_client makes a call to this endpoint when it doesn't have a local prover: + // https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L166 rpc GetBlobCommitment(BlobCommitmentRequest) returns (BlobCommitmentReply) {} - // GetPaymentState is a utility method to get the payment state of a given account. + // GetPaymentState is a utility method to get the payment state of a given account, at a given disperser. + // EigenDA's payment system for v2 is currently centralized, meaning that each disperser does its own accounting. + // A client wanting to disperse a blob would thus need to synchronize its local accounting state with that of the disperser. + // That typically only needs to be done once, and the state can be updated locally as the client disperses blobs. + // The accounting rules are simple and can be updated locally, but periodic checks with the disperser can't hurt. + // + // For an example usage, see how our disperser_client makes a call to this endpoint to populate its local accountant struct: + // https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L298 rpc GetPaymentState(GetPaymentStateRequest) returns (GetPaymentStateReply) {} } // Requests and Replys message DisperseBlobRequest { - // The data to be dispersed. - // The size of data must be <= 16MiB. Every 32 bytes of data is interpreted as an integer in big endian format - // where the lower address has more significant bits. The integer must stay in the valid range to be interpreted - // as a field element on the bn254 curve. The valid range is - // 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 - // If any one of the 32 bytes elements is outside the range, the whole request is deemed as invalid, and rejected. + // The encoded data to be dispersed to the EigenDA network. + // + // Validation rules: + // 1. The size of data must be <= 16MiB. + // 2. The data is allowed to not be a multiple of 32 bytes: the last chunk will be padded with zeros to make it so. + // 3. Every 32 bytes chunk (including the last after rule 2) must be a valid bid-endian serialized field element on the bn254 curve. + // The valid range for each 32 byte chunk is: 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 + // If rule 1 or 3 is violated, the whole request is deemed as invalid, and rejected. + // + // To encode your payload data into the correct blob format, you can make use of our codec: + // https://github.com/Layr-Labs/eigenda/blob/82192985a2d15b88d85a6090404b2595f4922bef/api/clients/codecs/default_blob_codec.go#L21 + // Most users will not need to interact with this low level codec directly however, given that the high-level eigenda_client does the encoding for you: + // https://github.com/Layr-Labs/eigenda/blob/master/api/clients/eigenda_client.go bytes data = 1; common.v2.BlobHeader blob_header = 2; } diff --git a/encoding/encoding.go b/encoding/encoding.go index b4af4ac2db..ef29e6c6b0 100644 --- a/encoding/encoding.go +++ b/encoding/encoding.go @@ -12,6 +12,9 @@ type Prover interface { // reconstruct the blob. EncodeAndProve(data []byte, params EncodingParams) (BlobCommitments, []*Frame, error) + // GetCommitmentsForPaddedLength takes in a byte slice representing a list of bn254 + // field elements (32 bytes each, except potentially the last element), + // pads the (potentially incomplete) last element with zeroes, and returns the commitments for the padded list. GetCommitmentsForPaddedLength(data []byte) (BlobCommitments, error) GetFrames(data []byte, params EncodingParams) ([]*Frame, error) diff --git a/encoding/kzg/prover/prover.go b/encoding/kzg/prover/prover.go index bdcce2ab92..041323ed51 100644 --- a/encoding/kzg/prover/prover.go +++ b/encoding/kzg/prover/prover.go @@ -212,6 +212,9 @@ func (e *Prover) GetFrames(data []byte, params encoding.EncodingParams) ([]*enco return chunks, nil } +// GetCommitmentsForPaddedLength takes in a byte slice representing a list of bn254 +// field elements (32 bytes each, except potentially the last element), +// pads the (potentially incomplete) last element with zeroes, and returns the commitments for the padded list. func (e *Prover) GetCommitmentsForPaddedLength(data []byte) (encoding.BlobCommitments, error) { symbols, err := rs.ToFrArray(data) if err != nil { diff --git a/encoding/rs/utils.go b/encoding/rs/utils.go index d1959380b3..0895a533b6 100644 --- a/encoding/rs/utils.go +++ b/encoding/rs/utils.go @@ -10,6 +10,15 @@ import ( "github.com/consensys/gnark-crypto/ecc/bn254/fr" ) +// ToFrArray deserializes a byte array into a list of bn254 field elements., +// where each 32-byte chunk needs to be a big-endian serialized bn254 field element. +// The last chunk is allowed to not have 32-bytes, and will be padded with zeroes +// on the right (so make sure that the last partial chunk represents a valid field element +// when padded with zeroes on the right and interpreted as big-endian). +// +// TODO: we should probably just force the data to be a multiple of 32 bytes. +// This would make the API and code simpler to read, and also allow the code +// to be auto-vectorized by the compiler (it probably isn't right now given the if inside the for loop). func ToFrArray(data []byte) ([]fr.Element, error) { numEle := GetNumElement(uint64(len(data)), encoding.BYTES_PER_SYMBOL) eles := make([]fr.Element, numEle) @@ -35,7 +44,9 @@ func ToFrArray(data []byte) ([]fr.Element, error) { return eles, nil } -// ToByteArray converts a list of Fr to a byte array +// ToByteArray serializes a slice of fields elements to a slice of bytes. +// The byte array is created by serializing each Fr element in big-endian format. +// It is the reverse operation of ToFrArray. func ToByteArray(dataFr []fr.Element, maxDataSize uint64) []byte { n := len(dataFr) dataSize := int(math.Min( From bd072d9f42d3188c70725bca0b95ddfcd7a7f8bc Mon Sep 17 00:00:00 2001 From: Samuel Laferriere Date: Mon, 13 Jan 2025 17:57:26 -0500 Subject: [PATCH 2/3] chore: make protoc --- api/docs/common.html | 13 ++--- api/docs/common.md | 18 ++++--- api/docs/disperser_v2.html | 35 ++++++++++---- api/docs/disperser_v2.md | 14 ++++-- api/docs/eigenda-protos.html | 48 +++++++++++++------ api/docs/eigenda-protos.md | 32 +++++++++---- api/grpc/common/common.pb.go | 20 ++++++-- api/grpc/disperser/v2/disperser_v2.pb.go | 20 +++++--- api/grpc/disperser/v2/disperser_v2_grpc.pb.go | 28 ++++++++++- 9 files changed, 170 insertions(+), 58 deletions(-) diff --git a/api/docs/common.html b/api/docs/common.html index 522008e9f9..9ee20dcd5b 100644 --- a/api/docs/common.html +++ b/api/docs/common.html @@ -209,7 +209,7 @@

common/common.proto

Top

BlobCommitment

-

BlobCommitment represents commitment of a specific blob, containing its

KZG commitment, degree proof, the actual degree, and data length in number of symbols.

+

BlobCommitment represents commitment of a specific blob, containing its

KZG commitment, degree proof, the actual degree, and data length in number of symbols (field elements).

It deserializes into https://github.com/Layr-Labs/eigenda/blob/ce89dab18d2f8f55004002e17dd3a18529277845/encoding/data.go#L27

See https://github.com/Layr-Labs/eigenda/blob/master/docs/spec/attestation/encoding.md#validation-via-kzg

to understand how this commitment is used to validate the blob.

@@ -222,28 +222,29 @@

BlobCommitment

- + - + - + - + @@ -254,7 +255,7 @@

BlobCommitment

G1Commitment

-

+

G1Commitment represents the serialized coordinates of a G1 KZG commitment.

We use gnark-crypto so adopt its serialization, which is big-endian. See:

https://github.com/Consensys/gnark-crypto/blob/779e884dabb38b92e677f4891286637a3d2e5734/ecc/bn254/fp/element.go#L862

commitment bytes

Concatenation of the x and y coordinates of `common.G1Commitment`.

length_commitment bytes

Serialization of the G2Commitment to the blob length.

length_proof bytes

Serialization of the G2Affine element representing the proof of the blob length.

length uint32

The length of the blob in symbols (field elements). +TODO: is this length always a power of 2? Are there any other characteristics that we should list? etc.

diff --git a/api/docs/common.md b/api/docs/common.md index 4f846da132..7445cbaf39 100644 --- a/api/docs/common.md +++ b/api/docs/common.md @@ -23,15 +23,19 @@ ### BlobCommitment BlobCommitment represents commitment of a specific blob, containing its -KZG commitment, degree proof, the actual degree, and data length in number of symbols. +KZG commitment, degree proof, the actual degree, and data length in number of symbols (field elements). +It deserializes into https://github.com/Layr-Labs/eigenda/blob/ce89dab18d2f8f55004002e17dd3a18529277845/encoding/data.go#L27 + +See https://github.com/Layr-Labs/eigenda/blob/master/docs/spec/attestation/encoding.md#validation-via-kzg +to understand how this commitment is used to validate the blob. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| commitment | [bytes](#bytes) | | | -| length_commitment | [bytes](#bytes) | | | -| length_proof | [bytes](#bytes) | | | -| length | [uint32](#uint32) | | | +| commitment | [bytes](#bytes) | | Concatenation of the x and y coordinates of `common.G1Commitment`. | +| length_commitment | [bytes](#bytes) | | Serialization of the G2Commitment to the blob length. | +| length_proof | [bytes](#bytes) | | Serialization of the G2Affine element representing the proof of the blob length. | +| length | [uint32](#uint32) | | The length of the blob in symbols (field elements). TODO: is this length always a power of 2? Are there any other characteristics that we should list? etc. | @@ -41,7 +45,9 @@ KZG commitment, degree proof, the actual degree, and data length in number of sy ### G1Commitment - +G1Commitment represents the serialized coordinates of a G1 KZG commitment. +We use gnark-crypto so adopt its serialization, which is big-endian. See: +https://github.com/Consensys/gnark-crypto/blob/779e884dabb38b92e677f4891286637a3d2e5734/ecc/bn254/fp/element.go#L862 | Field | Type | Label | Description | diff --git a/api/docs/disperser_v2.html b/api/docs/disperser_v2.html index b7d9f23109..fafbc8acc4 100644 --- a/api/docs/disperser_v2.html +++ b/api/docs/disperser_v2.html @@ -514,12 +514,19 @@

DisperseBlobRequest

- + @@ -868,14 +875,26 @@

Disperser

- + - + diff --git a/api/docs/disperser_v2.md b/api/docs/disperser_v2.md index eeadbd563c..9ee01e87b1 100644 --- a/api/docs/disperser_v2.md +++ b/api/docs/disperser_v2.md @@ -158,7 +158,11 @@ BlobVerificationInfo is the information needed to verify the inclusion of a blob | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| data | [bytes](#bytes) | | The data to be dispersed. The size of data must be <= 16MiB. Every 32 bytes of data is interpreted as an integer in big endian format where the lower address has more significant bits. The integer must stay in the valid range to be interpreted as a field element on the bn254 curve. The valid range is 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 If any one of the 32 bytes elements is outside the range, the whole request is deemed as invalid, and rejected. | +| data | [bytes](#bytes) | | The encoded data to be dispersed to the EigenDA network. + +Validation rules: 1. The size of data must be <= 16MiB. 2. The data is allowed to not be a multiple of 32 bytes: the last chunk will be padded with zeros to make it so. 3. Every 32 bytes chunk (including the last after rule 2) must be a valid bid-endian serialized field element on the bn254 curve. The valid range for each 32 byte chunk is: 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 If rule 1 or 3 is violated, the whole request is deemed as invalid, and rejected. + +To encode your payload data into the correct blob format, you can make use of our codec: https://github.com/Layr-Labs/eigenda/blob/82192985a2d15b88d85a6090404b2595f4922bef/api/clients/codecs/default_blob_codec.go#L21 Most users will not need to interact with this low level codec directly however, given that the high-level eigenda_client does the encoding for you: https://github.com/Layr-Labs/eigenda/blob/master/api/clients/eigenda_client.go | | blob_header | [common.v2.BlobHeader](#common-v2-BlobHeader) | | | @@ -312,8 +316,12 @@ Disperser defines the public APIs for dispersing blobs. | ----------- | ------------ | ------------- | ------------| | DisperseBlob | [DisperseBlobRequest](#disperser-v2-DisperseBlobRequest) | [DisperseBlobReply](#disperser-v2-DisperseBlobReply) | DisperseBlob accepts blob to disperse from clients. This executes the dispersal asynchronously, i.e. it returns once the request is accepted. The client could use GetBlobStatus() API to poll the the processing status of the blob. | | GetBlobStatus | [BlobStatusRequest](#disperser-v2-BlobStatusRequest) | [BlobStatusReply](#disperser-v2-BlobStatusReply) | GetBlobStatus is meant to be polled for the blob status. | -| GetBlobCommitment | [BlobCommitmentRequest](#disperser-v2-BlobCommitmentRequest) | [BlobCommitmentReply](#disperser-v2-BlobCommitmentReply) | GetBlobCommitment is a utility method that calculates commitment for a blob payload. | -| GetPaymentState | [GetPaymentStateRequest](#disperser-v2-GetPaymentStateRequest) | [GetPaymentStateReply](#disperser-v2-GetPaymentStateReply) | GetPaymentState is a utility method to get the payment state of a given account. | +| GetBlobCommitment | [BlobCommitmentRequest](#disperser-v2-BlobCommitmentRequest) | [BlobCommitmentReply](#disperser-v2-BlobCommitmentReply) | GetBlobCommitment is a utility method that calculates commitment for a blob payload. It is provided to help clients who are trying to construct a DisperseBlobRequest.blob_header and don't have the ability to calculate the commitment themselves (expensive operation which requires SRS points). + +For an example usage, see how our disperser_client makes a call to this endpoint when it doesn't have a local prover: https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L166 | +| GetPaymentState | [GetPaymentStateRequest](#disperser-v2-GetPaymentStateRequest) | [GetPaymentStateReply](#disperser-v2-GetPaymentStateReply) | GetPaymentState is a utility method to get the payment state of a given account, at a given disperser. EigenDA's payment system for v2 is currently centralized, meaning that each disperser does its own accounting. A client wanting to disperse a blob would thus need to synchronize its local accounting state with that of the disperser. That typically only needs to be done once, and the state can be updated locally as the client disperses blobs. The accounting rules are simple and can be updated locally, but periodic checks with the disperser can't hurt. + +For an example usage, see how our disperser_client makes a call to this endpoint to populate its local accountant struct: https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L298 | diff --git a/api/docs/eigenda-protos.html b/api/docs/eigenda-protos.html index 9fb7ed8a63..012169b50f 100644 --- a/api/docs/eigenda-protos.html +++ b/api/docs/eigenda-protos.html @@ -883,7 +883,7 @@

common/common.proto

Top

BlobCommitment

-

BlobCommitment represents commitment of a specific blob, containing its

KZG commitment, degree proof, the actual degree, and data length in number of symbols.

+

BlobCommitment represents commitment of a specific blob, containing its

KZG commitment, degree proof, the actual degree, and data length in number of symbols (field elements).

It deserializes into https://github.com/Layr-Labs/eigenda/blob/ce89dab18d2f8f55004002e17dd3a18529277845/encoding/data.go#L27

See https://github.com/Layr-Labs/eigenda/blob/master/docs/spec/attestation/encoding.md#validation-via-kzg

to understand how this commitment is used to validate the blob.

data bytes

The data to be dispersed. -The size of data must be <= 16MiB. Every 32 bytes of data is interpreted as an integer in big endian format -where the lower address has more significant bits. The integer must stay in the valid range to be interpreted -as a field element on the bn254 curve. The valid range is -0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 -If any one of the 32 bytes elements is outside the range, the whole request is deemed as invalid, and rejected.

The encoded data to be dispersed to the EigenDA network. + +Validation rules: + 1. The size of data must be <= 16MiB. + 2. The data is allowed to not be a multiple of 32 bytes: the last chunk will be padded with zeros to make it so. + 3. Every 32 bytes chunk (including the last after rule 2) must be a valid bid-endian serialized field element on the bn254 curve. + The valid range for each 32 byte chunk is: 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 +If rule 1 or 3 is violated, the whole request is deemed as invalid, and rejected. + +To encode your payload data into the correct blob format, you can make use of our codec: +https://github.com/Layr-Labs/eigenda/blob/82192985a2d15b88d85a6090404b2595f4922bef/api/clients/codecs/default_blob_codec.go#L21 +Most users will not need to interact with this low level codec directly however, given that the high-level eigenda_client does the encoding for you: +https://github.com/Layr-Labs/eigenda/blob/master/api/clients/eigenda_client.go

GetBlobCommitment BlobCommitmentRequest BlobCommitmentReply

GetBlobCommitment is a utility method that calculates commitment for a blob payload.

GetBlobCommitment is a utility method that calculates commitment for a blob payload. +It is provided to help clients who are trying to construct a DisperseBlobRequest.blob_header +and don't have the ability to calculate the commitment themselves (expensive operation which requires SRS points). + +For an example usage, see how our disperser_client makes a call to this endpoint when it doesn't have a local prover: +https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L166
GetPaymentState GetPaymentStateRequest GetPaymentStateReply

GetPaymentState is a utility method to get the payment state of a given account.

GetPaymentState is a utility method to get the payment state of a given account, at a given disperser. +EigenDA's payment system for v2 is currently centralized, meaning that each disperser does its own accounting. +A client wanting to disperse a blob would thus need to synchronize its local accounting state with that of the disperser. +That typically only needs to be done once, and the state can be updated locally as the client disperses blobs. +The accounting rules are simple and can be updated locally, but periodic checks with the disperser can't hurt. + +For an example usage, see how our disperser_client makes a call to this endpoint to populate its local accountant struct: +https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L298
@@ -896,28 +896,29 @@

BlobCommitment

- + - + - + - + @@ -928,7 +929,7 @@

BlobCommitment

G1Commitment

-

+

G1Commitment represents the serialized coordinates of a G1 KZG commitment.

We use gnark-crypto so adopt its serialization, which is big-endian. See:

https://github.com/Consensys/gnark-crypto/blob/779e884dabb38b92e677f4891286637a3d2e5734/ecc/bn254/fp/element.go#L862

commitment bytes

Concatenation of the x and y coordinates of `common.G1Commitment`.

length_commitment bytes

Serialization of the G2Commitment to the blob length.

length_proof bytes

Serialization of the G2Affine element representing the proof of the blob length.

length uint32

The length of the blob in symbols (field elements). +TODO: is this length always a power of 2? Are there any other characteristics that we should list? etc.

@@ -2208,12 +2209,19 @@

DisperseBlobRequest

- + @@ -2562,14 +2570,26 @@

Disperser

- + - + diff --git a/api/docs/eigenda-protos.md b/api/docs/eigenda-protos.md index 061eb81d1f..be0f431e79 100644 --- a/api/docs/eigenda-protos.md +++ b/api/docs/eigenda-protos.md @@ -247,15 +247,19 @@ https://github.com/Layr-Labs/eigenlayer-middleware/blob/master/src/interfaces/IB ### BlobCommitment BlobCommitment represents commitment of a specific blob, containing its -KZG commitment, degree proof, the actual degree, and data length in number of symbols. +KZG commitment, degree proof, the actual degree, and data length in number of symbols (field elements). +It deserializes into https://github.com/Layr-Labs/eigenda/blob/ce89dab18d2f8f55004002e17dd3a18529277845/encoding/data.go#L27 + +See https://github.com/Layr-Labs/eigenda/blob/master/docs/spec/attestation/encoding.md#validation-via-kzg +to understand how this commitment is used to validate the blob. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| commitment | [bytes](#bytes) | | | -| length_commitment | [bytes](#bytes) | | | -| length_proof | [bytes](#bytes) | | | -| length | [uint32](#uint32) | | | +| commitment | [bytes](#bytes) | | Concatenation of the x and y coordinates of `common.G1Commitment`. | +| length_commitment | [bytes](#bytes) | | Serialization of the G2Commitment to the blob length. | +| length_proof | [bytes](#bytes) | | Serialization of the G2Affine element representing the proof of the blob length. | +| length | [uint32](#uint32) | | The length of the blob in symbols (field elements). TODO: is this length always a power of 2? Are there any other characteristics that we should list? etc. | @@ -265,7 +269,9 @@ KZG commitment, degree proof, the actual degree, and data length in number of sy ### G1Commitment - +G1Commitment represents the serialized coordinates of a G1 KZG commitment. +We use gnark-crypto so adopt its serialization, which is big-endian. See: +https://github.com/Consensys/gnark-crypto/blob/779e884dabb38b92e677f4891286637a3d2e5734/ecc/bn254/fp/element.go#L862 | Field | Type | Label | Description | @@ -866,7 +872,11 @@ BlobVerificationInfo is the information needed to verify the inclusion of a blob | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | -| data | [bytes](#bytes) | | The data to be dispersed. The size of data must be <= 16MiB. Every 32 bytes of data is interpreted as an integer in big endian format where the lower address has more significant bits. The integer must stay in the valid range to be interpreted as a field element on the bn254 curve. The valid range is 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 If any one of the 32 bytes elements is outside the range, the whole request is deemed as invalid, and rejected. | +| data | [bytes](#bytes) | | The encoded data to be dispersed to the EigenDA network. + +Validation rules: 1. The size of data must be <= 16MiB. 2. The data is allowed to not be a multiple of 32 bytes: the last chunk will be padded with zeros to make it so. 3. Every 32 bytes chunk (including the last after rule 2) must be a valid bid-endian serialized field element on the bn254 curve. The valid range for each 32 byte chunk is: 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 If rule 1 or 3 is violated, the whole request is deemed as invalid, and rejected. + +To encode your payload data into the correct blob format, you can make use of our codec: https://github.com/Layr-Labs/eigenda/blob/82192985a2d15b88d85a6090404b2595f4922bef/api/clients/codecs/default_blob_codec.go#L21 Most users will not need to interact with this low level codec directly however, given that the high-level eigenda_client does the encoding for you: https://github.com/Layr-Labs/eigenda/blob/master/api/clients/eigenda_client.go | | blob_header | [common.v2.BlobHeader](#common-v2-BlobHeader) | | | @@ -1020,8 +1030,12 @@ Disperser defines the public APIs for dispersing blobs. | ----------- | ------------ | ------------- | ------------| | DisperseBlob | [DisperseBlobRequest](#disperser-v2-DisperseBlobRequest) | [DisperseBlobReply](#disperser-v2-DisperseBlobReply) | DisperseBlob accepts blob to disperse from clients. This executes the dispersal asynchronously, i.e. it returns once the request is accepted. The client could use GetBlobStatus() API to poll the the processing status of the blob. | | GetBlobStatus | [BlobStatusRequest](#disperser-v2-BlobStatusRequest) | [BlobStatusReply](#disperser-v2-BlobStatusReply) | GetBlobStatus is meant to be polled for the blob status. | -| GetBlobCommitment | [BlobCommitmentRequest](#disperser-v2-BlobCommitmentRequest) | [BlobCommitmentReply](#disperser-v2-BlobCommitmentReply) | GetBlobCommitment is a utility method that calculates commitment for a blob payload. | -| GetPaymentState | [GetPaymentStateRequest](#disperser-v2-GetPaymentStateRequest) | [GetPaymentStateReply](#disperser-v2-GetPaymentStateReply) | GetPaymentState is a utility method to get the payment state of a given account. | +| GetBlobCommitment | [BlobCommitmentRequest](#disperser-v2-BlobCommitmentRequest) | [BlobCommitmentReply](#disperser-v2-BlobCommitmentReply) | GetBlobCommitment is a utility method that calculates commitment for a blob payload. It is provided to help clients who are trying to construct a DisperseBlobRequest.blob_header and don't have the ability to calculate the commitment themselves (expensive operation which requires SRS points). + +For an example usage, see how our disperser_client makes a call to this endpoint when it doesn't have a local prover: https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L166 | +| GetPaymentState | [GetPaymentStateRequest](#disperser-v2-GetPaymentStateRequest) | [GetPaymentStateReply](#disperser-v2-GetPaymentStateReply) | GetPaymentState is a utility method to get the payment state of a given account, at a given disperser. EigenDA's payment system for v2 is currently centralized, meaning that each disperser does its own accounting. A client wanting to disperse a blob would thus need to synchronize its local accounting state with that of the disperser. That typically only needs to be done once, and the state can be updated locally as the client disperses blobs. The accounting rules are simple and can be updated locally, but periodic checks with the disperser can't hurt. + +For an example usage, see how our disperser_client makes a call to this endpoint to populate its local accountant struct: https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L298 | diff --git a/api/grpc/common/common.pb.go b/api/grpc/common/common.pb.go index 8f31656a02..5ab191fa43 100644 --- a/api/grpc/common/common.pb.go +++ b/api/grpc/common/common.pb.go @@ -20,6 +20,9 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// G1Commitment represents the serialized coordinates of a G1 KZG commitment. +// We use gnark-crypto so adopt its serialization, which is big-endian. See: +// https://github.com/Consensys/gnark-crypto/blob/779e884dabb38b92e677f4891286637a3d2e5734/ecc/bn254/fp/element.go#L862 type G1Commitment struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -78,16 +81,25 @@ func (x *G1Commitment) GetY() []byte { } // BlobCommitment represents commitment of a specific blob, containing its -// KZG commitment, degree proof, the actual degree, and data length in number of symbols. +// KZG commitment, degree proof, the actual degree, and data length in number of symbols (field elements). +// It deserializes into https://github.com/Layr-Labs/eigenda/blob/ce89dab18d2f8f55004002e17dd3a18529277845/encoding/data.go#L27 +// +// See https://github.com/Layr-Labs/eigenda/blob/master/docs/spec/attestation/encoding.md#validation-via-kzg +// to understand how this commitment is used to validate the blob. type BlobCommitment struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Commitment []byte `protobuf:"bytes,1,opt,name=commitment,proto3" json:"commitment,omitempty"` + // Concatenation of the x and y coordinates of `common.G1Commitment`. + Commitment []byte `protobuf:"bytes,1,opt,name=commitment,proto3" json:"commitment,omitempty"` + // Serialization of the G2Commitment to the blob length. LengthCommitment []byte `protobuf:"bytes,2,opt,name=length_commitment,json=lengthCommitment,proto3" json:"length_commitment,omitempty"` - LengthProof []byte `protobuf:"bytes,3,opt,name=length_proof,json=lengthProof,proto3" json:"length_proof,omitempty"` - Length uint32 `protobuf:"varint,4,opt,name=length,proto3" json:"length,omitempty"` + // Serialization of the G2Affine element representing the proof of the blob length. + LengthProof []byte `protobuf:"bytes,3,opt,name=length_proof,json=lengthProof,proto3" json:"length_proof,omitempty"` + // The length of the blob in symbols (field elements). + // TODO: is this length always a power of 2? Are there any other characteristics that we should list? etc. + Length uint32 `protobuf:"varint,4,opt,name=length,proto3" json:"length,omitempty"` } func (x *BlobCommitment) Reset() { diff --git a/api/grpc/disperser/v2/disperser_v2.pb.go b/api/grpc/disperser/v2/disperser_v2.pb.go index 13cdbf9287..1aa43eb79e 100644 --- a/api/grpc/disperser/v2/disperser_v2.pb.go +++ b/api/grpc/disperser/v2/disperser_v2.pb.go @@ -100,12 +100,20 @@ type DisperseBlobRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The data to be dispersed. - // The size of data must be <= 16MiB. Every 32 bytes of data is interpreted as an integer in big endian format - // where the lower address has more significant bits. The integer must stay in the valid range to be interpreted - // as a field element on the bn254 curve. The valid range is - // 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 - // If any one of the 32 bytes elements is outside the range, the whole request is deemed as invalid, and rejected. + // The encoded data to be dispersed to the EigenDA network. + // + // Validation rules: + // 1. The size of data must be <= 16MiB. + // 2. The data is allowed to not be a multiple of 32 bytes: the last chunk will be padded with zeros to make it so. + // 3. Every 32 bytes chunk (including the last after rule 2) must be a valid bid-endian serialized field element on the bn254 curve. + // The valid range for each 32 byte chunk is: 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 + // + // If rule 1 or 3 is violated, the whole request is deemed as invalid, and rejected. + // + // To encode your payload data into the correct blob format, you can make use of our codec: + // https://github.com/Layr-Labs/eigenda/blob/82192985a2d15b88d85a6090404b2595f4922bef/api/clients/codecs/default_blob_codec.go#L21 + // Most users will not need to interact with this low level codec directly however, given that the high-level eigenda_client does the encoding for you: + // https://github.com/Layr-Labs/eigenda/blob/master/api/clients/eigenda_client.go Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` BlobHeader *v2.BlobHeader `protobuf:"bytes,2,opt,name=blob_header,json=blobHeader,proto3" json:"blob_header,omitempty"` } diff --git a/api/grpc/disperser/v2/disperser_v2_grpc.pb.go b/api/grpc/disperser/v2/disperser_v2_grpc.pb.go index bd93c2ea2a..f384059a99 100644 --- a/api/grpc/disperser/v2/disperser_v2_grpc.pb.go +++ b/api/grpc/disperser/v2/disperser_v2_grpc.pb.go @@ -37,8 +37,20 @@ type DisperserClient interface { // GetBlobStatus is meant to be polled for the blob status. GetBlobStatus(ctx context.Context, in *BlobStatusRequest, opts ...grpc.CallOption) (*BlobStatusReply, error) // GetBlobCommitment is a utility method that calculates commitment for a blob payload. + // It is provided to help clients who are trying to construct a DisperseBlobRequest.blob_header + // and don't have the ability to calculate the commitment themselves (expensive operation which requires SRS points). + // + // For an example usage, see how our disperser_client makes a call to this endpoint when it doesn't have a local prover: + // https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L166 GetBlobCommitment(ctx context.Context, in *BlobCommitmentRequest, opts ...grpc.CallOption) (*BlobCommitmentReply, error) - // GetPaymentState is a utility method to get the payment state of a given account. + // GetPaymentState is a utility method to get the payment state of a given account, at a given disperser. + // EigenDA's payment system for v2 is currently centralized, meaning that each disperser does its own accounting. + // A client wanting to disperse a blob would thus need to synchronize its local accounting state with that of the disperser. + // That typically only needs to be done once, and the state can be updated locally as the client disperses blobs. + // The accounting rules are simple and can be updated locally, but periodic checks with the disperser can't hurt. + // + // For an example usage, see how our disperser_client makes a call to this endpoint to populate its local accountant struct: + // https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L298 GetPaymentState(ctx context.Context, in *GetPaymentStateRequest, opts ...grpc.CallOption) (*GetPaymentStateReply, error) } @@ -98,8 +110,20 @@ type DisperserServer interface { // GetBlobStatus is meant to be polled for the blob status. GetBlobStatus(context.Context, *BlobStatusRequest) (*BlobStatusReply, error) // GetBlobCommitment is a utility method that calculates commitment for a blob payload. + // It is provided to help clients who are trying to construct a DisperseBlobRequest.blob_header + // and don't have the ability to calculate the commitment themselves (expensive operation which requires SRS points). + // + // For an example usage, see how our disperser_client makes a call to this endpoint when it doesn't have a local prover: + // https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L166 GetBlobCommitment(context.Context, *BlobCommitmentRequest) (*BlobCommitmentReply, error) - // GetPaymentState is a utility method to get the payment state of a given account. + // GetPaymentState is a utility method to get the payment state of a given account, at a given disperser. + // EigenDA's payment system for v2 is currently centralized, meaning that each disperser does its own accounting. + // A client wanting to disperse a blob would thus need to synchronize its local accounting state with that of the disperser. + // That typically only needs to be done once, and the state can be updated locally as the client disperses blobs. + // The accounting rules are simple and can be updated locally, but periodic checks with the disperser can't hurt. + // + // For an example usage, see how our disperser_client makes a call to this endpoint to populate its local accountant struct: + // https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L298 GetPaymentState(context.Context, *GetPaymentStateRequest) (*GetPaymentStateReply, error) mustEmbedUnimplementedDisperserServer() } From b52788aca3629cc680c759741d622c85684d6376 Mon Sep 17 00:00:00 2001 From: Samuel Laferriere Date: Tue, 21 Jan 2025 18:08:08 -0500 Subject: [PATCH 3/3] docs: add missing documentation for BlobKey and BlobHeader.BlobKey() --- core/v2/serialization.go | 3 +++ core/v2/types.go | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/core/v2/serialization.go b/core/v2/serialization.go index fa36850b1a..279d0f5a00 100644 --- a/core/v2/serialization.go +++ b/core/v2/serialization.go @@ -27,6 +27,9 @@ type abiBlobCommitments struct { DataLength uint32 } +// BlobKey serializes the BlobHeader into a 32-byte unique identifier. +// This serialization must ALWAYS match the serialization that we perform onchain: +// https://github.com/Layr-Labs/eigenda/blob/a6dd724acdf732af483fd2d9a86325febe7ebdcd/contracts/src/libraries/EigenDAHasher.sol#L119 func (b *BlobHeader) BlobKey() (BlobKey, error) { versionType, err := abi.NewType("uint16", "", nil) if err != nil { diff --git a/core/v2/types.go b/core/v2/types.go index 41f58c6772..e824e488d6 100644 --- a/core/v2/types.go +++ b/core/v2/types.go @@ -31,6 +31,17 @@ func (c *Assignment) GetIndices() []uint32 { return indices } +// BlobKey is the unique identifier for a blob dispersal. +// +// It is computed as the Keccak256 hash of some serialization of the blob header +// where the PaymentHeader has been replaced with Hash(PaymentHeader), in order +// to be easily verifiable onchain. See the BlobKey method of BlobHeader for more +// details. +// +// It can be used to retrieve a blob from relays. +// +// Note that two blobs can have the same content but different headers, +// so they are allowed to both exist in the system. type BlobKey [32]byte func (b BlobKey) Hex() string {
data bytes

The data to be dispersed. -The size of data must be <= 16MiB. Every 32 bytes of data is interpreted as an integer in big endian format -where the lower address has more significant bits. The integer must stay in the valid range to be interpreted -as a field element on the bn254 curve. The valid range is -0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 -If any one of the 32 bytes elements is outside the range, the whole request is deemed as invalid, and rejected.

The encoded data to be dispersed to the EigenDA network. + +Validation rules: + 1. The size of data must be <= 16MiB. + 2. The data is allowed to not be a multiple of 32 bytes: the last chunk will be padded with zeros to make it so. + 3. Every 32 bytes chunk (including the last after rule 2) must be a valid bid-endian serialized field element on the bn254 curve. + The valid range for each 32 byte chunk is: 0 <= x < 21888242871839275222246405745257275088548364400416034343698204186575808495617 +If rule 1 or 3 is violated, the whole request is deemed as invalid, and rejected. + +To encode your payload data into the correct blob format, you can make use of our codec: +https://github.com/Layr-Labs/eigenda/blob/82192985a2d15b88d85a6090404b2595f4922bef/api/clients/codecs/default_blob_codec.go#L21 +Most users will not need to interact with this low level codec directly however, given that the high-level eigenda_client does the encoding for you: +https://github.com/Layr-Labs/eigenda/blob/master/api/clients/eigenda_client.go

GetBlobCommitment BlobCommitmentRequest BlobCommitmentReply

GetBlobCommitment is a utility method that calculates commitment for a blob payload.

GetBlobCommitment is a utility method that calculates commitment for a blob payload. +It is provided to help clients who are trying to construct a DisperseBlobRequest.blob_header +and don't have the ability to calculate the commitment themselves (expensive operation which requires SRS points). + +For an example usage, see how our disperser_client makes a call to this endpoint when it doesn't have a local prover: +https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L166
GetPaymentState GetPaymentStateRequest GetPaymentStateReply

GetPaymentState is a utility method to get the payment state of a given account.

GetPaymentState is a utility method to get the payment state of a given account, at a given disperser. +EigenDA's payment system for v2 is currently centralized, meaning that each disperser does its own accounting. +A client wanting to disperse a blob would thus need to synchronize its local accounting state with that of the disperser. +That typically only needs to be done once, and the state can be updated locally as the client disperses blobs. +The accounting rules are simple and can be updated locally, but periodic checks with the disperser can't hurt. + +For an example usage, see how our disperser_client makes a call to this endpoint to populate its local accountant struct: +https://github.com/Layr-Labs/eigenda/blob/6059c6a068298d11c41e50f5bcd208d0da44906a/api/clients/v2/disperser_client.go#L298