diff --git a/api/clients/codecs/blob_codec.go b/api/clients/codecs/blob_codec.go deleted file mode 100644 index 5be5190261..0000000000 --- a/api/clients/codecs/blob_codec.go +++ /dev/null @@ -1,48 +0,0 @@ -package codecs - -import ( - "fmt" -) - -type BlobEncodingVersion byte - -const ( - // This minimal blob encoding contains a 32 byte header = [0x00, version byte, uint32 len of data, 0x00, 0x00,...] - // followed by the encoded data [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] - DefaultBlobEncoding BlobEncodingVersion = 0x0 -) - -type BlobCodec interface { - DecodeBlob(encodedData []byte) ([]byte, error) - EncodeBlob(rawData []byte) ([]byte, error) -} - -func BlobEncodingVersionToCodec(version BlobEncodingVersion) (BlobCodec, error) { - switch version { - case DefaultBlobEncoding: - return DefaultBlobCodec{}, nil - default: - return nil, fmt.Errorf("unsupported blob encoding version: %x", version) - } -} - -func GenericDecodeBlob(data []byte) ([]byte, error) { - if len(data) <= 32 { - return nil, fmt.Errorf("data is not of length greater than 32 bytes: %d", len(data)) - } - // version byte is stored in [1], because [0] is always 0 to ensure the codecBlobHeader is a valid bn254 element - // see https://github.com/Layr-Labs/eigenda/blob/master/api/clients/codecs/default_blob_codec.go#L21 - // TODO: we should prob be working over a struct with methods such as GetBlobEncodingVersion() to prevent index errors - version := BlobEncodingVersion(data[1]) - codec, err := BlobEncodingVersionToCodec(version) - if err != nil { - return nil, err - } - - data, err = codec.DecodeBlob(data) - if err != nil { - return nil, fmt.Errorf("unable to decode blob: %w", err) - } - - return data, nil -} diff --git a/api/clients/codecs/blob_codec_test.go b/api/clients/codecs/blob_codec_test.go deleted file mode 100644 index 9e0867fee4..0000000000 --- a/api/clients/codecs/blob_codec_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package codecs_test - -import ( - "bytes" - "crypto/rand" - "math/big" - "testing" - - "github.com/Layr-Labs/eigenda/api/clients/codecs" -) - -// Helper function to generate a random byte slice of a given length -func randomByteSlice(length int64) []byte { - b := make([]byte, length) - _, err := rand.Read(b) - if err != nil { - panic(err) - } - return b -} - -// TestIFFTCodec tests the encoding and decoding of random byte streams -func TestIFFTCodec(t *testing.T) { - // Create an instance of the DefaultBlobEncodingCodec - codec := codecs.NewIFFTCodec(codecs.NewDefaultBlobCodec()) - - // Number of test iterations - const iterations = 100 - - for i := 0; i < iterations; i++ { - // Generate a random length for the byte slice - length, err := rand.Int(rand.Reader, big.NewInt(1024)) // Random length between 0 and 1023 - if err != nil { - panic(err) - } - originalData := randomByteSlice(length.Int64() + 1) // ensure it's not length 0 - - // Encode the original data - encodedData, err := codec.EncodeBlob(originalData) - if err != nil { - t.Fatalf("Iteration %d: failed to encode blob: %v", i, err) - } - - // Decode the encoded data - decodedData, err := codec.DecodeBlob(encodedData) - if err != nil { - t.Fatalf("Iteration %d: failed to decode blob: %v", i, err) - } - - // Compare the original data with the decoded data - if !bytes.Equal(originalData, decodedData) { - t.Fatalf("Iteration %d: original and decoded data do not match\nOriginal: %v\nDecoded: %v", i, originalData, decodedData) - } - } -} - -// TestIFFTCodec tests the encoding and decoding of random byte streams -func TestNoIFFTCodec(t *testing.T) { - // Create an instance of the DefaultBlobEncodingCodec - codec := codecs.NewNoIFFTCodec(codecs.NewDefaultBlobCodec()) - - // Number of test iterations - const iterations = 100 - - for i := 0; i < iterations; i++ { - // Generate a random length for the byte slice - length, err := rand.Int(rand.Reader, big.NewInt(1024)) // Random length between 0 and 1023 - if err != nil { - panic(err) - } - originalData := randomByteSlice(length.Int64() + 1) // ensure it's not length 0 - - // Encode the original data - encodedData, err := codec.EncodeBlob(originalData) - if err != nil { - t.Fatalf("Iteration %d: failed to encode blob: %v", i, err) - } - - // Decode the encoded data - decodedData, err := codec.DecodeBlob(encodedData) - if err != nil { - t.Fatalf("Iteration %d: failed to decode blob: %v", i, err) - } - - // Compare the original data with the decoded data - if !bytes.Equal(originalData, decodedData) { - t.Fatalf("Iteration %d: original and decoded data do not match\nOriginal: %v\nDecoded: %v", i, originalData, decodedData) - } - } -} diff --git a/api/clients/codecs/default_blob_codec.go b/api/clients/codecs/default_blob_codec.go deleted file mode 100644 index 6d3ec29944..0000000000 --- a/api/clients/codecs/default_blob_codec.go +++ /dev/null @@ -1,61 +0,0 @@ -package codecs - -import ( - "bytes" - "encoding/binary" - "fmt" - - "github.com/Layr-Labs/eigenda/encoding/utils/codec" -) - -type DefaultBlobCodec struct{} - -var _ BlobCodec = DefaultBlobCodec{} - -func NewDefaultBlobCodec() DefaultBlobCodec { - return DefaultBlobCodec{} -} - -// EncodeBlob can never return an error, but to maintain the interface it is included -// so that it can be swapped for the IFFTCodec without changing the interface -func (v DefaultBlobCodec) EncodeBlob(rawData []byte) ([]byte, error) { - codecBlobHeader := make([]byte, 32) - // first byte is always 0 to ensure the codecBlobHeader is a valid bn254 element - // encode version byte - codecBlobHeader[1] = byte(DefaultBlobEncoding) - - // encode length as uint32 - binary.BigEndian.PutUint32(codecBlobHeader[2:6], uint32(len(rawData))) // uint32 should be more than enough to store the length (approx 4gb) - - // encode raw data modulo bn254 - rawDataPadded := codec.ConvertByPaddingEmptyByte(rawData) - - // append raw data - encodedData := append(codecBlobHeader, rawDataPadded...) - - return encodedData, nil -} - -func (v DefaultBlobCodec) DecodeBlob(data []byte) ([]byte, error) { - if len(data) < 32 { - return nil, fmt.Errorf("blob does not contain 32 header bytes, meaning it is malformed") - } - - length := binary.BigEndian.Uint32(data[2:6]) - - // decode raw data modulo bn254 - decodedData := codec.RemoveEmptyByteFromPaddedBytes(data[32:]) - - // get non blob header data - reader := bytes.NewReader(decodedData) - rawData := make([]byte, length) - n, err := reader.Read(rawData) - if err != nil { - return nil, fmt.Errorf("failed to copy unpadded data into final buffer, length: %d, bytes read: %d", length, n) - } - if uint32(n) != length { - return nil, fmt.Errorf("data length does not match length prefix") - } - - return rawData, nil -} diff --git a/api/clients/codecs/ifft_codec.go b/api/clients/codecs/ifft_codec.go deleted file mode 100644 index 41f756e80c..0000000000 --- a/api/clients/codecs/ifft_codec.go +++ /dev/null @@ -1,39 +0,0 @@ -package codecs - -import "fmt" - -type IFFTCodec struct { - writeCodec BlobCodec -} - -var _ BlobCodec = IFFTCodec{} - -func NewIFFTCodec(writeCodec BlobCodec) IFFTCodec { - return IFFTCodec{ - writeCodec: writeCodec, - } -} - -func (v IFFTCodec) EncodeBlob(data []byte) ([]byte, error) { - var err error - data, err = v.writeCodec.EncodeBlob(data) - if err != nil { - // this cannot happen, because EncodeBlob never returns an error - return nil, fmt.Errorf("error encoding data: %w", err) - } - - return IFFT(data) -} - -func (v IFFTCodec) DecodeBlob(data []byte) ([]byte, error) { - if len(data) == 0 { - return nil, fmt.Errorf("blob has length 0, meaning it is malformed") - } - var err error - data, err = FFT(data) - if err != nil { - return nil, fmt.Errorf("error FFTing data: %w", err) - } - - return GenericDecodeBlob(data) -} diff --git a/api/clients/codecs/no_ifft_codec.go b/api/clients/codecs/no_ifft_codec.go deleted file mode 100644 index 372bbfbcd3..0000000000 --- a/api/clients/codecs/no_ifft_codec.go +++ /dev/null @@ -1,21 +0,0 @@ -package codecs - -type NoIFFTCodec struct { - writeCodec BlobCodec -} - -var _ BlobCodec = NoIFFTCodec{} - -func NewNoIFFTCodec(writeCodec BlobCodec) NoIFFTCodec { - return NoIFFTCodec{ - writeCodec: writeCodec, - } -} - -func (v NoIFFTCodec) EncodeBlob(data []byte) ([]byte, error) { - return v.writeCodec.EncodeBlob(data) -} - -func (v NoIFFTCodec) DecodeBlob(data []byte) ([]byte, error) { - return GenericDecodeBlob(data) -} diff --git a/api/clients/config.go b/api/clients/config.go index f4d9caa9fb..220dd303b6 100644 --- a/api/clients/config.go +++ b/api/clients/config.go @@ -5,7 +5,7 @@ import ( "log" "time" - "github.com/Layr-Labs/eigenda/api/clients/codecs" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" ) type EigenDAClientConfig struct { @@ -64,7 +64,7 @@ type EigenDAClientConfig struct { DisableTLS bool // The blob encoding version to use when writing blobs from the high level interface. - PutBlobEncodingVersion codecs.BlobEncodingVersion + PutBlobEncodingVersion codec.BlobEncodingVersion // Point verification mode does an IFFT on data before it is written, and does an FFT on data after it is read. // This makes it possible to open points on the KZG commitment to prove that the field elements correspond to diff --git a/api/clients/eigenda_client.go b/api/clients/eigenda_client.go index 3231f29541..e556c1cd89 100644 --- a/api/clients/eigenda_client.go +++ b/api/clients/eigenda_client.go @@ -10,13 +10,13 @@ import ( "net" "time" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/Layr-Labs/eigenda/api" - "github.com/Layr-Labs/eigenda/api/clients/codecs" grpcdisperser "github.com/Layr-Labs/eigenda/api/grpc/disperser" edasm "github.com/Layr-Labs/eigenda/contracts/bindings/EigenDAServiceManager" "github.com/Layr-Labs/eigenda/core" @@ -28,7 +28,7 @@ import ( type IEigenDAClient interface { GetBlob(ctx context.Context, batchHeaderHash []byte, blobIndex uint32) ([]byte, error) PutBlob(ctx context.Context, txData []byte) (*grpcdisperser.BlobInfo, error) - GetCodec() codecs.BlobCodec + GetPolynomialForm() codec.PolynomialForm Close() error } @@ -39,12 +39,12 @@ type IEigenDAClient interface { type EigenDAClient struct { // TODO: all of these should be private, to prevent users from using them directly, // which breaks encapsulation and makes it hard for us to do refactors or changes - Config EigenDAClientConfig - Log log.Logger - Client DisperserClient - ethClient *ethclient.Client - edasmCaller *edasm.ContractEigenDAServiceManagerCaller - Codec codecs.BlobCodec + Config EigenDAClientConfig + Log log.Logger + Client DisperserClient + ethClient *ethclient.Client + edasmCaller *edasm.ContractEigenDAServiceManagerCaller + PolynomialForm codec.PolynomialForm } var _ IEigenDAClient = &EigenDAClient{} @@ -115,30 +115,29 @@ func NewEigenDAClient(log log.Logger, config EigenDAClientConfig) (*EigenDAClien return nil, fmt.Errorf("new disperser-client: %w", err) } - lowLevelCodec, err := codecs.BlobEncodingVersionToCodec(config.PutBlobEncodingVersion) - if err != nil { - return nil, fmt.Errorf("create low level codec: %w", err) - } - - var codec codecs.BlobCodec + var polynomialForm codec.PolynomialForm if config.DisablePointVerificationMode { - codec = codecs.NewNoIFFTCodec(lowLevelCodec) + polynomialForm = codec.Eval } else { - codec = codecs.NewIFFTCodec(lowLevelCodec) + polynomialForm = codec.Coeff } return &EigenDAClient{ - Log: log, - Config: config, - Client: disperserClient, - ethClient: ethClient, - edasmCaller: edasmCaller, - Codec: codec, + Log: log, + Config: config, + Client: disperserClient, + ethClient: ethClient, + edasmCaller: edasmCaller, + PolynomialForm: polynomialForm, }, nil } -func (m *EigenDAClient) GetCodec() codecs.BlobCodec { - return m.Codec +// GetPolynomialForm returns the form of polynomials, as they are distributed in the system +// +// The polynomial form indicates how blobs must be constructed before dispersal, and how received blobs ought to be +// interpreted. +func (m *EigenDAClient) GetPolynomialForm() codec.PolynomialForm { + return m.PolynomialForm } // GetBlob retrieves a blob from the EigenDA service using the provided context, @@ -159,12 +158,17 @@ func (m *EigenDAClient) GetBlob(ctx context.Context, batchHeaderHash []byte, blo return nil, fmt.Errorf("blob has length zero - this should not be possible") } - decodedData, err := m.Codec.DecodeBlob(data) + encodedPayload, err := m.blobToEncodedPayload(data) if err != nil { - return nil, fmt.Errorf("error decoding blob: %w", err) + return nil, fmt.Errorf("blob to encoded payload: %w", err) } - return decodedData, nil + decodedPayload, err := codec.DecodePayload(encodedPayload) + if err != nil { + return nil, fmt.Errorf("error decoding payload: %w", err) + } + + return decodedPayload, nil } // PutBlob encodes and writes a blob to EigenDA, waiting for a desired blob status @@ -220,13 +224,9 @@ func (m *EigenDAClient) PutBlobAsync(ctx context.Context, data []byte) (resultCh func (m *EigenDAClient) putBlob(ctxFinality context.Context, rawData []byte, resultChan chan *grpcdisperser.BlobInfo, errChan chan error) { m.Log.Info("Attempting to disperse blob to EigenDA") - // encode blob - if m.Codec == nil { - errChan <- api.NewErrorInternal("codec not initialized") - return - } + encodedPayload := codec.EncodePayload(rawData) + blob, err := m.encodedPayloadToBlob(encodedPayload) - data, err := m.Codec.EncodeBlob(rawData) if err != nil { // Encode can only fail if there is something wrong with the data, so we return a 400 error errChan <- api.NewErrorInvalidArg(fmt.Sprintf("error encoding blob: %v", err)) @@ -240,7 +240,7 @@ func (m *EigenDAClient) putBlob(ctxFinality context.Context, rawData []byte, res // disperse blob // TODO: would be nice to add a trace-id key to the context, to be able to follow requests from batcher->proxy->eigenda // clients with a payment signer setting can disperse paid blobs - _, requestID, err := m.Client.DisperseBlobAuthenticated(ctxFinality, data, customQuorumNumbers) + _, requestID, err := m.Client.DisperseBlobAuthenticated(ctxFinality, blob, customQuorumNumbers) if err != nil { // DisperserClient returned error is already a grpc error which can be a 400 (eg rate limited) or 500, // so we wrap the error such that clients can still use grpc's status.FromError() function to get the status code. @@ -374,12 +374,12 @@ func (m *EigenDAClient) putBlob(ctxFinality context.Context, rawData []byte, res // Close simply calls Close() on the wrapped disperserClient, to close the grpc connection to the disperser server. // It is thread safe and can be called multiple times. -func (c *EigenDAClient) Close() error { - return c.Client.Close() +func (m *EigenDAClient) Close() error { + return m.Client.Close() } // getConfDeepBlockNumber returns the block number that is `depth` blocks behind the current block number. -func (m EigenDAClient) getConfDeepBlockNumber(ctx context.Context, depth uint64) (*big.Int, error) { +func (m *EigenDAClient) getConfDeepBlockNumber(ctx context.Context, depth uint64) (*big.Int, error) { curBlockNumber, err := m.ethClient.BlockNumber(ctx) if err != nil { return nil, fmt.Errorf("failed to get latest block number: %w", err) @@ -396,7 +396,7 @@ func (m EigenDAClient) getConfDeepBlockNumber(ctx context.Context, depth uint64) // batchIdConfirmedAtDepth checks if a batch ID has been confirmed at a certain depth. // It returns true if the batch ID has been confirmed at the given depth, and false otherwise, // or returns an error if any of the network calls fail. -func (m EigenDAClient) batchIdConfirmedAtDepth(ctx context.Context, batchId uint32, depth uint64) (bool, error) { +func (m *EigenDAClient) batchIdConfirmedAtDepth(ctx context.Context, batchId uint32, depth uint64) (bool, error) { confDeepBlockNumber, err := m.getConfDeepBlockNumber(ctx, depth) if err != nil { return false, fmt.Errorf("failed to get confirmation deep block number: %w", err) @@ -410,3 +410,43 @@ func (m EigenDAClient) batchIdConfirmedAtDepth(ctx context.Context, batchId uint } return true, nil } + +// blobToEncodedPayload accepts blob bytes and converts them into an encoded payload +// +// If the system is configured to distribute blobs in Eval form, the returned bytes exactly match the blob bytes. +// If the system is configured to distribute blobs in Coeff form, the blob is FFTed before being returned +func (m *EigenDAClient) blobToEncodedPayload(blob []byte) ([]byte, error) { + switch m.PolynomialForm { + case codec.Eval: + return blob, nil + case codec.Coeff: + encodedPayload, err := codec.FFT(blob) + if err != nil { + return nil, fmt.Errorf("fft: %w", err) + } + + return encodedPayload, nil + default: + return nil, fmt.Errorf("unknown polynomial form: %v", m.PolynomialForm) + } +} + +// encodedPayloadToBlob accepts encoded payload bytes and converts them into a blob +// +// If the system is configured to distribute blobs in Eval form, the returned bytes exactly match the input bytes +// If the system is configured to distribute blobs in Coeff form, the encoded payload is IFFTed before being returned +func (m *EigenDAClient) encodedPayloadToBlob(encodedPayload []byte) ([]byte, error) { + switch m.PolynomialForm { + case codec.Eval: + return encodedPayload, nil + case codec.Coeff: + coeffPolynomial, err := codec.IFFT(encodedPayload) + if err != nil { + return nil, fmt.Errorf("ifft: %w", err) + } + + return coeffPolynomial, nil + default: + return nil, fmt.Errorf("unknown polynomial form: %v", m.PolynomialForm) + } +} diff --git a/api/clients/eigenda_client_test.go b/api/clients/eigenda_client_test.go index 29435472be..131e0c28fc 100644 --- a/api/clients/eigenda_client_test.go +++ b/api/clients/eigenda_client_test.go @@ -7,11 +7,11 @@ import ( "time" "github.com/Layr-Labs/eigenda/api/clients" - "github.com/Layr-Labs/eigenda/api/clients/codecs" clientsmock "github.com/Layr-Labs/eigenda/api/clients/mock" "github.com/Layr-Labs/eigenda/api/grpc/common" grpcdisperser "github.com/Layr-Labs/eigenda/api/grpc/disperser" "github.com/Layr-Labs/eigenda/disperser" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/ethereum/go-ethereum/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -66,12 +66,12 @@ func TestPutRetrieveBlobIFFTSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codec.BlobEncodingVersion0, DisablePointVerificationMode: false, WaitForFinalization: true, }, - Client: disperserClient, - Codec: codecs.NewIFFTCodec(codecs.NewDefaultBlobCodec()), + Client: disperserClient, + PolynomialForm: codec.Coeff, } expectedBlob := []byte("dc49e7df326cfb2e7da5cf68f263e1898443ec2e862350606e7dfbda55ad10b5d61ed1d54baf6ae7a86279c1b4fa9c49a7de721dacb211264c1f5df31bade51c") blobInfo, err := eigendaClient.PutBlob(context.Background(), expectedBlob) @@ -84,78 +84,6 @@ func TestPutRetrieveBlobIFFTSuccess(t *testing.T) { require.Equal(t, expectedBlob, resultBlob) } -func TestPutRetrieveBlobIFFTNoDecodeSuccess(t *testing.T) { - disperserClient := clientsmock.NewMockDisperserClient() - expectedBlobStatus := disperser.Processing - (disperserClient.On("DisperseBlobAuthenticated", mock.Anything, mock.Anything, mock.Anything). - Return(&expectedBlobStatus, []byte("mock-request-id"), nil)) - (disperserClient.On("GetBlobStatus", mock.Anything, mock.Anything). - Return(&grpcdisperser.BlobStatusReply{Status: grpcdisperser.BlobStatus_PROCESSING}, nil).Once()) - (disperserClient.On("GetBlobStatus", mock.Anything, mock.Anything). - Return(&grpcdisperser.BlobStatusReply{Status: grpcdisperser.BlobStatus_DISPERSING}, nil).Once()) - (disperserClient.On("GetBlobStatus", mock.Anything, mock.Anything). - Return(&grpcdisperser.BlobStatusReply{Status: grpcdisperser.BlobStatus_CONFIRMED}, nil).Once()) - finalizedBlobInfo := &grpcdisperser.BlobInfo{ - BlobHeader: &grpcdisperser.BlobHeader{ - Commitment: &common.G1Commitment{X: []byte{0x00, 0x00, 0x00, 0x00}, Y: []byte{0x01, 0x00, 0x00, 0x00}}, - BlobQuorumParams: []*grpcdisperser.BlobQuorumParam{ - { - QuorumNumber: 0, - }, - { - QuorumNumber: 1, - }, - }, - }, - BlobVerificationProof: &grpcdisperser.BlobVerificationProof{ - BlobIndex: 100, - BatchMetadata: &grpcdisperser.BatchMetadata{ - BatchHeaderHash: []byte("mock-batch-header-hash"), - BatchHeader: &grpcdisperser.BatchHeader{ - ReferenceBlockNumber: 200, - }, - }, - }, - } - (disperserClient.On("GetBlobStatus", mock.Anything, mock.Anything). - Return(&grpcdisperser.BlobStatusReply{Status: grpcdisperser.BlobStatus_FINALIZED, Info: finalizedBlobInfo}, nil).Once()) - (disperserClient.On("RetrieveBlob", mock.Anything, mock.Anything, mock.Anything). - Return(nil, nil).Once()) // pass nil in as the return blob to tell the mock to return the corresponding blob - logger := log.NewLogger(log.DiscardHandler()) - ifftCodec := codecs.NewIFFTCodec(codecs.NewDefaultBlobCodec()) - eigendaClient := clients.EigenDAClient{ - Log: logger, - Config: clients.EigenDAClientConfig{ - RPC: "localhost:51001", - StatusQueryTimeout: 10 * time.Minute, - StatusQueryRetryInterval: 50 * time.Millisecond, - ResponseTimeout: 10 * time.Second, - CustomQuorumIDs: []uint{}, - SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", - DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, - DisablePointVerificationMode: false, - WaitForFinalization: true, - }, - Client: disperserClient, - Codec: ifftCodec, - } - expectedBlob := []byte("dc49e7df326cfb2e7da5cf68f263e1898443ec2e862350606e7dfbda55ad10b5d61ed1d54baf6ae7a86279c1b4fa9c49a7de721dacb211264c1f5df31bade51c") - blobInfo, err := eigendaClient.PutBlob(context.Background(), expectedBlob) - require.NoError(t, err) - require.NotNil(t, blobInfo) - assert.Equal(t, finalizedBlobInfo, blobInfo) - - resultBlob, err := eigendaClient.GetBlob(context.Background(), []byte("mock-batch-header-hash"), 100) - require.NoError(t, err) - encodedBlob, err := ifftCodec.EncodeBlob(resultBlob) - require.NoError(t, err) - - resultBlob, err = codecs.NewIFFTCodec(codecs.NewDefaultBlobCodec()).DecodeBlob(encodedBlob) - require.NoError(t, err) - require.Equal(t, expectedBlob, resultBlob) -} - func TestPutRetrieveBlobNoIFFTSuccess(t *testing.T) { disperserClient := clientsmock.NewMockDisperserClient() expectedBlobStatus := disperser.Processing @@ -204,12 +132,12 @@ func TestPutRetrieveBlobNoIFFTSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codec.BlobEncodingVersion0, DisablePointVerificationMode: true, WaitForFinalization: true, }, - Client: disperserClient, - Codec: codecs.NewNoIFFTCodec(codecs.NewDefaultBlobCodec()), + Client: disperserClient, + PolynomialForm: codec.Eval, } expectedBlob := []byte("dc49e7df326cfb2e7da5cf68f263e1898443ec2e862350606e7dfbda55ad10b5d61ed1d54baf6ae7a86279c1b4fa9c49a7de721dacb211264c1f5df31bade51c") blobInfo, err := eigendaClient.PutBlob(context.Background(), expectedBlob) @@ -237,11 +165,11 @@ func TestPutBlobFailDispersal(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codec.BlobEncodingVersion0, WaitForFinalization: true, }, - Client: disperserClient, - Codec: codecs.NewIFFTCodec(codecs.NewDefaultBlobCodec()), + Client: disperserClient, + PolynomialForm: codec.Coeff, } blobInfo, err := eigendaClient.PutBlob(context.Background(), []byte("hello")) require.Error(t, err) @@ -270,11 +198,11 @@ func TestPutBlobFailureInsufficentSignatures(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codec.BlobEncodingVersion0, WaitForFinalization: true, }, - Client: disperserClient, - Codec: codecs.NewIFFTCodec(codecs.NewDefaultBlobCodec()), + Client: disperserClient, + PolynomialForm: codec.Coeff, } blobInfo, err := eigendaClient.PutBlob(context.Background(), []byte("hello")) require.Error(t, err) @@ -303,11 +231,11 @@ func TestPutBlobFailureGeneral(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codec.BlobEncodingVersion0, WaitForFinalization: true, }, - Client: disperserClient, - Codec: codecs.NewIFFTCodec(codecs.NewDefaultBlobCodec()), + Client: disperserClient, + PolynomialForm: codec.Coeff, } blobInfo, err := eigendaClient.PutBlob(context.Background(), []byte("hello")) require.Error(t, err) @@ -336,11 +264,11 @@ func TestPutBlobFailureUnknown(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codec.BlobEncodingVersion0, WaitForFinalization: true, }, - Client: disperserClient, - Codec: codecs.NewIFFTCodec(codecs.NewDefaultBlobCodec()), + Client: disperserClient, + PolynomialForm: codec.Coeff, } blobInfo, err := eigendaClient.PutBlob(context.Background(), []byte("hello")) require.Error(t, err) @@ -371,11 +299,11 @@ func TestPutBlobFinalizationTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codec.BlobEncodingVersion0, WaitForFinalization: true, }, - Client: disperserClient, - Codec: codecs.NewIFFTCodec(codecs.NewDefaultBlobCodec()), + Client: disperserClient, + PolynomialForm: codec.Coeff, } blobInfo, err := eigendaClient.PutBlob(context.Background(), []byte("hello")) require.Error(t, err) @@ -431,11 +359,11 @@ func TestPutBlobIndividualRequestTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codec.BlobEncodingVersion0, WaitForFinalization: true, }, - Client: disperserClient, - Codec: codecs.NewIFFTCodec(codecs.NewDefaultBlobCodec()), + Client: disperserClient, + PolynomialForm: codec.Coeff, } blobInfo, err := eigendaClient.PutBlob(context.Background(), []byte("hello")) @@ -494,11 +422,11 @@ func TestPutBlobTotalTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codec.BlobEncodingVersion0, WaitForFinalization: true, }, - Client: disperserClient, - Codec: codecs.NewIFFTCodec(codecs.NewDefaultBlobCodec()), + Client: disperserClient, + PolynomialForm: codec.Coeff, } blobInfo, err := eigendaClient.PutBlob(context.Background(), []byte("hello")) diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index 09659d4332..66987519b0 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -1,9 +1,81 @@ package codec import ( + "bytes" + "encoding/binary" + "fmt" + "github.com/Layr-Labs/eigenda/encoding" ) +type BlobEncodingVersion byte + +const ( + // BlobEncodingVersion0 entails a 32 byte header = [0x00, version byte, big-endian uint32 len of payload, 0x00, 0x00,...] + // followed by the encoded data [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] + BlobEncodingVersion0 BlobEncodingVersion = 0x0 +) + +// EncodePayload accepts an arbitrary payload byte array, and encodes it. +// +// The returned bytes shall be interpreted as a polynomial in Eval form, where each contained field element of +// length 32 represents the evaluation of the polynomial at an expanded root of unity +// +// The returned bytes may or may not represent a blob. If the system is configured to distribute blobs in Coeff form, +// then the data returned from this function must be IFFTed to produce the final blob. If the system is configured to +// distribute blobs in Eval form, then the data returned from this function is the final blob representation. +// +// Example encoding: +// Payload header (32 bytes total) Encoded Payload Data +// [0x00, version byte, big-endian uint32 len of payload, 0x00, ...] + [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] +func EncodePayload(payload []byte) []byte { + payloadHeader := make([]byte, 32) + // first byte is always 0 to ensure the payloadHeader is a valid bn254 element + payloadHeader[1] = byte(BlobEncodingVersion0) // encode version byte + + // encode payload length as uint32 + binary.BigEndian.PutUint32( + payloadHeader[2:6], + uint32(len(payload))) // uint32 should be more than enough to store the length (approx 4gb) + + // encode payload modulo bn254 + // the resulting bytes subsequently may be treated as the evaluation of a polynomial + polynomialEval := ConvertByPaddingEmptyByte(payload) + + encodedPayload := append(payloadHeader, polynomialEval...) + + return encodedPayload +} + +// DecodePayload accepts bytes representing an encoded payload, and returns the decoded payload +// +// This function expects the parameter bytes to be a polynomial in Eval form. In other words, if blobs in the system +// are being distributed in Coeff form, a blob must be FFTed prior to being passed into the function. +func DecodePayload(encodedPayload []byte) ([]byte, error) { + if len(encodedPayload) < 32 { + return nil, fmt.Errorf("encoded payload does not contain 32 header bytes, meaning it is malformed") + } + + payloadLength := binary.BigEndian.Uint32(encodedPayload[2:6]) + + // decode raw data modulo bn254 + nonPaddedData := RemoveEmptyByteFromPaddedBytes(encodedPayload[32:]) + + reader := bytes.NewReader(nonPaddedData) + payload := make([]byte, payloadLength) + readLength, err := reader.Read(payload) + if err != nil { + return nil, fmt.Errorf( + "failed to copy unpadded data into final buffer, length: %d, bytes read: %d", + payloadLength, readLength) + } + if uint32(readLength) != payloadLength { + return nil, fmt.Errorf("data length does not match length prefix") + } + + return payload, nil +} + // ConvertByPaddingEmptyByte takes bytes and insert an empty byte at the front of every 31 byte. // The empty byte is padded at the low address, because we use big endian to interpret a field element. // This ensures every 32 bytes is within the valid range of a field element for bn254 curve. diff --git a/encoding/utils/codec/codec_test.go b/encoding/utils/codec/codec_test.go index 3137d7fe7b..85dc6bb4f2 100644 --- a/encoding/utils/codec/codec_test.go +++ b/encoding/utils/codec/codec_test.go @@ -1,9 +1,11 @@ package codec_test import ( + "bytes" "crypto/rand" "testing" + "github.com/Layr-Labs/eigenda/common/testutils/random" "github.com/Layr-Labs/eigenda/encoding/rs" "github.com/Layr-Labs/eigenda/encoding/utils/codec" "github.com/stretchr/testify/require" @@ -49,3 +51,33 @@ func TestSimplePaddingCodec_Fuzz(t *testing.T) { } } } + +// TestCodec tests the encoding and decoding of random byte streams +func TestPayloadEncoding(t *testing.T) { + testRandom := random.NewTestRandom(t) + + // Number of test iterations + const iterations = 100 + + for i := 0; i < iterations; i++ { + originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) + + // Encode the original data + encodedData := codec.EncodePayload(originalData) + + // Decode the encoded data + decodedData, err := codec.DecodePayload(encodedData) + if err != nil { + t.Fatalf("Iteration %d: failed to decode blob: %v", i, err) + } + + // Compare the original data with the decoded data + if !bytes.Equal(originalData, decodedData) { + t.Fatalf( + "Iteration %d: original and decoded data do not match\nOriginal: %v\nDecoded: %v", + i, + originalData, + decodedData) + } + } +} diff --git a/api/clients/codecs/fft.go b/encoding/utils/codec/fft.go similarity index 99% rename from api/clients/codecs/fft.go rename to encoding/utils/codec/fft.go index a190d29d4e..d0e9328dc9 100644 --- a/api/clients/codecs/fft.go +++ b/encoding/utils/codec/fft.go @@ -1,4 +1,4 @@ -package codecs +package codec import ( "fmt" diff --git a/encoding/utils/codec/fft_test.go b/encoding/utils/codec/fft_test.go new file mode 100644 index 0000000000..8ee67a120b --- /dev/null +++ b/encoding/utils/codec/fft_test.go @@ -0,0 +1,39 @@ +package codec + +import ( + "bytes" + "testing" + + "github.com/Layr-Labs/eigenda/common/testutils/random" + "github.com/stretchr/testify/require" +) + +// TestFFT checks that data can be IFFTed and FFTed repeatedly, always getting back the original data +func TestFFT(t *testing.T) { + testRandom := random.NewTestRandom(t) + + // Number of test iterations + iterations := testRandom.Intn(100) + 1 + + for i := 0; i < iterations; i++ { + originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) // ensure it's not length 0 + + encodedData := EncodePayload(originalData) + coeffPoly, err := IFFT(encodedData) + require.NoError(t, err) + + evalPoly, err := FFT(coeffPoly) + require.NoError(t, err) + + // Decode the encoded data + decodedData, err := DecodePayload(evalPoly) + if err != nil { + t.Fatalf("Iteration %d: failed to decode blob: %v", i, err) + } + + // Compare the original data with the decoded data + if !bytes.Equal(originalData, decodedData) { + t.Fatalf("Iteration %d: original and decoded data do not match\nOriginal: %v\nDecoded: %v", i, originalData, decodedData) + } + } +} diff --git a/encoding/utils/codec/polynomial_form.go b/encoding/utils/codec/polynomial_form.go new file mode 100644 index 0000000000..9f265ccf3f --- /dev/null +++ b/encoding/utils/codec/polynomial_form.go @@ -0,0 +1,12 @@ +package codec + +// PolynomialForm is an enum that represents the different ways that a blob polynomial may be represented +type PolynomialForm uint + +const ( + // Eval is short for "evaluation form". The field elements represent the evaluation at the polynomial's expanded + // roots of unity + Eval PolynomialForm = iota + // Coeff is short for "coefficient form". The field elements represent the coefficients of the polynomial + Coeff +)