From 062f4999c5133d46a580dad96d13e47b78c031a3 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Tue, 30 Jul 2024 14:02:38 +0200 Subject: [PATCH] feat(shwap): Shwap add readFrom Writeto methods to shwap ids (#3605) --- share/shwap/eds_id.go | 29 ++++++++++++++++++ share/shwap/eds_id_test.go | 18 ++++++++++++ share/shwap/namespace_data.go | 36 +++++++++++++++++++---- share/shwap/namespace_data_id.go | 29 ++++++++++++++++++ share/shwap/namespace_data_id_test.go | 20 +++++++++++++ share/shwap/row_id.go | 27 +++++++++++++++++ share/shwap/row_id_test.go | 20 +++++++++++++ share/shwap/row_namespace_data_id.go | 29 ++++++++++++++++++ share/shwap/row_namespace_data_id_test.go | 21 +++++++++++++ share/shwap/sample_id.go | 29 ++++++++++++++++++ share/shwap/sample_id_test.go | 20 +++++++++++++ 11 files changed, 272 insertions(+), 6 deletions(-) diff --git a/share/shwap/eds_id.go b/share/shwap/eds_id.go index 5ef37744ef..011a9be412 100644 --- a/share/shwap/eds_id.go +++ b/share/shwap/eds_id.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "errors" "fmt" + "io" ) // EdsIDSize defines the byte size of the EdsID. @@ -45,6 +46,24 @@ func EdsIDFromBinary(data []byte) (EdsID, error) { return eid, nil } +// ReadFrom reads the binary form of EdsID from the provided reader. +func (eid *EdsID) ReadFrom(r io.Reader) (int64, error) { + data := make([]byte, EdsIDSize) + n, err := r.Read(data) + if err != nil { + return int64(n), err + } + if n != EdsIDSize { + return int64(n), fmt.Errorf("EdsID: expected %d bytes, got %d", EdsIDSize, n) + } + id, err := EdsIDFromBinary(data) + if err != nil { + return int64(n), fmt.Errorf("EdsIDFromBinary: %w", err) + } + *eid = id + return int64(n), nil +} + // MarshalBinary encodes an EdsID into its binary form, primarily for storage or network // transmission. func (eid EdsID) MarshalBinary() ([]byte, error) { @@ -52,6 +71,16 @@ func (eid EdsID) MarshalBinary() ([]byte, error) { return eid.appendTo(data), nil } +// WriteTo writes the binary form of EdsID to the provided writer. +func (eid EdsID) WriteTo(w io.Writer) (int64, error) { + data, err := eid.MarshalBinary() + if err != nil { + return 0, err + } + n, err := w.Write(data) + return int64(n), err +} + // Validate checks the integrity of an EdsID's fields against the provided Root. // It ensures that the EdsID is not constructed with a zero Height and that the root is not nil. func (eid EdsID) Validate() error { diff --git a/share/shwap/eds_id_test.go b/share/shwap/eds_id_test.go index 553fd5b120..0b5cff1b90 100644 --- a/share/shwap/eds_id_test.go +++ b/share/shwap/eds_id_test.go @@ -1,6 +1,7 @@ package shwap import ( + "bytes" "testing" "github.com/stretchr/testify/assert" @@ -21,3 +22,20 @@ func TestEdsID(t *testing.T) { err = idOut.Validate() require.NoError(t, err) } + +func TestEdsIDReaderWriter(t *testing.T) { + id, err := NewEdsID(2) + require.NoError(t, err) + + buf := bytes.NewBuffer(nil) + n, err := id.WriteTo(buf) + require.NoError(t, err) + require.Equal(t, int64(EdsIDSize), n) + + eidOut := EdsID{} + n, err = eidOut.ReadFrom(buf) + require.NoError(t, err) + require.Equal(t, int64(EdsIDSize), n) + + require.EqualValues(t, id, eidOut) +} diff --git a/share/shwap/namespace_data.go b/share/shwap/namespace_data.go index 6ddc6a8798..62b157e783 100644 --- a/share/shwap/namespace_data.go +++ b/share/shwap/namespace_data.go @@ -1,9 +1,14 @@ package shwap import ( + "errors" "fmt" + "io" + + "github.com/celestiaorg/go-libp2p-messenger/serde" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/shwap/pb" ) // NamespacedData stores collections of RowNamespaceData, each representing shares and their proofs @@ -11,25 +16,44 @@ import ( type NamespacedData []RowNamespaceData // Flatten combines all shares from all rows within the namespace into a single slice. -func (ns NamespacedData) Flatten() []share.Share { +func (nd NamespacedData) Flatten() []share.Share { var shares []share.Share - for _, row := range ns { + for _, row := range nd { shares = append(shares, row.Shares...) } return shares } // Validate checks the integrity of the NamespacedData against a provided root and namespace. -func (ns NamespacedData) Validate(root *share.AxisRoots, namespace share.Namespace) error { +func (nd NamespacedData) Validate(root *share.AxisRoots, namespace share.Namespace) error { rowIdxs := share.RowsWithNamespace(root, namespace) - if len(rowIdxs) != len(ns) { - return fmt.Errorf("expected %d rows, found %d rows", len(rowIdxs), len(ns)) + if len(rowIdxs) != len(nd) { + return fmt.Errorf("expected %d rows, found %d rows", len(rowIdxs), len(nd)) } - for i, row := range ns { + for i, row := range nd { if err := row.Validate(root, namespace, rowIdxs[i]); err != nil { return fmt.Errorf("validating row: %w", err) } } return nil } + +// ReadFrom reads the binary form of NamespacedData from the provided reader. +func (nd *NamespacedData) ReadFrom(reader io.Reader) error { + var data []RowNamespaceData + for { + var pbRow pb.RowNamespaceData + _, err := serde.Read(reader, &pbRow) + if err != nil { + if errors.Is(err, io.EOF) { + // all rows have been read + *nd = data + return nil + } + return err + } + row := RowNamespaceDataFromProto(&pbRow) + data = append(data, row) + } +} diff --git a/share/shwap/namespace_data_id.go b/share/shwap/namespace_data_id.go index 7bc7b49bb7..d22c1879bc 100644 --- a/share/shwap/namespace_data_id.go +++ b/share/shwap/namespace_data_id.go @@ -2,6 +2,7 @@ package shwap import ( "fmt" + "io" "github.com/celestiaorg/celestia-node/share" ) @@ -58,6 +59,24 @@ func NamespaceDataIDFromBinary(data []byte) (NamespaceDataID, error) { return ndid, nil } +// ReadFrom reads the binary form of NamespaceDataID from the provided reader. +func (ndid *NamespaceDataID) ReadFrom(r io.Reader) (int64, error) { + data := make([]byte, NamespaceDataIDSize) + n, err := r.Read(data) + if err != nil { + return int64(n), err + } + if n != NamespaceDataIDSize { + return int64(n), fmt.Errorf("NamespaceDataID: expected %d bytes, got %d", NamespaceDataIDSize, n) + } + id, err := NamespaceDataIDFromBinary(data) + if err != nil { + return int64(n), fmt.Errorf("NamespaceDataIDFromBinary: %w", err) + } + *ndid = id + return int64(n), nil +} + // MarshalBinary encodes NamespaceDataID into binary form. // NOTE: Proto is avoided because // * Its size is not deterministic which is required for IPLD. @@ -67,6 +86,16 @@ func (ndid NamespaceDataID) MarshalBinary() ([]byte, error) { return ndid.appendTo(data), nil } +// WriteTo writes the binary form of NamespaceDataID to the provided writer. +func (ndid NamespaceDataID) WriteTo(w io.Writer) (int64, error) { + data, err := ndid.MarshalBinary() + if err != nil { + return 0, err + } + n, err := w.Write(data) + return int64(n), err +} + // Validate checks if the NamespaceDataID is valid. It checks the validity of the EdsID and the // DataNamespace. func (ndid NamespaceDataID) Validate() error { diff --git a/share/shwap/namespace_data_id_test.go b/share/shwap/namespace_data_id_test.go index 96f8b63bb3..da848b50bb 100644 --- a/share/shwap/namespace_data_id_test.go +++ b/share/shwap/namespace_data_id_test.go @@ -1,6 +1,7 @@ package shwap import ( + "bytes" "testing" "github.com/stretchr/testify/assert" @@ -25,3 +26,22 @@ func TestNamespaceDataID(t *testing.T) { err = sidOut.Validate() require.NoError(t, err) } + +func TestNamespaceDataIDReaderWriter(t *testing.T) { + ns := sharetest.RandV0Namespace() + + id, err := NewNamespaceDataID(1, ns) + require.NoError(t, err) + + buf := bytes.NewBuffer(nil) + n, err := id.WriteTo(buf) + require.NoError(t, err) + require.Equal(t, int64(NamespaceDataIDSize), n) + + ndidOut := NamespaceDataID{} + n, err = ndidOut.ReadFrom(buf) + require.NoError(t, err) + require.Equal(t, int64(NamespaceDataIDSize), n) + + require.EqualValues(t, id, ndidOut) +} diff --git a/share/shwap/row_id.go b/share/shwap/row_id.go index fd6ae49f5e..0c4a9233df 100644 --- a/share/shwap/row_id.go +++ b/share/shwap/row_id.go @@ -3,6 +3,7 @@ package shwap import ( "encoding/binary" "fmt" + "io" ) // RowIDSize defines the size in bytes of RowID, consisting of the size of EdsID and 2 bytes for @@ -55,12 +56,38 @@ func RowIDFromBinary(data []byte) (RowID, error) { return rid, nil } +func (rid *RowID) ReadFrom(r io.Reader) (int64, error) { + data := make([]byte, RowIDSize) + n, err := r.Read(data) + if err != nil { + return int64(n), err + } + if n != RowIDSize { + return int64(n), fmt.Errorf("RowID: expected %d bytes, got %d", RowIDSize, n) + } + id, err := RowIDFromBinary(data) + if err != nil { + return int64(n), fmt.Errorf("RowIDFromBinary: %w", err) + } + *rid = id + return int64(n), nil +} + // MarshalBinary encodes the RowID into a binary form for storage or network transmission. func (rid RowID) MarshalBinary() ([]byte, error) { data := make([]byte, 0, RowIDSize) return rid.appendTo(data), nil } +func (rid RowID) WriteTo(w io.Writer) (int64, error) { + data, err := rid.MarshalBinary() + if err != nil { + return 0, err + } + n, err := w.Write(data) + return int64(n), err +} + // Verify validates the RowID fields and verifies that RowIndex is within the bounds of // the square size func (rid RowID) Verify(edsSize int) error { diff --git a/share/shwap/row_id_test.go b/share/shwap/row_id_test.go index 1f8bf873f3..cb17ba21c2 100644 --- a/share/shwap/row_id_test.go +++ b/share/shwap/row_id_test.go @@ -1,6 +1,7 @@ package shwap import ( + "bytes" "testing" "github.com/stretchr/testify/assert" @@ -23,3 +24,22 @@ func TestRowID(t *testing.T) { err = idOut.Verify(edsSize) require.NoError(t, err) } + +func TestRowIDReaderWriter(t *testing.T) { + edsSize := 4 + + id, err := NewRowID(2, 1, edsSize) + require.NoError(t, err) + + buf := bytes.NewBuffer(nil) + n, err := id.WriteTo(buf) + require.NoError(t, err) + require.Equal(t, int64(RowIDSize), n) + + ridOut := RowID{} + n, err = ridOut.ReadFrom(buf) + require.NoError(t, err) + require.Equal(t, int64(RowIDSize), n) + + require.EqualValues(t, id, ridOut) +} diff --git a/share/shwap/row_namespace_data_id.go b/share/shwap/row_namespace_data_id.go index 4dfdf25090..041d83a66b 100644 --- a/share/shwap/row_namespace_data_id.go +++ b/share/shwap/row_namespace_data_id.go @@ -2,6 +2,7 @@ package shwap import ( "fmt" + "io" "github.com/celestiaorg/celestia-node/share" ) @@ -65,6 +66,24 @@ func RowNamespaceDataIDFromBinary(data []byte) (RowNamespaceDataID, error) { return rndid, nil } +// ReadFrom reads the binary form of RowNamespaceDataID from the provided reader. +func (s *RowNamespaceDataID) ReadFrom(r io.Reader) (int64, error) { + data := make([]byte, RowNamespaceDataIDSize) + n, err := r.Read(data) + if err != nil { + return int64(n), err + } + if n != RowNamespaceDataIDSize { + return int64(n), fmt.Errorf("RowNamespaceDataID: expected %d bytes, got %d", RowNamespaceDataIDSize, n) + } + id, err := RowNamespaceDataIDFromBinary(data) + if err != nil { + return int64(n), fmt.Errorf("RowNamespaceDataIDFromBinary: %w", err) + } + *s = id + return int64(n), nil +} + // MarshalBinary encodes RowNamespaceDataID into binary form. // NOTE: Proto is avoided because // * Its size is not deterministic which is required for IPLD. @@ -74,6 +93,16 @@ func (s RowNamespaceDataID) MarshalBinary() ([]byte, error) { return s.appendTo(data), nil } +// WriteTo writes the binary form of RowNamespaceDataID to the provided writer. +func (s RowNamespaceDataID) WriteTo(w io.Writer) (int64, error) { + data, err := s.MarshalBinary() + if err != nil { + return 0, err + } + n, err := w.Write(data) + return int64(n), err +} + // Verify validates the RowNamespaceDataID and verifies the embedded RowID. func (s RowNamespaceDataID) Verify(edsSize int) error { if err := s.RowID.Verify(edsSize); err != nil { diff --git a/share/shwap/row_namespace_data_id_test.go b/share/shwap/row_namespace_data_id_test.go index 72abbac512..ab382d18e7 100644 --- a/share/shwap/row_namespace_data_id_test.go +++ b/share/shwap/row_namespace_data_id_test.go @@ -1,6 +1,7 @@ package shwap import ( + "bytes" "testing" "github.com/stretchr/testify/assert" @@ -26,3 +27,23 @@ func TestRowNamespaceDataID(t *testing.T) { err = sidOut.Verify(edsSize) require.NoError(t, err) } + +func TestRowNamespaceDataIDReaderWriter(t *testing.T) { + edsSize := 4 + ns := sharetest.RandV0Namespace() + + id, err := NewRowNamespaceDataID(1, 1, ns, edsSize) + require.NoError(t, err) + + buf := bytes.NewBuffer(nil) + n, err := id.WriteTo(buf) + require.NoError(t, err) + require.Equal(t, int64(RowNamespaceDataIDSize), n) + + rndidOut := RowNamespaceDataID{} + n, err = rndidOut.ReadFrom(buf) + require.NoError(t, err) + require.Equal(t, int64(RowNamespaceDataIDSize), n) + + require.EqualValues(t, id, rndidOut) +} diff --git a/share/shwap/sample_id.go b/share/shwap/sample_id.go index 7bc885f5d6..818f4a51e6 100644 --- a/share/shwap/sample_id.go +++ b/share/shwap/sample_id.go @@ -3,6 +3,7 @@ package shwap import ( "encoding/binary" "fmt" + "io" ) // SampleIDSize defines the size of the SampleID in bytes, combining RowID size and 2 additional @@ -57,6 +58,24 @@ func SampleIDFromBinary(data []byte) (SampleID, error) { return sid, nil } +// ReadFrom reads the binary form of SampleID from the provided reader. +func (sid *SampleID) ReadFrom(r io.Reader) (int64, error) { + data := make([]byte, SampleIDSize) + n, err := r.Read(data) + if err != nil { + return int64(n), err + } + if n != SampleIDSize { + return int64(n), fmt.Errorf("SampleID: expected %d bytes, got %d", SampleIDSize, n) + } + id, err := SampleIDFromBinary(data) + if err != nil { + return int64(n), fmt.Errorf("SampleIDFromBinary: %w", err) + } + *sid = id + return int64(n), nil +} + // MarshalBinary encodes SampleID into binary form. // NOTE: Proto is avoided because // * Its size is not deterministic which is required for IPLD. @@ -66,6 +85,16 @@ func (sid SampleID) MarshalBinary() ([]byte, error) { return sid.appendTo(data), nil } +// WriteTo writes the binary form of SampleID to the provided writer. +func (sid SampleID) WriteTo(w io.Writer) (int64, error) { + data, err := sid.MarshalBinary() + if err != nil { + return 0, err + } + n, err := w.Write(data) + return int64(n), err +} + // Verify validates the SampleID and verifies that the ShareIndex is within the bounds of // the square size. func (sid SampleID) Verify(edsSize int) error { diff --git a/share/shwap/sample_id_test.go b/share/shwap/sample_id_test.go index 0a601df6c9..db56b82bb7 100644 --- a/share/shwap/sample_id_test.go +++ b/share/shwap/sample_id_test.go @@ -1,6 +1,7 @@ package shwap import ( + "bytes" "testing" "github.com/stretchr/testify/assert" @@ -23,3 +24,22 @@ func TestSampleID(t *testing.T) { err = idOut.Verify(edsSize) require.NoError(t, err) } + +func TestSampleIDReaderWriter(t *testing.T) { + edsSize := 4 + + id, err := NewSampleID(1, 1, 1, edsSize) + require.NoError(t, err) + + buf := bytes.NewBuffer(nil) + n, err := id.WriteTo(buf) + require.NoError(t, err) + require.Equal(t, int64(SampleIDSize), n) + + sidOut := SampleID{} + n, err = sidOut.ReadFrom(buf) + require.NoError(t, err) + require.Equal(t, int64(SampleIDSize), n) + + require.EqualValues(t, id, sidOut) +}