Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add GET header to return raw blob #222

Closed
wants to merge 1 commit into from
Closed

feat: add GET header to return raw blob #222

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions common/store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ type GeneratedKeyStore interface {
Store
// Get retrieves the given key if it's present in the key-value data store.
Get(ctx context.Context, key []byte) ([]byte, error)
// Get retrieves the given key if it's present in the key-value data store as the raw blob
GetRaw(ctx context.Context, key []byte) ([]byte, error)
// Put inserts the given value into the key-value data store.
Put(ctx context.Context, value []byte) (key []byte, err error)
}
Expand All @@ -77,6 +79,8 @@ type PrecomputedKeyStore interface {
Store
// Get retrieves the given key if it's present in the key-value data store.
Get(ctx context.Context, key []byte) ([]byte, error)
// Get retrieves the given key if it's present in the key-value data store.
GetRaw(ctx context.Context, key []byte) ([]byte, error)
// Put inserts the given value into the key-value data store.
Put(ctx context.Context, key []byte, value []byte) error
}
9 changes: 9 additions & 0 deletions mocks/manager.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 27 additions & 1 deletion server/handlers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,12 @@
return fmt.Errorf("failed to decode commitment %s: %w", rawCommitmentHex, err)
}

return svr.handleGetShared(r.Context(), w, commitment, commitmentMeta)
if r.Header.Get("raw") != "" {
fmt.Println("get raw", r.Header.Get("raw"))

Check failure on line 94 in server/handlers.go

View workflow job for this annotation

GitHub Actions / Linter 

use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
return svr.handleGetRawShared(r.Context(), w, commitment, commitmentMeta)
} else {

Check failure on line 96 in server/handlers.go

View workflow job for this annotation

GitHub Actions / Linter 

indent-error-flow: if block ends with a return statement, so drop this else and outdent its block (revive)
return svr.handleGetShared(r.Context(), w, commitment, commitmentMeta)
}
}

func (svr *Server) handleGetShared(ctx context.Context, w http.ResponseWriter, comm []byte, meta commitments.CommitmentMeta) error {
Expand All @@ -114,6 +119,27 @@
return nil
}

func (svr *Server) handleGetRawShared(ctx context.Context, w http.ResponseWriter, comm []byte, meta commitments.CommitmentMeta) error {
commitmentHex := hex.EncodeToString(comm)
svr.log.Info("Processing GetRaw request", "commitment", commitmentHex, "commitmentMeta", meta)
input, err := svr.sm.GetRaw(ctx, comm, meta.Mode)
if err != nil {
err = MetaError{
Err: fmt.Errorf("get request failed with commitment %v: %w", commitmentHex, err),
Meta: meta,
}
if errors.Is(err, ErrNotFound) {
http.Error(w, err.Error(), http.StatusNotFound)
} else {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
return err
}

svr.writeResponse(w, input)
return nil
}

// =================================================================================================
// POST ROUTES
// =================================================================================================
Expand Down
16 changes: 16 additions & 0 deletions store/generated_key/eigenda/eigenda.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,22 @@ func (e Store) Get(ctx context.Context, key []byte) ([]byte, error) {
return decodedBlob, nil
}

// ToDo still not correct impl
func (e Store) GetRaw(ctx context.Context, key []byte) ([]byte, error) {
Comment on lines +70 to +71
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm confused, what is this supposed to do? It's doing the exact same thing as Get() right now.

var cert verify.Certificate
err := rlp.DecodeBytes(key, &cert)
if err != nil {
return nil, fmt.Errorf("failed to decode DA cert to RLP format: %w", err)
}

decodedBlob, err := e.client.GetBlob(ctx, cert.BlobVerificationProof.BatchMetadata.BatchHeaderHash, cert.BlobVerificationProof.BlobIndex)
if err != nil {
return nil, fmt.Errorf("EigenDA client failed to retrieve decoded blob: %w", err)
}

return decodedBlob, nil
}

// Put disperses a blob for some pre-image and returns the associated RLP encoded certificate commit.
func (e Store) Put(ctx context.Context, value []byte) ([]byte, error) {
encodedBlob, err := e.client.GetCodec().EncodeBlob(value)
Expand Down
45 changes: 45 additions & 0 deletions store/generated_key/memstore/memstore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,51 @@
return e.codec.DecodeBlob(encodedBlob)
}

// Get fetches a value from the store.
func (e *MemStore) GetRaw(_ context.Context, commit []byte) ([]byte, error) {
fmt.Println("enter")

Check failure on line 136 in store/generated_key/memstore/memstore.go

View workflow job for this annotation

GitHub Actions / Linter 

use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
time.Sleep(e.config.GetLatency)
e.reads++
e.RLock()
defer e.RUnlock()

var cert verify.Certificate
err := rlp.DecodeBytes(commit, &cert)
if err != nil {
return nil, fmt.Errorf("failed to decode DA cert to RLP format: %w", err)
}

var encodedBlob []byte
var exists bool
if encodedBlob, exists = e.store[string(cert.BlobVerificationProof.InclusionProof)]; !exists {
return nil, fmt.Errorf("commitment key not found")
}

fmt.Println("encodedBlob", encodedBlob)

Check failure on line 154 in store/generated_key/memstore/memstore.go

View workflow job for this annotation

GitHub Actions / Linter 

use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)

// Don't need to do this really since it's a mock store
err = e.verifier.VerifyCommitment(cert.BlobHeader.Commitment, encodedBlob)
if err != nil {
return nil, err
}

decoded, err := e.codec.DecodeBlob(encodedBlob)
if err != nil {
return nil, err
}

fmt.Println("decoded", decoded)

Check failure on line 167 in store/generated_key/memstore/memstore.go

View workflow job for this annotation

GitHub Actions / Linter 

use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)

tmpCodec := codecs.NewNoIFFTCodec(codecs.NewDefaultBlobCodec())
codeced, err := tmpCodec.EncodeBlob(decoded)
if err != nil {
return nil, err
}

fmt.Println("GetRaw", codeced)

Check failure on line 175 in store/generated_key/memstore/memstore.go

View workflow job for this annotation

GitHub Actions / Linter 

use of `fmt.Println` forbidden by pattern `^(fmt\.Print(|f|ln)|print|println)$` (forbidigo)
return codeced, nil
}

// Put inserts a value into the store.
func (e *MemStore) Put(_ context.Context, value []byte) ([]byte, error) {
time.Sleep(e.config.PutLatency)
Expand Down
69 changes: 69 additions & 0 deletions store/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
// IManager ... read/write interface
type IManager interface {
Get(ctx context.Context, key []byte, cm commitments.CommitmentMode) ([]byte, error)
GetRaw(ctx context.Context, key []byte, cm commitments.CommitmentMode) ([]byte, error)
Put(ctx context.Context, cm commitments.CommitmentMode, key, value []byte) ([]byte, error)
}

Expand Down Expand Up @@ -108,6 +109,74 @@
}
}

func (m *Manager) GetRaw(ctx context.Context, key []byte, cm commitments.CommitmentMode) ([]byte, error) {
switch cm {
case commitments.OptimismKeccak:

if m.s3 == nil {
return nil, errors.New("expected S3 backend for OP keccak256 commitment type, but none configured")
}

// 1 - read blob from S3 backend
m.log.Debug("Retrieving data from S3 backend")
value, err := m.s3.GetRaw(ctx, key)
if err != nil {
return nil, err
}

// 2 - verify blob hash against commitment key digest
err = m.s3.Verify(ctx, key, value)
if err != nil {
return nil, err
}
return value, nil

case commitments.Standard, commitments.OptimismGeneric:
if m.eigenda == nil {
return nil, errors.New("expected EigenDA backend for DA commitment type, but none configured")
}

// 1 - read blob from cache if enabled
if m.secondary.CachingEnabled() {
m.log.Debug("Retrieving data from cached backends")
data, err := m.secondary.MultiSourceRead(ctx, key, false, m.eigenda.Verify)
if err == nil {
return data, nil
}

m.log.Warn("Failed to read from cache targets", "err", err)
}

// 2 - read blob from EigenDA
rawData, err := m.eigenda.GetRaw(ctx, key)

Check failure on line 151 in store/manager.go

View workflow job for this annotation

GitHub Actions / Linter 

ineffectual assignment to err (ineffassign)
data, err := m.eigenda.GetRaw(ctx, key)
if err == nil {
// verify
err = m.eigenda.Verify(ctx, key, data)
if err != nil {
return nil, err
}
return rawData, nil
}

// 3 - read blob from fallbacks if enabled and data is non-retrievable from EigenDA
if m.secondary.FallbackEnabled() {
data, err = m.secondary.MultiSourceRead(ctx, key, true, m.eigenda.Verify)

Check failure on line 164 in store/manager.go

View workflow job for this annotation

GitHub Actions / Linter 

ineffectual assignment to data (ineffassign)
if err != nil {
m.log.Error("Failed to read from fallback targets", "err", err)
return nil, err
}
} else {
return nil, err
}

return rawData, err

default:
return nil, errors.New("could not determine which storage backend to route to based on unknown commitment mode")
}
}

// Put ... inserts a value into a storage backend based on the commitment mode
func (m *Manager) Put(ctx context.Context, cm commitments.CommitmentMode, key, value []byte) ([]byte, error) {
var commit []byte
Expand Down
4 changes: 4 additions & 0 deletions store/precomputed_key/redis/redis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ func (r *Store) Get(ctx context.Context, key []byte) ([]byte, error) {
return []byte(value), nil
}

func (r *Store) GetRaw(ctx context.Context, key []byte) ([]byte, error) {
return r.Get(ctx, key)
}

// Put ... inserts a value into the Redis store
func (r *Store) Put(ctx context.Context, key []byte, value []byte) error {
return r.client.Set(ctx, string(key), string(value), r.eviction).Err()
Expand Down
4 changes: 4 additions & 0 deletions store/precomputed_key/s3/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,10 @@ func (s *Store) Get(ctx context.Context, key []byte) ([]byte, error) {
return data, nil
}

func (s *Store) GetRaw(ctx context.Context, key []byte) ([]byte, error) {
return s.Get(ctx, key)
}

func (s *Store) Put(ctx context.Context, key []byte, value []byte) error {
_, err := s.client.PutObject(ctx, s.cfg.Bucket, path.Join(s.cfg.Path, hex.EncodeToString(key)), bytes.NewReader(value), int64(len(value)), s.putObjectOptions)
if err != nil {
Expand Down
Loading