From 347157858a0cdb5ae92827c28018c6d23809205e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 07:17:23 +0000 Subject: [PATCH] poc --- common/store.go | 4 ++ mocks/manager.go | 9 ++++ server/handlers.go | 28 +++++++++- store/generated_key/eigenda/eigenda.go | 16 ++++++ store/generated_key/memstore/memstore.go | 45 ++++++++++++++++ store/manager.go | 69 ++++++++++++++++++++++++ store/precomputed_key/redis/redis.go | 4 ++ store/precomputed_key/s3/s3.go | 4 ++ 8 files changed, 178 insertions(+), 1 deletion(-) diff --git a/common/store.go b/common/store.go index 2d508538..7be2966e 100644 --- a/common/store.go +++ b/common/store.go @@ -69,6 +69,8 @@ type GeneratedKeyStore interface { Store // Get retrieves the given key if it's present in the key-value data store. Get(ctx context.Context, key []byte) ([]byte, error) + // Get retrieves the given key if it's present in the key-value data store as the raw blob + GetRaw(ctx context.Context, key []byte) ([]byte, error) // Put inserts the given value into the key-value data store. Put(ctx context.Context, value []byte) (key []byte, err error) } @@ -77,6 +79,8 @@ type PrecomputedKeyStore interface { Store // Get retrieves the given key if it's present in the key-value data store. Get(ctx context.Context, key []byte) ([]byte, error) + // Get retrieves the given key if it's present in the key-value data store. + GetRaw(ctx context.Context, key []byte) ([]byte, error) // Put inserts the given value into the key-value data store. Put(ctx context.Context, key []byte, value []byte) error } diff --git a/mocks/manager.go b/mocks/manager.go index 8aeb3352..d5b79d7d 100644 --- a/mocks/manager.go +++ b/mocks/manager.go @@ -44,6 +44,15 @@ func (m *MockIManager) Get(arg0 context.Context, arg1 []byte, arg2 commitments.C return ret0, ret1 } +// Get mocks base method. +func (m *MockIManager) GetRaw(arg0 context.Context, arg1 []byte, arg2 commitments.CommitmentMode) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRaw", arg0, arg1, arg2) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + // Get indicates an expected call of Get. func (mr *MockIManagerMockRecorder) Get(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() diff --git a/server/handlers.go b/server/handlers.go index d1acb29c..d0957761 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -90,7 +90,12 @@ func (svr *Server) handleGetOPGenericCommitment(w http.ResponseWriter, r *http.R return fmt.Errorf("failed to decode commitment %s: %w", rawCommitmentHex, err) } - return svr.handleGetShared(r.Context(), w, commitment, commitmentMeta) + if r.Header.Get("raw") != "" { + fmt.Println("get raw", r.Header.Get("raw")) + return svr.handleGetRawShared(r.Context(), w, commitment, commitmentMeta) + } else { + return svr.handleGetShared(r.Context(), w, commitment, commitmentMeta) + } } func (svr *Server) handleGetShared(ctx context.Context, w http.ResponseWriter, comm []byte, meta commitments.CommitmentMeta) error { @@ -114,6 +119,27 @@ func (svr *Server) handleGetShared(ctx context.Context, w http.ResponseWriter, c return nil } +func (svr *Server) handleGetRawShared(ctx context.Context, w http.ResponseWriter, comm []byte, meta commitments.CommitmentMeta) error { + commitmentHex := hex.EncodeToString(comm) + svr.log.Info("Processing GetRaw request", "commitment", commitmentHex, "commitmentMeta", meta) + input, err := svr.sm.GetRaw(ctx, comm, meta.Mode) + if err != nil { + err = MetaError{ + Err: fmt.Errorf("get request failed with commitment %v: %w", commitmentHex, err), + Meta: meta, + } + if errors.Is(err, ErrNotFound) { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return err + } + + svr.writeResponse(w, input) + return nil +} + // ================================================================================================= // POST ROUTES // ================================================================================================= diff --git a/store/generated_key/eigenda/eigenda.go b/store/generated_key/eigenda/eigenda.go index b67ce1d2..fcf6117c 100644 --- a/store/generated_key/eigenda/eigenda.go +++ b/store/generated_key/eigenda/eigenda.go @@ -67,6 +67,22 @@ func (e Store) Get(ctx context.Context, key []byte) ([]byte, error) { return decodedBlob, nil } +// ToDo still not correct impl +func (e Store) GetRaw(ctx context.Context, key []byte) ([]byte, error) { + var cert verify.Certificate + err := rlp.DecodeBytes(key, &cert) + if err != nil { + return nil, fmt.Errorf("failed to decode DA cert to RLP format: %w", err) + } + + decodedBlob, err := e.client.GetBlob(ctx, cert.BlobVerificationProof.BatchMetadata.BatchHeaderHash, cert.BlobVerificationProof.BlobIndex) + if err != nil { + return nil, fmt.Errorf("EigenDA client failed to retrieve decoded blob: %w", err) + } + + return decodedBlob, nil +} + // Put disperses a blob for some pre-image and returns the associated RLP encoded certificate commit. func (e Store) Put(ctx context.Context, value []byte) ([]byte, error) { encodedBlob, err := e.client.GetCodec().EncodeBlob(value) diff --git a/store/generated_key/memstore/memstore.go b/store/generated_key/memstore/memstore.go index 3707029d..5df33d5b 100644 --- a/store/generated_key/memstore/memstore.go +++ b/store/generated_key/memstore/memstore.go @@ -131,6 +131,51 @@ func (e *MemStore) Get(_ context.Context, commit []byte) ([]byte, error) { return e.codec.DecodeBlob(encodedBlob) } +// Get fetches a value from the store. +func (e *MemStore) GetRaw(_ context.Context, commit []byte) ([]byte, error) { + fmt.Println("enter") + time.Sleep(e.config.GetLatency) + e.reads++ + e.RLock() + defer e.RUnlock() + + var cert verify.Certificate + err := rlp.DecodeBytes(commit, &cert) + if err != nil { + return nil, fmt.Errorf("failed to decode DA cert to RLP format: %w", err) + } + + var encodedBlob []byte + var exists bool + if encodedBlob, exists = e.store[string(cert.BlobVerificationProof.InclusionProof)]; !exists { + return nil, fmt.Errorf("commitment key not found") + } + + fmt.Println("encodedBlob", encodedBlob) + + // Don't need to do this really since it's a mock store + err = e.verifier.VerifyCommitment(cert.BlobHeader.Commitment, encodedBlob) + if err != nil { + return nil, err + } + + decoded, err := e.codec.DecodeBlob(encodedBlob) + if err != nil { + return nil, err + } + + fmt.Println("decoded", decoded) + + tmpCodec := codecs.NewNoIFFTCodec(codecs.NewDefaultBlobCodec()) + codeced, err := tmpCodec.EncodeBlob(decoded) + if err != nil { + return nil, err + } + + fmt.Println("GetRaw", codeced) + return codeced, nil +} + // Put inserts a value into the store. func (e *MemStore) Put(_ context.Context, value []byte) ([]byte, error) { time.Sleep(e.config.PutLatency) diff --git a/store/manager.go b/store/manager.go index 93721d89..509d69a6 100644 --- a/store/manager.go +++ b/store/manager.go @@ -15,6 +15,7 @@ import ( // IManager ... read/write interface type IManager interface { Get(ctx context.Context, key []byte, cm commitments.CommitmentMode) ([]byte, error) + GetRaw(ctx context.Context, key []byte, cm commitments.CommitmentMode) ([]byte, error) Put(ctx context.Context, cm commitments.CommitmentMode, key, value []byte) ([]byte, error) } @@ -108,6 +109,74 @@ func (m *Manager) Get(ctx context.Context, key []byte, cm commitments.Commitment } } +func (m *Manager) GetRaw(ctx context.Context, key []byte, cm commitments.CommitmentMode) ([]byte, error) { + switch cm { + case commitments.OptimismKeccak: + + if m.s3 == nil { + return nil, errors.New("expected S3 backend for OP keccak256 commitment type, but none configured") + } + + // 1 - read blob from S3 backend + m.log.Debug("Retrieving data from S3 backend") + value, err := m.s3.GetRaw(ctx, key) + if err != nil { + return nil, err + } + + // 2 - verify blob hash against commitment key digest + err = m.s3.Verify(ctx, key, value) + if err != nil { + return nil, err + } + return value, nil + + case commitments.Standard, commitments.OptimismGeneric: + if m.eigenda == nil { + return nil, errors.New("expected EigenDA backend for DA commitment type, but none configured") + } + + // 1 - read blob from cache if enabled + if m.secondary.CachingEnabled() { + m.log.Debug("Retrieving data from cached backends") + data, err := m.secondary.MultiSourceRead(ctx, key, false, m.eigenda.Verify) + if err == nil { + return data, nil + } + + m.log.Warn("Failed to read from cache targets", "err", err) + } + + // 2 - read blob from EigenDA + rawData, err := m.eigenda.GetRaw(ctx, key) + data, err := m.eigenda.GetRaw(ctx, key) + if err == nil { + // verify + err = m.eigenda.Verify(ctx, key, data) + if err != nil { + return nil, err + } + return rawData, nil + } + + // 3 - read blob from fallbacks if enabled and data is non-retrievable from EigenDA + if m.secondary.FallbackEnabled() { + data, err = m.secondary.MultiSourceRead(ctx, key, true, m.eigenda.Verify) + if err != nil { + m.log.Error("Failed to read from fallback targets", "err", err) + return nil, err + } + } else { + return nil, err + } + + return rawData, err + + default: + return nil, errors.New("could not determine which storage backend to route to based on unknown commitment mode") + } +} + // Put ... inserts a value into a storage backend based on the commitment mode func (m *Manager) Put(ctx context.Context, cm commitments.CommitmentMode, key, value []byte) ([]byte, error) { var commit []byte diff --git a/store/precomputed_key/redis/redis.go b/store/precomputed_key/redis/redis.go index 4b30853e..78bf06a5 100644 --- a/store/precomputed_key/redis/redis.go +++ b/store/precomputed_key/redis/redis.go @@ -79,6 +79,10 @@ func (r *Store) Get(ctx context.Context, key []byte) ([]byte, error) { return []byte(value), nil } +func (r *Store) GetRaw(ctx context.Context, key []byte) ([]byte, error) { + return r.Get(ctx, key) +} + // Put ... inserts a value into the Redis store func (r *Store) Put(ctx context.Context, key []byte, value []byte) error { return r.client.Set(ctx, string(key), string(value), r.eviction).Err() diff --git a/store/precomputed_key/s3/s3.go b/store/precomputed_key/s3/s3.go index fc468192..66588520 100644 --- a/store/precomputed_key/s3/s3.go +++ b/store/precomputed_key/s3/s3.go @@ -112,6 +112,10 @@ func (s *Store) Get(ctx context.Context, key []byte) ([]byte, error) { return data, nil } +func (s *Store) GetRaw(ctx context.Context, key []byte) ([]byte, error) { + return s.Get(ctx, key) +} + func (s *Store) Put(ctx context.Context, key []byte, value []byte) error { _, err := s.client.PutObject(ctx, s.cfg.Bucket, path.Join(s.cfg.Path, hex.EncodeToString(key)), bytes.NewReader(value), int64(len(value)), s.putObjectOptions) if err != nil {