Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

internal/era: update block index format to be based on record offset #28959

Merged
merged 1 commit into from
Feb 9, 2024
Merged

internal/era: update block index format to be based on record offset #28959

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/utils/history_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ func TestHistoryImportAndExport(t *testing.T) {
for j := 0; it.Next(); j++ {
n := i*int(step) + j
if it.Error() != nil {
t.Fatalf("error reading block entry %d: %v", n, err)
t.Fatalf("error reading block entry %d: %v", n, it.Error())
}
block, receipts, err := it.BlockAndReceipts()
if err != nil {
Expand Down
24 changes: 10 additions & 14 deletions internal/era/builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ import (
// CompressedBody = { type: [0x04, 0x00], data: snappyFramed(rlp(body)) }
// CompressedReceipts = { type: [0x05, 0x00], data: snappyFramed(rlp(receipts)) }
// TotalDifficulty = { type: [0x06, 0x00], data: uint256(header.total_difficulty) }
// Accumulator = { type: [0x07, 0x00], data: accumulator-root }
// AccumulatorRoot = { type: [0x07, 0x00], data: accumulator-root }
// BlockIndex = { type: [0x32, 0x66], data: block-index }
//
// Accumulator is computed by constructing an SSZ list of header-records of length at most
Expand All @@ -64,8 +64,8 @@ import (
// block-index := starting-number | index | index | index ... | count
//
// starting-number is the first block number in the archive. Every index is a
// defined relative to index's location in the file. The total number of block
// entries in the file is recorded in count.
// defined relative to beginning of the record. The total number of block
// entries in the file is recorded with count.
//
// Due to the accumulator size limit of 8192, the maximum number of blocks in
// an Era1 batch is also 8192.
Expand Down Expand Up @@ -115,12 +115,14 @@ func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int)
func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error {
// Write Era1 version entry before first block.
if b.startNum == nil {
if err := writeVersion(b.w); err != nil {
n, err := b.w.Write(TypeVersion, nil)
if err != nil {
return err
}
n := number
b.startNum = &n
startNum := number
b.startNum = &startNum
b.startTd = new(big.Int).Sub(td, difficulty)
b.written += n
}
if len(b.indexes) >= MaxEra1Size {
return fmt.Errorf("exceeds maximum batch size of %d", MaxEra1Size)
Expand Down Expand Up @@ -169,7 +171,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err)
}
// Get beginning of index entry to calculate block relative offset.
base := int64(b.written + (3 * 8)) // skip e2store header (type, length) and start block
base := int64(b.written)

// Construct block index. Detailed format described in Builder
// documentation, but it is essentially encoded as:
Expand All @@ -186,7 +188,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
// relative offset, the corresponding block can be quickly read by
// performing a seek relative to the current position.
for i, offset := range b.indexes {
relative := int64(offset) - (base + int64(i)*8)
relative := int64(offset) - base
binary.LittleEndian.PutUint64(index[8+i*8:], uint64(relative))
}
binary.LittleEndian.PutUint64(index[8+count*8:], uint64(count))
Expand Down Expand Up @@ -220,9 +222,3 @@ func (b *Builder) snappyWrite(typ uint16, in []byte) error {
}
return nil
}

// writeVersion writes a version entry to e2store.
func writeVersion(w *e2store.Writer) error {
_, err := w.Write(TypeVersion, nil)
return err
}
15 changes: 8 additions & 7 deletions internal/era/era.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,20 +221,21 @@ func (e *Era) Count() uint64 {
// is the absolute block number desired.
func (e *Era) readOffset(n uint64) (int64, error) {
var (
firstIndex = -8 - int64(e.m.count)*8 // size of count - index entries
indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes
offOffset = e.m.length + firstIndex + indexOffset // offset of block offset
blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header
firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num
indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes
offOffset = firstIndex + indexOffset // offset of block offset
)
e.mu.Lock()
defer e.mu.Unlock()
clearBuffer(e.buf[:])
if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil {
return 0, err
}
// Since the block offset is relative from its location + size of index
// value (8), we need to add it to it's offset to get the block's
// absolute offset.
return offOffset + 8 + int64(binary.LittleEndian.Uint64(e.buf[:])), nil
// Since the block offset is relative from the start of the block index record
// we need to add the record offset to it's offset to get the block's absolute
// offset.
return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil
}

// newReader returns a snappy.Reader for the e2store entry value at off.
Expand Down
Loading