Skip to content

Commit

Permalink
Merge pull request #685 from 0xPolygonHermez/fractasy_hashdb_64
Browse files Browse the repository at this point in the history 
Fractasy hashdb 64
  • Loading branch information
@fractasy
fractasy authored Oct 27, 2023
2 parents 4e2d03d + 85658f6 commit 91f3c38
Show file tree
Hide file tree
Showing 6 changed files with 75 additions and 60 deletions.
4 changes: 2 additions & 2 deletions src/hashdb64/database_64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -401,7 +401,7 @@ zkresult Database64::WriteTree (const Goldilocks::Element (&oldRoot)[4], const v

headerPageNumber = 0;

//KeyValueHistoryPage::Print(versionData.keyValueHistoryPage, true, "version=" + to_string(version) + " ");
//KeyValueHistoryPage::Print(versionData.keyValueHistoryPage, true, "version=" + to_string(version) + " ");

return ZKR_SUCCESS;
}
Expand Down Expand Up @@ -465,6 +465,6 @@ zkresult Database64::ReadTree (const Goldilocks::Element (&root)[4], vector<KeyV

zkresult Database64::PrintTree (const string &root)
{
zklog.info("Database64::PrintTree()");
zklog.info("Database64::PrintTree() headerPageNumber=" + to_string(headerPageNumber));
return HeaderPage::KeyValueHistoryPrint(ctx, headerPageNumber, root);
}
10 changes: 4 additions & 6 deletions src/hashdb64/page/header_page.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ zkresult HeaderPage::GetFreePagesContainer (PageContext &ctx, const uint64_t hea
HeaderStruct * page = (HeaderStruct *)ctx.pageManager.getPageAddress(headerPageNumber);

// Call the specific method
return PageListPage::GetContainerPages (ctx, page->freePages, containerPages);
return PageListPage::GetContainerPages(ctx, page->freePages, containerPages);
}

zkresult HeaderPage::GetFreePages (PageContext &ctx, const uint64_t headerPageNumber, vector<uint64_t> (&freePages))
Expand All @@ -156,11 +156,9 @@ zkresult HeaderPage::CreateFreePages (PageContext &ctx, uint64_t &headerPageNumb
// Call the specific method
return PageListPage::CreatePages(ctx, page->freePages, freePages, containerPages);

return ZKR_SUCCESS;

}

zkresult HeaderPage::GetFirstUnusedPage (PageContext &ctx,const uint64_t headerPageNumber, uint64_t &firstUnusedPage)
zkresult HeaderPage::GetFirstUnusedPage (PageContext &ctx, const uint64_t headerPageNumber, uint64_t &firstUnusedPage)
{
// Get header page
HeaderStruct * page = (HeaderStruct *)ctx.pageManager.getPageAddress(headerPageNumber);
Expand All @@ -171,7 +169,7 @@ zkresult HeaderPage::GetFirstUnusedPage (PageContext &ctx,const uint64_t headerP
return ZKR_SUCCESS;
}

zkresult HeaderPage::SetFirstUnusedPage (PageContext &ctx,uint64_t &headerPageNumber, const uint64_t firstUnusedPage)
zkresult HeaderPage::SetFirstUnusedPage (PageContext &ctx, uint64_t &headerPageNumber, const uint64_t firstUnusedPage)
{
// Get an editable page
headerPageNumber = ctx.pageManager.editPage(headerPageNumber);
Expand Down Expand Up @@ -298,7 +296,7 @@ zkresult HeaderPage::KeyValueHistoryPrint (PageContext &ctx, const uint64_t head
// If root is empty, print the last version
if (root == "")
{
KeyValueHistoryPage::Print(ctx, headerPage->keyValueHistoryPage, false, "");
KeyValueHistoryPage::Print(ctx, headerPage->keyValueHistoryPage, true, "");
return ZKR_SUCCESS;
}

Expand Down
80 changes: 42 additions & 38 deletions src/hashdb64/page/key_value_history_page.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@ zkresult KeyValueHistoryPage::Write (PageContext &ctx, uint64_t &pageNumber, con
KeyValueHistoryStruct * page = (KeyValueHistoryStruct *)ctx.pageManager.getPageAddress(pageNumber);

// Get an editable version of the header page
//headerPageNumber = ctx.pageManager.editPage(headerPageNumber);
headerPageNumber = ctx.pageManager.editPage(headerPageNumber);
HeaderStruct *headerPage = (HeaderStruct *)ctx.pageManager.getPageAddress(headerPageNumber);

uint64_t insertionRawDataPage = headerPage->rawDataPage;
Expand Down Expand Up @@ -603,6 +603,7 @@ zkresult KeyValueHistoryPage::Write (PageContext &ctx, uint64_t &pageNumber, con
page->historyOffset += entrySize;

// Get an editable version of the header page
headerPageNumber = ctx.pageManager.editPage(headerPageNumber);
HeaderStruct *headerPage = (HeaderStruct *)ctx.pageManager.getPageAddress(headerPageNumber);

// Get the current rawDataPage and offset
Expand Down Expand Up @@ -638,11 +639,13 @@ zkresult KeyValueHistoryPage::Write (PageContext &ctx, uint64_t &pageNumber, con
newPage->keyValueEntry[newIndex][1] = page->keyValueEntry[index][1];
newPage->keyValueEntry[newIndex][2] = 0; // Invalidate hash, since level has changed

zkr = Write(ctx, newPageNumber, key, keyBits, version, value, level+1, headerPageNumber);

page->keyValueEntry[index][0] = uint64_t(2) << 60;
page->keyValueEntry[index][1] = newPageNumber;
page->keyValueEntry[index][2] = 0; // Invalidate hash, since now it is an intermediate node hash

return Write(ctx, newPageNumber, key, keyBits, version, value, level+1, headerPageNumber);
return zkr;
}

// Intermediate node
Expand Down Expand Up @@ -707,6 +710,7 @@ zkresult KeyValueHistoryPage::calculatePageHash (PageContext &ctx, uint64_t &pag
KeyValueHistoryStruct *page = (KeyValueHistoryStruct *)ctx.pageManager.getPageAddress(pageNumber);

// Get the header page
headerPageNumber = ctx.pageManager.editPage(headerPageNumber);
HeaderStruct *headerPage = (HeaderStruct *)ctx.pageManager.getPageAddress(headerPageNumber);

// Get the SMT level
Expand Down Expand Up @@ -944,36 +948,36 @@ void KeyValueHistoryPage::Print (PageContext &ctx, const uint64_t pageNumber, bo
case 1:
{
counters.leafNodes++;

uint64_t version = page->keyValueEntry[i][0] & U64Mask48;
// Read the key-value stored in raw data
uint64_t rawDataPage = page->keyValueEntry[i][1] & U64Mask48;
uint64_t rawDataOffset = page->keyValueEntry[i][1] >> 48;
string keyAndValue;
zkr = RawDataPage::Read(ctx, rawDataPage, rawDataOffset, 64, keyAndValue);
if (zkr != ZKR_SUCCESS)
{
zklog.error(prefix + "KeyValueHistoryPage::Print() failed calling RawDataPage::Read(64) result=" + zkresult2string(zkr) + " i=" + to_string(i) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset));
continue;
}

if (details)
// Read the hash stored in raw data, if it has been calculated
string hash;
if (page->keyValueEntry[i][2] != 0)
{
uint64_t version = page->keyValueEntry[i][0] & U64Mask48;
// Read the key-value stored in raw data
uint64_t rawDataPage = page->keyValueEntry[i][1] & U64Mask48;
uint64_t rawDataOffset = page->keyValueEntry[i][1] >> 48;
string keyAndValue;
zkr = RawDataPage::Read(ctx, rawDataPage, rawDataOffset, 64, keyAndValue);
counters.leafHashes++;
rawDataPage = page->keyValueEntry[i][2] & U64Mask48;
rawDataOffset = page->keyValueEntry[i][2] >> 48;
zkr = RawDataPage::Read(ctx, rawDataPage, rawDataOffset, 32, hash);
if (zkr != ZKR_SUCCESS)
{
zklog.error(prefix + "KeyValueHistoryPage::Print() failed calling RawDataPage::Read(64) result=" + zkresult2string(zkr) + " i=" + to_string(i) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset));
zklog.error(prefix + "KeyValueHistoryPage::Print() failed calling RawDataPage::Read(32) result=" + zkresult2string(zkr) + " i=" + to_string(i) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset));
continue;
}
}

// Read the hash stored in raw data
string hash;
if (page->keyValueEntry[i][2] != 0)
{
counters.leafHashes++;
rawDataPage = page->keyValueEntry[i][2] & U64Mask48;
rawDataOffset = page->keyValueEntry[i][2] >> 48;
zkr = RawDataPage::Read(ctx, rawDataPage, rawDataOffset, 32, hash);
if (zkr != ZKR_SUCCESS)
{
zklog.error(prefix + "KeyValueHistoryPage::Print() failed calling RawDataPage::Read(32) result=" + zkresult2string(zkr) + " i=" + to_string(i) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset));
continue;
}
}

if (details)
{
zklog.info(prefix + "i=" + to_string(i) + " key=" + ba2string(keyAndValue.substr(0, 32)) + " value=" + ba2string(keyAndValue.substr(32, 32)) + " hash=" + ba2string(hash) + " version=" + to_string(version));
}
continue;
Expand All @@ -987,23 +991,23 @@ void KeyValueHistoryPage::Print (PageContext &ctx, const uint64_t pageNumber, bo
uint64_t nextKeyValueHistoryPage = page->keyValueEntry[i][1] & U64Mask48;
nextKeyValueHistoryPages.emplace_back(nextKeyValueHistoryPage);

if (details)
string hash;
if (page->keyValueEntry[i][2] != 0)
{
string hash;
if (page->keyValueEntry[i][2] != 0)
counters.intermediateHashes++;
// Read the hash stored in raw data
uint64_t rawDataPage = page->keyValueEntry[i][2] & U64Mask48;
uint64_t rawDataOffset = page->keyValueEntry[i][2] >> 48;
zkr = RawDataPage::Read(ctx, rawDataPage, rawDataOffset, 32, hash);
if (zkr != ZKR_SUCCESS)
{
counters.intermediateHashes++;
// Read the hash stored in raw data
uint64_t rawDataPage = page->keyValueEntry[i][2] & U64Mask48;
uint64_t rawDataOffset = page->keyValueEntry[i][2] >> 48;
zkr = RawDataPage::Read(ctx, rawDataPage, rawDataOffset, 32, hash);
if (zkr != ZKR_SUCCESS)
{
zklog.error(prefix + "KeyValueHistoryPage::Print() failed calling RawDataPage::Read(32) result=" + zkresult2string(zkr) + " i=" + to_string(i) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset));
continue;
}
zklog.error(prefix + "KeyValueHistoryPage::Print() failed calling RawDataPage::Read(32) result=" + zkresult2string(zkr) + " i=" + to_string(i) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset));
continue;
}
}

if (details)
{
zklog.info(prefix + "i=" + to_string(i) + " nextKeyValueHistoryPage=" + to_string(nextKeyValueHistoryPage) + " hash=" + ba2string(hash));
}
continue;
Expand Down
8 changes: 4 additions & 4 deletions src/hashdb64/page/key_value_page.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ zkresult KeyValuePage::Read (PageContext &ctx, const uint64_t pageNumber, const
return Read(ctx, pageNumber, key, keyBits, value, 0);
}

zkresult KeyValuePage::Write (PageContext &ctx, uint64_t &pageNumber, const string &key, const vector<uint64_t> &keyBits, const string &value, const uint64_t level, const uint64_t headerPageNumber)
zkresult KeyValuePage::Write (PageContext &ctx, uint64_t &pageNumber, const string &key, const vector<uint64_t> &keyBits, const string &value, const uint64_t level, uint64_t &headerPageNumber)
{
// Check input parameters
if (level >= keyBits.size())
Expand Down Expand Up @@ -177,6 +177,7 @@ zkresult KeyValuePage::Write (PageContext &ctx, uint64_t &pageNumber, const stri
lengthKeyValue += value;

// Get header page data
headerPageNumber = ctx.pageManager.editPage(headerPageNumber);
HeaderStruct * headerPage = (HeaderStruct *)ctx.pageManager.getPageAddress(headerPageNumber);
uint64_t rawDataPage = headerPage->rawDataPage;
uint64_t rawDataOffset = RawDataPage::GetOffset(ctx, headerPage->rawDataPage);
Expand Down Expand Up @@ -317,7 +318,7 @@ zkresult KeyValuePage::Write (PageContext &ctx, uint64_t &pageNumber, const stri
return zkr;
}
//nextKeyValuePage could have changed, so we need to update it.
page->key[index] = nextKeyValuePage + (uint64_t(2) << 60);
page->key[index] = nextKeyValuePage | (uint64_t(2) << 60);

return ZKR_SUCCESS;
}
Expand All @@ -333,7 +334,7 @@ zkresult KeyValuePage::Write (PageContext &ctx, uint64_t &pageNumber, const stri
return ZKR_DB_ERROR;
}

zkresult KeyValuePage::Write (PageContext &ctx, uint64_t &pageNumber, const string &key, const string &value, const uint64_t headerPageNumber)
zkresult KeyValuePage::Write (PageContext &ctx, uint64_t &pageNumber, const string &key, const string &value, uint64_t &headerPageNumber)
{
// Check input parameters
if (key.size() == 0)
Expand All @@ -346,7 +347,6 @@ zkresult KeyValuePage::Write (PageContext &ctx, uint64_t &pageNumber, const stri
vector<uint64_t> keyBits;
splitKey9(key, keyBits);


//zklog.info("KeyValuePage::Write() key=" + ba2string(key) + " keyBits=" + to_string(keyBits[0]) + ":" + to_string(keyBits[1]) + ":" + to_string(keyBits[2]));

// Call Write with level=0
Expand Down
4 changes: 2 additions & 2 deletions src/hashdb64/page/key_value_page.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@ class KeyValuePage
private:

static zkresult Read (PageContext &ctx, const uint64_t pageNumber, const string &key, const vector<uint64_t> &keyBits, string &value, const uint64_t level);
static zkresult Write (PageContext &ctx, uint64_t &pageNumber, const string &key, const vector<uint64_t> &keyBits, const string &value, const uint64_t level, const uint64_t headerPageNumber);
static zkresult Write (PageContext &ctx, uint64_t &pageNumber, const string &key, const vector<uint64_t> &keyBits, const string &value, const uint64_t level, uint64_t &headerPageNumber);

public:

static zkresult InitEmptyPage (PageContext &ctx, const uint64_t pageNumber);
static zkresult Read (PageContext &ctx, const uint64_t pageNumber, const string &key, string &value);
static zkresult Write (PageContext &ctx, uint64_t &pageNumber, const string &key, const string &value, const uint64_t headerPageNumber);
static zkresult Write (PageContext &ctx, uint64_t &pageNumber, const string &key, const string &value, uint64_t &headerPageNumber);

static void Print (PageContext &ctx, const uint64_t pageNumber, bool details, const string &prefix, const uint64_t keySize);
};
Expand Down
29 changes: 21 additions & 8 deletions src/hashdb64/page/raw_data_page.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,7 @@ zkresult RawDataPage::Read (PageContext &ctx, const uint64_t _pageNumber, const
zkresult RawDataPage::Write (PageContext &ctx, uint64_t &pageNumber, const string &data)
{
// Get the pointer corresponding to the current page number
pageNumber = ctx.pageManager.editPage(pageNumber);
RawDataStruct * page = (RawDataStruct *)ctx.pageManager.getPageAddress(pageNumber);
RawDataStruct *page = (RawDataStruct *)ctx.pageManager.getPageAddress(pageNumber);

// Get page attributes
uint64_t offset = page->nextPageNumberAndOffset >> 48;
Expand All @@ -98,17 +97,17 @@ zkresult RawDataPage::Write (PageContext &ctx, uint64_t &pageNumber, const strin
// Check attributes
if (nextPage != 0)
{
zklog.error("RawDataPage::Write() found non-zero nextPage=" + to_string(nextPage) + " pageNumber=" + to_string(pageNumber) + " length=" + to_string(data.size()));
zklog.error("RawDataPage::Write() 1 found non-zero nextPage=" + to_string(nextPage) + " pageNumber=" + to_string(pageNumber) + " length=" + to_string(data.size()));
return ZKR_DB_ERROR;
}
if (offset < minOffset)
{
zklog.error("RawDataPage::Write() found too-small offset=" + to_string(offset) + " pageNumber=" + to_string(pageNumber) + " length=" + to_string(data.size()));
zklog.error("RawDataPage::Write() 1 found too-small offset=" + to_string(offset) + " pageNumber=" + to_string(pageNumber) + " length=" + to_string(data.size()));
return ZKR_DB_ERROR;
}
if (offset > maxOffset)
{
zklog.error("RawDataPage::Write() found too-big offset=" + to_string(offset) + " pageNumber=" + to_string(pageNumber) + " length=" + to_string(data.size()));
zklog.error("RawDataPage::Write() 1 found too-big offset=" + to_string(offset) + " pageNumber=" + to_string(pageNumber) + " length=" + to_string(data.size()));
return ZKR_DB_ERROR;
}

Expand All @@ -121,15 +120,29 @@ zkresult RawDataPage::Write (PageContext &ctx, uint64_t &pageNumber, const strin
uint64_t nextPageNumber = ctx.pageManager.getFreePage();
InitEmptyPage(ctx, nextPageNumber);
RawDataStruct * nextPage = (RawDataStruct *)ctx.pageManager.getPageAddress(nextPageNumber);
RawDataStruct * page = (RawDataStruct *)ctx.pageManager.getPageAddress(pageNumber);
page->nextPageNumberAndOffset = nextPageNumber | (maxOffset << 48);
nextPage->previousPageNumber = pageNumber;
pageNumber = nextPageNumber;
page = nextPage;
offset = minOffset;
}

// Get the pointer corresponding to the current page number
RawDataStruct * page = (RawDataStruct *)ctx.pageManager.getPageAddress(pageNumber);
// Check attributes
if (nextPage != 0)
{
zklog.error("RawDataPage::Write() 2 found non-zero nextPage=" + to_string(nextPage) + " pageNumber=" + to_string(pageNumber) + " length=" + to_string(data.size()));
return ZKR_DB_ERROR;
}
if (offset < minOffset)
{
zklog.error("RawDataPage::Write() 2 found too-small offset=" + to_string(offset) + " pageNumber=" + to_string(pageNumber) + " length=" + to_string(data.size()));
return ZKR_DB_ERROR;
}
if (offset > maxOffset)
{
zklog.error("RawDataPage::Write() 2 found too-big offset=" + to_string(offset) + " pageNumber=" + to_string(pageNumber) + " length=" + to_string(data.size()));
return ZKR_DB_ERROR;
}

// Calculate the amount of bytes to write this time
uint64_t pageRemainingBytes = maxOffset - offset;
Expand Down

0 comments on commit 91f3c38

Please sign in to comment.