Skip to content

Commit

Permalink
Merge pull request #652 from 0xPolygonHermez/fractasy_hashdb_64
Browse files Browse the repository at this point in the history 
Implement keyLevel in read Database64 methods.  Code and traces cleanup.
  • Loading branch information
@fractasy
fractasy authored Oct 17, 2023
2 parents 71f6be0 + 638ceb5 commit 6c1b348
Show file tree
Hide file tree
Showing 6 changed files with 153 additions and 17 deletions.
7 changes: 3 additions & 4 deletions src/hashdb64/database_64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,6 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc
{
zkresult zkr;

level = 128; // TODO: Return the right level

// Check that it has been initialized before
if (!bInitialized)
{
Expand Down Expand Up @@ -97,7 +95,7 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc
// Get the value
string keyString = fea2string(fr, key);
string keyBa = string2ba(keyString);
zkr = HeaderPage::KeyValueHistoryRead(versionData.keyValueHistoryPage, keyBa, version, value);
zkr = HeaderPage::KeyValueHistoryRead(versionData.keyValueHistoryPage, keyBa, version, value, level);
if (zkr != ZKR_SUCCESS)
{
zklog.error("Database64::readKV() faile calling HeaderPage::KeyValueHistoryRead() result=" + zkresult2string(zkr) + " root=" + rootString + " key=" + fea2string(fr, key));
Expand Down Expand Up @@ -429,11 +427,12 @@ zkresult Database64::ReadTree (const Goldilocks::Element (&root)[4], vector<KeyV
// Read all key-values
string keyString;
string key;
uint64_t level;
for (uint64_t i=0; i<keyValues.size(); i++)
{
keyString = fea2string(fr, keyValues[i].key);
key = string2ba(keyString);
zkr = HeaderPage::KeyValueHistoryRead(versionData.keyValueHistoryPage, key, version, keyValues[i].value);
zkr = HeaderPage::KeyValueHistoryRead(versionData.keyValueHistoryPage, key, version, keyValues[i].value, level);
if (zkr != ZKR_SUCCESS)
{
zklog.error("Database64::ReadTree() failed calling HeaderPage::KeyValueHistoryRead() result=" + zkresult2string(zkr) + " rootString=" + rootString + " version=" + to_string(version));
Expand Down
13 changes: 11 additions & 2 deletions src/hashdb64/page/header_page.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,10 +180,19 @@ zkresult HeaderPage::WriteVersionData (uint64_t &headerPageNumber, const uint64_
return KeyValuePage::Write(headerPage->versionDataPage, version2key(version), versionData2value(versionData), headerPageNumber);
}

zkresult HeaderPage::KeyValueHistoryRead (const uint64_t keyValueHistoryPage, const string &key, const uint64_t version, mpz_class &value)
zkresult HeaderPage::KeyValueHistoryRead (const uint64_t keyValueHistoryPage, const string &key, const uint64_t version, mpz_class &value, uint64_t &keyLevel)
{
// Call the specific method
return KeyValueHistoryPage::Read(keyValueHistoryPage, key, version, value);
return KeyValueHistoryPage::Read(keyValueHistoryPage, key, version, value, keyLevel);
}

zkresult HeaderPage::KeyValueHistoryReadLevel (uint64_t &headerPageNumber, const string &key, uint64_t &keyLevel)
{
// Get header page
HeaderStruct * headerPage = (HeaderStruct *)pageManager.getPageAddress(headerPageNumber);

// Call the specific method
return KeyValueHistoryPage::ReadLevel(headerPage->keyValueHistoryPage, key, keyLevel);
}

zkresult HeaderPage::KeyValueHistoryWrite (uint64_t &headerPageNumber, const string &key, const uint64_t version, const mpz_class &value)
Expand Down
5 changes: 3 additions & 2 deletions src/hashdb64/page/header_page.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,9 @@ class HeaderPage
static zkresult WriteVersionData ( uint64_t &headerPageNumber, const uint64_t &version, const VersionDataStruct &versionData);

// Key-Value-History methods
static zkresult KeyValueHistoryRead (const uint64_t keyValueHistoryPage, const string &key, const uint64_t version, mpz_class &value);
static zkresult KeyValueHistoryWrite ( uint64_t &headerPageNumber, const string &key, const uint64_t version, const mpz_class &value);
static zkresult KeyValueHistoryRead (const uint64_t keyValueHistoryPage, const string &key, const uint64_t version, mpz_class &value, uint64_t &keyLevel);
static zkresult KeyValueHistoryReadLevel ( uint64_t &headerPageNumber, const string &key, uint64_t &keyLevel);
static zkresult KeyValueHistoryWrite ( uint64_t &headerPageNumber, const string &key, const uint64_t version, const mpz_class &value);
static zkresult KeyValueHistoryCalculateHash (uint64_t &headerPageNumber, Goldilocks::Element (&hash)[4]);

// Program page methods
Expand Down
127 changes: 122 additions & 5 deletions src/hashdb64/page/key_value_history_page.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ zkresult KeyValueHistoryPage::InitEmptyPage (const uint64_t pageNumber)
return ZKR_SUCCESS;
}

zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t version, mpz_class &value, const uint64_t level)
zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t version, mpz_class &value, const uint64_t level, uint64_t &keyLevel)
{
zkassert(key.size() == 32);
zkassert(keyBits.size() == 43);
Expand All @@ -39,6 +39,7 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key
case 0:
{
value = 0;
keyLevel = (level + 1) * 6;
return ZKR_SUCCESS;
}
// Leaf node
Expand Down Expand Up @@ -74,12 +75,19 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key
{
//zklog.info("KeyValueHistoryPage::Read() found existing key=" + ba2string(keyValue.substr(0, 32)) + " != key=" + ba2string(key));
value = 0;

// Get the key level
keyLevel = (level + 1) * 6;

return ZKR_SUCCESS;
}

// Convert the value
ba2scalar((uint8_t *)keyValue.c_str() + 32, 32, value);

// Get the key level
keyLevel = (level + 1) * 6;

return ZKR_SUCCESS;
}

Expand All @@ -90,6 +98,10 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key
if (previousVersionOffset == 0)
{
value = 0;

// Get the key level
keyLevel = (level + 1) * 6;

return ZKR_SUCCESS;
}

Expand All @@ -110,11 +122,11 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key
case 2:
{
uint64_t nextPageNumber = page->keyValueEntry[index][1] & U64Mask48;
return Read(nextPageNumber, key, keyBits, version, value, level + 1);
return Read(nextPageNumber, key, keyBits, version, value, level + 1, keyLevel);
}
default:
{
zklog.error("KeyValuePage::Write() found invalid control=" + to_string(control) + " pageNumber=" + to_string(pageNumber));
zklog.error("KeyValuePage::Read() found invalid control=" + to_string(control) + " pageNumber=" + to_string(pageNumber));
return ZKR_DB_ERROR;
}
}
Expand All @@ -123,7 +135,7 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key
return ZKR_DB_KEY_NOT_FOUND;
}

zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key, const uint64_t version, mpz_class &value)
zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key, const uint64_t version, mpz_class &value, uint64_t &keyLevel)
{
zkassert(key.size() == 32);
zkassert((version & U64Mask48) == version);
Expand All @@ -136,7 +148,112 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key
string keyBits;
keyBits.append((char *)keyBitsArray, 43);

return Read(pageNumber, key, keyBits, version, value, 0);
return Read(pageNumber, key, keyBits, version, value, 0, keyLevel);
}

zkresult KeyValueHistoryPage::ReadLevel (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t level, uint64_t &keyLevel)
{
zkassert(key.size() == 32);
zkassert(keyBits.size() == 43);
zkassert(level < 43);

zkresult zkr;

// Get the data from this page
KeyValueHistoryStruct * page = (KeyValueHistoryStruct *)pageManager.getPageAddress(pageNumber);
uint64_t index = keyBits[level];
uint64_t control = page->keyValueEntry[index][0] >> 60;

// Check control
switch (control)
{
// Empty slot
case 0:
{
keyLevel = (level + 1) * 6;
return ZKR_SUCCESS;
}
// Leaf node
case 1:
{
uint64_t rawDataPage = page->keyValueEntry[index][1] & U64Mask48;
uint64_t rawDataOffset = page->keyValueEntry[index][1] >> 48;
string keyValue;
zkr = RawDataPage::Read(rawDataPage, rawDataOffset, 64, keyValue);
if (zkr != ZKR_SUCCESS)
{
zklog.error("KeyValueHistoryPage::ReadLevel() failed calling RawDataPage.Read result=" + zkresult2string(zkr) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset) + " key=" + ba2string(key) + " level=" + to_string(level) + " index=" + to_string(index));
return zkr;
}
if (keyValue.size() != 64)
{
zklog.error("KeyValueHistoryPage::ReadLevel() called RawDataPage.Read and got invalid length=" + to_string(keyValue.size()) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset) + " key=" + ba2string(key) + " level=" + to_string(level) + " index=" + to_string(index));
return zkr;
}

// If this is the same key
if (memcmp(key.c_str(), keyValue.c_str(), 32) != 0)
{
// Get the key level
keyLevel = (level + 1) * 6;

return ZKR_SUCCESS;
}

// If keys are different, we need to know how different they are
Goldilocks::Element keyFea[4];
string2fea(fr, ba2string(keyValue.substr(0, 32)), keyFea);
uint8_t foundKeyBitsArray[43];
splitKey6(fr, keyFea, foundKeyBitsArray);
string foundKeyBits;
foundKeyBits.append((char *)foundKeyBitsArray, 43);


// Find the first 6-bit set that is different
uint64_t i=0;
for (; i<43; i++)
{
if (keyBits[i] != foundKeyBits[i])
{
break;
}
}

// Set the level
keyLevel = (i + 1) * 6;

return ZKR_SUCCESS;
}
// Intermediate node
case 2:
{
uint64_t nextPageNumber = page->keyValueEntry[index][1] & U64Mask48;
return ReadLevel(nextPageNumber, key, keyBits, level + 1, keyLevel);
}
default:
{
zklog.error("KeyValuePage::ReadLevel() found invalid control=" + to_string(control) + " pageNumber=" + to_string(pageNumber));
return ZKR_DB_ERROR;
}
}

// Not found
return ZKR_DB_KEY_NOT_FOUND;
}

zkresult KeyValueHistoryPage::ReadLevel (const uint64_t pageNumber, const string &key, uint64_t &keyLevel)
{
zkassert(key.size() == 32);

// Get 256 key bits in SMT order, in sets of 6 bits
Goldilocks::Element keyFea[4];
string2fea(fr, ba2string(key), keyFea);
uint8_t keyBitsArray[43];
splitKey6(fr, keyFea, keyBitsArray);
string keyBits;
keyBits.append((char *)keyBitsArray, 43);

return ReadLevel(pageNumber, key, keyBits, 0, keyLevel);
}

zkresult KeyValueHistoryPage::Write (uint64_t &pageNumber, const string &key, const string &keyBits, const uint64_t version, const mpz_class &value, const uint64_t level, uint64_t &headerPageNumber)
Expand Down
6 changes: 4 additions & 2 deletions src/hashdb64/page/key_value_history_page.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,13 @@ class KeyValueHistoryPage
static const uint64_t minHistoryOffset = 8 + 8 + 64*3*8; // 1552
static const uint64_t maxHistoryOffset = 8 + 8 + 64*3*8 + 106*3*8; // 4096
private:
static zkresult Read (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t version, mpz_class &value, const uint64_t level);
static zkresult Read (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t version, mpz_class &value, const uint64_t level, uint64_t &keyLevel);
static zkresult ReadLevel (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t level, uint64_t &keyLevel);
static zkresult Write ( uint64_t &pageNumber, const string &key, const string &keyBits, const uint64_t version, const mpz_class &value, const uint64_t level, uint64_t &headerPageNumber);
public:
static zkresult InitEmptyPage (const uint64_t pageNumber);
static zkresult Read (const uint64_t pageNumber, const string &key, const uint64_t version, mpz_class &value);
static zkresult Read (const uint64_t pageNumber, const string &key, const uint64_t version, mpz_class &value, uint64_t &keyLevel);
static zkresult ReadLevel (const uint64_t pageNumber, const string &key, uint64_t &keyLevel);
static zkresult Write ( uint64_t &pageNumber, const string &key, const uint64_t version, const mpz_class &value, uint64_t &headerPageNumber);

static zkresult calculateHash (const uint64_t pageNumber, Goldilocks::Element (&hash)[4], uint64_t &headerPageNumber);
Expand Down
12 changes: 10 additions & 2 deletions src/hashdb64/page/key_value_page.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
#include "key_utils.hpp"
#include "constants.hpp"

//#define LOG_KEY_VALUE_PAGE

zkresult KeyValuePage::InitEmptyPage (const uint64_t pageNumber)
{
KeyValueStruct * page = (KeyValueStruct *)pageManager.getPageAddress(pageNumber);
Expand Down Expand Up @@ -82,6 +84,7 @@ zkresult KeyValuePage::Read (const uint64_t pageNumber, const string &key, const
return ZKR_DB_KEY_NOT_FOUND;
}

#ifdef LOG_KEY_VALUE_PAGE
zklog.info("KeyValuePage::Read() read length=" + to_string(length) +
" result=" + zkresult2string(zkr) +
" pageNumber=" + to_string(pageNumber) +
Expand All @@ -90,7 +93,8 @@ zkresult KeyValuePage::Read (const uint64_t pageNumber, const string &key, const
" key=" + ba2string(key) +
" rawDataPage=" + to_string(rawDataPage) +
" rawDataOffset=" + to_string(rawDataOffset));

#endif

value = rawData.substr(4 + key.size());

return ZKR_SUCCESS;
Expand Down Expand Up @@ -185,6 +189,7 @@ zkresult KeyValuePage::Write (uint64_t &pageNumber, const string &key, const vec
return zkr;
}

#ifdef LOG_KEY_VALUE_PAGE
zklog.info("KeyValuePage::Write() wrote length=" + to_string(length) +
" totalLength=" + to_string(lengthKeyValue.size()) +
" result=" + zkresult2string(zkr) +
Expand All @@ -196,6 +201,7 @@ zkresult KeyValuePage::Write (uint64_t &pageNumber, const string &key, const vec
" rawDataOffset=" + to_string(rawDataOffset) +
" leaving headerPage->rawDataPage=" + to_string(headerPage->rawDataPage) +
" headerPage->rawDataOffset=" + to_string(RawDataPage::GetOffset(headerPage->rawDataPage)));
#endif

// Update this entry as a leaf node (control = 1)
page->key[index] = (uint64_t(1)<<60) | ((rawDataOffset & U64Mask12) << 48) | (rawDataPage & U64Mask48);
Expand Down Expand Up @@ -282,8 +288,10 @@ zkresult KeyValuePage::Write (uint64_t &pageNumber, const string &key, const vec
}
uint64_t newIndex = existingKeyBits[level+1];
newPage->key[newIndex] = page->key[index];


#ifdef LOG_KEY_VALUE_PAGE
zklog.info("KeyValuePage::Write() moved existing key=" + ba2string(existingLengthAndKey.substr(4)) + " to new page=" + to_string(newPageNumber) + " at index=" + to_string(newIndex));
#endif

// Write the new key in the newly created page at the next level
zkr = Write(newPageNumber, key, keyBits, value, level+1, headerPageNumber);
Expand Down

0 comments on commit 6c1b348

Please sign in to comment.