Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement keyLevel in read Database64 methods. Code and traces cleanup. #652

Merged
merged 1 commit into from
Oct 17, 2023
Merged

Implement keyLevel in read Database64 methods. Code and traces cleanup. #652

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions src/hashdb64/database_64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,6 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc
{
zkresult zkr;

level = 128; // TODO: Return the right level

// Check that it has been initialized before
if (!bInitialized)
{
Expand Down Expand Up @@ -97,7 +95,7 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc
// Get the value
string keyString = fea2string(fr, key);
string keyBa = string2ba(keyString);
zkr = HeaderPage::KeyValueHistoryRead(versionData.keyValueHistoryPage, keyBa, version, value);
zkr = HeaderPage::KeyValueHistoryRead(versionData.keyValueHistoryPage, keyBa, version, value, level);
if (zkr != ZKR_SUCCESS)
{
zklog.error("Database64::readKV() faile calling HeaderPage::KeyValueHistoryRead() result=" + zkresult2string(zkr) + " root=" + rootString + " key=" + fea2string(fr, key));
Expand Down Expand Up @@ -429,11 +427,12 @@ zkresult Database64::ReadTree (const Goldilocks::Element (&root)[4], vector<KeyV
// Read all key-values
string keyString;
string key;
uint64_t level;
for (uint64_t i=0; i<keyValues.size(); i++)
{
keyString = fea2string(fr, keyValues[i].key);
key = string2ba(keyString);
zkr = HeaderPage::KeyValueHistoryRead(versionData.keyValueHistoryPage, key, version, keyValues[i].value);
zkr = HeaderPage::KeyValueHistoryRead(versionData.keyValueHistoryPage, key, version, keyValues[i].value, level);
if (zkr != ZKR_SUCCESS)
{
zklog.error("Database64::ReadTree() failed calling HeaderPage::KeyValueHistoryRead() result=" + zkresult2string(zkr) + " rootString=" + rootString + " version=" + to_string(version));
Expand Down
13 changes: 11 additions & 2 deletions src/hashdb64/page/header_page.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,10 +180,19 @@ zkresult HeaderPage::WriteVersionData (uint64_t &headerPageNumber, const uint64_
return KeyValuePage::Write(headerPage->versionDataPage, version2key(version), versionData2value(versionData), headerPageNumber);
}

zkresult HeaderPage::KeyValueHistoryRead (const uint64_t keyValueHistoryPage, const string &key, const uint64_t version, mpz_class &value)
zkresult HeaderPage::KeyValueHistoryRead (const uint64_t keyValueHistoryPage, const string &key, const uint64_t version, mpz_class &value, uint64_t &keyLevel)
{
// Call the specific method
return KeyValueHistoryPage::Read(keyValueHistoryPage, key, version, value);
return KeyValueHistoryPage::Read(keyValueHistoryPage, key, version, value, keyLevel);
}

zkresult HeaderPage::KeyValueHistoryReadLevel (uint64_t &headerPageNumber, const string &key, uint64_t &keyLevel)
{
// Get header page
HeaderStruct * headerPage = (HeaderStruct *)pageManager.getPageAddress(headerPageNumber);

// Call the specific method
return KeyValueHistoryPage::ReadLevel(headerPage->keyValueHistoryPage, key, keyLevel);
}

zkresult HeaderPage::KeyValueHistoryWrite (uint64_t &headerPageNumber, const string &key, const uint64_t version, const mpz_class &value)
Expand Down
5 changes: 3 additions & 2 deletions src/hashdb64/page/header_page.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,9 @@ class HeaderPage
static zkresult WriteVersionData ( uint64_t &headerPageNumber, const uint64_t &version, const VersionDataStruct &versionData);

// Key-Value-History methods
static zkresult KeyValueHistoryRead (const uint64_t keyValueHistoryPage, const string &key, const uint64_t version, mpz_class &value);
static zkresult KeyValueHistoryWrite ( uint64_t &headerPageNumber, const string &key, const uint64_t version, const mpz_class &value);
static zkresult KeyValueHistoryRead (const uint64_t keyValueHistoryPage, const string &key, const uint64_t version, mpz_class &value, uint64_t &keyLevel);
static zkresult KeyValueHistoryReadLevel ( uint64_t &headerPageNumber, const string &key, uint64_t &keyLevel);
static zkresult KeyValueHistoryWrite ( uint64_t &headerPageNumber, const string &key, const uint64_t version, const mpz_class &value);
static zkresult KeyValueHistoryCalculateHash (uint64_t &headerPageNumber, Goldilocks::Element (&hash)[4]);

// Program page methods
Expand Down
127 changes: 122 additions & 5 deletions src/hashdb64/page/key_value_history_page.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ zkresult KeyValueHistoryPage::InitEmptyPage (const uint64_t pageNumber)
return ZKR_SUCCESS;
}

zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t version, mpz_class &value, const uint64_t level)
zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t version, mpz_class &value, const uint64_t level, uint64_t &keyLevel)
{
zkassert(key.size() == 32);
zkassert(keyBits.size() == 43);
Expand All @@ -39,6 +39,7 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key
case 0:
{
value = 0;
keyLevel = (level + 1) * 6;
return ZKR_SUCCESS;
}
// Leaf node
Expand Down Expand Up @@ -74,12 +75,19 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key
{
//zklog.info("KeyValueHistoryPage::Read() found existing key=" + ba2string(keyValue.substr(0, 32)) + " != key=" + ba2string(key));
value = 0;

// Get the key level
keyLevel = (level + 1) * 6;

return ZKR_SUCCESS;
}

// Convert the value
ba2scalar((uint8_t *)keyValue.c_str() + 32, 32, value);

// Get the key level
keyLevel = (level + 1) * 6;

return ZKR_SUCCESS;
}

Expand All @@ -90,6 +98,10 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key
if (previousVersionOffset == 0)
{
value = 0;

// Get the key level
keyLevel = (level + 1) * 6;

return ZKR_SUCCESS;
}

Expand All @@ -110,11 +122,11 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key
case 2:
{
uint64_t nextPageNumber = page->keyValueEntry[index][1] & U64Mask48;
return Read(nextPageNumber, key, keyBits, version, value, level + 1);
return Read(nextPageNumber, key, keyBits, version, value, level + 1, keyLevel);
}
default:
{
zklog.error("KeyValuePage::Write() found invalid control=" + to_string(control) + " pageNumber=" + to_string(pageNumber));
zklog.error("KeyValuePage::Read() found invalid control=" + to_string(control) + " pageNumber=" + to_string(pageNumber));
return ZKR_DB_ERROR;
}
}
Expand All @@ -123,7 +135,7 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key
return ZKR_DB_KEY_NOT_FOUND;
}

zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key, const uint64_t version, mpz_class &value)
zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key, const uint64_t version, mpz_class &value, uint64_t &keyLevel)
{
zkassert(key.size() == 32);
zkassert((version & U64Mask48) == version);
Expand All @@ -136,7 +148,112 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key
string keyBits;
keyBits.append((char *)keyBitsArray, 43);

return Read(pageNumber, key, keyBits, version, value, 0);
return Read(pageNumber, key, keyBits, version, value, 0, keyLevel);
}

zkresult KeyValueHistoryPage::ReadLevel (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t level, uint64_t &keyLevel)
{
zkassert(key.size() == 32);
zkassert(keyBits.size() == 43);
zkassert(level < 43);

zkresult zkr;

// Get the data from this page
KeyValueHistoryStruct * page = (KeyValueHistoryStruct *)pageManager.getPageAddress(pageNumber);
uint64_t index = keyBits[level];
uint64_t control = page->keyValueEntry[index][0] >> 60;

// Check control
switch (control)
{
// Empty slot
case 0:
{
keyLevel = (level + 1) * 6;
return ZKR_SUCCESS;
}
// Leaf node
case 1:
{
uint64_t rawDataPage = page->keyValueEntry[index][1] & U64Mask48;
uint64_t rawDataOffset = page->keyValueEntry[index][1] >> 48;
string keyValue;
zkr = RawDataPage::Read(rawDataPage, rawDataOffset, 64, keyValue);
if (zkr != ZKR_SUCCESS)
{
zklog.error("KeyValueHistoryPage::ReadLevel() failed calling RawDataPage.Read result=" + zkresult2string(zkr) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset) + " key=" + ba2string(key) + " level=" + to_string(level) + " index=" + to_string(index));
return zkr;
}
if (keyValue.size() != 64)
{
zklog.error("KeyValueHistoryPage::ReadLevel() called RawDataPage.Read and got invalid length=" + to_string(keyValue.size()) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset) + " key=" + ba2string(key) + " level=" + to_string(level) + " index=" + to_string(index));
return zkr;
}

// If this is the same key
if (memcmp(key.c_str(), keyValue.c_str(), 32) != 0)
{
// Get the key level
keyLevel = (level + 1) * 6;

return ZKR_SUCCESS;
}

// If keys are different, we need to know how different they are
Goldilocks::Element keyFea[4];
string2fea(fr, ba2string(keyValue.substr(0, 32)), keyFea);
uint8_t foundKeyBitsArray[43];
splitKey6(fr, keyFea, foundKeyBitsArray);
string foundKeyBits;
foundKeyBits.append((char *)foundKeyBitsArray, 43);


// Find the first 6-bit set that is different
uint64_t i=0;
for (; i<43; i++)
{
if (keyBits[i] != foundKeyBits[i])
{
break;
}
}

// Set the level
keyLevel = (i + 1) * 6;

return ZKR_SUCCESS;
}
// Intermediate node
case 2:
{
uint64_t nextPageNumber = page->keyValueEntry[index][1] & U64Mask48;
return ReadLevel(nextPageNumber, key, keyBits, level + 1, keyLevel);
}
default:
{
zklog.error("KeyValuePage::ReadLevel() found invalid control=" + to_string(control) + " pageNumber=" + to_string(pageNumber));
return ZKR_DB_ERROR;
}
}

// Not found
return ZKR_DB_KEY_NOT_FOUND;
}

zkresult KeyValueHistoryPage::ReadLevel (const uint64_t pageNumber, const string &key, uint64_t &keyLevel)
{
zkassert(key.size() == 32);

// Get 256 key bits in SMT order, in sets of 6 bits
Goldilocks::Element keyFea[4];
string2fea(fr, ba2string(key), keyFea);
uint8_t keyBitsArray[43];
splitKey6(fr, keyFea, keyBitsArray);
string keyBits;
keyBits.append((char *)keyBitsArray, 43);

return ReadLevel(pageNumber, key, keyBits, 0, keyLevel);
}

zkresult KeyValueHistoryPage::Write (uint64_t &pageNumber, const string &key, const string &keyBits, const uint64_t version, const mpz_class &value, const uint64_t level, uint64_t &headerPageNumber)
Expand Down
6 changes: 4 additions & 2 deletions src/hashdb64/page/key_value_history_page.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,13 @@ class KeyValueHistoryPage
static const uint64_t minHistoryOffset = 8 + 8 + 64*3*8; // 1552
static const uint64_t maxHistoryOffset = 8 + 8 + 64*3*8 + 106*3*8; // 4096
private:
static zkresult Read (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t version, mpz_class &value, const uint64_t level);
static zkresult Read (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t version, mpz_class &value, const uint64_t level, uint64_t &keyLevel);
static zkresult ReadLevel (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t level, uint64_t &keyLevel);
static zkresult Write ( uint64_t &pageNumber, const string &key, const string &keyBits, const uint64_t version, const mpz_class &value, const uint64_t level, uint64_t &headerPageNumber);
public:
static zkresult InitEmptyPage (const uint64_t pageNumber);
static zkresult Read (const uint64_t pageNumber, const string &key, const uint64_t version, mpz_class &value);
static zkresult Read (const uint64_t pageNumber, const string &key, const uint64_t version, mpz_class &value, uint64_t &keyLevel);
static zkresult ReadLevel (const uint64_t pageNumber, const string &key, uint64_t &keyLevel);
static zkresult Write ( uint64_t &pageNumber, const string &key, const uint64_t version, const mpz_class &value, uint64_t &headerPageNumber);

static zkresult calculateHash (const uint64_t pageNumber, Goldilocks::Element (&hash)[4], uint64_t &headerPageNumber);
Expand Down
12 changes: 10 additions & 2 deletions src/hashdb64/page/key_value_page.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
#include "key_utils.hpp"
#include "constants.hpp"

//#define LOG_KEY_VALUE_PAGE

zkresult KeyValuePage::InitEmptyPage (const uint64_t pageNumber)
{
KeyValueStruct * page = (KeyValueStruct *)pageManager.getPageAddress(pageNumber);
Expand Down Expand Up @@ -82,6 +84,7 @@ zkresult KeyValuePage::Read (const uint64_t pageNumber, const string &key, const
return ZKR_DB_KEY_NOT_FOUND;
}

#ifdef LOG_KEY_VALUE_PAGE
zklog.info("KeyValuePage::Read() read length=" + to_string(length) +
" result=" + zkresult2string(zkr) +
" pageNumber=" + to_string(pageNumber) +
Expand All @@ -90,7 +93,8 @@ zkresult KeyValuePage::Read (const uint64_t pageNumber, const string &key, const
" key=" + ba2string(key) +
" rawDataPage=" + to_string(rawDataPage) +
" rawDataOffset=" + to_string(rawDataOffset));

#endif

value = rawData.substr(4 + key.size());

return ZKR_SUCCESS;
Expand Down Expand Up @@ -185,6 +189,7 @@ zkresult KeyValuePage::Write (uint64_t &pageNumber, const string &key, const vec
return zkr;
}

#ifdef LOG_KEY_VALUE_PAGE
zklog.info("KeyValuePage::Write() wrote length=" + to_string(length) +
" totalLength=" + to_string(lengthKeyValue.size()) +
" result=" + zkresult2string(zkr) +
Expand All @@ -196,6 +201,7 @@ zkresult KeyValuePage::Write (uint64_t &pageNumber, const string &key, const vec
" rawDataOffset=" + to_string(rawDataOffset) +
" leaving headerPage->rawDataPage=" + to_string(headerPage->rawDataPage) +
" headerPage->rawDataOffset=" + to_string(RawDataPage::GetOffset(headerPage->rawDataPage)));
#endif

// Update this entry as a leaf node (control = 1)
page->key[index] = (uint64_t(1)<<60) | ((rawDataOffset & U64Mask12) << 48) | (rawDataPage & U64Mask48);
Expand Down Expand Up @@ -282,8 +288,10 @@ zkresult KeyValuePage::Write (uint64_t &pageNumber, const string &key, const vec
}
uint64_t newIndex = existingKeyBits[level+1];
newPage->key[newIndex] = page->key[index];


#ifdef LOG_KEY_VALUE_PAGE
zklog.info("KeyValuePage::Write() moved existing key=" + ba2string(existingLengthAndKey.substr(4)) + " to new page=" + to_string(newPageNumber) + " at index=" + to_string(newIndex));
#endif

// Write the new key in the newly created page at the next level
zkr = Write(newPageNumber, key, keyBits, value, level+1, headerPageNumber);
Expand Down