From 638ceb561e36925c1ff76891fc74ee0775a927a7 Mon Sep 17 00:00:00 2001 From: fractasy Date: Tue, 17 Oct 2023 14:16:30 +0000 Subject: [PATCH] Implement keyLevel in read Database64 methods. Code and traces cleanup. --- src/hashdb64/database_64.cpp | 7 +- src/hashdb64/page/header_page.cpp | 13 +- src/hashdb64/page/header_page.hpp | 5 +- src/hashdb64/page/key_value_history_page.cpp | 127 ++++++++++++++++++- src/hashdb64/page/key_value_history_page.hpp | 6 +- src/hashdb64/page/key_value_page.cpp | 12 +- 6 files changed, 153 insertions(+), 17 deletions(-) diff --git a/src/hashdb64/database_64.cpp b/src/hashdb64/database_64.cpp index e8959d6be..45e42169d 100644 --- a/src/hashdb64/database_64.cpp +++ b/src/hashdb64/database_64.cpp @@ -63,8 +63,6 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc { zkresult zkr; - level = 128; // TODO: Return the right level - // Check that it has been initialized before if (!bInitialized) { @@ -97,7 +95,7 @@ zkresult Database64::readKV(const Goldilocks::Element (&root)[4], const Goldiloc // Get the value string keyString = fea2string(fr, key); string keyBa = string2ba(keyString); - zkr = HeaderPage::KeyValueHistoryRead(versionData.keyValueHistoryPage, keyBa, version, value); + zkr = HeaderPage::KeyValueHistoryRead(versionData.keyValueHistoryPage, keyBa, version, value, level); if (zkr != ZKR_SUCCESS) { zklog.error("Database64::readKV() faile calling HeaderPage::KeyValueHistoryRead() result=" + zkresult2string(zkr) + " root=" + rootString + " key=" + fea2string(fr, key)); @@ -429,11 +427,12 @@ zkresult Database64::ReadTree (const Goldilocks::Element (&root)[4], vectorkeyValueEntry[index][1] & U64Mask48; - return Read(nextPageNumber, key, keyBits, version, value, level + 1); + return Read(nextPageNumber, key, keyBits, version, value, level + 1, keyLevel); } default: { - zklog.error("KeyValuePage::Write() found invalid control=" + to_string(control) + " pageNumber=" + to_string(pageNumber)); + zklog.error("KeyValuePage::Read() found invalid control=" + to_string(control) + " pageNumber=" + to_string(pageNumber)); return ZKR_DB_ERROR; } } @@ -123,7 +135,7 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key return ZKR_DB_KEY_NOT_FOUND; } -zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key, const uint64_t version, mpz_class &value) +zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key, const uint64_t version, mpz_class &value, uint64_t &keyLevel) { zkassert(key.size() == 32); zkassert((version & U64Mask48) == version); @@ -136,7 +148,112 @@ zkresult KeyValueHistoryPage::Read (const uint64_t pageNumber, const string &key string keyBits; keyBits.append((char *)keyBitsArray, 43); - return Read(pageNumber, key, keyBits, version, value, 0); + return Read(pageNumber, key, keyBits, version, value, 0, keyLevel); +} + +zkresult KeyValueHistoryPage::ReadLevel (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t level, uint64_t &keyLevel) +{ + zkassert(key.size() == 32); + zkassert(keyBits.size() == 43); + zkassert(level < 43); + + zkresult zkr; + + // Get the data from this page + KeyValueHistoryStruct * page = (KeyValueHistoryStruct *)pageManager.getPageAddress(pageNumber); + uint64_t index = keyBits[level]; + uint64_t control = page->keyValueEntry[index][0] >> 60; + + // Check control + switch (control) + { + // Empty slot + case 0: + { + keyLevel = (level + 1) * 6; + return ZKR_SUCCESS; + } + // Leaf node + case 1: + { + uint64_t rawDataPage = page->keyValueEntry[index][1] & U64Mask48; + uint64_t rawDataOffset = page->keyValueEntry[index][1] >> 48; + string keyValue; + zkr = RawDataPage::Read(rawDataPage, rawDataOffset, 64, keyValue); + if (zkr != ZKR_SUCCESS) + { + zklog.error("KeyValueHistoryPage::ReadLevel() failed calling RawDataPage.Read result=" + zkresult2string(zkr) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset) + " key=" + ba2string(key) + " level=" + to_string(level) + " index=" + to_string(index)); + return zkr; + } + if (keyValue.size() != 64) + { + zklog.error("KeyValueHistoryPage::ReadLevel() called RawDataPage.Read and got invalid length=" + to_string(keyValue.size()) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset) + " key=" + ba2string(key) + " level=" + to_string(level) + " index=" + to_string(index)); + return zkr; + } + + // If this is the same key + if (memcmp(key.c_str(), keyValue.c_str(), 32) != 0) + { + // Get the key level + keyLevel = (level + 1) * 6; + + return ZKR_SUCCESS; + } + + // If keys are different, we need to know how different they are + Goldilocks::Element keyFea[4]; + string2fea(fr, ba2string(keyValue.substr(0, 32)), keyFea); + uint8_t foundKeyBitsArray[43]; + splitKey6(fr, keyFea, foundKeyBitsArray); + string foundKeyBits; + foundKeyBits.append((char *)foundKeyBitsArray, 43); + + + // Find the first 6-bit set that is different + uint64_t i=0; + for (; i<43; i++) + { + if (keyBits[i] != foundKeyBits[i]) + { + break; + } + } + + // Set the level + keyLevel = (i + 1) * 6; + + return ZKR_SUCCESS; + } + // Intermediate node + case 2: + { + uint64_t nextPageNumber = page->keyValueEntry[index][1] & U64Mask48; + return ReadLevel(nextPageNumber, key, keyBits, level + 1, keyLevel); + } + default: + { + zklog.error("KeyValuePage::ReadLevel() found invalid control=" + to_string(control) + " pageNumber=" + to_string(pageNumber)); + return ZKR_DB_ERROR; + } + } + + // Not found + return ZKR_DB_KEY_NOT_FOUND; +} + +zkresult KeyValueHistoryPage::ReadLevel (const uint64_t pageNumber, const string &key, uint64_t &keyLevel) +{ + zkassert(key.size() == 32); + + // Get 256 key bits in SMT order, in sets of 6 bits + Goldilocks::Element keyFea[4]; + string2fea(fr, ba2string(key), keyFea); + uint8_t keyBitsArray[43]; + splitKey6(fr, keyFea, keyBitsArray); + string keyBits; + keyBits.append((char *)keyBitsArray, 43); + + return ReadLevel(pageNumber, key, keyBits, 0, keyLevel); } zkresult KeyValueHistoryPage::Write (uint64_t &pageNumber, const string &key, const string &keyBits, const uint64_t version, const mpz_class &value, const uint64_t level, uint64_t &headerPageNumber) diff --git a/src/hashdb64/page/key_value_history_page.hpp b/src/hashdb64/page/key_value_history_page.hpp index 2c0249731..93f076152 100644 --- a/src/hashdb64/page/key_value_history_page.hpp +++ b/src/hashdb64/page/key_value_history_page.hpp @@ -29,11 +29,13 @@ class KeyValueHistoryPage static const uint64_t minHistoryOffset = 8 + 8 + 64*3*8; // 1552 static const uint64_t maxHistoryOffset = 8 + 8 + 64*3*8 + 106*3*8; // 4096 private: - static zkresult Read (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t version, mpz_class &value, const uint64_t level); + static zkresult Read (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t version, mpz_class &value, const uint64_t level, uint64_t &keyLevel); + static zkresult ReadLevel (const uint64_t pageNumber, const string &key, const string &keyBits, const uint64_t level, uint64_t &keyLevel); static zkresult Write ( uint64_t &pageNumber, const string &key, const string &keyBits, const uint64_t version, const mpz_class &value, const uint64_t level, uint64_t &headerPageNumber); public: static zkresult InitEmptyPage (const uint64_t pageNumber); - static zkresult Read (const uint64_t pageNumber, const string &key, const uint64_t version, mpz_class &value); + static zkresult Read (const uint64_t pageNumber, const string &key, const uint64_t version, mpz_class &value, uint64_t &keyLevel); + static zkresult ReadLevel (const uint64_t pageNumber, const string &key, uint64_t &keyLevel); static zkresult Write ( uint64_t &pageNumber, const string &key, const uint64_t version, const mpz_class &value, uint64_t &headerPageNumber); static zkresult calculateHash (const uint64_t pageNumber, Goldilocks::Element (&hash)[4], uint64_t &headerPageNumber); diff --git a/src/hashdb64/page/key_value_page.cpp b/src/hashdb64/page/key_value_page.cpp index 28ea96dba..d33ce5377 100644 --- a/src/hashdb64/page/key_value_page.cpp +++ b/src/hashdb64/page/key_value_page.cpp @@ -10,6 +10,8 @@ #include "key_utils.hpp" #include "constants.hpp" +//#define LOG_KEY_VALUE_PAGE + zkresult KeyValuePage::InitEmptyPage (const uint64_t pageNumber) { KeyValueStruct * page = (KeyValueStruct *)pageManager.getPageAddress(pageNumber); @@ -82,6 +84,7 @@ zkresult KeyValuePage::Read (const uint64_t pageNumber, const string &key, const return ZKR_DB_KEY_NOT_FOUND; } +#ifdef LOG_KEY_VALUE_PAGE zklog.info("KeyValuePage::Read() read length=" + to_string(length) + " result=" + zkresult2string(zkr) + " pageNumber=" + to_string(pageNumber) + @@ -90,7 +93,8 @@ zkresult KeyValuePage::Read (const uint64_t pageNumber, const string &key, const " key=" + ba2string(key) + " rawDataPage=" + to_string(rawDataPage) + " rawDataOffset=" + to_string(rawDataOffset)); - +#endif + value = rawData.substr(4 + key.size()); return ZKR_SUCCESS; @@ -185,6 +189,7 @@ zkresult KeyValuePage::Write (uint64_t &pageNumber, const string &key, const vec return zkr; } +#ifdef LOG_KEY_VALUE_PAGE zklog.info("KeyValuePage::Write() wrote length=" + to_string(length) + " totalLength=" + to_string(lengthKeyValue.size()) + " result=" + zkresult2string(zkr) + @@ -196,6 +201,7 @@ zkresult KeyValuePage::Write (uint64_t &pageNumber, const string &key, const vec " rawDataOffset=" + to_string(rawDataOffset) + " leaving headerPage->rawDataPage=" + to_string(headerPage->rawDataPage) + " headerPage->rawDataOffset=" + to_string(RawDataPage::GetOffset(headerPage->rawDataPage))); +#endif // Update this entry as a leaf node (control = 1) page->key[index] = (uint64_t(1)<<60) | ((rawDataOffset & U64Mask12) << 48) | (rawDataPage & U64Mask48); @@ -282,8 +288,10 @@ zkresult KeyValuePage::Write (uint64_t &pageNumber, const string &key, const vec } uint64_t newIndex = existingKeyBits[level+1]; newPage->key[newIndex] = page->key[index]; - + +#ifdef LOG_KEY_VALUE_PAGE zklog.info("KeyValuePage::Write() moved existing key=" + ba2string(existingLengthAndKey.substr(4)) + " to new page=" + to_string(newPageNumber) + " at index=" + to_string(newIndex)); +#endif // Write the new key in the newly created page at the next level zkr = Write(newPageNumber, key, keyBits, value, level+1, headerPageNumber);