From b989ecae722216f334b1e129366a388577e249df Mon Sep 17 00:00:00 2001 From: rickb80 Date: Thu, 12 Oct 2023 09:46:00 +0200 Subject: [PATCH 1/2] Add LevelTree and KVTree classes --- src/hashdb64/level_tree.cpp | 487 ++++++++++++++++++++++++++ src/hashdb64/level_tree.hpp | 82 +++++ src/hashdb64/level_tree_key_value.cpp | 99 ++++++ src/hashdb64/level_tree_key_value.hpp | 48 +++ 4 files changed, 716 insertions(+) create mode 100644 src/hashdb64/level_tree.cpp create mode 100644 src/hashdb64/level_tree.hpp create mode 100644 src/hashdb64/level_tree_key_value.cpp create mode 100644 src/hashdb64/level_tree_key_value.hpp diff --git a/src/hashdb64/level_tree.cpp b/src/hashdb64/level_tree.cpp new file mode 100644 index 000000000..85b7f1b42 --- /dev/null +++ b/src/hashdb64/level_tree.cpp @@ -0,0 +1,487 @@ +#include "level_tree.hpp" + +using namespace std; + +static const uint32_t mask16[] = {0x55555555, 0x33333333, 0x0F0F0F0F, 0x00FF00FF}; +static const uint64_t mask32[] = {0x5555555555555555, 0x3333333333333333, 0x0F0F0F0F0F0F0F0F, 0x00FF00FF00FF00FF, 0x0000FFFF0000FFFF}; +uint32_t interleave16(uint16_t a, uint16_t b); +uint64_t interleave32(uint32_t a, uint32_t b); + +LevelTree::LevelTree(uint64_t nBitsStep_, bool useInsertCouters_) +{ + postConstruct(nBitsStep_, useInsertCouters_); +} + +void LevelTree::postConstruct(uint64_t nBitsStep_, bool useInsertCounters_) +{ + assert(nBitsKey % nBitsStep_ == 0); + assert(nBitsStep_ <= 32); + nBitsStep = nBitsStep_; + stepMask = (1ULL << nBitsStep) - 1; + stepsPerKeyWord = 64 / nBitsStep; + nodeSize = 1 << nBitsStep; + nSteps = nBitsKey / nBitsStep; + useInsertCounters = useInsertCounters_; + if (useInsertCounters) + { + pileSlotSize = 5; + } + else + { + pileSlotSize = 4; + } + nodes.resize(4096 * nodeSize, 0); + nNodes = 1; + pile.resize(1024 * pileSlotSize, 0); + nKeys = 1; + emptyNodes.resize(512, 0); + nEmptyNodes = 0; + emptyKeys.resize(512, 0); + nEmptyKeys = 0; +} + +uint64_t LevelTree::insert(const uint64_t key[4], int64_t *pileIdx) +{ + uint64_t currentNodeIdx = 0; + int64_t nextNodeIdx; + uint64_t step; + uint64_t mixedKey[4]; + mixKey(key, mixedKey); + + for (step = 0; step < nSteps; step++) + { + uint64_t index = getStepIndex(mixedKey, step); + nextNodeIdx = nodes[currentNodeIdx * nodeSize + index]; + if (nextNodeIdx == 0) + { + int64_t auxPileIdx = addKey(mixedKey); + nodes[currentNodeIdx * nodeSize + index] = -auxPileIdx; + if (pileIdx != nullptr) + *pileIdx = auxPileIdx; + return step * nBitsStep + levelInNode(currentNodeIdx, index); + } + else if (nextNodeIdx > 0) + { + currentNodeIdx = nextNodeIdx; + } + else + { + uint64_t pileKey[4]; + getKey(-nextNodeIdx, pileKey); + if (pileKey[0] != mixedKey[0] || pileKey[1] != mixedKey[1] || pileKey[2] != mixedKey[2] || pileKey[3] != mixedKey[3]) + { + int64_t nextPileIdxKey = -addKey(mixedKey); + if (pileIdx != nullptr) + *pileIdx = -nextPileIdxKey; + int64_t nextPileIdxPKey = nextNodeIdx; + uint64_t nextIndexKey; + uint64_t nextIndexPKey; + do + { + ++step; + int64_t addedNodeIdx = addNode(); + nodes[currentNodeIdx * nodeSize + index] = addedNodeIdx; + + nextIndexKey = getStepIndex(mixedKey, step); + nextIndexPKey = getStepIndex(pileKey, step); + if (nextIndexKey != nextIndexPKey) + { + nodes[addedNodeIdx * nodeSize + nextIndexKey] = nextPileIdxKey; + nodes[addedNodeIdx * nodeSize + nextIndexPKey] = nextPileIdxPKey; + return step * nBitsStep + levelInNode(addedNodeIdx, nextIndexKey); + } + else + { + index = nextIndexKey; + currentNodeIdx = addedNodeIdx; + } + } while (step < nSteps); + } + else + { + if (useInsertCounters) + { + uint64_t offset = pileSlotSize * -nextNodeIdx; + pile[offset + 4]++; + } + if (pileIdx != nullptr) + *pileIdx = -nextNodeIdx; + return step * nBitsStep + levelInNode(currentNodeIdx, index); + } + break; + } + } + assert(0); // should never reach this point + return 0; +} + +uint64_t LevelTree::level(const uint64_t key[4], int64_t *pileIdx) +{ + uint64_t currentNodeIdx = 0; + int64_t nextNodeIdx; + uint64_t step; + uint64_t mixedKey[4]; + mixKey(key, mixedKey); + + for (step = 0; step < nSteps; step++) + { + uint64_t index = getStepIndex(mixedKey, step); + nextNodeIdx = nodes[currentNodeIdx * nodeSize + index]; + if (nextNodeIdx == 0) + { + if (pileIdx != nullptr) + { + *pileIdx = -1; + } + return step * nBitsStep + levelInNode(currentNodeIdx, index); + } + else if (nextNodeIdx > 0) + { + currentNodeIdx = nextNodeIdx; + } + else + { + uint64_t pileKey[4]; + getKey(-nextNodeIdx, pileKey); + uint64_t nbits = step * nBitsStep + levelInNode(currentNodeIdx, index); + uint64_t common = commonBits(mixedKey, pileKey, nbits); + if (pileIdx != nullptr) + { + if (common == nbits) + { + *pileIdx = -nextNodeIdx; + } + else + { + *pileIdx = -1; + } + } + return common; + } + } + assert(0); // should never reach this point + return 0; +} + +bool LevelTree::extract(const uint64_t key[4], int64_t *pileIdx) +{ + int64_t nextId; + vector nodeIds(nSteps); + nodeIds[0] = 0; + uint64_t mixedKey[4]; + mixKey(key, mixedKey); + + for (uint64_t step = 0; step < nSteps; step++) + { + uint64_t index = getStepIndex(mixedKey, step); + nextId = nodes[nodeIds[step] * nodeSize + index]; + + if (nextId == 0) + { + if (pileIdx != nullptr) + *pileIdx = -1; + return false; + } + else if (nextId > 0) + { + nodeIds[step + 1] = nextId; + } + else if (nextId < 0) + { + uint64_t pileKey[4]; + getKey(-nextId, pileKey); + if (pileKey[0] != mixedKey[0] || + pileKey[1] != mixedKey[1] || + pileKey[2] != mixedKey[2] || + pileKey[3] != mixedKey[3]) + { + if (pileIdx != nullptr) + *pileIdx = -1; + return false; + } + else + { + if (pileIdx != nullptr) + *pileIdx = -nextId; + if (removeKey(-nextId)) + { + nodes[nodeIds[step] * nodeSize + index] = 0; + } + + while (step > 0) + { + // Find if there is a unique neighbour at same node and with same level + uint64_t nneigh = 0; + uint64_t nodeIndexNeighbour = 0; + int64_t pileIndexNeighbour = 0; + for (uint64_t i = 0; i < nodeSize; ++i) + { + uint64_t aux = nodes[nodeIds[step] * nodeSize + i]; + if (aux != 0) + { + ++nneigh; + if (nneigh > 1) + { + return true; + } + pileIndexNeighbour = aux; + nodeIndexNeighbour = i; + } + } + if (nneigh == 1 && pileIndexNeighbour < 0) + { + nodes[nodeIds[step] * nodeSize + nodeIndexNeighbour] = 0; // REUSE THIS NODE POSITION LATTER + step--; + uint64_t keyNeighbour[4]; + getKey(-pileIndexNeighbour, keyNeighbour); + uint64_t indexPrev = getStepIndex(keyNeighbour, step); + nodes[nodeIds[step] * nodeSize + indexPrev] = pileIndexNeighbour; + } + else + { + return true; + } + } + return true; + } + } + } + assert(0); // should never reach this point + return false; +} + +// Returns node index +int64_t LevelTree::addNode() +{ + int64_t nodeId; + if (nEmptyNodes > 0) + { + nodeId = emptyNodes[nEmptyNodes - 1]; + nEmptyNodes--; + } + else + { + nodeId = nNodes; + nNodes++; + if (nodeSize * nNodes == nodes.size()) + { + nodes.resize(nodes.size() * 2, 0); + } + } + return nodeId; +} + +// Returns Key index +int64_t LevelTree::addKey(const uint64_t key[4]) +{ + int64_t keyId; + if (nEmptyKeys > 0) + { + keyId = emptyKeys[nEmptyKeys - 1]; + nEmptyKeys--; + } + else + { + keyId = nKeys; + nKeys++; + if (pileSlotSize * nKeys == pile.size()) + { + pile.resize(pile.size() * 2, 0); + } + } + uint64_t offset = pileSlotSize * keyId; + pile[offset] = key[0]; + pile[offset + 1] = key[1]; + pile[offset + 2] = key[2]; + pile[offset + 3] = key[3]; + if (useInsertCounters) + pile[offset + 4]++; + + return keyId; +} + +void LevelTree::removeNode(uint64_t nodeId) +{ + assert(nodeId < nNodes); + memset(&nodes[nodeId * nodeSize], 0, nodeSize * sizeof(int64_t)); + emptyNodes[nEmptyNodes++] = nodeId; + if (nEmptyNodes == emptyNodes.size()) + { + emptyNodes.resize(emptyNodes.size() * 2, 0); + } +} + +bool LevelTree::removeKey(uint64_t keyIdx) +{ + assert(keyIdx < nKeys); + + uint64_t offset = pileSlotSize * keyIdx; + assert((!useInsertCounters) || (pile[offset + 4] > 0)); + if ((!useInsertCounters) || (pile[offset + 4] == 1)) + { + pile[offset] = 0; + pile[offset + 1] = 0; + pile[offset + 2] = 0; + pile[offset + 3] = 0; + if (useInsertCounters) + pile[offset + 4] = 0; + emptyKeys[nEmptyKeys++] = keyIdx; + if (nEmptyKeys == emptyKeys.size()) + { + emptyKeys.resize(emptyKeys.size() * 2, 0); + } + return true; + } + else + { + pile[offset + 4]--; + return false; + } +} + +uint64_t LevelTree::levelInNode(uint64_t nodeId, uint64_t localIdx) +{ + uint16_t globalIdx = localIdx + nodeId * nodeSize; + assert(localIdx < nodeSize); + uint64_t auxIndex = localIdx; + uint64_t nbits = 0; + uint64_t start = nodeId * nodeSize; + for (uint64_t j = 0; j < nBitsStep; ++j) + { + uint64_t inc = 1 << (nBitsStep - j - 1); + if (auxIndex >= inc) + { + start = start + inc; + auxIndex = auxIndex - inc; + } + bool found = false; + for (uint64_t i = start; i < start + inc; ++i) + { + if (nodes[i] != 0 && i != globalIdx) + { + ++nbits; + found = true; + break; + } + } + if (!found) + { + return nbits; + } + } + return nbits; +} + +uint64_t LevelTree::commonBits(const uint64_t key1[4], const uint64_t key2[4], uint64_t checkedBits) +{ + uint64_t commonWords = checkedBits >> 6; + for (int i = commonWords; i < 4; ++i) + { + if (key1[i] == key2[i]) + commonWords++; + else + break; + } + if (commonWords == 4) + { + return checkedBits; + } + uint64_t w1 = key1[commonWords]; + uint64_t w2 = key2[commonWords]; + + int count = 0; + uint64_t mask = 1ULL << 63; + + while (mask > 0) + { + if ((w1 & mask) == (w2 & mask)) + { + count++; + mask >>= 1; + } + else + break; + } + return commonWords * 64 + count; +} + +void LevelTree::mixKey(const uint64_t inKey[4], uint64_t outKey[4]) +{ + uint16_t k00 = inKey[0] >> 48; + uint16_t k01 = inKey[0] >> 32; + uint16_t k02 = inKey[0] >> 16; + uint16_t k03 = inKey[0]; + + uint16_t k10 = inKey[1] >> 48; + uint16_t k11 = inKey[1] >> 32; + uint16_t k12 = inKey[1] >> 16; + uint16_t k13 = inKey[1]; + + uint16_t k20 = inKey[2] >> 48; + uint16_t k21 = inKey[2] >> 32; + uint16_t k22 = inKey[2] >> 16; + uint16_t k23 = inKey[2]; + + uint16_t k30 = inKey[3] >> 48; + uint16_t k31 = inKey[3] >> 32; + uint16_t k32 = inKey[3] >> 16; + uint16_t k33 = inKey[3]; + + // interleave16 + + uint32_t k00k20 = interleave16(k00, k20); + uint32_t k10k30 = interleave16(k10, k30); + + uint32_t k01k21 = interleave16(k01, k21); + uint32_t k11k31 = interleave16(k11, k31); + + uint32_t k02k22 = interleave16(k02, k22); + uint32_t k12k32 = interleave16(k12, k32); + + uint32_t k03k23 = interleave16(k03, k23); + uint32_t k13k33 = interleave16(k13, k33); + + // interleave32 + outKey[0] = interleave32(k00k20, k10k30); + outKey[1] = interleave32(k01k21, k11k31); + outKey[2] = interleave32(k02k22, k12k32); + outKey[3] = interleave32(k03k23, k13k33); +} + +uint32_t interleave16(uint16_t x, uint16_t y) +{ + uint32_t a = x; + uint32_t b = y; + + a = (a | (a << 8)) & mask16[3]; + a = (a | (a << 4)) & mask16[2]; + a = (a | (a << 2)) & mask16[1]; + a = (a | (a << 1)) & mask16[0]; + + b = (b | (b << 8)) & mask16[3]; + b = (b | (b << 4)) & mask16[2]; + b = (b | (b << 2)) & mask16[1]; + b = (b | (b << 1)) & mask16[0]; + + return b | (a << 1); +} + +uint64_t interleave32(uint32_t x, uint32_t y) +{ + uint64_t a = x; + uint64_t b = y; + + a = (a | (a << 16)) & mask32[4]; + a = (a | (a << 8)) & mask32[3]; + a = (a | (a << 4)) & mask32[2]; + a = (a | (a << 2)) & mask32[1]; + a = (a | (a << 1)) & mask32[0]; + + b = (b | (b << 16)) & mask32[4]; + b = (b | (b << 8)) & mask32[3]; + b = (b | (b << 4)) & mask32[2]; + b = (b | (b << 2)) & mask32[1]; + b = (b | (b << 1)) & mask32[0]; + + return b | (a << 1); +} diff --git a/src/hashdb64/level_tree.hpp b/src/hashdb64/level_tree.hpp new file mode 100644 index 000000000..a86a9e163 --- /dev/null +++ b/src/hashdb64/level_tree.hpp @@ -0,0 +1,82 @@ +#ifndef LEVEL_TREE_HPP +#define LEVEL_TREE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +// +// LevelTree +// +class LevelTree +{ + +public: + LevelTree(){}; + LevelTree(uint64_t nBitsStep_, bool useInsertCounters_ = false); + void virtual postConstruct(uint64_t nBitsStep_, bool useInsertCounters_ = false); + ~LevelTree(){}; + + // main methods + uint64_t insert(const uint64_t key[4], int64_t *pileIdx = nullptr); + bool extract(const uint64_t key[4], int64_t *pileIdx = nullptr); // pileIdx = -1 if not found + uint64_t level(const uint64_t key[4], int64_t *pileIdx = nullptr); // pileIdx = -1 if not found + +protected: + const uint64_t nBitsKey = 256; + uint64_t nBitsStep; + uint64_t stepMask; + uint64_t stepsPerKeyWord; + uint64_t nodeSize; + uint64_t nSteps; + uint64_t pileSlotSize; + bool useInsertCounters; + + vector nodes; + uint64_t nNodes; + + vector pile; + uint64_t nKeys; + + vector emptyNodes; + uint64_t nEmptyNodes; + + vector emptyKeys; + uint64_t nEmptyKeys; + + // auxiliary methods + int64_t addNode(); + void removeNode(uint64_t nodeId); + virtual int64_t addKey(const uint64_t key[4]); + bool removeKey(uint64_t keyId); + inline void getKey(uint64_t keyId, uint64_t key[4]); + inline uint64_t getStepIndex(uint64_t key[4], uint64_t step); + uint64_t levelInNode(uint64_t nodeId, uint64_t localIdx); + uint64_t commonBits(const uint64_t key1[4], const uint64_t key2[4], uint64_t checkedBits = 0); + void mixKey(const uint64_t inKey[4], uint64_t outKey[4]); +}; + +uint64_t LevelTree::getStepIndex(uint64_t key[4], uint64_t step) +{ + uint64_t indx = (step * nBitsStep) >> 6; + uint64_t stepInWord = step - indx * stepsPerKeyWord; + return (key[indx] >> (64 - ((stepInWord + 1) * nBitsStep))) & stepMask; +} +void LevelTree::getKey(uint64_t keyId, uint64_t key[4]) +{ + uint64_t offset = pileSlotSize * keyId; + key[0] = pile[offset]; + key[1] = pile[offset + 1]; + key[2] = pile[offset + 2]; + key[3] = pile[offset + 3]; +} + +#endif \ No newline at end of file diff --git a/src/hashdb64/level_tree_key_value.cpp b/src/hashdb64/level_tree_key_value.cpp new file mode 100644 index 000000000..16f062bca --- /dev/null +++ b/src/hashdb64/level_tree_key_value.cpp @@ -0,0 +1,99 @@ +#include "level_tree_key_value.hpp" + +KVTree::KVTree(uint64_t nBitsStep_) +{ + postConstruct(nBitsStep_); +} + +void KVTree::postConstruct(uint64_t nBitsStep_) +{ + assert(nBitsKey % nBitsStep_ == 0); + assert(nBitsStep_ <= 32); + nBitsStep = nBitsStep_; + stepMask = (1ULL << nBitsStep) - 1; + stepsPerKeyWord = 64 / nBitsStep; + nodeSize = 1 << nBitsStep; + nSteps = nBitsKey / nBitsStep; + useInsertCounters = true; + pileSlotSize = 6; + + nodes.resize(4096 * nodeSize, 0); + nNodes = 1; + pile.resize(1024 * pileSlotSize, 0); + nKeys = 1; + emptyNodes.resize(512, 0); + nEmptyNodes = 0; + emptyKeys.resize(512, 0); + nEmptyKeys = 0; + pileValues.resize(4096, {0, 0}); + nValues = 0; + emptyValues.resize(1024, 0); + nEmptyValues = 0; +} + +bool KVTree::read(const uint64_t key[4], mpz_class &value, uint64_t &level) +{ + int64_t pileIdx; + level = LevelTree::level(key, &pileIdx); + if (pileIdx == -1) + { + return false; + } + else + { + value = pileValues[pile[pileIdx * 6 + 5]].value; + return true; + } +} + +void KVTree::write(const uint64_t key[4], const mpz_class &value, uint64_t &level) +{ + int64_t pileIdx; + level = insert(key, &pileIdx); + addValue(pileIdx, value); +} + +bool KVTree::extract(const uint64_t key[4], mpz_class &value) +{ + int64_t pileIdx; + bool bfound = LevelTree::extract(key, &pileIdx); + if (bfound) + { + removeValue(pileIdx, value); + } + return bfound; +} +uint64_t KVTree::addValue(const uint64_t pileIdx, const mpz_class &value) +{ + int idx; + if (nEmptyValues > 0) + { + nEmptyValues--; + idx = emptyValues[nEmptyValues]; + } + else + { + idx = nValues; + nValues++; + if (nValues == pileValues.size()) + { + pileValues.resize(pileValues.size() * 2, {0, 0}); + } + } + pileValues[idx].value = value; + pileValues[idx].prev = pile[pileIdx * 6 + 5]; + pile[pileIdx * 6 + 5] = idx; + return idx; +} +void KVTree::removeValue(uint64_t pileIdx, mpz_class &value) +{ + uint64_t valIdx = pile[pileIdx * 6 + 5]; + value = pileValues[valIdx].value; + pile[pileIdx * 6 + 5] = pileValues[valIdx].prev; + emptyValues[nEmptyValues] = valIdx; + nEmptyValues++; + if (nEmptyValues == emptyValues.size()) + { + emptyValues.resize(emptyValues.size() * 2, 0); + } +} diff --git a/src/hashdb64/level_tree_key_value.hpp b/src/hashdb64/level_tree_key_value.hpp new file mode 100644 index 000000000..56d4ac42c --- /dev/null +++ b/src/hashdb64/level_tree_key_value.hpp @@ -0,0 +1,48 @@ +#ifndef KV_TREE_HPP +#define KV_TREE_HPP +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "level_tree.hpp" + +using namespace std; + +struct +{ + mpz_class value; + uint32_t prev; +} typedef ListedValue; + +// +// KVTree +// +class KVTree : public LevelTree +{ +public: + KVTree(){}; + KVTree(uint64_t nBitsStep_); + void postConstruct(uint64_t nBitsStep_); + ~KVTree(){}; + + bool read(const uint64_t key[4], mpz_class &value, uint64_t &level); + void write(const uint64_t key[4], const mpz_class &value, uint64_t &level); + bool extract(const uint64_t key[4], mpz_class &value); + +protected: + uint64_t addValue(const uint64_t pileIdx, const mpz_class &value); + void removeValue(const uint64_t pileIdx, mpz_class &value); + + vector pileValues; + uint64_t nValues = 0; + + vector emptyValues; + uint64_t nEmptyValues = 0; +}; + +#endif \ No newline at end of file From 947f8900744e819f4054df6221dbae866b61243a Mon Sep 17 00:00:00 2001 From: rickb80 Date: Thu, 12 Oct 2023 10:32:39 +0200 Subject: [PATCH 2/2] Add tests for KeyValueTree --- src/config/config.cpp | 3 + src/config/config.hpp | 1 + src/main.cpp | 6 + test/hashdb/key_value_tree_test.cpp | 849 ++++++++++++++++++++++++++++ test/hashdb/key_value_tree_test.hpp | 8 + 5 files changed, 867 insertions(+) create mode 100644 test/hashdb/key_value_tree_test.cpp create mode 100644 test/hashdb/key_value_tree_test.hpp diff --git a/src/config/config.cpp b/src/config/config.cpp index dd2339cab..affe96c87 100644 --- a/src/config/config.cpp +++ b/src/config/config.cpp @@ -147,6 +147,7 @@ void Config::load(json &config) ParseString(config, "checkTreeRoot", "CHECK_TREE_ROOT", checkTreeRoot, "auto"); ParseBool(config, "runDatabasePerformanceTest", "RUN_DATABASE_PERFORMANCE_TEST", runDatabasePerformanceTest, false); ParseBool(config, "runPageManagerTest", "RUN_PAGE_MANAGER_TEST", runPageManagerTest, false); + ParseBool(config, "runKeyValueTreeTest", "RUN_KEY_VALUE_TREE_TEST", runKeyValueTreeTest, false); ParseBool(config, "runSMT64Test", "RUN_SMT64_TEST", runSMT64Test, false); ParseBool(config, "runUnitTest", "RUN_UNIT_TEST", runUnitTest, false); @@ -366,6 +367,8 @@ void Config::print(void) zklog.info(" runDatabasePerformanceTest=true"); if (runPageManagerTest) zklog.info(" runPageManagerTest=true"); + if (runKeyValueTreeTest) + zklog.info(" runKeyValueTreeTest=true"); if (runSMT64Test) zklog.info(" runSMT64Test=true"); if (runUnitTest) diff --git a/src/config/config.hpp b/src/config/config.hpp index e8e12fb5f..41c41e968 100644 --- a/src/config/config.hpp +++ b/src/config/config.hpp @@ -44,6 +44,7 @@ class Config string checkTreeRoot; bool runDatabasePerformanceTest; bool runPageManagerTest; + bool runKeyValueTreeTest; bool runSMT64Test; bool runUnitTest; diff --git a/src/main.cpp b/src/main.cpp index 1543b05fe..b749e9c19 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -45,6 +45,7 @@ #include "smt_64_test.hpp" #include "sha256.hpp" #include "page_manager_test.hpp" +#include "key_value_tree_test.hpp" using namespace std; using json = nlohmann::json; @@ -492,6 +493,11 @@ int main(int argc, char **argv) { PageManagerTest(); } + // Test KeyValueTree + if (config.runKeyValueTreeTest) + { + KeyValueTreeTest(); + } // Test SMT64 if (config.runSMT64Test) diff --git a/test/hashdb/key_value_tree_test.cpp b/test/hashdb/key_value_tree_test.cpp new file mode 100644 index 000000000..c93636ac2 --- /dev/null +++ b/test/hashdb/key_value_tree_test.cpp @@ -0,0 +1,849 @@ +#include "key_value_tree_test.hpp" +#include "level_tree_key_value.hpp" +#include "level_tree.hpp" +#include "timer.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +void test_LevelTree_insertCounters() +{ + int level; + bool bfound; + // + // test LevelTree InsertCounters + // + // std::cout << "test LevelTree: useInsertCounters=true" << std::endl; + LevelTree tree(4, true); + std::string binaryStr0 = "1101001110101011010000111011100101010111101011010101101101010101"; + std::string binaryStr1 = "1011101010110011101100101011011000100100010010000010101101000111"; + std::string binaryStr2 = "1101111010100011110001101110000010101010101011101001001111111011"; + std::string binaryStr3 = "1001001110101110011000010010001010101011111100011111001110000110"; + + uint64_t key1[4]; + key1[0] = std::bitset<64>(binaryStr0).to_ullong(); + key1[1] = std::bitset<64>(binaryStr1).to_ullong(); + key1[2] = std::bitset<64>(binaryStr2).to_ullong(); + key1[3] = std::bitset<64>(binaryStr3).to_ullong(); + + level = tree.insert(key1); + assert(level == 0); + + // key2: 1 bits in common with key1 (bits of components are interleaved) + std::string binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + std::string binaryStr1_ = "0011101010110011101100101011011000100100010010000010101101000111"; + std::string binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + std::string binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key2[4]; + key2[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key2[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key2[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key2[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.insert(key2); + assert(level == 1); + bfound = tree.extract(key2); + assert(bfound == true); + level = tree.level(key1); + assert(level == 0); + level = tree.insert(key2); + assert(level == 1); + level = tree.insert(key2); + assert(level == 1); + level = tree.insert(key2); + assert(level == 1); + bfound = tree.extract(key2); + assert(bfound == true); + bfound = tree.extract(key2); + assert(bfound == true); + bfound = tree.extract(key2); + assert(bfound == true); + bfound = tree.extract(key2); + assert(bfound == false); + + // key3: 201 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000000101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key3[4]; + key3[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key3[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key3[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key3[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.insert(key3); + assert(level == 201); + bfound = tree.extract(key3); + assert(bfound == true); + level = tree.level(key1); + assert(level == 0); + level = tree.insert(key3); + assert(level == 201); + level = tree.insert(key3); + assert(level == 201); + level = tree.insert(key3); + assert(level == 201); + bfound = tree.extract(key3); + assert(bfound == true); + bfound = tree.extract(key3); + assert(bfound == true); + bfound = tree.extract(key3); + assert(bfound == true); + bfound = tree.extract(key3); + assert(bfound == false); +} + +void test_LevelTree(uint64_t nBitsStep) +{ + int level; + bool bfound; + int64_t pileIdx; + LevelTree tree(nBitsStep); + // + // test LevelTree.insert & LevelTree.extract + // + // std::cout << "test LevelTree: level, insert & extract with nBitsStep: " << nBitsStep << std::endl; + std::string binaryStr0 = "1101001110101011010000111011100101010111101011010101101101010101"; + std::string binaryStr1 = "1011101010110011101100101011011000100100010010000010101101000111"; + std::string binaryStr2 = "1101111010100011110001101110000010101010101011101001001111111011"; + std::string binaryStr3 = "1001001110101110011000010010001010101011111100011111001110000110"; + + uint64_t key1[4]; + key1[0] = std::bitset<64>(binaryStr0).to_ullong(); + key1[1] = std::bitset<64>(binaryStr1).to_ullong(); + key1[2] = std::bitset<64>(binaryStr2).to_ullong(); + key1[3] = std::bitset<64>(binaryStr3).to_ullong(); + + bfound = tree.extract(key1, &pileIdx); + assert(pileIdx == -1); + assert(bfound == false); + pileIdx = 0; + level = tree.level(key1, &pileIdx); + assert(pileIdx == -1); + assert(level == 0); + level = tree.insert(key1, &pileIdx); + assert(level == 0); + assert(pileIdx == 1); + pileIdx = 0; + level = tree.level(key1, &pileIdx); + assert(level == 0); + assert(pileIdx == 1); + + // key2: 0 bits in common with key1 (bits of components are interleaved) + std::string binaryStr0_ = "0101001110101011010000111011100101010111101011010101101101010101"; + std::string binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + std::string binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + std::string binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key2[4]; + key2[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key2[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key2[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key2[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.level(key2, &pileIdx); + assert(level == 0); + assert(pileIdx == -1); + pileIdx = 0; + bfound = tree.extract(key2, &pileIdx); + assert(bfound == false); + assert(pileIdx == -1); + level = tree.insert(key2, &pileIdx); + assert(level == 0); + assert(pileIdx == 2); + level = tree.insert(key1, &pileIdx); + assert(level == 0); + assert(pileIdx == 1); + + // key3: 1 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "0011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key3[4]; + key3[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key3[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key3[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key3[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + bfound = tree.extract(key3); + assert(bfound == false); + level = tree.level(key3); + assert(level == 1); + level = tree.insert(key3); + assert(level == 1); + level = tree.level(key1); + assert(level == 1); + + bfound = tree.extract(key3); + assert(bfound == true); + level = tree.insert(key1); + assert(level == 0); + + level = tree.insert(key3); + assert(level == 1); + level = tree.level(key1); + assert(level == 1); + + // key4: 9 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1001101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key4[4]; + key4[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key4[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key4[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key4[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + bfound = tree.extract(key4); + assert(bfound == false); + level = tree.level(key4); + assert(level == 9); + level = tree.insert(key4); + assert(level == 9); + level = tree.level(key1); + assert(level == 9); + + bfound = tree.extract(key4); + assert(bfound == true); + level = tree.insert(key1); + assert(level == 1); + level = tree.insert(key4); + assert(level == 9); + level = tree.level(key1); + assert(level == 9); + + // key5: 63 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101111011000010010001010101011111100011111001110000110"; + uint64_t key5[4]; + key5[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key5[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key5[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key5[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + bfound = tree.extract(key5); + assert(bfound == false); + level = tree.level(key5); + assert(level == 63); + level = tree.insert(key5); + assert(level == 63); + level = tree.insert(key1); + assert(level == 63); + + bfound = tree.extract(key5); + assert(bfound == true); + level = tree.insert(key1); + assert(level == 9); + + bfound = tree.extract(key4); + assert(bfound == true); + level = tree.level(key1); + assert(level == 1); + + // key6: 79 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011100010010001010101011111100011111001110000110"; + uint64_t key6[4]; + key6[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key6[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key6[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key6[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.level(key6); + assert(level == 79); + bfound = tree.extract(key6); + assert(bfound == false); + level = tree.insert(key6); + assert(level == 79); + level = tree.level(key1); + assert(level == 79); + + // key7: 187 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100111111001110000110"; + uint64_t key7[4]; + key7[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key7[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key7[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key7[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + bfound = tree.extract(key7); + assert(bfound == false); + level = tree.level(key7); + assert(level == 187); + level = tree.insert(key7); + assert(level == 187); + level = tree.level(key1); + assert(level == 187); + + // key8: 201 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000000101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key8[4]; + key8[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key8[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key8[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key8[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + bfound = tree.extract(key8); + assert(bfound == false); + level = tree.level(key8); + assert(level == 201); + level = tree.insert(key8); + assert(level == 201); + level = tree.level(key1); + assert(level == 201); + + bfound = tree.extract(key8); + assert(bfound == true); + level = tree.insert(key1); + assert(level == 187); + + bfound = tree.extract(key7); + assert(bfound == true); + level = tree.insert(key1); + assert(level == 79); + + bfound = tree.extract(key6); + assert(bfound == true); + level = tree.level(key1); + assert(level == 1); +} + +void test_LevelTree_resize() +{ + int level; + uint64_t nBitsStep = 4; + LevelTree tree(nBitsStep); + // + // test LevelTree.insert & LevelTree.extract + // + // std::cout << "test LevelTree: resize " << std::endl; + std::string binaryStr0 = "1101001110101011010000111011100101010111101011010101101101010101"; + std::string binaryStr1 = "1011101010110011101100101011011000100100010010000010101101000111"; + std::string binaryStr2 = "1101111010100011110001101110000010101010101011101001001111111011"; + std::string binaryStr3 = "1001001110101110011000010010001010101011111100011111001110000110"; + + uint64_t key1[4]; + key1[0] = std::bitset<64>(binaryStr0).to_ullong(); + key1[1] = std::bitset<64>(binaryStr1).to_ullong(); + key1[2] = std::bitset<64>(binaryStr2).to_ullong(); + key1[3] = std::bitset<64>(binaryStr3).to_ullong(); + + level = tree.level(key1); + assert(level == 0); + level = tree.insert(key1); + assert(level == 0); + level = tree.level(key1); + assert(level == 0); + + // key2: 0 bits in common with key1 (bits of components are interleaved) + std::string binaryStr0_ = "0101001110101011010000111011100101010111101011010101101101010101"; + std::string binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + std::string binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + std::string binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key2[4]; + key2[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key2[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key2[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key2[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.level(key2); + assert(level == 0); + level = tree.insert(key2); + assert(level == 0); + level = tree.insert(key1); + assert(level == 0); + + // key3: 1 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "0011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key3[4]; + key3[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key3[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key3[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key3[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.level(key3); + assert(level == 1); + level = tree.insert(key3); + assert(level == 1); + level = tree.level(key1); + assert(level == 1); + + tree.extract(key3); + level = tree.insert(key1); + assert(level == 0); + + level = tree.insert(key3); + assert(level == 1); + level = tree.level(key1); + assert(level == 1); + + // key4: 9 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1001101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key4[4]; + key4[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key4[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key4[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key4[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.level(key4); + assert(level == 9); + level = tree.insert(key4); + assert(level == 9); + level = tree.level(key1); + assert(level == 9); + + tree.extract(key4); + level = tree.insert(key1); + assert(level == 1); + level = tree.insert(key4); + assert(level == 9); + level = tree.level(key1); + assert(level == 9); + + // key5: 63 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101111011000010010001010101011111100011111001110000110"; + uint64_t key5[4]; + key5[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key5[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key5[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key5[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.level(key5); + assert(level == 63); + level = tree.insert(key5); + assert(level == 63); + level = tree.insert(key1); + assert(level == 63); + + tree.extract(key5); + level = tree.insert(key1); + assert(level == 9); + + tree.extract(key4); + level = tree.level(key1); + assert(level == 1); + + // key6: 79 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011100010010001010101011111100011111001110000110"; + uint64_t key6[4]; + key6[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key6[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key6[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key6[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.level(key6); + assert(level == 79); + level = tree.insert(key6); + assert(level == 79); + level = tree.level(key1); + assert(level == 79); + + // key7: 187 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100111111001110000110"; + uint64_t key7[4]; + key7[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key7[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key7[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key7[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.level(key7); + assert(level == 187); + level = tree.insert(key7); + assert(level == 187); + level = tree.level(key1); + assert(level == 187); + + // key8: 201 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000000101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key8[4]; + key8[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key8[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key8[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key8[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.level(key8); + assert(level == 201); + level = tree.insert(key8); + assert(level == 201); + level = tree.level(key1); + assert(level == 201); + + tree.extract(key8); + level = tree.insert(key1); + assert(level == 187); + + tree.extract(key7); + level = tree.insert(key1); + assert(level == 79); + + tree.extract(key6); + level = tree.level(key1); + assert(level == 1); +} + +void test_KVTree_asLevelTree() +{ + int level; + bool bfound; + KVTree tree(4); + // + // test LevelTree.insert & LevelTree.extract + // + // std::cout << "test KVTree: as LevelTree with nBitsStep: 4 " << std::endl; + std::string binaryStr0 = "1101001110101011010000111011100101010111101011010101101101010101"; + std::string binaryStr1 = "1011101010110011101100101011011000100100010010000010101101000111"; + std::string binaryStr2 = "1101111010100011110001101110000010101010101011101001001111111011"; + std::string binaryStr3 = "1001001110101110011000010010001010101011111100011111001110000110"; + + uint64_t key1[4]; + key1[0] = std::bitset<64>(binaryStr0).to_ullong(); + key1[1] = std::bitset<64>(binaryStr1).to_ullong(); + key1[2] = std::bitset<64>(binaryStr2).to_ullong(); + key1[3] = std::bitset<64>(binaryStr3).to_ullong(); + + bfound = tree.LevelTree::extract(key1); + assert(bfound == false); + level = tree.level(key1); + assert(level == 0); + level = tree.insert(key1); + assert(level == 0); + level = tree.level(key1); + assert(level == 0); + + // key2: 0 bits in common with key1 (bits of components are interleaved) + std::string binaryStr0_ = "0101001110101011010000111011100101010111101011010101101101010101"; + std::string binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + std::string binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + std::string binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key2[4]; + key2[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key2[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key2[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key2[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.level(key2); + assert(level == 0); + bfound = tree.LevelTree::extract(key2); + assert(bfound == false); + level = tree.insert(key2); + assert(level == 0); + level = tree.insert(key1); + assert(level == 0); + + // key3: 1 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "0011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key3[4]; + key3[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key3[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key3[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key3[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + bfound = tree.LevelTree::extract(key3); + assert(bfound == false); + level = tree.level(key3); + assert(level == 1); + level = tree.insert(key3); + assert(level == 1); + level = tree.level(key1); + assert(level == 1); + + bfound = tree.LevelTree::extract(key3); + assert(bfound == true); + level = tree.insert(key1); + assert(level == 0); + + level = tree.insert(key3); + assert(level == 1); + level = tree.level(key1); + assert(level == 1); + + // key4: 9 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1001101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key4[4]; + key4[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key4[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key4[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key4[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + bfound = tree.LevelTree::extract(key4); + assert(bfound == false); + level = tree.level(key4); + assert(level == 9); + level = tree.insert(key4); + assert(level == 9); + level = tree.level(key1); + assert(level == 9); + + bfound = tree.LevelTree::extract(key4); + assert(bfound == true); + level = tree.insert(key1); + assert(level == 1); + level = tree.insert(key4); + assert(level == 9); + level = tree.level(key1); + assert(level == 9); + + // key5: 63 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101111011000010010001010101011111100011111001110000110"; + uint64_t key5[4]; + key5[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key5[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key5[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key5[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + bfound = tree.LevelTree::extract(key5); + assert(bfound == false); + level = tree.level(key5); + assert(level == 63); + level = tree.insert(key5); + assert(level == 63); + level = tree.insert(key1); + assert(level == 63); + + bfound = tree.LevelTree::extract(key5); + assert(bfound == true); + level = tree.insert(key1); + assert(level == 9); + + bfound = tree.LevelTree::extract(key4); + assert(bfound == true); + level = tree.level(key1); + assert(level == 1); + + // key6: 79 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011100010010001010101011111100011111001110000110"; + uint64_t key6[4]; + key6[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key6[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key6[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key6[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + level = tree.level(key6); + assert(level == 79); + bfound = tree.LevelTree::extract(key6); + assert(bfound == false); + level = tree.insert(key6); + assert(level == 79); + level = tree.level(key1); + assert(level == 79); + + // key7: 187 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100111111001110000110"; + uint64_t key7[4]; + key7[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key7[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key7[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key7[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + bfound = tree.LevelTree::extract(key7); + assert(bfound == false); + level = tree.level(key7); + assert(level == 187); + level = tree.insert(key7); + assert(level == 187); + level = tree.level(key1); + assert(level == 187); + + // key8: 201 bits in common with key1 (bits of components are interleaved) + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000000101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key8[4]; + key8[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key8[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key8[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key8[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + bfound = tree.LevelTree::extract(key8); + assert(bfound == false); + level = tree.level(key8); + assert(level == 201); + level = tree.insert(key8); + assert(level == 201); + level = tree.level(key1); + assert(level == 201); + + bfound = tree.LevelTree::extract(key8); + assert(bfound == true); + level = tree.insert(key1); + assert(level == 187); + + bfound = tree.LevelTree::extract(key7); + assert(bfound == true); + level = tree.insert(key1); + assert(level == 79); + + bfound = tree.LevelTree::extract(key6); + assert(bfound == true); + level = tree.level(key1); + assert(level == 1); +} + +void test_KVTree(uint64_t nBitsStep) +{ + uint64_t level; + bool bfound; + KVTree kvtree(nBitsStep); + mpz_class value; + // + // test LevelTree.insert & LevelTree.extract + // + // std::cout << "test KVTree: level, insert & extract with nBitsStep: " << nBitsStep << std::endl; + + std::string binaryStr0 = "1101001110101011010000111011100101010111101011010101101101010101"; + std::string binaryStr1 = "1011101010110011101100101011011000100100010010000010101101000111"; + std::string binaryStr2 = "1101111010100011110001101110000010101010101011101001001111111011"; + std::string binaryStr3 = "1001001110101110011000010010001010101011111100011111001110000110"; + + uint64_t key1[4]; + key1[0] = std::bitset<64>(binaryStr0).to_ullong(); + key1[1] = std::bitset<64>(binaryStr1).to_ullong(); + key1[2] = std::bitset<64>(binaryStr2).to_ullong(); + key1[3] = std::bitset<64>(binaryStr3).to_ullong(); + + bfound = kvtree.read(key1, value, level); + assert(bfound == false); + bfound = kvtree.extract(key1, value); + assert(bfound == false); + level = kvtree.level(key1); + assert(level == 0); + kvtree.write(key1, 1, level); + bfound = kvtree.read(key1, value, level); + assert(bfound == true && value == 1 && level == 0); + + std::string binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + std::string binaryStr1_ = "1011101010110011101100101011011000100100010010000000101101000111"; + std::string binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + std::string binaryStr3_ = "1001001110101110011000010010001010101011111100011111001110000110"; + uint64_t key2[4]; + key2[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key2[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key2[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key2[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + kvtree.write(key2, 2, level); + assert(level == 201); + bfound = kvtree.read(key1, value, level); + assert(bfound == true && value == 1 && level == 201); + bfound = kvtree.read(key2, value, level); + assert(bfound == true && value == 2 && level == 201); + value = 0; + bfound = kvtree.extract(key2, value); + assert(bfound == true && value == 2); + bfound = kvtree.read(key1, value, level); + assert(bfound == true && value == 1 && level == 0); + + binaryStr0_ = "1101001110101011010000111011100101010111101011010101101101010101"; + binaryStr1_ = "1011101010110011101100101011011000100100010010000010101101000111"; + binaryStr2_ = "1101111010100011110001101110000010101010101011101001001111111011"; + binaryStr3_ = "1001001110101110011000010010001010101011111100111111001110000110"; + uint64_t key3[4]; + key3[0] = std::bitset<64>(binaryStr0_).to_ullong(); + key3[1] = std::bitset<64>(binaryStr1_).to_ullong(); + key3[2] = std::bitset<64>(binaryStr2_).to_ullong(); + key3[3] = std::bitset<64>(binaryStr3_).to_ullong(); + + kvtree.write(key3, 2, level); + assert(level == 187); + bfound = kvtree.read(key1, value, level); + assert(bfound == true && value == 1 && level == 187); + kvtree.write(key1, 0, level); + assert(level == 187); + bfound = kvtree.read(key1, value, level); + assert(bfound == true && value == 0 && level == 187); + kvtree.write(key1, 10, level); + assert(level == 187); + bfound = kvtree.read(key1, value, level); + assert(bfound == true && value == 10 && level == 187); + kvtree.write(key1, 20, level); + assert(level == 187); + bfound = kvtree.read(key1, value, level); + assert(bfound == true && value == 20 && level == 187); + bfound = kvtree.extract(key1, value); + assert(bfound == true && value == 20); + bfound = kvtree.read(key1, value, level); + assert(bfound == true && value == 10 && level == 187); + kvtree.write(key1, 20, level); + assert(level == 187); + bfound = kvtree.read(key1, value, level); + assert(bfound == true && value == 20 && level == 187); + bfound = kvtree.extract(key1, value); + assert(bfound == true && value == 20); + bfound = kvtree.extract(key1, value); + assert(bfound == true && value == 10); + bfound = kvtree.extract(key1, value); + assert(bfound == true && value == 0); + bfound = kvtree.extract(key1, value); + assert(bfound == true && value == 1); + bfound = kvtree.extract(key1, value); + assert(bfound == false); +} + +uint64_t KeyValueTreeTest(void) +{ + TimerStart(KEY_VALUE_TREE_TEST); + + test_LevelTree(2); + test_LevelTree(4); + test_LevelTree(8); + test_LevelTree_resize(); + test_LevelTree_insertCounters(); + test_KVTree_asLevelTree(); + test_KVTree(2); + test_KVTree(4); + test_KVTree(8); + + TimerStopAndLog(KEY_VALUE_TREE_TEST); + return 0; +} \ No newline at end of file diff --git a/test/hashdb/key_value_tree_test.hpp b/test/hashdb/key_value_tree_test.hpp new file mode 100644 index 000000000..f2da72b9a --- /dev/null +++ b/test/hashdb/key_value_tree_test.hpp @@ -0,0 +1,8 @@ +#ifndef KEY_VALUE_TREE_TEST_HPP +#define KEY_VALUE_TREE_TEST_HPP + +#include + +uint64_t KeyValueTreeTest(void); + +#endif \ No newline at end of file