Testing PM flexible memory allocation

0xPolygonHermez · Oct 23, 2023 · 1a410bf · 1a410bf
1 parent fe0a9cb
commit 1a410bf
Show file tree

Hide file tree

Showing 3 changed files with 67 additions and 26 deletions.
diff --git a/src/config/README.md b/src/config/README.md
@@ -74,7 +74,7 @@ The configuration parameters can be of different uses:
 |`hashDB64`|test|boolean|Use HashDB64 new database (experimental)|false|HASHDB64|
 |`kvDBMaxVersions`|production|u64|Maximum number of KV versionn in Database|131072|HASHDB64_MAX_VERSIONS|
 |`dbCacheSynchURL`|test|string|URL of the HashDB service to synchronize the Database cache (experimental)|""|DB_CACHE_SYNCH_URL|
-|`hashDBFileName`|test|string|core name used for the hashDB files (path,numbering and extension not included)|""|HASHDB_FILE_NAME|
+|`hashDBFileName`|test|string|core name used for the hashDB files (path,numbering and extension not included). If hashDBFileName is empty in-memory version of the hashDB is used (only for DEBUG purposes). |""|HASHDB_FILE_NAME|
 |`hashDBFileSize`|test|u64| hashDB files size in GB|128|HASHDB_FILE_SIZE|
 |`hashDBFolder`|test|string|folder containing the hashDB files|hashdb|HASHDB_FOLDER|
 |`aggregatorServerPort`|test|u16|Aggregator server GRPC port|50081|AGGREGATOR_SERVER_PORT|

diff --git a/test/hashdb/page_manager_test.cpp b/test/hashdb/page_manager_test.cpp
@@ -16,26 +16,31 @@
 #include "config.hpp"
 #include "header_page.hpp"
 
-#define TEST_FILE_IO 0
 
 uint64_t PageManagerTest (void)
 {
     TimerStart(PAGE_MANAGER_TEST);
     PageManagerAccuracyTest();
+    PageManagerDBResizeTest();
     //PageManagerPerformanceTest();
     TimerStopAndLog(PAGE_MANAGER_TEST);
     return 0;
 }
 
-uint64_t PageManagerPerformanceTest(void){
+uint64_t PageManagerPerformanceTest (void)
+{
 
     string fileName = "benchmark_file";
-    uint64_t fileSize = 1ULL<<37;
-    string folderName = "db";
+    uint64_t fileSize = 128;
+    string folderName = "pmtest";
     uint64_t numPositions = 20000;
     uint64_t numReps = 100;
     uint64_t printFreq = 10;
 
+    //remove folder if exists
+    std::string command = "rm -rf " + folderName;
+    system(command.c_str());
+
     // Create the state manager
     double start = omp_get_wtime();
 
@@ -71,7 +76,6 @@ uint64_t PageManagerPerformanceTest(void){
     double avgThroughputFlushLast = 0;
     double throughput = 0;
 
-
     for(uint64_t k=0; k<numReps;++k){
 
         //Generate randoms
@@ -83,25 +87,11 @@ uint64_t PageManagerPerformanceTest(void){
 
         //Change first value of each page
         start = omp_get_wtime();
-#if TEST_FILE_IO
-        char singlePage[4096];
-        uint64_t sum=0;
-        //#pragma omp parallel for num_threads(64)
-        for (uint64_t i = 0; i < numPositions; ++i) {
-            pageManagerFile.getPageAddressFile(position[i], singlePage);
-
-            //pageData[0] = pageData[2] + position[i];
-            sum += *reinterpret_cast<uint64_t*>(&singlePage[2]) + position[i];        }
-#else
         for (uint64_t i = 0; i < numPositions; ++i) {
             uint64_t* pageData = (uint64_t *)pageManagerFile.getPageAddress(position[i]);
             pageData[0] = pageData[2] + position[i];
         }
-#endif
         end = omp_get_wtime();
-#if TEST_FILE_IO
-        std::cout << "Sum: " << sum << std::endl;
-#endif
 
         throughput = numGBytes / (end-start);
         avgThroughputFirstWrite += throughput;
@@ -112,9 +102,7 @@ uint64_t PageManagerPerformanceTest(void){
         start = omp_get_wtime();
         for (uint64_t i = 0; i < numPositions; ++i) {
             uint64_t* pageData = (uint64_t *)pageManagerFile.getPageAddress(position[i]);
-#if !TEST_FILE_IO
-        assert(pageData[0] == (pageData[2] + position[i]) );
-#endif
+            assert(pageData[0] == (pageData[2] + position[i]) );
             pageData[1] = position[i];    
         }
         end = omp_get_wtime();
@@ -141,9 +129,7 @@ uint64_t PageManagerPerformanceTest(void){
         for (uint64_t i = 0; i < numPositions; ++i) {
             uint64_t* pageData = (uint64_t *)pageManagerFile2.getPageAddress(position[i]);
             if(position[i] != 0){
-#if !TEST_FILE_IO
-        assert(pageData[0] == (pageData[2] + position[i]) );
-#endif
+                assert(pageData[0] == (pageData[2] + position[i]) );
                 assert(pageData[1] == position[i] );
             }
         }
@@ -162,8 +148,13 @@ uint64_t PageManagerPerformanceTest(void){
             avgThroughputFlushLast = 0;
         }
     }
+
+    //delete folder
+    command = "rm -rf " + folderName;
+    system(command.c_str());
     return 0;
 }
+
 uint64_t PageManagerAccuracyTest (void)
 {
 
@@ -344,4 +335,53 @@ uint64_t PageManagerAccuracyTest (void)
 
 
     return 0;   
+}
+
+uint64_t PageManagerDBResizeTest (void)
+{
+    //
+    // Memory version
+    //
+    PageManager pageManagerMem;
+    Config configPM;
+    PageContext ctx(pageManagerMem, configPM);
+    pageManagerMem.init(ctx);
+    uint64_t initialFreePages = pageManagerMem.getNumFreePages();
+    for(uint64_t i=0; i<initialFreePages;++i){
+        pageManagerMem.getFreePage();
+    }
+    assert(pageManagerMem.getNumFreePages() == 0);
+    pageManagerMem.getFreePage();
+    assert(pageManagerMem.getNumFreePages() == initialFreePages+7);
+
+    //
+    // File version
+    //
+
+    //generate a file with 100 pages
+    const string fileName = "page_manager_test";
+    const string folderName = "pmtest";
+    const int file_size = 1;  //in GB
+
+    //delete folder (is exists)
+    std::string command = "rm -rf " + folderName;
+    system(command.c_str());
+
+    PageManager pageManagerFile;
+    Config configPMFile;
+    configPMFile.hashDBFileName = fileName;
+    configPMFile.hashDBFileSize = file_size;
+    configPMFile.hashDBFolder = folderName;
+    PageContext ctxf(pageManagerFile, configPMFile);
+    pageManagerFile.init(ctxf);
+
+    initialFreePages = pageManagerFile.getNumFreePages();
+    for(uint64_t i=0; i<initialFreePages;++i){
+        pageManagerFile.getFreePage();
+    }
+    assert(pageManagerFile.getNumFreePages() == 0);
+    pageManagerFile.getFreePage();
+    assert(pageManagerFile.getNumFreePages() == initialFreePages+7);
+
+    return 0;
 }
diff --git a/test/hashdb/page_manager_test.hpp b/test/hashdb/page_manager_test.hpp
@@ -6,5 +6,6 @@
 uint64_t PageManagerTest (void);
 uint64_t PageManagerAccuracyTest (void);
 uint64_t PageManagerPerformanceTest (void);
+uint64_t PageManagerDBResizeTest (void);
 
 #endif