[3.2] benchmarking framework and its use in benchmarking crypto primitives

CMakeLists.txt

@@ -182,6 +182,7 @@ add_subdirectory( scripts )
 add_subdirectory( unittests )
 add_subdirectory( tests )
 add_subdirectory( tools )
+add_subdirectory( benchmark )
 
 option(DISABLE_WASM_SPEC_TESTS "disable building of wasm spec unit tests" OFF)
 

benchmark/CMakeLists.txt

@@ -0,0 +1,7 @@
+file(GLOB BENCHMARK "*.cpp")
+add_executable( benchmark ${BENCHMARK} )
+
+target_link_libraries( benchmark fc Boost::program_options )
+target_include_directories( benchmark PUBLIC
+                            ${CMAKE_CURRENT_SOURCE_DIR}
+                          )

benchmark/alt_bn_128.cpp

@@ -0,0 +1,123 @@
+#include <iostream>
+
+#include <fc/crypto/alt_bn128.hpp>
+
+#include <benchmark.hpp>
+
+namespace benchmark {
+
+using bytes = std::vector<char>;
+using g1g2_pair = std::vector<std::string>;
+
+void add_benchmarking() {
+   std::vector<unsigned char>  point1_raw = {
+      12,175,215,144,98,95,151,228,179,85,31,170,172,159,40,255,250,252,68,28,235,65,172,180,69,164,153,29,187,239,220,201,   // x
+      18,102,219,76,79,148,120,28,39,149,42,172,41,248,120,249,255,69,42,51,160,239,13,219,239,183,77,174,217,158,130,10,};  // y
+   std::vector<unsigned char>  point2_raw = {
+      12,144,10,32,104,85,103,36,222,232,48,152,108,217,40,145,230,48,8,54,0,7,134,164,7,10,139,110,95,205,124,121,       // x
+      22,254,176,251,18,168,78,220,142,100,102,113,58,176,83,186,212,62,154,138,235,135,34,46,237,117,54,36,198,40,79,73,};   // y 
+
+   bytes point1, point2;
+   point1.insert(point1.begin(), point1_raw.begin(), point1_raw.end());
+   point2.insert(point2.begin(), point2_raw.begin(), point2_raw.end());
+
+   auto f = [&]() {
+      auto res = fc::alt_bn128_add(point1, point2);
+      if (std::holds_alternative<fc::alt_bn128_error>(res)) {
+         std::cout << "alt_bn128_add failed: " 
+            << (int)std::get<fc::alt_bn128_error>(res) << std::endl;
+      }
+   };
+
+   benchmarking("alt_bn128_add", f);
+}
+
+void mul_benchmarking() {
+   std::vector<unsigned char>  point_raw = {
+      12,175,215,144,98,95,151,228,179,85,31,170,172,159,40,255,250,252,68,28,235,65,172,180,69,164,153,29,187,239,220,201,   // x
+      18,102,219,76,79,148,120,28,39,149,42,172,41,248,120,249,255,69,42,51,160,239,13,219,239,183,77,174,217,158,130,10,}; // y
+   std::vector<unsigned char>  scaler_raw = {
+     25,62,182,170,104,140,135,90,37,150,0,77,2,77,146,71,54,101,113,69,177,216,157,4,229,213,33,215,169,99,150,91, }; // scaler size 256 bits
+
+   bytes point, scaler;
+   point.insert(point.begin(), point_raw.begin(), point_raw.end());
+   scaler.insert(scaler.begin(), scaler_raw.begin(), scaler_raw.end());
+
+   auto f = [&]() {
+      auto res = fc::alt_bn128_mul(point, scaler);
+      if (std::holds_alternative<fc::alt_bn128_error>(res)) {
+         std::cout << "alt_bn128_mul failed: "
+            << (int)std::get<fc::alt_bn128_error>(res) << std::endl;
+      }
+   };
+
+   benchmarking("alt_bn128_mul", f);
+}
+
+void pair_benchmarking() {
+   std::vector<unsigned char>  g1_g2_pairs_raw = {
+      /* pair 1 */
+      12,175,215,144,98,95,151,228,179,85,31,170,172,159,40,255,250,252,68,28,235,65,172,180,69,164,153,29,187,239,220,201,18,102,219,76,79,148,120,28,39,149,42,172,41,248,120,249,255,69,42,51,160,239,13,219,239,183,77,174,217,158,130,10,
+      46,24,234,176,148,82,198,136,44,30,157,18,217,25,49,241,63,197,2,151,14,150,136,210,114,4,74,124,145,225,131,139,13,60,0,50,236,103,39,15,150,226,246,189,209,113,0,46,151,39,41,6,87,119,228,213,45,225,29,234,161,110,43,87,3,237,74,227,93,23,171,129,49,118,228,173,154,111,168,220,161,46,140,103,155,56,168,207,254,90,228,76,188,232,25,34,44,102,159,35,193,216,177,31,131,247,226,19,170,222,26,227,86,37,81,149,202,144,188,126,120,168,9,195,32,169,147,161,
+      /* pair,2,*/
+      12,144,10,32,104,85,103,36,222,232,48,152,108,217,40,145,230,48,8,54,0,7,134,164,7,10,139,110,95,205,124,121,22,254,176,251,18,168,78,220,142,100,102,113,58,176,83,186,212,62,154,138,235,135,34,46,237,117,54,36,198,40,79,73,
+      0,186,148,242,227,252,208,242,165,128,1,160,59,2,6,195,30,54,69,118,8,28,223,92,254,98,110,219,90,92,7,113,26,8,119,98,122,61,119,91,216,107,29,179,122,218,203,6,84,109,119,140,44,12,57,157,203,90,14,108,218,54,94,44,42,77,118,169,232,192,189,60,129,213,156,157,215,160,91,214,138,130,119,145,167,89,237,149,225,85,151,33,89,181,224,235,2,91,213,225,37,226,214,14,255,84,254,117,142,211,28,164,78,254,64,11,126,217,25,124,203,220,52,181,39,32,78,132,
+      /* pair 3 */
+      48,58,116,183,56,3,62,111,104,127,0,7,202,204,146,2,236,192,214,213,231,100,12,17,104,47,76,49,149,39,35,179,2,162,218,104,52,130,61,134,223,111,123,225,186,122,171,2,144,236,110,29,26,142,113,183,238,44,130,30,76,212,52,15,
+     19,206,16,79,220,158,153,194,12,48,10,79,173,10,97,199,113,25,255,87,220,3,102,235,164,170,50,240,177,237,223,205,11,75,211,28,143,229,192,35,171,167,172,238,138,235,82,134,111,165,144,29,118,150,179,21,158,9,202,2,242,109,33,148,14,188,33,145,213,186,0,126,178,10,131,168,121,66,121,193,106,209,176,34,176,41,145,227,3,55,245,150,82,218,232,155,31,153,213,183,157,2,159,247,25,69,49,215,219,36,46,5,192,205,201,182,72,189,84,62,61,47,136,81,51,65,231,161,
+     /* pair 4 */
+     27,31,150,92,17,82,135,210,46,161,0,24,16,199,200,165,29,165,157,168,222,9,83,17,44,27,64,226,208,112,223,128,20,253,182,130,45,130,249,9,168,206,153,86,197,214,59,248,241,191,93,113,70,113,247,244,43,214,240,246,24,38,33,50,
+     23,76,241,195,191,99,69,237,123,173,212,42,74,85,138,108,39,80,105,135,226,11,84,237,39,73,180,224,42,230,246,76,26,61,86,57,253,213,223,2,93,42,38,186,38,38,206,38,209,138,153,181,57,89,16,187,234,2,110,23,12,14,252,240,32,231,237,147,16,62,220,6,160,64,154,167,57,238,243,112,255,35,80,36,100,173,21,58,96,244,77,245,75,202,24,63,35,178,46,215,219,69,120,221,70,119,111,195,145,45,109,84,206,76,254,205,98,167,30,72,117,5,128,129,156,128,196,192,
+     /* pair 5 */
+     33,230,106,254,43,77,114,126,9,205,25,63,9,170,62,237,149,102,198,184,60,31,185,213,103,120,71,249,28,106,238,107,1,133,60,180,58,152,18,177,142,248,46,186,90,34,94,200,235,158,242,255,209,32,143,133,102,195,19,107,153,42,207,228,
+     30,43,165,137,133,231,242,156,65,126,94,159,26,27,78,164,165,159,225,150,220,158,247,118,118,170,195,122,12,125,8,34,39,57,159,231,167,233,3,47,243,142,174,224,162,38,147,65,74,191,98,144,126,125,81,177,75,140,161,27,34,97,46,249,8,65,220,15,139,144,78,12,72,239,54,163,39,6,30,147,154,208,89,111,170,126,6,214,32,181,195,140,215,53,218,239,44,43,229,0,169,242,16,227,163,185,224,21,142,203,135,66,60,53,91,80,48,112,8,13,226,206,58,249,163,52,196,53,
+     /* pair 6 */
+     14,157,100,136,255,206,130,101,56,187,235,39,67,209,67,124,40,174,153,135,155,26,166,170,118,193,244,21,70,71,120,150,3,189,254,120,183,98,23,175,175,28,195,233,165,224,35,95,247,132,40,64,26,208,70,206,247,92,134,105,118,181,21,242,
+     16,159,200,208,93,99,234,139,8,222,239,67,3,187,15,125,5,144,190,196,146,171,15,218,92,36,37,130,2,201,241,182,36,31,104,164,95,29,73,231,87,50,174,142,209,72,31,75,48,13,70,237,118,193,39,85,97,88,228,50,77,163,209,112,7,202,214,212,235,15,224,247,229,179,101,15,72,130,125,8,62,227,46,20,109,53,26,168,10,237,222,85,169,133,181,196,25,36,112,182,41,208,39,132,100,181,199,189,44,99,238,140,6,187,19,197,99,93,243,63,159,132,211,0,230,118,153,54,
+     /* pair 7 */
+     31,60,154,8,188,169,224,107,69,183,223,15,60,32,61,51,81,226,85,38,72,88,241,202,216,204,253,98,0,250,116,167,21,255,0,81,80,3,108,196,240,102,114,33,33,122,116,155,188,19,232,217,42,217,54,195,91,187,88,188,249,218,240,146,
+     12,28,171,60,8,166,151,19,125,237,128,127,207,183,19,1,188,189,105,181,170,172,8,180,30,175,4,108,191,252,197,161,37,122,14,151,117,34,172,227,236,154,84,99,189,197,91,46,43,148,174,218,83,61,196,48,131,45,242,61,92,242,56,251,39,125,111,11,24,37,76,62,151,59,183,3,156,187,249,232,229,56,230,171,55,2,246,70,74,102,0,156,185,237,126,90,0,30,248,164,199,146,66,237,158,134,139,168,128,189,126,55,35,198,48,18,112,40,63,70,254,246,88,111,254,67,213,223,
+     /* pair 8 */
+     27,106,52,232,120,148,201,234,161,123,171,160,223,151,6,105,206,47,94,147,47,22,27,21,248,20,223,51,116,192,29,221,25,149,109,33,131,198,32,101,157,94,62,164,205,151,23,108,214,43,231,7,95,228,183,1,242,205,19,62,124,79,48,203,
+     37,186,150,204,63,241,145,219,169,250,247,76,247,18,135,175,140,58,209,133,113,144,185,191,101,197,252,253,50,126,188,177,45,103,105,194,21,229,145,210,89,135,244,248,238,64,102,155,129,232,252,32,208,82,51,131,113,100,25,174,253,19,211,18,6,77,66,232,173,42,146,120,106,72,92,144,51,233,117,184,113,60,33,118,209,83,119,93,170,253,126,41,58,13,85,111,30,233,117,238,183,18,219,38,216,163,244,98,219,254,156,189,57,246,220,178,17,190,88,84,13,41,8,236,181,155,85,198,
+     /* pair 9 */
+     25,62,182,170,104,140,135,90,37,150,0,77,2,77,146,71,54,101,113,69,177,216,157,4,229,213,33,215,169,99,150,91,5,185,157,10,152,12,220,171,39,188,1,48,87,129,192,101,36,179,99,212,123,207,13,76,89,78,8,154,75,239,117,189,
+     19,87,226,242,101,183,176,42,195,135,40,225,195,207,13,62,60,37,28,251,13,165,96,198,173,250,251,107,25,141,77,68,45,115,128,233,135,104,231,12,192,206,60,249,137,191,114,141,3,235,34,189,208,170,23,59,231,12,197,116,5,188,52,20,11,153,186,15,96,240,54,123,212,242,168,141,151,187,17,38,185,238,196,242,26,194,193,18,34,107,95,238,108,154,140,224,13,76,71,149,81,56,85,83,56,59,203,183,170,10,167,200,128,135,105,19,93,203,131,146,116,209,62,27,81,80,234,172,
+     /* pair 10 */
+     30,204,212,77,212,159,135,173,195,194,165,112,62,248,180,204,66,73,253,99,65,111,39,171,19,211,171,203,35,66,146,20,33,241,46,6,167,133,80,76,238,165,59,232,120,58,211,157,50,212,86,191,95,6,134,164,36,227,79,58,119,98,108,171,
+     48,49,12,141,166,151,158,56,136,255,197,138,114,195,39,59,71,236,82,57,149,249,170,55,187,95,193,171,14,124,45,87,7,157,47,178,153,237,194,157,142,194,100,14,40,51,61,201,244,93,61,196,154,59,14,135,209,72,102,186,14,228,228,152,40,246,109,82,93,249,92,105,191,121,77,108,20,87,3,87,167,173,171,255,189,34,155,239,218,95,181,153,222,20,120,195,27,28,47,82,113,3,218,129,54,210,185,165,206,99,126,61,217,19,237,5,12,90,148,246,128,231,63,53,37,223,204,195
+   };
+
+   // benchmarking 1 pair of points
+   bytes g1_g2_1_pair;
+   g1_g2_1_pair.insert(g1_g2_1_pair.begin(), g1_g2_pairs_raw.begin(), g1_g2_pairs_raw.begin() + 384);
+   auto f_1_pair = [&]() {
+      auto res = fc::alt_bn128_pair(g1_g2_1_pair, [](){});
+      if (std::holds_alternative<fc::alt_bn128_error>(res)) {
+         std::cout << "alt_bn128_pair 1 pair failed: "
+            << (int)std::get<fc::alt_bn128_error>(res) << std::endl;
+      }
+   };
+   benchmarking("alt_bn128_pair (1 pair)", f_1_pair);
+
+   // benchmarking 10 pair of points
+   bytes g1_g2_10_pairs;
+   g1_g2_10_pairs.insert(g1_g2_10_pairs.begin(), g1_g2_pairs_raw.begin(), g1_g2_pairs_raw.end());
+   auto f_10_pairs = [&]() {
+      auto res = fc::alt_bn128_pair(g1_g2_10_pairs, [](){});
+      if (std::holds_alternative<fc::alt_bn128_error>(res)) {
+         std::cout << "alt_bn128_pair 10 pairs failed: "
+            << (int)std::get<fc::alt_bn128_error>(res) << std::endl;
+      }
+   };
+
+   benchmarking("alt_bn128_pair (10 pairs)", f_10_pairs);
+}
+
+void alt_bn_128_benchmarking() {
+   add_benchmarking();
+   mul_benchmarking();
+   pair_benchmarking();
+}
+
+} // benchmark

benchmark/benchmark.cpp

@@ -0,0 +1,73 @@
+#include <chrono>
+#include <iostream>
+#include <iomanip>
+#include <locale>
+
+#include <benchmark.hpp>
+
+namespace benchmark {
+
+// update this map when a new module is supported
+// key is the name and value is the function doing benchmarking
+std::map<std::string, std::function<void()>> modules {
+   { "alt_bn_128", alt_bn_128_benchmarking },
+   { "modexp", modexp_benchmarking }
+};
+
+// values to control cout format
+constexpr auto name_width = 30;
+constexpr auto runs_width = 5;
+constexpr auto time_width = 14;
+constexpr auto ns_width = 3;
+
+uint32_t num_runs = 1;
+
+std::map<std::string, std::function<void()>> get_modules() {
+   return modules;
+}
+
+void set_num_runs(uint32_t runs) {
+   num_runs = runs;
+}
+
+void printt_header() {
+   std::cout << std::left << std::setw(name_width) << "function"
+      << std::setw(runs_width) << "runs"
+      << std::setw(time_width + ns_width) << std::right << "avg"
+      << std::setw(time_width + ns_width) << "min"
+      << std::setw(time_width + ns_width) << "max"
+      << std::endl << std::endl;
+}
+
+void print_results(std::string name, uint32_t runs, uint64_t total, uint64_t min, uint64_t max) {
+   std::cout.imbue(std::locale(""));
+   std::cout
+      << std::setw(name_width) << std::left << name
+      << std::setw(runs_width)  << runs
+      // std::fixed for not printing 1234 in 1.234e3.
+      // setprecision(0) for not printing fractions
+      << std::right << std::fixed << std::setprecision(0)
+      << std::setw(time_width) << total/runs << std::setw(ns_width) << " ns"
+      << std::setw(time_width) << min << std::setw(ns_width) << " ns"
+      << std::setw(time_width) << max << std::setw(ns_width) << " ns"
+      << std::endl;
+}
+
+void benchmarking(std::string name, const std::function<void()>& func) {
+   uint64_t total {0}, min {std::numeric_limits<uint64_t>::max()}, max {0};
+
+   for (auto i = 0U; i < num_runs; ++i) {
+      auto start_time = std::chrono::steady_clock::now();
+      func();
+      auto end_time = std::chrono::steady_clock::now();
+
+      uint64_t duration = std::chrono::duration_cast<std::chrono::nanoseconds>(end_time - start_time).count();
+      total += duration;
+      min = std::min(min, duration);
+      max = std::max(max, duration);
+   }
+
+   print_results(name, num_runs, total, min, max);
+}
+
+} // benchmark

benchmark/benchmark.hpp

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <functional>
+#include <map>
+
+namespace benchmark {
+
+void set_num_runs(uint32_t runs);
+std::map<std::string, std::function<void()>> get_modules();
+void printt_header();
+
+void alt_bn_128_benchmarking();
+void modexp_benchmarking();
+
+void benchmarking(std::string name, const std::function<void()>& func);
+
+} // benchmark

benchmark/main.cpp

@@ -0,0 +1,80 @@
+#include <iostream>
+
+#include <boost/program_options.hpp>
+
+#include <benchmark.hpp>
+
+namespace bpo = boost::program_options;
+using bpo::options_description;
+using bpo::variables_map;
+
+int main(int argc, char* argv[]) {
+   uint32_t num_runs = 1;
+   std::string module_name;
+
+   auto modules = benchmark::get_modules();
+
+   options_description cli ("benchmark command line options");
+   cli.add_options()
+      ("module,m", bpo::value<std::string>(), "the module to be benchmarked; if this option is not present, all modules are benchmarked.")
+      ("list,l", "list of supported modules")
+      ("runs,r", bpo::value<uint32_t>(&num_runs)->default_value(1000), "the number of runs per function")
+      ("help,h", "execute the benchmarked functions for the specified number of times, and report average, minimum, and maximum time in nanoseconds");
+
+   variables_map vmap;
+   try {
+      bpo::store(bpo::parse_command_line(argc, argv, cli), vmap);
+      bpo::notify(vmap);
+
+      if (vmap.count("help") > 0) {
+         cli.print(std::cerr);
+         return 0;
+      }
+
+      if (vmap.count("list") > 0) {
+         auto first = true;
+         std::cout << "Supported modules are ";
+         for (auto& [name, f]: modules) {
+            if (first) {
+               first = false;
+            } else {
+               std::cout << ", ";
+            }
+            std::cout << name;
+         }
+         std::cout << std::endl;
+         return 0;
+      }
+
+      if (vmap.count("module") > 0) {
+         module_name = vmap["module"].as<std::string>();
+         if (modules.find(module_name) == modules.end()) {
+            std::cout << module_name << " is not supported" << std::endl;
+            return 1;
+         }
+      }
+   } catch (bpo::unknown_option &ex) {
+      std::cerr << ex.what() << std::endl;
+      cli.print (std::cerr);
+      return 1;
+   } catch( ... ) {
+      std::cerr << "unknown exception" << std::endl;
+   }
+
+   benchmark::set_num_runs(num_runs);
+   benchmark::printt_header();
+
+   if (module_name.empty()) {
+      for (auto& [name, f]: modules) {
+         std::cout << name << ":" << std::endl;
+         f();
+         std::cout << std::endl;
+      }
+   } else {
+      std::cout << module_name << ":" << std::endl;
+      modules[module_name]();
+      std::cout << std::endl;
+   }
+
+   return 0;
+}

benchmark/modexp.cpp

@@ -0,0 +1,59 @@
+#include <fc/crypto/modular_arithmetic.hpp>
+#include <fc/utility.hpp>
+
+#include <random>
+
+#include <benchmark.hpp>
+
+namespace benchmark {
+
+void modexp_benchmarking() {
+   std::mt19937 r(0x11223344);
+
+   auto generate_random_bytes = [](std::mt19937& rand_eng, unsigned int num_bytes) {
+      std::vector<char> result(num_bytes);
+
+      uint_fast32_t v = 0;
+      for(int byte_pos = 0, end = result.size(); byte_pos < end; ++byte_pos) {
+          if ((byte_pos & 0x03) == 0) { // if divisible by 4
+              v = rand_eng();
+          }
+          result[byte_pos] = v & 0xFF;
+          v >>= 8;
+      }
+
+      return result;
+   };
+
+   static constexpr unsigned int start_num_bytes = 128; // 64
+   static constexpr unsigned int end_num_bytes   = 256; // 512
+   static constexpr unsigned int delta_num_bytes = 128; // 64
+
+   static_assert(start_num_bytes <= end_num_bytes);
+   static_assert(delta_num_bytes > 0);
+   static_assert((end_num_bytes - start_num_bytes) % delta_num_bytes == 0);
+
+   for (unsigned int n = start_num_bytes, slot = 0; n <= end_num_bytes; n += delta_num_bytes, ++slot) {
+      auto base     = generate_random_bytes(r, n);
+      auto exponent = generate_random_bytes(r, n);
+      auto modulus  = generate_random_bytes(r, n);
+
+      auto f = [&]() {
+         fc::modexp(base, exponent, modulus);
+      };
+
+      benchmarking(std::to_string(n*8) + "-bit width", f);
+   }
+
+   // Running the above benchmark (using commented values for num_trials and *_num_bytes) with a release build on an AMD 3.4 GHz CPU
+   // provides average durations for executing mod_exp for increasing bit sizes for the value.
+
+   // For example: with 512-bit values, the average duration is approximately 40 microseconds; with 1024-bit values, the average duration
+   // is approximately 260 microseconds; with 2048-bit values, the average duration is approximately 2 milliseconds; and, with 4096-bit 
+   // values, the average duration is approximately 14 milliseconds.
+
+   // It appears that a model of the average time that scales quadratically with the bit size fits the empirically generated data well.
+   // TODO: See if theoretical analysis of the modular exponentiation algorithm also justifies quadratic scaling.
+}
+
+} // benchmark