Merge pull request #350 from AntelopeIO/feature_performance_harness

[PH -> main] Performance Harness MVP [PH]

.cicd/platforms/ubuntu18.Dockerfile

@@ -13,10 +13,14 @@ RUN apt-get update && apt-get upgrade -y && \
                                                       llvm-7-dev                  \
                                                       ninja-build                 \
                                                       python3                     \
+                                                      python3-numpy               \
+                                                      python3-pip                 \
                                                       software-properties-common  \
                                                       zlib1g-dev                  \
                                                       zstd
 
+RUN python3 -m pip install dataclasses
+
 # GitHub's actions/checkout requires git 2.18+ but Ubuntu 18 only provides 2.17
 RUN add-apt-repository ppa:git-core/ppa && apt update && apt install -y git
 

.cicd/platforms/ubuntu20.Dockerfile

@@ -12,4 +12,5 @@ RUN apt-get update && apt-get upgrade -y && \
                                                       libssl-dev           \
                                                       llvm-11-dev          \
                                                       ninja-build          \
+                                                      python3-numpy        \
                                                       zstd

.cicd/platforms/ubuntu22.Dockerfile

@@ -12,4 +12,5 @@ RUN apt-get update && apt-get upgrade -y && \
                                                       libssl-dev           \
                                                       llvm-11-dev          \
                                                       ninja-build          \
+                                                      python3-numpy        \
                                                       zstd

README.md

@@ -138,7 +138,8 @@ sudo apt-get install -y \
         libcurl4-openssl-dev \
         libgmp-dev \
         libssl-dev \
-        llvm-11-dev
+        llvm-11-dev \
+        python3-numpy
 ```
 To build, make sure you are in the root of the `leap` repo, then run the following command:
 ```bash
@@ -165,7 +166,11 @@ sudo apt-get install -y \
         libssl-dev \
         llvm-7-dev \
         python3 \
+        python3-numpy \
+        python3-pip \
         zlib1g-dev
+
+python3 -m pip install dataclasses
 ```
 You need to build Boost from source on this distribution:
 ```bash

programs/eosio-launcher/main.cpp

@@ -214,6 +214,7 @@ class eosd_def {
   uint16_t     http_port;
   uint16_t     file_size;
   string       name;
+  string       dex;
   tn_node_def* node;
   string       host;
   string       p2p_endpoint;
@@ -425,6 +426,8 @@ struct launcher_def {
    string start_script;
    std::optional<uint32_t> max_block_cpu_usage;
    std::optional<uint32_t> max_transaction_cpu_usage;
+   std::string logging_level;
+   std::map<string,fc::log_level> logging_level_map;
    eosio::chain::genesis_state genesis_from_file;
 
    void assign_name (eosd_def &node, bool is_bios);
@@ -505,6 +508,8 @@ launcher_def::set_options (bpo::options_description &cfg) {
     ("script",bpo::value<string>(&start_script)->default_value("bios_boot.sh"),"the generated startup script name")
     ("max-block-cpu-usage",bpo::value<uint32_t>(),"Provide the \"max-block-cpu-usage\" value to use in the genesis.json file")
     ("max-transaction-cpu-usage",bpo::value<uint32_t>(),"Provide the \"max-transaction-cpu-usage\" value to use in the genesis.json file")
+    ("logging-level",bpo::value<string>(),"Provide the \"level\" value to use in the logging.json file ")
+    ("logging-level-map",bpo::value<string>(),"String of a dict which specifies \"level\" value to use in the logging.json file for specific nodes, matching based on node number. Ex: {\"bios\":\"off\",\"00\":\"info\"}")
         ;
 }
 
@@ -566,6 +571,21 @@ launcher_def::initialize (const variables_map &vmap) {
      max_transaction_cpu_usage = vmap["max-transaction-cpu-usage"].as<uint32_t>();
   }
 
+  if (vmap.count("logging-level")) {
+     logging_level = vmap["logging-level"].as<string>();
+  }
+  if (vmap.count("logging-level-map")) {
+     string llm_str = vmap["logging-level-map"].as<string>();
+     auto const regex = std::regex("\"(.*?)\":\"(.*?)\"");
+     for (auto it = std::sregex_iterator(llm_str.begin(), llm_str.end(), regex); it != std::sregex_iterator(); it++) {
+        std::smatch sm = *it;
+        fc::log_level ll;
+        fc::variant v(sm.str(2));
+        fc::from_variant(v, ll);
+        logging_level_map[sm.str(1)] = ll;
+     }
+  }
+
   genesis = vmap["genesis"].as<string>();
   if (vmap.count("host-map")) {
      host_map_file = vmap["host-map"].as<string>();
@@ -703,12 +723,14 @@ launcher_def::assign_name (eosd_def &node, bool is_bios) {
    if (is_bios) {
       node.name = "bios";
       node_cfg_name = "node_bios";
+      node.dex = "bios";
    }
    else {
       string dex = next_node < 10 ? "0":"";
       dex += boost::lexical_cast<string,int>(next_node++);
       node.name = network.name + dex;
       node_cfg_name = "node_" + dex;
+      node.dex = dex;
    }
   node.config_dir_name = (config_dir_base / node_cfg_name).string();
   node.data_dir_name = (data_dir_base / node_cfg_name).string();
@@ -1146,13 +1168,25 @@ launcher_def::write_logging_config_file(tn_node_def &node) {
   if (!bfs::exists(dd)) {
     bfs::create_directories(dd);
   }
+  fc::log_level ll = fc::log_level::debug;
+  if (logging_level != "") {
+     fc::variant v(logging_level);
+     fc::from_variant(v, ll);
+  }
 
   filename = dd / "logging.json";
 
+  if (!logging_level_map.empty()) {
+     auto it = logging_level_map.find(instance.dex);
+     if (it != logging_level_map.end()) {
+       ll = it->second;
+     }
+  }
+
   bfs::ofstream cfg(filename);
   if (!cfg.good()) {
-    cerr << "unable to open " << filename << " " << strerror(errno) << "\n";
-    exit (9);
+     cerr << "unable to open " << filename << " " << strerror(errno) << "\n";
+     exit (9);
   }
 
   auto log_config = fc::logging_config::default_config();
@@ -1167,49 +1201,49 @@ launcher_def::write_logging_config_file(tn_node_def &node) {
   }
 
   fc::logger_config p2p( "net_plugin_impl" );
-  p2p.level = fc::log_level::debug;
+  p2p.level = ll;
   p2p.appenders.push_back( "stderr" );
   if( gelf_enabled ) p2p.appenders.push_back( "net" );
   log_config.loggers.emplace_back( p2p );
 
   fc::logger_config http( "http_plugin" );
-  http.level = fc::log_level::debug;
+  http.level = ll;
   http.appenders.push_back( "stderr" );
   if( gelf_enabled ) http.appenders.push_back( "net" );
   log_config.loggers.emplace_back( http );
 
   fc::logger_config pp( "producer_plugin" );
-  pp.level = fc::log_level::debug;
+  pp.level = ll;
   pp.appenders.push_back( "stderr" );
   if( gelf_enabled ) pp.appenders.push_back( "net" );
   log_config.loggers.emplace_back( pp );
 
   fc::logger_config tt( "transaction_success_tracing" );
-  tt.level = fc::log_level::debug;
+  tt.level = ll;
   tt.appenders.push_back( "stderr" );
   if( gelf_enabled ) tt.appenders.push_back( "net" );
   log_config.loggers.emplace_back( tt );
 
   fc::logger_config tft( "transaction_failure_tracing" );
-  tft.level = fc::log_level::debug;
+  tft.level = ll;
   tft.appenders.push_back( "stderr" );
   if( gelf_enabled ) tft.appenders.push_back( "net" );
   log_config.loggers.emplace_back( tft );
 
   fc::logger_config tts( "transaction_trace_success" );
-  tts.level = fc::log_level::debug;
+  tts.level = ll;
   tts.appenders.push_back( "stderr" );
   if( gelf_enabled ) tts.appenders.push_back( "net" );
   log_config.loggers.emplace_back( tts );
 
    fc::logger_config ttf( "transaction_trace_failure" );
-   ttf.level = fc::log_level::debug;
+   ttf.level = ll;
    ttf.appenders.push_back( "stderr" );
    if( gelf_enabled ) ttf.appenders.push_back( "net" );
    log_config.loggers.emplace_back( ttf );
 
   fc::logger_config ta( "trace_api" );
-  ta.level = fc::log_level::debug;
+  ta.level = ll;
   ta.appenders.push_back( "stderr" );
   if( gelf_enabled ) ta.appenders.push_back( "net" );
   log_config.loggers.emplace_back( ta );
@@ -1603,6 +1637,11 @@ launcher_def::launch (eosd_def &instance, string &gts) {
      }
   }
 
+  //Always enable the trace_api_plugin on the bios node
+  if (instance.name == "bios") {
+    eosdcmd += "--plugin eosio::trace_api_plugin ";
+  }
+
   if( add_enable_stale_production ) {
     eosdcmd += "--enable-stale-production true ";
     add_enable_stale_production = false;

tests/CMakeLists.txt

@@ -92,6 +92,8 @@ else()
 endif()
 
 add_subdirectory( TestHarness )
+add_subdirectory( trx_generator )
+add_subdirectory( performance_tests )
 
 find_package(Threads)
 add_executable(ship_client ship_client.cpp)

tests/TestHarness/Cluster.py

@@ -82,7 +82,7 @@ class Cluster(object):
     # pylint: disable=too-many-arguments
     # walletd [True|False] Is keosd running. If not load the wallet plugin
     def __init__(self, walletd=False, localCluster=True, host="localhost", port=8888, walletHost="localhost", walletPort=9899
-                 , defproduceraPrvtKey=None, defproducerbPrvtKey=None, staging=False):
+                 , defproduceraPrvtKey=None, defproducerbPrvtKey=None, staging=False, loggingLevel="debug", loggingLevelDict={}):
         """Cluster container.
         walletd [True|False] Is wallet keosd running. If not load the wallet plugin
         localCluster [True|False] Is cluster local to host.
@@ -105,6 +105,8 @@ def __init__(self, walletd=False, localCluster=True, host="localhost", port=8888
         self.walletHost=walletHost
         self.walletPort=walletPort
         self.staging=staging
+        self.loggingLevel=loggingLevel
+        self.loggingLevelDict=loggingLevelDict
         # init accounts
         self.defProducerAccounts={}
         self.defproduceraAccount=self.defProducerAccounts["defproducera"]= Account("defproducera")
@@ -166,7 +168,8 @@ def setAlternateVersionLabels(self, file):
     # pylint: disable=too-many-statements
     def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="mesh", delay=1, onlyBios=False, dontBootstrap=False,
                totalProducers=None, sharedProducers=0, extraNodeosArgs="", useBiosBootFile=True, specificExtraNodeosArgs=None, onlySetProds=False,
-               pfSetupPolicy=PFSetupPolicy.FULL, alternateVersionLabelsFile=None, associatedNodeLabels=None, loadSystemContract=True):
+               pfSetupPolicy=PFSetupPolicy.FULL, alternateVersionLabelsFile=None, associatedNodeLabels=None, loadSystemContract=True, genesisPath=None,
+               maximumP2pPerHost=0, maximumClients=25, prodsEnableTraceApi=True):
         """Launch cluster.
         pnodes: producer nodes count
         unstartedNodes: non-producer nodes that are configured into the launch, but not started.  Should be included in totalNodes.
@@ -177,7 +180,7 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me
           delay 0 exposes a bootstrap bug where producer handover may have a large gap confusing nodes and bringing system to a halt.
         onlyBios: When true, only loads the bios contract (and not more full bootstrapping).
         dontBootstrap: When true, don't do any bootstrapping at all. (even bios is not uploaded)
-        extraNodeosArgs: string of arguments to pass through to each nodoes instance (via --nodeos flag on launcher)
+        extraNodeosArgs: string of arguments to pass through to each nodeos instance (via --nodeos flag on launcher)
         useBiosBootFile: determines which of two bootstrap methods is used (when both dontBootstrap and onlyBios are false).
           The default value of true uses the bios_boot.sh file generated by the launcher.
           A value of false uses manual bootstrapping in this script, which does not do things like stake votes for producers.
@@ -188,6 +191,10 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me
         alternateVersionLabelsFile: Supply an alternate version labels file to use with associatedNodeLabels.
         associatedNodeLabels: Supply a dictionary of node numbers to use an alternate label for a specific node.
         loadSystemContract: indicate whether the eosio.system contract should be loaded (setting this to False causes useBiosBootFile to be treated as False)
+        genesisPath: set the path to a specific genesis.json to use
+        maximumP2pPerHost:  Maximum number of client nodes from any single IP address. Defaults to totalNodes if not set.
+        maximumClients: Maximum number of clients from which connections are accepted, use 0 for no limit. Defaults to 25.
+        prodsEnableTraceApi: Determines whether producer nodes should have eosio::trace_api_plugin enabled. Defaults to True.
         """
         assert(isinstance(topo, str))
         assert PFSetupPolicy.isValid(pfSetupPolicy)
@@ -224,6 +231,10 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me
         if sharedProducers > 0:
             producerFlag += (" --shared-producers %d" % (sharedProducers))
 
+        if maximumP2pPerHost <= 0:
+            maximumP2pPerHost = totalNodes
+
+
         self.setAlternateVersionLabels(alternateVersionLabelsFile)
 
         tries = 30
@@ -233,22 +244,23 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me
                 return False
             tries = tries - 1
             time.sleep(2)
-
-        cmd="%s -p %s -n %s -d %s -i %s -f %s --unstarted-nodes %s" % (
+        loggingLevelDictString = json.dumps(self.loggingLevelDict, separators=(',', ':'))
+        cmd="%s -p %s -n %s -d %s -i %s -f %s --unstarted-nodes %s --logging-level %s --logging-level-map %s" % (
             Utils.EosLauncherPath, pnodes, totalNodes, delay, datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3],
-            producerFlag, unstartedNodes)
+            producerFlag, unstartedNodes, self.loggingLevel, loggingLevelDictString)
         cmdArr=cmd.split()
         if self.staging:
             cmdArr.append("--nogen")
 
-        nodeosArgs="--resource-monitor-not-shutdown-on-threshold-exceeded --max-transaction-time -1 --abi-serializer-max-time-ms 990000 --p2p-max-nodes-per-host %d" % (totalNodes)
+        nodeosArgs="--max-transaction-time -1 --abi-serializer-max-time-ms 990000 --p2p-max-nodes-per-host %d --max-clients %d" % (maximumP2pPerHost, maximumClients)
         if not self.walletd:
             nodeosArgs += " --plugin eosio::wallet_api_plugin"
         if Utils.Debug:
             nodeosArgs += " --contracts-console"
         if PFSetupPolicy.hasPreactivateFeature(pfSetupPolicy):
             nodeosArgs += " --plugin eosio::producer_api_plugin"
-        nodeosArgs += " --plugin eosio::trace_api_plugin "
+        if prodsEnableTraceApi:
+            nodeosArgs += " --plugin eosio::trace_api_plugin "
         if extraNodeosArgs.find("--trace-rpc-abi") == -1:
             nodeosArgs += " --trace-no-abis "
         httpMaxResponseTimeSet = False
@@ -275,10 +287,14 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me
             cmdArr.append("--nodeos")
             cmdArr.append(nodeosArgs)
 
-        cmdArr.append("--max-block-cpu-usage")
-        cmdArr.append(str(160000000))
-        cmdArr.append("--max-transaction-cpu-usage")
-        cmdArr.append(str(150000000))
+        if genesisPath is None:
+            cmdArr.append("--max-block-cpu-usage")
+            cmdArr.append(str(500000))
+            cmdArr.append("--max-transaction-cpu-usage")
+            cmdArr.append(str(475000))
+        else:
+            cmdArr.append("--genesis")
+            cmdArr.append(str(genesisPath))
 
         if associatedNodeLabels is not None:
             for nodeNum,label in associatedNodeLabels.items():
@@ -1549,7 +1565,7 @@ def cleanup(self):
 
 
     # Create accounts and validates that the last transaction is received on root node
-    def createAccounts(self, creator, waitForTransBlock=True, stakedDeposit=1000):
+    def createAccounts(self, creator, waitForTransBlock=True, stakedDeposit=1000, validationNodeIndex=0):
         if self.accounts is None:
             return True
 
@@ -1564,7 +1580,7 @@ def createAccounts(self, creator, waitForTransBlock=True, stakedDeposit=1000):
             transId=Node.getTransId(trans)
 
         if waitForTransBlock and transId is not None:
-            node=self.nodes[0]
+            node=self.nodes[validationNodeIndex]
             if Utils.Debug: Utils.Print("Wait for transaction id %s on server port %d." % ( transId, node.port))
             if node.waitForTransactionInBlock(transId) is False:
                 Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, node.port))