Perf analysis based on tsc delta measurements

cli/develop.h

@@ -1,7 +1,15 @@
 #pragma once
 
+#include <chrono>
+#include <cstdint>
+#include <string>
+#include <sys/shm.h>
+#include <thread>
+#include <vector>
+
 #include "common/icontrolplane.h"
 #include "common/idataplane.h"
+#include "common/tsc_deltas.h"
 
 #include "helper.h"
 
@@ -160,6 +168,204 @@ void counter(const uint32_t& counter_id,
 	table.print();
 }
 
+using namespace ::dataplane::perf;
+class tsc_monitoring_t
+{
+public:
+	void connect_shm()
+	{
+		interface::dataPlane dataplane;
+		const auto response = dataplane.get_shm_tsc_info();
+		std::map<key_t, void*> ipc_cache;
+
+		for (const auto& [core, socket, ipc_key, offset] : response)
+		{
+			(void)socket;
+			if (ipc_cache.find(ipc_key) == ipc_cache.end())
+			{
+				auto shmid = shmget(ipc_key, 0, 0);
+				if (shmid == -1)
+				{
+					throw std::string("shmget(") + std::to_string(ipc_key) + ", 0, 0) = " + std::strerror(errno);
+				}
+				auto shmaddr = shmat(shmid, NULL, SHM_RDONLY);
+				if (shmaddr == (void*)-1)
+				{
+					throw std::string("shmat(") + std::to_string(ipc_key) + ", NULL, 0) = " + std::strerror(errno);
+				}
+
+				ipc_cache[ipc_key] = shmaddr;
+			}
+
+			auto counter_addr = (tsc_deltas*)((intptr_t)ipc_cache[ipc_key] + offset);
+			worker_counters.emplace_back(core, counter_addr, tsc_deltas{}, overflow_store{});
+		}
+	}
+
+	void monitor()
+	{
+		connect_shm();
+		for (auto iter = 0;; iter++)
+		{
+			if (iter % 4 == 0)
+			{
+				insert_header();
+			}
+
+			for (auto& [core_id, counter, previous_value, overflow_store] : worker_counters)
+			{
+				const auto& counter_copy = *counter;
+				for (auto bin = 0; bin < YANET_TSC_BINS_N; bin++)
+				{
+					overflow_store.handle_overflow(counter_copy, previous_value, bin);
+					if (iter % 4 == 0)
+					{
+						insert_bin(counter_copy, overflow_store, bin, core_id);
+					}
+				}
+
+				previous_value = counter_copy;
+			}
+
+			if (iter % 4 == 0)
+			{
+				table.render();
+			}
+			std::this_thread::sleep_for(std::chrono::milliseconds(250));
+		}
+	}
+
+protected:
+	struct overflow_store;
+
+	void insert_header()
+	{
+		table.insert("core_id",
+		             "iter_num",
+		             "logicalPort_ingress",
+		             "acl_ingress4",
+		             "acl_ingress6",
+		             "tun64_ipv4",
+		             "tun64_ipv6",
+		             "route4",
+		             "route6",
+		             "decap",
+		             "nat64stateful_lan",
+		             "nat64stateful_wan",
+		             "nat64stateless_egress",
+		             "nat64stateless_ingress",
+		             "nat46clat_lan",
+		             "nat46clat_wan",
+		             "balancer",
+		             "balancer_icmp_reply",
+		             "balancer_icmp_forward",
+		             "route_tunnel4",
+		             "route_tunnel6",
+		             "acl_egress4",
+		             "acl_egress6",
+		             "logicalPort_egress",
+		             "controlPlane");
+	}
+
+	void insert_bin(const tsc_deltas& cnt, const overflow_store& of_store, int bin, uint32_t core_id)
+	{
+		table.insert(bin == 0 ? std::to_string(core_id) : std::string{},
+		             bin == 0 ? std::to_string(cnt.iter_num) : std::string{},
+		             of_store.logicalPort_ingress_handle[bin] + cnt.logicalPort_ingress_handle[bin],
+		             of_store.acl_ingress_handle4[bin] + cnt.acl_ingress_handle4[bin],
+		             of_store.acl_ingress_handle6[bin] + cnt.acl_ingress_handle6[bin],
+		             of_store.tun64_ipv4_handle[bin] + cnt.tun64_ipv4_handle[bin],
+		             of_store.tun64_ipv6_handle[bin] + cnt.tun64_ipv6_handle[bin],
+		             of_store.route_handle4[bin] + cnt.route_handle4[bin],
+		             of_store.route_handle6[bin] + cnt.route_handle6[bin],
+
+		             of_store.decap_handle[bin] + cnt.decap_handle[bin],
+		             of_store.nat64stateful_lan_handle[bin] + cnt.nat64stateful_lan_handle[bin],
+		             of_store.nat64stateful_wan_handle[bin] + cnt.nat64stateful_wan_handle[bin],
+		             of_store.nat64stateless_egress_handle[bin] + cnt.nat64stateless_egress_handle[bin],
+		             of_store.nat64stateless_ingress_handle[bin] + cnt.nat64stateless_ingress_handle[bin],
+		             of_store.nat46clat_lan_handle[bin] + cnt.nat46clat_lan_handle[bin],
+		             of_store.nat46clat_wan_handle[bin] + cnt.nat46clat_wan_handle[bin],
+		             of_store.balancer_handle[bin] + cnt.balancer_handle[bin],
+
+		             of_store.balancer_icmp_reply_handle[bin] + cnt.balancer_icmp_reply_handle[bin],
+		             of_store.balancer_icmp_forward_handle[bin] + cnt.balancer_icmp_forward_handle[bin],
+		             of_store.route_tunnel_handle4[bin] + cnt.route_tunnel_handle4[bin],
+		             of_store.route_tunnel_handle6[bin] + cnt.route_tunnel_handle6[bin],
+		             of_store.acl_egress_handle4[bin] + cnt.acl_egress_handle4[bin],
+		             of_store.acl_egress_handle6[bin] + cnt.acl_egress_handle6[bin],
+		             of_store.logicalPort_egress_handle[bin] + cnt.logicalPort_egress_handle[bin],
+		             of_store.controlPlane_handle[bin] + cnt.controlPlane_handle[bin]);
+	}
+
+	struct overflow_store
+	{
+		uint64_t logicalPort_ingress_handle[YANET_TSC_BINS_N];
+		uint64_t acl_ingress_handle4[YANET_TSC_BINS_N];
+		uint64_t acl_ingress_handle6[YANET_TSC_BINS_N];
+		uint64_t tun64_ipv4_handle[YANET_TSC_BINS_N];
+		uint64_t tun64_ipv6_handle[YANET_TSC_BINS_N];
+		uint64_t route_handle4[YANET_TSC_BINS_N];
+		uint64_t route_handle6[YANET_TSC_BINS_N];
+
+		uint64_t decap_handle[YANET_TSC_BINS_N];
+		uint64_t nat64stateful_lan_handle[YANET_TSC_BINS_N];
+		uint64_t nat64stateful_wan_handle[YANET_TSC_BINS_N];
+		uint64_t nat64stateless_egress_handle[YANET_TSC_BINS_N];
+		uint64_t nat64stateless_ingress_handle[YANET_TSC_BINS_N];
+		uint64_t nat46clat_lan_handle[YANET_TSC_BINS_N];
+		uint64_t nat46clat_wan_handle[YANET_TSC_BINS_N];
+		uint64_t balancer_handle[YANET_TSC_BINS_N];
+
+		uint64_t balancer_icmp_reply_handle[YANET_TSC_BINS_N];
+		uint64_t balancer_icmp_forward_handle[YANET_TSC_BINS_N];
+		uint64_t route_tunnel_handle4[YANET_TSC_BINS_N];
+		uint64_t route_tunnel_handle6[YANET_TSC_BINS_N];
+		uint64_t acl_egress_handle4[YANET_TSC_BINS_N];
+		uint64_t acl_egress_handle6[YANET_TSC_BINS_N];
+		uint64_t logicalPort_egress_handle[YANET_TSC_BINS_N];
+		uint64_t controlPlane_handle[YANET_TSC_BINS_N];
+
+		void handle_overflow(const tsc_deltas& cnt, const tsc_deltas& prev, int bin)
+		{
+			logicalPort_ingress_handle[bin] += (prev.logicalPort_ingress_handle[bin] > cnt.logicalPort_ingress_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			acl_ingress_handle4[bin] += (prev.acl_ingress_handle4[bin] > cnt.acl_ingress_handle4[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			acl_ingress_handle6[bin] += (prev.acl_ingress_handle6[bin] > cnt.acl_ingress_handle6[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			tun64_ipv4_handle[bin] += (prev.tun64_ipv4_handle[bin] > cnt.tun64_ipv4_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			tun64_ipv6_handle[bin] += (prev.tun64_ipv6_handle[bin] > cnt.tun64_ipv6_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			route_handle4[bin] += (prev.route_handle4[bin] > cnt.route_handle4[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			route_handle6[bin] += (prev.route_handle6[bin] > cnt.route_handle6[bin]) << sizeof(uint16_t) * CHAR_BIT;
+
+			decap_handle[bin] += (prev.decap_handle[bin] > cnt.decap_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			nat64stateful_lan_handle[bin] += (prev.nat64stateful_lan_handle[bin] > cnt.nat64stateful_lan_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			nat64stateful_wan_handle[bin] += (prev.nat64stateful_wan_handle[bin] > cnt.nat64stateful_wan_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			nat64stateless_egress_handle[bin] += (prev.nat64stateless_egress_handle[bin] > cnt.nat64stateless_egress_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			nat64stateless_ingress_handle[bin] += (prev.nat64stateless_ingress_handle[bin] > cnt.nat64stateless_ingress_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			nat46clat_lan_handle[bin] += (prev.nat46clat_lan_handle[bin] > cnt.nat46clat_lan_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			nat46clat_wan_handle[bin] += (prev.nat46clat_wan_handle[bin] > cnt.nat46clat_wan_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			balancer_handle[bin] += (prev.balancer_handle[bin] > cnt.balancer_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+
+			balancer_icmp_reply_handle[bin] += (prev.balancer_icmp_reply_handle[bin] > cnt.balancer_icmp_reply_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			balancer_icmp_forward_handle[bin] += (prev.balancer_icmp_forward_handle[bin] > cnt.balancer_icmp_forward_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			route_tunnel_handle4[bin] += (prev.route_tunnel_handle4[bin] > cnt.route_tunnel_handle4[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			route_tunnel_handle6[bin] += (prev.route_tunnel_handle6[bin] > cnt.route_tunnel_handle6[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			acl_egress_handle4[bin] += (prev.acl_egress_handle4[bin] > cnt.acl_egress_handle4[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			acl_egress_handle6[bin] += (prev.acl_egress_handle6[bin] > cnt.acl_egress_handle6[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			logicalPort_egress_handle[bin] += (prev.logicalPort_egress_handle[bin] > cnt.logicalPort_egress_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+			controlPlane_handle[bin] += (prev.controlPlane_handle[bin] > cnt.controlPlane_handle[bin]) << sizeof(uint16_t) * CHAR_BIT;
+		}
+	};
+
+	std::vector<std::tuple<uint32_t, tsc_deltas*, tsc_deltas, overflow_store>> worker_counters;
+	table_t table;
+};
+
+void tsc_monitoring()
+{
+	tsc_monitoring_t monitoring{};
+	monitoring.monitor();
+}
+
 }
 
 }

cli/helper.h

@@ -97,6 +97,18 @@ void fillValue(std::optional<TArg>& value, const std::string& string)
 	}
 }
 
+void fillValue(bool& value, const std::string& string)
+{
+	if (string == "false" || string == "true")
+	{
+		value = string == "true";
+	}
+	else
+	{
+		throw std::string("invalid argument, must be true or false");
+	}
+}
+
 void fillValue(uint8_t& value, const std::string& string)
 {
 	value = std::stoull(string, nullptr, 0);
@@ -427,6 +439,13 @@ class table_t
 		}
 	}
 
+	void render()
+	{
+		printf("\033[2J\033[H");
+		print_default();
+		table.clear();
+	}
+
 protected:
 	converter::config_t config;
 	std::vector<std::vector<std::string>> table;

cli/main.cpp

@@ -100,6 +100,10 @@ std::vector<std::tuple<std::string,
                     {},
                     {"show shm info", "", [](const auto& args) { call(show::shm_info, args); }},
                     {},
+                    {"tsc show shm info", "", [](const auto& args) { call(show::shm_tsc_info, args); }},
+                    {"tsc set state", "[true|false]", [](const auto& args) { call(show::shm_tsc_set_state, args); }},
+                    {"tsc set base", "[handle] [value]", [](const auto& args) { call(show::shm_tsc_set_base_value, args); }},
+                    {},
                     {"samples show", "", [](const auto& args) { call(show::samples, args); }},
                     {"samples dump", "", [](const auto& args) { call(show::samples_dump, args); }},
                     {},
@@ -111,6 +115,7 @@ std::vector<std::tuple<std::string,
                     {"dontdoit podumoi controlplane rib save", "", [](const auto& args) { call(rib::save, args); }},
                     {"dontdoit podumoi controlplane rib load", "", [](const auto& args) { call(rib::load, args); }},
                     {"dontdoit podumoi controlplane rib clear", "[protocol] <peer> <vrf> <priority>", [](const auto& args) { call(rib::clear, args); }},
+                    {"dontdoit podumoi tsc monitoring", "", [](const auto& args) { call(develop::dataplane::tsc_monitoring, args); }},
                     {},
                     {"telegraf unsafe", "", [](const auto& args) { call(telegraf::unsafe, args); }},
                     {"telegraf ports", "", [](const auto& args) { call(telegraf::ports_stats, args); }},

cli/show.h

@@ -6,6 +6,7 @@
 
 #include "common/icontrolplane.h"
 #include "common/idataplane.h"
+#include "common/tsc_deltas.h"
 #include "common/version.h"
 
 #include "helper.h"
@@ -850,4 +851,77 @@ inline void shm_info()
 	table.print();
 }
 
+void shm_tsc_info()
+{
+	interface::dataPlane dataplane;
+	const auto response = dataplane.get_shm_tsc_info();
+
+	table_t table;
+	table.insert("core id",
+	             "socket id",
+	             "ipc key",
+	             "offset");
+
+	for (const auto& [core, socket, ipc_key, offset] : response)
+	{
+		table.insert(core, socket, ipc_key, offset);
+	}
+
+	table.print();
+}
+
+void shm_tsc_set_state(bool state)
+{
+	interface::dataPlane dataplane;
+	common::idp::updateGlobalBase::request globalbase;
+	globalbase.emplace_back(common::idp::updateGlobalBase::requestType::tsc_state_update,
+	                        state);
+	dataplane.updateGlobalBase(globalbase);
+}
+
+using dataplane::perf::tsc_base_values;
+static const std::map<std::string, uint32_t> counter_name_to_offset = {
+        {"logicalPort_ingress_handle", offsetof(tsc_base_values, logicalPort_ingress_handle)},
+        {"acl_ingress_handle4", offsetof(tsc_base_values, acl_ingress_handle4)},
+        {"acl_ingress_handle6", offsetof(tsc_base_values, acl_ingress_handle6)},
+        {"tun64_ipv4_handle", offsetof(tsc_base_values, tun64_ipv4_handle)},
+        {"tun64_ipv6_handle", offsetof(tsc_base_values, tun64_ipv6_handle)},
+        {"route_handle4", offsetof(tsc_base_values, route_handle4)},
+        {"route_handle6", offsetof(tsc_base_values, route_handle6)},
+        {"decap_handle", offsetof(tsc_base_values, decap_handle)},
+        {"nat64stateful_lan_handle", offsetof(tsc_base_values, nat64stateful_lan_handle)},
+        {"nat64stateful_wan_handle", offsetof(tsc_base_values, nat64stateful_wan_handle)},
+        {"nat64stateless_egress_handle", offsetof(tsc_base_values, nat64stateless_egress_handle)},
+        {"nat64stateless_ingress_handle", offsetof(tsc_base_values, nat64stateless_ingress_handle)},
+        {"nat46clat_lan_handle", offsetof(tsc_base_values, nat46clat_lan_handle)},
+        {"nat46clat_wan_handle", offsetof(tsc_base_values, nat46clat_wan_handle)},
+        {"balancer_handle", offsetof(tsc_base_values, balancer_handle)},
+        {"balancer_icmp_reply_handle", offsetof(tsc_base_values, balancer_icmp_reply_handle)},
+        {"balancer_icmp_forward_handle", offsetof(tsc_base_values, balancer_icmp_forward_handle)},
+        {"route_tunnel_handle4", offsetof(tsc_base_values, route_tunnel_handle4)},
+        {"route_tunnel_handle6", offsetof(tsc_base_values, route_tunnel_handle6)},
+        {"acl_egress_handle4", offsetof(tsc_base_values, acl_egress_handle4)},
+        {"acl_egress_handle6", offsetof(tsc_base_values, acl_egress_handle6)},
+        {"logicalPort_egress_handle", offsetof(tsc_base_values, logicalPort_egress_handle)},
+        {"controlPlane_handle", offsetof(tsc_base_values, controlPlane_handle)},
+};
+
+void shm_tsc_set_base_value(std::string counter_name, uint32_t value)
+{
+	if (counter_name_to_offset.count(counter_name) != 0)
+	{
+		interface::dataPlane dataplane;
+		common::idp::updateGlobalBase::request globalbase;
+		globalbase.emplace_back(common::idp::updateGlobalBase::requestType::tscs_base_value_update,
+		                        common::idp::updateGlobalBase::tscs_base_value_update::request{counter_name_to_offset.at(counter_name), value});
+		dataplane.updateGlobalBase(globalbase);
+	}
+	else
+	{
+		std::string args;
+		std::for_each(counter_name_to_offset.cbegin(), counter_name_to_offset.cend(), [&](const auto& e) { args += " " + e.first; });
+		throw std::string("invalid argument: ") + counter_name + ", supported types:" + args;
+	}
+}
+
 }

common/config.release.h

@@ -74,3 +74,4 @@
 #define YANET_DEFAULT_IPC_SHMKEY (12345)
 #define YANET_CONFIG_KERNEL_INTERFACE_QUEUE_SIZE (4096)
 #define YANET_CONFIG_NAT46CLATS_SIZE (32)
+#define YANET_CONFIG_TSC_ACTIVE_STATE (0)

common/idataplane.h

@@ -208,6 +208,11 @@ class dataPlane
 		return get<common::idp::requestType::get_shm_info, common::idp::get_shm_info::response>();
 	}
 
+	auto get_shm_tsc_info() const
+	{
+		return get<common::idp::requestType::get_shm_tsc_info, common::idp::get_shm_tsc_info::response>();
+	}
+
 	auto dump_physical_port(const common::idp::dump_physical_port::request& request) const
 	{
 		return get<common::idp::requestType::dump_physical_port, eResult>(request);