Implemented HOMA_GRO_FAST_GRANTS optimization

PlatformLab · Nov 4, 2022 · 4577c9f · 4577c9f
1 parent 03c7dce
commit 4577c9f
Show file tree

Hide file tree

Showing 3 changed files with 69 additions and 13 deletions.
diff --git a/homa_impl.h b/homa_impl.h
@@ -1724,15 +1724,17 @@ struct homa {
 	/* Bits that can be specified for gro_policy. These were created for
 	 * testing, in order to evaluate various possible policies; you almost
 	 * certainly should not use any value other than HOMA_GRO_NORMAL.
-	 * The meaning of the bits is also somewhat complicated; if you really
-	 * want to know what they mean, read the code of homa_offload.c
-	 */
-	#define HOMA_GRO_BYPASS      1
-	#define HOMA_GRO_SAME_CORE   2
-	#define HOMA_GRO_IDLE        4
-	#define HOMA_GRO_NEXT        8
-	#define HOMA_GRO_IDLE_NEW    16
-	#define HOMA_GRO_NORMAL      HOMA_GRO_SAME_CORE|HOMA_GRO_IDLE_NEW
+	 * If you want to know what the bits mean, read the code of
+	 * homa_offload.c.
+	 */
+	#define HOMA_GRO_BYPASS          1
+	#define HOMA_GRO_SAME_CORE       2
+	#define HOMA_GRO_IDLE            4
+	#define HOMA_GRO_NEXT            8
+	#define HOMA_GRO_IDLE_NEW       16
+	#define HOMA_GRO_FAST_GRANTS    32
+	#define HOMA_GRO_NORMAL      (HOMA_GRO_SAME_CORE|HOMA_GRO_IDLE_NEW \
+			|HOMA_GRO_FAST_GRANTS)
 
 	/*
 	 * @gro_busy_usecs: try not to schedule SoftIRQ processing on a core

diff --git a/homa_offload.c b/homa_offload.c
@@ -134,13 +134,25 @@ struct sk_buff *homa_gro_receive(struct list_head *held_list,
 				"id %llu, offset %d, priority %d",
 				saddr, homa_local_id(h_new->common.sender_id),
 				ntohl(h_new->seg.offset), priority);
-	else if (h_new->common.type == GRANT)
+	else if (h_new->common.type == GRANT) {
 		tt_record4("homa_gro_receive got grant from 0x%x "
 				"id %llu, offset %d, priority %d",
 				saddr, homa_local_id(h_new->common.sender_id),
 				ntohl(((struct grant_header *) h_new)->offset),
 				priority);
-	else
+		/* The following optimization handles grants here at NAPI
+		 * level, bypassing the SoftIRQ mechanism (and avoiding the
+		 * delay of handing off to a different core). This makes
+		 * a significant difference in throughput for large
+		 * messages, especially when the system is loaded.
+		 */
+		if (homa->gro_policy & HOMA_GRO_FAST_GRANTS) {
+			homa_softirq(skb);
+
+			/* Indicates that we have freed skb. */
+			return ERR_PTR(-EINPROGRESS);
+		}
+	} else
 		tt_record4("homa_gro_receive got packet from 0x%x "
 				"id %llu, type 0x%x, priority %d",
 				saddr, homa_local_id(h_new->common.sender_id),

diff --git a/test/unit_homa_offload.c b/test/unit_homa_offload.c
@@ -14,6 +14,7 @@
  */
 
 #include "homa_impl.h"
+#include "homa_lcache.h"
 #define KSELFTEST_NOT_MAIN 1
 #include "kselftest_harness.h"
 #include "ccutils.h"
@@ -24,6 +25,7 @@ extern struct homa *homa;
 
 FIXTURE(homa_offload) {
 	struct homa homa;
+	struct homa_sock hsk;
 	struct in6_addr ip;
 	struct data_header header;
 	struct napi_struct napi;
@@ -34,7 +36,10 @@ FIXTURE_SETUP(homa_offload)
 {
 	int i;
 	homa_init(&self->homa);
-	homa_init(homa);
+	self->homa.flags |= HOMA_FLAG_DONT_THROTTLE;
+	self->homa.grant_threshold = self->homa.rtt_bytes;
+	homa = &self->homa;
+	mock_sock_init(&self->hsk, &self->homa, 99);
 	self->ip = unit_get_in_addr("196.168.0.1");
 	self->header = (struct data_header){.common = {
 			.sport = htons(40000), .dport = htons(99),
@@ -78,10 +83,47 @@ FIXTURE_TEARDOWN(homa_offload)
 	list_for_each_entry_safe(skb, tmp, &self->napi.gro_hash[2].list, list)
 		kfree_skb(skb);
 	homa_destroy(&self->homa);
-	homa_destroy(homa);
+	homa = NULL;
 	unit_teardown();
 }
 
+TEST_F(homa_offload, homa_gro_receive__fast_grant_optimization)
+{
+	struct in6_addr client_ip = unit_get_in_addr("196.168.0.1");
+	struct in6_addr server_ip = unit_get_in_addr("1.2.3.4");
+	int client_port = 40000;
+	__u64 client_id = 1234;
+	__u64 server_id = 1235;
+	struct homa_rpc *srpc = unit_server_rpc(&self->hsk, RPC_OUTGOING,
+			&client_ip, &server_ip, client_port, server_id, 100,
+			20000);
+	ASSERT_NE(NULL, srpc);
+	homa_xmit_data(srpc, false);
+	unit_log_clear();
+
+	struct grant_header h = {{.sport = htons(srpc->dport),
+	                .dport = htons(self->hsk.port),
+			.sender_id = cpu_to_be64(client_id),
+			.type = GRANT},
+		        .offset = htonl(12600),
+			.priority = 3};
+	self->homa.gro_policy = HOMA_GRO_FAST_GRANTS;
+	struct sk_buff *result = homa_gro_receive(&self->empty_list,
+			mock_skb_new(&client_ip, &h.common, 0, 0));
+	EXPECT_EQ(EINPROGRESS, -PTR_ERR(result));
+	EXPECT_EQ(12600, srpc->msgout.granted);
+	EXPECT_STREQ("xmit DATA 1400@11200", unit_log_get());
+
+	unit_log_clear();
+	h.offset = htonl(14000);
+	self->homa.gro_policy = 0;
+	struct sk_buff *skb = mock_skb_new(&client_ip, &h.common, 0, 0);
+	result = homa_gro_receive(&self->empty_list, skb);
+	EXPECT_EQ(NULL, result);
+	EXPECT_EQ(12600, srpc->msgout.granted);
+	EXPECT_STREQ("", unit_log_get());
+	kfree_skb(skb);
+}
 TEST_F(homa_offload, homa_gro_receive__no_held_skb)
 {
 	struct sk_buff *skb;