diff --git a/homa_impl.h b/homa_impl.h index f963faf5..404d85c8 100644 --- a/homa_impl.h +++ b/homa_impl.h @@ -1724,15 +1724,17 @@ struct homa { /* Bits that can be specified for gro_policy. These were created for * testing, in order to evaluate various possible policies; you almost * certainly should not use any value other than HOMA_GRO_NORMAL. - * The meaning of the bits is also somewhat complicated; if you really - * want to know what they mean, read the code of homa_offload.c - */ - #define HOMA_GRO_BYPASS 1 - #define HOMA_GRO_SAME_CORE 2 - #define HOMA_GRO_IDLE 4 - #define HOMA_GRO_NEXT 8 - #define HOMA_GRO_IDLE_NEW 16 - #define HOMA_GRO_NORMAL HOMA_GRO_SAME_CORE|HOMA_GRO_IDLE_NEW + * If you want to know what the bits mean, read the code of + * homa_offload.c. + */ + #define HOMA_GRO_BYPASS 1 + #define HOMA_GRO_SAME_CORE 2 + #define HOMA_GRO_IDLE 4 + #define HOMA_GRO_NEXT 8 + #define HOMA_GRO_IDLE_NEW 16 + #define HOMA_GRO_FAST_GRANTS 32 + #define HOMA_GRO_NORMAL (HOMA_GRO_SAME_CORE|HOMA_GRO_IDLE_NEW \ + |HOMA_GRO_FAST_GRANTS) /* * @gro_busy_usecs: try not to schedule SoftIRQ processing on a core diff --git a/homa_offload.c b/homa_offload.c index 05b79cfc..a373cbbd 100644 --- a/homa_offload.c +++ b/homa_offload.c @@ -134,13 +134,25 @@ struct sk_buff *homa_gro_receive(struct list_head *held_list, "id %llu, offset %d, priority %d", saddr, homa_local_id(h_new->common.sender_id), ntohl(h_new->seg.offset), priority); - else if (h_new->common.type == GRANT) + else if (h_new->common.type == GRANT) { tt_record4("homa_gro_receive got grant from 0x%x " "id %llu, offset %d, priority %d", saddr, homa_local_id(h_new->common.sender_id), ntohl(((struct grant_header *) h_new)->offset), priority); - else + /* The following optimization handles grants here at NAPI + * level, bypassing the SoftIRQ mechanism (and avoiding the + * delay of handing off to a different core). This makes + * a significant difference in throughput for large + * messages, especially when the system is loaded. + */ + if (homa->gro_policy & HOMA_GRO_FAST_GRANTS) { + homa_softirq(skb); + + /* Indicates that we have freed skb. */ + return ERR_PTR(-EINPROGRESS); + } + } else tt_record4("homa_gro_receive got packet from 0x%x " "id %llu, type 0x%x, priority %d", saddr, homa_local_id(h_new->common.sender_id), diff --git a/test/unit_homa_offload.c b/test/unit_homa_offload.c index 0f9f07d5..ae138927 100644 --- a/test/unit_homa_offload.c +++ b/test/unit_homa_offload.c @@ -14,6 +14,7 @@ */ #include "homa_impl.h" +#include "homa_lcache.h" #define KSELFTEST_NOT_MAIN 1 #include "kselftest_harness.h" #include "ccutils.h" @@ -24,6 +25,7 @@ extern struct homa *homa; FIXTURE(homa_offload) { struct homa homa; + struct homa_sock hsk; struct in6_addr ip; struct data_header header; struct napi_struct napi; @@ -34,7 +36,10 @@ FIXTURE_SETUP(homa_offload) { int i; homa_init(&self->homa); - homa_init(homa); + self->homa.flags |= HOMA_FLAG_DONT_THROTTLE; + self->homa.grant_threshold = self->homa.rtt_bytes; + homa = &self->homa; + mock_sock_init(&self->hsk, &self->homa, 99); self->ip = unit_get_in_addr("196.168.0.1"); self->header = (struct data_header){.common = { .sport = htons(40000), .dport = htons(99), @@ -78,10 +83,47 @@ FIXTURE_TEARDOWN(homa_offload) list_for_each_entry_safe(skb, tmp, &self->napi.gro_hash[2].list, list) kfree_skb(skb); homa_destroy(&self->homa); - homa_destroy(homa); + homa = NULL; unit_teardown(); } +TEST_F(homa_offload, homa_gro_receive__fast_grant_optimization) +{ + struct in6_addr client_ip = unit_get_in_addr("196.168.0.1"); + struct in6_addr server_ip = unit_get_in_addr("1.2.3.4"); + int client_port = 40000; + __u64 client_id = 1234; + __u64 server_id = 1235; + struct homa_rpc *srpc = unit_server_rpc(&self->hsk, RPC_OUTGOING, + &client_ip, &server_ip, client_port, server_id, 100, + 20000); + ASSERT_NE(NULL, srpc); + homa_xmit_data(srpc, false); + unit_log_clear(); + + struct grant_header h = {{.sport = htons(srpc->dport), + .dport = htons(self->hsk.port), + .sender_id = cpu_to_be64(client_id), + .type = GRANT}, + .offset = htonl(12600), + .priority = 3}; + self->homa.gro_policy = HOMA_GRO_FAST_GRANTS; + struct sk_buff *result = homa_gro_receive(&self->empty_list, + mock_skb_new(&client_ip, &h.common, 0, 0)); + EXPECT_EQ(EINPROGRESS, -PTR_ERR(result)); + EXPECT_EQ(12600, srpc->msgout.granted); + EXPECT_STREQ("xmit DATA 1400@11200", unit_log_get()); + + unit_log_clear(); + h.offset = htonl(14000); + self->homa.gro_policy = 0; + struct sk_buff *skb = mock_skb_new(&client_ip, &h.common, 0, 0); + result = homa_gro_receive(&self->empty_list, skb); + EXPECT_EQ(NULL, result); + EXPECT_EQ(12600, srpc->msgout.granted); + EXPECT_STREQ("", unit_log_get()); + kfree_skb(skb); +} TEST_F(homa_offload, homa_gro_receive__no_held_skb) { struct sk_buff *skb;