Add hijack_tcp sysctl option

homa_impl.h

@@ -2138,19 +2138,26 @@ struct homa {
 	 */
 	int max_gso_size;
 
-	/**
-	 * @max_gro_skbs: Maximum number of socket buffers that can be
-	 * aggregated by the GRO mechanism.  Set externally via sysctl.
-	 */
-	int max_gro_skbs;
-
 	/**
 	 * @gso_force_software: A non-zero value will cause Home to perform
 	 * segmentation in software using GSO; zero means ask the NIC to
 	 * perform TSO. Set externally via sysctl.
 	 */
 	int gso_force_software;
 
+	/**
+	 * @hijack_tcp: Non-zero means encapsulate outgoing Homa packets
+	 * as TCP packets (i.e. use TCP as the IP protocol). This makes TSO
+	 * and RSS work better. Set externally via sysctl.
+	 */
+	int hijack_tcp;
+
+	/**
+	 * @max_gro_skbs: Maximum number of socket buffers that can be
+	 * aggregated by the GRO mechanism.  Set externally via sysctl.
+	 */
+	int max_gro_skbs;
+
 	/**
 	 * @gro_policy: An OR'ed together collection of bits that determine
 	 * how Homa packets should be steered for SoftIRQ handling.  A value

homa_plumbing.c

@@ -289,6 +289,20 @@ static struct ctl_table homa_ctl_table[] = {
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec
 	},
+	{
+		.procname	= "hijack_tcp",
+		.data		= &homa_data.hijack_tcp,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec
+	},
+	{
+		.procname	= "hijack_tcp",
+		.data		= &homa_data.hijack_tcp,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec
+	},
 	{
 		.procname	= "link_mbps",
 		.data		= &homa_data.link_mbps,

homa_socktab.c

@@ -154,11 +154,8 @@ void homa_sock_init(struct homa_sock *hsk, struct homa *homa)
 		bucket->id = i + 1000000;
 	}
 	memset(&hsk->buffer_pool, 0, sizeof(hsk->buffer_pool));
-
-	/* This line will cause outgoing packets to be sent with TCP
-	 * as the IP protocol (so that TSO and RSS will work better).
-	 */
-	hsk->sock.sk_protocol = IPPROTO_TCP;
+	if (homa->hijack_tcp)
+		hsk->sock.sk_protocol = IPPROTO_TCP;
 	spin_unlock_bh(&socktab->write_lock);
 }
 

homa_utils.c

@@ -186,8 +186,9 @@ int homa_init(struct homa *homa)
 	homa->cycles_per_kbyte = 0;
 	homa->verbose = 0;
 	homa->max_gso_size = 10000;
-	homa->max_gro_skbs = 20;
 	homa->gso_force_software = 0;
+	homa->hijack_tcp = 0;
+	homa->max_gro_skbs = 20;
 	homa->gro_policy = HOMA_GRO_NORMAL;
 	homa->busy_usecs = 100;
 	homa->gro_busy_usecs = 5;

man/homa.7

@@ -354,6 +354,13 @@ If this value is nonzero, Homa will perform GSO in software instead of
 asking the NIC to perform TSO in hardware. This can be useful when running
 with NICs that refuse to perform TSO on Homa packets.
 .TP
+.IR hijack_tcp
+An integer value; if nonzero, Homa will transmit its packets as TCP
+packets (e.g., using IPPROTO_TCP instead of IPPROTO_HOMA). This allows Homa
+to make better use of NIC hardware support such as TSO and RSS, but it
+requires Homa to intercept all incoming TCP packets to see if they are
+actually Homa packets. Some might object to this interference with the
+rest of the Linux kernel.
 .TP
 .IR link_mbps
 An integer value specifying the bandwidth of this machine's uplink to

test/unit_homa_socktab.c

@@ -128,7 +128,18 @@ TEST_F(homa_socktab, homa_sock_init__ip_header_length)
 	homa_sock_destroy(&hsk_v4);
 	homa_sock_destroy(&hsk_v6);
 }
-
+TEST_F(homa_socktab, homa_sock_init__hijack_tcp)
+{
+	struct homa_sock hijack, no_hijack;
+        self->homa.hijack_tcp = 0;
+	mock_sock_init(&no_hijack, &self->homa, 0);
+        self->homa.hijack_tcp = 1;
+        mock_sock_init(&hijack, &self->homa, 0);
+        EXPECT_EQ(0, no_hijack.sock.sk_protocol);
+        EXPECT_EQ(IPPROTO_TCP, hijack.sock.sk_protocol);
+        homa_sock_destroy(&hijack);
+	homa_sock_destroy(&no_hijack);
+}
 
 TEST_F(homa_socktab, homa_sock_shutdown__basics)
 {