From 17c11030d7fda6537374b426b7e39f4fb5337716 Mon Sep 17 00:00:00 2001 From: John Ousterhout Date: Wed, 25 Sep 2024 17:31:19 -0700 Subject: [PATCH] Fix style problems found by checkpatch --- homa.h | 18 +- homa_api.c | 5 +- homa_grant.c | 70 ++--- homa_impl.h | 304 +++++++++++----------- homa_incoming.c | 143 +++++------ homa_offload.c | 68 +++-- homa_outgoing.c | 93 +++---- homa_peertab.c | 68 ++--- homa_plumbing.c | 220 ++++++++-------- homa_pool.c | 61 +++-- homa_skb.c | 53 ++-- homa_socktab.c | 25 +- homa_timer.c | 43 ++-- homa_utils.c | 526 ++++++++++++++------------------------ test/unit_homa_grant.c | 2 +- test/unit_homa_incoming.c | 2 +- test/unit_homa_outgoing.c | 2 +- test/unit_homa_plumbing.c | 14 +- test/unit_homa_pool.c | 4 +- test/unit_homa_timer.c | 2 +- test/unit_homa_utils.c | 2 +- test/unit_timetrace.c | 2 +- test/utils.c | 2 +- timetrace.c | 154 ++++++----- timetrace.h | 27 +- 25 files changed, 860 insertions(+), 1050 deletions(-) diff --git a/homa.h b/homa.h index 4eae954..c8ac00a 100644 --- a/homa.h +++ b/homa.h @@ -1,6 +1,4 @@ -/* Copyright (c) 2019-2022 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +/* SPDX-License-Identifier: BSD-2-Clause */ /* This file defines the kernel call interface for the Homa * transport protocol. @@ -57,11 +55,11 @@ extern "C" * Holds either an IPv4 or IPv6 address (smaller and easier to use than * sockaddr_storage). */ -typedef union sockaddr_in_union { +union sockaddr_in_union { struct sockaddr sa; struct sockaddr_in in4; struct sockaddr_in6 in6; -} sockaddr_in_union; +}; /** * struct homa_sendmsg_args - Provides information needed by Homa's @@ -122,7 +120,7 @@ struct homa_recvmsg_args { * always be set when peer information is available, which includes * some error cases. */ - sockaddr_in_union peer_addr; + union sockaddr_in_union peer_addr; /** * @num_bpages: (in/out) Number of valid entries in @bpage_offsets. @@ -216,16 +214,16 @@ struct homa_set_buf_args { extern int homa_abortp(int fd, struct homa_abort_args *args); extern int homa_send(int sockfd, const void *message_buf, - size_t length, const sockaddr_in_union *dest_addr, + size_t length, const union sockaddr_in_union *dest_addr, uint64_t *id, uint64_t completion_cookie); extern int homa_sendv(int sockfd, const struct iovec *iov, - int iovcnt, const sockaddr_in_union *dest_addr, + int iovcnt, const union sockaddr_in_union *dest_addr, uint64_t *id, uint64_t completion_cookie); extern ssize_t homa_reply(int sockfd, const void *message_buf, - size_t length, const sockaddr_in_union *dest_addr, + size_t length, const union sockaddr_in_union *dest_addr, uint64_t id); extern ssize_t homa_replyv(int sockfd, const struct iovec *iov, - int iovcnt, const sockaddr_in_union *dest_addr, + int iovcnt, const union sockaddr_in_union *dest_addr, uint64_t id); extern int homa_abort(int sockfd, uint64_t id, int error); diff --git a/homa_api.c b/homa_api.c index cdb719e..c2b31f4 100644 --- a/homa_api.c +++ b/homa_api.c @@ -1,6 +1,4 @@ -/* Copyright (c) 2019-2022 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause /* This file contains functions that implement the Homa API visible to * applications. It is intended to be part of the user-level run-time library. @@ -204,5 +202,6 @@ int homa_sendv(int sockfd, const struct iovec *iov, int iovcnt, int homa_abort(int sockfd, uint64_t id, int error) { struct homa_abort_args args = {id, error}; + return ioctl(sockfd, HOMAIOCABORT, &args); } diff --git a/homa_grant.c b/homa_grant.c index bf542e2..6bb8467 100644 --- a/homa_grant.c +++ b/homa_grant.c @@ -1,6 +1,4 @@ -/* Copyright (c) 2024 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause /* This file contains functions related to issuing grants for incoming * messages. @@ -39,14 +37,17 @@ inline int homa_grant_outranks(struct homa_rpc *rpc1, struct homa_rpc *rpc2) * may be possible to send out additional grants to some RPCs (doing * this is left to the caller). */ -inline int homa_grant_update_incoming(struct homa_rpc *rpc, struct homa *homa) { +inline int homa_grant_update_incoming(struct homa_rpc *rpc, struct homa *homa) +{ int incoming = rpc->msgin.granted - (rpc->msgin.length - rpc->msgin.bytes_remaining); + if (incoming < 0) incoming = 0; if (incoming != rpc->msgin.rec_incoming) { int delta = incoming - rpc->msgin.rec_incoming; int old = atomic_fetch_add(delta, &homa->total_incoming); + rpc->msgin.rec_incoming = incoming; return ((old >= homa->max_incoming) && ((old + delta) < homa->max_incoming)); @@ -76,6 +77,7 @@ void homa_grant_add_rpc(struct homa_rpc *rpc) * the peer's list. */ __u64 time = get_cycles(); + INC_METRIC(grantable_rpcs_integral, homa->num_grantable_rpcs * (time - homa->last_grantable_change)); homa->last_grantable_change = time; @@ -94,20 +96,22 @@ void homa_grant_add_rpc(struct homa_rpc *rpc) } } list_add_tail(&rpc->grantable_links, &peer->grantable_rpcs); - } else while (rpc != list_first_entry(&peer->grantable_rpcs, - struct homa_rpc, grantable_links)) { - /* Message is on the list, but its priority may have - * increased because of the recent packet arrival. If so, - * adjust its position in the list. - */ - candidate = list_prev_entry(rpc, grantable_links); - if (!homa_grant_outranks(rpc, candidate)) - goto position_peer; - __list_del_entry(&candidate->grantable_links); - list_add(&candidate->grantable_links, &rpc->grantable_links); + } else { + while (rpc != list_first_entry(&peer->grantable_rpcs, + struct homa_rpc, grantable_links)) { + /* Message is on the list, but its priority may have + * increased because of the recent packet arrival. If + * so, adjust its position in the list. + */ + candidate = list_prev_entry(rpc, grantable_links); + if (!homa_grant_outranks(rpc, candidate)) + goto position_peer; + __list_del_entry(&candidate->grantable_links); + list_add(&candidate->grantable_links, &rpc->grantable_links); + } } - position_peer: +position_peer: /* At this point rpc is positioned correctly on the list for its peer. * However, the peer may need to be added to, or moved upward on, * homa->grantable_peers. @@ -127,8 +131,8 @@ void homa_grant_add_rpc(struct homa_rpc *rpc) list_add_tail(&peer->grantable_links, &homa->grantable_peers); goto done; } - /* The peer is on Homa's list, but it may need to move upward. */ - while (peer != list_first_entry(&homa->grantable_peers, + /* The peer is on Homa's list, but it may need to move upward. */ + while (peer != list_first_entry(&homa->grantable_peers, struct homa_peer, grantable_links)) { struct homa_peer *prev_peer = list_prev_entry( peer, grantable_links); @@ -139,7 +143,7 @@ void homa_grant_add_rpc(struct homa_rpc *rpc) __list_del_entry(&prev_peer->grantable_links); list_add(&prev_peer->grantable_links, &peer->grantable_links); } - done: +done: } /** @@ -188,7 +192,7 @@ void homa_grant_remove_rpc(struct homa_rpc *rpc) */ head = list_first_entry(&peer->grantable_rpcs, struct homa_rpc, grantable_links); - while (peer != list_last_entry(&homa->grantable_peers, struct homa_peer, + while (peer != list_last_entry(&homa->grantable_peers, struct homa_peer, grantable_links)) { struct homa_peer *next_peer = list_next_entry( peer, grantable_links); @@ -245,10 +249,10 @@ int homa_grant_send(struct homa_rpc *rpc, struct homa *homa) grant.priority = rpc->msgin.priority; grant.resend_all = rpc->msgin.resend_all; rpc->msgin.resend_all = 0; - tt_record4("sending grant for id %llu, offset %d, priority %d, " - "increment %d", rpc->id, rpc->msgin.granted, - rpc->msgin.priority, increment); - homa_xmit_control(GRANT, &grant, sizeof(grant),rpc); + tt_record4("sending grant for id %llu, offset %d, priority %d, increment %d", + rpc->id, rpc->msgin.granted, rpc->msgin.priority, + increment); + homa_xmit_control(GRANT, &grant, sizeof(grant), rpc); return 1; } @@ -286,20 +290,20 @@ void homa_grant_check_rpc(struct homa_rpc *rpc) } if (rpc->msgin.granted >= rpc->msgin.length) { - homa_grant_update_incoming(rpc,homa); + homa_grant_update_incoming(rpc, homa); homa_rpc_unlock(rpc); goto done; } - tt_record4("homa_grant_check_rpc starting for id %d, granted %d, " - "recv_end %d, length %d", rpc->id, rpc->msgin.granted, - rpc->msgin.recv_end, rpc->msgin.length); + tt_record4("homa_grant_check_rpc starting for id %d, granted %d, recv_end %d, length %d", + rpc->id, rpc->msgin.granted, rpc->msgin.recv_end, + rpc->msgin.length); /* This message requires grants; if it is a new message, set up * granting. */ if (list_empty(&rpc->grantable_links)) { - homa_grant_update_incoming(rpc,homa); + homa_grant_update_incoming(rpc, homa); homa_grantable_lock(homa, 0); homa_grant_add_rpc(rpc); recalc = ((homa->num_active_rpcs < homa->max_overcommit) @@ -355,7 +359,7 @@ void homa_grant_check_rpc(struct homa_rpc *rpc) homa_rpc_unlock(rpc); if (recalc) homa_grant_recalc(homa, 0); - done: +done: tt_record1("homa_grant_check_rpc finished with id %d", rpc->id); } @@ -405,9 +409,8 @@ void homa_grant_recalc(struct homa *homa, int locked) atomic_inc(&homa->grant_recalc_count); /* Clear the existing grant calculation. */ - for (i = 0; i < homa->num_active_rpcs; i++) { + for (i = 0; i < homa->num_active_rpcs; i++) atomic_set(&homa->active_rpcs[i]->msgin.rank, -1); - } /* Recompute which RPCs we'll grant to and initialize info * about them. @@ -653,8 +656,7 @@ int homa_grantable_lock_slow(struct homa *homa, int recalc) } if (recalc && atomic_read(&homa->grant_recalc_count) != starting_count) { - tt_record("skipping wait for grantable lock: recalc " - "elsewhere"); + tt_record("skipping wait for grantable lock: recalc elsewhere"); break; } } diff --git a/homa_impl.h b/homa_impl.h index a0e68e0..b8ec220 100644 --- a/homa_impl.h +++ b/homa_impl.h @@ -1,6 +1,4 @@ -/* Copyright (c) 2019-2023 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +/* SPDX-License-Identifier: BSD-2-Clause */ /* This file contains definitions that are shared across the files * that implement Homa for Linux. @@ -50,14 +48,10 @@ #pragma GCC diagnostic warning "-Wpointer-sign" #pragma GCC diagnostic warning "-Wunused-variable" -#if LINUX_VERSION_CODE < KERNEL_VERSION(4,16,0) -typedef unsigned int __poll_t; -#endif - #ifdef __UNIT_TEST__ #undef alloc_pages #define alloc_pages mock_alloc_pages -extern struct page *mock_alloc_pages(gfp_t gfp, unsigned order); +extern struct page *mock_alloc_pages(gfp_t gfp, unsigned int order); #define compound_order mock_compound_order extern unsigned int mock_compound_order(struct page *page); @@ -74,12 +68,15 @@ extern struct task_struct *current_task; extern cycles_t mock_get_cycles(void); #define get_page mock_get_page - extern void mock_get_page(struct page *page); +extern void mock_get_page(struct page *page); #undef kmalloc #define kmalloc mock_kmalloc extern void *mock_kmalloc(size_t size, gfp_t flags); +#undef kmalloc_array +#define kmalloc_array(count, size, type) mock_kmalloc(count*size, type) + #define kthread_complete_and_exit(comp, code) #ifdef page_address @@ -317,7 +314,8 @@ struct common_header { /** * @checksum: not used by Homa, but must occupy the same bytes as - * the checksum in a TCP header (TSO may modify this?).*/ + * the checksum in a TCP header (TSO may modify this?). + */ __be16 checksum; /** @@ -335,7 +333,7 @@ struct common_header { * this RPC). */ __be64 sender_id; -} __attribute__((packed)); +} __packed; /** * struct homa_ack - Identifies an RPC that can be safely deleted by its @@ -359,7 +357,7 @@ struct homa_ack { /** @server_port: The server-side port for the RPC. */ __be16 server_port; -} __attribute__((packed)); +} __packed; /* struct data_header - Contains data for part or all of a Homa message. * An incoming packet consists of a data_header followed by message data. @@ -415,7 +413,7 @@ struct seg_header { * value will always be valid once the packet reaches homa_softirq. */ __be32 offset; -} __attribute__((packed)); +} __packed; struct data_header { struct common_header common; @@ -459,17 +457,14 @@ struct data_header { /** @seg: First of possibly many segments. */ struct seg_header seg; -} __attribute__((packed)); +} __packed; _Static_assert(sizeof(struct data_header) <= HOMA_MAX_HEADER, - "data_header too large for HOMA_MAX_HEADER; must " - "adjust HOMA_MAX_HEADER"); + "data_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER"); _Static_assert(sizeof(struct data_header) >= HOMA_MIN_PKT_LENGTH, - "data_header too small: Homa doesn't currently have code" - "to pad data packets"); + "data_header too small: Homa doesn't currently have codeto pad data packets"); _Static_assert(((sizeof(struct data_header) - sizeof(struct seg_header)) & 0x3) == 0, - " data_header length not a multiple of 4 bytes (required " - "for TCP/TSO compatibility"); + " data_header length not a multiple of 4 bytes (required for TCP/TSO compatibility"); /** * homa_data_len() - Returns the total number of bytes in a DATA packet @@ -511,10 +506,9 @@ struct grant_header { * that no packets have been successfully received). */ __u8 resend_all; -} __attribute__((packed)); +} __packed; _Static_assert(sizeof(struct grant_header) <= HOMA_MAX_HEADER, - "grant_header too large for HOMA_MAX_HEADER; must " - "adjust HOMA_MAX_HEADER"); + "grant_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER"); /** * struct resend_header - Wire format for RESEND packets. @@ -550,10 +544,9 @@ struct resend_header { * priority. */ __u8 priority; -} __attribute__((packed)); +} __packed; _Static_assert(sizeof(struct resend_header) <= HOMA_MAX_HEADER, - "resend_header too large for HOMA_MAX_HEADER; must " - "adjust HOMA_MAX_HEADER"); + "resend_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER"); /** * struct unknown_header - Wire format for UNKNOWN packets. @@ -567,10 +560,9 @@ _Static_assert(sizeof(struct resend_header) <= HOMA_MAX_HEADER, struct unknown_header { /** @common: Fields common to all packet types. */ struct common_header common; -} __attribute__((packed)); +} __packed; _Static_assert(sizeof(struct unknown_header) <= HOMA_MAX_HEADER, - "unknown_header too large for HOMA_MAX_HEADER; must " - "adjust HOMA_MAX_HEADER"); + "unknown_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER"); /** * struct busy_header - Wire format for BUSY packets. @@ -581,10 +573,9 @@ _Static_assert(sizeof(struct unknown_header) <= HOMA_MAX_HEADER, struct busy_header { /** @common: Fields common to all packet types. */ struct common_header common; -} __attribute__((packed)); +} __packed; _Static_assert(sizeof(struct busy_header) <= HOMA_MAX_HEADER, - "busy_header too large for HOMA_MAX_HEADER; must " - "adjust HOMA_MAX_HEADER"); + "busy_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER"); /** * struct cutoffs_header - Wire format for CUTOFFS packets. @@ -609,10 +600,9 @@ struct cutoffs_header { * this packet. */ __be16 cutoff_version; -} __attribute__((packed)); +} __packed; _Static_assert(sizeof(struct cutoffs_header) <= HOMA_MAX_HEADER, - "cutoffs_header too large for HOMA_MAX_HEADER; must " - "adjust HOMA_MAX_HEADER"); + "cutoffs_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER"); /** * struct freeze_header - Wire format for FREEZE packets. @@ -623,10 +613,9 @@ _Static_assert(sizeof(struct cutoffs_header) <= HOMA_MAX_HEADER, struct freeze_header { /** @common: Fields common to all packet types. */ struct common_header common; -} __attribute__((packed)); +} __packed; _Static_assert(sizeof(struct freeze_header) <= HOMA_MAX_HEADER, - "freeze_header too large for HOMA_MAX_HEADER; must " - "adjust HOMA_MAX_HEADER"); + "freeze_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER"); /** * struct need_ack_header - Wire format for NEED_ACK packets. @@ -637,10 +626,9 @@ _Static_assert(sizeof(struct freeze_header) <= HOMA_MAX_HEADER, struct need_ack_header { /** @common: Fields common to all packet types. */ struct common_header common; -} __attribute__((packed)); +} __packed; _Static_assert(sizeof(struct need_ack_header) <= HOMA_MAX_HEADER, - "need_ack_header too large for HOMA_MAX_HEADER; must " - "adjust HOMA_MAX_HEADER"); + "need_ack_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER"); /** * struct ack_header - Wire format for ACK packets. @@ -657,10 +645,9 @@ struct ack_header { __be16 num_acks; struct homa_ack acks[NUM_PEER_UNACKED_IDS]; -} __attribute__((packed)); +} __packed; _Static_assert(sizeof(struct ack_header) <= HOMA_MAX_HEADER, - "ack_header too large for HOMA_MAX_HEADER; must " - "adjust HOMA_MAX_HEADER"); + "ack_header too large for HOMA_MAX_HEADER; must adjust HOMA_MAX_HEADER"); /** * struct homa_message_out - Describes a message (either request or response) @@ -799,7 +786,7 @@ struct homa_message_in { * Never larger than @length. Note: once initialized, this * may not be modified without holding @homa->grantable_lock. */ - int granted; + int granted; /** * @rec_incoming: Number of bytes in homa->total_incoming currently @@ -896,7 +883,7 @@ struct homa_interest { * of a struct homa_interest. * @interest: Struct to initialize. */ -inline static void homa_interest_init(struct homa_interest *interest) +static inline void homa_interest_init(struct homa_interest *interest) { interest->thread = current; atomic_long_set(&interest->ready_rpc, 0); @@ -1132,10 +1119,11 @@ struct homa_rpc { * trace. * @rpc: RPC to validate. */ -inline static void homa_rpc_validate(struct homa_rpc *rpc) { +static inline void homa_rpc_validate(struct homa_rpc *rpc) +{ if (rpc->magic == HOMA_RPC_MAGIC) return; - printk(KERN_ERR "Accessing reaped Homa RPC!\n"); + pr_err("Accessing reaped Homa RPC!\n"); BUG(); } @@ -1158,7 +1146,7 @@ struct homa_socktab { * for socket lookups (RCU is used instead). Also used to * synchronize port allocation. */ - struct spinlock write_lock; + spinlock_t write_lock; /** * @buckets: Heads of chains for hash table buckets. Chains @@ -1221,7 +1209,7 @@ struct homa_rpc_bucket { * this bucket. This dual purpose permits clean and safe * deletion and garbage collection of RPCs. */ - struct spinlock lock; + spinlock_t lock; /** @rpcs: list of RPCs that hash to this bucket. */ struct hlist_head rpcs; @@ -1247,7 +1235,7 @@ struct homa_bpage { struct homa_cache_line cache_line; struct { /** @lock: to synchronize shared access. */ - struct spinlock lock; + spinlock_t lock; /** * @refs: Counts number of distinct uses of this @@ -1393,11 +1381,12 @@ struct homa_sock { * spin lock). See sync.txt for more on Homa's synchronization * strategy. */ - struct spinlock lock; + spinlock_t lock; /** * @last_locker: identifies the code that most recently acquired - * @lock successfully. Occasionally used for debugging. */ + * @lock successfully. Occasionally used for debugging. + */ char *last_locker; /** @@ -1552,7 +1541,7 @@ struct homa_peertab { * @write_lock: Synchronizes addition of new entries; not needed * for lookups (RCU is used instead). */ - struct spinlock write_lock; + spinlock_t write_lock; /** * @dead_dsts: List of dst_entries that are waiting to be deleted. @@ -1687,7 +1676,7 @@ struct homa_peer { /** * @ack_lock: used to synchronize access to @num_acks and @acks. */ - struct spinlock ack_lock; + spinlock_t ack_lock; }; /** @@ -1709,8 +1698,7 @@ enum homa_freeze_type { * use by a single NUMA node. Access to these objects is synchronized with * @homa->page_pool_mutex. */ -struct homa_page_pool -{ +struct homa_page_pool { /** @avail: Number of free pages currently in the pool. */ int avail; @@ -1751,13 +1739,13 @@ struct homa { * it could be a severe underestimate if there is competing traffic * from, say, TCP. Access only with atomic ops. */ - atomic64_t link_idle_time __attribute__((aligned(CACHE_LINE_SIZE))); + atomic64_t link_idle_time __aligned(CACHE_LINE_SIZE); /** * @grantable_lock: Used to synchronize access to grant-related * fields below, from @grantable_peers to @last_grantable_change. */ - struct spinlock grantable_lock __attribute__((aligned(CACHE_LINE_SIZE))); + spinlock_t grantable_lock __aligned(CACHE_LINE_SIZE); /** * @grantable_lock_time: get_cycles() time when grantable_lock @@ -1856,7 +1844,7 @@ struct homa { * @pacer_mutex: Ensures that only one instance of homa_pacer_xmit * runs at a time. Only used in "try" mode: never block on this. */ - struct spinlock pacer_mutex __attribute__((aligned(CACHE_LINE_SIZE))); + spinlock_t pacer_mutex __aligned(CACHE_LINE_SIZE); /** * @pacer_fifo_fraction: The fraction of time (in thousandths) when @@ -1882,7 +1870,7 @@ struct homa { * insert or remove an RPC from throttled_rpcs, must first acquire * the RPC's socket lock, then this lock. */ - struct spinlock throttle_lock; + spinlock_t throttle_lock; /** * @throttled_rpcs: Contains all homa_rpcs that have bytes ready @@ -1918,7 +1906,7 @@ struct homa { * a peer sends more bytes than granted (see synchronization note in * homa_send_grants for why we have to allow this possibility). */ - atomic_t total_incoming __attribute__((aligned(CACHE_LINE_SIZE))); + atomic_t total_incoming __aligned(CACHE_LINE_SIZE); /** * @next_client_port: A client port number to consider for the @@ -1926,23 +1914,23 @@ struct homa { * be in the range allocated for servers; must check before using. * This port may also be in use already; must check. */ - __u16 next_client_port __attribute__((aligned(CACHE_LINE_SIZE))); + __u16 next_client_port __aligned(CACHE_LINE_SIZE); /** * @port_map: Information about all open sockets. */ - struct homa_socktab port_map __attribute__((aligned(CACHE_LINE_SIZE))); + struct homa_socktab port_map __aligned(CACHE_LINE_SIZE); /** * @peertab: Info about all the other hosts we have communicated with. */ struct homa_peertab peers; - /** + /** * @page_pool_mutex: Synchronizes access to any/all of the page_pools * used for outgoing sk_buff data. */ - struct spinlock page_pool_mutex __attribute__((aligned(CACHE_LINE_SIZE))); + spinlock_t page_pool_mutex __aligned(CACHE_LINE_SIZE); /** * @skb_page_frees_per_sec: Rate at which to return pages from sk_buff @@ -1957,7 +1945,7 @@ struct homa { */ struct page **skb_pages_to_free; - /** + /** * @pages_to_free_slot: Maximum number of pages that can be * stored in skb_pages_to_free; */ @@ -2290,7 +2278,7 @@ struct homa { * @metrics_lock: Used to synchronize accesses to @metrics_active_opens * and updates to @metrics. */ - struct spinlock metrics_lock; + spinlock_t metrics_lock; /* * @metrics: a human-readable string containing recent values @@ -2298,7 +2286,7 @@ struct homa { * homa_append_metric. This string is kmalloc-ed; NULL means * homa_append_metric has never been called. */ - char* metrics; + char *metrics; /** @metrics_capacity: number of bytes available at metrics. */ size_t metrics_capacity; @@ -2547,7 +2535,8 @@ struct homa_metrics { __u64 send_cycles; /** @send_calls: total number of invocations of homa_semdmsg - * for requests. */ + * for requests. + */ __u64 send_calls; /** @@ -3068,17 +3057,17 @@ struct homa_core { */ __u64 last_app_active; - /** - * held_skb: last packet buffer known to be available for - * merging other packets into on this core (note: may not still - * be available), or NULL if none. - */ - struct sk_buff *held_skb; + /** + * held_skb: last packet buffer known to be available for + * merging other packets into on this core (note: may not still + * be available), or NULL if none. + */ + struct sk_buff *held_skb; /** * @held_bucket: the index, within napi->gro_hash, of the list - * containing @held_skb; undefined if @held_skb is NULL. Used to - * verify that @held_skb is still available. + * containing @held_skb; undefined if @held_skb is NULL. Used to + * verify that @held_skb is still available. */ int held_bucket; @@ -3101,7 +3090,7 @@ struct homa_core { */ int rpcs_locked; - /** + /** * @skb_page: a page of data available being used for skb frags. * This pointer is included in the page's reference count. */ @@ -3130,7 +3119,7 @@ struct homa_core { */ int num_stashed_pages; - /** + /** * @stashed_pages: use to prefetch from the cache all of the pages a * message will need with a single operation, to avoid having to * synchronize separately for each page. Note: these pages are all @@ -3168,7 +3157,7 @@ struct homa_skb_info { */ int data_bytes; - /** @seg_length: maximum number of data bytes in each GSO segment. */ + /** @seg_length: maximum number of data bytes in each GSO segment. */ int seg_length; /** @@ -3224,13 +3213,14 @@ static inline __u64 homa_local_id(__be64 sender_id) * @locker: Static string identifying the locking code. Normally ignored, * but used occasionally for diagnostics and debugging. */ -inline static void homa_bucket_lock(struct homa_rpc_bucket *bucket, - __u64 id, char *locker) +static inline void homa_bucket_lock(struct homa_rpc_bucket *bucket, + __u64 id, const char *locker) { int core = raw_smp_processor_id(); + if (!spin_trylock_bh(&bucket->lock)) homa_bucket_lock_slow(bucket, id); - homa_cores[core]->rpcs_locked ++; + homa_cores[core]->rpcs_locked++; BUG_ON(homa_cores[core]->rpcs_locked > 1); } @@ -3244,13 +3234,14 @@ inline static void homa_bucket_lock(struct homa_rpc_bucket *bucket, * Return: Nonzero if lock was successfully acquired, zero if it is * currently owned by someone else. */ -inline static int homa_bucket_try_lock(struct homa_rpc_bucket *bucket, - __u64 id, char *locker) +static inline int homa_bucket_try_lock(struct homa_rpc_bucket *bucket, + __u64 id, const char *locker) { int core = raw_smp_processor_id(); + if (!spin_trylock_bh(&bucket->lock)) return 0; - homa_cores[core]->rpcs_locked ++; + homa_cores[core]->rpcs_locked++; BUG_ON(homa_cores[core]->rpcs_locked > 1); return 1; } @@ -3260,7 +3251,7 @@ inline static int homa_bucket_try_lock(struct homa_rpc_bucket *bucket, * @bucket: Bucket to unlock. * @id: ID of the RPC that was using the lock. */ -inline static void homa_bucket_unlock(struct homa_rpc_bucket *bucket, __u64 id) +static inline void homa_bucket_unlock(struct homa_rpc_bucket *bucket, __u64 id) { homa_cores[raw_smp_processor_id()]->rpcs_locked--; spin_unlock_bh(&bucket->lock); @@ -3279,7 +3270,8 @@ inline static void homa_bucket_unlock(struct homa_rpc_bucket *bucket, __u64 id) * @locker: Static string identifying the locking code. Normally ignored, * but used occasionally for diagnostics and debugging. */ -inline static void homa_rpc_lock(struct homa_rpc *rpc, char *locker) { +static inline void homa_rpc_lock(struct homa_rpc *rpc, const char *locker) +{ homa_bucket_lock(rpc->bucket, rpc->id, locker); } @@ -3287,7 +3279,8 @@ inline static void homa_rpc_lock(struct homa_rpc *rpc, char *locker) { * homa_rpc_unlock() - Release the lock for an RPC. * @rpc: RPC to unlock. */ -inline static void homa_rpc_unlock(struct homa_rpc *rpc) { +static inline void homa_rpc_unlock(struct homa_rpc *rpc) +{ homa_bucket_unlock(rpc->bucket, rpc->id); } @@ -3321,7 +3314,7 @@ static inline struct homa_rpc_bucket *homa_client_rpc_bucket( */ static inline struct sk_buff **homa_next_skb(struct sk_buff *skb) { - return (struct sk_buff **) (skb_end_pointer(skb) - sizeof(char*)); + return (struct sk_buff **) (skb_end_pointer(skb) - sizeof(char *)); } /** @@ -3384,7 +3377,8 @@ static inline struct homa_sock *homa_sk(const struct sock *sk) * @locker: Static string identifying where the socket was locked; * used to track down deadlocks. */ -static inline void homa_sock_lock(struct homa_sock *hsk, char *locker) { +static inline void homa_sock_lock(struct homa_sock *hsk, const char *locker) +{ if (!spin_trylock_bh(&hsk->lock)) { // printk(KERN_NOTICE "Slow path for socket %d, last locker %s", // hsk->client_port, hsk->last_locker); @@ -3397,7 +3391,8 @@ static inline void homa_sock_lock(struct homa_sock *hsk, char *locker) { * homa_sock_unlock() - Release the lock for a socket. * @hsk: Socket to lock. */ -static inline void homa_sock_unlock(struct homa_sock *hsk) { +static inline void homa_sock_unlock(struct homa_sock *hsk) +{ spin_unlock_bh(&hsk->lock); } @@ -3408,9 +3403,8 @@ static inline void homa_sock_unlock(struct homa_sock *hsk) { */ static inline void homa_peer_lock(struct homa_peer *peer) { - if (!spin_trylock_bh(&peer->ack_lock)) { + if (!spin_trylock_bh(&peer->ack_lock)) homa_peer_lock_slow(peer); - } } /** @@ -3424,10 +3418,9 @@ static inline void homa_peer_unlock(struct homa_peer *peer) /** * homa_protect_rpcs() - Ensures that no RPCs will be reaped for a given - * socket until until homa_sock_unprotect is called. Typically - * used by functions that want to scan the active RPCs for a socket - * without holding the socket lock. Multiple calls to this function may - * be in effect at once. + * socket until homa_sock_unprotect is called. Typically used by functions + * that want to scan the active RPCs for a socket without holding the socket + * lock. Multiple calls to this function may be in effect at once. * @hsk: Socket whose RPCs should be protected. Must not be locked * by the caller; will be locked here. * @@ -3437,7 +3430,8 @@ static inline void homa_peer_unlock(struct homa_peer *peer) static inline int homa_protect_rpcs(struct homa_sock *hsk) { int result; - homa_sock_lock(hsk, "homa_sock_protect"); + + homa_sock_lock(hsk, __func__); result = !hsk->shutdown; if (result) atomic_inc(&hsk->protect_count); @@ -3496,9 +3490,8 @@ static inline void homa_grantable_unlock(struct homa *homa) */ static inline void homa_throttle_lock(struct homa *homa) { - if (!spin_trylock_bh(&homa->throttle_lock)) { + if (!spin_trylock_bh(&homa->throttle_lock)) homa_throttle_lock_slow(homa); - } } /** @@ -3526,7 +3519,9 @@ static inline bool skb_is_ipv6(const struct sk_buff *skb) static inline struct in6_addr ipv4_to_ipv6(__be32 ip4) { struct in6_addr ret = {}; - if (ip4 == INADDR_ANY) return in6addr_any; + + if (ip4 == INADDR_ANY) + return in6addr_any; ret.in6_u.u6_addr32[2] = htonl(0xffff); ret.in6_u.u6_addr32[3] = ip4; return ret; @@ -3548,7 +3543,7 @@ static inline __be32 ipv6_to_ipv4(const struct in6_addr ip6) * was IPv4, convert it to an IPv4-mapped IPv6 address. * @addr: Address to canonicalize. */ -static inline struct in6_addr canonical_ipv6_addr(const sockaddr_in_union *addr) +static inline struct in6_addr canonical_ipv6_addr(const union sockaddr_in_union *addr) { if (addr) { return (addr->sa.sa_family == AF_INET6) @@ -3587,6 +3582,7 @@ static inline bool is_mapped_ipv4(const struct in6_addr x) static inline bool is_homa_pkt(struct sk_buff *skb) { struct iphdr *iph = ip_hdr(skb); + return ((iph->protocol == IPPROTO_HOMA) || ((iph->protocol == IPPROTO_TCP) && (tcp_hdr(skb)->urg_ptr == htons(HOMA_TCP_URGENT)))); @@ -3605,8 +3601,8 @@ static inline __be32 tt_addr(const struct in6_addr x) } #ifdef __UNIT_TEST__ -extern void unit_log_printf(const char *separator, const char* format, ...) - __attribute__((format(printf, 2, 3))); +extern void unit_log_printf(const char *separator, const char *format, ...) + __printf(2, 3); #define UNIT_LOG unit_log_printf extern void unit_hook(char *id); #define UNIT_HOOK(msg) unit_hook(msg) @@ -3622,42 +3618,43 @@ extern void homa_ack_pkt(struct sk_buff *skb, struct homa_sock *hsk, struct homa_rpc *rpc); extern void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb); extern void homa_add_to_throttled(struct homa_rpc *rpc); -extern void homa_append_metric(struct homa *homa, const char* format, ...); +extern void homa_append_metric(struct homa *homa, const char *format, ...); extern int homa_backlog_rcv(struct sock *sk, struct sk_buff *skb); extern int homa_bind(struct socket *sk, struct sockaddr *addr, - int addr_len); + int addr_len); extern void homa_bucket_unlock(struct homa_rpc_bucket *bucket, __u64 id); extern void homa_check_rpc(struct homa_rpc *rpc); extern int homa_check_nic_queue(struct homa *homa, struct sk_buff *skb, - bool force); + bool force); extern struct homa_rpc *homa_choose_fifo_grant(struct homa *homa); extern struct homa_interest - *homa_choose_interest(struct homa *homa, struct list_head *head, - int offset); + *homa_choose_interest(struct homa *homa, struct list_head *head, + int offset); extern void homa_close(struct sock *sock, long timeout); extern int homa_copy_to_user(struct homa_rpc *rpc); extern void homa_cutoffs_pkt(struct sk_buff *skb, struct homa_sock *hsk); extern void homa_data_from_server(struct sk_buff *skb, - struct homa_rpc *crpc); + struct homa_rpc *crpc); extern void homa_data_pkt(struct sk_buff *skb, struct homa_rpc *rpc); extern void homa_destroy(struct homa *homa); extern int homa_diag_destroy(struct sock *sk, int err); extern int homa_disconnect(struct sock *sk, int flags); extern void homa_dispatch_pkts(struct sk_buff *skb, struct homa *homa); extern int homa_dointvec(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, loff_t *ppos); + void __user *buffer, size_t *lenp, loff_t *ppos); extern void homa_dst_refresh(struct homa_peertab *peertab, - struct homa_peer *peer, struct homa_sock *hsk); + struct homa_peer *peer, struct homa_sock *hsk); extern int homa_err_handler_v4(struct sk_buff *skb, u32 info); -extern int homa_err_handler_v6(struct sk_buff *skb, struct inet6_skb_parm * - , u8, u8, int, __be32); +extern int homa_err_handler_v6(struct sk_buff *skb, + struct inet6_skb_parm *opt, u8 type, u8 code, int offset, + __be32 info); extern int homa_fill_data_interleaved(struct homa_rpc *rpc, struct sk_buff *skb, struct iov_iter *iter); extern struct homa_rpc - *homa_find_client_rpc(struct homa_sock *hsk, __u64 id); + *homa_find_client_rpc(struct homa_sock *hsk, __u64 id); extern struct homa_rpc - *homa_find_server_rpc(struct homa_sock *hsk, + *homa_find_server_rpc(struct homa_sock *hsk, const struct in6_addr *saddr, __u16 sport, __u64 id); extern void homa_freeze(struct homa_rpc *rpc, enum homa_freeze_type type, char *format); @@ -3667,7 +3664,7 @@ extern struct homa_gap extern void homa_gap_retry(struct homa_rpc *rpc); extern int homa_get_port(struct sock *sk, unsigned short snum); extern int homa_getsockopt(struct sock *sk, int level, int optname, - char __user *optval, int __user *option); + char __user *optval, int __user *option); extern void homa_grant_add_rpc(struct homa_rpc *rpc); extern void homa_grant_check_rpc(struct homa_rpc *rpc); extern void homa_grant_find_oldest(struct homa *homa); @@ -3689,14 +3686,14 @@ extern void homa_gro_gen3(struct sk_buff *skb); extern void homa_gro_hook_tcp(void); extern void homa_gro_unhook_tcp(void); extern struct sk_buff - *homa_gro_receive(struct list_head *gro_list, - struct sk_buff *skb); + *homa_gro_receive(struct list_head *gro_list, + struct sk_buff *skb); extern struct sk_buff - *homa_gso_segment(struct sk_buff *skb, + *homa_gso_segment(struct sk_buff *skb, netdev_features_t features); extern int homa_hash(struct sock *sk); extern enum hrtimer_restart - homa_hrtimer(struct hrtimer *timer); + homa_hrtimer(struct hrtimer *timer); extern int homa_init(struct homa *homa); extern void homa_incoming_sysctl_changed(struct homa *homa); extern int homa_ioc_abort(struct sock *sk, int *karg); @@ -3711,12 +3708,12 @@ extern loff_t homa_metrics_lseek(struct file *file, loff_t offset, int whence); extern int homa_metrics_open(struct inode *inode, struct file *file); extern ssize_t homa_metrics_read(struct file *file, char __user *buffer, - size_t length, loff_t *offset); + size_t length, loff_t *offset); extern int homa_metrics_release(struct inode *inode, struct file *file); extern void homa_need_ack_pkt(struct sk_buff *skb, struct homa_sock *hsk, struct homa_rpc *rpc); extern struct sk_buff - *homa_new_data_packet(struct homa_rpc *rpc, + *homa_new_data_packet(struct homa_rpc *rpc, struct iov_iter *iter, int offset, int length, int max_seg_data); extern int homa_offload_end(void); @@ -3732,18 +3729,18 @@ extern struct homa_peer ** extern int homa_peertab_init(struct homa_peertab *peertab); extern void homa_peer_add_ack(struct homa_rpc *rpc); extern struct homa_peer - *homa_peer_find(struct homa_peertab *peertab, + *homa_peer_find(struct homa_peertab *peertab, const struct in6_addr *addr, struct inet_sock *inet); extern int homa_peer_get_acks(struct homa_peer *peer, int count, struct homa_ack *dst); extern struct dst_entry - *homa_peer_get_dst(struct homa_peer *peer, + *homa_peer_get_dst(struct homa_peer *peer, struct inet_sock *inet); extern void homa_peer_set_cutoffs(struct homa_peer *peer, int c0, int c1, - int c2, int c3, int c4, int c5, int c6, int c7); + int c2, int c3, int c4, int c5, int c6, int c7); extern void homa_peertab_gc_dsts(struct homa_peertab *peertab, __u64 now); extern __poll_t homa_poll(struct file *file, struct socket *sock, - struct poll_table_struct *wait); + struct poll_table_struct *wait); extern int homa_pool_allocate(struct homa_rpc *rpc); extern void homa_pool_check_waiting(struct homa_pool *pool); extern void homa_pool_destroy(struct homa_pool *pool); @@ -3760,20 +3757,20 @@ extern char *homa_print_ipv6_addr(const struct in6_addr *addr); extern char *homa_print_metrics(struct homa *homa); extern char *homa_print_packet(struct sk_buff *skb, char *buffer, int buf_len); extern char *homa_print_packet_short(struct sk_buff *skb, char *buffer, - int buf_len); + int buf_len); extern void homa_prios_changed(struct homa *homa); extern int homa_proc_read_metrics(char *buffer, char **start, off_t offset, - int count, int *eof, void *data); + int count, int *eof, void *data); extern int homa_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, - int flags, int *addr_len); + int flags, int *addr_len); extern int homa_register_interests(struct homa_interest *interest, - struct homa_sock *hsk, int flags, __u64 id); + struct homa_sock *hsk, int flags, __u64 id); extern void homa_rehash(struct sock *sk); extern void homa_remove_from_throttled(struct homa_rpc *rpc); extern void homa_resend_data(struct homa_rpc *rpc, int start, int end, - int priority); + int priority); extern void homa_resend_pkt(struct sk_buff *skb, struct homa_rpc *rpc, - struct homa_sock *hsk); + struct homa_sock *hsk); extern void homa_rpc_abort(struct homa_rpc *crpc, int error); extern void homa_rpc_acked(struct homa_sock *hsk, const struct in6_addr *saddr, struct homa_ack *ack); @@ -3785,17 +3782,17 @@ extern void homa_rpc_log_tt(struct homa_rpc *rpc); extern void homa_rpc_log_active(struct homa *homa, uint64_t id); extern void homa_rpc_log_active_tt(struct homa *homa, int freeze_count); extern struct homa_rpc - *homa_rpc_new_client(struct homa_sock *hsk, - const sockaddr_in_union *dest); + *homa_rpc_new_client(struct homa_sock *hsk, + const union sockaddr_in_union *dest); extern struct homa_rpc - *homa_rpc_new_server(struct homa_sock *hsk, + *homa_rpc_new_server(struct homa_sock *hsk, const struct in6_addr *source, struct data_header *h, int *created); extern int homa_rpc_reap(struct homa_sock *hsk, int count); extern void homa_send_ipis(void); extern int homa_sendmsg(struct sock *sk, struct msghdr *msg, size_t len); extern int homa_setsockopt(struct sock *sk, int level, int optname, - sockptr_t __user optval, unsigned int optlen); + sockptr_t __user optval, unsigned int optlen); extern int homa_shutdown(struct socket *sock, int how); extern int homa_skb_append_from_iter(struct homa *homa, struct sk_buff *skb, struct iov_iter *iter, int length); @@ -3821,52 +3818,51 @@ extern void homa_skb_page_pool_init(struct homa_page_pool *pool); extern void homa_skb_release_pages(struct homa *homa); extern void homa_skb_stash_pages(struct homa *homa, int length); extern int homa_snprintf(char *buffer, int size, int used, - const char *format, ...) - __attribute__((format(printf, 4, 5))); + const char *format, ...) __printf(4, 5); extern int homa_sock_bind(struct homa_socktab *socktab, - struct homa_sock *hsk, __u16 port); + struct homa_sock *hsk, __u16 port); extern void homa_sock_destroy(struct homa_sock *hsk); extern struct homa_sock * - homa_sock_find(struct homa_socktab *socktab, __u16 port); + homa_sock_find(struct homa_socktab *socktab, __u16 port); extern void homa_sock_init(struct homa_sock *hsk, struct homa *homa); extern void homa_sock_shutdown(struct homa_sock *hsk); extern int homa_socket(struct sock *sk); extern void homa_socktab_destroy(struct homa_socktab *socktab); extern void homa_socktab_init(struct homa_socktab *socktab); extern struct homa_sock - *homa_socktab_next(struct homa_socktab_scan *scan); + *homa_socktab_next(struct homa_socktab_scan *scan); extern struct homa_sock - *homa_socktab_start_scan(struct homa_socktab *socktab, - struct homa_socktab_scan *scan); + *homa_socktab_start_scan(struct homa_socktab *socktab, + struct homa_socktab_scan *scan); extern int homa_softirq(struct sk_buff *skb); extern void homa_spin(int ns); extern char *homa_symbol_for_state(struct homa_rpc *rpc); extern char *homa_symbol_for_type(uint8_t type); extern int homa_sysctl_softirq_cores(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, loff_t *ppos); + void __user *buffer, size_t *lenp, loff_t *ppos); extern struct sk_buff - *homa_tcp_gro_receive(struct list_head *held_list, + *homa_tcp_gro_receive(struct list_head *held_list, struct sk_buff *skb); extern void homa_timer(struct homa *homa); extern int homa_timer_main(void *transportInfo); extern void homa_unhash(struct sock *sk); extern void homa_unknown_pkt(struct sk_buff *skb, struct homa_rpc *rpc); extern int homa_unsched_priority(struct homa *homa, - struct homa_peer *peer, int length); + struct homa_peer *peer, int length); extern int homa_v4_early_demux(struct sk_buff *skb); extern int homa_v4_early_demux_handler(struct sk_buff *skb); extern int homa_validate_incoming(struct homa *homa, int verbose, int *link_errors); extern struct homa_rpc - *homa_wait_for_message(struct homa_sock *hsk, int flags, - __u64 id); + *homa_wait_for_message(struct homa_sock *hsk, int flags, + __u64 id); extern int homa_xmit_control(enum homa_packet_type type, void *contents, - size_t length, struct homa_rpc *rpc); + size_t length, struct homa_rpc *rpc); extern int __homa_xmit_control(void *contents, size_t length, - struct homa_peer *peer, struct homa_sock *hsk); + struct homa_peer *peer, struct homa_sock *hsk); extern void homa_xmit_data(struct homa_rpc *rpc, bool force); extern void __homa_xmit_data(struct sk_buff *skb, struct homa_rpc *rpc, - int priority); + int priority); extern void homa_xmit_unknown(struct sk_buff *skb, struct homa_sock *hsk); /** diff --git a/homa_incoming.c b/homa_incoming.c index 81c9dde..cbf7f7b 100644 --- a/homa_incoming.c +++ b/homa_incoming.c @@ -1,9 +1,8 @@ -/* Copyright (c) 2019-2023 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause /* This file contains functions that handle incoming Homa messages, including - * both receiving information for those messages and sending grants. */ + * both receiving information for those messages and sending grants. + */ #include "homa_impl.h" @@ -58,10 +57,11 @@ int homa_message_in_init(struct homa_rpc *rpc, int length, int unsched) * @end: Offset of byte just after the last one covered by the gap. * Return: Pointer to the new gap. */ -struct homa_gap * homa_gap_new(struct list_head *next, int start, int end) +struct homa_gap *homa_gap_new(struct list_head *next, int start, int end) { struct homa_gap *gap; - gap = (struct homa_gap *) kmalloc(sizeof(struct homa_gap), GFP_KERNEL); + + gap = kmalloc(sizeof(struct homa_gap), GFP_KERNEL); gap->start = start; gap->end = end; gap->time = get_cycles(); @@ -79,13 +79,11 @@ void homa_gap_retry(struct homa_rpc *rpc) struct homa_gap *gap; struct resend_header resend; - list_for_each_entry(gap, &rpc->msgin.gaps, links) - { + list_for_each_entry(gap, &rpc->msgin.gaps, links) { resend.offset = htonl(gap->start); resend.length = htonl(gap->end - gap->start); resend.priority = rpc->hsk->homa->num_priorities - 1; - tt_record3("homa_gap_retry sending RESEND for id %d, start %d, " - "end %d", + tt_record3("homa_gap_retry sending RESEND for id %d, start %d, end %d", rpc->id, gap->start, gap->end); homa_xmit_control(RESEND, &resend, sizeof(resend), rpc); } @@ -107,8 +105,7 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb) struct homa_gap *gap, *dummy, *gap2; if ((start + length) > rpc->msgin.length) { - tt_record3("Packet extended past message end; id %d, " - "offset %d, length %d", + tt_record3("Packet extended past message end; id %d, offset %d, length %d", rpc->id, start, length); goto discard; } @@ -130,19 +127,17 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb) * an existing gap. */ list_for_each_entry_safe(gap, dummy, &rpc->msgin.gaps, links) { - /* Is packet at the start of this gap? */ + /* Is packet at the start of this gap? */ if (start <= gap->start) { if (end <= gap->start) continue; if (start < gap->start) { - tt_record4("Packet overlaps gap start: id %d, " - "start %d, end %d, gap_start %d", + tt_record4("Packet overlaps gap start: id %d, start %d, end %d, gap_start %d", rpc->id, start, end, gap->start); goto discard; } if (end > gap->end) { - tt_record4("Packet overlaps gap end: id %d, " - "start %d, end %d, gap_end %d", + tt_record4("Packet overlaps gap end: id %d, start %d, end %d, gap_end %d", rpc->id, start, end, gap->start); goto discard; } @@ -154,15 +149,14 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb) goto keep; } - /* Is packet at the end of this gap? BTW, at this point we know + /* Is packet at the end of this gap? BTW, at this point we know * the packet can't cover the entire gap. */ if (end >= gap->end) { if (start >= gap->end) continue; if (end > gap->end) { - tt_record4("Packet overlaps gap end: id %d, " - "start %d, end %d, gap_end %d", + tt_record4("Packet overlaps gap end: id %d, start %d, end %d, gap_end %d", rpc->id, start, end, gap->start); goto discard; } @@ -177,18 +171,17 @@ void homa_add_packet(struct homa_rpc *rpc, struct sk_buff *skb) goto keep; } - discard: +discard: if (h->retransmit) INC_METRIC(resent_discards, 1); else INC_METRIC(packet_discards, 1); - tt_record4("homa_add_packet discarding packet for id %d, " - "offset %d, length %d, retransmit %d", + tt_record4("homa_add_packet discarding packet for id %d, offset %d, length %d, retransmit %d", rpc->id, start, length, h->retransmit); kfree_skb(skb); return; - keep: +keep: if (h->retransmit) INC_METRIC(resent_packets_used, 1); __skb_queue_tail(&rpc->msgin.packets, skb); @@ -229,6 +222,7 @@ int homa_copy_to_user(struct homa_rpc *rpc) */ while (true) { struct sk_buff *skb = __skb_dequeue(&rpc->msgin.packets); + if (skb != NULL) { skbs[n] = skb; n++; @@ -297,7 +291,7 @@ int homa_copy_to_user(struct homa_rpc *rpc) end_offset = offset + pkt_length; } - free_skbs: +free_skbs: if (end_offset != 0) { tt_record3("copied out bytes %d-%d for id %d", start_offset, end_offset, rpc->id); @@ -365,10 +359,9 @@ void homa_dispatch_pkts(struct sk_buff *skb, struct homa *homa) else icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0); - tt_record3("Discarding packet(s) for unknown port %u, " - "id %llu, type %d", dport, - homa_local_id(h->common.sender_id), - h->common.type); + tt_record3("Discarding packet(s) for unknown port %u, id %llu, type %d", + dport, homa_local_id(h->common.sender_id), + h->common.type); while (skb != NULL) { next = skb->next; kfree_skb(skb); @@ -377,9 +370,7 @@ void homa_dispatch_pkts(struct sk_buff *skb, struct homa *homa) return; } - /* Each iteration through through the following loop processes one - * packet. - */ + /* Each iteration through the following loop processes one packet. */ for (; skb != NULL; skb = next) { h = (struct data_header *) skb->data; next = skb->next; @@ -389,10 +380,10 @@ void homa_dispatch_pkts(struct sk_buff *skb, struct homa *homa) */ if (rpc != NULL) { int flags = atomic_read(&rpc->flags); + if (flags & APP_NEEDS_LOCK) { homa_rpc_unlock(rpc); - tt_record2("softirq released lock for id %d, " - "flags 0x%x", rpc->id, flags); + tt_record2("softirq released lock for id %d, flags 0x%x", rpc->id, flags); homa_spin(200); rpc = NULL; } @@ -406,12 +397,12 @@ void homa_dispatch_pkts(struct sk_buff *skb, struct homa *homa) int created; /* Create a new RPC if one doesn't - * already exist. */ + * already exist. + */ rpc = homa_rpc_new_server(hsk, &saddr, h, &created); if (IS_ERR(rpc)) { - printk(KERN_WARNING "homa_pkt_dispatch couldn't " - "create server rpc: error %lu", + pr_warn("homa_pkt_dispatch couldn't create server rpc: error %lu", -PTR_ERR(rpc)); INC_METRIC(server_cant_create_rpcs, 1); rpc = NULL; @@ -430,8 +421,7 @@ void homa_dispatch_pkts(struct sk_buff *skb, struct homa *homa) && (h->common.type != NEED_ACK) && (h->common.type != ACK) && (h->common.type != RESEND)) { - tt_record4("Discarding packet for unknown RPC, " - "id %u, type %d, peer 0x%x:%d", + tt_record4("Discarding packet for unknown RPC, id %u, type %d, peer 0x%x:%d", id, h->common.type, tt_addr(saddr), ntohs(h->common.sport)); @@ -447,7 +437,7 @@ void homa_dispatch_pkts(struct sk_buff *skb, struct homa *homa) } switch (h->common.type) { - case DATA: + case DATA: if (h->ack.client_id != 0) { /* Save the ack for processing later, when we * have released the RPC lock. @@ -504,7 +494,7 @@ void homa_dispatch_pkts(struct sk_buff *skb, struct homa *homa) } continue; - discard: +discard: kfree_skb(skb); } if (rpc != NULL) @@ -570,8 +560,7 @@ void homa_data_pkt(struct sk_buff *skb, struct homa_rpc *rpc) * exceed available cache space, resulting in poor * performance. */ - tt_record4("Dropping packet because no buffer space available: " - "id %d, offset %d, length %d, old incoming %d", + tt_record4("Dropping packet because no buffer space available: id %d, offset %d, length %d, old incoming %d", rpc->id, ntohl(h->seg.offset), homa_data_len(skb), rpc->msgin.granted); @@ -613,7 +602,7 @@ void homa_data_pkt(struct sk_buff *skb, struct homa_rpc *rpc) } return; - discard: +discard: kfree_skb(skb); UNIT_LOG("; ", "homa_data_pkt discarded packet"); } @@ -629,8 +618,7 @@ void homa_grant_pkt(struct sk_buff *skb, struct homa_rpc *rpc) struct grant_header *h = (struct grant_header *) skb->data; int new_offset = ntohl(h->offset); - tt_record4("processing grant for id %llu, offset %d, priority %d, " - "increment %d", + tt_record4("processing grant for id %llu, offset %d, priority %d, increment %d", homa_local_id(h->common.sender_id), ntohl(h->offset), h->priority, new_offset - rpc->msgout.granted); if (rpc->state == RPC_OUTGOING) { @@ -666,8 +654,7 @@ void homa_resend_pkt(struct sk_buff *skb, struct homa_rpc *rpc, struct busy_header busy; if (rpc == NULL) { - tt_record4("resend request for unknown id %d, peer 0x%x:%d, " - "offset %d; responding with UNKNOWN", + tt_record4("resend request for unknown id %d, peer 0x%x:%d, offset %d; responding with UNKNOWN", homa_local_id(h->common.sender_id), tt_addr(saddr), ntohs(h->common.sport), ntohl(h->offset)); @@ -691,9 +678,8 @@ void homa_resend_pkt(struct sk_buff *skb, struct homa_rpc *rpc, /* We have chosen not to transmit data from this message; * send BUSY instead. */ - tt_record3("sending BUSY from resend, id %d, offset %d, " - "granted %d", rpc->id, - rpc->msgout.next_xmit_offset, + tt_record3("sending BUSY from resend, id %d, offset %d, granted %d", + rpc->id, rpc->msgout.next_xmit_offset, rpc->msgout.granted); homa_xmit_control(BUSY, &busy, sizeof(busy), rpc); } else { @@ -709,7 +695,7 @@ void homa_resend_pkt(struct sk_buff *skb, struct homa_rpc *rpc, h->priority); } - done: +done: kfree_skb(skb); } @@ -728,31 +714,26 @@ void homa_unknown_pkt(struct sk_buff *skb, struct homa_rpc *rpc) /* It appears that everything we've already transmitted * has been lost; retransmit it. */ - tt_record4("Restarting id %d to server 0x%x:%d, " - "lost %d bytes", + tt_record4("Restarting id %d to server 0x%x:%d, lost %d bytes", rpc->id, tt_addr(rpc->peer->addr), rpc->dport, rpc->msgout.next_xmit_offset); - homa_freeze(rpc, RESTART_RPC, "Freezing because of " - "RPC restart, id %d, peer 0x%x"); + homa_freeze(rpc, RESTART_RPC, "Freezing because of RPC restart, id %d, peer 0x%x"); homa_resend_data(rpc, 0, rpc->msgout.next_xmit_offset, homa_unsched_priority(rpc->hsk->homa, rpc->peer, rpc->msgout.length)); goto done; } - printk(KERN_ERR "Received unknown for RPC id %llu, peer %s:%d " - "in bogus state %d; discarding unknown\n", + pr_err("Received unknown for RPC id %llu, peer %s:%d in bogus state %d; discarding unknown\n", rpc->id, homa_print_ipv6_addr(&rpc->peer->addr), rpc->dport, rpc->state); - tt_record4("Discarding unknown for RPC id %d, peer 0x%x:%d: " - "bad state %d", + tt_record4("Discarding unknown for RPC id %d, peer 0x%x:%d: bad state %d", rpc->id, tt_addr(rpc->peer->addr), rpc->dport, rpc->state); } else { if (rpc->hsk->homa->verbose) - printk(KERN_NOTICE "Freeing rpc id %llu from client " - "%s:%d: unknown to client", + pr_notice("Freeing rpc id %llu from client %s:%d: unknown to client", rpc->id, homa_print_ipv6_addr(&rpc->peer->addr), rpc->dport); @@ -779,7 +760,7 @@ void homa_cutoffs_pkt(struct sk_buff *skb, struct homa_sock *hsk) if (!IS_ERR(peer)) { peer->unsched_cutoffs[0] = INT_MAX; - for (i = 1; i unsched_cutoffs[i] = ntohl(h->unsched_cutoffs[i]); peer->cutoff_version = h->cutoff_version; } @@ -811,12 +792,10 @@ void homa_need_ack_pkt(struct sk_buff *skb, struct homa_sock *hsk, */ if ((rpc != NULL) && ((rpc->state != RPC_INCOMING) || rpc->msgin.bytes_remaining)) { - tt_record3("NEED_ACK arrived for id %d before message " - "received, state %d, remaining %d", + tt_record3("NEED_ACK arrived for id %d before message received, state %d, remaining %d", rpc->id, rpc->state, rpc->msgin.bytes_remaining); homa_freeze(rpc, NEED_ACK_MISSING_DATA, - "Freezing because NEED_ACK received before " - "message complete, id %d, peer 0x%x"); + "Freezing because NEED_ACK received before message complete, id %d, peer 0x%x"); goto done; } else { peer = homa_peer_find(&hsk->homa->peers, &saddr, &hsk->inet); @@ -837,10 +816,10 @@ void homa_need_ack_pkt(struct sk_buff *skb, struct homa_sock *hsk, ack.num_acks = htons(homa_peer_get_acks(peer, NUM_PEER_UNACKED_IDS, ack.acks)); __homa_xmit_control(&ack, sizeof(ack), peer, hsk); - tt_record3("Responded to NEED_ACK for id %d, peer %0x%x with %d " - "other acks", id, tt_addr(saddr), ntohs(ack.num_acks)); + tt_record3("Responded to NEED_ACK for id %d, peer %0x%x with %d other acks", + id, tt_addr(saddr), ntohs(ack.num_acks)); - done: +done: kfree_skb(skb); } @@ -1051,18 +1030,17 @@ void homa_abort_sock_rpcs(struct homa_sock *hsk, int error) homa_rpc_unlock(rpc); continue; } - tt_record4("homa_abort_sock_rpcs aborting id %u on port %d, " - "peer 0x%x, error %d", + tt_record4("homa_abort_sock_rpcs aborting id %u on port %d, peer 0x%x, error %d", rpc->id, hsk->port, tt_addr(rpc->peer->addr), error); - if (error) { + if (error) homa_rpc_abort(rpc, error); - } else + else homa_rpc_free(rpc); homa_rpc_unlock(rpc); } homa_unprotect_rpcs(hsk); - done: +done: rcu_read_unlock(); } @@ -1154,7 +1132,7 @@ int homa_register_interests(struct homa_interest *interest, homa_sock_unlock(hsk); return 0; - claim_rpc: +claim_rpc: list_del_init(&rpc->ready_links); if (!list_empty(&hsk->ready_requests) || !list_empty(&hsk->ready_responses)) { @@ -1164,7 +1142,8 @@ int homa_register_interests(struct homa_interest *interest, /* This flag is needed to keep the RPC from being reaped during the * gap between when we release the socket lock and we acquire the - * RPC lock.*/ + * RPC lock. + */ atomic_or(RPC_HANDING_OFF, &rpc->flags); homa_sock_unlock(hsk); if (!interest->locked) { @@ -1209,9 +1188,8 @@ struct homa_rpc *homa_wait_for_message(struct homa_sock *hsk, int flags, while (1) { error = homa_register_interests(&interest, hsk, flags, id); rpc = (struct homa_rpc *) atomic_long_read(&interest.ready_rpc); - if (rpc) { + if (rpc) goto found_rpc; - } if (error < 0) { result = ERR_PTR(error); goto found_rpc; @@ -1220,11 +1198,12 @@ struct homa_rpc *homa_wait_for_message(struct homa_sock *hsk, int flags, // tt_record3("Preparing to poll, socket %d, flags 0x%x, pid %d", // hsk->client_port, flags, current->pid); - /* There is no ready RPC so far. Clean up dead RPCs before + /* There is no ready RPC so far. Clean up dead RPCs before * going to sleep (or returning, if in nonblocking mode). */ while (1) { int reaper_result; + rpc = (struct homa_rpc *) atomic_long_read( &interest.ready_rpc); if (rpc) { @@ -1251,6 +1230,7 @@ struct homa_rpc *homa_wait_for_message(struct homa_sock *hsk, int flags, poll_start = now = get_cycles(); while (1) { __u64 blocked; + rpc = (struct homa_rpc *) atomic_long_read( &interest.ready_rpc); if (rpc) { @@ -1286,6 +1266,7 @@ struct homa_rpc *homa_wait_for_message(struct homa_sock *hsk, int flags, if (!rpc && !hsk->shutdown) { __u64 end; __u64 start = get_cycles(); + tt_record1("homa_wait_for_message sleeping, pid %d", current->pid); schedule(); @@ -1358,7 +1339,7 @@ struct homa_rpc *homa_wait_for_message(struct homa_sock *hsk, int flags, if (signal_pending(current)) return ERR_PTR(-EINTR); - /* No message and no error; try again. */ + /* No message and no error; try again. */ } done: diff --git a/homa_offload.c b/homa_offload.c index e94102e..55cb8a8 100644 --- a/homa_offload.c +++ b/homa_offload.c @@ -1,6 +1,4 @@ -/* Copyright (c) 2019-2023 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause /* This file implements GSO (Generic Segmentation Offload) and GRO (Generic * Receive Offload) for Homa. @@ -23,8 +21,8 @@ extern struct homa *homa; /* Pointers to TCP's net_offload structures. NULL means homa_gro_hook_tcp * hasn't been called yet. */ -const struct net_offload *tcp_net_offload = NULL; -const struct net_offload *tcp6_net_offload = NULL; +const struct net_offload *tcp_net_offload; +const struct net_offload *tcp6_net_offload; /* * Identical to *tcp_net_offload except that the gro_receive function @@ -42,6 +40,7 @@ int homa_offload_init(void) { int res1 = inet_add_offload(&homa_offload, IPPROTO_HOMA); int res2 = inet6_add_offload(&homa_offload, IPPROTO_HOMA); + return res1 ? res1 : res2; } @@ -55,6 +54,7 @@ int homa_offload_end(void) { int res1 = inet_del_offload(&homa_offload, IPPROTO_HOMA); int res2 = inet6_del_offload(&homa_offload, IPPROTO_HOMA); + return res1 ? res1 : res2; } @@ -68,7 +68,7 @@ void homa_gro_hook_tcp(void) if (tcp_net_offload != NULL) return; - printk(KERN_NOTICE "Homa setting up TCP hijacking\n"); + pr_notice("Homa setting up TCP hijacking\n"); tcp_net_offload = inet_offloads[IPPROTO_TCP]; hook_tcp_net_offload = *tcp_net_offload; hook_tcp_net_offload.callbacks.gro_receive = homa_tcp_gro_receive; @@ -89,7 +89,7 @@ void homa_gro_unhook_tcp(void) { if (tcp_net_offload == NULL) return; - printk(KERN_NOTICE "Homa cancelling TCP hijacking\n"); + pr_notice("Homa cancelling TCP hijacking\n"); inet_offloads[IPPROTO_TCP] = tcp_net_offload; tcp_net_offload = NULL; inet6_offloads[IPPROTO_TCP] = tcp6_net_offload; @@ -110,8 +110,8 @@ struct sk_buff *homa_tcp_gro_receive(struct list_head *held_list, struct common_header *h = (struct common_header *) skb_transport_header(skb); // tt_record4("homa_tcp_gro_receive got type 0x%x, flags 0x%x, " - // "urgent 0x%x, id %d", h->type, h->flags, - // ntohs(h->urgent), homa_local_id(h->sender_id)); + // "urgent 0x%x, id %d", h->type, h->flags, + // ntohs(h->urgent), homa_local_id(h->sender_id)); if ((h->flags != HOMA_TCP_FLAGS) || (ntohs(h->urgent) != HOMA_TCP_URGENT)) return tcp_net_offload->callbacks.gro_receive(held_list, skb); @@ -199,6 +199,7 @@ struct sk_buff *homa_gso_segment(struct sk_buff *skb, netdev_features_t features) { struct sk_buff *segs; + tt_record2("homa_gso_segment invoked, frags %d, headlen %d", skb_shinfo(skb)->nr_frags, skb_headlen(skb)); @@ -215,6 +216,7 @@ struct sk_buff *homa_gso_segment(struct sk_buff *skb, if (ip_hdr(segs)->version == 4) { struct sk_buff *seg; int i = 0; + for (seg = segs; seg != NULL; seg = seg->next) { ip_hdr(seg)->id = htons(i); i++; @@ -285,8 +287,7 @@ struct sk_buff *homa_gro_receive(struct list_head *held_list, ntohl(h_new->common.sequence)); h_new->seg.offset = h_new->common.sequence; } - tt_record4("homa_gro_receive got packet from 0x%x " - "id %llu, offset %d, priority %d", + tt_record4("homa_gro_receive got packet from 0x%x id %llu, offset %d, priority %d", saddr, homa_local_id(h_new->common.sender_id), ntohl(h_new->seg.offset), priority); if ((homa_data_len(skb) == ntohl(h_new->message_length)) @@ -296,8 +297,7 @@ struct sk_buff *homa_gro_receive(struct list_head *held_list, goto bypass; } } else if (h_new->common.type == GRANT) { - tt_record4("homa_gro_receive got grant from 0x%x " - "id %llu, offset %d, priority %d", + tt_record4("homa_gro_receive got grant from 0x%x id %llu, offset %d, priority %d", saddr, homa_local_id(h_new->common.sender_id), ntohl(((struct grant_header *) h_new)->offset), priority); @@ -312,8 +312,7 @@ struct sk_buff *homa_gro_receive(struct list_head *held_list, goto bypass; } } else - tt_record4("homa_gro_receive got packet from 0x%x " - "id %llu, type 0x%x, priority %d", + tt_record4("homa_gro_receive got packet from 0x%x id %llu, type 0x%x, priority %d", saddr, homa_local_id(h_new->common.sender_id), h_new->common.type, priority); @@ -339,7 +338,7 @@ struct sk_buff *homa_gro_receive(struct list_head *held_list, * packets from the list, so core->held_skb could be a * dangling pointer (or the skb could have been reused for * some other protocol). - */ + */ list_for_each_entry(held_skb, &napi->gro_hash[core->held_bucket].list, list) { int protocol; @@ -351,8 +350,7 @@ struct sk_buff *homa_gro_receive(struct list_head *held_list, else protocol = ip_hdr(held_skb)->protocol; if (protocol != IPPROTO_HOMA) { - tt_record3("homa_gro_receive held_skb 0x%0x%0x " - "isn't Homa: protocol %d", + tt_record3("homa_gro_receive held_skb 0x%0x%0x isn't Homa: protocol %d", SPLIT_64(held_skb), protocol); continue; } @@ -406,13 +404,13 @@ struct sk_buff *homa_gro_receive(struct list_head *held_list, if (likely(homa->gro_policy & HOMA_GRO_SAME_CORE)) homa_set_softirq_cpu(skb, raw_smp_processor_id()); - done: +done: homa_check_pacer(homa, 1); core->last_gro = get_cycles(); return result; - bypass: - /* Record SoftIRQ cycles in a different metric to reflect that +bypass: + /* Record SoftIRQ cycles in a different metric to reflect that * they happened during bypass. */ saved_softirq_metric = core->metrics.softirq_cycles; @@ -449,6 +447,7 @@ void homa_gro_gen2(struct sk_buff *skb) int candidate = this_core; __u64 now = get_cycles(); struct homa_core *core; + for (i = CORES_TO_CHECK; i > 0; i--) { candidate++; if (unlikely(candidate >= nr_cpu_ids)) @@ -458,8 +457,7 @@ void homa_gro_gen2(struct sk_buff *skb) continue; if ((core->last_gro + homa->busy_cycles) > now) continue; - tt_record3("homa_gro_gen2 chose core %d for id %d " - "offset %d", + tt_record3("homa_gro_gen2 chose core %d for id %d offset %d", candidate, homa_local_id(h->common.sender_id), ntohl(h->seg.offset)); break; @@ -469,16 +467,15 @@ void homa_gro_gen2(struct sk_buff *skb) * rotate among them. */ int offset = homa_cores[this_core]->softirq_offset; + offset += 1; if (offset > CORES_TO_CHECK) offset = 1; homa_cores[this_core]->softirq_offset = offset; candidate = this_core + offset; - while (candidate >= nr_cpu_ids) { + while (candidate >= nr_cpu_ids) candidate -= nr_cpu_ids; - } - tt_record3("homa_gro_gen2 chose core %d for id %d " - "offset %d (all cores busy)", + tt_record3("homa_gro_gen2 chose core %d for id %d offset %d (all cores busy)", candidate, homa_local_id(h->common.sender_id), ntohl(h->seg.offset)); } @@ -510,9 +507,9 @@ void homa_gro_gen3(struct sk_buff *skb) core = candidates[0]; for (i = 0; i < NUM_GEN3_SOFTIRQ_CORES; i++) { int candidate = candidates[i]; - if (candidate < 0) { + + if (candidate < 0) break; - } if (homa_cores[candidate]->last_app_active < busy_time) { core = candidate; break; @@ -543,9 +540,9 @@ int homa_gro_complete(struct sk_buff *skb, int hoffset) { struct data_header *h = (struct data_header *) skb_transport_header(skb); // tt_record4("homa_gro_complete type %d, id %d, offset %d, count %d", - // h->common.type, homa_local_id(h->common.sender_id), - // ntohl(h->seg.offset), - // NAPI_GRO_CB(skb)->count); + // h->common.type, homa_local_id(h->common.sender_id), + // ntohl(h->seg.offset), + // NAPI_GRO_CB(skb)->count); homa_cores[raw_smp_processor_id()]->held_skb = NULL; if (homa->gro_policy & HOMA_GRO_GEN3) { @@ -576,8 +573,7 @@ int homa_gro_complete(struct sk_buff *skb, int hoffset) } } homa_set_softirq_cpu(skb, best); - tt_record3("homa_gro_complete chose core %d for id %d " - "offset %d with IDLE policy", + tt_record3("homa_gro_complete chose core %d for id %d offset %d with IDLE policy", best, homa_local_id(h->common.sender_id), ntohl(h->seg.offset)); } else if (homa->gro_policy & HOMA_GRO_NEXT) { @@ -585,11 +581,11 @@ int homa_gro_complete(struct sk_buff *skb, int hoffset) * SoftIRQ processing. */ int target = raw_smp_processor_id() + 1; + if (unlikely(target >= nr_cpu_ids)) target = 0; homa_set_softirq_cpu(skb, target); - tt_record3("homa_gro_complete chose core %d for id %d " - "offset %d with NEXT policy", + tt_record3("homa_gro_complete chose core %d for id %d offset %d with NEXT policy", target, homa_local_id(h->common.sender_id), ntohl(h->seg.offset)); } diff --git a/homa_outgoing.c b/homa_outgoing.c index b8d9eaf..a428554 100644 --- a/homa_outgoing.c +++ b/homa_outgoing.c @@ -1,6 +1,4 @@ -/* Copyright (c) 2019-2023 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause /* This file contains functions related to the sender side of message * transmission. It also contains utility functions for sending packets. @@ -16,7 +14,7 @@ * @priority: Priority level for the packet; must be less than * HOMA_MAX_PRIORITIES. */ -inline static void set_priority(struct sk_buff *skb, struct homa_sock *hsk, +static inline void set_priority(struct sk_buff *skb, struct homa_sock *hsk, int priority) { /* Note: this code initially specified the priority in the VLAN @@ -76,6 +74,7 @@ int homa_fill_data_interleaved(struct homa_rpc *rpc, struct sk_buff *skb, */ while (1) { struct seg_header seg; + if (bytes_left < seg_length) seg_length = bytes_left; err = homa_skb_append_from_iter(rpc->hsk->homa, skb, iter, @@ -185,7 +184,7 @@ struct sk_buff *homa_new_data_packet(struct homa_rpc *rpc, } return skb; - error: +error: homa_skb_free_tx(rpc->hsk->homa, skb); return ERR_PTR(err); } @@ -204,7 +203,7 @@ struct sk_buff *homa_new_data_packet(struct homa_rpc *rpc, * Zero means the caller will initiate transmission after this * function returns. * - * Return: 0 for success, or a negative errno for failure. It is is possible + * Return: 0 for success, or a negative errno for failure. It is possible * for the RPC to be freed while this function is active. If that * happens, copying will cease, -EINVAL will be returned, and * rpc->state will be RPC_DEAD. @@ -264,8 +263,7 @@ int homa_message_out_fill(struct homa_rpc *rpc, struct iov_iter *iter, int xmit) homa_skb_stash_pages(rpc->hsk->homa, rpc->msgout.length); /* Each iteration of the loop below creates one GSO packet. */ - tt_record3("starting copy from user space for id %d, length %d, " - "unscheduled %d", + tt_record3("starting copy from user space for id %d, length %d, unscheduled %d", rpc->id, rpc->msgout.length, rpc->msgout.unscheduled); last_link = &rpc->msgout.packets; for (bytes_left = rpc->msgout.length; bytes_left > 0; ) { @@ -320,7 +318,7 @@ int homa_message_out_fill(struct homa_rpc *rpc, struct iov_iter *iter, int xmit) homa_xmit_data(rpc, false); return 0; - error: +error: atomic_andnot(RPC_COPYING_FROM_USER, &rpc->flags); return err; } @@ -343,6 +341,7 @@ int homa_xmit_control(enum homa_packet_type type, void *contents, size_t length, struct homa_rpc *rpc) { struct common_header *h = (struct common_header *) contents; + h->type = type; h->sport = htons(rpc->hsk->port); h->dport = htons(rpc->dport); @@ -414,26 +413,19 @@ int __homa_xmit_control(void *contents, size_t length, struct homa_peer *peer, */ if (refcount_read(&skb->users) > 1) { if (hsk->inet.sk.sk_family == AF_INET6) { - printk(KERN_NOTICE "ip6_xmit didn't free " - "Homa control packet (type %d) " - "after error %d\n", + pr_notice("ip6_xmit didn't free Homa control packet (type %d) after error %d\n", h->type, result); } else { - printk(KERN_NOTICE "ip_queue_xmit didn't free " - "Homa control packet (type %d) " - "after error %d\n", + pr_notice("ip_queue_xmit didn't free Homa control packet (type %d) after error %d\n", h->type, result); - tt_record2("ip_queue_xmit didn't free Homa " - "control packet (type %d) " - "after error %d\n", + tt_record2("ip_queue_xmit didn't free Homa control packet (type %d) after error %d\n", h->type, result); } } } txq = netdev_get_tx_queue(skb->dev, skb->queue_mapping); if (netif_tx_queue_stopped(txq)) - tt_record4("__homa_xmit_control found stopped txq for id %d, " - "qid %d, num_queued %d, limit %d", + tt_record4("__homa_xmit_control found stopped txq for id %d, qid %d, num_queued %d, limit %d", be64_to_cpu(h->sender_id), skb->queue_mapping, txq->dql.num_queued, txq->dql.adj_limit); @@ -458,8 +450,7 @@ void homa_xmit_unknown(struct sk_buff *skb, struct homa_sock *hsk) struct in6_addr saddr = skb_canonical_ipv6_saddr(skb); if (hsk->homa->verbose) - printk(KERN_NOTICE "sending UNKNOWN to peer " - "%s:%d for id %llu", + pr_notice("sending UNKNOWN to peer %s:%d for id %llu", homa_print_ipv6_addr(&saddr), ntohs(h->sport), homa_local_id(h->sender_id)); tt_record3("sending unknown to 0x%x:%d for id %llu", @@ -473,7 +464,7 @@ void homa_xmit_unknown(struct sk_buff *skb, struct homa_sock *hsk) unknown.common.sender_id = cpu_to_be64(homa_local_id(h->sender_id)); peer = homa_peer_find(&hsk->homa->peers, &saddr, &hsk->inet); if (!IS_ERR(peer)) - __homa_xmit_control(&unknown, sizeof(unknown), peer, hsk); + __homa_xmit_control(&unknown, sizeof(unknown), peer, hsk); } /** @@ -502,8 +493,7 @@ void homa_xmit_data(struct homa_rpc *rpc, bool force) struct sk_buff *skb = *rpc->msgout.next_xmit; if (rpc->msgout.next_xmit_offset >= rpc->msgout.granted) { - tt_record3("homa_xmit_data stopping at offset %d " - "for id %u: granted is %d", + tt_record3("homa_xmit_data stopping at offset %d for id %u: granted is %d", rpc->msgout.next_xmit_offset, rpc->id, rpc->msgout.granted); break; @@ -512,8 +502,7 @@ void homa_xmit_data(struct homa_rpc *rpc, bool force) if ((rpc->msgout.length - rpc->msgout.next_xmit_offset) >= homa->throttle_min_bytes) { if (!homa_check_nic_queue(homa, skb, force)) { - tt_record1("homa_xmit_data adding id %u to " - "throttle queue", rpc->id); + tt_record1("homa_xmit_data adding id %u to throttle queue", rpc->id); homa_add_to_throttled(rpc); break; } @@ -534,8 +523,7 @@ void homa_xmit_data(struct homa_rpc *rpc, bool force) __homa_xmit_data(skb, rpc, priority); txq = netdev_get_tx_queue(skb->dev, skb->queue_mapping); if (netif_tx_queue_stopped(txq)) - tt_record4("homa_xmit_data found stopped txq for id %d, " - "qid %d, num_queued %d, limit %d", + tt_record4("homa_xmit_data found stopped txq for id %d, qid %d, num_queued %d, limit %d", rpc->id, skb->queue_mapping, txq->dql.num_queued, txq->dql.adj_limit); force = false; @@ -575,8 +563,7 @@ void __homa_xmit_data(struct sk_buff *skb, struct homa_rpc *rpc, int priority) skb->csum_start = skb_transport_header(skb) - skb->head; skb->csum_offset = offsetof(struct common_header, checksum); if (rpc->hsk->inet.sk.sk_family == AF_INET6) { - tt_record4("calling ip6_xmit: wire_bytes %d, peer 0x%x, id %d, " - "offset %d", + tt_record4("calling ip6_xmit: wire_bytes %d, peer 0x%x, id %d, offset %d", homa_get_skb_info(skb)->wire_bytes, tt_addr(rpc->peer->addr), rpc->id, homa_info->offset); @@ -584,8 +571,7 @@ void __homa_xmit_data(struct sk_buff *skb, struct homa_rpc *rpc, int priority) 0, NULL, rpc->hsk->homa->priority_map[priority] << 4, 0); } else { - tt_record4("calling ip_queue_xmit: wire_bytes %d, peer 0x%x, " - "id %d, offset %d", + tt_record4("calling ip_queue_xmit: wire_bytes %d, peer 0x%x, id %d, offset %d", homa_get_skb_info(skb)->wire_bytes, tt_addr(rpc->peer->addr), rpc->id, homa_info->offset); @@ -593,14 +579,12 @@ void __homa_xmit_data(struct sk_buff *skb, struct homa_rpc *rpc, int priority) rpc->hsk->inet.tos = rpc->hsk->homa->priority_map[priority]<<5; err = ip_queue_xmit(&rpc->hsk->inet.sk, skb, &rpc->peer->flow); } - tt_record4("Finished queueing packet: rpc id %llu, offset %d, len %d, " - "qid %d", + tt_record4("Finished queueing packet: rpc id %llu, offset %d, len %d, qid %d", rpc->id, homa_info->offset, homa_get_skb_info(skb)->data_bytes, skb->queue_mapping); - if (err) { + if (err) INC_METRIC(data_xmit_errors, 1); - } INC_METRIC(packets_sent[0], 1); INC_METRIC(priority_bytes[priority], skb->len); INC_METRIC(priority_packets[priority], 1); @@ -669,8 +653,8 @@ void homa_resend_data(struct homa_rpc *rpc, int start, int end, - sizeof(struct seg_header)); if (unlikely(!new_skb)) { if (rpc->hsk->homa->verbose) - printk(KERN_NOTICE "homa_resend_data " - "couldn't allocate skb\n"); + pr_notice("%s couldn't allocate skb\n", + __func__); UNIT_LOG("; ", "skb allocation error"); goto resend_done; } @@ -689,11 +673,10 @@ void homa_resend_data(struct homa_rpc *rpc, int start, int end, err = homa_skb_append_from_skb(rpc->hsk->homa, new_skb, skb, seg_offset, seg_length); if (err != 0) { - printk(KERN_ERR "homa_resend_data got error %d " - "from homa_skb_append_from_skb\n", - err); - UNIT_LOG("; ", "homa_resend_data got error %d " - "while copying data", -err); + pr_err("%s got error %d from homa_skb_append_from_skb\n", + __func__, err); + UNIT_LOG("; ", "%s got error %d while copying data", + __func__, -err); kfree_skb(new_skb); goto resend_done; } @@ -842,7 +825,7 @@ int homa_pacer_main(void *transportInfo) void homa_pacer_xmit(struct homa *homa) { struct homa_rpc *rpc; - int i; + int i; /* Make sure only one instance of this function executes at a * time. @@ -914,8 +897,7 @@ void homa_pacer_xmit(struct homa *homa) } homa_throttle_unlock(homa); - tt_record4("pacer calling homa_xmit_data for rpc id %llu, " - "port %d, offset %d, bytes_left %d", + tt_record4("pacer calling homa_xmit_data for rpc id %llu, port %d, offset %d, bytes_left %d", rpc->id, rpc->hsk->port, rpc->msgout.next_xmit_offset, rpc->msgout.length - rpc->msgout.next_xmit_offset); @@ -931,8 +913,7 @@ void homa_pacer_xmit(struct homa *homa) */ homa_throttle_lock(homa); if (!list_empty(&rpc->throttled_links)) { - tt_record2("pacer removing id %d from " - "throttled list, offset %d", + tt_record2("pacer removing id %d from throttled list, offset %d", rpc->id, rpc->msgout.next_xmit_offset); list_del_rcu(&rpc->throttled_links); @@ -954,7 +935,7 @@ void homa_pacer_xmit(struct homa *homa) } homa_rpc_unlock(rpc); } - done: +done: spin_unlock_bh(&homa->pacer_mutex); } @@ -985,9 +966,8 @@ void homa_add_to_throttled(struct homa_rpc *rpc) int checks = 0; __u64 now; - if (!list_empty(&rpc->throttled_links)) { + if (!list_empty(&rpc->throttled_links)) return; - } now = get_cycles(); if (!list_empty(&homa->throttled_rpcs)) INC_METRIC(throttled_cycles, now - homa->throttle_add); @@ -997,6 +977,7 @@ void homa_add_to_throttled(struct homa_rpc *rpc) list_for_each_entry_rcu(candidate, &homa->throttled_rpcs, throttled_links) { int bytes_left_cand; + checks++; /* Watch out: the pacer might have just transmitted the last @@ -1049,13 +1030,13 @@ void homa_log_throttled(struct homa *homa) int rpcs = 0; int64_t bytes = 0; - printk(KERN_NOTICE "Printing throttled list\n"); + pr_notice("Printing throttled list\n"); homa_throttle_lock(homa); list_for_each_entry_rcu(rpc, &homa->throttled_rpcs, throttled_links) { rpcs++; if (!homa_bucket_try_lock(rpc->bucket, rpc->id, "homa_log_throttled")) { - printk(KERN_NOTICE "Skipping throttled RPC: locked\n"); + pr_notice("Skipping throttled RPC: locked\n"); continue; } if (*rpc->msgout.next_xmit != NULL) @@ -1066,6 +1047,6 @@ void homa_log_throttled(struct homa *homa) homa_rpc_unlock(rpc); } homa_throttle_unlock(homa); - printk(KERN_NOTICE "Finished printing throttle list: %d rpcs, " - "%lld bytes\n", rpcs, bytes); + pr_notice("Finished printing throttle list: %d rpcs, %lld bytes\n", + rpcs, bytes); } diff --git a/homa_peertab.c b/homa_peertab.c index ba4d07d..c9ac76e 100644 --- a/homa_peertab.c +++ b/homa_peertab.c @@ -1,6 +1,4 @@ -/* Copyright (c) 2019-2022 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause /* This file manages homa_peertab objects and is responsible for creating * and deleting homa_peer objects. @@ -21,15 +19,15 @@ int homa_peertab_init(struct homa_peertab *peertab) * an error. */ int i; + spin_lock_init(&peertab->write_lock); INIT_LIST_HEAD(&peertab->dead_dsts); - peertab->buckets = (struct hlist_head *) vmalloc( + peertab->buckets = vmalloc( HOMA_PEERTAB_BUCKETS * sizeof(*peertab->buckets)); if (!peertab->buckets) return -ENOMEM; - for (i = 0; i < HOMA_PEERTAB_BUCKETS; i++) { + for (i = 0; i < HOMA_PEERTAB_BUCKETS; i++) INIT_HLIST_HEAD(&peertab->buckets[i]); - } return 0; } @@ -45,6 +43,7 @@ void homa_peertab_destroy(struct homa_peertab *peertab) int i; struct homa_peer *peer; struct hlist_node *next; + if (!peertab->buckets) return; @@ -68,7 +67,7 @@ void homa_peertab_destroy(struct homa_peertab *peertab) * caller must free this. If there is an error, or if there * are no peers, NULL is returned. */ -struct homa_peer ** homa_peertab_get_peers(struct homa_peertab *peertab, +struct homa_peer **homa_peertab_get_peers(struct homa_peertab *peertab, int *num_peers) { int i, count; @@ -91,7 +90,7 @@ struct homa_peer ** homa_peertab_get_peers(struct homa_peertab *peertab, if (count == 0) return NULL; - result = (struct homa_peer **) kmalloc(count*sizeof(peer), GFP_KERNEL); + result = kmalloc_array(count, sizeof(peer), GFP_KERNEL); if (result == NULL) return NULL; *num_peers = count; @@ -149,16 +148,17 @@ struct homa_peer *homa_peer_find(struct homa_peertab *peertab, */ struct homa_peer *peer; struct dst_entry *dst; + // Should use siphash or jhash here: __u32 bucket = hash_32(addr->in6_u.u6_addr32[0], HOMA_PEERTAB_BUCKET_BITS); + bucket ^= hash_32(addr->in6_u.u6_addr32[1], HOMA_PEERTAB_BUCKET_BITS); bucket ^= hash_32(addr->in6_u.u6_addr32[2], HOMA_PEERTAB_BUCKET_BITS); bucket ^= hash_32(addr->in6_u.u6_addr32[3], HOMA_PEERTAB_BUCKET_BITS); hlist_for_each_entry_rcu(peer, &peertab->buckets[bucket], peertab_links) { - if (ipv6_addr_equal(&peer->addr, addr)) { + if (ipv6_addr_equal(&peer->addr, addr)) return peer; - } INC_METRIC(peer_hash_links, 1); } @@ -206,7 +206,7 @@ struct homa_peer *homa_peer_find(struct homa_peertab *peertab, spin_lock_init(&peer->ack_lock); INC_METRIC(peer_new_entries, 1); - done: +done: spin_unlock_bh(&peertab->write_lock); return peer; } @@ -228,8 +228,8 @@ void homa_dst_refresh(struct homa_peertab *peertab, struct homa_peer *peer, if (IS_ERR(dst)) { /* Retain the existing dst if we can't create a new one. */ if (hsk->homa->verbose) - printk(KERN_NOTICE "homa_refresh couldn't recreate " - "dst: error %ld", PTR_ERR(dst)); + pr_notice("%s couldn't recreate dst: error %ld", + __func__, PTR_ERR(dst)); INC_METRIC(peer_route_errors, 1); } else { struct homa_dead_dst *dead = (struct homa_dead_dst *) @@ -266,6 +266,7 @@ int homa_unsched_priority(struct homa *homa, struct homa_peer *peer, int length) { int i; + for (i = homa->num_priorities-1; ; i--) { if (peer->unsched_cutoffs[i] >= length) return i; @@ -287,6 +288,7 @@ struct dst_entry *homa_peer_get_dst(struct homa_peer *peer, memset(&peer->flow, 0, sizeof(peer->flow)); if (inet->sk.sk_family == AF_INET) { struct rtable *rt; + flowi4_init_output(&peer->flow.u.ip4, inet->sk.sk_bound_dev_if, inet->sk.sk_mark, inet->tos, RT_SCOPE_UNIVERSE, inet->sk.sk_protocol, 0, @@ -298,27 +300,26 @@ struct dst_entry *homa_peer_get_dst(struct homa_peer *peer, if (IS_ERR(rt)) return (struct dst_entry *)(PTR_ERR(rt)); return &rt->dst; - } else { - peer->flow.u.ip6.flowi6_oif = inet->sk.sk_bound_dev_if; - peer->flow.u.ip6.flowi6_iif = LOOPBACK_IFINDEX; - peer->flow.u.ip6.flowi6_mark = inet->sk.sk_mark; - peer->flow.u.ip6.flowi6_scope = RT_SCOPE_UNIVERSE; - peer->flow.u.ip6.flowi6_proto = inet->sk.sk_protocol; - peer->flow.u.ip6.flowi6_flags = 0; - peer->flow.u.ip6.flowi6_secid = 0; - peer->flow.u.ip6.flowi6_tun_key.tun_id = 0; - peer->flow.u.ip6.flowi6_uid = inet->sk.sk_uid; - peer->flow.u.ip6.daddr = peer->addr; - peer->flow.u.ip6.saddr = inet->pinet6->saddr; - peer->flow.u.ip6.fl6_dport = 0; - peer->flow.u.ip6.fl6_sport = 0; - peer->flow.u.ip6.mp_hash = 0; - peer->flow.u.ip6.__fl_common.flowic_tos = inet->tos; - peer->flow.u.ip6.flowlabel = ip6_make_flowinfo(inet->tos, 0); - security_sk_classify_flow(&inet->sk, &peer->flow.u.__fl_common); - return ip6_dst_lookup_flow(sock_net(&inet->sk), &inet->sk, - &peer->flow.u.ip6, NULL); } + peer->flow.u.ip6.flowi6_oif = inet->sk.sk_bound_dev_if; + peer->flow.u.ip6.flowi6_iif = LOOPBACK_IFINDEX; + peer->flow.u.ip6.flowi6_mark = inet->sk.sk_mark; + peer->flow.u.ip6.flowi6_scope = RT_SCOPE_UNIVERSE; + peer->flow.u.ip6.flowi6_proto = inet->sk.sk_protocol; + peer->flow.u.ip6.flowi6_flags = 0; + peer->flow.u.ip6.flowi6_secid = 0; + peer->flow.u.ip6.flowi6_tun_key.tun_id = 0; + peer->flow.u.ip6.flowi6_uid = inet->sk.sk_uid; + peer->flow.u.ip6.daddr = peer->addr; + peer->flow.u.ip6.saddr = inet->pinet6->saddr; + peer->flow.u.ip6.fl6_dport = 0; + peer->flow.u.ip6.fl6_sport = 0; + peer->flow.u.ip6.mp_hash = 0; + peer->flow.u.ip6.__fl_common.flowic_tos = inet->tos; + peer->flow.u.ip6.flowlabel = ip6_make_flowinfo(inet->tos, 0); + security_sk_classify_flow(&inet->sk, &peer->flow.u.__fl_common); + return ip6_dst_lookup_flow(sock_net(&inet->sk), &inet->sk, + &peer->flow.u.ip6, NULL); } /** @@ -357,6 +358,7 @@ void homa_peer_set_cutoffs(struct homa_peer *peer, int c0, int c1, int c2, void homa_peer_lock_slow(struct homa_peer *peer) { __u64 start = get_cycles(); + tt_record("beginning wait for peer lock"); spin_lock_bh(&peer->ack_lock); tt_record("ending wait for peer lock"); diff --git a/homa_plumbing.c b/homa_plumbing.c index ba820ff..743cb8a 100644 --- a/homa_plumbing.c +++ b/homa_plumbing.c @@ -1,6 +1,4 @@ -/* Copyright (c) 2019-2023 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause /* This file consists mostly of "glue" that hooks Homa into the rest of * the Linux kernel. The guts of the protocol are in other files. @@ -29,7 +27,7 @@ struct homa *homa = &homa_data; /* True means that the Homa module is in the process of unloading itself, * so everyone should clean up. */ -static bool exiting = false; +static bool exiting; /* Thread that runs timer code to detect lost packets and crashed peers. */ static struct task_struct *timer_kthread; @@ -190,7 +188,7 @@ static const struct proc_ops homa_metrics_pops = { }; /* Used to remove /proc/net/homa_metrics when the module is unloaded. */ -static struct proc_dir_entry *metrics_dir_entry = NULL; +static struct proc_dir_entry *metrics_dir_entry; /* Used to configure sysctl access to Homa configuration parameters.*/ static struct ctl_table homa_ctl_table[] = { @@ -520,18 +518,12 @@ static DECLARE_COMPLETION(timer_thread_done); * homa_load() - invoked when this module is loaded into the Linux kernel * Return: 0 on success, otherwise a negative errno. */ -static int __init homa_load(void) { +static int __init homa_load(void) +{ int status; - printk(KERN_NOTICE "Homa module loading\n"); - printk(KERN_NOTICE "Homa structure sizes: data_header %u, " - "seg_header %u, ack %u, " - "grant_header %u, peer %u, ip_hdr %u flowi %u " - "ipv6_hdr %u, flowi6 %u " - "tcp_sock %u homa_rpc %u sk_buff %u " - "rcvmsg_control %u sockaddr_in_union %u " - "HOMA_MAX_BPAGES %u NR_CPUS %u " - "nr_cpu_ids %u, MAX_NUMNODES %d\n", + pr_notice("Homa module loading\n"); + pr_notice("Homa structure sizes: data_header %u, seg_header %u, ack %u, grant_header %u, peer %u, ip_hdr %u flowi %u ipv6_hdr %u, flowi6 %u tcp_sock %u homa_rpc %u sk_buff %u rcvmsg_control %u union sockaddr_in_union %u HOMA_MAX_BPAGES %u NR_CPUS %u nr_cpu_ids %u, MAX_NUMNODES %d\n", sizeof32(struct data_header), sizeof32(struct seg_header), sizeof32(struct homa_ack), @@ -545,45 +537,43 @@ static int __init homa_load(void) { sizeof32(struct homa_rpc), sizeof32(struct sk_buff), sizeof32(struct homa_recvmsg_args), - sizeof32(sockaddr_in_union), + sizeof32(union sockaddr_in_union), HOMA_MAX_BPAGES, NR_CPUS, nr_cpu_ids, MAX_NUMNODES); status = proto_register(&homa_prot, 1); if (status != 0) { - printk(KERN_ERR "proto_register failed for homa_prot: %d\n", - status); + pr_err("proto_register failed for homa_prot: %d\n", status); goto out; } status = proto_register(&homav6_prot, 1); if (status != 0) { - printk(KERN_ERR "proto_register failed for homav6_prot: %d\n", - status); + pr_err("proto_register failed for homav6_prot: %d\n", status); goto out; } inet_register_protosw(&homa_protosw); inet6_register_protosw(&homav6_protosw); status = inet_add_protocol(&homa_protocol, IPPROTO_HOMA); if (status != 0) { - printk(KERN_ERR "inet_add_protocol failed in homa_load: %d\n", - status); + pr_err("inet_add_protocol failed in %s: %d\n", __func__, + status); goto out_cleanup; } status = inet6_add_protocol(&homav6_protocol, IPPROTO_HOMA); if (status != 0) { - printk(KERN_ERR "inet6_add_protocol failed in homa_load: %d\n", - status); + pr_err("inet6_add_protocol failed in %s: %d\n", __func__, + status); goto out_cleanup; } status = homa_init(homa); if (status) goto out_cleanup; - metrics_dir_entry = proc_create("homa_metrics", S_IRUGO, + metrics_dir_entry = proc_create("homa_metrics", 0444, init_net.proc_net, &homa_metrics_pops); if (!metrics_dir_entry) { - printk(KERN_ERR "couldn't create /proc/net/homa_metrics\n"); + pr_err("couldn't create /proc/net/homa_metrics\n"); status = -ENOMEM; goto out_cleanup; } @@ -591,21 +581,21 @@ static int __init homa_load(void) { homa_ctl_header = register_net_sysctl(&init_net, "net/homa", homa_ctl_table); if (!homa_ctl_header) { - printk(KERN_ERR "couldn't register Homa sysctl parameters\n"); + pr_err("couldn't register Homa sysctl parameters\n"); status = -ENOMEM; goto out_cleanup; } status = homa_offload_init(); if (status != 0) { - printk(KERN_ERR "Homa couldn't init offloads\n"); + pr_err("Homa couldn't init offloads\n"); goto out_cleanup; } timer_kthread = kthread_run(homa_timer_main, homa, "homa_timer"); if (IS_ERR(timer_kthread)) { status = PTR_ERR(timer_kthread); - printk(KERN_ERR "couldn't create homa pacer thread: error %d\n", + pr_err("couldn't create homa pacer thread: error %d\n", status); timer_kthread = NULL; goto out_cleanup; @@ -635,8 +625,9 @@ static int __init homa_load(void) { /** * homa_unload() - invoked when this module is unloaded from the Linux kernel. */ -static void __exit homa_unload(void) { - printk(KERN_NOTICE "Homa module unloading\n"); +static void __exit homa_unload(void) +{ + pr_notice("Homa module unloading\n"); exiting = true; tt_destroy(); @@ -645,7 +636,7 @@ static void __exit homa_unload(void) { if (timer_kthread) wake_up_process(timer_kthread); if (homa_offload_end() != 0) - printk(KERN_ERR "Homa couldn't stop offloads\n"); + pr_err("Homa couldn't stop offloads\n"); wait_for_completion(&timer_thread_done); unregister_net_sysctl_table(homa_ctl_header); proc_remove(metrics_dir_entry); @@ -674,21 +665,18 @@ module_exit(homa_unload); int homa_bind(struct socket *sock, struct sockaddr *addr, int addr_len) { struct homa_sock *hsk = homa_sk(sock->sk); - sockaddr_in_union *addr_in = (sockaddr_in_union *) addr; + union sockaddr_in_union *addr_in = (union sockaddr_in_union *) addr; int port = 0; - if (unlikely(addr->sa_family != sock->sk->sk_family)) { + if (unlikely(addr->sa_family != sock->sk->sk_family)) return -EAFNOSUPPORT; - } if (addr_in->in6.sin6_family == AF_INET6) { - if (addr_len < sizeof(struct sockaddr_in6)) { + if (addr_len < sizeof(struct sockaddr_in6)) return -EINVAL; - } port = ntohs(addr_in->in4.sin_port); } else if (addr_in->in4.sin_family == AF_INET) { - if (addr_len < sizeof(struct sockaddr_in)) { + if (addr_len < sizeof(struct sockaddr_in)) return -EINVAL; - } port = ntohs(addr_in->in6.sin6_port); } return homa_sock_bind(&homa->port_map, hsk, port); @@ -699,8 +687,10 @@ int homa_bind(struct socket *sock, struct sockaddr *addr, int addr_len) * @sk: Socket being closed * @timeout: ?? */ -void homa_close(struct sock *sk, long timeout) { +void homa_close(struct sock *sk, long timeout) +{ struct homa_sock *hsk = homa_sk(sk); + homa_sock_destroy(hsk); sk_common_release(sk); tt_record1("closed socket, port %d\n", hsk->port); @@ -731,9 +721,10 @@ int homa_shutdown(struct socket *sock, int how) * * Return: 0 on success, otherwise a negative errno. */ -int homa_disconnect(struct sock *sk, int flags) { - printk(KERN_WARNING "unimplemented disconnect invoked on Homa socket\n"); - return -ENOSYS; +int homa_disconnect(struct sock *sk, int flags) +{ + pr_warn("unimplemented disconnect invoked on Homa socket\n"); + return -EINVAL; } /** @@ -744,7 +735,8 @@ int homa_disconnect(struct sock *sk, int flags) { * * Return: 0 on success, otherwise a negative errno. */ -int homa_ioc_abort(struct sock *sk, int *karg) { +int homa_ioc_abort(struct sock *sk, int *karg) +{ int ret = 0; struct homa_sock *hsk = homa_sk(sk); struct homa_abort_args args; @@ -753,9 +745,8 @@ int homa_ioc_abort(struct sock *sk, int *karg) { if (unlikely(copy_from_user(&args, (void *) karg, sizeof(args)))) return -EFAULT; - if (args._pad1 || args._pad2[0] || args._pad2[1]) { + if (args._pad1 || args._pad2[0] || args._pad2[1]) return -EINVAL; - } if (args.id == 0) { homa_abort_sock_rpcs(hsk, -args.error); return 0; @@ -764,11 +755,10 @@ int homa_ioc_abort(struct sock *sk, int *karg) { rpc = homa_find_client_rpc(hsk, args.id); if (rpc == NULL) return -EINVAL; - if (args.error == 0) { + if (args.error == 0) homa_rpc_free(rpc); - } else { + else homa_rpc_abort(rpc, -args.error); - } homa_rpc_unlock(rpc); return ret; } @@ -782,7 +772,8 @@ int homa_ioc_abort(struct sock *sk, int *karg) { * * Return: 0 on success, otherwise a negative errno. */ -int homa_ioctl(struct sock *sk, int cmd, int *karg) { +int homa_ioctl(struct sock *sk, int cmd, int *karg) +{ int result; __u64 start = get_cycles(); @@ -793,13 +784,12 @@ int homa_ioctl(struct sock *sk, int cmd, int *karg) { INC_METRIC(abort_cycles, get_cycles() - start); break; case HOMAIOCFREEZE: - tt_record1("Freezing timetrace because of HOMAIOCFREEZE ioctl, " - "pid %d", current->pid); + tt_record1("Freezing timetrace because of HOMAIOCFREEZE ioctl, pid %d", current->pid); tt_freeze(); result = 0; break; default: - printk(KERN_NOTICE "Unknown Homa ioctl: %d\n", cmd); + pr_notice("Unknown Homa ioctl: %d\n", cmd); result = -EINVAL; break; } @@ -816,6 +806,7 @@ int homa_ioctl(struct sock *sk, int cmd, int *karg) { int homa_socket(struct sock *sk) { struct homa_sock *hsk = homa_sk(sk); + homa_sock_init(hsk, homa); return 0; } @@ -870,9 +861,10 @@ int homa_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, * Return: 0 on success, otherwise a negative errno. */ int homa_getsockopt(struct sock *sk, int level, int optname, - char __user *optval, int __user *option) { - printk(KERN_WARNING "unimplemented getsockopt invoked on Homa socket:" - " level %d, optname %d\n", level, optname); + char __user *optval, int __user *option) +{ + pr_warn("unimplemented getsockopt invoked on Homa socket: level %d, optname %d\n", + level, optname); return -EINVAL; } @@ -885,14 +877,15 @@ int homa_getsockopt(struct sock *sk, int level, int optname, * @length: Number of bytes of the message. * Return: 0 on success, otherwise a negative errno. */ -int homa_sendmsg(struct sock *sk, struct msghdr *msg, size_t length) { +int homa_sendmsg(struct sock *sk, struct msghdr *msg, size_t length) +{ struct homa_sock *hsk = homa_sk(sk); struct homa_sendmsg_args args; __u64 start = get_cycles(); __u64 finish; int result = 0; struct homa_rpc *rpc = NULL; - sockaddr_in_union *addr = (sockaddr_in_union *) msg->msg_name; + union sockaddr_in_union *addr = (union sockaddr_in_union *) msg->msg_name; homa_cores[raw_smp_processor_id()]->last_app_active = start; if (unlikely(!msg->msg_control_is_user)) { @@ -969,9 +962,8 @@ int homa_sendmsg(struct sock *sk, struct msghdr *msg, size_t length) { * this could be totally valid (e.g. client is * no longer interested in it). */ - tt_record2("homa_sendmsg error: RPC id %d, peer 0x%x, " - "doesn't exist", args.id, - tt_addr(canonical_dest)); + tt_record2("homa_sendmsg error: RPC id %d, peer 0x%x, doesn't exist", + args.id, tt_addr(canonical_dest)); return 0; } if (rpc->error) { @@ -979,8 +971,7 @@ int homa_sendmsg(struct sock *sk, struct msghdr *msg, size_t length) { goto error; } if (rpc->state != RPC_IN_SERVICE) { - tt_record2("homa_sendmsg error: RPC id %d in bad " - "state %d", rpc->id, rpc->state); + tt_record2("homa_sendmsg error: RPC id %d in bad state %d", rpc->id, rpc->state); homa_rpc_unlock(rpc); rpc = 0; result = -EINVAL; @@ -1081,18 +1072,17 @@ int homa_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, */ if (rpc->hsk->homa->freeze_type == SLOW_RPC) { uint64_t elapsed = (get_cycles() - rpc->start_cycles)>>10; + if ((elapsed <= hsk->homa->temp[1]) && (elapsed >= hsk->homa->temp[0]) && homa_is_client(rpc->id) && (rpc->msgin.length >= hsk->homa->temp[2]) && (rpc->msgin.length < hsk->homa->temp[3])) { - tt_record4("Long RTT: kcycles %d, id %d, peer 0x%x, " - "length %d", + tt_record4("Long RTT: kcycles %d, id %d, peer 0x%x, length %d", elapsed, rpc->id, tt_addr(rpc->peer->addr), rpc->msgin.length); - homa_freeze(rpc, SLOW_RPC, "Freezing because of long " - "elapsed time for RPC id %d, peer 0x%x"); + homa_freeze(rpc, SLOW_RPC, "Freezing because of long elapsed time for RPC id %d, peer 0x%x"); } } @@ -1106,12 +1096,14 @@ int homa_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, } if (sk->sk_family == AF_INET6) { struct sockaddr_in6 *in6 = msg->msg_name; + in6->sin6_family = AF_INET6; in6->sin6_port = htons(rpc->dport); in6->sin6_addr = rpc->peer->addr; *addr_len = sizeof(*in6); } else { struct sockaddr_in *in4 = msg->msg_name; + in4->sin_family = AF_INET; in4->sin_port = htons(rpc->dport); in4->sin_addr.s_addr = ipv6_to_ipv4( @@ -1141,7 +1133,7 @@ int homa_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, done: if (unlikely(copy_to_user(msg->msg_control, &control, sizeof(control)))) { /* Note: in this case the message's buffers will be leaked. */ - printk(KERN_NOTICE "homa_recvmsg couldn't copy back args\n"); + pr_notice("%s couldn't copy back args\n", __func__); result = -EFAULT; } @@ -1158,8 +1150,9 @@ int homa_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, * @sk: Socket for the operation * Return: ?? */ -int homa_hash(struct sock *sk) { - printk(KERN_WARNING "unimplemented hash invoked on Homa socket\n"); +int homa_hash(struct sock *sk) +{ + pr_warn("unimplemented hash invoked on Homa socket\n"); return 0; } @@ -1167,17 +1160,18 @@ int homa_hash(struct sock *sk) { * homa_unhash() - ??. * @sk: Socket for the operation */ -void homa_unhash(struct sock *sk) { - return; - printk(KERN_WARNING "unimplemented unhash invoked on Homa socket\n"); +void homa_unhash(struct sock *sk) +{ + pr_warn("unimplemented unhash invoked on Homa socket\n"); } /** * homa_rehash() - ??. * @sk: Socket for the operation */ -void homa_rehash(struct sock *sk) { - printk(KERN_WARNING "unimplemented rehash invoked on Homa socket\n"); +void homa_rehash(struct sock *sk) +{ + pr_warn("unimplemented rehash invoked on Homa socket\n"); } /** @@ -1187,7 +1181,8 @@ void homa_rehash(struct sock *sk) { * @snum: Unclear what this is. * Return: Zero for success, or a negative errno for an error. */ -int homa_get_port(struct sock *sk, unsigned short snum) { +int homa_get_port(struct sock *sk, unsigned short snum) +{ /* Homa always assigns ports immediately when a socket is created, * so there is nothing to do here. */ @@ -1200,9 +1195,10 @@ int homa_get_port(struct sock *sk, unsigned short snum) { * @err: ?? * Return: ?? */ -int homa_diag_destroy(struct sock *sk, int err) { - printk(KERN_WARNING "unimplemented diag_destroy invoked on Homa socket\n"); - return -ENOSYS; +int homa_diag_destroy(struct sock *sk, int err) +{ + pr_warn("unimplemented diag_destroy invoked on Homa socket\n"); + return -EINVAL; } @@ -1211,8 +1207,9 @@ int homa_diag_destroy(struct sock *sk, int err) { * @skb: Socket buffer. * Return: Always 0? */ -int homa_v4_early_demux(struct sk_buff *skb) { - printk(KERN_WARNING "unimplemented early_demux invoked on Homa socket\n"); +int homa_v4_early_demux(struct sk_buff *skb) +{ + pr_warn("unimplemented early_demux invoked on Homa socket\n"); return 0; } @@ -1221,8 +1218,9 @@ int homa_v4_early_demux(struct sk_buff *skb) { * @skb: Socket buffer. * @return: Always 0? */ -int homa_v4_early_demux_handler(struct sk_buff *skb) { - printk(KERN_WARNING "unimplemented early_demux_handler invoked on Homa socket\n"); +int homa_v4_early_demux_handler(struct sk_buff *skb) +{ + pr_warn("unimplemented early_demux_handler invoked on Homa socket\n"); return 0; } @@ -1232,11 +1230,12 @@ int homa_v4_early_demux_handler(struct sk_buff *skb) { * @skb: The incoming packet. * Return: Always 0 */ -int homa_softirq(struct sk_buff *skb) { +int homa_softirq(struct sk_buff *skb) +{ struct common_header *h; struct sk_buff *packets, *other_pkts, *next; struct sk_buff **prev_link, **other_link; - static __u64 last = 0; + static __u64 last; __u64 start; int header_offset; int first_packet = 1; @@ -1247,12 +1246,13 @@ int homa_softirq(struct sk_buff *skb) { homa_cores[raw_smp_processor_id()]->last_active = start; if ((start - last) > 1000000) { int scaled_ms = (int) (10*(start-last)/cpu_khz); + if ((scaled_ms >= 50) && (scaled_ms < 10000)) { // tt_record3("Gap in incoming packets: %d cycles " // "(%d.%1d ms)", // (int) (start - last), scaled_ms/10, // scaled_ms%10); -// printk(KERN_NOTICE "Gap in incoming packets: %llu " +// pr_notice("Gap in incoming packets: %llu " // "cycles, (%d.%1d ms)", (start - last), // scaled_ms/10, scaled_ms%10); } @@ -1271,6 +1271,7 @@ int homa_softirq(struct sk_buff *skb) { prev_link = &packets; for (skb = packets; skb != NULL; skb = next) { const struct in6_addr saddr = skb_canonical_ipv6_saddr(skb); + next = skb->next; /* Make the header available at skb->data, even if the packet @@ -1284,9 +1285,7 @@ int homa_softirq(struct sk_buff *skb) { pull_length = skb->len; if (!pskb_may_pull(skb, pull_length)) { if (homa->verbose) - printk(KERN_NOTICE "Homa can't handle fragmented " - "packet (no space for header); " - "discarding\n"); + pr_notice("Homa can't handle fragmented packet (no space for header); discarding\n"); UNIT_LOG("", "pskb discard"); goto discard; } @@ -1300,9 +1299,7 @@ int homa_softirq(struct sk_buff *skb) { || (h->type >= BOGUS) || (skb->len < header_lengths[h->type-DATA]))) { if (homa->verbose) - printk(KERN_WARNING - "Homa %s packet from %s too " - "short: %d bytes\n", + pr_warn("Homa %s packet from %s too short: %d bytes\n", homa_symbol_for_type(h->type), homa_print_ipv6_addr(&saddr), skb->len - header_offset); @@ -1311,8 +1308,7 @@ int homa_softirq(struct sk_buff *skb) { } if (first_packet) { - tt_record4("homa_softirq: first packet from 0x%x:%d, " - "id %llu, type %d", + tt_record4("homa_softirq: first packet from 0x%x:%d, id %llu, type %d", tt_addr(saddr), ntohs(h->sport), homa_local_id(h->sender_id), h->type); first_packet = 0; @@ -1324,8 +1320,7 @@ int homa_softirq(struct sk_buff *skb) { if (unlikely(h->type == FREEZE)) { if (!tt_frozen) { homa_rpc_log_active_tt(homa, 0); - tt_record4("Freezing because of request on " - "port %d from 0x%x:%d, id %d", + tt_record4("Freezing because of request on port %d from 0x%x:%d, id %d", ntohs(h->dport), tt_addr(saddr), ntohs(h->sport), homa_local_id(h->sender_id)); @@ -1348,7 +1343,7 @@ int homa_softirq(struct sk_buff *skb) { prev_link = &skb->next; continue; - discard: +discard: *prev_link = skb->next; kfree_skb(skb); } @@ -1412,7 +1407,7 @@ int homa_softirq(struct sk_buff *skb) { */ int homa_backlog_rcv(struct sock *sk, struct sk_buff *skb) { - printk(KERN_WARNING "unimplemented backlog_rcv invoked on Homa socket\n"); + pr_warn("unimplemented backlog_rcv invoked on Homa socket\n"); kfree_skb(skb); return 0; } @@ -1435,12 +1430,14 @@ int homa_err_handler_v4(struct sk_buff *skb, u32 info) if ((type == ICMP_DEST_UNREACH) && (code == ICMP_PORT_UNREACH)) { struct common_header *h; char *icmp = (char *) icmp_hdr(skb); + iph = (struct iphdr *) (icmp + sizeof(struct icmphdr)); h = (struct common_header *) (icmp + sizeof(struct icmphdr) + iph->ihl*4); homa_abort_rpcs(homa, &saddr, htons(h->dport), -ENOTCONN); } else if (type == ICMP_DEST_UNREACH) { int error; + if (code == ICMP_PROT_UNREACH) error = -EPROTONOSUPPORT; else @@ -1449,9 +1446,8 @@ int homa_err_handler_v4(struct sk_buff *skb, u32 info) iph->saddr, iph->daddr); homa_abort_rpcs(homa, &saddr, 0, error); } else { - printk(KERN_NOTICE "homa_err_handler_v4 invoked with " - "info %x, ICMP type %d, ICMP code %d\n", - info, type, code); + pr_notice("%s invoked with info %x, ICMP type %d, ICMP code %d\n", + __func__, info, type, code); } return 0; } @@ -1476,12 +1472,14 @@ int homa_err_handler_v6(struct sk_buff *skb, struct inet6_skb_parm *opt, if ((type == ICMPV6_DEST_UNREACH) && (code == ICMPV6_PORT_UNREACH)) { struct common_header *h; char *icmp = (char *) icmp_hdr(skb); + iph = (struct ipv6hdr *) (icmp + sizeof(struct icmphdr)); h = (struct common_header *) (icmp + sizeof(struct icmphdr) + HOMA_IPV6_HEADER_LENGTH); homa_abort_rpcs(homa, &iph->daddr, htons(h->dport), -ENOTCONN); } else if (type == ICMPV6_DEST_UNREACH) { int error; + if (code == ICMP_PROT_UNREACH) error = -EPROTONOSUPPORT; else @@ -1491,9 +1489,8 @@ int homa_err_handler_v6(struct sk_buff *skb, struct inet6_skb_parm *opt, homa_abort_rpcs(homa, &iph->daddr, 0, error); } else { if (homa->verbose) - printk(KERN_NOTICE "homa_err_handler_v6 invoked with " - "info %x, ICMP type %d, ICMP code %d\n", - info, type, code); + pr_notice("%s invoked with info %x, ICMP type %d, ICMP code %d\n", + __func__, info, type, code); } return 0; } @@ -1510,7 +1507,8 @@ int homa_err_handler_v6(struct sk_buff *skb, struct inet6_skb_parm *opt, * state of the socket. */ __poll_t homa_poll(struct file *file, struct socket *sock, - struct poll_table_struct *wait) { + struct poll_table_struct *wait) +{ struct sock *sk = sock->sk; __poll_t mask; @@ -1546,9 +1544,8 @@ int homa_metrics_open(struct inode *inode, struct file *file) * completely closed. */ spin_lock(&homa->metrics_lock); - if (homa->metrics_active_opens == 0) { + if (homa->metrics_active_opens == 0) homa_print_metrics(homa); - } homa->metrics_active_opens++; spin_unlock(&homa->metrics_lock); return 0; @@ -1628,6 +1625,7 @@ int homa_dointvec(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { int result; + result = proc_dointvec(table, write, buffer, lenp, ppos); if (write) { /* Don't worry which particular value changed; update @@ -1664,8 +1662,7 @@ int homa_dointvec(struct ctl_table *table, int write, else if (action == 5) tt_printk(); else if (action == 6) { - tt_record("Calling homa_rpc_log_active because " - "of action 6"); + tt_record("Calling homa_rpc_log_active because of action 6"); homa_rpc_log_active_tt(homa, 0); tt_record("Freezing because of action 6"); tt_freeze(); @@ -1676,7 +1673,7 @@ int homa_dointvec(struct ctl_table *table, int write, tt_record("Finished freezing cluster"); tt_freeze(); } else if (action == 8) { - printk(KERN_NOTICE"homa_total_incoming is %d\n", + pr_notice("homa_total_incoming is %d\n", atomic_read(&homa->total_incoming)); } else if (action == 9) { tt_print_file("/users/ouster/node.tt"); @@ -1709,7 +1706,7 @@ int homa_sysctl_softirq_cores(struct ctl_table *table, int write, int max_values, *values; max_values = (NUM_GEN3_SOFTIRQ_CORES + 1) * nr_cpu_ids; - values = (int *) kmalloc(max_values * sizeof(int), GFP_KERNEL); + values = kmalloc_array(max_values, sizeof(int), GFP_KERNEL); if (values == NULL) return -ENOMEM; @@ -1728,6 +1725,7 @@ int homa_sysctl_softirq_cores(struct ctl_table *table, int write, for (i = 0; i < max_values; i += NUM_GEN3_SOFTIRQ_CORES + 1) { int j; + if (values[i] < 0) break; core = homa_cores[values[i]]; diff --git a/homa_pool.c b/homa_pool.c index a8dec58..5ea9ab4 100644 --- a/homa_pool.c +++ b/homa_pool.c @@ -1,6 +1,4 @@ -/* Copyright (c) 2022-2023 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause #include "homa_impl.h" @@ -29,7 +27,8 @@ * The caller must own the lock for @pool->hsk. * @pool: Pool to update. */ -inline static void set_bpages_needed(struct homa_pool *pool) { +static inline void set_bpages_needed(struct homa_pool *pool) +{ struct homa_rpc *rpc = list_first_entry(&pool->hsk->waiting_for_bufs, struct homa_rpc, buf_links); pool->bpages_needed = (rpc->msgin.length + HOMA_BPAGE_SIZE - 1) @@ -61,15 +60,15 @@ int homa_pool_init(struct homa_sock *hsk, void *region, __u64 region_size) result = -EINVAL; goto error; } - pool->descriptors = (struct homa_bpage *) kmalloc( - pool->num_bpages * sizeof(struct homa_bpage), - GFP_ATOMIC); + pool->descriptors = kmalloc_array(pool->num_bpages, + sizeof(struct homa_bpage), GFP_ATOMIC); if (!pool->descriptors) { result = -ENOMEM; goto error; } for (i = 0; i < pool->num_bpages; i++) { struct homa_bpage *bp = &pool->descriptors[i]; + spin_lock_init(&bp->lock); atomic_set(&bp->refs, 0); bp->owner = -1; @@ -79,8 +78,8 @@ int homa_pool_init(struct homa_sock *hsk, void *region, __u64 region_size) pool->bpages_needed = INT_MAX; /* Allocate and initialize core-specific data. */ - pool->cores = (struct homa_pool_core *) kmalloc(nr_cpu_ids * - sizeof(struct homa_pool_core), GFP_ATOMIC); + pool->cores = kmalloc_array(nr_cpu_ids, sizeof(struct homa_pool_core), + GFP_ATOMIC); if (!pool->cores) { result = -ENOMEM; goto error; @@ -95,11 +94,9 @@ int homa_pool_init(struct homa_sock *hsk, void *region, __u64 region_size) return 0; - error: - if (pool->descriptors) - kfree(pool->descriptors); - if (pool->cores) - kfree(pool->cores); +error: + kfree(pool->descriptors); + kfree(pool->cores); pool->region = NULL; return result; } @@ -130,7 +127,7 @@ void homa_pool_destroy(struct homa_pool *pool) * of the allocated pages (and the expiration time is also * set). Otherwise the pages are left unowned. * Return: 0 for success, -1 if there wasn't enough free space in the pool. -*/ + */ int homa_pool_get_pages(struct homa_pool *pool, int num_pages, __u32 *pages, int set_owner) { @@ -163,6 +160,7 @@ int homa_pool_get_pages(struct homa_pool *pool, int num_pages, __u32 *pages, */ if (limit == 0) { int extra; + limit = pool->num_bpages - atomic_read(&pool->free_bpages); extra = limit>>2; @@ -295,7 +293,7 @@ int homa_pool_allocate(struct homa_rpc *rpc) goto allocate_partial; /* Can't use the current page; get another one. */ - new_page: +new_page: if (homa_pool_get_pages(pool, 1, pages, 1) != 0) { homa_pool_release_buffers(pool, rpc->msgin.num_bpages, rpc->msgin.bpage_offsets); @@ -305,15 +303,14 @@ int homa_pool_allocate(struct homa_rpc *rpc) core->page_hint = pages[0]; core->allocated = 0; - allocate_partial: +allocate_partial: rpc->msgin.bpage_offsets[rpc->msgin.num_bpages] = core->allocated + (core->page_hint << HOMA_BPAGE_SHIFT); rpc->msgin.num_bpages++; core->allocated += partial; - success: - tt_record4("Allocated %d bpage pointers on port %d for id %d, " - "free_bpages now %d", +success: + tt_record4("Allocated %d bpage pointers on port %d for id %d, free_bpages now %d", rpc->msgin.num_bpages, pool->hsk->port, rpc->id, atomic_read(&pool->free_bpages)); return 0; @@ -321,10 +318,10 @@ int homa_pool_allocate(struct homa_rpc *rpc) /* We get here if there wasn't enough buffer space for this * message; add the RPC to hsk->waiting_for_bufs. */ - out_of_space: +out_of_space: INC_METRIC(buffer_alloc_failures, 1); - tt_record4("Buffer allocation failed, port %d, id %d, length %d, " - "free_bpages %d", pool->hsk->port, rpc->id, + tt_record4("Buffer allocation failed, port %d, id %d, length %d, free_bpages %d", + pool->hsk->port, rpc->id, rpc->msgin.length, atomic_read(&pool->free_bpages)); homa_sock_lock(pool->hsk, "homa_pool_allocate"); @@ -336,7 +333,7 @@ int homa_pool_allocate(struct homa_rpc *rpc) } list_add_tail_rcu(&rpc->buf_links, &pool->hsk->waiting_for_bufs); - queued: +queued: set_bpages_needed(pool); homa_sock_unlock(pool->hsk); return 0; @@ -386,10 +383,11 @@ void homa_pool_release_buffers(struct homa_pool *pool, int num_buffers, return; for (i = 0; i < num_buffers; i++) { __u32 bpage_index = buffers[i] >> HOMA_BPAGE_SHIFT; - struct homa_bpage *bpage= &pool->descriptors[bpage_index]; + struct homa_bpage *bpage = &pool->descriptors[bpage_index]; + if (bpage_index < pool->num_bpages) { - if (atomic_dec_return(&bpage->refs) == 0) - atomic_inc(&pool->free_bpages); + if (atomic_dec_return(&bpage->refs) == 0) + atomic_inc(&pool->free_bpages); } } tt_record3("Released %d bpages, free_bpages for port %d now %d", @@ -413,6 +411,7 @@ void homa_pool_check_waiting(struct homa_pool *pool) #endif while (atomic_read(&pool->free_bpages) >= pool->bpages_needed) { struct homa_rpc *rpc; + homa_sock_lock(pool->hsk, "buffer pool"); if (list_empty(&pool->hsk->waiting_for_bufs)) { pool->bpages_needed = INT_MAX; @@ -429,8 +428,7 @@ void homa_pool_check_waiting(struct homa_pool *pool) * operation again. */ homa_sock_unlock(pool->hsk); - UNIT_LOG("; ", "rpc lock unavailable in " - "homa_pool_release_buffers"); + UNIT_LOG("; ", "rpc lock unavailable in %s", __func__); continue; } list_del_init(&rpc->buf_links); @@ -439,8 +437,7 @@ void homa_pool_check_waiting(struct homa_pool *pool) else set_bpages_needed(pool); homa_sock_unlock(pool->hsk); - tt_record4("Retrying buffer allocation for id %d, length %d, " - "free_bpages %d, new bpages_needed %d", + tt_record4("Retrying buffer allocation for id %d, length %d, free_bpages %d, new bpages_needed %d", rpc->id, rpc->msgin.length, atomic_read(&pool->free_bpages), pool->bpages_needed); @@ -452,4 +449,4 @@ void homa_pool_check_waiting(struct homa_pool *pool) } else homa_rpc_unlock(rpc); } -} \ No newline at end of file +} diff --git a/homa_skb.c b/homa_skb.c index e49455f..d3a40f1 100644 --- a/homa_skb.c +++ b/homa_skb.c @@ -1,6 +1,4 @@ -/* Copyright (c) 2024 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause /* This file contains functions for allocating and freeing sk_buffs. */ @@ -38,8 +36,10 @@ void homa_skb_page_pool_init(struct homa_page_pool *pool) void homa_skb_cleanup(struct homa *homa) { int i, j; + for (i = 0; i < nr_cpu_ids; i++) { struct homa_core *core = homa_cores[i]; + if (core->skb_page != NULL) { put_page(core->skb_page); core->skb_page = NULL; @@ -53,6 +53,7 @@ void homa_skb_cleanup(struct homa *homa) for (i = 0; i < MAX_NUMNODES; i++) { struct homa_numa *numa = homa_numas[i]; + if (!numa) continue; for (j = numa->page_pool.avail - 1; j >= 0; j--) @@ -120,7 +121,7 @@ void homa_skb_stash_pages(struct homa *homa, int length) spin_lock_bh(&homa->page_pool_mutex); while (pool->avail && (core->num_stashed_pages < pages_needed)) { pool->avail--; - if (pool->avail < pool->low_mark ) + if (pool->avail < pool->low_mark) pool->low_mark = pool->avail; core->stashed_pages[core->num_stashed_pages] = pool->pages[pool->avail]; @@ -191,10 +192,11 @@ void *homa_skb_extend_frags(struct homa *homa, struct sk_buff *skb, int *length) * @core: Allocate page in this core. * Return: True if successful, false if memory not available. */ -bool homa_skb_page_alloc(struct homa *homa, struct homa_core * core) +bool homa_skb_page_alloc(struct homa *homa, struct homa_core *core) { struct homa_page_pool *pool; - __u64 start; + __u64 start; + if (core->skb_page) { if (page_ref_count(core->skb_page) == 1) { /* The existing page is no longer in use, so we can @@ -219,6 +221,7 @@ bool homa_skb_page_alloc(struct homa *homa, struct homa_core * core) pool = &core->numa->page_pool; if (pool->avail) { struct homa_page_pool *pool = &core->numa->page_pool; + spin_lock_bh(&homa->page_pool_mutex); if (pool->avail) { pool->avail--; @@ -251,7 +254,7 @@ bool homa_skb_page_alloc(struct homa *homa, struct homa_core * core) core->page_size = core->page_inuse = 0; return false; - success: +success: return true; } @@ -300,8 +303,7 @@ int homa_skb_append_from_iter(struct homa *homa, struct sk_buff *skb, int chunk_length; char *dst; - while (length > 0) - { + while (length > 0) { chunk_length = length; dst = (char *) homa_skb_extend_frags(homa, skb, &chunk_length); if (!dst) @@ -321,7 +323,7 @@ int homa_skb_append_from_iter(struct homa *homa, struct sk_buff *skb, * @dst_skb: Data gets added to the end of this skb. * @src_skb: Data is copied out of this skb. * @offset: Offset within @src_skb of first byte to copy, relative - * to the transport header. + * to the transport header. * @length: Total number of bytes to copy; fewer bytes than this may * be copied if @src_skb isn't long enough to hold all of the * desired bytes. @@ -337,8 +339,7 @@ int homa_skb_append_from_skb(struct homa *homa, struct sk_buff *dst_skb, /* Copy bytes from the linear part of the source, if any. */ head_len = skb_tail_pointer(src_skb) - skb_transport_header(src_skb); - if (offset < head_len) - { + if (offset < head_len) { chunk_size = length; if (chunk_size > (head_len - offset)) chunk_size = head_len - offset; @@ -351,13 +352,12 @@ int homa_skb_append_from_skb(struct homa *homa, struct sk_buff *dst_skb, length -= chunk_size; } - /* Virtually copy bytes from source frags, if needed. */ + /* Virtually copy bytes from source frags, if needed. */ src_frag_offset = head_len; for (src_frags_left = src_shinfo->nr_frags, src_frag = &src_shinfo->frags[0]; - (src_frags_left > 0) && (length > 0); - src_frags_left--, src_frag_offset += skb_frag_size(src_frag), - src_frag++) - { + (src_frags_left > 0) && (length > 0); + src_frags_left--, src_frag_offset += skb_frag_size(src_frag), + src_frag++) { if (offset >= (src_frag_offset + skb_frag_size(src_frag))) continue; chunk_size = skb_frag_size(src_frag) - (offset - src_frag_offset); @@ -415,7 +415,8 @@ void homa_skb_free_many_tx(struct homa *homa, struct sk_buff **skbs, int count) if (refcount_read(&skb->users) != 1) { /* This sk_buff is still in use somewhere, so can't - * reclaim its pages. */ + * reclaim its pages. + */ kfree_skb(skb); continue; } @@ -423,6 +424,7 @@ void homa_skb_free_many_tx(struct homa *homa, struct sk_buff **skbs, int count) /* Reclaim cacheable pages. */ for (j = 0; j < shinfo->nr_frags; j++) { struct page *page = skb_frag_page(&shinfo->frags[j]); + if ((compound_order(page) == HOMA_SKB_PAGE_ORDER) && (page_ref_count(page) == 1)) { pages_to_cache[num_pages] = page; @@ -459,6 +461,7 @@ void homa_skb_cache_pages(struct homa *homa, struct page **pages, int count) #define LIMIT HOMA_PAGE_POOL_SIZE #endif int i; + spin_lock_bh(&homa->page_pool_mutex); for (i = 0; i < count; i++) { struct page *page = pages[i]; @@ -536,14 +539,14 @@ void homa_skb_release_pages(struct homa *homa) if (now < homa->skb_page_free_time) return; - /* Free pages every 0.5 second. */ + /* Free pages every 0.5 second. */ interval = cpu_khz*500; homa->skb_page_free_time = now + interval; release_max = homa->skb_page_frees_per_sec/2; if (homa->pages_to_free_slots < release_max) { if (homa->skb_pages_to_free != NULL) kfree(homa->skb_pages_to_free); - homa->skb_pages_to_free = kmalloc(release_max * + homa->skb_pages_to_free = kmalloc_array(release_max, sizeof(struct page *), GFP_KERNEL); homa->pages_to_free_slots = release_max; } @@ -553,12 +556,13 @@ void homa_skb_release_pages(struct homa *homa) spin_lock_bh(&homa->page_pool_mutex); for (i = 0; i < homa_num_numas; i++) { struct homa_page_pool *pool = &homa_numas[i]->page_pool; + if (pool->low_mark > max_low_mark) { max_low_mark = pool->low_mark; max_pool = pool; } - tt_record3("NUMA node %d has %d pages in skb page pool, " - "low mark %d", i, pool->avail, pool->low_mark); + tt_record3("NUMA node %d has %d pages in skb page pool, low mark %d", + i, pool->avail, pool->low_mark); pool->low_mark = pool->avail; } @@ -566,7 +570,7 @@ void homa_skb_release_pages(struct homa *homa) * releasing the lock, since freeing is expensive). */ min_pages = ((homa->skb_page_pool_min_kb * 1000) - + (HOMA_SKB_PAGE_SIZE - 1))/ HOMA_SKB_PAGE_SIZE; + + (HOMA_SKB_PAGE_SIZE - 1)) / HOMA_SKB_PAGE_SIZE; release = max_low_mark - min_pages; if (release > release_max) release = release_max; @@ -580,8 +584,9 @@ void homa_skb_release_pages(struct homa *homa) /* Free the pages that were collected. */ for (i = 0; i < release; i++) { struct page *page = homa->skb_pages_to_free[i]; + tt_record2("homa_skb_release_pages releasing page 0x%08x%08x", tt_hi(page), tt_lo(page)); put_page(page); } -} \ No newline at end of file +} diff --git a/homa_socktab.c b/homa_socktab.c index 4f71080..aa01bf1 100644 --- a/homa_socktab.c +++ b/homa_socktab.c @@ -1,6 +1,4 @@ -/* Copyright (c) 2019-2023 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause /* This file manages homa_socktab objects; it also implements several * operations on homa_sock objects, such as construction and destruction. @@ -15,10 +13,10 @@ void homa_socktab_init(struct homa_socktab *socktab) { int i; + spin_lock_init(&socktab->write_lock); - for (i = 0; i < HOMA_SOCKTAB_BUCKETS; i++) { + for (i = 0; i < HOMA_SOCKTAB_BUCKETS; i++) INIT_HLIST_HEAD(&socktab->buckets[i]); - } } /** @@ -78,6 +76,7 @@ struct homa_sock *homa_socktab_next(struct homa_socktab_scan *scan) { struct homa_sock *hsk; struct homa_socktab_links *links; + while (1) { while (scan->next == NULL) { scan->current_bucket++; @@ -118,12 +117,10 @@ void homa_sock_init(struct homa_sock *hsk, struct homa *homa) ? HOMA_IPV4_HEADER_LENGTH : HOMA_IPV6_HEADER_LENGTH; hsk->shutdown = false; while (1) { - if (homa->next_client_port < HOMA_MIN_DEFAULT_PORT) { + if (homa->next_client_port < HOMA_MIN_DEFAULT_PORT) homa->next_client_port = HOMA_MIN_DEFAULT_PORT; - } - if (!homa_sock_find(socktab, homa->next_client_port)) { + if (!homa_sock_find(socktab, homa->next_client_port)) break; - } homa->next_client_port++; } hsk->port = homa->next_client_port; @@ -143,12 +140,14 @@ void homa_sock_init(struct homa_sock *hsk, struct homa *homa) INIT_LIST_HEAD(&hsk->response_interests); for (i = 0; i < HOMA_CLIENT_RPC_BUCKETS; i++) { struct homa_rpc_bucket *bucket = &hsk->client_rpc_buckets[i]; + spin_lock_init(&bucket->lock); INIT_HLIST_HEAD(&bucket->rpcs); bucket->id = i; } for (i = 0; i < HOMA_SERVER_RPC_BUCKETS; i++) { struct homa_rpc_bucket *bucket = &hsk->server_rpc_buckets[i]; + spin_lock_init(&bucket->lock); INIT_HLIST_HEAD(&bucket->rpcs); bucket->id = i + 1000000; @@ -251,9 +250,8 @@ int homa_sock_bind(struct homa_socktab *socktab, struct homa_sock *hsk, if (port == 0) return result; - if (port >= HOMA_MIN_DEFAULT_PORT) { + if (port >= HOMA_MIN_DEFAULT_PORT) return -EINVAL; - } homa_sock_lock(hsk, "homa_sock_bind"); spin_lock_bh(&socktab->write_lock); if (hsk->shutdown) { @@ -273,7 +271,7 @@ int homa_sock_bind(struct homa_socktab *socktab, struct homa_sock *hsk, hsk->inet.inet_sport = htons(hsk->port); hlist_add_head_rcu(&hsk->socktab_links.hash_links, &socktab->buckets[homa_port_hash(port)]); - done: +done: spin_unlock_bh(&socktab->write_lock); homa_sock_unlock(hsk); return result; @@ -293,9 +291,11 @@ struct homa_sock *homa_sock_find(struct homa_socktab *socktab, __u16 port) { struct homa_socktab_links *link; struct homa_sock *result = NULL; + hlist_for_each_entry_rcu(link, &socktab->buckets[homa_port_hash(port)], hash_links) { struct homa_sock *hsk = link->sock; + if (hsk->port == port) { result = hsk; break; @@ -314,6 +314,7 @@ struct homa_sock *homa_sock_find(struct homa_socktab *socktab, __u16 port) void homa_sock_lock_slow(struct homa_sock *hsk) { __u64 start = get_cycles(); + tt_record("beginning wait for socket lock"); spin_lock_bh(&hsk->lock); tt_record("ending wait for socket lock"); diff --git a/homa_timer.c b/homa_timer.c index 9154016..f6f4b55 100644 --- a/homa_timer.c +++ b/homa_timer.c @@ -1,9 +1,8 @@ -/* Copyright (c) 2019-2023 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause /* This file handles timing-related functions for Homa, such as retries - * and timeouts. */ + * and timeouts. + */ #include "homa_impl.h" @@ -31,9 +30,9 @@ void homa_check_rpc(struct homa_rpc *rpc) if ((rpc->done_timer_ticks + homa->request_ack_ticks - 1 - homa->timer_ticks) & 1<<31) { struct need_ack_header h; + homa_xmit_control(NEED_ACK, &h, sizeof(h), rpc); - tt_record4("Sent NEED_ACK for RPC id %d to " - "peer 0x%x, port %d, ticks %d", + tt_record4("Sent NEED_ACK for RPC id %d to peer 0x%x, port %d, ticks %d", rpc->id, tt_addr(rpc->peer->addr), rpc->dport, homa->timer_ticks @@ -78,16 +77,14 @@ void homa_check_rpc(struct homa_rpc *rpc) return; if (rpc->silent_ticks >= homa->timeout_ticks) { INC_METRIC(rpc_timeouts, 1); - tt_record3("RPC id %d, peer 0x%x, aborted because of timeout, " - "state %d", + tt_record3("RPC id %d, peer 0x%x, aborted because of timeout, state %d", rpc->id, tt_addr(rpc->peer->addr), rpc->state); homa_rpc_log_active_tt(homa, 0); tt_record1("Freezing because of RPC abort (id %d)", rpc->id); homa_freeze_peers(homa); tt_freeze(); if (homa->verbose) - printk(KERN_NOTICE "RPC id %llu, peer %s, aborted " - "because of timeout, state %d\n", + pr_notice("RPC id %llu, peer %s, aborted because of timeout, state %d\n", rpc->id, homa_print_ipv6_addr(&rpc->peer->addr), rpc->state); @@ -121,8 +118,7 @@ void homa_check_rpc(struct homa_rpc *rpc) if (homa_is_client(rpc->id)) { us = "client"; them = "server"; - tt_record4("Sent RESEND for client RPC id %llu, server 0x%x:%d, " - "offset %d", + tt_record4("Sent RESEND for client RPC id %llu, server 0x%x:%d, offset %d", rpc->id, tt_addr(rpc->peer->addr), rpc->dport, rpc->msgin.recv_end); tt_record4("length %d, granted %d, rem %d, rec_incoming %d", @@ -132,8 +128,7 @@ void homa_check_rpc(struct homa_rpc *rpc) } else { us = "server"; them = "client"; - tt_record4("Sent RESEND for server RPC id %llu, client 0x%x:%d " - "offset %d", + tt_record4("Sent RESEND for server RPC id %llu, client 0x%x:%d offset %d", rpc->id, tt_addr(rpc->peer->addr), rpc->dport, rpc->msgin.recv_end); tt_record4("length %d, granted %d, rem %d, rec_incoming %d", @@ -142,8 +137,7 @@ void homa_check_rpc(struct homa_rpc *rpc) rpc->msgin.rec_incoming); } if (homa->verbose) - printk(KERN_NOTICE "Homa %s RESEND to %s %s:%d for id %llu, " - "offset %d, length %d\n", us, them, + pr_notice("Homa %s RESEND to %s %s:%d for id %llu, offset %d, length %d\n", us, them, homa_print_ipv6_addr(&rpc->peer->addr), rpc->dport, rpc->id, rpc->msgin.recv_end, rpc->msgin.granted - rpc->msgin.recv_end); @@ -165,8 +159,8 @@ void homa_timer(struct homa *homa) int total_incoming_rpcs = 0; int sum_incoming = 0; int sum_incoming_rec = 0; - static __u64 prev_grant_count = 0; - static int zero_count = 0; + static __u64 prev_grant_count; + static int zero_count; int core; __u64 total_grants; @@ -176,11 +170,11 @@ void homa_timer(struct homa *homa) total_grants = 0; for (core = 0; core < nr_cpu_ids; core++) { struct homa_metrics *m = &homa_cores[core]->metrics; + total_grants += m->packets_sent[GRANT-DATA]; } - tt_record4("homa_timer found total_incoming %d, num_grantable_rpcs %d, " - "num_active_rpcs %d, new grants %d", + tt_record4("homa_timer found total_incoming %d, num_grantable_rpcs %d, num_active_rpcs %d, new grants %d", atomic_read(&homa->total_incoming), homa->num_grantable_rpcs, homa->num_active_rpcs, @@ -189,7 +183,7 @@ void homa_timer(struct homa *homa) && (homa->num_grantable_rpcs > 20)) { zero_count++; if ((zero_count > 3) && !tt_frozen && 0) { - printk(KERN_ERR "homa timer found no grants going out\n"); + pr_err("%s found no grants going out\n", __func__); homa_rpc_log_active_tt(homa, 0); tt_record("freezing because no grants are going out"); homa_freeze_peers(homa); @@ -208,8 +202,10 @@ void homa_timer(struct homa *homa) while (hsk->dead_skbs >= homa->dead_buffs_limit) { /* If we get here, it means that homa_wait_for_message * isn't keeping up with RPC reaping, so we'll help - * out. See reap.txt for more info. */ + * out. See reap.txt for more info. + */ uint64_t start = get_cycles(); + tt_record("homa_timer calling homa_rpc_reap"); if (homa_rpc_reap(hsk, hsk->homa->reap_limit) == 0) break; @@ -253,8 +249,7 @@ void homa_timer(struct homa *homa) homa_unprotect_rpcs(hsk); } rcu_read_unlock(); - tt_record4("homa_timer found %d incoming RPCs, incoming sum %d, " - "rec_sum %d, homa->total_incoming %d", + tt_record4("homa_timer found %d incoming RPCs, incoming sum %d, rec_sum %d, homa->total_incoming %d", total_incoming_rpcs, sum_incoming, sum_incoming_rec, atomic_read(&homa->total_incoming)); diff --git a/homa_utils.c b/homa_utils.c index cfa1e02..df9cd0b 100644 --- a/homa_utils.c +++ b/homa_utils.c @@ -1,6 +1,4 @@ -/* Copyright (c) 2019-2023 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause /* This file contains miscellaneous utility functions for the Homa protocol. */ @@ -15,7 +13,7 @@ struct homa_core *homa_cores[NR_CPUS]; struct homa_numa *homa_numas[MAX_NUMNODES]; /* Total number of NUMA nodes actually defined in homa_numas. */ -int homa_num_numas = 0; +int homa_num_numas; /* Points to block of memory holding all homa_cores; used to free it. */ char *core_memory; @@ -35,6 +33,7 @@ int homa_init(struct homa *homa) size_t aligned_size; char *first; int i, err, num_numas; + _Static_assert(HOMA_MAX_PRIORITIES >= 8, "homa_init assumes at least 8 priority levels"); @@ -44,6 +43,7 @@ int homa_init(struct homa *homa) for (i = 0; i < nr_cpu_ids; i++) { struct homa_numa *numa; int n = cpu_to_node(i); + if (homa_numas[n]) continue; numa = kmalloc(sizeof(struct homa_numa), GFP_KERNEL); @@ -53,8 +53,7 @@ int homa_init(struct homa *homa) homa_num_numas = n+1; num_numas++; } - printk(KERN_NOTICE "Homa initialized %d homa_numas, highest " - "number %d\n", num_numas, homa_num_numas-1); + pr_notice("Homa initialized %d homa_numas, highest number %d\n", num_numas, homa_num_numas-1); /* Initialize core-specific info (if no-one else has already done it), * making sure that each core has private cache lines. @@ -63,8 +62,7 @@ int homa_init(struct homa *homa) aligned_size = (sizeof(struct homa_core) + 0x3f) & ~0x3f; core_memory = vmalloc(0x3f + (nr_cpu_ids*aligned_size)); if (!core_memory) { - printk(KERN_ERR "Homa couldn't allocate memory " - "for core-specific data\n"); + pr_err("Homa couldn't allocate memory for core-specific data\n"); return -ENOMEM; } first = (char *) (((__u64) core_memory + 0x3f) & ~0x3f); @@ -127,8 +125,7 @@ int homa_init(struct homa *homa) homa_socktab_init(&homa->port_map); err = homa_peertab_init(&homa->peers); if (err) { - printk(KERN_ERR "Couldn't initialize peer table (errno %d)\n", - -err); + pr_err("Couldn't initialize peer table (errno %d)\n", -err); return err; } spin_lock_init(&homa->page_pool_mutex); @@ -177,8 +174,7 @@ int homa_init(struct homa *homa) if (IS_ERR(homa->pacer_kthread)) { err = PTR_ERR(homa->pacer_kthread); homa->pacer_kthread = NULL; - printk(KERN_ERR "couldn't create homa pacer thread: error %d\n", - err); + pr_err("couldn't create homa pacer thread: error %d\n", err); return err; } homa->pacer_exit = false; @@ -214,6 +210,7 @@ int homa_init(struct homa *homa) void homa_destroy(struct homa *homa) { int i; + if (homa->pacer_kthread) { homa_pacer_stop(homa); wait_for_completion(&homa_pacer_kthread_done); @@ -226,6 +223,7 @@ void homa_destroy(struct homa *homa) for (i = 0; i < MAX_NUMNODES; i++) { struct homa_numa *numa = homa_numas[i]; + if (numa != NULL) { kfree(numa); homa_numas[i] = NULL; @@ -234,12 +232,10 @@ void homa_destroy(struct homa *homa) if (core_memory) { vfree(core_memory); core_memory = NULL; - for (i = 0; i < nr_cpu_ids; i++) { + for (i = 0; i < nr_cpu_ids; i++) homa_cores[i] = NULL; - } } - if (homa->metrics) - kfree(homa->metrics); + kfree(homa->metrics); } /** @@ -254,14 +250,14 @@ void homa_destroy(struct homa *homa) * caller must eventually unlock it. */ struct homa_rpc *homa_rpc_new_client(struct homa_sock *hsk, - const sockaddr_in_union *dest) + const union sockaddr_in_union *dest) { int err; struct homa_rpc *crpc; struct homa_rpc_bucket *bucket; struct in6_addr dest_addr_as_ipv6 = canonical_ipv6_addr(dest); - crpc = (struct homa_rpc *) kmalloc(sizeof(*crpc), GFP_KERNEL); + crpc = kmalloc(sizeof(*crpc), GFP_KERNEL); if (unlikely(!crpc)) return ERR_PTR(-ENOMEM); @@ -275,7 +271,7 @@ struct homa_rpc *homa_rpc_new_client(struct homa_sock *hsk, atomic_set(&crpc->grants_in_progress, 0); crpc->peer = homa_peer_find(&hsk->homa->peers, &dest_addr_as_ipv6, &hsk->inet); - if (unlikely(IS_ERR(crpc->peer))) { + if (IS_ERR(crpc->peer)) { tt_record("error in homa_peer_find"); err = PTR_ERR(crpc->peer); goto error; @@ -365,7 +361,7 @@ struct homa_rpc *homa_rpc_new_server(struct homa_sock *hsk, } /* Initialize fields that don't require the socket lock. */ - srpc = (struct homa_rpc *) kmalloc(sizeof(*srpc), GFP_KERNEL); + srpc = kmalloc(sizeof(*srpc), GFP_KERNEL); if (!srpc) { err = -ENOMEM; goto error; @@ -376,7 +372,7 @@ struct homa_rpc *homa_rpc_new_server(struct homa_sock *hsk, atomic_set(&srpc->flags, 0); atomic_set(&srpc->grants_in_progress, 0); srpc->peer = homa_peer_find(&hsk->homa->peers, source, &hsk->inet); - if (unlikely(IS_ERR(srpc->peer))) { + if (IS_ERR(srpc->peer)) { err = PTR_ERR(srpc->peer); goto error; } @@ -426,8 +422,7 @@ struct homa_rpc *homa_rpc_new_server(struct homa_sock *hsk, error: homa_bucket_unlock(bucket, id); - if (srpc) - kfree(srpc); + kfree(srpc); return ERR_PTR(err); } @@ -443,6 +438,7 @@ struct homa_rpc *homa_rpc_new_server(struct homa_sock *hsk, void homa_bucket_lock_slow(struct homa_rpc_bucket *bucket, __u64 id) { __u64 start = get_cycles(); + tt_record2("beginning wait for rpc lock, id %d (bucket %d)", id, bucket->id); spin_lock_bh(&bucket->lock); @@ -492,7 +488,7 @@ void homa_rpc_acked(struct homa_sock *hsk, const struct in6_addr *saddr, homa_rpc_unlock(rpc); } - done: +done: if (hsk->port != server_port) rcu_read_unlock(); } @@ -553,9 +549,8 @@ void homa_rpc_free(struct homa_rpc *rpc) while (1) { struct homa_gap *gap = list_first_entry_or_null( &rpc->msgin.gaps, struct homa_gap, links); - if (gap == NULL) { + if (gap == NULL) break; - } list_del(&gap->links); kfree(gap); } @@ -619,8 +614,7 @@ int homa_rpc_reap(struct homa_sock *hsk, int count) homa_sock_lock(hsk, "homa_rpc_reap"); if (atomic_read(&hsk->protect_count)) { INC_METRIC(disabled_reaps, 1); - tt_record2("homa_rpc_reap returning: protect_count " - "%d, dead_skbs %d", + tt_record2("homa_rpc_reap returning: protect_count %d, dead_skbs %d", atomic_read(&hsk->protect_count), hsk->dead_skbs); homa_sock_unlock(hsk); @@ -663,6 +657,7 @@ int homa_rpc_reap(struct homa_sock *hsk, int count) if (rpc->msgin.length >= 0) { while (1) { struct sk_buff *skb; + skb = skb_dequeue(&rpc->msgin.packets); if (!skb) break; @@ -684,7 +679,7 @@ int homa_rpc_reap(struct homa_sock *hsk, int count) /* Free all of the collected resources; release the socket * lock while doing this. */ - release: +release: hsk->dead_skbs -= num_skbs + rx_frees; result = !list_empty(&hsk->dead_rpcs) && ((num_skbs + num_rpcs) != 0); @@ -713,9 +708,8 @@ int homa_rpc_reap(struct homa_sock *hsk, int count) struct homa_gap *gap = list_first_entry_or_null( &rpc->msgin.gaps, struct homa_gap, links); - if (gap == NULL) { + if (gap == NULL) break; - } list_del(&gap->links); kfree(gap); } @@ -749,7 +743,8 @@ struct homa_rpc *homa_find_client_rpc(struct homa_sock *hsk, __u64 id) { struct homa_rpc *crpc; struct homa_rpc_bucket *bucket = homa_client_rpc_bucket(hsk, id); - homa_bucket_lock(bucket, id, "homa_find_client_rpc"); + + homa_bucket_lock(bucket, id, __func__); hlist_for_each_entry_rcu(crpc, &bucket->rpcs, hash_links) { if (crpc->id == id) return crpc; @@ -775,7 +770,8 @@ struct homa_rpc *homa_find_server_rpc(struct homa_sock *hsk, { struct homa_rpc *srpc; struct homa_rpc_bucket *bucket = homa_server_rpc_bucket(hsk, id); - homa_bucket_lock(bucket, id, "homa_find_server_rpc"); + + homa_bucket_lock(bucket, id, __func__); hlist_for_each_entry_rcu(srpc, &bucket->rpcs, hash_links) { if ((srpc->id == id) && (srpc->dport == sport) && ipv6_addr_equal(&srpc->peer->addr, saddr)) @@ -796,16 +792,13 @@ void homa_rpc_log(struct homa_rpc *rpc) char *peer = homa_print_ipv6_addr(&rpc->peer->addr); if (rpc->state == RPC_INCOMING) - printk(KERN_NOTICE "%s RPC INCOMING, id %llu, peer %s:%d, " - "%d/%d bytes received, incoming %d\n", + pr_notice("%s RPC INCOMING, id %llu, peer %s:%d, %d/%d bytes received, incoming %d\n", type, rpc->id, peer, rpc->dport, rpc->msgin.length - rpc->msgin.bytes_remaining, rpc->msgin.length, rpc->msgin.granted); else if (rpc->state == RPC_OUTGOING) { - printk(KERN_NOTICE "%s RPC OUTGOING, id %llu, peer %s:%d, " - "out length %d, left %d, granted %d, " - "in left %d, resend_ticks %u, silent_ticks %d\n", + pr_notice("%s RPC OUTGOING, id %llu, peer %s:%d, out length %d, left %d, granted %d, in left %d, resend_ticks %u, silent_ticks %d\n", type, rpc->id, peer, rpc->dport, rpc->msgout.length, rpc->msgout.length - rpc->msgout.next_xmit_offset, @@ -814,8 +807,7 @@ void homa_rpc_log(struct homa_rpc *rpc) rpc->resend_timer_ticks, rpc->silent_ticks); } else { - printk(KERN_NOTICE "%s RPC %s, id %llu, peer %s:%d, " - "incoming length %d, outgoing length %d\n", + pr_notice("%s RPC %s, id %llu, peer %s:%d, incoming length %d, outgoing length %d\n", type, homa_symbol_for_state(rpc), rpc->id, peer, rpc->dport, rpc->msgin.length, rpc->msgout.length); @@ -836,7 +828,7 @@ void homa_rpc_log_active(struct homa *homa, uint64_t id) struct homa_rpc *rpc; int count = 0; - printk("Logging active Homa RPCs:\n"); + pr_notice("Logging active Homa RPCs:\n"); rcu_read_lock(); for (hsk = homa_socktab_start_scan(&homa->port_map, &scan); hsk != NULL; hsk = homa_socktab_next(&scan)) { @@ -854,7 +846,7 @@ void homa_rpc_log_active(struct homa *homa, uint64_t id) homa_unprotect_rpcs(hsk); } rcu_read_unlock(); - printk("Finished logging active Homa RPCs: %d active RPCs\n", count); + pr_notice("Finished logging active Homa RPCs: %d active RPCs\n", count); } /** @@ -866,13 +858,11 @@ void homa_rpc_log_tt(struct homa_rpc *rpc) if (rpc->state == RPC_INCOMING) { int received = rpc->msgin.length - rpc->msgin.bytes_remaining; - tt_record4("Incoming RPC id %d, peer 0x%x, %d/%d bytes " - "received", + tt_record4("Incoming RPC id %d, peer 0x%x, %d/%d bytes received", rpc->id, tt_addr(rpc->peer->addr), received, rpc->msgin.length); if (1) - tt_record4("RPC id %d has incoming %d, " - "granted %d, prio %d", rpc->id, + tt_record4("RPC id %d has incoming %d, granted %d, prio %d", rpc->id, rpc->msgin.granted - received, rpc->msgin.granted, rpc->msgin.priority); tt_record4("RPC id %d: length %d, remaining %d, rank %d", @@ -886,8 +876,7 @@ void homa_rpc_log_tt(struct homa_rpc *rpc) tt_record2("RPC id %d has %d bpages allocated", rpc->id, rpc->msgin.num_bpages); } else if (rpc->state == RPC_OUTGOING) { - tt_record4("Outgoing RPC id %d, peer 0x%x, %d/%d bytes " - "sent", + tt_record4("Outgoing RPC id %d, peer 0x%x, %d/%d bytes sent", rpc->id, tt_addr(rpc->peer->addr), rpc->msgout.next_xmit_offset, rpc->msgout.length); @@ -927,18 +916,18 @@ void homa_rpc_log_active_tt(struct homa *homa, int freeze_count) continue; list_for_each_entry_rcu(rpc, &hsk->active_rpcs, active_links) { struct freeze_header freeze; + count++; homa_rpc_log_tt(rpc); - if (freeze_count == 0) { + if (freeze_count == 0) continue; - } if (rpc->state != RPC_INCOMING) continue; if (rpc->msgin.granted <= (rpc->msgin.length - rpc->msgin.bytes_remaining)) continue; freeze_count--; - printk(KERN_NOTICE "Emitting FREEZE in homa_rpc_log_active_tt\n"); + pr_notice("Emitting FREEZE in %s\n", __func__); homa_xmit_control(FREEZE, &freeze, sizeof(freeze), rpc); } homa_unprotect_rpcs(hsk); @@ -979,6 +968,7 @@ int homa_validate_incoming(struct homa *homa, int verbose, int *link_errors) continue; list_for_each_entry_rcu(rpc, &hsk->active_rpcs, active_links) { int incoming; + if (rpc->state != RPC_INCOMING) continue; incoming = rpc->msgin.granted - @@ -990,23 +980,19 @@ int homa_validate_incoming(struct homa *homa, int verbose, int *link_errors) continue; total_incoming += rpc->msgin.rec_incoming; if (verbose) - tt_record3("homa_validate_incoming: RPC id %d, " - "incoming %d, " - "rec_incoming %d", + tt_record3("homa_validate_incoming: RPC id %d, ncoming %d, rec_incoming %d", rpc->id, incoming, rpc->msgin.rec_incoming); if (rpc->msgin.granted >= rpc->msgin.length) continue; if (list_empty(&rpc->grantable_links)) { - tt_record1("homa_validate_incoming: RPC id %d " - "not linked in grantable list", + tt_record1("homa_validate_incoming: RPC id %d not linked in grantable list", rpc->id); *link_errors = 1; } if (list_empty(&rpc->grantable_links)) { - tt_record1("homa_validate_incoming: RPC id %d " - "peer not linked in grantable list", - rpc->id);\ + tt_record1("homa_validate_incoming: RPC id %d peer not linked in grantable list", + rpc->id); *link_errors = 1; } } @@ -1038,9 +1024,10 @@ char *homa_print_ipv4_addr(__be32 addr) #define NUM_BUFS_IPV4 4 #define BUF_SIZE_IPV4 30 static char buffers[NUM_BUFS_IPV4][BUF_SIZE_IPV4]; - static int next_buf = 0; + static int next_buf; __u32 a2 = ntohl(addr); char *buffer = buffers[next_buf]; + next_buf++; if (next_buf >= NUM_BUFS_IPV4) next_buf = 0; @@ -1064,23 +1051,27 @@ char *homa_print_ipv6_addr(const struct in6_addr *addr) #define NUM_BUFS (1 << 2) #define BUF_SIZE 64 static char buffers[NUM_BUFS][BUF_SIZE]; - static int next_buf = 0; + static int next_buf; char *buffer = buffers[next_buf]; + next_buf++; if (next_buf >= NUM_BUFS) next_buf = 0; #ifdef __UNIT_TEST__ struct in6_addr zero = {}; + if (ipv6_addr_equal(addr, &zero)) { snprintf(buffer, BUF_SIZE, "0.0.0.0"); } else if ((addr->s6_addr32[0] == 0) && (addr->s6_addr32[1] == 0) && (addr->s6_addr32[2] == htonl(0x0000ffff))) { __u32 a2 = ntohl(addr->s6_addr32[3]); + snprintf(buffer, BUF_SIZE, "%u.%u.%u.%u", (a2 >> 24) & 0xff, (a2 >> 16) & 0xff, (a2 >> 8) & 0xff, a2 & 0xff); } else { - const char *inet_ntop(int, const void *, char *, size_t); + const char *inet_ntop(int af, const void *src, char *dst, + size_t size); inet_ntop(AF_INET6, addr, buffer + 1, BUF_SIZE); buffer[0] = '['; strcat(buffer, "]"); @@ -1127,6 +1118,7 @@ char *homa_print_packet(struct sk_buff *skb, char *buffer, int buf_len) struct data_header *h = (struct data_header *) header; struct homa_skb_info *homa_info = homa_get_skb_info(skb); int data_left, i, seg_length, pos, offset; + if (skb_shinfo(skb)->gso_segs == 0) { seg_length = homa_data_len(skb); data_left = 0; @@ -1140,8 +1132,7 @@ char *homa_print_packet(struct sk_buff *skb, char *buffer, int buf_len) if (offset == -1) offset = ntohl(h->common.sequence); used = homa_snprintf(buffer, buf_len, used, - ", message_length %d, offset %d, " - "data_length %d, incoming %d", + ", message_length %d, offset %d, data_length %d, incoming %d", ntohl(h->message_length), offset, seg_length, ntohl(h->incoming)); if (ntohs(h->cutoff_version != 0)) @@ -1161,6 +1152,7 @@ char *homa_print_packet(struct sk_buff *skb, char *buffer, int buf_len) for (i = skb_shinfo(skb)->gso_segs - 1; i > 0; i--) { if (homa_info->seg_length < skb_shinfo(skb)->gso_size) { struct seg_header seg; + homa_skb_get(skb, &seg, pos, sizeof(seg)); offset = ntohl(seg.offset); } else { @@ -1178,6 +1170,7 @@ char *homa_print_packet(struct sk_buff *skb, char *buffer, int buf_len) case GRANT: { struct grant_header *h = (struct grant_header *) header; char *resend = (h->resend_all) ? ", resend_all" : ""; + used = homa_snprintf(buffer, buf_len, used, ", offset %d, grant_prio %u%s", ntohl(h->offset), h->priority, resend); @@ -1185,6 +1178,7 @@ char *homa_print_packet(struct sk_buff *skb, char *buffer, int buf_len) } case RESEND: { struct resend_header *h = (struct resend_header *) header; + used = homa_snprintf(buffer, buf_len, used, ", offset %d, length %d, resend_prio %u", ntohl(h->offset), ntohl(h->length), @@ -1199,6 +1193,7 @@ char *homa_print_packet(struct sk_buff *skb, char *buffer, int buf_len) break; case CUTOFFS: { struct cutoffs_header *h = (struct cutoffs_header *) header; + used = homa_snprintf(buffer, buf_len, used, ", cutoffs %d %d %d %d %d %d %d %d, version %u", ntohl(h->unsched_cutoffs[0]), @@ -1221,6 +1216,7 @@ char *homa_print_packet(struct sk_buff *skb, char *buffer, int buf_len) case ACK: { struct ack_header *h = (struct ack_header *) header; int i, count; + count = ntohs(h->num_acks); used = homa_snprintf(buffer, buf_len, used, ", acks"); for (i = 0; i < count; i++) { @@ -1278,6 +1274,7 @@ char *homa_print_packet_short(struct sk_buff *skb, char *buffer, int buf_len) for (i = skb_shinfo(skb)->gso_segs - 1; i > 0; i--) { if (homa_info->seg_length < skb_shinfo(skb)->gso_size) { struct seg_header seg; + homa_skb_get(skb, &seg, pos, sizeof(seg)); offset = ntohl(seg.offset); } else { @@ -1295,12 +1292,14 @@ char *homa_print_packet_short(struct sk_buff *skb, char *buffer, int buf_len) case GRANT: { struct grant_header *h = (struct grant_header *) header; char *resend = h->resend_all ? " resend_all" : ""; + snprintf(buffer, buf_len, "GRANT %d@%d%s", ntohl(h->offset), h->priority, resend); break; } case RESEND: { struct resend_header *h = (struct resend_header *) header; + snprintf(buffer, buf_len, "RESEND %d-%d@%d", ntohl(h->offset), ntohl(h->offset) + ntohl(h->length) - 1, h->priority); @@ -1321,7 +1320,6 @@ char *homa_print_packet_short(struct sk_buff *skb, char *buffer, int buf_len) case NEED_ACK: snprintf(buffer, buf_len, "NEED_ACK"); break; - break; case ACK: snprintf(buffer, buf_len, "ACK"); break; @@ -1358,7 +1356,7 @@ void homa_freeze_peers(struct homa *homa) return; } freeze.common.type = FREEZE; - freeze.common.sport = htons(hsk->port);; + freeze.common.sport = htons(hsk->port); freeze.common.dport = 0; freeze.common.flags = HOMA_TCP_FLAGS; freeze.common.urgent = htons(HOMA_TCP_URGENT); @@ -1367,8 +1365,7 @@ void homa_freeze_peers(struct homa *homa) tt_record1("Sending freeze to 0x%x", tt_addr(peers[i]->addr)); err = __homa_xmit_control(&freeze, sizeof(freeze), peers[i], hsk); if (err != 0) - tt_record2("homa_freeze_peers got error %d in xmit " - "to 0x%x\n", err, + tt_record2("homa_freeze_peers got error %d in xmit to 0x%x\n", err, tt_addr(peers[i]->addr)); } kfree(peers); @@ -1392,11 +1389,11 @@ void homa_freeze_peers(struct homa *homa) * Return: The number of characters now occupied in @buffer, not * including the terminating null character. */ -int homa_snprintf(char *buffer, int size, int used, const char* format, ...) +int homa_snprintf(char *buffer, int size, int used, const char *format, ...) { int new_chars; - va_list ap; + va_start(ap, format); if (used >= (size-1)) @@ -1420,6 +1417,7 @@ int homa_snprintf(char *buffer, int size, int used, const char* format, ...) char *homa_symbol_for_state(struct homa_rpc *rpc) { static char buffer[20]; + switch (rpc->state) { case RPC_OUTGOING: return "OUTGOING"; @@ -1446,6 +1444,7 @@ char *homa_symbol_for_state(struct homa_rpc *rpc) char *homa_symbol_for_type(uint8_t type) { static char buffer[20]; + switch (type) { case DATA: return "DATA"; @@ -1471,7 +1470,8 @@ char *homa_symbol_for_type(uint8_t type) * but (a) it's unlikely (this code only executes if the opcode is * bogus), (b) this is mostly for testing and debugging, and (c) the * code below ensures that the string cannot run past the end of the - * buffer, so the code is safe. */ + * buffer, so the code is safe. + */ snprintf(buffer, sizeof(buffer)-1, "unknown(%u)", type); buffer[sizeof(buffer)-1] = 0; return buffer; @@ -1485,7 +1485,7 @@ char *homa_symbol_for_type(uint8_t type) * new metric. Arguments after this provide the usual * values expected for printf-like functions. */ -void homa_append_metric(struct homa *homa, const char* format, ...) +void homa_append_metric(struct homa *homa, const char *format, ...) { char *new_buffer; size_t new_chars; @@ -1499,8 +1499,7 @@ void homa_append_metric(struct homa *homa, const char* format, ...) #endif homa->metrics = kmalloc(homa->metrics_capacity, GFP_KERNEL); if (!homa->metrics) { - printk(KERN_WARNING "homa_append_metric couldn't " - "allocate memory\n"); + pr_warn("%s couldn't allocate memory\n", __func__); return; } homa->metrics_length = 0; @@ -1523,8 +1522,7 @@ void homa_append_metric(struct homa *homa, const char* format, ...) homa->metrics_capacity *= 2; new_buffer = kmalloc(homa->metrics_capacity, GFP_KERNEL); if (!new_buffer) { - printk(KERN_WARNING "homa_append_metric couldn't " - "allocate memory\n"); + pr_warn("%s couldn't allocate memory\n", __func__); return; } memcpy(new_buffer, homa->metrics, homa->metrics_length); @@ -1548,25 +1546,21 @@ char *homa_print_metrics(struct homa *homa) homa->metrics_length = 0; homa_append_metric(homa, - "rdtsc_cycles %20llu " - "RDTSC cycle counter when metrics were gathered\n", + "rdtsc_cycles %20llu RDTSC cycle counter when metrics were gathered\n", get_cycles()); homa_append_metric(homa, - "cpu_khz %15llu " - "Clock rate for RDTSC counter, in khz\n", + "cpu_khz %15llu Clock rate for RDTSC counter, in khz\n", cpu_khz); for (core = 0; core < nr_cpu_ids; core++) { struct homa_metrics *m = &homa_cores[core]->metrics; __s64 delta; + homa_append_metric(homa, - "core %15d " - "Core id for following metrics\n", + "core %15d Core id for following metrics\n", core); for (i = 0; i < HOMA_NUM_SMALL_COUNTS; i++) { homa_append_metric(homa, - "msg_bytes_%-9d %15llu " - "Bytes in incoming messages containing " - "%d-%d bytes\n", + "msg_bytes_%-9d %15llu Bytes in incoming messages containing %d-%d bytes\n", (i+1)*64, m->small_msg_bytes[i], lower, (i+1)*64); lower = (i+1)*64 + 1; @@ -1574,145 +1568,111 @@ char *homa_print_metrics(struct homa *homa) for (i = (HOMA_NUM_SMALL_COUNTS*64)/1024; i < HOMA_NUM_MEDIUM_COUNTS; i++) { homa_append_metric(homa, - "msg_bytes_%-9d %15llu " - "Bytes in incoming messages containing " - "%d-%d bytes\n", + "msg_bytes_%-9d %15llu Bytes in incoming messages containing %d-%d bytes\n", (i+1)*1024, m->medium_msg_bytes[i], lower, (i+1)*1024); lower = (i+1)*1024 + 1; } homa_append_metric(homa, - "large_msg_count %15llu " - "# of incoming messages >= %d bytes\n", + "large_msg_count %15llu # of incoming messages >= %d bytes\n", m->large_msg_count, lower); homa_append_metric(homa, - "large_msg_bytes %15llu " - "Bytes in incoming messages >= %d bytes\n", + "large_msg_bytes %15llu Bytes in incoming messages >= %d bytes\n", m->large_msg_bytes, lower); homa_append_metric(homa, - "sent_msg_bytes %15llu " - "Total bytes in all outgoing messages\n", + "sent_msg_bytes %15llu otal bytes in all outgoing messages\n", m->sent_msg_bytes); for (i = DATA; i < BOGUS; i++) { char *symbol = homa_symbol_for_type(i); + homa_append_metric(homa, - "packets_sent_%-7s %15llu " - "%s packets sent\n", + "packets_sent_%-7s %15llu %s packets sent\n", symbol, m->packets_sent[i-DATA], symbol); } for (i = DATA; i < BOGUS; i++) { char *symbol = homa_symbol_for_type(i); + homa_append_metric(homa, - "packets_rcvd_%-7s %15llu " - "%s packets received\n", + "packets_rcvd_%-7s %15llu %s packets received\n", symbol, m->packets_received[i-DATA], symbol); } for (i = 0; i < HOMA_MAX_PRIORITIES; i++) { homa_append_metric(homa, - "priority%d_bytes %15llu " - "Bytes sent at priority %d " - "(including headers)\n", + "priority%d_bytes %15llu Bytes sent at priority %d (including headers)\n", i, m->priority_bytes[i], i); } for (i = 0; i < HOMA_MAX_PRIORITIES; i++) { homa_append_metric(homa, - "priority%d_packets %15llu " - "Packets sent at priority %d\n", + "priority%d_packets %15llu Packets sent at priority %d\n", i, m->priority_packets[i], i); } homa_append_metric(homa, - "skb_allocs %15llu " - "sk_buffs allocated\n", + "skb_allocs %15llu sk_buffs allocated\n", m->skb_allocs); homa_append_metric(homa, - "skb_alloc_cycles %15llu " - "Time spent allocating sk_buffs\n", + "skb_alloc_cycles %15llu Time spent allocating sk_buffs\n", m->skb_alloc_cycles); homa_append_metric(homa, - "skb_frees %15llu " - "Data sk_buffs freed in normal paths\n", + "skb_frees %15llu Data sk_buffs freed in normal paths\n", m->skb_frees); homa_append_metric(homa, - "skb_free_cycles %15llu " - "Time spent freeing data sk_buffs\n", + "skb_free_cycles %15llu Time spent freeing data sk_buffs\n", m->skb_free_cycles); homa_append_metric(homa, - "skb_page_allocs %15llu " - "Pages allocated for sk_buff frags\n", + "skb_page_allocs %15llu Pages allocated for sk_buff frags\n", m->skb_page_allocs); homa_append_metric(homa, - "skb_page_alloc_cycles %15llu " - "Time spent allocating pages for sk_buff frags\n", + "skb_page_alloc_cycles %15llu Time spent allocating pages for sk_buff frags\n", m->skb_page_alloc_cycles); homa_append_metric(homa, - "requests_received %15llu " - "Incoming request messages\n", + "requests_received %15llu Incoming request messages\n", m->requests_received); homa_append_metric(homa, - "requests_queued %15llu " - "Requests for which no thread was waiting\n", + "requests_queued %15llu Requests for which no thread was waiting\n", m->requests_queued); homa_append_metric(homa, - "responses_received %15llu " - "Incoming response messages\n", + "responses_received %15llu Incoming response messages\n", m->responses_received); homa_append_metric(homa, - "responses_queued %15llu " - "Responses for which no thread was waiting\n", + "responses_queued %15llu Responses for which no thread was waiting\n", m->responses_queued); homa_append_metric(homa, - "fast_wakeups %15llu " - "Messages received while polling\n", + "fast_wakeups %15llu Messages received while polling\n", m->fast_wakeups); homa_append_metric(homa, - "slow_wakeups %15llu " - "Messages received after thread went to sleep\n", + "slow_wakeups %15llu Messages received after thread went to sleep\n", m->slow_wakeups); homa_append_metric(homa, - "handoffs_thread_waiting %15llu " - "RPC handoffs to waiting threads (vs. queue)\n", + "handoffs_thread_waiting %15llu RPC handoffs to waiting threads (vs. queue)\n", m->handoffs_thread_waiting); homa_append_metric(homa, - "handoffs_alt_thread %15llu " - "RPC handoffs not to first on list (avoid busy " - "core)\n", + "handoffs_alt_thread %15llu RPC handoffs not to first on list (avoid busy core)\n", m->handoffs_alt_thread); homa_append_metric(homa, - "poll_cycles %15llu " - "Time spent polling for incoming messages\n", + "poll_cycles %15llu Time spent polling for incoming messages\n", m->poll_cycles); homa_append_metric(homa, - "softirq_calls %15llu " - "Calls to homa_softirq (i.e. # GRO pkts " - "received)\n", + "softirq_calls %15llu Calls to homa_softirq (i.e. # GRO pkts received)\n", m->softirq_calls); homa_append_metric(homa, - "softirq_cycles %15llu " - "Time spent in homa_softirq during SoftIRQ\n", + "softirq_cycles %15llu Time spent in homa_softirq during SoftIRQ\n", m->softirq_cycles); homa_append_metric(homa, - "bypass_softirq_cycles %15llu " - "Time spent in homa_softirq during bypass " - "from GRO\n", + "bypass_softirq_cycles %15llu Time spent in homa_softirq during bypass from GRO\n", m->bypass_softirq_cycles); homa_append_metric(homa, - "linux_softirq_cycles %15llu " - "Time spent in all Linux SoftIRQ\n", + "linux_softirq_cycles %15llu Time spent in all Linux SoftIRQ\n", m->linux_softirq_cycles); homa_append_metric(homa, - "napi_cycles %15llu " - "Time spent in NAPI-level packet handling\n", + "napi_cycles %15llu Time spent in NAPI-level packet handling\n", m->napi_cycles); homa_append_metric(homa, - "send_cycles %15llu " - "Time spent in homa_sendmsg for requests\n", + "send_cycles %15llu Time spent in homa_sendmsg for requests\n", m->send_cycles); homa_append_metric(homa, - "send_calls %15llu " - "Total invocations of homa_sendmsg for " - "requests\n", + "send_calls %15llu Total invocations of homa_sendmsg for equests\n", m->send_calls); // It is possible for us to get here at a time when a // thread has been blocked for a long time and has @@ -1724,331 +1684,233 @@ char *homa_print_metrics(struct homa *homa) if (delta < 0) delta = 0; homa_append_metric(homa, - "recv_cycles %15llu " - "Unblocked time spent in recvmsg kernel call\n", + "recv_cycles %15llu Unblocked time spent in recvmsg kernel call\n", delta); homa_append_metric(homa, - "recv_calls %15llu " - "Total invocations of recvmsg kernel call\n", + "recv_calls %15llu Total invocations of recvmsg kernel call\n", m->recv_calls); homa_append_metric(homa, - "blocked_cycles %15llu " - "Time spent blocked in homa_recvmsg\n", + "blocked_cycles %15llu Time spent blocked in homa_recvmsg\n", m->blocked_cycles); homa_append_metric(homa, - "reply_cycles %15llu " - "Time spent in homa_sendmsg for responses\n", + "reply_cycles %15llu Time spent in homa_sendmsg for responses\n", m->reply_cycles); homa_append_metric(homa, - "reply_calls %15llu " - "Total invocations of homa_sendmsg for " - "responses\n", + "reply_calls %15llu Total invocations of homa_sendmsg for responses\n", m->reply_calls); homa_append_metric(homa, - "abort_cycles %15llu " - "Time spent in homa_ioc_abort kernel call\n", + "abort_cycles %15llu Time spent in homa_ioc_abort kernel call\n", m->reply_cycles); homa_append_metric(homa, - "abort_calls %15llu " - "Total invocations of abort kernel call\n", + "abort_calls %15llu Total invocations of abort kernel call\n", m->reply_calls); homa_append_metric(homa, - "so_set_buf_cycles %15llu " - "Time spent in setsockopt SO_HOMA_SET_BUF\n", + "so_set_buf_cycles %15llu Time spent in setsockopt SO_HOMA_SET_BUF\n", m->so_set_buf_cycles); homa_append_metric(homa, - "so_set_buf_calls %15llu " - "Total invocations of setsockopt SO_HOMA_SET_BUF\n", + "so_set_buf_calls %15llu Total invocations of setsockopt SO_HOMA_SET_BUF\n", m->so_set_buf_calls); homa_append_metric(homa, - "grantable_lock_cycles %15llu " - "Time spent with homa->grantable_lock locked\n", + "grantable_lock_cycles %15llu Time spent with homa->grantable_lock locked\n", m->grantable_lock_cycles); homa_append_metric(homa, - "timer_cycles %15llu " - "Time spent in homa_timer\n", + "timer_cycles %15llu Time spent in homa_timer\n", m->timer_cycles); homa_append_metric(homa, - "timer_reap_cycles %15llu " - "Time in homa_timer spent reaping RPCs\n", + "timer_reap_cycles %15llu Time in homa_timer spent reaping RPCs\n", m->timer_reap_cycles); homa_append_metric(homa, - "data_pkt_reap_cycles %15llu " - "Time in homa_data_pkt spent reaping RPCs\n", + "data_pkt_reap_cycles %15llu Time in homa_data_pkt spent reaping RPCs\n", m->data_pkt_reap_cycles); homa_append_metric(homa, - "pacer_cycles %15llu " - "Time spent in homa_pacer_main\n", + "pacer_cycles %15llu Time spent in homa_pacer_main\n", m->pacer_cycles); homa_append_metric(homa, - "homa_cycles %15llu " - "Total time in all Homa-related functions\n", + "homa_cycles %15llu Total time in all Homa-related functions\n", m->softirq_cycles + m->napi_cycles + m->send_cycles + m->recv_cycles + m->reply_cycles - m->blocked_cycles + m->timer_cycles + m->pacer_cycles); homa_append_metric(homa, - "pacer_lost_cycles %15llu " - "Lost transmission time because pacer was " - "slow\n", + "pacer_lost_cycles %15llu Lost transmission time because pacer was slow\n", m->pacer_lost_cycles); homa_append_metric(homa, - "pacer_bytes %15llu " - "Bytes transmitted when the pacer was active\n", + "pacer_bytes %15llu Bytes transmitted when the pacer was active\n", m->pacer_bytes); homa_append_metric(homa, - "pacer_skipped_rpcs %15llu " - "Pacer aborts because of locked RPCs\n", + "pacer_skipped_rpcs %15llu Pacer aborts because of locked RPCs\n", m->pacer_skipped_rpcs); homa_append_metric(homa, - "pacer_needed_help %15llu " - "homa_pacer_xmit invocations from " - "homa_check_pacer\n", + "pacer_needed_help %15llu homa_pacer_xmit invocations from homa_check_pacer\n", m->pacer_needed_help); homa_append_metric(homa, - "throttled_cycles %15llu " - "Time when the throttled queue was nonempty\n", + "throttled_cycles %15llu Time when the throttled queue was nonempty\n", m->throttled_cycles); homa_append_metric(homa, - "resent_packets %15llu " - "DATA packets sent in response to RESENDs\n", + "resent_packets %15llu DATA packets sent in response to RESENDs\n", m->resent_packets); homa_append_metric(homa, - "peer_hash_links %15llu " - "Hash chain link traversals in peer table\n", + "peer_hash_links %15llu Hash chain link traversals in peer table\n", m->peer_hash_links); homa_append_metric(homa, - "peer_new_entries %15llu " - "New entries created in peer table\n", + "peer_new_entries %15llu New entries created in peer table\n", m->peer_new_entries); homa_append_metric(homa, - "peer_kmalloc_errors %15llu " - "kmalloc failures creating peer table " - "entries\n", + "peer_kmalloc_errors %15llu kmalloc failures creating peer table entries\n", m->peer_kmalloc_errors); homa_append_metric(homa, - "peer_route_errors %15llu " - "Routing failures creating peer table " - "entries\n", + "peer_route_errors %15llu Routing failures creating peer table entries\n", m->peer_route_errors); homa_append_metric(homa, - "control_xmit_errors %15llu " - "Errors sending control packets\n", + "control_xmit_errors %15llu Errors sending control packets\n", m->control_xmit_errors); homa_append_metric(homa, - "data_xmit_errors %15llu " - "Errors sending data packets\n", + "data_xmit_errors %15llu Errors sending data packets\n", m->data_xmit_errors); homa_append_metric(homa, - "unknown_rpcs %15llu " - "Non-grant packets discarded because RPC unknown\n", + "unknown_rpcs %15llu Non-grant packets discarded because RPC unknown\n", m->unknown_rpcs); homa_append_metric(homa, - "server_cant_create_rpcs %15llu " - "Packets discarded because server " - "couldn't create RPC\n", + "server_cant_create_rpcs %15llu Packets discarded because server couldn't create RPC\n", m->server_cant_create_rpcs); homa_append_metric(homa, - "unknown_packet_types %15llu " - "Packets discarded because of unsupported " - "type\n", + "unknown_packet_types %15llu Packets discarded because of unsupported type\n", m->unknown_packet_types); homa_append_metric(homa, - "short_packets %15llu " - "Packets discarded because too short\n", + "short_packets %15llu Packets discarded because too short\n", m->short_packets); homa_append_metric(homa, - "packet_discards %15llu " - "Non-resent packets discarded because data " - "already received\n", + "packet_discards %15llu Non-resent packets discarded because data already received\n", m->packet_discards); homa_append_metric(homa, - "resent_discards %15llu " - "Resent packets discarded because data " - "already received\n", + "resent_discards %15llu Resent packets discarded because data already received\n", m->resent_discards); homa_append_metric(homa, - "resent_packets_used %15llu " - "Retransmitted packets that were actually used\n", + "resent_packets_used %15llu Retransmitted packets that were actually used\n", m->resent_packets_used); homa_append_metric(homa, - "rpc_timeouts %15llu " - "RPCs aborted because peer was nonresponsive\n", + "rpc_timeouts %15llu RPCs aborted because peer was nonresponsive\n", m->rpc_timeouts); homa_append_metric(homa, - "server_rpc_discards %15llu " - "RPCs discarded by server because of errors\n", + "server_rpc_discards %15llu RPCs discarded by server because of errors\n", m->server_rpc_discards); homa_append_metric(homa, - "server_rpcs_unknown %15llu " - "RPCs aborted by server because unknown to " - "client\n", + "server_rpcs_unknown %15llu RPCs aborted by server because unknown to client\n", m->server_rpcs_unknown); homa_append_metric(homa, - "client_lock_misses %15llu " - "Bucket lock misses for client RPCs\n", + "client_lock_misses %15llu Bucket lock misses for client RPCs\n", m->client_lock_misses); homa_append_metric(homa, - "client_lock_miss_cycles %15llu " - "Time lost waiting for client bucket locks\n", + "client_lock_miss_cycles %15llu Time lost waiting for client bucket locks\n", m->client_lock_miss_cycles); homa_append_metric(homa, - "server_lock_misses %15llu " - "Bucket lock misses for server RPCs\n", + "server_lock_misses %15llu Bucket lock misses for server RPCs\n", m->server_lock_misses); homa_append_metric(homa, - "server_lock_miss_cycles %15llu " - "Time lost waiting for server bucket locks\n", + "server_lock_miss_cycles %15llu Time lost waiting for server bucket locks\n", m->server_lock_miss_cycles); homa_append_metric(homa, - "socket_lock_misses %15llu " - "Socket lock misses\n", + "socket_lock_misses %15llu Socket lock misses\n", m->socket_lock_misses); homa_append_metric(homa, - "socket_lock_miss_cycles %15llu " - "Time lost waiting for socket locks\n", + "socket_lock_miss_cycles %15llu Time lost waiting for socket locks\n", m->socket_lock_miss_cycles); homa_append_metric(homa, - "throttle_lock_misses %15llu " - "Throttle lock misses\n", + "throttle_lock_misses %15llu Throttle lock misses\n", m->throttle_lock_misses); homa_append_metric(homa, - "throttle_lock_miss_cycles %15llu " - "Time lost waiting for throttle locks\n", + "throttle_lock_miss_cycles %15llu Time lost waiting for throttle locks\n", m->throttle_lock_miss_cycles); homa_append_metric(homa, - "peer_ack_lock_misses %15llu " - "Misses on peer ack locks\n", + "peer_ack_lock_misses %15llu Misses on peer ack locks\n", m->peer_ack_lock_misses); homa_append_metric(homa, - "peer_ack_lock_miss_cycles %15llu " - "Time lost waiting for peer ack locks\n", + "peer_ack_lock_miss_cycles %15llu Time lost waiting for peer ack locks\n", m->peer_ack_lock_miss_cycles); homa_append_metric(homa, - "grantable_lock_misses %15llu " - "Grantable lock misses\n", + "grantable_lock_misses %15llu Grantable lock misses\n", m->grantable_lock_misses); homa_append_metric(homa, - "grantable_lock_miss_cycles%15llu " - "Time lost waiting for grantable lock\n", + "grantable_lock_miss_cycles%15llu Time lost waiting for grantable lock\n", m->grantable_lock_miss_cycles); homa_append_metric(homa, - "grantable_rpcs_integral %15llu " - "Integral of homa->num_grantable_rpcs*dt\n", + "grantable_rpcs_integral %15llu Integral of homa->num_grantable_rpcs*dt\n", m->grantable_rpcs_integral); homa_append_metric(homa, - "grant_recalc_calls %15llu " - "Number of calls to homa_grant_recalc\n", + "grant_recalc_calls %15llu Number of calls to homa_grant_recalc\n", m->grant_recalc_calls); homa_append_metric(homa, - "grant_recalc_cycles %15llu " - "Time spent in homa_grant_recalc\n", + "grant_recalc_cycles %15llu Time spent in homa_grant_recalc\n", m->grant_recalc_cycles); homa_append_metric(homa, - "grant_recalc_skips %15llu " - "Number of times homa_grant_recalc skipped " - "redundant work\n", + "grant_recalc_skips %15llu Number of times homa_grant_recalc skipped redundant work\n", m->grant_recalc_skips); homa_append_metric(homa, - "grant_recalc_loops %15llu " - "Number of times homa_grant_recalc looped back\n", + "grant_recalc_loops %15llu Number of times homa_grant_recalc looped back\n", m->grant_recalc_loops); homa_append_metric(homa, - "grant_priority_bumps %15llu " - "Number of times an RPC moved up in the grant " - "priority order\n", + "grant_priority_bumps %15llu Number of times an RPC moved up in the grant priority order\n", m->grant_priority_bumps); homa_append_metric(homa, - "fifo_grants %15llu " - "Grants issued using FIFO priority\n", + "fifo_grants %15llu Grants issued using FIFO priority\n", m->fifo_grants); homa_append_metric(homa, - "fifo_grants_no_incoming %15llu " - "FIFO grants to messages with no " - "outstanding grants\n", + "fifo_grants_no_incoming %15llu FIFO grants to messages with no outstanding grants\n", m->fifo_grants_no_incoming); homa_append_metric(homa, - "disabled_reaps %15llu " - "Reaper invocations that were disabled\n", + "disabled_reaps %15llu Reaper invocations that were disabled\n", m->disabled_reaps); homa_append_metric(homa, - "disabled_rpc_reaps %15llu " - "Disabled RPCs skipped by reaper\n", + "disabled_rpc_reaps %15llu Disabled RPCs skipped by reaper\n", m->disabled_rpc_reaps); homa_append_metric(homa, - "reaper_calls %15llu " - "Reaper invocations that were not disabled\n", + "reaper_calls %15llu Reaper invocations that were not disabled\n", m->reaper_calls); homa_append_metric(homa, - "reaper_dead_skbs %15llu " - "Sum of hsk->dead_skbs across all reaper " - "calls\n", + "reaper_dead_skbs %15llu Sum of hsk->dead_skbs across all reaper alls\n", m->reaper_dead_skbs); homa_append_metric(homa, - "forced_reaps %15llu " - "Reaps forced by accumulation of dead RPCs\n", + "forced_reaps %15llu Reaps forced by accumulation of dead RPCs\n", m->forced_reaps); homa_append_metric(homa, - "throttle_list_adds %15llu " - "Calls to homa_add_to_throttled\n", + "throttle_list_adds %15llu Calls to homa_add_to_throttled\n", m->throttle_list_adds); homa_append_metric(homa, - "throttle_list_checks %15llu " - "List elements checked in " - "homa_add_to_throttled\n", + "throttle_list_checks %15llu List elements checked in homa_add_to_throttled\n", m->throttle_list_checks); homa_append_metric(homa, - "ack_overflows %15llu " - "Explicit ACKs sent because peer->acks was " - "full\n", + "ack_overflows %15llu Explicit ACKs sent because peer->acks was full\n", m->ack_overflows); homa_append_metric(homa, - "ignored_need_acks %15llu " - "NEED_ACKs ignored because RPC result not " - "yet received\n", + "ignored_need_acks %15llu NEED_ACKs ignored because RPC result not yet received\n", m->ignored_need_acks); homa_append_metric(homa, - "bpage_reuses %15llu " - "Buffer page could be reused because ref " - "count was zero\n", + "bpage_reuses %15llu Buffer page could be reused because ref count was zero\n", m->bpage_reuses); homa_append_metric(homa, - "buffer_alloc_failures %15llu " - "homa_pool_allocate didn't find enough buffer " - "space for an RPC\n", + "buffer_alloc_failures %15llu homa_pool_allocate didn't find enough buffer space for an RPC\n", m->buffer_alloc_failures); homa_append_metric(homa, - "linux_pkt_alloc_bytes %15llu " - "Bytes allocated in new packets by NIC driver " - "due to cache overflows\n", + "linux_pkt_alloc_bytes %15llu Bytes allocated in new packets by NIC driver due to cache overflows\n", m->linux_pkt_alloc_bytes); homa_append_metric(homa, - "dropped_data_no_bufs %15llu " - "Data bytes dropped because app buffers full\n", + "dropped_data_no_bufs %15llu Data bytes dropped because app buffers full\n", m->dropped_data_no_bufs); homa_append_metric(homa, - "gen3_handoffs %15llu " - "GRO->SoftIRQ handoffs made by Gen3 balancer\n", + "gen3_handoffs %15llu GRO->SoftIRQ handoffs made by Gen3 balancer\n", m->gen3_handoffs); homa_append_metric(homa, - "gen3_alt_handoffs %15llu " - "Gen3 handoffs to secondary core (primary was " - "busy)\n", + "gen3_alt_handoffs %15llu Gen3 handoffs to secondary core (primary was busy)\n", m->gen3_alt_handoffs); homa_append_metric(homa, - "gro_grant_bypasses %15llu " - "Grant packets passed directly to homa_softirq " - "by homa_gro_receive\n", + "gro_grant_bypasses %15llu Grant packets passed directly to homa_softirq by homa_gro_receive\n", m->gro_grant_bypasses); homa_append_metric(homa, - "gro_data_bypasses %15llu " - "Data packets passed directly to homa_softirq " - "by homa_gro_receive\n", + "gro_data_bypasses %15llu Data packets passed directly to homa_softirq by homa_gro_receive\n", m->gro_data_bypasses); for (i = 0; i < NUM_TEMP_METRICS; i++) homa_append_metric(homa, - "temp%-2d %15llu " - "Temporary use in testing\n", + "temp%-2d %15llu Temporary use in testing\n", i, m->temp[i]); } @@ -2085,7 +1947,7 @@ void homa_prios_changed(struct homa *homa) homa->max_sched_prio = 0; break; } - if ((homa->unsched_cutoffs[i] >= HOMA_MAX_MESSAGE_LENGTH)) { + if (homa->unsched_cutoffs[i] >= HOMA_MAX_MESSAGE_LENGTH) { homa->max_sched_prio = i-1; break; } @@ -2100,10 +1962,11 @@ void homa_prios_changed(struct homa *homa) void homa_spin(int ns) { __u64 end; + end = get_cycles() + (ns*cpu_khz)/1000000; - while (get_cycles() < end) { + while (get_cycles() < end) /* Empty loop body.*/ - } + ; } /** @@ -2116,6 +1979,7 @@ void homa_spin(int ns) void homa_throttle_lock_slow(struct homa *homa) { __u64 start = get_cycles(); + tt_record("beginning wait for throttle lock"); spin_lock_bh(&homa->throttle_lock); tt_record("ending wait for throttle lock"); @@ -2141,14 +2005,16 @@ void homa_freeze(struct homa_rpc *rpc, enum homa_freeze_type type, char *format) if (!tt_frozen) { // struct freeze_header freeze; int dummy; - printk(KERN_NOTICE "freezing in homa_freeze with freeze_type %d\n", type); + + pr_notice("freezing in %s with freeze_type %d\n", __func__, + type); tt_record1("homa_freeze calling homa_rpc_log_active with freeze_type %d", type); homa_rpc_log_active_tt(rpc->hsk->homa, 0); homa_validate_incoming(rpc->hsk->homa, 1, &dummy); - printk(KERN_NOTICE "%s\n", format); + pr_notice("%s\n", format); tt_record2(format, rpc->id, tt_addr(rpc->peer->addr)); tt_freeze(); // homa_xmit_control(FREEZE, &freeze, sizeof(freeze), rpc); homa_freeze_peers(rpc->hsk->homa); } -} \ No newline at end of file +} diff --git a/test/unit_homa_grant.c b/test/unit_homa_grant.c index fcfc529..ce97e7d 100644 --- a/test/unit_homa_grant.c +++ b/test/unit_homa_grant.c @@ -42,7 +42,7 @@ FIXTURE(homa_grant) { int server_port; __u64 client_id; __u64 server_id; - sockaddr_in_union server_addr; + union sockaddr_in_union server_addr; struct homa homa; struct homa_sock hsk; struct data_header data; diff --git a/test/unit_homa_incoming.c b/test/unit_homa_incoming.c index 86e8866..4b7f306 100644 --- a/test/unit_homa_incoming.c +++ b/test/unit_homa_incoming.c @@ -129,7 +129,7 @@ FIXTURE(homa_incoming) { int server_port; __u64 client_id; __u64 server_id; - sockaddr_in_union server_addr; + union sockaddr_in_union server_addr; struct homa homa; struct homa_sock hsk; struct homa_sock hsk2; diff --git a/test/unit_homa_outgoing.c b/test/unit_homa_outgoing.c index 4ecd305..83002cb 100644 --- a/test/unit_homa_outgoing.c +++ b/test/unit_homa_outgoing.c @@ -38,7 +38,7 @@ FIXTURE(homa_outgoing) { __u64 server_id; struct homa homa; struct homa_sock hsk; - sockaddr_in_union server_addr; + union sockaddr_in_union server_addr; struct homa_peer *peer; }; FIXTURE_SETUP(homa_outgoing) diff --git a/test/unit_homa_plumbing.c b/test/unit_homa_plumbing.c index 556752f..6dbf40e 100644 --- a/test/unit_homa_plumbing.c +++ b/test/unit_homa_plumbing.c @@ -29,8 +29,8 @@ FIXTURE(homa_plumbing) { __u64 server_id; struct homa homa; struct homa_sock hsk; - sockaddr_in_union client_addr; - sockaddr_in_union server_addr; + union sockaddr_in_union client_addr; + union sockaddr_in_union server_addr; struct data_header data; int starting_skb_count; struct msghdr recvmsg_hdr; @@ -40,7 +40,7 @@ FIXTURE(homa_plumbing) { struct homa_sendmsg_args sendmsg_args; char buffer[2000]; sockptr_t optval; - sockaddr_in_union addr; + union sockaddr_in_union addr; }; FIXTURE_SETUP(homa_plumbing) { @@ -128,7 +128,7 @@ TEST_F(homa_plumbing, homa_bind__ipv6_address_too_short) homa_sock_destroy(&self->hsk); mock_sock_init(&self->hsk, &self->homa, 0); - sockaddr_in_union addr = {}; + union sockaddr_in_union addr = {}; addr.in6.sin6_family = AF_INET6; struct socket sock = {}; sock.sk = &self->hsk.inet.sk; @@ -142,7 +142,7 @@ TEST_F(homa_plumbing, homa_bind__ipv6_ok) homa_sock_destroy(&self->hsk); mock_sock_init(&self->hsk, &self->homa, 0); - sockaddr_in_union addr = {}; + union sockaddr_in_union addr = {}; addr.in6.sin6_family = AF_INET6; addr.in6.sin6_port = htons(123); struct socket sock = {}; @@ -158,7 +158,7 @@ TEST_F(homa_plumbing, homa_bind__ipv4_address_too_short) homa_sock_destroy(&self->hsk); mock_sock_init(&self->hsk, &self->homa, 0); - sockaddr_in_union addr = {}; + union sockaddr_in_union addr = {}; addr.in4.sin_family = AF_INET; struct socket sock = {}; sock.sk = &self->hsk.inet.sk; @@ -172,7 +172,7 @@ TEST_F(homa_plumbing, homa_bind__ipv4_ok) homa_sock_destroy(&self->hsk); mock_sock_init(&self->hsk, &self->homa, 0); - sockaddr_in_union addr = {}; + union sockaddr_in_union addr = {}; addr.in4.sin_family = AF_INET; addr.in4.sin_port = htons(345); struct socket sock = {}; diff --git a/test/unit_homa_pool.c b/test/unit_homa_pool.c index 3970d5b..43ea267 100644 --- a/test/unit_homa_pool.c +++ b/test/unit_homa_pool.c @@ -519,8 +519,8 @@ TEST_F(homa_pool, homa_pool_check_waiting__rpc_initially_locked) unit_log_clear(); atomic_set(&pool->free_bpages, 1); homa_pool_check_waiting(pool); - EXPECT_SUBSTR("rpc lock unavailable in homa_pool_release_buffers; " - "rpc lock unavailable in homa_pool_release_buffers", + EXPECT_SUBSTR("rpc lock unavailable in homa_pool_check_waiting; " + "rpc lock unavailable in homa_pool_check_waiting", unit_log_get()); EXPECT_EQ(1, crpc->msgin.num_bpages); EXPECT_TRUE(list_empty(&self->hsk.waiting_for_bufs)); diff --git a/test/unit_homa_timer.c b/test/unit_homa_timer.c index cd5193e..33f00a8 100644 --- a/test/unit_homa_timer.c +++ b/test/unit_homa_timer.c @@ -16,7 +16,7 @@ FIXTURE(homa_timer) { int server_port; __u64 client_id; __u64 server_id; - sockaddr_in_union server_addr; + union sockaddr_in_union server_addr; struct homa homa; struct homa_sock hsk; }; diff --git a/test/unit_homa_utils.c b/test/unit_homa_utils.c index fcc8221..aa03f90 100644 --- a/test/unit_homa_utils.c +++ b/test/unit_homa_utils.c @@ -21,7 +21,7 @@ FIXTURE(homa_utils) { __u64 server_id; struct homa homa; struct homa_sock hsk; - sockaddr_in_union server_addr; + union sockaddr_in_union server_addr; struct data_header data; struct homa_rpc *crpc; struct iovec iovec; diff --git a/test/unit_timetrace.c b/test/unit_timetrace.c index aae1c00..650e96e 100644 --- a/test/unit_timetrace.c +++ b/test/unit_timetrace.c @@ -90,7 +90,7 @@ TEST_F(timetrace, tt_record_buf__wraparound) TEST_F(timetrace, tt_find_oldest) { - int pos[NR_CPUS]; + int pos[nr_cpu_ids]; tt_buffer_size = 4; tt_record_buf(tt_buffers[0], 1500, "Buf0", 0, 0, 0, 0); diff --git a/test/utils.c b/test/utils.c index 841f7a2..8b4e3e6 100644 --- a/test/utils.c +++ b/test/utils.c @@ -34,7 +34,7 @@ struct homa_rpc *unit_client_rpc(struct homa_sock *hsk, int req_length, int resp_length) { int bytes_received; - sockaddr_in_union server_addr; + union sockaddr_in_union server_addr; int saved_id = atomic64_read(&hsk->homa->next_outgoing_id); server_addr.in6.sin6_family = AF_INET6; diff --git a/timetrace.c b/timetrace.c index e429786..7c44cdd 100644 --- a/timetrace.c +++ b/timetrace.c @@ -1,6 +1,4 @@ -/* Copyright (c) 2019-2023 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +// SPDX-License-Identifier: BSD-2-Clause #include "homa_impl.h" @@ -20,7 +18,7 @@ extern int *tt_linux_homa_temp; extern int tt_linux_homa_temp_default[]; extern void (*tt_linux_inc_metrics)(int metric, __u64 count); extern void (*tt_linux_record)(struct tt_buffer *buffer, __u64 timestamp, - const char* format, __u32 arg0, __u32 arg1, __u32 arg2, + const char *format, __u32 arg0, __u32 arg1, __u32 arg2, __u32 arg3); extern void tt_linux_skip_metrics(int metric, __u64 count); extern void (*tt_linux_printk)(void); @@ -40,7 +38,7 @@ extern void tt_inc_metric(int metric, __u64 count); * synchronization in tt_record, which improves performance significantly. * NR_CPUS is an overestimate of the actual number of cores; we use it * here, rather than nr_cpu_ids, because it allows for static allocation - * of this array. And + * of this array. */ struct tt_buffer *tt_buffers[NR_CPUS]; @@ -87,7 +85,7 @@ int tt_buffer_size = TT_BUF_SIZE; int tt_pf_storage = TT_PF_BUF_SIZE; /* Set during tests to disable "cpu_khz" line in trace output. */ -bool tt_test_no_khz = false; +bool tt_test_no_khz; /** * tt_init(): Enable time tracing, create /proc file for reading traces. @@ -105,16 +103,15 @@ int tt_init(char *proc_file, int *temp) { int i; - if (init) { + if (init) return 0; - } for (i = 0; i < nr_cpu_ids; i++) { struct tt_buffer *buffer; + buffer = kmalloc(sizeof(*buffer), GFP_KERNEL); if (buffer == NULL) { - printk(KERN_ERR "timetrace couldn't allocate " - "tt_buffers\n"); + pr_err("timetrace couldn't allocate tt_buffers\n"); goto error; } memset(buffer, 0, sizeof(*buffer)); @@ -122,10 +119,10 @@ int tt_init(char *proc_file, int *temp) } if (proc_file != NULL) { - tt_dir_entry = proc_create(proc_file, S_IRUGO, NULL, &tt_pops); + tt_dir_entry = proc_create(proc_file, 0444, NULL, &tt_pops); if (!tt_dir_entry) { - printk(KERN_ERR "couldn't create /proc/%s for timetrace " - "reading\n", proc_file); + pr_err("couldn't create /proc/%s for timetrace reading\n", + proc_file); goto error; } } else { @@ -138,9 +135,8 @@ int tt_init(char *proc_file, int *temp) init = true; #ifdef TT_KERNEL - for (i = 0; i < nr_cpu_ids; i++) { + for (i = 0; i < nr_cpu_ids; i++) tt_linux_buffers[i] = tt_buffers[i]; - } tt_linux_record = tt_record_buf; tt_linux_freeze = tt_freeze; tt_linux_freeze_count = &tt_freeze_count; @@ -155,7 +151,7 @@ int tt_init(char *proc_file, int *temp) return 0; - error: +error: for (i = 0; i < nr_cpu_ids; i++) { kfree(tt_buffers[i]); tt_buffers[i] = NULL; @@ -170,6 +166,7 @@ int tt_init(char *proc_file, int *temp) void tt_destroy(void) { int i; + spin_lock(&tt_lock); if (init) { init = false; @@ -186,9 +183,8 @@ void tt_destroy(void) tt_linux_record = ltt_record_nop; tt_linux_freeze = tt_linux_nop; tt_linux_freeze_count = &tt_linux_freeze_no_homa; - for (i = 0; i < nr_cpu_ids; i++) { + for (i = 0; i < nr_cpu_ids; i++) tt_linux_buffers[i] = NULL; - } tt_linux_inc_metrics = tt_linux_skip_metrics; tt_linux_printk = tt_linux_nop; tt_linux_dbg1 = (void (*)(char *, ...)) tt_linux_nop; @@ -214,7 +210,7 @@ void tt_freeze(void) if (tt_frozen) return; tt_record("timetrace frozen"); - printk(KERN_NOTICE "tt_freeze invoked\n"); + pr_notice("%s invoked\n", __func__); spin_lock(&tt_lock); if (!tt_frozen) { tt_frozen = true; @@ -243,10 +239,11 @@ void tt_freeze(void) * @arg3: Argument to use when printing a message about this event. */ void tt_record_buf(struct tt_buffer *buffer, __u64 timestamp, - const char* format, __u32 arg0, __u32 arg1, __u32 arg2, + const char *format, __u32 arg0, __u32 arg1, __u32 arg2, __u32 arg3) { struct tt_event *event; + if (unlikely(atomic_read(&tt_freeze_count) > 0)) { // In order to ensure that reads produce consistent // results, don't record concurrently (this could cause @@ -285,7 +282,7 @@ void tt_record_buf(struct tt_buffer *buffer, __u64 timestamp, */ void tt_find_oldest(int *pos) { - struct tt_buffer* buffer; + struct tt_buffer *buffer; int i; __u64 start_time = 0; @@ -297,10 +294,10 @@ void tt_find_oldest(int *pos) int index = (buffer->next_index + 1) & (tt_buffer_size-1); struct tt_event *event = &buffer->events[index]; + pos[i] = index; - if (event->timestamp > start_time) { + if (event->timestamp > start_time) start_time = event->timestamp; - } } } @@ -326,7 +323,7 @@ void tt_find_oldest(int *pos) */ int tt_proc_open(struct inode *inode, struct file *file) { - struct tt_proc_file* pf = NULL; + struct tt_proc_file *pf = NULL; int result = 0; spin_lock(&tt_lock); @@ -352,7 +349,7 @@ int tt_proc_open(struct inode *inode, struct file *file) "cpu_khz: %u\n", cpu_khz); } - done: +done: spin_unlock(&tt_lock); return result; } @@ -381,8 +378,8 @@ ssize_t tt_proc_read(struct file *file, char __user *user_buf, spin_lock(&tt_lock); if ((pf == NULL) || (pf->file != file)) { - printk(KERN_ERR "tt_metrics_read found damaged " - "private_data: 0x%p\n", file->private_data); + pr_err("tt_metrics_read found damaged private_data: 0x%p\n", + file->private_data); copied_to_user = -EINVAL; goto done; } @@ -403,16 +400,17 @@ ssize_t tt_proc_read(struct file *file, char __user *user_buf, /* Check all the traces to find the earliest available event. */ for (i = 0; i < nr_cpu_ids; i++) { struct tt_buffer *buffer = tt_buffers[i]; + event = &buffer->events[pf->pos[i]]; if ((pf->pos[i] != buffer->next_index) && (event->timestamp < earliest_time)) { - current_core = i; - earliest_time = event->timestamp; + current_core = i; + earliest_time = event->timestamp; } } if (current_core < 0) { - /* None of the traces have any more events to process. */ - goto flush; + /* None of the traces have any more events. */ + goto flush; } /* Format one event. */ @@ -420,13 +418,12 @@ ssize_t tt_proc_read(struct file *file, char __user *user_buf, pf->pos[current_core]]); available = tt_pf_storage - (pf->next_byte + pf->bytes_available - pf->msg_storage); - if (available == 0) { + if (available == 0) goto flush; - } entry_length = snprintf(pf->next_byte + pf->bytes_available, available, "%lu [C%02d] ", - (long unsigned int) event->timestamp, - current_core); + (unsigned long) event->timestamp, + current_core); if (available >= entry_length) entry_length += snprintf(pf->next_byte + pf->bytes_available + entry_length, @@ -437,7 +434,8 @@ ssize_t tt_proc_read(struct file *file, char __user *user_buf, /* Not enough room for this entry. */ if (pf->bytes_available == 0) { /* Even a full buffer isn't enough for - * this entry; truncate the entry. */ + * this entry; truncate the entry. + */ entry_length = available - 1; } else { goto flush; @@ -450,11 +448,10 @@ ssize_t tt_proc_read(struct file *file, char __user *user_buf, & (tt_buffer_size-1); continue; - flush: +flush: chunk_size = pf->bytes_available; - if (chunk_size > (length - copied_to_user)) { + if (chunk_size > (length - copied_to_user)) chunk_size = length - copied_to_user; - } if (chunk_size == 0) goto done; failed_to_copy = copy_to_user(user_buf + copied_to_user, @@ -473,7 +470,7 @@ ssize_t tt_proc_read(struct file *file, char __user *user_buf, } } - done: +done: spin_unlock(&tt_lock); return copied_to_user; } @@ -503,11 +500,11 @@ loff_t tt_proc_lseek(struct file *file, loff_t offset, int whence) int tt_proc_release(struct inode *inode, struct file *file) { int i; - struct tt_proc_file *pf = file->private_data; + if ((pf == NULL) || (pf->file != file)) { - printk(KERN_ERR "tt_metrics_release found damaged " - "private_data: 0x%p\n", file->private_data); + pr_err("tt_metrics_release found damaged private_data: 0x%p\n", + file->private_data); return -EINVAL; } @@ -528,6 +525,7 @@ int tt_proc_release(struct inode *inode, struct file *file) */ for (i = 0; i < nr_cpu_ids; i++) { struct tt_buffer *buffer = tt_buffers[i]; + buffer->events[tt_buffer_size-1].format = NULL; buffer->next_index = 0; } @@ -569,10 +567,8 @@ void tt_print_file(char *path) int bytes_used = 0; loff_t offset = 0; - printk(KERN_ERR "tt_print_file starting, file %s\n", path); - if (atomic_xchg(&active, 1)) { - printk(KERN_ERR "concurrent call to tt_print_file aborting\n"); + pr_err("concurrent call to %s aborting\n", __func__); return; } if (!init) @@ -580,13 +576,12 @@ void tt_print_file(char *path) filp = filp_open(path, O_WRONLY | O_CREAT, 0666); if (IS_ERR(filp)) { - printk(KERN_ERR "tt_print_file couldn't open %s: " - "error %ld\n", path, -PTR_ERR(filp)); + pr_err("%s couldn't open %s: error %ld\n", __func__, path, + -PTR_ERR(filp)); filp = NULL; goto done; } - tt_record("tt_print_file printing timetrace"); atomic_inc(&tt_freeze_count); tt_find_oldest(pos); @@ -604,16 +599,17 @@ void tt_print_file(char *path) /* Check all the traces to find the earliest available event. */ for (i = 0; i < nr_cpu_ids; i++) { struct tt_buffer *buffer = tt_buffers[i]; + event = &buffer->events[pos[i]]; if ((pos[i] != buffer->next_index) && (event->timestamp < earliest_time)) { - current_core = i; - earliest_time = event->timestamp; + current_core = i; + earliest_time = event->timestamp; } } if (current_core < 0) { - /* None of the traces have any more events to process. */ - break; + /* None of the traces have any more events. */ + break; } event = &(tt_buffers[current_core]->events[ pos[current_core]]); @@ -623,7 +619,7 @@ void tt_print_file(char *path) bytes_used += snprintf(buffer + bytes_used, sizeof(buffer) - bytes_used, "%lu [C%02d] ", - (long unsigned int) event->timestamp, + (unsigned long) event->timestamp, current_core); bytes_used += snprintf(buffer + bytes_used, sizeof(buffer) - bytes_used, @@ -637,9 +633,8 @@ void tt_print_file(char *path) err = kernel_write(filp, buffer, bytes_used, &offset); if (err < 0) { - printk(KERN_NOTICE "tt_print_file got " - "error %d writing %s\n", - -err, path); + pr_notice("%s got error %d writing %s\n", + __func__, -err, path); goto done; } bytes_used = 0; @@ -648,25 +643,22 @@ void tt_print_file(char *path) if (bytes_used > 0) { err = kernel_write(filp, buffer, bytes_used, &offset); if (err < 0) - printk(KERN_ERR "tt_print_file got error %d " - "writing %s\n", -err, path); + pr_err("%s got error %d writing %s\n", + __func__, -err, path); } - printk(KERN_ERR "tt_print_file finishing up\n"); - done: +done: if (filp != NULL) { err = vfs_fsync(filp, 0); if (err < 0) - printk(KERN_ERR "tt_print_file got error %d " - "in fsync\n", -err); + pr_err("%s got error %d in fsync\n", __func__, -err); err = filp_close(filp, NULL); if (err < 0) - printk(KERN_ERR "tt_print_file got error %d " - "in filp_close\n", -err); + pr_err("%s got error %d in filp_close\n", __func__, + -err); } atomic_dec(&tt_freeze_count); atomic_set(&active, 0); - printk(KERN_ERR "tt_print_file(%s) finished\n", path); } /** @@ -687,7 +679,7 @@ void tt_printk(void) static atomic_t active; if (atomic_xchg(&active, 1)) { - printk(KERN_NOTICE "concurrent call to tt_printk aborting\n"); + pr_notice("concurrent call to %s aborting\n", __func__); return; } if (!init) @@ -695,7 +687,7 @@ void tt_printk(void) atomic_inc(&tt_freeze_count); tt_find_oldest(pos); - printk(KERN_NOTICE "cpu_khz: %u\n", cpu_khz); + pr_notice("cpu_khz: %u\n", cpu_khz); /* Each iteration of this loop printk's one event. */ while (true) { @@ -708,16 +700,17 @@ void tt_printk(void) /* Check all the traces to find the earliest available event. */ for (i = 0; i < nr_cpu_ids; i++) { struct tt_buffer *buffer = tt_buffers[i]; + event = &buffer->events[pos[i]]; if ((pos[i] != buffer->next_index) && (event->timestamp < earliest_time)) { - current_core = i; - earliest_time = event->timestamp; + current_core = i; + earliest_time = event->timestamp; } } if (current_core < 0) { - /* None of the traces have any more events to process. */ - break; + /* None of the traces have any more events. */ + break; } event = &(tt_buffers[current_core]->events[ pos[current_core]]); @@ -726,8 +719,8 @@ void tt_printk(void) snprintf(msg, sizeof(msg), event->format, event->arg0, event->arg1, event->arg2, event->arg3); - printk(KERN_NOTICE "%lu [C%02d] %s\n", - (long unsigned int) event->timestamp, + pr_notice("%lu [C%02d] %s\n", + (unsigned long) event->timestamp, current_core, msg); } @@ -748,7 +741,7 @@ void tt_get_messages(char *buffer, size_t length) /* Index of the next entry to return from each tt_buffer (too * large to allocate on stack, so allocate dynamically). */ - int *pos = kmalloc(NR_CPUS * sizeof(int), GFP_KERNEL); + int *pos = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL); int printed = 0; *buffer = 0; @@ -767,16 +760,17 @@ void tt_get_messages(char *buffer, size_t length) /* Check all the traces to find the earliest available event. */ for (i = 0; i < nr_cpu_ids; i++) { struct tt_buffer *buffer = tt_buffers[i]; + event = &buffer->events[pos[i]]; if ((pos[i] != buffer->next_index) && (event->timestamp < earliest_time)) { - current_core = i; - earliest_time = event->timestamp; + current_core = i; + earliest_time = event->timestamp; } } if (current_core < 0) { - /* None of the traces have any more events to process. */ - break; + /* None of the traces have any more events. */ + break; } event = &(tt_buffers[current_core]->events[ pos[current_core]]); @@ -800,7 +794,7 @@ void tt_get_messages(char *buffer, size_t length) atomic_dec(&tt_freeze_count); - done: +done: kfree(pos); } diff --git a/timetrace.h b/timetrace.h index fdabf97..3c186a5 100644 --- a/timetrace.h +++ b/timetrace.h @@ -1,6 +1,4 @@ -/* Copyright (c) 2019-2023 Homa Developers - * SPDX-License-Identifier: BSD-1-Clause - */ +/* SPDX-License-Identifier: BSD-2-Clause */ #ifndef HOMA_TIMETRACE_H #define HOMA_TIMETRACE_H @@ -15,7 +13,7 @@ * Timetrace implements a circular buffer of entries, each of which * consists of a fine-grain timestamp, a short descriptive string, and * a few additional values. It's typically used to record times at - * various points in in kernel operations, in order to find performance + * various points in kernel operations, in order to find performance * bottlenecks. It can record a trace relatively efficiently (< 10ns as * of 6/2018), and the trace can be retrieved by user programs for * analysis by reading a file in /proc. @@ -33,7 +31,7 @@ struct tt_event { * Format string describing the event. NULL means that this * entry has never been occupied. */ - const char* format; + const char *format; /** * Up to 4 additional arguments that may be referenced by @@ -74,7 +72,7 @@ struct tt_buffer { */ struct tt_proc_file { /* Identifies a particular open file. */ - struct file* file; + struct file *file; /* Index of the next entry to return from each tt_buffer. */ int pos[NR_CPUS]; @@ -97,8 +95,8 @@ struct tt_proc_file { extern void tt_destroy(void); extern void tt_freeze(void); extern int tt_init(char *proc_file, int *temp); -extern void tt_record_buf(struct tt_buffer* buffer, __u64 timestamp, - const char* format, __u32 arg0, __u32 arg1, +extern void tt_record_buf(struct tt_buffer *buffer, __u64 timestamp, + const char *format, __u32 arg0, __u32 arg1, __u32 arg2, __u32 arg3); /* Private methods and variables: exposed so they can be accessed @@ -127,7 +125,7 @@ extern bool tt_test_no_khz; * the kernel. */ extern int64_t tt_debug_int64[100]; -extern void * tt_debug_ptr[100]; +extern void *tt_debug_ptr[100]; /** * tt_rdtsc(): return the current value of the fine-grain CPU cycle counter @@ -136,6 +134,7 @@ extern void * tt_debug_ptr[100]; static inline __u64 tt_rdtsc(void) { __u32 lo, hi; + __asm__ __volatile__("rdtsc" : "=a" (lo), "=d" (hi)); return (((__u64)hi << 32) | lo); } @@ -157,7 +156,7 @@ static inline __u64 tt_rdtsc(void) * @arg2 Argument to use when printing a message about this event. * @arg3 Argument to use when printing a message about this event. */ -static inline void tt_record4(const char* format, __u32 arg0, __u32 arg1, +static inline void tt_record4(const char *format, __u32 arg0, __u32 arg1, __u32 arg2, __u32 arg3) { #if ENABLE_TIME_TRACE @@ -165,7 +164,7 @@ static inline void tt_record4(const char* format, __u32 arg0, __u32 arg1, arg0, arg1, arg2, arg3); #endif } -static inline void tt_record3(const char* format, __u32 arg0, __u32 arg1, +static inline void tt_record3(const char *format, __u32 arg0, __u32 arg1, __u32 arg2) { #if ENABLE_TIME_TRACE @@ -173,21 +172,21 @@ static inline void tt_record3(const char* format, __u32 arg0, __u32 arg1, arg0, arg1, arg2, 0); #endif } -static inline void tt_record2(const char* format, __u32 arg0, __u32 arg1) +static inline void tt_record2(const char *format, __u32 arg0, __u32 arg1) { #if ENABLE_TIME_TRACE tt_record_buf(tt_buffers[raw_smp_processor_id()], get_cycles(), format, arg0, arg1, 0, 0); #endif } -static inline void tt_record1(const char* format, __u32 arg0) +static inline void tt_record1(const char *format, __u32 arg0) { #if ENABLE_TIME_TRACE tt_record_buf(tt_buffers[raw_smp_processor_id()], get_cycles(), format, arg0, 0, 0, 0); #endif } -static inline void tt_record(const char* format) +static inline void tt_record(const char *format) { #if ENABLE_TIME_TRACE tt_record_buf(tt_buffers[raw_smp_processor_id()], get_cycles(), format,