From df0daa5a9b4f26fd6dfd8e3630906cac01fc4324 Mon Sep 17 00:00:00 2001 From: John Ousterhout Date: Thu, 29 Aug 2024 09:27:00 -0700 Subject: [PATCH] Upgrade to run under Linux 6.10.6 * Renamed install script to install_homa to avoid conflicts with the Linux install program * Reworked the timetracing mechanism used by the main Linux kernel to avoid header file conflicts --- README.md | 10 +- cloudlab/bin/{install => install_homa} | 6 +- homa_grant.c | 4 +- homa_impl.h | 85 +++++++++------ homa_incoming.c | 5 +- homa_offload.c | 7 +- homa_plumbing.c | 31 +----- homa_pool.c | 2 +- homa_skb.c | 45 ++++---- test/mock.c | 145 +++++++++++++++---------- test/mock.h | 10 +- test/unit_homa_grant.c | 22 ++-- test/unit_homa_incoming.c | 49 ++++----- test/unit_homa_offload.c | 82 +++++++------- test/unit_homa_outgoing.c | 30 ++--- test/unit_homa_peertab.c | 16 +-- test/unit_homa_plumbing.c | 16 ++- test/unit_homa_pool.c | 53 ++++----- test/unit_homa_skb.c | 52 ++++----- test/unit_homa_socktab.c | 8 +- test/unit_homa_timer.c | 6 +- test/unit_homa_utils.c | 16 +-- test/utils.h | 4 + timetrace.c | 15 ++- 24 files changed, 381 insertions(+), 338 deletions(-) rename cloudlab/bin/{install => install_homa} (91%) diff --git a/README.md b/README.md index 0e1a6f9..913fd2b 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,10 @@ This repo contains an implementation of the Homa transport protocol as a Linux k - Please contact me if you have any problems using this repo; I'm happy to provide advice and support. -- The head is known to work under Linux 6.1.38. In the past, Homa has - run under several earlier versions of Linux, including 5.17.7, - 5.4.80, and 4.15.18. There is a separate branch for each of these - older versions, with a names such as linux_4.15.18. Older branches are +- The head is known to work under Linux 6.10.6. In the past, Homa has + run under several earlier versions of Linux. There is a separate branch + for each of these + older versions, with names such as linux_4.15.18. Older branches are out of date feature-wise: recent commits have not been back-ported to them. Other versions of Linux have not been tested and may require code changes (these upgrades rarely take more than a couple @@ -47,7 +47,7 @@ This repo contains an implementation of the Homa transport protocol as a Linux k invoke it with no parameters to install and configure Homa on the current machine. -- The script `cloudlab/bin/install` will copy relevant Homa files +- The script `cloudlab/bin/install_homa` will copy relevant Homa files across a cluster of machines and configure Homa on each node. It assumes that nodes have names `nodeN` where N is a small integer, and it also assumes that you have already run `make` both in the top-level directory and diff --git a/cloudlab/bin/install b/cloudlab/bin/install_homa similarity index 91% rename from cloudlab/bin/install rename to cloudlab/bin/install_homa index 120b694..eb89935 100755 --- a/cloudlab/bin/install +++ b/cloudlab/bin/install_homa @@ -7,7 +7,7 @@ # or more target machines; it also loads the Homa kernel module. # # Usage: -# install num_nodes [first] +# install_homa num_nodes [first] # # The "num_nodes" arguments indicates how many servers should be updated. # The "first" argument is optional; it is an integer identifying the @@ -24,7 +24,7 @@ if [ $# -eq 2 ]; then elif [ $# -eq 1 ]; then first=0 else - echo "Usage: install num_nodes [first]" + echo "Usage: install_homa num_nodes [first]" exit 1 fi last=`expr $first + $1 - 1` || true @@ -32,7 +32,7 @@ last=`expr $first + $1 - 1` || true for ((i = $first ; i <= $last; i++)); do node=node$i echo - echo '*** Installing on' $node '***' + echo '*** Installing Homa on' $node '***' rsync --ipv4 -e "ssh -4 -o StrictHostKeyChecking=no" -rtv ~/.bashrc ~/.bash_profile ~/.gdbinit $node: rsync --ipv4 -e "ssh -4 -o StrictHostKeyChecking=no" -rtv --exclude __pycache__ ~/bin/ $node:bin/ rsync --ipv4 -e "ssh -4 -o StrictHostKeyChecking=no" -rtv $root/homa.ko $root/util/cp_node $root/util/homa_prio $root/util/*.py $node:bin/ diff --git a/homa_grant.c b/homa_grant.c index 27df0b3..bf542e2 100644 --- a/homa_grant.c +++ b/homa_grant.c @@ -15,7 +15,7 @@ * @rpc1: First RPC to consider. * @rpc2: Second RPC to consider. */ -int inline homa_grant_outranks(struct homa_rpc *rpc1, struct homa_rpc *rpc2) +inline int homa_grant_outranks(struct homa_rpc *rpc1, struct homa_rpc *rpc2) { /* Fewest bytes remaining is the primary criterion; if those are * equal, then favor the older RPC. @@ -39,7 +39,7 @@ int inline homa_grant_outranks(struct homa_rpc *rpc1, struct homa_rpc *rpc2) * may be possible to send out additional grants to some RPCs (doing * this is left to the caller). */ -int inline homa_grant_update_incoming(struct homa_rpc *rpc, struct homa *homa) { +inline int homa_grant_update_incoming(struct homa_rpc *rpc, struct homa *homa) { int incoming = rpc->msgin.granted - (rpc->msgin.length - rpc->msgin.bytes_remaining); if (incoming < 0) diff --git a/homa_impl.h b/homa_impl.h index 9fd4686..a0e68e0 100644 --- a/homa_impl.h +++ b/homa_impl.h @@ -38,13 +38,15 @@ #include #include #include -#include #include +#include +#include #include #include #include #include #include +#include #pragma GCC diagnostic warning "-Wpointer-sign" #pragma GCC diagnostic warning "-Wunused-variable" @@ -53,53 +55,68 @@ typedef unsigned int __poll_t; #endif #ifdef __UNIT_TEST__ -#define spin_unlock mock_spin_unlock -extern void mock_spin_unlock(spinlock_t *lock); - -#undef get_cycles -#define get_cycles mock_get_cycles -extern cycles_t mock_get_cycles(void); +#undef alloc_pages +#define alloc_pages mock_alloc_pages +extern struct page *mock_alloc_pages(gfp_t gfp, unsigned order); -#define signal_pending(xxx) mock_signal_pending -extern int mock_signal_pending; - -#define rcu_read_lock mock_rcu_read_lock -extern void mock_rcu_read_lock(void); +#define compound_order mock_compound_order +extern unsigned int mock_compound_order(struct page *page); -#define rcu_read_unlock mock_rcu_read_unlock -extern void mock_rcu_read_unlock(void); +#define cpu_to_node mock_cpu_to_node +extern int mock_cpu_to_node(int cpu); #undef current #define current current_task +extern struct task_struct *current_task; -#define kthread_complete_and_exit(comp, code) +#undef get_cycles +#define get_cycles mock_get_cycles +extern cycles_t mock_get_cycles(void); +#define get_page mock_get_page + extern void mock_get_page(struct page *page); + +#undef kmalloc #define kmalloc mock_kmalloc extern void *mock_kmalloc(size_t size, gfp_t flags); -#define get_page mock_get_page -extern void mock_get_page(struct page *page); +#define kthread_complete_and_exit(comp, code) -#define put_page mock_put_page -extern void mock_put_page(struct page *page); +#ifdef page_address +#undef page_address +#endif +#define page_address(page) ((void *) page) -#define compound_order mock_compound_order -extern unsigned int mock_compound_order(struct page *page); +#define page_ref_count mock_page_refs +extern int mock_page_refs(struct page *page); #define page_to_nid mock_page_to_nid extern int mock_page_to_nid(struct page *page); -#define page_ref_count mock_page_refs -extern int mock_page_refs(struct page *page); +#define put_page mock_put_page +extern void mock_put_page(struct page *page); -#define cpu_to_node mock_cpu_to_node -extern int mock_cpu_to_node(int cpu); +#define rcu_read_lock mock_rcu_read_lock +extern void mock_rcu_read_lock(void); -#ifdef page_address -#undef page_address -#endif -#define page_address(page) ((void *) page) -#endif +#define rcu_read_unlock mock_rcu_read_unlock +extern void mock_rcu_read_unlock(void); + +#undef register_net_sysctl +#define register_net_sysctl mock_register_net_sysctl +extern struct ctl_table_header *mock_register_net_sysctl(struct net *net, + const char *path, struct ctl_table *table); + +#define signal_pending(xxx) mock_signal_pending +extern int mock_signal_pending; + +#define spin_unlock mock_spin_unlock +extern void mock_spin_unlock(spinlock_t *lock); + +#undef vmalloc +#define vmalloc mock_vmalloc +extern void *mock_vmalloc(size_t size); +#endif /* __UNIT_TEST__ */ /* Null out things that confuse VSCode Intellisense */ #ifdef __VSCODE__ @@ -879,7 +896,7 @@ struct homa_interest { * of a struct homa_interest. * @interest: Struct to initialize. */ -static void inline homa_interest_init(struct homa_interest *interest) +inline static void homa_interest_init(struct homa_interest *interest) { interest->thread = current; atomic_long_set(&interest->ready_rpc, 0); @@ -3682,8 +3699,8 @@ extern enum hrtimer_restart homa_hrtimer(struct hrtimer *timer); extern int homa_init(struct homa *homa); extern void homa_incoming_sysctl_changed(struct homa *homa); -extern int homa_ioc_abort(struct sock *sk, unsigned long arg); -extern int homa_ioctl(struct sock *sk, int cmd, unsigned long arg); +extern int homa_ioc_abort(struct sock *sk, int *karg); +extern int homa_ioctl(struct sock *sk, int cmd, int *karg); extern void homa_log_throttled(struct homa *homa); extern int homa_message_in_init(struct homa_rpc *rpc, int length, int unsched); @@ -3777,8 +3794,6 @@ extern struct homa_rpc extern int homa_rpc_reap(struct homa_sock *hsk, int count); extern void homa_send_ipis(void); extern int homa_sendmsg(struct sock *sk, struct msghdr *msg, size_t len); -extern int homa_sendpage(struct sock *sk, struct page *page, int offset, - size_t size, int flags); extern int homa_setsockopt(struct sock *sk, int level, int optname, sockptr_t __user optval, unsigned int optlen); extern int homa_shutdown(struct socket *sock, int how); diff --git a/homa_incoming.c b/homa_incoming.c index d5dbb3e..81c9dde 100644 --- a/homa_incoming.c +++ b/homa_incoming.c @@ -256,7 +256,6 @@ int homa_copy_to_user(struct homa_rpc *rpc) int pkt_length = homa_data_len(skbs[i]); int copied = 0; char *dst; - struct iovec iov; struct iov_iter iter; int buf_bytes, chunk_size; @@ -276,8 +275,8 @@ int homa_copy_to_user(struct homa_rpc *rpc) } chunk_size = buf_bytes; } - error = import_single_range(READ, dst, - chunk_size, &iov, &iter); + error = import_ubuf(READ, dst, chunk_size, + &iter); if (error) goto free_skbs; error = skb_copy_datagram_iter(skbs[i], diff --git a/homa_offload.c b/homa_offload.c index 502acde..a6e3405 100644 --- a/homa_offload.c +++ b/homa_offload.c @@ -140,13 +140,14 @@ static inline void homa_set_softirq_cpu(struct sk_buff *skb, int cpu) struct rps_sock_flow_table *sock_flow_table; int hash; - sock_flow_table = rcu_dereference(rps_sock_flow_table); + sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table); if (sock_flow_table == NULL) return; - hash = cpu + rps_cpu_mask + 1; + hash = cpu + net_hotdata.rps_cpu_mask + 1; if (sock_flow_table->ents[hash] != hash) { rcu_read_lock(); - sock_flow_table = rcu_dereference(rps_sock_flow_table); + sock_flow_table = rcu_dereference( + net_hotdata.rps_sock_flow_table); sock_flow_table->ents[hash] = hash; rcu_read_unlock(); } diff --git a/homa_plumbing.c b/homa_plumbing.c index 533d2d4..ba820ff 100644 --- a/homa_plumbing.c +++ b/homa_plumbing.c @@ -63,7 +63,6 @@ const struct proto_ops homa_proto_ops = { .sendmsg = inet_sendmsg, .recvmsg = inet_recvmsg, .mmap = sock_no_mmap, - .sendpage = sock_no_sendpage, .set_peek_off = sk_set_peek_off, }; @@ -85,7 +84,6 @@ const struct proto_ops homav6_proto_ops = { .sendmsg = inet_sendmsg, .recvmsg = inet_recvmsg, .mmap = sock_no_mmap, - .sendpage = sock_no_sendpage, .set_peek_off = sk_set_peek_off, }; @@ -108,7 +106,6 @@ struct proto homa_prot = { .getsockopt = homa_getsockopt, .sendmsg = homa_sendmsg, .recvmsg = homa_recvmsg, - .sendpage = homa_sendpage, .backlog_rcv = homa_backlog_rcv, .release_cb = ip4_datagram_release_cb, .hash = homa_hash, @@ -136,7 +133,6 @@ struct proto homav6_prot = { .getsockopt = homa_getsockopt, .sendmsg = homa_sendmsg, .recvmsg = homa_recvmsg, - .sendpage = homa_sendpage, .backlog_rcv = homa_backlog_rcv, .release_cb = ip6_datagram_release_cb, .hash = homa_hash, @@ -744,17 +740,17 @@ int homa_disconnect(struct sock *sk, int flags) { * homa_ioc_abort() - The top-level function for the ioctl that implements * the homa_abort user-level API. * @sk: Socket for this request. - * @arg: Used to pass information from user space. + * @karg: Used to pass information from user space. * * Return: 0 on success, otherwise a negative errno. */ -int homa_ioc_abort(struct sock *sk, unsigned long arg) { +int homa_ioc_abort(struct sock *sk, int *karg) { int ret = 0; struct homa_sock *hsk = homa_sk(sk); struct homa_abort_args args; struct homa_rpc *rpc; - if (unlikely(copy_from_user(&args, (void *) arg, sizeof(args)))) + if (unlikely(copy_from_user(&args, (void *) karg, sizeof(args)))) return -EFAULT; if (args._pad1 || args._pad2[0] || args._pad2[1]) { @@ -781,18 +777,18 @@ int homa_ioc_abort(struct sock *sk, unsigned long arg) { * homa_ioctl() - Implements the ioctl system call for Homa sockets. * @sk: Socket on which the system call was invoked. * @cmd: Identifier for a particular ioctl operation. - * @arg: Operation-specific argument; typically the address of a block + * @karg: Operation-specific argument; typically the address of a block * of data in user address space. * * Return: 0 on success, otherwise a negative errno. */ -int homa_ioctl(struct sock *sk, int cmd, unsigned long arg) { +int homa_ioctl(struct sock *sk, int cmd, int *karg) { int result; __u64 start = get_cycles(); switch (cmd) { case HOMAIOCABORT: - result = homa_ioc_abort(sk, arg); + result = homa_ioc_abort(sk, karg); INC_METRIC(abort_calls, 1); INC_METRIC(abort_cycles, get_cycles() - start); break; @@ -1157,21 +1153,6 @@ int homa_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, return result; } -/** - * homa_sendpage() - ??. - * @sk: Socket for the operation - * @page: ?? - * @offset: ?? - * @size: ?? - * @flags: ?? - * Return: 0 on success, otherwise a negative errno. - */ -int homa_sendpage(struct sock *sk, struct page *page, int offset, - size_t size, int flags) { - printk(KERN_WARNING "unimplemented sendpage invoked on Homa socket\n"); - return -ENOSYS; -} - /** * homa_hash() - ??. * @sk: Socket for the operation diff --git a/homa_pool.c b/homa_pool.c index 9c7ce73..a8dec58 100644 --- a/homa_pool.c +++ b/homa_pool.c @@ -29,7 +29,7 @@ * The caller must own the lock for @pool->hsk. * @pool: Pool to update. */ -static void inline set_bpages_needed(struct homa_pool *pool) { +inline static void set_bpages_needed(struct homa_pool *pool) { struct homa_rpc *rpc = list_first_entry(&pool->hsk->waiting_for_bufs, struct homa_rpc, buf_links); pool->bpages_needed = (rpc->msgin.length + HOMA_BPAGE_SIZE - 1) diff --git a/homa_skb.c b/homa_skb.c index 455b293..e49455f 100644 --- a/homa_skb.c +++ b/homa_skb.c @@ -13,6 +13,11 @@ extern int mock_max_skb_frags; #define HOMA_MAX_SKB_FRAGS MAX_SKB_FRAGS #endif +static inline void frag_page_set(skb_frag_t *frag, struct page *page) +{ + frag->netmem = page_to_netmem(page); +} + /** * homa_skb_page_pool_init() - Invoked when a struct homa is created to * initialize a page pool. @@ -144,15 +149,15 @@ void *homa_skb_extend_frags(struct homa *homa, struct sk_buff *skb, int *length) int actual_size = *length; /* Can we just extend the skb's last fragment? */ - if ((shinfo->nr_frags > 0) && (frag->bv_page == core->skb_page) + if ((shinfo->nr_frags > 0) && (skb_frag_page(frag) == core->skb_page) && (core->page_inuse < core->page_size) - && ((frag->bv_offset + frag->bv_len) + && ((frag->offset + skb_frag_size(frag)) == core->page_inuse)) { if ((core->page_size - core->page_inuse) < actual_size) actual_size = core->page_size - core->page_inuse; *length = actual_size; - frag->bv_len += actual_size; - result = page_address(frag->bv_page) + core->page_inuse; + skb_frag_size_add(frag, actual_size); + result = page_address(skb_frag_page(frag)) + core->page_inuse; core->page_inuse += actual_size; skb_len_add(skb, actual_size); return result; @@ -168,12 +173,12 @@ void *homa_skb_extend_frags(struct homa *homa, struct sk_buff *skb, int *length) actual_size = core->page_size - core->page_inuse; frag = &shinfo->frags[shinfo->nr_frags]; shinfo->nr_frags++; - frag->bv_page = core->skb_page; + frag_page_set(frag, core->skb_page); get_page(core->skb_page); - frag->bv_offset = core->page_inuse; + frag->offset = core->page_inuse; *length = actual_size; - frag->bv_len = actual_size; - result = page_address(frag->bv_page) + core->page_inuse; + skb_frag_size_set(frag, actual_size); + result = page_address(skb_frag_page(frag)) + core->page_inuse; core->page_inuse += actual_size; skb_len_add(skb, actual_size); return result; @@ -350,22 +355,23 @@ int homa_skb_append_from_skb(struct homa *homa, struct sk_buff *dst_skb, src_frag_offset = head_len; for (src_frags_left = src_shinfo->nr_frags, src_frag = &src_shinfo->frags[0]; (src_frags_left > 0) && (length > 0); - src_frags_left--, src_frag_offset += src_frag->bv_len, src_frag++) + src_frags_left--, src_frag_offset += skb_frag_size(src_frag), + src_frag++) { - if (offset >= (src_frag_offset + src_frag->bv_len)) + if (offset >= (src_frag_offset + skb_frag_size(src_frag))) continue; - chunk_size = src_frag->bv_len - (offset - src_frag_offset); + chunk_size = skb_frag_size(src_frag) - (offset - src_frag_offset); if (chunk_size > length) chunk_size = length; if (dst_shinfo->nr_frags == HOMA_MAX_SKB_FRAGS) return -EINVAL; dst_frag = &dst_shinfo->frags[dst_shinfo->nr_frags]; dst_shinfo->nr_frags++; - dst_frag->bv_page = src_frag->bv_page; - get_page(src_frag->bv_page); - dst_frag->bv_offset = src_frag->bv_offset + frag_page_set(dst_frag, skb_frag_page(src_frag)); + get_page(skb_frag_page(src_frag)); + dst_frag->offset = src_frag->offset + (offset - src_frag_offset); - dst_frag->bv_len = chunk_size; + skb_frag_size_set(dst_frag, chunk_size); offset += chunk_size; length -= chunk_size; skb_len_add(dst_skb, chunk_size); @@ -498,13 +504,14 @@ void homa_skb_get(struct sk_buff *skb, void *dest, int offset, int length) frag_offset = head_len; for (frags_left = shinfo->nr_frags, frag = &shinfo->frags[0]; (frags_left > 0) && (length > 0); - frags_left--, frag_offset += frag->bv_len, frag++) { - if (offset >= (frag_offset + frag->bv_len)) + frags_left--, + frag_offset += skb_frag_size(frag), frag++) { + if (offset >= (frag_offset + skb_frag_size(frag))) continue; - chunk_size = frag->bv_len - (offset - frag_offset); + chunk_size = skb_frag_size(frag) - (offset - frag_offset); if (chunk_size > length) chunk_size = length; - memcpy(dst, page_address(frag->bv_page) + frag->bv_offset + memcpy(dst, page_address(skb_frag_page(frag)) + frag->offset + (offset - frag_offset), chunk_size); offset += chunk_size; diff --git a/test/mock.c b/test/mock.c index abe99df..6e2b3e0 100644 --- a/test/mock.c +++ b/test/mock.c @@ -38,7 +38,7 @@ int mock_copy_data_errors = 0; int mock_copy_to_iter_errors = 0; int mock_copy_to_user_errors = 0; int mock_cpu_idle = 0; -int mock_import_single_range_errors = 0; +int mock_import_ubuf_errors = 0; int mock_import_iovec_errors = 0; int mock_ip6_xmit_errors = 0; int mock_ip_queue_xmit_errors = 0; @@ -136,9 +136,6 @@ bool mock_ipv6 = true; /* The value to use for mock_ipv6 in each test unless overridden. */ bool mock_ipv6_default; -/* Linux's idea of the current CPU number. */ -int cpu_number = 1; - /* List of priorities for all outbound packets. */ char mock_xmit_prios[1000]; int mock_xmit_prios_offset = 0; @@ -184,26 +181,16 @@ unsigned long page_offset_base = 0; unsigned long phys_base = 0; unsigned long vmemmap_base = 0; int __preempt_count = 0; +struct pcpu_hot pcpu_hot = {.cpu_number = 1}; char sock_flow_table[RPS_SOCK_FLOW_TABLE_SIZE(1024)]; -struct rps_sock_flow_table *rps_sock_flow_table - = (struct rps_sock_flow_table *) sock_flow_table; -__u32 rps_cpu_mask = 0x1f; +struct net_hotdata net_hotdata = { + .rps_cpu_mask = 0x1f, + .rps_sock_flow_table = (struct rps_sock_flow_table *) sock_flow_table +}; extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) {} -struct page *alloc_pages(gfp_t gfp, unsigned order) -{ - struct page *page; - if (mock_check_error(&mock_alloc_page_errors)) - return NULL; - page = (struct page *) malloc(PAGE_SIZE << order); - if (!pages_in_use) - pages_in_use = unit_hash_new(); - unit_hash_set(pages_in_use, page, (char *) 1); - return page; -} - struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags, int node) { @@ -257,7 +244,7 @@ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *iter) return 0; } while (bytes_left > 0) { - struct iovec *iov = (struct iovec *) iter->iov; + struct iovec *iov = (struct iovec *) iter_iov(iter); __u64 int_base = (__u64) iov->iov_base; size_t chunk_bytes = iov->iov_len; if (chunk_bytes > bytes_left) @@ -269,7 +256,7 @@ size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *iter) iov->iov_base = (void *) (int_base + chunk_bytes); iov->iov_len -= chunk_bytes; if (iov->iov_len == 0) - iter->iov++; + iter->__iov++; } return bytes; } @@ -336,8 +323,8 @@ void dst_release(struct dst_entry *dst) { if (!dst) return; - dst->__refcnt.counter--; - if (dst->__refcnt.counter > 0) + atomic_dec(&dst->__rcuref.refcnt); + if (atomic_read(&dst->__rcuref.refcnt) > 0) return; if (!routes_in_use || unit_hash_get(routes_in_use, dst) == NULL) { FAIL("dst_release on unknown route"); @@ -422,14 +409,11 @@ ssize_t import_iovec(int type, const struct iovec __user * uvector, return size; } -int import_single_range(int type, void __user *buf, size_t len, - struct iovec *iov, struct iov_iter *i) +int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i) { - if (mock_check_error(&mock_import_single_range_errors)) + if (mock_check_error(&mock_import_ubuf_errors)) return -EACCES; - iov->iov_base = buf; - iov->iov_len = len; - iov_iter_init(i, type, iov, 1, len); + iov_iter_ubuf(i, rw, buf, len); return 0; } @@ -540,7 +524,7 @@ void iov_iter_init(struct iov_iter *i, unsigned int direction, { direction &= READ | WRITE; i->iter_type = ITER_IOVEC | direction; - i->iov = iov; + i->__iov = iov; i->nr_segs = nr_segs; i->iov_offset = 0; i->count = count; @@ -572,7 +556,7 @@ struct dst_entry *ip6_dst_lookup_flow(struct net *net, const struct sock *sk, FAIL("malloc failed"); return ERR_PTR(-ENOMEM); } - route->dst.__refcnt.counter = 1; + atomic_set(&route->dst.__rcuref.refcnt, 1); route->dst.ops = &mock_dst_ops; route->dst.dev = &mock_net_device; route->dst.obsolete = 0; @@ -667,7 +651,7 @@ struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4, FAIL("malloc failed"); return ERR_PTR(-ENOMEM); } - route->dst.__refcnt.counter = 1; + atomic_set(&route->dst.__rcuref.refcnt, 1); route->dst.ops = &mock_dst_ops; route->dst.dev = &mock_net_device; route->dst.obsolete = 0; @@ -738,7 +722,7 @@ void kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) shinfo->frag_list = next; } for (i = 0; i < shinfo->nr_frags; i++) { - put_page(shinfo->frags[i].bv_page); + put_page(skb_frag_page(&shinfo->frags[i])); } free(skb->head); free(skb); @@ -917,14 +901,13 @@ int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) return 1; } -void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t) {} - -struct ctl_table_header *register_net_sysctl(struct net *net, - const char *path, struct ctl_table *table) +bool rcuref_get_slowpath(rcuref_t *ref) { - return NULL; + return true; } +void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t) {} + void release_sock(struct sock *sk) { mock_active_locks--; @@ -939,7 +922,8 @@ void schedule(void) UNIT_HOOK("schedule"); } -void security_sk_classify_flow(struct sock *sk, struct flowi_common *flic) {} +void security_sk_classify_flow(const struct sock *sk, + struct flowi_common *flic) {} void __show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) {} @@ -965,7 +949,7 @@ int skb_copy_datagram_iter(const struct sk_buff *from, int offset, return 0; } while (bytes_left > 0) { - struct iovec *iov = (struct iovec *) iter->iov; + struct iovec *iov = (struct iovec *) iter_iov(iter); __u64 int_base = (__u64) iov->iov_base; size_t chunk_bytes = iov->iov_len; if (chunk_bytes > bytes_left) @@ -980,7 +964,7 @@ int skb_copy_datagram_iter(const struct sk_buff *from, int offset, iov->iov_base = (void *) (int_base + chunk_bytes); iov->iov_len -= chunk_bytes; if (iov->iov_len == 0) - iter->iov++; + iter->__iov++; } return 0; } @@ -1049,8 +1033,8 @@ int sock_common_setsockopt(struct socket *sock, int level, int optname, return 0; } -int sock_no_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) +int sock_no_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) { return 0; } @@ -1108,21 +1092,6 @@ int vfs_fsync(struct file *file, int datasync) return 0; } -void *vmalloc(size_t size) -{ - if (mock_check_error(&mock_vmalloc_errors)) - return NULL; - void *block = malloc(size); - if (!block) { - FAIL("malloc failed"); - return NULL; - } - if (!vmallocs_in_use) - vmallocs_in_use = unit_hash_new(); - unit_hash_set(vmallocs_in_use, block, "used"); - return block; -} - void wait_for_completion(struct completion *x) {} long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, @@ -1151,6 +1120,22 @@ int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, return 0; } +/** + * mock_alloc_pages() - Called instead of alloc_pages when Homa is compiled + * for unit testing. + */ +struct page *mock_alloc_pages(gfp_t gfp, unsigned order) +{ + struct page *page; + if (mock_check_error(&mock_alloc_page_errors)) + return NULL; + page = (struct page *)malloc(PAGE_SIZE << order); + if (!pages_in_use) + pages_in_use = unit_hash_new(); + unit_hash_set(pages_in_use, page, (char *)1); + return page; +} + /** * mock_check_error() - Determines whether a method should simulate an error * return. @@ -1304,6 +1289,25 @@ void mock_rcu_read_unlock(void) mock_active_rcu_locks--; } +/** + * mock_register_net_sysctl() - Called instead of register_net_sysctl + * when Homa is compiled for unit testing. + */ +struct ctl_table_header *mock_register_net_sysctl(struct net *net, + const char *path, struct ctl_table *table) +{ + return NULL; +} + +/** + * mock_set_core() - Set internal state that indicates the "current core". + * @num: Integer identifier for a core. + */ +void mock_set_core(int num) +{ + pcpu_hot.cpu_number = num; +} + /** * mock_skb_new() - Allocate and return a packet buffer. The buffer is * initialized as if it just arrived from the network. @@ -1451,7 +1455,7 @@ void mock_spin_unlock(spinlock_t *lock) */ void mock_teardown(void) { - cpu_number = 1; + pcpu_hot.cpu_number = 1; cpu_khz = 1000000; mock_alloc_page_errors = 0; mock_alloc_skb_errors = 0; @@ -1461,7 +1465,7 @@ void mock_teardown(void) mock_cpu_idle = 0; mock_cycles = 0; mock_ipv6 = mock_ipv6_default; - mock_import_single_range_errors = 0; + mock_import_ubuf_errors = 0; mock_import_iovec_errors = 0; mock_ip6_xmit_errors = 0; mock_ip_queue_xmit_errors = 0; @@ -1535,3 +1539,24 @@ void mock_teardown(void) unit_hook_clear(); } + +/** + * mock_vmalloc() - Called instead of vmalloc when Homa is compiled + * for unit testing. + * @size: Number of bytes to allocate. + */ +void *mock_vmalloc(size_t size) +{ + if (mock_check_error(&mock_vmalloc_errors)) + return NULL; + void *block = malloc(size); + if (!block) + { + FAIL("malloc failed"); + return NULL; + } + if (!vmallocs_in_use) + vmallocs_in_use = unit_hash_new(); + unit_hash_set(vmallocs_in_use, block, "used"); + return block; +} diff --git a/test/mock.h b/test/mock.h index c2e19a7..20dcb2b 100644 --- a/test/mock.h +++ b/test/mock.h @@ -4,7 +4,6 @@ /* Functions for mocking that are exported to test code. */ -extern int cpu_number; extern int mock_alloc_page_errors; extern int mock_alloc_skb_errors; extern int mock_bpage_size; @@ -16,7 +15,7 @@ extern int mock_copy_to_user_errors; extern int mock_cpu_idle; extern cycles_t mock_cycles; extern int mock_import_iovec_errors; -extern int mock_import_single_range_errors; +extern int mock_import_ubuf_errors; extern int mock_ip6_xmit_errors; extern int mock_ip_queue_xmit_errors; extern bool mock_ipv6; @@ -40,6 +39,8 @@ extern int mock_vmalloc_errors; extern int mock_xmit_log_verbose; extern int mock_xmit_log_homa_info; +extern struct page * + mock_alloc_pages(gfp_t gfp, unsigned order); extern int mock_check_error(int *errorMask); extern void mock_clear_xmit_prios(void); extern void mock_data_ready(struct sock *sk); @@ -52,6 +53,10 @@ extern int mock_page_refs(struct page *page); extern void mock_put_page(struct page *page); extern void mock_rcu_read_lock(void); extern void mock_rcu_read_unlock(void); +extern struct ctl_table_header * + mock_register_net_sysctl(struct net *net, + const char *path, struct ctl_table *table); +extern void mock_set_core(int num); extern void mock_spin_lock(spinlock_t *lock); extern void mock_spin_unlock(spinlock_t *lock); extern int mock_skb_count(void); @@ -63,3 +68,4 @@ extern void mock_sock_destroy(struct homa_sock *hsk, extern void mock_sock_init(struct homa_sock *hsk, struct homa *homa, int port); extern void mock_teardown(void); +extern void *mock_vmalloc(size_t size); diff --git a/test/unit_homa_grant.c b/test/unit_homa_grant.c index 5b1bb52..fcfc529 100644 --- a/test/unit_homa_grant.c +++ b/test/unit_homa_grant.c @@ -192,7 +192,7 @@ TEST_F(homa_grant, homa_grant_add_rpc__update_metrics) mock_cycles = 200; test_rpc(self, 100, self->server_ip, 100000); EXPECT_EQ(4, self->homa.num_grantable_rpcs); - EXPECT_EQ(300, homa_cores[cpu_number]->metrics.grantable_rpcs_integral); + EXPECT_EQ(300, core_metrics.grantable_rpcs_integral); EXPECT_EQ(200, self->homa.last_grantable_change); } TEST_F(homa_grant, homa_grant_add_rpc__insert_in_peer_list) @@ -335,7 +335,7 @@ TEST_F(homa_grant, homa_grant_remove_rpc__update_metrics) homa_grant_remove_rpc(rpc); EXPECT_EQ(2, self->homa.num_grantable_rpcs); - EXPECT_EQ(300, homa_cores[cpu_number]->metrics.grantable_rpcs_integral); + EXPECT_EQ(300, core_metrics.grantable_rpcs_integral); EXPECT_EQ(200, self->homa.last_grantable_change); } TEST_F(homa_grant, homa_grant_remove_rpc__not_first_in_peer_list) @@ -739,7 +739,7 @@ TEST_F(homa_grant, homa_grant_recalc__basics) EXPECT_EQ(2, atomic_read(&rpc2->msgin.rank)); EXPECT_EQ(-1, atomic_read(&rpc4->msgin.rank)); - EXPECT_NE(0, homa_cores[cpu_number]->metrics.grant_recalc_cycles); + EXPECT_NE(0, core_metrics.grant_recalc_cycles); } TEST_F(homa_grant, homa_grant_recalc__already_locked) { @@ -763,7 +763,7 @@ TEST_F(homa_grant, homa_grant_recalc__skip_recalc) EXPECT_STREQ("", unit_log_get()); EXPECT_EQ(0, rpc->msgin.granted); EXPECT_EQ(2, atomic_read(&self->homa.grant_recalc_count)); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.grant_recalc_skips); + EXPECT_EQ(1, core_metrics.grant_recalc_skips); } TEST_F(homa_grant, homa_grant_recalc__clear_existing_active_rpcs) { @@ -871,14 +871,14 @@ TEST_F(homa_grant, homa_grant_recalc__rpc_fully_granted_but_skip_recalc) unit_hook_register(grantable_spinlock_hook); hook_homa = &self->homa; mock_trylock_errors = 0xfe0; - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.grant_recalc_skips); + EXPECT_EQ(0, core_metrics.grant_recalc_skips); homa_grant_recalc(&self->homa, 0); EXPECT_EQ(10000, rpc1->msgin.granted); EXPECT_EQ(10000, rpc2->msgin.granted); EXPECT_EQ(0, rpc3->msgin.granted); EXPECT_EQ(0, rpc4->msgin.granted); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.grant_recalc_skips); + EXPECT_EQ(1, core_metrics.grant_recalc_skips); } TEST_F(homa_grant, homa_grant_pick_rpcs__basics) @@ -1073,8 +1073,8 @@ TEST_F(homa_grant, homa_grantable_lock_slow__basics) EXPECT_EQ(1, homa_grantable_lock_slow(&self->homa, 0)); homa_grantable_unlock(&self->homa); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.grantable_lock_misses); - EXPECT_EQ(500, homa_cores[cpu_number]->metrics.grantable_lock_miss_cycles); + EXPECT_EQ(1, core_metrics.grantable_lock_misses); + EXPECT_EQ(500, core_metrics.grantable_lock_miss_cycles); } TEST_F(homa_grant, homa_grantable_lock_slow__recalc_count) { @@ -1086,12 +1086,12 @@ TEST_F(homa_grant, homa_grantable_lock_slow__recalc_count) EXPECT_EQ(0, homa_grantable_lock_slow(&self->homa, 1)); hook_homa = NULL; - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.grantable_lock_misses); - EXPECT_EQ(500, homa_cores[cpu_number]->metrics.grantable_lock_miss_cycles); + EXPECT_EQ(1, core_metrics.grantable_lock_misses); + EXPECT_EQ(500, core_metrics.grantable_lock_miss_cycles); /* Make sure the check only occurs if the recalc argument is set. */ mock_trylock_errors = 0xff; EXPECT_EQ(1, homa_grantable_lock_slow(&self->homa, 0)); - EXPECT_EQ(2, homa_cores[cpu_number]->metrics.grantable_lock_misses); + EXPECT_EQ(2, core_metrics.grantable_lock_misses); homa_grantable_unlock(&self->homa); } \ No newline at end of file diff --git a/test/unit_homa_incoming.c b/test/unit_homa_incoming.c index 003b04e..86e8866 100644 --- a/test/unit_homa_incoming.c +++ b/test/unit_homa_incoming.c @@ -222,11 +222,11 @@ TEST_F(homa_incoming, homa_message_in_init__update_metrics) EXPECT_EQ(0, homa_message_in_init(crpc, 0x3000, 0)); EXPECT_EQ(0, homa_message_in_init(crpc, 1000000, 0)); EXPECT_EQ(0, homa_message_in_init(crpc, 900000, 0)); - EXPECT_EQ(270, homa_cores[cpu_number]->metrics.small_msg_bytes[2]); - EXPECT_EQ(0xfff, homa_cores[cpu_number]->metrics.small_msg_bytes[63]); - EXPECT_EQ(0x3000, homa_cores[cpu_number]->metrics.medium_msg_bytes[11]); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.medium_msg_bytes[15]); - EXPECT_EQ(1900000, homa_cores[cpu_number]->metrics.large_msg_bytes); + EXPECT_EQ(270, core_metrics.small_msg_bytes[2]); + EXPECT_EQ(0xfff, core_metrics.small_msg_bytes[63]); + EXPECT_EQ(0x3000, core_metrics.medium_msg_bytes[11]); + EXPECT_EQ(0, core_metrics.medium_msg_bytes[15]); + EXPECT_EQ(1900000, core_metrics.large_msg_bytes); } TEST_F(homa_incoming, homa_gap_retry) @@ -567,21 +567,21 @@ TEST_F(homa_incoming, homa_add_packet__metrics) homa_add_packet(crpc, mock_skb_new(self->client_ip, &self->data.common, 1400, 0)); EXPECT_EQ(0, skb_queue_len(&crpc->msgin.packets)); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.resent_discards); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.packet_discards); + EXPECT_EQ(0, core_metrics.resent_discards); + EXPECT_EQ(1, core_metrics.packet_discards); self->data.retransmit = 1; homa_add_packet(crpc, mock_skb_new(self->client_ip, &self->data.common, 1400, 0)); EXPECT_EQ(0, skb_queue_len(&crpc->msgin.packets)); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.resent_discards); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.packet_discards); + EXPECT_EQ(1, core_metrics.resent_discards); + EXPECT_EQ(1, core_metrics.packet_discards); self->data.seg.offset = htonl(4200); homa_add_packet(crpc, mock_skb_new(self->client_ip, &self->data.common, 1400, 4200)); EXPECT_EQ(1, skb_queue_len(&crpc->msgin.packets)); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.resent_packets_used); + EXPECT_EQ(1, core_metrics.resent_packets_used); } TEST_F(homa_incoming, homa_copy_to_user__basics) @@ -718,7 +718,7 @@ TEST_F(homa_incoming, homa_copy_to_user__error_in_import_single_range) ASSERT_NE(NULL, crpc); unit_log_clear(); - mock_import_single_range_errors = 1; + mock_import_ubuf_errors = 1; EXPECT_EQ(13, -homa_copy_to_user(crpc)); EXPECT_STREQ("", unit_log_get()); EXPECT_EQ(0, skb_queue_len(&crpc->msgin.packets)); @@ -859,7 +859,7 @@ TEST_F(homa_incoming, homa_dispatch_pkts__cant_create_server_rpc) 1400, 0), &self->homa); EXPECT_EQ(0, unit_list_length(&self->hsk.active_rpcs)); EXPECT_EQ(0, mock_skb_count()); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.server_cant_create_rpcs); + EXPECT_EQ(1, core_metrics.server_cant_create_rpcs); } TEST_F(homa_incoming, homa_dispatch_pkts__existing_server_rpc) { @@ -922,7 +922,7 @@ TEST_F(homa_incoming, homa_dispatch_pkts__unknown_client_rpc) mock_xmit_log_verbose = 1; homa_dispatch_pkts(mock_skb_new(self->client_ip, &h.common, 0, 0), &self->homa); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.unknown_rpcs); + EXPECT_EQ(1, core_metrics.unknown_rpcs); } TEST_F(homa_incoming, homa_dispatch_pkts__unknown_server_rpc) { @@ -933,7 +933,7 @@ TEST_F(homa_incoming, homa_dispatch_pkts__unknown_server_rpc) mock_xmit_log_verbose = 1; homa_dispatch_pkts(mock_skb_new(self->client_ip, &h.common, 0, 0), &self->homa); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.unknown_rpcs); + EXPECT_EQ(0, core_metrics.unknown_rpcs); } TEST_F(homa_incoming, homa_dispatch_pkts__cutoffs_for_unknown_client_rpc) { @@ -1009,7 +1009,7 @@ TEST_F(homa_incoming, homa_dispatch_pkts__unknown_type) .dport = htons(self->hsk.port), .sender_id = cpu_to_be64(self->server_id), .type = 99}; homa_dispatch_pkts(mock_skb_new(self->client_ip, &h, 0, 0), &self->homa); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.unknown_packet_types); + EXPECT_EQ(1, core_metrics.unknown_packet_types); } TEST_F(homa_incoming, homa_dispatch_pkts__handle_ack) { @@ -1077,7 +1077,7 @@ TEST_F(homa_incoming, homa_dispatch_pkts__forced_reap) homa_dispatch_pkts(mock_skb_new(self->client_ip, &self->data.common, 1400, 0), &self->homa); EXPECT_EQ(31, self->hsk.dead_skbs); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.data_pkt_reap_cycles); + EXPECT_EQ(0, core_metrics.data_pkt_reap_cycles); /* Second packet: must reap. */ self->homa.dead_buffs_limit = 15; @@ -1085,7 +1085,7 @@ TEST_F(homa_incoming, homa_dispatch_pkts__forced_reap) homa_dispatch_pkts(mock_skb_new(self->client_ip, &self->data.common, 1400, 0), &self->homa); EXPECT_EQ(21, self->hsk.dead_skbs); - EXPECT_NE(0, homa_cores[cpu_number]->metrics.data_pkt_reap_cycles); + EXPECT_NE(0, core_metrics.data_pkt_reap_cycles); } TEST_F(homa_incoming, homa_data_pkt__basics) @@ -1104,7 +1104,7 @@ TEST_F(homa_incoming, homa_data_pkt__basics) EXPECT_EQ(200, crpc->msgin.bytes_remaining); EXPECT_EQ(1, skb_queue_len(&crpc->msgin.packets)); EXPECT_EQ(1600, crpc->msgin.granted); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.responses_received); + EXPECT_EQ(1, core_metrics.responses_received); } TEST_F(homa_incoming, homa_data_pkt__wrong_client_rpc_state) { @@ -1169,7 +1169,7 @@ TEST_F(homa_incoming, homa_data_pkt__no_buffers) atomic_set(&self->hsk.buffer_pool.free_bpages, 0); homa_data_pkt(mock_skb_new(self->server_ip, &self->data.common, 1400, 0), crpc); - EXPECT_EQ(1400, homa_cores[cpu_number]->metrics.dropped_data_no_bufs); + EXPECT_EQ(1400, core_metrics.dropped_data_no_bufs); EXPECT_EQ(0, skb_queue_len(&crpc->msgin.packets)); } TEST_F(homa_incoming, homa_data_pkt__update_delta) @@ -1598,7 +1598,7 @@ TEST_F(homa_incoming, homa_cutoffs__cant_find_peer) struct sk_buff *skb = mock_skb_new(self->server_ip, &h.common, 0, 0); mock_kmalloc_errors = 1; homa_cutoffs_pkt(skb, &self->hsk); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.peer_kmalloc_errors); + EXPECT_EQ(1, core_metrics.peer_kmalloc_errors); peer = homa_peer_find(&self->homa.peers, self->server_ip, &self->hsk.inet); ASSERT_FALSE(IS_ERR(peer)); @@ -1622,7 +1622,7 @@ TEST_F(homa_incoming, homa_need_ack_pkt__rpc_response_fully_received) &self->homa); EXPECT_STREQ("xmit ACK from 0.0.0.0:32768, dport 99, id 1234, acks", unit_log_get()); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.packets_received[ + EXPECT_EQ(1, core_metrics.packets_received[ NEED_ACK - DATA]); } TEST_F(homa_incoming, homa_need_ack_pkt__rpc_response_not_fully_received) @@ -1641,7 +1641,7 @@ TEST_F(homa_incoming, homa_need_ack_pkt__rpc_response_not_fully_received) homa_dispatch_pkts(mock_skb_new(self->server_ip, &h.common, 0, 0), &self->homa); EXPECT_STREQ("", unit_log_get()); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.packets_received[ + EXPECT_EQ(1, core_metrics.packets_received[ NEED_ACK - DATA]); } TEST_F(homa_incoming, homa_need_ack_pkt__rpc_not_incoming) @@ -1660,7 +1660,7 @@ TEST_F(homa_incoming, homa_need_ack_pkt__rpc_not_incoming) homa_dispatch_pkts(mock_skb_new(self->server_ip, &h.common, 0, 0), &self->homa); EXPECT_STREQ("", unit_log_get()); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.packets_received[ + EXPECT_EQ(1, core_metrics.packets_received[ NEED_ACK - DATA]); } TEST_F(homa_incoming, homa_need_ack_pkt__rpc_doesnt_exist) @@ -1701,8 +1701,7 @@ TEST_F(homa_incoming, homa_ack_pkt__target_rpc_exists) homa_dispatch_pkts(mock_skb_new(self->client_ip, &h.common, 0, 0), &self->homa); EXPECT_EQ(0, unit_list_length(&self->hsk2.active_rpcs)); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.packets_received[ - ACK - DATA]); + EXPECT_EQ(1, core_metrics.packets_received[ACK - DATA]); } TEST_F(homa_incoming, homa_ack_pkt__target_rpc_doesnt_exist) { diff --git a/test/unit_homa_offload.c b/test/unit_homa_offload.c index 70de2ca..ae409a5 100644 --- a/test/unit_homa_offload.c +++ b/test/unit_homa_offload.c @@ -87,7 +87,7 @@ FIXTURE_SETUP(homa_offload) /* Configure so core isn't considered too busy for bypasses. */ mock_cycles = 1000; self->homa.gro_busy_cycles = 500; - homa_cores[cpu_number]->last_gro = 400; + cur_core->last_gro = 400; } FIXTURE_TEARDOWN(homa_offload) { @@ -159,10 +159,10 @@ TEST_F(homa_offload, homa_tcp_gro_receive__pass_to_homa_ipv6) h->flags = HOMA_TCP_FLAGS; h->urgent = htons(HOMA_TCP_URGENT); NAPI_GRO_CB(skb)->same_flow = 0; - homa_cores[cpu_number]->held_skb = NULL; - homa_cores[cpu_number]->held_bucket = 99; + cur_core->held_skb = NULL; + cur_core->held_bucket = 99; EXPECT_EQ(NULL, homa_tcp_gro_receive(&self->empty_list, skb)); - EXPECT_EQ(skb, homa_cores[cpu_number]->held_skb); + EXPECT_EQ(skb, cur_core->held_skb); EXPECT_STREQ("", unit_log_get()); EXPECT_EQ(IPPROTO_HOMA, ipv6_hdr(skb)->nexthdr); kfree_skb(skb); @@ -181,10 +181,10 @@ TEST_F(homa_offload, homa_tcp_gro_receive__pass_to_homa_ipv4) h->flags = HOMA_TCP_FLAGS; h->urgent = htons(HOMA_TCP_URGENT); NAPI_GRO_CB(skb)->same_flow = 0; - homa_cores[cpu_number]->held_skb = NULL; - homa_cores[cpu_number]->held_bucket = 99; + cur_core->held_skb = NULL; + cur_core->held_bucket = 99; EXPECT_EQ(NULL, homa_tcp_gro_receive(&self->empty_list, skb)); - EXPECT_EQ(skb, homa_cores[cpu_number]->held_skb); + EXPECT_EQ(skb, cur_core->held_skb); EXPECT_STREQ("", unit_log_get()); EXPECT_EQ(IPPROTO_HOMA, ip_hdr(skb)->protocol); EXPECT_EQ(2303, ip_hdr(skb)->check); @@ -220,8 +220,8 @@ TEST_F(homa_offload, homa_gro_receive__update_offset_from_sequence) self->header.seg.offset = -1; skb = mock_skb_new(&self->ip, &self->header.common, 1400, 0); NAPI_GRO_CB(skb)->same_flow = 0; - homa_cores[cpu_number]->held_skb = NULL; - homa_cores[cpu_number]->held_bucket = 99; + cur_core->held_skb = NULL; + cur_core->held_bucket = 99; EXPECT_EQ(NULL, homa_gro_receive(&self->empty_list, skb)); h = (struct data_header *) skb_transport_header(skb); EXPECT_EQ(6000, htonl(h->seg.offset)); @@ -267,33 +267,33 @@ TEST_F(homa_offload, homa_gro_receive__HOMA_GRO_SHORT_BYPASS) skb = mock_skb_new(&self->ip, &h.common, 1400, 2000); struct sk_buff *result = homa_gro_receive(&self->empty_list, skb); EXPECT_EQ(0, -PTR_ERR(result)); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.gro_data_bypasses); + EXPECT_EQ(0, core_metrics.gro_data_bypasses); /* Second attempt: HOMA_GRO_SHORT_BYPASS enabled but message longer * than one packet. */ self->homa.gro_policy |= HOMA_GRO_SHORT_BYPASS; - homa_cores[cpu_number]->last_gro = 400; + cur_core->last_gro = 400; skb2 = mock_skb_new(&self->ip, &h.common, 1400, 2000); result = homa_gro_receive(&self->empty_list, skb2); EXPECT_EQ(0, -PTR_ERR(result)); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.gro_data_bypasses); + EXPECT_EQ(0, core_metrics.gro_data_bypasses); /* Third attempt: bypass should happen. */ h.message_length = htonl(1400); h.incoming = htonl(1400); - homa_cores[cpu_number]->last_gro = 400; + cur_core->last_gro = 400; skb3 = mock_skb_new(&self->ip, &h.common, 1400, 4000); result = homa_gro_receive(&self->empty_list, skb3); EXPECT_EQ(EINPROGRESS, -PTR_ERR(result)); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.gro_data_bypasses); + EXPECT_EQ(1, core_metrics.gro_data_bypasses); /* Third attempt: no bypass because core busy. */ - homa_cores[cpu_number]->last_gro = 600; + cur_core->last_gro = 600; skb4 = mock_skb_new(&self->ip, &h.common, 1400, 4000); result = homa_gro_receive(&self->empty_list, skb3); EXPECT_EQ(0, -PTR_ERR(result)); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.gro_data_bypasses); + EXPECT_EQ(1, core_metrics.gro_data_bypasses); kfree_skb(skb); kfree_skb(skb2); @@ -326,24 +326,24 @@ TEST_F(homa_offload, homa_gro_receive__fast_grant_optimization) struct sk_buff *skb = mock_skb_new(&client_ip, &h.common, 0, 0); struct sk_buff *result = homa_gro_receive(&self->empty_list, skb); EXPECT_EQ(0, -PTR_ERR(result)); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.gro_grant_bypasses); + EXPECT_EQ(0, core_metrics.gro_grant_bypasses); EXPECT_STREQ("", unit_log_get()); /* Second attempt: HOMA_FAST_GRANTS is enabled. */ self->homa.gro_policy = HOMA_GRO_FAST_GRANTS; - homa_cores[cpu_number]->last_gro = 400; + cur_core->last_gro = 400; struct sk_buff *skb2 = mock_skb_new(&client_ip, &h.common, 0, 0); result = homa_gro_receive(&self->empty_list, skb2); EXPECT_EQ(EINPROGRESS, -PTR_ERR(result)); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.gro_grant_bypasses); + EXPECT_EQ(1, core_metrics.gro_grant_bypasses); EXPECT_SUBSTR("xmit DATA 1400@10000", unit_log_get()); /* Third attempt: core is too busy for fast grants. */ - homa_cores[cpu_number]->last_gro = 600; + cur_core->last_gro = 600; struct sk_buff *skb3 = mock_skb_new(&client_ip, &h.common, 0, 0); result = homa_gro_receive(&self->empty_list, skb3); EXPECT_EQ(0, -PTR_ERR(result)); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.gro_grant_bypasses); + EXPECT_EQ(1, core_metrics.gro_grant_bypasses); kfree_skb(skb); kfree_skb(skb3); } @@ -354,13 +354,13 @@ TEST_F(homa_offload, homa_gro_receive__no_held_skb) self->header.seg.offset = htonl(6000); skb = mock_skb_new(&self->ip, &self->header.common, 1400, 0); NAPI_GRO_CB(skb)->same_flow = 0; - homa_cores[cpu_number]->held_skb = NULL; - homa_cores[cpu_number]->held_bucket = 99; + cur_core->held_skb = NULL; + cur_core->held_bucket = 99; EXPECT_EQ(NULL, homa_gro_receive(&self->empty_list, skb)); same_flow = NAPI_GRO_CB(skb)->same_flow; EXPECT_EQ(0, same_flow); - EXPECT_EQ(skb, homa_cores[cpu_number]->held_skb); - EXPECT_EQ(3, homa_cores[cpu_number]->held_bucket); + EXPECT_EQ(skb, cur_core->held_skb); + EXPECT_EQ(3, cur_core->held_bucket); kfree_skb(skb); } TEST_F(homa_offload, homa_gro_receive__empty_merge_list) @@ -370,21 +370,21 @@ TEST_F(homa_offload, homa_gro_receive__empty_merge_list) self->header.seg.offset = htonl(6000); skb = mock_skb_new(&self->ip, &self->header.common, 1400, 0); NAPI_GRO_CB(skb)->same_flow = 0; - homa_cores[cpu_number]->held_skb = skb; - homa_cores[cpu_number]->held_bucket = 3; + cur_core->held_skb = skb; + cur_core->held_bucket = 3; EXPECT_EQ(NULL, homa_gro_receive(&self->empty_list, skb)); same_flow = NAPI_GRO_CB(skb)->same_flow; EXPECT_EQ(0, same_flow); - EXPECT_EQ(skb, homa_cores[cpu_number]->held_skb); - EXPECT_EQ(3, homa_cores[cpu_number]->held_bucket); + EXPECT_EQ(skb, cur_core->held_skb); + EXPECT_EQ(3, cur_core->held_bucket); kfree_skb(skb); } TEST_F(homa_offload, homa_gro_receive__merge) { struct sk_buff *skb, *skb2; int same_flow; - homa_cores[cpu_number]->held_skb = self->skb2; - homa_cores[cpu_number]->held_bucket = 2; + cur_core->held_skb = self->skb2; + cur_core->held_bucket = 2; self->header.seg.offset = htonl(6000); self->header.common.sender_id = cpu_to_be64(1002); @@ -419,8 +419,8 @@ TEST_F(homa_offload, homa_gro_receive__max_gro_skbs) // First packet: fits below the limit. homa->max_gro_skbs = 3; - homa_cores[cpu_number]->held_skb = self->skb2; - homa_cores[cpu_number]->held_bucket = 2; + cur_core->held_skb = self->skb2; + cur_core->held_bucket = 2; self->header.seg.offset = htonl(6000); skb = mock_skb_new(&self->ip, &self->header.common, 1400, 0); homa_gro_receive(&self->napi.gro_hash[3].list, skb); @@ -444,7 +444,7 @@ TEST_F(homa_offload, homa_gro_receive__max_gro_skbs) // Third packet also hits the limit for skb, causing the bucket // to become empty. homa->max_gro_skbs = 2; - homa_cores[cpu_number]->held_skb = self->skb; + cur_core->held_skb = self->skb; skb = mock_skb_new(&self->ip, &self->header.common, 1400, 0); unit_log_clear(); EXPECT_EQ(EINPROGRESS, -PTR_ERR(homa_gro_receive( @@ -462,7 +462,7 @@ TEST_F(homa_offload, homa_gro_gen2) homa->gro_policy = HOMA_GRO_GEN2; mock_cycles = 1000; homa->busy_cycles = 100; - cpu_number = 5; + mock_set_core(5); atomic_set(&homa_cores[6]->softirq_backlog, 1); homa_cores[6]->last_gro = 0; atomic_set(&homa_cores[7]->softirq_backlog, 0); @@ -497,7 +497,7 @@ TEST_F(homa_offload, homa_gro_gen2) TEST_F(homa_offload, homa_gro_gen3__basics) { homa->gro_policy = HOMA_GRO_GEN3; - struct homa_core *core = homa_cores[cpu_number]; + struct homa_core *core = cur_core; core->gen3_softirq_cores[0] = 3; core->gen3_softirq_cores[1] = 7; core->gen3_softirq_cores[2] = 5; @@ -515,7 +515,7 @@ TEST_F(homa_offload, homa_gro_gen3__basics) TEST_F(homa_offload, homa_gro_gen3__stop_on_negative_core_id) { homa->gro_policy = HOMA_GRO_GEN3; - struct homa_core *core = homa_cores[cpu_number]; + struct homa_core *core = cur_core; core->gen3_softirq_cores[0] = 3; core->gen3_softirq_cores[1] = -1; core->gen3_softirq_cores[2] = 5; @@ -531,7 +531,7 @@ TEST_F(homa_offload, homa_gro_gen3__stop_on_negative_core_id) TEST_F(homa_offload, homa_gro_gen3__all_cores_busy_so_pick_first) { homa->gro_policy = HOMA_GRO_GEN3; - struct homa_core *core = homa_cores[cpu_number]; + struct homa_core *core = cur_core; core->gen3_softirq_cores[0] = 3; core->gen3_softirq_cores[1] = 7; core->gen3_softirq_cores[2] = 5; @@ -555,16 +555,16 @@ TEST_F(homa_offload, homa_gro_complete__GRO_IDLE) homa_cores[1]->last_active = 15; homa_cores[2]->last_active = 10; - cpu_number = 5; + mock_set_core(5); homa_gro_complete(self->skb, 0); EXPECT_EQ(1, self->skb->hash - 32); homa_cores[6]->last_active = 5; - cpu_number = 5; + mock_set_core(5); homa_gro_complete(self->skb, 0); EXPECT_EQ(6, self->skb->hash - 32); - cpu_number = 6; + mock_set_core(6); homa_gro_complete(self->skb, 0); EXPECT_EQ(2, self->skb->hash - 32); } diff --git a/test/unit_homa_outgoing.c b/test/unit_homa_outgoing.c index 7b04e88..4ecd305 100644 --- a/test/unit_homa_outgoing.c +++ b/test/unit_homa_outgoing.c @@ -559,7 +559,7 @@ TEST_F(homa_outgoing, __homa_xmit_control__ipv4_error) mock_ip_queue_xmit_errors = 1; EXPECT_EQ(ENETDOWN, -homa_xmit_control(GRANT, &h, sizeof(h), srpc)); EXPECT_STREQ("", unit_log_get()); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.control_xmit_errors); + EXPECT_EQ(1, core_metrics.control_xmit_errors); } TEST_F(homa_outgoing, __homa_xmit_control__ipv6_error) { @@ -583,7 +583,7 @@ TEST_F(homa_outgoing, __homa_xmit_control__ipv6_error) mock_ip6_xmit_errors = 1; EXPECT_EQ(ENETDOWN, -homa_xmit_control(GRANT, &h, sizeof(h), srpc)); EXPECT_STREQ("", unit_log_get()); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.control_xmit_errors); + EXPECT_EQ(1, core_metrics.control_xmit_errors); } TEST_F(homa_outgoing, homa_xmit_unknown) @@ -741,13 +741,13 @@ TEST_F(homa_outgoing, __homa_xmit_data__fill_dst) self->server_port, self->client_id, 1000, 1000); unit_log_clear(); dst = crpc->peer->dst; - old_refcount = dst->__refcnt.counter; + old_refcount = atomic_read(&dst->__rcuref.refcnt); skb_get(crpc->msgout.packets); __homa_xmit_data(crpc->msgout.packets, crpc, 6); EXPECT_STREQ("xmit DATA 1000@0", unit_log_get()); EXPECT_EQ(dst, skb_dst(crpc->msgout.packets)); - EXPECT_EQ(old_refcount+1, dst->__refcnt.counter); + EXPECT_EQ(old_refcount+1, atomic_read(&dst->__rcuref.refcnt)); } TEST_F(homa_outgoing, __homa_xmit_data__ipv4_transmit_error) { @@ -763,7 +763,7 @@ TEST_F(homa_outgoing, __homa_xmit_data__ipv4_transmit_error) mock_ip_queue_xmit_errors = 1; skb_get(crpc->msgout.packets); __homa_xmit_data(crpc->msgout.packets, crpc, 5); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.data_xmit_errors); + EXPECT_EQ(1, core_metrics.data_xmit_errors); } TEST_F(homa_outgoing, __homa_xmit_data__ipv6_transmit_error) { @@ -779,7 +779,7 @@ TEST_F(homa_outgoing, __homa_xmit_data__ipv6_transmit_error) mock_ip6_xmit_errors = 1; skb_get(crpc->msgout.packets); __homa_xmit_data(crpc->msgout.packets, crpc, 5); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.data_xmit_errors); + EXPECT_EQ(1, core_metrics.data_xmit_errors); } TEST_F(homa_outgoing, homa_resend_data__basics) @@ -984,8 +984,8 @@ TEST_F(homa_outgoing, homa_check_nic_queue__pacer_metrics) EXPECT_EQ(1, homa_check_nic_queue(&self->homa, crpc->msgout.packets, true)); EXPECT_EQ(10500, atomic64_read(&self->homa.link_idle_time)); - EXPECT_EQ(500, homa_cores[cpu_number]->metrics.pacer_bytes); - EXPECT_EQ(200, homa_cores[cpu_number]->metrics.pacer_lost_cycles); + EXPECT_EQ(500, core_metrics.pacer_bytes); + EXPECT_EQ(200, core_metrics.pacer_lost_cycles); } TEST_F(homa_outgoing, homa_check_nic_queue__queue_empty) { @@ -1141,7 +1141,7 @@ TEST_F(homa_outgoing, homa_pacer_xmit__rpc_locked) mock_trylock_errors = ~1; homa_pacer_xmit(&self->homa); EXPECT_STREQ("", unit_log_get()); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.pacer_skipped_rpcs); + EXPECT_EQ(1, core_metrics.pacer_skipped_rpcs); unit_log_clear(); mock_trylock_errors = 0; homa_pacer_xmit(&self->homa); @@ -1234,16 +1234,16 @@ TEST_F(homa_outgoing, homa_add_to_throttled__inc_metrics) self->server_port, self->client_id+4, 15000, 1000); homa_add_to_throttled(crpc1); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.throttle_list_adds); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.throttle_list_checks); + EXPECT_EQ(1, core_metrics.throttle_list_adds); + EXPECT_EQ(0, core_metrics.throttle_list_checks); homa_add_to_throttled(crpc2); - EXPECT_EQ(2, homa_cores[cpu_number]->metrics.throttle_list_adds); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.throttle_list_checks); + EXPECT_EQ(2, core_metrics.throttle_list_adds); + EXPECT_EQ(1, core_metrics.throttle_list_checks); homa_add_to_throttled(crpc3); - EXPECT_EQ(3, homa_cores[cpu_number]->metrics.throttle_list_adds); - EXPECT_EQ(3, homa_cores[cpu_number]->metrics.throttle_list_checks); + EXPECT_EQ(3, core_metrics.throttle_list_adds); + EXPECT_EQ(3, core_metrics.throttle_list_checks); } TEST_F(homa_outgoing, homa_remove_from_throttled) diff --git a/test/unit_homa_peertab.c b/test/unit_homa_peertab.c index 006312a..d1645ca 100644 --- a/test/unit_homa_peertab.c +++ b/test/unit_homa_peertab.c @@ -72,7 +72,7 @@ TEST_F(homa_peertab, homa_peer_find__basics) peer2 = homa_peer_find(&self->peertab, ip2222, &self->hsk.inet); EXPECT_NE(peer, peer2); - EXPECT_EQ(2, homa_cores[cpu_number]->metrics.peer_new_entries); + EXPECT_EQ(2, core_metrics.peer_new_entries); } static struct _test_data_homa_peertab *test_data; @@ -191,7 +191,7 @@ TEST_F(homa_peertab, homa_peer_find__kmalloc_error) peer = homa_peer_find(&self->peertab, ip3333, &self->hsk.inet); EXPECT_EQ(ENOMEM, -PTR_ERR(peer)); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.peer_kmalloc_errors); + EXPECT_EQ(1, core_metrics.peer_kmalloc_errors); } TEST_F(homa_peertab, homa_peer_find__route_error) { @@ -201,7 +201,7 @@ TEST_F(homa_peertab, homa_peer_find__route_error) peer = homa_peer_find(&self->peertab, ip3333, &self->hsk.inet); EXPECT_EQ(EHOSTUNREACH, -PTR_ERR(peer)); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.peer_route_errors); + EXPECT_EQ(1, core_metrics.peer_route_errors); } TEST_F(homa_peertab, homa_dst_refresh__basics) @@ -229,7 +229,7 @@ TEST_F(homa_peertab, homa_dst_refresh__routing_error) mock_route_errors = 1; homa_dst_refresh(&self->homa.peers, peer, &self->hsk); EXPECT_EQ(old_dst, peer->dst); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.peer_route_errors); + EXPECT_EQ(1, core_metrics.peer_route_errors); EXPECT_EQ(0, dead_count(&self->homa.peers)); } TEST_F(homa_peertab, homa_dst_refresh__malloc_error) @@ -324,15 +324,15 @@ TEST_F(homa_peertab, homa_peer_lock_slow) ASSERT_NE(NULL, peer); homa_peer_lock(peer); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.peer_ack_lock_misses); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.peer_ack_lock_miss_cycles); + EXPECT_EQ(0, core_metrics.peer_ack_lock_misses); + EXPECT_EQ(0, core_metrics.peer_ack_lock_miss_cycles); homa_peer_unlock(peer); mock_trylock_errors = 1; unit_hook_register(peer_spinlock_hook); homa_peer_lock(peer); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.peer_ack_lock_misses); - EXPECT_EQ(1000, homa_cores[cpu_number]->metrics.peer_ack_lock_miss_cycles); + EXPECT_EQ(1, core_metrics.peer_ack_lock_misses); + EXPECT_EQ(1000, core_metrics.peer_ack_lock_miss_cycles); homa_peer_unlock(peer); } diff --git a/test/unit_homa_plumbing.c b/test/unit_homa_plumbing.c index a671de4..556752f 100644 --- a/test/unit_homa_plumbing.c +++ b/test/unit_homa_plumbing.c @@ -189,7 +189,7 @@ TEST_F(homa_plumbing, homa_ioc_abort__basics) UNIT_OUTGOING, self->client_ip, self->server_ip, self->server_port, self->client_id, 10000, 200); ASSERT_NE(NULL, crpc); - EXPECT_EQ(0, homa_ioc_abort(&self->hsk.inet.sk, (unsigned long) &args)); + EXPECT_EQ(0, homa_ioc_abort(&self->hsk.inet.sk, (int *) &args)); EXPECT_EQ(RPC_DEAD, crpc->state); EXPECT_EQ(0, unit_list_length(&self->hsk.active_rpcs)); } @@ -197,8 +197,7 @@ TEST_F(homa_plumbing, homa_ioc_abort__cant_read_user_args) { struct homa_abort_args args = {self->client_id, 0}; mock_copy_data_errors = 1; - EXPECT_EQ(EFAULT, -homa_ioc_abort(&self->hsk.inet.sk, - (unsigned long) &args)); + EXPECT_EQ(EFAULT, -homa_ioc_abort(&self->hsk.inet.sk, (int *) &args)); } TEST_F(homa_plumbing, homa_ioc_abort__abort_multiple_rpcs) { @@ -211,7 +210,7 @@ TEST_F(homa_plumbing, homa_ioc_abort__abort_multiple_rpcs) self->server_port, self->client_id, 10000, 200); ASSERT_NE(NULL, crpc1); ASSERT_NE(NULL, crpc2); - EXPECT_EQ(0, homa_ioc_abort(&self->hsk.inet.sk, (unsigned long) &args)); + EXPECT_EQ(0, homa_ioc_abort(&self->hsk.inet.sk, (int *) &args)); EXPECT_EQ(-ECANCELED, crpc1->error); EXPECT_EQ(-ECANCELED, crpc2->error); EXPECT_EQ(2, unit_list_length(&self->hsk.active_rpcs)); @@ -219,8 +218,7 @@ TEST_F(homa_plumbing, homa_ioc_abort__abort_multiple_rpcs) TEST_F(homa_plumbing, homa_ioc_abort__nonexistent_rpc) { struct homa_abort_args args = {99, 0}; - EXPECT_EQ(EINVAL, -homa_ioc_abort(&self->hsk.inet.sk, - (unsigned long) &args)); + EXPECT_EQ(EINVAL, -homa_ioc_abort(&self->hsk.inet.sk, (int *) &args)); } TEST_F(homa_plumbing, homa_set_sock_opt__bad_level) @@ -270,7 +268,7 @@ TEST_F(homa_plumbing, homa_set_sock_opt__success) sizeof(struct homa_set_buf_args))); EXPECT_EQ(args.start, self->hsk.buffer_pool.region); EXPECT_EQ(64, self->hsk.buffer_pool.num_bpages); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.so_set_buf_calls); + EXPECT_EQ(1, core_metrics.so_set_buf_calls); } TEST_F(homa_plumbing, homa_sendmsg__args_not_in_user_space) @@ -688,7 +686,7 @@ TEST_F(homa_plumbing, homa_softirq__packet_too_short) skb->len -= 1; homa_softirq(skb); EXPECT_EQ(0, unit_list_length(&self->hsk.active_rpcs)); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.short_packets); + EXPECT_EQ(1, core_metrics.short_packets); } TEST_F(homa_plumbing, homa_softirq__bogus_packet_type) { @@ -697,7 +695,7 @@ TEST_F(homa_plumbing, homa_softirq__bogus_packet_type) skb = mock_skb_new(self->client_ip, &self->data.common, 1400, 1400); homa_softirq(skb); EXPECT_EQ(0, unit_list_length(&self->hsk.active_rpcs)); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.short_packets); + EXPECT_EQ(1, core_metrics.short_packets); } TEST_F(homa_plumbing, homa_softirq__process_short_messages_first) { diff --git a/test/unit_homa_pool.c b/test/unit_homa_pool.c index 628b131..3970d5b 100644 --- a/test/unit_homa_pool.c +++ b/test/unit_homa_pool.c @@ -61,7 +61,8 @@ static void change_owner_hook(char *id) return; if (!cur_pool) return; - cur_pool->descriptors[cur_pool->cores[cpu_number].page_hint].owner = -1; + cur_pool->descriptors[cur_pool->cores[raw_smp_processor_id()] + .page_hint].owner = -1; } TEST_F(homa_pool, homa_pool_set_bpages_needed) @@ -126,7 +127,7 @@ TEST_F(homa_pool, homa_pool_get_pages__basics) EXPECT_EQ(1, pages[1]); EXPECT_EQ(1, atomic_read(&pool->descriptors[1].refs)); EXPECT_EQ(-1, pool->descriptors[1].owner); - EXPECT_EQ(2, pool->cores[cpu_number].next_candidate); + EXPECT_EQ(2, pool->cores[raw_smp_processor_id()].next_candidate); EXPECT_EQ(98, atomic_read(&pool->free_bpages)); } TEST_F(homa_pool, homa_pool_get_pages__not_enough_space) @@ -143,7 +144,7 @@ TEST_F(homa_pool, homa_pool_get_pages__set_limit) struct homa_pool *pool = &self->hsk.buffer_pool; __u32 pages[10]; atomic_set(&pool->free_bpages, 62); - pool->cores[cpu_number].next_candidate = 49; + pool->cores[raw_smp_processor_id()].next_candidate = 49; EXPECT_EQ(0, homa_pool_get_pages(pool, 2, pages, 0)); EXPECT_EQ(49, pages[0]); EXPECT_EQ(0, pages[1]); @@ -153,7 +154,7 @@ TEST_F(homa_pool, homa_pool_get_pages__set_limit_with_MIN_EXTRA) struct homa_pool *pool = &self->hsk.buffer_pool; __u32 pages[10]; atomic_set(&pool->free_bpages, 92); - pool->cores[cpu_number].next_candidate = 13; + pool->cores[raw_smp_processor_id()].next_candidate = 13; EXPECT_EQ(0, homa_pool_get_pages(pool, 2, pages, 0)); EXPECT_EQ(13, pages[0]); EXPECT_EQ(0, pages[1]); @@ -234,9 +235,9 @@ TEST_F(homa_pool, homa_pool_allocate__basics) EXPECT_EQ(0, crpc->msgin.bpage_offsets[0]); EXPECT_EQ(-1, pool->descriptors[0].owner); EXPECT_EQ(2*HOMA_BPAGE_SIZE, crpc->msgin.bpage_offsets[2]); - EXPECT_EQ(2, pool->cores[cpu_number].page_hint); + EXPECT_EQ(2, pool->cores[raw_smp_processor_id()].page_hint); EXPECT_EQ(150000 - 2*HOMA_BPAGE_SIZE, - pool->cores[cpu_number].allocated); + pool->cores[raw_smp_processor_id()].allocated); } TEST_F(homa_pool, homa_pool_no_buffer_pool) { @@ -278,7 +279,7 @@ TEST_F(homa_pool, homa_pool_allocate__no_partial_page) TEST_F(homa_pool, homa_pool_allocate__owned_page_locked_and_page_stolen) { struct homa_pool *pool = &self->hsk.buffer_pool; - pool->cores[cpu_number].next_candidate = 2; + pool->cores[raw_smp_processor_id()].next_candidate = 2; atomic_set(&pool->free_bpages, 40); struct homa_rpc *crpc = unit_client_rpc(&self->hsk, UNIT_RCVD_ONE_PKT, &self->client_ip, &self->server_ip, @@ -286,7 +287,7 @@ TEST_F(homa_pool, homa_pool_allocate__owned_page_locked_and_page_stolen) ASSERT_NE(NULL, crpc); // First allocation just sets up a partially-allocated bpage. - EXPECT_EQ(2, pool->cores[cpu_number].page_hint); + EXPECT_EQ(2, pool->cores[raw_smp_processor_id()].page_hint); // Try a second allocation; the lock hook steals the partial bpage, // so a new one has to be allocated. @@ -296,8 +297,8 @@ TEST_F(homa_pool, homa_pool_allocate__owned_page_locked_and_page_stolen) EXPECT_EQ(0, homa_pool_allocate(crpc)); EXPECT_EQ(1, crpc->msgin.num_bpages); EXPECT_EQ(3*HOMA_BPAGE_SIZE, crpc->msgin.bpage_offsets[0]); - EXPECT_EQ(3, pool->cores[cpu_number].page_hint); - EXPECT_EQ(2000, pool->cores[cpu_number].allocated); + EXPECT_EQ(3, pool->cores[raw_smp_processor_id()].page_hint); + EXPECT_EQ(2000, pool->cores[raw_smp_processor_id()].allocated); EXPECT_EQ(1, -pool->descriptors[2].owner); EXPECT_EQ(1, pool->descriptors[3].owner); EXPECT_EQ(38, atomic_read(&pool->free_bpages)); @@ -305,40 +306,40 @@ TEST_F(homa_pool, homa_pool_allocate__owned_page_locked_and_page_stolen) TEST_F(homa_pool, homa_pool_allocate__page_wrap_around) { struct homa_pool *pool = &self->hsk.buffer_pool; - pool->cores[cpu_number].page_hint = 2; - pool->cores[cpu_number].allocated = HOMA_BPAGE_SIZE-1900; + pool->cores[raw_smp_processor_id()].page_hint = 2; + pool->cores[raw_smp_processor_id()].allocated = HOMA_BPAGE_SIZE-1900; atomic_set(&pool->descriptors[2].refs, 1); - pool->descriptors[2].owner = cpu_number; + pool->descriptors[2].owner = raw_smp_processor_id(); struct homa_rpc *crpc = unit_client_rpc(&self->hsk, UNIT_RCVD_ONE_PKT, &self->client_ip, &self->server_ip, 4000, 98, 1000, 2000); ASSERT_NE(NULL, crpc); - EXPECT_EQ(2, pool->cores[cpu_number].page_hint); + EXPECT_EQ(2, pool->cores[raw_smp_processor_id()].page_hint); EXPECT_EQ(1, crpc->msgin.num_bpages); EXPECT_EQ(2*HOMA_BPAGE_SIZE, crpc->msgin.bpage_offsets[0]); - EXPECT_EQ(2000, pool->cores[cpu_number].allocated); - EXPECT_EQ(cpu_number, pool->descriptors[2].owner); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.bpage_reuses); + EXPECT_EQ(2000, pool->cores[raw_smp_processor_id()].allocated); + EXPECT_EQ(raw_smp_processor_id(), pool->descriptors[2].owner); + EXPECT_EQ(1, core_metrics.bpage_reuses); } TEST_F(homa_pool, homa_pool_allocate__owned_page_overflow) { struct homa_pool *pool = &self->hsk.buffer_pool; - pool->cores[cpu_number].next_candidate = 2; + pool->cores[raw_smp_processor_id()].next_candidate = 2; atomic_set(&pool->free_bpages, 50); struct homa_rpc *crpc = unit_client_rpc(&self->hsk, UNIT_RCVD_ONE_PKT, &self->client_ip, &self->server_ip, 4000, 98, 1000, 2000); ASSERT_NE(NULL, crpc); - EXPECT_EQ(2, pool->cores[cpu_number].page_hint); + EXPECT_EQ(2, pool->cores[raw_smp_processor_id()].page_hint); crpc->msgin.num_bpages = 0; - pool->cores[cpu_number].allocated = HOMA_BPAGE_SIZE-1900; + pool->cores[raw_smp_processor_id()].allocated = HOMA_BPAGE_SIZE-1900; EXPECT_EQ(0, homa_pool_allocate(crpc)); EXPECT_EQ(1, crpc->msgin.num_bpages); EXPECT_EQ(3*HOMA_BPAGE_SIZE, crpc->msgin.bpage_offsets[0]); - EXPECT_EQ(3, pool->cores[cpu_number].page_hint); - EXPECT_EQ(2000, pool->cores[cpu_number].allocated); + EXPECT_EQ(3, pool->cores[raw_smp_processor_id()].page_hint); + EXPECT_EQ(2000, pool->cores[raw_smp_processor_id()].allocated); EXPECT_EQ(-1, pool->descriptors[2].owner); EXPECT_EQ(1, atomic_read(&pool->descriptors[2].refs)); EXPECT_EQ(1, pool->descriptors[3].owner); @@ -347,7 +348,7 @@ TEST_F(homa_pool, homa_pool_allocate__owned_page_overflow) TEST_F(homa_pool, homa_pool_allocate__reuse_owned_page) { struct homa_pool *pool = &self->hsk.buffer_pool; - pool->cores[cpu_number].next_candidate = 2; + pool->cores[raw_smp_processor_id()].next_candidate = 2; struct homa_rpc *crpc1 = unit_client_rpc(&self->hsk, UNIT_RCVD_ONE_PKT, &self->client_ip, &self->server_ip, 4000, 98, 1000, 2000); @@ -362,8 +363,8 @@ TEST_F(homa_pool, homa_pool_allocate__reuse_owned_page) EXPECT_EQ(1, crpc2->msgin.num_bpages); EXPECT_EQ(2*HOMA_BPAGE_SIZE + 2000, crpc2->msgin.bpage_offsets[0]); EXPECT_EQ(3, atomic_read(&pool->descriptors[2].refs)); - EXPECT_EQ(2, pool->cores[cpu_number].page_hint); - EXPECT_EQ(5000, pool->cores[cpu_number].allocated); + EXPECT_EQ(2, pool->cores[raw_smp_processor_id()].page_hint); + EXPECT_EQ(5000, pool->cores[raw_smp_processor_id()].allocated); } TEST_F(homa_pool, homa_pool_allocate__cant_allocate_partial_bpage) { @@ -404,7 +405,7 @@ TEST_F(homa_pool, homa_pool_allocate__out_of_space) rpc = list_next_entry(rpc, buf_links); EXPECT_EQ(100, rpc->id); EXPECT_TRUE(list_is_last(&rpc->buf_links, &self->hsk.waiting_for_bufs)); - EXPECT_EQ(3, homa_cores[cpu_number]->metrics.buffer_alloc_failures); + EXPECT_EQ(3, core_metrics.buffer_alloc_failures); EXPECT_EQ(1, pool->bpages_needed); } diff --git a/test/unit_homa_skb.c b/test/unit_homa_skb.c index 34259f0..6de0ef1 100644 --- a/test/unit_homa_skb.c +++ b/test/unit_homa_skb.c @@ -95,7 +95,7 @@ TEST_F(homa_skb, homa_skb_cleanup) core->skb_page = alloc_pages(GFP_KERNEL, 2); add_to_pool(&self->homa, 5, 2); add_to_pool(&self->homa, 4, 3); - cpu_number = 3; + mock_set_core(3); homa_skb_stash_pages(&self->homa, 2 * HOMA_SKB_PAGE_SIZE - 100); EXPECT_EQ(5, homa_cores[2]->numa->page_pool.avail); EXPECT_EQ(2, homa_cores[3]->numa->page_pool.avail); @@ -195,7 +195,7 @@ TEST_F(homa_skb, homa_skb_extend_frags__cant_merge_allocate_new_page) EXPECT_NE(NULL, p3); EXPECT_EQ(1000, length); EXPECT_EQ(2, skb_shinfo(self->skb)->nr_frags); - EXPECT_EQ(0, skb_shinfo(self->skb)->frags[1].bv_offset); + EXPECT_EQ(0, skb_shinfo(self->skb)->frags[1].offset); EXPECT_EQ(2000, self->skb->len); EXPECT_EQ(1000, core->page_inuse); @@ -223,7 +223,7 @@ TEST_F(homa_skb, homa_skb_extend_frags__cant_merge_use_same_page_reduce_length) EXPECT_EQ(p2 + 512, p3); EXPECT_EQ(512, length); EXPECT_EQ(2, skb_shinfo(self->skb)->nr_frags); - EXPECT_EQ(1536, skb_shinfo(self->skb)->frags[1].bv_offset); + EXPECT_EQ(1536, skb_shinfo(self->skb)->frags[1].offset); EXPECT_EQ(2048, core->page_inuse); kfree_skb(skb2); @@ -246,7 +246,7 @@ TEST_F(homa_skb, homa_skb_page_alloc__free_previous_page) } TEST_F(homa_skb, homa_skb_page_alloc__reuse_existing_page) { - struct homa_core *core = homa_cores[cpu_number]; + struct homa_core *core = cur_core; struct sk_buff *skb = homa_skb_new_tx(100); struct page *page; int length = 100; @@ -262,8 +262,8 @@ TEST_F(homa_skb, homa_skb_page_alloc__reuse_existing_page) } TEST_F(homa_skb, homa_skb_page_alloc__from_stash) { - struct homa_core *core = homa_cores[cpu_number]; - add_to_pool(&self->homa, 5, cpu_number); + struct homa_core *core = cur_core; + add_to_pool(&self->homa, 5, raw_smp_processor_id()); homa_skb_stash_pages(&self->homa, 3*HOMA_SKB_PAGE_SIZE - 100); EXPECT_TRUE(homa_skb_page_alloc(&self->homa, core)); EXPECT_NE(NULL, core->skb_page); @@ -273,8 +273,8 @@ TEST_F(homa_skb, homa_skb_page_alloc__from_stash) } TEST_F(homa_skb, homa_skb_page_alloc__from_pool) { - struct homa_core *core = homa_cores[cpu_number]; - add_to_pool(&self->homa, 5, cpu_number); + struct homa_core *core = cur_core; + add_to_pool(&self->homa, 5, raw_smp_processor_id()); EXPECT_EQ(5, core->numa->page_pool.avail); EXPECT_EQ(0, core->num_stashed_pages); EXPECT_TRUE(homa_skb_page_alloc(&self->homa, core)); @@ -283,8 +283,8 @@ TEST_F(homa_skb, homa_skb_page_alloc__from_pool) } TEST_F(homa_skb, homa_skb_page_alloc__pool_page_taken_while_locking) { - struct homa_core *core = homa_cores[cpu_number]; - add_to_pool(&self->homa, 1, cpu_number); + struct homa_core *core = cur_core; + add_to_pool(&self->homa, 1, raw_smp_processor_id()); EXPECT_EQ(1, core->numa->page_pool.avail); EXPECT_EQ(0, core->num_stashed_pages); hook_pool = &core->numa->page_pool; @@ -297,15 +297,15 @@ TEST_F(homa_skb, homa_skb_page_alloc__pool_page_taken_while_locking) } TEST_F(homa_skb, homa_skb_page_alloc__new_large_page) { - struct homa_core *core = homa_cores[cpu_number]; + struct homa_core *core = cur_core; mock_cycles = ~0; EXPECT_EQ(0, core->numa->page_pool.avail); EXPECT_EQ(0, core->num_stashed_pages); EXPECT_TRUE(homa_skb_page_alloc(&self->homa, core)); EXPECT_NE(NULL, core->skb_page); EXPECT_EQ(HOMA_SKB_PAGE_SIZE, core->page_size); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.skb_page_allocs); - EXPECT_NE(0, homa_cores[cpu_number]->metrics.skb_page_alloc_cycles); + EXPECT_EQ(1, core_metrics.skb_page_allocs); + EXPECT_NE(0, core_metrics.skb_page_alloc_cycles); } TEST_F(homa_skb, homa_skb_page_alloc__high_order_page_not_available) { @@ -317,8 +317,8 @@ TEST_F(homa_skb, homa_skb_page_alloc__high_order_page_not_available) EXPECT_NE(NULL, core->skb_page); EXPECT_EQ(PAGE_SIZE, core->page_size); EXPECT_EQ(0, core->page_inuse); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.skb_page_allocs); - EXPECT_NE(0, homa_cores[cpu_number]->metrics.skb_page_alloc_cycles); + EXPECT_EQ(1, core_metrics.skb_page_allocs); + EXPECT_NE(0, core_metrics.skb_page_alloc_cycles); } TEST_F(homa_skb, homa_skb_page_alloc__no_pages_available) { @@ -342,15 +342,15 @@ TEST_F(homa_skb, homa_skb_append_to_frag__basics) "0123456789ABCDEFGHIJ", 21)); EXPECT_EQ(2, shinfo->nr_frags); - EXPECT_EQ(10, shinfo->frags[0].bv_len); - char *p = ((char *) page_address(shinfo->frags[0].bv_page)) - + shinfo->frags[0].bv_offset; - p[shinfo->frags[0].bv_len] = 0; + EXPECT_EQ(10, skb_frag_size(&shinfo->frags[0])); + char *p = ((char *) page_address(skb_frag_page(&shinfo->frags[0]))) + + shinfo->frags[0].offset; + p[skb_frag_size(&shinfo->frags[0])] = 0; EXPECT_STREQ("abcd012345", p); - EXPECT_EQ(15, shinfo->frags[1].bv_len); - p = ((char *) page_address(shinfo->frags[1].bv_page)) - + shinfo->frags[1].bv_offset; + EXPECT_EQ(15, skb_frag_size(&shinfo->frags[1])); + p = ((char *) page_address(skb_frag_page(&shinfo->frags[1]))) + + shinfo->frags[1].offset; EXPECT_STREQ("6789ABCDEFGHIJ", p); } TEST_F(homa_skb, homa_skb_append_to_frag__no_memory) @@ -383,8 +383,8 @@ TEST_F(homa_skb, homa_skb_append_from_iter__basics) unit_log_get()); EXPECT_EQ(2, shinfo->nr_frags); - EXPECT_EQ(4096, shinfo->frags[0].bv_len); - EXPECT_EQ(904, shinfo->frags[1].bv_len); + EXPECT_EQ(4096, skb_frag_size(&shinfo->frags[0])); + EXPECT_EQ(904, skb_frag_size(&shinfo->frags[1])); } TEST_F(homa_skb, homa_skb_append_from_iter__no_memory) { @@ -513,7 +513,7 @@ TEST_F(homa_skb, homa_skb_free_many_tx__skb_ref_count_not_one) length = HOMA_SKB_PAGE_SIZE; homa_skb_extend_frags(&self->homa, skb, &length); EXPECT_EQ(HOMA_SKB_PAGE_SIZE, length); - page = skb_shinfo(skb)->frags[0].bv_page; + page = skb_frag_page(&skb_shinfo(skb)->frags[0]); EXPECT_EQ(2, page_ref_count(page)); skb_get(skb); EXPECT_EQ(2, refcount_read(&skb->users)); @@ -535,7 +535,7 @@ TEST_F(homa_skb, homa_skb_free_many_tx__check_page_order) homa_skb_extend_frags(&self->homa, skb, &length); } EXPECT_EQ(HOMA_SKB_PAGE_SIZE, length); - struct page *page = skb_shinfo(skb)->frags[2].bv_page; + struct page *page = skb_frag_page(&skb_shinfo(skb)->frags[2]); mock_compound_order_mask = 3; homa_skb_free_many_tx(&self->homa, &skb, 1); diff --git a/test/unit_homa_socktab.c b/test/unit_homa_socktab.c index ef7f018..646182a 100644 --- a/test/unit_homa_socktab.c +++ b/test/unit_homa_socktab.c @@ -294,13 +294,13 @@ TEST_F(homa_socktab, homa_sock_lock_slow) mock_cycles = ~0; homa_sock_lock(&self->hsk, "unit test"); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.socket_lock_misses); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.socket_lock_miss_cycles); + EXPECT_EQ(0, core_metrics.socket_lock_misses); + EXPECT_EQ(0, core_metrics.socket_lock_miss_cycles); homa_sock_unlock(&self->hsk); mock_trylock_errors = 1; homa_sock_lock(&self->hsk, "unit test"); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.socket_lock_misses); - EXPECT_NE(0, homa_cores[cpu_number]->metrics.socket_lock_miss_cycles); + EXPECT_EQ(1, core_metrics.socket_lock_misses); + EXPECT_NE(0, core_metrics.socket_lock_miss_cycles); homa_sock_unlock(&self->hsk); } \ No newline at end of file diff --git a/test/unit_homa_timer.c b/test/unit_homa_timer.c index 28c0015..cd5193e 100644 --- a/test/unit_homa_timer.c +++ b/test/unit_homa_timer.c @@ -136,11 +136,11 @@ TEST_F(homa_timer, homa_check_rpc__timeout) unit_log_clear(); crpc->silent_ticks = self->homa.timeout_ticks-1; homa_check_rpc(crpc); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.rpc_timeouts); + EXPECT_EQ(0, core_metrics.rpc_timeouts); EXPECT_EQ(0, crpc->error); crpc->silent_ticks = self->homa.timeout_ticks; homa_check_rpc(crpc); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.rpc_timeouts); + EXPECT_EQ(1, core_metrics.rpc_timeouts); EXPECT_EQ(ETIMEDOUT, -crpc->error); } TEST_F(homa_timer, homa_check_rpc__issue_resend) @@ -250,7 +250,7 @@ TEST_F(homa_timer, homa_timer__basics) unit_log_clear(); crpc->peer->outstanding_resends = self->homa.timeout_resends; homa_timer(&self->homa); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.rpc_timeouts); + EXPECT_EQ(1, core_metrics.rpc_timeouts); EXPECT_EQ(ETIMEDOUT, -crpc->error); } TEST_F(homa_timer, homa_timer__reap_dead_rpcs) diff --git a/test/unit_homa_utils.c b/test/unit_homa_utils.c index b5297ee..fcc8221 100644 --- a/test/unit_homa_utils.c +++ b/test/unit_homa_utils.c @@ -280,18 +280,18 @@ TEST_F(homa_utils, homa_bucket_lock_slow) ASSERT_FALSE(IS_ERR(srpc)); homa_rpc_unlock(srpc); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.client_lock_misses); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.client_lock_miss_cycles); + EXPECT_EQ(0, core_metrics.client_lock_misses); + EXPECT_EQ(0, core_metrics.client_lock_miss_cycles); homa_bucket_lock_slow(crpc->bucket, crpc->id); homa_rpc_unlock(crpc); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.client_lock_misses); - EXPECT_NE(0, homa_cores[cpu_number]->metrics.client_lock_miss_cycles); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.server_lock_misses); - EXPECT_EQ(0, homa_cores[cpu_number]->metrics.server_lock_miss_cycles); + EXPECT_EQ(1, core_metrics.client_lock_misses); + EXPECT_NE(0, core_metrics.client_lock_miss_cycles); + EXPECT_EQ(0, core_metrics.server_lock_misses); + EXPECT_EQ(0, core_metrics.server_lock_miss_cycles); homa_bucket_lock_slow(srpc->bucket, srpc->id); homa_rpc_unlock(srpc); - EXPECT_EQ(1, homa_cores[cpu_number]->metrics.server_lock_misses); - EXPECT_NE(0, homa_cores[cpu_number]->metrics.server_lock_miss_cycles); + EXPECT_EQ(1, core_metrics.server_lock_misses); + EXPECT_NE(0, core_metrics.server_lock_miss_cycles); } TEST_F(homa_utils, homa_rpc_acked__basics) diff --git a/test/utils.h b/test/utils.h index 8e23de7..f782266 100644 --- a/test/utils.h +++ b/test/utils.h @@ -30,6 +30,10 @@ enum unit_rpc_state { UNIT_IN_SERVICE = 24, }; +#define core_metrics homa_cores[raw_smp_processor_id()]->metrics + +#define cur_core homa_cores[raw_smp_processor_id()] + extern char *unit_ack_string(struct homa_ack *ack); extern struct homa_rpc *unit_client_rpc(struct homa_sock *hsk, diff --git a/timetrace.c b/timetrace.c index 737fd0c..e429786 100644 --- a/timetrace.c +++ b/timetrace.c @@ -9,18 +9,19 @@ * timetrace stubs; we will then connect the timetrace mechanism here with * those stubs to allow the rest of the kernel to log in our buffers. */ -#define TT_KERNEL 1 +//#define TT_KERNEL 1 #endif #ifdef TT_KERNEL -extern int tt_linux_buffer_mask; extern struct tt_buffer *tt_linux_buffers[]; extern void (*tt_linux_freeze)(void); extern atomic_t *tt_linux_freeze_count; extern atomic_t tt_linux_freeze_no_homa; extern int *tt_linux_homa_temp; extern int tt_linux_homa_temp_default[]; -extern void tt_inc_metric(int metric, __u64 count); extern void (*tt_linux_inc_metrics)(int metric, __u64 count); +extern void (*tt_linux_record)(struct tt_buffer *buffer, __u64 timestamp, + const char* format, __u32 arg0, __u32 arg1, __u32 arg2, + __u32 arg3); extern void tt_linux_skip_metrics(int metric, __u64 count); extern void (*tt_linux_printk)(void); extern void (*tt_linux_dbg1)(char *msg, ...); @@ -28,7 +29,12 @@ extern void (*tt_linux_dbg2)(char *msg, ...); extern void (*tt_linux_dbg3)(char *msg, ...); extern void tt_linux_nop(void); extern void homa_trace(__u64 u0, __u64 u1, int i0, int i1); + +extern void ltt_record_nop(struct tt_buffer *buffer, __u64 timestamp, + const char *format, __u32 arg0, __u32 arg1, + __u32 arg2, __u32 arg3); #endif +extern void tt_inc_metric(int metric, __u64 count); /* Separate buffers for each core: this eliminates the need for * synchronization in tt_record, which improves performance significantly. @@ -135,7 +141,7 @@ int tt_init(char *proc_file, int *temp) for (i = 0; i < nr_cpu_ids; i++) { tt_linux_buffers[i] = tt_buffers[i]; } - tt_linux_buffer_mask = TT_BUF_SIZE-1; + tt_linux_record = tt_record_buf; tt_linux_freeze = tt_freeze; tt_linux_freeze_count = &tt_freeze_count; tt_linux_inc_metrics = tt_inc_metric; @@ -177,6 +183,7 @@ void tt_destroy(void) tt_freeze_count.counter = 1; #ifdef TT_KERNEL + tt_linux_record = ltt_record_nop; tt_linux_freeze = tt_linux_nop; tt_linux_freeze_count = &tt_linux_freeze_no_homa; for (i = 0; i < nr_cpu_ids; i++) {