From 9f7a1423a068f0a03a43931f027fb68be4125e5c Mon Sep 17 00:00:00 2001 From: hao Date: Tue, 31 May 2022 19:46:09 -0400 Subject: [PATCH] Added F-Stack FreeBSD support (This is a part of the research work at RCSLab, University of Waterloo) --- README.md | 31 +++++++++++++++----- dpdk/lib/librte_eal/freebsd/include/rte_os.h | 28 ++++++++++++++++++ lib/Makefile | 20 +++++++++++-- lib/ff_dpdk_if.c | 3 +- 4 files changed, 72 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 00035d8c9..2d176e01a 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,9 @@ Currently, besides authorized DNS server of DNSPod, there are various products i yum install numactl-devel # on Centos #sudo apt-get install libnuma-dev # on Ubuntu + # Install dependencies (FreeBSD only) + #pkg install meson pkgconf py38-pyelftools + cd f-stack # Compile DPDK cd dpdk/ @@ -45,25 +48,27 @@ Currently, besides authorized DNS server of DNSPod, there are various products i ninja -C build ninja -C build install - # Set hugepage + # Set hugepage (Linux only) # single-node system echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - # or NUMA + # or NUMA (Linux only) echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages echo 1024 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages - # Using Hugepage with the DPDK + # Using Hugepage with the DPDK (Linux only) mkdir /mnt/huge mount -t hugetlbfs nodev /mnt/huge - # Close ASLR; it is necessary in multiple process + # Close ASLR; it is necessary in multiple process (Linux only) echo 0 > /proc/sys/kernel/randomize_va_space # Install python for running DPDK python scripts sudo apt install python # On ubuntu + #sudo pkg install python # On FreeBSD # Offload NIC + # For Linux: modprobe uio insmod /data/f-stack/dpdk/build/kernel/linux/igb_uio/igb_uio.ko insmod /data/f-stack/dpdk/build/kernel/linux/kni/rte_kni.ko carrier=on # carrier=on is necessary, otherwise need to be up `veth0` via `echo 1 > /sys/class/net/veth0/carrier` @@ -71,11 +76,21 @@ Currently, besides authorized DNS server of DNSPod, there are various products i ifconfig eth0 down python dpdk-devbind.py --bind=igb_uio eth0 # assuming that use 10GE NIC and eth0 + # For FreeBSD: + # Refer DPDK FreeBSD guide to set tunables in /boot/loader.conf + # Below is an example used for our testing machine + #echo "hw.nic_uio.bdfs=\"2:0:0\"" >> /boot/loader.conf + #echo "hw.contigmem.num_buffers=1" >> /boot/loader.conf + #echo "hw.contigmem.buffer_size=1073741824" >> /boot/loader.conf + #kldload contigmem + #kldload nic_uio + # On Ubuntu, use gawk instead of the default mawk. #sudo apt-get install gawk # or execute `sudo update-alternatives --config awk` to choose gawk. # Install dependencies for F-Stack - sudo apt install gcc make libssl-dev # On ubuntu + sudo apt install gcc make libssl-dev # On ubuntu + #sudo pkg install gcc gmake openssl pkgconf libepoll-shim # On FreeBSD # Upgrade pkg-config while version < 0.28 #cd /data @@ -92,14 +107,16 @@ Currently, besides authorized DNS server of DNSPod, there are various products i export FF_PATH=/data/f-stack export PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig:/usr/lib/pkgconfig cd /data/f-stack/lib/ - make + make # On Linux + #gmake # On FreeBSD # Install F-STACK # libfstack.a will be installed to /usr/local/lib # ff_*.h will be installed to /usr/local/include # start.sh will be installed to /usr/local/bin/ff_start # config.ini will be installed to /etc/f-stack.conf - make install + make install # On Linux + #gmake install # On FreeBSD #### Nginx diff --git a/dpdk/lib/librte_eal/freebsd/include/rte_os.h b/dpdk/lib/librte_eal/freebsd/include/rte_os.h index eeb750cd8..f73eebfdd 100644 --- a/dpdk/lib/librte_eal/freebsd/include/rte_os.h +++ b/dpdk/lib/librte_eal/freebsd/include/rte_os.h @@ -14,6 +14,22 @@ #include typedef cpuset_t rte_cpuset_t; +#if __FreeBSD_version >= 1301000 +#define RTE_CPU_AND(dst, src1, src2) do \ +{ \ + cpuset_t tmp; \ + CPU_COPY(src1, &tmp); \ + CPU_AND(&tmp, &tmp, src2); \ + CPU_COPY(&tmp, dst); \ +} while (0) +#define RTE_CPU_OR(dst, src1, src2) do \ +{ \ + cpuset_t tmp; \ + CPU_COPY(src1, &tmp); \ + CPU_OR(&tmp, &tmp, src2); \ + CPU_COPY(&tmp, dst); \ +} while (0) +#else #define RTE_CPU_AND(dst, src1, src2) do \ { \ cpuset_t tmp; \ @@ -28,6 +44,7 @@ typedef cpuset_t rte_cpuset_t; CPU_OR(&tmp, src2); \ CPU_COPY(&tmp, dst); \ } while (0) +#endif #define RTE_CPU_FILL(set) CPU_FILL(set) /* In FreeBSD 13 CPU_NAND macro is CPU_ANDNOT */ @@ -40,6 +57,15 @@ typedef cpuset_t rte_cpuset_t; CPU_COPY(&tmp, dst); \ } while (0) #else +#if __FreeBSD_version >= 1301000 +#define RTE_CPU_NOT(dst, src) do \ +{ \ + cpuset_t tmp; \ + CPU_FILL(&tmp); \ + CPU_ANDNOT(&tmp, &tmp, src); \ + CPU_COPY(&tmp, dst); \ +} while (0) +#else #define RTE_CPU_NOT(dst, src) do \ { \ cpuset_t tmp; \ @@ -47,6 +73,8 @@ typedef cpuset_t rte_cpuset_t; CPU_ANDNOT(&tmp, src); \ CPU_COPY(&tmp, dst); \ } while (0) + +#endif #endif #endif /* _RTE_OS_H_ */ diff --git a/lib/Makefile b/lib/Makefile index 57c160ba4..6c8720cb3 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -21,12 +21,19 @@ PREFIX_INCLUDE=/usr/local/include PREFIX_BIN=/usr/local/bin F-STACK_CONF=/etc/f-stack.conf F-STACK_VERSION=1.22 +TGT_OS=$(shell uname) +ifeq ($(TGT_OS),FreeBSD) +CC=gcc +endif HOST_OS:=$(shell uname -s) DEBUG=-O0 -gdwarf-2 -g3 -Wno-format-truncation +# No DPDK KNI support on FreeBSD +ifneq ($(TGT_OS),FreeBSD) FF_KNI=1 +endif #FF_FLOW_ISOLATE=1 @@ -59,6 +66,11 @@ INCLUDES+= -I./opt # Include search path for files that only include host OS headers HOST_INCLUDES= -I. +# Use libepoll shim on FreeBSD +ifeq ($(TGT_OS),FreeBSD) +HOST_INCLUDES+= -I/usr/local/include/libepoll-shim +endif + ifndef DEBUG HOST_CFLAGS = -O2 -frename-registers -funswitch-loops -fweb -Wno-format-truncation else @@ -524,11 +536,15 @@ EXTRA_TCP_STACKS_SRCS+= \ bbr.c endif + +ifneq ($(TGT_OS),FreeBSD) ifndef FF_KNI FF_HOST_SRCS+= \ ff_dpdk_kni.c -endif -endif +endif #FF_KNI +endif #FreeBSD OS Check + +endif #INET6 ifdef FF_IPFW NETIPFW_SRCS+= \ diff --git a/lib/ff_dpdk_if.c b/lib/ff_dpdk_if.c index 5a104d8f9..eeb13dd3f 100644 --- a/lib/ff_dpdk_if.c +++ b/lib/ff_dpdk_if.c @@ -1179,7 +1179,8 @@ protocol_filter(const void *data, uint16_t len) if(ether_type == RTE_ETHER_TYPE_ARP) return FILTER_ARP; -#ifdef INET6 +#if (!defined(__FreeBSD__) && defined(INET6) ) || \ + ( defined(__FreeBSD__) && defined(INET6) && defined(FF_KNI)) if (ether_type == RTE_ETHER_TYPE_IPV6) { return ff_kni_proto_filter(data, len, ether_type);