Use Ubuntu PF_RING package instead of submodule. (#91)

* Modified .dockerignore to improve effectiveness of docker caching
* Changed Dockerfile to use ntop pfring package
* Added a script to add ntop repository and install pfring and ZC on the host system (Debian and Ubuntu)
* Update docker default phantom_subnets.toml file
* Added some sanity checks into zbalance container
	* Check if hugepages number is 512
	* Check if ZC driver is loaded
	* If check not passed show some hints what can be wrong

.dockerignore

@@ -1,8 +1,5 @@
-docker
-!docker/*
-docker/Dockerfile
-docker/docker-compose*
-docker/.dockerignore
-docker/*.swp
-.dockerignore
+.git
 *.swp
+/docker/**
+!docker/*entrypoint.sh
+.dockerignore

docker/Dockerfile

@@ -1,57 +1,69 @@
-FROM ubuntu:20.04 as build_base
+#FROM ubuntu:20.04 as build_base
+FROM ubuntu:20.04 as build_base_go
 #    PATH="/opt/PF_RING/userland/examples_zc:$PATH"
-
-#wget make gcc bison flex
-RUN apt-get update && \
-    apt-get -y -q install wget git make gcc bison flex && \
-    git clone --recurse-submodule https://github.com/refraction-networking/conjure.git && \
-    cd /conjure/PF_RING/userland/lib && ./configure && make && \
-    cd /conjure/PF_RING/userland/libpcap && ./configure && make && \
-    cd /conjure/PF_RING/userland && ./configure && make && \
-    cp -r /conjure/PF_RING /opt/PF_RING && \
-    apt-get clean all
-
-FROM build_base as build_base_go
 ARG GO_VERSION=1.15.3
 ARG CUSTOM_BUILD
+ARG BRANCH=master
 ENV PATH="/usr/local/go/bin:/root/.cargo/bin:${PATH}" \
     GOPATH="/root/go" \
     GOROOT="/usr/local/go"
-COPY . /tmp/conjure
 
-
-# Install rust and go
-RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get -y -q install protobuf-compiler curl libssl-dev pkg-config libgmp3-dev libzmq3-dev && \
+# Install dependencies: including rust and go
+RUN apt-get update && \
+    DEBIAN_FRONTEND="noninteractive" apt-get -y -q install wget git make gcc bison flex protobuf-compiler curl libssl-dev pkg-config libgmp3-dev libzmq3-dev && \
+    apt-get clean all && \
     wget -q https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz && \
     tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz && \
     curl https://sh.rustup.rs -sSf -o install_rust.sh; sh install_rust.sh -y && \
     cargo install protobuf-codegen
 
+# Install PFRING to get libraries
+RUN apt-get install -y -q software-properties-common wget && \
+    add-apt-repository universe && \
+    wget https://packages.ntop.org/apt-stable/20.04/all/apt-ntop-stable.deb && \
+    apt install ./apt-ntop-stable.deb && \
+    apt-get clean all && \
+    apt-get update && \
+    apt-get install -y -q pfring && \
+    apt-get clean all 
+
+# Get go-reddis and make sure it's version is 7.4.0.
 RUN go get -u -d github.com/go-redis/redis || true && cd ${GOPATH}/src/github.com/go-redis/redis && git checkout tags/v7.4.0 -b v7-master 
-RUN    go get -u -d github.com/BurntSushi/toml 
+
+# Copy docker context dir. This is used as a source if CUSTOM_BUILD is enabled
+COPY . /tmp/conjure
+# Get Conjure or copy a directory Dockerfile is in. Switched by CUSTOM_BUILD var
 RUN    bash -c 'if [[ -z "$CUSTOM_BUILD" ]] ; then \
      go get -d github.com/refraction-networking/conjure/...  ; \
      else mkdir -p ${GOPATH}/src/github.com/refraction-networking; cp -r /tmp/conjure ${GOPATH}/src/github.com/refraction-networking/conjure ; \
      fi' 
-RUN    rm -rf ${GOPATH}/src/github.com/refracion-networking/conjure/PF_RING && cp -r /conjure/PF_RING ${GOPATH}/src/github.com/refraction-networking/conjure 
+
+# Checkout needed branch and compile
 RUN    cd /root/go/src/github.com/refraction-networking/conjure && \
+    git checkout ${BRANCH} && \
     go get ./... || true && \
     make 
 RUN    cp -r /root/go/src/github.com/refraction-networking/conjure /opt/conjure
 
 
 
-
 FROM ubuntu:20.04 as zbalance
 ENV CJ_IFACE=lo \
     CJ_CLUSTER_ID=98 \
     CJ_CORECOUNT=1 \
     CJ_COREBASE=0 \
     ZBALANCE_HASH_MODE=1
-COPY --from=build_base /opt/PF_RING /opt/PF_RING
+#COPY --from=build_base /opt/PF_RING /opt/PF_RING
+
+RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get -y -q install libelf1 
+
+COPY --from=build_base_go /usr/bin/zbalance_ipc /usr/bin/zbalance_ipc
 COPY ./docker/zbalance-entrypoint.sh /entrypoint.sh
 ENTRYPOINT ["bash", "/entrypoint.sh"]
 
+
+
+
 FROM ubuntu:20.04 as detector
 ENV CJ_CLUSTER_ID=98 \
     CJ_CORECOUNT=1 \
@@ -63,20 +75,23 @@ ENV CJ_CLUSTER_ID=98 \
     CJ_STATION_CONFIG=/opt/conjure/application/config.toml \
     CJ_IP4_ADDR=127.0.0.1 \
     CJ_IP6_ADDR=[::1]
-COPY --from=build_base_go /opt/conjure/dark-decoy /opt/conjure/dark-decoy
+#COPY --from=build_base_go /opt/conjure/dark-decoy /opt/conjure/dark-decoy
+COPY --from=build_base_go /opt/conjure/conjure /opt/conjure/conjure
 COPY --from=build_base_go /opt/conjure/application/config.toml /opt/conjure/application/config.toml
 COPY ./docker/detector-entrypoint.sh /entrypoint.sh
-# this list will be removed in a near future
-RUN touch /var/lib/dark-decoy.prefixes
+COPY --from=build_base_go /usr/local/lib/libpcap.so /usr/local/lib/libpcap.so
 
 RUN apt-get update && apt-get -y -q install libzmq3-dev iproute2 iptables && apt-get clean all
 ENTRYPOINT [ "/entrypoint.sh"]
 
+
+
+
 FROM ubuntu:20.04 as application
 ENV CJ_STATION_CONFIG=/opt/conjure/application/config.toml \
     PHANTOM_SUBNET_LOCATION=/opt/conjure/sysconfig/phantom_subnets.toml
 COPY --from=build_base_go /opt/conjure/application/application /opt/conjure/application/application
-RUN apt-get update && apt-get -y -q install libzmq3-dev
+RUN apt-get update && apt-get -y -q install libzmq3-dev && apt-get clean all
 COPY --from=build_base_go /opt/conjure/application/config.toml ${CJ_STATION_CONFIG}
 COPY --from=build_base_go /opt/conjure/application/lib/test/phantom_subnets.toml ${PHANTOM_SUBNET_LOCATION}
 #COPY ./docker/application-entrypoint.sh /entrypoint.sh

docker/detector-entrypoint.sh

@@ -5,7 +5,7 @@ OFFSET=${CJ_QUEUE_OFFSET:-2}
 
 cleanup() {
   echo $(ps aux)
-  start-stop-daemon --stop --oknodo --retry 15 -n dark-decoy
+  start-stop-daemon --stop --oknodo --retry 15 -n conjure
   #pkill dark-decoy
   echo $(ps aux)
   for CORE in `seq $OFFSET $((OFFSET + CORE_COUNT -1 ))`
@@ -65,7 +65,7 @@ do
   fi
 done
 echo "Prerequisite configuration complete."
-/opt/conjure/dark-decoy -c ${CJ_CLUSTER_ID} -o ${CJ_COREBASE} -n ${CJ_CORECOUNT} -l ${CJ_LOG_INTERVAL} -K ${CJ_PRIVKEY} -s ${CJ_SKIP_CORE} -z ${CJ_QUEUE_OFFSET} &
+/opt/conjure/conjure -c ${CJ_CLUSTER_ID} -o ${CJ_COREBASE} -n ${CJ_CORECOUNT} -l ${CJ_LOG_INTERVAL} -K ${CJ_PRIVKEY} -s ${CJ_SKIP_CORE} -z ${CJ_QUEUE_OFFSET} &
 wait $!
 cleanup
 

docker/install_pfring_package.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+OS=$(lsb_release -si)
+RELEASE=$(lsb_release -sr)
+CODENAME=$(lsb_release -sc)
+
+function no_support {
+        echo "Script does not support selected parameters or OS ${OS} ${CODENAME} ${RELEASE}."
+        echo "Check https://packages.ntop.org/apt-stable/ for more information."
+	exit 1
+}
+
+if [[ $EUID -ne 0 ]]; then
+     echo "This script must be run as root" 
+     exit 1
+fi
+
+case "$OS" in
+        Ubuntu)
+                echo "OS: Ubuntu"
+                case $RELEASE in
+                        20.04 | 18.04)
+                                apt-get install software-properties-common wget
+                                add-apt-repository universe
+                                wget https://packages.ntop.org/apt-stable/${RELEASE}/all/apt-ntop-stable.deb
+                                apt install ./apt-ntop-stable.deb
+                                ;;
+                        16.04)
+                                wget https://packages.ntop.org/apt-stable/${RELEASE}/all/apt-ntop-stable.deb
+                                apt install ./apt-ntop-stable.deb
+                                ;;
+                        *)
+                                no_support
+                                ;;
+                esac
+                ;;
+        Debian)
+                echo "OS: Debian"
+                case $CODENAME in
+                        buster | stretch)
+                                echo ${CODENAME} requires 'contrib' apt sources. Do you want to enable?
+                                read -p "[N/y]" enable_contrib
+                                if [ ${enable_contrib:-N} = 'y' ]
+                                then
+                                        echo "Enabling 'contrib' sources in /etc/apt/sources.list"
+                                        sed -i.bak -e '/contrib/ ! s/^deb.*debian\.org.*$/\0 contrib/' /etc/apt/sources.list
+                                        wget https://packages.ntop.org/apt-stable/${CODENAME}/all/apt-ntop-stable.deb
+                                        apt install ./apt-ntop-stable.deb
+                                else
+                                        no_support
+                                fi
+                                ;;
+                        jessie)
+                                wget https://packages.ntop.org/apt-stable/jessie/all/apt-ntop-stable.deb
+                                dpkg -i apt-ntop-stable.deb
+                                echo "deb http://archive.debian.org/debian jessie-backports main" >> /etc/apt/sources.list
+                                echo 'Acquire::Check-Valid-Until no;' > /etc/apt/apt.conf.d/99no-check-valid-until
+                                apt-get update && apt-get install libjson-c2
+                                ;;
+                        *)
+                                no_support
+                                ;;
+                esac
+                ;;
+        *)
+                no_support
+                ;;
+esac
+apt-get clean all
+apt-get update
+apt-get install pfring-dkms nprobe ntopng n2disk cento
+apt-get install pfring-drivers-zc-dkms

docker/phantom_subnets.toml

@@ -4,20 +4,20 @@
 	[Networks.1]
         Generation = 1
         [[Networks.1.WeightedSubnets]]
-            Weight = 9.0
+            Weight = 9
             Subnets = ["192.122.190.0/24", "2001:48a8:687f:1::/64"] 
 
     [Networks.2]
         Generation = 2
         [[Networks.2.WeightedSubnets]]
-            Weight = 1.0
+            Weight = 1
             Subnets = ["192.122.190.0/28", "2001:48a8:687f:1::/96"] 
 
     [Networks.957]
         Generation = 957
         [[Networks.957.WeightedSubnets]]
-            Weight = 9.0
+            Weight = 9
             Subnets = ["192.122.190.0/24", "2001:48a8:687f:1::/64"] 
         [[Networks.957.WeightedSubnets]]
-            Weight = 1.0
+            Weight = 1
             Subnets = ["141.219.0.0/16", "35.8.0.0/16"] 

docker/zbalance-entrypoint.sh

@@ -1,6 +1,39 @@
 #!/bin/bash
 set -e
 
+
+
+while [ $(sysctl -b vm.nr_hugepages) -lt 512 ]
+do
+	echo 'Please set number of hugepages to at least 512.'
+	echo ''
+	echo 'To check current value run:'
+	echo '	sysctl vm.nr_hugepages'
+	echo 'OR'
+	echo '	cat /proc/sys/vm/nr_hugepages'
+	echo ''
+	echo 'To set number of hugepages run:'
+	echo '	sysctl -w vm.nr_hugepages=512'
+	echo ''
+	echo 'To make this setting persistent run:'
+	echo '	echo "vm.nr_hugepages=512" >> /etc/sysctl.conf'
+	echo ''
+	echo ''
+	echo 'Sleeping for 10 seconds'
+	sleep 10
+done
+
+while [ ! $(cat "/proc/net/pf_ring/dev/${CJ_IFACE}/info" | grep ZC) ]
+do
+	echo 'Is ZC network drivers loaded? For instructions visit https://www.ntop.org/guides/pf_ring/get_started/packages_installation.html'
+	echo ''
+	echo 'To check for ZC driver run:'
+	echo '	cat /proc/net/pf_ring/dev/'${CJ_IFACE}'/info'
+	echo 'You should see "Polling Mode: ZC/NAPI"'
+	echo ''
+	sleep 10; 
+done
+
 # TD_IFACE could be a CSV list of interfaces.
 # Pull them apart to ensure each gets zc: prefix
 ifcarg=""
@@ -20,4 +53,4 @@ do
     fi
 done
 echo "Setting up with params: -i $ifcarg -c ${CJ_CLUSTER_ID} -n ${CJ_CORECOUNT} -m ${ZBALANCE_HASH_MODE} -g ${CJ_COREBASE}"
-/opt/PF_RING/userland/examples_zc/zbalance_ipc -i $ifcarg -c ${CJ_CLUSTER_ID} -n ${CJ_CORECOUNT} -m ${ZBALANCE_HASH_MODE} -g ${CJ_COREBASE}
+zbalance_ipc -i $ifcarg -c ${CJ_CLUSTER_ID} -n ${CJ_CORECOUNT} -m ${ZBALANCE_HASH_MODE} -g ${CJ_COREBASE}