From f8b76adc91bb21b531453ecac074cba345e233e7 Mon Sep 17 00:00:00 2001
From: jagathprakash <31057312+jagathprakash@users.noreply.github.com>
Date: Tue, 18 Apr 2023 10:25:15 -0400
Subject: [PATCH] [TEP-0089] SPIRE for non-falsifiable provenance. Setup the
test environment.
This PR is part of a set of PRs to enable non-falsifiable provenance using SPIRE.
This PR sets up the environment needed to enable this feature in E2E tests.
Note that the SPRIRE flag itself is not enabled, i.e. the feature itself is not enabled.
This PR is to test if adding the SPIRE environment does not break anything in E2E tests.
Signed-off-by: jagathprakash <31057312+jagathprakash@users.noreply.github.com>
---
test/e2e-common.sh | 59 +++++
test/e2e-tests.sh | 17 ++
.../patch/pipeline-controller-spire.json | 57 +++++
test/testdata/spire/config-spire.yaml | 17 ++
test/testdata/spire/spiffe-csi-driver.yaml | 20 ++
test/testdata/spire/spire-agent.yaml | 204 +++++++++++++++++
test/testdata/spire/spire-server.yaml | 205 ++++++++++++++++++
7 files changed, 579 insertions(+)
create mode 100644 test/testdata/patch/pipeline-controller-spire.json
create mode 100644 test/testdata/spire/config-spire.yaml
create mode 100644 test/testdata/spire/spiffe-csi-driver.yaml
create mode 100644 test/testdata/spire/spire-agent.yaml
create mode 100644 test/testdata/spire/spire-server.yaml
diff --git a/test/e2e-common.sh b/test/e2e-common.sh
index 968dd31ec2c..3968ff579f9 100755
--- a/test/e2e-common.sh
+++ b/test/e2e-common.sh
@@ -48,6 +48,65 @@ function install_pipeline_crd_version() {
verify_pipeline_installation
}
+function spire_apply() {
+ if [ $# -lt 2 -o "$1" != "-spiffeID" ]; then
+ echo "spire_apply requires a spiffeID as the first arg" >&2
+ exit 1
+ fi
+ show=$(kubectl exec -n spire deployment/spire-server -- \
+ /opt/spire/bin/spire-server entry show $1 $2)
+ if [ "$show" != "Found 0 entries" ]; then
+ # delete to recreate
+ entryid=$(echo "$show" | grep "^Entry ID" | cut -f2 -d:)
+ kubectl exec -n spire deployment/spire-server -- \
+ /opt/spire/bin/spire-server entry delete -entryID $entryid
+ fi
+ kubectl exec -n spire deployment/spire-server -- \
+ /opt/spire/bin/spire-server entry create "$@"
+}
+
+function install_spire() {
+ echo ">> Deploying Spire"
+ DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+ echo "Creating SPIRE namespace..."
+ kubectl create ns spire
+
+ echo "Applying SPIFFE CSI Driver configuration..."
+ kubectl apply -f "$DIR"/testdata/spire/spiffe-csi-driver.yaml
+
+ echo "Deploying SPIRE server"
+ kubectl apply -f "$DIR"/testdata/spire/spire-server.yaml
+
+ echo "Deploying SPIRE agent"
+ kubectl apply -f "$DIR"/testdata/spire/spire-agent.yaml
+
+ wait_until_pods_running spire || fail_test "SPIRE did not come up"
+
+ spire_apply \
+ -spiffeID spiffe://example.org/ns/spire/node/example \
+ -selector k8s_psat:cluster:example-cluster \
+ -selector k8s_psat:agent_ns:spire \
+ -selector k8s_psat:agent_sa:spire-agent \
+ -node
+ spire_apply \
+ -spiffeID spiffe://example.org/ns/tekton-pipelines/sa/tekton-pipelines-controller \
+ -parentID spiffe://example.org/ns/spire/node/example \
+ -selector k8s:ns:tekton-pipelines \
+ -selector k8s:pod-label:app:tekton-pipelines-controller \
+ -selector k8s:sa:tekton-pipelines-controller \
+ -admin
+}
+
+function patch_pipline_spire() {
+ kubectl patch \
+ deployment tekton-pipelines-controller \
+ -n tekton-pipelines \
+ --patch-file "$DIR"/testdata/patch/pipeline-controller-spire.json
+
+ verify_pipeline_installation
+}
+
function verify_pipeline_installation() {
# Make sure that everything is cleaned up in the current namespace.
delete_pipeline_resources
diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh
index 66d39911000..c110ee3be0f 100755
--- a/test/e2e-tests.sh
+++ b/test/e2e-tests.sh
@@ -41,6 +41,22 @@ install_pipeline_crd
failed=0
+function add_spire() {
+ local gate="$1"
+ if [ "$gate" != "alpha" ] && [ "$gate" != "stable" ] && [ "$gate" != "beta" ] ; then
+ printf "Invalid gate %s\n" ${gate}
+ exit 255
+ fi
+ if [ "$gate" == "alpha" ] ; then
+ DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+ printf "Setting up environment for alpha features"
+ install_spire
+ patch_pipline_spire
+ kubectl apply -n tekton-pipelines -f "$DIR"/testdata/spire/config-spire.yaml
+ failed=0
+ fi
+}
+
function set_feature_gate() {
local gate="$1"
local resolver="false"
@@ -91,6 +107,7 @@ function run_e2e() {
fi
}
+add_spire "$PIPELINE_FEATURE_GATE"
set_feature_gate "$PIPELINE_FEATURE_GATE"
set_result_extraction_method "$RESULTS_FROM"
run_e2e
diff --git a/test/testdata/patch/pipeline-controller-spire.json b/test/testdata/patch/pipeline-controller-spire.json
new file mode 100644
index 00000000000..27831e9bef6
--- /dev/null
+++ b/test/testdata/patch/pipeline-controller-spire.json
@@ -0,0 +1,57 @@
+{
+ "spec":{
+ "template":{
+ "spec":{
+ "$setElementOrder/containers":[
+ {
+ "name":"tekton-pipelines-controller"
+ }
+ ],
+ "$setElementOrder/volumes":[
+ {
+ "name":"config-logging"
+ },
+ {
+ "name":"config-registry-cert"
+ },
+ {
+ "name":"spiffe-workload-api"
+ }
+ ],
+ "containers":[
+ {
+ "$setElementOrder/volumeMounts":[
+ {
+ "mountPath":"/etc/config-logging"
+ },
+ {
+ "mountPath":"/etc/config-registry-cert"
+ },
+ {
+ "mountPath":"/spiffe-workload-api"
+ }
+ ],
+ "name":"tekton-pipelines-controller",
+ "volumeMounts":[
+ {
+ "mountPath":"/spiffe-workload-api",
+ "name":"spiffe-workload-api",
+ "readOnly":true
+ }
+ ]
+ }
+ ],
+ "volumes":[
+ {
+ "csi":{
+ "driver":"csi.spiffe.io",
+ "readOnly":true
+ },
+ "name":"spiffe-workload-api"
+ }
+ ]
+ }
+ }
+ }
+ }
+
\ No newline at end of file
diff --git a/test/testdata/spire/config-spire.yaml b/test/testdata/spire/config-spire.yaml
new file mode 100644
index 00000000000..30837a0e65d
--- /dev/null
+++ b/test/testdata/spire/config-spire.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: config-spire
+ namespace: tekton-pipelines
+ labels:
+ app.kubernetes.io/instance: default
+ app.kubernetes.io/part-of: tekton-pipelines
+data:
+ # spire-trust-domain specifies the SPIRE trust domain to use.
+ spire-trust-domain: "example.org"
+ # spire-socket-path specifies the SPIRE agent socket for SPIFFE workload API.
+ spire-socket-path: "unix:///spiffe-workload-api/spire-agent.sock"
+ # spire-server-addr specifies the SPIRE server address for workload/node registration.
+ spire-server-addr: "spire-server.spire.svc.cluster.local:8081"
+ # spire-node-alias-prefix specifies the SPIRE node alias prefix to use.
+ spire-node-alias-prefix: "/tekton-node/"
\ No newline at end of file
diff --git a/test/testdata/spire/spiffe-csi-driver.yaml b/test/testdata/spire/spiffe-csi-driver.yaml
new file mode 100644
index 00000000000..ca64e7aceae
--- /dev/null
+++ b/test/testdata/spire/spiffe-csi-driver.yaml
@@ -0,0 +1,20 @@
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+ name: "csi.spiffe.io"
+spec:
+ # Only ephemeral, inline volumes are supported. There is no need for a
+ # controller to provision and attach volumes.
+ attachRequired: false
+
+ # Request the pod information which the CSI driver uses to verify that an
+ # ephemeral mount was requested.
+ podInfoOnMount: true
+
+ # Don't change ownership on the contents of the mount since the Workload API
+ # Unix Domain Socket is typically open to all (i.e. 0777).
+ fsGroupPolicy: None
+
+ # Declare support for ephemeral volumes only.
+ volumeLifecycleModes:
+ - Ephemeral
\ No newline at end of file
diff --git a/test/testdata/spire/spire-agent.yaml b/test/testdata/spire/spire-agent.yaml
new file mode 100644
index 00000000000..67092e064ab
--- /dev/null
+++ b/test/testdata/spire/spire-agent.yaml
@@ -0,0 +1,204 @@
+# ServiceAccount for the SPIRE agent
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: spire-agent
+ namespace: spire
+
+---
+
+# Required cluster role to allow spire-agent to query k8s API server
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: spire-agent-cluster-role
+rules:
+- apiGroups: [""]
+ resources: ["pods", "nodes", "nodes/proxy"]
+ verbs: ["get"]
+
+---
+
+# Binds above cluster role to spire-agent service account
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: spire-agent-cluster-role-binding
+subjects:
+- kind: ServiceAccount
+ name: spire-agent
+ namespace: spire
+roleRef:
+ kind: ClusterRole
+ name: spire-agent-cluster-role
+ apiGroup: rbac.authorization.k8s.io
+
+
+---
+
+# ConfigMap for the SPIRE agent featuring:
+# 1) PSAT node attestation
+# 2) K8S Workload Attestation over the secure kubelet port
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: spire-agent
+ namespace: spire
+data:
+ agent.conf: |
+ agent {
+ data_dir = "/run/spire"
+ log_level = "DEBUG"
+ server_address = "spire-server"
+ server_port = "8081"
+ socket_path = "/run/spire/sockets/spire-agent.sock"
+ trust_bundle_path = "/run/spire/bundle/bundle.crt"
+ trust_domain = "example.org"
+ }
+ plugins {
+ NodeAttestor "k8s_psat" {
+ plugin_data {
+ cluster = "example-cluster"
+ }
+ }
+ KeyManager "memory" {
+ plugin_data {
+ }
+ }
+ WorkloadAttestor "k8s" {
+ plugin_data {
+ skip_kubelet_verification = true
+ }
+ }
+ }
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: spire-agent
+ namespace: spire
+ labels:
+ app: spire-agent
+spec:
+ selector:
+ matchLabels:
+ app: spire-agent
+ updateStrategy:
+ type: RollingUpdate
+ template:
+ metadata:
+ namespace: spire
+ labels:
+ app: spire-agent
+ spec:
+ hostPID: true
+ hostNetwork: true
+ dnsPolicy: ClusterFirstWithHostNet
+ serviceAccountName: spire-agent
+ containers:
+ - name: spire-agent
+ image: ghcr.io/spiffe/spire-agent:1.1.1
+ imagePullPolicy: IfNotPresent
+ args: ["-config", "/run/spire/config/agent.conf"]
+ volumeMounts:
+ - name: spire-config
+ mountPath: /run/spire/config
+ readOnly: true
+ - name: spire-bundle
+ mountPath: /run/spire/bundle
+ readOnly: true
+ - name: spire-token
+ mountPath: /var/run/secrets/tokens
+ - name: spire-agent-socket-dir
+ mountPath: /run/spire/sockets
+ # This is the container which runs the SPIFFE CSI driver.
+ - name: spiffe-csi-driver
+ image: ghcr.io/spiffe/spiffe-csi-driver:nightly
+ imagePullPolicy: IfNotPresent
+ args: [
+ "-workload-api-socket-dir", "/spire-agent-socket",
+ "-csi-socket-path", "/spiffe-csi/csi.sock",
+ ]
+ env:
+ # The CSI driver needs a unique node ID. The node name can be
+ # used for this purpose.
+ - name: MY_NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ volumeMounts:
+ # The volume containing the SPIRE agent socket. The SPIFFE CSI
+ # driver will mount this directory into containers.
+ - mountPath: /spire-agent-socket
+ name: spire-agent-socket-dir
+ readOnly: true
+ # The volume that will contain the CSI driver socket shared
+ # with the kubelet and the driver registrar.
+ - mountPath: /spiffe-csi
+ name: spiffe-csi-socket-dir
+ # The volume containing mount points for containers.
+ - mountPath: /var/lib/kubelet/pods
+ mountPropagation: Bidirectional
+ name: mountpoint-dir
+ securityContext:
+ privileged: true
+ # This container runs the CSI Node Driver Registrar which takes care
+ # of all the little details required to register a CSI driver with
+ # the kubelet.
+ - name: node-driver-registrar
+ image: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1
+ imagePullPolicy: IfNotPresent
+ args: [
+ "-csi-address", "/spiffe-csi/csi.sock",
+ "-kubelet-registration-path", "/var/lib/kubelet/plugins/csi.spiffe.io/csi.sock",
+ ]
+ volumeMounts:
+ # The registrar needs access to the SPIFFE CSI driver socket
+ - mountPath: /spiffe-csi
+ name: spiffe-csi-socket-dir
+ # The registrar needs access to the Kubelet plugin registration
+ # directory
+ - name: kubelet-plugin-registration-dir
+ mountPath: /registration
+ volumes:
+ - name: spire-config
+ configMap:
+ name: spire-agent
+ - name: spire-bundle
+ configMap:
+ name: spire-bundle
+ - name: spire-token
+ projected:
+ sources:
+ - serviceAccountToken:
+ path: spire-agent
+ expirationSeconds: 7200
+ audience: spire-server
+ # This volume is used to share the Workload API socket between the CSI
+ # driver and SPIRE agent. Note, an emptyDir volume could also be used,
+ # however, this can lead to broken bind mounts in the workload
+ # containers if the agent pod is restarted (since the emptyDir
+ # directory on the node that was mounted into workload containers by
+ # the CSI driver belongs to the old pod instance and is no longer
+ # valid).
+ - name: spire-agent-socket-dir
+ hostPath:
+ path: /run/spire/agent-sockets
+ type: DirectoryOrCreate
+ # This volume is where the socket for kubelet->driver communication lives
+ - name: spiffe-csi-socket-dir
+ hostPath:
+ path: /var/lib/kubelet/plugins/csi.spiffe.io
+ type: DirectoryOrCreate
+ # This volume is where the SPIFFE CSI driver mounts volumes
+ - name: mountpoint-dir
+ hostPath:
+ path: /var/lib/kubelet/pods
+ type: Directory
+ # This volume is where the node-driver-registrar registers the plugin
+ # with kubelet
+ - name: kubelet-plugin-registration-dir
+ hostPath:
+ path: /var/lib/kubelet/plugins_registry
+ type: Directory
\ No newline at end of file
diff --git a/test/testdata/spire/spire-server.yaml b/test/testdata/spire/spire-server.yaml
new file mode 100644
index 00000000000..b150f44266d
--- /dev/null
+++ b/test/testdata/spire/spire-server.yaml
@@ -0,0 +1,205 @@
+# ServiceAccount used by the SPIRE server.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: spire-server
+ namespace: spire
+
+---
+
+# Required cluster role to allow spire-server to query k8s API server
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: spire-server-cluster-role
+rules:
+- apiGroups: [""]
+ resources: ["nodes"]
+ verbs: ["get"]
+ # allow TokenReview requests (to verify service account tokens for PSAT
+ # attestation)
+- apiGroups: ["authentication.k8s.io"]
+ resources: ["tokenreviews"]
+ verbs: ["get", "create"]
+
+---
+
+# Binds above cluster role to spire-server service account
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: spire-server-cluster-role-binding
+ namespace: spire
+subjects:
+- kind: ServiceAccount
+ name: spire-server
+ namespace: spire
+roleRef:
+ kind: ClusterRole
+ name: spire-server-cluster-role
+ apiGroup: rbac.authorization.k8s.io
+
+---
+
+# Role for the SPIRE server
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ namespace: spire
+ name: spire-server-role
+rules:
+# allow "get" access to pods (to resolve selectors for PSAT attestation)
+- apiGroups: [""]
+ resources: ["pods"]
+ verbs: ["get"]
+ # allow access to "get" and "patch" the spire-bundle ConfigMap (for SPIRE
+ # agent bootstrapping, see the spire-bundle ConfigMap below)
+- apiGroups: [""]
+ resources: ["configmaps"]
+ resourceNames: ["spire-bundle"]
+ verbs: ["get", "patch"]
+
+---
+
+# RoleBinding granting the spire-server-role to the SPIRE server
+# service account.
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: spire-server-role-binding
+ namespace: spire
+subjects:
+- kind: ServiceAccount
+ name: spire-server
+ namespace: spire
+roleRef:
+ kind: Role
+ name: spire-server-role
+ apiGroup: rbac.authorization.k8s.io
+
+---
+
+# ConfigMap containing the latest trust bundle for the trust domain. It is
+# updated by SPIRE using the k8sbundle notifier plugin. SPIRE agents mount
+# this config map and use the certificate to bootstrap trust with the SPIRE
+# server during attestation.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: spire-bundle
+ namespace: spire
+
+---
+
+# ConfigMap containing the SPIRE server configuration.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: spire-server
+ namespace: spire
+data:
+ server.conf: |
+ server {
+ bind_address = "0.0.0.0"
+ bind_port = "8081"
+ trust_domain = "example.org"
+ data_dir = "/run/spire/data"
+ log_level = "DEBUG"
+ default_svid_ttl = "1h"
+ ca_ttl = "12h"
+ ca_subject {
+ country = ["US"]
+ organization = ["SPIFFE"]
+ common_name = ""
+ }
+ }
+ plugins {
+ DataStore "sql" {
+ plugin_data {
+ database_type = "sqlite3"
+ connection_string = "/run/spire/data/datastore.sqlite3"
+ }
+ }
+ NodeAttestor "k8s_psat" {
+ plugin_data {
+ clusters = {
+ "example-cluster" = {
+ service_account_allow_list = ["spire:spire-agent"]
+ }
+ }
+ }
+ }
+ KeyManager "disk" {
+ plugin_data {
+ keys_path = "/run/spire/data/keys.json"
+ }
+ }
+ Notifier "k8sbundle" {
+ plugin_data {
+ # This plugin updates the bundle.crt value in the spire:spire-bundle
+ # ConfigMap by default, so no additional configuration is necessary.
+ }
+ }
+ }
+ health_checks {
+ listener_enabled = true
+ bind_address = "0.0.0.0"
+ bind_port = "8080"
+ live_path = "/live"
+ ready_path = "/ready"
+ }
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: spire-server
+ namespace: spire
+ labels:
+ app: spire-server
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: spire-server
+ template:
+ metadata:
+ namespace: spire
+ labels:
+ app: spire-server
+ spec:
+ serviceAccountName: spire-server
+ shareProcessNamespace: true
+ containers:
+ - name: spire-server
+ image: ghcr.io/spiffe/spire-server:1.1.1
+ imagePullPolicy: IfNotPresent
+ args: ["-config", "/run/spire/config/server.conf"]
+ ports:
+ - containerPort: 8081
+ volumeMounts:
+ - name: spire-config
+ mountPath: /run/spire/config
+ readOnly: true
+ volumes:
+ - name: spire-config
+ configMap:
+ name: spire-server
+
+---
+
+# Service definition for SPIRE server defining the gRPC port.
+apiVersion: v1
+kind: Service
+metadata:
+ name: spire-server
+ namespace: spire
+spec:
+ type: NodePort
+ ports:
+ - name: grpc
+ port: 8081
+ targetPort: 8081
+ protocol: TCP
+ selector:
+ app: spire-server
\ No newline at end of file