SecureComms: Add support daemonConfig

Support configuring the APF Secure Comms from the CAA side including: - WN public Key - PP private key - Activating Secure Comms - inbouns and outbounds of th PP This is useful for activating Secure Comms from the CAA and without Trustee. It can be used for Testing without producing dedicated podvms which activate Secure Comms and set Inbounds/Outbounds by default. It can also be used for non-Coco peerpods. Signed-off-by: David Hadas <david.hadas@gmail.com>
confidential-containers · Nov 8, 2024 · fb72ef1 · fb72ef1
1 parent 8e08587
commit fb72ef1
Show file tree

Hide file tree

Showing 18 changed files with 282 additions and 152 deletions.
diff --git a/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go b/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go
@@ -93,7 +93,9 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
 		return nil, err
 	}
 
-	if secureComms {
+	if secureComms || cfg.daemonConfig.SecureComms {
+		var inbounds, outbounds []string
+
 		ppssh.Singleton()
 		host, port, err := net.SplitHostPort(cfg.listenAddr)
 		if err != nil {
@@ -103,15 +105,33 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
 			logger.Printf("Address %s is changed to 127.0.0.1:%s since secure-comms is enabled.", cfg.listenAddr, port)
 			cfg.listenAddr = "127.0.0.1:" + port
 		}
-		inbounds := append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...)
-		outbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...)
 
 		// Create a Client that will approach the api-server-rest service at the podns
 		// To obtain secrets from KBS, we approach the api-server-rest service which then approaches the CDH asking for a secret resource
 		// the CDH than contact the KBS (possibly after approaching Attestation Agent for a token) and the KBS serves the requested key
 		// The communication between the CDH (and Attestation Agent) and the KBS is performed via an SSH tunnel named  "KBS"
 		apic := apic.NewApiClient(API_SERVER_REST_PORT, cfg.podNamespace)
-		services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppssh.GetSecret(apic.GetKey), sshutil.SSHPORT))
+
+		ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(apic.GetKey))
+
+		if secureComms {
+			// CoCo in production
+			ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY)
+			ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY)
+		} else {
+			// Never here under CoCo in production
+			// Set secureComms using daemonConfig for testing
+			ppSecrets.SetKey(ppssh.WN_PUBLIC_KEY, cfg.daemonConfig.WnPublicKey)
+			ppSecrets.SetKey(ppssh.PP_PRIVATE_KEY, cfg.daemonConfig.PpPrivateKey)
+		}
+
+		inbounds = append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...)
+		inbounds = append(inbounds, strings.Split(cfg.daemonConfig.SecureCommsInbounds, ",")...)
+
+		outbounds = append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...)
+		outbounds = append(outbounds, strings.Split(cfg.daemonConfig.SecureCommsOutbounds, ",")...)
+
+		services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppSecrets, sshutil.SSHPORT))
 	} else {
 		if !disableTLS {
 			cfg.tlsConfig = &tlsConfig

diff --git a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go
@@ -9,9 +9,11 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor"
+	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy"
 	daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler/vxlan"
@@ -28,7 +30,7 @@ const (
 )
 
 type daemonConfig struct {
-	serverConfig adaptor.ServerConfig
+	serverConfig cloud.ServerConfig
 	networkConfig
 }
 
@@ -86,12 +88,14 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 	}
 
 	var (
-		disableTLS           bool
-		tlsConfig            tlsutil.TLSConfig
-		secureComms          bool
-		secureCommsInbounds  string
-		secureCommsOutbounds string
-		secureCommsKbsAddr   string
+		disableTLS             bool
+		tlsConfig              tlsutil.TLSConfig
+		secureComms            bool
+		secureCommsInbounds    string
+		secureCommsOutbounds   string
+		secureCommsPpInbounds  string
+		secureCommsPpOutbounds string
+		secureCommsKbsAddr     string
 	)
 
 	cmd.Parse(programName, os.Args[1:], func(flags *flag.FlagSet) {
@@ -112,8 +116,10 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 		flags.BoolVar(&tlsConfig.SkipVerify, "tls-skip-verify", false, "Skip TLS certificate verification - use it only for testing")
 		flags.BoolVar(&disableTLS, "disable-tls", false, "Disable TLS encryption - use it only for testing")
 		flags.BoolVar(&secureComms, "secure-comms", false, "Use SSH to secure communication between cluster and peer pods")
-		flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "Inbound tags for secure communication tunnels")
-		flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "Outbound tags for secure communication tunnels")
+		flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "WN Inbound tags for secure communication tunnels")
+		flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "WN Outbound tags for secure communication tunnels")
+		flags.StringVar(&secureCommsPpInbounds, "secure-comms-pp-inbounds", "", "PP Inbound tags for secure communication tunnels")
+		flags.StringVar(&secureCommsPpOutbounds, "secure-comms-pp-outbounds", "", "PP Outbound tags for secure communication tunnels")
 		flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.trustee-operator-system:8080", "Address of a Trustee Service for Secure-Comms")
 		flags.DurationVar(&cfg.serverConfig.ProxyTimeout, "proxy-timeout", proxy.DefaultProxyTimeout, "Maximum timeout in minutes for establishing agent proxy connection")
 
@@ -137,10 +143,15 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 		if err != nil {
 			return nil, fmt.Errorf("secure comms failed to initialize KubeMgr: %w", err)
 		}
+		if strings.EqualFold(secureCommsKbsAddr, "false") {
+			secureCommsKbsAddr = ""
+		}
 
 		cfg.serverConfig.SecureComms = true
 		cfg.serverConfig.SecureCommsInbounds = secureCommsInbounds
 		cfg.serverConfig.SecureCommsOutbounds = secureCommsOutbounds
+		cfg.serverConfig.SecureCommsPpInbounds = secureCommsPpInbounds
+		cfg.serverConfig.SecureCommsPpOutbounds = secureCommsPpOutbounds
 		cfg.serverConfig.SecureCommsKbsAddress = secureCommsKbsAddr
 	} else {
 		if !disableTLS {

diff --git a/src/cloud-api-adaptor/docs/SecureComms.md b/src/cloud-api-adaptor/docs/SecureComms.md
@@ -27,6 +27,34 @@ Once the "Kubernetes Phase" SSH channel is established, Secure Comms connects th
 
 See [Secure Comms Architecture Slides](./SecureComms.pdf) for more details.
 
+## Setup for for testing without Trustee (and for non-CoCo peerpods)
+
+### Deploy CAA
+Use any of the option for installing CAA depending on the cloud driver used.
+
+
+### Activate Secure-Comms feature from CAA side
+Activate Secure-Comms from CAA side by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.  
+
+```sh
+kubectl -n confidential-containers-system  get cm peer-pods-cm  -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
+```
+
+You may also include additional Inbounds and Outbounds configurations to the Adaptor side using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points.
+You may also add Inbounds and Outbounds configurations to the Forwarder side using the `SECURE_COMMS_PP_INBOUNDS` and `SECURE_COMMS_PP_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)
+
+Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example:
+```sh
+apiVersion: v1
+data:
+    ...
+    SECURE_COMMS: "true"
+    SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
+    SECURE_COMMS_PP_INBOUNDS: "KUBERNETES_PHASE:mytunnel:podns:6666"
+    SECURE_COMMS_KBS_ADDR: "false"
+    ...
+```
+
 ## Setup for CoCo with Trustee
 
 ### Deploy CAA
@@ -169,5 +197,4 @@ Alternatively, the client and server can be separately executed in independent t
 
 - Add DeleteResource() support in KBS, KBC, api-server-rest, than cleanup resources added by Secure Comms to KBS whenever a Peer Pod fail to be created or when a Peer Pod is terminated.
 - Add support for running the vxlan tunnel traffic via a Secure Comms SSH tunnel
-- Add support for non-confidential Peer Pods which do not go via an Attestation Phase.
 - Add support for KBS identities allowing a Peer Pod to register its own identity in KBS and replace the current Secure Comms mechanism which delivers a private key to the Peer Pod via the KBS
diff --git a/src/cloud-api-adaptor/entrypoint.sh b/src/cloud-api-adaptor/entrypoint.sh
@@ -24,6 +24,8 @@ optionals+=""
 [[ "${SECURE_COMMS}" == "true" ]] && optionals+="-secure-comms "
 [[ "${SECURE_COMMS_INBOUNDS}" ]] && optionals+="-secure-comms-inbounds ${SECURE_COMMS_INBOUNDS} "
 [[ "${SECURE_COMMS_OUTBOUNDS}" ]] && optionals+="-secure-comms-outbounds ${SECURE_COMMS_OUTBOUNDS} "
+[[ "${SECURE_COMMS_PP_INBOUNDS}" ]] && optionals+="-secure-comms-pp-inbounds ${SECURE_COMMS_PP_INBOUNDS} "
+[[ "${SECURE_COMMS_PP_OUTBOUNDS}" ]] && optionals+="-secure-comms-pp-outbounds ${SECURE_COMMS_PP_OUTBOUNDS} "
 [[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} "
 [[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} "
 

diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
@@ -16,6 +16,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/containerd/containerd/pkg/cri/annotations"
 	pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor"
@@ -27,6 +28,7 @@ import (
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/wnssh"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util"
+	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/tlsutil"
 	provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers"
 	putil "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util/cloudinit"
@@ -48,6 +50,24 @@ type InitData struct {
 	Data      map[string]string `toml:"data,omitempty"`
 }
 
+type ServerConfig struct {
+	TLSConfig               *tlsutil.TLSConfig
+	SocketPath              string
+	PauseImage              string
+	PodsDir                 string
+	ForwarderPort           string
+	ProxyTimeout            time.Duration
+	Initdata                string
+	EnableCloudConfigVerify bool
+	SecureComms             bool
+	SecureCommsInbounds     string
+	SecureCommsOutbounds    string
+	SecureCommsPpInbounds   string
+	SecureCommsPpOutbounds  string
+	SecureCommsKbsAddress   string
+	PeerPodsLimitPerNode    int
+}
+
 var logger = log.New(log.Writer(), "[adaptor/cloud] ", log.LstdFlags|log.Lmsgprefix)
 
 func (s *cloudService) addSandbox(sid sandboxID, sandbox *sandbox) error {
@@ -87,15 +107,20 @@ func (s *cloudService) removeSandbox(id sandboxID) error {
 }
 
 func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNode podnetwork.WorkerNode,
-	secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir,
-	daemonPort, initdata, sshport string) Service {
+	serverConfig *ServerConfig, sshport string) Service {
 	var err error
 	var sshClient *wnssh.SshClient
 
-	if secureComms {
-		inbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, strings.Split(secureCommsInbounds, ",")...)
-		outbounds := append([]string{"BOTH_PHASES:KBS:" + kbsAddress}, strings.Split(secureCommsOutbounds, ",")...)
-		sshClient, err = wnssh.InitSshClient(inbounds, outbounds, kbsAddress, sshport)
+	if serverConfig.SecureComms {
+		inbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, strings.Split(serverConfig.SecureCommsInbounds, ",")...)
+
+		var outbounds []string
+		outbounds = append(outbounds, strings.Split(serverConfig.SecureCommsOutbounds, ",")...)
+		if len(serverConfig.SecureCommsKbsAddress) > 0 {
+			outbounds = append(outbounds, "BOTH_PHASES:KBS:"+serverConfig.SecureCommsKbsAddress)
+		}
+
+		sshClient, err = wnssh.InitSshClient(inbounds, outbounds, serverConfig.SecureCommsKbsAddress, sshport)
 		if err != nil {
 			log.Fatalf("InitSshClient %v", err)
 		}
@@ -105,9 +130,7 @@ func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNo
 		provider:     provider,
 		proxyFactory: proxyFactory,
 		sandboxes:    map[sandboxID]*sandbox{},
-		podsDir:      podsDir,
-		daemonPort:   daemonPort,
-		initdata:     initdata,
+		serverConfig: serverConfig,
 		workerNode:   workerNode,
 		sshClient:    sshClient,
 	}
@@ -224,7 +247,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 		return nil, fmt.Errorf("failed to inspect netns %s: %w", netNSPath, err)
 	}
 
-	podDir := filepath.Join(s.podsDir, string(sid))
+	podDir := filepath.Join(s.serverConfig.PodsDir, string(sid))
 	if err := os.MkdirAll(podDir, os.ModePerm); err != nil {
 		return nil, fmt.Errorf("creating a pod directory: %s, %w", podDir, err)
 	}
@@ -268,6 +291,22 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 		daemonConfig.TLSServerKey = string(keyPEM)
 	}
 
+	var ci *wnssh.SshClientInstance
+
+	if s.sshClient != nil {
+		var ppPrivateKey []byte
+		ci, ppPrivateKey = s.sshClient.InitPP(context.Background(), string(sid))
+		if ci == nil {
+			return nil, fmt.Errorf("failed sshClient.InitPP")
+		}
+
+		daemonConfig.WnPublicKey = s.sshClient.GetWnPublicKey()
+		daemonConfig.PpPrivateKey = ppPrivateKey
+		daemonConfig.SecureCommsOutbounds = s.serverConfig.SecureCommsPpOutbounds
+		daemonConfig.SecureCommsInbounds = s.serverConfig.SecureCommsPpInbounds
+		daemonConfig.SecureComms = true
+	}
+
 	daemonJSON, err := json.MarshalIndent(daemonConfig, "", "    ")
 	if err != nil {
 		return nil, fmt.Errorf("generating JSON data: %w", err)
@@ -308,19 +347,19 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 	logger.Printf("initdata in Pod annotation: %s", initdataStr)
 
 	if initdataStr == "" {
-		logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.initdata)
-		initdataStr = s.initdata
+		logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.serverConfig.Initdata)
+		initdataStr = s.serverConfig.Initdata
 	}
 
 	if initdataStr != "" {
 		decodedBytes, err := base64.StdEncoding.DecodeString(initdataStr)
 		if err != nil {
-			return nil, fmt.Errorf("Error base64 decode initdata: %w", err)
+			return nil, fmt.Errorf("error base64 decode initdata: %w", err)
 		}
 		initdata := InitData{}
 		err = toml.Unmarshal(decodedBytes, &initdata)
 		if err != nil {
-			return nil, fmt.Errorf("Error unmarshalling initdata: %w", err)
+			return nil, fmt.Errorf("error unmarshalling initdata: %w", err)
 		}
 
 		cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{
@@ -330,14 +369,15 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 	}
 
 	sandbox := &sandbox{
-		id:           sid,
-		podName:      pod,
-		podNamespace: namespace,
-		netNSPath:    netNSPath,
-		agentProxy:   agentProxy,
-		podNetwork:   podNetworkConfig,
-		cloudConfig:  cloudConfig,
-		spec:         vmSpec,
+		id:            sid,
+		podName:       pod,
+		podNamespace:  namespace,
+		netNSPath:     netNSPath,
+		agentProxy:    agentProxy,
+		podNetwork:    podNetworkConfig,
+		cloudConfig:   cloudConfig,
+		spec:          vmSpec,
+		sshClientInst: ci,
 	}
 
 	if err := s.addSandbox(sid, sandbox); err != nil {
@@ -381,24 +421,16 @@ func (s *cloudService) StartVM(ctx context.Context, req *pb.StartVMRequest) (res
 	logger.Printf("created an instance %s for sandbox %s", instance.Name, sid)
 
 	instanceIP := instance.IPs[0].String()
-	forwarderPort := s.daemonPort
+	forwarderPort := s.serverConfig.ForwarderPort
 
 	if s.sshClient != nil {
-		ci := s.sshClient.InitPP(context.Background(), string(sid), instance.IPs)
-		if ci == nil {
-			return nil, fmt.Errorf("failed sshClient.InitPP")
-		}
-
-		if err := ci.Start(); err != nil {
+		if err := sandbox.sshClientInst.Start(instance.IPs); err != nil {
 			return nil, fmt.Errorf("failed SshClientInstance.Start: %w", err)
 		}
 
 		// Set agentProxy
 		instanceIP = "127.0.0.1"
-		forwarderPort = ci.GetPort("KATAAGENT")
-
-		// Set ci in sandbox
-		sandbox.sshClientInst = ci
+		forwarderPort = sandbox.sshClientInst.GetPort("KATAAGENT")
 	}
 
 	if err := s.workerNode.Setup(sandbox.netNSPath, instance.IPs, sandbox.podNetwork); err != nil {