Skip to content

Commit

Permalink
SecureComms: Add support daemonConfig
Browse files Browse the repository at this point in the history 
Support configuring the APF Secure Comms from the CAA side including:
 - WN public Key
 - PP private key
 - Activating Secure Comms
 - inbouns and outbounds of th PP

This is useful for activating Secure Comms from the CAA and without
Trustee. It can be used for Testing without producing dedicated podvms
which activate Secure Comms and set Inbounds/Outbounds by default.
It can also be used for non-Coco peerpods.

Signed-off-by: David Hadas <david.hadas@gmail.com>
  • Loading branch information
@davidhadas @davidhIBM
davidhadas authored and davidhIBM committed Oct 9, 2024
1 parent 04ba307 commit 4dbf850
Show file tree
Hide file tree
Showing 18 changed files with 341 additions and 164 deletions.
28 changes: 24 additions & 4 deletions src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,9 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
return nil, err
}

if secureComms {
if secureComms || cfg.daemonConfig.SecureComms {
var inbounds, outbounds []string

ppssh.Singleton()
host, port, err := net.SplitHostPort(cfg.listenAddr)
if err != nil {
Expand All @@ -103,15 +105,33 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
logger.Printf("Address %s is changed to 127.0.0.1:%s since secure-comms is enabled.", cfg.listenAddr, port)
cfg.listenAddr = "127.0.0.1:" + port
}
inbounds := append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...)
outbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...)

// Create a Client that will approach the api-server-rest service at the podns
// To obtain secrets from KBS, we approach the api-server-rest service which then approaches the CDH asking for a secret resource
// the CDH than contact the KBS (possibly after approaching Attestation Agent for a token) and the KBS serves the requested key
// The communication between the CDH (and Attestation Agent) and the KBS is performed via an SSH tunnel named "KBS"
apic := apic.NewApiClient(API_SERVER_REST_PORT, cfg.podNamespace)
services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppssh.GetSecret(apic.GetKey), sshutil.SSHPORT))

ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(apic.GetKey))

if secureComms {
// CoCo in production
ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY)
ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY)
} else {
// Never here under CoCo in production
// Set secureComms using daemonConfig for testing
ppSecrets.SetKey(ppssh.WN_PUBLIC_KEY, cfg.daemonConfig.WnPublicKey)
ppSecrets.SetKey(ppssh.PP_PRIVATE_KEY, cfg.daemonConfig.PpPrivateKey)
}

inbounds = append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...)
inbounds = append(inbounds, strings.Split(cfg.daemonConfig.SecureCommsInbounds, ",")...)

outbounds = append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...)
outbounds = append(outbounds, strings.Split(cfg.daemonConfig.SecureCommsOutbounds, ",")...)

services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppSecrets, sshutil.SSHPORT))
} else {
if !disableTLS {
cfg.tlsConfig = &tlsConfig
Expand Down
31 changes: 21 additions & 10 deletions src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ import (
"fmt"
"io"
"os"
"strings"

"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy"
daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder"
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler/vxlan"
Expand All @@ -28,7 +30,7 @@ const (
)

type daemonConfig struct {
serverConfig adaptor.ServerConfig
serverConfig cloud.ServerConfig
networkConfig
}

Expand Down Expand Up @@ -86,12 +88,14 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
}

var (
disableTLS bool
tlsConfig tlsutil.TLSConfig
secureComms bool
secureCommsInbounds string
secureCommsOutbounds string
secureCommsKbsAddr string
disableTLS bool
tlsConfig tlsutil.TLSConfig
secureComms bool
secureCommsInbounds string
secureCommsOutbounds string
secureCommsPpInbounds string
secureCommsPpOutbounds string
secureCommsKbsAddr string
)

cmd.Parse(programName, os.Args[1:], func(flags *flag.FlagSet) {
Expand All @@ -112,9 +116,11 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
flags.BoolVar(&tlsConfig.SkipVerify, "tls-skip-verify", false, "Skip TLS certificate verification - use it only for testing")
flags.BoolVar(&disableTLS, "disable-tls", false, "Disable TLS encryption - use it only for testing")
flags.BoolVar(&secureComms, "secure-comms", false, "Use SSH to secure communication between cluster and peer pods")
flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "Inbound tags for secure communication tunnels")
flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "Outbound tags for secure communication tunnels")
flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.kbs-operator-system:8080", "Address of a KBS Service for Secure-Comms")
flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "WN Inbound tags for secure communication tunnels")
flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "WN Outbound tags for secure communication tunnels")
flags.StringVar(&secureCommsPpInbounds, "secure-comms-pp-inbounds", "", "PP Inbound tags for secure communication tunnels")
flags.StringVar(&secureCommsPpOutbounds, "secure-comms-pp-outbounds", "", "PP Outbound tags for secure communication tunnels")
flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.trustee-operator-system:8080", "Address of a Trustee Service for Secure-Comms")
flags.DurationVar(&cfg.serverConfig.ProxyTimeout, "proxy-timeout", proxy.DefaultProxyTimeout, "Maximum timeout in minutes for establishing agent proxy connection")

flags.StringVar(&cfg.networkConfig.TunnelType, "tunnel-type", podnetwork.DefaultTunnelType, "Tunnel provider")
Expand All @@ -137,10 +143,15 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
if err != nil {
return nil, fmt.Errorf("secure comms failed to initialize KubeMgr: %w", err)
}
if strings.EqualFold(secureCommsKbsAddr, "false") {
secureCommsKbsAddr = ""
}

cfg.serverConfig.SecureComms = true
cfg.serverConfig.SecureCommsInbounds = secureCommsInbounds
cfg.serverConfig.SecureCommsOutbounds = secureCommsOutbounds
cfg.serverConfig.SecureCommsPpInbounds = secureCommsPpInbounds
cfg.serverConfig.SecureCommsPpOutbounds = secureCommsPpOutbounds
cfg.serverConfig.SecureCommsKbsAddress = secureCommsKbsAddr
} else {
if !disableTLS {
Expand Down
98 changes: 86 additions & 12 deletions src/cloud-api-adaptor/docs/SecureComms.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,35 @@ Once the "Kubernetes Phase" SSH channel is established, Secure Comms connects th

See [Secure Comms Architecture Slides](./SecureComms.pdf) for more details.

## Setup
## Setup for for testing without Trustee (and for non-CoCo peerpods)

### Deploy CAA
Use any of the option for installing CAA depending on the cloud driver used.


### Activate Secure-Comms feature from CAA side
Activate Secure-Comms from CAA side by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.

```sh
kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
```

You may also include additional Inbounds and Outbounds configurations to the Adaptor side using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points.
You may also add Inbounds and Outbounds configurations to the Forwarder side using the `SECURE_COMMS_PP_INBOUNDS` and `SECURE_COMMS_PP_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)

Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example:
```sh
apiVersion: v1
data:
...
SECURE_COMMS: "true"
SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
SECURE_COMMS_PP_INBOUNDS: "KUBERNETES_PHASE:mytunnel:6666"
SECURE_COMMS_KBS_ADDR: "false"
...
```

## Setup for CoCo with Trustee

### Deploy CAA
Use any of the option for installing CAA depending on the cloud driver used.
Expand All @@ -37,19 +65,25 @@ Deploy Trustee-Operator by following instructions at [trustee Operator Getting S

Make sure to uncomment the secret generation as recommended for both public and private key (`kbs-auth-public-key` and `kbs-client` secrets).

Copy the kbs-client secret from the `kbs-operator-system` namespace to the `confidential-containers-system` ns. This can be done using:
```sh
kubectl get secrets -n trustee-operator-system
NAME TYPE DATA AGE
kbs-auth-public-key Opaque 1 28h
kbs-client Opaque 1 28h
```

Copy the kbs-client secret from the `trustee-operator-system` namespace to the `confidential-containers-system` ns. This can be done using:

```sh
kubectl get secret kbs-client -n kbs-operator-system -o json|jq --arg ns "confidential-containers-system" 'del(.metadata["creationTimestamp","resourceVersion","selfLink","uid","annotations"]) | .metadata.namespace |= $ns' |kubectl apply -f -
kubectl get secret kbs-client -n trustee-operator-system -o json|jq --arg ns "confidential-containers-system" 'del(.metadata["creationTimestamp","resourceVersion","selfLink","uid","annotations"]) | .metadata.namespace |= $ns' |kubectl apply -f -
```

For a testing environment, you may need to change the policy of the KBS and AS using the KBS Client to allow all or fit your own policy. One way to do that is:

```sh
kubectl -n kbs-operator-system exec deployment/trustee-deployment --container as -it -- /bin/bash
sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego
kubectl -n trustee-operator-system exec deployment/trustee-deployment --container as -it -- sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego

kubectl -n kbs-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f -
kubectl -n trustee-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f -
```

### Build a podvm that enforces Secure-Comms
Expand All @@ -59,27 +93,68 @@ Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol-
ExecStart=/usr/local/bin/agent-protocol-forwarder -pod-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
```

You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. See more details regarding Inbounds and Outbounds below.
You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)

For example:
```sh
ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -secure-comms-inbounds KUBERNETES_PHASE:mytunnel:6666 -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
```

Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md).


### Activate CAA Secure-Comms feature
Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to add to the `peer-pods-cm` config map at the `confidential-containers-system` namespace:
Activate Secure-Comms of CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.

```sh
kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
```

Set InitData to point KBC services to IP address 127.0.0.1
```sh
cat <<EOF > /tmp/initdata.txt
algorithm = "sha384"
version = "0.1.0"
[data]
"aa.toml" = '''
[token_configs]
[token_configs.coco_as]
url = 'http://127.0.0.1:8080'
[token_configs.kbs]
url = 'http://127.0.0.1:8080'
'''
"cdh.toml" = '''
socket = 'unix:///run/confidential-containers/cdh.sock'
credentials = []
[kbc]
name = 'cc_kbc'
url = 'http://127.0.0.1:8080'
'''
EOF
export INITDATA=`base64 -w 0 /tmp/initdata.txt`
kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed 's/^INITDATA: .*/INITDATA: '$INITDATA'/'|kubectl apply -f -

```

You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)

Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example:
```sh
apiVersion: v1
data:
...
SECURE_COMMS: "true"
SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
...
```

You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. See more details regarding Inbounds and Outbounds below.

You may also set the KBS address using the `SECURE_COMMS_KBS_ADDR` config point.


### Adding named tunnels to the SSH channel
## Adding named tunnels to the SSH channel
Named tunnels can be added to the SSH channel. Adding a named tunnel requires adding an Inbound at one of the SSH channel peers and an Outbound at the other SSH channel peer. The Inbound and Outbound both carry the name of the tunnel being created.

|---------Tunnel----------|
Expand Down Expand Up @@ -117,5 +192,4 @@ Alternatively, the client and server can be separately executed in independent t

- Add DeleteResource() support in KBS, KBC, api-server-rest, than cleanup resources added by Secure Comms to KBS whenever a Peer Pod fail to be created or when a Peer Pod is terminated.
- Add support for running the vxlan tunnel traffic via a Secure Comms SSH tunnel
- Add support for non-confidential Peer Pods which do not go via an Attestation Phase.
- Add support for KBS identities allowing a Peer Pod to register its own identity in KBS and replace the current Secure Comms mechanism which delivers a private key to the Peer Pod via the KBS
2 changes: 2 additions & 0 deletions src/cloud-api-adaptor/entrypoint.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ optionals+=""
[[ "${SECURE_COMMS}" == "true" ]] && optionals+="-secure-comms "
[[ "${SECURE_COMMS_INBOUNDS}" ]] && optionals+="-secure-comms-inbounds ${SECURE_COMMS_INBOUNDS} "
[[ "${SECURE_COMMS_OUTBOUNDS}" ]] && optionals+="-secure-comms-outbounds ${SECURE_COMMS_OUTBOUNDS} "
[[ "${SECURE_COMMS_PP_INBOUNDS}" ]] && optionals+="-secure-comms-pp-inbounds ${SECURE_COMMS_PP_INBOUNDS} "
[[ "${SECURE_COMMS_PP_OUTBOUNDS}" ]] && optionals+="-secure-comms-pp-outbounds ${SECURE_COMMS_PP_OUTBOUNDS} "
[[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} "
[[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} "

Expand Down
Loading

0 comments on commit 4dbf850

Please sign in to comment.