Skip to content

Commit

Permalink
Enable connect-injector to talk to external servers. (#1323)
Browse files Browse the repository at this point in the history
  • Loading branch information
ishustava authored and thisisnotashwin committed Sep 19, 2022
1 parent 61d5ea3 commit 4fd8b5d
Show file tree
Hide file tree
Showing 7 changed files with 357 additions and 18 deletions.
8 changes: 7 additions & 1 deletion acceptance/framework/consul/helm_cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ type HelmCluster struct {
// a bootstrap token from a Kubernetes secret stored in the cluster.
ACLToken string

// SkipCheckForPreviousInstallations is a toggle for skipping the check
// if there are any previous installations of this Helm chart in the cluster.
SkipCheckForPreviousInstallations bool

ctx environment.TestContext
helmOptions *helm.Options
releaseName string
Expand Down Expand Up @@ -109,7 +113,9 @@ func (h *HelmCluster) Create(t *testing.T) {
})

// Fail if there are any existing installations of the Helm chart.
helpers.CheckForPriorInstallations(t, h.kubernetesClient, h.helmOptions, "consul-helm", "chart=consul-helm")
if !h.SkipCheckForPreviousInstallations {
helpers.CheckForPriorInstallations(t, h.kubernetesClient, h.helmOptions, "consul-helm", "chart=consul-helm")
}

chartName := config.HelmChartPath
if h.helmOptions.Version != config.HelmChartPath {
Expand Down
135 changes: 135 additions & 0 deletions acceptance/tests/connect/connect_external_servers_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
package connect

import (
"context"
"fmt"
"strconv"
"testing"

"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
"github.com/hashicorp/consul/api"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// TestConnectInject_ExternalServers tests that connect works when using external servers.
// It sets up an external Consul server in the same cluster but a different Helm installation
// and then treats this server as external.
func TestConnectInject_ExternalServers(t *testing.T) {
cases := []bool{false, true}

for _, secure := range cases {
caseName := fmt.Sprintf("secure: %t", secure)
t.Run(caseName, func(t *testing.T) {
cfg := suite.Config()
ctx := suite.Environment().DefaultContext(t)

serverHelmValues := map[string]string{
"global.acls.manageSystemACLs": strconv.FormatBool(secure),
"global.tls.enabled": strconv.FormatBool(secure),
}
serverReleaseName := helpers.RandomName()
consulServerCluster := consul.NewHelmCluster(t, serverHelmValues, ctx, cfg, serverReleaseName)

consulServerCluster.Create(t)

helmValues := map[string]string{
"server.enabled": "false",
"global.acls.manageSystemACLs": strconv.FormatBool(secure),

"global.tls.enabled": strconv.FormatBool(secure),

"connectInject.enabled": "true",

"externalServers.enabled": "true",
"externalServers.hosts[0]": fmt.Sprintf("%s-consul-server", serverReleaseName),
"externalServers.httpsPort": "8500",
}

if secure {
helmValues["global.tls.caCert.secretName"] = fmt.Sprintf("%s-consul-ca-cert", serverReleaseName)
helmValues["global.tls.caCert.secretKey"] = "tls.crt"
helmValues["global.acls.bootstrapToken.secretName"] = fmt.Sprintf("%s-consul-bootstrap-acl-token", serverReleaseName)
helmValues["global.acls.bootstrapToken.secretKey"] = "token"
helmValues["externalServers.httpsPort"] = "8501"
}

releaseName := helpers.RandomName()
consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName)
consulCluster.SkipCheckForPreviousInstallations = true

consulCluster.Create(t)

logger.Log(t, "creating static-server and static-client deployments")
k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject")
if cfg.EnableTransparentProxy {
k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy")
} else {
k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject")
}

// Check that both static-server and static-client have been injected and now have 2 containers.
for _, labelSelector := range []string{"app=static-server", "app=static-client"} {
podList, err := ctx.KubernetesClient(t).CoreV1().Pods(metav1.NamespaceAll).List(context.Background(), metav1.ListOptions{
LabelSelector: labelSelector,
})
require.NoError(t, err)
require.Len(t, podList.Items, 1)
require.Len(t, podList.Items[0].Spec.Containers, 2)
}

if secure {
consulClient, _ := consulServerCluster.SetupConsulClient(t, true)

logger.Log(t, "checking that the connection is not successful because there's no intention")
if cfg.EnableTransparentProxy {
k8s.CheckStaticServerConnectionFailing(t, ctx.KubectlOptions(t), StaticClientName, "http://static-server")
} else {
k8s.CheckStaticServerConnectionFailing(t, ctx.KubectlOptions(t), StaticClientName, "http://localhost:1234")
}

intention := &api.ServiceIntentionsConfigEntry{
Kind: api.ServiceIntentions,
Name: staticServerName,
Sources: []*api.SourceIntention{
{
Name: StaticClientName,
Action: api.IntentionActionAllow,
},
},
}

logger.Log(t, "creating intention")
_, _, err := consulClient.ConfigEntries().Set(intention, nil)
require.NoError(t, err)
}

logger.Log(t, "checking that connection is successful")
if cfg.EnableTransparentProxy {
k8s.CheckStaticServerConnectionSuccessful(t, ctx.KubectlOptions(t), StaticClientName, "http://static-server")
} else {
k8s.CheckStaticServerConnectionSuccessful(t, ctx.KubectlOptions(t), StaticClientName, "http://localhost:1234")
}

// Test that kubernetes readiness status is synced to Consul.
// Create the file so that the readiness probe of the static-server pod fails.
logger.Log(t, "testing k8s -> consul health checks sync by making the static-server unhealthy")
k8s.RunKubectl(t, ctx.KubectlOptions(t), "exec", "deploy/"+staticServerName, "--", "touch", "/tmp/unhealthy")

// The readiness probe should take a moment to be reflected in Consul, CheckStaticServerConnection will retry
// until Consul marks the service instance unavailable for mesh traffic, causing the connection to fail.
// We are expecting a "connection reset by peer" error because in a case of health checks,
// there will be no healthy proxy host to connect to. That's why we can't assert that we receive an empty reply
// from server, which is the case when a connection is unsuccessful due to intentions in other tests.
logger.Log(t, "checking that connection is unsuccessful")
if cfg.EnableTransparentProxy {
k8s.CheckStaticServerConnectionMultipleFailureMessages(t, ctx.KubectlOptions(t), StaticClientName, false, []string{"curl: (56) Recv failure: Connection reset by peer", "curl: (52) Empty reply from server", "curl: (7) Failed to connect to static-server port 80: Connection refused"}, "", "http://static-server.%s")
} else {
k8s.CheckStaticServerConnectionMultipleFailureMessages(t, ctx.KubectlOptions(t), StaticClientName, false, []string{"curl: (56) Recv failure: Connection reset by peer", "curl: (52) Empty reply from server"}, "", "http://localhost:1234")
}
})
}
}
2 changes: 0 additions & 2 deletions acceptance/tests/connect/connect_inject_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ func TestConnectInject(t *testing.T) {
clusterKind consul.ClusterKind
releaseName string
secure bool
autoEncrypt bool
}{
"Helm install without secure": {
clusterKind: consul.Helm,
Expand Down Expand Up @@ -60,7 +59,6 @@ func TestConnectInject(t *testing.T) {
connHelper := ConnectHelper{
ClusterKind: c.clusterKind,
Secure: c.secure,
AutoEncrypt: c.autoEncrypt,
ReleaseName: c.releaseName,
Ctx: ctx,
Cfg: cfg,
Expand Down
37 changes: 33 additions & 4 deletions charts/consul/templates/connect-inject-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -102,13 +102,15 @@ spec:
name: {{ .Values.connectInject.aclInjectToken.secretName }}
key: {{ .Values.connectInject.aclInjectToken.secretKey }}
{{- end }}
{{- if not .Values.externalServers.enabled }}
- name: CONSUL_HTTP_ADDR
{{- if .Values.global.tls.enabled }}
value: https://{{ template "consul.fullname" . }}-server.{{ .Release.Namespace }}.svc:8501
{{- else }}
value: http://{{ template "consul.fullname" . }}-server.{{ .Release.Namespace }}.svc:8500
{{- end }}
{{- if (and .Values.global.tls.enabled (not .Values.externalServers.useSystemRoots)) }}
{{- end }}
{{- if (and .Values.global.tls.enabled (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots))) }}
- name: CONSUL_CACERT
{{- if .Values.global.secretsBackend.vault.enabled }}
value: "/vault/secrets/serverca.crt"
Expand All @@ -132,6 +134,19 @@ spec:
-release-namespace="{{ .Release.Namespace }}" \
-resource-prefix={{ template "consul.fullname" . }} \
-listen=:8080 \
{{- if and .Values.externalServers.enabled (not .Values.externalServers.hosts) }}{{ fail "externalServers.hosts must be set if externalServers.enabled is true" }}{{ end -}}
{{- if .Values.externalServers.enabled }}
{{- if .Values.global.tls.enabled }}
-use-https \
{{- end }}
{{- range .Values.externalServers.hosts }}
-server-address={{ quote . }} \
{{- end }}
-server-port={{ .Values.externalServers.httpsPort }} \
{{- if .Values.externalServers.tlsServerName }}
-tls-server-name={{ .Values.externalServers.tlsServerName }} \
{{- end }}
{{- end }}
{{- if .Values.connectInject.transparentProxy.defaultEnabled }}
-default-enable-transparent-proxy=true \
{{- else }}
Expand Down Expand Up @@ -312,7 +327,7 @@ spec:
- mountPath: /consul/login
name: consul-data
readOnly: true
{{- if .Values.global.tls.enabled }}
{{- if and .Values.global.tls.enabled (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots))}}
- name: consul-ca-cert
mountPath: /consul/tls/ca
readOnly: true
Expand Down Expand Up @@ -349,13 +364,15 @@ spec:
initContainers:
- name: connect-injector-acl-init
env:
{{- if not .Values.externalServers.enabled }}
- name: CONSUL_HTTP_ADDR
{{- if .Values.global.tls.enabled }}
value: https://{{ template "consul.fullname" . }}-server.{{ .Release.Namespace }}.svc:8501
{{- else }}
value: http://{{ template "consul.fullname" . }}-server.{{ .Release.Namespace }}.svc:8500
{{- end }}
{{- if (and .Values.global.tls.enabled (not .Values.externalServers.useSystemRoots)) }}
{{- end }}
{{- if (and .Values.global.tls.enabled (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots))) }}
- name: CONSUL_CACERT
{{- if .Values.global.secretsBackend.vault.enabled }}
value: "/vault/secrets/serverca.crt"
Expand All @@ -368,7 +385,7 @@ spec:
- mountPath: /consul/login
name: consul-data
readOnly: false
{{- if .Values.global.tls.enabled }}
{{- if and .Values.global.tls.enabled (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots))}}
- name: consul-ca-cert
mountPath: /consul/tls/ca
readOnly: true
Expand All @@ -388,6 +405,18 @@ spec:
{{- if .Values.global.adminPartitions.enabled }}
-partition={{ .Values.global.adminPartitions.name }} \
{{- end }}
{{- if .Values.externalServers.enabled }}
{{- if .Values.global.tls.enabled }}
-use-https \
{{- end }}
{{- range .Values.externalServers.hosts }}
-server-address={{ quote . }} \
{{- end }}
-server-port={{ .Values.externalServers.httpsPort }} \
{{- if .Values.externalServers.tlsServerName }}
-tls-server-name={{ .Values.externalServers.tlsServerName }} \
{{- end }}
{{- end }}
-consul-api-timeout={{ .Values.global.consulAPITimeout }} \
-log-level={{ default .Values.global.logLevel .Values.connectInject.logLevel }} \
-log-json={{ .Values.global.logJSON }}
Expand Down
Loading

0 comments on commit 4fd8b5d

Please sign in to comment.