Make ECK IPv6 compatible

cmd/manager/main.go

@@ -34,6 +34,7 @@ import (
 	controllerscheme "github.com/elastic/cloud-on-k8s/pkg/controller/common/scheme"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/tracing"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/settings"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/enterprisesearch"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/kibana"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/license"
@@ -440,6 +441,7 @@ func startOperator(stopChan <-chan struct{}) error {
 	}
 	params := operator.Parameters{
 		Dialer:            dialer,
+		IPFamily:          net.ToIPFamily(os.Getenv(settings.EnvPodIP)),
 		OperatorNamespace: operatorNamespace,
 		OperatorInfo:      operatorInfo,
 		CACertRotation: certificates.RotationParams{

config/e2e/operator.yaml

@@ -259,6 +259,10 @@ spec:
           - name: ELASTIC_APM_ENVIRONMENT
             value: {{ .Pipeline }}-{{ .BuildNumber }}-{{ .Provider }}-{{ .ClusterName }}-{{ .KubernetesVersion }}-{{ .ElasticStackVersion }}
           {{end}}
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
         resources:
           limits:
             cpu: 1

hack/manifest-gen/assets/charts/eck/templates/statefulset.yaml

@@ -35,6 +35,10 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
           {{- if .Values.config.webhook.enabled }}
           - name: WEBHOOK_SECRET
             value: "{{ .Values.config.webhook.certsSecret }}"

pkg/apis/elasticsearch/v1/validations.go

@@ -117,7 +117,7 @@ func validSanIP(es *Elasticsearch) field.ErrorList {
 	if selfSignedCerts != nil {
 		for _, san := range selfSignedCerts.SubjectAlternativeNames {
 			if san.IP != "" {
-				ip := netutil.MaybeIPTo4(net.ParseIP(san.IP))
+				ip := netutil.IPToRFCForm(net.ParseIP(san.IP))
 				if ip == nil {
 					errs = append(errs, field.Invalid(field.NewPath("spec").Child("http", "tls", "selfSignedCertificate", "subjectAlternativeNames"), san.IP, invalidSanIPErrMsg))
 				}

pkg/apis/elasticsearch/v1beta1/validations.go

@@ -112,7 +112,7 @@ func validSanIP(es *Elasticsearch) field.ErrorList {
 	if selfSignedCerts != nil {
 		for _, san := range selfSignedCerts.SubjectAlternativeNames {
 			if san.IP != "" {
-				ip := netutil.MaybeIPTo4(net.ParseIP(san.IP))
+				ip := netutil.IPToRFCForm(net.ParseIP(san.IP))
 				if ip == nil {
 					errs = append(errs, field.Invalid(field.NewPath("spec").Child("http", "tls", "selfSignedCertificate", "subjectAlternativeNames"), san.IP, invalidSanIPErrMsg))
 				}

pkg/controller/common/certificates/http_reconcile.go

@@ -362,7 +362,7 @@ func createValidatedHTTPCertificateTemplate(
 				dnsNames = append(dnsNames, san.DNS)
 			}
 			if san.IP != "" {
-				ipAddresses = append(ipAddresses, netutil.MaybeIPTo4(net.ParseIP(san.IP)))
+				ipAddresses = append(ipAddresses, netutil.IPToRFCForm(net.ParseIP(san.IP)))
 			}
 		}
 	}

pkg/controller/common/operator/parameters.go

@@ -6,6 +6,7 @@ package operator
 
 import (
 	"go.elastic.co/apm"
+	corev1 "k8s.io/api/core/v1"
 
 	"github.com/elastic/cloud-on-k8s/pkg/about"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates"
@@ -20,6 +21,8 @@ type Parameters struct {
 	OperatorInfo about.OperatorInfo
 	// Dialer is used to create the Elasticsearch HTTP client.
 	Dialer net.Dialer
+	// IPFamily represents the IP family to use when creating configuration and services.
+	IPFamily corev1.IPFamily
 	// CACertRotation defines the rotation params for CA certificates.
 	CACertRotation certificates.RotationParams
 	// CertRotation defines the rotation params for non-CA certificates.

pkg/controller/elasticsearch/certificates/transport/csr.go

@@ -11,12 +11,14 @@ import (
 	"net"
 	"time"
 
+	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/sset"
+
+	netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 
 	esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates"
-	netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net"
 )
 
 // createValidatedCertificateTemplate validates a CSR and creates a certificate template.
@@ -90,8 +92,10 @@ func buildGeneralNames(
 		// add the transport service name for remote cluster connections initially connecting through the service
 		// the DNS name has to match the seed hosts configured in the remote cluster settings
 		{DNSName: fmt.Sprintf("%s.%s.svc", esv1.TransportService(cluster.Name), cluster.Namespace)},
-		{IPAddress: netutil.MaybeIPTo4(podIP)},
-		{IPAddress: net.ParseIP("127.0.0.1").To4()},
+		// add the resolvable DNS name of the Pod as published by Elasticsearch
+		{DNSName: sset.PodDNSName(pod)},
+		{IPAddress: netutil.IPToRFCForm(podIP)},
+		{IPAddress: netutil.IPToRFCForm(netutil.LoopbackFor(netutil.ToIPFamily(podIP.String())))},
 	}
 
 	return generalNames, nil

pkg/controller/elasticsearch/certificates/transport/csr_test.go

@@ -93,6 +93,7 @@ func Test_buildGeneralNames(t *testing.T) {
 				{OtherName: *otherName},
 				{DNSName: expectedCommonName},
 				{DNSName: expectedTransportSvcName},
+				{DNSName: "test-pod-name.test-sset"},
 				{IPAddress: net.ParseIP(testIP).To4()},
 				{IPAddress: net.ParseIP("127.0.0.1").To4()},
 			},

pkg/controller/elasticsearch/certificates/transport/transport_fixtures_test.go

@@ -11,6 +11,7 @@ import (
 	"crypto/x509/pkix"
 	"encoding/pem"
 
+	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -34,6 +35,9 @@ var (
 	testPod = corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "test-pod-name",
+			Labels: map[string]string{
+				label.StatefulSetNameLabelName: "test-sset",
+			},
 		},
 		Status: corev1.PodStatus{
 			PodIP: testIP,

pkg/controller/elasticsearch/driver/upscale_test.go

@@ -124,8 +124,8 @@ func TestHandleUpscaleAndSpecChanges(t *testing.T) {
 	require.Equal(t, pointer.Int32(4), sset2.Spec.Replicas)
 	comparison.RequireEqual(t, &updatedStatefulSets[1], &sset2)
 	// headless services should be created for both
-	require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: nodespec.HeadlessServiceName("sset1")}, &corev1.Service{}))
-	require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: nodespec.HeadlessServiceName("sset2")}, &corev1.Service{}))
+	require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: sset.HeadlessServiceName("sset1")}, &corev1.Service{}))
+	require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: sset.HeadlessServiceName("sset2")}, &corev1.Service{}))
 	// config should be created for both
 	require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: esv1.ConfigSecret("sset1")}, &corev1.Secret{}))
 	require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: esv1.ConfigSecret("sset2")}, &corev1.Secret{}))

pkg/controller/elasticsearch/nodespec/podspec.go

@@ -8,6 +8,8 @@ import (
 	"crypto/sha256"
 	"fmt"
 
+	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/sset"
+
 	corev1 "k8s.io/api/core/v1"
 
 	esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1"
@@ -58,7 +60,7 @@ func BuildPodTemplateSpec(
 		WithPorts(defaultContainerPorts).
 		WithReadinessProbe(*NewReadinessProbe()).
 		WithAffinity(DefaultAffinity(es.Name)).
-		WithEnv(DefaultEnvVars(es.Spec.HTTP, HeadlessServiceName(esv1.StatefulSet(es.Name, nodeSet.Name)))...).
+		WithEnv(DefaultEnvVars(es.Spec.HTTP, sset.HeadlessServiceName(esv1.StatefulSet(es.Name, nodeSet.Name)))...).
 		WithVolumes(volumes...).
 		WithVolumeMounts(volumeMounts...).
 		WithInitContainers(initContainers...).

pkg/controller/elasticsearch/nodespec/podspec_test.go

@@ -8,6 +8,8 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/sset"
+
 	commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1"
 	esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/defaults"
@@ -137,7 +139,7 @@ func TestBuildPodTemplateSpec(t *testing.T) {
 			Labels: map[string]string{
 				"common.k8s.elastic.co/type":                    "elasticsearch",
 				"elasticsearch.k8s.elastic.co/cluster-name":     "name",
-				"elasticsearch.k8s.elastic.co/config-hash":      "2311754148",
+				"elasticsearch.k8s.elastic.co/config-hash":      "3785137207",
 				"elasticsearch.k8s.elastic.co/http-scheme":      "https",
 				"elasticsearch.k8s.elastic.co/node-data":        "false",
 				"elasticsearch.k8s.elastic.co/node-ingest":      "true",
@@ -173,7 +175,7 @@ func TestBuildPodTemplateSpec(t *testing.T) {
 					},
 					Env: append(
 						[]corev1.EnvVar{{Name: "my-env", Value: "my-value"}},
-						DefaultEnvVars(sampleES.Spec.HTTP, HeadlessServiceName(esv1.StatefulSet(sampleES.Name, nodeSet.Name)))...),
+						DefaultEnvVars(sampleES.Spec.HTTP, sset.HeadlessServiceName(esv1.StatefulSet(sampleES.Name, nodeSet.Name)))...),
 					Resources:      DefaultResources,
 					VolumeMounts:   volumeMounts,
 					ReadinessProbe: NewReadinessProbe(),

pkg/controller/elasticsearch/nodespec/readiness_probe.go

@@ -64,9 +64,17 @@ else
   BASIC_AUTH=''
 fi
 
+# Check if we are using IPv6
+if [[ $POD_IP =~ .*:.* ]]; then
+  LOOPBACK=[::1] 
+else 
+  LOOPBACK=127.0.01
+fi
+
 # request Elasticsearch on /
-ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://127.0.0.1:9200/"
-status=$(curl -o /dev/null -w "%{http_code}" --max-time ${READINESS_PROBE_TIMEOUT} -XGET -s -k ${BASIC_AUTH} $ENDPOINT)
+# we are turning globbing off to allow for unescaped [] in case of IPv6
+ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://${LOOPBACK}:9200/"
+status=$(curl -o /dev/null -w "%{http_code}" --max-time ${READINESS_PROBE_TIMEOUT} -XGET -g -s -k ${BASIC_AUTH} $ENDPOINT)
 curl_rc=$?
 
 if [[ ${curl_rc} -ne 0 ]]; then

pkg/controller/elasticsearch/nodespec/statefulset.go

@@ -27,20 +27,14 @@ var (
 	f = false
 )
 
-// HeadlessServiceName returns the name of the headless service for the given StatefulSet.
-func HeadlessServiceName(ssetName string) string {
-	// just use the sset name
-	return ssetName
-}
-
 // HeadlessService returns a headless service for the given StatefulSet
 func HeadlessService(es *esv1.Elasticsearch, ssetName string) corev1.Service {
 	nsn := k8s.ExtractNamespacedName(es)
 
 	return corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: nsn.Namespace,
-			Name:      HeadlessServiceName(ssetName),
+			Name:      sset.HeadlessServiceName(ssetName),
 			Labels:    label.NewStatefulSetLabels(nsn, ssetName),
 		},
 		Spec: corev1.ServiceSpec{
@@ -54,6 +48,8 @@ func HeadlessService(es *esv1.Elasticsearch, ssetName string) corev1.Service {
 					Port:     network.HTTPPort,
 				},
 			},
+			// allow nodes to discover themselves via DNS while they are booting up ie. are not ready yet
+			PublishNotReadyAddresses: true,
 		},
 	}
 }
@@ -113,7 +109,7 @@ func BuildStatefulSet(
 			// use default revision history limit
 			RevisionHistoryLimit: nil,
 			// build a headless service per StatefulSet, matching the StatefulSet labels
-			ServiceName: HeadlessServiceName(statefulSetName),
+			ServiceName: sset.HeadlessServiceName(statefulSetName),
 			Selector: &metav1.LabelSelector{
 				MatchLabels: ssetSelector,
 			},

pkg/controller/elasticsearch/settings/masters.go

@@ -6,7 +6,6 @@ package settings
 
 import (
 	"context"
-	"fmt"
 	"reflect"
 	"sort"
 	"strings"
@@ -16,7 +15,7 @@ import (
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/tracing"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label"
-	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/network"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/sset"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume"
 	"github.com/elastic/cloud-on-k8s/pkg/utils/k8s"
 	"go.elastic.co/apm"
@@ -58,7 +57,7 @@ func UpdateSeedHostsConfigMap(
 		if len(master.Status.PodIP) > 0 { // do not add pod with no IPs
 			seedHosts = append(
 				seedHosts,
-				fmt.Sprintf("%s:%d", master.Status.PodIP, network.TransportPort),
+				sset.PodDNSName(master),
 			)
 		}
 	}

pkg/controller/elasticsearch/settings/masters_test.go

@@ -33,6 +33,7 @@ func newPodWithIP(name, ip string, master bool) corev1.Pod {
 		},
 	}
 	label.NodeTypesMasterLabelName.Set(master, p.Labels)
+	p.Labels[label.StatefulSetNameLabelName] = "test-sset"
 	return p
 }
 
@@ -97,7 +98,7 @@ func TestUpdateSeedHostsConfigMap(t *testing.T) {
 				es: es,
 			},
 			wantErr:         false,
-			expectedContent: "10.0.3.3:9300\n10.0.9.2:9300",
+			expectedContent: "master1.test-sset\nmaster3.test-sset",
 		},
 		{
 			name: "All masters have IPs, some nodes don't",
@@ -113,7 +114,7 @@ func TestUpdateSeedHostsConfigMap(t *testing.T) {
 				es: es,
 			},
 			wantErr:         false,
-			expectedContent: "10.0.3.3:9300\n10.0.6.5:9300\n10.0.9.2:9300",
+			expectedContent: "master1.test-sset\nmaster2.test-sset\nmaster3.test-sset",
 		},
 		{
 			name: "Ordering of pods should not matter",
@@ -127,7 +128,7 @@ func TestUpdateSeedHostsConfigMap(t *testing.T) {
 				es: es,
 			},
 			wantErr:         false,
-			expectedContent: "10.0.3.3:9300\n10.0.6.5:9300\n10.0.9.2:9300",
+			expectedContent: "master1.test-sset\nmaster2.test-sset\nmaster3.test-sset",
 		},
 	}
 	for _, tt := range tests {

pkg/controller/elasticsearch/settings/merged_config.go

@@ -52,9 +52,9 @@ func baseConfig(clusterName string, ver version.Version) *CanonicalConfig {
 		esv1.NodeName:    "${" + EnvPodName + "}",
 		esv1.ClusterName: clusterName,
 
-		// derive IP dynamically from the pod IP, injected as env var
-		esv1.NetworkPublishHost: "${" + EnvPodIP + "}",
-		esv1.NetworkHost:        "0.0.0.0",
+		// use the DNS name as the publish host
+		esv1.NetworkPublishHost: fmt.Sprintf("${%s}.${%s}", EnvPodName, HeadlessServiceName),
+		esv1.NetworkHost:        "0",
 
 		// allow ES to be aware of k8s node the pod is running on when allocating shards
 		esv1.ShardAwarenessAttributes: nodeAttrK8sNodeName,

pkg/controller/elasticsearch/settings/merged_config_test.go

@@ -159,7 +159,7 @@ func TestNewMergedESConfig(t *testing.T) {
 				cfgBytes, err := cfg.Render()
 				require.NoError(t, err)
 				// default config is still there
-				require.True(t, bytes.Contains(cfgBytes, []byte("publish_host: ${POD_IP}")))
+				require.True(t, bytes.Contains(cfgBytes, []byte("publish_host: ${POD_NAME}.${HEADLESS_SERVICE_NAME}")))
 				// but has been overridden
 				require.True(t, bytes.Contains(cfgBytes, []byte("seed_providers: something-else")))
 				require.Equal(t, 1, bytes.Count(cfgBytes, []byte("seed_providers:")))

pkg/controller/elasticsearch/sset/pod.go

@@ -20,6 +20,18 @@ import (
 	"github.com/elastic/cloud-on-k8s/pkg/utils/stringsutil"
 )
 
+// HeadlessServiceName returns the name of the headless service for the given StatefulSet.
+func HeadlessServiceName(ssetName string) string {
+	// just use the sset name
+	return ssetName
+}
+
+//PodDNSName returns the DNS resolvable name of the given pod resolvable within the namespace.
+func PodDNSName(pod corev1.Pod) string {
+	ssetName := pod.Labels[label.StatefulSetNameLabelName]
+	return fmt.Sprintf("%s.%s", pod.Name, HeadlessServiceName(ssetName))
+}
+
 // PodName returns the name of the pod with the given ordinal for this StatefulSet.
 func PodName(ssetName string, ordinal int32) string {
 	return fmt.Sprintf("%s-%d", ssetName, ordinal)