From 1875fe89fa125ed9b127439c9dbb264be7783608 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Fri, 21 Aug 2020 14:51:36 +0200 Subject: [PATCH 01/19] Support IPv6 in discovery configuration --- pkg/controller/elasticsearch/settings/masters.go | 5 +++-- .../elasticsearch/settings/masters_test.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pkg/controller/elasticsearch/settings/masters.go b/pkg/controller/elasticsearch/settings/masters.go index c64aab647a..185b6282fe 100644 --- a/pkg/controller/elasticsearch/settings/masters.go +++ b/pkg/controller/elasticsearch/settings/masters.go @@ -6,9 +6,10 @@ package settings import ( "context" - "fmt" + "net" "reflect" "sort" + "strconv" "strings" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" @@ -58,7 +59,7 @@ func UpdateSeedHostsConfigMap( if len(master.Status.PodIP) > 0 { // do not add pod with no IPs seedHosts = append( seedHosts, - fmt.Sprintf("%s:%d", master.Status.PodIP, network.TransportPort), + net.JoinHostPort(master.Status.PodIP, strconv.FormatInt(network.TransportPort, 10)), ) } } diff --git a/pkg/controller/elasticsearch/settings/masters_test.go b/pkg/controller/elasticsearch/settings/masters_test.go index 4c810f5210..704261325c 100644 --- a/pkg/controller/elasticsearch/settings/masters_test.go +++ b/pkg/controller/elasticsearch/settings/masters_test.go @@ -129,6 +129,20 @@ func TestUpdateSeedHostsConfigMap(t *testing.T) { wantErr: false, expectedContent: "10.0.3.3:9300\n10.0.6.5:9300\n10.0.9.2:9300", }, + { + name: "Can handle IPv6 addresses", + args: args{ + pods: []corev1.Pod{ // + newPodWithIP("master2", "fd00:10:244:0:2::3", true), + newPodWithIP("master3", "fd00:10:244:0:2::5", true), + newPodWithIP("master1", "fd00:10:244:0:2::2", true), + }, + c: k8s.WrappedFakeClient(), + es: es, + }, + wantErr: false, + expectedContent: "[fd00:10:244:0:2::2]:9300\n[fd00:10:244:0:2::3]:9300\n[fd00:10:244:0:2::5]:9300", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 8679497643882af1f6dadeb63d1639c10714df00 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Fri, 21 Aug 2020 16:29:34 +0200 Subject: [PATCH 02/19] Allow IPv6 in CSR (no tests) --- .../elasticsearch/certificates/transport/csr.go | 6 +++--- pkg/utils/net/ip.go | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/pkg/controller/elasticsearch/certificates/transport/csr.go b/pkg/controller/elasticsearch/certificates/transport/csr.go index 68ec91d20e..91306efe85 100644 --- a/pkg/controller/elasticsearch/certificates/transport/csr.go +++ b/pkg/controller/elasticsearch/certificates/transport/csr.go @@ -11,12 +11,12 @@ import ( "net" "time" + netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" - netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net" ) // createValidatedCertificateTemplate validates a CSR and creates a certificate template. @@ -90,8 +90,8 @@ func buildGeneralNames( // add the transport service name for remote cluster connections initially connecting through the service // the DNS name has to match the seed hosts configured in the remote cluster settings {DNSName: fmt.Sprintf("%s.%s.svc", esv1.TransportService(cluster.Name), cluster.Namespace)}, - {IPAddress: netutil.MaybeIPTo4(podIP)}, - {IPAddress: net.ParseIP("127.0.0.1").To4()}, + {IPAddress: netutil.IpToRFCForm(podIP)}, + {IPAddress: netutil.IpToRFCForm(netutil.LoopbackFor(podIP))}, } return generalNames, nil diff --git a/pkg/utils/net/ip.go b/pkg/utils/net/ip.go index d88b16948e..809ec046f8 100644 --- a/pkg/utils/net/ip.go +++ b/pkg/utils/net/ip.go @@ -15,3 +15,20 @@ func MaybeIPTo4(ipAddress net.IP) net.IP { } return ipAddress } + +// IpToRFCForm normalizes the IP address given to fit the expected network byte order octet form described in +// https://tools.ietf.org/html/rfc5280#section-4.2.1.6 +func IpToRFCForm(ip net.IP) net.IP { + if ip := ip.To4(); ip != nil { + return ip + } + return ip.To16() +} + +func LoopbackFor(ip net.IP) net.IP { + lb := net.IPv6loopback + if ip.To4() != nil { + lb = net.ParseIP("127.0.0.1") + } + return lb +} From fcdb4b9bd4b82c515b5ddf328d1a7289d3bade5e Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 23 Aug 2020 17:27:44 +0200 Subject: [PATCH 03/19] experimental bracketing --- pkg/controller/elasticsearch/settings/merged_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/elasticsearch/settings/merged_config.go b/pkg/controller/elasticsearch/settings/merged_config.go index bab0c013d6..c76219bbe1 100644 --- a/pkg/controller/elasticsearch/settings/merged_config.go +++ b/pkg/controller/elasticsearch/settings/merged_config.go @@ -53,7 +53,7 @@ func baseConfig(clusterName string, ver version.Version) *CanonicalConfig { esv1.ClusterName: clusterName, // derive IP dynamically from the pod IP, injected as env var - esv1.NetworkPublishHost: "${" + EnvPodIP + "}", + esv1.NetworkPublishHost: "[${" + EnvPodIP + "}]", esv1.NetworkHost: "0.0.0.0", // allow ES to be aware of k8s node the pod is running on when allocating shards From ad05b50c096aaad2974e12ed0f4be4142145e9ed Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 23 Aug 2020 18:11:09 +0200 Subject: [PATCH 04/19] experimental DNS based network.publish_host --- pkg/controller/elasticsearch/nodespec/statefulset.go | 2 ++ pkg/controller/elasticsearch/settings/merged_config.go | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/controller/elasticsearch/nodespec/statefulset.go b/pkg/controller/elasticsearch/nodespec/statefulset.go index f5507bb4be..d67c7b62dd 100644 --- a/pkg/controller/elasticsearch/nodespec/statefulset.go +++ b/pkg/controller/elasticsearch/nodespec/statefulset.go @@ -54,6 +54,8 @@ func HeadlessService(es *esv1.Elasticsearch, ssetName string) corev1.Service { Port: network.HTTPPort, }, }, + // allow nodes to discover themselves via DNS while they are booting up ie. are not ready yet + PublishNotReadyAddresses: true, }, } } diff --git a/pkg/controller/elasticsearch/settings/merged_config.go b/pkg/controller/elasticsearch/settings/merged_config.go index c76219bbe1..0d04d67462 100644 --- a/pkg/controller/elasticsearch/settings/merged_config.go +++ b/pkg/controller/elasticsearch/settings/merged_config.go @@ -52,9 +52,9 @@ func baseConfig(clusterName string, ver version.Version) *CanonicalConfig { esv1.NodeName: "${" + EnvPodName + "}", esv1.ClusterName: clusterName, - // derive IP dynamically from the pod IP, injected as env var - esv1.NetworkPublishHost: "[${" + EnvPodIP + "}]", - esv1.NetworkHost: "0.0.0.0", + // use the DNS name as the publish host + esv1.NetworkPublishHost: fmt.Sprintf("${%s}.${%s}", EnvPodName, HeadlessServiceName), + esv1.NetworkHost: "0", // allow ES to be aware of k8s node the pod is running on when allocating shards esv1.ShardAwarenessAttributes: nodeAttrK8sNodeName, From 7f79e5d399026bc645902aa82928ea03d3147279 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 23 Aug 2020 18:35:36 +0200 Subject: [PATCH 05/19] experimental addition of pods DNS name to certs DNSNames --- pkg/controller/elasticsearch/certificates/transport/csr.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/controller/elasticsearch/certificates/transport/csr.go b/pkg/controller/elasticsearch/certificates/transport/csr.go index 91306efe85..747fb7100e 100644 --- a/pkg/controller/elasticsearch/certificates/transport/csr.go +++ b/pkg/controller/elasticsearch/certificates/transport/csr.go @@ -11,6 +11,8 @@ import ( "net" "time" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec" netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -71,6 +73,9 @@ func buildGeneralNames( return nil, errors.Errorf("pod currently has no valid IP, found: [%s]", pod.Status.PodIP) } + ssetName := pod.Labels[label.StatefulSetNameLabelName] + svcName := nodespec.HeadlessServiceName(ssetName) + commonName := buildCertificateCommonName(pod, cluster.Name, cluster.Namespace) commonNameUTF8OtherName := &certificates.UTF8StringValuedOtherName{ @@ -90,6 +95,8 @@ func buildGeneralNames( // add the transport service name for remote cluster connections initially connecting through the service // the DNS name has to match the seed hosts configured in the remote cluster settings {DNSName: fmt.Sprintf("%s.%s.svc", esv1.TransportService(cluster.Name), cluster.Namespace)}, + // add the resolvable DNS name of the Pod as published by Elasticsearch + {DNSName: fmt.Sprintf("%s.%s", pod.Name, svcName)}, {IPAddress: netutil.IpToRFCForm(podIP)}, {IPAddress: netutil.IpToRFCForm(netutil.LoopbackFor(podIP))}, } From e71bea3009184c1928d64a68ffb13edba2351cec Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 23 Aug 2020 21:41:02 +0200 Subject: [PATCH 06/19] fix unit tests --- .../elasticsearch/certificates/transport/csr_test.go | 1 + .../certificates/transport/transport_fixtures_test.go | 4 ++++ pkg/controller/elasticsearch/nodespec/podspec_test.go | 2 +- pkg/controller/elasticsearch/settings/merged_config_test.go | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/controller/elasticsearch/certificates/transport/csr_test.go b/pkg/controller/elasticsearch/certificates/transport/csr_test.go index 5df0cb9096..894a340816 100644 --- a/pkg/controller/elasticsearch/certificates/transport/csr_test.go +++ b/pkg/controller/elasticsearch/certificates/transport/csr_test.go @@ -93,6 +93,7 @@ func Test_buildGeneralNames(t *testing.T) { {OtherName: *otherName}, {DNSName: expectedCommonName}, {DNSName: expectedTransportSvcName}, + {DNSName: "test-pod-name.test-sset"}, {IPAddress: net.ParseIP(testIP).To4()}, {IPAddress: net.ParseIP("127.0.0.1").To4()}, }, diff --git a/pkg/controller/elasticsearch/certificates/transport/transport_fixtures_test.go b/pkg/controller/elasticsearch/certificates/transport/transport_fixtures_test.go index 78baa293ed..1ea729daac 100644 --- a/pkg/controller/elasticsearch/certificates/transport/transport_fixtures_test.go +++ b/pkg/controller/elasticsearch/certificates/transport/transport_fixtures_test.go @@ -11,6 +11,7 @@ import ( "crypto/x509/pkix" "encoding/pem" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,6 +35,9 @@ var ( testPod = corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pod-name", + Labels: map[string]string{ + label.StatefulSetNameLabelName: "test-sset", + }, }, Status: corev1.PodStatus{ PodIP: testIP, diff --git a/pkg/controller/elasticsearch/nodespec/podspec_test.go b/pkg/controller/elasticsearch/nodespec/podspec_test.go index 2645fc717e..a4f9c0b660 100644 --- a/pkg/controller/elasticsearch/nodespec/podspec_test.go +++ b/pkg/controller/elasticsearch/nodespec/podspec_test.go @@ -137,7 +137,7 @@ func TestBuildPodTemplateSpec(t *testing.T) { Labels: map[string]string{ "common.k8s.elastic.co/type": "elasticsearch", "elasticsearch.k8s.elastic.co/cluster-name": "name", - "elasticsearch.k8s.elastic.co/config-hash": "2311754148", + "elasticsearch.k8s.elastic.co/config-hash": "3785137207", "elasticsearch.k8s.elastic.co/http-scheme": "https", "elasticsearch.k8s.elastic.co/node-data": "false", "elasticsearch.k8s.elastic.co/node-ingest": "true", diff --git a/pkg/controller/elasticsearch/settings/merged_config_test.go b/pkg/controller/elasticsearch/settings/merged_config_test.go index 3d6946ad06..bf534e7b0f 100644 --- a/pkg/controller/elasticsearch/settings/merged_config_test.go +++ b/pkg/controller/elasticsearch/settings/merged_config_test.go @@ -159,7 +159,7 @@ func TestNewMergedESConfig(t *testing.T) { cfgBytes, err := cfg.Render() require.NoError(t, err) // default config is still there - require.True(t, bytes.Contains(cfgBytes, []byte("publish_host: ${POD_IP}"))) + require.True(t, bytes.Contains(cfgBytes, []byte("publish_host: ${POD_NAME}.${HEADLESS_SERVICE_NAME}"))) // but has been overridden require.True(t, bytes.Contains(cfgBytes, []byte("seed_providers: something-else"))) require.Equal(t, 1, bytes.Count(cfgBytes, []byte("seed_providers:"))) From d55c409a23d3abb431e32a9dc86876f8566bc126 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Sun, 23 Aug 2020 21:45:09 +0200 Subject: [PATCH 07/19] remove IPv4 form from validations --- pkg/apis/elasticsearch/v1/validations.go | 2 +- pkg/apis/elasticsearch/v1beta1/validations.go | 2 +- pkg/controller/common/certificates/http_reconcile.go | 2 +- .../elasticsearch/certificates/transport/csr.go | 4 ++-- pkg/utils/net/ip.go | 12 ++---------- 5 files changed, 7 insertions(+), 15 deletions(-) diff --git a/pkg/apis/elasticsearch/v1/validations.go b/pkg/apis/elasticsearch/v1/validations.go index cfce99ddae..a13f5cc589 100644 --- a/pkg/apis/elasticsearch/v1/validations.go +++ b/pkg/apis/elasticsearch/v1/validations.go @@ -117,7 +117,7 @@ func validSanIP(es *Elasticsearch) field.ErrorList { if selfSignedCerts != nil { for _, san := range selfSignedCerts.SubjectAlternativeNames { if san.IP != "" { - ip := netutil.MaybeIPTo4(net.ParseIP(san.IP)) + ip := netutil.IPToRFCForm(net.ParseIP(san.IP)) if ip == nil { errs = append(errs, field.Invalid(field.NewPath("spec").Child("http", "tls", "selfSignedCertificate", "subjectAlternativeNames"), san.IP, invalidSanIPErrMsg)) } diff --git a/pkg/apis/elasticsearch/v1beta1/validations.go b/pkg/apis/elasticsearch/v1beta1/validations.go index 0188b04f56..b8647b301d 100644 --- a/pkg/apis/elasticsearch/v1beta1/validations.go +++ b/pkg/apis/elasticsearch/v1beta1/validations.go @@ -112,7 +112,7 @@ func validSanIP(es *Elasticsearch) field.ErrorList { if selfSignedCerts != nil { for _, san := range selfSignedCerts.SubjectAlternativeNames { if san.IP != "" { - ip := netutil.MaybeIPTo4(net.ParseIP(san.IP)) + ip := netutil.IPToRFCForm(net.ParseIP(san.IP)) if ip == nil { errs = append(errs, field.Invalid(field.NewPath("spec").Child("http", "tls", "selfSignedCertificate", "subjectAlternativeNames"), san.IP, invalidSanIPErrMsg)) } diff --git a/pkg/controller/common/certificates/http_reconcile.go b/pkg/controller/common/certificates/http_reconcile.go index eb9f862e84..47287a8103 100644 --- a/pkg/controller/common/certificates/http_reconcile.go +++ b/pkg/controller/common/certificates/http_reconcile.go @@ -362,7 +362,7 @@ func createValidatedHTTPCertificateTemplate( dnsNames = append(dnsNames, san.DNS) } if san.IP != "" { - ipAddresses = append(ipAddresses, netutil.MaybeIPTo4(net.ParseIP(san.IP))) + ipAddresses = append(ipAddresses, netutil.IPToRFCForm(net.ParseIP(san.IP))) } } } diff --git a/pkg/controller/elasticsearch/certificates/transport/csr.go b/pkg/controller/elasticsearch/certificates/transport/csr.go index 747fb7100e..14f64df122 100644 --- a/pkg/controller/elasticsearch/certificates/transport/csr.go +++ b/pkg/controller/elasticsearch/certificates/transport/csr.go @@ -97,8 +97,8 @@ func buildGeneralNames( {DNSName: fmt.Sprintf("%s.%s.svc", esv1.TransportService(cluster.Name), cluster.Namespace)}, // add the resolvable DNS name of the Pod as published by Elasticsearch {DNSName: fmt.Sprintf("%s.%s", pod.Name, svcName)}, - {IPAddress: netutil.IpToRFCForm(podIP)}, - {IPAddress: netutil.IpToRFCForm(netutil.LoopbackFor(podIP))}, + {IPAddress: netutil.IPToRFCForm(podIP)}, + {IPAddress: netutil.IPToRFCForm(netutil.LoopbackFor(podIP))}, } return generalNames, nil diff --git a/pkg/utils/net/ip.go b/pkg/utils/net/ip.go index 809ec046f8..803d69f8ea 100644 --- a/pkg/utils/net/ip.go +++ b/pkg/utils/net/ip.go @@ -8,17 +8,9 @@ import ( "net" ) -// MaybeIPTo4 attempts to convert the provided net.IP to a 4-byte representation if possible, otherwise does nothing. -func MaybeIPTo4(ipAddress net.IP) net.IP { - if ip := ipAddress.To4(); ip != nil { - return ip - } - return ipAddress -} - -// IpToRFCForm normalizes the IP address given to fit the expected network byte order octet form described in +// IPToRFCForm normalizes the IP address given to fit the expected network byte order octet form described in // https://tools.ietf.org/html/rfc5280#section-4.2.1.6 -func IpToRFCForm(ip net.IP) net.IP { +func IPToRFCForm(ip net.IP) net.IP { if ip := ip.To4(); ip != nil { return ip } From bbb183dfcdce211a43fcbaf835fae17694312ead Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 26 Aug 2020 11:57:00 +0200 Subject: [PATCH 08/19] Autodetect IP family and use for Kibana and Enterprise Search --- cmd/manager/main.go | 2 ++ config/e2e/operator.yaml | 4 ++++ .../charts/eck/templates/statefulset.yaml | 3 +++ pkg/controller/common/operator/parameters.go | 3 +++ pkg/controller/enterprisesearch/config.go | 14 ++++++++------ pkg/controller/enterprisesearch/config_test.go | 4 ++-- .../enterprisesearch_controller.go | 2 +- pkg/controller/kibana/config_settings.go | 10 ++++++---- pkg/controller/kibana/config_settings_test.go | 10 +++++----- pkg/controller/kibana/controller.go | 2 +- pkg/controller/kibana/driver.go | 5 ++++- pkg/controller/kibana/driver_test.go | 6 +++--- pkg/utils/net/ip.go | 18 ++++++++++++++++++ 13 files changed, 60 insertions(+), 23 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 8aa3386e34..bb224ac1e1 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -34,6 +34,7 @@ import ( controllerscheme "github.com/elastic/cloud-on-k8s/pkg/controller/common/scheme" "github.com/elastic/cloud-on-k8s/pkg/controller/common/tracing" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/settings" "github.com/elastic/cloud-on-k8s/pkg/controller/enterprisesearch" "github.com/elastic/cloud-on-k8s/pkg/controller/kibana" "github.com/elastic/cloud-on-k8s/pkg/controller/license" @@ -440,6 +441,7 @@ func startOperator(stopChan <-chan struct{}) error { } params := operator.Parameters{ Dialer: dialer, + IPFamily: net.AutodetectIPFamily(os.Getenv(settings.EnvPodIP)), OperatorNamespace: operatorNamespace, OperatorInfo: operatorInfo, CACertRotation: certificates.RotationParams{ diff --git a/config/e2e/operator.yaml b/config/e2e/operator.yaml index dce1f40699..3987b4f659 100644 --- a/config/e2e/operator.yaml +++ b/config/e2e/operator.yaml @@ -259,6 +259,10 @@ spec: - name: ELASTIC_APM_ENVIRONMENT value: {{ .Pipeline }}-{{ .BuildNumber }}-{{ .Provider }}-{{ .ClusterName }}-{{ .KubernetesVersion }}-{{ .ElasticStackVersion }} {{end}} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP resources: limits: cpu: 1 diff --git a/hack/manifest-gen/assets/charts/eck/templates/statefulset.yaml b/hack/manifest-gen/assets/charts/eck/templates/statefulset.yaml index 812496b78d..2d0a01657a 100644 --- a/hack/manifest-gen/assets/charts/eck/templates/statefulset.yaml +++ b/hack/manifest-gen/assets/charts/eck/templates/statefulset.yaml @@ -35,6 +35,9 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldPath: status.podIP {{- if .Values.config.webhook.enabled }} - name: WEBHOOK_SECRET value: "{{ .Values.config.webhook.certsSecret }}" diff --git a/pkg/controller/common/operator/parameters.go b/pkg/controller/common/operator/parameters.go index c29e4eb9e2..65e908491a 100644 --- a/pkg/controller/common/operator/parameters.go +++ b/pkg/controller/common/operator/parameters.go @@ -6,6 +6,7 @@ package operator import ( "go.elastic.co/apm" + corev1 "k8s.io/api/core/v1" "github.com/elastic/cloud-on-k8s/pkg/about" "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" @@ -20,6 +21,8 @@ type Parameters struct { OperatorInfo about.OperatorInfo // Dialer is used to create the Elasticsearch HTTP client. Dialer net.Dialer + // IPFamily represents the IP family to use when creating configuration and services. + IPFamily corev1.IPFamily // CACertRotation defines the rotation params for CA certificates. CACertRotation certificates.RotationParams // CertRotation defines the rotation params for non-CA certificates. diff --git a/pkg/controller/enterprisesearch/config.go b/pkg/controller/enterprisesearch/config.go index 92c2256d3a..c86d1d268c 100644 --- a/pkg/controller/enterprisesearch/config.go +++ b/pkg/controller/enterprisesearch/config.go @@ -8,6 +8,8 @@ import ( "fmt" "path/filepath" + "github.com/elastic/cloud-on-k8s/pkg/utils/net" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -52,8 +54,8 @@ func ReadinessProbeSecretVolume(ent entv1beta1.EnterpriseSearch) volume.SecretVo // The secret contains 2 entries: // - the Enterprise Search configuration file // - a bash script used as readiness probe -func ReconcileConfig(driver driver.Interface, ent entv1beta1.EnterpriseSearch) (corev1.Secret, error) { - cfg, err := newConfig(driver, ent) +func ReconcileConfig(driver driver.Interface, ent entv1beta1.EnterpriseSearch, ipFamily corev1.IPFamily) (corev1.Secret, error) { + cfg, err := newConfig(driver, ent, ipFamily) if err != nil { return corev1.Secret{}, err } @@ -141,7 +143,7 @@ func readinessProbeScript(ent entv1beta1.EnterpriseSearch, config *settings.Cano // - user-provided plaintext configuration // - user-provided secret configuration // In case of duplicate settings, the last one takes precedence. -func newConfig(driver driver.Interface, ent entv1beta1.EnterpriseSearch) (*settings.CanonicalConfig, error) { +func newConfig(driver driver.Interface, ent entv1beta1.EnterpriseSearch, ipFamily corev1.IPFamily) (*settings.CanonicalConfig, error) { reusedCfg, err := getOrCreateReusableSettings(driver.K8sClient(), ent) if err != nil { return nil, err @@ -163,7 +165,7 @@ func newConfig(driver driver.Interface, ent entv1beta1.EnterpriseSearch) (*setti if err != nil { return nil, err } - cfg := defaultConfig(ent) + cfg := defaultConfig(ent, ipFamily) // merge with user settings last so they take precedence err = cfg.MergeWith(reusedCfg, tlsCfg, associationCfg, userProvidedCfg, userProvidedSecretCfg) @@ -227,10 +229,10 @@ func parseConfigRef(driver driver.Interface, ent entv1beta1.EnterpriseSearch) (* return common.ParseConfigRef(driver, &ent, ent.Spec.ConfigRef, ConfigFilename) } -func defaultConfig(ent entv1beta1.EnterpriseSearch) *settings.CanonicalConfig { +func defaultConfig(ent entv1beta1.EnterpriseSearch, ipFamily corev1.IPFamily) *settings.CanonicalConfig { return settings.MustCanonicalConfig(map[string]interface{}{ "ent_search.external_url": fmt.Sprintf("%s://localhost:%d", ent.Spec.HTTP.Protocol(), HTTPPort), - "ent_search.listen_host": "0.0.0.0", + "ent_search.listen_host": net.InAddrAnyFor(ipFamily).String(), "filebeat_log_directory": LogVolumeMountPath, "log_directory": LogVolumeMountPath, "allow_es_settings_modification": true, diff --git a/pkg/controller/enterprisesearch/config_test.go b/pkg/controller/enterprisesearch/config_test.go index 7686e1c17f..a229a066cd 100644 --- a/pkg/controller/enterprisesearch/config_test.go +++ b/pkg/controller/enterprisesearch/config_test.go @@ -444,7 +444,7 @@ func TestReconcileConfig(t *testing.T) { } // secret metadata should be correct - got, err := ReconcileConfig(driver, tt.ent) + got, err := ReconcileConfig(driver, tt.ent, corev1.IPv4Protocol) require.NoError(t, err) assert.Equal(t, "sample-ent-config", got.Name) assert.Equal(t, "ns", got.Namespace) @@ -566,7 +566,7 @@ func TestReconcileConfig_ReadinessProbe(t *testing.T) { dynamicWatches: watches.NewDynamicWatches(), } - got, err := ReconcileConfig(driver, tt.ent) + got, err := ReconcileConfig(driver, tt.ent, corev1.IPv4Protocol) require.NoError(t, err) require.Contains(t, string(got.Data[ReadinessProbeFilename]), tt.wantCmd) diff --git a/pkg/controller/enterprisesearch/enterprisesearch_controller.go b/pkg/controller/enterprisesearch/enterprisesearch_controller.go index 79d1516acd..1d92b940cd 100644 --- a/pkg/controller/enterprisesearch/enterprisesearch_controller.go +++ b/pkg/controller/enterprisesearch/enterprisesearch_controller.go @@ -234,7 +234,7 @@ func (r *ReconcileEnterpriseSearch) doReconcile(ctx context.Context, ent entv1be return reconcile.Result{}, nil // will eventually retry once updated } - configSecret, err := ReconcileConfig(r, ent) + configSecret, err := ReconcileConfig(r, ent, r.IPFamily) if err != nil { return reconcile.Result{}, err } diff --git a/pkg/controller/kibana/config_settings.go b/pkg/controller/kibana/config_settings.go index 5f14f3f4f9..5bd81fcc29 100644 --- a/pkg/controller/kibana/config_settings.go +++ b/pkg/controller/kibana/config_settings.go @@ -8,6 +8,8 @@ import ( "context" "path" + "github.com/elastic/cloud-on-k8s/pkg/utils/net" + ucfg "github.com/elastic/go-ucfg" "github.com/pkg/errors" "go.elastic.co/apm" @@ -68,7 +70,7 @@ type CanonicalConfig struct { } // NewConfigSettings returns the Kibana configuration settings for the given Kibana resource. -func NewConfigSettings(ctx context.Context, client k8s.Client, kb kbv1.Kibana, v version.Version) (CanonicalConfig, error) { +func NewConfigSettings(ctx context.Context, client k8s.Client, kb kbv1.Kibana, v version.Version, ipFamily corev1.IPFamily) (CanonicalConfig, error) { span, _ := apm.StartSpan(ctx, "new_config_settings", tracing.SpanTypeApp) defer span.End() @@ -93,7 +95,7 @@ func NewConfigSettings(ctx context.Context, client k8s.Client, kb kbv1.Kibana, v return CanonicalConfig{}, err } - cfg := settings.MustCanonicalConfig(baseSettings(&kb)) + cfg := settings.MustCanonicalConfig(baseSettings(&kb, ipFamily)) kibanaTLSCfg := settings.MustCanonicalConfig(kibanaTLSSettings(kb)) versionSpecificCfg := VersionDefaults(&kb, v) @@ -222,10 +224,10 @@ func getOrCreateReusableSettings(c k8s.Client, kb kbv1.Kibana) (*settings.Canoni return settings.MustCanonicalConfig(r), nil } -func baseSettings(kb *kbv1.Kibana) map[string]interface{} { +func baseSettings(kb *kbv1.Kibana, ipFamily corev1.IPFamily) map[string]interface{} { conf := map[string]interface{}{ ServerName: kb.Name, - ServerHost: "0.0.0.0", + ServerHost: net.InAddrAnyFor(ipFamily).String(), XpackMonitoringUIContainerElasticsearchEnabled: true, } diff --git a/pkg/controller/kibana/config_settings_test.go b/pkg/controller/kibana/config_settings_test.go index 33f4e9703f..af032dda60 100644 --- a/pkg/controller/kibana/config_settings_test.go +++ b/pkg/controller/kibana/config_settings_test.go @@ -321,7 +321,7 @@ func TestNewConfigSettings(t *testing.T) { t.Run(tt.name, func(t *testing.T) { kb := tt.args.kb() v := version.From(7, 6, 0) - got, err := NewConfigSettings(context.Background(), tt.args.client, kb, v) + got, err := NewConfigSettings(context.Background(), tt.args.client, kb, v, corev1.IPv4Protocol) if tt.wantErr { require.Error(t, err) } @@ -347,7 +347,7 @@ func TestNewConfigSettingsCreateEncryptionKeys(t *testing.T) { client := k8s.WrapClient(fake.NewFakeClient()) kb := mkKibana() v := version.MustParse(kb.Spec.Version) - got, err := NewConfigSettings(context.Background(), client, kb, v) + got, err := NewConfigSettings(context.Background(), client, kb, v, corev1.IPv4Protocol) require.NoError(t, err) for _, key := range []string{XpackSecurityEncryptionKey, XpackReportingEncryptionKey, XpackEncryptedSavedObjectsEncryptionKey} { val, err := (*ucfg.Config)(got.CanonicalConfig).String(key, -1, settings.Options...) @@ -374,7 +374,7 @@ func TestNewConfigSettingsExistingEncryptionKey(t *testing.T) { } client := k8s.WrapClient(fake.NewFakeClient(existingSecret)) v := version.MustParse(kb.Spec.Version) - got, err := NewConfigSettings(context.Background(), client, kb, v) + got, err := NewConfigSettings(context.Background(), client, kb, v, corev1.IPv4Protocol) require.NoError(t, err) var gotCfg map[string]interface{} require.NoError(t, got.Unpack(&gotCfg)) @@ -403,7 +403,7 @@ func TestNewConfigSettingsExplicitEncryptionKey(t *testing.T) { kb.Spec.Config = &cfg client := k8s.WrapClient(fake.NewFakeClient()) v := version.MustParse(kb.Spec.Version) - got, err := NewConfigSettings(context.Background(), client, kb, v) + got, err := NewConfigSettings(context.Background(), client, kb, v, corev1.IPv4Protocol) require.NoError(t, err) val, err := (*ucfg.Config)(got.CanonicalConfig).String(XpackSecurityEncryptionKey, -1, settings.Options...) require.NoError(t, err) @@ -416,7 +416,7 @@ func TestNewConfigSettingsPre760(t *testing.T) { kb.Spec.Version = "7.5.0" client := k8s.WrapClient(fake.NewFakeClient()) v := version.MustParse(kb.Spec.Version) - got, err := NewConfigSettings(context.Background(), client, kb, v) + got, err := NewConfigSettings(context.Background(), client, kb, v, corev1.IPv4Protocol) require.NoError(t, err) assert.Equal(t, 0, len(got.CanonicalConfig.HasKeys([]string{XpackEncryptedSavedObjects}))) } diff --git a/pkg/controller/kibana/controller.go b/pkg/controller/kibana/controller.go index 455706db4b..b0fe606711 100644 --- a/pkg/controller/kibana/controller.go +++ b/pkg/controller/kibana/controller.go @@ -191,7 +191,7 @@ func (r *ReconcileKibana) doReconcile(ctx context.Context, request reconcile.Req return reconcile.Result{}, err } - driver, err := newDriver(r, r.dynamicWatches, r.recorder, kb) + driver, err := newDriver(r, r.dynamicWatches, r.recorder, kb, r.params.IPFamily) if err != nil { return reconcile.Result{}, tracing.CaptureError(ctx, err) } diff --git a/pkg/controller/kibana/driver.go b/pkg/controller/kibana/driver.go index 4095d7ab6a..c4df33c39e 100644 --- a/pkg/controller/kibana/driver.go +++ b/pkg/controller/kibana/driver.go @@ -44,6 +44,7 @@ type driver struct { dynamicWatches watches.DynamicWatches recorder record.EventRecorder version version.Version + ipFamily corev1.IPFamily } func (d *driver) DynamicWatches() watches.DynamicWatches { @@ -65,6 +66,7 @@ func newDriver( watches watches.DynamicWatches, recorder record.EventRecorder, kb *kbv1.Kibana, + ipFamily corev1.IPFamily, ) (*driver, error) { ver, err := version.Parse(kb.Spec.Version) if err != nil { @@ -83,6 +85,7 @@ func newDriver( dynamicWatches: watches, recorder: recorder, version: *ver, + ipFamily: ipFamily, }, nil } @@ -124,7 +127,7 @@ func (d *driver) Reconcile( return results // will eventually retry } - kbSettings, err := NewConfigSettings(ctx, d.client, *kb, d.version) + kbSettings, err := NewConfigSettings(ctx, d.client, *kb, d.version, d.ipFamily) if err != nil { return results.WithError(err) } diff --git a/pkg/controller/kibana/driver_test.go b/pkg/controller/kibana/driver_test.go index 0f46ba2cee..84bb51705b 100644 --- a/pkg/controller/kibana/driver_test.go +++ b/pkg/controller/kibana/driver_test.go @@ -188,7 +188,7 @@ func Test_getStrategyType(t *testing.T) { client = &failingClient{} } - d, err := newDriver(client, w, record.NewFakeRecorder(100), kb) + d, err := newDriver(client, w, record.NewFakeRecorder(100), kb, corev1.IPv4Protocol) assert.NoError(t, err) strategy, err := d.getStrategyType(kb) @@ -368,7 +368,7 @@ func TestDriverDeploymentParams(t *testing.T) { client := k8s.WrappedFakeClient(initialObjects...) w := watches.NewDynamicWatches() - d, err := newDriver(client, w, record.NewFakeRecorder(100), kb) + d, err := newDriver(client, w, record.NewFakeRecorder(100), kb, corev1.IPv4Protocol) require.NoError(t, err) got, err := d.deploymentParams(kb) @@ -414,7 +414,7 @@ func TestMinSupportedVersion(t *testing.T) { client := k8s.WrappedFakeClient(defaultInitialObjects()...) w := watches.NewDynamicWatches() - _, err := newDriver(client, w, record.NewFakeRecorder(100), kb) + _, err := newDriver(client, w, record.NewFakeRecorder(100), kb, corev1.IPv4Protocol) if tc.wantErr { require.Error(t, err) } else { diff --git a/pkg/utils/net/ip.go b/pkg/utils/net/ip.go index 803d69f8ea..deb163d2a7 100644 --- a/pkg/utils/net/ip.go +++ b/pkg/utils/net/ip.go @@ -6,6 +6,8 @@ package net import ( "net" + + corev1 "k8s.io/api/core/v1" ) // IPToRFCForm normalizes the IP address given to fit the expected network byte order octet form described in @@ -24,3 +26,19 @@ func LoopbackFor(ip net.IP) net.IP { } return lb } + +// InAddrAnyFor returns the special IP address to bind to any IP address (0.0.0.0 or ::) depending on IP family. +func InAddrAnyFor(ipFamily corev1.IPFamily) net.IP { + if ipFamily == corev1.IPv4Protocol { + return net.IPv4zero + } + return net.IPv6zero +} + +//AutodetectIPFamily tries to detect the IP family (IPv4 or IPv6) based on the given IP string. +func AutodetectIPFamily(ipStr string) corev1.IPFamily { + if ip := net.ParseIP(ipStr); len(ip) == net.IPv6len { + return corev1.IPv6Protocol + } + return corev1.IPv4Protocol +} From b1612804ccde0304d8cc82d357ee62df73d1e4e6 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 26 Aug 2020 14:22:44 +0200 Subject: [PATCH 09/19] update readiness probes to use correct loopback address --- cmd/manager/main.go | 2 +- .../certificates/transport/csr.go | 2 +- .../elasticsearch/nodespec/readiness_probe.go | 9 ++++++++- pkg/controller/enterprisesearch/config.go | 6 +++--- pkg/utils/net/ip.go | 20 ++++++++++++------- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index bb224ac1e1..ac1a968a8f 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -441,7 +441,7 @@ func startOperator(stopChan <-chan struct{}) error { } params := operator.Parameters{ Dialer: dialer, - IPFamily: net.AutodetectIPFamily(os.Getenv(settings.EnvPodIP)), + IPFamily: net.ToIPFamily(os.Getenv(settings.EnvPodIP)), OperatorNamespace: operatorNamespace, OperatorInfo: operatorInfo, CACertRotation: certificates.RotationParams{ diff --git a/pkg/controller/elasticsearch/certificates/transport/csr.go b/pkg/controller/elasticsearch/certificates/transport/csr.go index 14f64df122..bc42cfbd79 100644 --- a/pkg/controller/elasticsearch/certificates/transport/csr.go +++ b/pkg/controller/elasticsearch/certificates/transport/csr.go @@ -98,7 +98,7 @@ func buildGeneralNames( // add the resolvable DNS name of the Pod as published by Elasticsearch {DNSName: fmt.Sprintf("%s.%s", pod.Name, svcName)}, {IPAddress: netutil.IPToRFCForm(podIP)}, - {IPAddress: netutil.IPToRFCForm(netutil.LoopbackFor(podIP))}, + {IPAddress: netutil.IPToRFCForm(netutil.LoopbackFor(netutil.ToIPFamily(podIP.String())))}, } return generalNames, nil diff --git a/pkg/controller/elasticsearch/nodespec/readiness_probe.go b/pkg/controller/elasticsearch/nodespec/readiness_probe.go index d736c679cb..85ca8e3568 100644 --- a/pkg/controller/elasticsearch/nodespec/readiness_probe.go +++ b/pkg/controller/elasticsearch/nodespec/readiness_probe.go @@ -64,8 +64,15 @@ else BASIC_AUTH='' fi +# Check if we are using IPv6 +if [[ $POD_IP =~ .*:.* ]]; then + LOOPBACK=[::1] +else + LOOPBACK=127.0.01 +fi + # request Elasticsearch on / -ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://127.0.0.1:9200/" +ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://${LOOPBACK}:9200/" status=$(curl -o /dev/null -w "%{http_code}" --max-time ${READINESS_PROBE_TIMEOUT} -XGET -s -k ${BASIC_AUTH} $ENDPOINT) curl_rc=$? diff --git a/pkg/controller/enterprisesearch/config.go b/pkg/controller/enterprisesearch/config.go index c86d1d268c..afd90cdea1 100644 --- a/pkg/controller/enterprisesearch/config.go +++ b/pkg/controller/enterprisesearch/config.go @@ -65,7 +65,7 @@ func ReconcileConfig(driver driver.Interface, ent entv1beta1.EnterpriseSearch, i return corev1.Secret{}, err } - readinessProbeBytes, err := readinessProbeScript(ent, cfg) + readinessProbeBytes, err := readinessProbeScript(ent, cfg, ipFamily) if err != nil { return corev1.Secret{}, err } @@ -95,8 +95,8 @@ type partialConfigWithESAuth struct { } // readinessProbeScript returns a bash script that requests the health endpoint. -func readinessProbeScript(ent entv1beta1.EnterpriseSearch, config *settings.CanonicalConfig) ([]byte, error) { - url := fmt.Sprintf("%s://127.0.0.1:%d/api/ent/v1/internal/health", ent.Spec.HTTP.Protocol(), HTTPPort) +func readinessProbeScript(ent entv1beta1.EnterpriseSearch, config *settings.CanonicalConfig, ipFamily corev1.IPFamily) ([]byte, error) { + url := fmt.Sprintf("%s://%s/api/ent/v1/internal/health", ent.Spec.HTTP.Protocol(), net.LoopbackHostPort(ipFamily, HTTPPort)) // retrieve Elasticsearch user credentials from the aggregated config since it could be user-provided var esAuth partialConfigWithESAuth diff --git a/pkg/utils/net/ip.go b/pkg/utils/net/ip.go index deb163d2a7..ac5cf129ae 100644 --- a/pkg/utils/net/ip.go +++ b/pkg/utils/net/ip.go @@ -6,6 +6,7 @@ package net import ( "net" + "strconv" corev1 "k8s.io/api/core/v1" ) @@ -19,12 +20,17 @@ func IPToRFCForm(ip net.IP) net.IP { return ip.To16() } -func LoopbackFor(ip net.IP) net.IP { - lb := net.IPv6loopback - if ip.To4() != nil { - lb = net.ParseIP("127.0.0.1") +// LoopbackFor returns the loopback address for the given IP family. +func LoopbackFor(ipFamily corev1.IPFamily) net.IP { + if ipFamily == corev1.IPv4Protocol { + return net.ParseIP("127.0.0.1") } - return lb + return net.IPv6loopback +} + +// LoopbackHostPort formats a loopback address and port correctly for the given IP family. +func LoopbackHostPort(ipFamily corev1.IPFamily, port int) string { + return net.JoinHostPort(LoopbackFor(ipFamily).String(), strconv.Itoa(port)) } // InAddrAnyFor returns the special IP address to bind to any IP address (0.0.0.0 or ::) depending on IP family. @@ -35,8 +41,8 @@ func InAddrAnyFor(ipFamily corev1.IPFamily) net.IP { return net.IPv6zero } -//AutodetectIPFamily tries to detect the IP family (IPv4 or IPv6) based on the given IP string. -func AutodetectIPFamily(ipStr string) corev1.IPFamily { +// ToIPFamily tries to detect the IP family (IPv4 or IPv6) based on the given IP string. +func ToIPFamily(ipStr string) corev1.IPFamily { if ip := net.ParseIP(ipStr); len(ip) == net.IPv6len { return corev1.IPv6Protocol } From ecc9ed6e135af86b0bc4b0dd1a6615919327ed9c Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 26 Aug 2020 14:49:55 +0200 Subject: [PATCH 10/19] fix missing fieldRef in manifest-generator --- hack/manifest-gen/assets/charts/eck/templates/statefulset.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/manifest-gen/assets/charts/eck/templates/statefulset.yaml b/hack/manifest-gen/assets/charts/eck/templates/statefulset.yaml index 2d0a01657a..2a9d99ac42 100644 --- a/hack/manifest-gen/assets/charts/eck/templates/statefulset.yaml +++ b/hack/manifest-gen/assets/charts/eck/templates/statefulset.yaml @@ -37,7 +37,8 @@ spec: fieldPath: metadata.namespace - name: POD_IP valueFrom: - fieldPath: status.podIP + fieldRef: + fieldPath: status.podIP {{- if .Values.config.webhook.enabled }} - name: WEBHOOK_SECRET value: "{{ .Values.config.webhook.certsSecret }}" From cb550f5b6a4eaa90138c4197c0b1e63649903745 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 26 Aug 2020 15:06:29 +0200 Subject: [PATCH 11/19] turn globbing off in readiness script to allow for unescaped brackets --- pkg/controller/elasticsearch/nodespec/readiness_probe.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/controller/elasticsearch/nodespec/readiness_probe.go b/pkg/controller/elasticsearch/nodespec/readiness_probe.go index 85ca8e3568..a379e35616 100644 --- a/pkg/controller/elasticsearch/nodespec/readiness_probe.go +++ b/pkg/controller/elasticsearch/nodespec/readiness_probe.go @@ -66,14 +66,15 @@ fi # Check if we are using IPv6 if [[ $POD_IP =~ .*:.* ]]; then - LOOPBACK=[::1] + LOOPBACK=[::1] else LOOPBACK=127.0.01 fi # request Elasticsearch on / +# we are turning globbing off to allow for unescaped [] in case of IPv6 ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://${LOOPBACK}:9200/" -status=$(curl -o /dev/null -w "%{http_code}" --max-time ${READINESS_PROBE_TIMEOUT} -XGET -s -k ${BASIC_AUTH} $ENDPOINT) +status=$(curl -o /dev/null -w "%{http_code}" --max-time ${READINESS_PROBE_TIMEOUT} -XGET -g -s -k ${BASIC_AUTH} $ENDPOINT) curl_rc=$? if [[ ${curl_rc} -ne 0 ]]; then From 8a9b1e85e6fb64f7d467a4374f8eaad918bc3c84 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 26 Aug 2020 15:16:28 +0200 Subject: [PATCH 12/19] turn globbing off in readiness script to allow for unescaped brackets --- pkg/controller/enterprisesearch/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/enterprisesearch/config.go b/pkg/controller/enterprisesearch/config.go index afd90cdea1..1dc52635e9 100644 --- a/pkg/controller/enterprisesearch/config.go +++ b/pkg/controller/enterprisesearch/config.go @@ -120,8 +120,8 @@ func readinessProbeScript(ent entv1beta1.EnterpriseSearch, config *settings.Cano # request timeout can be overridden from an environment variable READINESS_PROBE_TIMEOUT=${READINESS_PROBE_TIMEOUT:=` + fmt.Sprintf("%d", ReadinessProbeTimeoutSec) + `} - # request the health endpoint and expect http status code 200 - status=$(curl -o /dev/null -w "%{http_code}" ` + url + ` ` + basicAuthArgs + ` -k -s --max-time ${READINESS_PROBE_TIMEOUT}) + # request the health endpoint and expect http status code 200. Turning globbing off for unescaped IPv6 addresses + status=$(curl -g -o /dev/null -w "%{http_code}" ` + url + ` ` + basicAuthArgs + ` -k -s --max-time ${READINESS_PROBE_TIMEOUT}) curl_rc=$? if [[ ${curl_rc} -ne 0 ]]; then From 6861f4274bd97d1aa221f5123bf655ebfde73329 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 26 Aug 2020 21:18:30 +0200 Subject: [PATCH 13/19] test fixes and test additions --- .../elasticsearch/settings/masters.go | 2 +- .../enterprisesearch/config_test.go | 29 ++- pkg/utils/net/ip.go | 12 +- pkg/utils/net/ip_test.go | 209 ++++++++++++++++++ 4 files changed, 239 insertions(+), 13 deletions(-) create mode 100644 pkg/utils/net/ip_test.go diff --git a/pkg/controller/elasticsearch/settings/masters.go b/pkg/controller/elasticsearch/settings/masters.go index 185b6282fe..b358ddb536 100644 --- a/pkg/controller/elasticsearch/settings/masters.go +++ b/pkg/controller/elasticsearch/settings/masters.go @@ -59,7 +59,7 @@ func UpdateSeedHostsConfigMap( if len(master.Status.PodIP) > 0 { // do not add pod with no IPs seedHosts = append( seedHosts, - net.JoinHostPort(master.Status.PodIP, strconv.FormatInt(network.TransportPort, 10)), + net.JoinHostPort(master.Status.PodIP, strconv.Itoa(network.TransportPort)), ) } } diff --git a/pkg/controller/enterprisesearch/config_test.go b/pkg/controller/enterprisesearch/config_test.go index a229a066cd..4e32849931 100644 --- a/pkg/controller/enterprisesearch/config_test.go +++ b/pkg/controller/enterprisesearch/config_test.go @@ -475,10 +475,11 @@ func TestReconcileConfig_ReadinessProbe(t *testing.T) { name string runtimeObjs []runtime.Object ent entv1beta1.EnterpriseSearch + ipFamily corev1.IPFamily wantCmd string }{ { - name: "create default readiness probe script (no es association)", + name: "create default readiness probe script (no es association, IPv4)", runtimeObjs: nil, ent: entv1beta1.EnterpriseSearch{ ObjectMeta: metav1.ObjectMeta{ @@ -486,7 +487,20 @@ func TestReconcileConfig_ReadinessProbe(t *testing.T) { Name: "sample", }, }, - wantCmd: `curl -o /dev/null -w "%{http_code}" https://127.0.0.1:3002/api/ent/v1/internal/health -k -s --max-time ${READINESS_PROBE_TIMEOUT}`, // no ES basic auth + ipFamily: corev1.IPv4Protocol, + wantCmd: `curl -g -o /dev/null -w "%{http_code}" https://127.0.0.1:3002/api/ent/v1/internal/health -k -s --max-time ${READINESS_PROBE_TIMEOUT}`, // no ES basic auth + }, + { + name: "create default readiness probe script (no es association, IPv6)", + runtimeObjs: nil, + ent: entv1beta1.EnterpriseSearch{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", + Name: "sample", + }, + }, + ipFamily: corev1.IPv6Protocol, + wantCmd: `curl -g -o /dev/null -w "%{http_code}" https://[::1]:3002/api/ent/v1/internal/health -k -s --max-time ${READINESS_PROBE_TIMEOUT}`, // no ES basic auth }, { name: "update existing readiness probe script if different", @@ -507,7 +521,8 @@ func TestReconcileConfig_ReadinessProbe(t *testing.T) { Name: "sample", }, }, - wantCmd: `curl -o /dev/null -w "%{http_code}" https://127.0.0.1:3002/api/ent/v1/internal/health -k -s --max-time ${READINESS_PROBE_TIMEOUT}`, // no ES basic auth + ipFamily: corev1.IPv4Protocol, + wantCmd: `curl -g -o /dev/null -w "%{http_code}" https://127.0.0.1:3002/api/ent/v1/internal/health -k -s --max-time ${READINESS_PROBE_TIMEOUT}`, // no ES basic auth }, { name: "with ES association: use ES user credentials", @@ -529,7 +544,8 @@ func TestReconcileConfig_ReadinessProbe(t *testing.T) { }, }, }, - wantCmd: `curl -o /dev/null -w "%{http_code}" https://127.0.0.1:3002/api/ent/v1/internal/health -u ns-sample-ent-user:password -k -s --max-time ${READINESS_PROBE_TIMEOUT}`, + ipFamily: corev1.IPv4Protocol, + wantCmd: `curl -g -o /dev/null -w "%{http_code}" https://127.0.0.1:3002/api/ent/v1/internal/health -u ns-sample-ent-user:password -k -s --max-time ${READINESS_PROBE_TIMEOUT}`, }, { name: "with es credentials in a user-provided config secret", @@ -555,7 +571,8 @@ func TestReconcileConfig_ReadinessProbe(t *testing.T) { }, }, }, - wantCmd: `curl -o /dev/null -w "%{http_code}" https://127.0.0.1:3002/api/ent/v1/internal/health -u myusername:mypassword -k -s --max-time ${READINESS_PROBE_TIMEOUT}`, + ipFamily: corev1.IPv4Protocol, + wantCmd: `curl -g -o /dev/null -w "%{http_code}" https://127.0.0.1:3002/api/ent/v1/internal/health -u myusername:mypassword -k -s --max-time ${READINESS_PROBE_TIMEOUT}`, }, } for _, tt := range tests { @@ -566,7 +583,7 @@ func TestReconcileConfig_ReadinessProbe(t *testing.T) { dynamicWatches: watches.NewDynamicWatches(), } - got, err := ReconcileConfig(driver, tt.ent, corev1.IPv4Protocol) + got, err := ReconcileConfig(driver, tt.ent, tt.ipFamily) require.NoError(t, err) require.Contains(t, string(got.Data[ReadinessProbeFilename]), tt.wantCmd) diff --git a/pkg/utils/net/ip.go b/pkg/utils/net/ip.go index ac5cf129ae..8840a18d66 100644 --- a/pkg/utils/net/ip.go +++ b/pkg/utils/net/ip.go @@ -22,10 +22,10 @@ func IPToRFCForm(ip net.IP) net.IP { // LoopbackFor returns the loopback address for the given IP family. func LoopbackFor(ipFamily corev1.IPFamily) net.IP { - if ipFamily == corev1.IPv4Protocol { - return net.ParseIP("127.0.0.1") + if ipFamily == corev1.IPv6Protocol { + return net.IPv6loopback } - return net.IPv6loopback + return net.ParseIP("127.0.0.1") } // LoopbackHostPort formats a loopback address and port correctly for the given IP family. @@ -43,8 +43,8 @@ func InAddrAnyFor(ipFamily corev1.IPFamily) net.IP { // ToIPFamily tries to detect the IP family (IPv4 or IPv6) based on the given IP string. func ToIPFamily(ipStr string) corev1.IPFamily { - if ip := net.ParseIP(ipStr); len(ip) == net.IPv6len { - return corev1.IPv6Protocol + if ip := net.ParseIP(ipStr); ip.To4() != nil { + return corev1.IPv4Protocol } - return corev1.IPv4Protocol + return corev1.IPv6Protocol } diff --git a/pkg/utils/net/ip_test.go b/pkg/utils/net/ip_test.go new file mode 100644 index 0000000000..7c4820e9a7 --- /dev/null +++ b/pkg/utils/net/ip_test.go @@ -0,0 +1,209 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package net + +import ( + "net" + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" +) + +func TestIPToRFCForm(t *testing.T) { + type args struct { + ip net.IP + } + tests := []struct { + name string + args args + want net.IP + }{ + { + name: "IPv4", + args: args{ + ip: net.IPv4(127, 0, 0, 1), + }, + want: net.IPv4(127, 0, 0, 1).To4(), + }, + { + name: "IPv4 mapped IPv6", + args: args{ + ip: net.ParseIP("::FFFF:129.144.52.38"), + }, + want: net.IPv4(129, 144, 52, 38).To4(), + }, + { + name: "IPv6", + args: args{ + ip: net.ParseIP("2001:4860:0:2001::68"), + }, + want: net.IP{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01, 0, 0, 0, 0, 0, 0, 0x00, 0x68}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IPToRFCForm(tt.args.ip); !reflect.DeepEqual(got, tt.want) { + t.Errorf("IPToRFCForm() = %v, want %v", len(got), len(tt.want)) + } + }) + } +} + +func TestInAddrAnyFor(t *testing.T) { + type args struct { + ipFamily corev1.IPFamily + } + tests := []struct { + name string + args args + want net.IP + }{ + { + name: "IPv6", + args: args{ + ipFamily: corev1.IPv6Protocol, + }, + want: net.IPv6zero, + }, + { + name: "IPv4", + args: args{ + ipFamily: corev1.IPv4Protocol, + }, + want: net.IPv4zero, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := InAddrAnyFor(tt.args.ipFamily); !reflect.DeepEqual(got, tt.want) { + t.Errorf("InAddrAnyFor() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLoopbackFor(t *testing.T) { + type args struct { + ipFamily corev1.IPFamily + } + tests := []struct { + name string + args args + want net.IP + }{ + { + name: "IPv4", + args: args{ + ipFamily: corev1.IPv4Protocol, + }, + want: net.ParseIP("127.0.0.1"), + }, + { + name: "IPv6", + args: args{ + ipFamily: corev1.IPv6Protocol, + }, + want: net.IPv6loopback, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := LoopbackFor(tt.args.ipFamily); !reflect.DeepEqual(got, tt.want) { + t.Errorf("LoopbackFor() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLoopbackHostPort(t *testing.T) { + type args struct { + ipFamily corev1.IPFamily + port int + } + tests := []struct { + name string + args args + want string + }{ + { + name: "IPv4", + args: args{ + ipFamily: corev1.IPv4Protocol, + port: 80, + }, + want: "127.0.0.1:80", + }, + { + name: "IPv6", + args: args{ + ipFamily: corev1.IPv6Protocol, + port: 80, + }, + want: "[::1]:80", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := LoopbackHostPort(tt.args.ipFamily, tt.args.port); got != tt.want { + t.Errorf("LoopbackHostPort() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestToIPFamily(t *testing.T) { + type args struct { + ipStr string + } + tests := []struct { + name string + args args + want corev1.IPFamily + }{ + { + name: "IPv4", + args: args{ + ipStr: "127.0.0.1", + }, + want: corev1.IPv4Protocol, + }, + { + name: "IPv6 all zeros", + args: args{ + ipStr: "::", + }, + want: corev1.IPv6Protocol, + }, + { + name: "IPv4 all zeros", + args: args{ + ipStr: "0.0.0.0", + }, + want: corev1.IPv4Protocol, + }, + { + name: "IPv4-mapped IPv6", + args: args{ + ipStr: "::FFFF:129.144.52.38", + }, + want: corev1.IPv4Protocol, + }, + { + name: "IPv6", + args: args{ + ipStr: "2001:0db8:0000:0000:0000:ff00:0042:8329", + }, + want: corev1.IPv6Protocol, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToIPFamily(tt.args.ipStr); got != tt.want { + t.Errorf("ToIPFamily() = %v, want %v", got, tt.want) + } + }) + } +} From 283aa79c190329c6667818b447ee9e3fb45b5ea8 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Thu, 27 Aug 2020 12:05:35 +0200 Subject: [PATCH 14/19] test IPv6 path in Kibana config --- pkg/controller/kibana/config_settings_test.go | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/pkg/controller/kibana/config_settings_test.go b/pkg/controller/kibana/config_settings_test.go index af032dda60..5690429c99 100644 --- a/pkg/controller/kibana/config_settings_test.go +++ b/pkg/controller/kibana/config_settings_test.go @@ -164,8 +164,9 @@ func TestNewConfigSettings(t *testing.T) { }, } type args struct { - client k8s.Client - kb func() kbv1.Kibana + client k8s.Client + kb func() kbv1.Kibana + ipFamily corev1.IPFamily } tests := []struct { name string @@ -174,13 +175,31 @@ func TestNewConfigSettings(t *testing.T) { wantErr bool }{ { - name: "default config", + name: "default config IPv4", args: args{ - client: k8s.WrappedFakeClient(existingSecret), - kb: mkKibana, + client: k8s.WrappedFakeClient(existingSecret), + kb: mkKibana, + ipFamily: corev1.IPv4Protocol, }, want: defaultConfig, }, + { + name: "default config IPv6", + args: args{ + client: k8s.WrappedFakeClient(existingSecret), + kb: mkKibana, + ipFamily: corev1.IPv6Protocol, + }, + want: func() []byte { + cfg, err := settings.ParseConfig(defaultConfig) + require.NoError(t, err, "cfg", cfg) + err = (*ucfg.Config)(cfg).SetString("server.host", -1, "::", settings.Options...) + require.NoError(t, err) + bytes, err := cfg.Render() + require.NoError(t, err) + return bytes + }(), + }, { name: "without TLS", args: args{ @@ -196,6 +215,7 @@ func TestNewConfigSettings(t *testing.T) { } return kb }, + ipFamily: corev1.IPv4Protocol, }, want: func() []byte { cfg, err := settings.ParseConfig(defaultConfig) @@ -244,6 +264,7 @@ func TestNewConfigSettings(t *testing.T) { }, }, )), + ipFamily: corev1.IPv4Protocol, }, want: func() []byte { cfg, err := settings.ParseConfig(defaultConfig) @@ -270,6 +291,7 @@ func TestNewConfigSettings(t *testing.T) { } return kb }, + ipFamily: corev1.IPv4Protocol, }, want: append(defaultConfig, []byte(`foo: bar`)...), }, @@ -294,6 +316,7 @@ func TestNewConfigSettings(t *testing.T) { } return kb }, + ipFamily: corev1.IPv4Protocol, }, want: append(defaultConfig, []byte(`logging.verbose: false`)...), }, @@ -313,6 +336,7 @@ func TestNewConfigSettings(t *testing.T) { kb := mkKibana() return kb }, + ipFamily: corev1.IPv4Protocol, }, want: defaultConfig, }, @@ -321,7 +345,7 @@ func TestNewConfigSettings(t *testing.T) { t.Run(tt.name, func(t *testing.T) { kb := tt.args.kb() v := version.From(7, 6, 0) - got, err := NewConfigSettings(context.Background(), tt.args.client, kb, v, corev1.IPv4Protocol) + got, err := NewConfigSettings(context.Background(), tt.args.client, kb, v, tt.args.ipFamily) if tt.wantErr { require.Error(t, err) } From 59c217f2869b6d97e2d850739154668653222d0e Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Thu, 27 Aug 2020 14:28:22 +0200 Subject: [PATCH 15/19] Move discovery to DNS names --- .../certificates/transport/csr.go | 9 +++----- .../elasticsearch/driver/upscale_test.go | 4 ++-- .../elasticsearch/nodespec/podspec.go | 4 +++- .../elasticsearch/nodespec/podspec_test.go | 4 +++- .../elasticsearch/nodespec/statefulset.go | 10 ++------- .../elasticsearch/settings/masters.go | 6 ++---- .../elasticsearch/settings/masters_test.go | 21 ++++--------------- pkg/controller/elasticsearch/sset/pod.go | 12 +++++++++++ 8 files changed, 31 insertions(+), 39 deletions(-) diff --git a/pkg/controller/elasticsearch/certificates/transport/csr.go b/pkg/controller/elasticsearch/certificates/transport/csr.go index bc42cfbd79..9c5d6ce1ea 100644 --- a/pkg/controller/elasticsearch/certificates/transport/csr.go +++ b/pkg/controller/elasticsearch/certificates/transport/csr.go @@ -11,8 +11,8 @@ import ( "net" "time" - "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label" - "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/sset" + netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -73,9 +73,6 @@ func buildGeneralNames( return nil, errors.Errorf("pod currently has no valid IP, found: [%s]", pod.Status.PodIP) } - ssetName := pod.Labels[label.StatefulSetNameLabelName] - svcName := nodespec.HeadlessServiceName(ssetName) - commonName := buildCertificateCommonName(pod, cluster.Name, cluster.Namespace) commonNameUTF8OtherName := &certificates.UTF8StringValuedOtherName{ @@ -96,7 +93,7 @@ func buildGeneralNames( // the DNS name has to match the seed hosts configured in the remote cluster settings {DNSName: fmt.Sprintf("%s.%s.svc", esv1.TransportService(cluster.Name), cluster.Namespace)}, // add the resolvable DNS name of the Pod as published by Elasticsearch - {DNSName: fmt.Sprintf("%s.%s", pod.Name, svcName)}, + {DNSName: sset.PodDNSName(pod)}, {IPAddress: netutil.IPToRFCForm(podIP)}, {IPAddress: netutil.IPToRFCForm(netutil.LoopbackFor(netutil.ToIPFamily(podIP.String())))}, } diff --git a/pkg/controller/elasticsearch/driver/upscale_test.go b/pkg/controller/elasticsearch/driver/upscale_test.go index 773519e408..9d6b742d09 100644 --- a/pkg/controller/elasticsearch/driver/upscale_test.go +++ b/pkg/controller/elasticsearch/driver/upscale_test.go @@ -124,8 +124,8 @@ func TestHandleUpscaleAndSpecChanges(t *testing.T) { require.Equal(t, pointer.Int32(4), sset2.Spec.Replicas) comparison.RequireEqual(t, &updatedStatefulSets[1], &sset2) // headless services should be created for both - require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: nodespec.HeadlessServiceName("sset1")}, &corev1.Service{})) - require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: nodespec.HeadlessServiceName("sset2")}, &corev1.Service{})) + require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: sset.HeadlessServiceName("sset1")}, &corev1.Service{})) + require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: sset.HeadlessServiceName("sset2")}, &corev1.Service{})) // config should be created for both require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: esv1.ConfigSecret("sset1")}, &corev1.Secret{})) require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: esv1.ConfigSecret("sset2")}, &corev1.Secret{})) diff --git a/pkg/controller/elasticsearch/nodespec/podspec.go b/pkg/controller/elasticsearch/nodespec/podspec.go index 6352a26144..fe1f968a80 100644 --- a/pkg/controller/elasticsearch/nodespec/podspec.go +++ b/pkg/controller/elasticsearch/nodespec/podspec.go @@ -8,6 +8,8 @@ import ( "crypto/sha256" "fmt" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/sset" + corev1 "k8s.io/api/core/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" @@ -58,7 +60,7 @@ func BuildPodTemplateSpec( WithPorts(defaultContainerPorts). WithReadinessProbe(*NewReadinessProbe()). WithAffinity(DefaultAffinity(es.Name)). - WithEnv(DefaultEnvVars(es.Spec.HTTP, HeadlessServiceName(esv1.StatefulSet(es.Name, nodeSet.Name)))...). + WithEnv(DefaultEnvVars(es.Spec.HTTP, sset.HeadlessServiceName(esv1.StatefulSet(es.Name, nodeSet.Name)))...). WithVolumes(volumes...). WithVolumeMounts(volumeMounts...). WithInitContainers(initContainers...). diff --git a/pkg/controller/elasticsearch/nodespec/podspec_test.go b/pkg/controller/elasticsearch/nodespec/podspec_test.go index a4f9c0b660..2d52c2995d 100644 --- a/pkg/controller/elasticsearch/nodespec/podspec_test.go +++ b/pkg/controller/elasticsearch/nodespec/podspec_test.go @@ -8,6 +8,8 @@ import ( "sort" "testing" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/sset" + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/common/defaults" @@ -173,7 +175,7 @@ func TestBuildPodTemplateSpec(t *testing.T) { }, Env: append( []corev1.EnvVar{{Name: "my-env", Value: "my-value"}}, - DefaultEnvVars(sampleES.Spec.HTTP, HeadlessServiceName(esv1.StatefulSet(sampleES.Name, nodeSet.Name)))...), + DefaultEnvVars(sampleES.Spec.HTTP, sset.HeadlessServiceName(esv1.StatefulSet(sampleES.Name, nodeSet.Name)))...), Resources: DefaultResources, VolumeMounts: volumeMounts, ReadinessProbe: NewReadinessProbe(), diff --git a/pkg/controller/elasticsearch/nodespec/statefulset.go b/pkg/controller/elasticsearch/nodespec/statefulset.go index d67c7b62dd..feaa6e024d 100644 --- a/pkg/controller/elasticsearch/nodespec/statefulset.go +++ b/pkg/controller/elasticsearch/nodespec/statefulset.go @@ -27,12 +27,6 @@ var ( f = false ) -// HeadlessServiceName returns the name of the headless service for the given StatefulSet. -func HeadlessServiceName(ssetName string) string { - // just use the sset name - return ssetName -} - // HeadlessService returns a headless service for the given StatefulSet func HeadlessService(es *esv1.Elasticsearch, ssetName string) corev1.Service { nsn := k8s.ExtractNamespacedName(es) @@ -40,7 +34,7 @@ func HeadlessService(es *esv1.Elasticsearch, ssetName string) corev1.Service { return corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: nsn.Namespace, - Name: HeadlessServiceName(ssetName), + Name: sset.HeadlessServiceName(ssetName), Labels: label.NewStatefulSetLabels(nsn, ssetName), }, Spec: corev1.ServiceSpec{ @@ -115,7 +109,7 @@ func BuildStatefulSet( // use default revision history limit RevisionHistoryLimit: nil, // build a headless service per StatefulSet, matching the StatefulSet labels - ServiceName: HeadlessServiceName(statefulSetName), + ServiceName: sset.HeadlessServiceName(statefulSetName), Selector: &metav1.LabelSelector{ MatchLabels: ssetSelector, }, diff --git a/pkg/controller/elasticsearch/settings/masters.go b/pkg/controller/elasticsearch/settings/masters.go index b358ddb536..4d6e27c967 100644 --- a/pkg/controller/elasticsearch/settings/masters.go +++ b/pkg/controller/elasticsearch/settings/masters.go @@ -6,10 +6,8 @@ package settings import ( "context" - "net" "reflect" "sort" - "strconv" "strings" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" @@ -17,7 +15,7 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/pkg/controller/common/tracing" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label" - "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/network" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/sset" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "go.elastic.co/apm" @@ -59,7 +57,7 @@ func UpdateSeedHostsConfigMap( if len(master.Status.PodIP) > 0 { // do not add pod with no IPs seedHosts = append( seedHosts, - net.JoinHostPort(master.Status.PodIP, strconv.Itoa(network.TransportPort)), + sset.PodDNSName(master), ) } } diff --git a/pkg/controller/elasticsearch/settings/masters_test.go b/pkg/controller/elasticsearch/settings/masters_test.go index 704261325c..0dcc252f69 100644 --- a/pkg/controller/elasticsearch/settings/masters_test.go +++ b/pkg/controller/elasticsearch/settings/masters_test.go @@ -33,6 +33,7 @@ func newPodWithIP(name, ip string, master bool) corev1.Pod { }, } label.NodeTypesMasterLabelName.Set(master, p.Labels) + p.Labels[label.StatefulSetNameLabelName] = "test-sset" return p } @@ -97,7 +98,7 @@ func TestUpdateSeedHostsConfigMap(t *testing.T) { es: es, }, wantErr: false, - expectedContent: "10.0.3.3:9300\n10.0.9.2:9300", + expectedContent: "master1.test-sset\nmaster3.test-sset", }, { name: "All masters have IPs, some nodes don't", @@ -113,7 +114,7 @@ func TestUpdateSeedHostsConfigMap(t *testing.T) { es: es, }, wantErr: false, - expectedContent: "10.0.3.3:9300\n10.0.6.5:9300\n10.0.9.2:9300", + expectedContent: "master1.test-sset\nmaster2.test-sset\nmaster3.test-sset", }, { name: "Ordering of pods should not matter", @@ -127,21 +128,7 @@ func TestUpdateSeedHostsConfigMap(t *testing.T) { es: es, }, wantErr: false, - expectedContent: "10.0.3.3:9300\n10.0.6.5:9300\n10.0.9.2:9300", - }, - { - name: "Can handle IPv6 addresses", - args: args{ - pods: []corev1.Pod{ // - newPodWithIP("master2", "fd00:10:244:0:2::3", true), - newPodWithIP("master3", "fd00:10:244:0:2::5", true), - newPodWithIP("master1", "fd00:10:244:0:2::2", true), - }, - c: k8s.WrappedFakeClient(), - es: es, - }, - wantErr: false, - expectedContent: "[fd00:10:244:0:2::2]:9300\n[fd00:10:244:0:2::3]:9300\n[fd00:10:244:0:2::5]:9300", + expectedContent: "master1.test-sset\nmaster2.test-sset\nmaster3.test-sset", }, } for _, tt := range tests { diff --git a/pkg/controller/elasticsearch/sset/pod.go b/pkg/controller/elasticsearch/sset/pod.go index 3ec1a011c7..61df280e8c 100644 --- a/pkg/controller/elasticsearch/sset/pod.go +++ b/pkg/controller/elasticsearch/sset/pod.go @@ -20,6 +20,18 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/utils/stringsutil" ) +// HeadlessServiceName returns the name of the headless service for the given StatefulSet. +func HeadlessServiceName(ssetName string) string { + // just use the sset name + return ssetName +} + +//PodDNSName returns the DNS resolvable name of the given pod resolvable within the namespace. +func PodDNSName(pod corev1.Pod) string { + ssetName := pod.Labels[label.StatefulSetNameLabelName] + return fmt.Sprintf("%s.%s", pod.Name, HeadlessServiceName(ssetName)) +} + // PodName returns the name of the pod with the given ordinal for this StatefulSet. func PodName(ssetName string, ordinal int32) string { return fmt.Sprintf("%s-%d", ssetName, ordinal) From e59e3aaa37d406a66dc6833e0aab978a9605595a Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Tue, 1 Sep 2020 13:40:40 +0200 Subject: [PATCH 16/19] workaround Enteprise Search IPv6 issue --- pkg/controller/enterprisesearch/config.go | 16 ++++++-- .../enterprisesearch/config_test.go | 39 ++++++++++++++++++- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/pkg/controller/enterprisesearch/config.go b/pkg/controller/enterprisesearch/config.go index 1dc52635e9..4c868df567 100644 --- a/pkg/controller/enterprisesearch/config.go +++ b/pkg/controller/enterprisesearch/config.go @@ -6,10 +6,9 @@ package enterprisesearch import ( "fmt" + "net" "path/filepath" - "github.com/elastic/cloud-on-k8s/pkg/utils/net" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,6 +25,7 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/common/volume" "github.com/elastic/cloud-on-k8s/pkg/controller/enterprisesearch/name" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" + netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net" ) const ( @@ -96,7 +96,7 @@ type partialConfigWithESAuth struct { // readinessProbeScript returns a bash script that requests the health endpoint. func readinessProbeScript(ent entv1beta1.EnterpriseSearch, config *settings.CanonicalConfig, ipFamily corev1.IPFamily) ([]byte, error) { - url := fmt.Sprintf("%s://%s/api/ent/v1/internal/health", ent.Spec.HTTP.Protocol(), net.LoopbackHostPort(ipFamily, HTTPPort)) + url := fmt.Sprintf("%s://%s/api/ent/v1/internal/health", ent.Spec.HTTP.Protocol(), netutil.LoopbackHostPort(ipFamily, HTTPPort)) // retrieve Elasticsearch user credentials from the aggregated config since it could be user-provided var esAuth partialConfigWithESAuth @@ -229,10 +229,18 @@ func parseConfigRef(driver driver.Interface, ent entv1beta1.EnterpriseSearch) (* return common.ParseConfigRef(driver, &ent, ent.Spec.ConfigRef, ConfigFilename) } +func inAddrAnyFor(ipFamily corev1.IPFamily) string { + if ipFamily == corev1.IPv4Protocol { + return net.IPv4zero.String() + } + // Enterprise Search even in its most recent version 7.9.0 cannot properly handle contracted IPv6 addresses like "::" + return "0:0:0:0:0:0:0:0" +} + func defaultConfig(ent entv1beta1.EnterpriseSearch, ipFamily corev1.IPFamily) *settings.CanonicalConfig { return settings.MustCanonicalConfig(map[string]interface{}{ "ent_search.external_url": fmt.Sprintf("%s://localhost:%d", ent.Spec.HTTP.Protocol(), HTTPPort), - "ent_search.listen_host": net.InAddrAnyFor(ipFamily).String(), + "ent_search.listen_host": inAddrAnyFor(ipFamily), "filebeat_log_directory": LogVolumeMountPath, "log_directory": LogVolumeMountPath, "allow_es_settings_modification": true, diff --git a/pkg/controller/enterprisesearch/config_test.go b/pkg/controller/enterprisesearch/config_test.go index 4e32849931..7d25a1884d 100644 --- a/pkg/controller/enterprisesearch/config_test.go +++ b/pkg/controller/enterprisesearch/config_test.go @@ -231,12 +231,13 @@ func TestReconcileConfig(t *testing.T) { name string runtimeObjs []runtime.Object ent entv1beta1.EnterpriseSearch + ipFamily corev1.IPFamily // we don't compare the exact secret content because some keys are random, but we at least check // all entries in this array exist in the reconciled secret, and there are not more rows wantSecretEntries []string }{ { - name: "create default config secret", + name: "create default config secret (IPv4)", runtimeObjs: nil, ent: entv1beta1.EnterpriseSearch{ ObjectMeta: metav1.ObjectMeta{ @@ -244,6 +245,7 @@ func TestReconcileConfig(t *testing.T) { Name: "sample", }, }, + ipFamily: corev1.IPv4Protocol, wantSecretEntries: []string{ "allow_es_settings_modification: true", "ent_search:", @@ -263,6 +265,35 @@ func TestReconcileConfig(t *testing.T) { "secret_session_key:", // don't check the actual secret session key }, }, + { + name: "create default config secret (IPv6)", + runtimeObjs: nil, + ent: entv1beta1.EnterpriseSearch{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", + Name: "sample", + }, + }, + ipFamily: corev1.IPv6Protocol, + wantSecretEntries: []string{ + "allow_es_settings_modification: true", + "ent_search:", + "external_url: https://localhost:3002", + "filebeat_log_directory: /var/log/enterprise-search", + "listen_host: \"0:0:0:0:0:0:0:0\"", + "log_directory: /var/log/enterprise-search", + "ssl:", + "certificate: /mnt/elastic-internal/http-certs/tls.crt", + "certificate_authorities:", + "- /mnt/elastic-internal/http-certs/ca.crt", + "enabled: true", + "key: /mnt/elastic-internal/http-certs/tls.key", + "secret_management:", + "encryption_keys:", + "-", // don't check the actual encryption key + "secret_session_key:", // don't check the actual secret session key + }, + }, { name: "update existing default config secret", runtimeObjs: []runtime.Object{ @@ -278,6 +309,7 @@ func TestReconcileConfig(t *testing.T) { }, }, }, + ipFamily: corev1.IPv4Protocol, ent: entv1beta1.EnterpriseSearch{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", @@ -323,6 +355,7 @@ func TestReconcileConfig(t *testing.T) { }, }, }, + ipFamily: corev1.IPv4Protocol, wantSecretEntries: []string{ "allow_es_settings_modification: true", "elasticsearch:", @@ -366,6 +399,7 @@ func TestReconcileConfig(t *testing.T) { }}, }, }, + ipFamily: corev1.IPv4Protocol, wantSecretEntries: []string{ "allow_es_settings_modification: true", "ent_search:", @@ -414,6 +448,7 @@ func TestReconcileConfig(t *testing.T) { }, }, }, + ipFamily: corev1.IPv4Protocol, wantSecretEntries: []string{ "allow_es_settings_modification: true", "ent_search:", @@ -444,7 +479,7 @@ func TestReconcileConfig(t *testing.T) { } // secret metadata should be correct - got, err := ReconcileConfig(driver, tt.ent, corev1.IPv4Protocol) + got, err := ReconcileConfig(driver, tt.ent, tt.ipFamily) require.NoError(t, err) assert.Equal(t, "sample-ent-config", got.Name) assert.Equal(t, "ns", got.Namespace) From d481ca5fd540f73201f61ad7b87831e4b7cf8847 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Wed, 2 Sep 2020 09:41:11 +0200 Subject: [PATCH 17/19] Revert "Move discovery to DNS names" This reverts commit 59c217f2869b6d97e2d850739154668653222d0e. --- .../certificates/transport/csr.go | 9 +++++--- .../elasticsearch/driver/upscale_test.go | 4 ++-- .../elasticsearch/nodespec/podspec.go | 4 +--- .../elasticsearch/nodespec/podspec_test.go | 4 +--- .../elasticsearch/nodespec/statefulset.go | 10 +++++++-- .../elasticsearch/settings/masters.go | 6 ++++-- .../elasticsearch/settings/masters_test.go | 21 +++++++++++++++---- pkg/controller/elasticsearch/sset/pod.go | 12 ----------- 8 files changed, 39 insertions(+), 31 deletions(-) diff --git a/pkg/controller/elasticsearch/certificates/transport/csr.go b/pkg/controller/elasticsearch/certificates/transport/csr.go index 9c5d6ce1ea..bc42cfbd79 100644 --- a/pkg/controller/elasticsearch/certificates/transport/csr.go +++ b/pkg/controller/elasticsearch/certificates/transport/csr.go @@ -11,8 +11,8 @@ import ( "net" "time" - "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/sset" - + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec" netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -73,6 +73,9 @@ func buildGeneralNames( return nil, errors.Errorf("pod currently has no valid IP, found: [%s]", pod.Status.PodIP) } + ssetName := pod.Labels[label.StatefulSetNameLabelName] + svcName := nodespec.HeadlessServiceName(ssetName) + commonName := buildCertificateCommonName(pod, cluster.Name, cluster.Namespace) commonNameUTF8OtherName := &certificates.UTF8StringValuedOtherName{ @@ -93,7 +96,7 @@ func buildGeneralNames( // the DNS name has to match the seed hosts configured in the remote cluster settings {DNSName: fmt.Sprintf("%s.%s.svc", esv1.TransportService(cluster.Name), cluster.Namespace)}, // add the resolvable DNS name of the Pod as published by Elasticsearch - {DNSName: sset.PodDNSName(pod)}, + {DNSName: fmt.Sprintf("%s.%s", pod.Name, svcName)}, {IPAddress: netutil.IPToRFCForm(podIP)}, {IPAddress: netutil.IPToRFCForm(netutil.LoopbackFor(netutil.ToIPFamily(podIP.String())))}, } diff --git a/pkg/controller/elasticsearch/driver/upscale_test.go b/pkg/controller/elasticsearch/driver/upscale_test.go index 9d6b742d09..773519e408 100644 --- a/pkg/controller/elasticsearch/driver/upscale_test.go +++ b/pkg/controller/elasticsearch/driver/upscale_test.go @@ -124,8 +124,8 @@ func TestHandleUpscaleAndSpecChanges(t *testing.T) { require.Equal(t, pointer.Int32(4), sset2.Spec.Replicas) comparison.RequireEqual(t, &updatedStatefulSets[1], &sset2) // headless services should be created for both - require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: sset.HeadlessServiceName("sset1")}, &corev1.Service{})) - require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: sset.HeadlessServiceName("sset2")}, &corev1.Service{})) + require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: nodespec.HeadlessServiceName("sset1")}, &corev1.Service{})) + require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: nodespec.HeadlessServiceName("sset2")}, &corev1.Service{})) // config should be created for both require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: esv1.ConfigSecret("sset1")}, &corev1.Secret{})) require.NoError(t, k8sClient.Get(types.NamespacedName{Namespace: "ns", Name: esv1.ConfigSecret("sset2")}, &corev1.Secret{})) diff --git a/pkg/controller/elasticsearch/nodespec/podspec.go b/pkg/controller/elasticsearch/nodespec/podspec.go index fe1f968a80..6352a26144 100644 --- a/pkg/controller/elasticsearch/nodespec/podspec.go +++ b/pkg/controller/elasticsearch/nodespec/podspec.go @@ -8,8 +8,6 @@ import ( "crypto/sha256" "fmt" - "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/sset" - corev1 "k8s.io/api/core/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" @@ -60,7 +58,7 @@ func BuildPodTemplateSpec( WithPorts(defaultContainerPorts). WithReadinessProbe(*NewReadinessProbe()). WithAffinity(DefaultAffinity(es.Name)). - WithEnv(DefaultEnvVars(es.Spec.HTTP, sset.HeadlessServiceName(esv1.StatefulSet(es.Name, nodeSet.Name)))...). + WithEnv(DefaultEnvVars(es.Spec.HTTP, HeadlessServiceName(esv1.StatefulSet(es.Name, nodeSet.Name)))...). WithVolumes(volumes...). WithVolumeMounts(volumeMounts...). WithInitContainers(initContainers...). diff --git a/pkg/controller/elasticsearch/nodespec/podspec_test.go b/pkg/controller/elasticsearch/nodespec/podspec_test.go index 2d52c2995d..a4f9c0b660 100644 --- a/pkg/controller/elasticsearch/nodespec/podspec_test.go +++ b/pkg/controller/elasticsearch/nodespec/podspec_test.go @@ -8,8 +8,6 @@ import ( "sort" "testing" - "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/sset" - commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/common/defaults" @@ -175,7 +173,7 @@ func TestBuildPodTemplateSpec(t *testing.T) { }, Env: append( []corev1.EnvVar{{Name: "my-env", Value: "my-value"}}, - DefaultEnvVars(sampleES.Spec.HTTP, sset.HeadlessServiceName(esv1.StatefulSet(sampleES.Name, nodeSet.Name)))...), + DefaultEnvVars(sampleES.Spec.HTTP, HeadlessServiceName(esv1.StatefulSet(sampleES.Name, nodeSet.Name)))...), Resources: DefaultResources, VolumeMounts: volumeMounts, ReadinessProbe: NewReadinessProbe(), diff --git a/pkg/controller/elasticsearch/nodespec/statefulset.go b/pkg/controller/elasticsearch/nodespec/statefulset.go index feaa6e024d..d67c7b62dd 100644 --- a/pkg/controller/elasticsearch/nodespec/statefulset.go +++ b/pkg/controller/elasticsearch/nodespec/statefulset.go @@ -27,6 +27,12 @@ var ( f = false ) +// HeadlessServiceName returns the name of the headless service for the given StatefulSet. +func HeadlessServiceName(ssetName string) string { + // just use the sset name + return ssetName +} + // HeadlessService returns a headless service for the given StatefulSet func HeadlessService(es *esv1.Elasticsearch, ssetName string) corev1.Service { nsn := k8s.ExtractNamespacedName(es) @@ -34,7 +40,7 @@ func HeadlessService(es *esv1.Elasticsearch, ssetName string) corev1.Service { return corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Namespace: nsn.Namespace, - Name: sset.HeadlessServiceName(ssetName), + Name: HeadlessServiceName(ssetName), Labels: label.NewStatefulSetLabels(nsn, ssetName), }, Spec: corev1.ServiceSpec{ @@ -109,7 +115,7 @@ func BuildStatefulSet( // use default revision history limit RevisionHistoryLimit: nil, // build a headless service per StatefulSet, matching the StatefulSet labels - ServiceName: sset.HeadlessServiceName(statefulSetName), + ServiceName: HeadlessServiceName(statefulSetName), Selector: &metav1.LabelSelector{ MatchLabels: ssetSelector, }, diff --git a/pkg/controller/elasticsearch/settings/masters.go b/pkg/controller/elasticsearch/settings/masters.go index 4d6e27c967..b358ddb536 100644 --- a/pkg/controller/elasticsearch/settings/masters.go +++ b/pkg/controller/elasticsearch/settings/masters.go @@ -6,8 +6,10 @@ package settings import ( "context" + "net" "reflect" "sort" + "strconv" "strings" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" @@ -15,7 +17,7 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/pkg/controller/common/tracing" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label" - "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/sset" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/network" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "go.elastic.co/apm" @@ -57,7 +59,7 @@ func UpdateSeedHostsConfigMap( if len(master.Status.PodIP) > 0 { // do not add pod with no IPs seedHosts = append( seedHosts, - sset.PodDNSName(master), + net.JoinHostPort(master.Status.PodIP, strconv.Itoa(network.TransportPort)), ) } } diff --git a/pkg/controller/elasticsearch/settings/masters_test.go b/pkg/controller/elasticsearch/settings/masters_test.go index 0dcc252f69..704261325c 100644 --- a/pkg/controller/elasticsearch/settings/masters_test.go +++ b/pkg/controller/elasticsearch/settings/masters_test.go @@ -33,7 +33,6 @@ func newPodWithIP(name, ip string, master bool) corev1.Pod { }, } label.NodeTypesMasterLabelName.Set(master, p.Labels) - p.Labels[label.StatefulSetNameLabelName] = "test-sset" return p } @@ -98,7 +97,7 @@ func TestUpdateSeedHostsConfigMap(t *testing.T) { es: es, }, wantErr: false, - expectedContent: "master1.test-sset\nmaster3.test-sset", + expectedContent: "10.0.3.3:9300\n10.0.9.2:9300", }, { name: "All masters have IPs, some nodes don't", @@ -114,7 +113,7 @@ func TestUpdateSeedHostsConfigMap(t *testing.T) { es: es, }, wantErr: false, - expectedContent: "master1.test-sset\nmaster2.test-sset\nmaster3.test-sset", + expectedContent: "10.0.3.3:9300\n10.0.6.5:9300\n10.0.9.2:9300", }, { name: "Ordering of pods should not matter", @@ -128,7 +127,21 @@ func TestUpdateSeedHostsConfigMap(t *testing.T) { es: es, }, wantErr: false, - expectedContent: "master1.test-sset\nmaster2.test-sset\nmaster3.test-sset", + expectedContent: "10.0.3.3:9300\n10.0.6.5:9300\n10.0.9.2:9300", + }, + { + name: "Can handle IPv6 addresses", + args: args{ + pods: []corev1.Pod{ // + newPodWithIP("master2", "fd00:10:244:0:2::3", true), + newPodWithIP("master3", "fd00:10:244:0:2::5", true), + newPodWithIP("master1", "fd00:10:244:0:2::2", true), + }, + c: k8s.WrappedFakeClient(), + es: es, + }, + wantErr: false, + expectedContent: "[fd00:10:244:0:2::2]:9300\n[fd00:10:244:0:2::3]:9300\n[fd00:10:244:0:2::5]:9300", }, } for _, tt := range tests { diff --git a/pkg/controller/elasticsearch/sset/pod.go b/pkg/controller/elasticsearch/sset/pod.go index 61df280e8c..3ec1a011c7 100644 --- a/pkg/controller/elasticsearch/sset/pod.go +++ b/pkg/controller/elasticsearch/sset/pod.go @@ -20,18 +20,6 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/utils/stringsutil" ) -// HeadlessServiceName returns the name of the headless service for the given StatefulSet. -func HeadlessServiceName(ssetName string) string { - // just use the sset name - return ssetName -} - -//PodDNSName returns the DNS resolvable name of the given pod resolvable within the namespace. -func PodDNSName(pod corev1.Pod) string { - ssetName := pod.Labels[label.StatefulSetNameLabelName] - return fmt.Sprintf("%s.%s", pod.Name, HeadlessServiceName(ssetName)) -} - // PodName returns the name of the pod with the given ordinal for this StatefulSet. func PodName(ssetName string, ordinal int32) string { return fmt.Sprintf("%s-%d", ssetName, ordinal) From c169ac4ce9ef04623a21a2d6d1f8c09d56db32f8 Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Fri, 4 Sep 2020 11:46:19 +0200 Subject: [PATCH 18/19] Add ip-family flag --- cmd/manager/main.go | 28 ++++++++++++++++++++- docs/operating-eck/operator-config.asciidoc | 1 + pkg/controller/common/operator/flags.go | 1 + 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index ac1a968a8f..bf8149258d 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -15,6 +15,8 @@ import ( "strings" "time" + corev1 "k8s.io/api/core/v1" + "github.com/elastic/cloud-on-k8s/pkg/about" apmv1 "github.com/elastic/cloud-on-k8s/pkg/apis/apm/v1" apmv1beta1 "github.com/elastic/cloud-on-k8s/pkg/apis/apm/v1beta1" @@ -178,6 +180,11 @@ func Command() *cobra.Command { false, "Enables a validating webhook server in the operator process.", ) + cmd.Flags().String( + operator.IPFamilyFlag, + "", + "Set the IP family to use. Possible values: IPv4, IPv6, \"\" (= auto-detect) ", + ) cmd.Flags().Bool( operator.ManageWebhookCertsFlag, true, @@ -421,6 +428,12 @@ func startOperator(stopChan <-chan struct{}) error { log.V(1).Info("Using certificate rotation parameters", operator.CertValidityFlag, certValidity, operator.CertRotateBeforeFlag, certRotateBefore) + ipFamily, err := chooseAndValidateIPFamily(viper.GetString(operator.IPFamilyFlag), net.ToIPFamily(os.Getenv(settings.EnvPodIP))) + if err != nil { + log.Error(err, "Invalid IP family parameter") + return err + } + // Setup a client to set the operator uuid config map clientset, err := kubernetes.NewForConfig(cfg) if err != nil { @@ -441,7 +454,7 @@ func startOperator(stopChan <-chan struct{}) error { } params := operator.Parameters{ Dialer: dialer, - IPFamily: net.ToIPFamily(os.Getenv(settings.EnvPodIP)), + IPFamily: ipFamily, OperatorNamespace: operatorNamespace, OperatorInfo: operatorInfo, CACertRotation: certificates.RotationParams{ @@ -496,6 +509,19 @@ func startOperator(stopChan <-chan struct{}) error { return nil } +func chooseAndValidateIPFamily(ipFamilyStr string, ipFamilyDefault corev1.IPFamily) (corev1.IPFamily, error) { + switch strings.ToLower(ipFamilyStr) { + case "": + return ipFamilyDefault, nil + case "ipv4": + return corev1.IPv4Protocol, nil + case "ipv6": + return corev1.IPv6Protocol, nil + default: + return ipFamilyDefault, fmt.Errorf("IP family can be one of: IPv4, IPv6 or \"\" to auto-detect, but was %s", ipFamilyStr) + } +} + func registerControllers(mgr manager.Manager, params operator.Parameters, accessReviewer rbac.AccessReviewer) error { controllers := []struct { name string diff --git a/docs/operating-eck/operator-config.asciidoc b/docs/operating-eck/operator-config.asciidoc index bae83081c5..3804a73d73 100644 --- a/docs/operating-eck/operator-config.asciidoc +++ b/docs/operating-eck/operator-config.asciidoc @@ -26,6 +26,7 @@ ECK can be configured using either command line flags or environment variables. |enable-tracing | false | Enable APM tracing in the operator process. Use environment variables to configure APM server URL, credentials, and so on. See link:https://www.elastic.co/guide/en/apm/agent/go/1.x/configuration.html[Apm Go Agent reference] for details. |enable-webhook | false | Enables a validating webhook server in the operator process. |enforce-rbac-on-refs| false | Enables restrictions on cross-namespace resource association through RBAC. +|ip-family|""| Set the IP family to use. Possible values: IPv4, IPv6, "" (= auto-detect) |log-verbosity |0 |Verbosity level of logs. `-2`=Error, `-1`=Warn, `0`=Info, `0` and above=Debug. |manage-webhook-certs |true |Enables automatic webhook certificate management. |max-concurrent-reconciles |3 | Maximum number of concurrent reconciles per controller (Elasticsearch, Kibana, APM Server). Affects the ability of the operator to process changes concurrently. diff --git a/pkg/controller/common/operator/flags.go b/pkg/controller/common/operator/flags.go index f5205ae1e9..26c900267a 100644 --- a/pkg/controller/common/operator/flags.go +++ b/pkg/controller/common/operator/flags.go @@ -18,6 +18,7 @@ const ( EnableTracingFlag = "enable-tracing" EnableWebhookFlag = "enable-webhook" EnforceRBACOnRefsFlag = "enforce-rbac-on-refs" + IPFamilyFlag = "ip-family" ManageWebhookCertsFlag = "manage-webhook-certs" MaxConcurrentReconcilesFlag = "max-concurrent-reconciles" MetricsPortFlag = "metrics-port" From eb3d4888819ccae5083a01f56d6b6ddc851f0f8c Mon Sep 17 00:00:00 2001 From: Peter Brachwitz Date: Fri, 4 Sep 2020 14:45:17 +0200 Subject: [PATCH 19/19] Reorg imports --- cmd/manager/main.go | 3 +-- .../common/certificates/http_reconcile.go | 11 +++++------ .../elasticsearch/certificates/transport/csr.go | 5 ++--- .../certificates/transport/csr_test.go | 5 ++--- .../transport/transport_fixtures_test.go | 5 ++--- .../elasticsearch/nodespec/podspec_test.go | 1 - .../elasticsearch/nodespec/readiness_probe.go | 2 +- .../elasticsearch/settings/merged_config_test.go | 3 +-- pkg/controller/enterprisesearch/config.go | 9 ++++----- pkg/controller/enterprisesearch/config_test.go | 11 +++++------ pkg/controller/kibana/config_settings.go | 16 +++++++--------- pkg/controller/kibana/config_settings_test.go | 11 +++++------ 12 files changed, 35 insertions(+), 47 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index bf8149258d..5973775862 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -15,8 +15,6 @@ import ( "strings" "time" - corev1 "k8s.io/api/core/v1" - "github.com/elastic/cloud-on-k8s/pkg/about" apmv1 "github.com/elastic/cloud-on-k8s/pkg/apis/apm/v1" apmv1beta1 "github.com/elastic/cloud-on-k8s/pkg/apis/apm/v1beta1" @@ -55,6 +53,7 @@ import ( "github.com/spf13/viper" "go.elastic.co/apm" "go.uber.org/automaxprocs/maxprocs" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" diff --git a/pkg/controller/common/certificates/http_reconcile.go b/pkg/controller/common/certificates/http_reconcile.go index 47287a8103..267793981c 100644 --- a/pkg/controller/common/certificates/http_reconcile.go +++ b/pkg/controller/common/certificates/http_reconcile.go @@ -14,18 +14,17 @@ import ( "strings" "time" + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/name" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler" + "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" + netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" - "github.com/elastic/cloud-on-k8s/pkg/controller/common/name" - "github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler" - "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" - netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net" ) // ReconcilePublicHTTPCerts reconciles the Secret containing the HTTP Certificate currently in use, and the CA of diff --git a/pkg/controller/elasticsearch/certificates/transport/csr.go b/pkg/controller/elasticsearch/certificates/transport/csr.go index bc42cfbd79..e63da377c8 100644 --- a/pkg/controller/elasticsearch/certificates/transport/csr.go +++ b/pkg/controller/elasticsearch/certificates/transport/csr.go @@ -11,14 +11,13 @@ import ( "net" "time" + esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/nodespec" netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - - esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" - "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" ) // createValidatedCertificateTemplate validates a CSR and creates a certificate template. diff --git a/pkg/controller/elasticsearch/certificates/transport/csr_test.go b/pkg/controller/elasticsearch/certificates/transport/csr_test.go index 894a340816..8f2bc48f62 100644 --- a/pkg/controller/elasticsearch/certificates/transport/csr_test.go +++ b/pkg/controller/elasticsearch/certificates/transport/csr_test.go @@ -9,12 +9,11 @@ import ( "net" "testing" + esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - - esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" - "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" ) // roundTripSerialize does a serialization round-trip of the certificate in order to make sure any extra extensions diff --git a/pkg/controller/elasticsearch/certificates/transport/transport_fixtures_test.go b/pkg/controller/elasticsearch/certificates/transport/transport_fixtures_test.go index 1ea729daac..44f0a57930 100644 --- a/pkg/controller/elasticsearch/certificates/transport/transport_fixtures_test.go +++ b/pkg/controller/elasticsearch/certificates/transport/transport_fixtures_test.go @@ -11,12 +11,11 @@ import ( "crypto/x509/pkix" "encoding/pem" + esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" - "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" ) // fixtures diff --git a/pkg/controller/elasticsearch/nodespec/podspec_test.go b/pkg/controller/elasticsearch/nodespec/podspec_test.go index a4f9c0b660..6b7c7eda6e 100644 --- a/pkg/controller/elasticsearch/nodespec/podspec_test.go +++ b/pkg/controller/elasticsearch/nodespec/podspec_test.go @@ -15,7 +15,6 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/initcontainer" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/settings" "github.com/go-test/deep" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" diff --git a/pkg/controller/elasticsearch/nodespec/readiness_probe.go b/pkg/controller/elasticsearch/nodespec/readiness_probe.go index a379e35616..39801fb57d 100644 --- a/pkg/controller/elasticsearch/nodespec/readiness_probe.go +++ b/pkg/controller/elasticsearch/nodespec/readiness_probe.go @@ -66,7 +66,7 @@ fi # Check if we are using IPv6 if [[ $POD_IP =~ .*:.* ]]; then - LOOPBACK=[::1] + LOOPBACK="[::1]" else LOOPBACK=127.0.01 fi diff --git a/pkg/controller/elasticsearch/settings/merged_config_test.go b/pkg/controller/elasticsearch/settings/merged_config_test.go index bf534e7b0f..a340315375 100644 --- a/pkg/controller/elasticsearch/settings/merged_config_test.go +++ b/pkg/controller/elasticsearch/settings/merged_config_test.go @@ -8,11 +8,10 @@ import ( "bytes" "testing" - "github.com/stretchr/testify/require" - commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/common/version" + "github.com/stretchr/testify/require" ) func TestNewMergedESConfig(t *testing.T) { diff --git a/pkg/controller/enterprisesearch/config.go b/pkg/controller/enterprisesearch/config.go index 4c868df567..9e242dfb11 100644 --- a/pkg/controller/enterprisesearch/config.go +++ b/pkg/controller/enterprisesearch/config.go @@ -9,11 +9,6 @@ import ( "net" "path/filepath" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" entv1beta1 "github.com/elastic/cloud-on-k8s/pkg/apis/enterprisesearch/v1beta1" "github.com/elastic/cloud-on-k8s/pkg/controller/association" @@ -26,6 +21,10 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/enterprisesearch/name" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" netutil "github.com/elastic/cloud-on-k8s/pkg/utils/net" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) const ( diff --git a/pkg/controller/enterprisesearch/config_test.go b/pkg/controller/enterprisesearch/config_test.go index 7d25a1884d..f0590f06c1 100644 --- a/pkg/controller/enterprisesearch/config_test.go +++ b/pkg/controller/enterprisesearch/config_test.go @@ -8,18 +8,17 @@ import ( "bytes" "testing" + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" + entv1beta1 "github.com/elastic/cloud-on-k8s/pkg/apis/enterprisesearch/v1beta1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/settings" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/watches" + "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" - - commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" - entv1beta1 "github.com/elastic/cloud-on-k8s/pkg/apis/enterprisesearch/v1beta1" - "github.com/elastic/cloud-on-k8s/pkg/controller/common/settings" - "github.com/elastic/cloud-on-k8s/pkg/controller/common/watches" - "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" ) func entWithConfigRef(secretName string) entv1beta1.EnterpriseSearch { diff --git a/pkg/controller/kibana/config_settings.go b/pkg/controller/kibana/config_settings.go index 5bd81fcc29..78d2976b8a 100644 --- a/pkg/controller/kibana/config_settings.go +++ b/pkg/controller/kibana/config_settings.go @@ -8,15 +8,6 @@ import ( "context" "path" - "github.com/elastic/cloud-on-k8s/pkg/utils/net" - - ucfg "github.com/elastic/go-ucfg" - "github.com/pkg/errors" - "go.elastic.co/apm" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/association" @@ -27,6 +18,13 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/common/version" "github.com/elastic/cloud-on-k8s/pkg/controller/common/volume" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" + "github.com/elastic/cloud-on-k8s/pkg/utils/net" + "github.com/elastic/go-ucfg" + "github.com/pkg/errors" + "go.elastic.co/apm" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" ) const ( diff --git a/pkg/controller/kibana/config_settings_test.go b/pkg/controller/kibana/config_settings_test.go index 5690429c99..16da4e345b 100644 --- a/pkg/controller/kibana/config_settings_test.go +++ b/pkg/controller/kibana/config_settings_test.go @@ -9,6 +9,11 @@ import ( "fmt" "testing" + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" + kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/settings" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/version" + "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" ucfg "github.com/elastic/go-ucfg" uyaml "github.com/elastic/go-ucfg/yaml" "github.com/go-test/deep" @@ -17,12 +22,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" - - commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" - kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" - "github.com/elastic/cloud-on-k8s/pkg/controller/common/settings" - "github.com/elastic/cloud-on-k8s/pkg/controller/common/version" - "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" ) var defaultConfig = []byte(`