Add TLS and basic authentication integration to Logstash API server

config/samples/logstash/logstash_pv.yml

@@ -4,7 +4,7 @@ metadata:
   name: d
 spec:
   count: 1
-  version: 8.8.0
+  version: 8.12.0
   config:
     queue.type: persisted
   pipelines:

docs/orchestrating-elastic-stack-applications/logstash.asciidoc

@@ -19,6 +19,7 @@ This section describes how to configure and deploy Logstash with ECK.
 ** <<{p}-logstash-volumes,Configuring Volumes>>
 ** <<{p}-logstash-pipelines-es,Using Elasticsearch in Logstash Pipelines>>
 ** <<{p}-logstash-expose-services,Exposing Services>>
+* <<{p}-logstash-securing-api,Securing Logstash API>>
 * <<{p}-logstash-configuration-examples,Configuration examples>>
 * <<{p}-logstash-advanced-configuration,Advanced Configuration>>
 ** <<{p}-logstash-jvm-options,Setting JVM Options>>
@@ -699,6 +700,121 @@ spec:
 The name of the container in the Pod template must be `logstash`.
 
 
+[id="{p}-logstash-securing-api"]
+== Securing Logstash API
+
+[id="{p}-logstash-https"]
+=== Enable HTTPS
+
+Access to the link:https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html#monitoring-api-security[Logstash Monitoring APIs] use HTTPS by default - the operator will set the values  `api.ssl.enabled: true`, `api.ssl.keystore.path` and `api.ssl.keystore.password`.
+
+You can further secure the {ls} Monitoring APIs by requiring HTTP Basic authentication by setting `api.auth.type: basic`, and providing the relevant credentials `api.auth.basic.username` and `api.auth.basic.password`:
+
+[source,yaml,subs="attributes,+macros,callouts"]
+----
+apiVersion: v1
+kind: Secret
+metadata:
+  name: logstash-api-secret
+stringData:
+  API_USERNAME: "AWESOME_USER"   <1>
+  API_PASSWORD: "T0p_Secret"     <1>
+---
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+  name: logstash-sample
+spec:
+  version: {version}
+  count: 1
+  config:
+    api.auth.type: basic
+    api.auth.basic.username: "${API_USERNAME}"   <3>
+    api.auth.basic.password: "${API_PASSWORD}"   <3>
+  podTemplate:
+    spec:
+      containers:
+        - name: logstash
+          envFrom:
+            - secretRef:
+                name: logstash-api-secret   <2>
+----
+<1> Store the username and password in a Secret.
+<2> Map the username and password to the environment variables of the Pod.
+<3> At Logstash startup, `${API_USERNAME}` and `${API_PASSWORD}` are replaced by the value of environment variables. Check link:https://www.elastic.co/guide/en/logstash/current/environment-variables.html[using environment variables] for more details.
+
+An alternative is to set up <<{p}-logstash-keystore, keystore>> to resolve `${API_USERNAME}` and `${API_PASSWORD}`
+
+NOTE: The variable substitution in `config` does not support the default value syntax.
+
+[id="{p}-logstash-http-tls-keystore"]
+=== TLS keystore
+
+The TLS Keystore is automatically generated and includes a certificate and a private key, with default password protection set to `changeit`.
+This password can be modified by configuring the `api.ssl.keystore.password` value.
+
+[source,yaml,subs="attributes"]
+----
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+  name: logstash-sample
+spec:
+  count: 1
+  version: {version}
+  config:
+    api.ssl.keystore.password: "${SSL_KEYSTORE_PASSWORD}"
+----
+
+
+[id="{p}-logstash-http-custom-tls"]
+=== Provide your own certificate
+
+If you want to use your own certificate, the required configuration is similar to Elasticsearch. Configure the certificate in `api` Service. Check <<{p}-custom-http-certificate>>.
+
+[source,yaml,subs="attributes,+macros,callouts"]
+----
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+  name: logstash-sample
+spec:
+  version: {version}
+  count: 1
+  elasticsearchRef:
+    name: "elasticsearch-sample"
+  services:
+    - name: api   <1>
+      tls:
+        certificate:
+          secretName: my-cert
+----
+<1> The service name `api` is reserved for {ls} monitoring endpoint.
+
+[id="{p}-logstash-http-disable-tls"]
+=== Disable TLS
+
+You can disable TLS by disabling the generation of the self-signed certificate in the API service definition
+
+[source,yaml,subs="attributes"]
+----
+apiVersion: logstash.k8s.elastic.co/v1alpha1
+kind: Logstash
+metadata:
+  name: logstash-sample
+spec:
+  version: {version}
+  count: 1
+  elasticsearchRef:
+    name: "elasticsearch-sample"
+  services:
+    - name: api
+      tls:
+        selfSignedCertificate:
+          disabled: true
+----
+
+
 [id="{p}-logstash-configuration-examples"]
 == Configuration examples
 
@@ -847,7 +963,7 @@ kind: Logstash
 metadata:
   name: logstash-sample
 spec:
-  version: 8.8.0
+  version: {version}
   count: 1
   pipelines:
     - pipeline.id: main

pkg/apis/logstash/v1alpha1/logstash_types.go

@@ -386,6 +386,21 @@ func (l *Logstash) MonitoringAssociation(esRef commonv1.ObjectSelector) commonv1
 	}
 }
 
+// APIServerService returns the user defined API Service
+func (l *Logstash) APIServerService() LogstashService {
+	for _, service := range l.Spec.Services {
+		if UserServiceName(l.Name, service.Name) == APIServiceName(l.Name) {
+			return service
+		}
+	}
+	return LogstashService{}
+}
+
+// APIServerTLSOptions returns the user defined TLSOptions of API Service
+func (l *Logstash) APIServerTLSOptions() commonv1.TLSOptions {
+	return l.APIServerService().TLS
+}
+
 func init() {
 	SchemeBuilder.Register(&Logstash{}, &LogstashList{})
 }

pkg/apis/logstash/v1alpha1/logstash_types_test.go

@@ -94,3 +94,65 @@ func TestLogstashMonitoringAssociation_AssociationConfAnnotationName(t *testing.
 		})
 	}
 }
+
+func TestLogstash_APIServerTLSOptions(t *testing.T) {
+	for _, tt := range []struct {
+		name     string
+		logstash Logstash
+		want     bool
+	}{
+		{
+			name: "default no service config enable TLS",
+			logstash: Logstash{
+				Spec: LogstashSpec{},
+			},
+			want: true,
+		},
+		{
+			name: "api service disable TLS",
+			logstash: Logstash{
+				Spec: LogstashSpec{
+					Services: []LogstashService{{
+						Name: "api",
+						TLS: commonv1.TLSOptions{
+							SelfSignedCertificate: &commonv1.SelfSignedCertificate{
+								Disabled: true,
+							},
+						},
+					}},
+				},
+			},
+			want: false,
+		},
+		{
+			name: "take api service from services",
+			logstash: Logstash{
+				Spec: LogstashSpec{
+					Services: []LogstashService{
+						{
+							Name: "strong_svc",
+							TLS: commonv1.TLSOptions{
+								SelfSignedCertificate: &commonv1.SelfSignedCertificate{
+									Disabled: false,
+								},
+							},
+						},
+						{
+							Name: "api",
+							TLS: commonv1.TLSOptions{
+								SelfSignedCertificate: &commonv1.SelfSignedCertificate{
+									Disabled: true,
+								},
+							},
+						},
+					},
+				},
+			},
+			want: false,
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.want, tt.logstash.APIServerTLSOptions().Enabled())
+		})
+	}
+}

pkg/apis/logstash/v1alpha1/validations.go

@@ -11,7 +11,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/validation/field"
 
 	commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1"
-	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/stackmon/validations"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/common/version"
 )
 
@@ -20,18 +19,12 @@ const (
 )
 
 var (
-	// MinStackMonVersion is the minimum version of Logstash to enable Stack Monitoring on an Elastic Stack application.
-	// This requirement comes from the fact that we configure Logstash to write logs to disk for Filebeat
-	// via the env var LOG_STYLE available from this version.
-	MinStackMonVersion = version.MustParse("8.7.0-SNAPSHOT")
-
 	defaultChecks = []func(*Logstash) field.ErrorList{
 		checkNoUnknownFields,
 		checkNameLength,
 		checkSupportedVersion,
 		checkSingleConfigSource,
 		checkESRefsNamed,
-		checkMonitoring,
 		checkAssociations,
 		checkSinglePipelineSource,
 	}
@@ -73,10 +66,6 @@ func checkSingleConfigSource(l *Logstash) field.ErrorList {
 	return nil
 }
 
-func checkMonitoring(l *Logstash) field.ErrorList {
-	return validations.Validate(l, l.Spec.Version, MinStackMonVersion)
-}
-
 func checkAssociations(l *Logstash) field.ErrorList {
 	monitoringPath := field.NewPath("spec").Child("monitoring")
 	err1 := commonv1.CheckAssociationRefs(monitoringPath.Child("metrics"), l.GetMonitoringMetricsRefs()...)

pkg/apis/logstash/v1alpha1/validations_test.go

@@ -214,7 +214,7 @@ func Test_checkSupportedVersion(t *testing.T) {
 		},
 		{
 			name:    "above min supported",
-			version: "8.7.1",
+			version: "8.12.0",
 			wantErr: false,
 		},
 	} {

pkg/apis/logstash/v1alpha1/webhook_test.go

@@ -26,7 +26,7 @@ func TestWebhook(t *testing.T) {
 			Object: func(t *testing.T, uid string) []byte {
 				t.Helper()
 				ls := mkLogstash(uid)
-				ls.Spec.Version = "8.7.0"
+				ls.Spec.Version = "8.12.0"
 				ls.Spec.Monitoring = commonv1.Monitoring{Metrics: commonv1.MetricsMonitoring{ElasticsearchRefs: []commonv1.ObjectSelector{{Name: "esmonname", Namespace: "esmonns"}}}}
 				return serialize(t, ls)
 			},
@@ -38,7 +38,7 @@ func TestWebhook(t *testing.T) {
 			Object: func(t *testing.T, uid string) []byte {
 				t.Helper()
 				ls := mkLogstash(uid)
-				ls.Spec.Version = "8.7.0"
+				ls.Spec.Version = "8.12.0"
 				ls.Spec.Monitoring = commonv1.Monitoring{
 					Metrics: commonv1.MetricsMonitoring{ElasticsearchRefs: []commonv1.ObjectSelector{{SecretName: "es1monname"}}},
 					Logs:    commonv1.LogsMonitoring{ElasticsearchRefs: []commonv1.ObjectSelector{{SecretName: "es2monname"}}},
@@ -48,7 +48,7 @@ func TestWebhook(t *testing.T) {
 			Check: test.ValidationWebhookSucceeded,
 		},
 		{
-			Name:      "invalid-version-for-stackmon",
+			Name:      "invalid-stack-version",
 			Operation: admissionv1beta1.Create,
 			Object: func(t *testing.T, uid string) []byte {
 				t.Helper()
@@ -58,7 +58,7 @@ func TestWebhook(t *testing.T) {
 				return serialize(t, ls)
 			},
 			Check: test.ValidationWebhookFailed(
-				`spec.version: Invalid value: "7.13.0": Unsupported version for Stack Monitoring. Required >= 8.7.0`,
+				`spec.version: Invalid value: "7.13.0": Unsupported version: version 7.13.0 is lower than the lowest supported version of 8.12.0`,
 			),
 		},
 		{
@@ -67,7 +67,7 @@ func TestWebhook(t *testing.T) {
 			Object: func(t *testing.T, uid string) []byte {
 				t.Helper()
 				ls := mkLogstash(uid)
-				ls.Spec.Version = "8.7.0"
+				ls.Spec.Version = "8.12.0"
 				ls.Spec.Monitoring = commonv1.Monitoring{
 					Metrics: commonv1.MetricsMonitoring{ElasticsearchRefs: []commonv1.ObjectSelector{{SecretName: "es1monname", Name: "xx"}}},
 					Logs:    commonv1.LogsMonitoring{ElasticsearchRefs: []commonv1.ObjectSelector{{SecretName: "es2monname"}}},
@@ -84,7 +84,7 @@ func TestWebhook(t *testing.T) {
 			Object: func(t *testing.T, uid string) []byte {
 				t.Helper()
 				ls := mkLogstash(uid)
-				ls.Spec.Version = "8.7.0"
+				ls.Spec.Version = "8.12.0"
 				ls.Spec.Monitoring = commonv1.Monitoring{
 					Metrics: commonv1.MetricsMonitoring{ElasticsearchRefs: []commonv1.ObjectSelector{{SecretName: "es1monname"}}},
 					Logs:    commonv1.LogsMonitoring{ElasticsearchRefs: []commonv1.ObjectSelector{{SecretName: "es2monname", ServiceName: "xx"}}},
@@ -109,7 +109,7 @@ func mkLogstash(uid string) *v1alpha1.Logstash {
 			UID:  types.UID(uid),
 		},
 		Spec: v1alpha1.LogstashSpec{
-			Version: "8.6.0",
+			Version: "8.12.0",
 		},
 	}
 }

pkg/controller/common/pod/spec.go

@@ -17,3 +17,12 @@ func ContainerByName(podSpec corev1.PodSpec, name string) *corev1.Container {
 	}
 	return nil
 }
+
+func InitContainerByName(podSpec corev1.PodSpec, name string) *corev1.Container {
+	for i, c := range podSpec.InitContainers {
+		if c.Name == name {
+			return &podSpec.InitContainers[i]
+		}
+	}
+	return nil
+}

pkg/controller/common/version/version.go

@@ -33,7 +33,7 @@ var (
 	// Due to bugfixes present in 7.14 that ECK depends on, this is the lowest version we support in Fleet mode.
 	SupportedFleetModeAgentVersions = MinMaxVersion{Min: MustParse("7.14.0-SNAPSHOT"), Max: From(8, 99, 99)}
 	SupportedMapsVersions           = MinMaxVersion{Min: From(7, 11, 0), Max: From(8, 99, 99)}
-	SupportedLogstashVersions       = MinMaxVersion{Min: From(8, 6, 0), Max: From(8, 99, 99)}
+	SupportedLogstashVersions       = MinMaxVersion{Min: From(8, 12, 0), Max: From(8, 99, 99)}
 
 	// minPreReleaseVersion is the lowest prerelease identifier as numeric prerelease takes precedence before
 	// alphanumeric ones and it can't have leading zeros.