libbeat/idxmgmt/ilm: fix alias creation (#26146)

When creating the initial index/write alias fails, don't
check the status code, just check if the alias exists
regardless of the error. This fixes a bug where alias
creation fails when the alias exists but points to another
index, and the initial index does not exist (e.g. due to
ILM deletion.)

(cherry picked from commit 7bd96c2)

CHANGELOG.next.asciidoc

@@ -111,6 +111,22 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - [Metricbeat][Kubernetes] Change cluster_ip field from ip to keyword. {pull}20571[20571]
 - The `o365input` and `o365` module now recover from an authentication problem or other fatal errors, instead of terminating. {pull}21258[21258]
 - Periodic metrics in logs will now report `libbeat.output.events.active` and `beat.memstats.rss`
+  as gauges (rather than counters). {pull}22877[22877]
+- Use PROGRAMDATA environment variable instead of C:\ProgramData for windows install service {pull}22874[22874]
+- Fix reporting of cgroup metrics when running under Docker {pull}22879[22879]
+- Fix typo in config docs {pull}23185[23185]
+- Add FAQ entry for madvdontneed variable {pull}23429[23429]
+- Fix panic due to unhandled DeletedFinalStateUnknown in k8s OnDelete {pull}23419[23419]
+- Fix error loop with runaway CPU use when the Kafka output encounters some connection errors {pull}23484[23484]
+- Fix ILM setup log reporting that a policy or an alias was created, even though the creation of any resource was disabled. {issue}24046[24046] {pull}24480[24480]
+- Fix ILM alias not being created if `setup.ilm.check_exists: false` and `setup.ilm.overwrite: true` has been configured. {pull}24480[24480]
+- Fix issue discovering docker containers and metadata after reconnections {pull}24318[24318]
+- Allow cgroup self-monitoring to see alternate `hostfs` paths {pull}24334[24334]
+- Fix 'make setup' instructions for a new beat {pull}24944[24944]
+- Fix out of date FreeBSD vagrantbox. {pull}25652[25652]
+- Fix handling of `file_selectors` in aws-s3 input. {pull}25792[25792]
+- Fix ILM alias creation when write alias exists and initial index does not exist {pull}26143[26143]
+- Include date separator in the filename prefix of `dateRotator` to make sure nothing gets purged accidentally {pull}26176[26176]
 
 *Auditbeat*
 

libbeat/idxmgmt/ilm/client_handler.go

@@ -191,19 +191,18 @@ func (h *ESClientHandler) CreateAlias(alias Alias) error {
 	}
 
 	// Note: actual aliases are accessible via the index
-	status, res, err := h.client.Request("PUT", "/"+firstIndex, "", nil, body)
-	if status == 400 {
-		// HasAlias fails if there is an index with the same name, that is
-		// what we want to check here.
-		_, err := h.HasAlias(alias.Name)
-		if err != nil {
+	if _, res, err := h.client.Request("PUT", "/"+firstIndex, "", nil, body); err != nil {
+		// Creating the index may fail for multiple reasons, e.g. because
+		// the index exists, or because the write alias exists and points
+		// to another index.
+		if ok, err := h.HasAlias(alias.Name); err != nil {
+			// HasAlias fails if there is an index with the same name.
 			return err
+		} else if ok {
+			return errOf(ErrAliasAlreadyExists)
 		}
-		return errOf(ErrAliasAlreadyExists)
-	} else if err != nil {
 		return wrapErrf(err, ErrAliasCreateFailed, "failed to create alias: %s", res)
 	}
-
 	return nil
 }
 

libbeat/idxmgmt/ilm/client_handler_integration_test.go

@@ -136,13 +136,44 @@ func TestESClientHandler_Alias(t *testing.T) {
 		assert.True(t, b)
 	})
 
-	t.Run("second create", func(t *testing.T) {
+	t.Run("create index exists", func(t *testing.T) {
 		alias := makeAlias("esch-alias-2create")
 		h := newESClientHandler(t)
 
 		err := h.CreateAlias(alias)
 		assert.NoError(t, err)
 
+		// Second time around creating the alias, ErrAliasAlreadyExists is returned:
+		// the initial index already exists and the write alias points to it.
+		err = h.CreateAlias(alias)
+		require.Error(t, err)
+		assert.Equal(t, ilm.ErrAliasAlreadyExists, ilm.ErrReason(err))
+
+		b, err := h.HasAlias(alias.Name)
+		assert.NoError(t, err)
+		assert.True(t, b)
+	})
+
+	t.Run("create alias exists", func(t *testing.T) {
+		alias := makeAlias("esch-alias-2create")
+		alias.Pattern = "000001" // no date math, so we get predictable index names
+		h := newESClientHandler(t)
+
+		err := h.CreateAlias(alias)
+		assert.NoError(t, err)
+
+		// Rollover, so write alias points at -000002.
+		es := newRawESClient(t)
+		_, _, err = es.Request("POST", "/"+alias.Name+"/_rollover", "", nil, nil)
+		require.NoError(t, err)
+
+		// Delete -000001, simulating ILM delete.
+		_, _, err = es.Request("DELETE", "/"+alias.Name+"-"+alias.Pattern, "", nil, nil)
+		require.NoError(t, err)
+
+		// Second time around creating the alias, ErrAliasAlreadyExists is returned:
+		// initial index does not exist, but the write alias exists and points to
+		// another index.
 		err = h.CreateAlias(alias)
 		require.Error(t, err)
 		assert.Equal(t, ilm.ErrAliasAlreadyExists, ilm.ErrReason(err))