Migrations: Don't auto-create temp index (elastic#158182)

Try to fix
elastic#156117 (comment)

Fixes a race condition that could cause intermittent upgrade migration
failures when Kibana connects to a single node Elasticsearch cluster.

Delete any items that are not applicable to this PR.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))
- [ ] Any UI touched in this PR does not create any new axe failures
(run axe in browser:
[FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/),
[Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This renders correctly on smaller devices using a responsive
layout. (You can test this [in your
browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [ ] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)

Delete this section if it is not applicable to this PR.

Before closing this PR, invite QA, stakeholders, and other developers to
identify risks that should be tested prior to the change/feature
release.

When forming the risk matrix, consider some of the following examples
and how they may potentially impact the change:

| Risk | Probability | Severity | Mitigation/Notes |

|---------------------------|-------------|----------|-------------------------|
| Multiple Spaces—unexpected behavior in non-default Kibana Space.
| Low | High | Integration tests will verify that all features are still
supported in non-default Kibana Space and when user switches between
spaces. |
| Multiple nodes—Elasticsearch polling might have race conditions
when multiple Kibana nodes are polling for the same tasks. | High | Low
| Tasks are idempotent, so executing them multiple times will not result
in logical error, but will degrade performance. To test for this case we
add plenty of unit tests around this logic and document manual testing
procedure. |
| Code should gracefully handle cases when feature X or plugin Y are
disabled. | Medium | High | Unit tests will verify that any feature flag
or plugin combination still results in our service operational. |
| [See more potential risk
examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) |

- [ ] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

(cherry picked from commit 8e7e263)

src/core/server/saved_objects/migrationsv2/actions/bulk_overwrite_transformed_documents.ts

@@ -50,6 +50,12 @@ export interface BulkOverwriteTransformedDocumentsParams {
   index: string;
   transformedDocs: SavedObjectsRawDoc[];
   refresh?: estypes.Refresh;
+  /**
+   * If true, we prevent Elasticsearch from auto-creating the index if it
+   * doesn't exist. We use the ES paramater require_alias: true so `index`
+   * must be an alias, otherwise the bulk index will fail.
+   */
+  useAliasToPreventAutoCreate?: boolean;
 }
 
 /**
@@ -62,6 +68,7 @@ export const bulkOverwriteTransformedDocuments =
     index,
     transformedDocs,
     refresh = false,
+    useAliasToPreventAutoCreate = false,
   }: BulkOverwriteTransformedDocumentsParams): TaskEither.TaskEither<
     | RetryableEsClientError
     | TargetIndexHadWriteBlock
@@ -83,7 +90,7 @@ export const bulkOverwriteTransformedDocuments =
         // mappings. Such tampering could lead to many other problems and is
         // probably unlikely so for now we'll accept this risk and wait till
         // system indices puts in place a hard control.
-        require_alias: false,
+        require_alias: useAliasToPreventAutoCreate,
         wait_for_active_shards: WAIT_FOR_ALL_SHARDS_TO_BE_ACTIVE,
         refresh,
         filter_path: ['items.*.error'],

src/core/server/saved_objects/migrationsv2/actions/integration_tests/actions.test.ts

@@ -70,6 +70,7 @@ describe('migration actions', () => {
     await createIndex({
       client,
       indexName: 'existing_index_with_docs',
+      aliases: ['existing_index_with_docs_alias'],
       mappings: {
         dynamic: true,
         properties: {},
@@ -133,7 +134,9 @@ describe('migration actions', () => {
       expect(res.right).toEqual(
         expect.objectContaining({
           existing_index_with_docs: {
-            aliases: {},
+            aliases: {
+              existing_index_with_docs_alias: {},
+            },
             mappings: expect.anything(),
             settings: expect.anything(),
           },
@@ -1519,6 +1522,30 @@ describe('migration actions', () => {
                 }
               `);
     });
+    it('resolves left index_not_found_exception if the index does not exist and useAliasToPreventAutoCreate=true', async () => {
+      const newDocs = [
+        { _source: { title: 'doc 5' } },
+        { _source: { title: 'doc 6' } },
+        { _source: { title: 'doc 7' } },
+      ] as unknown as SavedObjectsRawDoc[];
+      await expect(
+        bulkOverwriteTransformedDocuments({
+          client,
+          index: 'existing_index_with_docs_alias_that_does_not_exist',
+          useAliasToPreventAutoCreate: true,
+          transformedDocs: newDocs,
+          refresh: 'wait_for',
+        })()
+      ).resolves.toMatchInlineSnapshot(`
+          Object {
+            "_tag": "Left",
+            "left": Object {
+              "index": "existing_index_with_docs_alias_that_does_not_exist",
+              "type": "index_not_found_exception",
+            },
+          }
+      `);
+    });
     it('resolves left target_index_had_write_block if there are write_block errors', async () => {
       const newDocs = [
         { _source: { title: 'doc 5' } },

src/core/server/saved_objects/migrationsv2/initial_state.test.ts

@@ -72,6 +72,7 @@ describe('createInitialState', () => {
         },
       },
       tempIndex: '.kibana_task_manager_8.1.0_reindex_temp',
+      tempIndexAlias: '.kibana_task_manager_8.1.0_reindex_temp_alias',
       tempIndexMappings: {
         dynamic: false,
         properties: {

src/core/server/saved_objects/migrationsv2/initial_state.ts

@@ -73,6 +73,7 @@ export const createInitialState = ({
     versionAlias: `${indexPrefix}_${kibanaVersion}`,
     versionIndex: `${indexPrefix}_${kibanaVersion}_001`,
     tempIndex: `${indexPrefix}_${kibanaVersion}_reindex_temp`,
+    tempIndexAlias: `${indexPrefix}_${kibanaVersion}_reindex_temp_alias`,
     kibanaVersion,
     preMigrationScript: Option.fromNullable(preMigrationScript),
     targetIndexMappings: targetMappings,

src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts

@@ -90,7 +90,7 @@ describe('migration v2', function () {
     await root.preboot();
     await root.setup();
     await expect(root.start()).rejects.toMatchInlineSnapshot(
-      `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715277 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]`
+      `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715237 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]`
     );
 
     await retryAsync(
@@ -103,7 +103,7 @@ describe('migration v2', function () {
         expect(
           records.find((rec) =>
             rec.message.startsWith(
-              `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715277 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`
+              `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715237 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`
             )
           )
         ).toBeDefined();

src/core/server/saved_objects/migrationsv2/model/model.test.ts

@@ -81,6 +81,7 @@ describe('migrations v2 model', () => {
     versionAlias: '.kibana_7.11.0',
     versionIndex: '.kibana_7.11.0_001',
     tempIndex: '.kibana_7.11.0_reindex_temp',
+    tempIndexAlias: '.kibana_7.11.0_reindex_temp_alias',
     unusedTypesQuery: {
       bool: {
         must_not: [

src/core/server/saved_objects/migrationsv2/model/model.ts

@@ -504,7 +504,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
       if (stateP.corruptDocumentIds.length === 0 && stateP.transformErrors.length === 0) {
         const batches = createBatches(
           res.right.processedDocs,
-          stateP.tempIndex,
+          stateP.tempIndexAlias,
           stateP.maxBatchSizeBytes
         );
         if (Either.isRight(batches)) {

src/core/server/saved_objects/migrationsv2/next.ts

@@ -87,6 +87,7 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra
       Actions.createIndex({
         client,
         indexName: state.tempIndex,
+        aliases: [state.tempIndexAlias],
         mappings: state.tempIndexMappings,
       }),
     REINDEX_SOURCE_TO_TEMP_OPEN_PIT: (state: ReindexSourceToTempOpenPit) =>
@@ -110,8 +111,14 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra
     REINDEX_SOURCE_TO_TEMP_INDEX_BULK: (state: ReindexSourceToTempIndexBulk) =>
       Actions.bulkOverwriteTransformedDocuments({
         client,
-        index: state.tempIndex,
+        index: state.tempIndexAlias,
         transformedDocs: state.transformedDocBatches[state.currentBatch],
+        /*
+         * Since other nodes can delete the temp index while we're busy writing
+         * to it, we use the alias to prevent the auto-creation of the index if
+         * it doesn't exist.
+         */
+        useAliasToPreventAutoCreate: true,
         /**
          * Since we don't run a search against the target index, we disable "refresh" to speed up
          * the migration process.

src/core/server/saved_objects/migrationsv2/types.ts

@@ -117,10 +117,16 @@ export interface BaseState extends ControlState {
    */
   readonly versionIndex: string;
   /**
-   * An alias on the target index used as part of an "reindex block" that
-   * prevents lost deletes e.g. `.kibana_7.11.0_reindex`.
+   * A temporary index used as part of an "reindex block" that
+   * prevents lost deletes e.g. `.kibana_7.11.0_reindex_temp`.
    */
   readonly tempIndex: string;
+  /**
+   * An alias to the tempIndex used to prevent ES from auto-creating the temp
+   * index if one node deletes it while another writes to it
+   * e.g. `.kibana_7.11.0_reindex_temp_alias`.
+   */
+  readonly tempIndexAlias: string;
   /**
    * When reindexing we use a source query to exclude saved objects types which
    * are no longer used. These saved objects will still be kept in the outdated