Merge branch '7.17' into backport/7.17/pr-136419

elastic · Jul 26, 2022 · 11ed9a1 · 11ed9a1
2 parents b5a8918 + af01e8c
commit 11ed9a1
Show file tree

Hide file tree

Showing 43 changed files with 302 additions and 83 deletions.
diff --git a/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc b/docs/developer/plugin/migrating-legacy-plugins-examples.asciidoc
@@ -124,7 +124,7 @@ import { DemoPlugin } from './plugin';
 export const config = {
   schema: schema.object({
     enabled: schema.boolean({ defaultValue: true }),
-  });
+  }),
 }
 
 export const plugin = (initContext: PluginInitializerContext) => new DemoPlugin(initContext);

diff --git a/package.json b/package.json
@@ -581,7 +581,7 @@
     "@types/nock": "^10.0.3",
     "@types/node": "16.11.7",
     "@types/node-fetch": "^2.6.0",
-    "@types/node-forge": "^1.0.2",
+    "@types/node-forge": "^1.0.4",
     "@types/nodemailer": "^6.4.0",
     "@types/normalize-path": "^3.0.0",
     "@types/object-hash": "^1.3.0",
@@ -724,7 +724,7 @@
     "fetch-mock": "^7.3.9",
     "file-loader": "^4.2.0",
     "form-data": "^4.0.0",
-    "geckodriver": "^3.0.1",
+    "geckodriver": "^3.0.2",
     "glob-watcher": "5.0.3",
     "gulp": "4.0.2",
     "gulp-babel": "^8.0.0",
@@ -800,7 +800,7 @@
     "rxjs-marbles": "^5.0.6",
     "sass-loader": "^10.2.0",
     "sass-resources-loader": "^2.0.1",
-    "selenium-webdriver": "^4.3.0",
+    "selenium-webdriver": "^4.3.1",
     "serve-static": "1.14.1",
     "simple-git": "1.116.0",
     "sinon": "^7.4.2",
@@ -835,7 +835,7 @@
     "webpack-merge": "^4.2.2",
     "webpack-sources": "^1.4.1",
     "write-pkg": "^4.0.0",
-    "xml-crypto": "^2.1.3",
+    "xml-crypto": "^2.1.4",
     "xmlbuilder": "13.0.2",
     "yargs": "^15.4.1",
     "zlib": "^1.0.5"

diff --git a/packages/kbn-eslint-plugin-eslint/rules/module_migration.js b/packages/kbn-eslint-plugin-eslint/rules/module_migration.js
@@ -12,7 +12,8 @@ const KIBANA_ROOT = findKibanaRoot();
 
 function checkModuleNameNode(context, mappings, node, desc = 'Imported') {
   const mapping = mappings.find(
-    (mapping) => mapping.from === node.value || node.value.startsWith(`${mapping.from}/`)
+    (mapping) =>
+      mapping.from === node.value || (!mapping.exact && node.value.startsWith(`${mapping.from}/`))
   );
 
   if (!mapping) {
@@ -85,6 +86,10 @@ module.exports = {
             exclude: {
               type: 'array',
             },
+            exact: {
+              type: 'boolean',
+              default: false,
+            },
           },
           anyOf: [
             {

diff --git a/packages/kbn-eslint-plugin-eslint/rules/module_migration.test.js b/packages/kbn-eslint-plugin-eslint/rules/module_migration.test.js
@@ -37,6 +37,21 @@ ruleTester.run('@kbn/eslint/module-migration', rule, {
         ],
       ],
     },
+
+    {
+      code: dedent`
+        import "foo/bar"
+      `,
+      options: [
+        [
+          {
+            from: 'foo',
+            to: 'bar',
+            exact: true,
+          },
+        ],
+      ],
+    },
   ],
 
   invalid: [
@@ -148,5 +163,31 @@ ruleTester.run('@kbn/eslint/module-migration', rule, {
         import '../../common/foo'
       `,
     },
+
+    {
+      code: dedent`
+        import 'foo'
+        import 'foo/bar'
+      `,
+      options: [
+        [
+          {
+            from: 'foo',
+            to: 'bar',
+            exact: true,
+          },
+        ],
+      ],
+      errors: [
+        {
+          line: 1,
+          message: 'Imported module "foo" should be "bar"',
+        },
+      ],
+      output: dedent`
+        import 'bar'
+        import 'foo/bar'
+      `,
+    },
   ],
 });
diff --git a/packages/kbn-pm/src/utils/watch.test.ts b/packages/kbn-pm/src/utils/watch.test.ts
@@ -9,7 +9,8 @@
 import { EventEmitter } from 'events';
 import { waitUntilWatchIsReady } from './watch';
 
-describe('#waitUntilWatchIsReady', () => {
+// FLAKY: https://github.com/elastic/kibana/issues/135099
+describe.skip('#waitUntilWatchIsReady', () => {
   let buildOutputStream: EventEmitter;
   let completionHintPromise: Promise<string>;
   beforeEach(() => {

diff --git a/src/plugins/data/common/search/tabify/tabify_docs.ts b/src/plugins/data/common/search/tabify/tabify_docs.ts
@@ -94,7 +94,7 @@ export function flattenHit(hit: Hit, indexPattern?: IndexPattern, params?: Tabif
         continue;
       }
 
-      const hasValidMapping = field?.type !== 'conflict';
+      const hasValidMapping = field && field.type !== 'conflict';
       const isValue = !isPlainObject(val);
 
       if (hasValidMapping || isValue) {

diff --git a/test/api_integration/apis/saved_objects_management/find.ts b/test/api_integration/apis/saved_objects_management/find.ts
@@ -15,7 +15,8 @@ export default function ({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const kibanaServer = getService('kibanaServer');
 
-  describe('find', () => {
+  // Failing: See https://github.com/elastic/kibana/issues/132855
+  describe.skip('find', () => {
     let KIBANA_VERSION: string;
 
     before(async () => {

diff --git a/test/plugin_functional/test_suites/saved_objects_hidden_type/find.ts b/test/plugin_functional/test_suites/saved_objects_hidden_type/find.ts
@@ -13,7 +13,8 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
 
-  describe('find', () => {
+  // Failing: See https://github.com/elastic/kibana/issues/131830
+  describe.skip('find', () => {
     before(() =>
       esArchiver.load(
         'test/functional/fixtures/es_archiver/saved_objects_management/hidden_saved_objects'

diff --git a/x-pack/plugins/apm/common/fleet.ts b/x-pack/plugins/apm/common/fleet.ts
@@ -7,6 +7,6 @@
 
 export const POLICY_ELASTIC_AGENT_ON_CLOUD = 'policy-elastic-agent-on-cloud';
 
-export const SUPPORTED_APM_PACKAGE_VERSION = '7.16.0';
+export const SUPPORTED_APM_PACKAGE_VERSION = '7.17.0';
 
 export const ELASTIC_CLOUD_APM_AGENT_POLICY_ID = 'elastic-cloud-apm';
diff --git a/...ansaction_details/waterfall_with_summary/waterfall_container/Waterfall/waterfall_item.tsx b/...ansaction_details/waterfall_with_summary/waterfall_container/Waterfall/waterfall_item.tsx
@@ -238,11 +238,16 @@ function RelatedErrors({
   const theme = useTheme();
   const { query } = useApmParams('/services/{serviceName}/transactions/view');
 
+  let kuery = `${TRACE_ID} : "${item.doc.trace.id}"`;
+  if (item.doc.transaction?.id) {
+    kuery += ` and ${TRANSACTION_ID} : "${item.doc.transaction?.id}"`;
+  }
+
   const href = apmRouter.link(`/services/{serviceName}/errors`, {
     path: { serviceName: item.doc.service.name },
     query: {
       ...query,
-      kuery: `${TRACE_ID} : "${item.doc.trace.id}" and ${TRANSACTION_ID} : "${item.doc.transaction?.id}"`,
+      kuery,
     },
   });
 

diff --git a/x-pack/plugins/event_log/server/es/context.mock.ts b/x-pack/plugins/event_log/server/es/context.mock.ts
@@ -12,6 +12,8 @@ import { namesMock } from './names.mock';
 import { IClusterClientAdapter } from './cluster_client_adapter';
 import { clusterClientAdapterMock } from './cluster_client_adapter.mock';
 
+export const MOCK_RETRY_DELAY = 20;
+
 const createContextMock = () => {
   const mock: jest.Mocked<EsContext> & {
     esAdapter: jest.Mocked<IClusterClientAdapter>;
@@ -23,6 +25,7 @@ const createContextMock = () => {
     waitTillReady: jest.fn(async () => true),
     esAdapter: clusterClientAdapterMock.create(),
     initialized: true,
+    retryDelay: MOCK_RETRY_DELAY,
   };
   return mock;
 };

diff --git a/x-pack/plugins/event_log/server/es/context.ts b/x-pack/plugins/event_log/server/es/context.ts
@@ -12,14 +12,17 @@ import { initializeEs } from './init';
 import { ClusterClientAdapter, IClusterClientAdapter } from './cluster_client_adapter';
 import { createReadySignal, ReadySignal } from '../lib/ready_signal';
 
+export const RETRY_DELAY = 2000;
+
 export interface EsContext {
-  logger: Logger;
-  esNames: EsNames;
-  esAdapter: IClusterClientAdapter;
+  readonly logger: Logger;
+  readonly esNames: EsNames;
+  readonly esAdapter: IClusterClientAdapter;
   initialize(): void;
   shutdown(): Promise<void>;
   waitTillReady(): Promise<boolean>;
-  initialized: boolean;
+  readonly initialized: boolean;
+  readonly retryDelay: number;
 }
 
 export interface EsError {
@@ -44,12 +47,14 @@ class EsContextImpl implements EsContext {
   public esAdapter: IClusterClientAdapter;
   private readonly readySignal: ReadySignal<boolean>;
   public initialized: boolean;
+  public readonly retryDelay: number;
 
   constructor(params: EsContextCtorParams) {
     this.logger = params.logger;
     this.esNames = getEsNames(params.indexNameRoot, params.kibanaVersion);
     this.readySignal = createReadySignal();
     this.initialized = false;
+    this.retryDelay = RETRY_DELAY;
     this.esAdapter = new ClusterClientAdapter({
       logger: params.logger,
       elasticsearchClientPromise: params.elasticsearchClientPromise,

diff --git a/x-pack/plugins/event_log/server/es/init.test.ts b/x-pack/plugins/event_log/server/es/init.test.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { contextMock } from './context.mock';
+import { contextMock, MOCK_RETRY_DELAY } from './context.mock';
 import { initializeEs, parseIndexAliases } from './init';
 
 describe('initializeEs', () => {
@@ -454,3 +454,91 @@ describe('parseIndexAliases', () => {
     ]);
   });
 });
+
+describe('retries', () => {
+  let esContext = contextMock.create();
+  // set up context APIs to return defaults indicating already created
+  beforeEach(() => {
+    esContext = contextMock.create();
+    esContext.esAdapter.getExistingLegacyIndexTemplates.mockResolvedValue({});
+    esContext.esAdapter.getExistingIndices.mockResolvedValue({});
+    esContext.esAdapter.getExistingIndexAliases.mockResolvedValue({});
+    esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(true);
+    esContext.esAdapter.doesIndexTemplateExist.mockResolvedValue(true);
+    esContext.esAdapter.doesAliasExist.mockResolvedValue(true);
+  });
+
+  test('createIlmPolicyIfNotExists with 1 retry', async () => {
+    esContext.esAdapter.doesIlmPolicyExist.mockRejectedValueOnce(new Error('retry 1'));
+
+    const timeStart = Date.now();
+    await initializeEs(esContext);
+    const timeElapsed = Date.now() - timeStart;
+
+    expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY);
+
+    expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1);
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(2);
+    expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(1);
+    expect(esContext.esAdapter.doesAliasExist).toHaveBeenCalledTimes(1);
+
+    const prefix = `eventLog initialization operation failed and will be retried: createIlmPolicyIfNotExists`;
+    expect(esContext.logger.warn).toHaveBeenCalledTimes(1);
+    expect(esContext.logger.warn).toHaveBeenCalledWith(`${prefix}; 4 more times; error: retry 1`);
+  });
+
+  test('createIndexTemplateIfNotExists with 2 retries', async () => {
+    esContext.esAdapter.doesIndexTemplateExist.mockRejectedValueOnce(new Error('retry 2a'));
+    esContext.esAdapter.doesIndexTemplateExist.mockRejectedValueOnce(new Error('retry 2b'));
+
+    const timeStart = Date.now();
+    await initializeEs(esContext);
+    const timeElapsed = Date.now() - timeStart;
+
+    expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY * (1 + 2));
+
+    expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1);
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(1);
+    expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(3);
+    expect(esContext.esAdapter.doesAliasExist).toHaveBeenCalledTimes(1);
+
+    const prefix = `eventLog initialization operation failed and will be retried: createIndexTemplateIfNotExists`;
+    expect(esContext.logger.warn).toHaveBeenCalledTimes(2);
+    expect(esContext.logger.warn).toHaveBeenCalledWith(`${prefix}; 4 more times; error: retry 2a`);
+    expect(esContext.logger.warn).toHaveBeenCalledWith(`${prefix}; 3 more times; error: retry 2b`);
+  });
+
+  test('createInitialIndexIfNotExists', async () => {
+    esContext.esAdapter.doesAliasExist.mockRejectedValueOnce(new Error('retry 5a'));
+    esContext.esAdapter.doesAliasExist.mockRejectedValueOnce(new Error('retry 5b'));
+    esContext.esAdapter.doesAliasExist.mockRejectedValueOnce(new Error('retry 5c'));
+    esContext.esAdapter.doesAliasExist.mockRejectedValueOnce(new Error('retry 5d'));
+    esContext.esAdapter.doesAliasExist.mockRejectedValueOnce(new Error('retry 5e'));
+    // make sure it only tries 5 times - this one should not be reported
+    esContext.esAdapter.doesAliasExist.mockRejectedValueOnce(new Error('retry 5f'));
+
+    const timeStart = Date.now();
+    await initializeEs(esContext);
+    const timeElapsed = Date.now() - timeStart;
+
+    expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY * (1 + 2 + 4 + 8));
+
+    expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1);
+    expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(1);
+    expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(1);
+    expect(esContext.esAdapter.doesAliasExist).toHaveBeenCalledTimes(5);
+
+    const prefix = `eventLog initialization operation failed and will be retried: createInitialIndexIfNotExists`;
+    expect(esContext.logger.warn).toHaveBeenCalledTimes(5);
+    expect(esContext.logger.warn).toHaveBeenCalledWith(`${prefix}; 4 more times; error: retry 5a`);
+    expect(esContext.logger.warn).toHaveBeenCalledWith(`${prefix}; 3 more times; error: retry 5b`);
+    expect(esContext.logger.warn).toHaveBeenCalledWith(`${prefix}; 2 more times; error: retry 5c`);
+    expect(esContext.logger.warn).toHaveBeenCalledWith(`${prefix}; 1 more times; error: retry 5d`);
+    expect(esContext.logger.warn).toHaveBeenCalledWith(`${prefix}; 0 more times; error: retry 5e`);
+
+    expect(esContext.logger.error).toHaveBeenCalledTimes(1);
+    expect(esContext.logger.error).toHaveBeenCalledWith(
+      `error initializing elasticsearch resources: retry 5e`
+    );
+  });
+});
diff --git a/x-pack/plugins/event_log/server/es/init.ts b/x-pack/plugins/event_log/server/es/init.ts
@@ -9,9 +9,12 @@ import { asyncForEach } from '@kbn/std';
 import { IndicesIndexStatePrefixedSettings } from '@elastic/elasticsearch/api/types';
 import { estypes } from '@elastic/elasticsearch';
 import { groupBy } from 'lodash';
+import pRetry, { FailedAttemptError } from 'p-retry';
 import { getIlmPolicy, getIndexTemplate } from './documents';
 import { EsContext } from './context';
 
+const MAX_RETRY_DELAY = 30000;
+
 export async function initializeEs(esContext: EsContext): Promise<boolean> {
   esContext.logger.debug('initializing elasticsearch resources starting');
 
@@ -29,10 +32,33 @@ export async function initializeEs(esContext: EsContext): Promise<boolean> {
 async function initializeEsResources(esContext: EsContext) {
   const steps = new EsInitializationSteps(esContext);
 
-  await steps.setExistingAssetsToHidden();
-  await steps.createIlmPolicyIfNotExists();
-  await steps.createIndexTemplateIfNotExists();
-  await steps.createInitialIndexIfNotExists();
+  // today, setExistingAssetsToHidden() never throws, but just in case ...
+  await retry(steps.setExistingAssetsToHidden);
+  await retry(steps.createIlmPolicyIfNotExists);
+  await retry(steps.createIndexTemplateIfNotExists);
+  await retry(steps.createInitialIndexIfNotExists);
+
+  async function retry(stepMethod: () => Promise<void>): Promise<void> {
+    // call the step method with retry options via p-retry
+    await pRetry(() => stepMethod.call(steps), getRetryOptions(esContext, stepMethod.name));
+  }
+}
+
+function getRetryOptions(esContext: EsContext, operation: string) {
+  const logger = esContext.logger;
+  // should retry on the order of 2s, 4s, 8s, 16s
+  // see: https://github.com/tim-kos/node-retry#retryoperationoptions
+  return {
+    minTimeout: esContext.retryDelay,
+    maxTimeout: MAX_RETRY_DELAY,
+    retries: 4,
+    factor: 2,
+    randomize: true,
+    onFailedAttempt: (err: FailedAttemptError) => {
+      const message = `eventLog initialization operation failed and will be retried: ${operation}; ${err.retriesLeft} more times; error: ${err.message}`;
+      logger.warn(message);
+    },
+  };
 }
 
 export interface ParsedIndexAlias extends estypes.IndicesAliasDefinition {