[DO NOT MERGE] [Monitoring] Code to generate accurate mappings and update es_archives

packages/kbn-es-archiver/src/actions/rebuild_all.ts

@@ -23,7 +23,7 @@ import { Readable, Writable } from 'stream';
 import { fromNode } from 'bluebird';
 import { ToolingLog } from '@kbn/dev-utils';
 
-import { createPromiseFromStreams } from '../lib/streams';
+import { createPromiseFromStreams, createMapStream } from '../lib/streams';
 import {
   prioritizeMappings,
   readDirectory,
@@ -37,6 +37,11 @@ async function isDirectory(path: string): Promise<boolean> {
   return stats.isDirectory();
 }
 
+const esMappings = require('/Users/chris/Desktop/mappings/monitoring-es-*.json');
+const kibanaMappings = require('/Users/chris/Desktop/mappings/monitoring-kibana-*.json');
+const logstashMappings = require('/Users/chris/Desktop/mappings/monitoring-logstash-*.json');
+const beatsMappings = require('/Users/chris/Desktop/mappings/monitoring-beats-*.json');
+
 export async function rebuildAllAction({
   dataDir,
   log,
@@ -46,7 +51,10 @@ export async function rebuildAllAction({
   log: ToolingLog;
   rootDir?: string;
 }) {
-  const childNames = prioritizeMappings(await readDirectory(dataDir));
+  let childNames = prioritizeMappings(await readDirectory(dataDir));
+  if (dataDir === '/Users/chris/dev/repos/kibana/x-pack/test/functional/es_archives') {
+    childNames = ['monitoring']
+  }
   for (const childName of childNames) {
     const childPath = resolve(dataDir, childName);
 
@@ -67,7 +75,34 @@ export async function rebuildAllAction({
     await createPromiseFromStreams([
       createReadStream(childPath) as Readable,
       ...createParseArchiveStreams({ gzip }),
+      createMapStream<any>((item) => {
+        if (
+          item &&
+          item.type === 'index' &&
+          item.value.index &&
+          item.value.index.includes('monitoring')
+        ) {
+          let mapping = null;
+          if (item.value.index.includes('-es-')) {
+            mapping = esMappings;
+          } else if (item.value.index.includes('-kibana-')) {
+            mapping = kibanaMappings;
+          } else if (item.value.index.includes('-logstash-')) {
+            mapping = logstashMappings;
+          } else if (item.value.index.includes('-beats-')) {
+            mapping = beatsMappings;
+          }
+          if (mapping) {
+            log.info(`Updated for ${item.value.index}`);
+            item.value.mappings.properties = mapping.properties;
+          } else {
+            log.info(`MISSING for ${item.value.index}`);
+          }
+        }
+        return item;
+      }),
       ...createFormatArchiveStreams({ gzip }),
+
       createWriteStream(tempFile),
     ] as [Readable, ...Writable[]]);
 

x-pack/plugins/monitoring/server/alerts/base_alert.ts

@@ -215,7 +215,7 @@ export class BaseAlert {
     const availableCcs = this.config.ui.ccs.enabled ? await fetchAvailableCcs(callCluster) : [];
     // Support CCS use cases by querying to find available remote clusters
     // and then adding those to the index pattern we are searching against
-    let esIndexPattern = INDEX_PATTERN_ELASTICSEARCH;
+    let esIndexPattern = `${INDEX_PATTERN_ELASTICSEARCH},${this.config.ui.metricbeat.index}`;
     if (availableCcs) {
       esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs);
     }

x-pack/plugins/monitoring/server/alerts/cluster_health_alert.test.ts

@@ -66,7 +66,11 @@ describe('ClusterHealthAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-* ' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.test.ts

@@ -70,7 +70,11 @@ describe('CpuUsageAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-* ' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.ts

@@ -137,7 +137,7 @@ export class CpuUsageAlert extends BaseAlert {
     uiSettings: IUiSettingsClient,
     availableCcs: string[]
   ): Promise<AlertData[]> {
-    let esIndexPattern = INDEX_PATTERN_ELASTICSEARCH;
+    let esIndexPattern = `${INDEX_PATTERN_ELASTICSEARCH},${this.config.ui.metricbeat.index}`;
     if (availableCcs) {
       esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs);
     }

x-pack/plugins/monitoring/server/alerts/elasticsearch_version_mismatch_alert.test.ts

@@ -69,7 +69,11 @@ describe('ElasticsearchVersionMismatchAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-* ' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/alerts/kibana_version_mismatch_alert.test.ts

@@ -72,7 +72,11 @@ describe('KibanaVersionMismatchAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-* ' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/alerts/license_expiration_alert.test.ts

@@ -76,7 +76,11 @@ describe('LicenseExpirationAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-* ' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/alerts/logstash_version_mismatch_alert.test.ts

@@ -69,7 +69,11 @@ describe('LogstashVersionMismatchAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-* ' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/alerts/nodes_changed_alert.test.ts

@@ -82,7 +82,11 @@ describe('NodesChangedAlert', () => {
     });
     const monitoringCluster = null;
     const config = {
-      ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
+      ui: {
+        ccs: { enabled: true },
+        container: { elasticsearch: { enabled: false } },
+        metricbeat: { index: 'metricbeat-* ' },
+      },
     };
     const kibanaUrl = 'http://localhost:5601';
 

x-pack/plugins/monitoring/server/config.test.ts

@@ -86,6 +86,9 @@ describe('config schema', () => {
             "index": "filebeat-*",
           },
           "max_bucket_size": 10000,
+          "metricbeat": Object {
+            "index": "metricbeat-*",
+          },
           "min_interval_seconds": 10,
           "show_license_expiration": true,
         },

x-pack/plugins/monitoring/server/config.ts

@@ -29,6 +29,9 @@ export const configSchema = schema.object({
     logs: schema.object({
       index: schema.string({ defaultValue: 'filebeat-*' }),
     }),
+    metricbeat: schema.object({
+      index: schema.string({ defaultValue: 'metricbeat-*' }),
+    }),
     max_bucket_size: schema.number({ defaultValue: 10000 }),
     elasticsearch: monitoringElasticsearchConfigSchema,
     container: schema.object({

x-pack/plugins/monitoring/server/lib/cluster/get_index_patterns.js

@@ -15,7 +15,11 @@ import {
 
 export function getIndexPatterns(server, additionalPatterns = {}, ccs = '*') {
   const config = server.config();
-  const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs);
+  const esIndexPattern = prefixIndexPattern(
+    config,
+    `${INDEX_PATTERN_ELASTICSEARCH},${config.get('monitoring.ui.metricbeat.index')}`,
+    ccs
+  );
   const kbnIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_KIBANA, ccs);
   const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs);
   const beatsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_BEATS, ccs);

x-pack/plugins/monitoring/server/lib/create_query.js

@@ -57,7 +57,7 @@ export function createQuery(options) {
 
   let typeFilter;
   if (type) {
-    typeFilter = { term: { type } };
+    typeFilter = { bool: { should: [{ term: { type } }, { term: { 'metricset.name': type } }] } };
   }
 
   let clusterUuidFilter;

x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_node_summary.js

@@ -17,15 +17,20 @@ export function handleResponse(clusterState, shardStats, nodeUuid) {
   return (response) => {
     let nodeSummary = {};
     const nodeStatsHits = get(response, 'hits.hits', []);
-    const nodes = nodeStatsHits.map((hit) => hit._source.source_node); // using [0] value because query results are sorted desc per timestamp
+    const nodes = nodeStatsHits.map(
+      (hit) => hit._source.source_node || hit._source.elasticsearch.node
+    ); // using [0] value because query results are sorted desc per timestamp
     const node = nodes[0] || getDefaultNodeFromId(nodeUuid);
-    const sourceStats = get(response, 'hits.hits[0]._source.node_stats');
+    const sourceStats =
+      get(response, 'hits.hits[0]._source.node_stats') ||
+      get(response, 'hits.hits[0]._source.elasticsearch.node.stats');
     const clusterNode = get(clusterState, ['nodes', nodeUuid]);
     const stats = {
       resolver: nodeUuid,
       node_ids: nodes.map((node) => node.uuid),
       attributes: node.attributes,
-      transport_address: node.transport_address,
+      transport_address:
+        node.transport_address || get(response, 'hits.hits[0]._source.service.address'),
       name: node.name,
       type: node.type,
     };
@@ -45,10 +50,17 @@ export function handleResponse(clusterState, shardStats, nodeUuid) {
         totalShards: _shardStats.shardCount,
         indexCount: _shardStats.indexCount,
         documents: get(sourceStats, 'indices.docs.count'),
-        dataSize: get(sourceStats, 'indices.store.size_in_bytes'),
-        freeSpace: get(sourceStats, 'fs.total.available_in_bytes'),
-        totalSpace: get(sourceStats, 'fs.total.total_in_bytes'),
-        usedHeap: get(sourceStats, 'jvm.mem.heap_used_percent'),
+        dataSize:
+          get(sourceStats, 'indices.store.size_in_bytes') ||
+          get(sourceStats, 'indices.store.size.bytes'),
+        freeSpace:
+          get(sourceStats, 'fs.total.available_in_bytes') ||
+          get(sourceStats, 'fs.summary.available.bytes'),
+        totalSpace:
+          get(sourceStats, 'fs.total.total_in_bytes') || get(sourceStats, 'fs.summary.total.bytes'),
+        usedHeap:
+          get(sourceStats, 'jvm.mem.heap_used_percent') ||
+          get(sourceStats, 'jvm.mem.heap.used.pct'),
         status: i18n.translate('xpack.monitoring.es.nodes.onlineStatusLabel', {
           defaultMessage: 'Online',
         }),

x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/get_nodes.js

@@ -96,6 +96,7 @@ export async function getNodes(req, esIndexPattern, pageOfNodes, clusterStats, n
     },
     filterPath: [
       'hits.hits._source.source_node',
+      'hits.hits._source.elasticsearch.node',
       'aggregations.nodes.buckets.key',
       ...LISTING_METRICS_PATHS,
     ],

x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_nodes/map_nodes_info.js

@@ -17,25 +17,29 @@ export function mapNodesInfo(nodeHits, clusterStats, nodesShardCount) {
   const clusterState = get(clusterStats, 'cluster_state', { nodes: {} });
 
   return nodeHits.reduce((prev, node) => {
-    const sourceNode = get(node, '_source.source_node');
+    const sourceNode = get(node, '_source.source_node') || get(node, '_source.elasticsearch.node');
 
     const calculatedNodeType = calculateNodeType(sourceNode, get(clusterState, 'master_node'));
     const { nodeType, nodeTypeLabel, nodeTypeClass } = getNodeTypeClassLabel(
       sourceNode,
       calculatedNodeType
     );
-    const isOnline = !isUndefined(get(clusterState, ['nodes', sourceNode.uuid]));
+    const isOnline = !isUndefined(get(clusterState, ['nodes', sourceNode.uuid || sourceNode.id]));
 
     return {
       ...prev,
-      [sourceNode.uuid]: {
+      [sourceNode.uuid || sourceNode.id]: {
         name: sourceNode.name,
         transport_address: sourceNode.transport_address,
         type: nodeType,
         isOnline,
         nodeTypeLabel: nodeTypeLabel,
         nodeTypeClass: nodeTypeClass,
-        shardCount: get(nodesShardCount, `nodes[${sourceNode.uuid}].shardCount`, 0),
+        shardCount: get(
+          nodesShardCount,
+          `nodes[${sourceNode.uuid || sourceNode.id}].shardCount`,
+          0
+        ),
       },
     };
   }, {});

x-pack/plugins/monitoring/server/lib/setup/collection/__test__/get_collection_status.test.js

@@ -31,7 +31,12 @@ const mockReq = (
       },
       config() {
         return {
-          get: sinon.stub().withArgs('server.uuid').returns('kibana-1234'),
+          get: sinon
+            .stub()
+            .withArgs('server.uuid')
+            .returns('kibana-1234')
+            .withArgs('monitoring.ui.metricbeat.index')
+            .returns('metricbeat-*'),
         };
       },
       usage: {
@@ -191,6 +196,51 @@ describe('getCollectionStatus', () => {
     expect(result.elasticsearch.byUuid.es_1.isFullyMigrated).to.be(true);
   });
 
+  it('should handle some stack products as fully migrated under metricbeat-* indices', async () => {
+    const req = mockReq({
+      aggregations: {
+        indices: {
+          buckets: [
+            {
+              key: 'metricbeat-8.0.0',
+              es_uuids: { buckets: [{ key: 'es_1' }] },
+            },
+            {
+              key: '.monitoring-kibana-7-mb-2019',
+              kibana_uuids: { buckets: [{ key: 'kibana_1' }] },
+            },
+            {
+              key: '.monitoring-beats-7-2019',
+              beats_uuids: { buckets: [{ key: 'beats_1' }] },
+            },
+            {
+              key: '.monitoring-logstash-7-2019',
+              logstash_uuids: { buckets: [{ key: 'logstash_1' }] },
+            },
+          ],
+        },
+      },
+    });
+
+    const result = await getCollectionStatus(req, getIndexPatterns(req.server));
+
+    expect(result.kibana.totalUniqueInstanceCount).to.be(1);
+    expect(result.kibana.totalUniqueFullyMigratedCount).to.be(1);
+    expect(result.kibana.byUuid.kibana_1.isFullyMigrated).to.be(true);
+
+    expect(result.beats.totalUniqueInstanceCount).to.be(1);
+    expect(result.beats.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.beats.byUuid.beats_1.isInternalCollector).to.be(true);
+
+    expect(result.logstash.totalUniqueInstanceCount).to.be(1);
+    expect(result.logstash.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.logstash.byUuid.logstash_1.isInternalCollector).to.be(true);
+
+    expect(result.elasticsearch.totalUniqueInstanceCount).to.be(1);
+    expect(result.elasticsearch.totalUniqueFullyMigratedCount).to.be(1);
+    expect(result.elasticsearch.byUuid.es_1.isFullyMigrated).to.be(true);
+  });
+
   it('should handle some stack products as partially migrated', async () => {
     const req = mockReq({
       aggregations: {