-
Notifications
You must be signed in to change notification settings - Fork 8.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Monitoring] Fetch shard data more efficiently (#54028)
* For the nodes listing page, do not fetch shard data for indices * Optimize our shard queries for the index and node listing pages * This change isn't necessary * Rename file and function * Use optimized query for ml jobs and es overview * Apply to node/index detail page, and more renaming * Unnecessary change * Fix tests * Add basic tests Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
- Loading branch information
1 parent
921e933
commit 00f7cab
Showing
20 changed files
with
1,922 additions
and
433 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
93 changes: 93 additions & 0 deletions
93
.../plugins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| import { get } from 'lodash'; | ||
| import { checkParam } from '../../error_missing_required'; | ||
| import { createQuery } from '../../create_query'; | ||
| import { ElasticsearchMetric } from '../../metrics'; | ||
| import { calculateIndicesTotals } from './calculate_shard_stat_indices_totals'; | ||
|
|
||
| async function getUnassignedShardData(req, esIndexPattern, cluster) { | ||
| const config = req.server.config(); | ||
| const maxBucketSize = config.get('xpack.monitoring.max_bucket_size'); | ||
| const metric = ElasticsearchMetric.getMetricFields(); | ||
|
|
||
| const params = { | ||
| index: esIndexPattern, | ||
| size: 0, | ||
| ignoreUnavailable: true, | ||
| body: { | ||
| sort: { timestamp: { order: 'desc' } }, | ||
| query: createQuery({ | ||
| type: 'shards', | ||
| clusterUuid: cluster.cluster_uuid, | ||
| metric, | ||
| filters: [{ term: { state_uuid: get(cluster, 'cluster_state.state_uuid') } }], | ||
| }), | ||
| aggs: { | ||
| indices: { | ||
| terms: { | ||
| field: 'shard.index', | ||
| size: maxBucketSize, | ||
| }, | ||
| aggs: { | ||
| state: { | ||
| filter: { | ||
| terms: { | ||
| 'shard.state': ['UNASSIGNED', 'INITIALIZING'], | ||
| }, | ||
| }, | ||
| aggs: { | ||
| primary: { | ||
| terms: { | ||
| field: 'shard.primary', | ||
| size: 2, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); | ||
| return await callWithRequest(req, 'search', params); | ||
| } | ||
|
|
||
| export async function getIndicesUnassignedShardStats(req, esIndexPattern, cluster) { | ||
| checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getShardStats'); | ||
|
|
||
| const response = await getUnassignedShardData(req, esIndexPattern, cluster); | ||
| const indices = get(response, 'aggregations.indices.buckets', []).reduce((accum, bucket) => { | ||
| const index = bucket.key; | ||
| const states = get(bucket, 'state.primary.buckets', []); | ||
| const unassignedReplica = states | ||
| .filter(state => state.key_as_string === 'false') | ||
| .reduce((total, state) => total + state.doc_count, 0); | ||
| const unassignedPrimary = states | ||
| .filter(state => state.key_as_string === 'true') | ||
| .reduce((total, state) => total + state.doc_count, 0); | ||
|
|
||
| let status = 'green'; | ||
| if (unassignedReplica > 0) { | ||
| status = 'yellow'; | ||
| } | ||
| if (unassignedPrimary > 0) { | ||
| status = 'red'; | ||
| } | ||
|
|
||
| accum[index] = { | ||
| unassigned: { primary: unassignedPrimary, replica: unassignedReplica }, | ||
| status, | ||
| }; | ||
| return accum; | ||
| }, {}); | ||
|
|
||
| const indicesTotals = calculateIndicesTotals(indices); | ||
| return { indices, indicesTotals }; | ||
| } |
59 changes: 59 additions & 0 deletions
59
...ins/monitoring/server/lib/elasticsearch/shards/get_indices_unassigned_shard_stats.test.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| import { getIndicesUnassignedShardStats } from './get_indices_unassigned_shard_stats'; | ||
|
|
||
| describe('getIndicesUnassignedShardStats', () => { | ||
| it('should return the unassigned shard stats for indices', async () => { | ||
| const indices = { | ||
| 12345: { status: 'red', unassigned: { primary: 1, replica: 0 } }, | ||
| 6789: { status: 'yellow', unassigned: { primary: 0, replica: 1 } }, | ||
| absdf82: { status: 'green', unassigned: { primary: 0, replica: 0 } }, | ||
| }; | ||
|
|
||
| const req = { | ||
| server: { | ||
| config: () => ({ | ||
| get: () => {}, | ||
| }), | ||
| plugins: { | ||
| elasticsearch: { | ||
| getCluster: () => ({ | ||
| callWithRequest: () => ({ | ||
| aggregations: { | ||
| indices: { | ||
| buckets: Object.keys(indices).map(id => ({ | ||
| key: id, | ||
| state: { | ||
| primary: { | ||
| buckets: | ||
| indices[id].unassigned.primary || indices[id].unassigned.replica | ||
| ? [ | ||
| { | ||
| key_as_string: indices[id].unassigned.primary | ||
| ? 'true' | ||
| : 'false', | ||
| doc_count: 1, | ||
| }, | ||
| ] | ||
| : [], | ||
| }, | ||
| }, | ||
| })), | ||
| }, | ||
| }, | ||
| }), | ||
| }), | ||
| }, | ||
| }, | ||
| }, | ||
| }; | ||
| const esIndexPattern = '*'; | ||
| const cluster = {}; | ||
| const stats = await getIndicesUnassignedShardStats(req, esIndexPattern, cluster); | ||
| expect(stats.indices).toEqual(indices); | ||
| }); | ||
| }); |
53 changes: 53 additions & 0 deletions
53
x-pack/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| import { get } from 'lodash'; | ||
| import { checkParam } from '../../error_missing_required'; | ||
| import { createQuery } from '../../create_query'; | ||
| import { ElasticsearchMetric } from '../../metrics'; | ||
|
|
||
| async function getShardCountPerNode(req, esIndexPattern, cluster) { | ||
| const config = req.server.config(); | ||
| const maxBucketSize = config.get('xpack.monitoring.max_bucket_size'); | ||
| const metric = ElasticsearchMetric.getMetricFields(); | ||
|
|
||
| const params = { | ||
| index: esIndexPattern, | ||
| size: 0, | ||
| ignoreUnavailable: true, | ||
| body: { | ||
| sort: { timestamp: { order: 'desc' } }, | ||
| query: createQuery({ | ||
| type: 'shards', | ||
| clusterUuid: cluster.cluster_uuid, | ||
| metric, | ||
| filters: [{ term: { state_uuid: get(cluster, 'cluster_state.state_uuid') } }], | ||
| }), | ||
| aggs: { | ||
| nodes: { | ||
| terms: { | ||
| field: 'shard.node', | ||
| size: maxBucketSize, | ||
| }, | ||
| }, | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); | ||
| return await callWithRequest(req, 'search', params); | ||
| } | ||
|
|
||
| export async function getNodesShardCount(req, esIndexPattern, cluster) { | ||
| checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getShardStats'); | ||
|
|
||
| const response = await getShardCountPerNode(req, esIndexPattern, cluster); | ||
| const nodes = get(response, 'aggregations.nodes.buckets', []).reduce((accum, bucket) => { | ||
| accum[bucket.key] = { shardCount: bucket.doc_count }; | ||
| return accum; | ||
| }, {}); | ||
| return { nodes }; | ||
| } |
45 changes: 45 additions & 0 deletions
45
...k/legacy/plugins/monitoring/server/lib/elasticsearch/shards/get_nodes_shard_count.test.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| import { getNodesShardCount } from './get_nodes_shard_count'; | ||
|
|
||
| describe('getNodeShardCount', () => { | ||
| it('should return the shard count per node', async () => { | ||
| const nodes = { | ||
| 12345: { shardCount: 10 }, | ||
| 6789: { shardCount: 1 }, | ||
| absdf82: { shardCount: 20 }, | ||
| }; | ||
|
|
||
| const req = { | ||
| server: { | ||
| config: () => ({ | ||
| get: () => {}, | ||
| }), | ||
| plugins: { | ||
| elasticsearch: { | ||
| getCluster: () => ({ | ||
| callWithRequest: () => ({ | ||
| aggregations: { | ||
| nodes: { | ||
| buckets: Object.keys(nodes).map(id => ({ | ||
| key: id, | ||
| doc_count: nodes[id].shardCount, | ||
| })), | ||
| }, | ||
| }, | ||
| }), | ||
| }), | ||
| }, | ||
| }, | ||
| }, | ||
| }; | ||
| const esIndexPattern = '*'; | ||
| const cluster = {}; | ||
| const counts = await getNodesShardCount(req, esIndexPattern, cluster); | ||
| expect(counts.nodes).toEqual(nodes); | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.