diff --git a/x-pack/plugins/fleet/server/collectors/agent_logs_top_errors.test.ts b/x-pack/plugins/fleet/server/collectors/agent_logs_top_errors.test.ts new file mode 100644 index 0000000000000..c0a6f451446c2 --- /dev/null +++ b/x-pack/plugins/fleet/server/collectors/agent_logs_top_errors.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; + +import { getAgentLogsTopErrors } from './agent_logs_top_errors'; + +describe('getAgentLogsTopErrors', () => { + it('should return top 3 errors from 100 hits', async () => { + const esClientMock = { + search: jest.fn().mockImplementation((params) => { + if (params.index === 'logs-elastic_agent-*') + return { + hits: { + hits: [ + { + _source: { + message: 'error 2', + }, + }, + { + _source: { + message: 'error 2', + }, + }, + { + _source: { + message: 'error 3', + }, + }, + { + _source: { + message: 'error 3', + }, + }, + { + _source: { + message: 'error 3', + }, + }, + { + _source: { + message: 'error 1', + }, + }, + ], + }, + }; + else + return { + hits: { + hits: [ + { + _source: { + message: 'fleet server error 2', + }, + }, + { + _source: { + message: 'fleet server error 2', + }, + }, + { + _source: { + message: 'fleet server error 1', + }, + }, + ], + }, + }; + }), + } as unknown as ElasticsearchClient; + + const topErrors = await getAgentLogsTopErrors(esClientMock); + expect(topErrors).toEqual({ + agent_logs_top_errors: ['error 3', 'error 2', 'error 1'], + fleet_server_logs_top_errors: ['fleet server error 2', 'fleet server error 1'], + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/collectors/agent_logs_top_errors.ts b/x-pack/plugins/fleet/server/collectors/agent_logs_top_errors.ts index 562a217c2bd19..34a279ffc6f7e 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_logs_top_errors.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_logs_top_errors.ts @@ -7,6 +7,10 @@ import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { sortBy } from 'lodash'; + +import { DATA_TIERS } from '../../common/constants'; + import { appContextService } from '../services'; export interface AgentLogsData { @@ -29,10 +33,16 @@ export async function getAgentLogsTopErrors( const queryTopMessages = (index: string) => esClient.search({ index, - size: 0, + size: 100, + _source: ['message'], query: { bool: { filter: [ + { + terms: { + _tier: DATA_TIERS, + }, + }, { term: { 'log.level': 'error', @@ -48,35 +58,32 @@ export async function getAgentLogsTopErrors( ], }, }, - aggs: { - message_sample: { - sampler: { - shard_size: 200, - }, - aggs: { - categories: { - categorize_text: { - field: 'message', - size: 10, - }, - }, - }, - }, - }, }); - const transformBuckets = (resp: any) => - ((resp?.aggregations?.message_sample as any)?.categories?.buckets ?? []) + const getTopErrors = (resp: any) => { + const counts = (resp?.hits.hits ?? []).reduce((acc: any, curr: any) => { + if (!acc[curr._source.message]) { + acc[curr._source.message] = 0; + } + acc[curr._source.message]++; + return acc; + }, {}); + const top3 = sortBy( + Object.entries(counts).map(([key, value]) => ({ key, value })), + 'value' + ) .slice(0, 3) - .map((bucket: any) => bucket.key); + .reverse(); + return top3.map(({ key, value }) => key); + }; const agentResponse = await queryTopMessages('logs-elastic_agent-*'); const fleetServerResponse = await queryTopMessages('logs-elastic_agent.fleet_server-*'); return { - agent_logs_top_errors: transformBuckets(agentResponse), - fleet_server_logs_top_errors: transformBuckets(fleetServerResponse), + agent_logs_top_errors: getTopErrors(agentResponse), + fleet_server_logs_top_errors: getTopErrors(fleetServerResponse), }; } catch (error) { if (error.statusCode === 404) { diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index 901a049df8d73..c9922f78b70ce 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -21,6 +21,7 @@ import type { FleetServerUsage } from './fleet_server_collector'; import { getAgentPoliciesUsage } from './agent_policies'; import type { AgentPanicLogsData } from './agent_logs_panics'; import { getPanicLogsLastHour } from './agent_logs_panics'; +import { getAgentLogsTopErrors } from './agent_logs_top_errors'; export interface Usage { agents_enabled: boolean; @@ -64,8 +65,7 @@ export const fetchFleetUsage = async ( fleet_server_config: await getFleetServerConfig(soClient), agent_policies: await getAgentPoliciesUsage(soClient), ...(await getPanicLogsLastHour(esClient)), - // TODO removed top errors telemetry as it causes this issue: https://github.com/elastic/kibana/issues/148976 - // ...(await getAgentLogsTopErrors(esClient)), + ...(await getAgentLogsTopErrors(esClient)), }; return usage; }; diff --git a/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts index e95a6fce722a0..0589f567b971f 100644 --- a/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts +++ b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts @@ -400,8 +400,12 @@ describe('fleet usage telemetry', () => { message: 'stderr panic some other panic', }, ], - // agent_logs_top_errors: ['stderr panic close of closed channel'], - // fleet_server_logs_top_errors: ['failed to unenroll offline agents'], + agent_logs_top_errors: [ + 'stderr panic some other panic', + 'stderr panic close of closed channel', + 'this should not be included in metrics', + ], + fleet_server_logs_top_errors: ['failed to unenroll offline agents'], }) ); }); diff --git a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts index faf963798768b..f1172786c0b2b 100644 --- a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts +++ b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts @@ -247,4 +247,18 @@ export const fleetUsagesSchema: RootSchema = { }, }, }, + agent_logs_top_errors: { + type: 'array', + items: { + type: 'text', + _meta: { description: 'Top messages from agent error logs' }, + }, + }, + fleet_server_logs_top_errors: { + type: 'array', + items: { + type: 'text', + _meta: { description: 'Top messages from fleet server error logs' }, + }, + }, };