From 674a7502d139f543a4a9279447e1b21bedecd2d0 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:26:57 +0100 Subject: [PATCH] [Fleet] fix remote output health reporting if remote es output is default (#178857) ## Summary Closes https://github.com/elastic/kibana/issues/177927 Replaced "default" with real output id in full agent policy. This fixes the issue that the remote es health reporting was incorrect if the output was set as default. More explanation on the bug: https://github.com/elastic/kibana/issues/177927#issuecomment-2003410884 To verify: - create a remote es output and set as default (both data and monitoring) - create an agent policy that uses default output - enroll an agent - expect that the agent sends system and elastic-agent metrics/logs to remote es - verify that the remote es health badge shows up on UI image image ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../full_agent_policy.test.ts.snap | 28 ++++++++--------- .../agent_policies/full_agent_policy.test.ts | 30 +++++++++---------- .../agent_policies/full_agent_policy.ts | 6 ---- .../tests/fleet/input_only_package.spec.ts | 4 +-- .../agent_policy_datastream_permissions.ts | 4 ++- 5 files changed, 34 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap index 86f8126c1c45a..21192a1158009 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap @@ -12,7 +12,7 @@ Object { "logs": false, "metrics": true, "namespace": "default", - "use_output": "default", + "use_output": "test-id", }, "protection": Object { "enabled": false, @@ -35,7 +35,7 @@ Object { ], }, }, - "default": Object { + "test-id": Object { "_elastic_agent_checks": Object { "cluster": Array [ "monitor", @@ -61,7 +61,7 @@ Object { "preset": "balanced", "type": "elasticsearch", }, - "default": Object { + "test-id": Object { "hosts": Array [ "http://127.0.0.1:9201", ], @@ -106,13 +106,6 @@ Object { "id": "agent-policy", "inputs": Array [], "output_permissions": Object { - "default": Object { - "_elastic_agent_checks": Object { - "cluster": Array [ - "monitor", - ], - }, - }, "monitoring-output-id": Object { "_elastic_agent_checks": Object { "cluster": Array [ @@ -130,18 +123,25 @@ Object { ], }, }, + "test-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + }, }, "outputs": Object { - "default": Object { + "monitoring-output-id": Object { "hosts": Array [ - "http://127.0.0.1:9201", + "http://es-monitoring.co:9201", ], "preset": "balanced", "type": "elasticsearch", }, - "monitoring-output-id": Object { + "test-id": Object { "hosts": Array [ - "http://es-monitoring.co:9201", + "http://127.0.0.1:9201", ], "preset": "balanced", "type": "elasticsearch", diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts index a5cba9f9cab3e..09bf54773509f 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts @@ -197,7 +197,7 @@ describe('getFullAgentPolicy', () => { expect(agentPolicy).toMatchObject({ id: 'agent-policy', outputs: { - default: { + 'test-id': { type: 'elasticsearch', hosts: ['http://127.0.0.1:9201'], }, @@ -228,7 +228,7 @@ describe('getFullAgentPolicy', () => { expect(agentPolicy).toMatchObject({ id: 'agent-policy', outputs: { - default: { + 'test-id': { type: 'elasticsearch', hosts: ['http://127.0.0.1:9201'], }, @@ -244,7 +244,7 @@ describe('getFullAgentPolicy', () => { }, monitoring: { namespace: 'default', - use_output: 'default', + use_output: 'test-id', enabled: true, logs: true, metrics: false, @@ -264,7 +264,7 @@ describe('getFullAgentPolicy', () => { expect(agentPolicy).toMatchObject({ id: 'agent-policy', outputs: { - default: { + 'test-id': { type: 'elasticsearch', hosts: ['http://127.0.0.1:9201'], }, @@ -280,7 +280,7 @@ describe('getFullAgentPolicy', () => { }, monitoring: { namespace: 'default', - use_output: 'default', + use_output: 'test-id', enabled: true, logs: false, metrics: true, @@ -358,7 +358,7 @@ describe('getFullAgentPolicy', () => { expect(agentPolicy).toMatchSnapshot(); }); - it('should use "default" as the default policy id', async () => { + it('should use output id from default policy id', async () => { mockAgentPolicy({ id: 'policy', status: 'active', @@ -372,7 +372,7 @@ describe('getFullAgentPolicy', () => { const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy'); - expect(agentPolicy?.outputs.default).toBeDefined(); + expect(agentPolicy?.outputs['test-id']).toBeDefined(); }); it('should return the sourceURI from the agent policy', async () => { @@ -387,7 +387,7 @@ describe('getFullAgentPolicy', () => { expect(agentPolicy).toMatchObject({ id: 'agent-policy', outputs: { - default: { + 'test-id': { type: 'elasticsearch', hosts: ['http://127.0.0.1:9201'], }, @@ -403,7 +403,7 @@ describe('getFullAgentPolicy', () => { }, monitoring: { namespace: 'default', - use_output: 'default', + use_output: 'test-id', enabled: true, logs: false, metrics: true, @@ -427,7 +427,7 @@ describe('getFullAgentPolicy', () => { expect(agentPolicy).toMatchObject({ id: 'agent-policy', outputs: { - default: { + 'test-id': { type: 'elasticsearch', hosts: ['http://127.0.0.1:9201'], }, @@ -440,7 +440,7 @@ describe('getFullAgentPolicy', () => { agent: { monitoring: { namespace: 'default', - use_output: 'default', + use_output: 'test-id', enabled: true, logs: false, metrics: true, @@ -626,7 +626,7 @@ describe('getFullAgentPolicy', () => { }, ], type: 'test-logs', - use_output: 'default', + use_output: 'test-id', }, { data_stream: { @@ -652,11 +652,11 @@ describe('getFullAgentPolicy', () => { }, ], type: 'test-logs', - use_output: 'default', + use_output: 'test-id', }, ], output_permissions: { - default: { + 'test-id': { _elastic_agent_checks: { cluster: ['monitor'], }, @@ -679,7 +679,7 @@ describe('getFullAgentPolicy', () => { }, }, outputs: { - default: { + 'test-id': { hosts: ['http://127.0.0.1:9201'], preset: 'balanced', type: 'elasticsearch', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index 293090e9813fc..3a35eb9373cb6 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -32,7 +32,6 @@ import type { } from '../../../common/types'; import { agentPolicyService } from '../agent_policy'; import { dataTypes, kafkaCompressionType, outputType } from '../../../common/constants'; -import { DEFAULT_OUTPUT } from '../../constants'; import { getPackageInfo } from '../epm/packages'; import { pkgToPkgKey, splitPkgKey } from '../epm/registry'; @@ -491,13 +490,8 @@ export function transformOutputToFullPolicyOutput( /** * Get id used in full agent policy (sent to the agents) - * we use "default" for the default policy to avoid breaking changes */ function getOutputIdForAgentPolicy(output: Output) { - if (output.is_default) { - return DEFAULT_OUTPUT.name; - } - return output.id; } diff --git a/x-pack/test/apm_api_integration/tests/fleet/input_only_package.spec.ts b/x-pack/test/apm_api_integration/tests/fleet/input_only_package.spec.ts index 6d8cfb870d695..5d02209ca0aa6 100644 --- a/x-pack/test/apm_api_integration/tests/fleet/input_only_package.spec.ts +++ b/x-pack/test/apm_api_integration/tests/fleet/input_only_package.spec.ts @@ -88,13 +88,13 @@ export default function ApiTest(ftrProviderContext: FtrProviderContext) { registry.when('APM package policy', { config: 'basic', archives: [] }, () => { async function getAgentPolicyPermissions(agentPolicyId: string, packagePolicyId: string) { const res = await bettertest<{ - item: { output_permissions: { default: Record } }; + item: { output_permissions: { [key: string]: Record } }; }>({ pathname: `/api/fleet/agent_policies/${agentPolicyId}/full`, method: 'get', }); - return res.body.item.output_permissions.default[packagePolicyId]; + return Object.values(res.body.item.output_permissions)[0][packagePolicyId]; } describe('input only package', () => { diff --git a/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy_datastream_permissions.ts b/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy_datastream_permissions.ts index 9fe3f24b14a73..c786b806b1b14 100644 --- a/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy_datastream_permissions.ts +++ b/x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy_datastream_permissions.ts @@ -110,7 +110,9 @@ export default function (providerContext: FtrProviderContext) { .set('kbn-xsrf', 'xxxx'); // Check that the privileges are correct - expect(fullAgentPolicy.output_permissions.default[packagePolicyId].indices).to.eql([ + expect( + (Object.values(fullAgentPolicy.output_permissions)[0] as any)[packagePolicyId].indices + ).to.eql([ { names: ['logs-*-*'], privileges: ['auto_configure', 'create_doc'] }, { names: ['metrics-*-*'], privileges: ['auto_configure', 'create_doc'] }, ]);