[Fleet] partial revert using default as output id, only for elasticse…

…arch type (#179218)

## Summary

Related to #178857 and
#177927

It seems that using output id instead of "default" in full agent policy
had a higher impact than expected. There are a few places where agent
relies on the name "default".
([This](elastic/elastic-agent#4454) and
[this](elastic/elastic-agent#4453) pr)
Because of this, doing a partial revert, to keep using "default" for
elasticsearch output type to avoid breaking change.
However, for other types, using the output id. This will fix the
original issue of remote output health reporting.
I think it is a rarely used feature to use a non-elasticsearch output as
default, so it shouldn't have a big impact to not use "default" output
name for those.

To verify:
- create a remote es output and set as default (both data and
monitoring)
- create an agent policy that uses default output 
- enroll an agent
- expect that the agent sends system and elastic-agent metrics/logs to
remote es
- verify that the remote es health badge shows up on UI
- set elasticsearch output back as default
- verify that the agent policy has it as "default" in outputs section

<img width="704" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/ab46b00d-efc2-49e1-ad7f-9acd44b2a9e5">
<img width="1251" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/a07c0d78-9126-43d9-bd0e-a4df193d7e78">
<img width="1791" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/868a054b-2cae-42f3-8f60-f2bff3b29efd">

<img width="715" alt="image"
src="https://github.com/elastic/kibana/assets/90178898/721cd809-5f97-47e5-bf99-19f542d8ff83">



### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts

@@ -93,6 +93,15 @@ jest.mock('../output', () => {
       type: 'elasticsearch',
       hosts: ['http://127.0.0.1:9201'],
     },
+    'test-remote-id': {
+      id: 'test-remote-id',
+      is_default: true,
+      is_default_monitoring: true,
+      name: 'default',
+      // @ts-ignore
+      type: 'remote_elasticsearch',
+      hosts: ['http://127.0.0.1:9201'],
+    },
   };
   return {
     outputService: {
@@ -197,7 +206,7 @@ describe('getFullAgentPolicy', () => {
     expect(agentPolicy).toMatchObject({
       id: 'agent-policy',
       outputs: {
-        'test-id': {
+        default: {
           type: 'elasticsearch',
           hosts: ['http://127.0.0.1:9201'],
         },
@@ -228,7 +237,7 @@ describe('getFullAgentPolicy', () => {
     expect(agentPolicy).toMatchObject({
       id: 'agent-policy',
       outputs: {
-        'test-id': {
+        default: {
           type: 'elasticsearch',
           hosts: ['http://127.0.0.1:9201'],
         },
@@ -244,7 +253,7 @@ describe('getFullAgentPolicy', () => {
         },
         monitoring: {
           namespace: 'default',
-          use_output: 'test-id',
+          use_output: 'default',
           enabled: true,
           logs: true,
           metrics: false,
@@ -264,7 +273,7 @@ describe('getFullAgentPolicy', () => {
     expect(agentPolicy).toMatchObject({
       id: 'agent-policy',
       outputs: {
-        'test-id': {
+        default: {
           type: 'elasticsearch',
           hosts: ['http://127.0.0.1:9201'],
         },
@@ -280,7 +289,7 @@ describe('getFullAgentPolicy', () => {
         },
         monitoring: {
           namespace: 'default',
-          use_output: 'test-id',
+          use_output: 'default',
           enabled: true,
           logs: false,
           metrics: true,
@@ -358,7 +367,7 @@ describe('getFullAgentPolicy', () => {
     expect(agentPolicy).toMatchSnapshot();
   });
 
-  it('should use output id from default policy id', async () => {
+  it('should use "default" as the default policy id', async () => {
     mockAgentPolicy({
       id: 'policy',
       status: 'active',
@@ -372,7 +381,24 @@ describe('getFullAgentPolicy', () => {
 
     const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy');
 
-    expect(agentPolicy?.outputs['test-id']).toBeDefined();
+    expect(agentPolicy?.outputs.default).toBeDefined();
+  });
+
+  it('should use output id as the default policy id when remote elasticsearch', async () => {
+    mockAgentPolicy({
+      id: 'policy',
+      status: 'active',
+      package_policies: [],
+      is_managed: false,
+      namespace: 'default',
+      revision: 1,
+      data_output_id: 'test-remote-id',
+      monitoring_output_id: 'test-remote-id',
+    });
+
+    const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy');
+
+    expect(agentPolicy?.outputs['test-remote-id']).toBeDefined();
   });
 
   it('should return the sourceURI from the agent policy', async () => {
@@ -387,7 +413,7 @@ describe('getFullAgentPolicy', () => {
     expect(agentPolicy).toMatchObject({
       id: 'agent-policy',
       outputs: {
-        'test-id': {
+        default: {
           type: 'elasticsearch',
           hosts: ['http://127.0.0.1:9201'],
         },
@@ -403,7 +429,7 @@ describe('getFullAgentPolicy', () => {
         },
         monitoring: {
           namespace: 'default',
-          use_output: 'test-id',
+          use_output: 'default',
           enabled: true,
           logs: false,
           metrics: true,
@@ -427,7 +453,7 @@ describe('getFullAgentPolicy', () => {
     expect(agentPolicy).toMatchObject({
       id: 'agent-policy',
       outputs: {
-        'test-id': {
+        default: {
           type: 'elasticsearch',
           hosts: ['http://127.0.0.1:9201'],
         },
@@ -440,7 +466,7 @@ describe('getFullAgentPolicy', () => {
       agent: {
         monitoring: {
           namespace: 'default',
-          use_output: 'test-id',
+          use_output: 'default',
           enabled: true,
           logs: false,
           metrics: true,
@@ -626,7 +652,7 @@ describe('getFullAgentPolicy', () => {
             },
           ],
           type: 'test-logs',
-          use_output: 'test-id',
+          use_output: 'default',
         },
         {
           data_stream: {
@@ -652,11 +678,11 @@ describe('getFullAgentPolicy', () => {
             },
           ],
           type: 'test-logs',
-          use_output: 'test-id',
+          use_output: 'default',
         },
       ],
       output_permissions: {
-        'test-id': {
+        default: {
           _elastic_agent_checks: {
             cluster: ['monitor'],
           },
@@ -679,7 +705,7 @@ describe('getFullAgentPolicy', () => {
         },
       },
       outputs: {
-        'test-id': {
+        default: {
           hosts: ['http://127.0.0.1:9201'],
           preset: 'balanced',
           type: 'elasticsearch',

x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts

@@ -31,7 +31,12 @@ import type {
   PackageInfo,
 } from '../../../common/types';
 import { agentPolicyService } from '../agent_policy';
-import { dataTypes, kafkaCompressionType, outputType } from '../../../common/constants';
+import {
+  dataTypes,
+  DEFAULT_OUTPUT,
+  kafkaCompressionType,
+  outputType,
+} from '../../../common/constants';
 
 import { getPackageInfo } from '../epm/packages';
 import { pkgToPkgKey, splitPkgKey } from '../epm/registry';
@@ -490,8 +495,12 @@ export function transformOutputToFullPolicyOutput(
 
 /**
  * Get id used in full agent policy (sent to the agents)
+ * we use "default" for the default policy to avoid breaking changes
  */
 function getOutputIdForAgentPolicy(output: Output) {
+  if (output.is_default && output.type === outputType.Elasticsearch) {
+    return DEFAULT_OUTPUT.name;
+  }
   return output.id;
 }