Introduce Kibana task to deploy agentless connectors for 9.0 (elastic…

…#203973)

## Closes elastic/search-team#8508
## Closes elastic/search-team#8465

## Summary

This PR adds a background task for search_connectors plugin. This task
checks connector records and agentless package policies and sees if new
connector was added/old was deleted, and then adds/deletes package
policies for these connectors.

Scenario 1: a new connector was added by a user/API call

User creates an Elastic-managed connector:


https://github.com/user-attachments/assets/38296e48-b281-4b2b-9750-ab0a47334b55

When the user is done, a package policy is created by this background
task:


https://github.com/user-attachments/assets/12dbc33f-32bf-472d-b854-64588fc1e5b1

Scenario 2: a connector was deleted by a user/API call

User deletes an Elastic-managed connector:


https://github.com/user-attachments/assets/5997897e-fb9d-4199-8045-abe163264976

### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Jedr Blaszyk <jedrazb@gmail.com>

x-pack/platform/plugins/shared/fleet/server/mocks/index.ts

@@ -228,8 +228,10 @@ export const createPackagePolicyServiceMock = (): jest.Mocked<PackagePolicyClien
  */
 export const createMockAgentPolicyService = (): jest.Mocked<AgentPolicyServiceInterface> => {
   return {
+    create: jest.fn().mockReturnValue(Promise.resolve()),
     get: jest.fn().mockReturnValue(Promise.resolve()),
     list: jest.fn().mockReturnValue(Promise.resolve()),
+    delete: jest.fn().mockReturnValue(Promise.resolve()),
     getFullAgentPolicy: jest.fn().mockReturnValue(Promise.resolve()),
     getByIds: jest.fn().mockReturnValue(Promise.resolve()),
     turnOffAgentTamperProtections: jest.fn().mockReturnValue(Promise.resolve()),

x-pack/platform/plugins/shared/fleet/server/plugin.ts

@@ -816,16 +816,7 @@ export class FleetPlugin
         core.elasticsearch.client.asInternalUser,
         internalSoClient
       ),
-      agentPolicyService: {
-        get: agentPolicyService.get,
-        list: agentPolicyService.list,
-        getFullAgentPolicy: agentPolicyService.getFullAgentPolicy,
-        getByIds: agentPolicyService.getByIDs,
-        turnOffAgentTamperProtections:
-          agentPolicyService.turnOffAgentTamperProtections.bind(agentPolicyService),
-        fetchAllAgentPolicies: agentPolicyService.fetchAllAgentPolicies,
-        fetchAllAgentPolicyIds: agentPolicyService.fetchAllAgentPolicyIds,
-      },
+      agentPolicyService,
       packagePolicyService,
       registerExternalCallback: (type: ExternalCallback[0], callback: ExternalCallback[1]) => {
         return appContextService.addExternalCallback(type, callback);

x-pack/platform/plugins/shared/fleet/server/routes/agent_policy/handlers.ts

@@ -196,7 +196,7 @@ export const bulkGetAgentPoliciesHandler: FleetRequestHandler<
         'full query parameter require agent policies read permissions'
       );
     }
-    let items = await agentPolicyService.getByIDs(soClient, ids, {
+    let items = await agentPolicyService.getByIds(soClient, ids, {
       withPackagePolicies,
       ignoreMissing,
     });
@@ -687,7 +687,7 @@ export const GetListAgentPolicyOutputsHandler: FleetRequestHandler<
       body: { items: [] },
     });
   }
-  const agentPolicies = await agentPolicyService.getByIDs(soClient, ids, {
+  const agentPolicies = await agentPolicyService.getByIds(soClient, ids, {
     withPackagePolicies: true,
   });
 

x-pack/platform/plugins/shared/fleet/server/routes/settings/enrollment_settings_handler.test.ts

@@ -22,7 +22,7 @@ import {
 jest.mock('../../services', () => ({
   agentPolicyService: {
     get: jest.fn(),
-    getByIDs: jest.fn(),
+    getByIds: jest.fn(),
   },
   appContextService: {
     getInternalUserSOClientWithoutSpaceExtension: jest.fn(),

x-pack/platform/plugins/shared/fleet/server/routes/settings/settings_handler.test.ts

@@ -42,7 +42,7 @@ jest.mock('../../services', () => ({
   },
   agentPolicyService: {
     get: jest.fn(),
-    getByIDs: jest.fn(),
+    getByIds: jest.fn(),
   },
 }));
 

x-pack/platform/plugins/shared/fleet/server/services/agent_policy.test.ts

@@ -504,7 +504,7 @@ describe('Agent policy', () => {
     });
   });
 
-  describe('getByIDs', () => {
+  describe('getByIds', () => {
     it('should call audit logger', async () => {
       const soClient = savedObjectsClientMock.create();
 
@@ -525,7 +525,7 @@ describe('Agent policy', () => {
         ],
       });
 
-      await agentPolicyService.getByIDs(soClient, ['test-agent-policy-1', 'test-agent-policy-2']);
+      await agentPolicyService.getByIds(soClient, ['test-agent-policy-1', 'test-agent-policy-2']);
 
       expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenNthCalledWith(1, {
         action: 'get',

x-pack/platform/plugins/shared/fleet/server/services/agent_policy.ts

@@ -507,7 +507,7 @@ class AgentPolicyService {
     return agentPolicy;
   }
 
-  public async getByIDs(
+  public async getByIds(
     soClient: SavedObjectsClientContract,
     ids: Array<string | { id: string; spaceId?: string }>,
     options: { fields?: string[]; withPackagePolicies?: boolean; ignoreMissing?: boolean } = {}
@@ -1345,7 +1345,7 @@ class AgentPolicyService {
       });
     }
 
-    const policies = await agentPolicyService.getByIDs(soClient, agentPolicyIds);
+    const policies = await agentPolicyService.getByIds(soClient, agentPolicyIds);
     const policiesMap = keyBy(policies, 'id');
     const fullPolicies = await pMap(
       agentPolicyIds,

x-pack/platform/plugins/shared/fleet/server/services/agents/hosted_agent.test.ts

@@ -14,7 +14,7 @@ import { getHostedPolicies, isHostedAgent } from './hosted_agent';
 jest.mock('../agent_policy', () => {
   return {
     agentPolicyService: {
-      getByIDs: jest.fn().mockResolvedValue([
+      getByIds: jest.fn().mockResolvedValue([
         { id: 'hosted-policy', is_managed: true },
         { id: 'regular-policy', is_managed: false },
       ]),

x-pack/platform/plugins/shared/fleet/server/services/agents/hosted_agent.ts

@@ -20,7 +20,7 @@ export async function getHostedPolicies(
   );
 
   // get the agent policies for those ids
-  const agentPolicies = await agentPolicyService.getByIDs(soClient, Array.from(policyIdsToGet), {
+  const agentPolicies = await agentPolicyService.getByIds(soClient, Array.from(policyIdsToGet), {
     fields: ['is_managed'],
     ignoreMissing: true,
   });

x-pack/platform/plugins/shared/fleet/server/services/agents/update_agent_tags.test.ts

@@ -34,7 +34,7 @@ jest.mock('../agent_policy', () => {
   return {
     agentPolicyService: {
       getInactivityTimeouts: jest.fn().mockResolvedValue([]),
-      getByIDs: jest.fn().mockResolvedValue([{ id: 'hosted-agent-policy', is_managed: true }]),
+      getByIds: jest.fn().mockResolvedValue([{ id: 'hosted-agent-policy', is_managed: true }]),
       list: jest.fn().mockResolvedValue({ items: [] }),
     },
   };

x-pack/platform/plugins/shared/fleet/server/services/fleet_server/index.test.ts

@@ -204,7 +204,7 @@ describe('getFleetServerPolicies', () => {
       page: 1,
       perPage: mockPackagePolicies.length,
     });
-    (mockedAgentPolicyService.getByIDs as jest.Mock).mockResolvedValueOnce(mockFleetServerPolicies);
+    (mockedAgentPolicyService.getByIds as jest.Mock).mockResolvedValueOnce(mockFleetServerPolicies);
     const result = await getFleetServerPolicies(soClient);
     expect(result).toEqual(mockFleetServerPolicies);
   });

x-pack/platform/plugins/shared/fleet/server/services/fleet_server/index.ts

@@ -41,7 +41,7 @@ export const getFleetServerPolicies = async (
 
   // Retrieve associated agent policies
   const fleetServerAgentPolicies = fleetServerAgentPolicyIds.length
-    ? await agentPolicyService.getByIDs(
+    ? await agentPolicyService.getByIds(
         soClient,
         uniqBy(fleetServerAgentPolicyIds, (p) => p.id)
       )

x-pack/platform/plugins/shared/fleet/server/services/index.ts

@@ -15,10 +15,12 @@ export { getRegistryUrl } from './epm/registry/registry_url';
  */
 
 export interface AgentPolicyServiceInterface {
+  create: (typeof agentPolicyService)['create'];
   get: (typeof agentPolicyService)['get'];
   list: (typeof agentPolicyService)['list'];
+  delete: (typeof agentPolicyService)['delete'];
   getFullAgentPolicy: (typeof agentPolicyService)['getFullAgentPolicy'];
-  getByIds: (typeof agentPolicyService)['getByIDs'];
+  getByIds: (typeof agentPolicyService)['getByIds'];
   turnOffAgentTamperProtections: (typeof agentPolicyService)['turnOffAgentTamperProtections'];
   fetchAllAgentPolicyIds: (typeof agentPolicyService)['fetchAllAgentPolicyIds'];
   fetchAllAgentPolicies: (typeof agentPolicyService)['fetchAllAgentPolicies'];

x-pack/platform/plugins/shared/fleet/server/services/output.test.ts

@@ -316,7 +316,7 @@ describe('Output Service', () => {
   } as unknown as ReturnType<typeof mockedPackagePolicyService.list>;
 
   beforeEach(() => {
-    mockedAgentPolicyService.getByIDs.mockResolvedValue([]);
+    mockedAgentPolicyService.getByIds.mockResolvedValue([]);
     mockedAgentPolicyService.list.mockClear();
     mockedPackagePolicyService.list.mockReset();
     mockedAgentPolicyService.hasAPMIntegration.mockClear();
@@ -334,7 +334,7 @@ describe('Output Service', () => {
   });
 
   afterEach(() => {
-    mockedAgentPolicyService.getByIDs.mockClear();
+    mockedAgentPolicyService.getByIds.mockClear();
   });
 
   describe('create', () => {
@@ -688,7 +688,7 @@ describe('Output Service', () => {
         mockedPackagePolicyService.list.mockResolvedValue(
           mockedPackagePolicyWithFleetServerResolvedValue
         );
-        mockedAgentPolicyService.getByIDs.mockResolvedValue(
+        mockedAgentPolicyService.getByIds.mockResolvedValue(
           (await mockedAgentPolicyWithFleetServerResolvedValue).items
         );
 
@@ -727,7 +727,7 @@ describe('Output Service', () => {
         mockedPackagePolicyService.list.mockResolvedValue(
           mockedPackagePolicyWithSyntheticsResolvedValue
         );
-        mockedAgentPolicyService.getByIDs.mockResolvedValue(
+        mockedAgentPolicyService.getByIds.mockResolvedValue(
           (await mockedAgentPolicyWithSyntheticsResolvedValue).items
         );
 
@@ -845,7 +845,7 @@ describe('Output Service', () => {
         mockedPackagePolicyService.list.mockResolvedValue(
           mockedPackagePolicyWithFleetServerResolvedValue
         );
-        mockedAgentPolicyService.getByIDs.mockResolvedValue(
+        mockedAgentPolicyService.getByIds.mockResolvedValue(
           (await mockedAgentPolicyWithFleetServerResolvedValue).items
         );
 
@@ -884,7 +884,7 @@ describe('Output Service', () => {
         mockedPackagePolicyService.list.mockResolvedValue(
           mockedPackagePolicyWithSyntheticsResolvedValue
         );
-        mockedAgentPolicyService.getByIDs.mockResolvedValue(
+        mockedAgentPolicyService.getByIds.mockResolvedValue(
           (await mockedAgentPolicyWithSyntheticsResolvedValue).items
         );
 

x-pack/platform/plugins/shared/fleet/server/services/output.ts

@@ -176,7 +176,7 @@ async function getAgentPoliciesPerOutput(outputId?: string, isDefault?: boolean)
       }, [])
     ),
   ];
-  const agentPoliciesFromPackagePolicies = await agentPolicyService.getByIDs(
+  const agentPoliciesFromPackagePolicies = await agentPolicyService.getByIds(
     internalSoClientWithoutSpaceExtension,
     agentPolicyIdsFromPackagePolicies
   );
@@ -245,7 +245,7 @@ async function findPoliciesWithFleetServerOrSynthetics(outputId?: string, isDefa
     );
     const agentPolicyIds = _.uniq(packagePolicies.flatMap((p) => p.policy_ids));
     if (agentPolicyIds.length) {
-      agentPolicies = await agentPolicyService.getByIDs(
+      agentPolicies = await agentPolicyService.getByIds(
         internalSoClientWithoutSpaceExtension,
         agentPolicyIds
       );

x-pack/platform/plugins/shared/fleet/server/services/package_policy.test.ts

@@ -227,7 +227,7 @@ const mockAgentPolicyGet = (spaceIds: string[] = ['default']) => {
       });
     }
   );
-  mockAgentPolicyService.getByIDs.mockImplementation(
+  mockAgentPolicyService.getByIds.mockImplementation(
     // @ts-ignore
     (_soClient: SavedObjectsClientContract, ids: string[]) => {
       return Promise.resolve(

x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts

@@ -518,7 +518,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
 
     const agentPolicyIds = new Set(packagePolicies.flatMap((pkgPolicy) => pkgPolicy.policy_ids));
 
-    const agentPolicies = await agentPolicyService.getByIDs(soClient, [...agentPolicyIds]);
+    const agentPolicies = await agentPolicyService.getByIds(soClient, [...agentPolicyIds]);
     const agentPoliciesIndexById = indexBy('id', agentPolicies);
     for (const agentPolicy of agentPolicies) {
       validateIsNotHostedPolicy(agentPolicy, options?.force);
@@ -1551,7 +1551,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
         return acc;
       }, new Set());
 
-      const agentPolicies = await agentPolicyService.getByIDs(soClient, uniquePolicyIdsR);
+      const agentPolicies = await agentPolicyService.getByIds(soClient, uniquePolicyIdsR);
 
       for (const policyId of uniquePolicyIdsR) {
         const agentPolicy = agentPolicies.find((p) => p.id === policyId);

x-pack/platform/plugins/shared/fleet/server/services/security/uninstall_token_service/index.test.ts

@@ -217,7 +217,7 @@ describe('UninstallTokenService', () => {
     agentPolicyService.deployPolicies = jest.fn();
 
     getAgentPoliciesByIDsMock = jest.fn().mockResolvedValue([]);
-    agentPolicyService.getByIDs = getAgentPoliciesByIDsMock;
+    agentPolicyService.getByIds = getAgentPoliciesByIDsMock;
 
     if (scoppedInSpace) {
       soClientMock.getCurrentNamespace.mockReturnValue(scoppedInSpace);

x-pack/platform/plugins/shared/fleet/server/services/security/uninstall_token_service/index.ts

@@ -305,7 +305,7 @@ export class UninstallTokenService implements UninstallTokenServiceInterface {
   }
 
   private async getPolicyIdNameDictionary(policyIds: string[]): Promise<Record<string, string>> {
-    const agentPolicies = await agentPolicyService.getByIDs(this.soClient, policyIds, {
+    const agentPolicies = await agentPolicyService.getByIds(this.soClient, policyIds, {
       ignoreMissing: true,
     });
 
@@ -615,7 +615,7 @@ export class UninstallTokenService implements UninstallTokenServiceInterface {
     const batchSize = config?.setup?.agentPolicySchemaUpgradeBatchSize ?? 100;
 
     await asyncForEach(chunk(policyIds, batchSize), async (policyIdsBatch) => {
-      const policies = await agentPolicyService.getByIDs(
+      const policies = await agentPolicyService.getByIds(
         appContextService.getInternalUserSOClientWithoutSpaceExtension(),
         policyIds.map((id) => ({ id, spaceId: '*' }))
       );

x-pack/solutions/search/plugins/search_connectors/kibana.jsonc

@@ -17,8 +17,11 @@
       "search",
       "connectors"
     ],
-    "requiredPlugins": [],
-    "optionalPlugins": [],
-    "requiredBundles": []
+    "requiredPlugins": [
+      "licensing",
+      "taskManager",
+      "fleet"
+    ],
+    "optionalPlugins": []
   }
 }