[Enterprise Search] Attach ML Inference Pipeline - Pipeline re-use (#…

…143979) * added create ml inference pipeline parameters interface * updated NLP_CONFIG_KEYS to use common constant as source to match server code * attach existing ml inference pipeline Added the ability to choose an existing ml inference pipeline and attach it to the index. This will re-use the existing pipeline instead of creating a new one. * testing ml inference logic * test parseMlInferenceParametersFromPipeline
elastic · Oct 27, 2022 · 756916d · 756916d
1 parent 61505e5
commit 756916d
Show file tree

Hide file tree

Showing 18 changed files with 1,009 additions and 78 deletions.
diff --git a/x-pack/plugins/enterprise_search/common/ml_inference_pipeline/index.test.ts b/x-pack/plugins/enterprise_search/common/ml_inference_pipeline/index.test.ts
@@ -17,6 +17,7 @@ import {
   getMlModelTypesForModelConfig,
   getSetProcessorForInferenceType,
   SUPPORTED_PYTORCH_TASKS as LOCAL_SUPPORTED_PYTORCH_TASKS,
+  parseMlInferenceParametersFromPipeline,
 } from '.';
 
 const mockModel: MlTrainedModelConfig = {
@@ -198,3 +199,45 @@ describe('generateMlInferencePipelineBody lib function', () => {
     );
   });
 });
+
+describe('parseMlInferenceParametersFromPipeline', () => {
+  it('returns pipeline parameters from ingest pipeline', () => {
+    expect(
+      parseMlInferenceParametersFromPipeline('unit-test', {
+        processors: [
+          {
+            inference: {
+              field_map: {
+                body: 'text_field',
+              },
+              model_id: 'test-model',
+              target_field: 'ml.inference.test',
+            },
+          },
+        ],
+      })
+    ).toEqual({
+      destination_field: 'test',
+      model_id: 'test-model',
+      pipeline_name: 'unit-test',
+      source_field: 'body',
+    });
+  });
+  it('return null if pipeline missing inference processor', () => {
+    expect(parseMlInferenceParametersFromPipeline('unit-test', { processors: [] })).toBeNull();
+  });
+  it('return null if pipeline missing field_map', () => {
+    expect(
+      parseMlInferenceParametersFromPipeline('unit-test', {
+        processors: [
+          {
+            inference: {
+              model_id: 'test-model',
+              target_field: 'test',
+            },
+          },
+        ],
+      })
+    ).toBeNull();
+  });
+});
diff --git a/x-pack/plugins/enterprise_search/common/ml_inference_pipeline/index.ts b/x-pack/plugins/enterprise_search/common/ml_inference_pipeline/index.ts
@@ -5,9 +5,13 @@
  * 2.0.
  */
 
-import { IngestSetProcessor, MlTrainedModelConfig } from '@elastic/elasticsearch/lib/api/types';
+import {
+  IngestPipeline,
+  IngestSetProcessor,
+  MlTrainedModelConfig,
+} from '@elastic/elasticsearch/lib/api/types';
 
-import { MlInferencePipeline } from '../types/pipelines';
+import { MlInferencePipeline, CreateMlInferencePipelineParameters } from '../types/pipelines';
 
 // Getting an error importing this from @kbn/ml-plugin/common/constants/data_frame_analytics'
 // So defining it locally for now with a test to make sure it matches.
@@ -151,3 +155,25 @@ export const formatPipelineName = (rawName: string) =>
     .trim()
     .replace(/\s+/g, '_') // Convert whitespaces to underscores
     .toLowerCase();
+
+export const parseMlInferenceParametersFromPipeline = (
+  name: string,
+  pipeline: IngestPipeline
+): CreateMlInferencePipelineParameters | null => {
+  const processor = pipeline?.processors?.find((proc) => proc.inference !== undefined);
+  if (!processor || processor?.inference === undefined) {
+    return null;
+  }
+  const { inference: inferenceProcessor } = processor;
+  const sourceFields = Object.keys(inferenceProcessor.field_map ?? {});
+  const sourceField = sourceFields.length === 1 ? sourceFields[0] : null;
+  if (!sourceField) {
+    return null;
+  }
+  return {
+    destination_field: inferenceProcessor.target_field.replace('ml.inference.', ''),
+    model_id: inferenceProcessor.model_id,
+    pipeline_name: name,
+    source_field: sourceField,
+  };
+};
diff --git a/x-pack/plugins/enterprise_search/common/types/pipelines.ts b/x-pack/plugins/enterprise_search/common/types/pipelines.ts
@@ -64,3 +64,10 @@ export interface DeleteMlInferencePipelineResponse {
   deleted?: string;
   updated?: string;
 }
+
+export interface CreateMlInferencePipelineParameters {
+  destination_field?: string;
+  model_id: string;
+  pipeline_name: string;
+  source_field: string;
+}
diff --git a/...applications/enterprise_search_content/api/pipelines/attach_ml_inference_pipeline.test.ts b/...applications/enterprise_search_content/api/pipelines/attach_ml_inference_pipeline.test.ts
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { mockHttpValues } from '../../../__mocks__/kea_logic';
+
+import {
+  attachMlInferencePipeline,
+  AttachMlInferencePipelineApiLogicArgs,
+  AttachMlInferencePipelineResponse,
+} from './attach_ml_inference_pipeline';
+
+describe('AttachMlInferencePipelineApiLogic', () => {
+  const { http } = mockHttpValues;
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+  describe('createMlInferencePipeline', () => {
+    it('calls the api', async () => {
+      const response: Promise<AttachMlInferencePipelineResponse> = Promise.resolve({
+        addedToParentPipeline: true,
+        created: false,
+        id: 'unit-test',
+      });
+      http.post.mockReturnValue(response);
+
+      const args: AttachMlInferencePipelineApiLogicArgs = {
+        indexName: 'unit-test-index',
+        pipelineName: 'unit-test',
+      };
+      const result = await attachMlInferencePipeline(args);
+      expect(http.post).toHaveBeenCalledWith(
+        '/internal/enterprise_search/indices/unit-test-index/ml_inference/pipeline_processors/attach',
+        {
+          body: '{"pipeline_name":"unit-test"}',
+        }
+      );
+      expect(result).toEqual({
+        addedToParentPipeline: true,
+        created: false,
+        id: args.pipelineName,
+      });
+    });
+  });
+});
diff --git a/...blic/applications/enterprise_search_content/api/pipelines/attach_ml_inference_pipeline.ts b/...blic/applications/enterprise_search_content/api/pipelines/attach_ml_inference_pipeline.ts
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AttachMlInferencePipelineResponse } from '../../../../../common/types/pipelines';
+
+import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
+import { HttpLogic } from '../../../shared/http';
+
+export interface AttachMlInferencePipelineApiLogicArgs {
+  indexName: string;
+  pipelineName: string;
+}
+
+export type { AttachMlInferencePipelineResponse };
+
+export const attachMlInferencePipeline = async (
+  args: AttachMlInferencePipelineApiLogicArgs
+): Promise<AttachMlInferencePipelineResponse> => {
+  const route = `/internal/enterprise_search/indices/${args.indexName}/ml_inference/pipeline_processors/attach`;
+  const params = {
+    pipeline_name: args.pipelineName,
+  };
+
+  return await HttpLogic.values.http.post<AttachMlInferencePipelineResponse>(route, {
+    body: JSON.stringify(params),
+  });
+};
+
+export const AttachMlInferencePipelineApiLogic = createApiLogic(
+  ['attach_ml_inference_pipeline_api_logic'],
+  attachMlInferencePipeline
+);
diff --git a/...dels/create_ml_inference_pipeline.test.ts → ...ines/create_ml_inference_pipeline.test.ts b/...dels/create_ml_inference_pipeline.test.ts → ...ines/create_ml_inference_pipeline.test.ts
diff --git a/...ml_models/create_ml_inference_pipeline.ts → ...pipelines/create_ml_inference_pipeline.ts b/...ml_models/create_ml_inference_pipeline.ts → ...pipelines/create_ml_inference_pipeline.ts
@@ -4,6 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import { CreateMlInferencePipelineParameters } from '../../../../../common/types/pipelines';
 import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
 import { HttpLogic } from '../../../shared/http';
 
@@ -23,7 +24,7 @@ export const createMlInferencePipeline = async (
   args: CreateMlInferencePipelineApiLogicArgs
 ): Promise<CreateMlInferencePipelineResponse> => {
   const route = `/internal/enterprise_search/indices/${args.indexName}/ml_inference/pipeline_processors`;
-  const params = {
+  const params: CreateMlInferencePipelineParameters = {
     destination_field: args.destinationField,
     model_id: args.modelId,
     pipeline_name: args.pipelineName,

diff --git a/...cations/enterprise_search_content/api/pipelines/fetch_ml_inference_pipeline_processors.ts b/...cations/enterprise_search_content/api/pipelines/fetch_ml_inference_pipeline_processors.ts
@@ -9,10 +9,18 @@ import { InferencePipeline } from '../../../../../common/types/pipelines';
 import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
 import { HttpLogic } from '../../../shared/http';
 
-export const fetchMlInferencePipelineProcessors = async ({ indexName }: { indexName: string }) => {
+export interface FetchMlInferencePipelineProcessorsApiLogicArgs {
+  indexName: string;
+}
+
+export type FetchMlInferencePipelineProcessorsResponse = InferencePipeline[];
+
+export const fetchMlInferencePipelineProcessors = async ({
+  indexName,
+}: FetchMlInferencePipelineProcessorsApiLogicArgs) => {
   const route = `/internal/enterprise_search/indices/${indexName}/ml_inference/pipeline_processors`;
 
-  return await HttpLogic.values.http.get<InferencePipeline[]>(route);
+  return await HttpLogic.values.http.get<FetchMlInferencePipelineProcessorsResponse>(route);
 };
 
 export const FetchMlInferencePipelineProcessorsApiLogic = createApiLogic(

diff --git a/...blic/applications/enterprise_search_content/api/pipelines/fetch_ml_inference_pipelines.ts b/...blic/applications/enterprise_search_content/api/pipelines/fetch_ml_inference_pipelines.ts
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MlInferencePipeline } from '../../../../../common/types/pipelines';
+import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
+import { HttpLogic } from '../../../shared/http';
+
+export type FetchMlInferencePipelinesArgs = undefined;
+export type FetchMlInferencePipelinesResponse = Record<string, MlInferencePipeline | undefined>;
+
+export const fetchMlInferencePipelines = async () => {
+  const route = '/internal/enterprise_search/pipelines/ml_inference';
+
+  return await HttpLogic.values.http.get<FetchMlInferencePipelinesResponse>(route);
+};
+
+export const FetchMlInferencePipelinesApiLogic = createApiLogic(
+  ['fetch_ml_inference_pipelines_api_logic'],
+  fetchMlInferencePipelines
+);
diff --git a/...ontent/components/search_index/pipelines/ml_inference/add_ml_inference_pipeline_modal.tsx b/...ontent/components/search_index/pipelines/ml_inference/add_ml_inference_pipeline_modal.tsx
@@ -92,7 +92,7 @@ const AddProcessorContent: React.FC<AddMLInferencePipelineModalProps> = ({ onClo
       </EuiModalBody>
     );
   }
-  if (supportedMLModels === undefined || supportedMLModels?.length === 0) {
+  if (supportedMLModels.length === 0) {
     return <NoModelsPanel />;
   }
   return (
@@ -188,8 +188,10 @@ const ModalFooter: React.FC<AddMLInferencePipelineModalProps & { ingestionMethod
   onClose,
 }) => {
   const { addInferencePipelineModal: modal, isPipelineDataValid } = useValues(MLInferenceLogic);
-  const { createPipeline, setAddInferencePipelineStep } = useActions(MLInferenceLogic);
+  const { attachPipeline, createPipeline, setAddInferencePipelineStep } =
+    useActions(MLInferenceLogic);
 
+  const attachExistingPipeline = Boolean(modal.configuration.existingPipeline);
   let nextStep: AddInferencePipelineSteps | undefined;
   let previousStep: AddInferencePipelineSteps | undefined;
   switch (modal.step) {
@@ -239,6 +241,21 @@ const ModalFooter: React.FC<AddMLInferencePipelineModalProps & { ingestionMethod
             >
               {CONTINUE_BUTTON_LABEL}
             </EuiButton>
+          ) : attachExistingPipeline ? (
+            <EuiButton
+              color="primary"
+              data-telemetry-id={`entSearchContent-${ingestionMethod}-pipelines-addMlInference-attach`}
+              disabled={!isPipelineDataValid}
+              fill
+              onClick={attachPipeline}
+            >
+              {i18n.translate(
+                'xpack.enterpriseSearch.content.indices.transforms.addInferencePipelineModal.footer.attach',
+                {
+                  defaultMessage: 'Attach',
+                }
+              )}
+            </EuiButton>
           ) : (
             <EuiButton
               color="success"