Merge branch 'main' into update-ironbank-maintainers

docs/changelog/117589.yaml

@@ -0,0 +1,5 @@
+pr: 117589
+summary: "Add Inference Unified API for chat completions for OpenAI"
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/118064.yaml

@@ -0,0 +1,5 @@
+pr: 118064
+summary: Add Highlighter for Semantic Text Fields
+area: Highlighting
+type: feature
+issues: []

docs/reference/mapping/types/semantic-text.asciidoc

@@ -112,50 +112,43 @@ Trying to <<delete-inference-api,delete an {infer} endpoint>> that is used on a
 {infer-cap} endpoints have a limit on the amount of text they can process.
 To allow for large amounts of text to be used in semantic search, `semantic_text` automatically generates smaller passages if needed, called _chunks_.
 
-Each chunk will include the text subpassage and the corresponding embedding generated from it.
+Each chunk refers to a passage of the text and the corresponding embedding generated from it.
 When querying, the individual passages will be automatically searched for each document, and the most relevant passage will be used to compute a score.
 
 For more details on chunking and how to configure chunking settings, see <<infer-chunking-config, Configuring chunking>> in the Inference API documentation.
 
+Refer to <<semantic-search-semantic-text,this tutorial>> to learn more about
+semantic search using `semantic_text` and the `semantic` query.
 
 [discrete]
-[[semantic-text-structure]]
-==== `semantic_text` structure
+[[semantic-text-highlighting]]
+==== Extracting Relevant Fragments from Semantic Text
 
-Once a document is ingested, a `semantic_text` field will have the following structure:
+You can extract the most relevant fragments from a semantic text field by using the <<highlighting,highlight parameter>> in the <<search-search-api-request-body,Search API>>.
 
-[source,console-result]
+[source,console]
 ------------------------------------------------------------
-"inference_field": {
-  "text": "these are not the droids you're looking for", <1>
-  "inference": {
-    "inference_id": "my-elser-endpoint", <2>
-    "model_settings": { <3>
-      "task_type": "sparse_embedding"
+PUT test-index
+{
+    "query": {
+        "semantic": {
+            "field": "my_semantic_field"
+        }
     },
-    "chunks": [ <4>
-      {
-        "text": "these are not the droids you're looking for",
-        "embeddings": {
-          (...)
+    "highlight": {
+        "fields": {
+            "my_semantic_field": {
+                "type": "semantic",
+                "number_of_fragments": 2,  <1>
+                "order": "score"           <2>
+            }
         }
-      }
-    ]
-  }
+    }
 }
 ------------------------------------------------------------
-// TEST[skip:TBD]
-<1> The field will become an object structure to accommodate both the original
-text and the inference results.
-<2> The `inference_id` used to generate the embeddings.
-<3> Model settings, including the task type and dimensions/similarity if
-applicable.
-<4> Inference results will be grouped in chunks, each with its corresponding
-text and embeddings.
-
-Refer to <<semantic-search-semantic-text,this tutorial>> to learn more about
-semantic search using `semantic_text` and the `semantic` query.
-
+// TEST[skip:Requires inference endpoint]
+<1> Specifies the maximum number of fragments to return.
+<2> Sorts highlighted fragments by score when set to `score`. By default, fragments will be output in the order they appear in the field (order: none).
 
 [discrete]
 [[custom-indexing]]

muted-tests.yml

@@ -117,9 +117,6 @@ tests:
 - class: org.elasticsearch.xpack.deprecation.DeprecationHttpIT
   method: testDeprecatedSettingsReturnWarnings
   issue: https://github.com/elastic/elasticsearch/issues/108628
-- class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
-  method: testBottomFieldSort
-  issue: https://github.com/elastic/elasticsearch/issues/116249
 - class: org.elasticsearch.xpack.shutdown.NodeShutdownIT
   method: testAllocationPreventedForRemoval
   issue: https://github.com/elastic/elasticsearch/issues/116363
@@ -242,9 +239,6 @@ tests:
 - class: org.elasticsearch.packaging.test.ConfigurationTests
   method: test30SymlinkedDataPath
   issue: https://github.com/elastic/elasticsearch/issues/118111
-- class: org.elasticsearch.datastreams.ResolveClusterDataStreamIT
-  method: testClusterResolveWithDataStreamsUsingAlias
-  issue: https://github.com/elastic/elasticsearch/issues/118124
 - class: org.elasticsearch.packaging.test.KeystoreManagementTests
   method: test30KeystorePasswordFromFile
   issue: https://github.com/elastic/elasticsearch/issues/118123
@@ -266,6 +260,18 @@ tests:
 - class: org.elasticsearch.packaging.test.DebPreservationTests
   method: test40RestartOnUpgrade
   issue: https://github.com/elastic/elasticsearch/issues/118170
+- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
+  method: testInferDeploysDefaultRerank
+  issue: https://github.com/elastic/elasticsearch/issues/118184
+- class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT
+  method: testCancelRequestWhenFailingFetchingPages
+  issue: https://github.com/elastic/elasticsearch/issues/118193
+- class: org.elasticsearch.packaging.test.MemoryLockingTests
+  method: test20MemoryLockingEnabled
+  issue: https://github.com/elastic/elasticsearch/issues/118195
+- class: org.elasticsearch.packaging.test.ArchiveTests
+  method: test42AutoconfigurationNotTriggeredWhenNodeCannotBecomeMaster
+  issue: https://github.com/elastic/elasticsearch/issues/118196
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.xcontent.ToXContent;
 
+import java.util.Collections;
 import java.util.Iterator;
 
 public enum ChunkedToXContentHelper {
@@ -53,6 +54,14 @@ public static Iterator<ToXContent> field(String name, String value) {
         return Iterators.single(((builder, params) -> builder.field(name, value)));
     }
 
+    public static Iterator<ToXContent> optionalField(String name, String value) {
+        if (value == null) {
+            return Collections.emptyIterator();
+        } else {
+            return field(name, value);
+        }
+    }
+
     /**
      * Creates an Iterator of a single ToXContent object that serializes the given object as a single chunk. Just wraps {@link
      * Iterators#single}, but still useful because it avoids any type ambiguity.

server/src/main/java/org/elasticsearch/inference/InferenceService.java

@@ -112,6 +112,23 @@ void infer(
     );
 
     /**
+     * Perform completion inference on the model using the unified schema.
+     *
+     * @param model        The model
+     * @param request Parameters for the request
+     * @param timeout      The timeout for the request
+     * @param listener     Inference result listener
+     */
+    void unifiedCompletionInfer(
+        Model model,
+        UnifiedCompletionRequest request,
+        TimeValue timeout,
+        ActionListener<InferenceServiceResults> listener
+    );
+
+    /**
+     * Chunk long text.
+     *
      * @param model           The model
      * @param query           Inference query, mainly for re-ranking
      * @param input           Inference input

server/src/main/java/org/elasticsearch/inference/TaskType.java

@@ -38,6 +38,10 @@ public static TaskType fromString(String name) {
     }
 
     public static TaskType fromStringOrStatusException(String name) {
+        if (name == null) {
+            throw new ElasticsearchStatusException("Task type must not be null", RestStatus.BAD_REQUEST);
+        }
+
         try {
             TaskType taskType = TaskType.fromString(name);
             return Objects.requireNonNull(taskType);