Merge branch 'main' into index-metrics-version-conflicts

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -71,7 +71,7 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
             maybeSetActiveProcessorCount(nodeSettings),
             maybeSetReplayFile(distroType, isHotspot),
             maybeWorkaroundG1Bug(),
-            maybeAllowSecurityManager(),
+            maybeAllowSecurityManager(useEntitlements),
             maybeAttachEntitlementAgent(useEntitlements)
         ).flatMap(s -> s).toList();
     }
@@ -140,8 +140,8 @@ private static Stream<String> maybeWorkaroundG1Bug() {
     }
 
     @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    private static Stream<String> maybeAllowSecurityManager() {
-        if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
+    private static Stream<String> maybeAllowSecurityManager(boolean useEntitlements) {
+        if (useEntitlements == false && RuntimeVersionFeature.isSecurityManagerAvailable()) {
             // Will become conditional on useEntitlements once entitlements can run without SM
             return Stream.of("-Djava.security.manager=allow");
         }
@@ -167,12 +167,16 @@ private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlement
         } catch (IOException e) {
             throw new IllegalStateException("Failed to list entitlement jars in: " + dir, e);
         }
+        // We instrument classes in these modules to call the bridge. Because the bridge gets patched
+        // into java.base, we must export the bridge from java.base to these modules.
+        String modulesContainingEntitlementInstrumentation = "java.logging";
         return Stream.of(
             "-Des.entitlements.enabled=true",
             "-XX:+EnableDynamicAgentLoading",
             "-Djdk.attach.allowAttachSelf=true",
             "--patch-module=java.base=" + bridgeJar,
-            "--add-exports=java.base/org.elasticsearch.entitlement.bridge=org.elasticsearch.entitlement"
+            "--add-exports=java.base/org.elasticsearch.entitlement.bridge=org.elasticsearch.entitlement,"
+                + modulesContainingEntitlementInstrumentation
         );
     }
 }

docs/build.gradle

@@ -178,8 +178,9 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
 }
 
 tasks.named("yamlRestTest").configure {
+  def repoFolder = "${layout.buildDirectory.asFile.get()}/cluster/shared/repo"
   doFirst {
-    delete("${buildDir}/cluster/shared/repo")
+    delete(repoFolder)
   }
 }
 

docs/changelog/118774.yaml

@@ -0,0 +1,5 @@
+pr: 118774
+summary: Apply default k for knn query eagerly
+area: Vector Search
+type: bug
+issues: []

docs/changelog/118919.yaml

@@ -0,0 +1,5 @@
+pr: 118919
+summary: Remove unsupported timeout from rest-api-spec license API
+area: License
+type: bug
+issues: []

docs/changelog/118921.yaml

@@ -0,0 +1,5 @@
+pr: 118921
+summary: Add missing timeouts to rest-api-spec shutdown APIs
+area: Infra/Node Lifecycle
+type: bug
+issues: []

docs/changelog/118954.yaml

@@ -0,0 +1,5 @@
+pr: 118954
+summary: Add missing parameter to `xpack.info` rest-api-spec
+area: Infra/REST API
+type: bug
+issues: []

docs/changelog/119348.yaml

@@ -0,0 +1,5 @@
+pr: 119348
+summary: Auto-migrate `max_page_search_size`
+area: Transform
+type: enhancement
+issues: []

docs/changelog/119389.yaml

@@ -0,0 +1,5 @@
+pr: 119389
+summary: Restrict Connector APIs to manage/monitor_connector privileges
+area: Extract&Transform
+type: feature
+issues: []

docs/changelog/119564.yaml

@@ -0,0 +1,5 @@
+pr: 119564
+summary: Http stream activity tracker and exceptions handling
+area: Network
+type: enhancement
+issues: []

docs/changelog/119621.yaml

@@ -0,0 +1,5 @@
+pr: 119621
+summary: Enable node-level reduction by default
+area: ES|QL
+type: enhancement
+issues: []

docs/reference/inference/inference-apis.asciidoc

@@ -149,6 +149,7 @@ include::service-elser.asciidoc[]
 include::service-google-ai-studio.asciidoc[]
 include::service-google-vertex-ai.asciidoc[]
 include::service-hugging-face.asciidoc[]
+include::service-jinaai.asciidoc[]
 include::service-mistral.asciidoc[]
 include::service-openai.asciidoc[]
 include::service-watsonx-ai.asciidoc[]

docs/reference/inference/put-inference.asciidoc

@@ -78,6 +78,7 @@ Click the links to review the configuration details of the services:
 * <<infer-service-mistral,Mistral>> (`text_embedding`)
 * <<infer-service-openai,OpenAI>> (`completion`, `text_embedding`)
 * <<infer-service-watsonx-ai>> (`text_embedding`)
+* <<infer-service-jinaai,JinaAI>> (`text_embedding`, `rerank`)
 
 The {es} and ELSER services run on a {ml} node in your {es} cluster. The rest of
 the services connect to external providers.
@@ -93,4 +94,4 @@ When adaptive allocations are enabled:
 - The number of allocations scales up automatically when the load increases.
 - Allocations scale down to a minimum of 0 when the load decreases, saving resources.
 
-For more information about adaptive allocations and resources, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] documentation.
+For more information about adaptive allocations and resources, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] documentation.

docs/reference/inference/service-jinaai.asciidoc

@@ -0,0 +1,255 @@
+[[infer-service-jinaai]]
+=== JinaAI {infer} service
+
+Creates an {infer} endpoint to perform an {infer} task with the `jinaai` service.
+
+
+[discrete]
+[[infer-service-jinaai-api-request]]
+==== {api-request-title}
+
+`PUT /_inference/<task_type>/<inference_id>`
+
+[discrete]
+[[infer-service-jinaai-api-path-params]]
+==== {api-path-parms-title}
+
+`<inference_id>`::
+(Required, string)
+include::inference-shared.asciidoc[tag=inference-id]
+
+`<task_type>`::
+(Required, string)
+include::inference-shared.asciidoc[tag=task-type]
++
+--
+Available task types:
+
+* `text_embedding`,
+* `rerank`.
+--
+
+[discrete]
+[[infer-service-jinaai-api-request-body]]
+==== {api-request-body-title}
+
+`chunking_settings`::
+(Optional, object)
+include::inference-shared.asciidoc[tag=chunking-settings]
+
+`max_chunking_size`:::
+(Optional, integer)
+include::inference-shared.asciidoc[tag=chunking-settings-max-chunking-size]
+
+`overlap`:::
+(Optional, integer)
+include::inference-shared.asciidoc[tag=chunking-settings-overlap]
+
+`sentence_overlap`:::
+(Optional, integer)
+include::inference-shared.asciidoc[tag=chunking-settings-sentence-overlap]
+
+`strategy`:::
+(Optional, string)
+include::inference-shared.asciidoc[tag=chunking-settings-strategy]
+
+`service`::
+(Required, string)
+The type of service supported for the specified task type. In this case, 
+`jinaai`.
+
+`service_settings`::
+(Required, object)
+include::inference-shared.asciidoc[tag=service-settings]
++
+--
+These settings are specific to the `jinaai` service.
+--
+
+`api_key`:::
+(Required, string)
+A valid API key for your JinaAI account.
+You can find it at https://jina.ai/embeddings/.
++
+--
+include::inference-shared.asciidoc[tag=api-key-admonition]
+--
+
+`rate_limit`:::
+(Optional, object)
+The default rate limit for the `jinaai` service is 2000 requests per minute for all task types. 
+You can modify this using the `requests_per_minute` setting in your service settings:
++
+--
+include::inference-shared.asciidoc[tag=request-per-minute-example]
+
+More information about JinaAI's rate limits can be found in https://jina.ai/contact-sales/#rate-limit.
+--
++
+.`service_settings` for the `rerank` task type
+[%collapsible%closed]
+=====
+`model_id`::
+(Required, string)
+The name of the model to use for the {infer} task.
+To review the available `rerank` compatible models, refer to https://jina.ai/reranker.
+=====
++
+.`service_settings` for the `text_embedding` task type
+[%collapsible%closed]
+=====
+`model_id`:::
+(Optional, string)
+The name of the model to use for the {infer} task.
+To review the available `text_embedding` models, refer to the
+https://jina.ai/embeddings/.
+
+`similarity`:::
+(Optional, string)
+Similarity measure. One of `cosine`, `dot_product`, `l2_norm`.
+Defaults based on the `embedding_type` (`float` -> `dot_product`, `int8/byte` -> `cosine`).
+=====
+
+
+
+`task_settings`::
+(Optional, object)
+include::inference-shared.asciidoc[tag=task-settings]
++
+.`task_settings` for the `rerank` task type
+[%collapsible%closed]
+=====
+`return_documents`::
+(Optional, boolean)
+Specify whether to return doc text within the results.
+
+`top_n`::
+(Optional, integer)
+The number of most relevant documents to return, defaults to the number of the documents.
+If this {infer} endpoint is used in a `text_similarity_reranker` retriever query and `top_n` is set, it must be greater than or equal to `rank_window_size` in the query.
+=====
++
+.`task_settings` for the `text_embedding` task type
+[%collapsible%closed]
+=====
+`task`:::
+(Optional, string)
+Specifies the task passed to the model.
+Valid values are:
+* `classification`: use it for embeddings passed through a text classifier.
+* `clustering`: use it for the embeddings run through a clustering algorithm.
+* `ingest`: use it for storing document embeddings in a vector database.
+* `search`: use it for storing embeddings of search queries run against a vector database to find relevant documents.
+=====
+
+
+[discrete]
+[[inference-example-jinaai]]
+==== JinaAI service examples
+
+The following examples demonstrate how to create {infer} endpoints for `text_embeddings` and `rerank` tasks using the JinaAI service and use them in search requests.
+
+First, we create the `embeddings` service:
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/jinaai-embeddings
+{
+    "service": "jinaai",
+    "service_settings": {
+        "model_id": "jina-embeddings-v3",
+        "api_key": "<api_key>"
+    }
+}
+------------------------------------------------------------
+// TEST[skip:uses ML]
+
+Then, we create the `rerank` service:
+[source,console]
+------------------------------------------------------------
+PUT _inference/rerank/jinaai-rerank
+{
+    "service": "jinaai",
+    "service_settings": {
+        "api_key": "<api_key>",
+        "model_id": "jina-reranker-v2-base-multilingual"
+    },
+    "task_settings": {
+        "top_n": 10,
+        "return_documents": true
+    }
+}
+------------------------------------------------------------
+// TEST[skip:uses ML]
+
+Now we can create an index that will use `jinaai-embeddings` service to index the documents.
+
+[source,console]
+------------------------------------------------------------
+PUT jinaai-index
+{
+  "mappings": {
+    "properties": {
+      "content": {
+        "type": "semantic_text",
+        "inference_id": "jinaai-embeddings"
+      }
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:uses ML]
+
+[source,console]
+------------------------------------------------------------
+PUT jinaai-index/_bulk
+{ "index" : { "_index" : "jinaai-index", "_id" : "1" } }
+{"content": "Sarah Johnson is a talented marine biologist working at the Oceanographic Institute. Her groundbreaking research on coral reef ecosystems has garnered international attention and numerous accolades."}
+{ "index" : { "_index" : "jinaai-index", "_id" : "2" } }
+{"content": "She spends months at a time diving in remote locations, meticulously documenting the intricate relationships between various marine species. "}
+{ "index" : { "_index" : "jinaai-index", "_id" : "3" } }
+{"content": "Her dedication to preserving these delicate underwater environments has inspired a new generation of conservationists."}
+------------------------------------------------------------
+// TEST[skip:uses ML]
+
+Now, with the index created, we can search with and without the reranker service.
+
+[source,console]
+------------------------------------------------------------
+GET jinaai-index/_search 
+{
+  "query": {
+    "semantic": {
+      "field": "content",
+      "query": "who inspired taking care of the sea?"
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:uses ML]
+
+[source,console]
+------------------------------------------------------------
+POST jinaai-index/_search
+{
+  "retriever": {
+    "text_similarity_reranker": {
+      "retriever": {
+        "standard": {
+          "query": {
+            "semantic": {
+              "field": "content",
+              "query": "who inspired taking care of the sea?"
+            }
+          }
+        }
+      },
+      "field": "content",
+      "rank_window_size": 100,
+      "inference_id": "jinaai-rerank",
+      "inference_text": "who inspired taking care of the sea?"
+    }
+  }
+}
+------------------------------------------------------------
+// TEST[skip:uses ML]

docs/reference/query-dsl/knn-query.asciidoc

@@ -100,7 +100,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-query-vector-builde
 --
 (Optional, integer) The number of nearest neighbors to return from each shard.
 {es} collects `k` results from each shard, then merges them to find the global top results.
-This value must be less than or equal to `num_candidates`. Defaults to `num_candidates`.
+This value must be less than or equal to `num_candidates`. Defaults to search request size.
 --
 
 `num_candidates`::

docs/reference/search/search-your-data/highlighting.asciidoc

@@ -262,9 +262,11 @@ max_analyzed_offset:: By default, the maximum number of characters
 analyzed for a highlight request is bounded by the value defined in the
 <<index-max-analyzed-offset, `index.highlight.max_analyzed_offset`>> setting,
 and when the number of characters exceeds this limit an error is returned. If
-this setting is set to a non-negative value, the highlighting stops at this defined
+this setting is set to a positive value, the highlighting stops at this defined
 maximum limit, and the rest of the text is not processed, thus not highlighted and
-no error is returned. The <<max-analyzed-offset, `max_analyzed_offset`>> query setting
+no error is returned. If it is specifically set to -1 then the value of
+<<index-max-analyzed-offset, `index.highlight.max_analyzed_offset`>> is used instead.
+For values < -1 or 0, an error is returned. The <<max-analyzed-offset, `max_analyzed_offset`>> query setting
 does *not* override the <<index-max-analyzed-offset, `index.highlight.max_analyzed_offset`>>
 which prevails when it's set to lower value than the query setting.