Merge branch 'main' into query-built-in-roles

docs/changelog/110630.yaml

@@ -0,0 +1,5 @@
+pr: 110630
+summary: Telemetry for inference adaptive allocations
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/110816.yaml

@@ -0,0 +1,6 @@
+pr: 110816
+summary: GET _cluster/settings with include_defaults returns the expected fallback value if defined in elasticsearch.yml
+area: Infra/Settings
+type: bug
+issues:
+ - 110815

docs/changelog/110971.yaml

@@ -0,0 +1,5 @@
+pr: 110971
+summary: "Search in ES|QL: Add MATCH operator"
+area: ES|QL
+type: feature
+issues: []

docs/changelog/111071.yaml

@@ -0,0 +1,5 @@
+pr: 111071
+summary: Use native scalar scorer for int8_flat index
+area: Vector Search
+type: enhancement
+issues: []

docs/changelog/111290.yaml

@@ -0,0 +1,5 @@
+pr: 111290
+summary: Fix enrich policy runner exception handling on empty segments response
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/111311.yaml

@@ -0,0 +1,6 @@
+pr: 111311
+summary: Adding support for data streams with a match-all template
+area: Data streams
+type: bug
+issues:
+ - 111204

docs/changelog/111366.yaml

@@ -0,0 +1,6 @@
+pr: 111366
+summary: "[Inference API] Replace `model_id` with `inference_id` in inference API\
+  \ except when stored"
+area: Machine Learning
+type: bug
+issues: []

docs/reference/ingest/processors/geoip.asciidoc

@@ -64,12 +64,12 @@ depend on what has been found and which properties were configured in `propertie
 * If the GeoIP2 Domain database is used, then the following fields may be added under the `target_field`: `ip`, and `domain`.
 The fields actually added depend on what has been found and which properties were configured in `properties`.
 * If the GeoIP2 ISP database is used, then the following fields may be added under the `target_field`: `ip`, `asn`,
-`organization_name`, `network`, `isp`, `isp_organization`, `mobile_country_code`, and `mobile_network_code`. The fields actually added
+`organization_name`, `network`, `isp`, `isp_organization_name`, `mobile_country_code`, and `mobile_network_code`. The fields actually added
 depend on what has been found and which properties were configured in `properties`.
 * If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`,
 `country_iso_code`, `country_name`, `continent_code`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`,
 `location`, `asn`, `organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`,
-`residential_proxy`, `domain`, `isp`, `isp_organization`, `mobile_country_code`, `mobile_network_code`, `user_type`, and
+`residential_proxy`, `domain`, `isp`, `isp_organization_name`, `mobile_country_code`, `mobile_network_code`, `user_type`, and
 `connection_type`. The fields actually added  depend on what has been found and which properties were configured in `properties`.
 
 preview::["Do not use the GeoIP2 Anonymous IP, GeoIP2 Connection Type, GeoIP2 Domain, GeoIP2 ISP, and GeoIP2 Enterprise databases in production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."]

docs/reference/mapping/types/semantic-text.asciidoc

@@ -52,8 +52,8 @@ Use the <<put-inference-api>> to create the endpoint.
 The `inference_id` will not be validated when the mapping is created, but when documents are ingested into the index.
 When the first document is indexed, the `inference_id` will be used to generate underlying indexing structures for the field.
 
-WARNING: Removing an inference endpoint will cause ingestion of documents and semantic queries to fail on indices that define `semantic_text` fields with that inference endpoint as their `inference_id`.
-Please check that inference endpoints are not used in `semantic_text` fields before removal.
+WARNING: Removing an {infer} endpoint will cause ingestion of documents and semantic queries to fail on indices that define `semantic_text` fields with that {infer} endpoint as their `inference_id`.
+Trying to <<delete-inference-api,delete an {infer} endpoint>> that is used on a `semantic_text` field will result in an error.
 
 [discrete]
 [[auto-text-chunking]]
@@ -121,13 +121,19 @@ In case you want to customize data indexing, use the
 <<sparse-vector,`sparse_vector`>> or <<dense-vector,`dense_vector`>> field
 types and create an ingest pipeline with an
 <<inference-processor, {infer} processor>> to generate the embeddings.
-<<semantic-search-inference,This tutorial>> walks you through the process.
+<<semantic-search-inference,This tutorial>> walks you through the process. In
+these cases - when you use `sparse_vector` or `dense_vector` field types instead
+of the `semantic_text` field type to customize indexing - using the 
+<<query-dsl-semantic-query,`semantic_query`>> is not supported for querying the 
+field data.
+
 
 [discrete]
 [[update-script]]
 ==== Updates to `semantic_text` fields
 
-Updates that use scripts are not supported when the index contains a `semantic_text` field.
+Updates that use scripts are not supported for an index contains a `semantic_text` field.
+Even if the script targets non-`semantic_text` fields, the update will fail when the index contains a `semantic_text` field.
 
 
 [discrete]

docs/reference/query-dsl/semantic-query.asciidoc

@@ -128,6 +128,10 @@ If you want to fine-tune a search on a `semantic_text` field, you need to know t
 You can find the task type using the <<get-inference-api>>, and check the `task_type` associated with the {infer} service.
 Depending on the `task_type`, use either the <<query-dsl-sparse-vector-query,`sparse_vector`>> or the <<query-dsl-knn-query,`knn`>> query for greater flexibility and customization.
 
+NOTE: While it is possible to use the `sparse_vector` query or the `knn` query
+on a `semantic_text` field, it is not supported to use the `semantic_query` on a
+`sparse_vector` or `dense_vector` field type.
+
 
 [discrete]
 [[search-sparse-inference]]

docs/reference/search/retriever.asciidoc

@@ -209,11 +209,22 @@ GET /index/_search
 
 The `text_similarity_reranker` is a type of retriever that enhances search results by re-ranking documents based on semantic similarity to a specified inference text, using a machine learning model.
 
+[TIP]
+====
+Refer to <<semantic-reranking>> for a high level overview of semantic reranking.
+====
+
 ===== Prerequisites
 
 To use `text_similarity_reranker` you must first set up a `rerank` task using the <<put-inference-api, Create {infer} API>>.
 The `rerank` task should be set up with a machine learning model that can compute text similarity.
-Currently you can integrate directly with the Cohere Rerank endpoint using the <<infer-service-cohere,`cohere-rerank`>> task, or upload a model to {es} <<inference-example-eland,using Eland>>.
+
+Currently you can:
+
+* Integrate directly with the <<infer-service-cohere,Cohere Rerank inference endpoint>> using the `rerank` task type
+* Integrate directly with the <<infer-service-google-vertex-ai,Google Vertex AI inference endpoint>> using the `rerank` task type
+* Upload a model to {es} with {eland-docs}/machine-learning.html#ml-nlp-pytorch[Eland]
+** Then set up an <<inference-example-eland,{es} service inference endpoint>> with the `rerank` task type
 
 ===== Parameters
 
@@ -256,13 +267,13 @@ GET /index/_search
         "text_similarity_reranker": {
             "retriever": {
                 "standard": { ... }
-            }
-        },
-        "field": "text",
-        "inference_id": "my-cohere-rerank-model",
-        "inference_text": "Most famous landmark in Paris",
-        "rank_window_size": 100,
-        "min_score": 0.5
+            },
+            "field": "text",
+            "inference_id": "my-cohere-rerank-model",
+            "inference_text": "Most famous landmark in Paris",
+            "rank_window_size": 100,
+            "min_score": 0.5
+        }
     }
 }
 ----

docs/reference/search/search-your-data/retrievers-reranking/semantic-reranking.asciidoc

@@ -5,7 +5,7 @@ preview::[]
 
 [TIP]
 ====
-This overview focuses more on the high-level concepts and use cases for semantic reranking. For full implementation details on how to set up and use semantic reranking in {es}, see the <<retriever,reference documentation>> in the Search API docs.
+This overview focuses more on the high-level concepts and use cases for semantic reranking. For full implementation details on how to set up and use semantic reranking in {es}, see the <<text-similarity-reranker-retriever,reference documentation>> in the Search API docs.
 ====
 
 Rerankers improve the relevance of results from earlier-stage retrieval mechanisms.
@@ -89,11 +89,16 @@ In {es}, semantic rerankers are implemented using the {es} <<inference-apis,Infe
 
 To use semantic reranking in {es}, you need to:
 
-. Choose a reranking model. In addition to cross-encoder models running on {es} inference nodes, we also expose external models and services via the Inference API to semantic rerankers.
-** This includes cross-encoder models running in https://huggingface.co/inference-endpoints[HuggingFace Inference Endpoints] and the https://cohere.com/rerank[Cohere Rerank API].
-. Create a `rerank` task using the <<put-inference-api,{es} Inference API>>.
+. *Choose a reranking model*.
+Currently you can:
+
+** Integrate directly with the <<infer-service-cohere,Cohere Rerank inference endpoint>> using the `rerank` task type
+** Integrate directly with the <<infer-service-google-vertex-ai,Google Vertex AI inference endpoint>> using the `rerank` task type
+** Upload a model to {es} from Hugging Face with {eland-docs}/machine-learning.html#ml-nlp-pytorch[Eland]
+*** Then set up an <<inference-example-eland,{es} service inference endpoint>> with the `rerank` task type
+. *Create a `rerank` task using the <<put-inference-api,{es} Inference API>>*.
 The Inference API creates an inference endpoint and configures your chosen machine learning model to perform the reranking task.
-. Define a `text_similarity_reranker` retriever in your search request.
+. *Define a `text_similarity_reranker` retriever in your search request*.
 The retriever syntax makes it simple to configure both the retrieval and reranking of search results in a single API call.
 
 .*Example search request* with semantic reranker
@@ -127,20 +132,6 @@ POST _search
 // TEST[skip:TBD]
 ==============
 
-[discrete]
-[[semantic-reranking-types]]
-==== Supported reranking types
-
-The following `text_similarity_reranker` model configuration options are available.
-
-*Text similarity with cross-encoder*
-
-This solution uses a hosted or 3rd party inference service which relies on a cross-encoder model.
-The model receives the text fields from the _top-K_ documents, as well as the search query, and calculates scores directly, which are then used to rerank the documents.
-
-Used with the Cohere inference service rolled out in 8.13, turn on semantic reranking that works out of the box.
-Check out our https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/integrations/cohere/cohere-elasticsearch.ipynb[Python notebook] for using Cohere with {es}.
-
 [discrete]
 [[semantic-reranking-learn-more]]
 ==== Learn more

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APMMeterRegistry.java

@@ -38,6 +38,7 @@
 import org.elasticsearch.telemetry.metric.LongWithAttributes;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.locks.ReentrantLock;
@@ -96,11 +97,11 @@ public DoubleCounter getDoubleCounter(String name) {
     }
 
     @Override
-    public DoubleAsyncCounter registerDoubleAsyncCounter(
+    public DoubleAsyncCounter registerDoublesAsyncCounter(
         String name,
         String description,
         String unit,
-        Supplier<DoubleWithAttributes> observer
+        Supplier<Collection<DoubleWithAttributes>> observer
     ) {
         try (ReleasableLock lock = registerLock.acquire()) {
             return register(doubleAsynchronousCounters, new DoubleAsyncCounterAdapter(meter, name, description, unit, observer));
@@ -125,7 +126,12 @@ public DoubleUpDownCounter getDoubleUpDownCounter(String name) {
     }
 
     @Override
-    public DoubleGauge registerDoubleGauge(String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+    public DoubleGauge registerDoublesGauge(
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<DoubleWithAttributes>> observer
+    ) {
         try (ReleasableLock lock = registerLock.acquire()) {
             return register(doubleGauges, new DoubleGaugeAdapter(meter, name, description, unit, observer));
         }
@@ -156,7 +162,12 @@ public LongCounter registerLongCounter(String name, String description, String u
     }
 
     @Override
-    public LongAsyncCounter registerLongAsyncCounter(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+    public LongAsyncCounter registerLongsAsyncCounter(
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<LongWithAttributes>> observer
+    ) {
         try (ReleasableLock lock = registerLock.acquire()) {
             return register(longAsynchronousCounters, new LongAsyncCounterAdapter(meter, name, description, unit, observer));
         }
@@ -185,7 +196,7 @@ public LongUpDownCounter getLongUpDownCounter(String name) {
     }
 
     @Override
-    public LongGauge registerLongGauge(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+    public LongGauge registerLongsGauge(String name, String description, String unit, Supplier<Collection<LongWithAttributes>> observer) {
         try (ReleasableLock lock = registerLock.acquire()) {
             return register(longGauges, new LongGaugeAdapter(meter, name, description, unit, observer));
         }

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleAsyncCounterAdapter.java

@@ -15,12 +15,19 @@
 import org.elasticsearch.telemetry.metric.DoubleAsyncCounter;
 import org.elasticsearch.telemetry.metric.DoubleWithAttributes;
 
+import java.util.Collection;
 import java.util.Objects;
 import java.util.function.Supplier;
 
 public class DoubleAsyncCounterAdapter extends AbstractInstrument<ObservableDoubleCounter> implements DoubleAsyncCounter {
 
-    public DoubleAsyncCounterAdapter(Meter meter, String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+    public DoubleAsyncCounterAdapter(
+        Meter meter,
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<DoubleWithAttributes>> observer
+    ) {
         super(meter, new Builder(name, description, unit, observer));
     }
 
@@ -30,9 +37,9 @@ public void close() throws Exception {
     }
 
     private static class Builder extends AbstractInstrument.Builder<ObservableDoubleCounter> {
-        private final Supplier<DoubleWithAttributes> observer;
+        private final Supplier<Collection<DoubleWithAttributes>> observer;
 
-        private Builder(String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+        private Builder(String name, String description, String unit, Supplier<Collection<DoubleWithAttributes>> observer) {
             super(name, description, unit);
             this.observer = Objects.requireNonNull(observer);
         }

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.telemetry.apm.AbstractInstrument;
 import org.elasticsearch.telemetry.metric.DoubleWithAttributes;
 
+import java.util.Collection;
 import java.util.Objects;
 import java.util.function.Supplier;
 
@@ -24,7 +25,13 @@ public class DoubleGaugeAdapter extends AbstractInstrument<ObservableDoubleGauge
     implements
         org.elasticsearch.telemetry.metric.DoubleGauge {
 
-    public DoubleGaugeAdapter(Meter meter, String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+    public DoubleGaugeAdapter(
+        Meter meter,
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<DoubleWithAttributes>> observer
+    ) {
         super(meter, new Builder(name, description, unit, observer));
     }
 
@@ -34,9 +41,9 @@ public void close() throws Exception {
     }
 
     private static class Builder extends AbstractInstrument.Builder<ObservableDoubleGauge> {
-        private final Supplier<DoubleWithAttributes> observer;
+        private final Supplier<Collection<DoubleWithAttributes>> observer;
 
-        private Builder(String name, String description, String unit, Supplier<DoubleWithAttributes> observer) {
+        private Builder(String name, String description, String unit, Supplier<Collection<DoubleWithAttributes>> observer) {
             super(name, description, unit);
             this.observer = Objects.requireNonNull(observer);
         }

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongAsyncCounterAdapter.java

@@ -15,12 +15,19 @@
 import org.elasticsearch.telemetry.metric.LongAsyncCounter;
 import org.elasticsearch.telemetry.metric.LongWithAttributes;
 
+import java.util.Collection;
 import java.util.Objects;
 import java.util.function.Supplier;
 
 public class LongAsyncCounterAdapter extends AbstractInstrument<ObservableLongCounter> implements LongAsyncCounter {
 
-    public LongAsyncCounterAdapter(Meter meter, String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+    public LongAsyncCounterAdapter(
+        Meter meter,
+        String name,
+        String description,
+        String unit,
+        Supplier<Collection<LongWithAttributes>> observer
+    ) {
         super(meter, new Builder(name, description, unit, observer));
     }
 
@@ -30,9 +37,9 @@ public void close() throws Exception {
     }
 
     private static class Builder extends AbstractInstrument.Builder<ObservableLongCounter> {
-        private final Supplier<LongWithAttributes> observer;
+        private final Supplier<Collection<LongWithAttributes>> observer;
 
-        private Builder(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+        private Builder(String name, String description, String unit, Supplier<Collection<LongWithAttributes>> observer) {
             super(name, description, unit);
             this.observer = Objects.requireNonNull(observer);
         }

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java

@@ -14,14 +14,15 @@
 import org.elasticsearch.telemetry.apm.AbstractInstrument;
 import org.elasticsearch.telemetry.metric.LongWithAttributes;
 
+import java.util.Collection;
 import java.util.Objects;
 import java.util.function.Supplier;
 
 /**
  * LongGaugeAdapter wraps an otel ObservableLongGauge
  */
 public class LongGaugeAdapter extends AbstractInstrument<ObservableLongGauge> implements org.elasticsearch.telemetry.metric.LongGauge {
-    public LongGaugeAdapter(Meter meter, String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+    public LongGaugeAdapter(Meter meter, String name, String description, String unit, Supplier<Collection<LongWithAttributes>> observer) {
         super(meter, new Builder(name, description, unit, observer));
     }
 
@@ -31,11 +32,11 @@ public void close() throws Exception {
     }
 
     private static class Builder extends AbstractInstrument.Builder<ObservableLongGauge> {
-        private final Supplier<LongWithAttributes> observer;
+        private final Supplier<Collection<LongWithAttributes>> observer;
 
-        private Builder(String name, String description, String unit, Supplier<LongWithAttributes> observer) {
+        private Builder(String name, String description, String unit, Supplier<Collection<LongWithAttributes>> observer) {
             super(name, description, unit);
-            this.observer = Objects.requireNonNull(observer);
+            this.observer = observer;
         }
 
         @Override