Merge remote-tracking branch 'origin/main' into updates_by_script_ras

branches.json

@@ -4,6 +4,9 @@
     {
       "branch": "main"
     },
+    {
+      "branch": "8.15"
+    },
     {
       "branch": "8.14"
     },

build-tools-internal/build.gradle

@@ -274,10 +274,7 @@ dependencies {
     // ensuring brought asm version brought in by spock is up-to-date
     testImplementation buildLibs.asm
     integTestImplementation buildLibs.asm
-    integTestImplementation('org.ow2.asm:asm:9.6')
-    api("org.yaml:snakeyaml") {
-      version { strictly(versions.snakeyaml) }
-    }
+    api(buildLibs.snakeyaml)
   }
   // Forcefully downgrade the jackson platform as used in production
   api enforcedPlatform(buildLibs.jackson.platform)
@@ -314,7 +311,7 @@ dependencies {
   compileOnly buildLibs.checkstyle
   compileOnly buildLibs.reflections
 
-  implementation 'com.github.javaparser:javaparser-core:3.18.0'
+  implementation buildLibs.javaparser
 
   runtimeOnly "org.elasticsearch.gradle:reaper:$version"
   testImplementation buildLibs.checkstyle

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -26,13 +26,10 @@ develocity {
     if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') {
       publishing.onlyIf { true }
       server = 'https://gradle-enterprise.elastic.co'
-    } else {
-      publishing.onlyIf {
-        server.isPresent();
-      }
+    } else if( server.isPresent() == false) {
+      publishing.onlyIf { false }
     }
 
-
     background {
       tag OS.current().name()
       tag Architecture.current().name()

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -25,6 +25,7 @@
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.testing.Test;
@@ -33,16 +34,21 @@
 import java.util.List;
 import java.util.Map;
 
+import javax.inject.Inject;
+
 import static org.elasticsearch.gradle.util.FileUtils.mkdirs;
 import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
 
 /**
  * Applies commonly used settings to all Test tasks in the project
  */
-public class ElasticsearchTestBasePlugin implements Plugin<Project> {
+public abstract class ElasticsearchTestBasePlugin implements Plugin<Project> {
 
     public static final String DUMP_OUTPUT_ON_FAILURE_PROP_NAME = "dumpOutputOnFailure";
 
+    @Inject
+    protected abstract ProviderFactory getProviderFactory();
+
     @Override
     public void apply(Project project) {
         project.getPluginManager().apply(GradleTestPolicySetupPlugin.class);
@@ -150,13 +156,11 @@ public void execute(Task t) {
             // we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
             nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp"));
 
+            test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("tests.").get());
+            test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("es.").get());
+
             // TODO: remove setting logging level via system property
             test.systemProperty("tests.logger.level", "WARN");
-            System.getProperties().entrySet().forEach(entry -> {
-                if ((entry.getKey().toString().startsWith("tests.") || entry.getKey().toString().startsWith("es."))) {
-                    test.systemProperty(entry.getKey().toString(), entry.getValue());
-                }
-            });
 
             // TODO: remove this once ctx isn't added to update script params in 7.0
             test.systemProperty("es.scripting.update.ctx_in_params", "false");

docs/changelog/108395.yaml

@@ -0,0 +1,5 @@
+pr: 108395
+summary: "ESQL: change from quoting from backtick to quote"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/108733.yaml

@@ -0,0 +1,5 @@
+pr: 108733
+summary: Query Roles API
+area: Security
+type: feature
+issues: []

docs/changelog/109084.yaml

@@ -0,0 +1,5 @@
+pr: 109084
+summary: Add AVX-512 optimised vector distance functions for int7 on x64
+area: Search
+type: enhancement
+issues: []

docs/changelog/109813.yaml

@@ -0,0 +1,5 @@
+pr: 109813
+summary: Add text similarity reranker retriever
+area: Ranking
+type: feature
+issues: []

docs/changelog/109948.yaml

@@ -0,0 +1,5 @@
+pr: 109948
+summary: Automatically adjust `ignore_malformed` only for the @timestamp
+area: Mapping
+type: bug
+issues: []

docs/changelog/109993.yaml

@@ -0,0 +1,5 @@
+pr: 109993
+summary: "[ES|QL] `weighted_avg`"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/110096.yaml

@@ -0,0 +1,6 @@
+pr: 110096
+summary: Fix `ClassCastException` with MV_EXPAND on missing field
+area: ES|QL
+type: bug
+issues:
+ - 109974

docs/changelog/110233.yaml

@@ -0,0 +1,6 @@
+pr: 110233
+summary: Support k parameter for knn query
+area: Vector Search
+type: enhancement
+issues:
+ - 108473

docs/changelog/110236.yaml

@@ -0,0 +1,21 @@
+pr: 110236
+summary: '`ParseHeapRatioOrDeprecatedByteSizeValue` for `indices.breaker.total.limit`'
+area: Infra/Settings
+type: deprecation
+issues: []
+deprecation:
+  title: 'Deprecate absolute size values for `indices.breaker.total.limit` setting'
+  area: Cluster and node setting
+  details: Previously, the value of `indices.breaker.total.limit` could be specified as
+    an absolute size in bytes. This setting controls the overal amount of
+    memory the server is allowed to use before taking remedial actions. Setting
+    this to a specific number of bytes led to strange behaviour when the node
+    maximum heap size changed because the circut breaker limit would remain
+    unchanged. This would either leave the value too low, causing part of the
+    heap to remain unused; or it would leave the value too high, causing the
+    circuit breaker to be ineffective at preventing OOM errors.  The only
+    reasonable behaviour for this setting is that it scales with the size of
+    the heap, and so absolute byte limits are now deprecated.
+  impact: Users must change their configuration to specify a percentage instead of
+    an absolute number of bytes for `indices.breaker.total.limit`, or else
+    accept the default, which is already specified as a percentage.

docs/changelog/110251.yaml

@@ -0,0 +1,13 @@
+pr: 110251
+summary: Support index sorting with nested fields
+area: Logs
+type: enhancement
+issues:
+ - 107349
+highlight:
+  title: Index sorting on indexes with nested fields
+  body: |-
+    Index sorting is now supported for indexes with mappings containing nested objects.
+    The index sort spec (as specified by `index.sort.field`) can't contain any nested
+    fields, still.
+  notable: false

docs/changelog/110268.yaml

@@ -0,0 +1,6 @@
+pr: 110268
+summary: Disallow index.time_series.end_time setting from being set or updated in normal indices
+area: TSDB
+type: bug
+issues:
+ - 110265

docs/changelog/110334.yaml

@@ -0,0 +1,5 @@
+pr: 110334
+summary: Sentence Chunker
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/110337.yaml

@@ -0,0 +1,5 @@
+pr: 110337
+summary: Support `ignore_above` on keyword dimensions
+area: TSDB
+type: enhancement
+issues: []

docs/changelog/110338.yaml

@@ -0,0 +1,5 @@
+pr: 110338
+summary: Add `semantic_text` field type and `semantic` query
+area: Mapping
+type: feature
+issues: []

docs/changelog/110347.yaml

@@ -0,0 +1,5 @@
+pr: 110347
+summary: "ESQL: Renamed `TopList` to Top"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/110369.yaml

@@ -0,0 +1,6 @@
+pr: 110369
+summary: Run terms concurrently when cardinality is only lower than shard size
+area: Aggregations
+type: bug
+issues:
+ - 105505

docs/changelog/110395.yaml

@@ -0,0 +1,9 @@
+pr: 110395
+summary: Mark the Redact processor as Generally Available
+area: Ingest Node
+type: feature
+issues: []
+highlight:
+  title: The Redact processor is Generally Available
+  body: The Redact processor uses the Grok rules engine to obscure text in the input document matching the given Grok patterns. The Redact processor was initially released as Technical Preview in `8.7.0`, and is now released as Generally Available.
+  notable: true

docs/reference/esql/functions/aggregation-functions.asciidoc

@@ -18,8 +18,9 @@ The <<esql-stats-by>> command supports these aggregate functions:
 * <<esql-agg-percentile>>
 * experimental:[] <<esql-agg-st-centroid>>
 * <<esql-agg-sum>>
-* <<esql-top_list>>
+* <<esql-top>>
 * <<esql-agg-values>>
+* experimental:[] <<esql-agg-weighted-avg>>
 // end::agg_list[]
 
 include::avg.asciidoc[]
@@ -32,5 +33,6 @@ include::min.asciidoc[]
 include::percentile.asciidoc[]
 include::st_centroid_agg.asciidoc[]
 include::sum.asciidoc[]
-include::layout/top_list.asciidoc[]
+include::layout/top.asciidoc[]
 include::values.asciidoc[]
+include::weighted-avg.asciidoc[]

docs/reference/esql/functions/binary.asciidoc

@@ -7,6 +7,12 @@
 [.text-center]
 image::esql/functions/signature/equals.svg[Embedded,opts=inline]
 
+Check if two fields are equal. If either field is <<esql-multivalued-fields,multivalued>> then
+the result is `null`.
+
+NOTE: This is pushed to the underlying search index if one side of the comparison is constant
+      and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.
+
 Supported types:
 
 include::types/equals.asciidoc[]
@@ -15,6 +21,12 @@ include::types/equals.asciidoc[]
 [.text-center]
 image::esql/functions/signature/not_equals.svg[Embedded,opts=inline]
 
+Check if two fields are unequal. If either field is <<esql-multivalued-fields,multivalued>> then
+the result is `null`.
+
+NOTE: This is pushed to the underlying search index if one side of the comparison is constant
+      and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.
+
 Supported types:
 
 include::types/not_equals.asciidoc[]
@@ -23,55 +35,112 @@ include::types/not_equals.asciidoc[]
 [.text-center]
 image::esql/functions/signature/less_than.svg[Embedded,opts=inline]
 
+Check if one field is less than another. If either field is <<esql-multivalued-fields,multivalued>>
+then the result is `null`.
+
+NOTE: This is pushed to the underlying search index if one side of the comparison is constant
+      and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.
+
+Supported types:
+
 include::types/less_than.asciidoc[]
 
 ==== Less than or equal to `<=`
 [.text-center]
 image::esql/functions/signature/less_than_or_equal.svg[Embedded,opts=inline]
 
+Check if one field is less than or equal to another. If either field is <<esql-multivalued-fields,multivalued>>
+then the result is `null`.
+
+NOTE: This is pushed to the underlying search index if one side of the comparison is constant
+      and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.
+
+Supported types:
+
 include::types/less_than_or_equal.asciidoc[]
 
 ==== Greater than `>`
 [.text-center]
 image::esql/functions/signature/greater_than.svg[Embedded,opts=inline]
 
+Check if one field is greater than another. If either field is <<esql-multivalued-fields,multivalued>>
+then the result is `null`.
+
+NOTE: This is pushed to the underlying search index if one side of the comparison is constant
+      and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.
+
+Supported types:
+
 include::types/greater_than.asciidoc[]
 
 ==== Greater than or equal to `>=`
 [.text-center]
 image::esql/functions/signature/greater_than_or_equal.svg[Embedded,opts=inline]
 
+Check if one field is greater than or equal to another. If either field is <<esql-multivalued-fields,multivalued>>
+then the result is `null`.
+
+NOTE: This is pushed to the underlying search index if one side of the comparison is constant
+      and the other side is a field in the index that has both an <<mapping-index>> and <<doc-values>>.
+
+Supported types:
+
 include::types/greater_than_or_equal.asciidoc[]
 
 ==== Add `+`
 [.text-center]
 image::esql/functions/signature/add.svg[Embedded,opts=inline]
 
+Add two numbers together. If either field is <<esql-multivalued-fields,multivalued>>
+then the result is `null`.
+
+Supported types:
+
 include::types/add.asciidoc[]
 
 ==== Subtract `-`
 [.text-center]
 image::esql/functions/signature/sub.svg[Embedded,opts=inline]
 
+Subtract one number from another. If either field is <<esql-multivalued-fields,multivalued>>
+then the result is `null`.
+
+Supported types:
+
 include::types/sub.asciidoc[]
 
 ==== Multiply `*`
 [.text-center]
 image::esql/functions/signature/mul.svg[Embedded,opts=inline]
 
+Multiply two numbers together. If either field is <<esql-multivalued-fields,multivalued>>
+then the result is `null`.
+
+Supported types:
+
 include::types/mul.asciidoc[]
 
 ==== Divide `/`
 [.text-center]
 image::esql/functions/signature/div.svg[Embedded,opts=inline]
 
+Divide one number by another. If either field is <<esql-multivalued-fields,multivalued>>
+then the result is `null`.
+
 NOTE: Division of two integer types will yield an integer result, rounding towards 0.
       If you need floating point division, <<esql-cast-operator>> one of the arguments to a `DOUBLE`.
 
+Supported types:
+
 include::types/div.asciidoc[]
 
 ==== Modulus `%`
 [.text-center]
 image::esql/functions/signature/mod.svg[Embedded,opts=inline]
 
+Divide one number by another and return the remainder. If either field is <<esql-multivalued-fields,multivalued>>
+then the result is `null`.
+
+Supported types:
+
 include::types/mod.asciidoc[]