From a765f89c212c436ff36f554020365ea913728ac8 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Fri, 13 Dec 2024 12:59:14 +0100 Subject: [PATCH 01/24] Make search functions translation aware (#118355) * Introduce TranslationAware interface * Serialize query builder * Fix EsqlNodeSubclassTests * Add javadoc * Address review comments * Revert changes on making constructors private --- .../org/elasticsearch/TransportVersions.java | 1 + .../core/expression/TranslationAware.java | 20 ++++++++ .../TranslationAwareExpressionQuery.java | 35 ++++++++++++++ .../function/EsqlFunctionRegistry.java | 6 +-- .../function/fulltext/FullTextFunction.java | 46 ++++++++++++++++++- .../expression/function/fulltext/Kql.java | 38 ++++++++++++--- .../expression/function/fulltext/Match.java | 33 +++++++++++-- .../function/fulltext/QueryString.java | 42 ++++++++++++++--- .../expression/function/fulltext/Term.java | 33 +++++++++++-- .../planner/EsqlExpressionTranslators.java | 5 ++ 10 files changed, 234 insertions(+), 25 deletions(-) create mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TranslationAware.java create mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TranslationAwareExpressionQuery.java diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 4135b1f0b8e9a..388123e86c882 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -138,6 +138,7 @@ static TransportVersion def(int id) { public static final TransportVersion ADD_DATA_STREAM_OPTIONS_TO_TEMPLATES = def(8_805_00_0); public static final TransportVersion KNN_QUERY_RESCORE_OVERSAMPLE = def(8_806_00_0); public static final TransportVersion SEMANTIC_QUERY_LENIENT = def(8_807_00_0); + public static final TransportVersion ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS = def(8_808_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TranslationAware.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TranslationAware.java new file mode 100644 index 0000000000000..b1ac2b36314fa --- /dev/null +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TranslationAware.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.expression; + +import org.elasticsearch.xpack.esql.core.planner.TranslatorHandler; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; + +/** + * Expressions can implement this interface to control how they would be translated and pushed down as Lucene queries. + * When an expression implements {@link TranslationAware}, we call {@link #asQuery(TranslatorHandler)} to get the + * {@link Query} translation, instead of relying on the registered translators from EsqlExpressionTranslators. + */ +public interface TranslationAware { + Query asQuery(TranslatorHandler translatorHandler); +} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TranslationAwareExpressionQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TranslationAwareExpressionQuery.java new file mode 100644 index 0000000000000..92a42d3053b68 --- /dev/null +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TranslationAwareExpressionQuery.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.querydsl.query; + +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * Expressions that store their own {@link QueryBuilder} and implement + * {@link org.elasticsearch.xpack.esql.core.expression.TranslationAware} can use {@link TranslationAwareExpressionQuery} + * to wrap their {@link QueryBuilder}, instead of using the other existing {@link Query} implementations. + */ +public class TranslationAwareExpressionQuery extends Query { + private final QueryBuilder queryBuilder; + + public TranslationAwareExpressionQuery(Source source, QueryBuilder queryBuilder) { + super(source); + this.queryBuilder = queryBuilder; + } + + @Override + public QueryBuilder asBuilder() { + return queryBuilder; + } + + @Override + protected String innerToString() { + return queryBuilder.toString(); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 50d0d2438d8a1..1ccc22eb3a6a4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -416,7 +416,7 @@ private static FunctionDefinition[][] functions() { def(MvSum.class, MvSum::new, "mv_sum"), def(Split.class, Split::new, "split") }, // fulltext functions - new FunctionDefinition[] { def(Match.class, Match::new, "match"), def(QueryString.class, QueryString::new, "qstr") } }; + new FunctionDefinition[] { def(Match.class, bi(Match::new), "match"), def(QueryString.class, uni(QueryString::new), "qstr") } }; } @@ -426,9 +426,9 @@ private static FunctionDefinition[][] snapshotFunctions() { // The delay() function is for debug/snapshot environments only and should never be enabled in a non-snapshot build. // This is an experimental function and can be removed without notice. def(Delay.class, Delay::new, "delay"), - def(Kql.class, Kql::new, "kql"), + def(Kql.class, uni(Kql::new), "kql"), def(Rate.class, Rate::withUnresolvedTimestamp, "rate"), - def(Term.class, Term::new, "term") } }; + def(Term.class, bi(Term::new), "term") } }; } public EsqlFunctionRegistry snapshotRegistry() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index 78dc05af8f342..432d2d5f07429 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -8,14 +8,21 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Nullability; +import org.elasticsearch.xpack.esql.core.expression.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.Function; +import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator; +import org.elasticsearch.xpack.esql.core.planner.TranslatorHandler; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.querydsl.query.TranslationAwareExpressionQuery; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import java.util.List; +import java.util.Objects; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNullAndFoldable; @@ -26,13 +33,15 @@ * These functions needs to be pushed down to Lucene queries to be executed - there's no Evaluator for them, but depend on * {@link org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizer} to rewrite them into Lucene queries. */ -public abstract class FullTextFunction extends Function { +public abstract class FullTextFunction extends Function implements TranslationAware { private final Expression query; + private final QueryBuilder queryBuilder; - protected FullTextFunction(Source source, Expression query, List children) { + protected FullTextFunction(Source source, Expression query, List children, QueryBuilder queryBuilder) { super(source, children); this.query = query; + this.queryBuilder = queryBuilder; } @Override @@ -116,4 +125,37 @@ public Nullability nullable() { public String functionType() { return "function"; } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), queryBuilder); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + + return Objects.equals(queryBuilder, ((FullTextFunction) obj).queryBuilder); + } + + @Override + public Query asQuery(TranslatorHandler translatorHandler) { + if (queryBuilder != null) { + return new TranslationAwareExpressionQuery(source(), queryBuilder); + } + + ExpressionTranslator translator = translator(); + return translator.translate(this, translatorHandler); + } + + public QueryBuilder queryBuilder() { + return queryBuilder; + } + + @SuppressWarnings("rawtypes") + protected abstract ExpressionTranslator translator(); + + public abstract Expression replaceQueryBuilder(QueryBuilder queryBuilder); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java index c03902373c02e..1f7bcadd259a0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java @@ -7,16 +7,20 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.planner.EsqlExpressionTranslators; import org.elasticsearch.xpack.esql.querydsl.query.KqlQuery; import java.io.IOException; @@ -26,7 +30,7 @@ * Full text function that performs a {@link KqlQuery} . */ public class Kql extends FullTextFunction { - public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Kql", Kql::new); + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Kql", Kql::readFrom); @FunctionInfo( returnType = "boolean", @@ -42,17 +46,30 @@ public Kql( description = "Query string in KQL query string format." ) Expression queryString ) { - super(source, queryString, List.of(queryString)); + super(source, queryString, List.of(queryString), null); } - private Kql(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class)); + public Kql(Source source, Expression queryString, QueryBuilder queryBuilder) { + super(source, queryString, List.of(queryString), queryBuilder); + } + + private static Kql readFrom(StreamInput in) throws IOException { + Source source = Source.readFrom((PlanStreamInput) in); + Expression query = in.readNamedWriteable(Expression.class); + QueryBuilder queryBuilder = null; + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); + } + return new Kql(source, query, queryBuilder); } @Override public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteable(query()); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + out.writeOptionalNamedWriteable(queryBuilder()); + } } @Override @@ -62,12 +79,21 @@ public String getWriteableName() { @Override public Expression replaceChildren(List newChildren) { - return new Kql(source(), newChildren.get(0)); + return new Kql(source(), newChildren.get(0), queryBuilder()); } @Override protected NodeInfo info() { - return NodeInfo.create(this, Kql::new, query()); + return NodeInfo.create(this, Kql::new, query(), queryBuilder()); } + @Override + protected ExpressionTranslator translator() { + return new EsqlExpressionTranslators.KqlFunctionTranslator(); + } + + @Override + public Expression replaceQueryBuilder(QueryBuilder queryBuilder) { + return new Kql(source(), query(), queryBuilder); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java index 2b9a7c73a5853..0b2268fe1b022 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java @@ -8,15 +8,18 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.capabilities.Validatable; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator; import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -27,6 +30,7 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.planner.EsqlExpressionTranslators; import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; import java.io.IOException; @@ -109,7 +113,11 @@ public Match( description = "Value to find in the provided field." ) Expression matchQuery ) { - super(source, matchQuery, List.of(field, matchQuery)); + this(source, field, matchQuery, null); + } + + public Match(Source source, Expression field, Expression matchQuery, QueryBuilder queryBuilder) { + super(source, matchQuery, List.of(field, matchQuery), queryBuilder); this.field = field; } @@ -117,7 +125,11 @@ private static Match readFrom(StreamInput in) throws IOException { Source source = Source.readFrom((PlanStreamInput) in); Expression field = in.readNamedWriteable(Expression.class); Expression query = in.readNamedWriteable(Expression.class); - return new Match(source, field, query); + QueryBuilder queryBuilder = null; + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); + } + return new Match(source, field, query, queryBuilder); } @Override @@ -125,6 +137,9 @@ public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteable(field()); out.writeNamedWriteable(query()); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + out.writeOptionalNamedWriteable(queryBuilder()); + } } @Override @@ -224,12 +239,12 @@ public Object queryAsObject() { @Override public Expression replaceChildren(List newChildren) { - return new Match(source(), newChildren.get(0), newChildren.get(1)); + return new Match(source(), newChildren.get(0), newChildren.get(1), queryBuilder()); } @Override protected NodeInfo info() { - return NodeInfo.create(this, Match::new, field, query()); + return NodeInfo.create(this, Match::new, field, query(), queryBuilder()); } protected TypeResolutions.ParamOrdinal queryParamOrdinal() { @@ -245,6 +260,16 @@ public String functionType() { return isOperator() ? "operator" : super.functionType(); } + @Override + protected ExpressionTranslator translator() { + return new EsqlExpressionTranslators.MatchFunctionTranslator(); + } + + @Override + public Expression replaceQueryBuilder(QueryBuilder queryBuilder) { + return new Match(source(), field, query(), queryBuilder); + } + @Override public String functionName() { return isOperator() ? ":" : super.functionName(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java index bd79661534b76..ea21411d09173 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java @@ -7,10 +7,13 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator; import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -18,6 +21,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.planner.EsqlExpressionTranslators; import java.io.IOException; import java.util.List; @@ -27,7 +31,11 @@ */ public class QueryString extends FullTextFunction { - public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "QStr", QueryString::new); + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "QStr", + QueryString::readFrom + ); @FunctionInfo( returnType = "boolean", @@ -44,17 +52,30 @@ public QueryString( description = "Query string in Lucene query string format." ) Expression queryString ) { - super(source, queryString, List.of(queryString)); + super(source, queryString, List.of(queryString), null); } - private QueryString(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class)); + public QueryString(Source source, Expression queryString, QueryBuilder queryBuilder) { + super(source, queryString, List.of(queryString), queryBuilder); + } + + private static QueryString readFrom(StreamInput in) throws IOException { + Source source = Source.readFrom((PlanStreamInput) in); + Expression query = in.readNamedWriteable(Expression.class); + QueryBuilder queryBuilder = null; + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); + } + return new QueryString(source, query, queryBuilder); } @Override public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteable(query()); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + out.writeOptionalNamedWriteable(queryBuilder()); + } } @Override @@ -69,12 +90,21 @@ public String functionName() { @Override public Expression replaceChildren(List newChildren) { - return new QueryString(source(), newChildren.get(0)); + return new QueryString(source(), newChildren.get(0), queryBuilder()); } @Override protected NodeInfo info() { - return NodeInfo.create(this, QueryString::new, query()); + return NodeInfo.create(this, QueryString::new, query(), queryBuilder()); } + @Override + protected ExpressionTranslator translator() { + return new EsqlExpressionTranslators.QueryStringFunctionTranslator(); + } + + @Override + public Expression replaceQueryBuilder(QueryBuilder queryBuilder) { + return new QueryString(source(), query(), queryBuilder); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java index 125a5b02b6e1c..ff8085cd1b44b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java @@ -7,15 +7,18 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.capabilities.Validatable; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator; import org.elasticsearch.xpack.esql.core.querydsl.query.TermQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -23,6 +26,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.planner.EsqlExpressionTranslators; import java.io.IOException; import java.util.List; @@ -56,7 +60,11 @@ public Term( description = "Term you wish to find in the provided field." ) Expression termQuery ) { - super(source, termQuery, List.of(field, termQuery)); + this(source, field, termQuery, null); + } + + public Term(Source source, Expression field, Expression termQuery, QueryBuilder queryBuilder) { + super(source, termQuery, List.of(field, termQuery), queryBuilder); this.field = field; } @@ -64,7 +72,11 @@ private static Term readFrom(StreamInput in) throws IOException { Source source = Source.readFrom((PlanStreamInput) in); Expression field = in.readNamedWriteable(Expression.class); Expression query = in.readNamedWriteable(Expression.class); - return new Term(source, field, query); + QueryBuilder queryBuilder = null; + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); + } + return new Term(source, field, query, queryBuilder); } @Override @@ -72,6 +84,9 @@ public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteable(field()); out.writeNamedWriteable(query()); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + out.writeOptionalNamedWriteable(queryBuilder()); + } } @Override @@ -101,18 +116,28 @@ public void validate(Failures failures) { @Override public Expression replaceChildren(List newChildren) { - return new Term(source(), newChildren.get(0), newChildren.get(1)); + return new Term(source(), newChildren.get(0), newChildren.get(1), queryBuilder()); } @Override protected NodeInfo info() { - return NodeInfo.create(this, Term::new, field, query()); + return NodeInfo.create(this, Term::new, field, query(), queryBuilder()); } protected TypeResolutions.ParamOrdinal queryParamOrdinal() { return SECOND; } + @Override + protected ExpressionTranslator translator() { + return new EsqlExpressionTranslators.TermFunctionTranslator(); + } + + @Override + public Expression replaceQueryBuilder(QueryBuilder queryBuilder) { + return new Term(source(), field, query(), queryBuilder); + } + public Expression field() { return field; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index 7820f0f657f7f..43bbf9a5f4ff1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.Range; @@ -100,7 +101,11 @@ public final class EsqlExpressionTranslators { ); public static Query toQuery(Expression e, TranslatorHandler handler) { + if (e instanceof TranslationAware ta) { + return ta.asQuery(handler); + } Query translation = null; + for (ExpressionTranslator translator : QUERY_TRANSLATORS) { translation = translator.translate(e, handler); if (translation != null) { From ccdea4a21cd1fe195e5005d81f0a3a49e1f91c67 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 13 Dec 2024 13:41:46 +0100 Subject: [PATCH 02/24] ESQL: tests for LOOKUP JOIN with non-unique join keys (#118471) Add a csv dataset and tests for `LOOKUP JOIN` where the join keys are not unique. In particular, add tests that include MVs and nulls to see how `LOOKUP JOIN` treats these. --- .../xpack/esql/CsvTestsDataLoader.java | 3 + .../resources/languages_non_unique_key.csv | 10 +++ .../src/main/resources/lookup-join.csv-spec | 78 ++++++++++++++++++- 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_non_unique_key.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index dbeb54996733a..3b656ded94dd7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -63,6 +63,8 @@ public class CsvTestsDataLoader { private static final TestsDataset LANGUAGES = new TestsDataset("languages"); private static final TestsDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup") .withSetting("languages_lookup-settings.json"); + private static final TestsDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key") + .withData("languages_non_unique_key.csv"); private static final TestsDataset ALERTS = new TestsDataset("alerts"); private static final TestsDataset UL_LOGS = new TestsDataset("ul_logs"); private static final TestsDataset SAMPLE_DATA = new TestsDataset("sample_data"); @@ -114,6 +116,7 @@ public class CsvTestsDataLoader { Map.entry(APPS_SHORT.indexName, APPS_SHORT), Map.entry(LANGUAGES.indexName, LANGUAGES), Map.entry(LANGUAGES_LOOKUP.indexName, LANGUAGES_LOOKUP), + Map.entry(LANGUAGES_LOOKUP_NON_UNIQUE_KEY.indexName, LANGUAGES_LOOKUP_NON_UNIQUE_KEY), Map.entry(UL_LOGS.indexName, UL_LOGS), Map.entry(SAMPLE_DATA.indexName, SAMPLE_DATA), Map.entry(MV_SAMPLE_DATA.indexName, MV_SAMPLE_DATA), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_non_unique_key.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_non_unique_key.csv new file mode 100644 index 0000000000000..1578762f8d1cb --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_non_unique_key.csv @@ -0,0 +1,10 @@ +language_code:integer,language_name:keyword,country:keyword +1,English,Canada +1,English, +1,,United Kingdom +1,English,United States of America +2,German,[Germany,Austria] +2,German,Switzerland +2,German, +4,Quenya, +5,,Atlantis diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 12e333c0ed9f2..f6704d33934af 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -3,7 +3,6 @@ // Reuses the sample dataset and commands from enrich.csv-spec // -//TODO: this sometimes returns null instead of the looked up value (likely related to the execution order) basicOnTheDataNode required_capability: join_lookup_v5 @@ -102,6 +101,83 @@ emp_no:integer | language_code:integer | language_name:keyword 10003 | 4 | German ; +nonUniqueLeftKeyOnTheDataNode +required_capability: join_lookup_v5 + +FROM employees +| WHERE emp_no <= 10030 +| EVAL language_code = emp_no % 10 +| WHERE language_code < 3 +| LOOKUP JOIN languages_lookup ON language_code +| SORT emp_no +| KEEP emp_no, language_code, language_name +; + +emp_no:integer | language_code:integer | language_name:keyword +10001 |1 | English +10002 |2 | French +10010 |0 | null +10011 |1 | English +10012 |2 | French +10020 |0 | null +10021 |1 | English +10022 |2 | French +10030 |0 | null +; + +nonUniqueRightKeyOnTheDataNode +required_capability: join_lookup_v5 + +FROM employees +| EVAL language_code = emp_no % 10 +| LOOKUP JOIN languages_lookup_non_unique_key ON language_code +| WHERE emp_no > 10090 AND emp_no < 10096 +| SORT emp_no +| EVAL country = MV_SORT(country) +| KEEP emp_no, language_code, language_name, country +; + +emp_no:integer | language_code:integer | language_name:keyword | country:keyword +10091 | 1 | [English, English, English] | [Canada, United Kingdom, United States of America] +10092 | 2 | [German, German, German] | [Austria, Germany, Switzerland] +10093 | 3 | null | null +10094 | 4 | Quenya | null +10095 | 5 | null | Atlantis +; + +nonUniqueRightKeyOnTheCoordinator +required_capability: join_lookup_v5 + +FROM employees +| SORT emp_no +| LIMIT 5 +| EVAL language_code = emp_no % 10 +| LOOKUP JOIN languages_lookup_non_unique_key ON language_code +| EVAL country = MV_SORT(country) +| KEEP emp_no, language_code, language_name, country +; + +emp_no:integer | language_code:integer | language_name:keyword | country:keyword +10001 | 1 | [English, English, English] | [Canada, United Kingdom, United States of America] +10002 | 2 | [German, German, German] | [Austria, Germany, Switzerland] +10003 | 3 | null | null +10004 | 4 | Quenya | null +10005 | 5 | null | Atlantis +; + +nonUniqueRightKeyFromRow +required_capability: join_lookup_v5 + +ROW language_code = 2 +| LOOKUP JOIN languages_lookup_non_unique_key ON language_code +| DROP country.keyword +| EVAL country = MV_SORT(country) +; + +language_code:integer | language_name:keyword | country:keyword +2 | [German, German, German] | [Austria, Germany, Switzerland] +; + lookupIPFromRow required_capability: join_lookup_v5 From 71f98221cd8e0813e24d04f68cb03f1cc7e3f639 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 13 Dec 2024 14:09:06 +0100 Subject: [PATCH 03/24] Fix bug in InferenceUpgradeTestCase.getConfigsWithBreakingChangeHandling (#118624) We need to load the two fields from the same response. Otherwise, we can have a sort of race where we load "endpoints" from pre-8.15 as empty and then load "models" from a post-8.15 node also empty, resulting in an empty list because we took the wrong info from either response. closes #118163 --- .../xpack/application/InferenceUpgradeTestCase.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java index 58335eb53b366..d38503a884092 100644 --- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java +++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java @@ -19,7 +19,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Objects; import static org.elasticsearch.core.Strings.format; @@ -112,13 +111,10 @@ protected void put(String inferenceId, String modelConfig, TaskType taskType) th @SuppressWarnings("unchecked") // in version 8.15, there was a breaking change where "models" was renamed to "endpoints" LinkedList> getConfigsWithBreakingChangeHandling(TaskType testTaskType, String oldClusterId) throws IOException { - + var response = get(testTaskType, oldClusterId); LinkedList> configs; - configs = new LinkedList<>( - (List>) Objects.requireNonNullElse((get(testTaskType, oldClusterId).get("endpoints")), List.of()) - ); - configs.addAll(Objects.requireNonNullElse((List>) get(testTaskType, oldClusterId).get("models"), List.of())); - + configs = new LinkedList<>((List>) response.getOrDefault("endpoints", List.of())); + configs.addAll((List>) response.getOrDefault("models", List.of())); return configs; } } From 21b7afd6692105fc8f79b9c02ef4999c6e9a0b18 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:13:52 +0000 Subject: [PATCH 04/24] Update docker.elastic.co/wolfi/chainguard-base:latest Docker digest to 1b51ff6 (#117902) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Rene Groeschke --- .../main/java/org/elasticsearch/gradle/internal/DockerBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 0d7bcea168df8..d54eb798ce783 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -22,7 +22,7 @@ public enum DockerBase { // Chainguard based wolfi image with latest jdk // This is usually updated via renovatebot // spotless:off - WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:32f06b169bb4b0f257fbb10e8c8379f06d3ee1355c89b3327cb623781a29590e", + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:1b51ff6dba78c98d3e02b0cd64a8ce3238c7a40408d21e3af12a329d44db6f23", "-wolfi", "apk" ), From 528593b55f26005fec910bfb89f9468ec7798d1f Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:00:29 +0100 Subject: [PATCH 05/24] [DOCS] Link to Elastic Rerank model landing page (#118574) - Add link to Python notebook - Fix heading level --- docs/reference/inference/service-elasticsearch.asciidoc | 7 ++++++- docs/reference/search/retriever.asciidoc | 7 ++++++- .../search/search-your-data/retrievers-examples.asciidoc | 3 +-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/reference/inference/service-elasticsearch.asciidoc b/docs/reference/inference/service-elasticsearch.asciidoc index cd06e6d7b2f64..bf7e2976bbe63 100644 --- a/docs/reference/inference/service-elasticsearch.asciidoc +++ b/docs/reference/inference/service-elasticsearch.asciidoc @@ -153,7 +153,12 @@ For further details, refer to the {ml-docs}/ml-nlp-elser.html[ELSER model docume [[inference-example-elastic-reranker]] ==== Elastic Rerank via the `elasticsearch` service -The following example shows how to create an {infer} endpoint called `my-elastic-rerank` to perform a `rerank` task type using the built-in Elastic Rerank cross-encoder model. +The following example shows how to create an {infer} endpoint called `my-elastic-rerank` to perform a `rerank` task type using the built-in {ml-docs}/ml-nlp-rerank.html[Elastic Rerank] cross-encoder model. + +[TIP] +==== +Refer to this https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/search/12-semantic-reranking-elastic-rerank.ipynb[Python notebook] for an end-to-end example using Elastic Rerank. +==== The API request below will automatically download the Elastic Rerank model if it isn't already downloaded and then deploy the model. Once deployed, the model can be used for semantic re-ranking with a <>. diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc index cb04d4fb6fbf1..f20e9148bf5e7 100644 --- a/docs/reference/search/retriever.asciidoc +++ b/docs/reference/search/retriever.asciidoc @@ -442,7 +442,12 @@ If the child retriever already specifies any filters, then this top-level filter [[text-similarity-reranker-retriever-example-elastic-rerank]] ==== Example: Elastic Rerank -This examples demonstrates how to deploy the Elastic Rerank model and use it to re-rank search results using the `text_similarity_reranker` retriever. +[TIP] +==== +Refer to this https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/search/12-semantic-reranking-elastic-rerank.ipynb[Python notebook] for an end-to-end example using Elastic Rerank. +==== + +This example demonstrates how to deploy the {ml-docs}/ml-nlp-rerank.html[Elastic Rerank] model and use it to re-rank search results using the `text_similarity_reranker` retriever. Follow these steps: diff --git a/docs/reference/search/search-your-data/retrievers-examples.asciidoc b/docs/reference/search/search-your-data/retrievers-examples.asciidoc index ad1cc32dcee01..5cada8960aeab 100644 --- a/docs/reference/search/search-your-data/retrievers-examples.asciidoc +++ b/docs/reference/search/search-your-data/retrievers-examples.asciidoc @@ -1,9 +1,8 @@ [[retrievers-examples]] +=== Retrievers examples Learn how to combine different retrievers in these hands-on examples. -=== Retrievers examples - [discrete] [[retrievers-examples-setup]] ==== Add example data From 1e26791515184a965444ea45a300691d17cffb8d Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Fri, 13 Dec 2024 09:25:52 -0500 Subject: [PATCH 06/24] Esql bucket function for date nanos (#118474) This adds support for running the bucket function over a date nanos field. Code wise, this just delegates to DateTrunc, which already supports date nanos, so most of the PR is just tests and the auto-generated docs. Resolves #118031 --- docs/changelog/118474.yaml | 6 + .../functions/kibana/definition/bucket.json | 306 ++++++++++++++++++ .../esql/functions/types/bucket.asciidoc | 11 + .../src/main/resources/date_nanos.csv-spec | 74 +++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../expression/function/grouping/Bucket.java | 12 +- .../function/grouping/BucketTests.java | 91 +++++- 7 files changed, 498 insertions(+), 7 deletions(-) create mode 100644 docs/changelog/118474.yaml diff --git a/docs/changelog/118474.yaml b/docs/changelog/118474.yaml new file mode 100644 index 0000000000000..1b0c6942eb323 --- /dev/null +++ b/docs/changelog/118474.yaml @@ -0,0 +1,6 @@ +pr: 118474 +summary: Esql bucket function for date nanos +area: ES|QL +type: enhancement +issues: + - 118031 diff --git a/docs/reference/esql/functions/kibana/definition/bucket.json b/docs/reference/esql/functions/kibana/definition/bucket.json index 660e1be49fda9..18802f5ff8fef 100644 --- a/docs/reference/esql/functions/kibana/definition/bucket.json +++ b/docs/reference/esql/functions/kibana/definition/bucket.json @@ -310,6 +310,312 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "date_period", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "date", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "date", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "date", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "keyword", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "date", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "text", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "keyword", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "date", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "keyword", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "keyword", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "keyword", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "text", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "text", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "date", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "text", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "keyword", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "text", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "text", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "time_duration", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/types/bucket.asciidoc b/docs/reference/esql/functions/types/bucket.asciidoc index 172e84b6f7860..2e6985e6bc4ed 100644 --- a/docs/reference/esql/functions/types/bucket.asciidoc +++ b/docs/reference/esql/functions/types/bucket.asciidoc @@ -16,6 +16,17 @@ date | integer | text | date | date date | integer | text | keyword | date date | integer | text | text | date date | time_duration | | | date +date_nanos | date_period | | | date_nanos +date_nanos | integer | date | date | date_nanos +date_nanos | integer | date | keyword | date_nanos +date_nanos | integer | date | text | date_nanos +date_nanos | integer | keyword | date | date_nanos +date_nanos | integer | keyword | keyword | date_nanos +date_nanos | integer | keyword | text | date_nanos +date_nanos | integer | text | date | date_nanos +date_nanos | integer | text | keyword | date_nanos +date_nanos | integer | text | text | date_nanos +date_nanos | time_duration | | | date_nanos double | double | | | double double | integer | double | double | double double | integer | double | integer | double diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index 0d113c0422562..f4b5c98d596ae 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -549,6 +549,80 @@ yr:date_nanos | mo:date_nanos | mn:date_nanos 2023-01-01T00:00:00.000000000Z | 2023-10-01T00:00:00.000000000Z | 2023-10-23T12:10:00.000000000Z | 2023-10-23T12:15:03.360000000Z ; +Bucket Date nanos by Year +required_capability: date_trunc_date_nanos +required_capability: date_nanos_bucket + +FROM date_nanos +| WHERE millis > "2020-01-01" +| STATS ct = count(*) BY yr = BUCKET(nanos, 1 year); + +ct:long | yr:date_nanos +8 | 2023-01-01T00:00:00.000000000Z +; + +Bucket Date nanos by Year, range version +required_capability: date_trunc_date_nanos +required_capability: date_nanos_bucket + +FROM date_nanos +| WHERE millis > "2020-01-01" +| STATS ct = count(*) BY yr = BUCKET(nanos, 5, "1999-01-01", NOW()); + +ct:long | yr:date_nanos +8 | 2023-01-01T00:00:00.000000000Z +; + +Bucket Date nanos by Month +required_capability: date_trunc_date_nanos +required_capability: date_nanos_bucket + +FROM date_nanos +| WHERE millis > "2020-01-01" +| STATS ct = count(*) BY mo = BUCKET(nanos, 1 month); + +ct:long | mo:date_nanos +8 | 2023-10-01T00:00:00.000000000Z +; + +Bucket Date nanos by Month, range version +required_capability: date_trunc_date_nanos +required_capability: date_nanos_bucket + +FROM date_nanos +| WHERE millis > "2020-01-01" +| STATS ct = count(*) BY mo = BUCKET(nanos, 20, "2023-01-01", "2023-12-31"); + +ct:long | mo:date_nanos +8 | 2023-10-01T00:00:00.000000000Z +; + +Bucket Date nanos by Week, range version +required_capability: date_trunc_date_nanos +required_capability: date_nanos_bucket + +FROM date_nanos +| WHERE millis > "2020-01-01" +| STATS ct = count(*) BY mo = BUCKET(nanos, 55, "2023-01-01", "2023-12-31"); + +ct:long | mo:date_nanos +8 | 2023-10-23T00:00:00.000000000Z +; +Bucket Date nanos by 10 minutes +required_capability: date_trunc_date_nanos +required_capability: date_nanos_bucket + +FROM date_nanos +| WHERE millis > "2020-01-01" +| STATS ct = count(*) BY mn = BUCKET(nanos, 10 minutes); + +ct:long | mn:date_nanos +4 | 2023-10-23T13:50:00.000000000Z +1 | 2023-10-23T13:30:00.000000000Z +1 | 2023-10-23T12:20:00.000000000Z +2 | 2023-10-23T12:10:00.000000000Z +; + Add date nanos required_capability: date_nanos_add_subtract diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 4cf3162fcca3b..649ec1eba9785 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -367,6 +367,11 @@ public enum Cap { */ DATE_TRUNC_DATE_NANOS(), + /** + * Support date nanos values as the field argument to bucket + */ + DATE_NANOS_BUCKET(), + /** * support aggregations on date nanos */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java index 9e40b85fd6590..347d542f5212d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java @@ -90,7 +90,7 @@ public class Bucket extends GroupingFunction implements Validatable, TwoOptional private final Expression to; @FunctionInfo( - returnType = { "double", "date" }, + returnType = { "double", "date", "date_nanos" }, description = """ Creates groups of values - buckets - out of a datetime or numeric input. The size of the buckets can either be provided directly, or chosen based on a recommended count and values range.""", @@ -169,7 +169,7 @@ public Bucket( Source source, @Param( name = "field", - type = { "integer", "long", "double", "date" }, + type = { "integer", "long", "double", "date", "date_nanos" }, description = "Numeric or date expression from which to derive buckets." ) Expression field, @Param( @@ -241,7 +241,7 @@ public boolean foldable() { @Override public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { - if (field.dataType() == DataType.DATETIME) { + if (field.dataType() == DataType.DATETIME || field.dataType() == DataType.DATE_NANOS) { Rounding.Prepared preparedRounding; if (buckets.dataType().isWholeNumber()) { int b = ((Number) buckets.fold()).intValue(); @@ -314,8 +314,8 @@ private double pickRounding(int buckets, double from, double to) { } // supported parameter type combinations (1st, 2nd, 3rd, 4th): - // datetime, integer, string/datetime, string/datetime - // datetime, rounding/duration, -, - + // datetime/date_nanos, integer, string/datetime, string/datetime + // datetime/date_nanos, rounding/duration, -, - // numeric, integer, numeric, numeric // numeric, numeric, -, - @Override @@ -329,7 +329,7 @@ protected TypeResolution resolveType() { return TypeResolution.TYPE_RESOLVED; } - if (fieldType == DataType.DATETIME) { + if (fieldType == DataType.DATETIME || fieldType == DataType.DATE_NANOS) { TypeResolution resolution = isType( buckets, dt -> dt.isWholeNumber() || DataType.isTemporalAmount(dt), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java index 7e7d91cdf76f4..f01b06c23e8a8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java @@ -12,15 +12,19 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Rounding; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.logging.LogManager; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.hamcrest.Matcher; +import org.hamcrest.Matchers; import java.time.Duration; +import java.time.Instant; import java.time.Period; import java.util.ArrayList; import java.util.List; @@ -38,6 +42,7 @@ public BucketTests(@Name("TestCase") Supplier testCas public static Iterable parameters() { List suppliers = new ArrayList<>(); dateCases(suppliers, "fixed date", () -> DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-02-17T09:00:00.00Z")); + dateNanosCases(suppliers, "fixed date nanos", () -> DateUtils.toLong(Instant.parse("2023-02-17T09:00:00.00Z"))); dateCasesWithSpan( suppliers, "fixed date with period", @@ -54,6 +59,22 @@ public static Iterable parameters() { Duration.ofDays(1L), "[86400000 in Z][fixed]" ); + dateNanosCasesWithSpan( + suppliers, + "fixed date nanos with period", + () -> DateUtils.toLong(Instant.parse("2023-01-01T00:00:00.00Z")), + DataType.DATE_PERIOD, + Period.ofYears(1), + "[YEAR_OF_CENTURY in Z][fixed to midnight]" + ); + dateNanosCasesWithSpan( + suppliers, + "fixed date nanos with duration", + () -> DateUtils.toLong(Instant.parse("2023-02-17T09:00:00.00Z")), + DataType.TIME_DURATION, + Duration.ofDays(1L), + "[86400000 in Z][fixed]" + ); numberCases(suppliers, "fixed long", DataType.LONG, () -> 100L); numberCasesWithSpan(suppliers, "fixed long with span", DataType.LONG, () -> 100L); numberCases(suppliers, "fixed int", DataType.INTEGER, () -> 100); @@ -142,6 +163,62 @@ private static void dateCasesWithSpan( })); } + private static void dateNanosCasesWithSpan( + List suppliers, + String name, + LongSupplier date, + DataType spanType, + Object span, + String spanStr + ) { + suppliers.add(new TestCaseSupplier(name, List.of(DataType.DATE_NANOS, spanType), () -> { + List args = new ArrayList<>(); + args.add(new TestCaseSupplier.TypedData(date.getAsLong(), DataType.DATE_NANOS, "field")); + args.add(new TestCaseSupplier.TypedData(span, spanType, "buckets").forceLiteral()); + return new TestCaseSupplier.TestCase( + args, + Matchers.startsWith("DateTruncDateNanosEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding["), + DataType.DATE_NANOS, + resultsMatcher(args) + ); + })); + } + + private static void dateNanosCases(List suppliers, String name, LongSupplier date) { + for (DataType fromType : DATE_BOUNDS_TYPE) { + for (DataType toType : DATE_BOUNDS_TYPE) { + suppliers.add(new TestCaseSupplier(name, List.of(DataType.DATE_NANOS, DataType.INTEGER, fromType, toType), () -> { + List args = new ArrayList<>(); + args.add(new TestCaseSupplier.TypedData(date.getAsLong(), DataType.DATE_NANOS, "field")); + // TODO more "from" and "to" and "buckets" + args.add(new TestCaseSupplier.TypedData(50, DataType.INTEGER, "buckets").forceLiteral()); + args.add(dateBound("from", fromType, "2023-02-01T00:00:00.00Z")); + args.add(dateBound("to", toType, "2023-03-01T09:00:00.00Z")); + return new TestCaseSupplier.TestCase( + args, + Matchers.startsWith("DateTruncDateNanosEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding["), + DataType.DATE_NANOS, + resultsMatcher(args) + ); + })); + // same as above, but a low bucket count and datetime bounds that match it (at hour span) + suppliers.add(new TestCaseSupplier(name, List.of(DataType.DATE_NANOS, DataType.INTEGER, fromType, toType), () -> { + List args = new ArrayList<>(); + args.add(new TestCaseSupplier.TypedData(date.getAsLong(), DataType.DATE_NANOS, "field")); + args.add(new TestCaseSupplier.TypedData(4, DataType.INTEGER, "buckets").forceLiteral()); + args.add(dateBound("from", fromType, "2023-02-17T09:00:00Z")); + args.add(dateBound("to", toType, "2023-02-17T12:00:00Z")); + return new TestCaseSupplier.TestCase( + args, + Matchers.startsWith("DateTruncDateNanosEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding["), + DataType.DATE_NANOS, + equalTo(Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).build().prepareForUnknown().round(date.getAsLong())) + ); + })); + } + } + } + private static final DataType[] NUMBER_BOUNDS_TYPES = new DataType[] { DataType.INTEGER, DataType.LONG, DataType.DOUBLE }; private static void numberCases(List suppliers, String name, DataType numberType, Supplier number) { @@ -221,7 +298,19 @@ private static TestCaseSupplier.TypedData keywordDateLiteral(String name, DataTy private static Matcher resultsMatcher(List typedData) { if (typedData.get(0).type() == DataType.DATETIME) { long millis = ((Number) typedData.get(0).data()).longValue(); - return equalTo(Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).build().prepareForUnknown().round(millis)); + long expected = Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).build().prepareForUnknown().round(millis); + LogManager.getLogger(getTestClass()).info("Expected: " + Instant.ofEpochMilli(expected)); + LogManager.getLogger(getTestClass()).info("Input: " + Instant.ofEpochMilli(millis)); + return equalTo(expected); + } + if (typedData.get(0).type() == DataType.DATE_NANOS) { + long nanos = ((Number) typedData.get(0).data()).longValue(); + long expected = DateUtils.toNanoSeconds( + Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).build().prepareForUnknown().round(DateUtils.toMilliSeconds(nanos)) + ); + LogManager.getLogger(getTestClass()).info("Expected: " + DateUtils.toInstant(expected)); + LogManager.getLogger(getTestClass()).info("Input: " + DateUtils.toInstant(nanos)); + return equalTo(expected); } return equalTo(((Number) typedData.get(0).data()).doubleValue()); } From a583a38518f321d6092637e66151fc0bd1fe4d1d Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 13 Dec 2024 15:38:52 +0100 Subject: [PATCH 07/24] fix typo in muted CSV test for scoring in ES|QL (#118665) --- muted-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/muted-tests.yml b/muted-tests.yml index d5b933b96d73b..36dfc306b0147 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -174,7 +174,7 @@ tests: - class: "org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT" method: "test {scoring.*}" issue: https://github.com/elastic/elasticsearch/issues/117641 -- class: "org.elasticsearch.xpack.esql.qa.mixed.MultilusterEsqlSpecIT" +- class: "org.elasticsearch.xpack.esql.qa.mixed.MultiClusterEsqlSpecIT" method: "test {scoring.*}" issue: https://github.com/elastic/elasticsearch/issues/118460 - class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT" From 44a231acf2471f39497b7740ddc349306c97cd6a Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:53:33 -0500 Subject: [PATCH 08/24] [Inference API] Replace ElasticsearchTimeoutException with ElasticsearchStatusException (#118618) * Replace ElasticsearchTimeoutException with ElasticsearchTimeoutException with 408 status to avoid cuasing 503s --- .../xpack/inference/external/http/sender/RequestTask.java | 8 ++++++-- .../external/http/sender/HttpRequestSenderTests.java | 8 +++++--- .../external/http/sender/RequestExecutorServiceTests.java | 5 +++-- .../inference/external/http/sender/RequestTaskTests.java | 5 +++-- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java index 9ccb93a0858ae..e5c29adeb9176 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java @@ -7,13 +7,14 @@ package org.elasticsearch.xpack.inference.external.http.sender; -import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ListenerTimeouts; import org.elasticsearch.common.Strings; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import java.util.Objects; @@ -64,7 +65,10 @@ private ActionListener getListener( threadPool.executor(UTILITY_THREAD_POOL_NAME), notificationListener, (ignored) -> notificationListener.onFailure( - new ElasticsearchTimeoutException(Strings.format("Request timed out waiting to be sent after [%s]", timeout)) + new ElasticsearchStatusException( + Strings.format("Request timed out waiting to be sent after [%s]", timeout), + RestStatus.REQUEST_TIMEOUT + ) ) ); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java index 79f6aa8164b75..b3e7db6009204 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.inference.external.http.sender; import org.apache.http.HttpHeaders; -import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -162,12 +162,13 @@ public void testHttpRequestSender_Throws_WhenATimeoutOccurs() throws Exception { PlainActionFuture listener = new PlainActionFuture<>(); sender.send(RequestManagerTests.createMock(), new DocumentsOnlyInput(List.of()), TimeValue.timeValueNanos(1), listener); - var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT)); + var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( thrownException.getMessage(), is(format("Request timed out waiting to be sent after [%s]", TimeValue.timeValueNanos(1))) ); + assertThat(thrownException.status().getStatus(), is(408)); } } @@ -187,12 +188,13 @@ public void testHttpRequestSenderWithTimeout_Throws_WhenATimeoutOccurs() throws PlainActionFuture listener = new PlainActionFuture<>(); sender.send(RequestManagerTests.createMock(), new DocumentsOnlyInput(List.of()), TimeValue.timeValueNanos(1), listener); - var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT)); + var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( thrownException.getMessage(), is(format("Request timed out waiting to be sent after [%s]", TimeValue.timeValueNanos(1))) ); + assertThat(thrownException.status().getStatus(), is(408)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java index e09e4968571e5..7e29fad56812d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.inference.external.http.sender; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; @@ -238,12 +238,13 @@ public void testExecute_CallsOnFailure_WhenRequestTimesOut() { var listener = new PlainActionFuture(); service.execute(RequestManagerTests.createMock(), new DocumentsOnlyInput(List.of()), TimeValue.timeValueNanos(1), listener); - var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT)); + var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( thrownException.getMessage(), is(format("Request timed out waiting to be sent after [%s]", TimeValue.timeValueNanos(1))) ); + assertThat(thrownException.status().getStatus(), is(408)); } public void testExecute_PreservesThreadContext() throws InterruptedException, ExecutionException, TimeoutException { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTaskTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTaskTests.java index c839c266e9320..e37a1a213569e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTaskTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTaskTests.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.inference.external.http.sender; -import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.core.TimeValue; @@ -86,13 +86,14 @@ public void testRequest_ReturnsTimeoutException() { listener ); - var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT)); + var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( thrownException.getMessage(), is(format("Request timed out waiting to be sent after [%s]", TimeValue.timeValueMillis(1))) ); assertTrue(requestTask.hasCompleted()); assertTrue(requestTask.getRequestCompletedFunction().get()); + assertThat(thrownException.status().getStatus(), is(408)); } public void testRequest_DoesNotCallOnFailureTwiceWhenTimingOut() throws Exception { From 1bad1cf6b2256f594361059a8090d2707aa54001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Fri, 13 Dec 2024 16:21:42 +0100 Subject: [PATCH 09/24] Implementing the correct exit functions (Runtime) (#118657) --- .../entitlement/bridge/EntitlementChecker.java | 6 +++++- .../qa/common/RestEntitlementsCheckAction.java | 17 ++++++++++++----- .../api/ElasticsearchEntitlementChecker.java | 7 ++++++- .../Java23ElasticsearchEntitlementChecker.java | 4 ++-- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index ad0f14bcf4478..a6b8a31fc3894 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -13,7 +13,11 @@ import java.net.URLStreamHandlerFactory; public interface EntitlementChecker { - void check$java_lang_System$exit(Class callerClass, int status); + + // Exit the JVM process + void check$$exit(Class callerClass, Runtime runtime, int status); + + void check$$halt(Class callerClass, Runtime runtime, int status); // URLClassLoader ctor void check$java_net_URLClassLoader$(Class callerClass, URL[] urls); diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java index e63fa4f3b726b..1ac4a7506eacb 100644 --- a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java +++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java @@ -47,14 +47,21 @@ static CheckAction serverAndPlugin(Runnable action) { } private static final Map checkActions = Map.ofEntries( - entry("system_exit", CheckAction.serverOnly(RestEntitlementsCheckAction::systemExit)), + entry("runtime_exit", CheckAction.serverOnly(RestEntitlementsCheckAction::runtimeExit)), + entry("runtime_halt", CheckAction.serverOnly(RestEntitlementsCheckAction::runtimeHalt)), entry("create_classloader", CheckAction.serverAndPlugin(RestEntitlementsCheckAction::createClassLoader)) ); - @SuppressForbidden(reason = "Specifically testing System.exit") - private static void systemExit() { - logger.info("Calling System.exit(123);"); - System.exit(123); + @SuppressForbidden(reason = "Specifically testing Runtime.exit") + private static void runtimeExit() { + logger.info("Calling Runtime.exit;"); + Runtime.getRuntime().exit(123); + } + + @SuppressForbidden(reason = "Specifically testing Runtime.halt") + private static void runtimeHalt() { + logger.info("Calling Runtime.halt;"); + Runtime.getRuntime().halt(123); } private static void createClassLoader() { diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index aa63b630ed7cd..a5ca0543ad15a 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -28,7 +28,12 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { } @Override - public void check$java_lang_System$exit(Class callerClass, int status) { + public void check$$exit(Class callerClass, Runtime runtime, int status) { + policyManager.checkExitVM(callerClass); + } + + @Override + public void check$$halt(Class callerClass, Runtime runtime, int status) { policyManager.checkExitVM(callerClass); } diff --git a/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java index d0f9f4f48609c..912d76ecfc01a 100644 --- a/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java @@ -19,8 +19,8 @@ public Java23ElasticsearchEntitlementChecker(PolicyManager policyManager) { } @Override - public void check$java_lang_System$exit(Class callerClass, int status) { + public void check$$exit(Class callerClass, Runtime runtime, int status) { // TODO: this is just an example, we shouldn't really override a method implemented in the superclass - super.check$java_lang_System$exit(callerClass, status); + super.check$$exit(callerClass, runtime, status); } } From 5411b93d493ddc81682b49cf6cb9bac2607c4f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Fri, 13 Dec 2024 16:24:54 +0100 Subject: [PATCH 10/24] Entitlements tools: public callers finder (#116257) * WIP: Tool to find all public caller from a starting list of (JDK) methods. * Add public-callers-finder tool, extract common stuff to common module * Adjustments to visibility/functions and classes and modules to print out * Spotless * Missing gradle configuration * Add details in README as requested in PR * Update ASM version * Including protected methods --- .../entitlement/tools/ExternalAccess.java | 68 ++++++ .../entitlement/tools/Utils.java | 37 +++- .../tools/public-callers-finder/README.md | 50 +++++ .../tools/public-callers-finder/build.gradle | 61 ++++++ .../licenses/asm-LICENSE.txt | 26 +++ .../licenses/asm-NOTICE.txt | 1 + .../FindUsagesClassVisitor.java | 141 +++++++++++++ .../tools/publiccallersfinder/Main.java | 197 ++++++++++++++++++ .../{src => }/README.md | 0 .../tools/securitymanager/scanner/Main.java | 50 ++--- .../scanner/SecurityCheckClassVisitor.java | 22 +- 11 files changed, 603 insertions(+), 50 deletions(-) create mode 100644 libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/ExternalAccess.java create mode 100644 libs/entitlement/tools/public-callers-finder/README.md create mode 100644 libs/entitlement/tools/public-callers-finder/build.gradle create mode 100644 libs/entitlement/tools/public-callers-finder/licenses/asm-LICENSE.txt create mode 100644 libs/entitlement/tools/public-callers-finder/licenses/asm-NOTICE.txt create mode 100644 libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/FindUsagesClassVisitor.java create mode 100644 libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/Main.java rename libs/entitlement/tools/securitymanager-scanner/{src => }/README.md (100%) diff --git a/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/ExternalAccess.java b/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/ExternalAccess.java new file mode 100644 index 0000000000000..cd049a91fa4da --- /dev/null +++ b/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/ExternalAccess.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.tools; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.stream.Collectors; + +public enum ExternalAccess { + PUBLIC_CLASS, + PUBLIC_METHOD, + PROTECTED_METHOD; + + private static final String DELIMITER = ":"; + + public static String toString(EnumSet externalAccesses) { + return externalAccesses.stream().map(Enum::toString).collect(Collectors.joining(DELIMITER)); + } + + public static EnumSet fromPermissions( + boolean packageExported, + boolean publicClass, + boolean publicMethod, + boolean protectedMethod + ) { + if (publicMethod && protectedMethod) { + throw new IllegalArgumentException(); + } + + EnumSet externalAccesses = EnumSet.noneOf(ExternalAccess.class); + if (publicMethod) { + externalAccesses.add(ExternalAccess.PUBLIC_METHOD); + } else if (protectedMethod) { + externalAccesses.add(ExternalAccess.PROTECTED_METHOD); + } + + if (packageExported && publicClass) { + externalAccesses.add(ExternalAccess.PUBLIC_CLASS); + } + return externalAccesses; + } + + public static boolean isExternallyAccessible(EnumSet access) { + return access.contains(ExternalAccess.PUBLIC_CLASS) + && (access.contains(ExternalAccess.PUBLIC_METHOD) || access.contains(ExternalAccess.PROTECTED_METHOD)); + } + + public static EnumSet fromString(String accessAsString) { + if ("PUBLIC".equals(accessAsString)) { + return EnumSet.of(ExternalAccess.PUBLIC_CLASS, ExternalAccess.PUBLIC_METHOD); + } + if ("PUBLIC-METHOD".equals(accessAsString)) { + return EnumSet.of(ExternalAccess.PUBLIC_METHOD); + } + if ("PRIVATE".equals(accessAsString)) { + return EnumSet.noneOf(ExternalAccess.class); + } + + return EnumSet.copyOf(Arrays.stream(accessAsString.split(DELIMITER)).map(ExternalAccess::valueOf).toList()); + } +} diff --git a/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java b/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java index c72e550a529cd..c6a71f55db4c6 100644 --- a/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java +++ b/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java @@ -11,16 +11,28 @@ import java.io.IOException; import java.lang.module.ModuleDescriptor; +import java.net.URI; import java.nio.file.FileSystem; +import java.nio.file.FileSystems; import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; public class Utils { - public static Map> findModuleExports(FileSystem fs) throws IOException { + private static final Set EXCLUDED_MODULES = Set.of( + "java.desktop", + "jdk.jartool", + "jdk.jdi", + "java.security.jgss", + "jdk.jshell" + ); + + private static Map> findModuleExports(FileSystem fs) throws IOException { var modulesExports = new HashMap>(); try (var stream = Files.walk(fs.getPath("modules"))) { stream.filter(p -> p.getFileName().toString().equals("module-info.class")).forEach(x -> { @@ -42,4 +54,27 @@ public static Map> findModuleExports(FileSystem fs) throws I return modulesExports; } + public interface JdkModuleConsumer { + void accept(String moduleName, List moduleClasses, Set moduleExports); + } + + public static void walkJdkModules(JdkModuleConsumer c) throws IOException { + + FileSystem fs = FileSystems.getFileSystem(URI.create("jrt:/")); + + var moduleExports = Utils.findModuleExports(fs); + + try (var stream = Files.walk(fs.getPath("modules"))) { + var modules = stream.filter(x -> x.toString().endsWith(".class")) + .collect(Collectors.groupingBy(x -> x.subpath(1, 2).toString())); + + for (var kv : modules.entrySet()) { + var moduleName = kv.getKey(); + if (Utils.EXCLUDED_MODULES.contains(moduleName) == false) { + var thisModuleExports = moduleExports.get(moduleName); + c.accept(moduleName, kv.getValue(), thisModuleExports); + } + } + } + } } diff --git a/libs/entitlement/tools/public-callers-finder/README.md b/libs/entitlement/tools/public-callers-finder/README.md new file mode 100644 index 0000000000000..794576b3409a8 --- /dev/null +++ b/libs/entitlement/tools/public-callers-finder/README.md @@ -0,0 +1,50 @@ +This tool scans the JDK on which it is running. It takes a list of methods (compatible with the output of the `securitymanager-scanner` tool), and looks for the "public surface" of these methods (i.e. any class/method accessible from regular Java code that calls into the original list, directly or transitively). + +It acts basically as a recursive "Find Usages" in Intellij, stopping at the first fully accessible point (public method on a public class). +The tool scans every method in every class inside the same java module; e.g. +if you have a private method `File#normalizedList`, it will scan `java.base` to find +public methods like `File#list(String)`, `File#list(FilenameFilter, String)` and +`File#listFiles(File)`. + +The tool considers implemented interfaces (directly); e.g. if we're looking at a +method `C.m`, where `C implements I`, it will look for calls to `I.m`. It will +also consider (indirectly) calls to `S.m` (where `S` is a supertype of `C`), as +it treats calls to `super` in `S.m` as regular calls (e.g. `example() -> S.m() -> C.m()`). + + +In order to run the tool, use: +```shell +./gradlew :libs:entitlement:tools:public-callers-finder:run [] +``` +Where `input-file` is a CSV file (columns separated by `TAB`) that contains the following columns: +Module name +1. unused +2. unused +3. unused +4. Fully qualified class name (ASM style, with `/` separators) +5. Method name +6. Method descriptor (ASM signature) +7. Visibility (PUBLIC/PUBLIC-METHOD/PRIVATE) + +And `bubble-up-from-public` is a boolean (`true|false`) indicating if the code should stop at the first public method (`false`: default, recommended) or continue to find usages recursively even after reaching the "public surface". + +The output of the tool is another CSV file, with one line for each entry-point, columns separated by `TAB` + +1. Module name +2. File name (from source root) +3. Line number +4. Fully qualified class name (ASM style, with `/` separators) +5. Method name +6. Method descriptor (ASM signature) +7. Visibility (PUBLIC/PUBLIC-METHOD/PRIVATE) +8. Original caller Module name +9. Original caller Class name (ASM style, with `/` separators) +10. Original caller Method name +11. Original caller Visibility + +Examples: +``` +java.base DeleteOnExitHook.java 50 java/io/DeleteOnExitHook$1 run ()V PUBLIC java.base java/io/File delete PUBLIC +java.base ZipFile.java 254 java/util/zip/ZipFile (Ljava/io/File;ILjava/nio/charset/Charset;)V PUBLIC java.base java/io/File delete PUBLIC +java.logging FileHandler.java 279 java/util/logging/FileHandler ()V PUBLIC java.base java/io/File delete PUBLIC +``` diff --git a/libs/entitlement/tools/public-callers-finder/build.gradle b/libs/entitlement/tools/public-callers-finder/build.gradle new file mode 100644 index 0000000000000..083b1a43b9794 --- /dev/null +++ b/libs/entitlement/tools/public-callers-finder/build.gradle @@ -0,0 +1,61 @@ +plugins { + id 'application' +} + +apply plugin: 'elasticsearch.build' +apply plugin: 'elasticsearch.publish' + +tasks.named("dependencyLicenses").configure { + mapping from: /asm-.*/, to: 'asm' +} + +group = 'org.elasticsearch.entitlement.tools' + +ext { + javaMainClass = "org.elasticsearch.entitlement.tools.publiccallersfinder.Main" +} + +application { + mainClass.set(javaMainClass) + applicationDefaultJvmArgs = [ + '--add-exports', 'java.base/sun.security.util=ALL-UNNAMED', + '--add-opens', 'java.base/java.lang=ALL-UNNAMED', + '--add-opens', 'java.base/java.net=ALL-UNNAMED', + '--add-opens', 'java.base/java.net.spi=ALL-UNNAMED', + '--add-opens', 'java.base/java.util.concurrent=ALL-UNNAMED', + '--add-opens', 'java.base/javax.crypto=ALL-UNNAMED', + '--add-opens', 'java.base/javax.security.auth=ALL-UNNAMED', + '--add-opens', 'java.base/jdk.internal.logger=ALL-UNNAMED', + '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED', + '--add-opens', 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED', + '--add-opens', 'java.logging/java.util.logging=ALL-UNNAMED', + '--add-opens', 'java.logging/sun.util.logging.internal=ALL-UNNAMED', + '--add-opens', 'java.naming/javax.naming.ldap.spi=ALL-UNNAMED', + '--add-opens', 'java.rmi/sun.rmi.runtime=ALL-UNNAMED', + '--add-opens', 'jdk.dynalink/jdk.dynalink=ALL-UNNAMED', + '--add-opens', 'jdk.dynalink/jdk.dynalink.linker=ALL-UNNAMED', + '--add-opens', 'java.desktop/sun.awt=ALL-UNNAMED', + '--add-opens', 'java.sql.rowset/javax.sql.rowset.spi=ALL-UNNAMED', + '--add-opens', 'java.sql/java.sql=ALL-UNNAMED', + '--add-opens', 'java.xml.crypto/com.sun.org.apache.xml.internal.security.utils=ALL-UNNAMED' + ] +} + +repositories { + mavenCentral() +} + +dependencies { + compileOnly(project(':libs:core')) + implementation 'org.ow2.asm:asm:9.7.1' + implementation 'org.ow2.asm:asm-util:9.7.1' + implementation(project(':libs:entitlement:tools:common')) +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} + +tasks.named("thirdPartyAudit").configure { + ignoreMissingClasses() +} diff --git a/libs/entitlement/tools/public-callers-finder/licenses/asm-LICENSE.txt b/libs/entitlement/tools/public-callers-finder/licenses/asm-LICENSE.txt new file mode 100644 index 0000000000000..afb064f2f2666 --- /dev/null +++ b/libs/entitlement/tools/public-callers-finder/licenses/asm-LICENSE.txt @@ -0,0 +1,26 @@ +Copyright (c) 2012 France Télécom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/libs/entitlement/tools/public-callers-finder/licenses/asm-NOTICE.txt b/libs/entitlement/tools/public-callers-finder/licenses/asm-NOTICE.txt new file mode 100644 index 0000000000000..8d1c8b69c3fce --- /dev/null +++ b/libs/entitlement/tools/public-callers-finder/licenses/asm-NOTICE.txt @@ -0,0 +1 @@ + diff --git a/libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/FindUsagesClassVisitor.java b/libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/FindUsagesClassVisitor.java new file mode 100644 index 0000000000000..6f136d0977e3f --- /dev/null +++ b/libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/FindUsagesClassVisitor.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.tools.publiccallersfinder; + +import org.elasticsearch.entitlement.tools.ExternalAccess; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Type; + +import java.lang.constant.ClassDesc; +import java.lang.reflect.AccessFlag; +import java.util.EnumSet; +import java.util.Set; + +import static org.objectweb.asm.Opcodes.ACC_PROTECTED; +import static org.objectweb.asm.Opcodes.ACC_PUBLIC; +import static org.objectweb.asm.Opcodes.ASM9; + +class FindUsagesClassVisitor extends ClassVisitor { + + private int classAccess; + private boolean accessibleViaInterfaces; + + record MethodDescriptor(String className, String methodName, String methodDescriptor) {} + + record EntryPoint( + String moduleName, + String source, + int line, + String className, + String methodName, + String methodDescriptor, + EnumSet access + ) {} + + interface CallerConsumer { + void accept(String source, int line, String className, String methodName, String methodDescriptor, EnumSet access); + } + + private final Set moduleExports; + private final MethodDescriptor methodToFind; + private final CallerConsumer callers; + private String className; + private String source; + + protected FindUsagesClassVisitor(Set moduleExports, MethodDescriptor methodToFind, CallerConsumer callers) { + super(ASM9); + this.moduleExports = moduleExports; + this.methodToFind = methodToFind; + this.callers = callers; + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + super.visit(version, access, name, signature, superName, interfaces); + this.className = name; + this.classAccess = access; + if (interfaces.length > 0) { + this.accessibleViaInterfaces = findAccessibility(interfaces, moduleExports); + } + } + + private static boolean findAccessibility(String[] interfaces, Set moduleExports) { + var accessibleViaInterfaces = false; + for (var interfaceName : interfaces) { + if (moduleExports.contains(getPackageName(interfaceName))) { + var interfaceType = Type.getObjectType(interfaceName); + try { + var clazz = Class.forName(interfaceType.getClassName()); + if (clazz.accessFlags().contains(AccessFlag.PUBLIC)) { + accessibleViaInterfaces = true; + } + } catch (ClassNotFoundException ignored) {} + } + } + return accessibleViaInterfaces; + } + + @Override + public void visitSource(String source, String debug) { + super.visitSource(source, debug); + this.source = source; + } + + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + return new FindUsagesMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions), name, descriptor, access); + } + + private static String getPackageName(String className) { + return ClassDesc.ofInternalName(className).packageName(); + } + + private class FindUsagesMethodVisitor extends MethodVisitor { + + private final String methodName; + private int line; + private final String methodDescriptor; + private final int methodAccess; + + protected FindUsagesMethodVisitor(MethodVisitor mv, String methodName, String methodDescriptor, int methodAccess) { + super(ASM9, mv); + this.methodName = methodName; + this.methodDescriptor = methodDescriptor; + this.methodAccess = methodAccess; + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + + if (methodToFind.className.equals(owner)) { + if (methodToFind.methodName.equals(name)) { + if (methodToFind.methodDescriptor == null || methodToFind.methodDescriptor.equals(descriptor)) { + EnumSet externalAccess = ExternalAccess.fromPermissions( + moduleExports.contains(getPackageName(className)), + accessibleViaInterfaces || (classAccess & ACC_PUBLIC) != 0, + (methodAccess & ACC_PUBLIC) != 0, + (methodAccess & ACC_PROTECTED) != 0 + ); + callers.accept(source, line, className, methodName, methodDescriptor, externalAccess); + } + } + } + } + + @Override + public void visitLineNumber(int line, Label start) { + super.visitLineNumber(line, start); + this.line = line; + } + } +} diff --git a/libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/Main.java b/libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/Main.java new file mode 100644 index 0000000000000..60b3a3c9f3c8e --- /dev/null +++ b/libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/Main.java @@ -0,0 +1,197 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.tools.publiccallersfinder; + +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.tools.ExternalAccess; +import org.elasticsearch.entitlement.tools.Utils; +import org.objectweb.asm.ClassReader; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class Main { + + private static final String SEPARATOR = "\t"; + + record CallChain(FindUsagesClassVisitor.EntryPoint entryPoint, CallChain next) {} + + interface UsageConsumer { + void usageFound(CallChain originalEntryPoint, CallChain newMethod); + } + + private static void findTransitiveUsages( + Collection firstLevelCallers, + List classesToScan, + Set moduleExports, + boolean bubbleUpFromPublic, + UsageConsumer usageConsumer + ) { + for (var caller : firstLevelCallers) { + var methodsToCheck = new ArrayDeque<>(Set.of(caller)); + var methodsSeen = new HashSet(); + + while (methodsToCheck.isEmpty() == false) { + var methodToCheck = methodsToCheck.removeFirst(); + var m = methodToCheck.entryPoint(); + var visitor2 = new FindUsagesClassVisitor( + moduleExports, + new FindUsagesClassVisitor.MethodDescriptor(m.className(), m.methodName(), m.methodDescriptor()), + (source, line, className, methodName, methodDescriptor, access) -> { + var newMethod = new CallChain( + new FindUsagesClassVisitor.EntryPoint( + m.moduleName(), + source, + line, + className, + methodName, + methodDescriptor, + access + ), + methodToCheck + ); + + var notSeenBefore = methodsSeen.add(newMethod.entryPoint()); + if (notSeenBefore) { + if (ExternalAccess.isExternallyAccessible(access)) { + usageConsumer.usageFound(caller.next(), newMethod); + } + if (access.contains(ExternalAccess.PUBLIC_METHOD) == false || bubbleUpFromPublic) { + methodsToCheck.add(newMethod); + } + } + } + ); + + for (var classFile : classesToScan) { + try { + ClassReader cr = new ClassReader(Files.newInputStream(classFile)); + cr.accept(visitor2, 0); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + } + } + + private static void identifyTopLevelEntryPoints( + FindUsagesClassVisitor.MethodDescriptor methodToFind, + String methodToFindModule, + EnumSet methodToFindAccess, + boolean bubbleUpFromPublic + ) throws IOException { + + Utils.walkJdkModules((moduleName, moduleClasses, moduleExports) -> { + var originalCallers = new ArrayList(); + var visitor = new FindUsagesClassVisitor( + moduleExports, + methodToFind, + (source, line, className, methodName, methodDescriptor, access) -> originalCallers.add( + new CallChain( + new FindUsagesClassVisitor.EntryPoint(moduleName, source, line, className, methodName, methodDescriptor, access), + new CallChain( + new FindUsagesClassVisitor.EntryPoint( + methodToFindModule, + "", + 0, + methodToFind.className(), + methodToFind.methodName(), + methodToFind.methodDescriptor(), + methodToFindAccess + ), + null + ) + ) + ) + ); + + for (var classFile : moduleClasses) { + try { + ClassReader cr = new ClassReader(Files.newInputStream(classFile)); + cr.accept(visitor, 0); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + originalCallers.stream().filter(c -> ExternalAccess.isExternallyAccessible(c.entryPoint().access())).forEach(c -> { + var originalCaller = c.next(); + printRow(getEntryPointString(c.entryPoint().moduleName(), c.entryPoint()), getOriginalEntryPointString(originalCaller)); + }); + var firstLevelCallers = bubbleUpFromPublic ? originalCallers : originalCallers.stream().filter(Main::isNotFullyPublic).toList(); + + if (firstLevelCallers.isEmpty() == false) { + findTransitiveUsages( + firstLevelCallers, + moduleClasses, + moduleExports, + bubbleUpFromPublic, + (originalEntryPoint, newMethod) -> printRow( + getEntryPointString(moduleName, newMethod.entryPoint()), + getOriginalEntryPointString(originalEntryPoint) + ) + ); + } + }); + } + + private static boolean isNotFullyPublic(CallChain c) { + return (c.entryPoint().access().contains(ExternalAccess.PUBLIC_CLASS) + && c.entryPoint().access().contains(ExternalAccess.PUBLIC_METHOD)) == false; + } + + @SuppressForbidden(reason = "This tool prints the CSV to stdout") + private static void printRow(String entryPointString, String originalEntryPoint) { + System.out.println(entryPointString + SEPARATOR + originalEntryPoint); + } + + private static String getEntryPointString(String moduleName, FindUsagesClassVisitor.EntryPoint e) { + return moduleName + SEPARATOR + e.source() + SEPARATOR + e.line() + SEPARATOR + e.className() + SEPARATOR + e.methodName() + + SEPARATOR + e.methodDescriptor() + SEPARATOR + ExternalAccess.toString(e.access()); + } + + private static String getOriginalEntryPointString(CallChain originalCallChain) { + return originalCallChain.entryPoint().moduleName() + SEPARATOR + originalCallChain.entryPoint().className() + SEPARATOR + + originalCallChain.entryPoint().methodName() + SEPARATOR + ExternalAccess.toString(originalCallChain.entryPoint().access()); + } + + interface MethodDescriptorConsumer { + void accept(FindUsagesClassVisitor.MethodDescriptor methodDescriptor, String moduleName, EnumSet access) + throws IOException; + } + + private static void parseCsv(Path csvPath, MethodDescriptorConsumer methodConsumer) throws IOException { + var lines = Files.readAllLines(csvPath); + for (var l : lines) { + var tokens = l.split(SEPARATOR); + var moduleName = tokens[0]; + var className = tokens[3]; + var methodName = tokens[4]; + var methodDescriptor = tokens[5]; + var access = ExternalAccess.fromString(tokens[6]); + methodConsumer.accept(new FindUsagesClassVisitor.MethodDescriptor(className, methodName, methodDescriptor), moduleName, access); + } + } + + public static void main(String[] args) throws IOException { + var csvFilePath = Path.of(args[0]); + boolean bubbleUpFromPublic = args.length >= 2 && Boolean.parseBoolean(args[1]); + parseCsv(csvFilePath, (method, module, access) -> identifyTopLevelEntryPoints(method, module, access, bubbleUpFromPublic)); + } +} diff --git a/libs/entitlement/tools/securitymanager-scanner/src/README.md b/libs/entitlement/tools/securitymanager-scanner/README.md similarity index 100% rename from libs/entitlement/tools/securitymanager-scanner/src/README.md rename to libs/entitlement/tools/securitymanager-scanner/README.md diff --git a/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/Main.java b/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/Main.java index bea49e0296e67..7c2dd69d60f0c 100644 --- a/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/Main.java +++ b/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/Main.java @@ -10,47 +10,35 @@ package org.elasticsearch.entitlement.tools.securitymanager.scanner; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.tools.ExternalAccess; import org.elasticsearch.entitlement.tools.Utils; import org.objectweb.asm.ClassReader; import java.io.IOException; -import java.net.URI; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.util.HashMap; import java.util.List; -import java.util.Set; public class Main { - static final Set excludedModules = Set.of("java.desktop"); - private static void identifySMChecksEntryPoints() throws IOException { - FileSystem fs = FileSystems.getFileSystem(URI.create("jrt:/")); - - var moduleExports = Utils.findModuleExports(fs); - var callers = new HashMap>(); var visitor = new SecurityCheckClassVisitor(callers); - try (var stream = Files.walk(fs.getPath("modules"))) { - stream.filter(x -> x.toString().endsWith(".class")).forEach(x -> { - var moduleName = x.subpath(1, 2).toString(); - if (excludedModules.contains(moduleName) == false) { - try { - ClassReader cr = new ClassReader(Files.newInputStream(x)); - visitor.setCurrentModule(moduleName, moduleExports.get(moduleName)); - var path = x.getNameCount() > 3 ? x.subpath(2, x.getNameCount() - 1).toString() : ""; - visitor.setCurrentSourcePath(path); - cr.accept(visitor, 0); - } catch (IOException e) { - throw new RuntimeException(e); - } + Utils.walkJdkModules((moduleName, moduleClasses, moduleExports) -> { + for (var classFile : moduleClasses) { + try { + ClassReader cr = new ClassReader(Files.newInputStream(classFile)); + visitor.setCurrentModule(moduleName, moduleExports); + var path = classFile.getNameCount() > 3 ? classFile.subpath(2, classFile.getNameCount() - 1).toString() : ""; + visitor.setCurrentSourcePath(path); + cr.accept(visitor, 0); + } catch (IOException e) { + throw new RuntimeException(e); } - }); - } + } + }); printToStdout(callers); } @@ -68,16 +56,8 @@ private static void printToStdout(HashMap excludedClasses = Set.of(SECURITY_MANAGER_INTERNAL_NAME); - enum ExternalAccess { - CLASS, - METHOD - } - record CallerInfo( String moduleName, String source, @@ -208,15 +205,12 @@ public void visitMethodInsn(int opcode, String owner, String name, String descri || opcode == INVOKEDYNAMIC) { if (SECURITY_MANAGER_INTERNAL_NAME.equals(owner)) { - EnumSet externalAccesses = EnumSet.noneOf(ExternalAccess.class); - if (moduleExports.contains(getPackageName(className))) { - if ((methodAccess & ACC_PUBLIC) != 0) { - externalAccesses.add(ExternalAccess.METHOD); - } - if ((classAccess & ACC_PUBLIC) != 0) { - externalAccesses.add(ExternalAccess.CLASS); - } - } + EnumSet externalAccesses = ExternalAccess.fromPermissions( + moduleExports.contains(getPackageName(className)), + (classAccess & ACC_PUBLIC) != 0, + (methodAccess & ACC_PUBLIC) != 0, + (methodAccess & ACC_PROTECTED) != 0 + ); if (name.equals("checkPermission")) { var callers = callerInfoByMethod.computeIfAbsent(name, ignored -> new ArrayList<>()); From 0441555503593cf40fcb04b6edc49da3734a0738 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Fri, 13 Dec 2024 16:46:25 +0100 Subject: [PATCH 11/24] ESQL: Disable grok.OverwriteName* on pre-8.13 BWC tests (#118655) This prevents two tests in `grok` and `dissect` suites - `overwriteName` and `overwriteNameWhere` and one in the `stats` suite - `byStringAndLongWithAlias` - to run against pre-8.13.0 versions. Reason being that coordinators prior to that version can generate invalid node plans, that'd fail (verification) on 8.18+ nodes. --- .../qa/testFixtures/src/main/resources/dissect.csv-spec | 6 ++++-- .../esql/qa/testFixtures/src/main/resources/grok.csv-spec | 6 ++++-- .../esql/qa/testFixtures/src/main/resources/stats.csv-spec | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec index 38f09d2e3c56e..cde5427bf37d6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec @@ -223,7 +223,8 @@ null | null | null ; -overwriteName +// the query is incorrectly physically plan (fails the verification) in pre-8.13.0 versions +overwriteName#[skip:-8.12.99] from employees | sort emp_no asc | eval full_name = concat(first_name, " ", last_name) | dissect full_name "%{emp_no} %{b}" | keep full_name, emp_no, b | limit 3; full_name:keyword | emp_no:keyword | b:keyword @@ -244,7 +245,8 @@ emp_no:integer | first_name:keyword | rest:keyword ; -overwriteNameWhere +// the query is incorrectly physically plan (fails the verification) in pre-8.13.0 versions +overwriteNameWhere#[skip:-8.12.99] from employees | sort emp_no asc | eval full_name = concat(first_name, " ", last_name) | dissect full_name "%{emp_no} %{b}" | where emp_no == "Bezalel" | keep full_name, emp_no, b | limit 3; full_name:keyword | emp_no:keyword | b:keyword diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec index 98c88d06caa75..eece1bdfbffa4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec @@ -199,7 +199,8 @@ null | null | null ; -overwriteName +// the query is incorrectly physically plan (fails the verification) in pre-8.13.0 versions +overwriteName#[skip:-8.12.99] from employees | sort emp_no asc | eval full_name = concat(first_name, " ", last_name) | grok full_name "%{WORD:emp_no} %{WORD:b}" | keep full_name, emp_no, b | limit 3; full_name:keyword | emp_no:keyword | b:keyword @@ -209,7 +210,8 @@ Parto Bamford | Parto | Bamford ; -overwriteNameWhere +// the query is incorrectly physically plan (fails the verification) in pre-8.13.0 versions +overwriteNameWhere#[skip:-8.12.99] from employees | sort emp_no asc | eval full_name = concat(first_name, " ", last_name) | grok full_name "%{WORD:emp_no} %{WORD:b}" | where emp_no == "Bezalel" | keep full_name, emp_no, b | limit 3; full_name:keyword | emp_no:keyword | b:keyword diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index d76f4c05d955f..100c0d716d65c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -564,7 +564,8 @@ c:long | gender:keyword | trunk_worked_seconds:long 0 | null | 200000000 ; -byStringAndLongWithAlias +// the query is incorrectly physically plan (fails the verification) in pre-8.13.0 versions +byStringAndLongWithAlias#[skip:-8.12.99] FROM employees | EVAL trunk_worked_seconds = avg_worked_seconds / 100000000 * 100000000 | RENAME gender as g, trunk_worked_seconds as tws From cf7cb4bbb95996136a49340cc72acb850a703b44 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 13 Dec 2024 15:58:16 +0000 Subject: [PATCH 12/24] Bump versions after 8.17.0 release --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 6 +++--- .buildkite/pipelines/periodic.yml | 10 +++++----- .ci/bwcVersions | 2 +- .ci/snapshotBwcVersions | 2 +- server/src/main/java/org/elasticsearch/Version.java | 1 + .../resources/org/elasticsearch/TransportVersions.csv | 1 + .../org/elasticsearch/index/IndexVersions.csv | 1 + 8 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 6c8b8edfcbac1..6e15d64154960 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -56,7 +56,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"] + BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 2fbcd075b9719..9619de3c2c98b 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -301,8 +301,8 @@ steps: env: BWC_VERSION: 8.16.2 - - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0 + - label: "{{matrix.image}} / 8.17.1 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.1 timeout_in_minutes: 300 matrix: setup: @@ -315,7 +315,7 @@ steps: machineType: custom-16-32768 buildDirectory: /dev/shm/bk env: - BWC_VERSION: 8.17.0 + BWC_VERSION: 8.17.1 - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 94c9020a794a2..f2d169cd2b30d 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -325,8 +325,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.17.0 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.0#bwcTest + - label: 8.17.1 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.1#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -335,7 +335,7 @@ steps: buildDirectory: /dev/shm/bk preemptible: true env: - BWC_VERSION: 8.17.0 + BWC_VERSION: 8.17.1 retry: automatic: - exit_status: "-1" @@ -448,7 +448,7 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk21 - BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"] + BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -490,7 +490,7 @@ steps: ES_RUNTIME_JAVA: - openjdk21 - openjdk23 - BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"] + BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 79de891452117..3cb983373138f 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -16,6 +16,6 @@ BWC_VERSION: - "8.14.3" - "8.15.5" - "8.16.2" - - "8.17.0" + - "8.17.1" - "8.18.0" - "9.0.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 5514fc376a285..e05c0774c9819 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,5 +1,5 @@ BWC_VERSION: - "8.16.2" - - "8.17.0" + - "8.17.1" - "8.18.0" - "9.0.0" diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index f03505de310d5..47c43eadcfb03 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -192,6 +192,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_16_1 = new Version(8_16_01_99); public static final Version V_8_16_2 = new Version(8_16_02_99); public static final Version V_8_17_0 = new Version(8_17_00_99); + public static final Version V_8_17_1 = new Version(8_17_01_99); public static final Version V_8_18_0 = new Version(8_18_00_99); public static final Version V_9_0_0 = new Version(9_00_00_99); public static final Version CURRENT = V_9_0_0; diff --git a/server/src/main/resources/org/elasticsearch/TransportVersions.csv b/server/src/main/resources/org/elasticsearch/TransportVersions.csv index faeb7fe848159..08db0822dfef5 100644 --- a/server/src/main/resources/org/elasticsearch/TransportVersions.csv +++ b/server/src/main/resources/org/elasticsearch/TransportVersions.csv @@ -135,3 +135,4 @@ 8.15.5,8702003 8.16.0,8772001 8.16.1,8772004 +8.17.0,8797002 diff --git a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv index 1fc8bd8648ad6..afe696f31d323 100644 --- a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv +++ b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv @@ -135,3 +135,4 @@ 8.15.5,8512000 8.16.0,8518000 8.16.1,8518000 +8.17.0,8521000 From a68269b1763b9ed843d4e778373f38233c426aa5 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 13 Dec 2024 15:59:29 +0000 Subject: [PATCH 13/24] Prune changelogs after 8.17.0 release --- docs/changelog/104683.yaml | 5 ----- docs/changelog/112881.yaml | 5 ----- docs/changelog/112989.yaml | 5 ----- docs/changelog/113194.yaml | 5 ----- docs/changelog/113713.yaml | 5 ----- docs/changelog/113920.yaml | 5 ----- docs/changelog/114334.yaml | 7 ------- docs/changelog/114482.yaml | 5 ----- docs/changelog/114484.yaml | 6 ------ docs/changelog/114620.yaml | 5 ----- docs/changelog/114665.yaml | 6 ------ docs/changelog/114681.yaml | 6 ------ docs/changelog/114742.yaml | 5 ----- docs/changelog/114819.yaml | 6 ------ docs/changelog/114855.yaml | 5 ----- docs/changelog/114862.yaml | 5 ----- docs/changelog/114869.yaml | 5 ----- docs/changelog/114899.yaml | 5 ----- docs/changelog/114924.yaml | 5 ----- docs/changelog/114934.yaml | 6 ------ docs/changelog/114964.yaml | 6 ------ docs/changelog/115041.yaml | 6 ------ docs/changelog/115091.yaml | 7 ------- docs/changelog/115102.yaml | 6 ------ docs/changelog/115142.yaml | 6 ------ docs/changelog/115266.yaml | 6 ------ docs/changelog/115359.yaml | 6 ------ docs/changelog/115414.yaml | 9 --------- docs/changelog/115585.yaml | 6 ------ docs/changelog/115640.yaml | 6 ------ docs/changelog/115655.yaml | 5 ----- docs/changelog/115678.yaml | 5 ----- docs/changelog/115687.yaml | 5 ----- docs/changelog/115744.yaml | 6 ------ docs/changelog/115792.yaml | 5 ----- docs/changelog/115797.yaml | 6 ------ docs/changelog/115807.yaml | 5 ----- docs/changelog/115812.yaml | 5 ----- docs/changelog/115814.yaml | 6 ------ docs/changelog/115858.yaml | 5 ----- docs/changelog/115994.yaml | 5 ----- docs/changelog/116021.yaml | 6 ------ docs/changelog/116082.yaml | 5 ----- docs/changelog/116128.yaml | 5 ----- docs/changelog/116211.yaml | 5 ----- docs/changelog/116325.yaml | 5 ----- docs/changelog/116346.yaml | 5 ----- docs/changelog/116348.yaml | 5 ----- docs/changelog/116431.yaml | 5 ----- docs/changelog/116437.yaml | 5 ----- docs/changelog/116447.yaml | 5 ----- docs/changelog/116515.yaml | 5 ----- docs/changelog/116583.yaml | 7 ------- docs/changelog/116591.yaml | 5 ----- docs/changelog/116656.yaml | 6 ------ docs/changelog/116664.yaml | 6 ------ docs/changelog/116689.yaml | 10 ---------- docs/changelog/116809.yaml | 5 ----- docs/changelog/116819.yaml | 5 ----- docs/changelog/116931.yaml | 5 ----- docs/changelog/116953.yaml | 6 ------ docs/changelog/116957.yaml | 5 ----- docs/changelog/116962.yaml | 5 ----- docs/changelog/116980.yaml | 6 ------ docs/changelog/117080.yaml | 5 ----- docs/changelog/117105.yaml | 6 ------ docs/changelog/117189.yaml | 5 ----- docs/changelog/117213.yaml | 6 ------ docs/changelog/117271.yaml | 5 ----- docs/changelog/117294.yaml | 5 ----- docs/changelog/117297.yaml | 5 ----- docs/changelog/117312.yaml | 5 ----- docs/changelog/117316.yaml | 5 ----- docs/changelog/117350.yaml | 5 ----- docs/changelog/117404.yaml | 5 ----- docs/changelog/117503.yaml | 6 ------ docs/changelog/117551.yaml | 5 ----- docs/changelog/117575.yaml | 5 ----- docs/changelog/117595.yaml | 5 ----- docs/changelog/117657.yaml | 5 ----- docs/changelog/117762.yaml | 6 ------ docs/changelog/117792.yaml | 6 ------ docs/changelog/117842.yaml | 5 ----- docs/changelog/117865.yaml | 5 ----- docs/changelog/117914.yaml | 5 ----- docs/changelog/117920.yaml | 6 ------ docs/changelog/117953.yaml | 5 ----- docs/changelog/118354.yaml | 5 ----- docs/changelog/118370.yaml | 6 ------ docs/changelog/118378.yaml | 5 ----- 90 files changed, 493 deletions(-) delete mode 100644 docs/changelog/104683.yaml delete mode 100644 docs/changelog/112881.yaml delete mode 100644 docs/changelog/112989.yaml delete mode 100644 docs/changelog/113194.yaml delete mode 100644 docs/changelog/113713.yaml delete mode 100644 docs/changelog/113920.yaml delete mode 100644 docs/changelog/114334.yaml delete mode 100644 docs/changelog/114482.yaml delete mode 100644 docs/changelog/114484.yaml delete mode 100644 docs/changelog/114620.yaml delete mode 100644 docs/changelog/114665.yaml delete mode 100644 docs/changelog/114681.yaml delete mode 100644 docs/changelog/114742.yaml delete mode 100644 docs/changelog/114819.yaml delete mode 100644 docs/changelog/114855.yaml delete mode 100644 docs/changelog/114862.yaml delete mode 100644 docs/changelog/114869.yaml delete mode 100644 docs/changelog/114899.yaml delete mode 100644 docs/changelog/114924.yaml delete mode 100644 docs/changelog/114934.yaml delete mode 100644 docs/changelog/114964.yaml delete mode 100644 docs/changelog/115041.yaml delete mode 100644 docs/changelog/115091.yaml delete mode 100644 docs/changelog/115102.yaml delete mode 100644 docs/changelog/115142.yaml delete mode 100644 docs/changelog/115266.yaml delete mode 100644 docs/changelog/115359.yaml delete mode 100644 docs/changelog/115414.yaml delete mode 100644 docs/changelog/115585.yaml delete mode 100644 docs/changelog/115640.yaml delete mode 100644 docs/changelog/115655.yaml delete mode 100644 docs/changelog/115678.yaml delete mode 100644 docs/changelog/115687.yaml delete mode 100644 docs/changelog/115744.yaml delete mode 100644 docs/changelog/115792.yaml delete mode 100644 docs/changelog/115797.yaml delete mode 100644 docs/changelog/115807.yaml delete mode 100644 docs/changelog/115812.yaml delete mode 100644 docs/changelog/115814.yaml delete mode 100644 docs/changelog/115858.yaml delete mode 100644 docs/changelog/115994.yaml delete mode 100644 docs/changelog/116021.yaml delete mode 100644 docs/changelog/116082.yaml delete mode 100644 docs/changelog/116128.yaml delete mode 100644 docs/changelog/116211.yaml delete mode 100644 docs/changelog/116325.yaml delete mode 100644 docs/changelog/116346.yaml delete mode 100644 docs/changelog/116348.yaml delete mode 100644 docs/changelog/116431.yaml delete mode 100644 docs/changelog/116437.yaml delete mode 100644 docs/changelog/116447.yaml delete mode 100644 docs/changelog/116515.yaml delete mode 100644 docs/changelog/116583.yaml delete mode 100644 docs/changelog/116591.yaml delete mode 100644 docs/changelog/116656.yaml delete mode 100644 docs/changelog/116664.yaml delete mode 100644 docs/changelog/116689.yaml delete mode 100644 docs/changelog/116809.yaml delete mode 100644 docs/changelog/116819.yaml delete mode 100644 docs/changelog/116931.yaml delete mode 100644 docs/changelog/116953.yaml delete mode 100644 docs/changelog/116957.yaml delete mode 100644 docs/changelog/116962.yaml delete mode 100644 docs/changelog/116980.yaml delete mode 100644 docs/changelog/117080.yaml delete mode 100644 docs/changelog/117105.yaml delete mode 100644 docs/changelog/117189.yaml delete mode 100644 docs/changelog/117213.yaml delete mode 100644 docs/changelog/117271.yaml delete mode 100644 docs/changelog/117294.yaml delete mode 100644 docs/changelog/117297.yaml delete mode 100644 docs/changelog/117312.yaml delete mode 100644 docs/changelog/117316.yaml delete mode 100644 docs/changelog/117350.yaml delete mode 100644 docs/changelog/117404.yaml delete mode 100644 docs/changelog/117503.yaml delete mode 100644 docs/changelog/117551.yaml delete mode 100644 docs/changelog/117575.yaml delete mode 100644 docs/changelog/117595.yaml delete mode 100644 docs/changelog/117657.yaml delete mode 100644 docs/changelog/117762.yaml delete mode 100644 docs/changelog/117792.yaml delete mode 100644 docs/changelog/117842.yaml delete mode 100644 docs/changelog/117865.yaml delete mode 100644 docs/changelog/117914.yaml delete mode 100644 docs/changelog/117920.yaml delete mode 100644 docs/changelog/117953.yaml delete mode 100644 docs/changelog/118354.yaml delete mode 100644 docs/changelog/118370.yaml delete mode 100644 docs/changelog/118378.yaml diff --git a/docs/changelog/104683.yaml b/docs/changelog/104683.yaml deleted file mode 100644 index d4f40b59cfd91..0000000000000 --- a/docs/changelog/104683.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 104683 -summary: "Feature: re-structure document ID generation favoring _id inverted index compression" -area: Logs -type: enhancement -issues: [] diff --git a/docs/changelog/112881.yaml b/docs/changelog/112881.yaml deleted file mode 100644 index a8a0d542f8201..0000000000000 --- a/docs/changelog/112881.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112881 -summary: "ESQL: Remove parent from `FieldAttribute`" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/112989.yaml b/docs/changelog/112989.yaml deleted file mode 100644 index 364f012f94420..0000000000000 --- a/docs/changelog/112989.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112989 -summary: Upgrade Bouncy Castle FIPS dependencies -area: Security -type: upgrade -issues: [] diff --git a/docs/changelog/113194.yaml b/docs/changelog/113194.yaml deleted file mode 100644 index 132659321c65e..0000000000000 --- a/docs/changelog/113194.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 113194 -summary: Add Search Phase APM metrics -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/113713.yaml b/docs/changelog/113713.yaml deleted file mode 100644 index c5478c95e464d..0000000000000 --- a/docs/changelog/113713.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 113713 -summary: Adding inference endpoint validation for `AzureAiStudioService` -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/113920.yaml b/docs/changelog/113920.yaml deleted file mode 100644 index 4699ae6d7dd65..0000000000000 --- a/docs/changelog/113920.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 113920 -summary: Add initial support for `semantic_text` field type -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/114334.yaml b/docs/changelog/114334.yaml deleted file mode 100644 index d0fefe40c6970..0000000000000 --- a/docs/changelog/114334.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 114334 -summary: Don't return TEXT type for functions that take TEXT -area: ES|QL -type: bug -issues: - - 111537 - - 114333 diff --git a/docs/changelog/114482.yaml b/docs/changelog/114482.yaml deleted file mode 100644 index a5e2e981f7adc..0000000000000 --- a/docs/changelog/114482.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114482 -summary: Remove snapshot build restriction for match and qstr functions -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/114484.yaml b/docs/changelog/114484.yaml deleted file mode 100644 index 48f54ad0218bb..0000000000000 --- a/docs/changelog/114484.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 114484 -summary: Add `docvalue_fields` Support for `dense_vector` Fields -area: Search -type: enhancement -issues: - - 108470 diff --git a/docs/changelog/114620.yaml b/docs/changelog/114620.yaml deleted file mode 100644 index 92498db92061f..0000000000000 --- a/docs/changelog/114620.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114620 -summary: "ES|QL: add metrics for functions" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/114665.yaml b/docs/changelog/114665.yaml deleted file mode 100644 index b90bb799bd896..0000000000000 --- a/docs/changelog/114665.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 114665 -summary: Fixing remote ENRICH by pushing the Enrich inside `FragmentExec` -area: ES|QL -type: bug -issues: - - 105095 diff --git a/docs/changelog/114681.yaml b/docs/changelog/114681.yaml deleted file mode 100644 index 2a9901114e56f..0000000000000 --- a/docs/changelog/114681.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 114681 -summary: "Support for unsigned 64 bit numbers in Cpu stats" -area: Infra/Core -type: enhancement -issues: - - 112274 diff --git a/docs/changelog/114742.yaml b/docs/changelog/114742.yaml deleted file mode 100644 index 5bd3dad4400b8..0000000000000 --- a/docs/changelog/114742.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114742 -summary: Adding support for additional mapping to simulate ingest API -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/114819.yaml b/docs/changelog/114819.yaml deleted file mode 100644 index f8d03f7024801..0000000000000 --- a/docs/changelog/114819.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 114819 -summary: Don't use a `BytesStreamOutput` to copy keys in `BytesRefBlockHash` -area: EQL -type: bug -issues: - - 114599 diff --git a/docs/changelog/114855.yaml b/docs/changelog/114855.yaml deleted file mode 100644 index daa6b985a14cf..0000000000000 --- a/docs/changelog/114855.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114855 -summary: Add query rules retriever -area: Relevance -type: enhancement -issues: [ ] diff --git a/docs/changelog/114862.yaml b/docs/changelog/114862.yaml deleted file mode 100644 index fb5f05fb8e2f9..0000000000000 --- a/docs/changelog/114862.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114862 -summary: "[Inference API] Add API to get configuration of inference services" -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/114869.yaml b/docs/changelog/114869.yaml deleted file mode 100644 index 755418e7ce4d9..0000000000000 --- a/docs/changelog/114869.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114869 -summary: Standardize error code when bulk body is invalid -area: CRUD -type: bug -issues: [] diff --git a/docs/changelog/114899.yaml b/docs/changelog/114899.yaml deleted file mode 100644 index 399aa5cf35409..0000000000000 --- a/docs/changelog/114899.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114899 -summary: "ES|QL: Fix stats by constant expression" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/114924.yaml b/docs/changelog/114924.yaml deleted file mode 100644 index 536f446ef790d..0000000000000 --- a/docs/changelog/114924.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114924 -summary: Reducing error-level stack trace logging for normal events in `GeoIpDownloader` -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/114934.yaml b/docs/changelog/114934.yaml deleted file mode 100644 index 68628993b1c80..0000000000000 --- a/docs/changelog/114934.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 114934 -summary: "[ES|QL] To_DatePeriod and To_TimeDuration return better error messages on\ - \ `union_type` fields" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/114964.yaml b/docs/changelog/114964.yaml deleted file mode 100644 index 8274aeb76a937..0000000000000 --- a/docs/changelog/114964.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 114964 -summary: Add a `monitor_stats` privilege and allow that privilege for remote cluster - privileges -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/115041.yaml b/docs/changelog/115041.yaml deleted file mode 100644 index f4c047c1569ec..0000000000000 --- a/docs/changelog/115041.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115041 -summary: Increase default `queue_capacity` to 10_000 and decrease max `queue_capacity` - to 100_000 -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/115091.yaml b/docs/changelog/115091.yaml deleted file mode 100644 index 762bcca5e8c52..0000000000000 --- a/docs/changelog/115091.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 115091 -summary: Added stricter range type checks and runtime warnings for ENRICH -area: ES|QL -type: bug -issues: - - 107357 - - 116799 diff --git a/docs/changelog/115102.yaml b/docs/changelog/115102.yaml deleted file mode 100644 index f679bb6c223a6..0000000000000 --- a/docs/changelog/115102.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115102 -summary: Watch Next Run Interval Resets On Shard Move or Node Restart -area: Watcher -type: bug -issues: - - 111433 diff --git a/docs/changelog/115142.yaml b/docs/changelog/115142.yaml deleted file mode 100644 index 2af968ae156da..0000000000000 --- a/docs/changelog/115142.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115142 -summary: Attempt to clean up index before remote transfer -area: Recovery -type: enhancement -issues: - - 104473 diff --git a/docs/changelog/115266.yaml b/docs/changelog/115266.yaml deleted file mode 100644 index 1d7fb1368c0e8..0000000000000 --- a/docs/changelog/115266.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115266 -summary: ES|QL CCS uses `skip_unavailable` setting for handling disconnected remote - clusters -area: ES|QL -type: enhancement -issues: [ 114531 ] diff --git a/docs/changelog/115359.yaml b/docs/changelog/115359.yaml deleted file mode 100644 index 65b3086dfc8d0..0000000000000 --- a/docs/changelog/115359.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115359 -summary: Adding support for simulate ingest mapping adddition for indices with mappings - that do not come from templates -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/115414.yaml b/docs/changelog/115414.yaml deleted file mode 100644 index 7475b765bb30e..0000000000000 --- a/docs/changelog/115414.yaml +++ /dev/null @@ -1,9 +0,0 @@ -pr: 115414 -summary: Mitigate IOSession timeouts -area: Machine Learning -type: bug -issues: - - 114385 - - 114327 - - 114105 - - 114232 diff --git a/docs/changelog/115585.yaml b/docs/changelog/115585.yaml deleted file mode 100644 index 02eecfc3d7d2b..0000000000000 --- a/docs/changelog/115585.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115459 -summary: Adds access to flags no_sub_matches and no_overlapping_matches to hyphenation-decompounder-tokenfilter -area: Search -type: enhancement -issues: - - 97849 diff --git a/docs/changelog/115640.yaml b/docs/changelog/115640.yaml deleted file mode 100644 index 5c4a943a9697d..0000000000000 --- a/docs/changelog/115640.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115640 -summary: Fix NPE on plugin sync -area: Infra/CLI -type: bug -issues: - - 114818 diff --git a/docs/changelog/115655.yaml b/docs/changelog/115655.yaml deleted file mode 100644 index 7184405867657..0000000000000 --- a/docs/changelog/115655.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115655 -summary: Better sizing `BytesRef` for Strings in Queries -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/115678.yaml b/docs/changelog/115678.yaml deleted file mode 100644 index 31240eae1ebb4..0000000000000 --- a/docs/changelog/115678.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115678 -summary: "ESQL: extract common filter from aggs" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/115687.yaml b/docs/changelog/115687.yaml deleted file mode 100644 index 1180b4627c635..0000000000000 --- a/docs/changelog/115687.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115687 -summary: Add default ILM policies and switch to ILM for apm-data plugin -area: Data streams -type: feature -issues: [] diff --git a/docs/changelog/115744.yaml b/docs/changelog/115744.yaml deleted file mode 100644 index 9b8c91e59f451..0000000000000 --- a/docs/changelog/115744.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115744 -summary: Use `SearchStats` instead of field.isAggregatable in data node planning -area: ES|QL -type: bug -issues: - - 115737 diff --git a/docs/changelog/115792.yaml b/docs/changelog/115792.yaml deleted file mode 100644 index 2945a64e3043a..0000000000000 --- a/docs/changelog/115792.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115792 -summary: Add ES|QL `bit_length` function -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/115797.yaml b/docs/changelog/115797.yaml deleted file mode 100644 index 8adf51887c28a..0000000000000 --- a/docs/changelog/115797.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115797 -summary: Enable `_tier` based coordinator rewrites for all indices (not just mounted - indices) -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/115807.yaml b/docs/changelog/115807.yaml deleted file mode 100644 index d17cabca4bd03..0000000000000 --- a/docs/changelog/115807.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115807 -summary: "[Inference API] Improve chunked results error message" -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/115812.yaml b/docs/changelog/115812.yaml deleted file mode 100644 index c45c97041eb00..0000000000000 --- a/docs/changelog/115812.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115812 -summary: "Prohibit changes to index mode, source, and sort settings during resize" -area: Logs -type: bug -issues: [] diff --git a/docs/changelog/115814.yaml b/docs/changelog/115814.yaml deleted file mode 100644 index 34f1213272d6f..0000000000000 --- a/docs/changelog/115814.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115814 -summary: "[ES|QL] Implicit casting string literal to intervals" -area: ES|QL -type: enhancement -issues: - - 115352 diff --git a/docs/changelog/115858.yaml b/docs/changelog/115858.yaml deleted file mode 100644 index 0c0408fa656f8..0000000000000 --- a/docs/changelog/115858.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115858 -summary: "ESQL: optimise aggregations filtered by false/null into evals" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/115994.yaml b/docs/changelog/115994.yaml deleted file mode 100644 index ac090018c8a12..0000000000000 --- a/docs/changelog/115994.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115994 -summary: Add logsdb telemetry -area: Logs -type: enhancement -issues: [] diff --git a/docs/changelog/116021.yaml b/docs/changelog/116021.yaml deleted file mode 100644 index 58c84b26805b2..0000000000000 --- a/docs/changelog/116021.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 116021 -summary: Fields caps does not honour ignore_unavailable -area: Search -type: bug -issues: - - 107767 diff --git a/docs/changelog/116082.yaml b/docs/changelog/116082.yaml deleted file mode 100644 index 35ca5fb1ea82e..0000000000000 --- a/docs/changelog/116082.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116082 -summary: Add support for bitwise inner-product in painless -area: Vector Search -type: enhancement -issues: [] diff --git a/docs/changelog/116128.yaml b/docs/changelog/116128.yaml deleted file mode 100644 index 7c38c0529c50d..0000000000000 --- a/docs/changelog/116128.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116128 -summary: Add num docs and size to logsdb telemetry -area: Logs -type: enhancement -issues: [] diff --git a/docs/changelog/116211.yaml b/docs/changelog/116211.yaml deleted file mode 100644 index 6f55b1b2fef34..0000000000000 --- a/docs/changelog/116211.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116211 -summary: Use underlying `ByteBuf` `refCount` for `ReleasableBytesReference` -area: Network -type: bug -issues: [] diff --git a/docs/changelog/116325.yaml b/docs/changelog/116325.yaml deleted file mode 100644 index b8cd16dc85773..0000000000000 --- a/docs/changelog/116325.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116325 -summary: Adjust analyze limit exception to be a `bad_request` -area: Analysis -type: bug -issues: [] diff --git a/docs/changelog/116346.yaml b/docs/changelog/116346.yaml deleted file mode 100644 index 1dcace88a98c0..0000000000000 --- a/docs/changelog/116346.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116346 -summary: "[ESQL] Fix Binary Comparisons on Date Nanos" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/116348.yaml b/docs/changelog/116348.yaml deleted file mode 100644 index 927ffc5a6121d..0000000000000 --- a/docs/changelog/116348.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116348 -summary: "ESQL: Honor skip_unavailable setting for nonmatching indices errors at planning time" -area: ES|QL -type: enhancement -issues: [ 114531 ] diff --git a/docs/changelog/116431.yaml b/docs/changelog/116431.yaml deleted file mode 100644 index 50c6baf1d01c7..0000000000000 --- a/docs/changelog/116431.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116431 -summary: Adds support for `input_type` field to Vertex inference service -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/116437.yaml b/docs/changelog/116437.yaml deleted file mode 100644 index 94c2464db9980..0000000000000 --- a/docs/changelog/116437.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116437 -summary: Ensure class resource stream is closed in `ResourceUtils` -area: Indices APIs -type: enhancement -issues: [] diff --git a/docs/changelog/116447.yaml b/docs/changelog/116447.yaml deleted file mode 100644 index 8c0cea4b54578..0000000000000 --- a/docs/changelog/116447.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116447 -summary: Adding a deprecation info API warning for data streams with old indices -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/116515.yaml b/docs/changelog/116515.yaml deleted file mode 100644 index 6c0d473361e52..0000000000000 --- a/docs/changelog/116515.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116515 -summary: Esql/lookup join grammar -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/116583.yaml b/docs/changelog/116583.yaml deleted file mode 100644 index 3dc8337fe5b86..0000000000000 --- a/docs/changelog/116583.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 116583 -summary: Fix NPE in `EnrichLookupService` on mixed clusters with <8.14 versions -area: ES|QL -type: bug -issues: - - 116529 - - 116544 diff --git a/docs/changelog/116591.yaml b/docs/changelog/116591.yaml deleted file mode 100644 index 60ef241e197b3..0000000000000 --- a/docs/changelog/116591.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116591 -summary: "Add support for `BYTE_LENGTH` scalar function" -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/116656.yaml b/docs/changelog/116656.yaml deleted file mode 100644 index eb5d5a1cfc201..0000000000000 --- a/docs/changelog/116656.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 116656 -summary: _validate does not honour ignore_unavailable -area: Search -type: bug -issues: - - 116594 diff --git a/docs/changelog/116664.yaml b/docs/changelog/116664.yaml deleted file mode 100644 index 36915fca39731..0000000000000 --- a/docs/changelog/116664.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 116664 -summary: Hides `hugging_face_elser` service from the `GET _inference/_services API` -area: Machine Learning -type: bug -issues: - - 116644 diff --git a/docs/changelog/116689.yaml b/docs/changelog/116689.yaml deleted file mode 100644 index 0b1d1646868aa..0000000000000 --- a/docs/changelog/116689.yaml +++ /dev/null @@ -1,10 +0,0 @@ -pr: 116689 -summary: Deprecate `_source.mode` in mappings -area: Mapping -type: deprecation -issues: [] -deprecation: - title: Deprecate `_source.mode` in mappings - area: Mapping - details: Configuring `_source.mode` in mappings is deprecated and will be removed in future versions. Use `index.mapping.source.mode` index setting instead. - impact: Use `index.mapping.source.mode` index setting instead diff --git a/docs/changelog/116809.yaml b/docs/changelog/116809.yaml deleted file mode 100644 index 61dbeb233d576..0000000000000 --- a/docs/changelog/116809.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116809 -summary: "Distinguish `LicensedFeature` by family field" -area: License -type: bug -issues: [] diff --git a/docs/changelog/116819.yaml b/docs/changelog/116819.yaml deleted file mode 100644 index afe06c583fe55..0000000000000 --- a/docs/changelog/116819.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116819 -summary: ESQL - Add match operator (:) -area: Search -type: feature -issues: [] diff --git a/docs/changelog/116931.yaml b/docs/changelog/116931.yaml deleted file mode 100644 index 8b31d236ff137..0000000000000 --- a/docs/changelog/116931.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116931 -summary: Enable built-in Inference Endpoints and default for Semantic Text -area: "Machine Learning" -type: enhancement -issues: [] diff --git a/docs/changelog/116953.yaml b/docs/changelog/116953.yaml deleted file mode 100644 index 33616510d8fd0..0000000000000 --- a/docs/changelog/116953.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 116953 -summary: Fix false positive date detection with trailing dot -area: Mapping -type: bug -issues: - - 116946 diff --git a/docs/changelog/116957.yaml b/docs/changelog/116957.yaml deleted file mode 100644 index 1020190de180d..0000000000000 --- a/docs/changelog/116957.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116957 -summary: Propagate scoring function through random sampler -area: Machine Learning -type: bug -issues: [ 110134 ] diff --git a/docs/changelog/116962.yaml b/docs/changelog/116962.yaml deleted file mode 100644 index 8f16b00e3f9fc..0000000000000 --- a/docs/changelog/116962.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116962 -summary: "Add special case for elastic reranker in inference API" -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/116980.yaml b/docs/changelog/116980.yaml deleted file mode 100644 index 140324fd40b92..0000000000000 --- a/docs/changelog/116980.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 116980 -summary: "ESQL: Fix sorts containing `_source`" -area: ES|QL -type: bug -issues: - - 116659 diff --git a/docs/changelog/117080.yaml b/docs/changelog/117080.yaml deleted file mode 100644 index 5909f966e0fa2..0000000000000 --- a/docs/changelog/117080.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117080 -summary: Esql Enable Date Nanos (tech preview) -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/117105.yaml b/docs/changelog/117105.yaml deleted file mode 100644 index de56c4d521a62..0000000000000 --- a/docs/changelog/117105.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117105 -summary: Fix long metric deserialize & add - auto-resize needs to be set manually -area: CCS -type: bug -issues: - - 116914 diff --git a/docs/changelog/117189.yaml b/docs/changelog/117189.yaml deleted file mode 100644 index e89c2d81506d9..0000000000000 --- a/docs/changelog/117189.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117189 -summary: Fix deberta tokenizer bug caused by bug in normalizer -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/117213.yaml b/docs/changelog/117213.yaml deleted file mode 100644 index 3b4cd0cee966c..0000000000000 --- a/docs/changelog/117213.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117213 -summary: Fix reconstituting version string from components -area: Ingest Node -type: bug -issues: - - 116950 diff --git a/docs/changelog/117271.yaml b/docs/changelog/117271.yaml deleted file mode 100644 index 1a328279b9635..0000000000000 --- a/docs/changelog/117271.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117271 -summary: Don't skip shards in coord rewrite if timestamp is an alias -area: Search -type: bug -issues: [] diff --git a/docs/changelog/117294.yaml b/docs/changelog/117294.yaml deleted file mode 100644 index f6e80690de7ff..0000000000000 --- a/docs/changelog/117294.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117294 -summary: Always Emit Inference ID in Semantic Text Mapping -area: Mapping -type: bug -issues: [] diff --git a/docs/changelog/117297.yaml b/docs/changelog/117297.yaml deleted file mode 100644 index 4a0051bbae644..0000000000000 --- a/docs/changelog/117297.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117297 -summary: Fix CCS exchange when multi cluster aliases point to same cluster -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/117312.yaml b/docs/changelog/117312.yaml deleted file mode 100644 index 302b91388ef2b..0000000000000 --- a/docs/changelog/117312.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117312 -summary: Add missing `async_search` query parameters to rest-api-spec -area: Search -type: bug -issues: [] diff --git a/docs/changelog/117316.yaml b/docs/changelog/117316.yaml deleted file mode 100644 index 69474d68a8190..0000000000000 --- a/docs/changelog/117316.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117316 -summary: Fix validation of SORT by aggregate functions -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/117350.yaml b/docs/changelog/117350.yaml deleted file mode 100644 index dca54f2037a87..0000000000000 --- a/docs/changelog/117350.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117350 -summary: "Improve halfbyte transposition performance, marginally improving bbq performance" -area: Vector Search -type: enhancement -issues: [] diff --git a/docs/changelog/117404.yaml b/docs/changelog/117404.yaml deleted file mode 100644 index 0bab171956ca9..0000000000000 --- a/docs/changelog/117404.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117404 -summary: Correct bit * byte and bit * float script comparisons -area: Vector Search -type: bug -issues: [] diff --git a/docs/changelog/117503.yaml b/docs/changelog/117503.yaml deleted file mode 100644 index d48741262b581..0000000000000 --- a/docs/changelog/117503.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117503 -summary: Fix COUNT filter pushdown -area: ES|QL -type: bug -issues: - - 115522 diff --git a/docs/changelog/117551.yaml b/docs/changelog/117551.yaml deleted file mode 100644 index 081dd9203d82a..0000000000000 --- a/docs/changelog/117551.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117551 -summary: Fix stats by constant expresson with alias -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/117575.yaml b/docs/changelog/117575.yaml deleted file mode 100644 index 781444ae97be5..0000000000000 --- a/docs/changelog/117575.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117575 -summary: Fix enrich cache size setting name -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/117595.yaml b/docs/changelog/117595.yaml deleted file mode 100644 index 9360c372ac97e..0000000000000 --- a/docs/changelog/117595.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117595 -summary: Fix for Deberta tokenizer when input sequence exceeds 512 tokens -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/117657.yaml b/docs/changelog/117657.yaml deleted file mode 100644 index 0a72e9dabe9e8..0000000000000 --- a/docs/changelog/117657.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117657 -summary: Ignore cancellation exceptions -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/117762.yaml b/docs/changelog/117762.yaml deleted file mode 100644 index 123432e0f0507..0000000000000 --- a/docs/changelog/117762.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117762 -summary: "Parse the contents of dynamic objects for [subobjects:false]" -area: Mapping -type: bug -issues: - - 117544 diff --git a/docs/changelog/117792.yaml b/docs/changelog/117792.yaml deleted file mode 100644 index 2d7ddda1ace40..0000000000000 --- a/docs/changelog/117792.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117792 -summary: Address mapping and compute engine runtime field issues -area: Mapping -type: bug -issues: - - 117644 diff --git a/docs/changelog/117842.yaml b/docs/changelog/117842.yaml deleted file mode 100644 index 9b528a158288c..0000000000000 --- a/docs/changelog/117842.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117842 -summary: Limit size of `Literal#toString` -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/117865.yaml b/docs/changelog/117865.yaml deleted file mode 100644 index 33dc497725f92..0000000000000 --- a/docs/changelog/117865.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117865 -summary: Fix BWC for ES|QL cluster request -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/117914.yaml b/docs/changelog/117914.yaml deleted file mode 100644 index da58ed7bb04b7..0000000000000 --- a/docs/changelog/117914.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117914 -summary: Fix for propagating filters from compound to inner retrievers -area: Ranking -type: bug -issues: [] diff --git a/docs/changelog/117920.yaml b/docs/changelog/117920.yaml deleted file mode 100644 index 1bfddabd4462d..0000000000000 --- a/docs/changelog/117920.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117920 -summary: Wait for the worker service to shutdown before closing task processor -area: Machine Learning -type: bug -issues: - - 117563 diff --git a/docs/changelog/117953.yaml b/docs/changelog/117953.yaml deleted file mode 100644 index 62f0218b1cdc7..0000000000000 --- a/docs/changelog/117953.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117953 -summary: Acquire stats searcher for data stream stats -area: Data streams -type: bug -issues: [] diff --git a/docs/changelog/118354.yaml b/docs/changelog/118354.yaml deleted file mode 100644 index e2d72db121276..0000000000000 --- a/docs/changelog/118354.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 118354 -summary: Fix log message format bugs -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/118370.yaml b/docs/changelog/118370.yaml deleted file mode 100644 index e6a429448e493..0000000000000 --- a/docs/changelog/118370.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 118370 -summary: Fix concurrency issue with `ReinitializingSourceProvider` -area: Mapping -type: bug -issues: - - 118238 diff --git a/docs/changelog/118378.yaml b/docs/changelog/118378.yaml deleted file mode 100644 index d6c388b671968..0000000000000 --- a/docs/changelog/118378.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 118378 -summary: Opt into extra data stream resolution -area: ES|QL -type: bug -issues: [] From 950db572219e326d2986c744b20430dbfbd01a43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3zala?= <377355+jozala@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:27:24 +0100 Subject: [PATCH 14/24] [test] Avoid running the NoImds test on AWS (#118675) Disabled the NoImds test on AWS EC2 instance where it fails because the AWS metadata are available, which is not expected by this test. --- .../ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java index 602a98e17970d..73213090b6f93 100644 --- a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java +++ b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java @@ -9,6 +9,8 @@ package org.elasticsearch.discovery.ec2; +import com.amazonaws.util.EC2MetadataUtils; + import org.elasticsearch.client.Request; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; @@ -29,6 +31,8 @@ protected String getTestRestCluster() { } public void testAvailabilityZoneAttribute() throws IOException { + assumeTrue("test only in non-AWS environment", EC2MetadataUtils.getInstanceId() == null); + final var nodesInfoResponse = assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/_nodes/_all/_none"))); for (final var nodeId : nodesInfoResponse.evaluateMapKeys("nodes")) { assertNull(nodesInfoResponse.evaluateExact("nodes", nodeId, "attributes", "aws_availability_zone")); From 54e839b11068b9ea455befe1ce88d569b0f2c937 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 13 Dec 2024 17:42:08 +0100 Subject: [PATCH 15/24] ESQL: Fix LogicalPlanOptimizerTests testPlanSanityCheckWithBinaryPlans (#118672) --- muted-tests.yml | 3 --- .../xpack/esql/optimizer/LogicalPlanOptimizerTests.java | 3 +-- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 36dfc306b0147..e0d011c1bf239 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -317,9 +317,6 @@ tests: - class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests method: testInvalidJSON issue: https://github.com/elastic/elasticsearch/issues/116521 -- class: org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizerTests - method: testPlanSanityCheckWithBinaryPlans - issue: https://github.com/elastic/elasticsearch/issues/118656 # Examples: # diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 7e498eb6654b9..d97a8bb2bc27f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -4911,8 +4911,7 @@ public void testPlanSanityCheckWithBinaryPlans() throws Exception { """); var project = as(plan, Project.class); - var limit = as(project.child(), Limit.class); - var join = as(limit.child(), Join.class); + var join = as(project.child(), Join.class); var joinWithInvalidLeftPlan = join.replaceChildren(join.right(), join.right()); IllegalStateException e = expectThrows(IllegalStateException.class, () -> logicalOptimizer.optimize(joinWithInvalidLeftPlan)); From 6c56c32f7a872acda86232c542315f28215d24b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Slobodan=20Adamovi=C4=87?= Date: Fri, 13 Dec 2024 18:40:23 +0100 Subject: [PATCH 16/24] Grant necessary Kibana application privileges to `reporting_user` role (#118058) Previously, Kibana was authorizing (and granting application privileges) to create reports, simply based on the `reporting_user` role name. This PR makes these application privileges explicitly granted to the `reporting_user` role. --- docs/changelog/118058.yaml | 5 ++ .../authorization/built-in-roles.asciidoc | 11 ++- .../authz/store/ReservedRolesStore.java | 29 +++++-- .../authz/store/ReservedRolesStoreTests.java | 76 ++++++++++++++++++- 4 files changed, 106 insertions(+), 15 deletions(-) create mode 100644 docs/changelog/118058.yaml diff --git a/docs/changelog/118058.yaml b/docs/changelog/118058.yaml new file mode 100644 index 0000000000000..d5fad346d4d85 --- /dev/null +++ b/docs/changelog/118058.yaml @@ -0,0 +1,5 @@ +pr: 118058 +summary: Grant necessary Kibana application privileges to `reporting_user` role +area: Authorization +type: enhancement +issues: [] diff --git a/docs/reference/security/authorization/built-in-roles.asciidoc b/docs/reference/security/authorization/built-in-roles.asciidoc index d730587e7db17..13812b915dc5e 100644 --- a/docs/reference/security/authorization/built-in-roles.asciidoc +++ b/docs/reference/security/authorization/built-in-roles.asciidoc @@ -161,12 +161,11 @@ Grants the minimum privileges required to write data into the monitoring indices Grants the minimum privileges required to collect monitoring data for the {stack}. [[built-in-roles-reporting-user]] `reporting_user`:: -Grants the specific privileges required for users of {reporting} other than those -required to use {kib}. This role grants access to the reporting indices; each -user has access to only their own reports. -Reporting users should also be assigned additional roles that grant -{kibana-ref}/xpack-security-authorization.html[access to {kib}] as well as read -access to the <> that will be used to generate reports. +Grants the necessary privileges required to use {reporting} features in {kib}, +including generating and downloading reports. This role implicitly grants access +to all Kibana reporting features, with each user having access only to their own reports. +Note that reporting users should also be assigned additional roles that grant read access +to the <> that will be used to generate reports. [[built-in-roles-rollup-admin]] `rollup_admin`:: Grants `manage_rollup` cluster privileges, which enable you to manage and execute all rollup actions. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java index fc14ec6811014..bdaf75203ee5d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java @@ -301,25 +301,40 @@ private static Map initializeReservedRoles() { "Grants access to manage all index templates and all ingest pipeline configurations." ) ), - // reporting_user doesn't have any privileges in Elasticsearch, and Kibana authorizes privileges based on this role entry( "reporting_user", new RoleDescriptor( "reporting_user", null, null, + new RoleDescriptor.ApplicationResourcePrivileges[] { + RoleDescriptor.ApplicationResourcePrivileges.builder() + .application("kibana-.kibana") + .resources("*") + .privileges( + "feature_discover.minimal_read", + "feature_discover.generate_report", + "feature_dashboard.minimal_read", + "feature_dashboard.generate_report", + "feature_dashboard.download_csv_report", + "feature_canvas.minimal_read", + "feature_canvas.generate_report", + "feature_visualize.minimal_read", + "feature_visualize.generate_report" + ) + .build() }, null, null, - null, - MetadataUtils.getDeprecatedReservedMetadata("Please use Kibana feature privileges instead"), + MetadataUtils.DEFAULT_RESERVED_METADATA, null, null, null, null, - "Grants the specific privileges required for users of X-Pack reporting other than those required to use Kibana. " - + "This role grants access to the reporting indices; each user has access to only their own reports. " - + "Reporting users should also be assigned additional roles that grant access to Kibana as well as read access " - + "to the indices that will be used to generate reports." + "Grants the necessary privileges required to use reporting features in Kibana, " + + "including generating and downloading reports. " + + "This role implicitly grants access to all Kibana reporting features, " + + "with each user having access only to their own reports. Note that reporting users should also be assigned " + + "additional roles that grant read access to the indices that will be used to generate reports." ) ), entry(KibanaSystemUser.ROLE_NAME, kibanaSystemRoleDescriptor(KibanaSystemUser.ROLE_NAME)), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index b69b0ece89960..1b9a65d12d8d9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -2646,12 +2646,57 @@ public void testReportingUserRole() { RoleDescriptor roleDescriptor = ReservedRolesStore.roleDescriptor("reporting_user"); assertNotNull(roleDescriptor); assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true)); - assertThat(roleDescriptor.getMetadata(), hasEntry("_deprecated", true)); + + final String applicationName = "kibana-.kibana"; + + final Set applicationPrivilegeNames = Set.of( + "feature_discover.minimal_read", + "feature_discover.generate_report", + "feature_dashboard.minimal_read", + "feature_dashboard.generate_report", + "feature_dashboard.download_csv_report", + "feature_canvas.minimal_read", + "feature_canvas.generate_report", + "feature_visualize.minimal_read", + "feature_visualize.generate_report" + ); + + final Set allowedApplicationActionPatterns = Set.of( + "login:", + "app:discover", + "app:canvas", + "app:kibana", + "ui:catalogue/canvas", + "ui:navLinks/canvas", + "ui:catalogue/discover", + "ui:navLinks/discover", + "ui:navLinks/kibana", + "saved_object:index-pattern/*", + "saved_object:search/*", + "saved_object:query/*", + "saved_object:config/*", + "saved_object:config/get", + "saved_object:config/find", + "saved_object:config-global/*", + "saved_object:telemetry/*", + "saved_object:canvas-workpad/*", + "saved_object:canvas-element/*", + "saved_object:url/*", + "ui:discover/show" + ); + + final List applicationPrivilegeDescriptors = new ArrayList<>(); + for (String appPrivilegeName : applicationPrivilegeNames) { + applicationPrivilegeDescriptors.add( + new ApplicationPrivilegeDescriptor(applicationName, appPrivilegeName, allowedApplicationActionPatterns, Map.of()) + ); + } Role reportingUserRole = Role.buildFromRoleDescriptor( roleDescriptor, new FieldPermissionsCache(Settings.EMPTY), - RESTRICTED_INDICES + RESTRICTED_INDICES, + applicationPrivilegeDescriptors ); assertThat(reportingUserRole.cluster().check(TransportClusterHealthAction.NAME, request, authentication), is(false)); assertThat(reportingUserRole.cluster().check(ClusterStateAction.NAME, request, authentication), is(false)); @@ -2723,6 +2768,33 @@ public void testReportingUserRole() { assertNoAccessAllowed(reportingUserRole, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES); assertNoAccessAllowed(reportingUserRole, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2)); + + applicationPrivilegeNames.forEach(appPrivilege -> { + assertThat( + reportingUserRole.application() + .grants( + ApplicationPrivilegeTests.createPrivilege( + applicationName, + appPrivilege, + allowedApplicationActionPatterns.toArray(new String[0]) + ), + "*" + ), + is(true) + ); + }); + assertThat( + reportingUserRole.application() + .grants( + ApplicationPrivilegeTests.createPrivilege( + "kibana-.*", + "feature_random.minimal_read", + allowedApplicationActionPatterns.toArray(new String[0]) + ), + "*" + ), + is(false) + ); } public void testSuperuserRole() { From c9d8a3a2b6bff2130634ec1e572cdf2ccea203e4 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 13 Dec 2024 13:47:51 -0500 Subject: [PATCH 17/24] Add replica handling to the ILM MountSnapshotStep (#118687) --- .../xpack/core/ilm/AsyncWaitStep.java | 3 + .../xpack/core/ilm/DeleteAction.java | 6 +- .../xpack/core/ilm/DownsampleAction.java | 3 +- .../xpack/core/ilm/ForceMergeAction.java | 3 +- .../xpack/core/ilm/MountSnapshotStep.java | 29 ++-- .../xpack/core/ilm/ReadOnlyAction.java | 3 +- .../core/ilm/SearchableSnapshotAction.java | 7 +- .../xpack/core/ilm/ShrinkAction.java | 3 +- .../core/ilm/TimeseriesLifecycleType.java | 13 +- .../WaitUntilTimeSeriesEndTimePassesStep.java | 5 +- .../core/ilm/MountSnapshotStepTests.java | 161 ++++++++++++------ .../ilm/SearchableSnapshotActionTests.java | 95 ++++------- ...UntilTimeSeriesEndTimePassesStepTests.java | 9 +- 13 files changed, 182 insertions(+), 158 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncWaitStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncWaitStep.java index 6a72af5bce5e9..fc5e8d473b763 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncWaitStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncWaitStep.java @@ -8,6 +8,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.xcontent.ToXContentObject; @@ -20,6 +21,7 @@ */ public abstract class AsyncWaitStep extends Step { + @Nullable private final Client client; public AsyncWaitStep(StepKey key, StepKey nextStepKey, Client client) { @@ -27,6 +29,7 @@ public AsyncWaitStep(StepKey key, StepKey nextStepKey, Client client) { this.client = client; } + @Nullable protected Client getClient() { return client; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteAction.java index 8712cefac5d31..6c2ab86995a6d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteAction.java @@ -93,8 +93,7 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, cleanSnapshotKey, - Instant::now, - client + Instant::now ); CleanupSnapshotStep cleanupSnapshotStep = new CleanupSnapshotStep(cleanSnapshotKey, deleteStepKey, client); DeleteStep deleteStep = new DeleteStep(deleteStepKey, nextStepKey, client); @@ -108,8 +107,7 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, deleteStepKey, - Instant::now, - client + Instant::now ); DeleteStep deleteStep = new DeleteStep(deleteStepKey, nextStepKey, client); return List.of(waitForNoFollowersStep, waitUntilTimeSeriesEndTimeStep, deleteStep); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DownsampleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DownsampleAction.java index 697f948e47832..6ce9e05e4a464 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DownsampleAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DownsampleAction.java @@ -200,8 +200,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, readOnlyKey, - Instant::now, - client + Instant::now ); // Mark source index as read-only ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, generateDownsampleIndexNameKey, client); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java index f8f4ce2bb0354..ac398bccb64e4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java @@ -162,8 +162,7 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, codecChange ? closeKey : forceMergeKey, - Instant::now, - client + Instant::now ); // Indices already in this step key when upgrading need to know how to move forward but stop making the index diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java index 7d045f2950e1b..82d41b91fea4f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java @@ -41,6 +41,7 @@ public class MountSnapshotStep extends AsyncRetryDuringSnapshotActionStep { private final MountSearchableSnapshotRequest.Storage storageType; @Nullable private final Integer totalShardsPerNode; + private final int replicas; public MountSnapshotStep( StepKey key, @@ -48,7 +49,8 @@ public MountSnapshotStep( Client client, String restoredIndexPrefix, MountSearchableSnapshotRequest.Storage storageType, - @Nullable Integer totalShardsPerNode + @Nullable Integer totalShardsPerNode, + int replicas ) { super(key, nextStepKey, client); this.restoredIndexPrefix = restoredIndexPrefix; @@ -57,16 +59,10 @@ public MountSnapshotStep( throw new IllegalArgumentException("[" + SearchableSnapshotAction.TOTAL_SHARDS_PER_NODE.getPreferredName() + "] must be >= 1"); } this.totalShardsPerNode = totalShardsPerNode; - } - public MountSnapshotStep( - StepKey key, - StepKey nextStepKey, - Client client, - String restoredIndexPrefix, - MountSearchableSnapshotRequest.Storage storageType - ) { - this(key, nextStepKey, client, restoredIndexPrefix, storageType, null); + // this isn't directly settable by the user, so validation by assertion is sufficient + assert replicas >= 0 : "number of replicas must be gte zero, but was [" + replicas + "]"; + this.replicas = replicas; } @Override @@ -87,6 +83,10 @@ public Integer getTotalShardsPerNode() { return totalShardsPerNode; } + public int getReplicas() { + return replicas; + } + @Override void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentClusterState, ActionListener listener) { String indexName = indexMetadata.getIndex().getName(); @@ -162,11 +162,13 @@ void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentCl } final Settings.Builder settingsBuilder = Settings.builder(); - overrideTierPreference(this.getKey().phase()).ifPresent(override -> settingsBuilder.put(DataTier.TIER_PREFERENCE, override)); if (totalShardsPerNode != null) { settingsBuilder.put(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), totalShardsPerNode); } + if (replicas > 0) { + settingsBuilder.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicas); + } final MountSearchableSnapshotRequest mountSearchableSnapshotRequest = new MountSearchableSnapshotRequest( TimeValue.MAX_VALUE, @@ -245,7 +247,7 @@ String[] ignoredIndexSettings() { @Override public int hashCode() { - return Objects.hash(super.hashCode(), restoredIndexPrefix, storageType, totalShardsPerNode); + return Objects.hash(super.hashCode(), restoredIndexPrefix, storageType, totalShardsPerNode, replicas); } @Override @@ -260,6 +262,7 @@ public boolean equals(Object obj) { return super.equals(obj) && Objects.equals(restoredIndexPrefix, other.restoredIndexPrefix) && Objects.equals(storageType, other.storageType) - && Objects.equals(totalShardsPerNode, other.totalShardsPerNode); + && Objects.equals(totalShardsPerNode, other.totalShardsPerNode) + && Objects.equals(replicas, other.replicas); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReadOnlyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReadOnlyAction.java index 2b03dc77eb5b6..b36156842acf5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReadOnlyAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReadOnlyAction.java @@ -67,8 +67,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, readOnlyKey, - Instant::now, - client + Instant::now ); ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, nextStepKey, client); return List.of(checkNotWriteIndexStep, waitUntilTimeSeriesEndTimeStep, readOnlyStep); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index f585575534b76..b746ee8ea7c07 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -113,6 +113,7 @@ public String getSnapshotRepository() { return snapshotRepository; } + @Nullable public Integer getTotalShardsPerNode() { return totalShardsPerNode; } @@ -230,8 +231,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey, XPac WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, skipGeneratingSnapshotKey, - Instant::now, - client + Instant::now ); // When generating a snapshot, we either jump to the force merge step, or we skip the @@ -318,7 +318,8 @@ public List toSteps(Client client, String phase, StepKey nextStepKey, XPac client, getRestoredIndexPrefix(mountSnapshotKey), storageType, - totalShardsPerNode + totalShardsPerNode, + 0 ); WaitForIndexColorStep waitForGreenIndexHealthStep = new WaitForIndexColorStep( waitForGreenRestoredIndexKey, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java index 70ec5da1d8a2a..f7478518613e2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java @@ -231,8 +231,7 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, readOnlyKey, - Instant::now, - client + Instant::now ); ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, checkTargetShardsCountKey, client); CheckTargetShardsCountStep checkTargetShardsCountStep = new CheckTargetShardsCountStep( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index 0fd280f440f39..10a4c7086a0cc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -27,7 +27,6 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Represents the lifecycle of an index from creation to deletion. A @@ -49,7 +48,7 @@ public class TimeseriesLifecycleType implements LifecycleType { static final String DELETE_PHASE = "delete"; public static final List ORDERED_VALID_PHASES = List.of(HOT_PHASE, WARM_PHASE, COLD_PHASE, FROZEN_PHASE, DELETE_PHASE); - public static final List ORDERED_VALID_HOT_ACTIONS = Stream.of( + public static final List ORDERED_VALID_HOT_ACTIONS = List.of( SetPriorityAction.NAME, UnfollowAction.NAME, RolloverAction.NAME, @@ -58,8 +57,8 @@ public class TimeseriesLifecycleType implements LifecycleType { ShrinkAction.NAME, ForceMergeAction.NAME, SearchableSnapshotAction.NAME - ).filter(Objects::nonNull).toList(); - public static final List ORDERED_VALID_WARM_ACTIONS = Stream.of( + ); + public static final List ORDERED_VALID_WARM_ACTIONS = List.of( SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, @@ -68,8 +67,8 @@ public class TimeseriesLifecycleType implements LifecycleType { MigrateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME - ).filter(Objects::nonNull).toList(); - public static final List ORDERED_VALID_COLD_ACTIONS = Stream.of( + ); + public static final List ORDERED_VALID_COLD_ACTIONS = List.of( SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, @@ -78,7 +77,7 @@ public class TimeseriesLifecycleType implements LifecycleType { AllocateAction.NAME, MigrateAction.NAME, FreezeAction.NAME - ).filter(Objects::nonNull).toList(); + ); public static final List ORDERED_VALID_FROZEN_ACTIONS = List.of(UnfollowAction.NAME, SearchableSnapshotAction.NAME); public static final List ORDERED_VALID_DELETE_ACTIONS = List.of(WaitForSnapshotAction.NAME, DeleteAction.NAME); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStep.java index 50a7d48672c8e..3e190a26dd961 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStep.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.ilm; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; @@ -33,8 +32,8 @@ public class WaitUntilTimeSeriesEndTimePassesStep extends AsyncWaitStep { public static final String NAME = "check-ts-end-time-passed"; private final Supplier nowSupplier; - public WaitUntilTimeSeriesEndTimePassesStep(StepKey key, StepKey nextStepKey, Supplier nowSupplier, Client client) { - super(key, nextStepKey, client); + public WaitUntilTimeSeriesEndTimePassesStep(StepKey key, StepKey nextStepKey, Supplier nowSupplier) { + super(key, nextStepKey, null); this.nowSupplier = nowSupplier; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java index 8ca7a00ab0948..7ccdb1a27326a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.snapshots.RestoreInfo; import org.elasticsearch.test.client.NoOpClient; @@ -42,7 +43,7 @@ public MountSnapshotStep createRandomInstance() { String restoredIndexPrefix = randomAlphaOfLength(10); MountSearchableSnapshotRequest.Storage storage = randomStorageType(); Integer totalShardsPerNode = randomTotalShardsPerNode(true); - return new MountSnapshotStep(stepKey, nextStepKey, client, restoredIndexPrefix, storage, totalShardsPerNode); + return new MountSnapshotStep(stepKey, nextStepKey, client, restoredIndexPrefix, storage, totalShardsPerNode, 0); } public static MountSearchableSnapshotRequest.Storage randomStorageType() { @@ -61,7 +62,8 @@ protected MountSnapshotStep copyInstance(MountSnapshotStep instance) { instance.getClient(), instance.getRestoredIndexPrefix(), instance.getStorage(), - instance.getTotalShardsPerNode() + instance.getTotalShardsPerNode(), + instance.getReplicas() ); } @@ -72,7 +74,8 @@ public MountSnapshotStep mutateInstance(MountSnapshotStep instance) { String restoredIndexPrefix = instance.getRestoredIndexPrefix(); MountSearchableSnapshotRequest.Storage storage = instance.getStorage(); Integer totalShardsPerNode = instance.getTotalShardsPerNode(); - switch (between(0, 4)) { + int replicas = instance.getReplicas(); + switch (between(0, 5)) { case 0: key = new StepKey(key.phase(), key.action(), key.name() + randomAlphaOfLength(5)); break; @@ -94,10 +97,13 @@ public MountSnapshotStep mutateInstance(MountSnapshotStep instance) { case 4: totalShardsPerNode = totalShardsPerNode == null ? 1 : totalShardsPerNode + randomIntBetween(1, 100); break; + case 5: + replicas = replicas == 0 ? 1 : 0; // swap between 0 and 1 + break; default: throw new AssertionError("Illegal randomisation branch"); } - return new MountSnapshotStep(key, nextKey, instance.getClient(), restoredIndexPrefix, storage, totalShardsPerNode); + return new MountSnapshotStep(key, nextKey, instance.getClient(), restoredIndexPrefix, storage, totalShardsPerNode, replicas); } public void testCreateWithInvalidTotalShardsPerNode() throws Exception { @@ -111,7 +117,8 @@ public void testCreateWithInvalidTotalShardsPerNode() throws Exception { client, RESTORED_INDEX_PREFIX, randomStorageType(), - invalidTotalShardsPerNode + invalidTotalShardsPerNode, + 0 ) ); assertEquals("[total_shards_per_node] must be >= 1", exception.getMessage()); @@ -195,14 +202,18 @@ public void testPerformAction() throws Exception { indexName, RESTORED_INDEX_PREFIX, indexName, - new String[] { LifecycleSettings.LIFECYCLE_NAME } + new String[] { LifecycleSettings.LIFECYCLE_NAME }, + null, + 0 ); MountSnapshotStep step = new MountSnapshotStep( randomStepKey(), randomStepKey(), client, RESTORED_INDEX_PREFIX, - randomStorageType() + randomStorageType(), + null, + 0 ); performActionAndWait(step, indexMetadata, clusterState, null); } @@ -237,7 +248,9 @@ public void testResponseStatusHandling() throws Exception { randomStepKey(), clientPropagatingOKResponse, RESTORED_INDEX_PREFIX, - randomStorageType() + randomStorageType(), + null, + 0 ); performActionAndWait(step, indexMetadata, clusterState, null); } @@ -252,7 +265,9 @@ public void testResponseStatusHandling() throws Exception { randomStepKey(), clientPropagatingACCEPTEDResponse, RESTORED_INDEX_PREFIX, - randomStorageType() + randomStorageType(), + null, + 0 ); performActionAndWait(step, indexMetadata, clusterState, null); } @@ -289,47 +304,49 @@ public void testMountWithPartialAndRestoredPrefix() throws Exception { ); } - public void doTestMountWithoutSnapshotIndexNameInState(String prefix) throws Exception { - { - String indexNameSnippet = randomAlphaOfLength(10); - String indexName = prefix + indexNameSnippet; - String policyName = "test-ilm-policy"; - Map ilmCustom = new HashMap<>(); - String snapshotName = indexName + "-" + policyName; - ilmCustom.put("snapshot_name", snapshotName); - String repository = "repository"; - ilmCustom.put("snapshot_repository", repository); + private void doTestMountWithoutSnapshotIndexNameInState(String prefix) throws Exception { + String indexNameSnippet = randomAlphaOfLength(10); + String indexName = prefix + indexNameSnippet; + String policyName = "test-ilm-policy"; + Map ilmCustom = new HashMap<>(); + String snapshotName = indexName + "-" + policyName; + ilmCustom.put("snapshot_name", snapshotName); + String repository = "repository"; + ilmCustom.put("snapshot_repository", repository); - IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) - .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName)) - .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, ilmCustom) - .numberOfShards(randomIntBetween(1, 5)) - .numberOfReplicas(randomIntBetween(0, 5)); - IndexMetadata indexMetadata = indexMetadataBuilder.build(); + IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) + .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName)) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, ilmCustom) + .numberOfShards(randomIntBetween(1, 5)) + .numberOfReplicas(randomIntBetween(0, 5)); + IndexMetadata indexMetadata = indexMetadataBuilder.build(); - ClusterState clusterState = ClusterState.builder(emptyClusterState()) - .metadata(Metadata.builder().put(indexMetadata, true).build()) - .build(); + ClusterState clusterState = ClusterState.builder(emptyClusterState()) + .metadata(Metadata.builder().put(indexMetadata, true).build()) + .build(); - try (var threadPool = createThreadPool()) { - final var client = getRestoreSnapshotRequestAssertingClient( - threadPool, - repository, - snapshotName, - indexName, - RESTORED_INDEX_PREFIX, - indexNameSnippet, - new String[] { LifecycleSettings.LIFECYCLE_NAME } - ); - MountSnapshotStep step = new MountSnapshotStep( - randomStepKey(), - randomStepKey(), - client, - RESTORED_INDEX_PREFIX, - randomStorageType() - ); - performActionAndWait(step, indexMetadata, clusterState, null); - } + try (var threadPool = createThreadPool()) { + final var client = getRestoreSnapshotRequestAssertingClient( + threadPool, + repository, + snapshotName, + indexName, + RESTORED_INDEX_PREFIX, + indexNameSnippet, + new String[] { LifecycleSettings.LIFECYCLE_NAME }, + null, + 0 + ); + MountSnapshotStep step = new MountSnapshotStep( + randomStepKey(), + randomStepKey(), + client, + RESTORED_INDEX_PREFIX, + randomStorageType(), + null, + 0 + ); + performActionAndWait(step, indexMetadata, clusterState, null); } } @@ -361,7 +378,11 @@ public void testIgnoreTotalShardsPerNodeInFrozenPhase() throws Exception { indexName, RESTORED_INDEX_PREFIX, indexName, - new String[] { LifecycleSettings.LIFECYCLE_NAME, ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey() } + new String[] { + LifecycleSettings.LIFECYCLE_NAME, + ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey() }, + null, + 0 ); MountSnapshotStep step = new MountSnapshotStep( new StepKey(TimeseriesLifecycleType.FROZEN_PHASE, randomAlphaOfLength(10), randomAlphaOfLength(10)), @@ -369,13 +390,14 @@ public void testIgnoreTotalShardsPerNodeInFrozenPhase() throws Exception { client, RESTORED_INDEX_PREFIX, randomStorageType(), - null + null, + 0 ); performActionAndWait(step, indexMetadata, clusterState, null); } } - public void testDoNotIgnoreTotalShardsPerNodeIfSet() throws Exception { + public void testDoNotIgnoreTotalShardsPerNodeAndReplicasIfSet() throws Exception { String indexName = randomAlphaOfLength(10); String policyName = "test-ilm-policy"; Map ilmCustom = new HashMap<>(); @@ -395,6 +417,9 @@ public void testDoNotIgnoreTotalShardsPerNodeIfSet() throws Exception { .metadata(Metadata.builder().put(indexMetadata, true).build()) .build(); + final Integer totalShardsPerNode = randomTotalShardsPerNode(false); + final int replicas = randomIntBetween(1, 5); + try (var threadPool = createThreadPool()) { final var client = getRestoreSnapshotRequestAssertingClient( threadPool, @@ -403,7 +428,9 @@ public void testDoNotIgnoreTotalShardsPerNodeIfSet() throws Exception { indexName, RESTORED_INDEX_PREFIX, indexName, - new String[] { LifecycleSettings.LIFECYCLE_NAME } + new String[] { LifecycleSettings.LIFECYCLE_NAME }, + totalShardsPerNode, + replicas ); MountSnapshotStep step = new MountSnapshotStep( new StepKey(TimeseriesLifecycleType.FROZEN_PHASE, randomAlphaOfLength(10), randomAlphaOfLength(10)), @@ -411,7 +438,8 @@ public void testDoNotIgnoreTotalShardsPerNodeIfSet() throws Exception { client, RESTORED_INDEX_PREFIX, randomStorageType(), - randomTotalShardsPerNode(false) + totalShardsPerNode, + replicas ); performActionAndWait(step, indexMetadata, clusterState, null); } @@ -439,7 +467,9 @@ private NoOpClient getRestoreSnapshotRequestAssertingClient( String indexName, String restoredIndexPrefix, String expectedSnapshotIndexName, - String[] expectedIgnoredIndexSettings + String[] expectedIgnoredIndexSettings, + @Nullable Integer totalShardsPerNode, + int replicas ) { return new NoOpClient(threadPool) { @Override @@ -462,6 +492,31 @@ protected void assertThat(mountSearchableSnapshotRequest.mountedIndexName(), is(restoredIndexPrefix + indexName)); assertThat(mountSearchableSnapshotRequest.snapshotIndexName(), is(expectedSnapshotIndexName)); + if (totalShardsPerNode != null) { + Integer totalShardsPerNodeSettingValue = ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.get( + mountSearchableSnapshotRequest.indexSettings() + ); + assertThat(totalShardsPerNodeSettingValue, is(totalShardsPerNode)); + } else { + assertThat( + mountSearchableSnapshotRequest.indexSettings() + .hasValue(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey()), + is(false) + ); + } + + if (replicas > 0) { + Integer numberOfReplicasSettingValue = IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get( + mountSearchableSnapshotRequest.indexSettings() + ); + assertThat(numberOfReplicasSettingValue, is(replicas)); + } else { + assertThat( + mountSearchableSnapshotRequest.indexSettings().hasValue(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey()), + is(false) + ); + } + // invoke the awaiting listener with a very generic 'response', just to fulfill the contract listener.onResponse((Response) new RestoreSnapshotResponse((RestoreInfo) null)); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java index ca219fdde3d57..5304b7885f96c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java @@ -14,6 +14,8 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; import static org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction.NAME; import static org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction.TOTAL_SHARDS_PER_NODE; @@ -29,40 +31,23 @@ public void testToSteps() { StepKey nextStepKey = new StepKey(phase, randomAlphaOfLengthBetween(1, 5), randomAlphaOfLengthBetween(1, 5)); List steps = action.toSteps(null, phase, nextStepKey, null); - assertThat(steps.size(), is(action.isForceMergeIndex() ? 19 : 17)); - - List expectedSteps = action.isForceMergeIndex() - ? expectedStepKeysWithForceMerge(phase) - : expectedStepKeysNoForceMerge(phase); - - assertThat(steps.get(0).getKey(), is(expectedSteps.get(0))); - assertThat(steps.get(1).getKey(), is(expectedSteps.get(1))); - assertThat(steps.get(2).getKey(), is(expectedSteps.get(2))); - assertThat(steps.get(3).getKey(), is(expectedSteps.get(3))); - assertThat(steps.get(4).getKey(), is(expectedSteps.get(4))); - assertThat(steps.get(5).getKey(), is(expectedSteps.get(5))); - assertThat(steps.get(6).getKey(), is(expectedSteps.get(6))); - assertThat(steps.get(7).getKey(), is(expectedSteps.get(7))); - assertThat(steps.get(8).getKey(), is(expectedSteps.get(8))); - assertThat(steps.get(9).getKey(), is(expectedSteps.get(9))); - assertThat(steps.get(10).getKey(), is(expectedSteps.get(10))); - assertThat(steps.get(11).getKey(), is(expectedSteps.get(11))); - assertThat(steps.get(12).getKey(), is(expectedSteps.get(12))); - assertThat(steps.get(13).getKey(), is(expectedSteps.get(13))); - assertThat(steps.get(14).getKey(), is(expectedSteps.get(14))); - assertThat(steps.get(15).getKey(), is(expectedSteps.get(15))); - - if (action.isForceMergeIndex()) { - assertThat(steps.get(16).getKey(), is(expectedSteps.get(16))); - assertThat(steps.get(17).getKey(), is(expectedSteps.get(17))); - CreateSnapshotStep createSnapshotStep = (CreateSnapshotStep) steps.get(9); - assertThat(createSnapshotStep.getNextKeyOnIncomplete(), is(expectedSteps.get(8))); - validateWaitForDataTierStep(phase, steps, 10, 11); - } else { - CreateSnapshotStep createSnapshotStep = (CreateSnapshotStep) steps.get(7); - assertThat(createSnapshotStep.getNextKeyOnIncomplete(), is(expectedSteps.get(6))); - validateWaitForDataTierStep(phase, steps, 8, 9); + + List expectedSteps = expectedStepKeys(phase, action.isForceMergeIndex()); + assertThat(steps.size(), is(expectedSteps.size())); + for (int i = 0; i < expectedSteps.size(); i++) { + assertThat("steps match expectation at index " + i, steps.get(i).getKey(), is(expectedSteps.get(i))); + } + + int index = -1; + for (int i = 0; i < expectedSteps.size(); i++) { + if (expectedSteps.get(i).name().equals(CreateSnapshotStep.NAME)) { + index = i; + break; + } } + CreateSnapshotStep createSnapshotStep = (CreateSnapshotStep) steps.get(index); + assertThat(createSnapshotStep.getNextKeyOnIncomplete(), is(expectedSteps.get(index - 1))); + validateWaitForDataTierStep(phase, steps, index + 1, index + 2); } private void validateWaitForDataTierStep(String phase, List steps, int waitForDataTierStepIndex, int mountStepIndex) { @@ -108,15 +93,15 @@ public void testCreateWithInvalidTotalShardsPerNode() { assertEquals("[" + TOTAL_SHARDS_PER_NODE.getPreferredName() + "] must be >= 1", exception.getMessage()); } - private List expectedStepKeysWithForceMerge(String phase) { - return List.of( + private List expectedStepKeys(String phase, boolean forceMergeIndex) { + return Stream.of( new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_ACTION_STEP), new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME), new StepKey(phase, NAME, WaitForNoFollowersStep.NAME), new StepKey(phase, NAME, WaitUntilTimeSeriesEndTimePassesStep.NAME), new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_GENERATE_AND_CLEAN), - new StepKey(phase, NAME, ForceMergeStep.NAME), - new StepKey(phase, NAME, SegmentCountStep.NAME), + forceMergeIndex ? new StepKey(phase, NAME, ForceMergeStep.NAME) : null, + forceMergeIndex ? new StepKey(phase, NAME, SegmentCountStep.NAME) : null, new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME), new StepKey(phase, NAME, CleanupSnapshotStep.NAME), new StepKey(phase, NAME, CreateSnapshotStep.NAME), @@ -129,29 +114,7 @@ private List expectedStepKeysWithForceMerge(String phase) { new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME), new StepKey(phase, NAME, DeleteStep.NAME), new StepKey(phase, NAME, SwapAliasesAndDeleteSourceIndexStep.NAME) - ); - } - - private List expectedStepKeysNoForceMerge(String phase) { - return List.of( - new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_ACTION_STEP), - new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME), - new StepKey(phase, NAME, WaitForNoFollowersStep.NAME), - new StepKey(phase, NAME, WaitUntilTimeSeriesEndTimePassesStep.NAME), - new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_GENERATE_AND_CLEAN), - new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME), - new StepKey(phase, NAME, CleanupSnapshotStep.NAME), - new StepKey(phase, NAME, CreateSnapshotStep.NAME), - new StepKey(phase, NAME, WaitForDataTierStep.NAME), - new StepKey(phase, NAME, MountSnapshotStep.NAME), - new StepKey(phase, NAME, WaitForIndexColorStep.NAME), - new StepKey(phase, NAME, CopyExecutionStateStep.NAME), - new StepKey(phase, NAME, CopySettingsStep.NAME), - new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_DATASTREAM_CHECK_KEY), - new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME), - new StepKey(phase, NAME, DeleteStep.NAME), - new StepKey(phase, NAME, SwapAliasesAndDeleteSourceIndexStep.NAME) - ); + ).filter(Objects::nonNull).toList(); } @Override @@ -172,8 +135,16 @@ protected Writeable.Reader instanceReader() { @Override protected SearchableSnapshotAction mutateInstance(SearchableSnapshotAction instance) { return switch (randomIntBetween(0, 2)) { - case 0 -> new SearchableSnapshotAction(randomAlphaOfLengthBetween(5, 10), instance.isForceMergeIndex()); - case 1 -> new SearchableSnapshotAction(instance.getSnapshotRepository(), instance.isForceMergeIndex() == false); + case 0 -> new SearchableSnapshotAction( + randomAlphaOfLengthBetween(5, 10), + instance.isForceMergeIndex(), + instance.getTotalShardsPerNode() + ); + case 1 -> new SearchableSnapshotAction( + instance.getSnapshotRepository(), + instance.isForceMergeIndex() == false, + instance.getTotalShardsPerNode() + ); case 2 -> new SearchableSnapshotAction( instance.getSnapshotRepository(), instance.isForceMergeIndex(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java index 8ca6c0016a791..15bbbe7446429 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java @@ -30,7 +30,7 @@ public class WaitUntilTimeSeriesEndTimePassesStepTests extends AbstractStepTestC protected WaitUntilTimeSeriesEndTimePassesStep createRandomInstance() { Step.StepKey stepKey = randomStepKey(); Step.StepKey nextStepKey = randomStepKey(); - return new WaitUntilTimeSeriesEndTimePassesStep(stepKey, nextStepKey, Instant::now, client); + return new WaitUntilTimeSeriesEndTimePassesStep(stepKey, nextStepKey, Instant::now); } @Override @@ -42,12 +42,12 @@ protected WaitUntilTimeSeriesEndTimePassesStep mutateInstance(WaitUntilTimeSerie case 0 -> key = new Step.StepKey(key.phase(), key.action(), key.name() + randomAlphaOfLength(5)); case 1 -> nextKey = new Step.StepKey(nextKey.phase(), nextKey.action(), nextKey.name() + randomAlphaOfLength(5)); } - return new WaitUntilTimeSeriesEndTimePassesStep(key, nextKey, Instant::now, client); + return new WaitUntilTimeSeriesEndTimePassesStep(key, nextKey, Instant::now); } @Override protected WaitUntilTimeSeriesEndTimePassesStep copyInstance(WaitUntilTimeSeriesEndTimePassesStep instance) { - return new WaitUntilTimeSeriesEndTimePassesStep(instance.getKey(), instance.getNextStepKey(), Instant::now, client); + return new WaitUntilTimeSeriesEndTimePassesStep(instance.getKey(), instance.getNextStepKey(), Instant::now); } public void testEvaluateCondition() { @@ -68,8 +68,7 @@ public void testEvaluateCondition() { WaitUntilTimeSeriesEndTimePassesStep step = new WaitUntilTimeSeriesEndTimePassesStep( randomStepKey(), randomStepKey(), - () -> currentTime, - client + () -> currentTime ); { // end_time has lapsed already so condition must be met From dcadb08b57bc65ac5f989ad60e18eac9fcce42c4 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 13 Dec 2024 13:37:07 -0600 Subject: [PATCH 18/24] Unmuting XPackRestTest migrate reindex tests (#118407) These were fixed by https://github.com/elastic/elasticsearch/pull/118382 Closes #118401 Closes #118272 Closes #118273 Closes #118274 --- muted-tests.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index e0d011c1bf239..1255739e818be 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -294,18 +294,6 @@ tests: - class: org.elasticsearch.docker.test.DockerYmlTestSuiteIT method: test {p0=/11_nodes/Test cat nodes output} issue: https://github.com/elastic/elasticsearch/issues/118397 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=migrate/20_reindex_status/Test get reindex status with nonexistent task id} - issue: https://github.com/elastic/elasticsearch/issues/118401 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=migrate/10_reindex/Test Reindex With Nonexistent Data Stream} - issue: https://github.com/elastic/elasticsearch/issues/118274 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=migrate/10_reindex/Test Reindex With Bad Data Stream Name} - issue: https://github.com/elastic/elasticsearch/issues/118272 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=migrate/10_reindex/Test Reindex With Unsupported Mode} - issue: https://github.com/elastic/elasticsearch/issues/118273 - class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT method: testEveryActionIsEitherOperatorOnlyOrNonOperator issue: https://github.com/elastic/elasticsearch/issues/118220 From 908cf9ab768c4854e51926f28b62fd2108c69aa7 Mon Sep 17 00:00:00 2001 From: Quentin Deschamps Date: Fri, 13 Dec 2024 20:54:48 +0100 Subject: [PATCH 19/24] Fix moving function linear weighted avg (#118516) Fix moving function linear weighted avg --- docs/changelog/118516.yaml | 6 ++++ modules/aggregations/build.gradle | 1 + .../test/aggregations/moving_fn.yml | 31 +++++++++++++------ .../action/search/SearchCapabilities.java | 3 ++ .../pipeline/MovingFunctions.java | 4 +-- .../MovFnWhitelistedFunctionTests.java | 2 +- 6 files changed, 34 insertions(+), 13 deletions(-) create mode 100644 docs/changelog/118516.yaml diff --git a/docs/changelog/118516.yaml b/docs/changelog/118516.yaml new file mode 100644 index 0000000000000..8a618a6d6cfd7 --- /dev/null +++ b/docs/changelog/118516.yaml @@ -0,0 +1,6 @@ +pr: 118435 +summary: Fix moving function linear weighted avg +area: Aggregations +type: bug +issues: + - 113751 diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle index 94fdddf6d711a..3b5fb6ddecde9 100644 --- a/modules/aggregations/build.gradle +++ b/modules/aggregations/build.gradle @@ -48,4 +48,5 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("aggregations/date_agg_per_day_of_week/Date aggregartion per day of week", "week-date behaviour has changed") task.skipTest("aggregations/time_series/Configure with no synthetic source", "temporary until backport") task.skipTest("aggregations/percentiles_hdr_metric/Negative values test", "returned exception has changed") + task.skipTest("aggregations/moving_fn/linearWeightedAvg", "math was wrong in previous versions") }) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml index cd6feb601b1df..3abad87d57907 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml @@ -255,6 +255,17 @@ linearWeightedAvg: - skip: features: close_to + - requires: + test_runner_features: [capabilities] + + - requires: + capabilities: + - method: POST + path: /_search + parameters: [method, path, parameters, capabilities] + capabilities: [moving_fn_right_math] + reason: "math not fixed yet" + - do: search: index: no_gaps @@ -275,11 +286,11 @@ linearWeightedAvg: - match: { hits.total.value: 6 } - length: { aggregations.@timestamp.buckets: 6 } - is_false: aggregations.@timestamp.buckets.0.d.value - - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 0.500, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.250, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.000, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.250, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 3.500, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.667, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.333, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 3.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.667, error: 0.0005 } } - do: search: @@ -301,11 +312,11 @@ linearWeightedAvg: - match: { hits.total.value: 6 } - length: { aggregations.@timestamp.buckets: 6 } - is_false: aggregations.@timestamp.buckets.0.d.value - - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 0.500, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.250, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.143, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.286, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 3.429, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.667, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.333, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.667, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } --- ewma: diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index 86304c8c4bde2..06f8f8f3c1be6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -42,6 +42,8 @@ private SearchCapabilities() {} private static final String RANK_VECTORS_SCRIPT_ACCESS = "rank_vectors_script_access"; /** Initial support for rank-vectors maxSim functions access. */ private static final String RANK_VECTORS_SCRIPT_MAX_SIM = "rank_vectors_script_max_sim_with_bugfix"; + /** Fixed the math in {@code moving_fn}'s {@code linearWeightedAvg}. */ + private static final String MOVING_FN_RIGHT_MATH = "moving_fn_right_math"; private static final String RANDOM_SAMPLER_WITH_SCORED_SUBAGGS = "random_sampler_with_scored_subaggs"; private static final String OPTIMIZED_SCALAR_QUANTIZATION_BBQ = "optimized_scalar_quantization_bbq"; @@ -59,6 +61,7 @@ private SearchCapabilities() {} capabilities.add(RANDOM_SAMPLER_WITH_SCORED_SUBAGGS); capabilities.add(OPTIMIZED_SCALAR_QUANTIZATION_BBQ); capabilities.add(KNN_QUANTIZED_VECTOR_RESCORE); + capabilities.add(MOVING_FN_RIGHT_MATH); if (RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()) { capabilities.add(RANK_VECTORS_FIELD_MAPPER); capabilities.add(RANK_VECTORS_SCRIPT_ACCESS); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovingFunctions.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovingFunctions.java index 02e3c76e5e793..46584c171d16c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovingFunctions.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovingFunctions.java @@ -100,7 +100,7 @@ public static double stdDev(double[] values, double avg) { */ public static double linearWeightedAvg(double[] values) { double avg = 0; - long totalWeight = 1; + long totalWeight = 0; long current = 1; for (double v : values) { @@ -110,7 +110,7 @@ public static double linearWeightedAvg(double[] values) { current += 1; } } - return totalWeight == 1 ? Double.NaN : avg / totalWeight; + return totalWeight == 0 ? Double.NaN : avg / totalWeight; } /** diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnWhitelistedFunctionTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnWhitelistedFunctionTests.java index 69173957aebab..3bc458880db0a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnWhitelistedFunctionTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnWhitelistedFunctionTests.java @@ -326,7 +326,7 @@ public void testLinearMovAvg() { } double avg = 0; - long totalWeight = 1; + long totalWeight = 0; long current = 1; for (double value : window) { From b4610c8e26f292f73545b659a0bcdba7022ad1d0 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 13 Dec 2024 21:28:11 +0100 Subject: [PATCH 20/24] Remove needless sending of OriginalIndices in SearchFreeContextRequest (#117245) We don't need to use this request, the handler for freeing of scroll requests literally goes to the same transport handler and doesn't come with the list of indices. The original security need for keeping the list of indices around is long gone. --- .../action/IndicesRequestIT.java | 13 +--- .../search/AbstractSearchAsyncAction.java | 6 +- .../action/search/DfsQueryPhase.java | 6 +- .../action/search/SearchPhase.java | 6 +- .../action/search/SearchTransportService.java | 70 ++----------------- .../AbstractSearchAsyncActionTests.java | 6 +- .../action/search/MockSearchPhaseContext.java | 2 +- .../action/search/SearchAsyncActionTests.java | 15 +++- .../test/ESSingleNodeTestCase.java | 4 +- .../xpack/security/authz/RBACEngine.java | 26 +++---- 10 files changed, 44 insertions(+), 110 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java index 8bedf436e3698..f5860cedcd989 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java @@ -556,11 +556,7 @@ public void testUpdateSettings() { } public void testSearchQueryThenFetch() throws Exception { - interceptTransportActions( - SearchTransportService.QUERY_ACTION_NAME, - SearchTransportService.FETCH_ID_ACTION_NAME, - SearchTransportService.FREE_CONTEXT_ACTION_NAME - ); + interceptTransportActions(SearchTransportService.QUERY_ACTION_NAME, SearchTransportService.FETCH_ID_ACTION_NAME); String[] randomIndicesOrAliases = randomIndicesOrAliases(); for (int i = 0; i < randomIndicesOrAliases.length; i++) { @@ -580,16 +576,13 @@ public void testSearchQueryThenFetch() throws Exception { SearchTransportService.QUERY_ACTION_NAME, SearchTransportService.FETCH_ID_ACTION_NAME ); - // free context messages are not necessarily sent, but if they are, check their indices - assertIndicesSubsetOptionalRequests(Arrays.asList(searchRequest.indices()), SearchTransportService.FREE_CONTEXT_ACTION_NAME); } public void testSearchDfsQueryThenFetch() throws Exception { interceptTransportActions( SearchTransportService.DFS_ACTION_NAME, SearchTransportService.QUERY_ID_ACTION_NAME, - SearchTransportService.FETCH_ID_ACTION_NAME, - SearchTransportService.FREE_CONTEXT_ACTION_NAME + SearchTransportService.FETCH_ID_ACTION_NAME ); String[] randomIndicesOrAliases = randomIndicesOrAliases(); @@ -611,8 +604,6 @@ public void testSearchDfsQueryThenFetch() throws Exception { SearchTransportService.QUERY_ID_ACTION_NAME, SearchTransportService.FETCH_ID_ACTION_NAME ); - // free context messages are not necessarily sent, but if they are, check their indices - assertIndicesSubsetOptionalRequests(Arrays.asList(searchRequest.indices()), SearchTransportService.FREE_CONTEXT_ACTION_NAME); } private static void assertSameIndices(IndicesRequest originalRequest, String... actions) { diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index 800193e258dba..47abfe266c524 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -711,7 +711,7 @@ private void raisePhaseFailure(SearchPhaseExecutionException exception) { try { SearchShardTarget searchShardTarget = entry.getSearchShardTarget(); Transport.Connection connection = getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId()); - sendReleaseSearchContext(entry.getContextId(), connection, getOriginalIndices(entry.getShardIndex())); + sendReleaseSearchContext(entry.getContextId(), connection); } catch (Exception inner) { inner.addSuppressed(exception); logger.trace("failed to release context", inner); @@ -727,10 +727,10 @@ private void raisePhaseFailure(SearchPhaseExecutionException exception) { * @see org.elasticsearch.search.fetch.FetchSearchResult#getContextId() * */ - void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) { + void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection) { assert isPartOfPointInTime(contextId) == false : "Must not release point in time context [" + contextId + "]"; if (connection != null) { - searchTransportService.sendFreeContext(connection, contextId, originalIndices); + searchTransportService.sendFreeContext(connection, contextId, ActionListener.noop()); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java index 285dd0a22fd7e..cc8c4becea9a9 100644 --- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java @@ -119,11 +119,7 @@ public void onFailure(Exception exception) { // the query might not have been executed at all (for example because thread pool rejected // execution) and the search context that was created in dfs phase might not be released. // release it again to be in the safe side - context.sendReleaseSearchContext( - querySearchRequest.contextId(), - connection, - context.getOriginalIndices(shardIndex) - ); + context.sendReleaseSearchContext(querySearchRequest.contextId(), connection); } } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java index 986f7210c0d1b..7d849a72abf9d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java @@ -97,11 +97,7 @@ protected static void releaseIrrelevantSearchContext(SearchPhaseResult searchPha context.getLogger().trace("trying to release search context [{}]", phaseResult.getContextId()); SearchShardTarget shardTarget = phaseResult.getSearchShardTarget(); Transport.Connection connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()); - context.sendReleaseSearchContext( - phaseResult.getContextId(), - connection, - context.getOriginalIndices(phaseResult.getShardIndex()) - ); + context.sendReleaseSearchContext(phaseResult.getContextId(), connection); } catch (Exception e) { context.getLogger().trace("failed to release context", e); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java index 8444a92b24432..cfc2e1bcdaf2b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java @@ -13,12 +13,10 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; -import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.get.TransportGetTaskAction; import org.elasticsearch.action.support.ChannelActionListener; -import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -124,24 +122,6 @@ public SearchTransportService( this.responseWrapper = responseWrapper; } - private static final ActionListenerResponseHandler SEND_FREE_CONTEXT_LISTENER = - new ActionListenerResponseHandler<>( - ActionListener.noop(), - SearchFreeContextResponse::readFrom, - TransportResponseHandler.TRANSPORT_WORKER - ); - - public void sendFreeContext(Transport.Connection connection, final ShardSearchContextId contextId, OriginalIndices originalIndices) { - transportService.sendRequest( - connection, - FREE_CONTEXT_ACTION_NAME, - new SearchFreeContextRequest(originalIndices, contextId), - TransportRequestOptions.EMPTY, - // no need to respond if it was freed or not - SEND_FREE_CONTEXT_LISTENER - ); - } - public void sendFreeContext( Transport.Connection connection, ShardSearchContextId contextId, @@ -370,43 +350,6 @@ private static class ClearScrollContextsRequest extends TransportRequest { } } - static class SearchFreeContextRequest extends ScrollFreeContextRequest implements IndicesRequest { - private final OriginalIndices originalIndices; - - SearchFreeContextRequest(OriginalIndices originalIndices, ShardSearchContextId id) { - super(id); - this.originalIndices = originalIndices; - } - - SearchFreeContextRequest(StreamInput in) throws IOException { - super(in); - originalIndices = OriginalIndices.readOriginalIndices(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - OriginalIndices.writeOriginalIndices(originalIndices, out); - } - - @Override - public String[] indices() { - if (originalIndices == null) { - return null; - } - return originalIndices.indices(); - } - - @Override - public IndicesOptions indicesOptions() { - if (originalIndices == null) { - return null; - } - return originalIndices.indicesOptions(); - } - - } - public static class SearchFreeContextResponse extends TransportResponse { private static final SearchFreeContextResponse FREED = new SearchFreeContextResponse(true); @@ -456,12 +399,13 @@ public static void registerRequestHandler(TransportService transportService, Sea SearchFreeContextResponse::readFrom ); - transportService.registerRequestHandler( - FREE_CONTEXT_ACTION_NAME, - freeContextExecutor, - SearchFreeContextRequest::new, - freeContextHandler - ); + // TODO: remove this handler once the lowest compatible version stops using it + transportService.registerRequestHandler(FREE_CONTEXT_ACTION_NAME, freeContextExecutor, in -> { + var res = new ScrollFreeContextRequest(in); + // this handler exists for BwC purposes only, we don't need the original indices to free the context + OriginalIndices.readOriginalIndices(in); + return res; + }, freeContextHandler); TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, false, SearchFreeContextResponse::readFrom); transportService.registerRequestHandler( diff --git a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java index 725a4583d104a..71bf2a47cfa47 100644 --- a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java @@ -112,11 +112,7 @@ long buildTookInMillis() { } @Override - public void sendReleaseSearchContext( - ShardSearchContextId contextId, - Transport.Connection connection, - OriginalIndices originalIndices - ) { + public void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection) { releasedContexts.add(contextId); } diff --git a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java index 7a38858d8477a..cf65d756811ad 100644 --- a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java +++ b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java @@ -155,7 +155,7 @@ protected void executePhaseOnShard( } @Override - public void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) { + public void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection) { releasedSearchContexts.add(contextId); } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java index b4ddd48172d01..2361beb7ad036 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java @@ -296,7 +296,11 @@ public void testFanOutAndCollect() throws InterruptedException { AtomicInteger numFreedContext = new AtomicInteger(); SearchTransportService transportService = new SearchTransportService(null, null, null) { @Override - public void sendFreeContext(Transport.Connection connection, ShardSearchContextId contextId, OriginalIndices originalIndices) { + public void sendFreeContext( + Transport.Connection connection, + ShardSearchContextId contextId, + ActionListener listener + ) { numFreedContext.incrementAndGet(); assertTrue(nodeToContextMap.containsKey(connection.getNode())); assertTrue(nodeToContextMap.get(connection.getNode()).remove(contextId)); @@ -363,7 +367,7 @@ public void run() { for (int i = 0; i < results.getNumShards(); i++) { TestSearchPhaseResult result = results.getAtomicArray().get(i); assertEquals(result.node.getId(), result.getSearchShardTarget().getNodeId()); - sendReleaseSearchContext(result.getContextId(), new MockConnection(result.node), OriginalIndices.NONE); + sendReleaseSearchContext(result.getContextId(), new MockConnection(result.node)); } responseListener.onResponse(testResponse); if (latchTriggered.compareAndSet(false, true) == false) { @@ -421,8 +425,13 @@ public void testFanOutAndFail() throws InterruptedException { ); AtomicInteger numFreedContext = new AtomicInteger(); SearchTransportService transportService = new SearchTransportService(null, null, null) { + @Override - public void sendFreeContext(Transport.Connection connection, ShardSearchContextId contextId, OriginalIndices originalIndices) { + public void sendFreeContext( + Transport.Connection connection, + ShardSearchContextId contextId, + ActionListener listener + ) { assertNotNull(contextId); numFreedContext.incrementAndGet(); assertTrue(nodeToContextMap.containsKey(connection.getNode())); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 459d5573d7c12..63334bd70306f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -78,7 +78,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.elasticsearch.action.search.SearchTransportService.FREE_CONTEXT_ACTION_NAME; +import static org.elasticsearch.action.search.SearchTransportService.FREE_CONTEXT_SCROLL_ACTION_NAME; import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING; import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING; import static org.elasticsearch.test.NodeRoles.dataNode; @@ -482,7 +482,7 @@ protected void ensureNoInitializingShards() { */ protected void ensureAllFreeContextActionsAreConsumed() throws Exception { logger.info("--> waiting for all free_context tasks to complete within a reasonable time"); - safeGet(clusterAdmin().prepareListTasks().setActions(FREE_CONTEXT_ACTION_NAME + "*").setWaitForCompletion(true).execute()); + safeGet(clusterAdmin().prepareListTasks().setActions(FREE_CONTEXT_SCROLL_ACTION_NAME + "*").setWaitForCompletion(true).execute()); } /** diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index fa6187798da25..2353d710059ff 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -129,6 +129,19 @@ public class RBACEngine implements AuthorizationEngine { private static final String DELETE_SUB_REQUEST_REPLICA = TransportDeleteAction.NAME + "[r]"; private static final Logger logger = LogManager.getLogger(RBACEngine.class); + + private static final Set SCROLL_RELATED_ACTIONS = Set.of( + TransportSearchScrollAction.TYPE.name(), + SearchTransportService.FETCH_ID_SCROLL_ACTION_NAME, + SearchTransportService.QUERY_FETCH_SCROLL_ACTION_NAME, + SearchTransportService.QUERY_SCROLL_ACTION_NAME, + SearchTransportService.FREE_CONTEXT_ACTION_NAME, + SearchTransportService.FREE_CONTEXT_SCROLL_ACTION_NAME, + TransportClearScrollAction.NAME, + "indices:data/read/sql/close_cursor", + SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME + ); + private final Settings settings; private final CompositeRolesStore rolesStore; private final FieldPermissionsCache fieldPermissionsCache; @@ -320,7 +333,7 @@ public void authorizeIndexAction( // need to validate that the action is allowed and then move on listener.onResponse(role.checkIndicesAction(action) ? IndexAuthorizationResult.EMPTY : IndexAuthorizationResult.DENIED); } else if (request instanceof IndicesRequest == false) { - if (isScrollRelatedAction(action)) { + if (SCROLL_RELATED_ACTIONS.contains(action)) { // scroll is special // some APIs are indices requests that are not actually associated with indices. For example, // search scroll request, is categorized under the indices context, but doesn't hold indices names @@ -1000,17 +1013,6 @@ public int hashCode() { } } - private static boolean isScrollRelatedAction(String action) { - return action.equals(TransportSearchScrollAction.TYPE.name()) - || action.equals(SearchTransportService.FETCH_ID_SCROLL_ACTION_NAME) - || action.equals(SearchTransportService.QUERY_FETCH_SCROLL_ACTION_NAME) - || action.equals(SearchTransportService.QUERY_SCROLL_ACTION_NAME) - || action.equals(SearchTransportService.FREE_CONTEXT_SCROLL_ACTION_NAME) - || action.equals(TransportClearScrollAction.NAME) - || action.equals("indices:data/read/sql/close_cursor") - || action.equals(SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME); - } - private static boolean isAsyncRelatedAction(String action) { return action.equals(SubmitAsyncSearchAction.NAME) || action.equals(GetAsyncSearchAction.NAME) From 0a6ce27825d15a7e8294a99ee1aca52eb05f8ae0 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Fri, 13 Dec 2024 15:12:25 -0600 Subject: [PATCH 21/24] Add ReindexDatastreamIndexAction (#116996) Add an action to reindex a single index from a source index to a destination index. Unlike the reindex action, this action copies settings and mappings from the source index to the dest index before performing the reindex. This action is part of work to reindex data streams and will be called on each of the backing indices within a data stream. --- docs/changelog/116996.yaml | 5 + .../metadata/MetadataCreateIndexService.java | 48 ++- .../action/ReindexDatastreamIndexIT.java | 408 ++++++++++++++++++ .../xpack/migrate/MigratePlugin.java | 3 + .../action/ReindexDataStreamAction.java | 2 +- .../action/ReindexDataStreamIndexAction.java | 121 ++++++ ...ReindexDataStreamIndexTransportAction.java | 211 +++++++++ .../ReindexDatastreamIndexRequestTests.java | 29 ++ .../ReindexDatastreamIndexResponseTests.java | 29 ++ .../xpack/security/operator/Constants.java | 1 + 10 files changed, 841 insertions(+), 16 deletions(-) create mode 100644 docs/changelog/116996.yaml create mode 100644 x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexIT.java create mode 100644 x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java create mode 100644 x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java create mode 100644 x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexRequestTests.java create mode 100644 x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexResponseTests.java diff --git a/docs/changelog/116996.yaml b/docs/changelog/116996.yaml new file mode 100644 index 0000000000000..59f59355131bf --- /dev/null +++ b/docs/changelog/116996.yaml @@ -0,0 +1,5 @@ +pr: 116996 +summary: Initial work on `ReindexDatastreamIndexAction` +area: Data streams +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 75c2c06f36c8e..a99822c9de00c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1675,23 +1675,11 @@ static void prepareResizeIndexSettings( throw new IllegalStateException("unknown resize type is " + type); } - final Settings.Builder builder = Settings.builder(); + final Settings.Builder builder; if (copySettings) { - // copy all settings and non-copyable settings and settings that have already been set (e.g., from the request) - for (final String key : sourceMetadata.getSettings().keySet()) { - final Setting setting = indexScopedSettings.get(key); - if (setting == null) { - assert indexScopedSettings.isPrivateSetting(key) : key; - } else if (setting.getProperties().contains(Setting.Property.NotCopyableOnResize)) { - continue; - } - // do not override settings that have already been set (for example, from the request) - if (indexSettingsBuilder.keys().contains(key)) { - continue; - } - builder.copy(key, sourceMetadata.getSettings()); - } + builder = copySettingsFromSource(true, sourceMetadata.getSettings(), indexScopedSettings, indexSettingsBuilder); } else { + builder = Settings.builder(); final Predicate sourceSettingsPredicate = (s) -> (s.startsWith("index.similarity.") || s.startsWith("index.analysis.") || s.startsWith("index.sort.") @@ -1709,6 +1697,36 @@ static void prepareResizeIndexSettings( } } + public static Settings.Builder copySettingsFromSource( + boolean copyPrivateSettings, + Settings sourceSettings, + IndexScopedSettings indexScopedSettings, + Settings.Builder indexSettingsBuilder + ) { + final Settings.Builder builder = Settings.builder(); + for (final String key : sourceSettings.keySet()) { + final Setting setting = indexScopedSettings.get(key); + if (setting == null) { + assert indexScopedSettings.isPrivateSetting(key) : key; + if (copyPrivateSettings == false) { + continue; + } + } else if (setting.getProperties().contains(Setting.Property.NotCopyableOnResize)) { + continue; + } else if (setting.isPrivateIndex()) { + if (copyPrivateSettings == false) { + continue; + } + } + // do not override settings that have already been set (for example, from the request) + if (indexSettingsBuilder.keys().contains(key)) { + continue; + } + builder.copy(key, sourceSettings); + } + return builder; + } + /** * Returns a default number of routing shards based on the number of shards of the index. The default number of routing shards will * allow any index to be split at least once and at most 10 times by a factor of two. The closer the number or shards gets to 1024 diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexIT.java new file mode 100644 index 0000000000000..e492f035da866 --- /dev/null +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexIT.java @@ -0,0 +1,408 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.action; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.migrate.MigratePlugin; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.REINDEX_DATA_STREAM_FEATURE_FLAG; +import static org.hamcrest.Matchers.equalTo; + +public class ReindexDatastreamIndexIT extends ESIntegTestCase { + + private static final String MAPPING = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + + @Override + protected Collection> nodePlugins() { + return List.of(MigratePlugin.class, ReindexPlugin.class, MockTransportService.TestPlugin.class, DataStreamsPlugin.class); + } + + public void testDestIndexDeletedIfExists() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + // empty source index + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + + // dest index with docs + var destIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + indicesAdmin().create(new CreateIndexRequest(destIndex)).actionGet(); + indexDocs(destIndex, 10); + assertHitCount(prepareSearch(destIndex).setSize(0), 10); + + // call reindex + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet(); + + // verify that dest still exists, but is now empty + assertTrue(indexExists(destIndex)); + assertHitCount(prepareSearch(destIndex).setSize(0), 0); + } + + public void testDestIndexNameSet() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + + // call reindex + var response = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet(); + + var expectedDestIndexName = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + assertEquals(expectedDestIndexName, response.getDestIndex()); + } + + public void testDestIndexContainsDocs() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + // source index with docs + var numDocs = randomIntBetween(1, 100); + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + indexDocs(sourceIndex, numDocs); + + // call reindex + var response = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet(); + indicesAdmin().refresh(new RefreshRequest(response.getDestIndex())).actionGet(); + + // verify that dest contains docs + assertHitCount(prepareSearch(response.getDestIndex()).setSize(0), numDocs); + } + + public void testSetSourceToReadOnly() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + // empty source index + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + + // call reindex + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet(); + + // assert that write to source fails + var indexReq = new IndexRequest(sourceIndex).source(jsonBuilder().startObject().field("field", "1").endObject()); + assertThrows(ClusterBlockException.class, () -> client().index(indexReq).actionGet()); + assertHitCount(prepareSearch(sourceIndex).setSize(0), 0); + } + + public void testSettingsAddedBeforeReindex() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + // start with a static setting + var numShards = randomIntBetween(1, 10); + var staticSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build(); + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex, staticSettings)).get(); + + // update with a dynamic setting + var numReplicas = randomIntBetween(0, 10); + var dynamicSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas).build(); + indicesAdmin().updateSettings(new UpdateSettingsRequest(dynamicSettings, sourceIndex)).actionGet(); + + // call reindex + var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet() + .getDestIndex(); + + // assert both static and dynamic settings set on dest index + var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); + assertEquals(numReplicas, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS))); + assertEquals(numShards, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS))); + } + + public void testMappingsAddedToDestIndex() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + String mapping = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(mapping)).actionGet(); + + // call reindex + var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet() + .getDestIndex(); + + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + Map mappings = mappingsResponse.mappings(); + var destMappings = mappings.get(destIndex).sourceAsMap(); + var sourceMappings = mappings.get(sourceIndex).sourceAsMap(); + + assertEquals(sourceMappings, destMappings); + // sanity check specific value from dest mapping + assertEquals("text", XContentMapValues.extractValue("properties.foo1.type", destMappings)); + } + + public void testReadOnlyAddedBack() { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + // Create source index with read-only and/or block-writes + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + boolean isReadOnly = randomBoolean(); + boolean isBlockWrites = randomBoolean(); + var settings = Settings.builder() + .put(IndexMetadata.SETTING_READ_ONLY, isReadOnly) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, isBlockWrites) + .build(); + indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet(); + + // call reindex + var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet() + .getDestIndex(); + + // assert read-only settings added back to dest index + var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); + assertEquals(isReadOnly, Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY))); + assertEquals(isBlockWrites, Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE))); + + removeReadOnly(sourceIndex); + removeReadOnly(destIndex); + } + + public void testSettingsAndMappingsFromTemplate() throws IOException { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var numShards = randomIntBetween(1, 10); + var numReplicas = randomIntBetween(0, 10); + + var settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas) + .build(); + + // Create template with settings and mappings + var template = ComposableIndexTemplate.builder() + .indexPatterns(List.of("logs-*")) + .template(new Template(settings, CompressedXContent.fromJSON(MAPPING), null)) + .build(); + var request = new TransportPutComposableIndexTemplateAction.Request("logs-template"); + request.indexTemplate(template); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + + var sourceIndex = "logs-" + randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex)).actionGet(); + + // call reindex + var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet() + .getDestIndex(); + + // verify settings from templates copied to dest index + { + var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); + assertEquals(numReplicas, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS))); + assertEquals(numShards, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS))); + } + + // verify mappings from templates copied to dest index + { + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var destMappings = mappingsResponse.mappings().get(destIndex).sourceAsMap(); + var sourceMappings = mappingsResponse.mappings().get(sourceIndex).sourceAsMap(); + assertEquals(sourceMappings, destMappings); + // sanity check specific value from dest mapping + assertEquals("text", XContentMapValues.extractValue("properties.foo1.type", destMappings)); + } + } + + private static final String TSDB_MAPPING = """ + { + "_doc":{ + "properties": { + "@timestamp" : { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""; + + private static final String TSDB_DOC = """ + { + "@timestamp": "$time", + "metricset": "pod", + "k8s": { + "pod": { + "name": "dog", + "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", + "ip": "10.10.55.3", + "network": { + "tx": 1434595272, + "rx": 530605511 + } + } + } + } + """; + + public void testTsdbStartEndSet() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var templateSettings = Settings.builder().put("index.mode", "time_series"); + if (randomBoolean()) { + templateSettings.put("index.routing_path", "metricset"); + } + var mapping = new CompressedXContent(TSDB_MAPPING); + + // create template + var request = new TransportPutComposableIndexTemplateAction.Request("id"); + request.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of("k8s*")) + .template(new Template(templateSettings.build(), mapping, null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + + // index doc + Instant time = Instant.now(); + String backingIndexName; + { + var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); + indexRequest.source(TSDB_DOC.replace("$time", formatInstant(time)), XContentType.JSON); + var indexResponse = client().index(indexRequest).actionGet(); + backingIndexName = indexResponse.getIndex(); + } + + var sourceSettings = indicesAdmin().getIndex(new GetIndexRequest().indices(backingIndexName)) + .actionGet() + .getSettings() + .get(backingIndexName); + Instant startTime = IndexSettings.TIME_SERIES_START_TIME.get(sourceSettings); + Instant endTime = IndexSettings.TIME_SERIES_END_TIME.get(sourceSettings); + + // sanity check start/end time on source + assertNotNull(startTime); + assertNotNull(endTime); + assertTrue(endTime.isAfter(startTime)); + + // force a rollover so can call reindex and delete + var rolloverRequest = new RolloverRequest("k8s", null); + var rolloverResponse = indicesAdmin().rolloverIndex(rolloverRequest).actionGet(); + rolloverResponse.getNewIndex(); + + // call reindex on the original backing index + var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(backingIndexName)) + .actionGet() + .getDestIndex(); + + var destSettings = indicesAdmin().getIndex(new GetIndexRequest().indices(destIndex)).actionGet().getSettings().get(destIndex); + var destStart = IndexSettings.TIME_SERIES_START_TIME.get(destSettings); + var destEnd = IndexSettings.TIME_SERIES_END_TIME.get(destSettings); + + assertEquals(startTime, destStart); + assertEquals(endTime, destEnd); + } + + // TODO more logsdb/tsdb specific tests + // TODO more data stream specific tests (how are data streams indices are different from regular indices?) + // TODO check other IndexMetadata fields that need to be fixed after the fact + // TODO what happens if don't have necessary perms for a given index? + + private static void removeReadOnly(String index) { + var settings = Settings.builder() + .put(IndexMetadata.SETTING_READ_ONLY, false) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, false) + .build(); + assertAcked(indicesAdmin().updateSettings(new UpdateSettingsRequest(settings, index)).actionGet()); + } + + private static void indexDocs(String index, int numDocs) { + BulkRequest bulkRequest = new BulkRequest(); + for (int i = 0; i < numDocs; i++) { + String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); + bulkRequest.add( + new IndexRequest(index).opType(DocWriteRequest.OpType.CREATE) + .id(i + "") + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) + ); + } + BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat(bulkResponse.getItems().length, equalTo(numDocs)); + indicesAdmin().refresh(new RefreshRequest(index)).actionGet(); + } + + private static String formatInstant(Instant instant) { + return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); + } + + private static String getIndexUUID(String index) { + return indicesAdmin().getIndex(new GetIndexRequest().indices(index)) + .actionGet() + .getSettings() + .get(index) + .get(IndexMetadata.SETTING_INDEX_UUID); + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java index 26f8e57102a4d..f42d05727b9fd 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java @@ -37,6 +37,8 @@ import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusAction; import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusTransportAction; import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction; +import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexAction; +import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexTransportAction; import org.elasticsearch.xpack.migrate.action.ReindexDataStreamTransportAction; import org.elasticsearch.xpack.migrate.rest.RestCancelReindexDataStreamAction; import org.elasticsearch.xpack.migrate.rest.RestGetMigrationReindexStatusAction; @@ -84,6 +86,7 @@ public List getRestHandlers( actions.add(new ActionHandler<>(ReindexDataStreamAction.INSTANCE, ReindexDataStreamTransportAction.class)); actions.add(new ActionHandler<>(GetMigrationReindexStatusAction.INSTANCE, GetMigrationReindexStatusTransportAction.class)); actions.add(new ActionHandler<>(CancelReindexDataStreamAction.INSTANCE, CancelReindexDataStreamTransportAction.class)); + actions.add(new ActionHandler<>(ReindexDataStreamIndexAction.INSTANCE, ReindexDataStreamIndexTransportAction.class)); } return actions; } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java index 39d4170f6e712..b10bea9e54230 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java @@ -47,7 +47,7 @@ public class ReindexDataStreamAction extends ActionType { + + public static final String NAME = "indices:admin/data_stream/index/reindex"; + + public static final ActionType INSTANCE = new ReindexDataStreamIndexAction(); + + private ReindexDataStreamIndexAction() { + super(NAME); + } + + public static class Request extends ActionRequest implements IndicesRequest { + + private final String sourceIndex; + + public Request(String sourceIndex) { + this.sourceIndex = sourceIndex; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.sourceIndex = in.readString(); + } + + public static Request readFrom(StreamInput in) throws IOException { + return new Request(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(sourceIndex); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public String getSourceIndex() { + return sourceIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(sourceIndex, request.sourceIndex); + } + + @Override + public int hashCode() { + return Objects.hash(sourceIndex); + } + + @Override + public String[] indices() { + return new String[] { sourceIndex }; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + } + } + + public static class Response extends ActionResponse { + private final String destIndex; + + public Response(String destIndex) { + this.destIndex = destIndex; + } + + public Response(StreamInput in) throws IOException { + super(in); + this.destIndex = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(destIndex); + } + + public String getDestIndex() { + return destIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(destIndex, response.destIndex); + } + + @Override + public int hashCode() { + return Objects.hash(destIndex); + } + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java new file mode 100644 index 0000000000000..8863c45691c92 --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java @@ -0,0 +1,211 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.migrate.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.ReindexAction; +import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +public class ReindexDataStreamIndexTransportAction extends HandledTransportAction< + ReindexDataStreamIndexAction.Request, + ReindexDataStreamIndexAction.Response> { + + private static final Logger logger = LogManager.getLogger(ReindexDataStreamIndexTransportAction.class); + + private static final Set SETTINGS_TO_ADD_BACK = Set.of(IndexMetadata.SETTING_BLOCKS_WRITE, IndexMetadata.SETTING_READ_ONLY); + + private static final IndicesOptions IGNORE_MISSING_OPTIONS = IndicesOptions.fromOptions(true, true, false, false); + private final ClusterService clusterService; + private final Client client; + private final IndexScopedSettings indexScopedSettings; + + @Inject + public ReindexDataStreamIndexTransportAction( + TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + Client client, + IndexScopedSettings indexScopedSettings + ) { + super( + ReindexDataStreamIndexAction.NAME, + false, + transportService, + actionFilters, + ReindexDataStreamIndexAction.Request::new, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC) + ); + this.clusterService = clusterService; + this.client = client; + this.indexScopedSettings = indexScopedSettings; + } + + @Override + protected void doExecute( + Task task, + ReindexDataStreamIndexAction.Request request, + ActionListener listener + ) { + var sourceIndexName = request.getSourceIndex(); + var destIndexName = generateDestIndexName(sourceIndexName); + IndexMetadata sourceIndex = clusterService.state().getMetadata().index(sourceIndexName); + Settings settingsBefore = sourceIndex.getSettings(); + + var hasOldVersion = ReindexDataStreamAction.getOldIndexVersionPredicate(clusterService.state().metadata()); + if (hasOldVersion.test(sourceIndex.getIndex()) == false) { + logger.warn( + "Migrating index [{}] with version [{}] is unnecessary as its version is not before [{}]", + sourceIndexName, + sourceIndex.getCreationVersion(), + ReindexDataStreamAction.MINIMUM_WRITEABLE_VERSION_AFTER_UPGRADE + ); + } + + SubscribableListener.newForked(l -> setBlockWrites(sourceIndexName, l)) + .andThen(l -> deleteDestIfExists(destIndexName, l)) + .andThen(l -> createIndex(sourceIndex, destIndexName, l)) + .andThen(l -> reindex(sourceIndexName, destIndexName, l)) + .andThen(l -> updateSettings(settingsBefore, destIndexName, l)) + .andThenApply(ignored -> new ReindexDataStreamIndexAction.Response(destIndexName)) + .addListener(listener); + } + + private void setBlockWrites(String sourceIndexName, ActionListener listener) { + logger.debug("Setting write block on source index [{}]", sourceIndexName); + final Settings readOnlySettings = Settings.builder().put(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey(), true).build(); + var updateSettingsRequest = new UpdateSettingsRequest(readOnlySettings, sourceIndexName); + client.admin().indices().updateSettings(updateSettingsRequest, new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse response) { + if (response.isAcknowledged()) { + listener.onResponse(null); + } else { + var errorMessage = String.format(Locale.ROOT, "Could not set read-only on source index [%s]", sourceIndexName); + listener.onFailure(new ElasticsearchException(errorMessage)); + } + } + + @Override + public void onFailure(Exception e) { + if (e instanceof ClusterBlockException || e.getCause() instanceof ClusterBlockException) { + // It's fine if read-only is already set + listener.onResponse(null); + } else { + listener.onFailure(e); + } + } + }); + } + + private void deleteDestIfExists(String destIndexName, ActionListener listener) { + logger.debug("Attempting to delete index [{}]", destIndexName); + var deleteIndexRequest = new DeleteIndexRequest(destIndexName).indicesOptions(IGNORE_MISSING_OPTIONS) + .masterNodeTimeout(TimeValue.MAX_VALUE); + var errorMessage = String.format(Locale.ROOT, "Failed to acknowledge delete of index [%s]", destIndexName); + client.admin().indices().delete(deleteIndexRequest, failIfNotAcknowledged(listener, errorMessage)); + } + + private void createIndex(IndexMetadata sourceIndex, String destIndexName, ActionListener listener) { + logger.debug("Creating destination index [{}] for source index [{}]", destIndexName, sourceIndex.getIndex().getName()); + + // Create destination with subset of source index settings that can be added before reindex + var settings = getPreSettings(sourceIndex); + + var sourceMapping = sourceIndex.mapping(); + Map mapping = sourceMapping != null ? sourceMapping.rawSourceAsMap() : Map.of(); + var createIndexRequest = new CreateIndexRequest(destIndexName).settings(settings).mapping(mapping); + + var errorMessage = String.format(Locale.ROOT, "Could not create index [%s]", destIndexName); + client.admin().indices().create(createIndexRequest, failIfNotAcknowledged(listener, errorMessage)); + } + + private void reindex(String sourceIndexName, String destIndexName, ActionListener listener) { + logger.debug("Reindex to destination index [{}] from source index [{}]", destIndexName, sourceIndexName); + var reindexRequest = new ReindexRequest(); + reindexRequest.setSourceIndices(sourceIndexName); + reindexRequest.getSearchRequest().allowPartialSearchResults(false); + reindexRequest.getSearchRequest().source().fetchSource(true); + reindexRequest.setDestIndex(destIndexName); + client.execute(ReindexAction.INSTANCE, reindexRequest, listener); + } + + private void updateSettings(Settings settingsBefore, String destIndexName, ActionListener listener) { + logger.debug("Adding settings from source index that could not be added before reindex"); + + Settings postSettings = getPostSettings(settingsBefore); + if (postSettings.isEmpty()) { + listener.onResponse(null); + return; + } + + var updateSettingsRequest = new UpdateSettingsRequest(postSettings, destIndexName); + var errorMessage = String.format(Locale.ROOT, "Could not update settings on index [%s]", destIndexName); + client.admin().indices().updateSettings(updateSettingsRequest, failIfNotAcknowledged(listener, errorMessage)); + } + + // Filter source index settings to subset of settings that can be included during reindex. + // Similar to the settings filtering done when reindexing for upgrade in Kibana + // https://github.com/elastic/kibana/blob/8a8363f02cc990732eb9cbb60cd388643a336bed/x-pack + // /plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts#L155 + private Settings getPreSettings(IndexMetadata sourceIndex) { + // filter settings that will be added back later + var filtered = sourceIndex.getSettings().filter(settingName -> SETTINGS_TO_ADD_BACK.contains(settingName) == false); + + // filter private and non-copyable settings + var builder = MetadataCreateIndexService.copySettingsFromSource(false, filtered, indexScopedSettings, Settings.builder()); + return builder.build(); + } + + private Settings getPostSettings(Settings settingsBefore) { + return settingsBefore.filter(SETTINGS_TO_ADD_BACK::contains); + } + + public static String generateDestIndexName(String sourceIndex) { + return "migrated-" + sourceIndex; + } + + private static ActionListener failIfNotAcknowledged( + ActionListener listener, + String errorMessage + ) { + return listener.delegateFailureAndWrap((delegate, response) -> { + if (response.isAcknowledged()) { + delegate.onResponse(null); + } + throw new ElasticsearchException(errorMessage); + }); + } +} diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexRequestTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexRequestTests.java new file mode 100644 index 0000000000000..a057056474ef1 --- /dev/null +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexRequestTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.action; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexAction.Request; + +public class ReindexDatastreamIndexRequestTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return Request::new; + } + + @Override + protected Request createTestInstance() { + return new Request(randomAlphaOfLength(20)); + } + + @Override + protected Request mutateInstance(Request instance) { + return new ReindexDataStreamIndexAction.Request(randomValueOtherThan(instance.getSourceIndex(), () -> randomAlphaOfLength(20))); + } +} diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexResponseTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexResponseTests.java new file mode 100644 index 0000000000000..752e173585f0e --- /dev/null +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexResponseTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.action; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexAction.Response; + +public class ReindexDatastreamIndexResponseTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return Response::new; + } + + @Override + protected Response createTestInstance() { + return new Response(randomAlphaOfLength(20)); + } + + @Override + protected Response mutateInstance(Response instance) { + return new Response(randomValueOtherThan(instance.getDestIndex(), () -> randomAlphaOfLength(20))); + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index b139d1526ec20..c07e4b2c541a2 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -638,6 +638,7 @@ public class Constants { "internal:admin/indices/prevalidate_shard_path", "internal:index/metadata/migration_version/update", new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/migration/reindex_status" : null, + new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/index/reindex" : null, new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/reindex" : null, new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/reindex_cancel" : null, "internal:admin/repository/verify", From e454d74de4e7eaf7054c3992632b4004e864ca7c Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Fri, 13 Dec 2024 13:13:44 -0800 Subject: [PATCH 22/24] Add optional pull request check for running with entitlements enabled (#118693) --- .../pipelines/pull-request/part-1-entitlements.yml | 11 +++++++++++ .../pipelines/pull-request/part-2-entitlements.yml | 11 +++++++++++ .../pipelines/pull-request/part-3-entitlements.yml | 11 +++++++++++ .../pipelines/pull-request/part-4-entitlements.yml | 11 +++++++++++ .../pipelines/pull-request/part-5-entitlements.yml | 11 +++++++++++ .../gradle/testclusters/ElasticsearchNode.java | 6 ++---- 6 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 .buildkite/pipelines/pull-request/part-1-entitlements.yml create mode 100644 .buildkite/pipelines/pull-request/part-2-entitlements.yml create mode 100644 .buildkite/pipelines/pull-request/part-3-entitlements.yml create mode 100644 .buildkite/pipelines/pull-request/part-4-entitlements.yml create mode 100644 .buildkite/pipelines/pull-request/part-5-entitlements.yml diff --git a/.buildkite/pipelines/pull-request/part-1-entitlements.yml b/.buildkite/pipelines/pull-request/part-1-entitlements.yml new file mode 100644 index 0000000000000..abb9edf67484f --- /dev/null +++ b/.buildkite/pipelines/pull-request/part-1-entitlements.yml @@ -0,0 +1,11 @@ +config: + allow-labels: "test-entitlements" +steps: + - label: part-1-entitlements + command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart1 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk diff --git a/.buildkite/pipelines/pull-request/part-2-entitlements.yml b/.buildkite/pipelines/pull-request/part-2-entitlements.yml new file mode 100644 index 0000000000000..ef889f3819c5b --- /dev/null +++ b/.buildkite/pipelines/pull-request/part-2-entitlements.yml @@ -0,0 +1,11 @@ +config: + allow-labels: "test-entitlements" +steps: + - label: part-2-entitlements + command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart2 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk diff --git a/.buildkite/pipelines/pull-request/part-3-entitlements.yml b/.buildkite/pipelines/pull-request/part-3-entitlements.yml new file mode 100644 index 0000000000000..c31ae5e6a4ce3 --- /dev/null +++ b/.buildkite/pipelines/pull-request/part-3-entitlements.yml @@ -0,0 +1,11 @@ +config: + allow-labels: "test-entitlements" +steps: + - label: part-3-entitlements + command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart3 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk diff --git a/.buildkite/pipelines/pull-request/part-4-entitlements.yml b/.buildkite/pipelines/pull-request/part-4-entitlements.yml new file mode 100644 index 0000000000000..5817dacbad80b --- /dev/null +++ b/.buildkite/pipelines/pull-request/part-4-entitlements.yml @@ -0,0 +1,11 @@ +config: + allow-labels: "test-entitlements" +steps: + - label: part-4-entitlements + command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart4 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk diff --git a/.buildkite/pipelines/pull-request/part-5-entitlements.yml b/.buildkite/pipelines/pull-request/part-5-entitlements.yml new file mode 100644 index 0000000000000..5a92282361576 --- /dev/null +++ b/.buildkite/pipelines/pull-request/part-5-entitlements.yml @@ -0,0 +1,11 @@ +config: + allow-labels: "test-entitlements" +steps: + - label: part-5-entitlements + command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart5 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 4cb67e249b0b0..2f0f85bc6e159 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -876,10 +876,8 @@ private void startElasticsearchProcess() { // Don't inherit anything from the environment for as that would lack reproducibility environment.clear(); environment.putAll(getESEnvironment()); - if (cliJvmArgs.isEmpty() == false) { - String cliJvmArgsString = String.join(" ", cliJvmArgs); - environment.put("CLI_JAVA_OPTS", cliJvmArgsString); - } + String cliJvmArgsString = String.join(" ", cliJvmArgs); + environment.put("CLI_JAVA_OPTS", cliJvmArgsString + " " + System.getProperty("tests.jvm.argline", "")); // Direct the stderr to the ES log file. This should capture any jvm problems to start. // Stdout is discarded because ES duplicates the log file to stdout when run in the foreground. From ac0ba24847a6f09ef8686659cd16ca74bc1d79d1 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Fri, 13 Dec 2024 13:17:56 -0800 Subject: [PATCH 23/24] ESQL: Push down filter passed lookup join (#118410) Improve the planner to detect filters that can be pushed down 'through' a LOOKUP JOIN by determining the conditions scoped to the left/main side and moving them closer to the source. Relates #118305 --- docs/changelog/118410.yaml | 5 + .../core/expression/predicate/Predicates.java | 2 +- .../src/main/resources/lookup-join.csv-spec | 65 +++++ .../logical/PushDownAndCombineFilters.java | 55 ++++ .../optimizer/LogicalPlanOptimizerTests.java | 245 +++++++++++++++++- 5 files changed, 369 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/118410.yaml diff --git a/docs/changelog/118410.yaml b/docs/changelog/118410.yaml new file mode 100644 index 0000000000000..ccc7f71ee2e1c --- /dev/null +++ b/docs/changelog/118410.yaml @@ -0,0 +1,5 @@ +pr: 118410 +summary: Push down filter passed lookup join +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Predicates.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Predicates.java index e63cc1fcf25fe..32f7e181933b4 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Predicates.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Predicates.java @@ -61,7 +61,7 @@ public static Expression combineAnd(List exps) { * * using the given combiner. * - * While a bit longer, this method creates a balanced tree as oppose to a plain + * While a bit longer, this method creates a balanced tree as opposed to a plain * recursive approach which creates an unbalanced one (either to the left or right). */ private static Expression combine(List exps, BiFunction combiner) { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index f6704d33934af..c39f4ae7b4e0c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -418,3 +418,68 @@ count:long | type:keyword 3 | Success 1 | Disconnected ; + +// +// Filtering tests +// + +lookupWithFilterOnLeftSideField +required_capability: join_lookup_v5 + +FROM employees +| EVAL language_code = languages +| LOOKUP JOIN languages_lookup ON language_code +| SORT emp_no +| KEEP emp_no, language_code, language_name +| WHERE emp_no >= 10091 AND emp_no < 10094 +; + +emp_no:integer | language_code:integer | language_name:keyword +10091 | 3 | Spanish +10092 | 1 | English +10093 | 3 | Spanish +; + +lookupMessageWithFilterOnRightSideField-Ignore +required_capability: join_lookup_v5 + +FROM sample_data +| LOOKUP JOIN message_types_lookup ON message +| WHERE type == "Error" +| KEEP @timestamp, client_ip, event_duration, message, type +| SORT @timestamp DESC +; + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | type:keyword +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | Error +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | Error +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | Error +; + +lookupWithFieldAndRightSideAfterStats +required_capability: join_lookup_v5 + +FROM sample_data +| LOOKUP JOIN message_types_lookup ON message +| STATS count = count(message) BY type +| WHERE type == "Error" +; + +count:long | type:keyword +3 | Error +; + +lookupWithFieldOnJoinKey-Ignore +required_capability: join_lookup_v5 + +FROM employees +| EVAL language_code = languages +| LOOKUP JOIN languages_lookup ON language_code +| WHERE language_code > 1 AND language_name IS NOT NULL +| KEEP emp_no, language_code, language_name +; + +emp_no:integer | language_code:integer | language_name:keyword +10001 | 2 | French +10003 | 4 | German +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java index 15e49c22a44db..9ec902e729f54 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; +import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Filter; @@ -23,6 +24,8 @@ import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.plan.logical.join.Join; +import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; import java.util.ArrayList; import java.util.List; @@ -76,11 +79,63 @@ protected LogicalPlan rule(Filter filter) { } else if (child instanceof OrderBy orderBy) { // swap the filter with its child plan = orderBy.replaceChild(filter.with(orderBy.child(), condition)); + } else if (child instanceof Join join) { + return pushDownPastJoin(filter, join); } // cannot push past a Limit, this could change the tailing result set returned return plan; } + private record ScopedFilter(List commonFilters, List leftFilters, List rightFilters) {} + + // split the filter condition in 3 parts: + // 1. filter scoped to the left + // 2. filter scoped to the right + // 3. filter that requires both sides to be evaluated + private static ScopedFilter scopeFilter(List filters, LogicalPlan left, LogicalPlan right) { + List rest = new ArrayList<>(filters); + List leftFilters = new ArrayList<>(); + List rightFilters = new ArrayList<>(); + + AttributeSet leftOutput = left.outputSet(); + AttributeSet rightOutput = right.outputSet(); + + // first remove things that are left scoped only + rest.removeIf(f -> f.references().subsetOf(leftOutput) && leftFilters.add(f)); + // followed by right scoped only + rest.removeIf(f -> f.references().subsetOf(rightOutput) && rightFilters.add(f)); + return new ScopedFilter(rest, leftFilters, rightFilters); + } + + private static LogicalPlan pushDownPastJoin(Filter filter, Join join) { + LogicalPlan plan = filter; + // pushdown only through LEFT joins + // TODO: generalize this for other join types + if (join.config().type() == JoinTypes.LEFT) { + LogicalPlan left = join.left(); + LogicalPlan right = join.right(); + + // split the filter condition in 3 parts: + // 1. filter scoped to the left + // 2. filter scoped to the right + // 3. filter that requires both sides to be evaluated + ScopedFilter scoped = scopeFilter(Predicates.splitAnd(filter.condition()), left, right); + // push the left scoped filter down to the left child, keep the rest intact + if (scoped.leftFilters.size() > 0) { + // push the filter down to the left child + left = new Filter(left.source(), left, Predicates.combineAnd(scoped.leftFilters)); + // update the join with the new left child + join = (Join) join.replaceLeft(left); + + // keep the remaining filters in place, otherwise return the new join; + Expression remainingFilter = Predicates.combineAnd(CollectionUtils.combine(scoped.commonFilters, scoped.rightFilters)); + plan = remainingFilter != null ? filter.with(join, remainingFilter) : join; + } + } + // ignore the rest of the join + return plan; + } + private static Function NO_OP = expression -> expression; private static LogicalPlan maybePushDownPastUnary( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index d97a8bb2bc27f..0a01dfc088cfb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.Build; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.lucene.BytesRefs; @@ -217,6 +218,11 @@ public static void init() { enrichResolution = new EnrichResolution(); AnalyzerTestUtils.loadEnrichPolicyResolution(enrichResolution, "languages_idx", "id", "languages_idx", "mapping-languages.json"); + var lookupMapping = loadMapping("mapping-languages.json"); + IndexResolution lookupResolution = IndexResolution.valid( + new EsIndex("language_code", lookupMapping, Map.of("language_code", IndexMode.LOOKUP)) + ); + // Most tests used data from the test index, so we load it here, and use it in the plan() function. mapping = loadMapping("mapping-basic.json"); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); @@ -5740,7 +5746,7 @@ public void testLookupSimple() { String query = """ FROM test | RENAME languages AS int - | LOOKUP int_number_names ON int"""; + | LOOKUP_?? int_number_names ON int"""; if (Build.current().isSnapshot() == false) { var e = expectThrows(ParsingException.class, () -> analyze(query)); assertThat(e.getMessage(), containsString("line 3:3: mismatched input 'LOOKUP' expecting {")); @@ -5820,7 +5826,7 @@ public void testLookupStats() { String query = """ FROM test | RENAME languages AS int - | LOOKUP int_number_names ON int + | LOOKUP_?? int_number_names ON int | STATS MIN(emp_no) BY name"""; if (Build.current().isSnapshot() == false) { var e = expectThrows(ParsingException.class, () -> analyze(query)); @@ -5889,6 +5895,241 @@ public void testLookupStats() { ); } + // + // Lookup JOIN + // + + /** + * Filter on join keys should be pushed down + * Expects + * Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang + * uage_code{r}#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]] + * \_Join[LEFT,[language_code{r}#4],[language_code{r}#4],[language_code{f}#18]] + * |_EsqlProject[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang + * uages{f}#10 AS language_code, last_name{f}#11, long_noidx{f}#17, salary{f}#12]] + * | \_Limit[1000[INTEGER]] + * | \_Filter[languages{f}#10 > 1[INTEGER]] + * | \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsRelation[language_code][LOOKUP][language_code{f}#18, language_name{f}#19] + */ + public void testLookupJoinPushDownFilterOnJoinKeyWithRename() { + String query = """ + FROM test + | RENAME languages AS language_code + | LOOKUP JOIN language_code ON language_code + | WHERE language_code > 1 + """; + var plan = optimizedPlan(query); + + var project = as(plan, Project.class); + var join = as(project.child(), Join.class); + assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); + project = as(join.left(), Project.class); + var limit = as(project.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(1000)); + var filter = as(limit.child(), Filter.class); + // assert that the rename has been undone + var op = as(filter.condition(), GreaterThan.class); + var field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("languages")); + + var literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(1)); + + var leftRel = as(filter.child(), EsRelation.class); + var rightRel = as(join.right(), EsRelation.class); + } + + /** + * Filter on on left side fields (outside the join key) should be pushed down + * Expects + * Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang + * uage_code{r}#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]] + * \_Join[LEFT,[language_code{r}#4],[language_code{r}#4],[language_code{f}#18]] + * |_EsqlProject[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang + * uages{f}#10 AS language_code, last_name{f}#11, long_noidx{f}#17, salary{f}#12]] + * | \_Limit[1000[INTEGER]] + * | \_Filter[emp_no{f}#7 > 1[INTEGER]] + * | \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsRelation[language_code][LOOKUP][language_code{f}#18, language_name{f}#19] + */ + public void testLookupJoinPushDownFilterOnLeftSideField() { + String query = """ + FROM test + | RENAME languages AS language_code + | LOOKUP JOIN language_code ON language_code + | WHERE emp_no > 1 + """; + + var plan = optimizedPlan(query); + + var project = as(plan, Project.class); + var join = as(project.child(), Join.class); + assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); + project = as(join.left(), Project.class); + + var limit = as(project.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(1000)); + var filter = as(limit.child(), Filter.class); + var op = as(filter.condition(), GreaterThan.class); + var field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("emp_no")); + + var literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(1)); + + var leftRel = as(filter.child(), EsRelation.class); + var rightRel = as(join.right(), EsRelation.class); + } + + /** + * Filter works on the right side fields and thus cannot be pushed down + * Expects + * Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang + * uage_code{r}#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]] + * \_Limit[1000[INTEGER]] + * \_Filter[language_name{f}#19 == [45 6e 67 6c 69 73 68][KEYWORD]] + * \_Join[LEFT,[language_code{r}#4],[language_code{r}#4],[language_code{f}#18]] + * |_EsqlProject[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang + * uages{f}#10 AS language_code, last_name{f}#11, long_noidx{f}#17, salary{f}#12]] + * | \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsRelation[language_code][LOOKUP][language_code{f}#18, language_name{f}#19] + */ + public void testLookupJoinPushDownDisabledForLookupField() { + String query = """ + FROM test + | RENAME languages AS language_code + | LOOKUP JOIN language_code ON language_code + | WHERE language_name == "English" + """; + + var plan = optimizedPlan(query); + + var project = as(plan, Project.class); + var limit = as(project.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(1000)); + + var filter = as(limit.child(), Filter.class); + var op = as(filter.condition(), Equals.class); + var field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("language_name")); + var literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(new BytesRef("English"))); + + var join = as(filter.child(), Join.class); + assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); + project = as(join.left(), Project.class); + + var leftRel = as(project.child(), EsRelation.class); + var rightRel = as(join.right(), EsRelation.class); + } + + /** + * Split the conjunction into pushable and non pushable filters. + * Expects + * Project[[_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, gender{f}#10, hire_date{f}#15, job{f}#16, job.raw{f}#17, lan + * guage_code{r}#4, last_name{f}#12, long_noidx{f}#18, salary{f}#13, language_name{f}#20]] + * \_Limit[1000[INTEGER]] + * \_Filter[language_name{f}#20 == [45 6e 67 6c 69 73 68][KEYWORD]] + * \_Join[LEFT,[language_code{r}#4],[language_code{r}#4],[language_code{f}#19]] + * |_EsqlProject[[_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, gender{f}#10, hire_date{f}#15, job{f}#16, job.raw{f}#17, lan + * guages{f}#11 AS language_code, last_name{f}#12, long_noidx{f}#18, salary{f}#13]] + * | \_Filter[emp_no{f}#8 > 1[INTEGER]] + * | \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..] + * \_EsRelation[language_code][LOOKUP][language_code{f}#19, language_name{f}#20] + */ + public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightField() { + String query = """ + FROM test + | RENAME languages AS language_code + | LOOKUP JOIN language_code ON language_code + | WHERE language_name == "English" AND emp_no > 1 + """; + + var plan = optimizedPlan(query); + + var project = as(plan, Project.class); + var limit = as(project.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(1000)); + // filter kept in place, working on the right side + var filter = as(limit.child(), Filter.class); + EsqlBinaryComparison op = as(filter.condition(), Equals.class); + var field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("language_name")); + var literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(new BytesRef("English"))); + + var join = as(filter.child(), Join.class); + assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); + project = as(join.left(), Project.class); + // filter pushed down + filter = as(project.child(), Filter.class); + op = as(filter.condition(), GreaterThan.class); + field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("emp_no")); + + literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(1)); + + var leftRel = as(filter.child(), EsRelation.class); + var rightRel = as(join.right(), EsRelation.class); + + } + + /** + * Disjunctions however keep the filter in place, even on pushable fields + * Expects + * Project[[_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, gender{f}#10, hire_date{f}#15, job{f}#16, job.raw{f}#17, lan + * guage_code{r}#4, last_name{f}#12, long_noidx{f}#18, salary{f}#13, language_name{f}#20]] + * \_Limit[1000[INTEGER]] + * \_Filter[language_name{f}#20 == [45 6e 67 6c 69 73 68][KEYWORD] OR emp_no{f}#8 > 1[INTEGER]] + * \_Join[LEFT,[language_code{r}#4],[language_code{r}#4],[language_code{f}#19]] + * |_EsqlProject[[_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, gender{f}#10, hire_date{f}#15, job{f}#16, job.raw{f}#17, lan + * guages{f}#11 AS language_code, last_name{f}#12, long_noidx{f}#18, salary{f}#13]] + * | \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..] + * \_EsRelation[language_code][LOOKUP][language_code{f}#19, language_name{f}#20] + */ + public void testLookupJoinPushDownDisabledForDisjunctionBetweenLeftAndRightField() { + String query = """ + FROM test + | RENAME languages AS language_code + | LOOKUP JOIN language_code ON language_code + | WHERE language_name == "English" OR emp_no > 1 + """; + + var plan = optimizedPlan(query); + + var project = as(plan, Project.class); + var limit = as(project.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(1000)); + + var filter = as(limit.child(), Filter.class); + var or = as(filter.condition(), Or.class); + EsqlBinaryComparison op = as(or.left(), Equals.class); + // OR left side + var field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("language_name")); + var literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(new BytesRef("English"))); + // OR right side + op = as(or.right(), GreaterThan.class); + field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("emp_no")); + literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(1)); + + var join = as(filter.child(), Join.class); + assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); + project = as(join.left(), Project.class); + + var leftRel = as(project.child(), EsRelation.class); + var rightRel = as(join.right(), EsRelation.class); + } + + // + // + // + public void testTranslateMetricsWithoutGrouping() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "METRICS k8s max(rate(network.total_bytes_in))"; From 42792812c17646413f6346c2b4acbac35dfef44f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3zala?= <377355+jozala@users.noreply.github.com> Date: Fri, 13 Dec 2024 22:49:52 +0100 Subject: [PATCH 24/24] [ci] Add oraclelinux-9 to matrix in packaging and platform jobs (#118565) --- .buildkite/pipelines/periodic-packaging.template.yml | 1 + .buildkite/pipelines/periodic-packaging.yml | 1 + .buildkite/pipelines/periodic-platform-support.yml | 1 + .buildkite/pipelines/pull-request/packaging-tests-unix.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml index b00ef51ec8f07..c42c999e08c65 100644 --- a/.buildkite/pipelines/periodic-packaging.template.yml +++ b/.buildkite/pipelines/periodic-packaging.template.yml @@ -10,6 +10,7 @@ steps: - debian-12 - opensuse-leap-15 - oraclelinux-8 + - oraclelinux-9 - sles-15 - ubuntu-2004 - ubuntu-2204 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 9619de3c2c98b..3c74806315922 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -11,6 +11,7 @@ steps: - debian-12 - opensuse-leap-15 - oraclelinux-8 + - oraclelinux-9 - sles-15 - ubuntu-2004 - ubuntu-2204 diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index 33e9a040b22cf..63d422da0f53f 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -10,6 +10,7 @@ steps: - debian-12 - opensuse-leap-15 - oraclelinux-8 + - oraclelinux-9 - sles-15 - ubuntu-2004 - ubuntu-2204 diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml index 3b7973d05a338..a4ad053953509 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml @@ -13,6 +13,7 @@ steps: - debian-12 - opensuse-leap-15 - oraclelinux-8 + - oraclelinux-9 - sles-15 - ubuntu-2004 - ubuntu-2204