diff --git a/docs/changelog/104132.yaml b/docs/changelog/104132.yaml new file mode 100644 index 0000000000000..87fe94ddcfcea --- /dev/null +++ b/docs/changelog/104132.yaml @@ -0,0 +1,5 @@ +pr: 104132 +summary: Add support for the `simple_query_string` to the Query API Key API +area: Security +type: enhancement +issues: [] diff --git a/docs/reference/rest-api/security/query-api-key.asciidoc b/docs/reference/rest-api/security/query-api-key.asciidoc index 67b0b7bfac58d..a08a8fd1858b6 100644 --- a/docs/reference/rest-api/security/query-api-key.asciidoc +++ b/docs/reference/rest-api/security/query-api-key.asciidoc @@ -54,7 +54,7 @@ The query supports a subset of query types, including <>, <>, <>, <>, <>, <>, <>, <>, -and <>. +<>, and <> + You can query the following public values associated with an API key. + @@ -92,9 +92,13 @@ Username of the API key owner. Realm name of the API key owner. `metadata`:: -Metadata field associated with the API key, such as `metadata.my_field`. Because -metadata is stored as a <> field type, all fields act like -`keyword` fields when querying and sorting. +Metadata field associated with the API key, such as `metadata.my_field`. +Metadata is internally indexed as a <> field type. +This means that all fields act like `keyword` fields when querying and sorting. +It's not possible to refer to a subset of metadata fields using wildcard +patterns, e.g. `metadata.field*`, even for query types that support field +name patterns. Lastly, all the metadata fields can be searched together when +simply mentioning `metadata` (not followed by any dot and sub-field name). NOTE: You cannot query the role descriptors of an API key. ==== diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java index 18d9dcdc822e5..e552befc267c8 100644 --- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java @@ -32,6 +32,7 @@ import java.util.stream.IntStream; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; @@ -130,8 +131,9 @@ public void testQuery() throws IOException { }); // Search for fields outside of the allowlist fails - assertQueryError(API_KEY_ADMIN_AUTH_HEADER, 400, """ + ResponseException responseException = assertQueryError(API_KEY_ADMIN_AUTH_HEADER, 400, """ { "query": { "prefix": {"api_key_hash": "{PBKDF2}10000$"} } }"""); + assertThat(responseException.getMessage(), containsString("Field [api_key_hash] is not allowed for API Key query")); // Search for fields that are not allowed in Query DSL but used internally by the service itself final String fieldName = randomFrom("doc_type", "api_key_invalidated", "invalidation_time"); @@ -429,6 +431,130 @@ public void testSort() throws IOException { assertQueryError(authHeader, 400, "{\"sort\":[\"" + invalidFieldName + "\"]}"); } + public void testSimpleQueryStringQuery() throws IOException { + String batmanUserCredentials = createUser("batman", new String[] { "api_key_user_role" }); + final List apiKeyIds = new ArrayList<>(); + apiKeyIds.add(createApiKey("key1-user", null, null, Map.of("label", "prod"), API_KEY_USER_AUTH_HEADER).v1()); + apiKeyIds.add(createApiKey("key1-admin", null, null, Map.of("label", "prod"), API_KEY_ADMIN_AUTH_HEADER).v1()); + apiKeyIds.add(createApiKey("key2-user", null, null, Map.of("value", 42, "label", "prod"), API_KEY_USER_AUTH_HEADER).v1()); + apiKeyIds.add(createApiKey("key2-admin", null, null, Map.of("value", 42, "label", "prod"), API_KEY_ADMIN_AUTH_HEADER).v1()); + apiKeyIds.add(createApiKey("key3-user", null, null, Map.of("value", 42, "hero", true), API_KEY_USER_AUTH_HEADER).v1()); + apiKeyIds.add(createApiKey("key3-admin", null, null, Map.of("value", 42, "hero", true), API_KEY_ADMIN_AUTH_HEADER).v1()); + apiKeyIds.add(createApiKey("key4-batman", null, null, Map.of("hero", true), batmanUserCredentials).v1()); + apiKeyIds.add(createApiKey("key5-batman", null, null, Map.of("hero", true), batmanUserCredentials).v1()); + + assertQuery( + API_KEY_ADMIN_AUTH_HEADER, + """ + {"query": {"simple_query_string": {"query": "key*", "fields": ["no_such_field_pattern*"]}}}""", + apiKeys -> assertThat(apiKeys, is(empty())) + ); + assertQuery( + API_KEY_ADMIN_AUTH_HEADER, + """ + {"query": {"simple_query_string": {"query": "prod 42 true", "fields": ["metadata.*"]}}}""", + apiKeys -> assertThat(apiKeys, is(empty())) + ); + // disallowed fields are silently ignored for the simple query string query type + assertQuery( + API_KEY_ADMIN_AUTH_HEADER, + """ + {"query": {"simple_query_string": {"query": "ke*", "fields": ["x*", "api_key_hash"]}}}""", + apiKeys -> assertThat(apiKeys, is(empty())) + ); + assertQuery( + API_KEY_ADMIN_AUTH_HEADER, + """ + {"query": {"simple_query_string": {"query": "prod 42 true", "fields": ["wild*", "metadata"]}}}""", + apiKeys -> assertThat(apiKeys.stream().map(k -> (String) k.get("id")).toList(), containsInAnyOrder(apiKeyIds.toArray())) + ); + assertQuery( + API_KEY_ADMIN_AUTH_HEADER, + """ + {"query": {"simple_query_string": {"query": "key* +rest" }}}""", + apiKeys -> assertThat(apiKeys.stream().map(k -> (String) k.get("id")).toList(), containsInAnyOrder(apiKeyIds.toArray())) + ); + assertQuery( + API_KEY_ADMIN_AUTH_HEADER, + """ + {"query": {"simple_query_string": {"query": "-prod", "fields": ["metadata"]}}}""", + apiKeys -> assertThat( + apiKeys.stream().map(k -> (String) k.get("id")).toList(), + containsInAnyOrder(apiKeyIds.get(4), apiKeyIds.get(5), apiKeyIds.get(6), apiKeyIds.get(7)) + ) + ); + assertQuery( + API_KEY_ADMIN_AUTH_HEADER, + """ + {"query": {"simple_query_string": {"query": "-42", "fields": ["meta*", "whatever*"]}}}""", + apiKeys -> assertThat( + apiKeys.stream().map(k -> (String) k.get("id")).toList(), + containsInAnyOrder(apiKeyIds.get(0), apiKeyIds.get(1), apiKeyIds.get(6), apiKeyIds.get(7)) + ) + ); + assertQuery( + API_KEY_ADMIN_AUTH_HEADER, + """ + {"query": {"simple_query_string": {"query": "-rest term_which_does_not_exist"}}}""", + apiKeys -> assertThat(apiKeys, is(empty())) + ); + assertQuery( + API_KEY_ADMIN_AUTH_HEADER, + """ + {"query": {"simple_query_string": {"query": "+default_file +api_key_user", "fields": ["us*", "rea*"]}}}""", + apiKeys -> assertThat( + apiKeys.stream().map(k -> (String) k.get("id")).toList(), + containsInAnyOrder(apiKeyIds.get(0), apiKeyIds.get(2), apiKeyIds.get(4)) + ) + ); + assertQuery( + API_KEY_ADMIN_AUTH_HEADER, + """ + {"query": {"simple_query_string": {"query": "default_fie~4", "fields": ["*"]}}}""", + apiKeys -> assertThat( + apiKeys.stream().map(k -> (String) k.get("id")).toList(), + containsInAnyOrder( + apiKeyIds.get(0), + apiKeyIds.get(1), + apiKeyIds.get(2), + apiKeyIds.get(3), + apiKeyIds.get(4), + apiKeyIds.get(5) + ) + ) + ); + assertQuery( + API_KEY_ADMIN_AUTH_HEADER, + """ + {"query": {"simple_query_string": {"query": "+prod +42", + "fields": ["metadata.label", "metadata.value", "metadata.hero"]}}}""", + apiKeys -> assertThat( + apiKeys.stream().map(k -> (String) k.get("id")).toList(), + containsInAnyOrder(apiKeyIds.get(2), apiKeyIds.get(3)) + ) + ); + assertQuery(batmanUserCredentials, """ + {"query": {"simple_query_string": {"query": "+prod key*", "fields": ["name", "username", "metadata"], + "default_operator": "AND"}}}""", apiKeys -> assertThat(apiKeys, is(empty()))); + assertQuery( + batmanUserCredentials, + """ + {"query": {"simple_query_string": {"query": "+true +key*", "fields": ["name", "username", "metadata"], + "default_operator": "AND"}}}""", + apiKeys -> assertThat( + apiKeys.stream().map(k -> (String) k.get("id")).toList(), + containsInAnyOrder(apiKeyIds.get(6), apiKeyIds.get(7)) + ) + ); + assertQuery( + batmanUserCredentials, + """ + {"query": {"bool": {"must": [{"term": {"name": {"value":"key5-batman"}}}, + {"simple_query_string": {"query": "default_native"}}]}}}""", + apiKeys -> assertThat(apiKeys.stream().map(k -> (String) k.get("id")).toList(), containsInAnyOrder(apiKeyIds.get(7))) + ); + } + public void testExistsQuery() throws IOException, InterruptedException { final String authHeader = randomFrom(API_KEY_ADMIN_AUTH_HEADER, API_KEY_USER_AUTH_HEADER); @@ -530,12 +656,13 @@ private int collectApiKeys(List> apiKeyInfos, Request reques return actualSize; } - private void assertQueryError(String authHeader, int statusCode, String body) throws IOException { + private ResponseException assertQueryError(String authHeader, int statusCode, String body) throws IOException { final Request request = new Request("GET", "/_security/_query/api_key"); request.setJsonEntity(body); request.setOptions(request.getOptions().toBuilder().addHeader(HttpHeaders.AUTHORIZATION, authHeader)); final ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(statusCode)); + return responseException; } private void assertQuery(String authHeader, String body, Consumer>> apiKeysVerifier) throws IOException { diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index 0d5a757f65084..3833a6466c67c 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -732,6 +732,8 @@ public void testQueryCrossClusterApiKeysByType() throws IOException { for (String restTypeQuery : List.of(""" {"query": {"term": {"type": "rest" }}}""", """ {"query": {"bool": {"must_not": {"term": {"type": "cross_cluster"}}}}}""", """ + {"query": {"simple_query_string": {"query": "re* rest -cross_cluster", "fields": ["ty*"]}}}""", """ + {"query": {"simple_query_string": {"query": "-cross*", "fields": ["type"]}}}""", """ {"query": {"prefix": {"type": "re" }}}""", """ {"query": {"wildcard": {"type": "r*t" }}}""", """ {"query": {"range": {"type": {"gte": "raaa", "lte": "rzzz"}}}}""")) { @@ -747,6 +749,8 @@ public void testQueryCrossClusterApiKeysByType() throws IOException { for (String crossClusterTypeQuery : List.of(""" {"query": {"term": {"type": "cross_cluster" }}}""", """ {"query": {"bool": {"must_not": {"term": {"type": "rest"}}}}}""", """ + {"query": {"simple_query_string": {"query": "cro* cross_cluster -re*", "fields": ["ty*"]}}}""", """ + {"query": {"simple_query_string": {"query": "-re*", "fields": ["type"]}}}""", """ {"query": {"prefix": {"type": "cro" }}}""", """ {"query": {"wildcard": {"type": "*oss_*er" }}}""", """ {"query": {"range": {"type": {"gte": "cross", "lte": "zzzz"}}}}""")) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java index 5cb6573c8b5dc..9f7b84e4a2698 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java @@ -13,20 +13,25 @@ import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.PrefixQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.index.query.SimpleQueryStringBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.index.search.QueryParserHelper; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.security.authc.ApiKeyService; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import java.util.function.Consumer; @@ -45,6 +50,7 @@ public class ApiKeyBoolQueryBuilder extends BoolQueryBuilder { "invalidation_time", "creation_time", "expiration_time", + "metadata_flattened", "creator.principal", "creator.realm" ); @@ -160,6 +166,36 @@ private static QueryBuilder doProcess(QueryBuilder qb, Consumer fieldNam newQuery.to(query.to()).includeUpper(query.includeUpper()); } return newQuery.boost(query.boost()); + } else if (qb instanceof final SimpleQueryStringBuilder simpleQueryStringBuilder) { + if (simpleQueryStringBuilder.fields().isEmpty()) { + simpleQueryStringBuilder.field("*"); + } + // override lenient if querying all the fields, because, due to different field mappings, + // the query parsing will almost certainly fail otherwise + if (QueryParserHelper.hasAllFieldsWildcard(simpleQueryStringBuilder.fields().keySet())) { + simpleQueryStringBuilder.lenient(true); + } + Map requestedFields = new HashMap<>(simpleQueryStringBuilder.fields()); + simpleQueryStringBuilder.fields().clear(); + for (Map.Entry requestedFieldNameOrPattern : requestedFields.entrySet()) { + for (String translatedField : ApiKeyFieldNameTranslators.translatePattern(requestedFieldNameOrPattern.getKey())) { + simpleQueryStringBuilder.fields() + .compute( + translatedField, + (k, v) -> (v == null) ? requestedFieldNameOrPattern.getValue() : v * requestedFieldNameOrPattern.getValue() + ); + fieldNameVisitor.accept(translatedField); + } + } + if (simpleQueryStringBuilder.fields().isEmpty()) { + // A SimpleQueryStringBuilder with empty fields() will eventually produce a SimpleQueryString query + // that accesses all the fields, including disallowed ones. + // Instead, the behavior we're after is that a query that accesses only disallowed fields should + // not match any docs. + return new MatchNoneQueryBuilder(); + } else { + return simpleQueryStringBuilder; + } } else { throw new IllegalArgumentException("Query type [" + qb.getName() + "] is not supported for API Key query"); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyFieldNameTranslators.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyFieldNameTranslators.java index c204ec031b18c..29bf3ca5dd045 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyFieldNameTranslators.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyFieldNameTranslators.java @@ -7,7 +7,11 @@ package org.elasticsearch.xpack.security.support; +import org.elasticsearch.common.regex.Regex; + +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.function.Function; import static org.elasticsearch.xpack.security.action.apikey.TransportQueryApiKeyAction.API_KEY_TYPE_RUNTIME_MAPPING_FIELD; @@ -22,13 +26,15 @@ public class ApiKeyFieldNameTranslators { FIELD_NAME_TRANSLATORS = List.of( new ExactFieldNameTranslator(s -> "creator.principal", "username"), new ExactFieldNameTranslator(s -> "creator.realm", "realm_name"), - new ExactFieldNameTranslator(Function.identity(), "name"), + new ExactFieldNameTranslator(s -> "name", "name"), new ExactFieldNameTranslator(s -> API_KEY_TYPE_RUNTIME_MAPPING_FIELD, "type"), new ExactFieldNameTranslator(s -> "creation_time", "creation"), new ExactFieldNameTranslator(s -> "expiration_time", "expiration"), new ExactFieldNameTranslator(s -> "api_key_invalidated", "invalidated"), new ExactFieldNameTranslator(s -> "invalidation_time", "invalidation"), - new PrefixFieldNameTranslator(s -> "metadata_flattened" + s.substring(8), "metadata.") + // allows querying on all metadata values as keywords because "metadata_flattened" is a flattened field type + new ExactFieldNameTranslator(s -> "metadata_flattened", "metadata"), + new PrefixFieldNameTranslator(s -> "metadata_flattened." + s.substring("metadata.".length()), "metadata.") ); } @@ -37,6 +43,9 @@ public class ApiKeyFieldNameTranslators { * It throws an exception if the field name is not explicitly allowed. */ public static String translate(String fieldName) { + if (Regex.isSimpleMatchPattern(fieldName)) { + throw new IllegalArgumentException("Field name pattern [" + fieldName + "] is not allowed for API Key query"); + } for (FieldNameTranslator translator : FIELD_NAME_TRANSLATORS) { if (translator.supports(fieldName)) { return translator.translate(fieldName); @@ -45,6 +54,25 @@ public static String translate(String fieldName) { throw new IllegalArgumentException("Field [" + fieldName + "] is not allowed for API Key query"); } + /** + * Translates a query level field name pattern to the matching index level field names. + * The result can be the empty set, if the pattern doesn't match any of the allowed index level field names. + * If the pattern is actually a concrete field name rather than a pattern, + * it is also translated, but only if the query level field name is allowed, otherwise an exception is thrown. + */ + public static Set translatePattern(String fieldNameOrPattern) { + Set indexFieldNames = new HashSet<>(); + for (FieldNameTranslator translator : FIELD_NAME_TRANSLATORS) { + if (translator.supports(fieldNameOrPattern)) { + indexFieldNames.add(translator.translate(fieldNameOrPattern)); + } + } + // It's OK to "translate" to the empty set the concrete disallowed or unknown field names, because + // the SimpleQueryString query type is lenient in the sense that it ignores unknown fields and field name patterns, + // so this preprocessing can ignore them too. + return indexFieldNames; + } + abstract static class FieldNameTranslator { private final Function translationFunc; @@ -69,8 +97,12 @@ static class ExactFieldNameTranslator extends FieldNameTranslator { } @Override - public boolean supports(String fieldName) { - return name.equals(fieldName); + public boolean supports(String fieldNameOrPattern) { + if (Regex.isSimpleMatchPattern(fieldNameOrPattern)) { + return Regex.simpleMatch(fieldNameOrPattern, name); + } else { + return name.equals(fieldNameOrPattern); + } } } @@ -83,8 +115,16 @@ static class PrefixFieldNameTranslator extends FieldNameTranslator { } @Override - boolean supports(String fieldName) { - return fieldName.startsWith(prefix); + boolean supports(String fieldNamePrefix) { + // a pattern can generally match a prefix in multiple ways + // moreover, it's not possible to iterate the concrete fields matching the prefix + if (Regex.isSimpleMatchPattern(fieldNamePrefix)) { + // this means that e.g. `metadata.*` and `metadata.x*` are expanded to the empty list, + // rather than be replaced with `metadata_flattened.*` and `metadata_flattened.x*` + // (but, in any case, `metadata_flattened.*` and `metadata.x*` are going to be ignored) + return false; + } + return fieldNamePrefix.startsWith(prefix); } } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java index 235657a30e11f..4064d9f0ce4da 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java @@ -12,12 +12,14 @@ import org.elasticsearch.index.query.DistanceFeatureQueryBuilder; import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.MultiTermQueryBuilder; import org.elasticsearch.index.query.PrefixQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.index.query.SimpleQueryStringBuilder; import org.elasticsearch.index.query.SpanQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; @@ -42,6 +44,7 @@ import static org.elasticsearch.test.LambdaMatchers.falseWith; import static org.elasticsearch.test.LambdaMatchers.trueWith; import static org.elasticsearch.xpack.security.support.ApiKeyFieldNameTranslators.FIELD_NAME_TRANSLATORS; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -162,10 +165,10 @@ public void testFieldNameTranslation() { // metadata { - final List queryFields = new ArrayList<>(); + List queryFields = new ArrayList<>(); final String metadataKey = randomAlphaOfLengthBetween(3, 8); final TermQueryBuilder q1 = QueryBuilders.termQuery("metadata." + metadataKey, randomAlphaOfLengthBetween(3, 8)); - final ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(q1, queryFields::add, authentication); + ApiKeyBoolQueryBuilder apiKeyQb1 = ApiKeyBoolQueryBuilder.build(q1, queryFields::add, authentication); assertThat(queryFields, hasItem("doc_type")); assertThat(queryFields, hasItem("metadata_flattened." + metadataKey)); if (authentication != null && authentication.isApiKey() == false) { @@ -174,6 +177,22 @@ public void testFieldNameTranslation() { } assertCommonFilterQueries(apiKeyQb1, authentication); assertThat(apiKeyQb1.must().get(0), equalTo(QueryBuilders.termQuery("metadata_flattened." + metadataKey, q1.value()))); + + queryFields = new ArrayList<>(); + String queryStringQuery = randomAlphaOfLength(8); + SimpleQueryStringBuilder q2 = QueryBuilders.simpleQueryStringQuery(queryStringQuery).field("metadata"); + apiKeyQb1 = ApiKeyBoolQueryBuilder.build(q2, queryFields::add, authentication); + assertThat(queryFields, hasItem("doc_type")); + assertThat(queryFields, hasItem("metadata_flattened")); + if (authentication != null && authentication.isApiKey() == false) { + assertThat(queryFields, hasItem("creator.principal")); + assertThat(queryFields, hasItem("creator.realm")); + } + assertCommonFilterQueries(apiKeyQb1, authentication); + assertThat( + apiKeyQb1.must().get(0), + equalTo(QueryBuilders.simpleQueryStringQuery(queryStringQuery).field("metadata_flattened")) + ); } // username @@ -233,6 +252,70 @@ public void testFieldNameTranslation() { assertCommonFilterQueries(apiKeyQb5, authentication); assertThat(apiKeyQb5.must().get(0), equalTo(QueryBuilders.termQuery("expiration_time", q5.value()))); } + + // type + { + final List queryFields = new ArrayList<>(); + float fieldBoost = randomFloat(); + final SimpleQueryStringBuilder q5 = QueryBuilders.simpleQueryStringQuery("q=42").field("type", fieldBoost); + final ApiKeyBoolQueryBuilder apiKeyQb5 = ApiKeyBoolQueryBuilder.build(q5, queryFields::add, authentication); + assertThat(queryFields, hasItem("doc_type")); + assertThat(queryFields, hasItem("runtime_key_type")); // "type" translation + if (authentication != null && authentication.isApiKey() == false) { + assertThat(queryFields, hasItem("creator.principal")); + assertThat(queryFields, hasItem("creator.realm")); + } + assertCommonFilterQueries(apiKeyQb5, authentication); + assertThat( + apiKeyQb5.must().get(0), + equalTo(QueryBuilders.simpleQueryStringQuery("q=42").field("runtime_key_type", fieldBoost)) + ); + } + + // test them all together + { + final List queryFields = new ArrayList<>(); + final SimpleQueryStringBuilder q6 = QueryBuilders.simpleQueryStringQuery("+OK -NOK maybe~3") + .field("username") + .field("realm_name") + .field("name") + .field("type") + .field("creation") + .field("expiration") + .field("invalidated") + .field("invalidation") + .field("metadata") + .field("metadata.inner"); + final ApiKeyBoolQueryBuilder apiKeyQb6 = ApiKeyBoolQueryBuilder.build(q6, queryFields::add, authentication); + assertThat(queryFields, hasItem("doc_type")); + assertThat(queryFields, hasItem("creator.principal")); + assertThat(queryFields, hasItem("creator.realm")); + assertThat(queryFields, hasItem("name")); + assertThat(queryFields, hasItem("runtime_key_type")); // "type" translation + assertThat(queryFields, hasItem("creation_time")); + assertThat(queryFields, hasItem("expiration_time")); + assertThat(queryFields, hasItem("api_key_invalidated")); + assertThat(queryFields, hasItem("invalidation_time")); + assertThat(queryFields, hasItem("metadata_flattened")); + assertThat(queryFields, hasItem("metadata_flattened.inner")); + assertCommonFilterQueries(apiKeyQb6, authentication); + assertThat( + apiKeyQb6.must().get(0), + equalTo( + QueryBuilders.simpleQueryStringQuery("+OK -NOK maybe~3") + .field("creator.principal") + .field("creator.realm") + .field("name") + .field("runtime_key_type") + .field("creation_time") + .field("expiration_time") + .field("api_key_invalidated") + .field("invalidation_time") + .field("metadata_flattened") + .field("metadata_flattened.inner") + ) + ); + } } public void testAllowListOfFieldNames() { @@ -254,16 +337,48 @@ public void testAllowListOfFieldNames() { "creator.metadata" ); - final QueryBuilder q1 = randomValueOtherThanMany( - q -> q.getClass() == IdsQueryBuilder.class || q.getClass() == MatchAllQueryBuilder.class, - () -> randomSimpleQuery(fieldName) - ); - final IllegalArgumentException e1 = expectThrows( - IllegalArgumentException.class, - () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication) - ); + { + final QueryBuilder q1 = randomValueOtherThanMany( + q -> q.getClass() == IdsQueryBuilder.class + || q.getClass() == MatchAllQueryBuilder.class + || q.getClass() == SimpleQueryStringBuilder.class, + () -> randomSimpleQuery(fieldName) + ); + final IllegalArgumentException e1 = expectThrows( + IllegalArgumentException.class, + () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication) + ); + assertThat(e1.getMessage(), containsString("Field [" + fieldName + "] is not allowed for API Key query")); + } - assertThat(e1.getMessage(), containsString("Field [" + fieldName + "] is not allowed for API Key query")); + // also wrapped in a boolean query + { + final QueryBuilder q1 = randomValueOtherThanMany( + q -> q.getClass() == IdsQueryBuilder.class + || q.getClass() == MatchAllQueryBuilder.class + || q.getClass() == SimpleQueryStringBuilder.class, + () -> randomSimpleQuery(fieldName) + ); + final BoolQueryBuilder q2 = QueryBuilders.boolQuery(); + if (randomBoolean()) { + if (randomBoolean()) { + q2.filter(q1); + } else { + q2.must(q1); + } + } else { + if (randomBoolean()) { + q2.should(q1); + } else { + q2.mustNot(q1); + } + } + IllegalArgumentException e2 = expectThrows( + IllegalArgumentException.class, + () -> ApiKeyBoolQueryBuilder.build(q2, ignored -> {}, authentication) + ); + assertThat(e2.getMessage(), containsString("Field [" + fieldName + "] is not allowed for API Key query")); + } } public void testTermsLookupIsNotAllowed() { @@ -294,7 +409,6 @@ public void testDisallowedQueryTypes() { QueryBuilders.constantScoreQuery(mock(QueryBuilder.class)), QueryBuilders.boostingQuery(mock(QueryBuilder.class), mock(QueryBuilder.class)), QueryBuilders.queryStringQuery("q=a:42"), - QueryBuilders.simpleQueryStringQuery(randomAlphaOfLength(5)), QueryBuilders.combinedFieldsQuery(randomAlphaOfLength(5)), QueryBuilders.disMaxQuery(), QueryBuilders.distanceFeatureQuery( @@ -332,6 +446,29 @@ public void testDisallowedQueryTypes() { () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication) ); assertThat(e1.getMessage(), containsString("Query type [" + q1.getName() + "] is not supported for API Key query")); + + // also wrapped in a boolean query + { + final BoolQueryBuilder q2 = QueryBuilders.boolQuery(); + if (randomBoolean()) { + if (randomBoolean()) { + q2.filter(q1); + } else { + q2.must(q1); + } + } else { + if (randomBoolean()) { + q2.should(q1); + } else { + q2.mustNot(q1); + } + } + IllegalArgumentException e2 = expectThrows( + IllegalArgumentException.class, + () -> ApiKeyBoolQueryBuilder.build(q2, ignored -> {}, authentication) + ); + assertThat(e2.getMessage(), containsString("Query type [" + q1.getName() + "] is not supported for API Key query")); + } } public void testWillSetAllowedFields() throws IOException { @@ -378,6 +515,222 @@ public void testWillFilterForApiKeyId() { assertThat(apiKeyQb.filter(), hasItem(QueryBuilders.idsQuery().addIds(apiKeyId))); } + public void testSimpleQueryStringFieldPatternTranslation() { + String queryStringQuery = randomAlphaOfLength(8); + Authentication authentication = randomBoolean() ? AuthenticationTests.randomAuthentication(null, null) : null; + // no field translates to all the allowed fields + { + List queryFields = new ArrayList<>(); + SimpleQueryStringBuilder q = QueryBuilders.simpleQueryStringQuery(queryStringQuery); + ApiKeyBoolQueryBuilder apiKeyQb = ApiKeyBoolQueryBuilder.build(q, queryFields::add, authentication); + assertThat( + queryFields.subList(0, 9), + containsInAnyOrder( + "creator.principal", + "creator.realm", + "name", + "runtime_key_type", + "creation_time", + "expiration_time", + "api_key_invalidated", + "invalidation_time", + "metadata_flattened" + ) + ); + assertThat(queryFields.get(9), is("doc_type")); + assertThat( + apiKeyQb.must().get(0), + equalTo( + QueryBuilders.simpleQueryStringQuery(queryStringQuery) + .field("creator.principal") + .field("creator.realm") + .field("name") + .field("runtime_key_type") + .field("creation_time") + .field("expiration_time") + .field("api_key_invalidated") + .field("invalidation_time") + .field("metadata_flattened") + .lenient(true) + ) + ); + } + // * matches all fields + { + List queryFields = new ArrayList<>(); + float fieldBoost = Math.abs(randomFloat()); + SimpleQueryStringBuilder q = QueryBuilders.simpleQueryStringQuery(queryStringQuery).field("*", fieldBoost); + ApiKeyBoolQueryBuilder apiKeyQb = ApiKeyBoolQueryBuilder.build(q, queryFields::add, authentication); + assertThat( + queryFields.subList(0, 9), + containsInAnyOrder( + "creator.principal", + "creator.realm", + "name", + "runtime_key_type", + "creation_time", + "expiration_time", + "api_key_invalidated", + "invalidation_time", + "metadata_flattened" + ) + ); + assertThat(queryFields.get(9), is("doc_type")); + assertThat( + apiKeyQb.must().get(0), + equalTo( + QueryBuilders.simpleQueryStringQuery(queryStringQuery) + .field("creator.principal", fieldBoost) + .field("creator.realm", fieldBoost) + .field("name", fieldBoost) + .field("runtime_key_type", fieldBoost) + .field("creation_time", fieldBoost) + .field("expiration_time", fieldBoost) + .field("api_key_invalidated", fieldBoost) + .field("invalidation_time", fieldBoost) + .field("metadata_flattened", fieldBoost) + .lenient(true) + ) + ); + } + // pattern that matches a subset of fields + { + List queryFields = new ArrayList<>(); + float fieldBoost = Math.abs(randomFloat()); + boolean lenient = randomBoolean(); + SimpleQueryStringBuilder q = QueryBuilders.simpleQueryStringQuery(queryStringQuery).field("i*", fieldBoost).lenient(lenient); + ApiKeyBoolQueryBuilder apiKeyQb = ApiKeyBoolQueryBuilder.build(q, queryFields::add, authentication); + assertThat(queryFields.subList(0, 2), containsInAnyOrder("api_key_invalidated", "invalidation_time")); + assertThat(queryFields.get(2), is("doc_type")); + assertThat( + apiKeyQb.must().get(0), + equalTo( + QueryBuilders.simpleQueryStringQuery(queryStringQuery) + .field("api_key_invalidated", fieldBoost) + .field("invalidation_time", fieldBoost) + .lenient(lenient) + ) + ); + } + // multi pattern that matches a subset of fields + { + List queryFields = new ArrayList<>(); + float boost1 = randomFrom(2.0f, 4.0f, 8.0f); + float boost2 = randomFrom(2.0f, 4.0f, 8.0f); + float boost3 = randomFrom(2.0f, 4.0f, 8.0f); + SimpleQueryStringBuilder q = QueryBuilders.simpleQueryStringQuery(queryStringQuery) + .field("i*", boost1) + .field("u*", boost2) + .field("user*", boost3); + ApiKeyBoolQueryBuilder apiKeyQb = ApiKeyBoolQueryBuilder.build(q, queryFields::add, authentication); + assertThat(queryFields.subList(0, 3), containsInAnyOrder("creator.principal", "api_key_invalidated", "invalidation_time")); + assertThat(queryFields.get(4), is("doc_type")); + assertThat( + apiKeyQb.must().get(0), + equalTo( + QueryBuilders.simpleQueryStringQuery(queryStringQuery) + .field("api_key_invalidated", boost1) + .field("invalidation_time", boost1) + .field("creator.principal", boost2 * boost3) + .lenient(false) + ) + ); + + // wildcards don't expand under metadata.* + queryFields = new ArrayList<>(); + q = QueryBuilders.simpleQueryStringQuery(queryStringQuery) + .field("rea*", boost1) + .field("t*", boost1) + .field("ty*", boost2) + .field("me*", boost2) + .field("metadata.*", boost3) + .field("metadata.x*", boost3); + apiKeyQb = ApiKeyBoolQueryBuilder.build(q, queryFields::add, authentication); + assertThat( + queryFields.subList(0, 4), + containsInAnyOrder("creator.realm", "runtime_key_type", "metadata_flattened", "runtime_key_type") + ); + assertThat(queryFields.get(4), is("doc_type")); + assertThat( + apiKeyQb.must().get(0), + equalTo( + QueryBuilders.simpleQueryStringQuery(queryStringQuery) + .field("creator.realm", boost1) + .field("runtime_key_type", boost1 * boost2) + .field("metadata_flattened", boost2) + .lenient(false) + ) + ); + } + // patterns that don't match anything + { + List queryFields = new ArrayList<>(); + float boost1 = randomFrom(2.0f, 4.0f, 8.0f); + float boost2 = randomFrom(2.0f, 4.0f, 8.0f); + float boost3 = randomFrom(2.0f, 4.0f, 8.0f); + SimpleQueryStringBuilder q = QueryBuilders.simpleQueryStringQuery(queryStringQuery) + .field("field_that_does_not*", boost1) + .field("what*", boost2) + .field("aiaiaiai*", boost3); + ApiKeyBoolQueryBuilder apiKeyQb = ApiKeyBoolQueryBuilder.build(q, queryFields::add, authentication); + assertThat(queryFields.get(0), is("doc_type")); + if (authentication != null) { + assertThat(queryFields.get(1), is("creator.principal")); + assertThat(queryFields.get(2), is("creator.realm")); + assertThat(queryFields.size(), is(3)); + } else { + assertThat(queryFields.size(), is(1)); + } + assertThat(apiKeyQb.must().get(0), equalTo(new MatchNoneQueryBuilder())); + } + // disallowed or unknown field is silently ignored + { + List queryFields = new ArrayList<>(); + float boost1 = randomFrom(2.0f, 4.0f, 8.0f); + float boost2 = randomFrom(2.0f, 4.0f, 8.0f); + SimpleQueryStringBuilder q = QueryBuilders.simpleQueryStringQuery(queryStringQuery) + .field("field_that_does_not*", boost1) + .field("unknown_field", boost2); + ApiKeyBoolQueryBuilder apiKeyQb = ApiKeyBoolQueryBuilder.build(q, queryFields::add, authentication); + assertThat(queryFields.get(0), is("doc_type")); + if (authentication != null) { + assertThat(queryFields.get(1), is("creator.principal")); + assertThat(queryFields.get(2), is("creator.realm")); + assertThat(queryFields.size(), is(3)); + } else { + assertThat(queryFields.size(), is(1)); + } + assertThat(apiKeyQb.must().get(0), equalTo(new MatchNoneQueryBuilder())); + + // translated field + queryFields = new ArrayList<>(); + String translatedField = randomFrom( + "creator.principal", + "creator.realm", + "runtime_key_type", + "creation_time", + "expiration_time", + "api_key_invalidated", + "invalidation_time", + "metadata_flattened" + ); + SimpleQueryStringBuilder q2 = QueryBuilders.simpleQueryStringQuery(queryStringQuery) + .field(translatedField, boost1) + .field("field_that_does_not*", boost2); + apiKeyQb = ApiKeyBoolQueryBuilder.build(q2, queryFields::add, authentication); + assertThat(queryFields.get(0), is("doc_type")); + if (authentication != null) { + assertThat(queryFields.get(1), is("creator.principal")); + assertThat(queryFields.get(2), is("creator.realm")); + assertThat(queryFields.size(), is(3)); + } else { + assertThat(queryFields.size(), is(1)); + } + + assertThat(apiKeyQb.must().get(0), equalTo(new MatchNoneQueryBuilder())); + } + } + private void testAllowedIndexFieldName(Predicate predicate) { final String allowedField = randomFrom( "doc_type", @@ -419,18 +772,23 @@ private void assertCommonFilterQueries(ApiKeyBoolQueryBuilder qb, Authentication ); } - private QueryBuilder randomSimpleQuery(String name) { - return switch (randomIntBetween(0, 7)) { - case 0 -> QueryBuilders.termQuery(name, randomAlphaOfLengthBetween(3, 8)); - case 1 -> QueryBuilders.termsQuery(name, randomArray(1, 3, String[]::new, () -> randomAlphaOfLengthBetween(3, 8))); + private QueryBuilder randomSimpleQuery(String fieldName) { + return switch (randomIntBetween(0, 8)) { + case 0 -> QueryBuilders.termQuery(fieldName, randomAlphaOfLengthBetween(3, 8)); + case 1 -> QueryBuilders.termsQuery(fieldName, randomArray(1, 3, String[]::new, () -> randomAlphaOfLengthBetween(3, 8))); case 2 -> QueryBuilders.idsQuery().addIds(randomArray(1, 3, String[]::new, () -> randomAlphaOfLength(22))); - case 3 -> QueryBuilders.prefixQuery(name, "prod-"); - case 4 -> QueryBuilders.wildcardQuery(name, "prod-*-east-*"); + case 3 -> QueryBuilders.prefixQuery(fieldName, "prod-"); + case 4 -> QueryBuilders.wildcardQuery(fieldName, "prod-*-east-*"); case 5 -> QueryBuilders.matchAllQuery(); - case 6 -> QueryBuilders.existsQuery(name); - default -> QueryBuilders.rangeQuery(name) + case 6 -> QueryBuilders.existsQuery(fieldName); + case 7 -> QueryBuilders.rangeQuery(fieldName) .from(Instant.now().minus(1, ChronoUnit.DAYS).toEpochMilli(), randomBoolean()) .to(Instant.now().toEpochMilli(), randomBoolean()); + case 8 -> QueryBuilders.simpleQueryStringQuery("+rest key*") + .field(fieldName) + .lenient(randomBoolean()) + .analyzeWildcard(randomBoolean()); + default -> throw new IllegalStateException("illegal switch case"); }; }