Add support for the simple_query_string to the Query API Key API (#…

…104132)

This adds support for the simple_query_string query type to the Query API key Information API.
In addition, this also adds support for querying all the API Key metadata fields simultaneously,
rather than requiring each to be specified, such as metadata.x, metadata.y, etc.

Relates: #101691

docs/changelog/104132.yaml

@@ -0,0 +1,5 @@
+pr: 104132
+summary: Add support for the `simple_query_string` to the Query API Key API
+area: Security
+type: enhancement
+issues: []

docs/reference/rest-api/security/query-api-key.asciidoc

@@ -54,7 +54,7 @@ The query supports a subset of query types, including
 <<query-dsl-match-all-query,`match_all`>>, <<query-dsl-bool-query,`bool`>>,
 <<query-dsl-term-query,`term`>>, <<query-dsl-terms-query,`terms`>>, <<query-dsl-ids-query,`ids`>>,
 <<query-dsl-prefix-query,`prefix`>>, <<query-dsl-wildcard-query,`wildcard`>>, <<query-dsl-exists-query,`exists`>>,
-and <<query-dsl-range-query,`range`>>.
+<<query-dsl-range-query,`range`>>, and <<query-dsl-simple-query-string-query,`simple query string`>>
 +
 You can query the following public values associated with an API key.
 +
@@ -92,9 +92,13 @@ Username of the API key owner.
 Realm name of the API key owner.
 
 `metadata`::
-Metadata field associated with the API key, such as `metadata.my_field`. Because
-metadata is stored as a <<flattened,flattened>> field type, all fields act like
-`keyword` fields when querying and sorting.
+Metadata field associated with the API key, such as `metadata.my_field`.
+Metadata is internally indexed as a <<flattened,flattened>> field type.
+This means that all fields act like `keyword` fields when querying and sorting.
+It's not possible to refer to a subset of metadata fields using wildcard
+patterns, e.g. `metadata.field*`, even for query types that support field
+name patterns. Lastly, all the metadata fields can be searched together when
+simply mentioning `metadata` (not followed by any dot and sub-field name).
 
 NOTE: You cannot query the role descriptors of an API key.
 ====

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java

@@ -32,6 +32,7 @@
 import java.util.stream.IntStream;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
@@ -130,8 +131,9 @@ public void testQuery() throws IOException {
         });
 
         // Search for fields outside of the allowlist fails
-        assertQueryError(API_KEY_ADMIN_AUTH_HEADER, 400, """
+        ResponseException responseException = assertQueryError(API_KEY_ADMIN_AUTH_HEADER, 400, """
             { "query": { "prefix": {"api_key_hash": "{PBKDF2}10000$"} } }""");
+        assertThat(responseException.getMessage(), containsString("Field [api_key_hash] is not allowed for API Key query"));
 
         // Search for fields that are not allowed in Query DSL but used internally by the service itself
         final String fieldName = randomFrom("doc_type", "api_key_invalidated", "invalidation_time");
@@ -429,6 +431,130 @@ public void testSort() throws IOException {
         assertQueryError(authHeader, 400, "{\"sort\":[\"" + invalidFieldName + "\"]}");
     }
 
+    public void testSimpleQueryStringQuery() throws IOException {
+        String batmanUserCredentials = createUser("batman", new String[] { "api_key_user_role" });
+        final List<String> apiKeyIds = new ArrayList<>();
+        apiKeyIds.add(createApiKey("key1-user", null, null, Map.of("label", "prod"), API_KEY_USER_AUTH_HEADER).v1());
+        apiKeyIds.add(createApiKey("key1-admin", null, null, Map.of("label", "prod"), API_KEY_ADMIN_AUTH_HEADER).v1());
+        apiKeyIds.add(createApiKey("key2-user", null, null, Map.of("value", 42, "label", "prod"), API_KEY_USER_AUTH_HEADER).v1());
+        apiKeyIds.add(createApiKey("key2-admin", null, null, Map.of("value", 42, "label", "prod"), API_KEY_ADMIN_AUTH_HEADER).v1());
+        apiKeyIds.add(createApiKey("key3-user", null, null, Map.of("value", 42, "hero", true), API_KEY_USER_AUTH_HEADER).v1());
+        apiKeyIds.add(createApiKey("key3-admin", null, null, Map.of("value", 42, "hero", true), API_KEY_ADMIN_AUTH_HEADER).v1());
+        apiKeyIds.add(createApiKey("key4-batman", null, null, Map.of("hero", true), batmanUserCredentials).v1());
+        apiKeyIds.add(createApiKey("key5-batman", null, null, Map.of("hero", true), batmanUserCredentials).v1());
+
+        assertQuery(
+            API_KEY_ADMIN_AUTH_HEADER,
+            """
+                {"query": {"simple_query_string": {"query": "key*", "fields": ["no_such_field_pattern*"]}}}""",
+            apiKeys -> assertThat(apiKeys, is(empty()))
+        );
+        assertQuery(
+            API_KEY_ADMIN_AUTH_HEADER,
+            """
+                {"query": {"simple_query_string": {"query": "prod 42 true", "fields": ["metadata.*"]}}}""",
+            apiKeys -> assertThat(apiKeys, is(empty()))
+        );
+        // disallowed fields are silently ignored for the simple query string query type
+        assertQuery(
+            API_KEY_ADMIN_AUTH_HEADER,
+            """
+                {"query": {"simple_query_string": {"query": "ke*", "fields": ["x*", "api_key_hash"]}}}""",
+            apiKeys -> assertThat(apiKeys, is(empty()))
+        );
+        assertQuery(
+            API_KEY_ADMIN_AUTH_HEADER,
+            """
+                {"query": {"simple_query_string": {"query": "prod 42 true", "fields": ["wild*", "metadata"]}}}""",
+            apiKeys -> assertThat(apiKeys.stream().map(k -> (String) k.get("id")).toList(), containsInAnyOrder(apiKeyIds.toArray()))
+        );
+        assertQuery(
+            API_KEY_ADMIN_AUTH_HEADER,
+            """
+                {"query": {"simple_query_string": {"query": "key* +rest" }}}""",
+            apiKeys -> assertThat(apiKeys.stream().map(k -> (String) k.get("id")).toList(), containsInAnyOrder(apiKeyIds.toArray()))
+        );
+        assertQuery(
+            API_KEY_ADMIN_AUTH_HEADER,
+            """
+                {"query": {"simple_query_string": {"query": "-prod", "fields": ["metadata"]}}}""",
+            apiKeys -> assertThat(
+                apiKeys.stream().map(k -> (String) k.get("id")).toList(),
+                containsInAnyOrder(apiKeyIds.get(4), apiKeyIds.get(5), apiKeyIds.get(6), apiKeyIds.get(7))
+            )
+        );
+        assertQuery(
+            API_KEY_ADMIN_AUTH_HEADER,
+            """
+                {"query": {"simple_query_string": {"query": "-42", "fields": ["meta*", "whatever*"]}}}""",
+            apiKeys -> assertThat(
+                apiKeys.stream().map(k -> (String) k.get("id")).toList(),
+                containsInAnyOrder(apiKeyIds.get(0), apiKeyIds.get(1), apiKeyIds.get(6), apiKeyIds.get(7))
+            )
+        );
+        assertQuery(
+            API_KEY_ADMIN_AUTH_HEADER,
+            """
+                {"query": {"simple_query_string": {"query": "-rest term_which_does_not_exist"}}}""",
+            apiKeys -> assertThat(apiKeys, is(empty()))
+        );
+        assertQuery(
+            API_KEY_ADMIN_AUTH_HEADER,
+            """
+                {"query": {"simple_query_string": {"query": "+default_file +api_key_user", "fields": ["us*", "rea*"]}}}""",
+            apiKeys -> assertThat(
+                apiKeys.stream().map(k -> (String) k.get("id")).toList(),
+                containsInAnyOrder(apiKeyIds.get(0), apiKeyIds.get(2), apiKeyIds.get(4))
+            )
+        );
+        assertQuery(
+            API_KEY_ADMIN_AUTH_HEADER,
+            """
+                {"query": {"simple_query_string": {"query": "default_fie~4", "fields": ["*"]}}}""",
+            apiKeys -> assertThat(
+                apiKeys.stream().map(k -> (String) k.get("id")).toList(),
+                containsInAnyOrder(
+                    apiKeyIds.get(0),
+                    apiKeyIds.get(1),
+                    apiKeyIds.get(2),
+                    apiKeyIds.get(3),
+                    apiKeyIds.get(4),
+                    apiKeyIds.get(5)
+                )
+            )
+        );
+        assertQuery(
+            API_KEY_ADMIN_AUTH_HEADER,
+            """
+                {"query": {"simple_query_string": {"query": "+prod +42",
+                "fields": ["metadata.label", "metadata.value", "metadata.hero"]}}}""",
+            apiKeys -> assertThat(
+                apiKeys.stream().map(k -> (String) k.get("id")).toList(),
+                containsInAnyOrder(apiKeyIds.get(2), apiKeyIds.get(3))
+            )
+        );
+        assertQuery(batmanUserCredentials, """
+            {"query": {"simple_query_string": {"query": "+prod key*", "fields": ["name", "username", "metadata"],
+            "default_operator": "AND"}}}""", apiKeys -> assertThat(apiKeys, is(empty())));
+        assertQuery(
+            batmanUserCredentials,
+            """
+                {"query": {"simple_query_string": {"query": "+true +key*", "fields": ["name", "username", "metadata"],
+                "default_operator": "AND"}}}""",
+            apiKeys -> assertThat(
+                apiKeys.stream().map(k -> (String) k.get("id")).toList(),
+                containsInAnyOrder(apiKeyIds.get(6), apiKeyIds.get(7))
+            )
+        );
+        assertQuery(
+            batmanUserCredentials,
+            """
+                {"query": {"bool": {"must": [{"term": {"name": {"value":"key5-batman"}}},
+                {"simple_query_string": {"query": "default_native"}}]}}}""",
+            apiKeys -> assertThat(apiKeys.stream().map(k -> (String) k.get("id")).toList(), containsInAnyOrder(apiKeyIds.get(7)))
+        );
+    }
+
     public void testExistsQuery() throws IOException, InterruptedException {
         final String authHeader = randomFrom(API_KEY_ADMIN_AUTH_HEADER, API_KEY_USER_AUTH_HEADER);
 
@@ -530,12 +656,13 @@ private int collectApiKeys(List<Map<String, Object>> apiKeyInfos, Request reques
         return actualSize;
     }
 
-    private void assertQueryError(String authHeader, int statusCode, String body) throws IOException {
+    private ResponseException assertQueryError(String authHeader, int statusCode, String body) throws IOException {
         final Request request = new Request("GET", "/_security/_query/api_key");
         request.setJsonEntity(body);
         request.setOptions(request.getOptions().toBuilder().addHeader(HttpHeaders.AUTHORIZATION, authHeader));
         final ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(request));
         assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(statusCode));
+        return responseException;
     }
 
     private void assertQuery(String authHeader, String body, Consumer<List<Map<String, Object>>> apiKeysVerifier) throws IOException {

x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java

@@ -732,6 +732,8 @@ public void testQueryCrossClusterApiKeysByType() throws IOException {
         for (String restTypeQuery : List.of("""
             {"query": {"term": {"type": "rest" }}}""", """
             {"query": {"bool": {"must_not": {"term": {"type": "cross_cluster"}}}}}""", """
+            {"query": {"simple_query_string": {"query": "re* rest -cross_cluster", "fields": ["ty*"]}}}""", """
+            {"query": {"simple_query_string": {"query": "-cross*", "fields": ["type"]}}}""", """
             {"query": {"prefix": {"type": "re" }}}""", """
             {"query": {"wildcard": {"type": "r*t" }}}""", """
             {"query": {"range": {"type": {"gte": "raaa", "lte": "rzzz"}}}}""")) {
@@ -747,6 +749,8 @@ public void testQueryCrossClusterApiKeysByType() throws IOException {
         for (String crossClusterTypeQuery : List.of("""
             {"query": {"term": {"type": "cross_cluster" }}}""", """
             {"query": {"bool": {"must_not": {"term": {"type": "rest"}}}}}""", """
+            {"query": {"simple_query_string": {"query": "cro* cross_cluster -re*", "fields": ["ty*"]}}}""", """
+            {"query": {"simple_query_string": {"query": "-re*", "fields": ["type"]}}}""", """
             {"query": {"prefix": {"type": "cro" }}}""", """
             {"query": {"wildcard": {"type": "*oss_*er" }}}""", """
             {"query": {"range": {"type": {"gte": "cross", "lte": "zzzz"}}}}""")) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java

@@ -13,20 +13,25 @@
 import org.elasticsearch.index.query.ExistsQueryBuilder;
 import org.elasticsearch.index.query.IdsQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
 import org.elasticsearch.index.query.PrefixQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.index.query.SimpleQueryStringBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.index.query.WildcardQueryBuilder;
+import org.elasticsearch.index.search.QueryParserHelper;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
 import org.elasticsearch.xpack.security.authc.ApiKeyService;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Consumer;
 
@@ -45,6 +50,7 @@ public class ApiKeyBoolQueryBuilder extends BoolQueryBuilder {
         "invalidation_time",
         "creation_time",
         "expiration_time",
+        "metadata_flattened",
         "creator.principal",
         "creator.realm"
     );
@@ -160,6 +166,36 @@ private static QueryBuilder doProcess(QueryBuilder qb, Consumer<String> fieldNam
                 newQuery.to(query.to()).includeUpper(query.includeUpper());
             }
             return newQuery.boost(query.boost());
+        } else if (qb instanceof final SimpleQueryStringBuilder simpleQueryStringBuilder) {
+            if (simpleQueryStringBuilder.fields().isEmpty()) {
+                simpleQueryStringBuilder.field("*");
+            }
+            // override lenient if querying all the fields, because, due to different field mappings,
+            // the query parsing will almost certainly fail otherwise
+            if (QueryParserHelper.hasAllFieldsWildcard(simpleQueryStringBuilder.fields().keySet())) {
+                simpleQueryStringBuilder.lenient(true);
+            }
+            Map<String, Float> requestedFields = new HashMap<>(simpleQueryStringBuilder.fields());
+            simpleQueryStringBuilder.fields().clear();
+            for (Map.Entry<String, Float> requestedFieldNameOrPattern : requestedFields.entrySet()) {
+                for (String translatedField : ApiKeyFieldNameTranslators.translatePattern(requestedFieldNameOrPattern.getKey())) {
+                    simpleQueryStringBuilder.fields()
+                        .compute(
+                            translatedField,
+                            (k, v) -> (v == null) ? requestedFieldNameOrPattern.getValue() : v * requestedFieldNameOrPattern.getValue()
+                        );
+                    fieldNameVisitor.accept(translatedField);
+                }
+            }
+            if (simpleQueryStringBuilder.fields().isEmpty()) {
+                // A SimpleQueryStringBuilder with empty fields() will eventually produce a SimpleQueryString query
+                // that accesses all the fields, including disallowed ones.
+                // Instead, the behavior we're after is that a query that accesses only disallowed fields should
+                // not match any docs.
+                return new MatchNoneQueryBuilder();
+            } else {
+                return simpleQueryStringBuilder;
+            }
         } else {
             throw new IllegalArgumentException("Query type [" + qb.getName() + "] is not supported for API Key query");
         }