use terms in set for concrete IPs keep disjunctions over ranges. fix #16200 #16202
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,175 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.search; | ||
|
|
||
| import org.opensearch.action.bulk.BulkRequestBuilder; | ||
| import org.opensearch.action.search.SearchResponse; | ||
| import org.opensearch.common.network.InetAddresses; | ||
| import org.opensearch.common.xcontent.XContentFactory; | ||
| import org.opensearch.core.xcontent.XContentBuilder; | ||
| import org.opensearch.index.query.QueryBuilders; | ||
| import org.opensearch.test.OpenSearchSingleNodeTestCase; | ||
| import org.hamcrest.MatcherAssert; | ||
|
|
||
| import java.io.IOException; | ||
| import java.net.InetAddress; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
|
|
||
| import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; | ||
| import static org.hamcrest.Matchers.equalTo; | ||
|
|
||
| public class SearchIpFieldTermsTests extends OpenSearchSingleNodeTestCase { | ||
|
|
||
| public static final boolean IPv4_ONLY = true; | ||
| static String defaultIndexName = "test"; | ||
|
|
||
| public void testMassive() throws Exception { | ||
| XContentBuilder xcb = createMapping(); | ||
| client().admin().indices().prepareCreate(defaultIndexName).setMapping(xcb).get(); | ||
| ensureGreen(); | ||
|
|
||
| BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); | ||
|
|
||
| int cidrs = 0; | ||
| int ips = 0; | ||
| List<String> toQuery = new ArrayList<>(); | ||
| for (int i = 0; ips <= 1024 && i < 1000000; i++) { | ||
| final String ip; | ||
| final int prefix; | ||
| if (IPv4_ONLY) { | ||
| ip = generateRandomIPv4(); | ||
|
There was a problem hiding this comment. it's just IP4 test, no IP6. Is IP6 necessary, is it worth to combine 6th and 4th in the single test? |
||
| prefix = 8 + random().nextInt(24); // CIDR prefix for IPv4 | ||
| } else { | ||
| ip = generateRandomIPv6(); | ||
| prefix = 32 + random().nextInt(97); // CIDR prefix for IPv6 | ||
| } | ||
|
|
||
| bulkRequestBuilder.add(client().prepareIndex(defaultIndexName).setSource(Map.of("addr", ip))); | ||
|
|
||
| final String termToQuery; | ||
| if (cidrs < 1024 - 1 && random().nextBoolean()) { | ||
| termToQuery = ip + "/" + prefix; | ||
| cidrs++; | ||
| } else { | ||
| termToQuery = ip; | ||
| ips++; | ||
| } | ||
| toQuery.add(termToQuery); | ||
| } | ||
| int addMatches = 0; | ||
| for (int i = 0; i < atLeast(100); i++) { | ||
| final String ip; | ||
| if (IPv4_ONLY) { | ||
| ip = generateRandomIPv4(); | ||
| } else { | ||
| ip = generateRandomIPv6(); | ||
| } | ||
| bulkRequestBuilder.add(client().prepareIndex(defaultIndexName).setSource(Map.of("addr", ip))); | ||
| boolean match = false; | ||
| for (String termQ : toQuery) { | ||
| boolean isCidr = termQ.contains("/"); | ||
| if ((isCidr && isIPInCIDR(ip, termQ)) || (!isCidr && termQ.equals(ip))) { | ||
| match = true; | ||
| break; | ||
| } | ||
| } | ||
| if (match) { | ||
| addMatches++; | ||
| } else { | ||
| break; // single mismatch is enough. | ||
| } | ||
| } | ||
|
|
||
| bulkRequestBuilder.setRefreshPolicy(IMMEDIATE).get(); | ||
| SearchResponse result = client().prepareSearch(defaultIndexName).setQuery(QueryBuilders.termsQuery("addr", toQuery)).get(); | ||
| MatcherAssert.assertThat(Objects.requireNonNull(result.getHits().getTotalHits()).value, equalTo((long) cidrs + ips + addMatches)); | ||
| } | ||
|
|
||
| // Converts an IP string (either IPv4 or IPv6) to a byte array | ||
| private static byte[] ipToBytes(String ip) { | ||
| InetAddress inetAddress = InetAddresses.forString(ip); | ||
| return inetAddress.getAddress(); | ||
| } | ||
|
|
||
| // Checks if an IP is within a given CIDR (works for both IPv4 and IPv6) | ||
| private static boolean isIPInCIDR(String ip, String cidr) { | ||
| String[] cidrParts = cidr.split("/"); | ||
| String cidrIp = cidrParts[0]; | ||
| int prefixLength = Integer.parseInt(cidrParts[1]); | ||
|
|
||
| byte[] ipBytes = ipToBytes(ip); | ||
| byte[] cidrIpBytes = ipToBytes(cidrIp); | ||
|
|
||
| // Calculate how many full bytes and how many bits are in the mask | ||
| int fullBytes = prefixLength / 8; | ||
| int extraBits = prefixLength % 8; | ||
|
|
||
| // Compare full bytes | ||
| for (int i = 0; i < fullBytes; i++) { | ||
| if (ipBytes[i] != cidrIpBytes[i]) { | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| // Compare extra bits (if any) | ||
| if (extraBits > 0) { | ||
| int mask = 0xFF << (8 - extraBits); | ||
| return (ipBytes[fullBytes] & mask) == (cidrIpBytes[fullBytes] & mask); | ||
| } | ||
|
|
||
| return true; | ||
| } | ||
|
|
||
| // Generate a random IPv4 address | ||
| private String generateRandomIPv4() { | ||
| return String.join( | ||
| ".", | ||
| String.valueOf(random().nextInt(256)), | ||
| String.valueOf(random().nextInt(256)), | ||
| String.valueOf(random().nextInt(256)), | ||
| String.valueOf(random().nextInt(256)) | ||
| ); | ||
| } | ||
|
|
||
| // Generate a random IPv6 address | ||
| private static String generateRandomIPv6() { | ||
| StringBuilder ipv6 = new StringBuilder(); | ||
| for (int i = 0; i < 8; i++) { | ||
| ipv6.append(Integer.toHexString(random().nextInt(0xFFFF + 1))); | ||
| if (i < 7) { | ||
| ipv6.append(":"); | ||
| } | ||
| } | ||
| return ipv6.toString(); | ||
| } | ||
|
|
||
| private XContentBuilder createMapping() throws IOException { | ||
| return XContentFactory.jsonBuilder() | ||
| .startObject() | ||
| .startObject("properties") | ||
| .startObject("addr") | ||
| .field("type", "ip") | ||
| .startObject("fields") | ||
| .startObject("idx") | ||
| .field("type", "ip") | ||
| .field("doc_values", false) | ||
| .endObject() | ||
| .startObject("dv") | ||
| .field("type", "ip") | ||
| .field("index", false) | ||
| .endObject() | ||
| .endObject() | ||
| .endObject() | ||
| .endObject() | ||
| .endObject(); | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It might be too heavy testing approach for the current change. However, such depth is necessary for the second stage (when ranges will be combined at PointsVisitor level).
There might be just a
IpFieldTypeTestasserting resulting BooleanQuery has set of IP points. WDYT?