diff --git a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java index 7c7c700728074..516b57342ebb9 100644 --- a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java +++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java @@ -162,7 +162,7 @@ private EhcacheDiskCache(Builder builder) { this.ehCacheEventListener = new EhCacheEventListener(builder.getRemovalListener(), builder.getWeigher()); this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder); List dimensionNames = Objects.requireNonNull(builder.dimensionNames, "Dimension names can't be null"); - this.statsHolder = new StatsHolder(dimensionNames); + this.statsHolder = new StatsHolder(dimensionNames, EhcacheDiskCacheFactory.EHCACHE_DISK_CACHE_NAME); } @SuppressWarnings({ "rawtypes" }) diff --git a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java index 510a143b144d5..82acd0de5743a 100644 --- a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java +++ b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java @@ -20,13 +20,19 @@ import org.opensearch.common.cache.RemovalNotification; import org.opensearch.common.cache.serializer.BytesReferenceSerializer; import org.opensearch.common.cache.serializer.Serializer; +import org.opensearch.common.cache.stats.CacheStats; import org.opensearch.common.cache.store.config.CacheConfig; import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.bytes.CompositeBytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.env.NodeEnvironment; import org.opensearch.test.OpenSearchSingleNodeTestCase; @@ -797,6 +803,75 @@ public void testInvalidate() throws Exception { } } + // Modified from OpenSearchOnHeapCacheTests.java + public void testInvalidateWithDropDimensions() throws Exception { + Settings settings = Settings.builder().build(); + List dimensionNames = List.of("dim1", "dim2"); + try (NodeEnvironment env = newNodeEnvironment(settings)) { + ICache ehCacheDiskCachingTier = new EhcacheDiskCache.Builder().setThreadPoolAlias("ehcacheTest") + .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setKeySerializer(new StringSerializer()) + .setValueSerializer(new StringSerializer()) + .setDimensionNames(dimensionNames) + .setKeyType(String.class) + .setValueType(String.class) + .setCacheType(CacheType.INDICES_REQUEST_CACHE) + .setSettings(settings) + .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES * 20) // bigger so no evictions happen + .setExpireAfterAccess(TimeValue.MAX_VALUE) + .setRemovalListener(new MockRemovalListener<>()) + .setWeigher((key, value) -> 1) + .build(); + + List> keysAdded = new ArrayList<>(); + + for (int i = 0; i < 20; i++) { + ICacheKey key = new ICacheKey<>(UUID.randomUUID().toString(), getRandomDimensions(dimensionNames)); + keysAdded.add(key); + ehCacheDiskCachingTier.put(key, UUID.randomUUID().toString()); + } + + ICacheKey keyToDrop = keysAdded.get(0); + + Map xContentMap = getStatsXContentMap(ehCacheDiskCachingTier.stats(), dimensionNames); + List xContentMapKeys = getXContentMapKeys(keyToDrop, dimensionNames); + Map individualSnapshotMap = (Map) getValueFromNestedXContentMap(xContentMap, xContentMapKeys); + assertNotNull(individualSnapshotMap); + assertEquals(5, individualSnapshotMap.size()); // Assert all 5 stats are present and not null + for (Map.Entry entry : individualSnapshotMap.entrySet()) { + Integer value = (Integer) entry.getValue(); + assertNotNull(value); + } + + keyToDrop.setDropStatsForDimensions(true); + ehCacheDiskCachingTier.invalidate(keyToDrop); + + // Now assert the stats are gone for any key that has this combination of dimensions, but still there otherwise + xContentMap = getStatsXContentMap(ehCacheDiskCachingTier.stats(), dimensionNames); + for (ICacheKey keyAdded : keysAdded) { + xContentMapKeys = getXContentMapKeys(keyAdded, dimensionNames); + individualSnapshotMap = (Map) getValueFromNestedXContentMap(xContentMap, xContentMapKeys); + if (keyAdded.dimensions.equals(keyToDrop.dimensions)) { + assertNull(individualSnapshotMap); + } else { + assertNotNull(individualSnapshotMap); + } + } + + ehCacheDiskCachingTier.close(); + } + } + + private List getRandomDimensions(List dimensionNames) { + Random rand = Randomness.get(); + int bound = 3; + List result = new ArrayList<>(); + for (String dimName : dimensionNames) { + result.add(String.valueOf(rand.nextInt(bound))); + } + return result; + } + private static String generateRandomString(int length) { String characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; StringBuilder randomString = new StringBuilder(length); @@ -831,6 +906,42 @@ private ToLongBiFunction, String> getWeigher() { }; } + // Helper functions duplicated from server.test; we can't add a dependency on that module + private static Map getStatsXContentMap(CacheStats cacheStats, List levels) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + Map paramMap = Map.of("level", String.join(",", levels)); + ToXContent.Params params = new ToXContent.MapParams(paramMap); + + builder.startObject(); + cacheStats.toXContent(builder, params); + builder.endObject(); + + String resultString = builder.toString(); + return XContentHelper.convertToMap(MediaTypeRegistry.JSON.xContent(), resultString, true); + } + + private List getXContentMapKeys(ICacheKey iCacheKey, List dimensionNames) { + List result = new ArrayList<>(); + assert iCacheKey.dimensions.size() == dimensionNames.size(); + for (int i = 0; i < dimensionNames.size(); i++) { + result.add(dimensionNames.get(i)); + result.add(iCacheKey.dimensions.get(i)); + } + return result; + } + + public static Object getValueFromNestedXContentMap(Map xContentMap, List keys) { + Map current = xContentMap; + for (int i = 0; i < keys.size() - 1; i++) { + Object next = current.get(keys.get(i)); + if (next == null) { + return null; + } + current = (Map) next; + } + return current.get(keys.get(keys.size() - 1)); + } + static class MockRemovalListener implements RemovalListener, V> { CounterMetric evictionMetric = new CounterMetric(); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java index 52b4dad553180..1fd8486699ed2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java @@ -34,17 +34,30 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.opensearch.action.admin.cluster.node.stats.NodeStats; +import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; +import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchType; import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.common.cache.service.NodeCacheStats; +import org.opensearch.common.cache.stats.CacheStats; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.cache.request.RequestCacheStats; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.shard.IndexShard; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.histogram.Histogram; @@ -52,6 +65,7 @@ import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import org.opensearch.test.hamcrest.OpenSearchAssertions; +import java.io.IOException; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -59,6 +73,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; @@ -677,6 +692,34 @@ public void testCacheWithInvalidation() throws Exception { assertCacheState(client, "index", 1, 2); } + public void testCacheStatsAPI() throws Exception { + final String nodeId = internalCluster().startNode(); + Client client = client(nodeId); + assertAcked( + client.admin() + .indices() + .prepareCreate("index") + .setMapping("k", "type=keyword") + .setSettings( + Settings.builder() + .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ) + .get() + ); + indexRandom(true, client.prepareIndex("index").setSource("k", "hello")); + ensureSearchable("index"); + SearchResponse resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get(); + assertSearchResponse(resp); + OpenSearchAssertions.assertAllSuccessful(resp); + assertThat(resp.getHits().getTotalHits().value, equalTo(1L)); + + Map xContentMap = getNodeCacheStatsXContentMap(client, nodeId, List.of()); + int j = 0; + + } + private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) { RequestCacheStats requestCacheStats = client.admin() .indices() @@ -694,4 +737,26 @@ private static void assertCacheState(Client client, String index, long expectedH } + private static Map getNodeCacheStatsXContentMap(Client client, String nodeId, List aggregationLevels) throws IOException { + + + NodesStatsResponse nodeStatsResponse = client.admin().cluster() + .prepareNodesStats("data:true") + .addMetric(NodesStatsRequest.Metric.CACHE_STATS.metricName()) + .get(); + Map intermediate = nodeStatsResponse.getNodesMap(); + NodeCacheStats ncs = nodeStatsResponse.getNodes().get(0).getNodeCacheStats(); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + Map paramMap = Map.of("level", String.join(",", aggregationLevels)); + ToXContent.Params params = new ToXContent.MapParams(paramMap); + + builder.startObject(); + ncs.toXContent(builder, params); + builder.endObject(); + + String resultString = builder.toString(); + return XContentHelper.convertToMap(MediaTypeRegistry.JSON.xContent(), resultString, true); + } + } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java index 8562a7eb37709..ac2daf57f248b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java @@ -39,6 +39,7 @@ import org.opensearch.cluster.routing.WeightedRoutingStats; import org.opensearch.cluster.service.ClusterManagerThrottlingStats; import org.opensearch.common.Nullable; +import org.opensearch.common.cache.service.NodeCacheStats; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.indices.breaker.AllCircuitBreakerStats; @@ -158,6 +159,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { @Nullable private AdmissionControlStats admissionControlStats; + @Nullable + private NodeCacheStats nodeCacheStats; + public NodeStats(StreamInput in) throws IOException { super(in); timestamp = in.readVLong(); @@ -234,6 +238,11 @@ public NodeStats(StreamInput in) throws IOException { } else { admissionControlStats = null; } + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + nodeCacheStats = in.readOptionalWriteable(NodeCacheStats::new); + } else { + nodeCacheStats = null; + } } public NodeStats( @@ -264,7 +273,8 @@ public NodeStats( @Nullable SearchPipelineStats searchPipelineStats, @Nullable SegmentReplicationRejectionStats segmentReplicationRejectionStats, @Nullable RepositoriesStats repositoriesStats, - @Nullable AdmissionControlStats admissionControlStats + @Nullable AdmissionControlStats admissionControlStats, + @Nullable NodeCacheStats nodeCacheStats ) { super(node); this.timestamp = timestamp; @@ -294,6 +304,7 @@ public NodeStats( this.segmentReplicationRejectionStats = segmentReplicationRejectionStats; this.repositoriesStats = repositoriesStats; this.admissionControlStats = admissionControlStats; + this.nodeCacheStats = nodeCacheStats; } public long getTimestamp() { @@ -451,6 +462,11 @@ public AdmissionControlStats getAdmissionControlStats() { return admissionControlStats; } + @Nullable + public NodeCacheStats getNodeCacheStats() { + return nodeCacheStats; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -506,6 +522,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeOptionalWriteable(admissionControlStats); } + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalWriteable(nodeCacheStats); + } } @Override @@ -609,6 +628,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (getAdmissionControlStats() != null) { getAdmissionControlStats().toXContent(builder, params); } + if (getNodeCacheStats() != null) { + getNodeCacheStats().toXContent(builder, params); + } return builder; } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java index 1af56f10b95ee..379836cf442e3 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -219,7 +219,8 @@ public enum Metric { RESOURCE_USAGE_STATS("resource_usage_stats"), SEGMENT_REPLICATION_BACKPRESSURE("segment_replication_backpressure"), REPOSITORIES("repositories"), - ADMISSION_CONTROL("admission_control"); + ADMISSION_CONTROL("admission_control"), + CACHE_STATS("caches"); private String metricName; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index 1df73d3b4394d..2e93e5e7841cb 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -128,7 +128,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) { NodesStatsRequest.Metric.RESOURCE_USAGE_STATS.containedIn(metrics), NodesStatsRequest.Metric.SEGMENT_REPLICATION_BACKPRESSURE.containedIn(metrics), NodesStatsRequest.Metric.REPOSITORIES.containedIn(metrics), - NodesStatsRequest.Metric.ADMISSION_CONTROL.containedIn(metrics) + NodesStatsRequest.Metric.ADMISSION_CONTROL.containedIn(metrics), + NodesStatsRequest.Metric.CACHE_STATS.containedIn(metrics) ); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 9c5dcc9e9de3f..e4f483f796f44 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -172,6 +172,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false, false, false, + false, false ); List shardsStats = new ArrayList<>(); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java index a7d9f95b80f7b..cbde1637ea575 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java @@ -34,6 +34,7 @@ import org.opensearch.Version; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.cache.CacheType; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -63,6 +64,9 @@ public class CommonStatsFlags implements Writeable, Cloneable { private boolean includeAllShardIndexingPressureTrackers = false; private boolean includeOnlyTopIndexingPressureMetrics = false; + // Used for metric CACHE_STATS, to determine which caches to report stats for + private EnumSet includeCaches = EnumSet.noneOf(CacheType.class); + /** * @param flags flags to set. If no flags are supplied, default flags will be set. */ @@ -91,6 +95,9 @@ public CommonStatsFlags(StreamInput in) throws IOException { includeUnloadedSegments = in.readBoolean(); includeAllShardIndexingPressureTrackers = in.readBoolean(); includeOnlyTopIndexingPressureMetrics = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + includeCaches = in.readEnumSet(CacheType.class); + } } @Override @@ -111,6 +118,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(includeUnloadedSegments); out.writeBoolean(includeAllShardIndexingPressureTrackers); out.writeBoolean(includeOnlyTopIndexingPressureMetrics); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeEnumSet(includeCaches); + } } /** @@ -125,6 +135,7 @@ public CommonStatsFlags all() { includeUnloadedSegments = false; includeAllShardIndexingPressureTrackers = false; includeOnlyTopIndexingPressureMetrics = false; + includeCaches = EnumSet.noneOf(CacheType.class); return this; } @@ -140,6 +151,7 @@ public CommonStatsFlags clear() { includeUnloadedSegments = false; includeAllShardIndexingPressureTrackers = false; includeOnlyTopIndexingPressureMetrics = false; + includeCaches = EnumSet.noneOf(CacheType.class); return this; } @@ -151,6 +163,10 @@ public Flag[] getFlags() { return flags.toArray(new Flag[0]); } + public EnumSet getIncludeCaches() { + return includeCaches; + } + /** * Sets specific search group stats to retrieve the stats for. Mainly affects search * when enabled. @@ -206,6 +222,16 @@ public CommonStatsFlags includeOnlyTopIndexingPressureMetrics(boolean includeOnl return this; } + public CommonStatsFlags includeCacheType(CacheType cacheType) { + includeCaches.add(cacheType); + return this; + } + + public CommonStatsFlags includeAllCacheTypes() { + includeCaches = EnumSet.allOf(CacheType.class); + return this; + } + public boolean includeUnloadedSegments() { return this.includeUnloadedSegments; } diff --git a/server/src/main/java/org/opensearch/common/cache/CacheType.java b/server/src/main/java/org/opensearch/common/cache/CacheType.java index c5aeb7cd1fa40..61442db148067 100644 --- a/server/src/main/java/org/opensearch/common/cache/CacheType.java +++ b/server/src/main/java/org/opensearch/common/cache/CacheType.java @@ -10,20 +10,46 @@ import org.opensearch.common.annotation.ExperimentalApi; +import java.util.HashSet; +import java.util.Set; + /** * Cache types available within OpenSearch. */ @ExperimentalApi public enum CacheType { - INDICES_REQUEST_CACHE("indices.requests.cache"); + INDICES_REQUEST_CACHE("indices.requests.cache", "request_cache"); private final String settingPrefix; + private final String apiRepresentation; - CacheType(String settingPrefix) { + CacheType(String settingPrefix, String representation) { this.settingPrefix = settingPrefix; + this.apiRepresentation = representation; } public String getSettingPrefix() { return settingPrefix; } + + public String getApiRepresentation() { + return apiRepresentation; + } + + public static CacheType getByRepresentation(String representation) { + for (CacheType cacheType : values()) { + if (cacheType.apiRepresentation.equals(representation)) { + return cacheType; + } + } + throw new IllegalArgumentException("No CacheType with representation = " + representation); + } + + public static Set allRepresentations() { + Set reprs = new HashSet<>(); + for (CacheType cacheType : values()) { + reprs.add(cacheType.apiRepresentation); + } + return reprs; + } } diff --git a/server/src/main/java/org/opensearch/common/cache/ICacheKey.java b/server/src/main/java/org/opensearch/common/cache/ICacheKey.java index 0eb34952e78f1..e1aa9b1c5466c 100644 --- a/server/src/main/java/org/opensearch/common/cache/ICacheKey.java +++ b/server/src/main/java/org/opensearch/common/cache/ICacheKey.java @@ -10,7 +10,6 @@ import org.opensearch.common.annotation.ExperimentalApi; -import java.util.Collections; import java.util.List; /** diff --git a/server/src/main/java/org/opensearch/common/cache/service/CacheService.java b/server/src/main/java/org/opensearch/common/cache/service/CacheService.java index b6710e5e4b424..e450fc65fe351 100644 --- a/server/src/main/java/org/opensearch/common/cache/service/CacheService.java +++ b/server/src/main/java/org/opensearch/common/cache/service/CacheService.java @@ -8,10 +8,12 @@ package org.opensearch.common.cache.service; +import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.cache.CacheType; import org.opensearch.common.cache.ICache; import org.opensearch.common.cache.settings.CacheSettings; +import org.opensearch.common.cache.stats.CacheStats; import org.opensearch.common.cache.store.OpenSearchOnHeapCache; import org.opensearch.common.cache.store.config.CacheConfig; import org.opensearch.common.settings.Setting; @@ -19,6 +21,7 @@ import org.opensearch.common.util.FeatureFlags; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; /** @@ -62,4 +65,12 @@ public ICache createCache(CacheConfig config, CacheType cache cacheTypeMap.put(cacheType, iCache); return iCache; } + + public NodeCacheStats stats(CommonStatsFlags flags) { + LinkedHashMap statsMap = new LinkedHashMap<>(); + for (CacheType type : cacheTypeMap.keySet()) { + statsMap.put(type, cacheTypeMap.get(type).stats()); // TODO: We need to force some ordering on cacheTypeMap + } + return new NodeCacheStats(statsMap, flags); + } } diff --git a/server/src/main/java/org/opensearch/common/cache/service/NodeCacheStats.java b/server/src/main/java/org/opensearch/common/cache/service/NodeCacheStats.java new file mode 100644 index 0000000000000..43ed8c0d0fcd6 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/cache/service/NodeCacheStats.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.cache.service; + +import org.opensearch.action.admin.indices.stats.CommonStatsFlags; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.cache.CacheType; +import org.opensearch.common.cache.stats.CacheStats; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +/** + * A class creating XContent responses to cache stats API requests. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class NodeCacheStats implements ToXContentFragment, Writeable { + private final LinkedHashMap statsByCache; + private final CommonStatsFlags flags; + + public NodeCacheStats(LinkedHashMap statsByCache, CommonStatsFlags flags) { + this.statsByCache = statsByCache; + this.flags = flags; + } + + public NodeCacheStats(StreamInput in) throws IOException { + this.flags = new CommonStatsFlags(in); + Map readMap = in.readMap(i -> i.readEnum(CacheType.class), CacheStats::readFromStreamWithClassName); + this.statsByCache = new LinkedHashMap<>(readMap); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + flags.writeTo(out); + out.writeMap(statsByCache, StreamOutput::writeEnum, (o, cacheStats) -> cacheStats.writeToWithClassName(o)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + for (CacheType type : statsByCache.keySet()) { + if (flags.getIncludeCaches().contains(type)) { + builder.startObject(type.getApiRepresentation()); + statsByCache.get(type).toXContent(builder, params); + builder.endObject(); + } + } + return builder; + } + + @Override + public boolean equals(Object o) { + if (o == null) { + return false; + } + if (o.getClass() != NodeCacheStats.class) { + return false; + } + NodeCacheStats other = (NodeCacheStats) o; + return statsByCache.equals(other.statsByCache) && flags.getIncludeCaches().equals(other.flags.getIncludeCaches()); + } + + @Override + public int hashCode() { + return Objects.hash(statsByCache, flags); + } +} diff --git a/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java b/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java index a552b13aa5f84..1e4ae24eb88a1 100644 --- a/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java +++ b/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java @@ -9,7 +9,12 @@ package org.opensearch.common.cache.stats; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentFragment; + +import java.io.IOException; /** * Interface for access to any cache stats. Allows accessing stats by dimension values. @@ -18,7 +23,7 @@ * @opensearch.experimental */ @ExperimentalApi -public interface CacheStats extends Writeable {// TODO: also extends ToXContentFragment (in API PR) +public interface CacheStats extends Writeable, ToXContentFragment { // Method to get all 5 values at once CacheStatsCounterSnapshot getTotalStats(); @@ -33,4 +38,18 @@ public interface CacheStats extends Writeable {// TODO: also extends ToXContentF long getTotalSizeInBytes(); long getTotalEntries(); + + // Used for the readFromStream method to allow deserialization of generic CacheStats objects. + String getClassName(); + + void writeToWithClassName(StreamOutput out) throws IOException; + + static CacheStats readFromStreamWithClassName(StreamInput in) throws IOException { + String className = in.readString(); + + if (className.equals(MultiDimensionCacheStats.CLASS_NAME)) { + return new MultiDimensionCacheStats(in); + } + return null; + } } diff --git a/server/src/main/java/org/opensearch/common/cache/stats/CacheStatsCounterSnapshot.java b/server/src/main/java/org/opensearch/common/cache/stats/CacheStatsCounterSnapshot.java index 3057edd8b2afc..b987eadfa751f 100644 --- a/server/src/main/java/org/opensearch/common/cache/stats/CacheStatsCounterSnapshot.java +++ b/server/src/main/java/org/opensearch/common/cache/stats/CacheStatsCounterSnapshot.java @@ -12,6 +12,9 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; import java.util.Objects; @@ -22,7 +25,7 @@ * @opensearch.experimental */ @ExperimentalApi -public class CacheStatsCounterSnapshot implements Writeable { // TODO: Make this extend ToXContent (in API PR) +public class CacheStatsCounterSnapshot implements Writeable, ToXContent { private final long hits; private final long misses; private final long evictions; @@ -100,4 +103,26 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(hits, misses, evictions, sizeInBytes, entries); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // We don't write the header in CacheStatsResponse's toXContent, because it doesn't know the name of aggregation it's part of + builder.humanReadableField(Fields.MEMORY_SIZE_IN_BYTES, Fields.MEMORY_SIZE, new ByteSizeValue(sizeInBytes)); + builder.field(Fields.EVICTIONS, evictions); + builder.field(Fields.HIT_COUNT, hits); + builder.field(Fields.MISS_COUNT, misses); + builder.field(Fields.ENTRIES, entries); + return builder; + } + + static final class Fields { + static final String MEMORY_SIZE = "size"; // TODO: Bad name - think of something better + static final String MEMORY_SIZE_IN_BYTES = "size_in_bytes"; + // TODO: This might not be memory as it could be partially on disk, so I've changed it, but should it be consistent with the earlier + // field? + static final String EVICTIONS = "evictions"; + static final String HIT_COUNT = "hit_count"; + static final String MISS_COUNT = "miss_count"; + static final String ENTRIES = "entries"; + } } diff --git a/server/src/main/java/org/opensearch/common/cache/stats/DimensionNode.java b/server/src/main/java/org/opensearch/common/cache/stats/DimensionNode.java index 8e89eae80ce79..49f74ea941247 100644 --- a/server/src/main/java/org/opensearch/common/cache/stats/DimensionNode.java +++ b/server/src/main/java/org/opensearch/common/cache/stats/DimensionNode.java @@ -91,10 +91,6 @@ void resetSizeAndEntries() { } DimensionNode getChild(String dimensionValue) { // , boolean createIfAbsent, boolean createMapInChild - /*return children.computeIfAbsent( - dimensionValue, - (key) -> createIfAbsent ? new DimensionNode(dimensionValue, createMapInChild) : null - );*/ return children.get(dimensionValue); } diff --git a/server/src/main/java/org/opensearch/common/cache/stats/MultiDimensionCacheStats.java b/server/src/main/java/org/opensearch/common/cache/stats/MultiDimensionCacheStats.java index ea4dfcc818350..4ce7210e7775e 100644 --- a/server/src/main/java/org/opensearch/common/cache/stats/MultiDimensionCacheStats.java +++ b/server/src/main/java/org/opensearch/common/cache/stats/MultiDimensionCacheStats.java @@ -10,11 +10,13 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.TreeMap; /** @@ -29,11 +31,21 @@ public class MultiDimensionCacheStats implements CacheStats { final MDCSDimensionNode statsRoot; final List dimensionNames; - public MultiDimensionCacheStats(MDCSDimensionNode statsRoot, List dimensionNames) { + // The name of the cache type producing these stats. Returned in API response. + final String storeName; + public static String STORE_NAME_FIELD = "store_name"; + + public static String CLASS_NAME = "multidimension"; + + public MultiDimensionCacheStats(MDCSDimensionNode statsRoot, List dimensionNames, String storeName) { this.statsRoot = statsRoot; this.dimensionNames = dimensionNames; + this.storeName = storeName; } + /** + * Should not be used with StreamOutputs produced using writeToWithClassName. + */ public MultiDimensionCacheStats(StreamInput in) throws IOException { // Because we write in preorder order, the parent of the next node we read will always be one of the ancestors // of the last node we read. This allows us to avoid ambiguity if nodes have the same dimension value, without @@ -48,9 +60,10 @@ public MultiDimensionCacheStats(StreamInput in) throws IOException { // Finally, update sum-of-children stats for the root node CacheStatsCounter totalStats = new CacheStatsCounter(); for (MDCSDimensionNode child : statsRoot.children.values()) { - totalStats.add(child.getStats()); + totalStats.add(child.getStatsSnapshot()); } statsRoot.setStats(totalStats.snapshot()); + this.storeName = in.readString(); } @Override @@ -62,13 +75,14 @@ public void writeTo(StreamOutput out) throws IOException { writeDimensionNodeRecursive(out, child, 1); } out.writeBoolean(false); // Write false to signal there are no more nodes + out.writeString(storeName); } private void writeDimensionNodeRecursive(StreamOutput out, MDCSDimensionNode node, int depth) throws IOException { out.writeBoolean(true); // Signals there is a following node to deserialize out.writeVInt(depth); out.writeString(node.getDimensionValue()); - node.getStats().writeTo(out); + node.getStatsSnapshot().writeTo(out); if (node.hasChildren()) { // Not a leaf node @@ -111,7 +125,7 @@ private List readAndAttachDimensionNode(StreamInput in, List< @Override public CacheStatsCounterSnapshot getTotalStats() { - return statsRoot.getStats(); + return statsRoot.getStatsSnapshot(); } @Override @@ -139,13 +153,24 @@ public long getTotalEntries() { return getTotalStats().getEntries(); } + @Override + public String getClassName() { + return CLASS_NAME; + } + + @Override + public void writeToWithClassName(StreamOutput out) throws IOException { + out.writeString(getClassName()); + writeTo(out); + } + /** * Returns a new tree containing the stats aggregated by the levels passed in. The root node is a dummy node, * whose name and value are null. The new tree only has dimensions matching the levels passed in. */ MDCSDimensionNode aggregateByLevels(List levels) { List filteredLevels = filterLevels(levels); - MDCSDimensionNode newRoot = new MDCSDimensionNode(null, statsRoot.getStats()); + MDCSDimensionNode newRoot = new MDCSDimensionNode(null, statsRoot.getStatsSnapshot()); newRoot.createChildrenMap(); for (MDCSDimensionNode child : statsRoot.children.values()) { aggregateByLevelsHelper(newRoot, child, filteredLevels, 0); @@ -169,13 +194,13 @@ void aggregateByLevelsHelper( MDCSDimensionNode nodeInNewTree = parentInNewTree.children.get(dimensionValue); if (nodeInNewTree == null) { // Create new node with stats matching the node from the original tree - nodeInNewTree = new MDCSDimensionNode(dimensionValue, currentInOriginalTree.getStats()); + nodeInNewTree = new MDCSDimensionNode(dimensionValue, currentInOriginalTree.getStatsSnapshot()); parentInNewTree.children.put(dimensionValue, nodeInNewTree); } else { // Otherwise increment existing stats CacheStatsCounterSnapshot newStats = CacheStatsCounterSnapshot.addSnapshots( - nodeInNewTree.getStats(), - currentInOriginalTree.getStats() + nodeInNewTree.getStatsSnapshot(), + currentInOriginalTree.getStatsSnapshot() ); nodeInNewTree.setStats(newStats); } @@ -208,6 +233,62 @@ private List filterLevels(List levels) { return filtered; } + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // Always show total stats, regardless of levels + getTotalStats().toXContent(builder, params); + + List levels = getLevels(params); + if (levels == null) { + // display total stats only + return builder; + } + + List filteredLevels = filterLevels(levels); + toXContentForLevels(builder, params, filteredLevels); + // Also add the store name for the cache that produced the stats + builder.field(STORE_NAME_FIELD, storeName); + return builder; + } + + XContentBuilder toXContentForLevels(XContentBuilder builder, Params params, List levels) throws IOException { + MDCSDimensionNode aggregated = aggregateByLevels(levels); + // Depth -1 corresponds to the dummy root node, which has no dimension value and only has children + toXContentForLevelsHelper(-1, aggregated, levels, builder, params); + return builder; + + } + + private void toXContentForLevelsHelper(int depth, MDCSDimensionNode current, List levels, XContentBuilder builder, Params params) + throws IOException { + if (depth >= 0) { + builder.startObject(current.dimensionValue); + } + + if (depth == levels.size() - 1) { + // This is a leaf node + current.getStatsSnapshot().toXContent(builder, params); + } else { + builder.startObject(levels.get(depth + 1)); + for (MDCSDimensionNode nextNode : current.children.values()) { + toXContentForLevelsHelper(depth + 1, nextNode, levels, builder, params); + } + builder.endObject(); + } + + if (depth >= 0) { + builder.endObject(); + } + } + + private List getLevels(Params params) { + String levels = params.param("level"); + if (levels == null) { + return null; + } + return List.of(levels.split(",")); + } + // A version of DimensionNode which uses an ordered TreeMap and holds immutable CacheStatsCounterSnapshot as its stats. // TODO: Make this extend from DimensionNode? static class MDCSDimensionNode { @@ -238,7 +319,7 @@ protected Map getChildren() { return children; } - public CacheStatsCounterSnapshot getStats() { + public CacheStatsCounterSnapshot getStatsSnapshot() { return stats; } @@ -260,5 +341,43 @@ MDCSDimensionNode getStatsRoot() { return statsRoot; } - // TODO (in API PR): Produce XContent based on aggregateByLevels() + @Override + public boolean equals(Object o) { + if (o == null || o.getClass() != MultiDimensionCacheStats.class) { + return false; + } + MultiDimensionCacheStats other = (MultiDimensionCacheStats) o; + if (!dimensionNames.equals(other.dimensionNames) || !storeName.equals(other.storeName)) { + return false; + } + return equalsHelper(statsRoot, other.getStatsRoot()); + } + + private boolean equalsHelper(MDCSDimensionNode thisNode, MDCSDimensionNode otherNode) { + if (!thisNode.getStatsSnapshot().equals(otherNode.getStatsSnapshot())) { + return false; + } + if (thisNode.children == null && otherNode.children == null) { + // TODO: Simplify this logic once we inherit from normal DimensionNode and have the static empty map thing + return true; + } + if (thisNode.children == null || otherNode.children == null || !thisNode.getChildren().keySet().equals(otherNode.getChildren().keySet())) { + return false; + } + boolean allChildrenMatch = true; + for (String childValue : thisNode.getChildren().keySet()) { + allChildrenMatch = equalsHelper(thisNode.children.get(childValue), otherNode.children.get(childValue)); + if (!allChildrenMatch) { + return false; + } + } + return allChildrenMatch; + } + + @Override + public int hashCode() { + // Should be sufficient to hash based on the total stats value (found in the root node) + return Objects.hash(statsRoot.stats, dimensionNames); + } + } diff --git a/server/src/main/java/org/opensearch/common/cache/stats/StatsHolder.java b/server/src/main/java/org/opensearch/common/cache/stats/StatsHolder.java index dc161efb05597..f3fbc42b38021 100644 --- a/server/src/main/java/org/opensearch/common/cache/stats/StatsHolder.java +++ b/server/src/main/java/org/opensearch/common/cache/stats/StatsHolder.java @@ -17,6 +17,8 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; +import org.opensearch.common.annotation.ExperimentalApi; + import static org.opensearch.common.cache.stats.MultiDimensionCacheStats.MDCSDimensionNode; /** @@ -29,6 +31,7 @@ * * @opensearch.experimental */ +@ExperimentalApi public class StatsHolder { // The list of permitted dimensions. Should be ordered from "outermost" to "innermost", as you would like to @@ -41,8 +44,12 @@ public class StatsHolder { // No lock is needed to edit stats on existing nodes. private final Lock lock = new ReentrantLock(); - public StatsHolder(List dimensionNames) { + // The name of the cache type using these stats + private final String storeName; + + public StatsHolder(List dimensionNames, String storeName) { this.dimensionNames = dimensionNames; + this.storeName = storeName; this.statsRoot = new DimensionNode(null, true); // The root node has no dimension value associated with it, only children } @@ -160,7 +167,7 @@ public CacheStats getCacheStats() { getCacheStatsHelper(child, snapshot); } } - return new MultiDimensionCacheStats(snapshot, dimensionNames); + return new MultiDimensionCacheStats(snapshot, dimensionNames, storeName); } private void getCacheStatsHelper(DimensionNode currentNodeInOriginalTree, MDCSDimensionNode parentInNewTree) { diff --git a/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java b/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java index 2e60072d07ed2..2a68d83456ace 100644 --- a/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java +++ b/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java @@ -62,7 +62,7 @@ public OpenSearchOnHeapCache(Builder builder) { } cache = cacheBuilder.build(); this.dimensionNames = Objects.requireNonNull(builder.dimensionNames, "Dimension names can't be null"); - this.statsHolder = new StatsHolder(dimensionNames); + this.statsHolder = new StatsHolder(dimensionNames, OpenSearchOnHeapCacheFactory.NAME); this.removalListener = builder.getRemovalListener(); this.weigher = builder.getWeigher(); } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 7fa2b6c8ff497..628381beda3f9 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -1180,7 +1180,8 @@ protected Node( resourceUsageCollectorService, segmentReplicationStatsTracker, repositoryService, - admissionControlService + admissionControlService, + cacheService ); final SearchService searchService = newSearchService( diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java index 15cc8f3d20bb3..1eb38ea63ad5a 100644 --- a/server/src/main/java/org/opensearch/node/NodeService.java +++ b/server/src/main/java/org/opensearch/node/NodeService.java @@ -41,6 +41,7 @@ import org.opensearch.cluster.routing.WeightedRoutingStats; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; +import org.opensearch.common.cache.service.CacheService; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; import org.opensearch.common.util.io.IOUtils; @@ -99,6 +100,7 @@ public class NodeService implements Closeable { private final RepositoriesService repositoriesService; private final AdmissionControlService admissionControlService; private final SegmentReplicationStatsTracker segmentReplicationStatsTracker; + private final CacheService cacheService; NodeService( Settings settings, @@ -125,7 +127,8 @@ public class NodeService implements Closeable { ResourceUsageCollectorService resourceUsageCollectorService, SegmentReplicationStatsTracker segmentReplicationStatsTracker, RepositoriesService repositoriesService, - AdmissionControlService admissionControlService + AdmissionControlService admissionControlService, + CacheService cacheService ) { this.settings = settings; this.threadPool = threadPool; @@ -154,6 +157,7 @@ public class NodeService implements Closeable { clusterService.addStateApplier(ingestService); clusterService.addStateApplier(searchPipelineService); this.segmentReplicationStatsTracker = segmentReplicationStatsTracker; + this.cacheService = cacheService; } public NodeInfo info( @@ -236,7 +240,8 @@ public NodeStats stats( boolean resourceUsageStats, boolean segmentReplicationTrackerStats, boolean repositoriesStats, - boolean admissionControl + boolean admissionControl, + boolean cacheService ) { // for indices stats we want to include previous allocated shards stats as well (it will // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats) @@ -268,7 +273,8 @@ public NodeStats stats( searchPipelineStats ? this.searchPipelineService.stats() : null, segmentReplicationTrackerStats ? this.segmentReplicationStatsTracker.getTotalRejectionStats() : null, repositoriesStats ? this.repositoriesService.getRepositoriesStats() : null, - admissionControl ? this.admissionControlService.stats() : null + admissionControl ? this.admissionControlService.stats() : null, + cacheService ? this.cacheService.stats(indices) : null ); } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java index 66b9afda06eb6..f62eaeb37f41f 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java @@ -36,6 +36,7 @@ import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.action.admin.indices.stats.CommonStatsFlags.Flag; import org.opensearch.client.node.NodeClient; +import org.opensearch.common.cache.CacheType; import org.opensearch.core.common.Strings; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; @@ -175,6 +176,25 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC nodesStatsRequest.indices(flags); } + } else if (metrics.contains("caches")) { + // Extract the list of caches we want to get stats for from the submetrics (which we get from index_metric) + Set cacheMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", "_all")); + CommonStatsFlags cacheFlags = new CommonStatsFlags(); + cacheFlags.clear(); + if (cacheMetrics.size() == 1 && cacheMetrics.contains("_all")) { + cacheFlags.includeAllCacheTypes(); + } else { + for (String cacheName : cacheMetrics) { + try { + cacheFlags.includeCacheType(CacheType.getByRepresentation(cacheName)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + unrecognized(request, Set.of(cacheName), CacheType.allRepresentations(), "cache type") + ); + } + } + } + nodesStatsRequest.indices(cacheFlags); } else if (request.hasParam("index_metric")) { throw new IllegalArgumentException( String.format( diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java index 1b8b6243aa805..468085b4dd6ca 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -42,6 +42,12 @@ import org.opensearch.cluster.routing.WeightedRoutingStats; import org.opensearch.cluster.service.ClusterManagerThrottlingStats; import org.opensearch.cluster.service.ClusterStateStats; +import org.opensearch.common.cache.CacheType; +import org.opensearch.common.cache.service.NodeCacheStats; +import org.opensearch.common.cache.stats.CacheStats; +import org.opensearch.common.cache.stats.CacheStatsCounterSnapshot; +import org.opensearch.common.cache.stats.MultiDimensionCacheStats; +import org.opensearch.common.cache.stats.StatsHolder; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.metrics.OperationStats; import org.opensearch.common.settings.ClusterSettings; @@ -87,6 +93,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -577,6 +584,13 @@ public void testSerialization() throws IOException { deserializedAdmissionControllerStats.getRejectionCount().get(AdmissionControlActionType.INDEXING.getType()) ); } + NodeCacheStats nodeCacheStats = nodeStats.getNodeCacheStats(); + NodeCacheStats deserializedNodeCacheStats = deserializedNodeStats.getNodeCacheStats(); + if (nodeCacheStats == null) { + assertNull(deserializedNodeCacheStats); + } else { + assertEquals(nodeCacheStats, deserializedNodeCacheStats); + } } } } @@ -928,6 +942,62 @@ public void apply(String action, AdmissionControlActionType admissionControlActi NodeIndicesStats indicesStats = getNodeIndicesStats(remoteStoreStats); + NodeCacheStats nodeCacheStats = null; + if (frequently()) { + int numIndices = randomIntBetween(1, 10); + int numShardsPerIndex = randomIntBetween(1, 50); + //Map snapshotMap = new HashMap<>(); + + List dimensionNames = List.of("index", "shard", "tier"); + StatsHolder statsHolder = new StatsHolder(dimensionNames, "dummyStoreName"); + for (int indexNum = 0; indexNum < numIndices; indexNum++) { + String indexName = "index" + indexNum; + for (int shardNum = 0; shardNum < numShardsPerIndex; shardNum++) { + String shardName = "[" + indexName + "][" + shardNum + "]"; + for (String tierName : new String[] { "dummy_tier_1", "dummy_tier_2" }) { + /*CacheStatsCounterSnapshot response = new CacheStatsCounterSnapshot( + randomInt(100), + randomInt(100), + randomInt(100), + randomInt(100), + randomInt(100) + );*/ + List dimensionValues = List.of(indexName, shardName, tierName); + for (int i = 0; i < randomInt(20); i++){ + statsHolder.incrementHits(dimensionValues); + } + for (int i = 0; i < randomInt(20); i++){ + statsHolder.incrementMisses(dimensionValues); + } + for (int i = 0; i < randomInt(20); i++){ + statsHolder.incrementEvictions(dimensionValues); + } + statsHolder.incrementSizeInBytes(dimensionValues, randomInt(20)); + for (int i = 0; i < randomInt(20); i++){ + statsHolder.incrementEntries(dimensionValues); + } + /*snapshotMap.put( + new StatsHolder.Key(List.of( + new CacheStatsDimension("testIndexDimensionName", indexName), + new CacheStatsDimension("testShardDimensionName", shardName), + new CacheStatsDimension("testTierDimensionName", tierName) + )), + response);*/ + } + } + } + CommonStatsFlags flags = new CommonStatsFlags(); + for (CacheType cacheType : CacheType.values()) { + if (frequently()) { + flags.includeCacheType(cacheType); + } + } + CacheStats cacheStats = statsHolder.getCacheStats(); + LinkedHashMap cacheStatsMap = new LinkedHashMap<>(); + cacheStatsMap.put(CacheType.INDICES_REQUEST_CACHE, cacheStats); + nodeCacheStats = new NodeCacheStats(cacheStatsMap, flags); + } + // TODO: Only remote_store based aspects of NodeIndicesStats are being tested here. // It is possible to test other metrics in NodeIndicesStats as well since it extends Writeable now return new NodeStats( @@ -958,7 +1028,8 @@ public void apply(String action, AdmissionControlActionType admissionControlActi null, segmentReplicationRejectionStats, null, - admissionControlStats + admissionControlStats, + nodeCacheStats ); } diff --git a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java index ff47ec3015697..5539dd26dd52d 100644 --- a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java +++ b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java @@ -194,6 +194,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ), new NodeStats( @@ -224,6 +225,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ), new NodeStats( @@ -254,6 +256,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ) ); @@ -315,6 +318,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ), new NodeStats( @@ -345,6 +349,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ), new NodeStats( @@ -375,6 +380,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ) ); diff --git a/server/src/test/java/org/opensearch/common/cache/stats/MultiDimensionCacheStatsTests.java b/server/src/test/java/org/opensearch/common/cache/stats/MultiDimensionCacheStatsTests.java index ca9354e663e14..34d30f7ab3552 100644 --- a/server/src/test/java/org/opensearch/common/cache/stats/MultiDimensionCacheStatsTests.java +++ b/server/src/test/java/org/opensearch/common/cache/stats/MultiDimensionCacheStatsTests.java @@ -10,8 +10,13 @@ import org.opensearch.common.Randomness; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; @@ -22,11 +27,13 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.function.BiConsumer; public class MultiDimensionCacheStatsTests extends OpenSearchTestCase { + private final String storeName = "dummy_store"; public void testSerialization() throws Exception { List dimensionNames = List.of("dim1", "dim2", "dim3"); - StatsHolder statsHolder = new StatsHolder(dimensionNames); + StatsHolder statsHolder = new StatsHolder(dimensionNames, storeName); Map> usedDimensionValues = getUsedDimensionValues(statsHolder, 10); populateStats(statsHolder, usedDimensionValues, 100, 10); MultiDimensionCacheStats stats = (MultiDimensionCacheStats) statsHolder.getCacheStats(); @@ -37,20 +44,39 @@ public void testSerialization() throws Exception { MultiDimensionCacheStats deserialized = new MultiDimensionCacheStats(is); assertEquals(stats.dimensionNames, deserialized.dimensionNames); - List> pathsInOriginal = new ArrayList<>(); - getAllPathsInTree(stats.getStatsRoot(), new ArrayList<>(), pathsInOriginal); - for (List path : pathsInOriginal) { - MultiDimensionCacheStats.MDCSDimensionNode originalNode = getNode(path, stats.statsRoot); - MultiDimensionCacheStats.MDCSDimensionNode deserializedNode = getNode(path, deserialized.statsRoot); - assertNotNull(deserializedNode); - assertEquals(originalNode.getDimensionValue(), deserializedNode.getDimensionValue()); - assertEquals(originalNode.getStats(), deserializedNode.getStats()); - } + assertEquals(stats.storeName, deserialized.storeName); + + os = new BytesStreamOutput(); + stats.writeToWithClassName(os); + is = new BytesStreamInput(BytesReference.toBytes(os.bytes())); + CacheStats deserializedViaCacheStats = CacheStats.readFromStreamWithClassName(is); + assertEquals(MultiDimensionCacheStats.class, deserializedViaCacheStats.getClass()); + + assertEquals(stats, deserialized); + assertEquals(stats, deserializedViaCacheStats); + } + + public void testEquals() throws Exception { + List dimensionNames = List.of("dim1", "dim2", "dim3"); + StatsHolder statsHolder = new StatsHolder(dimensionNames, storeName); + StatsHolder differentStoreNameStatsHolder = new StatsHolder(dimensionNames, "nonMatchingStoreName"); + StatsHolder nonMatchingStatsHolder = new StatsHolder(dimensionNames, storeName); + Map> usedDimensionValues = getUsedDimensionValues(statsHolder, 10); + populateStats(List.of(statsHolder, differentStoreNameStatsHolder), usedDimensionValues, 100, 10); + populateStats(nonMatchingStatsHolder, usedDimensionValues, 100, 10); + MultiDimensionCacheStats stats = (MultiDimensionCacheStats) statsHolder.getCacheStats(); + + MultiDimensionCacheStats secondStats = (MultiDimensionCacheStats) statsHolder.getCacheStats(); + assertEquals(stats, secondStats); + MultiDimensionCacheStats nonMatchingStats = (MultiDimensionCacheStats) nonMatchingStatsHolder.getCacheStats(); + assertNotEquals(stats, nonMatchingStats); + MultiDimensionCacheStats differentStoreNameStats = (MultiDimensionCacheStats) differentStoreNameStatsHolder.getCacheStats(); + assertNotEquals(stats, differentStoreNameStats); } public void testAddAndGet() throws Exception { List dimensionNames = List.of("dim1", "dim2", "dim3", "dim4"); - StatsHolder statsHolder = new StatsHolder(dimensionNames); + StatsHolder statsHolder = new StatsHolder(dimensionNames, storeName); Map> usedDimensionValues = getUsedDimensionValues(statsHolder, 10); Map, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 1000, 10); MultiDimensionCacheStats stats = (MultiDimensionCacheStats) statsHolder.getCacheStats(); @@ -61,7 +87,7 @@ public void testAddAndGet() throws Exception { CacheStatsCounterSnapshot actualStatsHolder = StatsHolderTests.getNode(dimensionValues, statsHolder.getStatsRoot()) .getStatsSnapshot(); - CacheStatsCounterSnapshot actualCacheStats = getNode(dimensionValues, stats.getStatsRoot()).getStats(); + CacheStatsCounterSnapshot actualCacheStats = getNode(dimensionValues, stats.getStatsRoot()).getStatsSnapshot(); assertEquals(expectedCounter.snapshot(), actualStatsHolder); assertEquals(expectedCounter.snapshot(), actualCacheStats); @@ -85,20 +111,20 @@ public void testAddAndGet() throws Exception { public void testEmptyDimsList() throws Exception { // If the dimension list is empty, the tree should have only the root node containing the total stats. - StatsHolder statsHolder = new StatsHolder(List.of()); + StatsHolder statsHolder = new StatsHolder(List.of(), storeName); Map> usedDimensionValues = getUsedDimensionValues(statsHolder, 100); populateStats(statsHolder, usedDimensionValues, 10, 100); MultiDimensionCacheStats stats = (MultiDimensionCacheStats) statsHolder.getCacheStats(); MultiDimensionCacheStats.MDCSDimensionNode statsRoot = stats.getStatsRoot(); assertEquals(0, statsRoot.children.size()); - assertEquals(stats.getTotalStats(), statsRoot.getStats()); + assertEquals(stats.getTotalStats(), statsRoot.getStatsSnapshot()); } public void testAggregateByAllDimensions() throws Exception { // Aggregating with all dimensions as levels should just give us the same values that were in the original map List dimensionNames = List.of("dim1", "dim2", "dim3", "dim4"); - StatsHolder statsHolder = new StatsHolder(dimensionNames); + StatsHolder statsHolder = new StatsHolder(dimensionNames, storeName); Map> usedDimensionValues = getUsedDimensionValues(statsHolder, 10); Map, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 1000, 10); MultiDimensionCacheStats stats = (MultiDimensionCacheStats) statsHolder.getCacheStats(); @@ -109,14 +135,14 @@ public void testAggregateByAllDimensions() throws Exception { for (String dimValue : expectedEntry.getKey()) { dimensionValues.add(dimValue); } - assertEquals(expectedEntry.getValue().snapshot(), getNode(dimensionValues, aggregated).getStats()); + assertEquals(expectedEntry.getValue().snapshot(), getNode(dimensionValues, aggregated).getStatsSnapshot()); } assertSumOfChildrenStats(aggregated); } public void testAggregateBySomeDimensions() throws Exception { List dimensionNames = List.of("dim1", "dim2", "dim3", "dim4"); - StatsHolder statsHolder = new StatsHolder(dimensionNames); + StatsHolder statsHolder = new StatsHolder(dimensionNames, storeName); Map> usedDimensionValues = getUsedDimensionValues(statsHolder, 10); Map, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 1000, 10); MultiDimensionCacheStats stats = (MultiDimensionCacheStats) statsHolder.getCacheStats(); @@ -142,13 +168,81 @@ public void testAggregateBySomeDimensions() throws Exception { expectedCounter.add(expected.get(expectedDims)); } } - assertEquals(expectedCounter.snapshot(), aggEntry.getValue().getStats()); + assertEquals(expectedCounter.snapshot(), aggEntry.getValue().getStatsSnapshot()); } assertSumOfChildrenStats(aggregated); } } } + public void testXContentForLevels() throws Exception { + List dimensionNames = List.of("A", "B", "C"); + + StatsHolder statsHolder = new StatsHolder(dimensionNames, storeName); + StatsHolderTests.populateStatsHolderFromStatsValueMap(statsHolder, Map.of( + List.of("A1", "B1", "C1"), new CacheStatsCounter(1, 1, 1, 1, 1), + List.of("A1", "B1", "C2"), new CacheStatsCounter(2, 2, 2, 2, 2), + List.of("A1", "B2", "C1"), new CacheStatsCounter(3, 3, 3, 3, 3), + List.of("A2", "B1", "C3"), new CacheStatsCounter(4, 4, 4, 4, 4) + )); + MultiDimensionCacheStats stats = (MultiDimensionCacheStats) statsHolder.getCacheStats(); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + ToXContent.Params params = ToXContent.EMPTY_PARAMS; + + builder.startObject(); + stats.toXContentForLevels(builder, params, List.of("A", "B", "C")); + builder.endObject(); + String resultString = builder.toString(); + Map result = XContentHelper.convertToMap(MediaTypeRegistry.JSON.xContent(), resultString, true); + + Map> fieldNamesMap = Map.of( + CacheStatsCounterSnapshot.Fields.MEMORY_SIZE_IN_BYTES, + (counter, value) -> counter.sizeInBytes.inc(value), + CacheStatsCounterSnapshot.Fields.EVICTIONS, + (counter, value) -> counter.evictions.inc(value), + CacheStatsCounterSnapshot.Fields.HIT_COUNT, + (counter, value) -> counter.hits.inc(value), + CacheStatsCounterSnapshot.Fields.MISS_COUNT, + (counter, value) -> counter.misses.inc(value), + CacheStatsCounterSnapshot.Fields.ENTRIES, + (counter, value) -> counter.entries.inc(value) + ); + + Map, MultiDimensionCacheStats.MDCSDimensionNode> leafNodes = getAllLeafNodes(stats.getStatsRoot()); + for (Map.Entry, MultiDimensionCacheStats.MDCSDimensionNode> entry : leafNodes.entrySet()) { + List xContentKeys = new ArrayList<>(); + for (int i = 0; i < dimensionNames.size(); i++) { + xContentKeys.add(dimensionNames.get(i)); + xContentKeys.add(entry.getKey().get(i)); + } + CacheStatsCounter counterFromXContent = new CacheStatsCounter(); + + for (Map.Entry> fieldNamesEntry : fieldNamesMap.entrySet()) { + List fullXContentKeys = new ArrayList<>(xContentKeys); + fullXContentKeys.add(fieldNamesEntry.getKey()); + int valueInXContent = (int) getValueFromNestedXContentMap(result, fullXContentKeys); + BiConsumer incrementer = fieldNamesEntry.getValue(); + incrementer.accept(counterFromXContent, valueInXContent); + } + + CacheStatsCounterSnapshot expected = entry.getValue().getStatsSnapshot(); + assertEquals(counterFromXContent.snapshot(), expected); + } + } + + public static Object getValueFromNestedXContentMap(Map xContentMap, List keys) { + Map current = xContentMap; + for (int i = 0; i < keys.size() - 1; i++) { + Object next = current.get(keys.get(i)); + if (next == null) { + return null; + } + current = (Map) next; + } + return current.get(keys.get(keys.size() - 1)); + } + // Get a map from the list of dimension values to the corresponding leaf node. private Map, MultiDimensionCacheStats.MDCSDimensionNode> getAllLeafNodes(MultiDimensionCacheStats.MDCSDimensionNode root) { Map, MultiDimensionCacheStats.MDCSDimensionNode> result = new HashMap<>(); @@ -176,9 +270,9 @@ private void assertSumOfChildrenStats(MultiDimensionCacheStats.MDCSDimensionNode if (current.hasChildren()) { CacheStatsCounter expectedTotal = new CacheStatsCounter(); for (MultiDimensionCacheStats.MDCSDimensionNode child : current.children.values()) { - expectedTotal.add(child.getStats()); + expectedTotal.add(child.getStatsSnapshot()); } - assertEquals(expectedTotal.snapshot(), current.getStats()); + assertEquals(expectedTotal.snapshot(), current.getStatsSnapshot()); for (MultiDimensionCacheStats.MDCSDimensionNode child : current.children.values()) { assertSumOfChildrenStats(child); } @@ -202,52 +296,47 @@ static Map, CacheStatsCounter> populateStats( Map> usedDimensionValues, int numDistinctValuePairs, int numRepetitionsPerValue + ) throws InterruptedException { + return populateStats(List.of(statsHolder), usedDimensionValues, numDistinctValuePairs, numRepetitionsPerValue); + } + + static Map, CacheStatsCounter> populateStats( + List statsHolders, + Map> usedDimensionValues, + int numDistinctValuePairs, + int numRepetitionsPerValue ) throws InterruptedException { Map, CacheStatsCounter> expected = new ConcurrentHashMap<>(); + for (StatsHolder statsHolder : statsHolders) { + assertEquals(statsHolders.get(0).getDimensionNames(), statsHolder.getDimensionNames()); + } Thread[] threads = new Thread[numDistinctValuePairs]; CountDownLatch countDownLatch = new CountDownLatch(numDistinctValuePairs); Random rand = Randomness.get(); List> dimensionsForThreads = new ArrayList<>(); for (int i = 0; i < numDistinctValuePairs; i++) { - dimensionsForThreads.add(getRandomDimList(statsHolder.getDimensionNames(), usedDimensionValues, true, rand)); + dimensionsForThreads.add(getRandomDimList(statsHolders.get(0).getDimensionNames(), usedDimensionValues, true, rand)); int finalI = i; threads[i] = new Thread(() -> { - Random threadRand = Randomness.get(); // TODO: This always has the same seed for each thread, causing only 1 set of values + Random threadRand = Randomness.get(); List dimensions = dimensionsForThreads.get(finalI); expected.computeIfAbsent(dimensions, (key) -> new CacheStatsCounter()); - - for (int j = 0; j < numRepetitionsPerValue; j++) { - int numHitIncrements = threadRand.nextInt(10); - for (int k = 0; k < numHitIncrements; k++) { - statsHolder.incrementHits(dimensions); - expected.get(dimensions).hits.inc(); - } - int numMissIncrements = threadRand.nextInt(10); - for (int k = 0; k < numMissIncrements; k++) { - statsHolder.incrementMisses(dimensions); - expected.get(dimensions).misses.inc(); - } - int numEvictionIncrements = threadRand.nextInt(10); - for (int k = 0; k < numEvictionIncrements; k++) { - statsHolder.incrementEvictions(dimensions); - expected.get(dimensions).evictions.inc(); - } - int numMemorySizeIncrements = threadRand.nextInt(10); - for (int k = 0; k < numMemorySizeIncrements; k++) { - long memIncrementAmount = threadRand.nextInt(5000); - statsHolder.incrementSizeInBytes(dimensions, memIncrementAmount); - expected.get(dimensions).sizeInBytes.inc(memIncrementAmount); - } - int numEntryIncrements = threadRand.nextInt(9) + 1; - for (int k = 0; k < numEntryIncrements; k++) { - statsHolder.incrementEntries(dimensions); - expected.get(dimensions).entries.inc(); - } - int numEntryDecrements = threadRand.nextInt(numEntryIncrements); - for (int k = 0; k < numEntryDecrements; k++) { - statsHolder.decrementEntries(dimensions); - expected.get(dimensions).entries.dec(); + for (StatsHolder statsHolder : statsHolders) { + for (int j = 0; j < numRepetitionsPerValue; j++) { + CacheStatsCounter statsToInc = new CacheStatsCounter( + threadRand.nextInt(10), + threadRand.nextInt(10), + threadRand.nextInt(10), + threadRand.nextInt(5000), + threadRand.nextInt(10) + ); + expected.get(dimensions).hits.inc(statsToInc.getHits()); + expected.get(dimensions).misses.inc(statsToInc.getMisses()); + expected.get(dimensions).evictions.inc(statsToInc.getEvictions()); + expected.get(dimensions).sizeInBytes.inc(statsToInc.getSizeInBytes()); + expected.get(dimensions).entries.inc(statsToInc.getEntries()); + StatsHolderTests.populateStatsHolderFromStatsValueMap(statsHolder, Map.of(dimensions, statsToInc)); } } countDownLatch.countDown(); @@ -277,22 +366,6 @@ private static List getRandomDimList( return result; } - private void getAllPathsInTree( - MultiDimensionCacheStats.MDCSDimensionNode currentNode, - List pathToCurrentNode, - List> allPaths - ) { - allPaths.add(pathToCurrentNode); - if (currentNode.getChildren() != null && !currentNode.getChildren().isEmpty()) { - // not a leaf node - for (MultiDimensionCacheStats.MDCSDimensionNode child : currentNode.getChildren().values()) { - List pathToChild = new ArrayList<>(pathToCurrentNode); - pathToChild.add(child.getDimensionValue()); - getAllPathsInTree(child, pathToChild, allPaths); - } - } - } - private MultiDimensionCacheStats.MDCSDimensionNode getNode( List dimensionValues, MultiDimensionCacheStats.MDCSDimensionNode root diff --git a/server/src/test/java/org/opensearch/common/cache/stats/StatsHolderTests.java b/server/src/test/java/org/opensearch/common/cache/stats/StatsHolderTests.java index 05e5851ce9a50..dba3846fa4630 100644 --- a/server/src/test/java/org/opensearch/common/cache/stats/StatsHolderTests.java +++ b/server/src/test/java/org/opensearch/common/cache/stats/StatsHolderTests.java @@ -22,10 +22,11 @@ public class StatsHolderTests extends OpenSearchTestCase { // Since StatsHolder does not expose getter methods for aggregating stats, // we test the incrementing functionality in combination with MultiDimensionCacheStats, // in MultiDimensionCacheStatsTests.java. + private final String storeName = "dummy_store"; public void testReset() throws Exception { List dimensionNames = List.of("dim1", "dim2"); - StatsHolder statsHolder = new StatsHolder(dimensionNames); + StatsHolder statsHolder = new StatsHolder(dimensionNames, storeName); Map> usedDimensionValues = getUsedDimensionValues(statsHolder, 10); Map, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 100, 10); @@ -44,7 +45,7 @@ public void testReset() throws Exception { public void testDropStatsForDimensions() throws Exception { List dimensionNames = List.of("dim1", "dim2"); - StatsHolder statsHolder = new StatsHolder(dimensionNames); + StatsHolder statsHolder = new StatsHolder(dimensionNames, storeName); // Create stats for the following dimension sets List> populatedStats = List.of(List.of("A1", "B1"), List.of("A2", "B2"), List.of("A2", "B3")); @@ -80,7 +81,7 @@ public void testDropStatsForDimensions() throws Exception { public void testCount() throws Exception { List dimensionNames = List.of("dim1", "dim2"); - StatsHolder statsHolder = new StatsHolder(dimensionNames); + StatsHolder statsHolder = new StatsHolder(dimensionNames, storeName); Map> usedDimensionValues = getUsedDimensionValues(statsHolder, 10); Map, CacheStatsCounter> expected = populateStats(statsHolder, usedDimensionValues, 100, 10); @@ -93,7 +94,7 @@ public void testCount() throws Exception { public void testConcurrentRemoval() throws Exception { List dimensionNames = List.of("dim1", "dim2"); - StatsHolder statsHolder = new StatsHolder(dimensionNames); + StatsHolder statsHolder = new StatsHolder(dimensionNames, storeName); // Create stats for the following dimension sets List> populatedStats = List.of(List.of("A1", "B1"), List.of("A2", "B2"), List.of("A2", "B3")); @@ -151,4 +152,24 @@ static DimensionNode getNode(List dimensionValues, DimensionNode root) { } return current; } + + static void populateStatsHolderFromStatsValueMap(StatsHolder statsHolder, Map, CacheStatsCounter> statsMap) { + for (Map.Entry, CacheStatsCounter> entry : statsMap.entrySet()) { + CacheStatsCounter stats = entry.getValue(); + List dims = entry.getKey(); + for (int i = 0; i < stats.getHits(); i++) { + statsHolder.incrementHits(dims); + } + for (int i = 0; i < stats.getMisses(); i++) { + statsHolder.incrementMisses(dims); + } + for (int i = 0; i < stats.getEvictions(); i++) { + statsHolder.incrementEvictions(dims); + } + statsHolder.incrementSizeInBytes(dims, stats.getSizeInBytes()); + for (int i = 0; i < stats.getEntries(); i++) { + statsHolder.incrementEntries(dims); + } + } + } } diff --git a/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java b/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java index 84d4c823e640b..1c6017aa9fb0c 100644 --- a/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java +++ b/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java @@ -8,20 +8,31 @@ package org.opensearch.common.cache.store; +import org.opensearch.common.Randomness; import org.opensearch.common.cache.CacheType; import org.opensearch.common.cache.ICache; import org.opensearch.common.cache.ICacheKey; import org.opensearch.common.cache.LoadAwareCacheLoader; import org.opensearch.common.cache.RemovalListener; import org.opensearch.common.cache.RemovalNotification; +import org.opensearch.common.cache.stats.CacheStats; +import org.opensearch.common.cache.stats.MultiDimensionCacheStatsTests; import org.opensearch.common.cache.store.config.CacheConfig; import org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings; import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.OpenSearchTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Random; import java.util.UUID; import static org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES_KEY; @@ -96,6 +107,73 @@ private OpenSearchOnHeapCache getCache(int maxSizeKeys, MockRemo return (OpenSearchOnHeapCache) onHeapCacheFactory.create(cacheConfig, CacheType.INDICES_REQUEST_CACHE, null); } + public void testInvalidateWithDropDimensions() throws Exception { + MockRemovalListener listener = new MockRemovalListener<>(); + int maxKeys = 50; + OpenSearchOnHeapCache cache = getCache(maxKeys, listener); + + List> keysAdded = new ArrayList<>(); + + for (int i = 0; i < maxKeys - 5; i++) { + ICacheKey key = new ICacheKey<>(UUID.randomUUID().toString(), getRandomDimensions()); + keysAdded.add(key); + cache.computeIfAbsent(key, getLoadAwareCacheLoader()); + } + + ICacheKey keyToDrop = keysAdded.get(0); + + Map xContentMap = getStatsXContentMap(cache.stats(), dimensionNames); + List xContentMapKeys = getXContentMapKeys(keyToDrop, dimensionNames); + Map individualSnapshotMap = (Map) MultiDimensionCacheStatsTests.getValueFromNestedXContentMap( + xContentMap, + xContentMapKeys + ); + assertNotNull(individualSnapshotMap); + assertEquals(5, individualSnapshotMap.size()); // Assert all 5 stats are present and not null + for (Map.Entry entry : individualSnapshotMap.entrySet()) { + Integer value = (Integer) entry.getValue(); + assertNotNull(value); + } + + keyToDrop.setDropStatsForDimensions(true); + cache.invalidate(keyToDrop); + + // Now assert the stats are gone for any key that has this combination of dimensions, but still there otherwise + xContentMap = getStatsXContentMap(cache.stats(), dimensionNames); + for (ICacheKey keyAdded : keysAdded) { + xContentMapKeys = getXContentMapKeys(keyAdded, dimensionNames); + individualSnapshotMap = (Map) MultiDimensionCacheStatsTests.getValueFromNestedXContentMap( + xContentMap, + xContentMapKeys + ); + if (keyAdded.dimensions.equals(keyToDrop.dimensions)) { + assertNull(individualSnapshotMap); + } else { + assertNotNull(individualSnapshotMap); + } + } + } + + private List getXContentMapKeys(ICacheKey iCacheKey, List dimensionNames) { + List result = new ArrayList<>(); + assert iCacheKey.dimensions.size() == dimensionNames.size(); + for (int i = 0; i < dimensionNames.size(); i++) { + result.add(dimensionNames.get(i)); + result.add(iCacheKey.dimensions.get(i)); + } + return result; + } + + private List getRandomDimensions() { + Random rand = Randomness.get(); + int bound = 3; + List result = new ArrayList<>(); + for (String dimName : dimensionNames) { + result.add(String.valueOf(rand.nextInt(bound))); + } + return result; + } + private static class MockRemovalListener implements RemovalListener, V> { CounterMetric numRemovals; @@ -109,6 +187,20 @@ public void onRemoval(RemovalNotification, V> notification) { } } + // Public as this is used in other tests as well + public static Map getStatsXContentMap(CacheStats cacheStats, List levels) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + Map paramMap = Map.of("level", String.join(",", levels)); + ToXContent.Params params = new ToXContent.MapParams(paramMap); + + builder.startObject(); + cacheStats.toXContent(builder, params); + builder.endObject(); + + String resultString = builder.toString(); + return XContentHelper.convertToMap(MediaTypeRegistry.JSON.xContent(), resultString, true); + } + private ICacheKey getICacheKey(String key) { List dims = new ArrayList<>(); for (String dimName : dimensionNames) { diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java index 6143eeb5f13e4..9d4d2ba4aa52d 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java @@ -45,12 +45,15 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.CheckedSupplier; import org.opensearch.common.cache.ICacheKey; import org.opensearch.common.cache.RemovalNotification; import org.opensearch.common.cache.RemovalReason; import org.opensearch.common.cache.module.CacheModule; import org.opensearch.common.cache.service.CacheService; +import org.opensearch.common.cache.stats.MultiDimensionCacheStatsTests; +import org.opensearch.common.cache.store.OpenSearchOnHeapCacheTests; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lucene.index.OpenSearchDirectoryReader; import org.opensearch.common.settings.Settings; @@ -70,6 +73,7 @@ import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardState; +import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.node.Node; import org.opensearch.test.OpenSearchSingleNodeTestCase; import org.opensearch.threadpool.ThreadPool; @@ -77,11 +81,15 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; +import static org.opensearch.indices.IndicesRequestCache.INDEX_DIMENSION_NAME; import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING; +import static org.opensearch.indices.IndicesRequestCache.SHARD_ID_DIMENSION_NAME; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -753,6 +761,125 @@ public void testCacheCleanupBasedOnStaleThreshold_StalenessLesserThanThreshold() terminate(threadPool); } + public void testClosingIndexWipesStats() throws Exception { + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + // Create two indices each with multiple shards + int numShards = 3; + Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build(); + String indexToKeepName = "test"; + String indexToCloseName = "test2"; + IndexService indexToKeep = createIndex(indexToKeepName, indexSettings); + IndexService indexToClose = createIndex(indexToCloseName, indexSettings); + for (int i = 0; i < numShards; i++) { + // Check we can get all the shards we expect + assertNotNull(indexToKeep.getShard(i)); + assertNotNull(indexToClose.getShard(i)); + } + ThreadPool threadPool = getThreadPool(); + Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.001%").build(); + IndicesRequestCache cache = new IndicesRequestCache(settings, (shardId -> { + IndexService indexService = null; + try { + indexService = indicesService.indexServiceSafe(shardId.getIndex()); + } catch (IndexNotFoundException ex) { + return Optional.empty(); + } + try { + return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id()))); + } catch (ShardNotFoundException ex) { + return Optional.empty(); + } + }), new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(), threadPool); + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig()); + + writer.addDocument(newDoc(0, "foo")); + TermQueryBuilder termQuery = new TermQueryBuilder("id", "0"); + BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false); + if (randomBoolean()) { + writer.flush(); + IOUtils.close(writer); + writer = new IndexWriter(dir, newIndexWriterConfig()); + } + writer.updateDocument(new Term("id", "0"), newDoc(0, "bar")); + DirectoryReader secondReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1)); + + List readersToClose = new ArrayList<>(); + List readersToKeep = new ArrayList<>(); + // Put entries into the cache for each shard + for (IndexService indexService : new IndexService[] { indexToKeep, indexToClose }) { + for (int i = 0; i < numShards; i++) { + IndexShard indexShard = indexService.getShard(i); + IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard); + DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), indexShard.shardId()); + if (indexService == indexToClose) { + readersToClose.add(reader); + } else { + readersToKeep.add(reader); + } + Loader loader = new Loader(reader, 0); + cache.getOrCompute(entity, loader, reader, termBytes); + } + } + + // Check resulting stats + List dimensionNames = List.of(INDEX_DIMENSION_NAME, SHARD_ID_DIMENSION_NAME); + Map xContentMap = OpenSearchOnHeapCacheTests.getStatsXContentMap(cache.getCacheStats(), dimensionNames); + List> initialXContentPaths = new ArrayList<>(); + for (IndexService indexService : new IndexService[] { indexToKeep, indexToClose }) { + for (int i = 0; i < numShards; i++) { + ShardId shardId = indexService.getShard(i).shardId(); + List xContentPath = List.of( + INDEX_DIMENSION_NAME, + shardId.getIndexName(), + SHARD_ID_DIMENSION_NAME, + shardId.toString() + ); + initialXContentPaths.add(xContentPath); + Map individualSnapshotMap = (Map) MultiDimensionCacheStatsTests + .getValueFromNestedXContentMap(xContentMap, xContentPath); + assertNotNull(individualSnapshotMap); + // check the values are not empty by confirming entries != 0, this should always be true since the missed value is loaded + // into the cache + assertNotEquals(0, (int) individualSnapshotMap.get("entries")); + } + } + + // Delete an index + indexToClose.close("test_deletion", true); + // This actually closes the shards associated with the readers, which is necessary for cache cleanup logic + // In this UT, manually close the readers as well; could not figure out how to connect all this up in a UT so that + // we could get readers that were properly connected to an index's directory + for (DirectoryReader reader : readersToClose) { + IOUtils.close(reader); + } + // Trigger cache cleanup + cache.cacheCleanupManager.cleanCache(); + + // Now stats for the closed index should be gone + xContentMap = OpenSearchOnHeapCacheTests.getStatsXContentMap(cache.getCacheStats(), dimensionNames); + for (List path : initialXContentPaths) { + Map individualSnapshotMap = (Map) MultiDimensionCacheStatsTests.getValueFromNestedXContentMap( + xContentMap, + path + ); + if (path.get(1).equals(indexToCloseName)) { + assertNull(individualSnapshotMap); + } else { + assertNotNull(individualSnapshotMap); + // check the values are not empty by confirming entries != 0, this should always be true since the missed value is loaded + // into the cache + assertNotEquals(0, (int) individualSnapshotMap.get("entries")); + } + } + + for (DirectoryReader reader : readersToKeep) { + IOUtils.close(reader); + } + IOUtils.close(secondReader, writer, dir, cache); + terminate(threadPool); + } + public void testEviction() throws Exception { final ByteSizeValue size; { diff --git a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java index 1ad6083074025..35ca5d80aeb4e 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java @@ -124,7 +124,8 @@ List adjustNodesStats(List nodesStats) { nodeStats.getSearchPipelineStats(), nodeStats.getSegmentReplicationRejectionStats(), nodeStats.getRepositoriesStats(), - nodeStats.getAdmissionControlStats() + nodeStats.getAdmissionControlStats(), + nodeStats.getNodeCacheStats() ); }).collect(Collectors.toList()); } diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index c2b964aa96212..ca80c65e58522 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -2736,6 +2736,7 @@ public void ensureEstimatedStats() { false, false, false, + false, false ); assertThat(