From c74d37205f59fdd5b56223bb07956eafdae4ae1e Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 20 Jun 2022 16:13:09 -0400 Subject: [PATCH 01/11] Add processors to autoscaling capacity response --- .../xpack/autoscaling/Autoscaling.java | 6 +- ...TransportGetAutoscalingCapacityAction.java | 6 +- .../AutoscalingCalculateCapacityService.java | 21 +- .../capacity/AutoscalingCapacity.java | 95 ++++++-- .../FixedAutoscalingDeciderService.java | 14 +- ...=> AutoscalingMemoryAndProcessorInfo.java} | 12 +- ...scalingMemoryAndProcessorInfoService.java} | 119 ++++++---- .../capacity/memory/MemoryAndProcessors.java | 44 ++++ .../FrozenExistenceDeciderService.java | 6 +- .../shards/FrozenShardsDeciderService.java | 5 +- .../storage/FrozenStorageDeciderService.java | 5 +- .../ProactiveStorageDeciderService.java | 4 +- .../ReactiveStorageDeciderService.java | 19 +- .../autoscaling/AutoscalingTestCase.java | 32 ++- ...oscalingCalculateCapacityServiceTests.java | 26 ++- ...scalingCapacityWireSerializationTests.java | 74 ++++-- .../AutoscalingDeciderResultsTests.java | 39 +++- .../FixedAutoscalingDeciderServiceTests.java | 19 +- ...ngMemoryAndProcessorInfoServiceTests.java} | 220 +++++++++++++----- .../ReactiveStorageDeciderDecisionTests.java | 13 +- .../MlAutoscalingDeciderService.java | 6 +- .../ml/autoscaling/NativeMemoryCapacity.java | 8 +- .../MlAutoscalingDeciderServiceTests.java | 52 +++-- .../ml/autoscaling/MlScalingReasonTests.java | 3 +- 24 files changed, 612 insertions(+), 236 deletions(-) rename x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/{AutoscalingMemoryInfo.java => AutoscalingMemoryAndProcessorInfo.java} (52%) rename x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/{AutoscalingMemoryInfoService.java => AutoscalingMemoryAndProcessorInfoService.java} (55%) create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/MemoryAndProcessors.java rename x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/{AutoscalingMemoryInfoServiceTests.java => AutoscalingMemoryAndProcessorInfoServiceTests.java} (64%) diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java index fa252d0763feb..af7e288cebe51 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java @@ -50,7 +50,7 @@ import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResult; import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderService; import org.elasticsearch.xpack.autoscaling.capacity.FixedAutoscalingDeciderService; -import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryInfoService; +import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryAndProcessorInfoService; import org.elasticsearch.xpack.autoscaling.existence.FrozenExistenceDeciderService; import org.elasticsearch.xpack.autoscaling.rest.RestDeleteAutoscalingPolicyHandler; import org.elasticsearch.xpack.autoscaling.rest.RestGetAutoscalingCapacityHandler; @@ -119,13 +119,13 @@ public Collection createComponents( return List.of( new AutoscalingCalculateCapacityService.Holder(this), autoscalingLicenseChecker, - new AutoscalingMemoryInfoService(clusterService, client) + new AutoscalingMemoryAndProcessorInfoService(clusterService, client) ); } @Override public List> getSettings() { - return List.of(AutoscalingMemoryInfoService.FETCH_TIMEOUT); + return List.of(AutoscalingMemoryAndProcessorInfoService.FETCH_TIMEOUT); } @Override diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java index 10b5d610ea1cd..06cf78234263a 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java @@ -27,7 +27,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingLicenseChecker; import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingCalculateCapacityService; -import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryInfoService; +import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryAndProcessorInfoService; import java.util.Objects; @@ -40,7 +40,7 @@ public class TransportGetAutoscalingCapacityAction extends TransportMasterNodeAc private final AutoscalingCalculateCapacityService capacityService; private final ClusterInfoService clusterInfoService; private final SnapshotsInfoService snapshotsInfoService; - private final AutoscalingMemoryInfoService memoryInfoService; + private final AutoscalingMemoryAndProcessorInfoService memoryInfoService; private final AutoscalingLicenseChecker autoscalingLicenseChecker; private final CapacityResponseCache responseCache = new CapacityResponseCache<>( run -> threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(run), @@ -57,7 +57,7 @@ public TransportGetAutoscalingCapacityAction( final AutoscalingCalculateCapacityService.Holder capacityServiceHolder, final ClusterInfoService clusterInfoService, final SnapshotsInfoService snapshotsInfoService, - final AutoscalingMemoryInfoService memoryInfoService, + final AutoscalingMemoryAndProcessorInfoService memoryInfoService, final AllocationDeciders allocationDeciders, final AutoscalingLicenseChecker autoscalingLicenseChecker ) { diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java index 0afea2078b05d..72a94e630af3c 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java @@ -23,12 +23,14 @@ import org.elasticsearch.xpack.autoscaling.Autoscaling; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; import org.elasticsearch.xpack.autoscaling.action.PolicyValidator; -import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryInfo; +import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryAndProcessorInfo; +import org.elasticsearch.xpack.autoscaling.capacity.memory.MemoryAndProcessors; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicy; import java.util.Collections; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -107,7 +109,7 @@ public SortedMap calculate( ClusterState state, ClusterInfo clusterInfo, SnapshotShardSizeInfo shardSizeInfo, - AutoscalingMemoryInfo memoryInfo, + AutoscalingMemoryAndProcessorInfo memoryInfo, Runnable ensureNotCancelled ) { AutoscalingMetadata autoscalingMetadata = state.metadata().custom(AutoscalingMetadata.NAME); @@ -134,7 +136,7 @@ private AutoscalingDeciderResults calculateForPolicy( ClusterState state, ClusterInfo clusterInfo, SnapshotShardSizeInfo shardSizeInfo, - AutoscalingMemoryInfo memoryInfo, + AutoscalingMemoryAndProcessorInfo memoryInfo, Runnable ensureNotCancelled ) { if (hasUnknownRoles(policy)) { @@ -191,7 +193,7 @@ DefaultAutoscalingDeciderContext createContext( ClusterState state, ClusterInfo clusterInfo, SnapshotShardSizeInfo shardSizeInfo, - AutoscalingMemoryInfo memoryInfo, + AutoscalingMemoryAndProcessorInfo memoryInfo, Runnable ensureNotCancelled ) { return new DefaultAutoscalingDeciderContext(roles, state, clusterInfo, shardSizeInfo, memoryInfo, ensureNotCancelled); @@ -223,7 +225,7 @@ static class DefaultAutoscalingDeciderContext implements AutoscalingDeciderConte private final ClusterState state; private final ClusterInfo clusterInfo; private final SnapshotShardSizeInfo snapshotShardSizeInfo; - private final AutoscalingMemoryInfo memoryInfo; + private final AutoscalingMemoryAndProcessorInfo memoryInfo; private final SortedSet currentNodes; private final AutoscalingCapacity currentCapacity; private final boolean currentCapacityAccurate; @@ -234,7 +236,7 @@ static class DefaultAutoscalingDeciderContext implements AutoscalingDeciderConte ClusterState state, ClusterInfo clusterInfo, SnapshotShardSizeInfo snapshotShardSizeInfo, - AutoscalingMemoryInfo memoryInfo, + AutoscalingMemoryAndProcessorInfo memoryInfo, Runnable ensureNotCancelled ) { this.roles = roles.stream().map(DiscoveryNodeRole::getRoleFromRoleName).collect(Sets.toUnmodifiableSortedSet()); @@ -295,7 +297,7 @@ private boolean nodeHasAccurateCapacity(DiscoveryNode node) { } } - return memoryInfo.get(node) != null; + return memoryInfo.get(node).isPresent(); } private AutoscalingCapacity calculateCurrentCapacity() { @@ -319,10 +321,11 @@ private AutoscalingCapacity.AutoscalingResources resourcesFor(DiscoveryNode node ) : 0L; - Long memory = memoryInfo.get(node); + Optional memoryAndProcessors = memoryInfo.get(node); return new AutoscalingCapacity.AutoscalingResources( storage == -1 ? ByteSizeValue.ZERO : new ByteSizeValue(storage), - memory == null ? ByteSizeValue.ZERO : new ByteSizeValue(memory) + memoryAndProcessors.map(MemoryAndProcessors::memory).map(ByteSizeValue::new).orElse(ByteSizeValue.ZERO), + memoryAndProcessors.map(MemoryAndProcessors::processors).orElse(0) ); } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java index f14732c713f7c..03295d680d06d 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java @@ -7,16 +7,19 @@ package org.elasticsearch.xpack.autoscaling.capacity; +import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.Objects; +import java.util.OptionalInt; /** * Represents current/required capacity of a single tier. @@ -29,18 +32,28 @@ public class AutoscalingCapacity implements ToXContent, Writeable { public static class AutoscalingResources implements ToXContent, Writeable { private final ByteSizeValue storage; private final ByteSizeValue memory; + private final Integer processors; - public static final AutoscalingResources ZERO = new AutoscalingResources(new ByteSizeValue(0), new ByteSizeValue(0)); + public static final AutoscalingResources ZERO = new AutoscalingResources(new ByteSizeValue(0), new ByteSizeValue(0), 0); - public AutoscalingResources(ByteSizeValue storage, ByteSizeValue memory) { - assert storage != null || memory != null; + public AutoscalingResources(ByteSizeValue storage, ByteSizeValue memory, Integer processors) { + assert storage != null || memory != null || processors != null; this.storage = storage; this.memory = memory; + if (processors != null && processors < 0) { + throw new IllegalArgumentException("[processors] must be a non-negative number; provided [" + processors + "]"); + } + this.processors = processors; } public AutoscalingResources(StreamInput in) throws IOException { this.storage = in.readOptionalWriteable(ByteSizeValue::new); this.memory = in.readOptionalWriteable(ByteSizeValue::new); + if (in.getVersion().onOrAfter(Version.V_8_4_0)) { + this.processors = in.readOptionalVInt(); + } else { + this.processors = null; + } } public ByteSizeValue storage() { @@ -51,6 +64,15 @@ public ByteSizeValue memory() { return memory; } + public OptionalInt processors() { + return processors == null ? OptionalInt.empty() : OptionalInt.of(processors); + } + + @Nullable + Integer getProcessors() { + return processors; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -60,6 +82,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (memory != null) { builder.field("memory", memory.getBytes()); } + if (processors != null) { + builder.field("processors", processors); + } builder.endObject(); return builder; } @@ -73,6 +98,9 @@ public boolean isFragment() { public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(storage); out.writeOptionalWriteable(memory); + if (out.getVersion().onOrAfter(Version.V_8_4_0)) { + out.writeOptionalVInt(processors); + } } public static AutoscalingResources max(AutoscalingResources sm1, AutoscalingResources sm2) { @@ -83,7 +111,11 @@ public static AutoscalingResources max(AutoscalingResources sm1, AutoscalingReso return sm1; } - return new AutoscalingResources(max(sm1.storage, sm2.storage), max(sm1.memory, sm2.memory)); + return new AutoscalingResources( + max(sm1.storage, sm2.storage), + max(sm1.memory, sm2.memory), + max(sm1.processors, sm2.processors) + ); } public static AutoscalingResources sum(AutoscalingResources sm1, AutoscalingResources sm2) { @@ -94,7 +126,11 @@ public static AutoscalingResources sum(AutoscalingResources sm1, AutoscalingReso return sm1; } - return new AutoscalingResources(add(sm1.storage, sm2.storage), add(sm1.memory, sm2.memory)); + return new AutoscalingResources( + add(sm1.storage, sm2.storage), + add(sm1.memory, sm2.memory), + add(sm1.processors, sm2.processors) + ); } private static ByteSizeValue max(ByteSizeValue v1, ByteSizeValue v2) { @@ -119,17 +155,41 @@ private static ByteSizeValue add(ByteSizeValue v1, ByteSizeValue v2) { return new ByteSizeValue(v1.getBytes() + v2.getBytes()); } + private static Integer max(Integer v1, Integer v2) { + if (v1 == null) { + return v2; + } + if (v2 == null) { + return v1; + } + + return v1.compareTo(v2) < 0 ? v2 : v1; + } + + private static Integer add(Integer v1, Integer v2) { + if (v1 == null) { + return v2; + } + if (v2 == null) { + return v1; + } + + return v1 + v2; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; AutoscalingResources that = (AutoscalingResources) o; - return Objects.equals(storage, that.storage) && Objects.equals(memory, that.memory); + return Objects.equals(storage, that.storage) + && Objects.equals(memory, that.memory) + && Objects.equals(processors, that.processors); } @Override public int hashCode() { - return Objects.hash(storage, memory); + return Objects.hash(storage, memory, processors); } @Override @@ -147,7 +207,10 @@ public AutoscalingCapacity(AutoscalingResources total, AutoscalingResources node || total.memory != null : "Cannot provide node memory without total memory"; assert node == null || node.storage == null // implies - || total.storage != null : "Cannot provide node storage without total memory"; + || total.storage != null : "Cannot provide node storage without total storage"; + assert node == null || node.processors == null + // implies + || total.processors != null : "Cannot provide node processors without total processors"; this.total = total; this.node = node; @@ -226,12 +289,12 @@ public Builder capacity(AutoscalingCapacity capacity) { return this; } - public Builder total(Long storage, Long memory) { - return total(byteSizeValue(storage), byteSizeValue(memory)); + public Builder total(Long storage, Long memory, Integer processors) { + return total(byteSizeValue(storage), byteSizeValue(memory), processors); } - public Builder total(ByteSizeValue storage, ByteSizeValue memory) { - return total(new AutoscalingResources(storage, memory)); + public Builder total(ByteSizeValue storage, ByteSizeValue memory, Integer processors) { + return total(new AutoscalingResources(storage, memory, processors)); } public Builder total(AutoscalingResources total) { @@ -239,12 +302,12 @@ public Builder total(AutoscalingResources total) { return this; } - public Builder node(Long storage, Long memory) { - return node(byteSizeValue(storage), byteSizeValue(memory)); + public Builder node(Long storage, Long memory, Integer processors) { + return node(byteSizeValue(storage), byteSizeValue(memory), processors); } - public Builder node(ByteSizeValue storage, ByteSizeValue memory) { - return node(new AutoscalingResources(storage, memory)); + public Builder node(ByteSizeValue storage, ByteSizeValue memory, Integer processors) { + return node(new AutoscalingResources(storage, memory, processors)); } public Builder node(AutoscalingResources node) { diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java index 69e1589f5eedc..b3ef8f9f6f107 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java @@ -26,6 +26,7 @@ public class FixedAutoscalingDeciderService implements AutoscalingDeciderService public static final Setting STORAGE = Setting.byteSizeSetting("storage", ByteSizeValue.ofBytes(-1)); public static final Setting MEMORY = Setting.byteSizeSetting("memory", ByteSizeValue.ofBytes(-1)); + public static final Setting PROCESSORS = Setting.intSetting("processors", 1, 0); public static final Setting NODES = Setting.intSetting("nodes", 1, 0); @Inject @@ -44,10 +45,11 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider AutoscalingCapacity requiredCapacity; ByteSizeValue storage = STORAGE.exists(configuration) ? STORAGE.get(configuration) : null; ByteSizeValue memory = MEMORY.exists(configuration) ? MEMORY.get(configuration) : null; + Integer processors = PROCESSORS.exists(configuration) ? PROCESSORS.get(configuration) : null; if (storage != null || memory != null) { requiredCapacity = AutoscalingCapacity.builder() - .total(totalCapacity(storage, nodes), totalCapacity(memory, nodes)) - .node(storage, memory) + .total(totalCapacity(storage, nodes), totalCapacity(memory, nodes), totalCapacity(processors, nodes)) + .node(storage, memory, processors) .build(); } else { requiredCapacity = null; @@ -64,6 +66,14 @@ private static ByteSizeValue totalCapacity(ByteSizeValue nodeCapacity, int nodes } } + private static Integer totalCapacity(Integer nodeCapacity, int nodes) { + if (nodeCapacity != null) { + return nodeCapacity * nodes; + } else { + return null; + } + } + @Override public List> deciderSettings() { return List.of(STORAGE, MEMORY, NODES); diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfo.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfo.java similarity index 52% rename from x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfo.java rename to x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfo.java index cfb81cc79e952..a1946fe1349eb 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfo.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfo.java @@ -9,13 +9,15 @@ import org.elasticsearch.cluster.node.DiscoveryNode; -public interface AutoscalingMemoryInfo { - AutoscalingMemoryInfo EMPTY = n -> null; +import java.util.Optional; + +public interface AutoscalingMemoryAndProcessorInfo { + AutoscalingMemoryAndProcessorInfo EMPTY = n -> Optional.empty(); /** - * Get the memory use for the indicated node. Returns null if not available (new, fetching or failed). + * Get the memory and processor use for the indicated node. Returns null if not available (new, fetching or failed). * @param node the node to get info for - * @return info for node. + * @return memory and processor info for node if possible */ - Long get(DiscoveryNode node); + Optional get(DiscoveryNode node); } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfoService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfoService.java similarity index 55% rename from x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfoService.java rename to x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfoService.java index 1f0d6cbf46d21..d8442f81f398a 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfoService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfoService.java @@ -11,9 +11,9 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.FailedNodeException; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; @@ -22,42 +22,47 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.monitor.os.OsInfo; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicy; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicyMetadata; import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.stream.StreamSupport; import static java.util.stream.Collectors.toUnmodifiableMap; -public class AutoscalingMemoryInfoService { +public class AutoscalingMemoryAndProcessorInfoService { public static final Setting FETCH_TIMEOUT = Setting.timeSetting( "xpack.autoscaling.memory.monitor.timeout", TimeValue.timeValueSeconds(15), Setting.Property.Dynamic, Setting.Property.NodeScope ); - private static final Long FETCHING_SENTINEL = Long.MIN_VALUE; + private static final MemoryAndProcessors FETCHING_SENTINEL = new MemoryAndProcessors(Long.MIN_VALUE, Integer.MIN_VALUE); - private static final Logger logger = LogManager.getLogger(AutoscalingMemoryInfoService.class); + private static final Logger logger = LogManager.getLogger(AutoscalingMemoryAndProcessorInfoService.class); - private volatile Map nodeToMemory = Map.of(); + private volatile Map nodeToMemory = Map.of(); private volatile TimeValue fetchTimeout; private final Client client; private final Object mutex = new Object(); @Inject - public AutoscalingMemoryInfoService(ClusterService clusterService, Client client) { + public AutoscalingMemoryAndProcessorInfoService(ClusterService clusterService, Client client) { this.client = client; this.fetchTimeout = FETCH_TIMEOUT.get(clusterService.getSettings()); if (DiscoveryNode.isMasterNode(clusterService.getSettings())) { @@ -115,34 +120,75 @@ private void sendToMissingNodes(Function nodeLookup, Set< new NodesStatsRequest(missingNodes.stream().map(DiscoveryNode::getId).toArray(String[]::new)).clear() .addMetric(NodesStatsRequest.Metric.OS.metricName()) .timeout(fetchTimeout), - new ActionListener() { - @Override - public void onResponse(NodesStatsResponse nodesStatsResponse) { - synchronized (mutex) { - Map builder = new HashMap<>(nodeToMemory); - nodesStatsResponse.failures() - .stream() - .map(FailedNodeException::nodeId) - .map(nodeLookup) - .map(DiscoveryNode::getEphemeralId) - .forEach(builder::remove); - - nodesStatsResponse.getNodes().forEach(nodeStats -> addNodeStats(builder, nodeStats)); - nodeToMemory = Collections.unmodifiableMap(builder); - } - } - - @Override - public void onFailure(Exception e) { + ActionListener.wrap( + nodesStatsResponse -> client.admin() + .cluster() + .nodesInfo( + // Only gather info for nodes that provided their stats + new NodesInfoRequest( + nodesStatsResponse.getNodes() + .stream() + .map(BaseNodeResponse::getNode) + .map(DiscoveryNode::getId) + .toArray(String[]::new) + ).clear().addMetric(NodesInfoRequest.Metric.OS.metricName()).timeout(fetchTimeout), + ActionListener.wrap(nodesInfoResponse -> { + final Map builderBuilder = Maps.newHashMapWithExpectedSize( + nodesStatsResponse.getNodes().size() + ); + nodesStatsResponse.getNodes() + .forEach( + nodeStats -> builderBuilder.put( + nodeStats.getNode().getEphemeralId(), + MemoryAndProcessors.builder() + .setMemory(nodeStats.getOs().getMem().getAdjustedTotal().getBytes()) + ) + ); + nodesInfoResponse.getNodes() + .forEach( + nodeInfo -> builderBuilder.computeIfPresent( + nodeInfo.getNode().getEphemeralId(), + (n, b) -> b.setProcessors(nodeInfo.getInfo(OsInfo.class).getAllocatedProcessors()) + ) + ); + synchronized (mutex) { + Map builder = new HashMap<>(nodeToMemory); + // Remove all from the builder that failed getting info and stats + Stream.concat(nodesStatsResponse.failures().stream(), nodesInfoResponse.failures().stream()) + .map(FailedNodeException::nodeId) + .map(nodeLookup) + .map(DiscoveryNode::getEphemeralId) + .forEach(builder::remove); + + // we might add nodes that already died here, + // but those will be removed on next cluster state update anyway and is only a small waste. + builderBuilder.forEach((nodeEphemeralId, memoryProcessorBuilder) -> { + if (memoryProcessorBuilder.canBuild()) { + builder.put(nodeEphemeralId, memoryProcessorBuilder.build()); + } + }); + nodeToMemory = Collections.unmodifiableMap(builder); + } + }, e -> { + synchronized (mutex) { + var builder = new HashMap<>(nodeToMemory); + missingNodes.stream().map(DiscoveryNode::getEphemeralId).forEach(builder::remove); + nodeToMemory = Collections.unmodifiableMap(builder); + } + + logger.warn(() -> String.format(Locale.ROOT, "Unable to obtain processor info from [%s]", missingNodes), e); + }) + ), + e -> { synchronized (mutex) { var builder = new HashMap<>(nodeToMemory); missingNodes.stream().map(DiscoveryNode::getEphemeralId).forEach(builder::remove); nodeToMemory = Collections.unmodifiableMap(builder); } - logger.warn("Unable to obtain memory info from [{}]", missingNodes); + logger.warn(() -> String.format(Locale.ROOT, "Unable to obtain memory info from [%s]", missingNodes), e); } - } + ) ); } @@ -178,21 +224,14 @@ private void retainAliveNodes(Set currentNodes) { } } - private void addNodeStats(Map builder, NodeStats nodeStats) { - // we might add nodes that already died here, but those will be removed on next cluster state update anyway and is only a small - // waste. - builder.put(nodeStats.getNode().getEphemeralId(), nodeStats.getOs().getMem().getAdjustedTotal().getBytes()); - } - - public AutoscalingMemoryInfo snapshot() { - final Map nodeToMemoryRef = this.nodeToMemory; + public AutoscalingMemoryAndProcessorInfo snapshot() { + final Map nodeToMemoryRef = this.nodeToMemory; return node -> { - Long result = nodeToMemoryRef.get(node.getEphemeralId()); - // noinspection NumberEquality + MemoryAndProcessors result = nodeToMemoryRef.get(node.getEphemeralId()); if (result == FETCHING_SENTINEL) { - return null; + return Optional.empty(); } else { - return result; + return Optional.ofNullable(result); } }; } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/MemoryAndProcessors.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/MemoryAndProcessors.java new file mode 100644 index 0000000000000..b6ffc73df0eec --- /dev/null +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/MemoryAndProcessors.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.autoscaling.capacity.memory; + +/** + * Record for containing memory and processors for given node + * @param memory node total memory + * @param processors allocated processors + */ +public record MemoryAndProcessors(long memory, int processors) { + + static Builder builder() { + return new Builder(); + } + + static class Builder { + private Long memory; + private Integer processors; + + public Builder setMemory(long memory) { + this.memory = memory; + return this; + } + + public Builder setProcessors(int processors) { + this.processors = processors; + return this; + } + + boolean canBuild() { + return memory != null && processors != null; + } + + MemoryAndProcessors build() { + assert memory != null && processors != null : "unexpected null values when building node memory and processors information"; + return new MemoryAndProcessors(memory, processors); + } + } +} diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderService.java index 6b1035c9b7b23..35406102eb2f4 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderService.java @@ -53,10 +53,10 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider .collect(Collectors.toList()); AutoscalingCapacity.Builder builder = AutoscalingCapacity.builder(); if (indicesNeedingFrozen.size() > 0) { - builder.total(MINIMUM_FROZEN_STORAGE, MINIMUM_FROZEN_MEMORY); - builder.node(MINIMUM_FROZEN_STORAGE, MINIMUM_FROZEN_MEMORY); + builder.total(MINIMUM_FROZEN_STORAGE, MINIMUM_FROZEN_MEMORY, null); + builder.node(MINIMUM_FROZEN_STORAGE, MINIMUM_FROZEN_MEMORY, null); } else { - builder.total(0L, 0L); + builder.total(0L, 0L, 0); } return new AutoscalingDeciderResult(builder.build(), new FrozenExistenceReason(indicesNeedingFrozen)); diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/shards/FrozenShardsDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/shards/FrozenShardsDeciderService.java index 18044de7b5954..cbd88b7407e49 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/shards/FrozenShardsDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/shards/FrozenShardsDeciderService.java @@ -53,7 +53,10 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider // we assume that nodes do not grow beyond 64GB here. int shards = countFrozenShards(context.state().metadata()); long memory = shards * MEMORY_PER_SHARD.get(configuration).getBytes(); - return new AutoscalingDeciderResult(AutoscalingCapacity.builder().total(null, memory).build(), new FrozenShardsReason(shards)); + return new AutoscalingDeciderResult( + AutoscalingCapacity.builder().total(null, memory, null).build(), + new FrozenShardsReason(shards) + ); } static int countFrozenShards(Metadata metadata) { diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderService.java index 5c96e9029c530..f252dd21f7643 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderService.java @@ -47,7 +47,10 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider .sum(); long storageSize = (long) (PERCENTAGE.get(configuration) * dataSetSize) / 100; - return new AutoscalingDeciderResult(AutoscalingCapacity.builder().total(storageSize, null).build(), new FrozenReason(dataSetSize)); + return new AutoscalingDeciderResult( + AutoscalingCapacity.builder().total(storageSize, null, null).build(), + new FrozenReason(dataSetSize) + ); } static long estimateSize(IndexMetadata imd, ClusterInfo info) { diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java index 114b0b87ad850..5b076273f66cb 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java @@ -80,8 +80,8 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider assert maxShardSize >= 0; String message = ReactiveStorageDeciderService.message(unassignedBytes, assignedBytes); AutoscalingCapacity requiredCapacity = AutoscalingCapacity.builder() - .total(autoscalingCapacity.total().storage().getBytes() + unassignedBytes + assignedBytes, null) - .node(maxShardSize, null) + .total(autoscalingCapacity.total().storage().getBytes() + unassignedBytes + assignedBytes, null, null) + .node(maxShardSize, null, null) .build(); return new AutoscalingDeciderResult( requiredCapacity, diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java index fbe59764371ef..ff99eb8632dc9 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java @@ -120,8 +120,8 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider assert maxShardSize >= 0; String message = message(unassignedBytes, assignedBytes); AutoscalingCapacity requiredCapacity = AutoscalingCapacity.builder() - .total(autoscalingCapacity.total().storage().getBytes() + unassignedBytes + assignedBytes, null) - .node(maxShardSize, null) + .total(autoscalingCapacity.total().storage().getBytes() + unassignedBytes + assignedBytes, null, null) + .node(maxShardSize, null, null) .build(); return new AutoscalingDeciderResult( requiredCapacity, @@ -175,7 +175,7 @@ static Optional singleNoDecision(Decision decision, Predicate .stream() .filter(single -> single.type() == Decision.Type.NO) .filter(predicate) - .collect(Collectors.toList()); + .toList(); if (nos.size() == 1) { return Optional.ofNullable(nos.get(0).label()); @@ -431,8 +431,7 @@ private long getExpectedShardSize(ShardRouting shard) { } long unmovableSize(String nodeId, Collection shards) { - ClusterInfo clusterInfo = this.info; - DiskUsage diskUsage = clusterInfo.getNodeMostAvailableDiskUsages().get(nodeId); + DiskUsage diskUsage = this.info.getNodeMostAvailableDiskUsages().get(nodeId); if (diskUsage == null) { // do not want to scale up then, since this should only happen when node has just joined (clearly edge case). return 0; @@ -455,7 +454,7 @@ private long thresholdFromPercentage(Double percentage, DiskUsage diskUsage) { } Stream nodesInTier(RoutingNodes routingNodes) { - return nodeIds.stream().map(n -> routingNodes.node(n)); + return nodeIds.stream().map(routingNodes::node); } private static class SingleForecast { @@ -501,7 +500,7 @@ public AllocationState forecast(long forecastWindow, long now) { .map(IndexAbstraction.DataStream.class::cast) .map(ds -> forecast(state.metadata(), ds, forecastWindow, now)) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .toList(); if (singleForecasts.isEmpty()) { return this; } @@ -659,11 +658,7 @@ public Long getShardSize(ShardRouting shardRouting) { @Override public long getShardSize(ShardRouting shardRouting, long defaultValue) { Long shardSize = super.getShardSize(shardRouting); - if (shardSize != null) { - return shardSize; - } else { - return delegate.getShardSize(shardRouting, defaultValue); - } + return Objects.requireNonNullElseGet(shardSize, () -> delegate.getShardSize(shardRouting, defaultValue)); } @Override diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java index cfddd5994243f..c18f1dfc7d3e7 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java @@ -25,7 +25,6 @@ import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicyMetadata; import java.util.BitSet; -import java.util.List; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; @@ -34,6 +33,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; public abstract class AutoscalingTestCase extends ESTestCase { @@ -61,7 +61,9 @@ public static AutoscalingCapacity randomAutoscalingCapacity() { AutoscalingCapacity.AutoscalingResources total = randomNullValueAutoscalingResources(); return new AutoscalingCapacity( total, - randomBoolean() ? randomNullValueAutoscalingResources(total.storage() != null, total.memory() != null) : null + randomBoolean() + ? randomNullValueAutoscalingResources(total.storage() != null, total.memory() != null, total.processors().isPresent()) + : null ); } @@ -70,20 +72,26 @@ protected static AutoscalingCapacity randomNullableAutoscalingCapacity() { } protected static AutoscalingCapacity.AutoscalingResources randomAutoscalingResources() { - return new AutoscalingCapacity.AutoscalingResources(randomByteSizeValue(), randomByteSizeValue()); + return new AutoscalingCapacity.AutoscalingResources(randomByteSizeValue(), randomByteSizeValue(), randomInt(128)); } private static AutoscalingCapacity.AutoscalingResources randomNullValueAutoscalingResources() { - return randomNullValueAutoscalingResources(true, true); - } - - public static AutoscalingCapacity.AutoscalingResources randomNullValueAutoscalingResources(boolean allowStorage, boolean allowMemory) { - assert allowMemory || allowStorage; - boolean addStorage = (allowStorage && randomBoolean()) || allowMemory == false; - boolean addMemory = (allowMemory && randomBoolean()) || addStorage == false; + return randomNullValueAutoscalingResources(true, true, true); + } + + public static AutoscalingCapacity.AutoscalingResources randomNullValueAutoscalingResources( + boolean allowStorage, + boolean allowMemory, + boolean allowProcessors + ) { + assert allowMemory || allowStorage || allowProcessors; + boolean addStorage = (allowStorage && randomBoolean()) || (allowMemory == false && allowProcessors == false); + boolean addMemory = (allowMemory && randomBoolean()) || (addStorage == false && allowProcessors == false); + boolean addProcessors = (allowProcessors && randomBoolean()) || (addStorage == false && addMemory == false); return new AutoscalingCapacity.AutoscalingResources( addStorage ? randomByteSizeValue() : null, - addMemory ? randomByteSizeValue() : null + addMemory ? randomByteSizeValue() : null, + addProcessors ? randomInt(128) : null ); } @@ -113,7 +121,7 @@ public static ByteSizeValue randomNullableByteSizeValue() { public static SortedMap randomAutoscalingDeciders() { return new TreeMap<>( - List.of(randomFixedDecider()).stream().collect(Collectors.toMap(d -> FixedAutoscalingDeciderService.NAME, Function.identity())) + Stream.of(randomFixedDecider()).collect(Collectors.toMap(d -> FixedAutoscalingDeciderService.NAME, Function.identity())) ); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java index 4e6965486a5d5..fe6739efc1f04 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java @@ -22,7 +22,8 @@ import org.elasticsearch.snapshots.SnapshotShardSizeInfo; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; import org.elasticsearch.xpack.autoscaling.AutoscalingTestCase; -import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryInfo; +import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryAndProcessorInfo; +import org.elasticsearch.xpack.autoscaling.capacity.memory.MemoryAndProcessors; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicy; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicyMetadata; @@ -31,6 +32,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -64,7 +66,7 @@ public void testMultiplePoliciesFixedCapacity() { state, ClusterInfo.EMPTY, null, - AutoscalingMemoryInfo.EMPTY, + AutoscalingMemoryAndProcessorInfo.EMPTY, () -> {} ); assertThat(resultsMap.keySet(), equalTo(policyNames)); @@ -125,7 +127,7 @@ public String name() { .build(); assertThat( - service.calculate(state, ClusterInfo.EMPTY, SnapshotShardSizeInfo.EMPTY, AutoscalingMemoryInfo.EMPTY, () -> {}) + service.calculate(state, ClusterInfo.EMPTY, SnapshotShardSizeInfo.EMPTY, AutoscalingMemoryAndProcessorInfo.EMPTY, () -> {}) .get("test") .results() .keySet(), @@ -148,16 +150,18 @@ private SortedMap randomFixedDeciders() { private AutoscalingCapacity calculateFixedDeciderCapacity(Settings configuration) { ByteSizeValue storage = configuration.getAsBytesSize(FixedAutoscalingDeciderService.STORAGE.getKey(), null); ByteSizeValue memory = configuration.getAsBytesSize(FixedAutoscalingDeciderService.MEMORY.getKey(), null); + Integer processors = configuration.getAsInt(FixedAutoscalingDeciderService.PROCESSORS.getKey(), null); int nodes = FixedAutoscalingDeciderService.NODES.get(configuration); ByteSizeValue totalStorage = storage != null ? new ByteSizeValue(storage.getBytes() * nodes) : null; ByteSizeValue totalMemory = memory != null ? new ByteSizeValue(memory.getBytes() * nodes) : null; + Integer totalProcessors = processors != null ? processors * nodes : null; if (totalStorage == null && totalMemory == null) { return null; } else { return new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(totalStorage, totalMemory), - new AutoscalingCapacity.AutoscalingResources(storage, memory) + new AutoscalingCapacity.AutoscalingResources(totalStorage, totalMemory, totalProcessors), + new AutoscalingCapacity.AutoscalingResources(storage, memory, processors) ); } } @@ -175,7 +179,7 @@ public void testContext() { state, info, snapshotShardSizeInfo, - n -> randomNonNegativeLong(), + n -> Optional.of(new MemoryAndProcessors(randomNonNegativeLong(), randomInt(64))), () -> {} ); @@ -200,12 +204,12 @@ public void testContext() { state, info, null, - n -> memory, + n -> Optional.of(new MemoryAndProcessors(memory, randomInt(64))), () -> {} ); assertThat(context.nodes().size(), equalTo(1)); - assertThat(context.nodes(), equalTo(state.nodes().stream().collect(Collectors.toSet()))); + assertThat(context.nodes(), equalTo(new HashSet<>(state.nodes()))); if (hasDataRole) { assertNull(context.currentCapacity()); } else { @@ -258,7 +262,7 @@ public void testContext() { state, info, null, - n -> memory, + n -> Optional.of(new MemoryAndProcessors(memory, randomInt(64))), () -> {} ); @@ -279,7 +283,7 @@ public void testContext() { state, info, null, - AutoscalingMemoryInfo.EMPTY, + AutoscalingMemoryAndProcessorInfo.EMPTY, () -> {} ); assertThat(context.nodes(), equalTo(expectedNodes)); @@ -304,7 +308,7 @@ public void testContext() { state, info, null, - n -> memory, + n -> Optional.of(new MemoryAndProcessors(memory, randomInt(64))), () -> {} ); assertThat(context.nodes(), equalTo(expectedNodes)); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacityWireSerializationTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacityWireSerializationTests.java index d69942db88615..8ddcf2b7d97fd 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacityWireSerializationTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacityWireSerializationTests.java @@ -12,8 +12,6 @@ import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.autoscaling.AutoscalingTestCase; -import java.io.IOException; - public class AutoscalingCapacityWireSerializationTests extends AbstractWireSerializingTestCase { @Override protected Writeable.Reader instanceReader() { @@ -26,27 +24,38 @@ protected AutoscalingCapacity createTestInstance() { } @Override - protected AutoscalingCapacity mutateInstance(AutoscalingCapacity instance) throws IOException { + protected AutoscalingCapacity mutateInstance(AutoscalingCapacity instance) { AutoscalingCapacity.Builder builder = AutoscalingCapacity.builder().capacity(instance); - if (randomBoolean()) { // mutate total - boolean hasBothStorageAndMemory = instance.total().memory() != null && instance.total().storage() != null; + boolean hasAllMetrics = instance.total().memory() != null + && instance.total().storage() != null + && instance.total().processors().isPresent(); if (randomBoolean()) { builder.total( randomByteSize( - hasBothStorageAndMemory && (instance.node() == null || instance.node().storage() == null), + hasAllMetrics && (instance.node() == null || instance.node().storage() == null), instance.total().storage() ), - instance.total().memory() + instance.total().memory(), + instance.total().getProcessors() ); - } else { + } else if (randomBoolean()) { builder.total( instance.total().storage(), randomByteSize( - hasBothStorageAndMemory && (instance.node() == null || instance.node().memory() == null), + hasAllMetrics && (instance.node() == null || instance.node().memory() == null), instance.total().memory() - ) + ), + instance.total().getProcessors() + ); + } else { + builder.total( + instance.total().storage(), + instance.total().memory(), + hasAllMetrics && (instance.node() == null || instance.node().processors().isEmpty()) && randomBoolean() + ? null + : randomIntBetween(1, 64) + instance.total().processors().orElse(0) ); } } else { @@ -55,13 +64,50 @@ protected AutoscalingCapacity mutateInstance(AutoscalingCapacity instance) throw builder.node( AutoscalingTestCase.randomNullValueAutoscalingResources( instance.total().storage() != null, - instance.total().memory() != null + instance.total().memory() != null, + instance.total().processors().isPresent() ) ); - } else if (randomBoolean() && instance.total().storage() != null || instance.total().memory() == null) { - builder.node(randomByteSize(instance.node().memory() != null, instance.node().storage()), instance.node().memory()); + } else if (randomBoolean() && instance.total().storage() != null) { + builder.node( + randomByteSize(instance.node().memory() != null || instance.node().processors().isPresent(), instance.node().storage()), + instance.node().memory(), + instance.node().getProcessors() + ); + } else if (randomBoolean() && instance.total().memory() != null) { + builder.node( + instance.node().storage(), + randomByteSize(instance.node().storage() != null || instance.node().processors().isPresent(), instance.node().memory()), + instance.node().getProcessors() + ); + } else if (instance.total().processors().isPresent()) { + builder.node( + instance.node().storage(), + instance.node().memory(), + randomBoolean() + && (instance.node().storage() != null || instance.node().memory() != null) + && instance.node().processors().isPresent() + ? null + : randomIntBetween(1, 64) + instance.node().processors().orElse(0) + ); } else { - builder.node(instance.node().storage(), randomByteSize(instance.node().storage() != null, instance.node().memory())); + ByteSizeValue newStorage = instance.total().storage() != null + ? randomByteSize( + instance.node().memory() != null || instance.node().processors().isPresent(), + instance.node().storage() + ) + : null; + ByteSizeValue newMem = instance.total().memory() != null + ? randomByteSize(newStorage != null || instance.node().processors().isPresent(), instance.node().memory()) + : null; + builder.node( + newStorage, + newMem, + randomBoolean() && (newMem != null || newStorage != null) && instance.node().processors().isPresent() ? null + : instance.total().processors().isPresent() && randomBoolean() + ? randomIntBetween(1, 64) + instance.node().processors().orElse(0) + : null + ); } } return builder.build(); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java index c0ff016559399..b260733f34c08 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java @@ -57,21 +57,25 @@ public void testRequiredCapacity() { boolean node = randomBoolean(); boolean storage = randomBoolean(); boolean memory = randomBoolean() || storage == false; + boolean processor = randomBoolean() || memory == false; - AutoscalingCapacity large = randomCapacity(node, storage, memory, 1000, 2000); + AutoscalingCapacity large = randomCapacity(node, storage, memory, processor, 1000, 2000); List autoscalingCapacities = new ArrayList<>(); autoscalingCapacities.add(large); - IntStream.range(0, 10).mapToObj(i -> randomCapacity(node, storage, memory, 0, 1000)).forEach(autoscalingCapacities::add); + IntStream.range(0, 10).mapToObj(i -> randomCapacity(node, storage, memory, processor, 0, 1000)).forEach(autoscalingCapacities::add); Randomness.shuffle(autoscalingCapacities); verifyRequiredCapacity(large, autoscalingCapacities.toArray(AutoscalingCapacity[]::new)); - AutoscalingCapacity largerStorage = randomCapacity(node, true, false, 2000, 3000); - verifySingleMetricLarger(node, largerStorage, large, autoscalingCapacities, largerStorage); + AutoscalingCapacity largerStorage = randomCapacity(node, true, false, false, 2000, 3000); + verifySingleMetricLarger(node, largerStorage, large, large, autoscalingCapacities, largerStorage); - AutoscalingCapacity largerMemory = randomCapacity(node, false, true, 2000, 3000); - verifySingleMetricLarger(node, large, largerMemory, autoscalingCapacities, largerMemory); + AutoscalingCapacity largerMemory = randomCapacity(node, false, true, false, 2000, 3000); + verifySingleMetricLarger(node, large, largerMemory, large, autoscalingCapacities, largerMemory); + + AutoscalingCapacity largerProcessor = randomCapacity(node, false, false, true, 2000, 3000); + verifySingleMetricLarger(node, large, large, largerProcessor, autoscalingCapacities, largerProcessor); } public void testToXContent() { @@ -120,6 +124,7 @@ private void verifySingleMetricLarger( boolean node, AutoscalingCapacity expectedStorage, AutoscalingCapacity expectedMemory, + AutoscalingCapacity expectedProcessor, List other, AutoscalingCapacity larger ) { @@ -127,9 +132,13 @@ private void verifySingleMetricLarger( autoscalingCapacities.add(larger); Randomness.shuffle(autoscalingCapacities); AutoscalingCapacity.Builder expectedBuilder = AutoscalingCapacity.builder() - .total(expectedStorage.total().storage(), expectedMemory.total().memory()); + .total(expectedStorage.total().storage(), expectedMemory.total().memory(), expectedProcessor.total().getProcessors()); if (node) { - expectedBuilder.node(expectedStorage.node().storage(), expectedMemory.node().memory()); + expectedBuilder.node( + expectedStorage.node().storage(), + expectedMemory.node().memory(), + expectedProcessor.node().getProcessors() + ); } verifyRequiredCapacity(expectedBuilder.build(), autoscalingCapacities.toArray(AutoscalingCapacity[]::new)); } @@ -152,11 +161,19 @@ private void verifyRequiredCapacity(AutoscalingCapacity expected, AutoscalingCap ); } - private AutoscalingCapacity randomCapacity(boolean node, boolean storage, boolean memory, int lower, int upper) { + private AutoscalingCapacity randomCapacity(boolean node, boolean storage, boolean memory, boolean processor, int lower, int upper) { AutoscalingCapacity.Builder builder = AutoscalingCapacity.builder(); - builder.total(storage ? randomLongBetween(lower, upper) : null, memory ? randomLongBetween(lower, upper) : null); + builder.total( + storage ? randomLongBetween(lower, upper) : null, + memory ? randomLongBetween(lower, upper) : null, + processor ? randomIntBetween(lower, upper) : null + ); if (node) { - builder.node(storage ? randomLongBetween(lower, upper) : null, memory ? randomLongBetween(lower, upper) : null); + builder.node( + storage ? randomLongBetween(lower, upper) : null, + memory ? randomLongBetween(lower, upper) : null, + processor ? randomIntBetween(lower, upper) : null + ); } return builder.build(); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderServiceTests.java index ae27476de9d75..9b2eebae2727e 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderServiceTests.java @@ -14,7 +14,6 @@ public class FixedAutoscalingDeciderServiceTests extends AutoscalingTestCase { public void testScale() { - Settings.Builder configurationBuilder = Settings.builder(); int nodes = randomIntBetween(1, 1000); if (randomBoolean()) { @@ -26,18 +25,28 @@ public void testScale() { ByteSizeValue storage = randomNullableByteSizeValue(); ByteSizeValue memory = storage != null ? randomNullableByteSizeValue() : randomByteSizeValue(); + Integer processors = memory != null && randomBoolean() ? null : randomInt(64); if (storage != null) { configurationBuilder.put(FixedAutoscalingDeciderService.STORAGE.getKey(), storage); } if (memory != null) { configurationBuilder.put(FixedAutoscalingDeciderService.MEMORY.getKey(), memory); } - verify(configurationBuilder.build(), AutoscalingCapacity.builder().node(storage, memory).total(storage, memory).build()); + if (processors != null) { + configurationBuilder.put(FixedAutoscalingDeciderService.PROCESSORS.getKey(), processors); + } + verify( + configurationBuilder.build(), + AutoscalingCapacity.builder().node(storage, memory, processors).total(storage, memory, processors).build() + ); configurationBuilder.put(FixedAutoscalingDeciderService.NODES.getKey(), nodes); verify( configurationBuilder.build(), - AutoscalingCapacity.builder().node(storage, memory).total(multiply(storage, nodes), multiply(memory, nodes)).build() + AutoscalingCapacity.builder() + .node(storage, memory, processors) + .total(multiply(storage, nodes), multiply(memory, nodes), multiply(processors, nodes)) + .build() ); } @@ -51,4 +60,8 @@ private void verify(Settings configuration, AutoscalingCapacity expected) { private ByteSizeValue multiply(ByteSizeValue bytes, int nodes) { return bytes == null ? null : new ByteSizeValue(bytes.getBytes() * nodes); } + + private Integer multiply(Integer processors, int nodes) { + return processors == null ? null : processors * nodes; + } } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfoServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfoServiceTests.java similarity index 64% rename from x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfoServiceTests.java rename to x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfoServiceTests.java index d24d9dd03d16f..a7c55bccdea86 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryInfoServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfoServiceTests.java @@ -7,12 +7,17 @@ package org.elasticsearch.xpack.autoscaling.capacity.memory; +import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; @@ -31,6 +36,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.monitor.os.OsInfo; import org.elasticsearch.monitor.os.OsStats; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; @@ -54,6 +60,8 @@ import java.util.stream.IntStream; import java.util.stream.Stream; +import static org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryAndProcessorInfoService.FETCH_TIMEOUT; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -61,10 +69,10 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class AutoscalingMemoryInfoServiceTests extends AutoscalingTestCase { +public class AutoscalingMemoryAndProcessorInfoServiceTests extends AutoscalingTestCase { private NodeStatsClient client; - private AutoscalingMemoryInfoService service; + private AutoscalingMemoryAndProcessorInfoService service; private TimeValue fetchTimeout; private AutoscalingMetadata autoscalingMetadata; private Metadata metadata; @@ -81,16 +89,13 @@ public void setUp() throws Exception { settings = Settings.EMPTY; } else { fetchTimeout = TimeValue.timeValueMillis(randomLongBetween(1, 10000)); - settings = Settings.builder().put(AutoscalingMemoryInfoService.FETCH_TIMEOUT.getKey(), fetchTimeout).build(); + settings = Settings.builder().put(FETCH_TIMEOUT.getKey(), fetchTimeout).build(); } when(clusterService.getSettings()).thenReturn(settings); - Set> settingsSet = Sets.union( - ClusterSettings.BUILT_IN_CLUSTER_SETTINGS, - Set.of(AutoscalingMemoryInfoService.FETCH_TIMEOUT) - ); + Set> settingsSet = Sets.union(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS, Set.of(FETCH_TIMEOUT)); ClusterSettings clusterSettings = new ClusterSettings(settings, settingsSet); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); - service = new AutoscalingMemoryInfoService(clusterService, client); + service = new AutoscalingMemoryAndProcessorInfoService(clusterService, client); autoscalingMetadata = randomAutoscalingMetadataOfPolicyCount(between(1, 8)); metadata = Metadata.builder().putCustom(AutoscalingMetadata.NAME, autoscalingMetadata).build(); } @@ -131,17 +136,25 @@ public void testAddRemoveNode() { .collect(Collectors.toList()), failures ); - client.respond(response, () -> { + NodesInfoResponse responseInfo = new NodesInfoResponse( + ClusterName.DEFAULT, + succeedingNodes.stream().map(n -> infoForNode(n, randomIntBetween(0, 64))).collect(Collectors.toList()), + List.of() + ); + client.respondStats(response, () -> { Sets.union(missingNodes, Sets.difference(previousNodes, nodes)) - .forEach(n -> { assertThat(service.snapshot().get(n), nullValue()); }); - Sets.intersection(previousSucceededNodes, nodes).forEach(n -> assertThat(service.snapshot().get(n), notNullValue())); + .forEach(n -> assertThat(service.snapshot().get(n).isEmpty(), is(true))); + Sets.intersection(previousSucceededNodes, nodes).forEach(n -> assertThat(service.snapshot().get(n).isPresent(), is(true))); + }); + client.respondInfo(responseInfo, () -> { + }); service.onClusterChanged(new ClusterChangedEvent("test", state, previousState)); client.assertNoResponder(); - assertMatchesResponse(succeedingNodes, response); - failingNodes.forEach(n -> { assertThat(service.snapshot().get(n), nullValue()); }); + assertMatchesResponse(succeedingNodes, response, responseInfo); + failingNodes.forEach(n -> assertThat(service.snapshot().get(n).isEmpty(), is(true))); previousNodes.clear(); previousNodes.addAll(nodes); @@ -159,7 +172,7 @@ public void testNotMaster() { // client throws if called. service.onClusterChanged(new ClusterChangedEvent("test", state, ClusterState.EMPTY_STATE)); - nodes.forEach(n -> assertThat(service.snapshot().get(n), nullValue())); + nodes.forEach(n -> assertThat(service.snapshot().get(n).isEmpty(), is(true))); } public void testNoLongerMaster() { @@ -173,12 +186,17 @@ public void testNoLongerMaster() { nodes.stream().map(n -> statsForNode(n, randomLongBetween(0, Long.MAX_VALUE / 1000))).collect(Collectors.toList()), List.of() ); + NodesInfoResponse responseInfo = new NodesInfoResponse( + ClusterName.DEFAULT, + nodes.stream().map(n -> infoForNode(n, randomIntBetween(0, 64))).collect(Collectors.toList()), + List.of() + ); - client.respond(response, () -> {}); + client.respondStats(response, () -> {}); + client.respondInfo(responseInfo, () -> {}); service.onClusterChanged(new ClusterChangedEvent("test", masterState, ClusterState.EMPTY_STATE)); client.assertNoResponder(); - - assertMatchesResponse(nodes, response); + assertMatchesResponse(nodes, response, responseInfo); ClusterState notMasterState = ClusterState.builder(masterState) .nodes(DiscoveryNodes.builder(masterState.nodes()).masterNodeId(null)) @@ -187,26 +205,57 @@ public void testNoLongerMaster() { // client throws if called. service.onClusterChanged(new ClusterChangedEvent("test", notMasterState, masterState)); - nodes.forEach(n -> assertThat(service.snapshot().get(n), nullValue())); + nodes.forEach(n -> assertThat(service.snapshot().get(n).isEmpty(), is(true))); } - public void testFails() { + public void testStatsFails() { Set nodes = IntStream.range(0, between(1, 10)).mapToObj(n -> newNode("test_" + n)).collect(Collectors.toSet()); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).nodes(discoveryNodesBuilder(nodes, true)).metadata(metadata).build(); - client.respond((r, listener) -> listener.onFailure(randomFrom(new IllegalStateException(), new RejectedExecutionException()))); + client.respondStats((r, listener) -> listener.onFailure(randomFrom(new IllegalStateException(), new RejectedExecutionException()))); service.onClusterChanged(new ClusterChangedEvent("test", state, ClusterState.EMPTY_STATE)); - nodes.forEach(n -> assertThat(service.snapshot().get(n), nullValue())); + nodes.forEach(n -> assertThat(service.snapshot().get(n).isEmpty(), is(true))); NodesStatsResponse response = new NodesStatsResponse( ClusterName.DEFAULT, nodes.stream().map(n -> statsForNode(n, randomLongBetween(0, Long.MAX_VALUE / 1000))).collect(Collectors.toList()), List.of() ); + NodesInfoResponse responseInfo = new NodesInfoResponse( + ClusterName.DEFAULT, + nodes.stream().map(n -> infoForNode(n, randomIntBetween(0, 64))).collect(Collectors.toList()), + List.of() + ); // implicit retry on cluster state update. - client.respond(response, () -> {}); + client.respondStats(response, () -> {}); + client.respondInfo(responseInfo, () -> {}); + service.onClusterChanged(new ClusterChangedEvent("test", state, state)); + client.assertNoResponder(); + } + + public void testInfoFails() { + Set nodes = IntStream.range(0, between(1, 10)).mapToObj(n -> newNode("test_" + n)).collect(Collectors.toSet()); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).nodes(discoveryNodesBuilder(nodes, true)).metadata(metadata).build(); + NodesStatsResponse response = new NodesStatsResponse( + ClusterName.DEFAULT, + nodes.stream().map(n -> statsForNode(n, randomLongBetween(0, Long.MAX_VALUE / 1000))).collect(Collectors.toList()), + List.of() + ); + client.respondStats(response, () -> {}); + client.respondInfo((r, listener) -> listener.onFailure(randomFrom(new IllegalStateException(), new RejectedExecutionException()))); + service.onClusterChanged(new ClusterChangedEvent("test", state, ClusterState.EMPTY_STATE)); + nodes.forEach(n -> assertThat(service.snapshot().get(n).isEmpty(), is(true))); + NodesInfoResponse responseInfo = new NodesInfoResponse( + ClusterName.DEFAULT, + nodes.stream().map(n -> infoForNode(n, randomIntBetween(0, 64))).collect(Collectors.toList()), + List.of() + ); + + // implicit retry on cluster state update. + client.respondStats(response, () -> {}); + client.respondInfo(responseInfo, () -> {}); service.onClusterChanged(new ClusterChangedEvent("test", state, state)); client.assertNoResponder(); } @@ -221,11 +270,18 @@ public void testRestartNode() { List.of() ); - client.respond(response, () -> {}); + NodesInfoResponse responseInfo = new NodesInfoResponse( + ClusterName.DEFAULT, + nodes.stream().map(n -> infoForNode(n, randomIntBetween(0, 64))).collect(Collectors.toList()), + List.of() + ); + + client.respondStats(response, () -> {}); + client.respondInfo(responseInfo, () -> {}); service.onClusterChanged(new ClusterChangedEvent("test", state, ClusterState.EMPTY_STATE)); client.assertNoResponder(); - assertMatchesResponse(nodes, response); + assertMatchesResponse(nodes, response, responseInfo); Set restartedNodes = randomValueOtherThan( nodes, @@ -243,14 +299,21 @@ public void testRestartNode() { List.of() ); - client.respond(restartedStatsResponse, () -> {}); + NodesInfoResponse restartedInfoResponse = new NodesInfoResponse( + ClusterName.DEFAULT, + Sets.difference(restartedNodes, nodes).stream().map(n -> infoForNode(n, randomIntBetween(0, 64))).collect(Collectors.toList()), + List.of() + ); + + client.respondStats(restartedStatsResponse, () -> {}); + client.respondInfo(restartedInfoResponse, () -> {}); service.onClusterChanged(new ClusterChangedEvent("test", restartedState, state)); client.assertNoResponder(); - assertMatchesResponse(Sets.intersection(restartedNodes, nodes), response); - assertMatchesResponse(Sets.difference(restartedNodes, nodes), restartedStatsResponse); + assertMatchesResponse(Sets.intersection(restartedNodes, nodes), response, responseInfo); + assertMatchesResponse(Sets.difference(restartedNodes, nodes), restartedStatsResponse, restartedInfoResponse); - Sets.difference(nodes, restartedNodes).forEach(n -> assertThat(service.snapshot().get(n), nullValue())); + Sets.difference(nodes, restartedNodes).forEach(n -> assertThat(service.snapshot().get(n).isEmpty(), is(true))); } public void testConcurrentStateUpdate() throws Exception { @@ -262,9 +325,14 @@ public void testConcurrentStateUpdate() throws Exception { nodes.stream().map(n -> statsForNode(n, randomLongBetween(0, Long.MAX_VALUE / 1000))).collect(Collectors.toList()), List.of() ); + NodesInfoResponse nodesInfoResponse = new NodesInfoResponse( + ClusterName.DEFAULT, + nodes.stream().map(n -> infoForNode(n, randomIntBetween(0, 64))).collect(Collectors.toList()), + List.of() + ); List threads = new ArrayList<>(); - client.respond((request, listener) -> { + client.respondStats((request, listener) -> { CountDownLatch latch = new CountDownLatch(1); threads.add(startThread(() -> { try { @@ -280,11 +348,12 @@ public void testConcurrentStateUpdate() throws Exception { latch.countDown(); })); }); + client.respondInfo((r, l) -> l.onResponse(nodesInfoResponse)); service.onClusterChanged(new ClusterChangedEvent("test", state, ClusterState.EMPTY_STATE)); - client.assertNoResponder(); for (Thread thread : threads) { thread.join(10000); } + client.assertNoResponder(); threads.forEach(t -> assertThat(t.isAlive(), is(false))); } @@ -326,15 +395,19 @@ private Set randomIrrelevantRoles(Set> relevantRo .collect(Collectors.toSet()); } - public void assertMatchesResponse(Set nodes, NodesStatsResponse response) { - nodes.forEach( - n -> { - assertThat( - service.snapshot().get(n), - equalTo(response.getNodesMap().get(n.getId()).getOs().getMem().getAdjustedTotal().getBytes()) - ); - } - ); + public void assertMatchesResponse(Set nodes, NodesStatsResponse response, NodesInfoResponse infoResponse) { + nodes.forEach(n -> { + assertThat(service.snapshot().get(n).isPresent(), is(true)); + assertThat( + service.snapshot().get(n).get(), + equalTo( + new MemoryAndProcessors( + response.getNodesMap().get(n.getId()).getOs().getMem().getAdjustedTotal().getBytes(), + infoResponse.getNodesMap().get(n.getId()).getInfo(OsInfo.class).getAllocatedProcessors() + ) + ) + ); + }); } private Thread startThread(Runnable runnable) { @@ -372,15 +445,37 @@ private static NodeStats statsForNode(DiscoveryNode node, long memory) { ); } + private static NodeInfo infoForNode(DiscoveryNode node, int processors) { + OsInfo osInfo = new OsInfo(randomLong(), processors, processors, null, null, null, null); + return new NodeInfo(Version.CURRENT, Build.CURRENT, node, null, osInfo, null, null, null, null, null, null, null, null, null); + } + private class NodeStatsClient extends NoOpClient { - private BiConsumer> responder; + private BiConsumer> responderStats; + private BiConsumer> responderInfo; private NodeStatsClient() { super(getTestName()); } - public void respond(NodesStatsResponse response, Runnable whileFetching) { - respond((request, listener) -> { + public void respondInfo(NodesInfoResponse response, Runnable whileFetching) { + respondInfo((request, listener) -> { + assertThat( + Set.of(request.nodesIds()), + Matchers.equalTo( + Stream.concat( + response.getNodesMap().keySet().stream(), + response.failures().stream().map(FailedNodeException::nodeId) + ).collect(Collectors.toSet()) + ) + ); + whileFetching.run(); + listener.onResponse(response); + }); + } + + public void respondStats(NodesStatsResponse response, Runnable whileFetching) { + respondStats((request, listener) -> { assertThat( Set.of(request.nodesIds()), Matchers.equalTo( @@ -395,9 +490,14 @@ public void respond(NodesStatsResponse response, Runnable whileFetching) { }); } - public void respond(BiConsumer> responderValue) { + public void respondStats(BiConsumer> responderValue) { assertThat(responderValue, notNullValue()); - this.responder = responderValue; + this.responderStats = responderValue; + } + + public void respondInfo(BiConsumer> responderValue) { + assertThat(responderValue, notNullValue()); + this.responderInfo = responderValue; } @Override @@ -406,19 +506,31 @@ protected void Request request, ActionListener listener ) { - assertThat(action, Matchers.sameInstance(NodesStatsAction.INSTANCE)); - NodesStatsRequest nodesStatsRequest = (NodesStatsRequest) request; - assertThat(nodesStatsRequest.timeout(), equalTo(fetchTimeout)); - assertThat(responder, notNullValue()); - BiConsumer> responderValue = this.responder; - this.responder = null; - @SuppressWarnings("unchecked") - ActionListener statsListener = (ActionListener) listener; - responderValue.accept(nodesStatsRequest, statsListener); + assertThat(action, anyOf(Matchers.sameInstance(NodesStatsAction.INSTANCE), Matchers.sameInstance(NodesInfoAction.INSTANCE))); + if (action == NodesStatsAction.INSTANCE) { + NodesStatsRequest nodesStatsRequest = (NodesStatsRequest) request; + assertThat(nodesStatsRequest.timeout(), equalTo(fetchTimeout)); + assertThat(responderStats, notNullValue()); + BiConsumer> responderValue = this.responderStats; + this.responderStats = null; + @SuppressWarnings("unchecked") + ActionListener statsListener = (ActionListener) listener; + responderValue.accept(nodesStatsRequest, statsListener); + } else { + NodesInfoRequest nodesInfoRequest = (NodesInfoRequest) request; + assertThat(nodesInfoRequest.timeout(), equalTo(fetchTimeout)); + assertThat(responderInfo, notNullValue()); + BiConsumer> responderValue = this.responderInfo; + this.responderInfo = null; + @SuppressWarnings("unchecked") + ActionListener infoListener = (ActionListener) listener; + responderValue.accept(nodesInfoRequest, infoListener); + } } public void assertNoResponder() { - assertThat(responder, nullValue()); + assertThat(responderInfo, nullValue()); + assertThat(responderStats, nullValue()); } } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java index 52f35ad02cfc6..539ebfb65c369 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java @@ -549,9 +549,18 @@ static AutoscalingCapacity randomCurrentCapacity() { if (randomInt(4) > 0) { // we only rely on storage. boolean includeMemory = randomBoolean(); + boolean includeProcessors = randomBoolean(); return AutoscalingCapacity.builder() - .total(randomByteSizeValue(), includeMemory ? randomByteSizeValue() : null) - .node(randomByteSizeValue(), includeMemory ? randomByteSizeValue() : null) + .total( + randomByteSizeValue(), + includeMemory || includeProcessors ? randomByteSizeValue() : null, + includeProcessors ? randomInt(64) : null + ) + .node( + randomByteSizeValue(), + includeMemory || includeProcessors ? randomByteSizeValue() : null, + includeProcessors ? randomInt(64) : null + ) .build(); } else { return null; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java index ae6f5f7f11292..17450f6276162 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderService.java @@ -724,11 +724,13 @@ static AutoscalingCapacity ensureScaleDown(AutoscalingCapacity scaleDownResult, AutoscalingCapacity newCapacity = new AutoscalingCapacity( new AutoscalingCapacity.AutoscalingResources( currentCapacity.total().storage(), - ByteSizeValue.ofBytes(Math.min(scaleDownResult.total().memory().getBytes(), currentCapacity.total().memory().getBytes())) + ByteSizeValue.ofBytes(Math.min(scaleDownResult.total().memory().getBytes(), currentCapacity.total().memory().getBytes())), + null ), new AutoscalingCapacity.AutoscalingResources( currentCapacity.node().storage(), - ByteSizeValue.ofBytes(Math.min(scaleDownResult.node().memory().getBytes(), currentCapacity.node().memory().getBytes())) + ByteSizeValue.ofBytes(Math.min(scaleDownResult.node().memory().getBytes(), currentCapacity.node().memory().getBytes())), + null ) ); if (scaleDownResult.node().memory().getBytes() - newCapacity.node().memory().getBytes() > ACCEPTABLE_DIFFERENCE diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NativeMemoryCapacity.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NativeMemoryCapacity.java index 4b6e93eb5799e..2a200c6e7b1d4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NativeMemoryCapacity.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/NativeMemoryCapacity.java @@ -137,8 +137,8 @@ public AutoscalingCapacity autoscalingCapacity( ); } return new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ZERO), - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ZERO) + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ZERO, null), + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ZERO, null) ); } @@ -228,8 +228,8 @@ public AutoscalingCapacity autoscalingCapacity( // The assertion above should hold, but the Math.max below catches the case with inconsistent // inputs plus any bugs that weren't caught in tests. return new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofBytes(Math.max(requiredTierSize, requiredNodeSize))), - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofBytes(requiredNodeSize)) + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofBytes(Math.max(requiredTierSize, requiredNodeSize)), null), + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofBytes(requiredNodeSize), null) ); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java index c73b273b9fc31..7602ca69307cf 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingDeciderServiceTests.java @@ -195,8 +195,8 @@ public void testScalingEdgeCase() { MlScalingReason.Builder reasonBuilder = new MlScalingReason.Builder().setPassedConfiguration(Settings.EMPTY) .setCurrentMlCapacity( AutoscalingCapacity.builder() - .node(null, AUTO_NODE_TIERS_NO_MONITORING.get(0).v1()) - .total(null, AUTO_NODE_TIERS_NO_MONITORING.get(0).v1()) + .node(null, AUTO_NODE_TIERS_NO_MONITORING.get(0).v1(), null) + .total(null, AUTO_NODE_TIERS_NO_MONITORING.get(0).v1(), null) .build() ); MlAutoscalingDeciderService service = buildService(); @@ -242,7 +242,7 @@ public void testScalingEdgeCase() { .build() ); reasonBuilder = new MlScalingReason.Builder().setPassedConfiguration(Settings.EMPTY) - .setCurrentMlCapacity(AutoscalingCapacity.builder().node(null, 2147483648L).total(null, 2147483648L).build()); + .setCurrentMlCapacity(AutoscalingCapacity.builder().node(null, 2147483648L, null).total(null, 2147483648L, null).build()); AutoscalingDeciderResult result = service.checkForScaleDown( nodeForScaleDown, ByteSizeValue.ofMb(200).getBytes() + Job.PROCESS_MEMORY_OVERHEAD.getBytes(), @@ -317,7 +317,7 @@ public void testScaleStability() { new NativeMemoryCapacity(lowerTierMemoryForMl, lowerTierMemoryForMl, lowerTierJvmSize), new MlScalingReason.Builder().setPassedConfiguration(Settings.EMPTY) .setCurrentMlCapacity( - AutoscalingCapacity.builder().node(null, lowerTierNodeSize).total(null, lowerTierNodeSize).build() + AutoscalingCapacity.builder().node(null, lowerTierNodeSize, null).total(null, lowerTierNodeSize, null).build() ) ).orElseThrow(); @@ -349,7 +349,9 @@ public void testScaleStability() { maxJobSize, new NativeMemoryCapacity(scaledUpBytesForMl, scaledUpBytesForMl, scaledUpJvmSize), new MlScalingReason.Builder().setPassedConfiguration(Settings.EMPTY) - .setCurrentMlCapacity(AutoscalingCapacity.builder().node(null, scaledUpSize).total(null, scaledUpSize).build()) + .setCurrentMlCapacity( + AutoscalingCapacity.builder().node(null, scaledUpSize, null).total(null, scaledUpSize, null).build() + ) ); // If scale down is present, we don't want to drop below our current tier. // If we do, that means that for the same jobs we scaled with, we calculated something incorrectly. @@ -1184,18 +1186,18 @@ public void testEnsureScaleDown() { assertThat( MlAutoscalingDeciderService.ensureScaleDown( new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(8)), - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(1)) + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(8), null), + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(1), null) ), new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4)), - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(2)) + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4), null), + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(2), null) ) ), equalTo( new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4)), - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(1)) + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4), null), + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(1), null) ) ) ); @@ -1203,18 +1205,18 @@ public void testEnsureScaleDown() { assertThat( MlAutoscalingDeciderService.ensureScaleDown( new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(8)), - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(3)) + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(8), null), + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(3), null) ), new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4)), - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(2)) + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4), null), + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(2), null) ) ), equalTo( new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4)), - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(2)) + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4), null), + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(2), null) ) ) ); @@ -1222,18 +1224,18 @@ public void testEnsureScaleDown() { assertThat( MlAutoscalingDeciderService.ensureScaleDown( new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4)), - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(3)) + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(4), null), + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(3), null) ), new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(3)), - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(2)) + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(3), null), + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(2), null) ) ), equalTo( new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(3)), - new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(2)) + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(3), null), + new AutoscalingCapacity.AutoscalingResources(null, ByteSizeValue.ofGb(2), null) ) ) ); @@ -1299,8 +1301,8 @@ public void testScale_WithNoScaleUpButWaitingJobs() { .put(MlAutoscalingDeciderService.NUM_ANOMALY_JOBS_IN_QUEUE.getKey(), maxWaitingAnomaly) .build(); AutoscalingCapacity autoscalingCapacity = new AutoscalingCapacity( - new AutoscalingCapacity.AutoscalingResources(ByteSizeValue.ofGb(1), ByteSizeValue.ofGb(1)), - new AutoscalingCapacity.AutoscalingResources(ByteSizeValue.ofGb(1), ByteSizeValue.ofGb(1)) + new AutoscalingCapacity.AutoscalingResources(ByteSizeValue.ofGb(1), ByteSizeValue.ofGb(1), null), + new AutoscalingCapacity.AutoscalingResources(ByteSizeValue.ofGb(1), ByteSizeValue.ofGb(1), null) ); DeciderContext deciderContext = new DeciderContext(clusterState, autoscalingCapacity); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlScalingReasonTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlScalingReasonTests.java index d9057606b14ba..bd02e2e797559 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlScalingReasonTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlScalingReasonTests.java @@ -51,7 +51,8 @@ protected MlScalingReason createTestInstance() { protected static AutoscalingCapacity.AutoscalingResources randomAutoscalingResources() { return new AutoscalingCapacity.AutoscalingResources( ByteSizeValue.ofBytes(randomLongBetween(10, ByteSizeValue.ofGb(10).getBytes())), - ByteSizeValue.ofBytes(randomLongBetween(10, ByteSizeValue.ofGb(10).getBytes())) + ByteSizeValue.ofBytes(randomLongBetween(10, ByteSizeValue.ofGb(10).getBytes())), + null ); } From f60f8e6b0b3bdb498bc705e7900c55a47dd35e26 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Tue, 21 Jun 2022 12:31:51 -0400 Subject: [PATCH 02/11] Update docs/changelog/87895.yaml --- docs/changelog/87895.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/87895.yaml diff --git a/docs/changelog/87895.yaml b/docs/changelog/87895.yaml new file mode 100644 index 0000000000000..4cb3419ead676 --- /dev/null +++ b/docs/changelog/87895.yaml @@ -0,0 +1,5 @@ +pr: 87895 +summary: Add processors to autoscaling capacity response +area: Autoscaling +type: enhancement +issues: [] From c478fa4700052fdf773e2368b15fb94c4bdb3d0b Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 22 Jun 2022 10:37:34 -0400 Subject: [PATCH 03/11] addressing pr comments --- .../autoscaling/get_autoscaling_capacity.yml | 46 +++++++++++++++++++ .../xpack/autoscaling/Autoscaling.java | 2 +- ...TransportGetAutoscalingCapacityAction.java | 10 ++-- .../AutoscalingCalculateCapacityService.java | 4 +- .../FixedAutoscalingDeciderService.java | 44 +++++++++++++++--- .../AutoscalingMemoryAndProcessorInfo.java | 2 +- ...oscalingMemoryAndProcessorInfoService.java | 36 +++++++-------- .../MemoryAndProcessors.java | 6 +-- .../ReactiveStorageDeciderService.java | 6 ++- .../autoscaling/AutoscalingTestCase.java | 7 ++- ...oscalingCalculateCapacityServiceTests.java | 21 +++++++-- .../AutoscalingDeciderResultsTests.java | 5 +- .../FixedAutoscalingDeciderServiceTests.java | 4 +- ...ingMemoryAndProcessorInfoServiceTests.java | 4 +- .../ReactiveStorageDeciderDecisionTests.java | 12 +---- 15 files changed, 148 insertions(+), 61 deletions(-) rename x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/{memory => memoryandprocessors}/AutoscalingMemoryAndProcessorInfo.java (91%) rename x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/{memory => memoryandprocessors}/AutoscalingMemoryAndProcessorInfoService.java (89%) rename x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/{memory => memoryandprocessors}/MemoryAndProcessors.java (87%) rename x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/{memory => memoryandprocessors}/AutoscalingMemoryAndProcessorInfoServiceTests.java (99%) diff --git a/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/get_autoscaling_capacity.yml b/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/get_autoscaling_capacity.yml index cc0513d626158..ef2d997de2d30 100644 --- a/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/get_autoscaling_capacity.yml +++ b/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/get_autoscaling_capacity.yml @@ -71,6 +71,52 @@ - gte: { policies.my_autoscaling_policy.current_capacity.node.memory: 0 } - length: { policies.my_autoscaling_policy.current_nodes: 1 } + # test cleanup + - do: + autoscaling.delete_autoscaling_policy: + name: my_autoscaling_policy +--- +"Test get fixed autoscaling capacity with processors`": + - do: + autoscaling.put_autoscaling_policy: + name: my_autoscaling_policy + body: + # voting_only requires master to start so we are sure no nodes match + roles: ["voting_only"] + deciders: + fixed: + storage: 1337b + memory: 7331b + processors: 2 + nodes: 10 + + - match: { "acknowledged": true } + + - do: + autoscaling.get_autoscaling_capacity: {} + + - match: { policies.my_autoscaling_policy.required_capacity.total.storage: 13370 } + - match: { policies.my_autoscaling_policy.required_capacity.total.memory: 73310 } + - match: { policies.my_autoscaling_policy.required_capacity.total.processors: 20 } + - match: { policies.my_autoscaling_policy.required_capacity.node.storage: 1337 } + - match: { policies.my_autoscaling_policy.required_capacity.node.memory: 7331 } + - match: { policies.my_autoscaling_policy.required_capacity.node.processors: 2 } + - match: { policies.my_autoscaling_policy.current_capacity.total.storage: 0 } + - match: { policies.my_autoscaling_policy.current_capacity.total.memory: 0 } + - match: { policies.my_autoscaling_policy.current_capacity.total.processors: 0 } + - match: { policies.my_autoscaling_policy.current_capacity.node.storage: 0 } + - match: { policies.my_autoscaling_policy.current_capacity.node.memory: 0 } + - match: { policies.my_autoscaling_policy.current_capacity.node.processors: 0 } + - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.total.storage: 13370 } + - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.total.memory: 73310 } + - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.total.processors: 20 } + - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.node.storage: 1337 } + - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.node.memory: 7331 } + - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.node.processors: 2 } + - match: { policies.my_autoscaling_policy.deciders.fixed.reason_summary: "fixed storage [1.3kb] memory [7.1kb] nodes [10] processors [2]" } + - length: { policies.my_autoscaling_policy.current_nodes: 0 } + + # test cleanup - do: autoscaling.delete_autoscaling_policy: diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java index af7e288cebe51..e1762d1ed40c3 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java @@ -50,7 +50,7 @@ import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResult; import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderService; import org.elasticsearch.xpack.autoscaling.capacity.FixedAutoscalingDeciderService; -import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryAndProcessorInfoService; +import org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.AutoscalingMemoryAndProcessorInfoService; import org.elasticsearch.xpack.autoscaling.existence.FrozenExistenceDeciderService; import org.elasticsearch.xpack.autoscaling.rest.RestDeleteAutoscalingPolicyHandler; import org.elasticsearch.xpack.autoscaling.rest.RestGetAutoscalingCapacityHandler; diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java index 06cf78234263a..459e970cf6651 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java @@ -27,7 +27,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingLicenseChecker; import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingCalculateCapacityService; -import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryAndProcessorInfoService; +import org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.AutoscalingMemoryAndProcessorInfoService; import java.util.Objects; @@ -40,7 +40,7 @@ public class TransportGetAutoscalingCapacityAction extends TransportMasterNodeAc private final AutoscalingCalculateCapacityService capacityService; private final ClusterInfoService clusterInfoService; private final SnapshotsInfoService snapshotsInfoService; - private final AutoscalingMemoryAndProcessorInfoService memoryInfoService; + private final AutoscalingMemoryAndProcessorInfoService memoryAndProcessorInfoService; private final AutoscalingLicenseChecker autoscalingLicenseChecker; private final CapacityResponseCache responseCache = new CapacityResponseCache<>( run -> threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(run), @@ -57,7 +57,7 @@ public TransportGetAutoscalingCapacityAction( final AutoscalingCalculateCapacityService.Holder capacityServiceHolder, final ClusterInfoService clusterInfoService, final SnapshotsInfoService snapshotsInfoService, - final AutoscalingMemoryAndProcessorInfoService memoryInfoService, + final AutoscalingMemoryAndProcessorInfoService memoryAndProcessorInfoService, final AllocationDeciders allocationDeciders, final AutoscalingLicenseChecker autoscalingLicenseChecker ) { @@ -73,7 +73,7 @@ public TransportGetAutoscalingCapacityAction( ThreadPool.Names.SAME ); this.snapshotsInfoService = snapshotsInfoService; - this.memoryInfoService = memoryInfoService; + this.memoryAndProcessorInfoService = memoryAndProcessorInfoService; this.capacityService = capacityServiceHolder.get(allocationDeciders); this.clusterInfoService = clusterInfoService; this.autoscalingLicenseChecker = Objects.requireNonNull(autoscalingLicenseChecker); @@ -104,7 +104,7 @@ private GetAutoscalingCapacityAction.Response computeCapacity(Runnable ensureNot clusterService.state(), clusterInfoService.getClusterInfo(), snapshotsInfoService.snapshotShardSizes(), - memoryInfoService.snapshot(), + memoryAndProcessorInfoService.snapshot(), ensureNotCancelled ) ); diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java index 72a94e630af3c..c6a1a63d04f79 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java @@ -23,8 +23,8 @@ import org.elasticsearch.xpack.autoscaling.Autoscaling; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; import org.elasticsearch.xpack.autoscaling.action.PolicyValidator; -import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryAndProcessorInfo; -import org.elasticsearch.xpack.autoscaling.capacity.memory.MemoryAndProcessors; +import org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.AutoscalingMemoryAndProcessorInfo; +import org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.MemoryAndProcessors; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicy; import java.util.Collections; diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java index b3ef8f9f6f107..3e3ce81d40d9d 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.autoscaling.capacity; +import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; @@ -18,7 +19,9 @@ import java.io.IOException; import java.util.List; +import java.util.Locale; import java.util.Objects; +import java.util.Optional; public class FixedAutoscalingDeciderService implements AutoscalingDeciderService { @@ -46,7 +49,7 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider ByteSizeValue storage = STORAGE.exists(configuration) ? STORAGE.get(configuration) : null; ByteSizeValue memory = MEMORY.exists(configuration) ? MEMORY.get(configuration) : null; Integer processors = PROCESSORS.exists(configuration) ? PROCESSORS.get(configuration) : null; - if (storage != null || memory != null) { + if (storage != null || memory != null || processors != null) { requiredCapacity = AutoscalingCapacity.builder() .total(totalCapacity(storage, nodes), totalCapacity(memory, nodes), totalCapacity(processors, nodes)) .node(storage, memory, processors) @@ -55,7 +58,7 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider requiredCapacity = null; } - return new AutoscalingDeciderResult(requiredCapacity, new FixedReason(storage, memory, nodes)); + return new AutoscalingDeciderResult(requiredCapacity, new FixedReason(storage, memory, nodes, processors)); } private static ByteSizeValue totalCapacity(ByteSizeValue nodeCapacity, int nodes) { @@ -76,7 +79,7 @@ private static Integer totalCapacity(Integer nodeCapacity, int nodes) { @Override public List> deciderSettings() { - return List.of(STORAGE, MEMORY, NODES); + return List.of(STORAGE, MEMORY, NODES, PROCESSORS); } @Override @@ -98,23 +101,41 @@ public static class FixedReason implements AutoscalingDeciderResult.Reason { private final ByteSizeValue storage; private final ByteSizeValue memory; + private final Integer processors; private final int nodes; - public FixedReason(ByteSizeValue storage, ByteSizeValue memory, int nodes) { + public FixedReason(ByteSizeValue storage, ByteSizeValue memory, int nodes, Integer processors) { this.storage = storage; this.memory = memory; this.nodes = nodes; + this.processors = processors; + if (processors != null && processors < 0) { + throw new IllegalArgumentException("[processors] must be a non-negative number"); + } } public FixedReason(StreamInput in) throws IOException { this.storage = in.readOptionalWriteable(ByteSizeValue::new); this.memory = in.readOptionalWriteable(ByteSizeValue::new); this.nodes = in.readInt(); + if (in.getVersion().onOrAfter(Version.V_8_4_0)) { + this.processors = in.readOptionalVInt(); + } else { + this.processors = null; + } } @Override public String summary() { - return "fixed storage [" + storage + "] memory [" + memory + "] nodes [" + nodes + "]"; + return String.format( + Locale.ROOT, + // We allow processors to be optional in the output for API backwards compatibility + "fixed storage [%s] memory [%s] nodes [%d]%s", + storage, + memory, + nodes, + Optional.ofNullable(processors).map(i -> " processors [" + i + "]").orElse("") + ); } @Override @@ -127,6 +148,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(storage); out.writeOptionalWriteable(memory); out.writeInt(nodes); + if (out.getVersion().onOrAfter(Version.V_8_4_0)) { + out.writeOptionalVInt(processors); + } } @Override @@ -135,6 +159,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("storage", storage); builder.field("memory", memory); builder.field("nodes", nodes); + if (processors != null) { + builder.field("processors", nodes); + } builder.endObject(); return builder; } @@ -144,12 +171,15 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; FixedReason that = (FixedReason) o; - return nodes == that.nodes && Objects.equals(storage, that.storage) && Objects.equals(memory, that.memory); + return nodes == that.nodes + && Objects.equals(storage, that.storage) + && Objects.equals(memory, that.memory) + && Objects.equals(processors, that.processors); } @Override public int hashCode() { - return Objects.hash(storage, memory, nodes); + return Objects.hash(storage, memory, nodes, processors); } } } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfo.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfo.java similarity index 91% rename from x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfo.java rename to x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfo.java index a1946fe1349eb..18b5d29dce856 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfo.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfo.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.autoscaling.capacity.memory; +package org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors; import org.elasticsearch.cluster.node.DiscoveryNode; diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfoService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfoService.java similarity index 89% rename from x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfoService.java rename to x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfoService.java index d8442f81f398a..00a9c6e2893a9 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfoService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfoService.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.autoscaling.capacity.memory; +package org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -114,6 +114,13 @@ private Set addMissingNodes(Set nodes) { } private void sendToMissingNodes(Function nodeLookup, Set missingNodes) { + final Runnable onError = () -> { + synchronized (mutex) { + var builder = new HashMap<>(nodeToMemory); + missingNodes.stream().map(DiscoveryNode::getEphemeralId).forEach(builder::remove); + nodeToMemory = Collections.unmodifiableMap(builder); + } + }; client.admin() .cluster() .nodesStats( @@ -144,13 +151,14 @@ private void sendToMissingNodes(Function nodeLookup, Set< .setMemory(nodeStats.getOs().getMem().getAdjustedTotal().getBytes()) ) ); - nodesInfoResponse.getNodes() - .forEach( - nodeInfo -> builderBuilder.computeIfPresent( - nodeInfo.getNode().getEphemeralId(), - (n, b) -> b.setProcessors(nodeInfo.getInfo(OsInfo.class).getAllocatedProcessors()) - ) + nodesInfoResponse.getNodes().forEach(nodeInfo -> { + assert builderBuilder.containsKey(nodeInfo.getNode().getEphemeralId()) + : "unexpected missing node when setting processors [" + nodeInfo.getNode().getEphemeralId() + "]"; + builderBuilder.computeIfPresent( + nodeInfo.getNode().getEphemeralId(), + (n, b) -> b.setProcessors(nodeInfo.getInfo(OsInfo.class).getAllocatedProcessors()) ); + }); synchronized (mutex) { Map builder = new HashMap<>(nodeToMemory); // Remove all from the builder that failed getting info and stats @@ -170,22 +178,12 @@ private void sendToMissingNodes(Function nodeLookup, Set< nodeToMemory = Collections.unmodifiableMap(builder); } }, e -> { - synchronized (mutex) { - var builder = new HashMap<>(nodeToMemory); - missingNodes.stream().map(DiscoveryNode::getEphemeralId).forEach(builder::remove); - nodeToMemory = Collections.unmodifiableMap(builder); - } - + onError.run(); logger.warn(() -> String.format(Locale.ROOT, "Unable to obtain processor info from [%s]", missingNodes), e); }) ), e -> { - synchronized (mutex) { - var builder = new HashMap<>(nodeToMemory); - missingNodes.stream().map(DiscoveryNode::getEphemeralId).forEach(builder::remove); - nodeToMemory = Collections.unmodifiableMap(builder); - } - + onError.run(); logger.warn(() -> String.format(Locale.ROOT, "Unable to obtain memory info from [%s]", missingNodes), e); } ) diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/MemoryAndProcessors.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/MemoryAndProcessors.java similarity index 87% rename from x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/MemoryAndProcessors.java rename to x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/MemoryAndProcessors.java index b6ffc73df0eec..e16e363a1cc37 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memory/MemoryAndProcessors.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/MemoryAndProcessors.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.autoscaling.capacity.memory; +package org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors; /** * Record for containing memory and processors for given node @@ -22,12 +22,12 @@ static class Builder { private Long memory; private Integer processors; - public Builder setMemory(long memory) { + Builder setMemory(long memory) { this.memory = memory; return this; } - public Builder setProcessors(int processors) { + Builder setProcessors(int processors) { this.processors = processors; return this; } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java index ff99eb8632dc9..864bcbed63e48 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java @@ -658,7 +658,11 @@ public Long getShardSize(ShardRouting shardRouting) { @Override public long getShardSize(ShardRouting shardRouting, long defaultValue) { Long shardSize = super.getShardSize(shardRouting); - return Objects.requireNonNullElseGet(shardSize, () -> delegate.getShardSize(shardRouting, defaultValue)); + if (shardSize != null) { + return shardSize; + } else { + return delegate.getShardSize(shardRouting, defaultValue); + } } @Override diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java index c18f1dfc7d3e7..7b2a3e715c365 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java @@ -45,7 +45,12 @@ public static AutoscalingDeciderResult randomAutoscalingDeciderResult() { protected static AutoscalingDeciderResult randomAutoscalingDeciderResultWithCapacity(AutoscalingCapacity capacity) { return new AutoscalingDeciderResult( capacity, - new FixedAutoscalingDeciderService.FixedReason(randomNullableByteSizeValue(), randomNullableByteSizeValue(), randomInt(1000)) + new FixedAutoscalingDeciderService.FixedReason( + randomNullableByteSizeValue(), + randomNullableByteSizeValue(), + randomInt(1000), + randomInt(64) + ) ); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java index fe6739efc1f04..759f863c5eae5 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java @@ -22,8 +22,8 @@ import org.elasticsearch.snapshots.SnapshotShardSizeInfo; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; import org.elasticsearch.xpack.autoscaling.AutoscalingTestCase; -import org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryAndProcessorInfo; -import org.elasticsearch.xpack.autoscaling.capacity.memory.MemoryAndProcessors; +import org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.AutoscalingMemoryAndProcessorInfo; +import org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.MemoryAndProcessors; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicy; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicyMetadata; @@ -83,11 +83,21 @@ public void testMultiplePoliciesFixedCapacity() { assertThat(deciderResult.requiredCapacity(), equalTo(requiredCapacity)); ByteSizeValue storage = configuration.getAsBytesSize(FixedAutoscalingDeciderService.STORAGE.getKey(), null); ByteSizeValue memory = configuration.getAsMemory(FixedAutoscalingDeciderService.MEMORY.getKey(), null); + Integer processors = configuration.getAsInt(FixedAutoscalingDeciderService.PROCESSORS.getKey(), null); int nodes = FixedAutoscalingDeciderService.NODES.get(configuration); - assertThat(deciderResult.reason(), equalTo(new FixedAutoscalingDeciderService.FixedReason(storage, memory, nodes))); + assertThat(deciderResult.reason(), equalTo(new FixedAutoscalingDeciderService.FixedReason(storage, memory, nodes, processors))); assertThat( deciderResult.reason().summary(), - equalTo("fixed storage [" + storage + "] memory [" + memory + "] nodes [" + nodes + "]") + equalTo( + "fixed storage [" + + storage + + "] memory [" + + memory + + "] nodes [" + + nodes + + "]" + + (processors == null ? "" : " processors [" + processors + "]") + ) ); // there is no nodes in any tier. @@ -143,6 +153,9 @@ private SortedMap randomFixedDeciders() { if (randomBoolean()) { settings.put(FixedAutoscalingDeciderService.MEMORY.getKey(), randomByteSizeValue()); } + if (randomBoolean()) { + settings.put(FixedAutoscalingDeciderService.PROCESSORS.getKey(), randomInt(64)); + } settings.put(FixedAutoscalingDeciderService.NODES.getKey(), randomIntBetween(1, 10)); return new TreeMap<>(Map.of(FixedAutoscalingDeciderService.NAME, settings.build())); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java index b260733f34c08..375547e8071c0 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java @@ -56,9 +56,8 @@ public void testRequiredCapacity() { boolean node = randomBoolean(); boolean storage = randomBoolean(); - boolean memory = randomBoolean() || storage == false; - boolean processor = randomBoolean() || memory == false; - + boolean memory = randomBoolean(); + boolean processor = randomBoolean() || (storage == false && memory == false); AutoscalingCapacity large = randomCapacity(node, storage, memory, processor, 1000, 2000); List autoscalingCapacities = new ArrayList<>(); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderServiceTests.java index 9b2eebae2727e..9f5ee172467d8 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderServiceTests.java @@ -24,8 +24,8 @@ public void testScale() { configurationBuilder = Settings.builder(); ByteSizeValue storage = randomNullableByteSizeValue(); - ByteSizeValue memory = storage != null ? randomNullableByteSizeValue() : randomByteSizeValue(); - Integer processors = memory != null && randomBoolean() ? null : randomInt(64); + ByteSizeValue memory = randomNullableByteSizeValue(); + Integer processors = (memory != null || storage != null) && randomBoolean() ? null : randomInt(64); if (storage != null) { configurationBuilder.put(FixedAutoscalingDeciderService.STORAGE.getKey(), storage); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfoServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfoServiceTests.java similarity index 99% rename from x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfoServiceTests.java rename to x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfoServiceTests.java index a7c55bccdea86..b2d87bda0c7ac 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memory/AutoscalingMemoryAndProcessorInfoServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfoServiceTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.autoscaling.capacity.memory; +package org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors; import org.elasticsearch.Build; import org.elasticsearch.Version; @@ -60,7 +60,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; -import static org.elasticsearch.xpack.autoscaling.capacity.memory.AutoscalingMemoryAndProcessorInfoService.FETCH_TIMEOUT; +import static org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.AutoscalingMemoryAndProcessorInfoService.FETCH_TIMEOUT; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java index 539ebfb65c369..7a1d5022a10b7 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java @@ -551,16 +551,8 @@ static AutoscalingCapacity randomCurrentCapacity() { boolean includeMemory = randomBoolean(); boolean includeProcessors = randomBoolean(); return AutoscalingCapacity.builder() - .total( - randomByteSizeValue(), - includeMemory || includeProcessors ? randomByteSizeValue() : null, - includeProcessors ? randomInt(64) : null - ) - .node( - randomByteSizeValue(), - includeMemory || includeProcessors ? randomByteSizeValue() : null, - includeProcessors ? randomInt(64) : null - ) + .total(randomByteSizeValue(), includeMemory ? randomByteSizeValue() : null, includeProcessors ? randomInt(64) : null) + .node(randomByteSizeValue(), includeMemory ? randomByteSizeValue() : null, includeProcessors ? randomInt(64) : null) .build(); } else { return null; From ba35cddb8f1e026ab096ed2aa84c6fd59a798c69 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 22 Jun 2022 11:51:59 -0400 Subject: [PATCH 04/11] renaming --- .../xpack/autoscaling/Autoscaling.java | 6 +-- ...TransportGetAutoscalingCapacityAction.java | 6 +-- .../AutoscalingCalculateCapacityService.java | 37 ++++++++++------- .../capacity/AutoscalingCapacity.java | 9 ++-- .../AutoscalingNodeInfo.java} | 8 ++-- .../AutoscalingNodeInfoService.java} | 30 ++++++-------- .../NodeInfo.java} | 8 ++-- .../autoscaling/AutoscalingTestCase.java | 2 +- ...oscalingCalculateCapacityServiceTests.java | 20 ++++----- ...scalingCapacityWireSerializationTests.java | 41 +++++++++---------- .../AutoscalingDeciderResultsTests.java | 12 ++---- .../AutoscalingNodeInfoServiceTests.java} | 32 +++++++++++---- 12 files changed, 110 insertions(+), 101 deletions(-) rename x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/{memoryandprocessors/AutoscalingMemoryAndProcessorInfo.java => nodeinfo/AutoscalingNodeInfo.java} (69%) rename x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/{memoryandprocessors/AutoscalingMemoryAndProcessorInfoService.java => nodeinfo/AutoscalingNodeInfoService.java} (88%) rename x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/{memoryandprocessors/MemoryAndProcessors.java => nodeinfo/NodeInfo.java} (81%) rename x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/{memoryandprocessors/AutoscalingMemoryAndProcessorInfoServiceTests.java => nodeinfo/AutoscalingNodeInfoServiceTests.java} (96%) diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java index e1762d1ed40c3..57e565c165ba6 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java @@ -50,7 +50,7 @@ import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResult; import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderService; import org.elasticsearch.xpack.autoscaling.capacity.FixedAutoscalingDeciderService; -import org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.AutoscalingMemoryAndProcessorInfoService; +import org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.AutoscalingNodeInfoService; import org.elasticsearch.xpack.autoscaling.existence.FrozenExistenceDeciderService; import org.elasticsearch.xpack.autoscaling.rest.RestDeleteAutoscalingPolicyHandler; import org.elasticsearch.xpack.autoscaling.rest.RestGetAutoscalingCapacityHandler; @@ -119,13 +119,13 @@ public Collection createComponents( return List.of( new AutoscalingCalculateCapacityService.Holder(this), autoscalingLicenseChecker, - new AutoscalingMemoryAndProcessorInfoService(clusterService, client) + new AutoscalingNodeInfoService(clusterService, client) ); } @Override public List> getSettings() { - return List.of(AutoscalingMemoryAndProcessorInfoService.FETCH_TIMEOUT); + return List.of(AutoscalingNodeInfoService.FETCH_TIMEOUT); } @Override diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java index 459e970cf6651..9093a71a5af1c 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java @@ -27,7 +27,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.autoscaling.AutoscalingLicenseChecker; import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingCalculateCapacityService; -import org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.AutoscalingMemoryAndProcessorInfoService; +import org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.AutoscalingNodeInfoService; import java.util.Objects; @@ -40,7 +40,7 @@ public class TransportGetAutoscalingCapacityAction extends TransportMasterNodeAc private final AutoscalingCalculateCapacityService capacityService; private final ClusterInfoService clusterInfoService; private final SnapshotsInfoService snapshotsInfoService; - private final AutoscalingMemoryAndProcessorInfoService memoryAndProcessorInfoService; + private final AutoscalingNodeInfoService memoryAndProcessorInfoService; private final AutoscalingLicenseChecker autoscalingLicenseChecker; private final CapacityResponseCache responseCache = new CapacityResponseCache<>( run -> threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(run), @@ -57,7 +57,7 @@ public TransportGetAutoscalingCapacityAction( final AutoscalingCalculateCapacityService.Holder capacityServiceHolder, final ClusterInfoService clusterInfoService, final SnapshotsInfoService snapshotsInfoService, - final AutoscalingMemoryAndProcessorInfoService memoryAndProcessorInfoService, + final AutoscalingNodeInfoService memoryAndProcessorInfoService, final AllocationDeciders allocationDeciders, final AutoscalingLicenseChecker autoscalingLicenseChecker ) { diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java index c6a1a63d04f79..7556b8509e17b 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java @@ -23,8 +23,8 @@ import org.elasticsearch.xpack.autoscaling.Autoscaling; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; import org.elasticsearch.xpack.autoscaling.action.PolicyValidator; -import org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.AutoscalingMemoryAndProcessorInfo; -import org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.MemoryAndProcessors; +import org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.AutoscalingNodeInfo; +import org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.NodeInfo; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicy; import java.util.Collections; @@ -109,7 +109,7 @@ public SortedMap calculate( ClusterState state, ClusterInfo clusterInfo, SnapshotShardSizeInfo shardSizeInfo, - AutoscalingMemoryAndProcessorInfo memoryInfo, + AutoscalingNodeInfo autoscalingNodeInfo, Runnable ensureNotCancelled ) { AutoscalingMetadata autoscalingMetadata = state.metadata().custom(AutoscalingMetadata.NAME); @@ -121,7 +121,14 @@ public SortedMap calculate( .map( e -> Tuple.tuple( e.getKey(), - calculateForPolicy(e.getValue().policy(), state, clusterInfo, shardSizeInfo, memoryInfo, ensureNotCancelled) + calculateForPolicy( + e.getValue().policy(), + state, + clusterInfo, + shardSizeInfo, + autoscalingNodeInfo, + ensureNotCancelled + ) ) ) .collect(Collectors.toMap(Tuple::v1, Tuple::v2)) @@ -136,7 +143,7 @@ private AutoscalingDeciderResults calculateForPolicy( ClusterState state, ClusterInfo clusterInfo, SnapshotShardSizeInfo shardSizeInfo, - AutoscalingMemoryAndProcessorInfo memoryInfo, + AutoscalingNodeInfo autoscalingNodeInfo, Runnable ensureNotCancelled ) { if (hasUnknownRoles(policy)) { @@ -152,7 +159,7 @@ private AutoscalingDeciderResults calculateForPolicy( state, clusterInfo, shardSizeInfo, - memoryInfo, + autoscalingNodeInfo, ensureNotCancelled ); SortedMap results = deciders.entrySet() @@ -193,10 +200,10 @@ DefaultAutoscalingDeciderContext createContext( ClusterState state, ClusterInfo clusterInfo, SnapshotShardSizeInfo shardSizeInfo, - AutoscalingMemoryAndProcessorInfo memoryInfo, + AutoscalingNodeInfo autoscalingNodeInfo, Runnable ensureNotCancelled ) { - return new DefaultAutoscalingDeciderContext(roles, state, clusterInfo, shardSizeInfo, memoryInfo, ensureNotCancelled); + return new DefaultAutoscalingDeciderContext(roles, state, clusterInfo, shardSizeInfo, autoscalingNodeInfo, ensureNotCancelled); } /** @@ -225,7 +232,7 @@ static class DefaultAutoscalingDeciderContext implements AutoscalingDeciderConte private final ClusterState state; private final ClusterInfo clusterInfo; private final SnapshotShardSizeInfo snapshotShardSizeInfo; - private final AutoscalingMemoryAndProcessorInfo memoryInfo; + private final AutoscalingNodeInfo autoscalingNodeInfo; private final SortedSet currentNodes; private final AutoscalingCapacity currentCapacity; private final boolean currentCapacityAccurate; @@ -236,7 +243,7 @@ static class DefaultAutoscalingDeciderContext implements AutoscalingDeciderConte ClusterState state, ClusterInfo clusterInfo, SnapshotShardSizeInfo snapshotShardSizeInfo, - AutoscalingMemoryAndProcessorInfo memoryInfo, + AutoscalingNodeInfo autoscalingNodeInfo, Runnable ensureNotCancelled ) { this.roles = roles.stream().map(DiscoveryNodeRole::getRoleFromRoleName).collect(Sets.toUnmodifiableSortedSet()); @@ -245,7 +252,7 @@ static class DefaultAutoscalingDeciderContext implements AutoscalingDeciderConte this.state = state; this.clusterInfo = clusterInfo; this.snapshotShardSizeInfo = snapshotShardSizeInfo; - this.memoryInfo = memoryInfo; + this.autoscalingNodeInfo = autoscalingNodeInfo; this.currentNodes = state.nodes() .stream() .filter(this::rolesFilter) @@ -297,7 +304,7 @@ private boolean nodeHasAccurateCapacity(DiscoveryNode node) { } } - return memoryInfo.get(node).isPresent(); + return autoscalingNodeInfo.get(node).isPresent(); } private AutoscalingCapacity calculateCurrentCapacity() { @@ -321,11 +328,11 @@ private AutoscalingCapacity.AutoscalingResources resourcesFor(DiscoveryNode node ) : 0L; - Optional memoryAndProcessors = memoryInfo.get(node); + Optional memoryAndProcessors = autoscalingNodeInfo.get(node); return new AutoscalingCapacity.AutoscalingResources( storage == -1 ? ByteSizeValue.ZERO : new ByteSizeValue(storage), - memoryAndProcessors.map(MemoryAndProcessors::memory).map(ByteSizeValue::new).orElse(ByteSizeValue.ZERO), - memoryAndProcessors.map(MemoryAndProcessors::processors).orElse(0) + memoryAndProcessors.map(NodeInfo::memory).map(ByteSizeValue::new).orElse(ByteSizeValue.ZERO), + memoryAndProcessors.map(NodeInfo::processors).orElse(0) ); } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java index 03295d680d06d..2599917c83f7a 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.Objects; -import java.util.OptionalInt; /** * Represents current/required capacity of a single tier. @@ -56,20 +55,18 @@ public AutoscalingResources(StreamInput in) throws IOException { } } + @Nullable public ByteSizeValue storage() { return storage; } + @Nullable public ByteSizeValue memory() { return memory; } - public OptionalInt processors() { - return processors == null ? OptionalInt.empty() : OptionalInt.of(processors); - } - @Nullable - Integer getProcessors() { + public Integer processors() { return processors; } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfo.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfo.java similarity index 69% rename from x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfo.java rename to x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfo.java index 18b5d29dce856..2d6e62a38da03 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfo.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfo.java @@ -5,19 +5,19 @@ * 2.0. */ -package org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors; +package org.elasticsearch.xpack.autoscaling.capacity.nodeinfo; import org.elasticsearch.cluster.node.DiscoveryNode; import java.util.Optional; -public interface AutoscalingMemoryAndProcessorInfo { - AutoscalingMemoryAndProcessorInfo EMPTY = n -> Optional.empty(); +public interface AutoscalingNodeInfo { + AutoscalingNodeInfo EMPTY = n -> Optional.empty(); /** * Get the memory and processor use for the indicated node. Returns null if not available (new, fetching or failed). * @param node the node to get info for * @return memory and processor info for node if possible */ - Optional get(DiscoveryNode node); + Optional get(DiscoveryNode node); } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfoService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java similarity index 88% rename from x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfoService.java rename to x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java index 00a9c6e2893a9..00f913a3a587f 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfoService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors; +package org.elasticsearch.xpack.autoscaling.capacity.nodeinfo; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,29 +40,28 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import java.util.stream.StreamSupport; import static java.util.stream.Collectors.toUnmodifiableMap; -public class AutoscalingMemoryAndProcessorInfoService { +public class AutoscalingNodeInfoService { public static final Setting FETCH_TIMEOUT = Setting.timeSetting( "xpack.autoscaling.memory.monitor.timeout", TimeValue.timeValueSeconds(15), Setting.Property.Dynamic, Setting.Property.NodeScope ); - private static final MemoryAndProcessors FETCHING_SENTINEL = new MemoryAndProcessors(Long.MIN_VALUE, Integer.MIN_VALUE); + private static final NodeInfo FETCHING_SENTINEL = new NodeInfo(Long.MIN_VALUE, Integer.MIN_VALUE); - private static final Logger logger = LogManager.getLogger(AutoscalingMemoryAndProcessorInfoService.class); + private static final Logger logger = LogManager.getLogger(AutoscalingNodeInfoService.class); - private volatile Map nodeToMemory = Map.of(); + private volatile Map nodeToMemory = Map.of(); private volatile TimeValue fetchTimeout; private final Client client; private final Object mutex = new Object(); @Inject - public AutoscalingMemoryAndProcessorInfoService(ClusterService clusterService, Client client) { + public AutoscalingNodeInfoService(ClusterService clusterService, Client client) { this.client = client; this.fetchTimeout = FETCH_TIMEOUT.get(clusterService.getSettings()); if (DiscoveryNode.isMasterNode(clusterService.getSettings())) { @@ -140,15 +139,14 @@ private void sendToMissingNodes(Function nodeLookup, Set< .toArray(String[]::new) ).clear().addMetric(NodesInfoRequest.Metric.OS.metricName()).timeout(fetchTimeout), ActionListener.wrap(nodesInfoResponse -> { - final Map builderBuilder = Maps.newHashMapWithExpectedSize( + final Map builderBuilder = Maps.newHashMapWithExpectedSize( nodesStatsResponse.getNodes().size() ); nodesStatsResponse.getNodes() .forEach( nodeStats -> builderBuilder.put( nodeStats.getNode().getEphemeralId(), - MemoryAndProcessors.builder() - .setMemory(nodeStats.getOs().getMem().getAdjustedTotal().getBytes()) + NodeInfo.builder().setMemory(nodeStats.getOs().getMem().getAdjustedTotal().getBytes()) ) ); nodesInfoResponse.getNodes().forEach(nodeInfo -> { @@ -160,7 +158,7 @@ private void sendToMissingNodes(Function nodeLookup, Set< ); }); synchronized (mutex) { - Map builder = new HashMap<>(nodeToMemory); + Map builder = new HashMap<>(nodeToMemory); // Remove all from the builder that failed getting info and stats Stream.concat(nodesStatsResponse.failures().stream(), nodesInfoResponse.failures().stream()) .map(FailedNodeException::nodeId) @@ -211,9 +209,7 @@ private Set toRoles(SortedSet roleNames) { private void retainAliveNodes(Set currentNodes) { assert Thread.holdsLock(mutex); Set ephemeralIds = currentNodes.stream().map(DiscoveryNode::getEphemeralId).collect(Collectors.toSet()); - Set toRemove = StreamSupport.stream(nodeToMemory.keySet().spliterator(), false) - .filter(Predicate.not(ephemeralIds::contains)) - .collect(Collectors.toSet()); + Set toRemove = nodeToMemory.keySet().stream().filter(Predicate.not(ephemeralIds::contains)).collect(Collectors.toSet()); if (toRemove.isEmpty() == false) { nodeToMemory = nodeToMemory.entrySet() .stream() @@ -222,10 +218,10 @@ private void retainAliveNodes(Set currentNodes) { } } - public AutoscalingMemoryAndProcessorInfo snapshot() { - final Map nodeToMemoryRef = this.nodeToMemory; + public AutoscalingNodeInfo snapshot() { + final Map nodeToMemoryRef = this.nodeToMemory; return node -> { - MemoryAndProcessors result = nodeToMemoryRef.get(node.getEphemeralId()); + NodeInfo result = nodeToMemoryRef.get(node.getEphemeralId()); if (result == FETCHING_SENTINEL) { return Optional.empty(); } else { diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/MemoryAndProcessors.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/NodeInfo.java similarity index 81% rename from x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/MemoryAndProcessors.java rename to x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/NodeInfo.java index e16e363a1cc37..99e33b5c16252 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/MemoryAndProcessors.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/NodeInfo.java @@ -5,14 +5,14 @@ * 2.0. */ -package org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors; +package org.elasticsearch.xpack.autoscaling.capacity.nodeinfo; /** * Record for containing memory and processors for given node * @param memory node total memory * @param processors allocated processors */ -public record MemoryAndProcessors(long memory, int processors) { +public record NodeInfo(long memory, int processors) { static Builder builder() { return new Builder(); @@ -36,9 +36,9 @@ boolean canBuild() { return memory != null && processors != null; } - MemoryAndProcessors build() { + NodeInfo build() { assert memory != null && processors != null : "unexpected null values when building node memory and processors information"; - return new MemoryAndProcessors(memory, processors); + return new NodeInfo(memory, processors); } } } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java index 7b2a3e715c365..af66978f0d00c 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java @@ -67,7 +67,7 @@ public static AutoscalingCapacity randomAutoscalingCapacity() { return new AutoscalingCapacity( total, randomBoolean() - ? randomNullValueAutoscalingResources(total.storage() != null, total.memory() != null, total.processors().isPresent()) + ? randomNullValueAutoscalingResources(total.storage() != null, total.memory() != null, total.processors() != null) : null ); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java index 759f863c5eae5..38a6eda2da0d4 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java @@ -22,8 +22,8 @@ import org.elasticsearch.snapshots.SnapshotShardSizeInfo; import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; import org.elasticsearch.xpack.autoscaling.AutoscalingTestCase; -import org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.AutoscalingMemoryAndProcessorInfo; -import org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.MemoryAndProcessors; +import org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.AutoscalingNodeInfo; +import org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.NodeInfo; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicy; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicyMetadata; @@ -66,7 +66,7 @@ public void testMultiplePoliciesFixedCapacity() { state, ClusterInfo.EMPTY, null, - AutoscalingMemoryAndProcessorInfo.EMPTY, + AutoscalingNodeInfo.EMPTY, () -> {} ); assertThat(resultsMap.keySet(), equalTo(policyNames)); @@ -137,7 +137,7 @@ public String name() { .build(); assertThat( - service.calculate(state, ClusterInfo.EMPTY, SnapshotShardSizeInfo.EMPTY, AutoscalingMemoryAndProcessorInfo.EMPTY, () -> {}) + service.calculate(state, ClusterInfo.EMPTY, SnapshotShardSizeInfo.EMPTY, AutoscalingNodeInfo.EMPTY, () -> {}) .get("test") .results() .keySet(), @@ -169,7 +169,7 @@ private AutoscalingCapacity calculateFixedDeciderCapacity(Settings configuration ByteSizeValue totalMemory = memory != null ? new ByteSizeValue(memory.getBytes() * nodes) : null; Integer totalProcessors = processors != null ? processors * nodes : null; - if (totalStorage == null && totalMemory == null) { + if (totalStorage == null && totalMemory == null && totalProcessors == null) { return null; } else { return new AutoscalingCapacity( @@ -192,7 +192,7 @@ public void testContext() { state, info, snapshotShardSizeInfo, - n -> Optional.of(new MemoryAndProcessors(randomNonNegativeLong(), randomInt(64))), + n -> Optional.of(new NodeInfo(randomNonNegativeLong(), randomInt(64))), () -> {} ); @@ -217,7 +217,7 @@ public void testContext() { state, info, null, - n -> Optional.of(new MemoryAndProcessors(memory, randomInt(64))), + n -> Optional.of(new NodeInfo(memory, randomInt(64))), () -> {} ); @@ -275,7 +275,7 @@ public void testContext() { state, info, null, - n -> Optional.of(new MemoryAndProcessors(memory, randomInt(64))), + n -> Optional.of(new NodeInfo(memory, randomInt(64))), () -> {} ); @@ -296,7 +296,7 @@ public void testContext() { state, info, null, - AutoscalingMemoryAndProcessorInfo.EMPTY, + AutoscalingNodeInfo.EMPTY, () -> {} ); assertThat(context.nodes(), equalTo(expectedNodes)); @@ -321,7 +321,7 @@ public void testContext() { state, info, null, - n -> Optional.of(new MemoryAndProcessors(memory, randomInt(64))), + n -> Optional.of(new NodeInfo(memory, randomInt(64))), () -> {} ); assertThat(context.nodes(), equalTo(expectedNodes)); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacityWireSerializationTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacityWireSerializationTests.java index 8ddcf2b7d97fd..19c933b8c2fda 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacityWireSerializationTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacityWireSerializationTests.java @@ -12,6 +12,8 @@ import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.autoscaling.AutoscalingTestCase; +import java.util.Optional; + public class AutoscalingCapacityWireSerializationTests extends AbstractWireSerializingTestCase { @Override protected Writeable.Reader instanceReader() { @@ -30,7 +32,7 @@ protected AutoscalingCapacity mutateInstance(AutoscalingCapacity instance) { // mutate total boolean hasAllMetrics = instance.total().memory() != null && instance.total().storage() != null - && instance.total().processors().isPresent(); + && instance.total().processors() != null; if (randomBoolean()) { builder.total( randomByteSize( @@ -38,7 +40,7 @@ protected AutoscalingCapacity mutateInstance(AutoscalingCapacity instance) { instance.total().storage() ), instance.total().memory(), - instance.total().getProcessors() + instance.total().processors() ); } else if (randomBoolean()) { builder.total( @@ -47,15 +49,15 @@ protected AutoscalingCapacity mutateInstance(AutoscalingCapacity instance) { hasAllMetrics && (instance.node() == null || instance.node().memory() == null), instance.total().memory() ), - instance.total().getProcessors() + instance.total().processors() ); } else { builder.total( instance.total().storage(), instance.total().memory(), - hasAllMetrics && (instance.node() == null || instance.node().processors().isEmpty()) && randomBoolean() + hasAllMetrics && (instance.node() == null || instance.node().processors() == null) && randomBoolean() ? null - : randomIntBetween(1, 64) + instance.total().processors().orElse(0) + : randomIntBetween(1, 64) + Optional.ofNullable(instance.total().processors()).orElse(0) ); } } else { @@ -65,47 +67,44 @@ protected AutoscalingCapacity mutateInstance(AutoscalingCapacity instance) { AutoscalingTestCase.randomNullValueAutoscalingResources( instance.total().storage() != null, instance.total().memory() != null, - instance.total().processors().isPresent() + instance.total().processors() != null ) ); } else if (randomBoolean() && instance.total().storage() != null) { builder.node( - randomByteSize(instance.node().memory() != null || instance.node().processors().isPresent(), instance.node().storage()), + randomByteSize(instance.node().memory() != null || instance.node().processors() != null, instance.node().storage()), instance.node().memory(), - instance.node().getProcessors() + instance.node().processors() ); } else if (randomBoolean() && instance.total().memory() != null) { builder.node( instance.node().storage(), - randomByteSize(instance.node().storage() != null || instance.node().processors().isPresent(), instance.node().memory()), - instance.node().getProcessors() + randomByteSize(instance.node().storage() != null || instance.node().processors() != null, instance.node().memory()), + instance.node().processors() ); - } else if (instance.total().processors().isPresent()) { + } else if (instance.total().processors() != null) { builder.node( instance.node().storage(), instance.node().memory(), randomBoolean() && (instance.node().storage() != null || instance.node().memory() != null) - && instance.node().processors().isPresent() + && instance.node().processors() != null ? null - : randomIntBetween(1, 64) + instance.node().processors().orElse(0) + : randomIntBetween(1, 64) + Optional.ofNullable(instance.node().processors()).orElse(0) ); } else { ByteSizeValue newStorage = instance.total().storage() != null - ? randomByteSize( - instance.node().memory() != null || instance.node().processors().isPresent(), - instance.node().storage() - ) + ? randomByteSize(instance.node().memory() != null || instance.node().processors() != null, instance.node().storage()) : null; ByteSizeValue newMem = instance.total().memory() != null - ? randomByteSize(newStorage != null || instance.node().processors().isPresent(), instance.node().memory()) + ? randomByteSize(newStorage != null || instance.node().processors() != null, instance.node().memory()) : null; builder.node( newStorage, newMem, - randomBoolean() && (newMem != null || newStorage != null) && instance.node().processors().isPresent() ? null - : instance.total().processors().isPresent() && randomBoolean() - ? randomIntBetween(1, 64) + instance.node().processors().orElse(0) + randomBoolean() && (newMem != null || newStorage != null) && instance.node().processors() != null ? null + : instance.total().processors() != null && randomBoolean() + ? randomIntBetween(1, 64) + Optional.ofNullable(instance.node().processors()).orElse(0) : null ); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java index 375547e8071c0..12db5657dc127 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java @@ -9,11 +9,11 @@ import org.elasticsearch.common.Randomness; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.autoscaling.AutoscalingTestCase; @@ -110,7 +110,7 @@ public Map toMap(ToXContent tox) { } try ( XContentParser parser = XContentType.JSON.xContent() - .createParser(NamedXContentRegistry.EMPTY, null, BytesReference.bytes(builder).streamInput()) + .createParser(XContentParserConfiguration.EMPTY, BytesReference.bytes(builder).streamInput()) ) { return parser.map(); } @@ -131,13 +131,9 @@ private void verifySingleMetricLarger( autoscalingCapacities.add(larger); Randomness.shuffle(autoscalingCapacities); AutoscalingCapacity.Builder expectedBuilder = AutoscalingCapacity.builder() - .total(expectedStorage.total().storage(), expectedMemory.total().memory(), expectedProcessor.total().getProcessors()); + .total(expectedStorage.total().storage(), expectedMemory.total().memory(), expectedProcessor.total().processors()); if (node) { - expectedBuilder.node( - expectedStorage.node().storage(), - expectedMemory.node().memory(), - expectedProcessor.node().getProcessors() - ); + expectedBuilder.node(expectedStorage.node().storage(), expectedMemory.node().memory(), expectedProcessor.node().processors()); } verifyRequiredCapacity(expectedBuilder.build(), autoscalingCapacities.toArray(AutoscalingCapacity[]::new)); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfoServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoServiceTests.java similarity index 96% rename from x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfoServiceTests.java rename to x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoServiceTests.java index b2d87bda0c7ac..9b7d60663566c 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/memoryandprocessors/AutoscalingMemoryAndProcessorInfoServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoServiceTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors; +package org.elasticsearch.xpack.autoscaling.capacity.nodeinfo; import org.elasticsearch.Build; import org.elasticsearch.Version; @@ -14,7 +14,6 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; @@ -60,7 +59,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; -import static org.elasticsearch.xpack.autoscaling.capacity.memoryandprocessors.AutoscalingMemoryAndProcessorInfoService.FETCH_TIMEOUT; +import static org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.AutoscalingNodeInfoService.FETCH_TIMEOUT; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -69,10 +68,10 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class AutoscalingMemoryAndProcessorInfoServiceTests extends AutoscalingTestCase { +public class AutoscalingNodeInfoServiceTests extends AutoscalingTestCase { private NodeStatsClient client; - private AutoscalingMemoryAndProcessorInfoService service; + private AutoscalingNodeInfoService service; private TimeValue fetchTimeout; private AutoscalingMetadata autoscalingMetadata; private Metadata metadata; @@ -95,7 +94,7 @@ public void setUp() throws Exception { Set> settingsSet = Sets.union(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS, Set.of(FETCH_TIMEOUT)); ClusterSettings clusterSettings = new ClusterSettings(settings, settingsSet); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); - service = new AutoscalingMemoryAndProcessorInfoService(clusterService, client); + service = new AutoscalingNodeInfoService(clusterService, client); autoscalingMetadata = randomAutoscalingMetadataOfPolicyCount(between(1, 8)); metadata = Metadata.builder().putCustom(AutoscalingMetadata.NAME, autoscalingMetadata).build(); } @@ -401,7 +400,7 @@ public void assertMatchesResponse(Set nodes, NodesStatsResponse r assertThat( service.snapshot().get(n).get(), equalTo( - new MemoryAndProcessors( + new NodeInfo( response.getNodesMap().get(n.getId()).getOs().getMem().getAdjustedTotal().getBytes(), infoResponse.getNodesMap().get(n.getId()).getInfo(OsInfo.class).getAllocatedProcessors() ) @@ -445,9 +444,24 @@ private static NodeStats statsForNode(DiscoveryNode node, long memory) { ); } - private static NodeInfo infoForNode(DiscoveryNode node, int processors) { + private static org.elasticsearch.action.admin.cluster.node.info.NodeInfo infoForNode(DiscoveryNode node, int processors) { OsInfo osInfo = new OsInfo(randomLong(), processors, processors, null, null, null, null); - return new NodeInfo(Version.CURRENT, Build.CURRENT, node, null, osInfo, null, null, null, null, null, null, null, null, null); + return new org.elasticsearch.action.admin.cluster.node.info.NodeInfo( + Version.CURRENT, + Build.CURRENT, + node, + null, + osInfo, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); } private class NodeStatsClient extends NoOpClient { From e3bd75d7396e9ad7073ae5afe7d5e4e2be74b583 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 23 Jun 2022 08:12:35 -0400 Subject: [PATCH 05/11] switch processors to a float --- .../autoscaling/get_autoscaling_capacity.yml | 2 +- .../AutoscalingCalculateCapacityService.java | 2 +- .../capacity/AutoscalingCapacity.java | 26 +++++++++---------- .../FixedAutoscalingDeciderService.java | 14 +++++----- .../capacity/nodeinfo/NodeInfo.java | 6 ++--- .../FrozenExistenceDeciderService.java | 2 +- .../autoscaling/AutoscalingTestCase.java | 6 ++--- ...oscalingCalculateCapacityServiceTests.java | 6 ++--- ...scalingCapacityWireSerializationTests.java | 6 ++--- .../AutoscalingDeciderResultsTests.java | 4 +-- .../FixedAutoscalingDeciderServiceTests.java | 4 +-- .../ReactiveStorageDeciderDecisionTests.java | 23 +++++++--------- 12 files changed, 49 insertions(+), 52 deletions(-) diff --git a/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/get_autoscaling_capacity.yml b/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/get_autoscaling_capacity.yml index ef2d997de2d30..fdc8af1d880b4 100644 --- a/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/get_autoscaling_capacity.yml +++ b/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/get_autoscaling_capacity.yml @@ -113,7 +113,7 @@ - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.node.storage: 1337 } - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.node.memory: 7331 } - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.node.processors: 2 } - - match: { policies.my_autoscaling_policy.deciders.fixed.reason_summary: "fixed storage [1.3kb] memory [7.1kb] nodes [10] processors [2]" } + - match: { policies.my_autoscaling_policy.deciders.fixed.reason_summary: "fixed storage [1.3kb] memory [7.1kb] nodes [10] processors [2.0]" } - length: { policies.my_autoscaling_policy.current_nodes: 0 } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java index 7556b8509e17b..c57e085b3fb3f 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java @@ -332,7 +332,7 @@ private AutoscalingCapacity.AutoscalingResources resourcesFor(DiscoveryNode node return new AutoscalingCapacity.AutoscalingResources( storage == -1 ? ByteSizeValue.ZERO : new ByteSizeValue(storage), memoryAndProcessors.map(NodeInfo::memory).map(ByteSizeValue::new).orElse(ByteSizeValue.ZERO), - memoryAndProcessors.map(NodeInfo::processors).orElse(0) + memoryAndProcessors.map(NodeInfo::processors).orElse(0f) ); } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java index 2599917c83f7a..09b8988249239 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java @@ -31,15 +31,15 @@ public class AutoscalingCapacity implements ToXContent, Writeable { public static class AutoscalingResources implements ToXContent, Writeable { private final ByteSizeValue storage; private final ByteSizeValue memory; - private final Integer processors; + private final Float processors; - public static final AutoscalingResources ZERO = new AutoscalingResources(new ByteSizeValue(0), new ByteSizeValue(0), 0); + public static final AutoscalingResources ZERO = new AutoscalingResources(new ByteSizeValue(0), new ByteSizeValue(0), 0.0f); - public AutoscalingResources(ByteSizeValue storage, ByteSizeValue memory, Integer processors) { + public AutoscalingResources(ByteSizeValue storage, ByteSizeValue memory, Float processors) { assert storage != null || memory != null || processors != null; this.storage = storage; this.memory = memory; - if (processors != null && processors < 0) { + if (processors != null && processors < 0.0f) { throw new IllegalArgumentException("[processors] must be a non-negative number; provided [" + processors + "]"); } this.processors = processors; @@ -49,7 +49,7 @@ public AutoscalingResources(StreamInput in) throws IOException { this.storage = in.readOptionalWriteable(ByteSizeValue::new); this.memory = in.readOptionalWriteable(ByteSizeValue::new); if (in.getVersion().onOrAfter(Version.V_8_4_0)) { - this.processors = in.readOptionalVInt(); + this.processors = in.readOptionalFloat(); } else { this.processors = null; } @@ -66,7 +66,7 @@ public ByteSizeValue memory() { } @Nullable - public Integer processors() { + public Float processors() { return processors; } @@ -96,7 +96,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(storage); out.writeOptionalWriteable(memory); if (out.getVersion().onOrAfter(Version.V_8_4_0)) { - out.writeOptionalVInt(processors); + out.writeOptionalFloat(processors); } } @@ -152,7 +152,7 @@ private static ByteSizeValue add(ByteSizeValue v1, ByteSizeValue v2) { return new ByteSizeValue(v1.getBytes() + v2.getBytes()); } - private static Integer max(Integer v1, Integer v2) { + private static Float max(Float v1, Float v2) { if (v1 == null) { return v2; } @@ -163,7 +163,7 @@ private static Integer max(Integer v1, Integer v2) { return v1.compareTo(v2) < 0 ? v2 : v1; } - private static Integer add(Integer v1, Integer v2) { + private static Float add(Float v1, Float v2) { if (v1 == null) { return v2; } @@ -286,11 +286,11 @@ public Builder capacity(AutoscalingCapacity capacity) { return this; } - public Builder total(Long storage, Long memory, Integer processors) { + public Builder total(Long storage, Long memory, Float processors) { return total(byteSizeValue(storage), byteSizeValue(memory), processors); } - public Builder total(ByteSizeValue storage, ByteSizeValue memory, Integer processors) { + public Builder total(ByteSizeValue storage, ByteSizeValue memory, Float processors) { return total(new AutoscalingResources(storage, memory, processors)); } @@ -299,11 +299,11 @@ public Builder total(AutoscalingResources total) { return this; } - public Builder node(Long storage, Long memory, Integer processors) { + public Builder node(Long storage, Long memory, Float processors) { return node(byteSizeValue(storage), byteSizeValue(memory), processors); } - public Builder node(ByteSizeValue storage, ByteSizeValue memory, Integer processors) { + public Builder node(ByteSizeValue storage, ByteSizeValue memory, Float processors) { return node(new AutoscalingResources(storage, memory, processors)); } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java index 3e3ce81d40d9d..1ce8b25a04db1 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java @@ -29,7 +29,7 @@ public class FixedAutoscalingDeciderService implements AutoscalingDeciderService public static final Setting STORAGE = Setting.byteSizeSetting("storage", ByteSizeValue.ofBytes(-1)); public static final Setting MEMORY = Setting.byteSizeSetting("memory", ByteSizeValue.ofBytes(-1)); - public static final Setting PROCESSORS = Setting.intSetting("processors", 1, 0); + public static final Setting PROCESSORS = Setting.floatSetting("processors", 1f, 0f); public static final Setting NODES = Setting.intSetting("nodes", 1, 0); @Inject @@ -48,7 +48,7 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider AutoscalingCapacity requiredCapacity; ByteSizeValue storage = STORAGE.exists(configuration) ? STORAGE.get(configuration) : null; ByteSizeValue memory = MEMORY.exists(configuration) ? MEMORY.get(configuration) : null; - Integer processors = PROCESSORS.exists(configuration) ? PROCESSORS.get(configuration) : null; + Float processors = PROCESSORS.exists(configuration) ? PROCESSORS.get(configuration) : null; if (storage != null || memory != null || processors != null) { requiredCapacity = AutoscalingCapacity.builder() .total(totalCapacity(storage, nodes), totalCapacity(memory, nodes), totalCapacity(processors, nodes)) @@ -69,7 +69,7 @@ private static ByteSizeValue totalCapacity(ByteSizeValue nodeCapacity, int nodes } } - private static Integer totalCapacity(Integer nodeCapacity, int nodes) { + private static Float totalCapacity(Float nodeCapacity, int nodes) { if (nodeCapacity != null) { return nodeCapacity * nodes; } else { @@ -101,10 +101,10 @@ public static class FixedReason implements AutoscalingDeciderResult.Reason { private final ByteSizeValue storage; private final ByteSizeValue memory; - private final Integer processors; + private final Float processors; private final int nodes; - public FixedReason(ByteSizeValue storage, ByteSizeValue memory, int nodes, Integer processors) { + public FixedReason(ByteSizeValue storage, ByteSizeValue memory, int nodes, Float processors) { this.storage = storage; this.memory = memory; this.nodes = nodes; @@ -119,7 +119,7 @@ public FixedReason(StreamInput in) throws IOException { this.memory = in.readOptionalWriteable(ByteSizeValue::new); this.nodes = in.readInt(); if (in.getVersion().onOrAfter(Version.V_8_4_0)) { - this.processors = in.readOptionalVInt(); + this.processors = in.readOptionalFloat(); } else { this.processors = null; } @@ -149,7 +149,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(memory); out.writeInt(nodes); if (out.getVersion().onOrAfter(Version.V_8_4_0)) { - out.writeOptionalVInt(processors); + out.writeOptionalFloat(processors); } } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/NodeInfo.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/NodeInfo.java index 99e33b5c16252..553ba8c7b0266 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/NodeInfo.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/NodeInfo.java @@ -12,7 +12,7 @@ * @param memory node total memory * @param processors allocated processors */ -public record NodeInfo(long memory, int processors) { +public record NodeInfo(long memory, float processors) { static Builder builder() { return new Builder(); @@ -20,14 +20,14 @@ static Builder builder() { static class Builder { private Long memory; - private Integer processors; + private Float processors; Builder setMemory(long memory) { this.memory = memory; return this; } - Builder setProcessors(int processors) { + Builder setProcessors(float processors) { this.processors = processors; return this; } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderService.java index 35406102eb2f4..6929f3f4d9198 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderService.java @@ -56,7 +56,7 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider builder.total(MINIMUM_FROZEN_STORAGE, MINIMUM_FROZEN_MEMORY, null); builder.node(MINIMUM_FROZEN_STORAGE, MINIMUM_FROZEN_MEMORY, null); } else { - builder.total(0L, 0L, 0); + builder.total(0L, 0L, 0f); } return new AutoscalingDeciderResult(builder.build(), new FrozenExistenceReason(indicesNeedingFrozen)); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java index af66978f0d00c..24f7e52018df9 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingTestCase.java @@ -49,7 +49,7 @@ protected static AutoscalingDeciderResult randomAutoscalingDeciderResultWithCapa randomNullableByteSizeValue(), randomNullableByteSizeValue(), randomInt(1000), - randomInt(64) + (float) randomInt(64) ) ); } @@ -77,7 +77,7 @@ protected static AutoscalingCapacity randomNullableAutoscalingCapacity() { } protected static AutoscalingCapacity.AutoscalingResources randomAutoscalingResources() { - return new AutoscalingCapacity.AutoscalingResources(randomByteSizeValue(), randomByteSizeValue(), randomInt(128)); + return new AutoscalingCapacity.AutoscalingResources(randomByteSizeValue(), randomByteSizeValue(), (float) randomInt(128)); } private static AutoscalingCapacity.AutoscalingResources randomNullValueAutoscalingResources() { @@ -96,7 +96,7 @@ public static AutoscalingCapacity.AutoscalingResources randomNullValueAutoscalin return new AutoscalingCapacity.AutoscalingResources( addStorage ? randomByteSizeValue() : null, addMemory ? randomByteSizeValue() : null, - addProcessors ? randomInt(128) : null + addProcessors ? (float) randomInt(128) : null ); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java index 38a6eda2da0d4..a394f5f098a60 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java @@ -83,7 +83,7 @@ public void testMultiplePoliciesFixedCapacity() { assertThat(deciderResult.requiredCapacity(), equalTo(requiredCapacity)); ByteSizeValue storage = configuration.getAsBytesSize(FixedAutoscalingDeciderService.STORAGE.getKey(), null); ByteSizeValue memory = configuration.getAsMemory(FixedAutoscalingDeciderService.MEMORY.getKey(), null); - Integer processors = configuration.getAsInt(FixedAutoscalingDeciderService.PROCESSORS.getKey(), null); + Float processors = configuration.getAsFloat(FixedAutoscalingDeciderService.PROCESSORS.getKey(), null); int nodes = FixedAutoscalingDeciderService.NODES.get(configuration); assertThat(deciderResult.reason(), equalTo(new FixedAutoscalingDeciderService.FixedReason(storage, memory, nodes, processors))); assertThat( @@ -163,11 +163,11 @@ private SortedMap randomFixedDeciders() { private AutoscalingCapacity calculateFixedDeciderCapacity(Settings configuration) { ByteSizeValue storage = configuration.getAsBytesSize(FixedAutoscalingDeciderService.STORAGE.getKey(), null); ByteSizeValue memory = configuration.getAsBytesSize(FixedAutoscalingDeciderService.MEMORY.getKey(), null); - Integer processors = configuration.getAsInt(FixedAutoscalingDeciderService.PROCESSORS.getKey(), null); + Float processors = configuration.getAsFloat(FixedAutoscalingDeciderService.PROCESSORS.getKey(), null); int nodes = FixedAutoscalingDeciderService.NODES.get(configuration); ByteSizeValue totalStorage = storage != null ? new ByteSizeValue(storage.getBytes() * nodes) : null; ByteSizeValue totalMemory = memory != null ? new ByteSizeValue(memory.getBytes() * nodes) : null; - Integer totalProcessors = processors != null ? processors * nodes : null; + Float totalProcessors = processors != null ? processors * nodes : null; if (totalStorage == null && totalMemory == null && totalProcessors == null) { return null; diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacityWireSerializationTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacityWireSerializationTests.java index 19c933b8c2fda..2053449f5d408 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacityWireSerializationTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacityWireSerializationTests.java @@ -57,7 +57,7 @@ protected AutoscalingCapacity mutateInstance(AutoscalingCapacity instance) { instance.total().memory(), hasAllMetrics && (instance.node() == null || instance.node().processors() == null) && randomBoolean() ? null - : randomIntBetween(1, 64) + Optional.ofNullable(instance.total().processors()).orElse(0) + : randomIntBetween(1, 64) + Optional.ofNullable(instance.total().processors()).orElse(0f) ); } } else { @@ -90,7 +90,7 @@ protected AutoscalingCapacity mutateInstance(AutoscalingCapacity instance) { && (instance.node().storage() != null || instance.node().memory() != null) && instance.node().processors() != null ? null - : randomIntBetween(1, 64) + Optional.ofNullable(instance.node().processors()).orElse(0) + : randomIntBetween(1, 64) + Optional.ofNullable(instance.node().processors()).orElse(0f) ); } else { ByteSizeValue newStorage = instance.total().storage() != null @@ -104,7 +104,7 @@ protected AutoscalingCapacity mutateInstance(AutoscalingCapacity instance) { newMem, randomBoolean() && (newMem != null || newStorage != null) && instance.node().processors() != null ? null : instance.total().processors() != null && randomBoolean() - ? randomIntBetween(1, 64) + Optional.ofNullable(instance.node().processors()).orElse(0) + ? randomIntBetween(1, 64) + Optional.ofNullable(instance.node().processors()).orElse(0f) : null ); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java index 12db5657dc127..dc6c08f530a0b 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsTests.java @@ -161,13 +161,13 @@ private AutoscalingCapacity randomCapacity(boolean node, boolean storage, boolea builder.total( storage ? randomLongBetween(lower, upper) : null, memory ? randomLongBetween(lower, upper) : null, - processor ? randomIntBetween(lower, upper) : null + processor ? (float) randomIntBetween(lower, upper) : null ); if (node) { builder.node( storage ? randomLongBetween(lower, upper) : null, memory ? randomLongBetween(lower, upper) : null, - processor ? randomIntBetween(lower, upper) : null + processor ? (float) randomIntBetween(lower, upper) : null ); } return builder.build(); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderServiceTests.java index 9f5ee172467d8..c2bf99eb4ad9a 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderServiceTests.java @@ -25,7 +25,7 @@ public void testScale() { ByteSizeValue storage = randomNullableByteSizeValue(); ByteSizeValue memory = randomNullableByteSizeValue(); - Integer processors = (memory != null || storage != null) && randomBoolean() ? null : randomInt(64); + Float processors = (memory != null || storage != null) && randomBoolean() ? null : (float) randomInt(64); if (storage != null) { configurationBuilder.put(FixedAutoscalingDeciderService.STORAGE.getKey(), storage); } @@ -61,7 +61,7 @@ private ByteSizeValue multiply(ByteSizeValue bytes, int nodes) { return bytes == null ? null : new ByteSizeValue(bytes.getBytes() * nodes); } - private Integer multiply(Integer processors, int nodes) { + private Float multiply(Float processors, int nodes) { return processors == null ? null : processors * nodes; } } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java index 7a1d5022a10b7..8be3e2c481fd2 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java @@ -526,7 +526,8 @@ private void startRandomShards() { List started = RoutingNodesHelper.shardsWithState(allocation.routingNodes(), ShardRoutingState.STARTED); if (started.isEmpty() == false) { ShardRouting toMove = randomFrom(started); - Set candidates = StreamSupport.stream(allocation.routingNodes().spliterator(), false) + Set candidates = allocation.routingNodes() + .stream() .filter(n -> allocation.deciders().canAllocate(toMove, n, allocation) == Decision.YES) .collect(Collectors.toSet()); if (candidates.isEmpty() == false) { @@ -537,10 +538,6 @@ private void startRandomShards() { }); } - private TestAutoscalingDeciderContext createContext(DiscoveryNodeRole role) { - return createContext(state, Set.of(role)); - } - private static TestAutoscalingDeciderContext createContext(ClusterState state, Set roles) { return new TestAutoscalingDeciderContext(state, roles, randomCurrentCapacity()); } @@ -551,8 +548,12 @@ static AutoscalingCapacity randomCurrentCapacity() { boolean includeMemory = randomBoolean(); boolean includeProcessors = randomBoolean(); return AutoscalingCapacity.builder() - .total(randomByteSizeValue(), includeMemory ? randomByteSizeValue() : null, includeProcessors ? randomInt(64) : null) - .node(randomByteSizeValue(), includeMemory ? randomByteSizeValue() : null, includeProcessors ? randomInt(64) : null) + .total( + randomByteSizeValue(), + includeMemory ? randomByteSizeValue() : null, + includeProcessors ? (float) randomInt(64) : null + ) + .node(randomByteSizeValue(), includeMemory ? randomByteSizeValue() : null, includeProcessors ? (float) randomInt(64) : null) .build(); } else { return null; @@ -686,12 +687,8 @@ static DiscoveryNode newDataNode(DiscoveryNodeRole role, String nodeName) { } private static String randomNodeId(RoutingNodes routingNodes, DiscoveryNodeRole role) { - return randomFrom( - StreamSupport.stream(routingNodes.spliterator(), false) - .map(RoutingNode::node) - .filter(n -> n.getRoles().contains(role)) - .collect(Collectors.toSet()) - ).getId(); + return randomFrom(routingNodes.stream().map(RoutingNode::node).filter(n -> n.getRoles().contains(role)).collect(Collectors.toSet())) + .getId(); } private static Set shardIds(Iterable candidateShards) { From 3bdf22f3a8e28fd5b767ea1c075c1868f3cabf36 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 23 Jun 2022 08:14:00 -0400 Subject: [PATCH 06/11] fixing line --- .../xpack/autoscaling/capacity/AutoscalingCapacity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java index 09b8988249239..4229df4e2155d 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java @@ -33,7 +33,7 @@ public static class AutoscalingResources implements ToXContent, Writeable { private final ByteSizeValue memory; private final Float processors; - public static final AutoscalingResources ZERO = new AutoscalingResources(new ByteSizeValue(0), new ByteSizeValue(0), 0.0f); + public static final AutoscalingResources ZERO = new AutoscalingResources(ByteSizeValue.ZERO, ByteSizeValue.ZERO, 0.0f); public AutoscalingResources(ByteSizeValue storage, ByteSizeValue memory, Float processors) { assert storage != null || memory != null || processors != null; From 83c9c931b73836a6182a9fcb69655eb79ad58e4e Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 23 Jun 2022 08:16:03 -0400 Subject: [PATCH 07/11] fixing parameter name --- .../action/TransportGetAutoscalingCapacityAction.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java index 9093a71a5af1c..e58109e5dc1ff 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/action/TransportGetAutoscalingCapacityAction.java @@ -40,7 +40,7 @@ public class TransportGetAutoscalingCapacityAction extends TransportMasterNodeAc private final AutoscalingCalculateCapacityService capacityService; private final ClusterInfoService clusterInfoService; private final SnapshotsInfoService snapshotsInfoService; - private final AutoscalingNodeInfoService memoryAndProcessorInfoService; + private final AutoscalingNodeInfoService nodeInfoService; private final AutoscalingLicenseChecker autoscalingLicenseChecker; private final CapacityResponseCache responseCache = new CapacityResponseCache<>( run -> threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(run), @@ -57,7 +57,7 @@ public TransportGetAutoscalingCapacityAction( final AutoscalingCalculateCapacityService.Holder capacityServiceHolder, final ClusterInfoService clusterInfoService, final SnapshotsInfoService snapshotsInfoService, - final AutoscalingNodeInfoService memoryAndProcessorInfoService, + final AutoscalingNodeInfoService nodeInfoService, final AllocationDeciders allocationDeciders, final AutoscalingLicenseChecker autoscalingLicenseChecker ) { @@ -73,7 +73,7 @@ public TransportGetAutoscalingCapacityAction( ThreadPool.Names.SAME ); this.snapshotsInfoService = snapshotsInfoService; - this.memoryAndProcessorInfoService = memoryAndProcessorInfoService; + this.nodeInfoService = nodeInfoService; this.capacityService = capacityServiceHolder.get(allocationDeciders); this.clusterInfoService = clusterInfoService; this.autoscalingLicenseChecker = Objects.requireNonNull(autoscalingLicenseChecker); @@ -104,7 +104,7 @@ private GetAutoscalingCapacityAction.Response computeCapacity(Runnable ensureNot clusterService.state(), clusterInfoService.getClusterInfo(), snapshotsInfoService.snapshotShardSizes(), - memoryAndProcessorInfoService.snapshot(), + nodeInfoService.snapshot(), ensureNotCancelled ) ); From 65d65a04ea05d65d121d03e03255179da42fbfd7 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 27 Jun 2022 14:24:56 -0400 Subject: [PATCH 08/11] addressing pr comments --- .../autoscaling/get_autoscaling_capacity.yml | 6 +-- .../AutoscalingCalculateCapacityService.java | 28 ++++++------ .../FixedAutoscalingDeciderService.java | 7 ++- .../nodeinfo/AutoscalingNodeInfo.java | 41 ++++++++++++----- .../nodeinfo/AutoscalingNodeInfoService.java | 17 +++---- .../nodeinfo/AutoscalingNodesInfo.java | 23 ++++++++++ .../capacity/nodeinfo/NodeInfo.java | 44 ------------------- ...oscalingCalculateCapacityServiceTests.java | 16 +++---- ... => AutoscalingNodesInfoServiceTests.java} | 4 +- x-pack/plugin/build.gradle | 5 ++- 10 files changed, 97 insertions(+), 94 deletions(-) create mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfo.java delete mode 100644 x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/NodeInfo.java rename x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/{AutoscalingNodeInfoServiceTests.java => AutoscalingNodesInfoServiceTests.java} (99%) diff --git a/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/get_autoscaling_capacity.yml b/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/get_autoscaling_capacity.yml index fdc8af1d880b4..e23909287e6ab 100644 --- a/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/get_autoscaling_capacity.yml +++ b/x-pack/plugin/autoscaling/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/autoscaling/get_autoscaling_capacity.yml @@ -36,7 +36,7 @@ - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.total.memory: 73310 } - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.node.storage: 1337 } - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.node.memory: 7331 } - - match: { policies.my_autoscaling_policy.deciders.fixed.reason_summary: "fixed storage [1.3kb] memory [7.1kb] nodes [10]" } + - match: { policies.my_autoscaling_policy.deciders.fixed.reason_summary: "fixed storage [1.3kb] memory [7.1kb] processors [null] nodes [10]" } - length: { policies.my_autoscaling_policy.current_nodes: 0 } @@ -76,7 +76,7 @@ autoscaling.delete_autoscaling_policy: name: my_autoscaling_policy --- -"Test get fixed autoscaling capacity with processors`": +"Test get fixed autoscaling capacity with processors": - do: autoscaling.put_autoscaling_policy: name: my_autoscaling_policy @@ -113,7 +113,7 @@ - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.node.storage: 1337 } - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.node.memory: 7331 } - match: { policies.my_autoscaling_policy.deciders.fixed.required_capacity.node.processors: 2 } - - match: { policies.my_autoscaling_policy.deciders.fixed.reason_summary: "fixed storage [1.3kb] memory [7.1kb] nodes [10] processors [2.0]" } + - match: { policies.my_autoscaling_policy.deciders.fixed.reason_summary: "fixed storage [1.3kb] memory [7.1kb] processors [2.0] nodes [10]" } - length: { policies.my_autoscaling_policy.current_nodes: 0 } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java index c57e085b3fb3f..600d0a78e1f29 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; import org.elasticsearch.xpack.autoscaling.action.PolicyValidator; import org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.AutoscalingNodeInfo; -import org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.NodeInfo; +import org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.AutoscalingNodesInfo; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicy; import java.util.Collections; @@ -109,7 +109,7 @@ public SortedMap calculate( ClusterState state, ClusterInfo clusterInfo, SnapshotShardSizeInfo shardSizeInfo, - AutoscalingNodeInfo autoscalingNodeInfo, + AutoscalingNodesInfo autoscalingNodesInfo, Runnable ensureNotCancelled ) { AutoscalingMetadata autoscalingMetadata = state.metadata().custom(AutoscalingMetadata.NAME); @@ -126,7 +126,7 @@ public SortedMap calculate( state, clusterInfo, shardSizeInfo, - autoscalingNodeInfo, + autoscalingNodesInfo, ensureNotCancelled ) ) @@ -143,7 +143,7 @@ private AutoscalingDeciderResults calculateForPolicy( ClusterState state, ClusterInfo clusterInfo, SnapshotShardSizeInfo shardSizeInfo, - AutoscalingNodeInfo autoscalingNodeInfo, + AutoscalingNodesInfo autoscalingNodesInfo, Runnable ensureNotCancelled ) { if (hasUnknownRoles(policy)) { @@ -159,7 +159,7 @@ private AutoscalingDeciderResults calculateForPolicy( state, clusterInfo, shardSizeInfo, - autoscalingNodeInfo, + autoscalingNodesInfo, ensureNotCancelled ); SortedMap results = deciders.entrySet() @@ -200,10 +200,10 @@ DefaultAutoscalingDeciderContext createContext( ClusterState state, ClusterInfo clusterInfo, SnapshotShardSizeInfo shardSizeInfo, - AutoscalingNodeInfo autoscalingNodeInfo, + AutoscalingNodesInfo autoscalingNodesInfo, Runnable ensureNotCancelled ) { - return new DefaultAutoscalingDeciderContext(roles, state, clusterInfo, shardSizeInfo, autoscalingNodeInfo, ensureNotCancelled); + return new DefaultAutoscalingDeciderContext(roles, state, clusterInfo, shardSizeInfo, autoscalingNodesInfo, ensureNotCancelled); } /** @@ -232,7 +232,7 @@ static class DefaultAutoscalingDeciderContext implements AutoscalingDeciderConte private final ClusterState state; private final ClusterInfo clusterInfo; private final SnapshotShardSizeInfo snapshotShardSizeInfo; - private final AutoscalingNodeInfo autoscalingNodeInfo; + private final AutoscalingNodesInfo autoscalingNodesInfo; private final SortedSet currentNodes; private final AutoscalingCapacity currentCapacity; private final boolean currentCapacityAccurate; @@ -243,7 +243,7 @@ static class DefaultAutoscalingDeciderContext implements AutoscalingDeciderConte ClusterState state, ClusterInfo clusterInfo, SnapshotShardSizeInfo snapshotShardSizeInfo, - AutoscalingNodeInfo autoscalingNodeInfo, + AutoscalingNodesInfo autoscalingNodesInfo, Runnable ensureNotCancelled ) { this.roles = roles.stream().map(DiscoveryNodeRole::getRoleFromRoleName).collect(Sets.toUnmodifiableSortedSet()); @@ -252,7 +252,7 @@ static class DefaultAutoscalingDeciderContext implements AutoscalingDeciderConte this.state = state; this.clusterInfo = clusterInfo; this.snapshotShardSizeInfo = snapshotShardSizeInfo; - this.autoscalingNodeInfo = autoscalingNodeInfo; + this.autoscalingNodesInfo = autoscalingNodesInfo; this.currentNodes = state.nodes() .stream() .filter(this::rolesFilter) @@ -304,7 +304,7 @@ private boolean nodeHasAccurateCapacity(DiscoveryNode node) { } } - return autoscalingNodeInfo.get(node).isPresent(); + return autoscalingNodesInfo.get(node).isPresent(); } private AutoscalingCapacity calculateCurrentCapacity() { @@ -328,11 +328,11 @@ private AutoscalingCapacity.AutoscalingResources resourcesFor(DiscoveryNode node ) : 0L; - Optional memoryAndProcessors = autoscalingNodeInfo.get(node); + Optional memoryAndProcessors = autoscalingNodesInfo.get(node); return new AutoscalingCapacity.AutoscalingResources( storage == -1 ? ByteSizeValue.ZERO : new ByteSizeValue(storage), - memoryAndProcessors.map(NodeInfo::memory).map(ByteSizeValue::new).orElse(ByteSizeValue.ZERO), - memoryAndProcessors.map(NodeInfo::processors).orElse(0f) + memoryAndProcessors.map(AutoscalingNodeInfo::memory).map(ByteSizeValue::new).orElse(ByteSizeValue.ZERO), + memoryAndProcessors.map(AutoscalingNodeInfo::processors).orElse(0f) ); } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java index 1ce8b25a04db1..c8feb6575e9f0 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Locale; import java.util.Objects; -import java.util.Optional; public class FixedAutoscalingDeciderService implements AutoscalingDeciderService { @@ -130,11 +129,11 @@ public String summary() { return String.format( Locale.ROOT, // We allow processors to be optional in the output for API backwards compatibility - "fixed storage [%s] memory [%s] nodes [%d]%s", + "fixed storage [%s] memory [%s] processors [%s] nodes [%d]", storage, memory, - nodes, - Optional.ofNullable(processors).map(i -> " processors [" + i + "]").orElse("") + processors, + nodes ); } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfo.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfo.java index 2d6e62a38da03..3f84ec16cbec6 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfo.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfo.java @@ -7,17 +7,38 @@ package org.elasticsearch.xpack.autoscaling.capacity.nodeinfo; -import org.elasticsearch.cluster.node.DiscoveryNode; +/** + * Record for containing memory and processors for given node + * @param memory node total memory + * @param processors allocated processors + */ +public record AutoscalingNodeInfo(long memory, float processors) { + + static Builder builder() { + return new Builder(); + } + + static class Builder { + private Long memory; + private Float processors; + + Builder setMemory(long memory) { + this.memory = memory; + return this; + } -import java.util.Optional; + Builder setProcessors(float processors) { + this.processors = processors; + return this; + } -public interface AutoscalingNodeInfo { - AutoscalingNodeInfo EMPTY = n -> Optional.empty(); + boolean canBuild() { + return memory != null && processors != null; + } - /** - * Get the memory and processor use for the indicated node. Returns null if not available (new, fetching or failed). - * @param node the node to get info for - * @return memory and processor info for node if possible - */ - Optional get(DiscoveryNode node); + AutoscalingNodeInfo build() { + assert memory != null && processors != null : "unexpected null values when building node memory and processors information"; + return new AutoscalingNodeInfo(memory, processors); + } + } } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java index 00f913a3a587f..d80fa350ccee6 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoService.java @@ -50,11 +50,11 @@ public class AutoscalingNodeInfoService { Setting.Property.Dynamic, Setting.Property.NodeScope ); - private static final NodeInfo FETCHING_SENTINEL = new NodeInfo(Long.MIN_VALUE, Integer.MIN_VALUE); + private static final AutoscalingNodeInfo FETCHING_SENTINEL = new AutoscalingNodeInfo(Long.MIN_VALUE, Integer.MIN_VALUE); private static final Logger logger = LogManager.getLogger(AutoscalingNodeInfoService.class); - private volatile Map nodeToMemory = Map.of(); + private volatile Map nodeToMemory = Map.of(); private volatile TimeValue fetchTimeout; private final Client client; @@ -139,14 +139,15 @@ private void sendToMissingNodes(Function nodeLookup, Set< .toArray(String[]::new) ).clear().addMetric(NodesInfoRequest.Metric.OS.metricName()).timeout(fetchTimeout), ActionListener.wrap(nodesInfoResponse -> { - final Map builderBuilder = Maps.newHashMapWithExpectedSize( + final Map builderBuilder = Maps.newHashMapWithExpectedSize( nodesStatsResponse.getNodes().size() ); nodesStatsResponse.getNodes() .forEach( nodeStats -> builderBuilder.put( nodeStats.getNode().getEphemeralId(), - NodeInfo.builder().setMemory(nodeStats.getOs().getMem().getAdjustedTotal().getBytes()) + AutoscalingNodeInfo.builder() + .setMemory(nodeStats.getOs().getMem().getAdjustedTotal().getBytes()) ) ); nodesInfoResponse.getNodes().forEach(nodeInfo -> { @@ -158,7 +159,7 @@ private void sendToMissingNodes(Function nodeLookup, Set< ); }); synchronized (mutex) { - Map builder = new HashMap<>(nodeToMemory); + Map builder = new HashMap<>(nodeToMemory); // Remove all from the builder that failed getting info and stats Stream.concat(nodesStatsResponse.failures().stream(), nodesInfoResponse.failures().stream()) .map(FailedNodeException::nodeId) @@ -218,10 +219,10 @@ private void retainAliveNodes(Set currentNodes) { } } - public AutoscalingNodeInfo snapshot() { - final Map nodeToMemoryRef = this.nodeToMemory; + public AutoscalingNodesInfo snapshot() { + final Map nodeToMemoryRef = this.nodeToMemory; return node -> { - NodeInfo result = nodeToMemoryRef.get(node.getEphemeralId()); + AutoscalingNodeInfo result = nodeToMemoryRef.get(node.getEphemeralId()); if (result == FETCHING_SENTINEL) { return Optional.empty(); } else { diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfo.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfo.java new file mode 100644 index 0000000000000..6a32e847654c3 --- /dev/null +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfo.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.autoscaling.capacity.nodeinfo; + +import org.elasticsearch.cluster.node.DiscoveryNode; + +import java.util.Optional; + +public interface AutoscalingNodesInfo { + AutoscalingNodesInfo EMPTY = n -> Optional.empty(); + + /** + * Get the memory and processor use for the indicated node. Returns null if not available (new, fetching or failed). + * @param node the node to get info for + * @return memory and processor info for node if possible + */ + Optional get(DiscoveryNode node); +} diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/NodeInfo.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/NodeInfo.java deleted file mode 100644 index 553ba8c7b0266..0000000000000 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/NodeInfo.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.autoscaling.capacity.nodeinfo; - -/** - * Record for containing memory and processors for given node - * @param memory node total memory - * @param processors allocated processors - */ -public record NodeInfo(long memory, float processors) { - - static Builder builder() { - return new Builder(); - } - - static class Builder { - private Long memory; - private Float processors; - - Builder setMemory(long memory) { - this.memory = memory; - return this; - } - - Builder setProcessors(float processors) { - this.processors = processors; - return this; - } - - boolean canBuild() { - return memory != null && processors != null; - } - - NodeInfo build() { - assert memory != null && processors != null : "unexpected null values when building node memory and processors information"; - return new NodeInfo(memory, processors); - } - } -} diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java index a394f5f098a60..e2ffc93f4b01c 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java @@ -23,7 +23,7 @@ import org.elasticsearch.xpack.autoscaling.AutoscalingMetadata; import org.elasticsearch.xpack.autoscaling.AutoscalingTestCase; import org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.AutoscalingNodeInfo; -import org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.NodeInfo; +import org.elasticsearch.xpack.autoscaling.capacity.nodeinfo.AutoscalingNodesInfo; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicy; import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicyMetadata; @@ -66,7 +66,7 @@ public void testMultiplePoliciesFixedCapacity() { state, ClusterInfo.EMPTY, null, - AutoscalingNodeInfo.EMPTY, + AutoscalingNodesInfo.EMPTY, () -> {} ); assertThat(resultsMap.keySet(), equalTo(policyNames)); @@ -137,7 +137,7 @@ public String name() { .build(); assertThat( - service.calculate(state, ClusterInfo.EMPTY, SnapshotShardSizeInfo.EMPTY, AutoscalingNodeInfo.EMPTY, () -> {}) + service.calculate(state, ClusterInfo.EMPTY, SnapshotShardSizeInfo.EMPTY, AutoscalingNodesInfo.EMPTY, () -> {}) .get("test") .results() .keySet(), @@ -192,7 +192,7 @@ public void testContext() { state, info, snapshotShardSizeInfo, - n -> Optional.of(new NodeInfo(randomNonNegativeLong(), randomInt(64))), + n -> Optional.of(new AutoscalingNodeInfo(randomNonNegativeLong(), randomInt(64))), () -> {} ); @@ -217,7 +217,7 @@ public void testContext() { state, info, null, - n -> Optional.of(new NodeInfo(memory, randomInt(64))), + n -> Optional.of(new AutoscalingNodeInfo(memory, randomInt(64))), () -> {} ); @@ -275,7 +275,7 @@ public void testContext() { state, info, null, - n -> Optional.of(new NodeInfo(memory, randomInt(64))), + n -> Optional.of(new AutoscalingNodeInfo(memory, randomInt(64))), () -> {} ); @@ -296,7 +296,7 @@ public void testContext() { state, info, null, - AutoscalingNodeInfo.EMPTY, + AutoscalingNodesInfo.EMPTY, () -> {} ); assertThat(context.nodes(), equalTo(expectedNodes)); @@ -321,7 +321,7 @@ public void testContext() { state, info, null, - n -> Optional.of(new NodeInfo(memory, randomInt(64))), + n -> Optional.of(new AutoscalingNodeInfo(memory, randomInt(64))), () -> {} ); assertThat(context.nodes(), equalTo(expectedNodes)); diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java similarity index 99% rename from x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoServiceTests.java rename to x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java index 9b7d60663566c..e5006ac12b36b 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodeInfoServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java @@ -68,7 +68,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class AutoscalingNodeInfoServiceTests extends AutoscalingTestCase { +public class AutoscalingNodesInfoServiceTests extends AutoscalingTestCase { private NodeStatsClient client; private AutoscalingNodeInfoService service; @@ -400,7 +400,7 @@ public void assertMatchesResponse(Set nodes, NodesStatsResponse r assertThat( service.snapshot().get(n).get(), equalTo( - new NodeInfo( + new AutoscalingNodeInfo( response.getNodesMap().get(n.getId()).getOs().getMem().getAdjustedTotal().getBytes(), infoResponse.getNodesMap().get(n.getId()).getInfo(OsInfo.class).getAllocatedProcessors() ) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 534215cc7a765..6cda78a2be979 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -1,7 +1,6 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.internal.info.BuildParams -import org.elasticsearch.gradle.internal.test.RestIntegTestTask import org.elasticsearch.gradle.util.GradleUtils apply plugin: 'elasticsearch.internal-yaml-rest-test' @@ -84,6 +83,10 @@ tasks.named("yamlRestTestV7CompatTest").configure { } tasks.named("yamlRestTestV7CompatTransform").configure { task -> + task.skipTest( + "autoscaling/get_autoscaling_capacity/Test get fixed autoscaling capacity", + "behavior changed #87895" + ) task.skipTest( "vectors/10_dense_vector_basic/Deprecated function signature", "to support it, it would require to almost revert back the #48725 and complicate the code" From 1a9eb77c448b434a4ca8d161f699959668de94a7 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 27 Jun 2022 14:50:32 -0400 Subject: [PATCH 09/11] fixing tests --- .../capacity/AutoscalingCalculateCapacityServiceTests.java | 3 ++- x-pack/plugin/build.gradle | 5 +---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java index e2ffc93f4b01c..20ab08bb96d1b 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java @@ -93,10 +93,11 @@ public void testMultiplePoliciesFixedCapacity() { + storage + "] memory [" + memory + + "] processors [" + + processors + "] nodes [" + nodes + "]" - + (processors == null ? "" : " processors [" + processors + "]") ) ); diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 6cda78a2be979..9bfeb8291cb2a 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -79,14 +79,11 @@ tasks.named("yamlRestTestV7CompatTest").configure { 'unsigned_long/50_script_values/script_score query', 'unsigned_long/50_script_values/Script query', 'data_stream/140_data_stream_aliases/Fix IndexNotFoundException error when handling remove alias action', + "autoscaling/get_autoscaling_capacity/Test get fixed autoscaling capacity", ].join(',') } tasks.named("yamlRestTestV7CompatTransform").configure { task -> - task.skipTest( - "autoscaling/get_autoscaling_capacity/Test get fixed autoscaling capacity", - "behavior changed #87895" - ) task.skipTest( "vectors/10_dense_vector_basic/Deprecated function signature", "to support it, it would require to almost revert back the #48725 and complicate the code" From 6d6d6edc238405d7eab08f36ec3c99c31f165295 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 27 Jun 2022 14:56:02 -0400 Subject: [PATCH 10/11] fixing format --- .../AutoscalingCalculateCapacityServiceTests.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java index 20ab08bb96d1b..0ea41e147cec7 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java @@ -88,17 +88,7 @@ public void testMultiplePoliciesFixedCapacity() { assertThat(deciderResult.reason(), equalTo(new FixedAutoscalingDeciderService.FixedReason(storage, memory, nodes, processors))); assertThat( deciderResult.reason().summary(), - equalTo( - "fixed storage [" - + storage - + "] memory [" - + memory - + "] processors [" - + processors - + "] nodes [" - + nodes - + "]" - ) + equalTo("fixed storage [" + storage + "] memory [" + memory + "] processors [" + processors + "] nodes [" + nodes + "]") ); // there is no nodes in any tier. From a58d21df39e914d4de6d03d74644f8c62fc38ffb Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 27 Jun 2022 15:10:59 -0400 Subject: [PATCH 11/11] muting test --- x-pack/plugin/autoscaling/qa/rest/build.gradle | 6 ++++++ x-pack/plugin/build.gradle | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/autoscaling/qa/rest/build.gradle b/x-pack/plugin/autoscaling/qa/rest/build.gradle index bfdd8ac4401b1..143e271412d4f 100644 --- a/x-pack/plugin/autoscaling/qa/rest/build.gradle +++ b/x-pack/plugin/autoscaling/qa/rest/build.gradle @@ -13,6 +13,12 @@ restResources { } } +tasks.named("yamlRestTestV7CompatTest").configure { + systemProperty 'tests.rest.blacklist', [ + "autoscaling/get_autoscaling_capacity/Test get fixed autoscaling capacity", + ].join(',') +} + testClusters.configureEach { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'true' diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 9bfeb8291cb2a..1c77629af888c 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -79,7 +79,6 @@ tasks.named("yamlRestTestV7CompatTest").configure { 'unsigned_long/50_script_values/script_score query', 'unsigned_long/50_script_values/Script query', 'data_stream/140_data_stream_aliases/Fix IndexNotFoundException error when handling remove alias action', - "autoscaling/get_autoscaling_capacity/Test get fixed autoscaling capacity", ].join(',') }