Skip to content

Commit

Permalink
Add info item for worker metrics.
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 522009172
Change-Id: I59ba1290c6a36a04aafb81f698e49616f057fae0
  • Loading branch information
meisterT authored and copybara-github committed Apr 5, 2023
1 parent 2aa2e0e commit a165187
Show file tree
Hide file tree
Showing 5 changed files with 104 additions and 12 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@
// limitations under the License.
package com.google.devtools.build.lib.metrics;

import static com.google.common.collect.ImmutableList.toImmutableList;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.eventbus.AllowConcurrentEvents;
import com.google.common.eventbus.Subscribe;
Expand All @@ -40,7 +38,6 @@
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.PackageMetrics;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.TargetMetrics;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.TimingMetrics;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.WorkerMetrics;
import com.google.devtools.build.lib.buildtool.BuildPrecompleteEvent;
import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent;
import com.google.devtools.build.lib.clock.BlazeClock;
Expand All @@ -54,7 +51,6 @@
import com.google.devtools.build.lib.runtime.SpawnStats;
import com.google.devtools.build.lib.skyframe.ExecutionFinishedEvent;
import com.google.devtools.build.lib.skyframe.TopLevelStatusEvents.TopLevelTargetPendingExecutionEvent;
import com.google.devtools.build.lib.worker.WorkerMetric;
import com.google.devtools.build.lib.worker.WorkerMetricsCollector;
import com.google.devtools.build.skyframe.SkyframeGraphStatsEvent;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
Expand Down Expand Up @@ -208,12 +204,6 @@ private void postBuildMetricsEvent() {
env.getEventBus().post(new BuildMetricsEvent(createBuildMetrics()));
}

private ImmutableList<WorkerMetrics> createWorkerMetrics() {
return WorkerMetricsCollector.instance().collectMetrics().stream()
.map(WorkerMetric::toProto)
.collect(toImmutableList());
}

private BuildMetrics createBuildMetrics() {
BuildMetrics.Builder buildMetrics =
BuildMetrics.newBuilder()
Expand All @@ -225,7 +215,7 @@ private BuildMetrics createBuildMetrics() {
.setCumulativeMetrics(createCumulativeMetrics())
.setArtifactMetrics(artifactMetrics.build())
.setBuildGraphMetrics(buildGraphMetrics.build())
.addAllWorkerMetrics(createWorkerMetrics());
.addAllWorkerMetrics(WorkerMetricsCollector.instance().createWorkerMetricsProto());

NetworkMetrics networkMetrics = NetworkMetricsCollector.instance().collectMetrics();
if (networkMetrics != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import com.google.devtools.build.lib.runtime.commands.info.StarlarkSemanticsInfoItem;
import com.google.devtools.build.lib.runtime.commands.info.UsedHeapSizeAfterGcInfoItem;
import com.google.devtools.build.lib.runtime.commands.info.UsedHeapSizeInfoItem;
import com.google.devtools.build.lib.runtime.commands.info.WorkerMetricsInfoItem;
import com.google.devtools.build.lib.runtime.commands.info.WorkspaceInfoItem;
import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
Expand Down Expand Up @@ -291,7 +292,8 @@ private static Map<String, InfoItem> getHardwiredInfoItemMap(
new DefaultsPackageInfoItem(),
new BuildLanguageInfoItem(),
new DefaultPackagePathInfoItem(commandOptions),
new StarlarkSemanticsInfoItem(commandOptions));
new StarlarkSemanticsInfoItem(commandOptions),
new WorkerMetricsInfoItem());
ImmutableMap.Builder<String, InfoItem> result = new ImmutableMap.Builder<>();
for (InfoItem item : hardwiredInfoItems) {
result.put(item.getName(), item);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info",
"//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration",
"//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/packages",
"//src/main/java/com/google/devtools/build/lib/packages/semantics",
Expand All @@ -27,6 +28,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/util:debug-logger-configurator",
"//src/main/java/com/google/devtools/build/lib/util:string",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/worker:worker_metric",
"//src/main/java/com/google/devtools/common/options",
"//src/main/java/net/starlark/java/eval",
"//src/main/protobuf:build_java_proto",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// Copyright 2023 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.runtime.commands.info;

import static java.util.stream.Collectors.joining;

import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.WorkerMetrics;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.WorkerMetrics.WorkerStats;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import com.google.devtools.build.lib.runtime.InfoItem;
import com.google.devtools.build.lib.worker.WorkerMetricsCollector;
import java.util.List;

/** Info item for persistent worker metrics. */
public final class WorkerMetricsInfoItem extends InfoItem {
public WorkerMetricsInfoItem() {
super("worker_metrics", "persistent worker metrics", true);
}

@Override
public byte[] get(
Supplier<BuildConfigurationValue> configurationSupplier, CommandEnvironment env) {

ImmutableList<WorkerMetrics> proto =
WorkerMetricsCollector.instance().createWorkerMetricsProto();
if (proto.isEmpty()) {
return print("No persistent workers active.");
} else {
StringBuilder stringBuilder = new StringBuilder();
for (WorkerMetrics workerMetrics : proto) {
stringBuilder
.append("- ")
.append("pid ")
.append(workerMetrics.getProcessId())
.append(", ")
.append(workerMetrics.getMnemonic())
.append(", ");
List<WorkerStats> workerStats = workerMetrics.getWorkerStatsList();
if (!workerStats.isEmpty()) {
WorkerStats lastWorkerStats = Iterables.getLast(workerStats);
long currentTimeMillis = env.getClock().currentTimeMillis();
long diffSeconds =
(currentTimeMillis - lastWorkerStats.getLastActionStartTimeInMs()) / 1000;
long minutesAgo = diffSeconds / 60;
long remainingSeconds = diffSeconds - 60 * minutesAgo;
stringBuilder
.append(lastWorkerStats.getWorkerMemoryInKb() / 1024)
.append("MB, ")
.append("last action ");
if (minutesAgo > 0) {
stringBuilder.append(minutesAgo).append("m ");
}
stringBuilder.append(remainingSeconds).append("s ago, ");
}
if (workerMetrics.getIsSandbox()) {
stringBuilder.append("sandboxed, ");
}
if (workerMetrics.getIsMultiplex()) {
stringBuilder.append("multiplexed, ");
}
String workerIds =
workerMetrics.getWorkerIdsList().stream()
.sorted()
.map(e -> e.toString())
.collect(joining(", "));
if (workerMetrics.getWorkerIdsList().size() == 1) {
stringBuilder.append("id ").append(workerIds).append("\n");
} else {
stringBuilder.append("ids [").append(workerIds).append("]\n");
}
}
return print(stringBuilder.toString());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,15 @@

package com.google.devtools.build.lib.worker;

import static com.google.common.collect.ImmutableList.toImmutableList;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.VerifyException;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildMetrics.WorkerMetrics;
import com.google.devtools.build.lib.clock.Clock;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.util.PsInfoCollector;
Expand Down Expand Up @@ -162,6 +165,10 @@ public ImmutableList<WorkerMetric> collectMetrics() {
return updateMetricsCache(workerMetrics.build(), collectionTime).metrics;
}

public ImmutableList<WorkerMetrics> createWorkerMetricsProto() {
return collectMetrics().stream().map(WorkerMetric::toProto).collect(toImmutableList());
}

public void clear() {
processIdToWorkerProperties.clear();
workerLastCallTime.clear();
Expand Down

0 comments on commit a165187

Please sign in to comment.