diff --git a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java index 8d2e931214fff5..3afd5480375db6 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java +++ b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java @@ -41,9 +41,7 @@ */ class ByteStreamBuildEventArtifactUploader implements BuildEventArtifactUploader { - private final ListeningExecutorService uploadExecutor = - MoreExecutors.listeningDecorator( - Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())); + private final ListeningExecutorService uploadExecutor; private final Context ctx; private final ByteStreamUploader uploader; private final String remoteServerInstanceName; @@ -51,8 +49,11 @@ class ByteStreamBuildEventArtifactUploader implements BuildEventArtifactUploader private final AtomicBoolean shutdown = new AtomicBoolean(); ByteStreamBuildEventArtifactUploader( - ByteStreamUploader uploader, String remoteServerName, Context ctx, - @Nullable String remoteInstanceName) { + ByteStreamUploader uploader, + String remoteServerName, + Context ctx, + @Nullable String remoteInstanceName, + int maxUploadThreads) { this.uploader = Preconditions.checkNotNull(uploader); String remoteServerInstanceName = Preconditions.checkNotNull(remoteServerName); if (!Strings.isNullOrEmpty(remoteInstanceName)) { @@ -60,6 +61,10 @@ class ByteStreamBuildEventArtifactUploader implements BuildEventArtifactUploader } this.ctx = ctx; this.remoteServerInstanceName = remoteServerInstanceName; + // Limit the maximum threads number to 1000 (chosen arbitrarily) + this.uploadExecutor = + MoreExecutors.listeningDecorator( + Executors.newFixedThreadPool(Math.min(maxUploadThreads, 1000))); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java index 8c57f40604b604..53157d50ef27a6 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java +++ b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java @@ -41,7 +41,11 @@ class ByteStreamBuildEventArtifactUploaderFactory implements @Override public BuildEventArtifactUploader create(CommandEnvironment env) { - return new ByteStreamBuildEventArtifactUploader(uploader.retain(), remoteServerName, ctx, - remoteInstanceName); + return new ByteStreamBuildEventArtifactUploader( + uploader.retain(), + remoteServerName, + ctx, + remoteInstanceName, + env.getOptions().getOptions(RemoteOptions.class).buildEventUploadMaxThreads); } } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java index 180e8a94095c6c..9f59acd86349d8 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java @@ -252,6 +252,14 @@ public final class RemoteOptions extends OptionsBase { + "symlinks and represent them as files. See #6631 for details.") public boolean incompatibleRemoteSymlinks; + @Option( + name = "build_event_upload_max_threads", + defaultValue = "100", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "The number of threads used to do build event uploads. Capped at 1000.") + public int buildEventUploadMaxThreads; + @Deprecated @Option( name = "remote_allow_symlink_upload", diff --git a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java index 69c107bfbc33b3..08e1c9593bab87 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java @@ -145,7 +145,7 @@ public void uploadsShouldWork() throws Exception { new ByteStreamUploader("instance", refCntChannel, null, 3, retrier); ByteStreamBuildEventArtifactUploader artifactUploader = new ByteStreamBuildEventArtifactUploader( - uploader, "localhost", withEmptyMetadata, "instance"); + uploader, "localhost", withEmptyMetadata, "instance", /* maxUploadThreads= */ 100); PathConverter pathConverter = artifactUploader.upload(filesToUpload).get(); for (Path file : filesToUpload.keySet()) { @@ -171,7 +171,7 @@ public void testUploadDirectoryDoesNotCrash() throws Exception { ByteStreamUploader uploader = mock(ByteStreamUploader.class); ByteStreamBuildEventArtifactUploader artifactUploader = new ByteStreamBuildEventArtifactUploader( - uploader, "localhost", withEmptyMetadata, "instance"); + uploader, "localhost", withEmptyMetadata, "instance", /* maxUploadThreads= */ 100); PathConverter pathConverter = artifactUploader.upload(filesToUpload).get(); assertThat(pathConverter.apply(dir)).isNull(); artifactUploader.shutdown(); @@ -234,7 +234,7 @@ public void onCompleted() { new ByteStreamUploader("instance", refCntChannel, null, 3, retrier); ByteStreamBuildEventArtifactUploader artifactUploader = new ByteStreamBuildEventArtifactUploader( - uploader, "localhost", withEmptyMetadata, "instance"); + uploader, "localhost", withEmptyMetadata, "instance", /* maxUploadThreads= */ 100); try { artifactUploader.upload(filesToUpload).get();