Add --build_event_upload_max_threads option

This is a follow up for 7fc967c, related issue #6806

RELNOTES: None
PiperOrigin-RevId: 223944430

src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploader.java

@@ -41,25 +41,30 @@
  */
 class ByteStreamBuildEventArtifactUploader implements BuildEventArtifactUploader {
 
-  private final ListeningExecutorService uploadExecutor =
-      MoreExecutors.listeningDecorator(
-          Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()));
+  private final ListeningExecutorService uploadExecutor;
   private final Context ctx;
   private final ByteStreamUploader uploader;
   private final String remoteServerInstanceName;
 
   private final AtomicBoolean shutdown = new AtomicBoolean();
 
   ByteStreamBuildEventArtifactUploader(
-      ByteStreamUploader uploader, String remoteServerName, Context ctx,
-      @Nullable String remoteInstanceName) {
+      ByteStreamUploader uploader,
+      String remoteServerName,
+      Context ctx,
+      @Nullable String remoteInstanceName,
+      int maxUploadThreads) {
     this.uploader = Preconditions.checkNotNull(uploader);
     String remoteServerInstanceName = Preconditions.checkNotNull(remoteServerName);
     if (!Strings.isNullOrEmpty(remoteInstanceName)) {
       remoteServerInstanceName += "/" + remoteInstanceName;
     }
     this.ctx = ctx;
     this.remoteServerInstanceName = remoteServerInstanceName;
+    // Limit the maximum threads number to 1000 (chosen arbitrarily)
+    this.uploadExecutor =
+        MoreExecutors.listeningDecorator(
+            Executors.newFixedThreadPool(Math.min(maxUploadThreads, 1000)));
   }
 
   @Override

src/main/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderFactory.java

@@ -41,7 +41,11 @@ class ByteStreamBuildEventArtifactUploaderFactory implements
 
   @Override
   public BuildEventArtifactUploader create(CommandEnvironment env) {
-    return new ByteStreamBuildEventArtifactUploader(uploader.retain(), remoteServerName, ctx,
-        remoteInstanceName);
+    return new ByteStreamBuildEventArtifactUploader(
+        uploader.retain(),
+        remoteServerName,
+        ctx,
+        remoteInstanceName,
+        env.getOptions().getOptions(RemoteOptions.class).buildEventUploadMaxThreads);
   }
 }

src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java

@@ -252,6 +252,14 @@ public final class RemoteOptions extends OptionsBase {
               + "symlinks and represent them as files. See #6631 for details.")
   public boolean incompatibleRemoteSymlinks;
 
+  @Option(
+      name = "build_event_upload_max_threads",
+      defaultValue = "100",
+      documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      help = "The number of threads used to do build event uploads. Capped at 1000.")
+  public int buildEventUploadMaxThreads;
+
   @Deprecated
   @Option(
       name = "remote_allow_symlink_upload",

src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java

@@ -145,7 +145,7 @@ public void uploadsShouldWork() throws Exception {
         new ByteStreamUploader("instance", refCntChannel, null, 3, retrier);
     ByteStreamBuildEventArtifactUploader artifactUploader =
         new ByteStreamBuildEventArtifactUploader(
-            uploader, "localhost", withEmptyMetadata, "instance");
+            uploader, "localhost", withEmptyMetadata, "instance", /* maxUploadThreads= */ 100);
 
     PathConverter pathConverter = artifactUploader.upload(filesToUpload).get();
     for (Path file : filesToUpload.keySet()) {
@@ -171,7 +171,7 @@ public void testUploadDirectoryDoesNotCrash() throws Exception {
     ByteStreamUploader uploader = mock(ByteStreamUploader.class);
     ByteStreamBuildEventArtifactUploader artifactUploader =
         new ByteStreamBuildEventArtifactUploader(
-            uploader, "localhost", withEmptyMetadata, "instance");
+            uploader, "localhost", withEmptyMetadata, "instance", /* maxUploadThreads= */ 100);
     PathConverter pathConverter = artifactUploader.upload(filesToUpload).get();
     assertThat(pathConverter.apply(dir)).isNull();
     artifactUploader.shutdown();
@@ -234,7 +234,7 @@ public void onCompleted() {
         new ByteStreamUploader("instance", refCntChannel, null, 3, retrier);
     ByteStreamBuildEventArtifactUploader artifactUploader =
         new ByteStreamBuildEventArtifactUploader(
-            uploader, "localhost", withEmptyMetadata, "instance");
+            uploader, "localhost", withEmptyMetadata, "instance", /* maxUploadThreads= */ 100);
 
     try {
       artifactUploader.upload(filesToUpload).get();