remote: Add support for Google Cloud Storage.

Add support for Google Cloud Storage (GCS) as a HTTP caching backend.
This commit mainly adds the infrastructure necessary to authenticate
to GCS.

Using GCS as a caching backend works as follows:
  1) Create a new GCS bucket.
  2) Create a service account that can read and write to the GCS bucket
     and generate a JSON credentials file for it.
  3) Invoke Bazel as follows:
    bazel build
      --remote_rest_cache=https://storage.googleapis.com/<bucket>
      --auth_enabled
      --auth_scope=https://www.googleapis.com/auth/devstorage.read_write
      --auth_credentials=</path/to/creds.json>

I'll add simplification's and docs in a subsequent commit.

Change-Id: Ie827d7946a2193b97ea7d9aa72eb15f09de2164d
PiperOrigin-RevId: 179406380

src/main/java/com/google/devtools/build/lib/authandtls/GrpcUtils.java → src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java

@@ -14,6 +14,7 @@
 
 package com.google.devtools.build.lib.authandtls;
 
+import com.google.auth.Credentials;
 import com.google.auth.oauth2.GoogleCredentials;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
@@ -33,15 +34,13 @@
 import java.io.InputStream;
 import javax.annotation.Nullable;
 
-/**
- * Utility methods for using {@link AuthAndTLSOptions} with gRPC.
- */
-public final class GrpcUtils {
+/** Utility methods for using {@link AuthAndTLSOptions} with Google Cloud. */
+public final class GoogleAuthUtils {
 
   /**
    * Create a new gRPC {@link ManagedChannel}.
    *
-   * @throws IOException  in case the channel can't be constructed.
+   * @throws IOException in case the channel can't be constructed.
    */
   public static ManagedChannel newChannel(String target, AuthAndTLSOptions options)
       throws IOException {
@@ -54,8 +53,7 @@ public static ManagedChannel newChannel(String target, AuthAndTLSOptions options
     try {
       NettyChannelBuilder builder =
           NettyChannelBuilder.forTarget(target)
-              .negotiationType(
-                  options.tlsEnabled ? NegotiationType.TLS : NegotiationType.PLAINTEXT)
+              .negotiationType(options.tlsEnabled ? NegotiationType.TLS : NegotiationType.PLAINTEXT)
               .loadBalancerFactory(RoundRobinLoadBalancerFactory.getInstance());
       if (sslContext != null) {
         builder.sslContext(sslContext);
@@ -77,8 +75,7 @@ private static SslContext createSSlContext(@Nullable String rootCert) throws IOE
       try {
         return GrpcSslContexts.forClient().build();
       } catch (Exception e) {
-        String message = "Failed to init TLS infrastructure: "
-            + e.getMessage();
+        String message = "Failed to init TLS infrastructure: " + e.getMessage();
         throw new IOException(message, e);
       }
     } else {
@@ -95,30 +92,54 @@ private static SslContext createSSlContext(@Nullable String rootCert) throws IOE
   /**
    * Create a new {@link CallCredentials} object.
    *
-   * @throws IOException  in case the call credentials can't be constructed.
+   * @throws IOException in case the call credentials can't be constructed.
    */
   public static CallCredentials newCallCredentials(AuthAndTLSOptions options) throws IOException {
+    Credentials creds = newCredentials(options);
+    if (creds != null) {
+      return MoreCallCredentials.from(creds);
+    }
+    return null;
+  }
+
+  @VisibleForTesting
+  public static CallCredentials newCallCredentials(
+      @Nullable InputStream credentialsFile, @Nullable String authScope) throws IOException {
+    Credentials creds = newCredentials(credentialsFile, authScope);
+    if (creds != null) {
+      return MoreCallCredentials.from(creds);
+    }
+    return null;
+  }
+
+  /**
+   * Create a new {@link Credentials} object.
+   *
+   * @throws IOException in case the credentials can't be constructed.
+   */
+  public static Credentials newCredentials(AuthAndTLSOptions options) throws IOException {
     if (!options.authEnabled) {
       return null;
     }
 
     if (options.authCredentials != null) {
       // Credentials from file
       try (InputStream authFile = new FileInputStream(options.authCredentials)) {
-        return newCallCredentials(authFile, options.authScope);
+        return newCredentials(authFile, options.authScope);
       } catch (FileNotFoundException e) {
-        String message = String.format("Could not open auth credentials file '%s': %s",
-            options.authCredentials, e.getMessage());
+        String message =
+            String.format(
+                "Could not open auth credentials file '%s': %s",
+                options.authCredentials, e.getMessage());
         throw new IOException(message, e);
       }
     }
     // Google Application Default Credentials
-    return newCallCredentials(null, options.authScope);
+    return newCredentials(null, options.authScope);
   }
 
-  @VisibleForTesting
-  public static CallCredentials newCallCredentials(@Nullable InputStream credentialsFile,
-      @Nullable String authScope) throws IOException {
+  private static Credentials newCredentials(
+      @Nullable InputStream credentialsFile, @Nullable String authScope) throws IOException {
     try {
       GoogleCredentials creds =
           credentialsFile == null
@@ -127,10 +148,9 @@ public static CallCredentials newCallCredentials(@Nullable InputStream credentia
       if (authScope != null) {
         creds = creds.createScoped(ImmutableList.of(authScope));
       }
-      return MoreCallCredentials.from(creds);
+      return creds;
     } catch (IOException e) {
-      String message = "Failed to init auth credentials: "
-          + e.getMessage();
+      String message = "Failed to init auth credentials: " + e.getMessage();
       throw new IOException(message, e);
     }
   }

src/main/java/com/google/devtools/build/lib/buildeventservice/BazelBuildEventServiceModule.java

@@ -16,7 +16,7 @@
 
 import com.google.common.collect.ImmutableSet;
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
-import com.google.devtools.build.lib.authandtls.GrpcUtils;
+import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
 import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceClient;
 import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceGrpcClient;
 import java.io.IOException;
@@ -37,8 +37,8 @@ protected Class<BuildEventServiceOptions> optionsClass() {
   protected BuildEventServiceClient createBesClient(BuildEventServiceOptions besOptions,
       AuthAndTLSOptions authAndTLSOptions) throws IOException {
     return new BuildEventServiceGrpcClient(
-        GrpcUtils.newChannel(besOptions.besBackend, authAndTLSOptions),
-        GrpcUtils.newCallCredentials(authAndTLSOptions));
+        GoogleAuthUtils.newChannel(besOptions.besBackend, authAndTLSOptions),
+        GoogleAuthUtils.newCallCredentials(authAndTLSOptions));
   }
 
   @Override

src/main/java/com/google/devtools/build/lib/remote/BUILD

@@ -32,6 +32,7 @@ java_library(
         "//src/main/java/com/google/devtools/common/options",
         "//third_party:apache_httpclient",
         "//third_party:apache_httpcore",
+        "//third_party:api_client",
         "//third_party:auth",
         "//third_party:gson",
         "//third_party:guava",

src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java

@@ -17,7 +17,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
-import com.google.devtools.build.lib.authandtls.GrpcUtils;
+import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
 import com.google.devtools.build.lib.buildeventstream.PathConverter;
 import com.google.devtools.build.lib.buildtool.BuildRequest;
 import com.google.devtools.build.lib.events.Event;
@@ -33,7 +33,6 @@
 import com.google.devtools.common.options.OptionsBase;
 import com.google.devtools.common.options.OptionsProvider;
 import com.google.devtools.remoteexecution.v1test.Digest;
-import io.grpc.CallCredentials;
 import io.grpc.Channel;
 import java.io.IOException;
 import java.util.logging.Logger;
@@ -109,39 +108,47 @@ public void beforeCommand(CommandEnvironment env) {
       return;
     }
 
-
     try {
       boolean remoteOrLocalCache = SimpleBlobStoreFactory.isRemoteCacheOptions(remoteOptions);
       boolean grpcCache = GrpcRemoteCache.isRemoteCacheOptions(remoteOptions);
 
       RemoteRetrier retrier =
           new RemoteRetrier(
               remoteOptions, RemoteRetrier.RETRIABLE_GRPC_ERRORS, Retrier.ALLOW_ALL_CALLS);
-      CallCredentials creds = GrpcUtils.newCallCredentials(authAndTlsOptions);
       // TODO(davido): The naming is wrong here. "Remote"-prefix in RemoteActionCache class has no
       // meaning.
       final RemoteActionCache cache;
       if (remoteOrLocalCache) {
         cache =
             new SimpleBlobStoreActionCache(
-                SimpleBlobStoreFactory.create(remoteOptions, env.getWorkingDirectory()),
+                SimpleBlobStoreFactory.create(
+                    remoteOptions,
+                    GoogleAuthUtils.newCredentials(authAndTlsOptions),
+                    env.getWorkingDirectory()),
                 digestUtil);
       } else if (grpcCache || remoteOptions.remoteExecutor != null) {
         // If a remote executor but no remote cache is specified, assume both at the same target.
         String target = grpcCache ? remoteOptions.remoteCache : remoteOptions.remoteExecutor;
-        Channel ch = GrpcUtils.newChannel(target, authAndTlsOptions);
-        cache = new GrpcRemoteCache(ch, creds, remoteOptions, retrier, digestUtil);
+        Channel ch = GoogleAuthUtils.newChannel(target, authAndTlsOptions);
+        cache =
+            new GrpcRemoteCache(
+                ch,
+                GoogleAuthUtils.newCallCredentials(authAndTlsOptions),
+                remoteOptions,
+                retrier,
+                digestUtil);
       } else {
         cache = null;
       }
 
       final GrpcRemoteExecutor executor;
       if (remoteOptions.remoteExecutor != null) {
-        executor = new GrpcRemoteExecutor(
-            GrpcUtils.newChannel(remoteOptions.remoteExecutor, authAndTlsOptions),
-            creds,
-            remoteOptions.remoteTimeout,
-            retrier);
+        executor =
+            new GrpcRemoteExecutor(
+                GoogleAuthUtils.newChannel(remoteOptions.remoteExecutor, authAndTlsOptions),
+                GoogleAuthUtils.newCallCredentials(authAndTlsOptions),
+                remoteOptions.remoteTimeout,
+                retrier);
       } else {
         executor = null;
       }

src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java

@@ -16,6 +16,7 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.google.auth.Credentials;
 import com.google.devtools.build.lib.remote.blobstore.OnDiskBlobStore;
 import com.google.devtools.build.lib.remote.blobstore.RestBlobStore;
 import com.google.devtools.build.lib.remote.blobstore.SimpleBlobStore;
@@ -32,9 +33,13 @@ public final class SimpleBlobStoreFactory {
 
   private SimpleBlobStoreFactory() {}
 
-  public static SimpleBlobStore createRest(RemoteOptions options) throws IOException {
-    return new RestBlobStore(options.remoteRestCache, options.restCachePoolSize,
-        (int) TimeUnit.SECONDS.toMillis(options.remoteTimeout));
+  public static SimpleBlobStore createRest(RemoteOptions options, Credentials creds)
+      throws IOException {
+    return new RestBlobStore(
+        options.remoteRestCache,
+        options.restCachePoolSize,
+        (int) TimeUnit.SECONDS.toMillis(options.remoteTimeout),
+        creds);
   }
 
   public static SimpleBlobStore createLocalDisk(RemoteOptions options, Path workingDirectory)
@@ -43,10 +48,11 @@ public static SimpleBlobStore createLocalDisk(RemoteOptions options, Path workin
         workingDirectory.getRelative(checkNotNull(options.experimentalLocalDiskCachePath)));
   }
 
-  public static SimpleBlobStore create(RemoteOptions options, @Nullable Path workingDirectory)
+  public static SimpleBlobStore create(
+      RemoteOptions options, @Nullable Credentials creds, @Nullable Path workingDirectory)
       throws IOException {
     if (isRestUrlOptions(options)) {
-      return createRest(options);
+      return createRest(options, creds);
     }
     if (workingDirectory != null && isLocalDiskCache(options)) {
       return createLocalDisk(options, workingDirectory);