[WIP] Enable populating java/kotlin/android artifacts for darwin (M1) from linux

src/main/java/com/google/devtools/build/lib/analysis/config/OutputDirectories.java

@@ -239,10 +239,7 @@ private static String buildCpuIdentifier(
       CoreOptions options, @Nullable PlatformOptions platformOptions) {
     if (options.platformInOutputDir && platformOptions != null) {
       Label targetPlatform = platformOptions.computeTargetPlatform();
-      // Only use non-default platforms.
-      if (!PlatformOptions.platformIsDefault(targetPlatform)) {
-        return targetPlatform.getName();
-      }
+      return targetPlatform.getName();
     }
 
     // Fall back to using the CPU.

src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java

@@ -17,6 +17,7 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Strings.isNullOrEmpty;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
 import static com.google.common.util.concurrent.Futures.immediateFuture;
 import static com.google.common.util.concurrent.Futures.transform;
@@ -175,6 +176,25 @@ public class RemoteExecutionService {
   private final AtomicBoolean shutdown = new AtomicBoolean(false);
   private final AtomicBoolean buildInterrupted = new AtomicBoolean(false);
 
+  private boolean isActionCompatibleWithXPlatformCache(Spawn spawn) {
+    String repo = spawn.getResourceOwner().getOwner().getLabel().getPackageIdentifier().getRepository().getName();
+    if (!repo.isEmpty()) {
+      return false;
+    }
+    String mnemonic = spawn.getResourceOwner().getMnemonic();
+    return remoteOptions.remoteXPlatSupportedMnemonics.contains(mnemonic);
+  }
+
+  private boolean isInputRemovedForXPlatform(PathFragment pathFragment) {
+    String path = pathFragment.getPathString();
+    return remoteOptions.remoteXPlatRemovedInputs.stream().anyMatch(s -> path.indexOf(s) >= 0);
+  }
+
+  private boolean isInputIgnoredForXPlatform(PathFragment pathFragment) {
+    String path = pathFragment.getPathString();
+    return remoteOptions.remoteXPlatIgnoredInputs.stream().anyMatch(s -> path.indexOf(s) >= 0);
+  }
+
   public RemoteExecutionService(
       Executor executor,
       Reporter reporter,
@@ -218,7 +238,8 @@ static Command buildCommand(
       List<String> arguments,
       ImmutableMap<String, String> env,
       @Nullable Platform platform,
-      RemotePathResolver remotePathResolver) {
+      RemotePathResolver remotePathResolver,
+      boolean actionCompatibleWithXPlatformCache) {
     Command.Builder command = Command.newBuilder();
     ArrayList<String> outputFiles = new ArrayList<>();
     ArrayList<String> outputDirectories = new ArrayList<>();
@@ -239,6 +260,11 @@ static Command buildCommand(
       command.setPlatform(platform);
     }
     for (String arg : arguments) {
+      if (actionCompatibleWithXPlatformCache) {
+        // Ensure the command line matches linux for M1
+        arg = arg.replace("macos_aarch64", "linux");
+        arg = arg.replace("darwin_arm64", "linux");
+      }
       command.addArguments(decodeBytestringUtf8(arg));
     }
     // Sorting the environment pairs by variable name.
@@ -392,6 +418,17 @@ private MerkleTree buildInputMerkleTree(
       SortedMap<PathFragment, ActionInput> inputMap =
           remotePathResolver.getInputMapping(
               context, /* willAccessRepeatedly= */ !remoteOptions.remoteDiscardMerkleTrees);
+
+      // Compute removed and ignored inputs, to make actionkey hash computation output match between platform.
+      boolean useXPlatformCache = isActionCompatibleWithXPlatformCache(spawn);
+      ImmutableSet<PathFragment> removedInputs = useXPlatformCache ? inputMap.keySet().stream().filter(this::isInputRemovedForXPlatform).collect(toImmutableSet()) : ImmutableSet.of();
+      ImmutableSet<PathFragment> ignoredInputs = useXPlatformCache ? inputMap.keySet().stream().filter(this::isInputIgnoredForXPlatform).collect(toImmutableSet()) : ImmutableSet.of();
+
+      // Remove inputs only present on specific platform, so that they won't contribute to actionkey hash.
+      for (PathFragment fragment : removedInputs) {
+        inputMap.remove(fragment);
+      }
+
       if (!outputDirMap.isEmpty()) {
         // The map returned by getInputMapping is mutable, but must not be mutated here as it is
         // shared with all other strategies.
@@ -406,7 +443,8 @@ private MerkleTree buildInputMerkleTree(
           context.getMetadataProvider(),
           execRoot,
           context.getPathResolver(),
-          digestUtil);
+          digestUtil,
+          ignoredInputs);
     }
   }
 
@@ -470,7 +508,7 @@ public MerkleTree uncachedBuildMerkleTreeVisitor(
   }
 
   @Nullable
-  private static ByteString buildSalt(Spawn spawn) {
+  private ByteString buildSalt(Spawn spawn) {
     String workspace =
         spawn.getExecutionInfo().get(ExecutionRequirements.DIFFERENTIATE_WORKSPACE_CACHE);
     if (workspace != null) {
@@ -479,10 +517,13 @@ private static ByteString buildSalt(Spawn spawn) {
               .addProperties(
                   Platform.Property.newBuilder().setName("workspace").setValue(workspace).build())
               .build();
-      return platform.toByteString();
+      ByteString salt = platform.toByteString();
+      return remoteOptions.remoteActionKeySalt.isEmpty() ?
+              salt : salt.concat(ByteString.copyFromUtf8(remoteOptions.remoteActionKeySalt));
     }
 
-    return null;
+    return remoteOptions.remoteActionKeySalt.isEmpty() ?
+            null : ByteString.copyFromUtf8(remoteOptions.remoteActionKeySalt);
   }
 
   /**
@@ -528,14 +569,14 @@ public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context
       } else {
         platform = PlatformUtils.getPlatformProto(spawn, remoteOptions);
       }
-
       Command command =
           buildCommand(
               spawn.getOutputFiles(),
               spawn.getArguments(),
               spawn.getEnvironment(),
               platform,
-              remotePathResolver);
+              remotePathResolver,
+              isActionCompatibleWithXPlatformCache(spawn));
       Digest commandHash = digestUtil.compute(command);
       Action action =
           Utils.buildAction(
@@ -1400,7 +1441,7 @@ public void onError(@NonNull Throwable e) {
               });
     } else {
       try (SilentCloseable c =
-          Profiler.instance().profile(ProfilerTask.UPLOAD_TIME, "upload outputs")) {
+          Profiler.instance().profile(ProfilerTask.UPLOAD_TIME, "upload outputs for " + action.getActionKey().getDigest().getHash())) {
         UploadManifest manifest = buildUploadManifest(action, spawnResult);
         manifest.upload(action.getRemoteActionExecutionContext(), remoteCache, reporter);
       } catch (IOException e) {

src/main/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutor.java

@@ -143,7 +143,7 @@ public ExecutionResult execute(
     ActionKey actionKey = new ActionKey(actionDigest);
     CachedActionResult cachedActionResult;
     try (SilentCloseable c =
-        Profiler.instance().profile(ProfilerTask.REMOTE_CACHE_CHECK, "check cache hit")) {
+        Profiler.instance().profile(ProfilerTask.REMOTE_CACHE_CHECK, "check cache hit for " + actionKey.getDigest().getHash())) {
       cachedActionResult =
           remoteCache.downloadActionResult(context, actionKey, /* inlineOutErr= */ true);
     }

src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java

@@ -105,9 +105,15 @@ public CacheHandle lookup(Spawn spawn, SpawnExecutionContext context)
       // This is done via a thread-local variable.
       try {
         RemoteActionResult result;
-        try (SilentCloseable c = prof.profile(ProfilerTask.REMOTE_CACHE_CHECK, "check cache hit")) {
+        try (SilentCloseable c = prof.profile(ProfilerTask.REMOTE_CACHE_CHECK, "check cache hit for " + action.getActionKey().getDigest().getHash())) {
           result = remoteExecutionService.lookupCache(action);
         }
+
+        String verb = (result != null && result.getExitCode() == 0) ? "HIT" : "MISS";
+        if (options.remotePrintExecutionMessages == RemoteOptions.ExecutionMessagePrintMode.ALL) {
+          System.out.println("RemoteSpawnCache: Lookup " + verb + " with key " + action.getActionKey().getDigest().getHash() + " for action " + action.getSpawn().getResourceOwner().getOwner().getLabel() + " [" + action.getSpawn().getResourceOwner().getMnemonic() + "]");
+        }
+
         // In case the remote cache returned a failed action (exit code != 0) we treat it as a
         // cache miss
         if (result != null && result.getExitCode() == 0) {

src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java

@@ -203,7 +203,7 @@ public SpawnResult exec(Spawn spawn, SpawnExecutionContext context)
 
       // Try to lookup the action in the action cache.
       RemoteActionResult cachedResult;
-      try (SilentCloseable c = prof.profile(ProfilerTask.REMOTE_CACHE_CHECK, "check cache hit")) {
+      try (SilentCloseable c = prof.profile(ProfilerTask.REMOTE_CACHE_CHECK, "check cache hit for " + action.getActionKey().getDigest().getHash())) {
         cachedResult = acceptCachedResult ? remoteExecutionService.lookupCache(action) : null;
       }
 

src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java

@@ -66,7 +66,7 @@ static DirectoryTree fromActionInputs(
       DigestUtil digestUtil)
       throws IOException {
     return fromActionInputs(
-        inputs, ImmutableSet.of(), metadataProvider, execRoot, artifactPathResolver, digestUtil);
+        inputs, ImmutableSet.of(), metadataProvider, execRoot, artifactPathResolver, digestUtil, ImmutableSet.of());
   }
 
   static DirectoryTree fromActionInputs(
@@ -75,12 +75,13 @@ static DirectoryTree fromActionInputs(
       MetadataProvider metadataProvider,
       Path execRoot,
       ArtifactPathResolver artifactPathResolver,
-      DigestUtil digestUtil)
+      DigestUtil digestUtil,
+      Set<PathFragment> ignoredInputs)
       throws IOException {
     Map<PathFragment, DirectoryNode> tree = new HashMap<>();
     int numFiles =
         buildFromActionInputs(
-            inputs, toolInputs, metadataProvider, execRoot, artifactPathResolver, digestUtil, tree);
+            inputs, toolInputs, metadataProvider, execRoot, artifactPathResolver, digestUtil, ignoredInputs, tree);
     return new DirectoryTree(tree, numFiles);
   }
 
@@ -142,6 +143,7 @@ private static int buildFromActionInputs(
       Path execRoot,
       ArtifactPathResolver artifactPathResolver,
       DigestUtil digestUtil,
+      Set<PathFragment> ignoredInputs,
       Map<PathFragment, DirectoryNode> tree)
       throws IOException {
     return build(
@@ -170,6 +172,12 @@ private static int buildFromActionInputs(
             case REGULAR_FILE:
               {
                 Digest d = DigestUtil.buildDigest(metadata.getDigest(), metadata.getSize());
+
+                if (ignoredInputs.contains(path)) {
+                  // Digest differs per platform, default to 0 for all platforms.
+                  d = Digest.getDefaultInstance();
+                }
+
                 Path inputPath = artifactPathResolver.toPath(input);
                 boolean childAdded =
                     currDir.addChild(
@@ -188,6 +196,7 @@ private static int buildFromActionInputs(
                   execRoot,
                   artifactPathResolver,
                   digestUtil,
+                  ignoredInputs,
                   tree);
 
             case SYMLINK:

src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java

@@ -245,17 +245,18 @@ public static MerkleTree build(
    * @param digestUtil a hashing utility
    */
   public static MerkleTree build(
-      SortedMap<PathFragment, ActionInput> inputs,
-      Set<PathFragment> toolInputs,
-      MetadataProvider metadataProvider,
-      Path execRoot,
-      ArtifactPathResolver artifactPathResolver,
-      DigestUtil digestUtil)
-      throws IOException {
+          SortedMap<PathFragment, ActionInput> inputs,
+          Set<PathFragment> toolInputs,
+          MetadataProvider metadataProvider,
+          Path execRoot,
+          ArtifactPathResolver artifactPathResolver,
+          DigestUtil digestUtil,
+          Set<PathFragment> ignoredInputs)
+          throws IOException {
     try (SilentCloseable c = Profiler.instance().profile("MerkleTree.build(ActionInput)")) {
       DirectoryTree tree =
-          DirectoryTreeBuilder.fromActionInputs(
-              inputs, toolInputs, metadataProvider, execRoot, artifactPathResolver, digestUtil);
+              DirectoryTreeBuilder.fromActionInputs(
+                      inputs, toolInputs, metadataProvider, execRoot, artifactPathResolver, digestUtil, ignoredInputs);
       return build(tree, digestUtil);
     }
   }

src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java

@@ -638,6 +638,56 @@ public RemoteOutputsStrategyConverter() {
               + "`all` to print always.")
   public ExecutionMessagePrintMode remotePrintExecutionMessages;
 
+  @Option(
+          name = "remote_actionkey_salt",
+          defaultValue = "",
+          documentationCategory = OptionDocumentationCategory.REMOTE,
+          effectTags = {OptionEffectTag.UNKNOWN},
+          help = "Salt used when hashing remote actionkeys. Used for remote cache hits troubleshooting, or invalidating " +
+                  "entire cache when updating files not tracked by actions but causing different artifact outputs (see " +
+                  "support for xplat artifacts")
+  public String remoteActionKeySalt;
+
+  @Option(
+          name = "remote_xplat_supported_mnemonics",
+          converter = Converters.CommaSeparatedOptionListConverter.class,
+          defaultValue = "null",
+          documentationCategory = OptionDocumentationCategory.REMOTE,
+          effectTags = {OptionEffectTag.UNKNOWN},
+          help =
+                  "Specify the list of mnemonics whose artifact outputs should be treated as platform independant." +
+                          "This is used to support x-platform cache population",
+          allowMultiple = true)
+  public List<String> remoteXPlatSupportedMnemonics;
+
+  @Option(
+          name = "remote_xplat_removed_inputs",
+          converter = Converters.CommaSeparatedOptionListConverter.class,
+          defaultValue = "null",
+          documentationCategory = OptionDocumentationCategory.REMOTE,
+          effectTags = {OptionEffectTag.UNKNOWN},
+          help =
+                  "Specify the actions inputs that should be removed. They will be removed if their paths contains any " +
+                          "of the specified value here. Removing input from an action can be used to make action key hash " +
+                          "platform independent, at the risk of not rebuilding this action if this input only changes. For such" +
+                          "changes, developers need to bump the salt to re-hash every actions",
+          allowMultiple = true)
+  public List<String> remoteXPlatRemovedInputs;
+
+  @Option(
+          name = "remote_xplat_ignored_inputs",
+          converter = Converters.CommaSeparatedOptionListConverter.class,
+          defaultValue = "null",
+          documentationCategory = OptionDocumentationCategory.REMOTE,
+          effectTags = {OptionEffectTag.UNKNOWN},
+          help =
+                  "Specify the actions inputs that should be ignored. They will used a default (0) hash as these inputs " +
+                          "exist on different platform, but have different size/hash. Ignoring input from an action can be used to make action key hash " +
+                          "platform independent, at the risk of not rebuilding this action if this input only changes. For such" +
+                          "changes, developers need to bump the salt to re-hash every actions",
+          allowMultiple = true)
+  public List<String> remoteXPlatIgnoredInputs;
+
   @Option(
       name = "incompatible_remote_downloader_send_all_headers",
       defaultValue = "true",