Generate index for objective-c code. Remap and move index to deriveddata.

.gitignore

@@ -1,3 +1,4 @@
 bazel-*
 .idea
-tests/macos/xcodeproj/**/xcuserdata
+tests/macos/xcodeproj/**/xcuserdata
+**/*.xcodeproj/bazelinstallers/index-import

rules/library.bzl

@@ -568,6 +568,8 @@ def apple_library(name, library_tools = {}, export_private_headers = True, names
 
     objc_library_data = library_tools["wrap_resources_in_filegroup"](name = objc_libname + "_data", srcs = data)
     objc_copts.append("-I.")
+
+    objc_copts.extend(("-index-store-path", "$(GENDIR)/rules_ios_apple_library_objc.indexstore"))
     objc_library(
         name = objc_libname,
         srcs = objc_sources + objc_private_hdrs + objc_non_exported_hdrs,

rules/repositories.bzl

@@ -59,6 +59,24 @@ def rules_ios_dependencies():
         ],
     )
 
+    _maybe(
+        http_archive,
+        name = "com_github_lyft_index_import",
+        build_file_content = """\
+load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
+
+native_binary(
+    name = "index_import",
+    src = "index-import",
+    out = "index-import",
+    visibility = ["//visibility:public"],
+)
+""",
+        canonical_id = "index-import-5.1.1.3",
+        sha256 = "0946012f1557a72fcbc65bd3f2e9b2a99d5b1e57e6db43a5ad0c969bddd9e072",
+        urls = ["https://github.com/lyft/index-import/releases/download/5.1.1.3/index-import.zip"],
+    )
+
     _maybe(
         http_archive,
         name = "com_github_yonaskolb_xcodegen",

rules/xcodeproj.bzl

@@ -185,7 +185,8 @@ def _xcodeproj_impl(ctx):
         "BAZEL_PATH": ctx.attr.bazel_path,
         "BAZEL_WORKSPACE_ROOT": "$SRCROOT/%s" % script_dot_dots,
         "BAZEL_STUBS_DIR": "$PROJECT_FILE_PATH/bazelstubs",
-        "BAZEL_INSTALLER": "$BAZEL_STUBS_DIR/%s" % ctx.executable.installer.short_path,
+        "BAZEL_INSTALLERS_DIR": "$PROJECT_FILE_PATH/bazelinstallers",
+        "BAZEL_INSTALLER": "$BAZEL_INSTALLERS_DIR/%s" % ctx.executable.installer.basename,
         "CC": "$BAZEL_STUBS_DIR/clang-stub",
         "CXX": "$CC",
         "CLANG_ANALYZER_EXEC": "$CC",
@@ -227,14 +228,14 @@ def _xcodeproj_impl(ctx):
             "PRODUCT_NAME": target_info.name,
             "BAZEL_BIN_SUBDIR": target_info.bazel_bin_subdir,
             "MACH_O_TYPE": target_macho_type,
+            "CLANG_ENABLE_MODULES": "YES",
         }
 
         if target_info.product_type == "application":
             target_settings["INFOPLIST_FILE"] = "$BAZEL_STUBS_DIR/Info-stub.plist"
             target_settings["PRODUCT_BUNDLE_IDENTIFIER"] = target_info.bundle_id
         if target_info.product_type == "bundle.unit-test":
             target_settings["SUPPORTS_MACCATALYST"] = False
-
         target_dependencies = []
         test_host_appname = getattr(target_info, "test_host_appname", None)
         if test_host_appname:
@@ -315,15 +316,17 @@ $BAZEL_INSTALLER
     install_script = ctx.actions.declare_file(
         "%s-install-xcodeproj.sh" % ctx.attr.name,
     )
-
+    installer_runfile_paths = [i.short_path for i in ctx.attr.installer[DefaultInfo].default_runfiles.files.to_list()]
     ctx.actions.expand_template(
         template = ctx.file._xcodeproj_installer_template,
         output = install_script,
         substitutions = {
             "$(project_short_path)": project.short_path,
             "$(project_full_path)": project.path,
+            "$(installer_runfile_short_paths)": " ".join(installer_runfile_paths),
             "$(installer_short_path)": ctx.executable.installer.short_path,
             "$(clang_stub_short_path)": ctx.executable.clang_stub.short_path,
+            "$(index_import_short_path)": ctx.executable.index_import.short_path,
             "$(clang_stub_ld_path)": ctx.executable.ld_stub.short_path,
             "$(clang_stub_swiftc_path)": ctx.executable.swiftc_stub.short_path,
             "$(print_json_leaf_nodes_path)": ctx.executable.print_json_leaf_nodes.short_path,
@@ -343,6 +346,7 @@ $BAZEL_INSTALLER
                 direct = ctx.files.build_wrapper +
                          ctx.files.installer +
                          ctx.files.clang_stub +
+                         ctx.files.index_import +
                          ctx.files.ld_stub +
                          ctx.files.swiftc_stub +
                          ctx.files._infoplist_stub +
@@ -367,6 +371,7 @@ xcodeproj = rule(
         "_workspace_xcsettings": attr.label(executable = False, default = Label("//tools/xcodeproj_shims:WorkspaceSettings.xcsettings"), allow_single_file = ["xcsettings"]),
         "output_processor": attr.label(executable = True, default = Label("//tools/xcodeproj_shims:output-processor.rb"), cfg = "host", allow_single_file = True),
         "_xcodegen": attr.label(executable = True, default = Label("@com_github_yonaskolb_xcodegen//:xcodegen"), cfg = "host"),
+        "index_import": attr.label(executable = True, default = Label("@com_github_lyft_index_import//:index_import"), cfg = "host"),
         "clang_stub": attr.label(executable = True, default = Label("//tools/xcodeproj_shims:clang-stub"), cfg = "host"),
         "ld_stub": attr.label(executable = True, default = Label("//tools/xcodeproj_shims:ld-stub"), cfg = "host"),
         "swiftc_stub": attr.label(executable = True, default = Label("//tools/xcodeproj_shims:swiftc-stub"), cfg = "host"),

tests/macos/xcodeproj/Single-Application-Project-AllTargets.xcodeproj/bazelinstallers/_indexstore.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Import Bazel built indexstores into Xcode. See https://github.com/lyft/index-import
+#
+# This makes use of a number of Xcode's provided environment variables. These
+# file remappings are not specific to this project, the apply to all projects
+# built with Bazel and rules_swift.
+
+# Example: /private/var/tmp/_bazel_<username>/<hash>/execroot/<workspacename>
+readonly bazel_root="^/private/var/tmp/_bazel_.+?/.+?/execroot/[^/]+"
+readonly bazel_bin="^(?:$bazel_root/)?bazel-out/.+?/bin"
+
+# Object file paths are fundamental to how Xcode loads from the indexstore. If
+# the `OutputFile` does not match what Xcode looks for, then those parts of the
+# indexstore will not be found and used.
+readonly bazel_swift_object="$bazel_bin/.*/(.+?)(?:_swift)?_objs/.*/(.+?)[.]swift[.]o$"
+readonly bazel_objc_object="$bazel_bin/.*/_objs/(?:arc/|non_arc/)?(.+?)-(?:objc|cpp)/(.+?)[.]o$"
+readonly xcode_object="$CONFIGURATION_TEMP_DIR/\$1.build/Objects-normal/$ARCHS/\$2.o"
+
+# Bazel built `.swiftmodule` files are copied to `DerivedData`. Since modules
+# are referenced by indexstores, their paths are remapped.
+readonly bazel_module="$bazel_bin/.*/(.+?)[.]swiftmodule$"
+readonly xcode_module="$BUILT_PRODUCTS_DIR/\$1.swiftmodule/$ARCHS.swiftmodule"
+
+# External dependencies are available via the `bazel-<workspacename>` symlink.
+# This remapping, in combination with `xcode-index-preferences.json`, enables
+# index features for external dependencies, such as jump-to-definition.
+readonly bazel_external="$bazel_root/external"
+readonly xcode_external="$BAZEL_WORKSPACE_ROOT/bazel-$(basename "$SRCROOT")/external"
+
+
+$BAZEL_INSTALLERS_DIR/index-import \
+    -remap "$bazel_module=$xcode_module" \
+    -remap "$bazel_swift_object=$xcode_object" \
+    -remap "$bazel_objc_object=$xcode_object" \
+    -remap "$bazel_external=$xcode_external" \
+    -remap "$bazel_root=$BAZEL_WORKSPACE_ROOT" \
+    -remap "^([^//])=$BAZEL_WORKSPACE_ROOT/\$1" \
+    "$@" \
+    "$BUILD_DIR"/../../Index/DataStore

tests/macos/xcodeproj/Single-Application-Project-AllTargets.xcodeproj/bazelinstallers/_swiftmodule.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -euo pipefail
+
+for module in "$@"; do
+    doc="${module%.swiftmodule}.swiftdoc"
+    module_name=$(basename "$module")
+    module_bundle="$BUILT_PRODUCTS_DIR/$module_name"
+    mkdir -p "$module_bundle"
+
+    cp "$module" "$module_bundle/$NATIVE_ARCH_ACTUAL.swiftmodule"
+    cp "$doc" "$module_bundle/$NATIVE_ARCH_ACTUAL.swiftdoc"
+
+    ios_module_name="$NATIVE_ARCH_ACTUAL-$LLVM_TARGET_TRIPLE_VENDOR-$SWIFT_PLATFORM_TARGET_PREFIX$LLVM_TARGET_TRIPLE_SUFFIX"
+    cp "$module" "$module_bundle/$ios_module_name.swiftmodule"
+    cp "$doc" "$module_bundle/$ios_module_name.swiftdoc"
+
+    chmod -R +w "$module_bundle"
+done

tests/macos/xcodeproj/Single-Application-Project-AllTargets.xcodeproj/bazelinstallers/indexstores.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# See `_indexstore.sh` for full details.
+
+# Make sure add these to bazel build target copts for objective-c
+find "$BAZEL_WORKSPACE_ROOT/bazel-out"/*/bin/ \
+     -type d \
+     -name "*.indexstore" \
+     -print0 \
+    | xargs -0 "$BAZEL_INSTALLERS_DIR/_indexstore.sh"
+
+echo "Finish remapping index files"

tests/macos/xcodeproj/Single-Application-Project-AllTargets.xcodeproj/bazelinstallers/install.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Install the runnable products so that Xcode can run it. This includes `.app`s,
+# `.xctest`s, and also command line binaries.
+
+# TODO: Lyft uses this script, because Xcode calls `ditto` on the `.swiftmodule`/`.swiftdoc` files of the top-level
+# target.
+# "$SRCROOT"/bazel/installers/xcode-artifacts.sh
+
+set -eux
+
+
+diagnostics_dir="$TARGET_BUILD_DIR/../../../bazel-xcode-diagnostics/"
+mkdir -p $diagnostics_dir
+
+# Delete all logfiles that are older than 7 days
+find $diagnostics_dir -type f -atime +7d -delete
+date_suffix="$(date +%Y%m%d.%H%M%S%L)"
+
+case ${PRODUCT_TYPE} in
+    com.apple.product-type.framework)
+        input="bazel-bin/$BAZEL_BIN_SUBDIR/${TARGET_NAME}/${FULL_PRODUCT_NAME}"
+        ;;
+    com.apple.product-type.bundle.unit-test)
+        input="bazel-bin/$BAZEL_BIN_SUBDIR/$TARGET_NAME.__internal__.__test_bundle_archive-root/$TARGET_NAME${WRAPPER_SUFFIX:-}"
+        ;;
+    com.apple.product-type.application)
+        input="bazel-bin/$BAZEL_BIN_SUBDIR/${TARGET_NAME}_archive-root/Payload/$TARGET_NAME${WRAPPER_SUFFIX:-}"
+        ;;
+    *)
+        echo "Error: Installing ${TARGET_NAME} of type ${PRODUCT_TYPE} is unsupported" >&2
+        exit 1
+        ;;
+esac
+output="$TARGET_BUILD_DIR/$FULL_PRODUCT_NAME"
+
+mkdir -p "$(dirname "$output")"
+
+if [[ -d $input ]]; then
+    # Copy bundle contents, into the destination bundle.
+    # This avoids self-nesting, like: Foo.app/Foo.app
+    input+="/"
+fi
+
+
+rsync \
+    --recursive --chmod=u+w --delete \
+    "$input" "$output" > $diagnostics_dir/rsync-stdout-$date_suffix.log 2> $diagnostics_dir/rsync-stderr-$date_suffix.log
+
+
+# Part of the build intermediary output will be swiftmodule files
+# which XCode will use for indexing. Let's keep those.
+$BAZEL_INSTALLERS_DIR/swiftmodules.sh > $diagnostics_dir/swiftmodules-stdout-$date_suffix.log 2> $diagnostics_dir/swiftmodules-stderr-$date_suffix.log &
+$BAZEL_INSTALLERS_DIR/indexstores.sh > $diagnostics_dir/indexstores-stdout-$date_suffix.log 2> $diagnostics_dir/indexstores-stderr-$date_suffix.log &
+$BAZEL_INSTALLERS_DIR/lldb-settings.sh  > $diagnostics_dir/lldb-stdout-$date_suffix.log 2> $diagnostics_dir/lldb-stderr-$date_suffix.log &
+
+

tests/macos/xcodeproj/Single-Application-Project-AllTargets.xcodeproj/bazelinstallers/installer

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Install the runnable products so that Xcode can run it. This includes `.app`s,
+# `.xctest`s, and also command line binaries.
+
+# TODO: Lyft uses this script, because Xcode calls `ditto` on the `.swiftmodule`/`.swiftdoc` files of the top-level
+# target.
+# "$SRCROOT"/bazel/installers/xcode-artifacts.sh
+
+set -eux
+
+
+diagnostics_dir="$TARGET_BUILD_DIR/../../../bazel-xcode-diagnostics/"
+mkdir -p $diagnostics_dir
+
+# Delete all logfiles that are older than 7 days
+find $diagnostics_dir -type f -atime +7d -delete
+date_suffix="$(date +%Y%m%d.%H%M%S%L)"
+
+case ${PRODUCT_TYPE} in
+    com.apple.product-type.framework)
+        input="bazel-bin/$BAZEL_BIN_SUBDIR/${TARGET_NAME}/${FULL_PRODUCT_NAME}"
+        ;;
+    com.apple.product-type.bundle.unit-test)
+        input="bazel-bin/$BAZEL_BIN_SUBDIR/$TARGET_NAME.__internal__.__test_bundle_archive-root/$TARGET_NAME${WRAPPER_SUFFIX:-}"
+        ;;
+    com.apple.product-type.application)
+        input="bazel-bin/$BAZEL_BIN_SUBDIR/${TARGET_NAME}_archive-root/Payload/$TARGET_NAME${WRAPPER_SUFFIX:-}"
+        ;;
+    *)
+        echo "Error: Installing ${TARGET_NAME} of type ${PRODUCT_TYPE} is unsupported" >&2
+        exit 1
+        ;;
+esac
+output="$TARGET_BUILD_DIR/$FULL_PRODUCT_NAME"
+
+mkdir -p "$(dirname "$output")"
+
+if [[ -d $input ]]; then
+    # Copy bundle contents, into the destination bundle.
+    # This avoids self-nesting, like: Foo.app/Foo.app
+    input+="/"
+fi
+
+
+rsync \
+    --recursive --chmod=u+w --delete \
+    "$input" "$output" > $diagnostics_dir/rsync-stdout-$date_suffix.log 2> $diagnostics_dir/rsync-stderr-$date_suffix.log
+
+
+# Part of the build intermediary output will be swiftmodule files
+# which XCode will use for indexing. Let's keep those.
+$BAZEL_INSTALLERS_DIR/swiftmodules.sh > $diagnostics_dir/swiftmodules-stdout-$date_suffix.log 2> $diagnostics_dir/swiftmodules-stderr-$date_suffix.log &
+$BAZEL_INSTALLERS_DIR/indexstores.sh > $diagnostics_dir/indexstores-stdout-$date_suffix.log 2> $diagnostics_dir/indexstores-stderr-$date_suffix.log &
+$BAZEL_INSTALLERS_DIR/lldb-settings.sh  > $diagnostics_dir/lldb-stdout-$date_suffix.log 2> $diagnostics_dir/lldb-stderr-$date_suffix.log &
+
+

tests/macos/xcodeproj/Single-Application-Project-AllTargets.xcodeproj/bazelinstallers/lldb-settings.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -euo pipefail
+
+
+# Bazel uses `-debug-prefix-map` to strip the bazel build directory from the
+# paths embedded in debug info. This means the debug info contains _project
+# relative_ paths instead of Bazel absolute paths.
+#
+# However, Xcode sets breakpoints via _project absolute_ paths, which are not
+# the paths in the debug info. This lldb setting ensures that _project relative_
+# paths are remapped to _project absolute_ paths.
+#
+# NOTE: In order to use this, add this line to `~/.lldbinit`:
+#
+#     command source ~/.lldbinit-source-map
+cat <<-END > ~/.lldbinit-source-map
+settings set target.source-map ./ "$SRCROOT/"
+END

tests/macos/xcodeproj/Single-Application-Project-AllTargets.xcodeproj/bazelinstallers/swiftmodules.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Copy Bazel build `.swiftmodule` files to `DerivedData`. This is used by Xcode
+# and its indexing.
+find "$BAZEL_WORKSPACE_ROOT/bazel-out"/*/bin/ \
+    -name "*.swiftmodule" \
+     -not -path "*/_swift_module_cache/*" \
+     -print0 \
+    | xargs -0 "$BAZEL_INSTALLERS_DIR/_swiftmodule.sh"

tests/macos/xcodeproj/Single-Application-Project-AllTargets.xcodeproj/project.pbxproj

@@ -272,6 +272,7 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				BAZEL_BIN_SUBDIR = /tests/macos/xcodeproj;
+				CLANG_ENABLE_MODULES = YES;
 				IPHONEOS_DEPLOYMENT_TARGET = 11.0;
 				MACH_O_TYPE = "$(inherited)";
 				PRODUCT_NAME = "Single-Application-UnitTests";
@@ -284,6 +285,7 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				BAZEL_BIN_SUBDIR = /rules/test_host_app;
+				CLANG_ENABLE_MODULES = YES;
 				INFOPLIST_FILE = "$BAZEL_STUBS_DIR/Info-stub.plist";
 				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
 				MACH_O_TYPE = "$(inherited)";
@@ -297,6 +299,7 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				BAZEL_BIN_SUBDIR = /rules/test_host_app;
+				CLANG_ENABLE_MODULES = YES;
 				INFOPLIST_FILE = "$BAZEL_STUBS_DIR/Info-stub.plist";
 				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
 				MACH_O_TYPE = "$(inherited)";
@@ -310,7 +313,8 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				BAZEL_BUILD_EXEC = "$BAZEL_STUBS_DIR/build-wrapper";
-				BAZEL_INSTALLER = $BAZEL_STUBS_DIR/tools/xcodeproj_shims/installer;
+				BAZEL_INSTALLER = $BAZEL_INSTALLERS_DIR/installer;
+				BAZEL_INSTALLERS_DIR = $PROJECT_FILE_PATH/bazelinstallers;
 				BAZEL_OUTPUT_PROCESSOR = "$BAZEL_STUBS_DIR/output-processor.rb";
 				BAZEL_PATH = bazelisk;
 				BAZEL_STUBS_DIR = $PROJECT_FILE_PATH/bazelstubs;
@@ -333,7 +337,8 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				BAZEL_BUILD_EXEC = "$BAZEL_STUBS_DIR/build-wrapper";
-				BAZEL_INSTALLER = $BAZEL_STUBS_DIR/tools/xcodeproj_shims/installer;
+				BAZEL_INSTALLER = $BAZEL_INSTALLERS_DIR/installer;
+				BAZEL_INSTALLERS_DIR = $PROJECT_FILE_PATH/bazelinstallers;
 				BAZEL_OUTPUT_PROCESSOR = "$BAZEL_STUBS_DIR/output-processor.rb";
 				BAZEL_PATH = bazelisk;
 				BAZEL_STUBS_DIR = $PROJECT_FILE_PATH/bazelstubs;
@@ -356,6 +361,7 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				BAZEL_BIN_SUBDIR = /tests/macos/xcodeproj;
+				CLANG_ENABLE_MODULES = YES;
 				IPHONEOS_DEPLOYMENT_TARGET = 11.0;
 				MACH_O_TYPE = "$(inherited)";
 				PRODUCT_NAME = "Single-Application-RunnableTestSuite";
@@ -368,6 +374,7 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				BAZEL_BIN_SUBDIR = /tests/macos/xcodeproj;
+				CLANG_ENABLE_MODULES = YES;
 				IPHONEOS_DEPLOYMENT_TARGET = 11.0;
 				MACH_O_TYPE = "$(inherited)";
 				PRODUCT_NAME = "Single-Application-RunnableTestSuite";
@@ -380,6 +387,7 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				BAZEL_BIN_SUBDIR = /tests/macos/xcodeproj;
+				CLANG_ENABLE_MODULES = YES;
 				IPHONEOS_DEPLOYMENT_TARGET = 11.0;
 				MACH_O_TYPE = "$(inherited)";
 				PRODUCT_NAME = "Single-Application-UnitTests";