Add source_output_file_map_aspect and DataStore symlink logic

BazelExtensions/xchammerconfig.bzl

@@ -56,13 +56,22 @@ def target_config(
 def project_config(
         paths,  # [String]?
         build_bazel_platform_options = None,  # [String: [String]]?
+        bazel_build_service_config = None, # [String: XCHammerTargetConfig]?
         generate_transitive_xcode_targets = None,  # Bool
         generate_xcode_schemes = None,  # Bool
         xcconfig_overrides = None):  # : [String: String]?
-    return struct(paths = paths, buildBazelPlatformOptions = build_bazel_platform_options, generateTransitiveXcodeTargets = generate_transitive_xcode_targets, generateXcodeSchemes = generate_xcode_schemes, xcconfigOverrides = xcconfig_overrides)
+    return struct(paths = paths, buildBazelPlatformOptions = build_bazel_platform_options, bazelBuildServiceConfig = bazel_build_service_config, generateTransitiveXcodeTargets = generate_transitive_xcode_targets, generateXcodeSchemes = generate_xcode_schemes, xcconfigOverrides = xcconfig_overrides)
 
 def xchammer_config(
         targets,  # [String]
         projects,  # [String: XCHammerProjectConfig]
         target_config = None):  # [String: XCHammerTargetConfig]?
     return struct(targets = targets, targetConfig = target_config, projects = projects)
+
+def bazel_build_service_config(
+        bep_path = None, # String?
+        indexing_enabled = False, # Bool
+        index_store_path = None, # String?
+        indexing_data_dir = None, # String?
+        progress_bar_enabled = False): # Bool
+    return struct(bepPath = bep_path, indexingEnabled = indexing_enabled, indexStorePath = index_store_path, indexingDataDir = indexing_data_dir, progressBarEnabled = progress_bar_enabled)

BazelExtensions/xcode_configuration_provider.bzl

@@ -154,8 +154,79 @@ def _install_action(ctx, infos, itarget):
     )
     return [output]
 
+SourceOutputFileMapInfo = provider(
+    doc = "...",
+    fields = {
+        "mapping": "Dictionary where keys are source file paths and values are at the respective .o file under bazel-out",
+    },
+)
+
+def _source_output_file_map(target, ctx):
+    """
+    Maps source code files to respective `.o` object file under bazel-out. Output group is used for indexing in xcbuildkit.
+    """
+    source_output_file_map = ctx.actions.declare_file("{}_source_output_file_map.json".format(target.label.name))
+
+    mapping = {}
+    objc_srcs = []
+    objc_objects = []
+
+    # List of source files to be mapped to output files
+    if hasattr(ctx.rule.attr, "srcs"):
+        objc_srcs = [
+            f
+            for source_file in ctx.rule.attr.srcs
+            for f in source_file.files.to_list()
+            # Handling objc only for now
+            if f.path.endswith((".m", ".mm", ".c", ".cc", ".cpp"))
+            # TODO: handle swift
+        ]
+
+    # Get compilation outputs if present
+    if OutputGroupInfo in target:
+        if hasattr(target[OutputGroupInfo], "compilation_outputs"):
+            objc_objects.extend(target[OutputGroupInfo].compilation_outputs.to_list())
+
+    # Map source to output file
+    if len(objc_srcs):
+        if len(objc_srcs) != len(objc_objects):
+            fail("[ERROR] Unexpected number of object files")
+        for src in objc_srcs:
+            basename_without_ext = src.basename.replace(".%s" % src.extension, "")
+            obj = [o for o in objc_objects if "%s.o" % basename_without_ext == o.basename]
+            if len(obj) != 1:
+                fail("Failed to find single object file for source %s. Found: %s" % (src, obj))
+
+            obj = obj[0]
+            mapping["/{}".format(src.path)] = obj.path
+
+    # Collect info from deps
+    deps = getattr(ctx.rule.attr, "deps", [])
+    transitive_jsons = []
+    for dep in deps:
+        # Collect mappings from deps
+        if SourceOutputFileMapInfo in dep:
+            for k, v in dep[SourceOutputFileMapInfo].mapping.items():
+                mapping[k] = v
+        # Collect generated JSON files from deps
+        if OutputGroupInfo in dep:
+            if hasattr(dep[OutputGroupInfo], "source_output_file_map"):
+                transitive_jsons.append(dep[OutputGroupInfo].source_output_file_map)
+
+    # Writes JSON
+    ctx.actions.write(source_output_file_map, json.encode(mapping))
+
+    return [
+        OutputGroupInfo(
+            source_output_file_map = depset([source_output_file_map], transitive = transitive_jsons),
+        ),
+        SourceOutputFileMapInfo(
+            mapping = mapping
+        ),
+    ]
 def _xcode_build_sources_aspect_impl(itarget, ctx):
     """ Install Xcode project dependencies into the source root.
+
     This is required as by default, Bazel only installs genfiles for those
     genfiles which are passed to the Bazel command line.
     """
@@ -192,7 +263,7 @@ def _xcode_build_sources_aspect_impl(itarget, ctx):
             ),
         ),
         XcodeBuildSourceInfo(values = depset(infos), transitive=[depset(compacted_transitive_files)]),
-    ]
+    ] + _source_output_file_map(itarget, ctx)
 
 
 # Note, that for "pure" Xcode builds we build swiftmodules with Xcode, so we

BazelExtensions/xcodeproject.bzl

@@ -9,16 +9,16 @@ load(
     "@xchammer//:BazelExtensions/xchammerconfig.bzl",
     "xchammer_config",
     "gen_xchammer_config",
-    "project_config"
+    "project_config",
 )
 
 load(
     "@xchammer//:BazelExtensions/xcode_configuration_provider.bzl",
     "XcodeProjectTargetInfo",
     "XcodeConfigurationAspectInfo",
     "target_config_aspect",
-    "xcode_build_sources_aspect",
     "XcodeBuildSourceInfo",
+    "SourceOutputFileMapInfo",
 )
 
 non_hermetic_execution_requirements = { "no-cache": "1", "no-remote": "1", "local": "1", "no-sandbox": "1" }
@@ -41,6 +41,7 @@ def _xcode_project_impl(ctx):
     # Collect Target configuration JSON from deps
     # Then, merge them to a list
     aggregate_target_config = {}
+
     for dep in ctx.attr.targets:
         if XcodeConfigurationAspectInfo in dep:
             for info in dep[XcodeConfigurationAspectInfo].values:
@@ -146,12 +147,11 @@ def _xcode_project_impl(ctx):
         execution_requirements = { "local": "1" }
     )
 
-
 _xcode_project = rule(
     implementation=_xcode_project_impl,
     attrs={
         "targets": attr.label_list(
-            aspects=[tulsi_sources_aspect, target_config_aspect]
+            aspects=[tulsi_sources_aspect, target_config_aspect],
         ),
         "project_name": attr.string(),
         "bazel": attr.string(default="bazel"),
@@ -169,11 +169,13 @@ get_srcroot = "\"$(cat ../../DO_NOT_BUILD_HERE)/\""
 def _install_xcode_project_impl(ctx):
     xcodeproj = ctx.attr.xcodeproj.files.to_list()[0]
     output_proj = "$SRCROOT/" + xcodeproj.basename
+
     command = [
         "SRCROOT=" + get_srcroot,
         "ditto " + xcodeproj.path + " " + output_proj,
         "sed -i '' \"s,__BAZEL_EXEC_ROOT__,$PWD,g\" " + output_proj + "/XCHammerAssets/bazel_build_settings.py",
         "sed -i '' \"s,__BAZEL_OUTPUT_BASE__,$(dirname $(dirname $PWD)),g\" " + output_proj + "/XCHammerAssets/bazel_build_settings.py",
+        "sed -i '' \"s,__BAZEL_EXEC_ROOT__,$PWD,g\" " + output_proj + "/XCHammerAssets/bazel_build_service_setup.sh",
         # This is kind of a hack for reference bazel relative to the source
         # directory, as bazel_build_settings.py doesn't sub Xcode build
         # settings.
@@ -185,6 +187,7 @@ def _install_xcode_project_impl(ctx):
         "(rm -f $SRCROOT/external && ln -sf $PWD/../../external $SRCROOT/external)",
         'echo "' + output_proj + '" > ' + ctx.outputs.out.path,
     ]
+
     ctx.actions.run_shell(
         inputs=ctx.attr.xcodeproj.files,
         command=";".join(command),
@@ -196,11 +199,12 @@ def _install_xcode_project_impl(ctx):
 
 _install_xcode_project = rule(
     implementation=_install_xcode_project_impl,
-    attrs={"xcodeproj": attr.label(mandatory=True)},
+    attrs={
+        "xcodeproj": attr.label(mandatory=True),
+    },
     outputs={"out": "%{name}.dummy"},
 )
 
-
 def xcode_project(**kwargs):
     """ Generate an Xcode project
 
@@ -225,7 +229,6 @@ def xcode_project(**kwargs):
         proj_args["project_name"] = kwargs["name"]
 
     # Build an XCHammer config Based on inputs
-    targets_json = [str(t) for t in kwargs.get("targets")]
     if "target_config" in  proj_args:
         str_dict = {}
         for k in proj_args["target_config"]:
@@ -236,7 +239,7 @@ def xcode_project(**kwargs):
         proj_args["target_config"] =  "{}"
 
     proj_args["name"] = rule_name + "_impl"
-    proj_args["project_config"] = proj_args["project_config"].to_json() if "project_config" in  proj_args else None
+    proj_args["project_config"] = proj_args["project_config"].to_json() if "project_config" in proj_args else None
 
     _xcode_project(**proj_args)
 

Sources/XCHammer/Generator.swift

@@ -88,7 +88,7 @@ enum Generator {
         let overrides = getRepositoryOverrides(genOptions: genOptions)
         let bazelArgs: [String] = [
             "--aspects @xchammer//:BazelExtensions/xcode_configuration_provider.bzl%pure_xcode_build_sources_aspect",
-            "--output_groups=xcode_project_deps"
+            "--output_groups=xcode_project_deps,+source_output_file_map",
         ] + overrides + labels.map { $0.value }
 
         // We retry.sh the bazel command so if Xcode updates, the build still works
@@ -430,7 +430,7 @@ enum Generator {
             ] + overrides + [
                 // Build xcode_project_deps for targets in question.
                 "--aspects @xchammer//:BazelExtensions/xcode_configuration_provider.bzl%xcode_build_sources_aspect",
-                "--output_groups=+xcode_project_deps"
+                "--output_groups=+xcode_project_deps,+source_output_file_map",
             ]
 
             let buildOptions = (targetConfig?.buildBazelOptions ?? "") + " " +

Sources/XCHammer/XCBuildSettings.swift

@@ -185,6 +185,12 @@ enum XCSettingCodingKey: String, CodingKey {
     case sdkRoot = "SDKROOT"
     case targetedDeviceFamily = "TARGETED_DEVICE_FAMILY"
 
+    // Build Service configs (xcbuildkit)
+    case buildServiceBEPPath = "BUILD_SERVICE_BEP_PATH"
+    case buildServiceIndexingEnabled = "BUILD_SERVICE_INDEXING_ENABLED"
+    case buildServiceIndexStorePath = "BUILD_SERVICE_INDEX_STORE_PATH"
+    case buildServiceIndexingDataDir = "BUILD_SERVICE_INDEXING_DATA_DIR"
+    case buildServiceProgressBarEnabled = "BUILD_SERVICE_PROGRESS_BAR_ENABLED"
 }
 
 
@@ -237,6 +243,11 @@ struct XCBuildSettings: Encodable {
     var targetedDeviceFamily: OrderedArray<String> = OrderedArray.empty
     var isBazel: First<String> = First("NO")
     var diagnosticFlags: [String] = []
+    var buildServiceBEPPath: First<String>?
+    var buildServiceIndexingEnabled: First<String>?
+    var buildServiceIndexStorePath: First<String>?
+    var buildServiceIndexingDataDir: First<String>?
+    var buildServiceProgressBarEnabled: First<String>?
 
 
     func encode(to encoder: Encoder) throws {
@@ -309,6 +320,13 @@ struct XCBuildSettings: Encodable {
         // XCHammer only supports Xcode projects at the root directory
         try container.encode("$SOURCE_ROOT", forKey: .tulsiWR)
         try container.encode(diagnosticFlags.joined(separator: " "), forKey: .diagnosticFlags)
+
+        // Build Service (xcbuildkit)
+        try buildServiceBEPPath.map { try container.encode($0.v, forKey: .buildServiceBEPPath) }
+        try buildServiceIndexingEnabled.map { try container.encode($0.v, forKey: .buildServiceIndexingEnabled) }
+        try buildServiceIndexStorePath.map { try container.encode($0.v, forKey: .buildServiceIndexStorePath) }
+        try buildServiceIndexingDataDir.map { try container.encode($0.v, forKey: .buildServiceIndexingDataDir) }
+        try buildServiceProgressBarEnabled.map { try container.encode($0.v, forKey: .buildServiceProgressBarEnabled) }
     }
 }
 
@@ -366,7 +384,12 @@ extension XCBuildSettings: Monoid {
             targetedDeviceFamily: lhs.targetedDeviceFamily <>
                 rhs.targetedDeviceFamily,
             isBazel: lhs.isBazel <> rhs.isBazel,
-            diagnosticFlags: lhs.diagnosticFlags <> rhs.diagnosticFlags
+            diagnosticFlags: lhs.diagnosticFlags <> rhs.diagnosticFlags,
+            buildServiceBEPPath: lhs.buildServiceBEPPath <> rhs.buildServiceBEPPath,
+            buildServiceIndexingEnabled: lhs.buildServiceIndexingEnabled <> rhs.buildServiceIndexingEnabled,
+            buildServiceIndexStorePath: lhs.buildServiceIndexStorePath <> rhs.buildServiceIndexStorePath,
+            buildServiceIndexingDataDir: lhs.buildServiceIndexingDataDir <> rhs.buildServiceIndexingDataDir,
+            buildServiceProgressBarEnabled: lhs.buildServiceProgressBarEnabled <> rhs.buildServiceProgressBarEnabled
         )
     }
 

Sources/XCHammer/XCHammerConfig.swift

@@ -137,6 +137,9 @@ public struct XCHammerProjectConfig: Codable {
     /// serialized. Spaces and escapes matter.
     public let buildBazelPlatformOptions: [String: [String]]?
 
+    /// Allows one to configure and pass information to the Build Service
+    public let bazelBuildServiceConfig: BazelBuildServiceConfig?
+
     /// Enable generation of transitive Xcode targets.
     /// Defaults to `true`
     /// @note this is _generally_ required for Xcode projects to build with
@@ -193,9 +196,19 @@ public struct XCHammerProjectConfig: Codable {
         xcconfigOverrides = (try container.decodeIfPresent(
                 [String: String].self, forKey: .xcconfigOverrides)) ?? nil
 
+        bazelBuildServiceConfig = try container.decodeIfPresent(
+                BazelBuildServiceConfig.self, forKey: .bazelBuildServiceConfig)
     }
 }
 
+public struct BazelBuildServiceConfig: Codable {
+    public let bepPath: String?
+    public let indexStorePath: String?
+    public let indexingEnabled: Bool
+    public let indexingDataDir: String?
+    public let progressBarEnabled: Bool
+}
+
 public struct XCHammerConfig: Codable {
     /// Labels for all targets. 
     /// Transitve dependencies are converted into targets unless excluded by

Sources/XCHammer/XcodeTarget.swift

@@ -1499,9 +1499,18 @@ public class XcodeTarget: Hashable, Equatable {
         settings.pythonPath =
             First("${PYTHONPATH}:$(PROJECT_FILE_PATH)/XCHammerAssets")
         settings <>= getDeploymentTargetSettings()
+        // Build Service configs (xcbuildkit)
+        if let bazelBuildServiceConfig = genOptions.config.projects[genOptions.projectName]?.bazelBuildServiceConfig {
+            settings.buildServiceBEPPath = First(bazelBuildServiceConfig.bepPath ?? "")
+            settings.buildServiceIndexingEnabled = First(bazelBuildServiceConfig.indexingEnabled ? "YES" : "NO")
+            settings.buildServiceIndexStorePath = First(bazelBuildServiceConfig.indexStorePath ?? "")
+            settings.buildServiceIndexingDataDir = First(bazelBuildServiceConfig.indexingDataDir ?? "")
+            settings.buildServiceProgressBarEnabled = First(bazelBuildServiceConfig.progressBarEnabled ? "YES" : "NO")
+        }
+
+        let bazelScript = ProjectSpec.BuildScript(path: nil, script: getScriptContent(), name: "Bazel build")
+        let buildServiceSetupScript = ProjectSpec.BuildScript(path: nil, script: "$PROJECT_FILE_PATH/XCHammerAssets/bazel_build_service_setup.sh", name: "Build Service Setup")
 
-        let bazelScript = ProjectSpec.BuildScript(path: nil, script: getScriptContent(),
-                name: "Bazel build")
         return ProjectSpec.Target(
             name: xcTargetName,
             type: PBXProductType(rawValue: productType.rawValue)!,
@@ -1510,6 +1519,7 @@ public class XcodeTarget: Hashable, Equatable {
             configFiles: getXCConfigFiles(for: self),
             sources: sources,
             dependencies: [],
+            preBuildScripts: [buildServiceSetupScript],
             postBuildScripts: [bazelScript]
         )
     }

WORKSPACE

@@ -67,7 +67,7 @@ xchammer_dependencies()
 # https://github.com/bazelbuild/bazel/issues/1550
 git_repository(
     name = "xcbuildkit",
-    commit = "b2f0e4dd5a572b7029db3cf791d0897977f96a80",
+    commit = "4c366afb48cb78caed268d483e3cdb308dfc1794",
     remote = "https://github.com/jerrymarino/xcbuildkit.git",
 )
 

XCHammerAssets/bazel_build_service_setup.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Values set at project generation time, for reference check `BazelExtensions/xcodeproject.bzl` => `_install_xcode_project` rule
+BUILD_SERVICE_BAZEL_EXEC_ROOT=__BAZEL_EXEC_ROOT__
+
+# Check `BazelExtensions/source_output_file_map_aspect.bzl`, xcbuildkit needs to know what pattern to look for to pre-load indexing information
+BUILD_SERVICE_SOURCE_OUTPUT_FILE_MAP_SUFFIX=source_output_file_map.json
+
+# Build service binary checks
+XCODE_CONTENTS=$(dirname $DEVELOPER_DIR)
+BUILD_SERVICE=$XCODE_CONTENTS/SharedFrameworks/XCBuild.framework/PlugIns/XCBBuildService.bundle/Contents/MacOS/XCBBuildService
+
+# if the build service does not exist at this location we messed something up
+if [[ ! -f $BUILD_SERVICE ]]; then
+    echo "Could not find build service at $BUILD_SERVICE. Check your Xcode installation."
+    exit 1
+fi
+
+# If the build service is not a symlink, xcbuildkit is not installed so there's nothing to do
+if [[ ! -L $BUILD_SERVICE ]]; then
+    echo "Build service not installed. Nothing to do."
+    exit 0
+fi
+
+# Ensure this folder exists, used by xcbuildkit to hold cached indexing data
+mkdir -p $BUILD_SERVICE_INDEXING_DATA_DIR
+
+# xcbuildkit expects a config file called `xcbuildkit.config` under path/to/foo.xcodeproj
+BUILD_SERVICE_CONFIG_PATH=$PROJECT_FILE_PATH/xcbuildkit.config
+
+cat >$BUILD_SERVICE_CONFIG_PATH <<EOL
+BUILD_SERVICE_INDEXING_ENABLED=$BUILD_SERVICE_INDEXING_ENABLED
+BUILD_SERVICE_INDEX_STORE_PATH=$BUILD_SERVICE_INDEX_STORE_PATH
+BUILD_SERVICE_INDEXING_DATA_DIR=$BUILD_SERVICE_INDEXING_DATA_DIR
+BUILD_SERVICE_PROGRESS_BAR_ENABLED=$BUILD_SERVICE_PROGRESS_BAR_ENABLED
+BUILD_SERVICE_BEP_PATH=$BUILD_SERVICE_BEP_PATH
+BUILD_SERVICE_BAZEL_EXEC_ROOT=$BUILD_SERVICE_BAZEL_EXEC_ROOT
+BUILD_SERVICE_SOURCE_OUTPUT_FILE_MAP_SUFFIX=$BUILD_SERVICE_SOURCE_OUTPUT_FILE_MAP_SUFFIX
+EOL
+
+echo "[INFO] Wrote xcbuildkit config file at $BUILD_SERVICE_CONFIG_PATH"

third_party/repositories.bzl

@@ -263,7 +263,7 @@ def xchammer_dependencies():
         # release, then adds changes to this tag for the Bazel release in
         # question
         # Persisted on github tag=rules_ios-5.0.0,
-        commit = "0dd73ce2950b81934c2bb67bbad09332dbabdee9",
+        commit = "a90a2925d24cc02174188a9365bc84f9c2cb37f4",
     )
 
     namespaced_new_git_repository(