Add feature for partially explicit modules (bazelbuild#763)

As of Swift 5.6 and this PR swiftlang/swift#39887
the new json format used for entirely explicit module builds can be used
without entirely disabling implicit modules. This provides an
alternative to VFS overlays for discovering dependencies without `-I`
search paths. This is theoretically about the same but I expect this
file format to have better support in general than VFS overlays for
things like the new incremental compilation support, which currently
doesn't support VFS overlays, and this shouldn't have any downsides vs
VFS files.

This format does support some more keys than just the 2 we pass, as far
as I can tell they are unused today, so I'm opting not to pass them. We
may have to revisit this in the future.

This kind of cherry picks 1666670 but I
kept the VFS feature for now for legacy users.

Author: keith

swift/internal/BUILD

@@ -45,6 +45,7 @@ bzl_library(
         ":debugging",
         ":derived_files",
         ":developer_dirs",
+        ":explicit_module_map_file",
         ":feature_names",
         ":features",
         ":providers",
@@ -388,6 +389,11 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "explicit_module_map_file",
+    srcs = ["explicit_module_map_file.bzl"],
+)
+
 bzl_library(
     name = "feature_names",
     srcs = ["feature_names.bzl"],

swift/internal/compiling.bzl

@@ -24,6 +24,7 @@ load(
     "run_toolchain_action",
     "swift_action_names",
 )
+load(":explicit_module_map_file.bzl", "write_explicit_swift_module_map_file")
 load(":derived_files.bzl", "derived_files")
 load(
     ":feature_names.bzl",
@@ -62,6 +63,7 @@ load(
     "SWIFT_FEATURE_SUPPORTS_SYSTEM_MODULE_FLAG",
     "SWIFT_FEATURE_SYSTEM_MODULE",
     "SWIFT_FEATURE_USE_C_MODULES",
+    "SWIFT_FEATURE_USE_EXPLICIT_SWIFT_MODULE_MAP",
     "SWIFT_FEATURE_USE_GLOBAL_INDEX_STORE",
     "SWIFT_FEATURE_USE_GLOBAL_MODULE_CACHE",
     "SWIFT_FEATURE_USE_OLD_DRIVER",
@@ -875,7 +877,10 @@ def compile_action_configs(
                 swift_action_names.DUMP_AST,
             ],
             configurators = [_dependencies_swiftmodules_configurator],
-            not_features = [SWIFT_FEATURE_VFSOVERLAY],
+            not_features = [
+                [SWIFT_FEATURE_VFSOVERLAY],
+                [SWIFT_FEATURE_USE_EXPLICIT_SWIFT_MODULE_MAP],
+            ],
         ),
         swift_toolchain_config.action_config(
             actions = [
@@ -888,6 +893,17 @@ def compile_action_configs(
             ],
             features = [SWIFT_FEATURE_VFSOVERLAY],
         ),
+        swift_toolchain_config.action_config(
+            actions = [
+                swift_action_names.COMPILE,
+                swift_action_names.DERIVE_FILES,
+                swift_action_names.DUMP_AST,
+            ],
+            configurators = [
+                _explicit_swift_module_map_configurator,
+            ],
+            features = [SWIFT_FEATURE_USE_EXPLICIT_SWIFT_MODULE_MAP],
+        ),
     ])
 
     #### Search paths for framework dependencies
@@ -1620,6 +1636,21 @@ def _dependencies_swiftmodules_vfsoverlay_configurator(prerequisites, args):
         inputs = swiftmodules + [prerequisites.vfsoverlay_file],
     )
 
+def _explicit_swift_module_map_configurator(prerequisites, args):
+    """Adds the explicit Swift module map file to the command line."""
+    args.add_all(
+        [
+            "-explicit-swift-module-map-file",
+            prerequisites.explicit_swift_module_map_file,
+        ],
+        before_each = "-Xfrontend",
+    )
+    return swift_toolchain_config.config_result(
+        inputs = prerequisites.transitive_swiftmodules + [
+            prerequisites.explicit_swift_module_map_file,
+        ],
+    )
+
 def _module_name_configurator(prerequisites, args):
     """Adds the module name flag to the command line."""
     args.add("-module-name", prerequisites.module_name)
@@ -2081,11 +2112,32 @@ def compile(
     else:
         vfsoverlay_file = None
 
+    if is_feature_enabled(
+        feature_configuration = feature_configuration,
+        feature_name = SWIFT_FEATURE_USE_EXPLICIT_SWIFT_MODULE_MAP,
+    ):
+        if vfsoverlay_file:
+            fail("Cannot use both `swift.vfsoverlay` and `swift.use_explicit_swift_module_map` features at the same time.")
+
+        # Generate the JSON file that contains the manifest of Swift
+        # dependencies.
+        explicit_swift_module_map_file = actions.declare_file(
+            "{}.swift-explicit-module-map.json".format(target_name),
+        )
+        write_explicit_swift_module_map_file(
+            actions = actions,
+            explicit_swift_module_map_file = explicit_swift_module_map_file,
+            module_contexts = transitive_modules,
+        )
+    else:
+        explicit_swift_module_map_file = None
+
     prerequisites = struct(
         additional_inputs = additional_inputs,
         bin_dir = feature_configuration._bin_dir,
         cc_compilation_context = merged_providers.cc_info.compilation_context,
         defines = sets.to_list(defines_set),
+        explicit_swift_module_map_file = explicit_swift_module_map_file,
         developer_dirs = swift_toolchain.developer_dirs,
         genfiles_dir = feature_configuration._genfiles_dir,
         is_swift = True,

swift/internal/explicit_module_map_file.bzl

@@ -0,0 +1,50 @@
+# Copyright 2022 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generates the JSON manifest used to pass Swift modules to the compiler."""
+
+def write_explicit_swift_module_map_file(
+        *,
+        actions,
+        explicit_swift_module_map_file,
+        module_contexts):
+    """Generates the JSON-formatted explicit module map file.
+
+    This file is a manifest that contains the path information for all the
+    Swift modules from dependencies that are needed to compile a particular
+    module.
+
+    Args:
+        actions: The object used to register actions.
+        explicit_swift_module_map_file: A `File` to which the generated JSON
+            will be written.
+        module_contexts: A list of module contexts that provide the Swift
+            dependencies for the compilation.
+    """
+    module_descriptions = []
+
+    for module_context in module_contexts:
+        if not module_context.swift:
+            continue
+
+        swift_context = module_context.swift
+        module_description = {"moduleName": module_context.name}
+        if swift_context.swiftmodule:
+            module_description["modulePath"] = swift_context.swiftmodule.path
+        module_descriptions.append(module_description)
+
+    actions.write(
+        content = json.encode(module_descriptions),
+        output = explicit_swift_module_map_file,
+    )

swift/internal/feature_names.bzl

@@ -166,6 +166,10 @@ SWIFT_FEATURE_REWRITE_GENERATED_HEADER = "swift.rewrite_generated_header"
 # them.
 SWIFT_FEATURE_USE_C_MODULES = "swift.use_c_modules"
 
+# If enabled, Swift modules for dependencies will be passed to the compiler
+# using a JSON file instead of `-I` search paths.
+SWIFT_FEATURE_USE_EXPLICIT_SWIFT_MODULE_MAP = "swift.use_explicit_swift_module_map"
+
 # If enabled, Swift compilation actions will use the same global Clang module
 # cache used by Objective-C compilation actions. This is disabled by default
 # because under some circumstances Clang module cache corruption can cause the

test/features_tests.bzl

@@ -38,6 +38,22 @@ file_prefix_xcode_remap_test = make_action_command_line_test_rule(
     },
 )
 
+vfsoverlay_test = make_action_command_line_test_rule(
+    config_settings = {
+        "//command_line_option:features": [
+            "swift.vfsoverlay",
+        ],
+    },
+)
+
+explicit_swift_module_map_test = make_action_command_line_test_rule(
+    config_settings = {
+        "//command_line_option:features": [
+            "swift.use_explicit_swift_module_map",
+        ],
+    },
+)
+
 def features_test_suite(name):
     """Test suite for various features.
 
@@ -47,13 +63,16 @@ def features_test_suite(name):
     default_test(
         name = "{}_default_test".format(name),
         tags = [name],
-        expected_argv = ["-emit-object"],
+        expected_argv = [
+            "-emit-object",
+            "-I$(BIN_DIR)/test/fixtures/basic",
+        ],
         not_expected_argv = [
             "-file-prefix-map",
             "-Xwrapped-swift=-file-prefix-pwd-is-dot",
         ],
         mnemonic = "SwiftCompile",
-        target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple",
+        target_under_test = "@build_bazel_rules_swift//test/fixtures/basic:second",
     )
 
     file_prefix_map_test(
@@ -95,3 +114,33 @@ def features_test_suite(name):
         mnemonic = "SwiftCompile",
         target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple",
     )
+
+    vfsoverlay_test(
+        name = "{}_vfsoverlay_test".format(name),
+        tags = [name],
+        expected_argv = [
+            "-Xfrontend -vfsoverlay$(BIN_DIR)/test/fixtures/basic/second.vfsoverlay.yaml",
+            "-I/__build_bazel_rules_swift/swiftmodules",
+        ],
+        not_expected_argv = [
+            "-I$(BIN_DIR)/test/fixtures/basic",
+            "-explicit-swift-module-map-file",
+        ],
+        mnemonic = "SwiftCompile",
+        target_under_test = "@build_bazel_rules_swift//test/fixtures/basic:second",
+    )
+
+    explicit_swift_module_map_test(
+        name = "{}_explicit_swift_module_map_test".format(name),
+        tags = [name],
+        expected_argv = [
+            "-Xfrontend -explicit-swift-module-map-file -Xfrontend $(BIN_DIR)/test/fixtures/basic/second.swift-explicit-module-map.json",
+        ],
+        not_expected_argv = [
+            "-I$(BIN_DIR)/test/fixtures/basic",
+            "-I/__build_bazel_rules_swift/swiftmodules",
+            "-Xfrontend -vfsoverlay$(BIN_DIR)/test/fixtures/basic/second.vfsoverlay.yaml",
+        ],
+        mnemonic = "SwiftCompile",
+        target_under_test = "@build_bazel_rules_swift//test/fixtures/basic:second",
+    )

test/fixtures/basic/BUILD

@@ -0,0 +1,21 @@
+load("//swift:swift.bzl", "swift_library")
+load("//test/fixtures:common.bzl", "FIXTURE_TAGS")
+
+package(
+    default_visibility = ["//test:__subpackages__"],
+)
+
+licenses(["notice"])
+
+swift_library(
+    name = "first",
+    srcs = ["first.swift"],
+    tags = FIXTURE_TAGS,
+)
+
+swift_library(
+    name = "second",
+    srcs = ["second.swift"],
+    tags = FIXTURE_TAGS,
+    deps = ["first"],
+)

test/fixtures/basic/first.swift

@@ -0,0 +1,3 @@
+public func foo() -> String {
+    return "foo"
+}

test/fixtures/basic/second.swift

@@ -0,0 +1,5 @@
+import first
+
+public func bar() -> String {
+    return foo()
+}

test/rules/action_command_line_test.bzl

@@ -77,7 +77,9 @@ def _action_command_line_test_impl(ctx):
     # end and look for arguments followed by a trailing space so that having
     # `-foo` in the expected list doesn't match `-foobar`, for example.
     concatenated_args = " ".join(action.argv) + " "
+    bin_dir = analysistest.target_bin_dir_path(env)
     for expected in ctx.attr.expected_argv:
+        expected = expected.replace("$(BIN_DIR)", bin_dir)
         if expected + " " not in concatenated_args and expected + "=" not in concatenated_args:
             unittest.fail(
                 env,
@@ -88,6 +90,7 @@ def _action_command_line_test_impl(ctx):
                 ),
             )
     for not_expected in ctx.attr.not_expected_argv:
+        not_expected = not_expected.replace("$(BIN_DIR)", bin_dir)
         if not_expected + " " in concatenated_args or not_expected + "=" in concatenated_args:
             unittest.fail(
                 env,