Skip to content

Commit

Permalink
Optional outputs (bazelbuild#291)
Browse files Browse the repository at this point in the history 
* Convert all Dep dependencies to be addressable by label at test time.

* Add files explicitly to runfiles from the data attribute. This is more explicit and resilient that using collect_default.

* New output scheme for the JvmCompilationTask, if an output isn't requested do not create it.

* Break apart compileAll to allow stage independent compilations.
Make jdeps optional, as it is not currently consumed so we shouldn't do the work.
Update test framework to handle the output scheme.
If kotlin srcs are absent, skip kotlin compilation.


* Introduce CompilationArgs to improve compilation argument readability

Issue: bazelbuild/issues/232
  • Loading branch information
@jongerrish
jongerrish authored Mar 2, 2020
1 parent 97d984c commit c2145df
Show file tree
Hide file tree
Showing 13 changed files with 790 additions and 525 deletions.
227 changes: 142 additions & 85 deletions kotlin/internal/jvm/compile.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ load(

# INTERNAL ACTIONS #####################################################################################################
def _fold_jars_action(ctx, rule_kind, output_jar, input_jars):
"""Set up an action to Fold the input jars into a normalized ouput jar."""
"""Set up an action to Fold the input jars into a normalized output jar."""
args = ctx.actions.args()
args.add_all([
"--normalize",
Expand Down Expand Up @@ -114,12 +114,8 @@ def _build_resourcejar_action(ctx):
)
return resources_jar_output

# MAIN ACTIONS #########################################################################################################
def _output_dir_path(ctx, aspect, dir_name):
return "_kotlinc/%s_%s/%s_%s" % (ctx.label.name, aspect, ctx.label.name, dir_name)

def _partition_srcs(srcs):
"""Partition sources for the jvm aspect."""
def _partitioned_srcs(srcs):
"""Creates a struct of srcs sorted by extension. Fails if there are no sources."""
kt_srcs = []
java_srcs = []
src_jars = []
Expand All @@ -132,72 +128,156 @@ def _partition_srcs(srcs):
elif f.path.endswith(".srcjar"):
src_jars.append(f)

if not kt_srcs and not java_srcs and not src_jars:
fail("no sources provided")

return struct(
kt = depset(kt_srcs),
java = depset(java_srcs),
all_srcs = depset(kt_srcs + java_srcs),
src_jars = depset(src_jars),
kt = kt_srcs,
java = java_srcs,
all_srcs = kt_srcs + java_srcs,
src_jars = src_jars,
)

def kt_jvm_compile_action(ctx, rule_kind, output_jar):
"""This macro sets up a compile action for a Kotlin jar.
def _output_dir_path(ctx, aspect, dir_name):
return "_kotlinc/%s_%s/%s_%s" % (ctx.label.name, aspect, ctx.label.name, dir_name)

Args:
rule_kind: The rule kind --e.g., `kt_jvm_library`.
output_jar: The jar file that this macro will use as the output.
Returns:
A struct containing the providers JavaInfo (`java`) and `kt` (KtJvmInfo). This struct is not intended to be
used as a legacy provider -- rather the caller should transform the result.
"""
toolchain = ctx.toolchains[_TOOLCHAIN_TYPE]
def _compiler_directories(ctx):
"""Creates a dict of the necessary compiler directories for generating compile actions"""
return dict(
classdir = _output_dir_path(ctx, "jvm", "classes"),
kotlin_generated_classdir = _output_dir_path(ctx, "jvm", "generated_classes"),
sourcegendir = _output_dir_path(ctx, "jvm", "sourcegenfiles"),
tempdir = _output_dir_path(ctx, "jvm", "temp"),
)

srcs = _partition_srcs(ctx.files.srcs)
if not srcs.kt and not srcs.java and not srcs.src_jars:
fail("no sources provided")
def _compiler_toolchains(ctx):
"""Creates a struct of the relevant compilation toolchains"""
return struct(
kt = ctx.toolchains[_TOOLCHAIN_TYPE],
)

# TODO extract and move this into common. Need to make it generic first.
friends = getattr(ctx.attr, "friends", [])
deps = [d[JavaInfo] for d in friends + ctx.attr.deps] + [toolchain.jvm_stdlibs]
compile_jars = java_common.merge(deps).transitive_compile_time_jars
def _compiler_friends(ctx, friends):
"""Creates a struct of friends meta data"""

# TODO extract and move this into common. Need to make it generic first.
if len(friends) == 0:
module_name = _utils.derive_module_name(ctx)
friend_paths = depset()
return struct(
targets = [],
module_name = _utils.derive_module_name(ctx),
paths = [],
)
elif len(friends) == 1:
if friends[0][_KtJvmInfo] == None:
fail("only kotlin dependencies can be friends")
elif ctx.attr.module_name:
fail("if friends has been set then module_name cannot be provided")
else:
friend_paths = depset([j.path for j in friends[0][JavaInfo].compile_jars.to_list()])
module_name = friends[0][_KtJvmInfo].module_name
return struct(
targets = friends,
paths = friends[0][JavaInfo].compile_jars,
module_name = friends[0][_KtJvmInfo].module_name,
)
else:
fail("only one friend is possible")

classes_directory = _output_dir_path(ctx, "jvm", "classes")
generated_classes_directory = _output_dir_path(ctx, "jvm", "generated_classes")
sourcegen_directory = _output_dir_path(ctx, "jvm", "sourcegenfiles")
temp_directory = _output_dir_path(ctx, "jvm", "temp")
def _compiler_deps(toolchains, friend, deps):
"""Encapsulates compiler dependency metadata."""
dep_infos = [d[JavaInfo] for d in friend.targets + deps] + [toolchains.kt.jvm_stdlibs]
return struct(
deps = dep_infos,
compile_jars = depset(
transitive = [
d.compile_jars
for d in dep_infos
] + [
d.transitive_compile_time_jars
for d in dep_infos
],
),
)

args = _utils.init_args(ctx, rule_kind, module_name)
def _java_info_to_compile_jars(target):
i = target[JavaInfo]
if i == None:
return None
return i.compile_jars

args.add("--classdir", classes_directory)
args.add("--sourcegendir", sourcegen_directory)
args.add("--tempdir", temp_directory)
args.add("--kotlin_generated_classdir", generated_classes_directory)
# MAIN ACTIONS #########################################################################################################

args.add("--output", output_jar)
args.add("--kotlin_output_jdeps", ctx.outputs.jdeps)
args.add("--kotlin_output_srcjar", ctx.outputs.srcjar)
def kt_jvm_compile_action(ctx, rule_kind, output_jar):
"""This macro sets up a compile action for a Kotlin jar.
Args:
rule_kind: The rule kind --e.g., `kt_jvm_library`.
output_jar: The jar file that this macro will use as the output.
Returns:
A struct containing the providers JavaInfo (`java`) and `kt` (KtJvmInfo). This struct is not intended to be
used as a legacy provider -- rather the caller should transform the result.
"""
toolchains = _compiler_toolchains(ctx)
dirs = _compiler_directories(ctx)
srcs = _partitioned_srcs(ctx.files.srcs)
friend = _compiler_friends(ctx, friends = getattr(ctx.attr, "friends", []))
compile_deps = _compiler_deps(toolchains, friend, deps = ctx.attr.deps)
plugins = _plugin_mappers.targets_to_kt_plugins(ctx.attr.plugins + ctx.attr.deps)
_run_kt_builder_action(
ctx = ctx,
rule_kind = rule_kind,
toolchains = toolchains,
dirs = dirs,
srcs = srcs,
friend = friend,
compile_deps = compile_deps,
plugins = plugins,
outputs = {
"output": output_jar,
"kotlin_output_jdeps": ctx.outputs.jdeps,
"kotlin_output_srcjar": ctx.outputs.srcjar,
}
)

return struct(
java = JavaInfo(
output_jar = ctx.outputs.jar,
compile_jar = ctx.outputs.jar,
source_jar = ctx.outputs.srcjar,
# jdeps = ctx.outputs.jdeps,
deps = compile_deps.deps,
runtime_deps = [d[JavaInfo] for d in ctx.attr.runtime_deps],
exports = [d[JavaInfo] for d in getattr(ctx.attr, "exports", [])],
neverlink = getattr(ctx.attr, "neverlink", False),
),
kt = _KtJvmInfo(
srcs = ctx.files.srcs,
module_name = friend.module_name,
language_version = toolchains.kt.api_version,
# intellij aspect needs this.
outputs = struct(
jdeps = ctx.outputs.jdeps,
jars = [struct(
class_jar = ctx.outputs.jar,
ijar = None,
source_jars = [ctx.outputs.srcjar],
)],
),
),
)

args.add("--kotlin_friend_paths", "\n".join(friend_paths.to_list()))

args.add_all("--classpath", compile_jars)
def _run_kt_builder_action(ctx, rule_kind, toolchains, dirs, srcs, friend, compile_deps, plugins, outputs):
"""Creates a KotlinBuilder action invocation."""
args = _utils.init_args(ctx, rule_kind, friend.module_name)

for f, path in dirs.items() + outputs.items():
args.add("--" + f, path)

args.add_all("--classpath", compile_deps.compile_jars)
args.add_all("--sources", srcs.all_srcs, omit_if_empty = True)
args.add_all("--source_jars", srcs.src_jars, omit_if_empty = True)

args.add_joined("--kotlin_friend_paths", friend.paths, join_with = "\n")

# Collect and prepare plugin descriptor for the worker.
plugins = _plugin_mappers.targets_to_kt_plugins(ctx.attr.plugins + ctx.attr.deps)
args.add_all(
"--processors",
plugins,
Expand All @@ -213,57 +293,34 @@ def kt_jvm_compile_action(ctx, rule_kind, output_jar):

progress_message = "Compiling Kotlin to JVM %s { kt: %d, java: %d, srcjars: %d }" % (
ctx.label,
len(srcs.kt.to_list()),
len(srcs.java.to_list()),
len(srcs.src_jars.to_list()),
len(srcs.kt),
len(srcs.java),
len(srcs.src_jars),
)

tools, input_manifests = ctx.resolve_tools(
tools = [
toolchains.kt.kotlinbuilder,
toolchains.kt.kotlin_home,
],
)

tools, _, input_manifests = ctx.resolve_command(tools = [toolchain.kotlinbuilder, toolchain.kotlin_home])
ctx.actions.run(
mnemonic = "KotlinCompile",
inputs = depset(ctx.files.srcs, transitive = [compile_jars]),
inputs = depset(ctx.files.srcs, transitive = [compile_deps.compile_jars]),
tools = tools,
outputs = [
output_jar,
ctx.outputs.jdeps,
ctx.outputs.srcjar,
],
executable = toolchain.kotlinbuilder.files_to_run.executable,
input_manifests = input_manifests,
outputs = [f for f in outputs.values()],
executable = toolchains.kt.kotlinbuilder.files_to_run.executable,
execution_requirements = {"supports-workers": "1"},
arguments = [args],
progress_message = progress_message,
input_manifests = input_manifests,
env = {
"LC_CTYPE": "en_US.UTF-8", # For Java source files
},
)

return struct(
java = JavaInfo(
output_jar = ctx.outputs.jar,
compile_jar = ctx.outputs.jar,
source_jar = ctx.outputs.srcjar,
# jdeps = ctx.outputs.jdeps,
deps = deps,
runtime_deps = [d[JavaInfo] for d in ctx.attr.runtime_deps],
exports = [d[JavaInfo] for d in getattr(ctx.attr, "exports", [])],
neverlink = getattr(ctx.attr, "neverlink", False),
),
kt = _KtJvmInfo(
srcs = ctx.files.srcs,
module_name = module_name,
language_version = toolchain.api_version,
# intelij aspect needs this.
outputs = struct(
jdeps = ctx.outputs.jdeps,
jars = [struct(
class_jar = ctx.outputs.jar,
ijar = None,
source_jars = [ctx.outputs.srcjar],
)],
),
),
)


def kt_jvm_produce_jar_actions(ctx, rule_kind):
"""Setup The actions to compile a jar and if any resources or resource_jars were provided to merge these in with the
Expand Down
4 changes: 4 additions & 0 deletions kotlin/internal/jvm/impl.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,11 @@ def _make_providers(ctx, providers, transitive_files = depset(order = "default")
DefaultInfo(
files = depset([ctx.outputs.jar]),
runfiles = ctx.runfiles(
# explicitly include data files, otherwise they appear to be missing
files = ctx.files.data,
transitive_files = transitive_files,
# continue to use collect_default until proper transitive data collecting is
# implmented.
collect_default = True,
),
),
Expand Down
Loading

0 comments on commit c2145df

Please sign in to comment.