Merge pull request #620 from ChrisCummins/feature/600-build-time-codegen

Remove build-time codegen using Python

WORKSPACE

@@ -277,7 +277,7 @@ http_archive(
 http_archive(
     name = "csmith",
     build_file_content = all_content,
-    sha256 = "86d08c19a1f123054ed9350b7962edaad7d46612de0e07c10b73e578911156fd",
+    sha256 = "9d024a6b202f6a1b9e01351218a85888c06b67b837fe4c6f8ef5bd522fae098c",
     strip_prefix = "csmith-csmith-2.3.0",
     urls = [
         "https://github.com/ChrisCummins/csmith/archive/refs/tags/csmith-2.3.0.tar.gz",

compiler_gym/envs/llvm/service/passes/extract_passes_from_llvm_source_tree.py → build_tools/llvm/legacy_pass_manager/extract_passes_from_llvm_source_tree.py

@@ -212,7 +212,7 @@ def handle_file(source_path: Path) -> Tuple[Path, List[Pass]]:
     """Parse the passes declared in a file."""
     assert str(source_path).endswith(".cpp"), f"Unexpected file type: {source_path}"
 
-    header = Path("include/llvm/" + str(source_path)[len("lib") : -len("cpp")] + "h")
+    header = Path("llvm/" + str(source_path)[len("lib") : -len("cpp")] + "h")
     if not header.is_file():
         header = ""
 

compiler_gym/envs/llvm/service/passes/make_action_space_genfiles.py → build_tools/llvm/legacy_pass_manager/make_action_space_genfiles.py

@@ -117,7 +117,9 @@ def make_action_sources(pass_iterator, outpath: Path):
     flags_path = Path(outpath / "flags.txt")
     descriptions_path = Path(outpath / "flag_descriptions.txt")
 
-    with open(switch_path, "w") as switch_f, open(enum_path, "w") as enum_f:
+    with open(switch_path, "w", encoding="utf-8") as switch_f, open(
+        enum_path, "w", encoding="utf-8"
+    ) as enum_f:
         print("enum class LlvmAction {", file=enum_f)
         print("#define HANDLE_ACTION(action, handlePass) \\", file=switch_f)
         print("  switch (action) {  \\", file=switch_f)
@@ -130,30 +132,18 @@ def make_action_sources(pass_iterator, outpath: Path):
     logger.debug("Generated %s", switch_path.name)
     logger.debug("Generated %s", enum_path.name)
 
-    with open(include_path, "w") as f:
+    with open(include_path, "w", encoding="utf-8") as f:
         print("#pragma once", file=f)
         for header in sorted(headers):
             print(f'#include "{header}"', file=f)
 
-        # Inject an ad-hoc workaround for the non-standard constructor of the
-        # EarlyCSEMemSSAPass.
-        print(
-            """
-namespace llvm {
-FunctionPass* createEarlyCSEMemSSAPass() {
-  return createEarlyCSEPass(/*UseMemorySSA=*/true);
-}
-} // namespace llvm
-""",
-            file=f,
-        )
     logger.debug("Generated %s", include_path.name)
 
-    with open(flags_path, "w") as f:
+    with open(flags_path, "w", encoding="utf-8") as f:
         print("\n".join(p.flag for p in passes), file=f)
     logger.debug("Generated %s", flags_path.name)
 
-    with open(descriptions_path, "w") as f:
+    with open(descriptions_path, "w", encoding="utf-8") as f:
         print("\n".join(p.description for p in passes), file=f)
     logger.debug("Generated %s", descriptions_path.name)
 

compiler_gym/envs/llvm/make_specs.py → build_tools/llvm/legacy_pass_manager/make_specs.py

@@ -11,23 +11,9 @@
 # TODO: As we add support for more compilers we could generalize this script
 # to work with other compiler services rather than hardcoding to LLVM.
 import sys
-import types
 from pathlib import Path
 
-# TODO(github.com/facebookresearch/CompilerGym/issues/506): Avoids circular
-# dependency during specs.py generation, because it is imported from
-# compiler_gym.envs.llvm before being generated.
-sys.modules["compiler_gym.envs.llvm.is_making_specs"] = types.ModuleType(
-    "compiler_gym.envs.llvm.is_making_specs"
-)
-
-from compiler_gym.envs.llvm.llvm_env import LlvmEnv  # noqa: E402
-from compiler_gym.util.runfiles_path import runfiles_path  # noqa: E402
-
-with open(
-    runfiles_path("compiler_gym/envs/llvm/service/passes/flag_descriptions.txt")
-) as f:
-    _FLAG_DESCRIPTIONS = [ln.rstrip() for ln in f.readlines()]
+from compiler_gym.envs.llvm.llvm_env import LlvmEnv
 
 # The maximum number of seconds to wait before timing out.
 TIMEOUT_SECONDS = 300
@@ -41,12 +27,17 @@ def timeout_handler(signum, frame):
 
 
 def main(argv):
-    assert len(argv) == 3, "Usage: make_specs.py <service_binary> <output_path>"
-    service_path, output_path = argv[1:]
+    assert (
+        len(argv) == 3
+    ), "Usage: make_specs.py <service_binary> <flag_descriptions> <output_path>"
+    service_path, flag_descriptions, output_path = argv[1:]
 
     signal.signal(signal.SIGALRM, timeout_handler)
     signal.alarm(TIMEOUT_SECONDS)
 
+    with open(flag_descriptions) as f:
+        flag_descriptions = [ln.rstrip() for ln in f.readlines()]
+
     with LlvmEnv(Path(service_path)) as env:
         with open(output_path, "w") as f:
             print("from enum import Enum", file=f)
@@ -65,7 +56,7 @@ def main(argv):
                 print(f'    {enum_name} = "{name}"', file=f)
             print(file=f)
             print("class action_descriptions(Enum):", file=f)
-            for name, description in zip(env.action_space.names, _FLAG_DESCRIPTIONS):
+            for name, description in zip(env.action_space.names, flag_descriptions):
                 enum_name = "".join([x.capitalize() for x in name[1:].split("-")])
                 sanitized_description = description.replace('" "', "")
                 sanitized_description = sanitized_description.replace('"', "")

compiler_gym/envs/llvm/BUILD

@@ -8,7 +8,7 @@ py_library(
     name = "llvm",
     srcs = [
         "__init__.py",
-        ":specs.py",
+        "specs.py",
     ],
     data = ["//compiler_gym/envs/llvm/service"],
     visibility = ["//visibility:public"],
@@ -68,24 +68,3 @@ py_library(
         "//compiler_gym/views",
     ],
 )
-
-genrule(
-    name = "specs",
-    srcs = [
-        "//compiler_gym/envs/llvm/service",
-        "//compiler_gym/envs/llvm/service:compiler_gym-llvm-service",
-    ],
-    outs = ["specs.py"],
-    cmd = "$(location :make_specs) $(location //compiler_gym/envs/llvm/service:compiler_gym-llvm-service) $@",
-    tools = [":make_specs"],
-)
-
-py_binary(
-    name = "make_specs",
-    srcs = ["make_specs.py"],
-    data = ["//compiler_gym/envs/llvm/service/passes:flag_descriptions.txt"],
-    deps = [
-        ":llvm_env",
-        "//compiler_gym/util",
-    ],
-)

compiler_gym/envs/llvm/CMakeLists.txt

@@ -91,14 +91,3 @@ cg_genrule(
     compiler_gym::envs::llvm::service::service
     compiler_gym::envs::llvm::service::compiler_gym-llvm-service
 )
-
-cg_py_binary(
-  NAME
-    make_specs
-  SRCS
-    "make_specs.py"
-  DATA "${CMAKE_CURRENT_BINARY_DIR}/service/passes/flag_descriptions.txt"
-  DEPS
-    ::llvm_env
-    compiler_gym::util::util
-)

compiler_gym/envs/llvm/service/ActionSpace.h

@@ -10,7 +10,7 @@ namespace compiler_gym::llvm_service {
 
 // This generated file defines the LlvmAction enum which lists the available
 // LLVM transforms. Generated by //compiler_gym/envs/llvm/service/passes:action-genfiles.
-#include "compiler_gym/envs/llvm/service/passes/ActionEnum.h"  // @donotremove
+#include "compiler_gym/envs/llvm/service/passes/10.0.0/ActionEnum.h"  // @donotremove
 
 /**
  * The available action spaces for LLVM.

compiler_gym/envs/llvm/service/BUILD

@@ -76,9 +76,9 @@ cc_library(
     ],
     hdrs = [
         "ActionSpace.h",
-        "//compiler_gym/envs/llvm/service/passes:ActionEnum.h",
     ],
     deps = [
+        "//compiler_gym/envs/llvm/service/passes/10.0.0:headers",
         "//compiler_gym/service/proto:compiler_gym_service_cc",
         "//compiler_gym/util:EnumUtil",
         "//compiler_gym/util:Unreachable",
@@ -212,8 +212,7 @@ cc_library(
     srcs = ["LlvmSession.cc"],
     hdrs = [
         "LlvmSession.h",
-        "//compiler_gym/envs/llvm/service/passes:ActionHeaders.h",
-        "//compiler_gym/envs/llvm/service/passes:ActionSwitch.h",
+        "//compiler_gym/envs/llvm/service/passes/10.0.0:headers",
     ],
     copts = [
         "-DGOOGLE_PROTOBUF_NO_RTTI",

compiler_gym/envs/llvm/service/CMakeLists.txt

@@ -30,10 +30,10 @@ cg_cc_library(
     ActionSpace
   HDRS
     "ActionSpace.h"
-    "$<TARGET_PROPERTY:compiler_gym__envs__llvm__service__passes__actions_genfiles,BINARY_DIR>/ActionEnum.h"
   SRCS
     "ActionSpace.cc"
   DEPS
+    compiler_gym::envs::llvm::service::passes::10.0.0::headers
     compiler_gym::service::proto::compiler_gym_service_cc
     compiler_gym::util::EnumUtil
     compiler_gym::util::Unreachable

compiler_gym/envs/llvm/service/LlvmSession.cc

@@ -22,8 +22,8 @@
 #include "compiler_gym/envs/llvm/service/Cost.h"
 #include "compiler_gym/envs/llvm/service/Observation.h"
 #include "compiler_gym/envs/llvm/service/ObservationSpaces.h"
-#include "compiler_gym/envs/llvm/service/passes/ActionHeaders.h"
-#include "compiler_gym/envs/llvm/service/passes/ActionSwitch.h"
+#include "compiler_gym/envs/llvm/service/passes/10.0.0/ActionHeaders.h"
+#include "compiler_gym/envs/llvm/service/passes/10.0.0/ActionSwitch.h"
 #include "compiler_gym/third_party/autophase/InstCount.h"
 #include "compiler_gym/third_party/llvm/InstCount.h"
 #include "compiler_gym/util/EnumUtil.h"

compiler_gym/envs/llvm/service/passes/10.0.0/ActionEnum.h

@@ -0,0 +1,134 @@
+// Copyright (c) Facebook, Inc. and its affiliates.
+//
+// This source code is licensed under the MIT license found in the LICENSE file
+// in the root directory of this source tree.
+//
+// This file was generated automatically the script
+// build_tools/llvm/legacy_pass_manager/make_action_space_genfiles.py.
+
+enum class LlvmAction {
+  ADD_DISCRIMINATORS,
+  ADCE,
+  AGGRESSIVE_INSTCOMBINE,
+  ALIGNMENT_FROM_ASSUMPTIONS,
+  ALWAYS_INLINE,
+  ARGPROMOTION,
+  ATTRIBUTOR,
+  BARRIER,
+  BDCE,
+  BREAK_CRIT_EDGES,
+  SIMPLIFYCFG,
+  CALLSITE_SPLITTING,
+  CALLED_VALUE_PROPAGATION,
+  CANONICALIZE_ALIASES,
+  CONSTHOIST,
+  CONSTMERGE,
+  CONSTPROP,
+  CORO_CLEANUP,
+  CORO_EARLY,
+  CORO_ELIDE,
+  CORO_SPLIT,
+  CORRELATED_PROPAGATION,
+  CROSS_DSO_CFI,
+  DEADARGELIM,
+  DCE,
+  DIE,
+  DSE,
+  REG2MEM,
+  DIV_REM_PAIRS,
+  EARLY_CSE_MEMSSA,
+  EARLY_CSE,
+  ELIM_AVAIL_EXTERN,
+  EE_INSTRUMENT,
+  FLATTENCFG,
+  FLOAT2INT,
+  FORCEATTRS,
+  INLINE,
+  INSERT_GCOV_PROFILING,
+  GVN_HOIST,
+  GVN,
+  GLOBALDCE,
+  GLOBALOPT,
+  GLOBALSPLIT,
+  GUARD_WIDENING,
+  HOTCOLDSPLIT,
+  IPCONSTPROP,
+  IPSCCP,
+  INDVARS,
+  IRCE,
+  INFER_ADDRESS_SPACES,
+  INFERATTRS,
+  INJECT_TLI_MAPPINGS,
+  INSTSIMPLIFY,
+  INSTCOMBINE,
+  INSTNAMER,
+  JUMP_THREADING,
+  LCSSA,
+  LICM,
+  LIBCALLS_SHRINKWRAP,
+  LOAD_STORE_VECTORIZER,
+  LOOP_DATA_PREFETCH,
+  LOOP_DELETION,
+  LOOP_DISTRIBUTE,
+  LOOP_FUSION,
+  LOOP_GUARD_WIDENING,
+  LOOP_IDIOM,
+  LOOP_INSTSIMPLIFY,
+  LOOP_INTERCHANGE,
+  LOOP_LOAD_ELIM,
+  LOOP_PREDICATION,
+  LOOP_REROLL,
+  LOOP_ROTATE,
+  LOOP_SIMPLIFYCFG,
+  LOOP_SIMPLIFY,
+  LOOP_SINK,
+  LOOP_REDUCE,
+  LOOP_UNROLL_AND_JAM,
+  LOOP_UNROLL,
+  LOOP_UNSWITCH,
+  LOOP_VECTORIZE,
+  LOOP_VERSIONING_LICM,
+  LOOP_VERSIONING,
+  LOWERATOMIC,
+  LOWER_CONSTANT_INTRINSICS,
+  LOWER_EXPECT,
+  LOWER_GUARD_INTRINSIC,
+  LOWERINVOKE,
+  LOWER_MATRIX_INTRINSICS,
+  LOWERSWITCH,
+  LOWER_WIDENABLE_CONDITION,
+  MEMCPYOPT,
+  MERGEFUNC,
+  MERGEICMPS,
+  MLDST_MOTION,
+  SANCOV,
+  NAME_ANON_GLOBALS,
+  NARY_REASSOCIATE,
+  NEWGVN,
+  PGO_MEMOP_OPT,
+  PARTIAL_INLINER,
+  PARTIALLY_INLINE_LIBCALLS,
+  POST_INLINE_EE_INSTRUMENT,
+  FUNCTIONATTRS,
+  MEM2REG,
+  PRUNE_EH,
+  REASSOCIATE,
+  REDUNDANT_DBG_INST_ELIM,
+  RPO_FUNCTIONATTRS,
+  REWRITE_STATEPOINTS_FOR_GC,
+  SCCP,
+  SLP_VECTORIZER,
+  SROA,
+  SCALARIZER,
+  SEPARATE_CONST_OFFSET_FROM_GEP,
+  SIMPLE_LOOP_UNSWITCH,
+  SINK,
+  SPECULATIVE_EXECUTION,
+  SLSR,
+  STRIP_DEAD_PROTOTYPES,
+  STRIP_DEBUG_DECLARE,
+  STRIP_NONDEBUG,
+  STRIP,
+  TAILCALLELIM,
+  MERGERETURN,
+};