Merge pull request #567 from ChrisCummins/feature/cmake-tweaks

[cmake] Handful of small tweaks

WORKSPACE

@@ -40,25 +40,6 @@ http_archive(
     urls = ["https://github.com/google/glog/archive/v0.4.0.tar.gz"],
 )
 
-# C++ subprocess management. https://github.com/arun11299/cpp-subprocess
-
-http_archive(
-    name = "subprocess",
-    build_file_content = """
-cc_library(
-    name = "subprocess",
-    hdrs = ["subprocess.hpp"],
-    include_prefix = "subprocess",
-    visibility = ["//visibility:public"],
-)
-""",
-    sha256 = "886df0a814a7bb7a3fdeead22f75400abd8d3235b81d05817bc8c1125eeebb8f",
-    strip_prefix = "cpp-subprocess-2.0",
-    urls = [
-        "https://github.com/arun11299/cpp-subprocess/archive/v2.0.tar.gz",
-    ],
-)
-
 # === LLVM ===
 
 http_archive(

build_tools/cmake/FindCsmith.cmake

@@ -49,7 +49,7 @@ if (Csmith_EXECUTABLE)
 
   get_filename_component(Csmith_ROOT_DIR "${Csmith_EXECUTABLE}" DIRECTORY)
   get_filename_component(Csmith_ROOT_DIR "${Csmith_ROOT_DIR}/.." ABSOLUTE)
-  set(Csmith_ROOT_DIR "${Csmith_ROOT_DIR}" CACHE string "Path to the root installation directory of Csmith.")
+  set(Csmith_ROOT_DIR "${Csmith_ROOT_DIR}" CACHE STRING "Path to the root installation directory of Csmith.")
 endif()
 find_path(Csmith_INCLUDE_DIRS csmith.h PATH_SUFFIXES csmith csmith-2.3.0)
 find_library(Csmith_LIBRARIES csmith)

compiler_gym/envs/llvm/service/Benchmark.h

@@ -14,9 +14,9 @@
 #include "compiler_gym/envs/llvm/service/Cost.h"
 #include "compiler_gym/service/proto/compiler_gym_service.pb.h"
 #include "compiler_gym/util/Subprocess.h"
-#include "include/llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/ModuleSummaryIndex.h"
 
 namespace compiler_gym::llvm_service {
 

compiler_gym/envs/llvm/service/CMakeLists.txt

@@ -7,24 +7,13 @@ cg_add_all_subdirs()
 
 set(_DEPS "compiler_gym-llvm-service")
 if(DARWIN)
-  list(APPEND _DEPS ::libLLVMPolly)
-  #TODO(boian): figure out what is this target.
   #list(APPEND _DEPS "@llvm//:darwin")
 endif()
 cg_filegroup(
   NAME "service"
   DEPENDS ${_DEPS}
 )
 
-cg_genrule(
-  NAME libLLVMPolly
-  OUTS "libLLVMPolly.so"
-  COMMAND
-    "cp $<TARGET_FILE:LLVMPolly> $@"
-  ABS_DEPENDS
-    LLVMPolly
-)
-
 cg_cc_binary(
   NAME
     compiler_gym-llvm-service
@@ -210,7 +199,6 @@ cg_cc_library(
     ProGraML::graph::format::node_link_graph
     ProGraML::ir::llvm::llvm-10
     ProGraML::proto::programl_cc
-    Subprocess::libsubprocess
     CpuInfo::cpuinfo
     Clog::libclog
   INCLUDES

compiler_gym/envs/llvm/service/passes/BUILD

@@ -2,7 +2,7 @@
 # and converting them to an action space for reinforcement learning.
 #
 # These scripts are used to programatically generate C++ headers and sources
-# that are then used to compiler the C++ LLVM compiler service.
+# that are then used to compile the LLVM compiler service.
 load("@rules_python//python:defs.bzl", "py_binary", "py_library")
 
 genrule(

compiler_gym/envs/llvm/service/passes/config.py

@@ -5,13 +5,12 @@
 """Configuration for building an action space from a list of LLVM passes."""
 from common import Pass
 
-# A set of headers that must be included.
-EXTRA_LLVM_HEADERS = {
-    "include/llvm/LinkAllPasses.h",
-    "include/llvm/Transforms/Coroutines.h",
-    "include/llvm/Transforms/IPO.h",
-    "include/llvm/Transforms/Scalar.h",
-    "include/llvm/Transforms/Utils.h",
+# A set of headers that must be included to use the generated pass list.
+LLVM_ACTION_INCLUDES = {
+    "llvm/LinkAllPasses.h",
+    # A handle of coroutine utility passes are not pulled in by the
+    # LinkAllPasses.h header.
+    "llvm/Transforms/Coroutines.h",
 }
 
 # A mapping from the name of a pass as defined in a INITIALIZE_PASS(name, ...)

compiler_gym/envs/llvm/service/passes/extract_passes_from_llvm_source_tree.py

@@ -18,7 +18,7 @@
 This implements a not-very-good parser for the INITIALIZE_PASS() family of
 macros, which are used in the LLVM sources to declare a pass using it's name,
 flag, and docstring. Parsing known macros like this is fragile and likely to
-break as the LLVM sources evolve. Currently only tested on LLVM 10.0.
+break as the LLVM sources evolve. Currently only tested on LLVM 10.0 and 13.0.1.
 
 A more robust solution would be to parse the C++ sources and extract all classes
 which inherit from ModulePass etc.
@@ -28,6 +28,7 @@
 import logging
 import os
 import re
+import shlex
 import subprocess
 import sys
 from pathlib import Path
@@ -56,6 +57,24 @@ def parse_initialize_pass(
     source_path: Path, header: Optional[str], input_source: str, defines: Dict[str, str]
 ) -> Iterable[Pass]:
     """A shitty parser for INITIALIZE_PASS() macro invocations.."""
+    # ****************************************************
+    #           __        _
+    #         _/  \    _(\(o
+    #        /     \  /  _  ^^^o
+    #       /   !   \/  ! '!!!v'
+    #      !  !  \ _' ( \____
+    #      ! . \ _!\   \===^\)
+    #       \ \_!  / __!
+    #        \!   /    \
+    #  (\_      _/   _\ )
+    #   \ ^^--^^ __-^ /(__
+    #    ^^----^^    "^--v'
+    #
+    #           HERE BE DRAGONS!
+    #
+    # TODO(cummins): Take this code out back and shoot it.
+    # ****************************************************
+
     # Squish down to a single line.
     source = re.sub(r"\n\s*", " ", input_source, re.MULTILINE)
     # Contract multi-spaces to single space.
@@ -74,6 +93,7 @@ def parse_initialize_pass(
     start = 0
     in_quotes = False
     in_comment = False
+    substr = ""
     for i in range(len(source)):
         if (
             not in_comment
@@ -82,6 +102,7 @@ def parse_initialize_pass(
             and source[i + 1] == "*"
         ):
             in_comment = True
+            substr += source[start:i].strip()
         if (
             in_comment
             and source[i] == "*"
@@ -93,9 +114,11 @@ def parse_initialize_pass(
         if source[i] == '"':
             in_quotes = not in_quotes
         if not in_quotes and source[i] == ",":
-            components.append(source[start:i].strip())
+            substr += source[start:i].strip()
+            components.append(substr)
+            substr = ""
             start = i + 2
-    components.append(source[start:].strip())
+    components.append(substr + source[start:].strip())
     if len(components) != 5:
         raise ParseError(
             f"Expected 5 components, found {len(components)}", source, components
@@ -108,14 +131,18 @@ def parse_initialize_pass(
     if not name:
         raise ParseError(f"Empty name: `{name}`", source, components)
 
-    while arg in defines:
-        arg = defines[arg]
+    # Dodgy code to combine adjacent strings with macro expansion. For example,
+    # 'DEBUG_TYPE "-foo"'.
+    arg_components = shlex.split(arg)
+    for i, _ in enumerate(arg_components):
+        while arg_components[i] in defines:
+            arg_components[i] = defines[arg_components[i]]
+    arg = " ".join(arg_components)
+    if arg[0] == '"' and arg[-1] == '"':
+        arg = arg[1:-1]
+
     while name in defines:
         name = defines[name]
-
-    if not (arg[0] == '"' and arg[-1] == '"'):
-        raise ParseError(f"Could not interpret arg `{arg}`", source, components)
-    arg = arg[1:-1]
     if not (name[0] == '"' and name[-1] == '"'):
         raise ParseError(f"Could not interpret name `{name}`", source, components)
     name = name[1:-1]

compiler_gym/envs/llvm/service/passes/make_action_space_genfiles.py

@@ -18,8 +18,8 @@
     Example:
 
         #pragma once
-        #include "include/llvm/LinkAllPasses.h"
-        #include "include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
+        #include "llvm/LinkAllPasses.h"
+        #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
         ...
 
     This file includes the set of LLVM headers that must be included to use the
@@ -82,15 +82,19 @@
 from pathlib import Path
 
 from common import Pass
-from config import EXTRA_LLVM_HEADERS
+from config import LLVM_ACTION_INCLUDES
 
 logger = logging.getLogger(__name__)
 
 
 def process_pass(pass_, headers, enum_f, switch_f):
     """Extract and process transform passes in header."""
     if pass_.header:
-        headers.add(pass_.header)
+        # Strip a leading "include/" from the header path.
+        header = pass_.header
+        if header.startswith("include/"):
+            header = header[len("include/") :]
+        headers.add(header)
 
     # The name of the pass in UPPER_PASCAL_CASE.
     enum_name = pass_.flag[1:].replace("-", "_").upper()
@@ -103,7 +107,7 @@ def process_pass(pass_, headers, enum_f, switch_f):
 def make_action_sources(pass_iterator, outpath: Path):
     """Generate the enum and switch content."""
     total_passes = 0
-    headers = set(EXTRA_LLVM_HEADERS)
+    headers = set(LLVM_ACTION_INCLUDES)
 
     passes = sorted(list(pass_iterator), key=lambda p: p.name)
 

compiler_gym/third_party/autophase/CMakeLists.txt

@@ -52,12 +52,3 @@ cg_cc_binary(
   DEFINES
     ${LLVM_DEFINITIONS}
 )
-
-cg_genrule(
-  NAME libLLVMPolly
-  OUTS "libLLVMPolly.so"
-  COMMAND
-    "cp $<TARGET_FILE:LLVMPolly> $@"
-  ABS_DEPENDS
-    LLVMPolly
-)

compiler_gym/third_party/llvm/CMakeLists.txt

@@ -33,15 +33,6 @@ cg_cc_binary(
     ${LLVM_DEFINITIONS}
 )
 
-cg_genrule(
-  NAME libLLVMPolly
-  OUTS "libLLVMPolly.so"
-  COMMAND
-    "cp $<TARGET_FILE:LLVMPolly> $@"
-  ABS_DEPENDS
-    LLVMPolly
-)
-
 cg_py_library(
   NAME
     instcount