Merge pull request ldc-developers#4430 from JohanEngelen/sema_plugin

Add support for Semantic Analysis plugins

Author: kinke

.github/actions/3-build-cross/action.yml

@@ -145,4 +145,4 @@ runs:
           ${{ inputs.cmake_flags }}
           ${{ inputs.with_pgo == 'true' && '-DDFLAGS_LDC=-fprofile-use=../pgo-ldc/merged.profdata' || '' }}
           ${{ env.CROSS_CMAKE_FLAGS }}
-        build_targets: ldc2 ldmd2 ldc-build-runtime ldc-profdata ldc-prune-cache timetrace2txt
+        build_targets: ldc2 ldmd2 ldc-build-runtime ldc-build-plugin ldc-profdata ldc-prune-cache timetrace2txt

.github/actions/5-install/action.yml

@@ -22,6 +22,7 @@ runs:
         else
           mkdir -p install/bin
           cp build-cross/bin/{ldc2,ldmd2,ldc-build-runtime,ldc-profdata,ldc-prune-cache,timetrace2txt} install/bin/
+          cp build-cross/bin/ldc-build-plugin install/bin/ || true
           cp -R build-cross-libs/lib install/
           cp build-cross/lib/{libldc_rt.*,libLTO-ldc.dylib,LLVMgold-ldc.so} install/lib/ || true
           mkdir install/etc

CHANGELOG.md

@@ -1,5 +1,34 @@
 # LDC master
 
+#### Big news
+- Frontend, druntime and Phobos are at version [2.103.1](https://dlang.org/changelog/2.103.0.html), incl. new command-line option `-verror-supplements`. (#4345)
+- The `--plugin` commandline option now also accepts semantic analysis plugins. Semantic analysis plugins are recognized by exporting the symbol: `extern(C) void runSemanticAnalysis(Module m)`. The plugin's `runSemanticAnalysis` function is called for each module, after all other semantic analysis steps (also after DCompute SemA), just before object codegen. (#4430)
+- New tool `ldc-build-plugin` that helps compiling user plugins. It downloads the correct LDC source version (if its not already available), and calls LDC with the correct commandline flags to build a plugin. (#4430)
+- New commandline option `-femit-local-var-lifetime` that enables variable lifetime (scope) annotation to LLVM IR codegen. Lifetime annotation enables stack memory reuse for local variables with non-overlapping scope. (#4395)
+- C files are now automatically preprocessed using the external C compiler (configurable via `-gcc` or the `CC` environment variable, and `-Xcc` for extra flags). Extra preprocessor flags (e.g., include dirs and manual defines) can be added via new command-line option `-P`. (#4417)
+  - Windows: If `clang-cl.exe` is on `PATH`, it is preferred over Microsoft's `cl.exe` by default (e.g., to avoid printing the C source file name to stderr during preprocessing).
+- Less pedantic checks for conflicting C(++) function declarations when compiling multiple modules to a single object file ('Error: Function type does not match previously declared function with the same mangled name'). The error now only appears if an object file actually references multiple conflicting functions. (#4420)
+- New command-line option `--fcf-protection`, which enables Intel's Control-Flow Enforcement Technology (CET). (#4437)
+
+#### Platform support
+
+#### Bug fixes
+- Handle potential lambda mangle collisions across separately compiled object files (and the linker then silently picking an arbitrary implementation). Lambdas (and their nested global variables) are now internal to each referencing object file (`static` linkage in C). (#4415)
+
+# LDC 1.32.2 (2023-05-12)
+
+#### Big news
+- New command-line option `--fwarn-stack-size=<threshold>` with LLVM 13+. (#4378)
+- New command-line option `--fsplit-stack` for incremental stack allocations, see https://llvm.org/docs/SegmentedStacks.html. (#4379)
+  - New UDA `ldc.attributes.noSplitStack` disables it on a per-function basis. (#4382)
+- New command-line option `--indent` for the `timetrace2txt` tool. (#4391)
+
+#### Bug fixes
+- Fix potentially huge compile slowdowns with `-g` and LLVM 15+. (#4354, #4393)
+- Treat *all* LLVM warnings as regular warnings (e.g., errors with `-w`). Requires LLVM 13+. (#4384)
+
+# LDC 1.32.1 (2023-04-17)
+
 #### Big news
 - The prebuilt Linux packages are now generated on a Ubuntu 20.04 box, so the min required `glibc` version has been raised from 2.26 to 2.31. (#4367)
 

CMakeLists.txt

@@ -246,17 +246,6 @@ if(CMAKE_COMPILER_IS_GNUCXX OR (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang"))
     endif()
 endif()
 
-if(NOT WIN32 AND NOT CYGWIN)
-    # Unify symbol visibility with LLVM to silence linker warning "direct access in function X to global
-    # weak symbol Y means the weak symbol cannot be overridden at runtime. This was likely caused by
-    # different translation units being compiled with different visibility settings."
-    # See LLVM's cmake/modules/HandleLLVMOptions.cmake.
-    check_cxx_compiler_flag("-fvisibility-inlines-hidden" SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG)
-    if (${SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG})
-        append("-fvisibility-inlines-hidden" LDC_CXXFLAGS)
-    endif()
-endif()
-
 if(MSVC)
     # Remove flags here, for exceptions and RTTI.
     # CL.EXE complains to override flags like "/GR /GR-".
@@ -632,6 +621,22 @@ if(LDC_WITH_LLD)
     endif()
 endif()
 
+if(NOT DEFINED LDC_LINK_MANUALLY)
+    if(MSVC)
+        # Use the D host compiler for linking D executables.
+        set(LDC_LINK_MANUALLY OFF)
+    else()
+        # On Unix-like systems, default to having CMake link the D executables via the C++ compiler.
+        # (Using the D compiler needs -Xcc and -gcc support, see file BuildDExecutable.cmake.)
+        set(LDC_LINK_MANUALLY ON)
+    endif()
+endif()
+if(LDC_LINK_MANUALLY AND NOT DEFINED D_LINKER_ARGS)
+    include(ExtractDMDSystemLinker)
+    message(STATUS "Host D compiler linker program: ${D_LINKER_COMMAND}")
+    message(STATUS "Host D compiler linker flags: ${D_LINKER_ARGS}")
+endif()
+
 # Plugin support
 if(UNIX)
     set(LDC_ENABLE_PLUGINS_DEFAULT ON)
@@ -643,7 +648,16 @@ if(LDC_ENABLE_PLUGINS)
     add_definitions(-DLDC_ENABLE_PLUGINS)
 
     if(APPLE)
-        # No extra link flags needed.
+        # Need to disable dead_strip with LDC host compilers.
+        if("${D_COMPILER_ID}" STREQUAL "LDMD")
+            if(LDC_LINK_MANUALLY)
+                # suboptimal - applies to all D executables (incl. ldmd2, ldc-build-runtime, ldc-prune-cache)
+                list(REMOVE_ITEM D_LINKER_ARGS "-Wl,-dead_strip")
+            else()
+                # just for ldc2 (and ldc2-unittest)
+                append("-disable-linker-strip-dead" DFLAGS_LDC)
+            endif()
+        endif()
     elseif(UNIX)
         # For plugin support, we need to link with --export-dynamic on Unix.
         # Make sure the linker supports --export-dynamic (on Solaris it is not supported and also not needed).
@@ -654,26 +668,26 @@ if(LDC_ENABLE_PLUGINS)
 
         if(LINKER_ACCEPTS_EXPORT_DYNAMIC_FLAG)
             set(LDC_LINKERFLAG_LIST "${LDC_LINKERFLAG_LIST};-Wl,--export-dynamic")
+        else()
+            message(WARNING "Linker does not accept --export-dynamic, user plugins may give missing symbol errors upon load")
         endif()
     endif()
 endif()
 message(STATUS "-- Building LDC with plugin support (LDC_ENABLE_PLUGINS): ${LDC_ENABLE_PLUGINS}")
+message(STATUS "-- Linking LDC with flags: ${ALTERNATIVE_MALLOC_O};${LDC_LINKERFLAG_LIST}")
 
-if(NOT DEFINED LDC_LINK_MANUALLY)
-    if(MSVC)
-        # Use the D host compiler for linking D executables.
-        set(LDC_LINK_MANUALLY OFF)
-    else()
-        # On Unix-like systems, default to having CMake link the D executables via the C++ compiler.
-        # (Using the D compiler needs -Xcc and -gcc support, see file BuildDExecutable.cmake.)
-        set(LDC_LINK_MANUALLY ON)
+if(NOT WIN32 AND NOT CYGWIN)
+    # Unify symbol visibility with LLVM to silence linker warning "direct access in function X to global
+    # weak symbol Y means the weak symbol cannot be overridden at runtime. This was likely caused by
+    # different translation units being compiled with different visibility settings."
+    # See LLVM's cmake/modules/HandleLLVMOptions.cmake.
+    check_cxx_compiler_flag("-fvisibility-inlines-hidden" SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG)
+    if (LDC_ENABLE_PLUGINS AND NOT APPLE)
+        # For plugins, we shouldn't apply this flag because it hides the inline methods of e.g. Visitor. On macOS it's OK to add.
+    elseif (${SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG})
+        append("-fvisibility-inlines-hidden" LDC_CXXFLAGS)
     endif()
 endif()
-if(LDC_LINK_MANUALLY AND NOT DEFINED D_LINKER_ARGS)
-    include(ExtractDMDSystemLinker)
-    message(STATUS "Host D compiler linker program: ${D_LINKER_COMMAND}")
-    message(STATUS "Host D compiler linker flags: ${D_LINKER_ARGS}")
-endif()
 
 build_d_executable(
     "${LDC_EXE}"

cmake/Modules/BuildDExecutable.cmake

@@ -25,6 +25,7 @@ endmacro()
 # - DFLAGS_BASE
 # - LDC_LINK_MANUALLY
 # - D_LINKER_ARGS
+# - LDC_ENABLE_PLUGINS
 function(build_d_executable target_name output_exe d_src_files compiler_args linker_args extra_compile_deps link_deps compile_separately)
     set(dflags "${D_COMPILER_FLAGS} ${DFLAGS_BASE} ${compiler_args}")
     if(UNIX)
@@ -40,7 +41,9 @@ function(build_d_executable target_name output_exe d_src_files compiler_args lin
         # Compile all D modules to a single object.
         set(object_file ${PROJECT_BINARY_DIR}/obj/${target_name}${CMAKE_CXX_OUTPUT_EXTENSION})
         # Default to -linkonce-templates with LDMD host compiler, to speed-up optimization.
-        if("${D_COMPILER_ID}" STREQUAL "LDMD")
+        if("${target_name}" STREQUAL "ldc2" AND LDC_ENABLE_PLUGINS)
+            # For plugin support we need ldc2's symbols to be global, don't use -linkonce-templates.
+        elseif("${D_COMPILER_ID}" STREQUAL "LDMD")
             set(dflags -linkonce-templates ${dflags})
         endif()
         add_custom_command(

driver/plugins.cpp

@@ -9,6 +9,9 @@
 //
 // Implements functionality related to plugins (`-plugin=...`).
 //
+// Note: plugins can be LLVM-plugins (to be registered with the pass manager)
+// or dlang-plugins for semantic analysis.
+//
 //===----------------------------------------------------------------------===//
 
 #include "driver/plugins.h"
@@ -17,6 +20,8 @@
 
 #include "dmd/errors.h"
 #include "dmd/globals.h"
+#include "dmd/module.h"
+#include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/DynamicLibrary.h"
 
@@ -27,23 +32,120 @@ cl::list<std::string>
     pluginFiles("plugin", cl::CommaSeparated, cl::desc("Plugins to load."),
                 cl::value_desc("dynamic_library.so,lib2.so"));
 
+struct SemaPlugin {
+  llvm::sys::DynamicLibrary library;
+  void (*runSemanticAnalysis)(Module *);
+
+  SemaPlugin(const llvm::sys::DynamicLibrary &library,
+             void (*runSemanticAnalysis)(Module *))
+      : library(library), runSemanticAnalysis(runSemanticAnalysis) {}
+};
+
+llvm::SmallVector<SemaPlugin, 1> sema_plugins;
+
+} // anonymous namespace
+
+// Tries to load plugin as SemanticAnalysis. Returns true on 'success', i.e. no
+// further attempts needed.
+bool loadSemanticAnalysisPlugin(const std::string &filename) {
+  std::string errorString;
+  auto library = llvm::sys::DynamicLibrary::getPermanentLibrary(
+      filename.c_str(), &errorString);
+  if (!library.isValid()) {
+    error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
+          errorString.c_str());
+    return true; // No success, but no need to try loading again as LLVM plugin.
+  }
+
+  // SemanticAnalysis plugins need to export the `runSemanticAnalysis` function.
+  void *runSemanticAnalysisFnPtr =
+      library.getAddressOfSymbol("runSemanticAnalysis");
+
+  // If the symbol isn't found, this is probably an LLVM plugin.
+  if (!runSemanticAnalysisFnPtr)
+    return false;
+
+  sema_plugins.emplace_back(
+      library, reinterpret_cast<void (*)(Module *)>(runSemanticAnalysisFnPtr));
+  return true;
+}
+
+/// Loads plugin for the legacy pass manager. The static constructor of
+/// the plugin should take care of the plugins registering themself with the
+/// rest of LDC/LLVM.
+void loadLLVMPluginLegacyPM(const std::string &filename) {
+  std::string errorString;
+  if (llvm::sys::DynamicLibrary::LoadLibraryPermanently(filename.c_str(),
+                                                        &errorString)) {
+    error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
+          errorString.c_str());
+  }
+}
+
+#if LDC_LLVM_VER >= 1400
+
+namespace {
+llvm::SmallVector<llvm::PassPlugin, 1> llvm_plugins;
+
+/// Loads plugin for the new pass manager. The plugin will need to be
+/// added explicitly when building the optimization pipeline.
+void loadLLVMPluginNewPM(const std::string &filename) {
+
+  auto plugin = llvm::PassPlugin::Load(filename);
+  if (!plugin) {
+    error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
+          llvm::toString(plugin.takeError()).c_str());
+    return;
+  }
+  llvm_plugins.emplace_back(plugin.get());
+}
+
 } // anonymous namespace
 
-/// Loads all plugins. The static constructor of each plugin should take care of
-/// the plugins registering themself with the rest of LDC/LLVM.
+#endif // LDC_LLVM_VER >= 1400
+
+void loadLLVMPlugin(const std::string &filename) {
+#if LDC_LLVM_VER >= 1400
+  if (opts::isUsingLegacyPassManager())
+    loadLLVMPluginLegacyPM(filename);
+  else
+    loadLLVMPluginNewPM(filename);
+#else
+  loadLLVMPluginLegacyPM(filename);
+#endif
+}
+
 void loadAllPlugins() {
   for (auto &filename : pluginFiles) {
-    std::string errorString;
-    if (llvm::sys::DynamicLibrary::LoadLibraryPermanently(filename.c_str(),
-                                                          &errorString)) {
-      error(Loc(), "Error loading plugin '%s': %s", filename.c_str(),
-            errorString.c_str());
-    }
+    // First attempt to load plugin as SemanticAnalysis plugin. If unsuccesfull,
+    // load as LLVM plugin.
+    auto success = loadSemanticAnalysisPlugin(filename);
+    if (!success)
+      loadLLVMPlugin(filename);
+  }
+}
+
+void registerAllPluginsWithPassBuilder(llvm::PassBuilder &PB) {
+#if LDC_LLVM_VER >= 1400
+  for (auto &plugin : llvm_plugins) {
+    plugin.registerPassBuilderCallbacks(PB);
+  }
+#endif
+}
+
+void runAllSemanticAnalysisPlugins(Module *m) {
+  for (auto &plugin : sema_plugins) {
+    assert(plugin.runSemanticAnalysis);
+    plugin.runSemanticAnalysis(m);
   }
 }
 
 #else // #if LDC_ENABLE_PLUGINS
 
+class Module;
+
 void loadAllPlugins() {}
+void registerAllPluginsWithPassBuilder(llvm::PassBuilder &) {}
+void runAllSemanticAnalysisPlugins(Module *m) {}
 
 #endif // LDC_ENABLE_PLUGINS

gen/semantic.d

@@ -15,13 +15,20 @@ import dmd.dmodule;
 
 extern(C++) void dcomputeSemanticAnalysis(Module m);
 extern(C) int hasComputeAttr(Dsymbol m);
+extern(C++) void runAllSemanticAnalysisPlugins(Module m);
 
 extern(C++) void extraLDCSpecificSemanticAnalysis(ref Modules modules)
 {
+    // First finish DCompute SemA for all modules, before calling plugins.
     foreach(m; modules[])
     {
-        if (hasComputeAttr(m))
+        if (hasComputeAttr(m)) {
             dcomputeSemanticAnalysis(m);
+        }
+    }
+
+    foreach(m; modules[])
+    {
+        runAllSemanticAnalysisPlugins(m);
     }
-    
 }

tests/CMakeLists.txt

@@ -1,6 +1,7 @@
 set( LDC2_BIN          ${PROJECT_BINARY_DIR}/bin/${LDC_EXE} )
 set( LDCPROFDATA_BIN   ${PROJECT_BINARY_DIR}/bin/ldc-profdata )
 set( LDCPRUNECACHE_BIN ${PROJECT_BINARY_DIR}/bin/${LDCPRUNECACHE_EXE} )
+set( LDCBUILDPLUGIN_BIN ${PROJECT_BINARY_DIR}/bin/${LDC_BUILD_PLUGIN_EXE} )
 set( TIMETRACE2TXT_BIN ${PROJECT_BINARY_DIR}/bin/${TIMETRACE2TXT_EXE} )
 set( LLVM_TOOLS_DIR    ${LLVM_ROOT_DIR}/bin )
 set( LDC2_BIN_DIR      ${PROJECT_BINARY_DIR}/bin )

tests/lit.site.cfg.in

@@ -17,10 +17,12 @@ OFF = False
 config.ldc2_bin            = "@LDC2_BIN@"
 config.ldcprofdata_bin     = "@LDCPROFDATA_BIN@"
 config.ldcprunecache_bin   = "@LDCPRUNECACHE_BIN@"
+config.ldcbuildplugin_bin  = "@LDCBUILDPLUGIN_BIN@"
 config.timetrace2txt_bin   = "@TIMETRACE2TXT_BIN@"
 config.ldc2_bin_dir        = "@LDC2_BIN_DIR@"
 config.ldc2_lib_dir        = "@LDC2_LIB_DIR@"
 config.ldc2_runtime_dir    = "@RUNTIME_DIR@"
+config.ldc2_source_dir     = "@PROJECT_SOURCE_DIR@"
 config.test_source_root    = "@TESTS_IR_DIR@"
 config.llvm_tools_dir      = "@LLVM_TOOLS_DIR@"
 config.llvm_version        = @LDC_LLVM_VER@
@@ -156,6 +158,7 @@ config.substitutions.append( ('%ldc', config.ldc2_bin) )
 config.substitutions.append( ('%gnu_make', config.gnu_make_bin) )
 config.substitutions.append( ('%profdata', config.ldcprofdata_bin) )
 config.substitutions.append( ('%prunecache', config.ldcprunecache_bin) )
+config.substitutions.append( ('%buildplugin', config.ldcbuildplugin_bin + " --ldcSrcDir=" + config.ldc2_source_dir ) )
 config.substitutions.append( ('%timetrace2txt', config.timetrace2txt_bin) )
 config.substitutions.append( ('%llvm-spirv', os.path.join(config.llvm_tools_dir, 'llvm-spirv')) )
 config.substitutions.append( ('%runtimedir', config.ldc2_runtime_dir ) )

tests/plugins/basic_sema_plugin.d

@@ -0,0 +1,26 @@
+// REQUIRES: Plugins
+
+// For some reason this test fails with missing symbol linking issues (or crash) with macOS on Intel x86 (but not for all CI testers...)
+// UNSUPPORTED: Darwin && host_X86
+
+// RUN: split-file %s %t --leading-lines
+// RUN: %buildplugin %t/plugin.d -of=%t/plugin%so --buildDir=%t/build
+// RUN: %ldc -wi -c -o- --plugin=%t/plugin%so %t/testcase.d 2>&1 | FileCheck %t/testcase.d
+
+//--- plugin.d
+import dmd.dmodule : Module;
+import dmd.errors;
+import dmd.location;
+
+extern(C) void runSemanticAnalysis(Module m) {
+    if (m.md) {
+        warning(m.md.loc, "It works!");
+    }
+}
+
+//--- testcase.d
+// CHECK: testcase.d([[@LINE+1]]): Warning: It works!
+module testcase;
+int testfunction(int i) {
+    return i * 2;
+}