[lld-macho] Initial groundwork for -bitcode_bundle

This diff creates an empty XAR file and copies it into
`__LLVM,__bundle`. Follow-up work will actually populate the contents of
that XAR.

Reviewed By: #lld-macho, gkm

Differential Revision: https://reviews.llvm.org/D100650

Author: int3

lld/MachO/Config.h

@@ -88,6 +88,7 @@ struct Configuration {
   bool saveTemps = false;
   bool adhocCodesign = false;
   bool emitFunctionStarts = false;
+  bool emitBitcodeBundle = false;
   bool timeTraceEnabled = false;
   uint32_t headerPad;
   uint32_t dylibCompatibilityVersion = 0;

lld/MachO/Driver.cpp

@@ -31,6 +31,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/BinaryFormat/Magic.h"
+#include "llvm/Config/config.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Option/ArgList.h"
@@ -958,6 +959,12 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
   config->demangle = args.hasArg(OPT_demangle);
   config->implicitDylibs = !args.hasArg(OPT_no_implicit_dylibs);
   config->emitFunctionStarts = !args.hasArg(OPT_no_function_starts);
+  config->emitBitcodeBundle = args.hasArg(OPT_bitcode_bundle);
+
+#ifndef HAVE_LIBXAR
+  if (config->emitBitcodeBundle)
+    error("-bitcode_bundle unsupported because LLD wasn't built with libxar");
+#endif
 
   if (const Arg *arg = args.getLastArg(OPT_install_name)) {
     if (config->outputType != MH_DYLIB)

lld/MachO/InputSection.h

@@ -101,6 +101,7 @@ constexpr const char stubs[] = "__stubs";
 constexpr const char stubHelper[] = "__stub_helper";
 constexpr const char laSymbolPtr[] = "__la_symbol_ptr";
 constexpr const char data[] = "__data";
+constexpr const char bitcodeBundle[] = "__bundle";
 
 } // namespace section_names
 

lld/MachO/Options.td

@@ -545,7 +545,6 @@ def grp_bitcode : OptionGroup<"bitcode">, HelpText<"BITCODE BUILD FLOW">;
 
 def bitcode_bundle : Flag<["-"], "bitcode_bundle">,
     HelpText<"Generate an embedded bitcode bundle in the __LLVM,__bundle section of the output">,
-    Flags<[HelpHidden]>,
     Group<grp_bitcode>;
 def bitcode_hide_symbols : Flag<["-"], "bitcode_hide_symbols">,
     HelpText<"With -bitcode_bundle, hide all non-exported symbols from output bitcode bundle.">,

lld/MachO/OutputSection.h

@@ -47,7 +47,6 @@ class OutputSection {
   // Unneeded sections are omitted entirely (header and body).
   virtual bool isNeeded() const { return true; }
 
-  // Specifically finalizes addresses and section size, not content.
   virtual void finalize() {
     // TODO investigate refactoring synthetic section finalization logic into
     // overrides of this function.

lld/MachO/OutputSegment.h

@@ -24,6 +24,7 @@ constexpr const char linkEdit[] = "__LINKEDIT";
 constexpr const char dataConst[] = "__DATA_CONST";
 constexpr const char ld[] = "__LD"; // output only with -r
 constexpr const char dwarf[] = "__DWARF";
+constexpr const char llvm[] = "__LLVM";
 
 } // namespace segment_names
 

lld/MachO/SyntheticSections.cpp

@@ -20,6 +20,7 @@
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Config/config.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/LEB128.h"
@@ -30,6 +31,11 @@
 #include <sys/mman.h>
 #endif
 
+#ifdef HAVE_LIBXAR
+#include <fcntl.h>
+#include <xar/xar.h>
+#endif
+
 using namespace llvm;
 using namespace llvm::MachO;
 using namespace llvm::support;
@@ -1033,6 +1039,58 @@ void CodeSignatureSection::writeTo(uint8_t *buf) const {
   memset(id + fileName.size(), 0, fileNamePad);
 }
 
+BitcodeBundleSection::BitcodeBundleSection()
+    : SyntheticSection(segment_names::llvm, section_names::bitcodeBundle) {}
+
+class ErrorCodeWrapper {
+public:
+  ErrorCodeWrapper(std::error_code ec) : errorCode(ec.value()) {}
+  ErrorCodeWrapper(int ec) : errorCode(ec) {}
+  operator int() const { return errorCode; }
+
+private:
+  int errorCode;
+};
+
+#define CHECK_EC(exp)                                                          \
+  do {                                                                         \
+    ErrorCodeWrapper ec(exp);                                                  \
+    if (ec)                                                                    \
+      fatal(Twine("operation failed with error code ") + Twine(ec) + #exp);    \
+  } while (0);
+
+void BitcodeBundleSection::finalize() {
+#ifdef HAVE_LIBXAR
+  using namespace llvm::sys::fs;
+  CHECK_EC(createTemporaryFile("bitcode-bundle", "xar", xarPath));
+
+  xar_t xar(xar_open(xarPath.data(), O_RDWR));
+  if (!xar)
+    fatal("failed to open XAR temporary file at " + xarPath);
+  CHECK_EC(xar_opt_set(xar, XAR_OPT_COMPRESSION, XAR_OPT_VAL_NONE));
+  // FIXME: add more data to XAR
+  CHECK_EC(xar_close(xar));
+
+  file_size(xarPath, xarSize);
+#endif // defined(HAVE_LIBXAR)
+}
+
+void BitcodeBundleSection::writeTo(uint8_t *buf) const {
+  using namespace llvm::sys::fs;
+  file_t handle =
+      CHECK(openNativeFile(xarPath, CD_OpenExisting, FA_Read, OF_None),
+            "failed to open XAR file");
+  std::error_code ec;
+  mapped_file_region xarMap(handle, mapped_file_region::mapmode::readonly,
+                            xarSize, 0, ec);
+  if (ec)
+    fatal("failed to map XAR file");
+  memcpy(buf, xarMap.const_data(), xarSize);
+
+  closeFile(handle);
+  remove(xarPath);
+}
+
 void macho::createSyntheticSymbols() {
   auto addHeaderSymbol = [](const char *name) {
     symtab->addSynthetic(name, in.header->isec, 0,

lld/MachO/SyntheticSections.h

@@ -488,6 +488,18 @@ class CodeSignatureSection : public LinkEditSection {
   void writeHashes(uint8_t *buf) const;
 };
 
+class BitcodeBundleSection : public SyntheticSection {
+public:
+  BitcodeBundleSection();
+  uint64_t getSize() const override { return xarSize; }
+  void finalize() override;
+  void writeTo(uint8_t *buf) const override;
+
+private:
+  llvm::SmallString<261> xarPath;
+  uint64_t xarSize;
+};
+
 static_assert((CodeSignatureSection::blobHeadersSize % 8) == 0, "");
 static_assert((CodeSignatureSection::fixedHeadersSize % 8) == 0, "");
 

lld/MachO/Writer.cpp

@@ -803,6 +803,8 @@ template <class LP> void Writer::createOutputSections() {
     codeSignatureSection = make<CodeSignatureSection>();
   if (config->emitFunctionStarts)
     functionStartsSection = make<FunctionStartsSection>();
+  if (config->emitBitcodeBundle)
+    make<BitcodeBundleSection>();
 
   switch (config->outputType) {
   case MH_EXECUTE:

lld/test/MachO/bitcode-bundle.ll

@@ -0,0 +1,42 @@
+; REQUIRES: x86, xar
+; RUN: rm -rf %t; split-file %s %t
+; RUN: opt -module-summary %t/test.ll -o %t/test.o
+; RUN: opt -module-summary %t/foo.ll -o %t/foo.o
+; RUN: %lld -lSystem -bitcode_bundle %t/test.o %t/foo.o -o %t/test
+; RUN: llvm-objdump --macho --section=__LLVM,__bundle %t/test | FileCheck %s
+
+; CHECK:      Contents of (__LLVM,__bundle) section
+; CHECK-NEXT: For (__LLVM,__bundle) section: xar header
+; CHECK-NEXT:                   magic XAR_HEADER_MAGIC
+; CHECK-NEXT:                    size 28
+; CHECK-NEXT:                 version 1
+; CHECK-NEXT:   toc_length_compressed
+; CHECK-NEXT: toc_length_uncompressed
+; CHECK-NEXT:               cksum_alg XAR_CKSUM_SHA1
+; CHECK-NEXT: For (__LLVM,__bundle) section: xar table of contents:
+; CHECK-NEXT: <?xml version="1.0" encoding="UTF-8"?>
+; CHECK-NEXT: <xar>
+; CHECK-NEXT:  <toc>
+; CHECK-NEXT:   <checksum style="sha1">
+; CHECK-NEXT:    <size>20</size>
+; CHECK-NEXT:    <offset>0</offset>
+; CHECK-NEXT:   </checksum>
+; CHECK-NEXT:   <creation-time>{{.*}}</creation-time>
+; CHECK-NEXT:  </toc>
+; CHECK-NEXT: </xar>
+
+;--- foo.ll
+target triple = "x86_64-apple-darwin"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo() {
+  ret void
+}
+
+;--- test.ll
+target triple = "x86_64-apple-darwin"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @main() {
+  ret void
+}

lld/test/MachO/invalid/no-libxar.ll

@@ -0,0 +1,12 @@
+; REQUIRES: x86
+; UNSUPPORTED: xar
+; RUN: opt -module-summary %s -o %t.o
+; RUN: not %lld -lSystem -bitcode_bundle %t.o -o /dev/null 2>&1 | FileCheck %s
+; CHECK: error: -bitcode_bundle unsupported because LLD wasn't built with libxar
+
+target triple = "x86_64-apple-darwin"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @main() {
+  ret void
+}

lld/test/lit.cfg.py

@@ -90,6 +90,9 @@
         config.have_libxml2):
     config.available_features.add('manifest_tool')
 
+if config.have_libxar:
+    config.available_features.add('xar')
+
 if config.have_libxml2:
     config.available_features.add('libxml2')
 

lld/test/lit.site.cfg.py.in

@@ -15,6 +15,7 @@ config.lld_tools_dir = "@LLD_TOOLS_DIR@"
 config.target_triple = "@TARGET_TRIPLE@"
 config.python_executable = "@Python3_EXECUTABLE@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_libxar = @HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.sizeof_void_p = @CMAKE_SIZEOF_VOID_P@
 config.ld_lld_default_mingw = @LLD_DEFAULT_LD_LLD_IS_MINGW@

lld/tools/lld/CMakeLists.txt

@@ -20,6 +20,10 @@ target_link_libraries(lld
   lldWasm
   )
 
+if(HAVE_LIBXAR)
+  target_link_libraries(lld PRIVATE ${XAR_LIB})
+endif()
+
 install(TARGETS lld
   RUNTIME DESTINATION bin)