Propagate x_facebook_sources field when merging source maps

Summary: Preserves Facebook-specific source file metadata (encoded in the optional `x_facebook_sources` field) when it is found in an input source map. We do the minimal amount of work needed to pair each source filename ( = entry in the `sources` array) with its corresponding "metadata blob" ( = entry in the `x_facebook_sources`); the code is intentionally agnostic of what this blob represents or how it is encoded, beyond the fact that `null` represents missing metadata. In order to have these opaque JSON values around for source map generation, we wrap them in instances of a new class, `JSONSharedValue`, which holds a shared reference to the underlying allocator of a JSON value, keeping it alive after the end of source map parsing. Reviewed By: rubennorte Differential Revision: D14951856 fbshipit-source-id: 952779cc0c2fd0fcd89e063485d25fb3ed67b592
facebook · Aug 2, 2019 · a8dafdf · a8dafdf
1 parent fee91d0
commit a8dafdf
Show file tree

Hide file tree

Showing 8 changed files with 327 additions and 58 deletions.
diff --git a/include/hermes/Parser/JSONParser.h b/include/hermes/Parser/JSONParser.h
@@ -12,6 +12,7 @@
 #include <functional>
 #include <iterator>
 #include <map>
+#include <memory>
 #include <utility>
 
 #include "hermes/Parser/JSLexer.h"
@@ -653,6 +654,30 @@ class JSONParser {
   llvm::Optional<JSONValue *> parseObject();
 };
 
+/// A holder class for a JSONValue backed by a shared allocator.
+class JSONSharedValue {
+ public:
+  using Allocator = hermes::BumpPtrAllocator;
+
+ private:
+  const JSONValue *value_;
+  std::shared_ptr<const Allocator> allocator_;
+
+ public:
+  JSONSharedValue(
+      const JSONValue *value,
+      std::shared_ptr<const Allocator> allocator)
+      : value_(value), allocator_(std::move(allocator)) {}
+
+  const JSONValue *operator*() const {
+    return value_;
+  }
+
+  const JSONValue *operator->() const {
+    return value_;
+  }
+};
+
 }; // namespace parser
 }; // namespace hermes
 

diff --git a/include/hermes/SourceMap/SourceMap.h b/include/hermes/SourceMap/SourceMap.h
@@ -7,8 +7,10 @@
 #ifndef HERMES_SUPPORT_SOURCEMAP_H
 #define HERMES_SUPPORT_SOURCEMAP_H
 
+#include "hermes/Parser/JSONParser.h"
 #include "hermes/Support/OptValue.h"
 
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include <vector>
@@ -86,21 +88,32 @@ class SourceMap {
     Segment() = default;
   };
   using SegmentList = std::vector<Segment>;
+  using MetadataEntry = parser::JSONSharedValue;
+  using MetadataList = std::vector<llvm::Optional<MetadataEntry>>;
 
   SourceMap(
       const std::string &sourceRoot,
       std::vector<std::string> &&sources,
-      std::vector<SegmentList> &&lines)
+      std::vector<SegmentList> &&lines,
+      MetadataList &&sourcesMetadata)
       : sourceRoot_(sourceRoot),
         sources_(std::move(sources)),
-        lines_(std::move(lines)) {}
+        lines_(std::move(lines)),
+        sourcesMetadata_(std::move(sourcesMetadata)) {}
 
   /// Query source map text location for \p line and \p column.
-  /// In both the input and output of this function, Line and column numbers
+  /// In both the input and output of this function, line and column numbers
   /// are 1-based.
   llvm::Optional<SourceMapTextLocation> getLocationForAddress(
       uint32_t line,
-      uint32_t column);
+      uint32_t column) const;
+
+  /// Query source map segment for \p line and \p column.
+  /// The line and column arguments are 1-based (but note that the return value
+  /// has 0-based line and column indices).
+  llvm::Optional<SourceMap::Segment> getSegmentForAddress(
+      uint32_t line,
+      uint32_t column) const;
 
   /// \return a list of original sources used by “mappings” entry.
   /// For testing.
@@ -112,14 +125,21 @@ class SourceMap {
     return sourceFullPath;
   }
 
- private:
   /// \return source file path with root combined for source \p index.
   std::string getSourceFullPath(uint32_t index) const {
     assert(index < sources_.size() && "index out-of-range for sources_");
     // TODO: more sophisticated path concat handling.
     return sourceRoot_ + sources_[index];
   }
 
+  /// \return metadata for source \p index, if we have any.
+  llvm::Optional<MetadataEntry> getSourceMetadata(uint32_t index) const {
+    if (index >= sourcesMetadata_.size()) {
+      return llvm::None;
+    }
+    return sourcesMetadata_[index];
+  }
+
  private:
   /// An optional source root, useful for relocating source files on a server or
   /// removing repeated values in the “sources” entry.  This value is prepended
@@ -132,6 +152,10 @@ class SourceMap {
 
   /// The list of segments in VLQ scheme.
   std::vector<SegmentList> lines_;
+
+  /// Metadata for each source keyed by source index. Represents the
+  /// x_facebook_sources field in the JSON source map.
+  MetadataList sourcesMetadata_;
 };
 
 } // namespace hermes

diff --git a/include/hermes/SourceMap/SourceMapGenerator.h b/include/hermes/SourceMap/SourceMapGenerator.h
@@ -46,10 +46,12 @@ class SourceMapGenerator {
   }
 
   /// Adds the source filename to filenameTable_ if it doesn't already exist.
+  /// If \p metadata is provided and is non-null, it becomes the metadata entry
+  /// associated with this source (even if the source existed previously).
   /// \return the index of \p filename in filenameTable_.
-  uint32_t addSource(llvm::StringRef filename) {
-    return filenameTable_.insert(filename);
-  }
+  uint32_t addSource(
+      llvm::StringRef filename,
+      llvm::Optional<SourceMap::MetadataEntry> metadata = llvm::None);
 
   /// Output the given source map as JSON.
   void outputAsJSON(llvm::raw_ostream &OS) const;
@@ -93,26 +95,20 @@ class SourceMapGenerator {
   /// and return a new generator which contains a merged representation.
   SourceMapGenerator mergedWithInputSourceMaps() const;
 
-  /// \return the input source map location for \p seg if the input source map
+  /// \return the input source map segment for \p seg if the input source map
   /// exists and has a valid location for \p seg, else return llvm::None.
-  llvm::Optional<SourceMapTextLocation> getInputLocationForSegment(
-      const SourceMap::Segment &seg) const {
-    if (seg.representedLocation.hasValue()) {
-      assert(
-          seg.representedLocation->sourceIndex >= 0 && "Negative source index");
+  llvm::Optional<std::pair<SourceMap::Segment, const SourceMap *>>
+  getInputSegmentForSegment(const SourceMap::Segment &seg) const;
+
+  bool hasSourcesMetadata() const;
+
+  /// \return metadata for source \index, if we have any.
+  llvm::Optional<SourceMap::MetadataEntry> getSourceMetadata(
+      uint32_t index) const {
+    if (index >= sourcesMetadata_.size()) {
+      return llvm::None;
     }
-    // True iff inputSourceMaps_ has a valid source map for
-    // seg.representedLocation->sourceIndex.
-    bool hasInput = seg.representedLocation.hasValue() &&
-        (uint32_t)seg.representedLocation->sourceIndex <
-            inputSourceMaps_.size() &&
-        inputSourceMaps_[seg.representedLocation->sourceIndex] != nullptr;
-
-    return hasInput ? inputSourceMaps_[seg.representedLocation->sourceIndex]
-                          ->getLocationForAddress(
-                              seg.representedLocation->lineIndex + 1,
-                              seg.representedLocation->columnIndex + 1)
-                    : llvm::None;
+    return sourcesMetadata_[index];
   }
 
   /// The list of symbol names, populating the names field.
@@ -128,6 +124,10 @@ class SourceMapGenerator {
 
   /// Map from {filename => source index}.
   StringSetVector filenameTable_{};
+
+  /// Metadata for each source keyed by source index. Represents the
+  /// x_facebook_sources field in the JSON source map.
+  SourceMap::MetadataList sourcesMetadata_;
 };
 
 } // namespace hermes

diff --git a/lib/SourceMap/SourceMap.cpp b/lib/SourceMap/SourceMap.cpp
@@ -12,7 +12,27 @@ namespace hermes {
 
 llvm::Optional<SourceMapTextLocation> SourceMap::getLocationForAddress(
     uint32_t line,
-    uint32_t column) {
+    uint32_t column) const {
+  auto seg = this->getSegmentForAddress(line, column);
+  // Unmapped location
+  if (!seg.hasValue() || !seg->representedLocation.hasValue()) {
+    return llvm::None;
+  }
+  // parseSegment() should have validated this.
+  assert(
+      (size_t)seg->representedLocation->sourceIndex < sources_.size() &&
+      "SourceIndex is out-of-range.");
+  std::string fileName =
+      getSourceFullPath(seg->representedLocation->sourceIndex);
+  return SourceMapTextLocation{
+      std::move(fileName),
+      (uint32_t)seg->representedLocation->lineIndex + 1,
+      (uint32_t)seg->representedLocation->columnIndex + 1};
+}
+
+llvm::Optional<SourceMap::Segment> SourceMap::getSegmentForAddress(
+    uint32_t line,
+    uint32_t column) const {
   if (line == 0 || line > lines_.size()) {
     return llvm::None;
   }
@@ -37,27 +57,14 @@ llvm::Optional<SourceMapTextLocation> SourceMap::getLocationForAddress(
       [](uint32_t column, const Segment &seg) {
         return column < (uint32_t)seg.generatedColumn;
       });
-  // The found sentinal segment is the first one. No covering segment.
+  // The found sentinel segment is the first one. No covering segment.
   if (segIter == segments.begin()) {
     return llvm::None;
   }
   // Move back one slot.
   const Segment &target =
       segIter == segments.end() ? segments.back() : *(--segIter);
-  // Unmapped location
-  if (!target.representedLocation.hasValue()) {
-    return llvm::None;
-  }
-  // parseSegment() should have validated this.
-  assert(
-      (size_t)target.representedLocation->sourceIndex < sources_.size() &&
-      "SourceIndex is out-of-range.");
-  std::string fileName =
-      getSourceFullPath(target.representedLocation->sourceIndex);
-  return SourceMapTextLocation{
-      std::move(fileName),
-      (uint32_t)target.representedLocation->lineIndex + 1,
-      (uint32_t)target.representedLocation->columnIndex + 1};
+  return target;
 }
 
 } // namespace hermes
diff --git a/lib/SourceMap/SourceMapGenerator.cpp b/lib/SourceMap/SourceMapGenerator.cpp
@@ -27,6 +27,60 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, VLQ vlq) {
 }
 } // namespace
 
+uint32_t SourceMapGenerator::addSource(
+    llvm::StringRef filename,
+    llvm::Optional<SourceMap::MetadataEntry> metadata) {
+  uint32_t index = filenameTable_.insert(filename);
+  if (sourcesMetadata_.size() <= index) {
+    sourcesMetadata_.resize(index + 1);
+  }
+  if (metadata.hasValue() &&
+      metadata.getValue()->getKind() != parser::JSONKind::Null) {
+    sourcesMetadata_[index] = metadata.getValue();
+  }
+  return index;
+}
+
+llvm::Optional<std::pair<SourceMap::Segment, const SourceMap *>>
+SourceMapGenerator::getInputSegmentForSegment(
+    const SourceMap::Segment &seg) const {
+  if (seg.representedLocation.hasValue()) {
+    assert(
+        seg.representedLocation->sourceIndex >= 0 && "Negative source index");
+  }
+  // True iff inputSourceMaps_ has a valid source map for
+  // seg.representedLocation->sourceIndex.
+  bool hasInput = seg.representedLocation.hasValue() &&
+      (uint32_t)seg.representedLocation->sourceIndex <
+          inputSourceMaps_.size() &&
+      inputSourceMaps_[seg.representedLocation->sourceIndex] != nullptr;
+
+  if (!hasInput) {
+    return llvm::None;
+  }
+
+  const SourceMap *inputMap =
+      inputSourceMaps_[seg.representedLocation->sourceIndex].get();
+  auto inputSeg = inputMap->getSegmentForAddress(
+      seg.representedLocation->lineIndex + 1,
+      seg.representedLocation->columnIndex + 1);
+  if (!inputSeg.hasValue()) {
+    return llvm::None;
+  }
+
+  return std::make_pair(inputSeg.getValue(), inputMap);
+}
+
+bool SourceMapGenerator::hasSourcesMetadata() const {
+  for (const auto &entry : sourcesMetadata_) {
+    if (entry.hasValue() &&
+        entry.getValue()->getKind() != parser::JSONKind::Null) {
+      return true;
+    }
+  }
+  return false;
+}
+
 SourceMapGenerator::State SourceMapGenerator::encodeSourceLocations(
     const SourceMapGenerator::State &lastState,
     llvm::ArrayRef<SourceMap::Segment> segments,
@@ -93,24 +147,36 @@ SourceMapGenerator SourceMapGenerator::mergedWithInputSourceMaps() const {
 
     for (const auto &seg : lines_[i]) {
       SourceMap::Segment newSeg = seg;
+      newSeg.representedLocation = llvm::None;
 
-      if (auto loc = getInputLocationForSegment(seg)) {
+      if (auto pair = getInputSegmentForSegment(seg)) {
         // We have an input source map and were able to find a merged source
         // location.
-        assert(loc->line >= 1 && "line numbers in debug info must be 1-based");
-        assert(
-            loc->column >= 1 && "column numbers in debug info must be 1-based");
-        newSeg.representedLocation = SourceMap::Segment::SourceLocation(
-            merged.addSource(loc->fileName), loc->line - 1, loc->column - 1
-            // TODO: Handle name index
-        );
-      } else {
+        auto inputSeg = pair->first;
+        auto inputMap = pair->second;
+        if (inputSeg.representedLocation.hasValue()) {
+          auto loc = inputSeg.representedLocation.getValue();
+          // Our _output_ sourceRoot is empty, so make sure to canonicalize
+          // the path based on the input map's sourceRoot.
+          std::string filename = inputMap->getSourceFullPath(loc.sourceIndex);
+
+          newSeg.representedLocation = SourceMap::Segment::SourceLocation(
+              merged.addSource(
+                  filename, inputMap->getSourceMetadata(loc.sourceIndex)),
+              loc.lineIndex,
+              loc.columnIndex
+              // TODO: Handle name index
+          );
+        }
+      }
+      if (!newSeg.representedLocation.hasValue() &&
+          seg.representedLocation.hasValue()) {
         // Failed to find a merge location. Use the existing location,
         // but copy over the source file name.
-        if (seg.representedLocation.hasValue()) {
-          newSeg.representedLocation->sourceIndex =
-              merged.addSource(sources[seg.representedLocation->sourceIndex]);
-        }
+        newSeg.representedLocation = seg.representedLocation;
+        newSeg.representedLocation->sourceIndex = merged.addSource(
+            sources[seg.representedLocation->sourceIndex],
+            getSourceMetadata(seg.representedLocation->sourceIndex));
       }
 
       newLine.push_back(std::move(newSeg));
@@ -142,6 +208,19 @@ void SourceMapGenerator::outputAsJSONImpl(llvm::raw_ostream &OS) const {
   json.emitValues(llvm::makeArrayRef(getSources()));
   json.closeArray();
 
+  if (hasSourcesMetadata()) {
+    json.emitKey("x_facebook_sources");
+    json.openArray();
+    for (const auto &source : sourcesMetadata_) {
+      if (source.hasValue()) {
+        source.getValue()->emitInto(json);
+      } else {
+        json.emitNullValue();
+      }
+    }
+    json.closeArray();
+  }
+
   json.emitKeyValue("mappings", getVLQMappingsString());
   json.closeDict();
   OS.flush();