std.parseJson - a builtin for parsing json (google#553)

Author: sbarzowski

.travis.yml

@@ -1,10 +1,32 @@
-language: cpp
-compiler:
-  - gcc
-  - clang
+language: generic
+matrix:
+  include:
+    - os: linux
+      env: COMPILER_NAME=gcc CXX=g++-4.9 CC=gcc-4.9
+      addons:
+        apt:
+          packages:
+            - g++-4.9
+          sources: &sources
+            - llvm-toolchain-precise-3.8
+            - ubuntu-toolchain-r-test
+    - os: linux
+      env: COMPILER_NAME=clang CXX=clang++-3.8 CC=clang-3.8
+      addons:
+        apt:
+          packages:
+            - clang-3.8
+          sources: *sources
+    - os: osx
+      osx_image: xcode8
+      env: CC=gcc-4.9 && CXX=g++-4.9
+    - os: osx
+      osx_image: xcode8
+      env: CC=clang && CXX=clang++
 before_install:
   - echo $LANG
   - echo $LC_ALL
+  - $CXX --version
 before_script:
 script:
 #    - rvm get head  # Workaround 'shell_session_update: command not found'
@@ -27,7 +49,4 @@ notifications:
     channels:
       - "chat.freenode.net#jsonnet"
     template:
-      - "%{repository}/%{branch} (%{commit} - %{author}): %{message}"
-os:
-  - linux
-  - osx
+      - "%{repository}/%{branch} (%{commit} - %{author}): %{message}"

CMakeLists.txt

@@ -87,6 +87,7 @@ link_directories(${GLOBAL_OUTPUT_PATH})
 include_directories(
     include
     third_party/md5
+    third_party/json
     core)
 
 install(DIRECTORY include DESTINATION include)

MANIFEST.in

@@ -1,2 +1,2 @@
-include LICENSE include/*.h core/*.cpp core/*.h python/*.c third_party/md5/*.cpp third_party/md5/*.h stdlib/std.jsonnet Makefile
+include LICENSE include/*.h core/*.cpp core/*.h python/*.c third_party/json*.hpp third_party/md5/*.cpp third_party/md5/*.h stdlib/std.jsonnet Makefile
 #recursive-include test_suite examples gc_stress benchmarks editors

Makefile

@@ -29,9 +29,9 @@ OD ?= od
 
 OPT ?= -O3
 
-CXXFLAGS ?= -g $(OPT) -Wall -Wextra -Woverloaded-virtual -pedantic -std=c++0x -fPIC -Iinclude -Ithird_party/md5
+CXXFLAGS ?= -g $(OPT) -Wall -Wextra -Woverloaded-virtual -pedantic -std=c++0x -fPIC -Iinclude -Ithird_party/md5 -Ithird_party/json
 CFLAGS ?= -g $(OPT) -Wall -Wextra -pedantic -std=c99 -fPIC -Iinclude
-MAKEDEPENDFLAGS ?= -Iinclude -Ithird_party/md5
+MAKEDEPENDFLAGS ?= -Iinclude -Ithird_party/md5 -Ithird_party/json
 EMCXXFLAGS = $(CXXFLAGS) -g0 -Os --memory-init-file 0 -s DISABLE_EXCEPTION_CATCHING=0 -s OUTLINING_LIMIT=10000 -s RESERVED_FUNCTION_POINTERS=20
 EMCFLAGS = $(CFLAGS) --memory-init-file 0 -s DISABLE_EXCEPTION_CATCHING=0
 LDFLAGS ?=
@@ -86,7 +86,8 @@ ALL_HEADERS = \
 	include/libjsonnet.h \
 	include/libjsonnet_fmt.h \
 	include/libjsonnet++.h \
-	third_party/md5/md5.h
+	third_party/md5/md5.h \
+	third_party/json/json.hpp
 
 default: jsonnet
 

core/BUILD

@@ -32,6 +32,7 @@ cc_library(
         "//include:libjsonnet",
         "//include:libjsonnet_fmt",
         "//stdlib:std",
+        "//third_party/json:json",
         "//third_party/md5:libmd5",
     ],
     linkopts = ["-lm"],

core/desugarer.cpp

@@ -34,7 +34,7 @@ struct BuiltinDecl {
     std::vector<UString> params;
 };
 
-static unsigned long max_builtin = 34;
+static unsigned long max_builtin = 35;
 BuiltinDecl jsonnet_builtin_decl(unsigned long builtin)
 {
     switch (builtin) {
@@ -73,6 +73,7 @@ BuiltinDecl jsonnet_builtin_decl(unsigned long builtin)
         case 32: return {U"asciiLower", {U"str"}};
         case 33: return {U"asciiUpper", {U"str"}};
         case 34: return {U"join", {U"sep", U"arr"}};
+        case 35: return {U"parseJson", {U"str"}};
         default:
             std::cerr << "INTERNAL ERROR: Unrecognized builtin function: " << builtin << std::endl;
             std::abort();

core/vm.cpp

@@ -23,13 +23,16 @@ limitations under the License.
 
 #include "desugarer.h"
 #include "json.h"
+#include "json.hpp"
 #include "md5.h"
 #include "parser.h"
 #include "state.h"
 #include "static_analysis.h"
 #include "string_utils.h"
 #include "vm.h"
 
+using json = nlohmann::json;
+
 namespace {
 
 /** Turn a path e.g. "/a/b/c" into a dir, e.g. "/a/b/".  If there is no path returns "".
@@ -870,6 +873,7 @@ class Interpreter {
         builtins["asciiLower"] = &Interpreter::builtinAsciiLower;
         builtins["asciiUpper"] = &Interpreter::builtinAsciiUpper;
         builtins["join"] = &Interpreter::builtinJoin;
+        builtins["parseJson"] = &Interpreter::builtinParseJson;
     }
 
     /** Clean up the heap, stack, stash, and builtin function ASTs. */
@@ -1435,6 +1439,75 @@ class Interpreter {
         return nullptr;
     }
 
+    const AST *builtinParseJson(const LocationRange &loc, const std::vector<Value> &args)
+    {
+        validateBuiltinArgs(loc, "parseJson", args, {Value::STRING});
+
+        std::string value = encode_utf8(static_cast<HeapString *>(args[0].v.h)->value);
+
+        auto j = json::parse(value);
+
+        bool filled;
+
+        otherJsonToHeap(j, filled, scratch);
+
+        return nullptr;
+    }
+
+    void otherJsonToHeap(const json &v, bool &filled, Value &attach) {
+        // In order to not anger the garbage collector, assign to attach immediately after
+        // making the heap object.
+        switch (v.type()) {
+            case json::value_t::string:
+                attach = makeString(decode_utf8(v.get<std::string>()));
+                filled = true;
+                break;
+
+            case json::value_t::boolean:
+                attach = makeBoolean(v.get<bool>());
+                filled = true;
+                break;
+
+            case json::value_t::number_integer:
+            case json::value_t::number_unsigned:
+            case json::value_t::number_float:
+                attach = makeNumber(v.get<double>());
+                filled = true;
+                break;
+
+            case json::value_t::null:
+                attach = makeNull();
+                filled = true;
+                break;
+
+            case json::value_t::array:{
+                attach = makeArray(std::vector<HeapThunk *>(v.size()));
+                filled = true;
+                auto *arr = static_cast<HeapArray *>(attach.v.h);
+                for (size_t i = 0; i < v.size(); ++i) {
+                    arr->elements[i] = makeHeap<HeapThunk>(idArrayElement, nullptr, 0, nullptr);
+                    otherJsonToHeap(v[i], arr->elements[i]->filled, arr->elements[i]->content);
+                }
+            } break;
+
+            case json::value_t::object: {
+                attach = makeObject<HeapComprehensionObject>(
+                    BindingFrame{}, jsonObjVar, idJsonObjVar, BindingFrame{});
+                filled = true;
+                auto *obj = static_cast<HeapComprehensionObject *>(attach.v.h);
+                for (auto it = v.begin(); it != v.end(); ++it) {
+                    auto *thunk = makeHeap<HeapThunk>(idJsonObjVar, nullptr, 0, nullptr);
+                    obj->compValues[alloc->makeIdentifier(decode_utf8(it.key()))] = thunk;
+                    otherJsonToHeap(it.value(), thunk->filled, thunk->content);
+                }
+            } break;
+
+            case json::value_t::discarded: {
+                abort();
+            }
+        }
+    }
+
     void joinString(UString &running, const Value &sep, unsigned idx, const Value &elt)
     {
         if (elt.t == Value::NULL_TYPE) {