Add support for JSON export (dgraph-io#3309)

* Parse query args in export request url. * Handle import of empty JSON arrays. * Add format field to ExportRequest protobuf message. * Add format arg to ExportOverNetwork(). * Add language tags to JSON live load test. * Add note about JSON exports to documentation. * Add export JSON test. * Refactor out common code from JSON and RDF export tests. * Try to put all export format related settings in one spot. * Replace buf.WriteString()s with fmt.Fprintf()s. * Make toRDF and toJSON methods instead of functions. * Change all buf.Write* to fmt.Fprint* for consistency. * Fix logic bug. * Refactor RDF exporter to share more code with JSON exporter. * Remove unused function. * Add test of request arg handling. * Set HTTP status in response. * Changes suggested by PR review.
dna2fork · Jul 19, 2019 · d6ebfc8 · d6ebfc8
1 parent b318cf9
commit d6ebfc8
Show file tree

Hide file tree

Showing 17 changed files with 735 additions and 438 deletions.
diff --git a/chunker/chunk.go b/chunker/chunk.go
@@ -166,7 +166,10 @@ func (jsonChunker) Chunk(r *bufio.Reader) (*bytes.Buffer, error) {
 	if err != nil {
 		return out, err
 	}
-	if ch != '{' {
+	if ch == ']' {
+		// Handle loading an empty JSON array ("[]") without error.
+		return nil, io.EOF
+	} else if ch != '{' {
 		return nil, fmt.Errorf("Expected JSON map start. Found: %v", string(ch))
 	}
 	x.Check2(out.WriteRune(ch))

diff --git a/dgraph/cmd/alpha/admin.go b/dgraph/cmd/alpha/admin.go
@@ -60,8 +60,25 @@ func exportHandler(w http.ResponseWriter, r *http.Request) {
 	if !handlerInit(w, r, http.MethodGet) {
 		return
 	}
-	// Export logic can be moved to dgraphzero.
-	if err := worker.ExportOverNetwork(context.Background()); err != nil {
+	if err := r.ParseForm(); err != nil {
+		x.SetHttpStatus(w, http.StatusBadRequest, "Parse of export request failed.")
+		return
+	}
+
+	format := worker.DefaultExportFormat
+	if vals, ok := r.Form["format"]; ok {
+		if len(vals) > 1 {
+			x.SetHttpStatus(w, http.StatusBadRequest,
+				"Only one export format may be specified.")
+			return
+		}
+		format = worker.NormalizeExportFormat(vals[0])
+		if format == "" {
+			x.SetHttpStatus(w, http.StatusBadRequest, "Invalid export format.")
+			return
+		}
+	}
+	if err := worker.ExportOverNetwork(context.Background(), format); err != nil {
 		x.SetStatus(w, err.Error(), "Export failed.")
 		return
 	}

diff --git a/dgraph/cmd/alpha/http.go b/dgraph/cmd/alpha/http.go
@@ -213,6 +213,7 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
 		x.SetStatusWithData(w, x.Error, err.Error())
 		return
 	}
+
 	out.WriteRune('{')
 	writeEntry("data", resp.Json)
 	out.WriteRune(',')

diff --git a/dgraph/cmd/live/load-json/family.json b/dgraph/cmd/live/load-json/family.json
@@ -4,7 +4,11 @@
     "name":"Homer",
     "age":"38",
     "role":"father",
-    "parent_to": [
+    "role|gender":"male",
+    "role@es":"padre",
+    "role@fr":"père",
+    "role@hi":"पिता",
+    "parent_to":[
       { "uid":"_:b" },
       { "uid":"_:l" },
       { "uid":"_:m2" }
@@ -15,8 +19,12 @@
     "name":"Marge",
     "age":"34",
     "role":"mother",
+    "role|gender":"female",
+    "role@es":"madre",
+    "role@fr":"mère",
+    "role@hi":"मां",
     "aka":"Midge",
-    "parent_to": [
+    "parent_to":[
       { "uid":"_:b" },
       { "uid":"_:l" },
       { "uid":"_:m2" }
@@ -27,9 +35,13 @@
     "name":"Bart",
     "age":"10",
     "role":"son",
+    "role|gender":"male",
+    "role@es":"hijo",
+    "role@fr":"fils",
+    "role@hi":"बेटा",
     "aka":"El Barto",
     "carries":"slingshot",
-    "sibling_of": [
+    "sibling_of":[
       { "uid":"_:l" },
       { "uid":"_:m2" }
     ]
@@ -39,8 +51,12 @@
     "name":"Lisa",
     "age":"8",
     "role":"daughter",
+    "role|gender":"female",
+    "role@es":"hija",
+    "role@fr":"fille",
+    "role@hi":"बेटी",
     "carries":"saxomophone",
-    "sibling_of": [
+    "sibling_of":[
       { "uid":"_:b" },
       { "uid":"_:m2" }
     ]
@@ -50,8 +66,13 @@
     "name":"Maggie",
     "age":"1",
     "role":"daughter",
+    "role|gender":"female",
+    "role|generation":3,
+    "role@es":"hija",
+    "role@fr":"fille",
+    "role@hi":"बेटी",
     "carries":"pacifier",
-    "sibling_of": [
+    "sibling_of":[
       { "uid":"_:b" },
       { "uid":"_:l" }
     ]
@@ -60,9 +81,13 @@
     "uid":"_:a",
     "name":"Abraham",
     "age":"83",
-    "role":"father",
+    "role":"grandfather",
+    "role|gender":"male",
+    "role@es":"abuelo",
+    "role@fr":"grand-père",
+    "role@hi":"दादा",
     "aka":"Grampa",
-    "parent_to": [
+    "parent_to":[
       { "uid":"_:h" }
     ]
   }

diff --git a/dgraph/cmd/live/load-json/family.schema b/dgraph/cmd/live/load-json/family.schema
@@ -1,6 +1,6 @@
 name:string @index(term) .
 age: int .
-role:string @index(term) .
+role:string @index(term) @lang .
 aka:string @index(term) .
 carries:string @index(term) .
 parent_to: [uid] @reverse .

diff --git a/dgraph/cmd/live/load-json/family1.json b/dgraph/cmd/live/load-json/family1.json
@@ -4,6 +4,8 @@
     "name":"Homer",
     "age":"38",
     "role":"father",
+    "role|gender":"male",
+    "role@es":"padre",
     "parent_to": [
       { "uid":"_:b" },
       { "uid":"_:l" },

diff --git a/dgraph/cmd/live/load-json/family2.json b/dgraph/cmd/live/load-json/family2.json
@@ -27,6 +27,9 @@
     "name":"Maggie",
     "age":"1",
     "role":"daughter",
+    "role|gender":"female",
+    "role|generation":3,
+    "role@es":"hija",
     "carries":"pacifier",
     "sibling_of": [
       { "uid":"_:b" },

diff --git a/dgraph/cmd/live/load-json/load_test.go b/dgraph/cmd/live/load-json/load_test.go
@@ -47,19 +47,22 @@ func checkLoadedData(t *testing.T) {
 			q(func: anyofterms(name, "Homer")) {
 				name
 				age
-				role
+				role @facets(gender,generation)
+				role@es
 			}
 		}
 	`)
 	require.NoError(t, err)
 	z.CompareJSON(t, `
 		{
-		    "q": [
+			"q": [
 					{
 					"name": "Homer",
 					"age": 38,
-					"role": "father"
-			    }
+					"role": "father",
+					"role@es": "padre",
+					"role|gender": "male"
+				}
 			]
 		}
 	`, string(resp.GetJson()))
@@ -68,25 +71,42 @@ func checkLoadedData(t *testing.T) {
 		{
 			q(func: anyofterms(name, "Maggie")) {
 				name
-				role
+				role @facets(gender,generation)
+				role@es
 				carries
 			}
 		}
 	`)
 	require.NoError(t, err)
 	z.CompareJSON(t, `
 		{
-		    "q": [
+			"q": [
 				{
 					"name": "Maggie",
 					"role": "daughter",
-					"carries": "pacifier"
-			    }
+					"role@es": "hija",
+					"carries": "pacifier",
+					"role|gender": "female",
+					"role|generation": 3
+				}
 			]
 		}
 	`, string(resp.GetJson()))
 }
 
+func TestLiveLoadJSONFileEmpty(t *testing.T) {
+	z.DropAll(t, dg)
+
+	pipeline := [][]string{
+		{"echo", "[]"},
+		{os.ExpandEnv("$GOPATH/bin/dgraph"), "live",
+			"--schema", testDataDir + "/family.schema", "--files", "/dev/stdin",
+			"--alpha", alphaService},
+	}
+	err := z.Pipeline(pipeline)
+	require.NoError(t, err, "live loading JSON file ran successfully")
+}
+
 func TestLiveLoadJSONFile(t *testing.T) {
 	z.DropAll(t, dg)
 

diff --git a/dgraph/cmd/live/run.go b/dgraph/cmd/live/run.go
@@ -224,7 +224,7 @@ func (l *loader) processLoadFile(ctx context.Context, rd *bufio.Reader, ck chunk
 // batches (each one containing opt.batchSize entries) and sends the batches
 // to the loader.reqs channel
 func (l *loader) processChunk(chunkBuf *bytes.Buffer, ck chunker.Chunker) {
-	if chunkBuf == nil && chunkBuf.Len() == 0 {
+	if chunkBuf == nil || chunkBuf.Len() == 0 {
 		return
 	}
 

diff --git a/protos/pb.proto b/protos/pb.proto
@@ -488,9 +488,10 @@ message BackupRequest {
 }
 
 message ExportRequest {
-	uint32 group_id = 1;  // Group id to back up.
-	uint64 read_ts  = 2;
-	int64 unix_ts   = 3;
+	uint32  group_id = 1;  // Group id to back up.
+	uint64  read_ts  = 2;
+	int64   unix_ts  = 3;
+	string  format   = 4;
 }
 
 // vim: noexpandtab sw=2 ts=2