feat(CBOR): added experimental support for CBOR data format

We've added very early support for the CBOR file format. Lots to come on this
subject, but if you have CBOR data lying around, try adding it!

TFW updating the qri repo to add CBOR involes only documentation & test
changes.

api/server_test.go

@@ -142,9 +142,9 @@ func TestServerRoutes(t *testing.T) {
 					t.Logf("error creating test dirpath: %s", err.Error())
 					continue
 				}
-				path := filepath.Join(dirpath, fmt.Sprintf("case_%d.json", i))
+				path := filepath.Join(dirpath, c.resBodyPath)
 				if err := ioutil.WriteFile(path, gotBody, os.ModePerm); err != nil {
-					t.Logf("erorr writing test file: %s", err.Error())
+					t.Logf("error writing test file: %s", err.Error())
 					continue
 				}
 				t.Logf("error written to: %s", path)

api/testdata/listResponse.json

@@ -24020,6 +24020,81 @@
         }
       }
     },
+    {
+      "peername": "peer",
+      "name": "flourinated_compounds_in_fast_food_packaging",
+      "path": "/map/QmU9YD3gLZjBLHhe9MEftyBtMEW6tky2XczTbCVq2q84cY",
+      "dataset": {
+        "abstract": "/map/QmPUAqhiqdGDZUourgARtmMqNKWmuTPcy5JSxasvRPdvor",
+        "commit": {
+          "qri": "cm:0",
+          "signature": "5LB6oyPr5VjMbTcwGRTqxmBkULqoJf3uYFM7ovoHkSLdfA94hPGa8uMveXE6YAxZYAB5kaKNWcEXg1oSwCKuXqVzvmjkPJPZs6J3LBYP1GdogZNMUxz9btrYAaS6qEcu4WwnTrhfr77wKc6mTDEzgnTR6tai78AjceXeEFpBppQ1xKXX3vn5ofmeRpoFVAKUN52FZra5x6swzncPaD8TzpFb4u59kHhVy9iKZ9xUKBmAAYPvSwJBJafYgSF5fR5sEYfjCbs6GXSDgpsRhdpGgqFpezUJL4u6ghCH5HbFtjEVGCiHFq5WRATDQMQSv1YNogKo9fhLPLkvKRqipRwsswSwTdEVXS",
+          "timestamp": "2001-01-01T01:01:01.000000001Z",
+          "title": "initial commit"
+        },
+        "dataPath": "/map/QmcN7FHgCbP9MfUmurGuNh3X1DNU4B9LbjXg3r2b3VBrQZ",
+        "meta": {
+          "description": "Paper samples, paper extracts (known), paper extracts (unknown). \n\nThis dataset is associated with the following publication:\nSchaider, L., S. Balan, A. Blum, D. Andrews, M. Strynar, M. Dickinson, D. Lunderberg, J. Lang, and G. Peaslee. Fluorinated Compounds in U.S. Fast Food Packaging.   Environmental Science \u0026amp; Technology Letters. American Chemical Society, Washington, DC, USA, 4(3): 105\u0026ndash;111, (2017).",
+          "qri": "md:0",
+          "title": "Fluorinated Compounds in U.S. Fast Food Packaging"
+        },
+        "qri": "ds:0",
+        "structure": {
+          "checksum": "QmcN7FHgCbP9MfUmurGuNh3X1DNU4B9LbjXg3r2b3VBrQZ",
+          "entries": 25,
+          "errCount": 1,
+          "format": "cbor",
+          "length": 1785,
+          "qri": "st:0",
+          "schema": {
+            "items": {
+              "items": [
+                {
+                  "title": "sample",
+                  "type": "string"
+                },
+                {
+                  "title": "comments",
+                  "type": "string"
+                },
+                {
+                  "description": "length of sample in cm",
+                  "title": "length_cm",
+                  "type": "integer"
+                },
+                {
+                  "description": "width of sample in cm",
+                  "title": "width_cm",
+                  "type": "integer"
+                },
+                {
+                  "description": "area of sample in cm2",
+                  "title": "area_cm",
+                  "type": "integer"
+                },
+                {
+                  "description": "vial tare in grams",
+                  "title": "vial_tare_g",
+                  "type": "number"
+                },
+                {
+                  "description": "mass of vial tare with paper in grams",
+                  "title": "vial_with_paper_g",
+                  "type": "number"
+                },
+                {
+                  "description": "mass of paper in grams",
+                  "title": "mass_g",
+                  "type": "number"
+                }
+              ],
+              "type": "array"
+            },
+            "type": "array"
+          }
+        }
+      }
+    },
     {
       "peername": "peer",
       "name": "movies",

cmd/add.go

@@ -33,8 +33,9 @@ to qri is made public on the distributed web when you run qri connect.
 When adding data, you can supply metadata and dataset structure, but it’s not 
 required. qri does what it can to infer the details you don’t provide. 
 add currently supports two data formats:
-- CSV (Comma Separated Values)
+- CSV  (Comma Separated Values)
 - JSON (Javascript Object Notation)
+- CBOR (Concise Binary Object Representation)
 
 Once you’ve added data, you can use the export command to pull the data out of 
 qri, change the data outside of qri, and use the save command to record those 

cmd/export.go

@@ -198,5 +198,5 @@ func init() {
 	// exportCmd.Flags().BoolVarP(&exportCmdVis, "vis-conf", "c", false, "export viz config file")
 
 	// TODO - get format conversion up & running
-	// exportCmd.Flags().StringP("format", "f", "csv", "set output format [csv,json]")
+	// exportCmd.Flags().StringP("format", "f", "csv", "set output format [csv,json,cbor]")
 }

core/datasets_test.go

@@ -68,7 +68,7 @@ func TestDatasetRequestsInit(t *testing.T) {
 
 func TestDatasetRequestsList(t *testing.T) {
 	var (
-		movies, counter, cities, craigslist repo.DatasetRef
+		movies, counter, cities, craigslist, flourinated repo.DatasetRef
 	)
 
 	mr, err := testrepo.NewTestRepo()
@@ -93,6 +93,8 @@ func TestDatasetRequestsList(t *testing.T) {
 			cities = ref
 		case "craigslist":
 			craigslist = ref
+		case "flourinated_compounds_in_fast_food_packaging":
+			flourinated = ref
 		}
 	}
 
@@ -103,9 +105,9 @@ func TestDatasetRequestsList(t *testing.T) {
 	}{
 		{&ListParams{OrderBy: "", Limit: 1, Offset: 0}, nil, ""},
 		{&ListParams{OrderBy: "chaos", Limit: 1, Offset: -50}, nil, ""},
-		{&ListParams{OrderBy: "", Limit: 30, Offset: 0}, []repo.DatasetRef{cities, counter, craigslist, movies}, ""},
-		{&ListParams{OrderBy: "timestamp", Limit: 30, Offset: 0}, []repo.DatasetRef{cities, counter, craigslist, movies}, ""},
-		{&ListParams{Peername: "me", OrderBy: "timestamp", Limit: 30, Offset: 0}, []repo.DatasetRef{cities, counter, craigslist, movies}, ""},
+		{&ListParams{OrderBy: "", Limit: 30, Offset: 0}, []repo.DatasetRef{cities, counter, craigslist, flourinated, movies}, ""},
+		{&ListParams{OrderBy: "timestamp", Limit: 30, Offset: 0}, []repo.DatasetRef{cities, counter, craigslist, flourinated, movies}, ""},
+		{&ListParams{Peername: "me", OrderBy: "timestamp", Limit: 30, Offset: 0}, []repo.DatasetRef{cities, counter, craigslist, flourinated, movies}, ""},
 		// TODO: re-enable {&ListParams{OrderBy: "name", Limit: 30, Offset: 0}, []*repo.DatasetRef{cities, counter, movies}, ""},
 	}
 

repo/test/test_repo.go

@@ -28,7 +28,7 @@ func init() {
 
 // NewTestRepo generates a repository usable for testing purposes
 func NewTestRepo() (mr repo.Repo, err error) {
-	datasets := []string{"movies", "cities", "counter", "craigslist"}
+	datasets := []string{"movies", "cities", "counter", "craigslist", "flourinated_compounds_in_fast_food_packaging"}
 	p := &profile.Profile{
 		Peername: "peer",
 	}

repo/test/testdata/flourinated_compounds_in_fast_food_packaging/input.dataset.json

@@ -0,0 +1,61 @@
+{
+  "commit" : {
+    "qri" : "cm:0",
+    "title" : "initial commit",
+    "timestamp": "2017-05-01T01:00:00.000Z"
+  },
+  "meta" : {
+    "title" : "Fluorinated Compounds in U.S. Fast Food Packaging",
+    "description" : "Paper samples, paper extracts (known), paper extracts (unknown). \n\nThis dataset is associated with the following publication:\nSchaider, L., S. Balan, A. Blum, D. Andrews, M. Strynar, M. Dickinson, D. Lunderberg, J. Lang, and G. Peaslee. Fluorinated Compounds in U.S. Fast Food Packaging.   Environmental Science \u0026amp; Technology Letters. American Chemical Society, Washington, DC, USA, 4(3): 105\u0026ndash;111, (2017)."
+  },
+  "structure": {
+    "qri": "st:0",
+    "format": "cbor",
+    "schema": {
+      "type": "array",
+      "items": {
+        "type": "array",
+        "items": [
+          {
+            "type": "string",
+            "title": "sample"
+          },
+          {
+            "type": "string",
+            "title": "comments"
+          },
+          {
+            "type": "integer",
+            "title": "length_cm",
+            "description": "length of sample in cm"
+          },
+          {
+            "type": "integer",
+            "title": "width_cm",
+            "description": "width of sample in cm"
+          },
+          {
+            "type": "integer",
+            "title": "area_cm",
+            "description": "area of sample in cm2"
+          },
+          {
+            "type": "number",
+            "title": "vial_tare_g",
+            "description": "vial tare in grams"
+          },
+          {
+            "type": "number",
+            "title": "vial_with_paper_g",
+            "description": "mass of vial tare with paper in grams"
+          },
+          {
+            "type": "number",
+            "title": "mass_g",
+            "description": "mass of paper in grams"
+          }
+        ]
+      }
+    }
+  }
+}