fix(Validate): completely overhaul validation to make it work properly

So, turns out a *bunch* of stuff needed fixing when it comes to valiation. We've gone through and cleanup up the following to make validation a better experience. jsonpointer: * added a new "Descendant" method on a pointer to generate child paths jsonschema: * complete overhaul of Validation interface to support multiple errors, consistent error formatting, and added a jsonpointer path to the location of each error dataset/dsio: * added a new "Entry" struct to represent top-level entries in a dataset * overhauled dsio to remove vals package. dsio now works with go prinitive types * CSVReader has been completely reworked to convert entry rows to the types specified by the Structure schema of the reader (this was the root cause of #290) dataset/validate: * added a new Data method that consumes a dsio.EntryReader to validate data that auto-converts incoming data to JSON for validation (a necessary step to work with JSONSchemas) dataset/dsfs: * dsfs.CreateDataset now uses validate.Data dataset: * updated all packages to use new dsio interfaces qri/core: * dataset.Validate now is mainly a wrapper around validate.Data This sets the stage for some exciting stuff in the future, and makes qri work properly / better today. I'm particularly excited to work with the new dsio Reader/Writer interfaces, as they set the stage for filtering & sorting subsections of datasets on the fly closes #290
qri-io · Mar 9, 2018 · 8af5653 · 8af5653
1 parent d6d683d
commit 8af5653
Show file tree

Hide file tree

Showing 21 changed files with 209 additions and 24,074 deletions.
diff --git a/api/server_test.go b/api/server_test.go
@@ -67,31 +67,31 @@ func TestServerRoutes(t *testing.T) {
 
  // get dataset
  {"GET", "/me/family_relationships", "", "getResponseFamilyRelationships.json", 200},
- {"GET", "/me/family_relationships/at/map/QmfMTGQKjCcy5nEJwMCQNaqNHVGFTyMdhvWk9pQUXSBD3Z", "", "getResponseFamilyRelationships.json", 200},
- {"GET", "/at/map/QmfMTGQKjCcy5nEJwMCQNaqNHVGFTyMdhvWk9pQUXSBD3Z", "", "getResponseFamilyRelationships.json", 200},
+ {"GET", "/me/family_relationships/at/map/QmY5Rm6HjXd2buGVbA5LLj7EPuMcbTiXoh9KRBRGEHxwvA", "", "getResponseFamilyRelationships.json", 200},
+ {"GET", "/at/map/QmY5Rm6HjXd2buGVbA5LLj7EPuMcbTiXoh9KRBRGEHxwvA", "", "getResponseFamilyRelationships.json", 200},
 
  {"POST", "/rename", "renameRequest.json", "renameResponse.json", 200},
 
- {"GET", "/export/me/cities", "", "", 200},
+ // {"GET", "/export/me/cities", "", "", 200},
 
  {"POST", "/save/me/cities", "saveMetaRequest.json", "saveMetaResponse.json", 200},
 
  // history
  {"GET", "/history/me/cities", "", "historyResponse.json", 200},
- {"GET", "/history/me/cities/at/map/QmVZJTaRsiSc5UkpBSzH4yFQXvKKWGkjvGh5rNzueWDNQa", "", "historyResponsePath.json", 200},
- {"GET", "/history/at/map/QmY6DkbrhHEBSMb1DqqPSUhuVo3BoNadAEGDxSveBZKDrk", "", "historyResponseAt.json", 200},
+ {"GET", "/history/me/cities/at/map/QmUC8bnchH1TjWTxzvPzLkksdH7cX4EpvNkXmeK1GSqStV", "", "historyResponsePath.json", 200},
+ {"GET", "/history/at/map/QmUC8bnchH1TjWTxzvPzLkksdH7cX4EpvNkXmeK1GSqStV", "", "historyResponseAt.json", 200},
 
- {"GET", "/export/me/cities", "", "", 200},
- {"GET", "/export/me/cities/at/map/QmVZJTaRsiSc5UkpBSzH4yFQXvKKWGkjvGh5rNzueWDNQa", "", "", 200},
- {"GET", "/export/at/map/QmVZJTaRsiSc5UkpBSzH4yFQXvKKWGkjvGh5rNzueWDNQa", "", "", 200},
+ /*14*/ {"GET", "/export/me/cities", "", "", 200},
+ {"GET", "/export/me/cities/at/map/QmUC8bnchH1TjWTxzvPzLkksdH7cX4EpvNkXmeK1GSqStV", "", "", 200},
+ {"GET", "/export/at/map/QmUC8bnchH1TjWTxzvPzLkksdH7cX4EpvNkXmeK1GSqStV", "", "", 200},
 
  // diff
  {"GET", "/diff", "diffRequest.json", "diffResponse.json", 200},
  {"GET", "/diff", "diffRequestPlusMinusColor.json", "diffResponsePlusMinusColor.json", 200},
 
  // remove
- {"POST", "/remove/me/cities/at/map/QmY6DkbrhHEBSMb1DqqPSUhuVo3BoNadAEGDxSveBZKDrk", "", "removeResponseWithPath.json", 200},
- {"POST", "/remove/at/map/QmRLpDXcrccTzNCwWXyyHRZCmSzDxPoMX91KtLW3szmmkX", "", "removeResponseByPath.json", 200},
+ {"POST", "/remove/me/cities/at/map/QmUC8bnchH1TjWTxzvPzLkksdH7cX4EpvNkXmeK1GSqStV", "", "removeResponseWithPath.json", 200},
+ {"POST", "/remove/at/map/QmY5Rm6HjXd2buGVbA5LLj7EPuMcbTiXoh9KRBRGEHxwvA", "", "removeResponseByPath.json", 200},
 
  {"GET", "/connect/", "", "", 400},
 

diff --git a/api/testdata/addResponseFromURL.json b/api/testdata/addResponseFromURL.json
@@ -26,7 +26,7 @@
  "structure": {
  "checksum": "QmXGZq5E3HmHzNHuoF9bHuCNaBSxpVtazPbCAQMcMnQqAG",
  "entries": 2,
- "errCount": 1,
+ "errCount": 0,
  "format": "csv",
  "formatConfig": {
  "headerRow": true

diff --git a/api/testdata/diffRequest.json b/api/testdata/diffRequest.json
@@ -1 +1 @@
-{"left":"me/cities@/map/QmbnworAqqiMBGRoQGfnseRJmsK31UGSAFtWD3Doh4b6xq", "right":"me/cities@/map/QmVZJTaRsiSc5UkpBSzH4yFQXvKKWGkjvGh5rNzueWDNQa"}
+{"left":"me/cities@/map/QmYeLPcKPZEyvQzhVgCDcJ8kWjAipwbuw5bB4kFAkrCnmA", "right":"me/cities@/map/QmUC8bnchH1TjWTxzvPzLkksdH7cX4EpvNkXmeK1GSqStV"}
diff --git a/api/testdata/diffRequestPlusMinusColor.json b/api/testdata/diffRequestPlusMinusColor.json
@@ -1 +1 @@
-{"left":"me/cities@/map/QmbnworAqqiMBGRoQGfnseRJmsK31UGSAFtWD3Doh4b6xq", "right":"me/cities@/map/QmVZJTaRsiSc5UkpBSzH4yFQXvKKWGkjvGh5rNzueWDNQa","format":"plusMinusColor"}
+{"left":"me/cities@/map/QmYeLPcKPZEyvQzhVgCDcJ8kWjAipwbuw5bB4kFAkrCnmA", "right":"me/cities@/map/QmUC8bnchH1TjWTxzvPzLkksdH7cX4EpvNkXmeK1GSqStV","format":"plusMinusColor"}
diff --git a/api/testdata/diffResponse.json b/api/testdata/diffResponse.json
@@ -1,64 +1,13 @@
 {
  "data": {
  "meta": [
- {
- "Position": "keywords",
- "OldValue": null,
- "NewValue": [
- "keyword1",
- "keyword2"
- ]
- },
  {
  "Position": "title",
- "Value": "test title"
+ "OldValue": "test title",
+ "NewValue": "Updated Title"
  }
  ],
- "structure": [
- {
- "Position": "checksum",
- "OldValue": "QmVYgdpvgnq3FABZFVWUgxr7UCwNSRJz97vBU9YX5g5pQ4",
- "NewValue": "QmY3dTEQLQFe3VTMi646N5NK2TRijPtCS2fi85z72owHPD",
- "Diff": [
- {}
- ]
- },
- {
- "Position": "entries",
- "OldValue": 5,
- "NewValue": 6
- },
- {
- "Position": "length",
- "OldValue": 154,
- "NewValue": 186
- },
- {
- "Position": "schema",
- "Deltas": [
- {
- "Position": "items",
- "Deltas": [
- {
- "Position": "items",
- "Deltas": [
- {
- "Position": 3,
- "Deltas": [
- {
- "Position": "type",
- "OldValue": "boolean",
- "NewValue": "string"
- }
- ]
- }
- ]
- }
- ]
- }
- ]
- }
- ]
+ "structure": []
  },
  "meta": {
  "code": 200

diff --git a/api/testdata/diffResponsePlusMinusColor.json b/api/testdata/diffResponsePlusMinusColor.json
@@ -1,5 +1,5 @@
 {
- "data": " {\n\u001b[30;41m- \"checksum\": \"QmVYgdpvgnq3FABZFVWUgxr7UCwNSRJz97vBU9YX5g5pQ4\",\u001b[0m\n\u001b[30;42m+ \"checksum\": \"QmY3dTEQLQFe3VTMi646N5NK2TRijPtCS2fi85z72owHPD\",\u001b[0m\n\u001b[30;41m- \"entries\": 5,\u001b[0m\n\u001b[30;42m+ \"entries\": 6,\u001b[0m\n \"errCount\": 1,\n \"format\": \"csv\",\n \"formatConfig\": {\n \"headerRow\": true\n },\n\u001b[30;41m- \"length\": 154,\u001b[0m\n\u001b[30;42m+ \"length\": 186,\u001b[0m\n \"qri\": \"st:0\",\n \"schema\": {\n \"items\": {\n \"items\": [\n 0: {\n \"title\": \"city\",\n \"type\": \"string\"\n },\n 1: {\n \"title\": \"pop\",\n \"type\": \"integer\"\n },\n 2: {\n \"title\": \"avg_age\",\n \"type\": \"number\"\n },\n 3: {\n \"title\": \"in_usa\",\n\u001b[30;41m- \"type\": \"boolean\"\u001b[0m\n\u001b[30;42m+ \"type\": \"string\"\u001b[0m\n }\n ],\n \"type\": \"array\"\n },\n \"type\": \"array\"\n }\n }\n",
+ "data": " {\n \"description\": \"test description\",\n \"keywords\": [\n : \"keyword1\",\n : \"keyword2\"\n ],\n \"license\": {\n \"type\": \"\",\n \"url\": \"\"\n },\n \"qri\": \"md:0\",\n\u001b[30;41m- \"title\": \"test title\"\u001b[0m\n\u001b[30;42m+ \"title\": \"Updated Title\"\u001b[0m\n }\n",
  "meta": {
  "code": 200
  }

diff --git a/api/testdata/getResponseFamilyRelationships.json b/api/testdata/getResponseFamilyRelationships.json
@@ -3,7 +3,7 @@
  "peername": "peer",
  "name": "family_relationships",
  "peerID": "QmZePf5LeXow3RW5U1AgEiNbW46YnRGhZ7HPvm1UmPFPwt",
- "path": "/map/QmfMTGQKjCcy5nEJwMCQNaqNHVGFTyMdhvWk9pQUXSBD3Z",
+ "path": "/map/QmY5Rm6HjXd2buGVbA5LLj7EPuMcbTiXoh9KRBRGEHxwvA",
  "dataset": {
  "abstract": "/map/Qmb3n8FvgDbLoU9d7e3vo1UAyVkwV1RnqXUqPKC3Rj2Ej7",
  "commit": {
@@ -22,7 +22,7 @@
  "structure": {
  "checksum": "QmXGZq5E3HmHzNHuoF9bHuCNaBSxpVtazPbCAQMcMnQqAG",
  "entries": 2,
- "errCount": 1,
+ "errCount": 0,
  "format": "csv",
  "formatConfig": {
  "headerRow": true

diff --git a/api/testdata/historyResponse.json b/api/testdata/historyResponse.json
@@ -4,7 +4,7 @@
  "peername": "peer",
  "name": "cities",
  "peerID": "QmZePf5LeXow3RW5U1AgEiNbW46YnRGhZ7HPvm1UmPFPwt",
- "path": "/map/QmY6DkbrhHEBSMb1DqqPSUhuVo3BoNadAEGDxSveBZKDrk",
+ "path": "/map/QmUC8bnchH1TjWTxzvPzLkksdH7cX4EpvNkXmeK1GSqStV",
  "dataset": {
  "abstract": {
  "qri": "ds:0",
@@ -38,12 +38,12 @@
  "qri": "md:0",
  "title": "Updated Title"
  },
- "previousPath": "/map/QmVZJTaRsiSc5UkpBSzH4yFQXvKKWGkjvGh5rNzueWDNQa",
+ "previousPath": "/map/QmYeLPcKPZEyvQzhVgCDcJ8kWjAipwbuw5bB4kFAkrCnmA",
  "qri": "ds:0",
  "structure": {
  "checksum": "QmY3dTEQLQFe3VTMi646N5NK2TRijPtCS2fi85z72owHPD",
  "entries": 6,
- "errCount": 1,
+ "errCount": 0,
  "format": "csv",
  "formatConfig": {
  "headerRow": true
@@ -81,7 +81,7 @@
  "peername": "peer",
  "name": "cities",
  "peerID": "QmZePf5LeXow3RW5U1AgEiNbW46YnRGhZ7HPvm1UmPFPwt",
- "path": "/map/QmVZJTaRsiSc5UkpBSzH4yFQXvKKWGkjvGh5rNzueWDNQa",
+ "path": "/map/QmYeLPcKPZEyvQzhVgCDcJ8kWjAipwbuw5bB4kFAkrCnmA",
  "dataset": {
  "abstract": {
  "qri": "ds:0",
@@ -115,12 +115,12 @@
  "qri": "md:0",
  "title": "test title"
  },
- "previousPath": "/map/QmQyHfc5iro93F5caS9XhkzEf5o6uq1Zz2jxc3Qu2cWNP2",
+ "previousPath": "/map/QmbvCThNst2f2JqG9VLorbqaDpH2fXo6AawFHAoZgqd1iX",
  "qri": "ds:0",
  "structure": {
  "checksum": "QmY3dTEQLQFe3VTMi646N5NK2TRijPtCS2fi85z72owHPD",
  "entries": 6,
- "errCount": 1,
+ "errCount": 0,
  "format": "csv",
  "formatConfig": {
  "headerRow": true
@@ -158,7 +158,7 @@
  "peername": "peer",
  "name": "cities",
  "peerID": "QmZePf5LeXow3RW5U1AgEiNbW46YnRGhZ7HPvm1UmPFPwt",
- "path": "/map/QmQyHfc5iro93F5caS9XhkzEf5o6uq1Zz2jxc3Qu2cWNP2",
+ "path": "/map/QmbvCThNst2f2JqG9VLorbqaDpH2fXo6AawFHAoZgqd1iX",
  "dataset": {
  "abstract": {
  "qri": "ds:0",
@@ -188,12 +188,12 @@
  },
  "qri": "md:0"
  },
- "previousPath": "/map/QmbnworAqqiMBGRoQGfnseRJmsK31UGSAFtWD3Doh4b6xq",
+ "previousPath": "/map/QmdDXJjs6rBQGFUCgT3aGTv3ScXSKR5RrupmfLLch8uXPj",
  "qri": "ds:0",
  "structure": {
  "checksum": "QmY3dTEQLQFe3VTMi646N5NK2TRijPtCS2fi85z72owHPD",
  "entries": 6,
- "errCount": 1,
+ "errCount": 0,
  "format": "csv",
  "formatConfig": {
  "headerRow": true
@@ -231,7 +231,7 @@
  "peername": "peer",
  "name": "cities",
  "peerID": "QmZePf5LeXow3RW5U1AgEiNbW46YnRGhZ7HPvm1UmPFPwt",
- "path": "/map/QmbnworAqqiMBGRoQGfnseRJmsK31UGSAFtWD3Doh4b6xq",
+ "path": "/map/QmdDXJjs6rBQGFUCgT3aGTv3ScXSKR5RrupmfLLch8uXPj",
  "dataset": {
  "abstract": {
  "qri": "ds:0",
@@ -263,7 +263,7 @@
  "structure": {
  "checksum": "QmVYgdpvgnq3FABZFVWUgxr7UCwNSRJz97vBU9YX5g5pQ4",
  "entries": 5,
- "errCount": 1,
+ "errCount": 0,
  "format": "csv",
  "formatConfig": {
  "headerRow": true

diff --git a/api/testdata/historyResponseAt.json b/api/testdata/historyResponseAt.json
@@ -4,7 +4,7 @@
  "peername": "peer",
  "name": "cities",
  "peerID": "QmZePf5LeXow3RW5U1AgEiNbW46YnRGhZ7HPvm1UmPFPwt",
- "path": "/map/QmY6DkbrhHEBSMb1DqqPSUhuVo3BoNadAEGDxSveBZKDrk",
+ "path": "/map/QmUC8bnchH1TjWTxzvPzLkksdH7cX4EpvNkXmeK1GSqStV",
  "dataset": {
  "abstract": {
  "qri": "ds:0",
@@ -38,12 +38,12 @@
  "qri": "md:0",
  "title": "Updated Title"
  },
- "previousPath": "/map/QmVZJTaRsiSc5UkpBSzH4yFQXvKKWGkjvGh5rNzueWDNQa",
+ "previousPath": "/map/QmYeLPcKPZEyvQzhVgCDcJ8kWjAipwbuw5bB4kFAkrCnmA",
  "qri": "ds:0",
  "structure": {
  "checksum": "QmY3dTEQLQFe3VTMi646N5NK2TRijPtCS2fi85z72owHPD",
  "entries": 6,
- "errCount": 1,
+ "errCount": 0,
  "format": "csv",
  "formatConfig": {
  "headerRow": true
@@ -81,7 +81,7 @@
  "peername": "peer",
  "name": "cities",
  "peerID": "QmZePf5LeXow3RW5U1AgEiNbW46YnRGhZ7HPvm1UmPFPwt",
- "path": "/map/QmVZJTaRsiSc5UkpBSzH4yFQXvKKWGkjvGh5rNzueWDNQa",
+ "path": "/map/QmYeLPcKPZEyvQzhVgCDcJ8kWjAipwbuw5bB4kFAkrCnmA",
  "dataset": {
  "abstract": {
  "qri": "ds:0",
@@ -115,12 +115,12 @@
  "qri": "md:0",
  "title": "test title"
  },
- "previousPath": "/map/QmQyHfc5iro93F5caS9XhkzEf5o6uq1Zz2jxc3Qu2cWNP2",
+ "previousPath": "/map/QmbvCThNst2f2JqG9VLorbqaDpH2fXo6AawFHAoZgqd1iX",
  "qri": "ds:0",
  "structure": {
  "checksum": "QmY3dTEQLQFe3VTMi646N5NK2TRijPtCS2fi85z72owHPD",
  "entries": 6,
- "errCount": 1,
+ "errCount": 0,
  "format": "csv",
  "formatConfig": {
  "headerRow": true
@@ -158,7 +158,7 @@
  "peername": "peer",
  "name": "cities",
  "peerID": "QmZePf5LeXow3RW5U1AgEiNbW46YnRGhZ7HPvm1UmPFPwt",
- "path": "/map/QmQyHfc5iro93F5caS9XhkzEf5o6uq1Zz2jxc3Qu2cWNP2",
+ "path": "/map/QmbvCThNst2f2JqG9VLorbqaDpH2fXo6AawFHAoZgqd1iX",
  "dataset": {
  "abstract": {
  "qri": "ds:0",
@@ -188,12 +188,12 @@
  },
  "qri": "md:0"
  },
- "previousPath": "/map/QmbnworAqqiMBGRoQGfnseRJmsK31UGSAFtWD3Doh4b6xq",
+ "previousPath": "/map/QmdDXJjs6rBQGFUCgT3aGTv3ScXSKR5RrupmfLLch8uXPj",
  "qri": "ds:0",
  "structure": {
  "checksum": "QmY3dTEQLQFe3VTMi646N5NK2TRijPtCS2fi85z72owHPD",
  "entries": 6,
- "errCount": 1,
+ "errCount": 0,
  "format": "csv",
  "formatConfig": {
  "headerRow": true
@@ -231,7 +231,7 @@
  "peername": "peer",
  "name": "cities",
  "peerID": "QmZePf5LeXow3RW5U1AgEiNbW46YnRGhZ7HPvm1UmPFPwt",
- "path": "/map/QmbnworAqqiMBGRoQGfnseRJmsK31UGSAFtWD3Doh4b6xq",
+ "path": "/map/QmdDXJjs6rBQGFUCgT3aGTv3ScXSKR5RrupmfLLch8uXPj",
  "dataset": {
  "abstract": {
  "qri": "ds:0",
@@ -263,7 +263,7 @@
  "structure": {
  "checksum": "QmVYgdpvgnq3FABZFVWUgxr7UCwNSRJz97vBU9YX5g5pQ4",
  "entries": 5,
- "errCount": 1,
+ "errCount": 0,
  "format": "csv",
  "formatConfig": {
  "headerRow": true
@@ -302,6 +302,6 @@
  "code": 200
  },
  "pagination": {
- "nextUrl": "/history/at/map/QmY6DkbrhHEBSMb1DqqPSUhuVo3BoNadAEGDxSveBZKDrk"
+ "nextUrl": "/history/at/map/QmUC8bnchH1TjWTxzvPzLkksdH7cX4EpvNkXmeK1GSqStV"
  }
 }