Skip to content

Commit

Permalink
Merge "FAB-2010 CouchDB Query Data Wrapper fails"
Browse files Browse the repository at this point in the history
  • Loading branch information
@binhn
binhn authored and Gerrit Code Review committed Feb 15, 2017
2 parents 99f50e8 + 1504eaa commit f590121
Show file tree
Hide file tree
Showing 4 changed files with 473 additions and 96 deletions.
75 changes: 68 additions & 7 deletions core/ledger/kvledger/txmgmt/statedb/commontests/test_common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,11 +183,27 @@ func TestQuery(t *testing.T, dbProvider statedb.VersionedDBProvider) {
db.Open()
defer db.Close()
batch := statedb.NewUpdateBatch()
jsonValue1 := "{\"asset_name\": \"marble1\",\"color\": \"blue\",\"size\": 35,\"owner\": \"tom\"}"
jsonValue1 := "{\"asset_name\": \"marble1\",\"color\": \"blue\",\"size\": 1,\"owner\": \"tom\"}"
batch.Put("ns1", "key1", []byte(jsonValue1), version.NewHeight(1, 1))
jsonValue2 := "{\"asset_name\": \"marble1\",\"color\": \"blue\",\"size\": 35,\"owner\": \"jerry\"}"
jsonValue2 := "{\"asset_name\": \"marble2\",\"color\": \"blue\",\"size\": 2,\"owner\": \"jerry\"}"
batch.Put("ns1", "key2", []byte(jsonValue2), version.NewHeight(1, 2))
savePoint := version.NewHeight(2, 5)
jsonValue3 := "{\"asset_name\": \"marble3\",\"color\": \"blue\",\"size\": 3,\"owner\": \"fred\"}"
batch.Put("ns1", "key3", []byte(jsonValue3), version.NewHeight(1, 3))
jsonValue4 := "{\"asset_name\": \"marble4\",\"color\": \"blue\",\"size\": 4,\"owner\": \"martha\"}"
batch.Put("ns1", "key4", []byte(jsonValue4), version.NewHeight(1, 4))
jsonValue5 := "{\"asset_name\": \"marble5\",\"color\": \"blue\",\"size\": 5,\"owner\": \"fred\"}"
batch.Put("ns1", "key5", []byte(jsonValue5), version.NewHeight(1, 5))
jsonValue6 := "{\"asset_name\": \"marble6\",\"color\": \"blue\",\"size\": 6,\"owner\": \"elaine\"}"
batch.Put("ns1", "key6", []byte(jsonValue6), version.NewHeight(1, 6))
jsonValue7 := "{\"asset_name\": \"marble7\",\"color\": \"blue\",\"size\": 7,\"owner\": \"fred\"}"
batch.Put("ns1", "key7", []byte(jsonValue7), version.NewHeight(1, 7))
jsonValue8 := "{\"asset_name\": \"marble8\",\"color\": \"blue\",\"size\": 8,\"owner\": \"elaine\"}"
batch.Put("ns1", "key8", []byte(jsonValue8), version.NewHeight(1, 8))
jsonValue9 := "{\"asset_name\": \"marble9\",\"color\": \"green\",\"size\": 9,\"owner\": \"fred\"}"
batch.Put("ns1", "key9", []byte(jsonValue9), version.NewHeight(1, 9))
jsonValue10 := "{\"asset_name\": \"marble10\",\"color\": \"green\",\"size\": 10,\"owner\": \"mary\"}"
batch.Put("ns1", "key10", []byte(jsonValue10), version.NewHeight(1, 12))
savePoint := version.NewHeight(2, 12)
db.ApplyUpdates(batch, savePoint)

// query for owner=jerry
Expand All @@ -200,8 +216,8 @@ func TestQuery(t *testing.T, dbProvider statedb.VersionedDBProvider) {
testutil.AssertNotNil(t, queryResult1)
versionedQueryRecord := queryResult1.(*statedb.VersionedQueryRecord)
stringRecord := string(versionedQueryRecord.Record)
bFoundJerry := strings.Contains(stringRecord, "jerry")
testutil.AssertEquals(t, bFoundJerry, true)
bFoundRecord := strings.Contains(stringRecord, "jerry")
testutil.AssertEquals(t, bFoundRecord, true)

// verify no more results
queryResult2, err := itr.Next()
Expand Down Expand Up @@ -231,12 +247,57 @@ func TestQuery(t *testing.T, dbProvider statedb.VersionedDBProvider) {
testutil.AssertNotNil(t, queryResult1)
versionedQueryRecord = queryResult1.(*statedb.VersionedQueryRecord)
stringRecord = string(versionedQueryRecord.Record)
bFoundJerry = strings.Contains(stringRecord, "jerry")
testutil.AssertEquals(t, bFoundJerry, true)
bFoundRecord = strings.Contains(stringRecord, "jerry")
testutil.AssertEquals(t, bFoundRecord, true)

// verify no more results
queryResult2, err = itr.Next()
testutil.AssertNoError(t, err, "")
testutil.AssertNil(t, queryResult2)

// query with complex selector
itr, err = db.ExecuteQuery("{\"selector\":{\"$and\":[{\"size\":{\"$gt\": 5}},{\"size\":{\"$lt\":8}},{\"$not\":{\"size\":6}}]}}")
testutil.AssertNoError(t, err, "")

// verify one fred result
queryResult1, err = itr.Next()
testutil.AssertNoError(t, err, "")
testutil.AssertNotNil(t, queryResult1)
versionedQueryRecord = queryResult1.(*statedb.VersionedQueryRecord)
stringRecord = string(versionedQueryRecord.Record)
bFoundRecord = strings.Contains(stringRecord, "fred")
testutil.AssertEquals(t, bFoundRecord, true)

// verify no more results
queryResult2, err = itr.Next()
testutil.AssertNoError(t, err, "")
testutil.AssertNil(t, queryResult2)

// query with embedded implicit "AND" and explicit "OR"
itr, err = db.ExecuteQuery("{\"selector\":{\"color\":\"green\",\"$or\":[{\"owner\":\"fred\"},{\"owner\":\"mary\"}]}}")
testutil.AssertNoError(t, err, "")

// verify one green result
queryResult1, err = itr.Next()
testutil.AssertNoError(t, err, "")
testutil.AssertNotNil(t, queryResult1)
versionedQueryRecord = queryResult1.(*statedb.VersionedQueryRecord)
stringRecord = string(versionedQueryRecord.Record)
bFoundRecord = strings.Contains(stringRecord, "green")
testutil.AssertEquals(t, bFoundRecord, true)

// verify another green result
queryResult2, err = itr.Next()
testutil.AssertNoError(t, err, "")
testutil.AssertNotNil(t, queryResult2)
versionedQueryRecord = queryResult2.(*statedb.VersionedQueryRecord)
stringRecord = string(versionedQueryRecord.Record)
bFoundRecord = strings.Contains(stringRecord, "green")
testutil.AssertEquals(t, bFoundRecord, true)

// verify no more results
queryResult3, err = itr.Next()
testutil.AssertNoError(t, err, "")
testutil.AssertNil(t, queryResult3)

}
196 changes: 112 additions & 84 deletions core/ledger/kvledger/txmgmt/statedb/statecouchdb/query_wrapper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,137 +3,165 @@ package statecouchdb
import (
"encoding/json"
"fmt"
"strings"
"reflect"
)

//ApplyQueryWrapper parses the query string passed to CouchDB
//the wrapper prepends the wrapper "data." to all fields specified in the query
//All fields in the selector must have "data." prepended to the field names
//Fields listed in fields key will have "data." prepended
//Fields in the sort key will have "data." prepended
func ApplyQueryWrapper(queryString string) []byte {
var dataWrapper = "data"
var jsonQueryFields = "fields"

//create a generic map for the query json
jsonQuery := make(map[string]interface{})
var validOperators = []string{"$and", "$or", "$not", "$nor", "$all", "$elemMatch",
"$lt", "$lte", "$eq", "$ne", "$gte", "$gt", "$exits", "$type", "$in", "$nin",
"$size", "$mod", "$regex"}

//unmarshal the selected json into the generic map
json.Unmarshal([]byte(queryString), &jsonQuery)
/*
ApplyQueryWrapper parses the query string passed to CouchDB
the wrapper prepends the wrapper "data." to all fields specified in the query
All fields in the selector must have "data." prepended to the field names
Fields listed in fields key will have "data." prepended
Fields in the sort key will have "data." prepended
//iterate through the JSON query
for jsonKey, jsonValue := range jsonQuery {
Example:
//create a case for the data types found in the JSON query
switch jsonQueryPart := jsonValue.(type) {
Source Query:
{"selector":{"owner": {"$eq": "tom"}},
"fields": ["owner", "asset_name", "color", "size"],
"sort": ["size", "color"], "limit": 10, "skip": 0}
//if the type is an array, then this is either the "fields" or "sort" part of the query
case []interface{}:
Result Wrapped Query:
{"selector":{"data.owner":{"$eq":"tom"}},
"fields": ["data.owner","data.asset_name","data.color","data.size","_id","version"],
"sort":["data.size","data.color"],"limit":10,"skip":0}
//check to see if this is a "fields" or "sort" array
//if jsonKey == jsonQueryFields || jsonKey == jsonQuerySort {
if jsonKey == jsonQueryFields {
//iterate through the names and add the data wrapper for each field
for itemKey, fieldName := range jsonQueryPart {
*/
func ApplyQueryWrapper(queryString string) string {

//add "data" wrapper to each field definition
jsonQueryPart[itemKey] = fmt.Sprintf("%v.%v", dataWrapper, fieldName)
}
//create a generic map for the query json
jsonQueryMap := make(map[string]interface{})

//Add the "_id" and "version" fields, these are needed by default
if jsonKey == jsonQueryFields {
//unmarshal the selected json into the generic map
json.Unmarshal([]byte(queryString), &jsonQueryMap)

jsonQueryPart = append(jsonQueryPart, "_id")
jsonQueryPart = append(jsonQueryPart, "version")
//traverse through the json query and wrap any field names
processAndWrapQuery(jsonQueryMap)

//Overwrite the query fields if the "_id" field has been added
jsonQuery[jsonQueryFields] = jsonQueryPart
}
//process the query and add the version and fields if fields are specified
for jsonKey, jsonValue := range jsonQueryMap {

//Add the "_id" and "version" fields, these are needed by default
if jsonKey == jsonQueryFields {

//check to see if this is an interface map
if reflect.TypeOf(jsonValue).String() == "[]interface {}" {

//Add the "_id" and "version" fields, these are needed by default
//Overwrite the query fields if the "_id" field has been added
jsonQueryMap[jsonQueryFields] = append(jsonValue.([]interface{}), "_id", "version")
}

if jsonKey == jsonQuerySort {
}
}

//Marshal the updated json query
editedQuery, _ := json.Marshal(jsonQueryMap)

logger.Debugf("Rewritten query with data wrapper: %s", editedQuery)

return string(editedQuery)

}

//iterate through the names and add the data wrapper for each field
for sortItemKey, sortField := range jsonQueryPart {
func processAndWrapQuery(jsonQueryMap map[string]interface{}) {

//create a case for the data types found in the JSON query
switch sortFieldType := sortField.(type) {
//iterate through the JSON query
for _, jsonValue := range jsonQueryMap {

//if the type is string, then this is a simple array of field names.
//Add the datawrapper to the field name
case string:
//create a case for the data types found in the JSON query
switch jsonValueType := jsonValue.(type) {

//simple case, update the existing array item with the updated name
jsonQueryPart[sortItemKey] = fmt.Sprintf("%v.%v", dataWrapper, sortField)
case string:
//intercept the string case and prevent the []interface{} case from
//incorrectly processing the string

case interface{}:
case float64:
//intercept the float64 case and prevent the []interface{} case from
//incorrectly processing the float64

//this case is a little more complicated. Here we need to
//iterate over the mapped field names since this is an array of objects
//example: {"fieldname":"desc"}
for key, itemValue := range sortField.(map[string]interface{}) {
//delete the mapping for the field definition, since we have to change the
//value of the key
delete(sortField.(map[string]interface{}), key)
//if the type is an array, then iterate through the items
case []interface{}:

//add the key back into the map with the field name wrapped with then "data" wrapper
sortField.(map[string]interface{})[fmt.Sprintf("%v.%v", dataWrapper, key)] = itemValue
}
//iterate the the items in the array
for itemKey, itemValue := range jsonValueType {

default:
switch itemValue.(type) {

logger.Debugf("The type %v was not recognized as a valid sort field type.", sortFieldType)
case string:

}
//This is a simple string, so wrap the field and replace in the array
jsonValueType[itemKey] = fmt.Sprintf("%v.%v", dataWrapper, itemValue)

}
}
case []interface{}:

case interface{}:
//This is a array, so traverse to the next level
processAndWrapQuery(itemValue.(map[string]interface{}))

//if this is the "selector", the field names need to be mapped with the
//data wrapper
if jsonKey == jsonQuerySelector {
case interface{}:

processSelector(jsonQueryPart.(map[string]interface{}))
//process this part as a map
processInterfaceMap(itemValue.(map[string]interface{}))

}
}

default:
case interface{}:

logger.Debugf("The value %v was not recognized as a valid selector field.", jsonKey)
//process this part as a map
processInterfaceMap(jsonValue.(map[string]interface{}))

}
}
}

//Marshal the updated json query
editedQuery, _ := json.Marshal(jsonQuery)
//processInterfaceMap processes an interface map and wraps field names or traverses the next level of the json query
func processInterfaceMap(jsonFragment map[string]interface{}) {

logger.Debugf("Rewritten query with data wrapper: %s", editedQuery)
//iterate the the item in the map
for keyVal, itemVal := range jsonFragment {

return editedQuery
//check to see if the key is an operator
if arrayContains(validOperators, keyVal) {

}
//if this is an operator, traverse the next level of the json query
processAndWrapQuery(jsonFragment)

//processSelector is a recursion function for traversing the selector part of the query
func processSelector(selectorFragment map[string]interface{}) {
} else {

//iterate through the top level definitions
for itemKey, itemValue := range selectorFragment {
//if this is not an operator, this is a field name and needs to be wrapped
wrapFieldName(jsonFragment, keyVal, itemVal)

}
}
}

//check to see if the itemKey starts with a $. If so, this indicates an operator
if strings.HasPrefix(fmt.Sprintf("%s", itemKey), "$") {
//wrapFieldName "wraps" the field name with the data wrapper, and replaces the key in the json fragment
func wrapFieldName(jsonFragment map[string]interface{}, key string, value interface{}) {

processSelector(itemValue.(map[string]interface{}))
//delete the mapping for the field definition, since we have to change the
//value of the key
delete(jsonFragment, key)

} else {
//add the key back into the map with the field name wrapped with then "data" wrapper
jsonFragment[fmt.Sprintf("%v.%v", dataWrapper, key)] = value

//delete the mapping for the field definition, since we have to change the
//value of the key
delete(selectorFragment, itemKey)
//add the key back into the map with the field name wrapped with then "data" wrapper
selectorFragment[fmt.Sprintf("%v.%v", dataWrapper, itemKey)] = itemValue
}

}
//arrayContains is a function to detect if a soure array of strings contains the selected string
//for this application, it is used to determine if a string is a valid CouchDB operator
func arrayContains(sourceArray []string, selectItem string) bool {
set := make(map[string]struct{}, len(sourceArray))
for _, s := range sourceArray {
set[s] = struct{}{}
}
_, ok := set[selectItem]
return ok
}
Loading

0 comments on commit f590121

Please sign in to comment.