diff --git a/README.md b/README.md index 05d025d..4e36677 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ API Documentation is available at http://godoc.org/gopkg.in/qntfy/kazaam.v3. ## Features Kazaam is primarily designed to be used as a library for transforming arbitrary JSON. -It ships with nine built-in transform types, and fifteen built-in converter types, +It ships with ten built-in transform types, and fifteen built-in converter types, described below, which provide significant flexibility in reshaping JSON data. Also included when you `go get` Kazaam, is a binary implementation, `kazaam` that can be used for @@ -45,6 +45,7 @@ need arises. Kazaam currently supports the following built-in transforms: - shift +- steps - concat - coalesce - extract @@ -157,6 +158,33 @@ within the arguments are preserved, if the whitespace around the arguments is re Arguments are passed to the converter functions as a single string, and will require the converter function to parse out any meaningful parameters. +### Steps +The steps transform performs a series of shift transforms with each step working on the ouptput from the last step. This +transform is very similar to the shift transform, and takes the same optional parameters. + +The following example produces the same results as the `Shift` transform example presented earlier. The only difference +is that the each of the steps are guaranteed to transform in the specified order. + +```json +{ + "operation": "steps", + "spec": { + "steps": [ + { + "object.id": "doc.uid" + }, + { + "gid2": "doc.guid[1]" + }, + { + "allGuids": "doc.guidObjects[*].id" + } + ] + } +} +``` + + ### Concat The concat transform allows the combination of fields and literal strings into a single string value. ```json @@ -519,6 +547,9 @@ Kazaam currently supports the following built-in Conveters: `substr []` | converts a string value to a substring value `trim` | converts a string value by removing the leading and trailing whitespace characters `upper` | converts a string value to uppercase characters +`len` | converts a string to an integer value equal to the length of the string +`splitn ` | splits a string by a delimiter string and returns the Nth token (1 based) +`eqs ` | returns `true` or `false` based on whether the value matches the parameter ### Converter Examples ### @@ -950,6 +981,83 @@ produces: } ``` +#### Len #### + +Returns the length of a string value + +Argument | Description +---------|------------ + +example: +```json +{ + "operation": "shift", + "spec": { + "output": "tests.test_string | len" + } +} +``` + +produces: +```json +{ + "output": 19 +} +``` + +#### Splitn #### + +Returns the Nth token of a string split by a delimiter string + +Argument | Description +---------|------------ +string | delimiter string +number | one based position of token to return + +example: +```json +{ + "operation": "shift", + "spec": { + "output": "tests.test_string | splitn o 2" + } +} +``` + +produces: +```json +{ + "output": "wn f" +} +``` + +#### Eqs #### + +Returns `true` or `false` based on whether the value equals the parameter + +Argument | Description +---------|------------ +any | value to compare + + +example: +```json +{ + "operation": "shift", + "spec": { + "output": "tests.test_string | eqs \"The quick brown fox\"" + } +} +``` + +produces: +```json +{ + "output": true +} +``` + + ## Usage diff --git a/converter/eqs.go b/converter/eqs.go new file mode 100644 index 0000000..c32112c --- /dev/null +++ b/converter/eqs.go @@ -0,0 +1,28 @@ +package converter + +import ( + "bytes" + "github.com/mbordner/kazaam/transform" +) + +type Eqs struct { + ConverterBase +} + +func (c *Eqs) Convert(jsonData []byte, value []byte, args []byte) (newValue []byte, err error) { + + var argsValue *transform.JSONValue + argsValue, err = c.NewJSONValue(args) + if err != nil { + return + } + + argsBytes := []byte(argsValue.GetStringValue()) + + if bytes.Equal(value, argsBytes) == true { + return []byte("true"), nil + } + + return []byte("false"), nil + +} diff --git a/converter/eqs_test.go b/converter/eqs_test.go new file mode 100644 index 0000000..414e8d9 --- /dev/null +++ b/converter/eqs_test.go @@ -0,0 +1,37 @@ +package converter + +import ( + "strconv" + "testing" + "github.com/mbordner/kazaam/registry" +) + +func TestEqs_Convert(t *testing.T) { + + registry.RegisterConverter("eqs", &Eqs{}) + c := registry.GetConverter("eqs") + + table := []struct { + value string + arguments string + expected string + }{ + {`"The quick brown fox jumps over the lazy dog"`, `"The quick brown fox jumps over the lazy dog"`, `true`,}, + {`42`,`42`,`true`,}, + } + + for _, test := range table { + v, e := c.Convert(getTestData(), []byte(test.value), []byte(strconv.Quote(test.arguments))) + + if e != nil { + t.Error("error running convert function") + } + + if string(v) != test.expected { + t.Error("unexpected result from convert") + t.Log("Expected: {}", test.expected) + t.Log("Actual: {}", string(v)) + } + } + +} \ No newline at end of file diff --git a/converter/len.go b/converter/len.go new file mode 100644 index 0000000..3be4045 --- /dev/null +++ b/converter/len.go @@ -0,0 +1,32 @@ +package converter + +import ( + "fmt" + "github.com/mbordner/kazaam/transform" + "github.com/pkg/errors" +) + +type Len struct { + ConverterBase +} + +func (c *Len) Convert(jsonData []byte, value []byte, args []byte) (newValue []byte, err error) { + newValue = value + + var jsonValue *transform.JSONValue + jsonValue, err = c.NewJSONValue(value) + if err != nil { + return + } + + if jsonValue.IsString() == false { + err = errors.New("value must be string for len converter") + return + } + + origValue := jsonValue.GetStringValue() + + newValue = []byte(fmt.Sprintf("%d", len(origValue))) + + return +} diff --git a/converter/len_test.go b/converter/len_test.go new file mode 100644 index 0000000..644d146 --- /dev/null +++ b/converter/len_test.go @@ -0,0 +1,37 @@ +package converter + +import ( + "github.com/mbordner/kazaam/registry" + "strconv" + "testing" +) + +func TestLen_Convert(t *testing.T) { + + registry.RegisterConverter("len", &Len{}) + c := registry.GetConverter("len") + + table := []struct { + value string + arguments string + expected string + }{ + {`"The quick brown fox jumps over the lazy dog"`, ``, `43`,}, + {`"the lazy dog"`, ``, `12`,}, + } + + for _, test := range table { + v, e := c.Convert(getTestData(), []byte(test.value), []byte(strconv.Quote(test.arguments))) + + if e != nil { + t.Error("error running convert function") + } + + if string(v) != test.expected { + t.Error("unexpected result from convert") + t.Log("Expected: {}", test.expected) + t.Log("Actual: {}", string(v)) + } + } + +} diff --git a/converter/splitn.go b/converter/splitn.go new file mode 100644 index 0000000..9d0315e --- /dev/null +++ b/converter/splitn.go @@ -0,0 +1,65 @@ +package converter + +import ( + "errors" + "github.com/mbordner/kazaam/transform" + "regexp" + "strconv" + "strings" +) + +type Splitn struct { + ConverterBase +} + +// |substr start end , end is optional, and will be the last char sliced's index + 1, +// start is the start index and required +func (c *Splitn) Convert(jsonData []byte, value []byte, args []byte) (newValue []byte, err error) { + newValue = value + + var jsonValue, argsValue *transform.JSONValue + jsonValue, err = c.NewJSONValue(value) + if err != nil { + return + } + + argsValue, err = transform.NewJSONValue(args) + if err != nil { + return + } + + if jsonValue.IsString() == false || argsValue.IsString() == false { + err = errors.New("invalid value or arguments for splintn converter") + return + } + + var re *regexp.Regexp + re, err = regexp.Compile(`(?Us)^(?:\s*)(.+)(?:\s*)(\d+)*(?:\s*)$`) + if err != nil { + return + } + + argsString := argsValue.GetStringValue() + origValue := jsonValue.GetStringValue() + + var n int64 + + newValue = []byte("null") + + if matches := re.FindStringSubmatch(argsString); matches != nil { + n, err = strconv.ParseInt(matches[2], 10, 64) + if err != nil { + return + } + + vals := strings.SplitN(origValue,matches[1],int(n)+1) + + if len(vals) >= int(n) { + newValue = []byte(strconv.Quote(vals[int(n)-1])) + } + + } + + + return +} diff --git a/converter/splitn_test.go b/converter/splitn_test.go new file mode 100644 index 0000000..f108810 --- /dev/null +++ b/converter/splitn_test.go @@ -0,0 +1,38 @@ +package converter + +import ( + "github.com/mbordner/kazaam/registry" + "strconv" + "testing" +) + +func TestSplitn_Convert(t *testing.T) { + + registry.RegisterConverter("splitn", &Splitn{}) + c := registry.GetConverter("splitn") + + table := []struct { + value string + arguments string + expected string + }{ + {`"aazbbzcczdd""`, `z 4`, `"dd"`,}, + {`"abc|def|ghi|jkl|mno"`, `| 2`, `"def"`,}, + {"\"abc\\ndef\\nghi\\njkl\\nmno\"", "\n 5", `"mno"`,}, + } + + for _, test := range table { + v, e := c.Convert(getTestData(), []byte(test.value), []byte(strconv.Quote(test.arguments))) + + if e != nil { + t.Error("error running convert function") + } + + if string(v) != test.expected { + t.Error("unexpected result from convert") + t.Log("Expected: {}", test.expected) + t.Log("Actual: {}", string(v)) + } + } + +} diff --git a/kazaam.go b/kazaam.go index 73e4f4e..788f34d 100644 --- a/kazaam.go +++ b/kazaam.go @@ -8,10 +8,10 @@ import ( "fmt" "strings" - "github.com/qntfy/jsonparser" "github.com/mbordner/kazaam/converter" "github.com/mbordner/kazaam/registry" "github.com/mbordner/kazaam/transform" + "github.com/qntfy/jsonparser" ) // TransformFunc defines the contract that any Transform function implementation @@ -40,6 +40,7 @@ func init() { "coalesce": transform.Coalesce, "timestamp": transform.Timestamp, "uuid": transform.UUID, + "steps": transform.Steps, } defaultConverters = map[string]registry.Converter{ @@ -58,6 +59,9 @@ func init() { "floor": &converter.Floor{}, "format": &converter.Format{}, "div": &converter.Div{}, + "len": &converter.Len{}, + "splitn": &converter.Splitn{}, + "eqs": &converter.Eqs{}, } for name, conv := range defaultConverters { diff --git a/kazaam_test.go b/kazaam_test.go index f5a8341..a85575e 100644 --- a/kazaam_test.go +++ b/kazaam_test.go @@ -38,7 +38,7 @@ func TestReregisterKazaamTransform(t *testing.T) { } func TestDefaultTransformsSetCardinarily(t *testing.T) { - if len(defaultSpecTypes) != 9 { + if len(defaultSpecTypes) != 10 { t.Error("Unexpected number of default transforms. Missing tests?") } } diff --git a/transform/steps.go b/transform/steps.go new file mode 100644 index 0000000..0cad779 --- /dev/null +++ b/transform/steps.go @@ -0,0 +1,90 @@ +package transform + +import ( + "fmt" +) + +// Shift moves values from one provided json path to another in raw []byte. +func Steps(spec *Config, data []byte) ([]byte, error) { + var outData []byte + if spec.InPlace { + outData = data + } else { + outData = []byte(`{}`) + } + + if steps, ok := (*spec.Spec)["steps"]; ok { + + for _, s := range steps.([]interface{}) { + stepSpec := s.(map[string]interface{}) + + for k, v := range stepSpec { + array := true + var keyList []string + + // check if `v` is a string or list and build a list of keys to evaluate + switch v.(type) { + case string: + keyList = append(keyList, v.(string)) + array = false + case []interface{}: + for _, vItem := range v.([]interface{}) { + vItemStr, found := vItem.(string) + if !found { + return nil, ParseError(fmt.Sprintf("Warn: Unable to coerce element to json string: %v", vItem)) + } + keyList = append(keyList, vItemStr) + } + default: + return nil, ParseError(fmt.Sprintf("Warn: Unknown type in message for key: %s", k)) + } + + // iterate over keys to evaluate + // Note: this could be sped up significantly (especially for large shift transforms) + // by using `jsonparser.EachKey()` to iterate through data once and pick up all the + // needed underlying data. It would be a non-trivial update since you'd have to make + // recursive calls and keep track of all the key paths at each level. + // Currently we iterate at worst once per key in spec, with a better design it would be once + // per spec. + for _, v := range keyList { + var dataForV []byte + var err error + + // grab the data + if v == "$" { + dataForV = data + } else { + dataForV, err = getJSONRaw(data, v, spec.Require) + if err != nil { + if _, ok := err.(CPathSkipError); ok { // was a conditional path, + continue + } + return nil, err + } + } + + // if array flag set, encapsulate data + if array { + // bookend() is destructive to underlying slice, need to copy. + // extra capacity saves an allocation and copy during bookend. + tmp := make([]byte, len(dataForV), len(dataForV)+2) + copy(tmp, dataForV) + dataForV = bookend(tmp, '[', ']') + } + // Note: following pattern from current Shift() - if multiple elements are included in an array, + // they will each successively overwrite each other and only the last element will be included + // in the transformed data. + outData, err = setJSONRaw(outData, dataForV, k) + if err != nil { + return nil, err + } + } + } + + data = outData + } + + } + + return outData, nil +} diff --git a/transform/steps_test.go b/transform/steps_test.go new file mode 100644 index 0000000..26d6805 --- /dev/null +++ b/transform/steps_test.go @@ -0,0 +1,19 @@ +package transform + +import "testing" + +func TestSteps(t *testing.T) { + jsonOut := `{"example":{"old":{"value":3}},"Rating":3}` + spec := `{"steps":[{"example.old": "rating.example"},{"Rating": "example.old.value"}]}` + + cfg := getConfig(spec, false) + kazaamOut, _ := getTransformTestWrapper(Steps, cfg, `{"rating":{"example":{"value":3},"primary":{"value":3}}}`) + areEqual, _ := checkJSONBytesEqual(kazaamOut, []byte(jsonOut)) + + if !areEqual { + t.Error("Transformed data does not match expectation.") + t.Log("Expected: ", jsonOut) + t.Log("Actual: ", string(kazaamOut)) + t.FailNow() + } +} \ No newline at end of file