CVL Changes #6: Customized Xpath Engine integration (sonic-net#27)

Adding support for evaluating xpath expression (leafref, must, when expression) using customized open source xpath, xmlquery and jsonquery.

cvl/cvl.go

@@ -27,6 +27,7 @@ import (
  log "github.com/golang/glog"
  "github.com/go-redis/redis"
  "github.com/antchfx/xmlquery"
+ "github.com/antchfx/xpath"
  "github.com/antchfx/jsonquery"
  "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser"
  . "github.com/Azure/sonic-mgmt-common/cvl/internal/util"
@@ -177,6 +178,15 @@ func init() {
  SetTrace(true)
  }
 
+ xpath.SetKeyGetClbk(func(listName string) []string {
+ if modelInfo.tableInfo[listName] != nil {
+ return modelInfo.tableInfo[listName].keys
+ }
+
+ return nil
+ })
+
+
  ConfigFileSyncHandler()
 
  cvlCfgMap := ReadConfFile()
@@ -214,6 +224,14 @@ func init() {
  }
 
  dbCacheSet(false, "PORT", 0)
+
+ xpath.SetLogCallback(func(fmt string, args ...interface{}) {
+ if !IsTraceLevelSet(TRACE_SEMANTIC) {
+ return
+ }
+
+ TRACE_LOG(INFO_API, TRACE_SEMANTIC, "XPATH: " + fmt, args...)
+ })
 }
 
 func Debug(on bool) {

cvl/cvl_semantics.go

@@ -22,6 +22,7 @@ package cvl
 import (
  "strings"
  "encoding/xml"
+ "encoding/json"
  "github.com/antchfx/xmlquery"
  "github.com/antchfx/jsonquery"
  "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser"
@@ -691,6 +692,97 @@ func (c *CVL) setOperation(op CVLOperation) {
  }
 }
 
+//Add given YANG data buffer to Yang Validator
+//redisKeys - Set of redis keys
+//redisKeyFilter - Redis key filter in glob style pattern
+//keyNames - Names of all keys separated by "|"
+//predicate - Condition on keys/fields
+//fields - Fields to retrieve, separated by "|"
+//Return "," separated list of leaf nodes if only one leaf is requested
+//One leaf is used as xpath query result in other nested xpath
+func (c *CVL) addDepYangData(redisKeys []string, redisKeyFilter,
+ keyNames, predicate, fields, count string) string {
+
+ var v interface{}
+ tmpPredicate := ""
+
+ //Get filtered Redis data based on lua script 
+ //filter derived from Xpath predicate
+ if (predicate != "") {
+ tmpPredicate = "return (" + predicate + ")"
+ }
+
+ cfgData, err := luaScripts["filter_entries"].Run(redisClient, []string{},
+ redisKeyFilter, keyNames, tmpPredicate, fields, count).Result()
+
+ singleLeaf := "" //leaf data for single leaf
+
+ TRACE_LOG(INFO_API, TRACE_SEMANTIC, "addDepYangData() with redisKeyFilter=%s, " +
+ "predicate=%s, fields=%s, returned cfgData = %s, err=%v",
+ redisKeyFilter, predicate, fields, cfgData, err)
+
+ if (cfgData == nil) {
+ return ""
+ }
+
+ //Parse the JSON map received from lua script
+ b := []byte(cfgData.(string))
+ if err := json.Unmarshal(b, &v); err != nil {
+ return ""
+ }
+
+ var dataMap map[string]interface{} = v.(map[string]interface{})
+
+ dataTop, _ := jsonquery.ParseJsonMap(&dataMap)
+
+ for jsonNode := dataTop.FirstChild; jsonNode != nil; jsonNode=jsonNode.NextSibling {
+ //Generate YANG data for Yang Validator from Redis JSON
+ topYangNode, _ := c.generateYangListData(jsonNode, false)
+
+ if topYangNode == nil {
+ continue
+ }
+
+ if (topYangNode.FirstChild != nil) &&
+ (topYangNode.FirstChild.FirstChild != nil) {
+ //Add attribute mentioning that data is from db
+ addAttrNode(topYangNode.FirstChild.FirstChild, "db", "")
+ }
+
+ //Build single leaf data requested
+ singleLeaf = ""
+ for redisKey := topYangNode.FirstChild.FirstChild;
+ redisKey != nil; redisKey = redisKey.NextSibling {
+
+ for field := redisKey.FirstChild; field != nil;
+ field = field.NextSibling {
+ if (field.Data == fields) {
+ //Single field requested
+ singleLeaf = singleLeaf + field.FirstChild.Data + ","
+ break
+ }
+ }
+ }
+
+ //Merge with main YANG data cache
+ doc := &xmlquery.Node{Type: xmlquery.DocumentNode}
+ doc.FirstChild = topYangNode
+ doc.LastChild = topYangNode
+ topYangNode.Parent = doc
+ if c.mergeYangData(c.yv.root, doc) != CVL_SUCCESS {
+ continue
+ }
+ }
+
+
+ //remove last comma in case mulitple values returned
+ if (singleLeaf != "") {
+ return singleLeaf[:len(singleLeaf) - 1]
+ }
+
+ return ""
+}
+
 //Check delete constraint for leafref if key/field is deleted
 func (c *CVL) checkDeleteConstraint(cfgData []CVLEditConfigData,
  tableName, keyVal, field string) CVLRetCode {

cvl/internal/util/util.go

@@ -133,6 +133,10 @@ func IsTraceSet() bool {
  }
 }
 
+func IsTraceLevelSet(tracelevel CVLTraceLevel) bool {
+ return (cvlTraceFlags & (uint32)(tracelevel)) != 0
+}
+
 func TRACE_LEVEL_LOG(level log.Level, tracelevel CVLTraceLevel, fmtStr string, args ...interface{}) {
 
  if (IsTraceSet() == false) {

patches/apply.sh

@@ -43,6 +43,8 @@ patch -d ${DEST_DIR}/github.com/openconfig -p1 < ${PATCH_DIR}/ygot/ygot.patch
 patch -d ${DEST_DIR}/github.com/openconfig/goyang -p1 < ${PATCH_DIR}/goyang/goyang.patch
 
 patch -d ${DEST_DIR}/github.com/antchfx/jsonquery -p1 < ${PATCH_DIR}/jsonquery.patch
+patch -d ${DEST_DIR}/github.com/antchfx/xmlquery -p1 < ${PATCH_DIR}/xmlquery.patch
+patch -d ${DEST_DIR}/github.com/antchfx/xpath -p1 < ${PATCH_DIR}/xpath.patch
 
 patch -d ${DEST_DIR}/github.com/golang/glog -p1 < ${PATCH_DIR}/glog.patch
 

patches/jsonquery.patch

@@ -1,8 +1,70 @@
 diff --git a/node.go b/node.go
-index 76032bb..db73a1e 100644
+index 76032bb..f6103d9 100644
 --- a/node.go
 +++ b/node.go
-@@ -155,3 +155,9 @@ func Parse(r io.Reader) (*Node, error) {
+@@ -8,6 +8,7 @@ import (
+ "net/http"
+ "sort"
+ "strconv"
++ "strings"
+ )
+
+ // A NodeType is the type of a Node.
+@@ -110,6 +111,29 @@ func parseValue(x interface{}, top *Node, level int) {
+ addNode(n)
+ parseValue(vv, n, level+1)
+ }
++ case map[string]string:
++ var keys []string
++ for key := range v {
++ keys = append(keys, key)
++ }
++ sort.Strings(keys)
++ for _, key := range keys {
++ tmpKey := key
++ var tmpVal interface{}
++ tmpVal = v[key]
++ if (strings.HasSuffix(key, "@")) {
++ tmpKey = key[:len(key) - 1]
++ tmpValArr := []interface{}{}
++ for _, val := range strings.Split(v[key], ",") {
++ tmpValArr = append(tmpValArr, val)
++ }
++ tmpVal = tmpValArr
++ }
++
++ n := &Node{Data: tmpKey, Type: ElementNode, level: level}
++ addNode(n)
++ parseValue(tmpVal, n, level+1)
++ }
+ case map[string]interface{}:
+ // The Go’s map iteration order is random.
+ // (https://blog.golang.org/go-maps-in-action#Iteration-order)
+@@ -119,9 +143,21 @@ func parseValue(x interface{}, top *Node, level int) {
+ }
+ sort.Strings(keys)
+ for _, key := range keys {
+- n := &Node{Data: key, Type: ElementNode, level: level}
++ tmpKey := key
++ var tmpVal interface{}
++ tmpVal = v[key]
++ if (strings.HasSuffix(key, "@")) {
++ tmpKey = key[:len(key) - 1]
++ tmpValArr := []interface{}{}
++ for _, val := range strings.Split(v[key].(string), ",") {
++ tmpValArr = append(tmpValArr, val)
++ }
++ tmpVal = tmpValArr
++ }
++
++ n := &Node{Data: tmpKey, Type: ElementNode, level: level}
+ addNode(n)
+- parseValue(v[key], n, level+1)
++ parseValue(tmpVal, n, level+1)
+ }
+ case string:
+ n := &Node{Data: v, Type: TextNode, level: level}
+@@ -155,3 +191,9 @@ func Parse(r io.Reader) (*Node, error) {
  }
  return parse(b)
  }
@@ -12,3 +74,22 @@ index 76032bb..db73a1e 100644
 + parseValue(*jsonMap, doc, 1)
 + return doc, nil
 +}
+diff --git a/query.go b/query.go
+index d105962..e8db1d6 100644
+--- a/query.go
++++ b/query.go
+@@ -120,6 +120,14 @@ func (a *NodeNavigator) MoveToRoot() {
+ a.cur = a.root
+ }
+
++func (a *NodeNavigator) MoveToContext() {
++ return
++}
++
++func (a *NodeNavigator) CurrentPrefix() string {
++ return ""
++}
++
+ func (a *NodeNavigator) MoveToParent() bool {
+ if n := a.cur.Parent; n != nil {
+ a.cur = n

patches/xmlquery.patch

@@ -0,0 +1,94 @@
+diff --git a/node.go b/node.go
+index e86c0c3..028867c 100644
+--- a/node.go
++++ b/node.go
+@@ -48,7 +48,7 @@ type Node struct {
+
+ // InnerText returns the text between the start and end tags of the object.
+ func (n *Node) InnerText() string {
+- var output func(*bytes.Buffer, *Node)
++ /*var output func(*bytes.Buffer, *Node)
+ output = func(buf *bytes.Buffer, n *Node) {
+ switch n.Type {
+ case TextNode:
+@@ -64,7 +64,18 @@ func (n *Node) InnerText() string {
+
+ var buf bytes.Buffer
+ output(&buf, n)
+- return buf.String()
++ return buf.String()*/
++
++ if (n.Type == TextNode) {
++ return n.Data
++ } else if (n.Type == ElementNode) &&
++ (n.FirstChild != nil) &&
++ (n.FirstChild.Type == TextNode) {
++ return n.FirstChild.Data
++ }
++
++
++ return ""
+ }
+
+ func (n *Node) sanitizedData(preserveSpaces bool) string {
+diff --git a/query.go b/query.go
+index 146c2a4..f21b61b 100644
+--- a/query.go
++++ b/query.go
+@@ -49,6 +49,29 @@ func CreateXPathNavigator(top *Node) *NodeNavigator {
+ return &NodeNavigator{curr: top, root: top, attr: -1}
+ }
+
++//Evaluate XPath expression, the expression should evaluate to true or false
++func Eval(top, ctx *Node, exp *xpath.Expr) bool {
++ if exp == nil {
++ return false
++ }
++
++ v := exp.Evaluate(&NodeNavigator{curr: ctx, ctxt: ctx, root: top, attr: -1})
++
++ switch val := v.(type) {
++ case bool:
++ return val
++ case string:
++ return (val != "")
++ case float64:
++ return (val != 0)
++ case *xpath.NodeIterator:
++ return (val != nil)
++ }
++
++ //return v.(bool)
++ return false
++}
++
+ func getCurrentNode(it *xpath.NodeIterator) *Node {
+ n := it.Current().(*NodeNavigator)
+ if n.NodeType() == xpath.AttributeNode {
+@@ -145,7 +168,7 @@ func FindEachWithBreak(top *Node, expr string, cb func(int, *Node) bool) {
+ }
+
+ type NodeNavigator struct {
+- root, curr *Node
++ root, curr, ctxt *Node
+ attr int
+ }
+
+@@ -212,6 +235,17 @@ func (x *NodeNavigator) MoveToRoot() {
+ x.curr = x.root
+ }
+
++func (x *NodeNavigator) MoveToContext() {
++ x.curr = x.ctxt
++}
++
++func (x *NodeNavigator) CurrentPrefix() string {
++ if (x.ctxt != nil) {
++ return x.ctxt.Prefix
++ }
++ return ""
++}
++
+ func (x *NodeNavigator) MoveToParent() bool {
+ if x.attr != -1 {
+ x.attr = -1