Add Expression Inliner for includes

expr/include.go

@@ -0,0 +1,82 @@
+package expr
+
+import (
+	"fmt"
+
+	u "github.com/araddon/gou"
+
+	"github.com/araddon/qlbridge/lex"
+)
+
+const (
+	maxIncludeDepth = 100
+)
+
+var (
+
+	// If we hit max depth
+	ErrMaxDepth = fmt.Errorf("Recursive Evaluation Error")
+)
+
+// InlineIncludes take an expression and resolve any includes so that
+// the included expression is "Inline"
+func InlineIncludes(ctx Includer, n Node) (Node, error) {
+	return inlineIncludesDepth(ctx, n, 0)
+}
+func inlineIncludesDepth(ctx Includer, arg Node, depth int) (Node, error) {
+	if depth > maxIncludeDepth {
+		return nil, ErrMaxDepth
+	}
+
+	switch n := arg.(type) {
+	case NodeArgs:
+		args := n.ChildrenArgs()
+		for i, narg := range args {
+			newNode, err := inlineIncludesDepth(ctx, narg, depth+1)
+			if err != nil {
+				return nil, err
+			}
+			if newNode != nil {
+				args[i] = newNode
+			}
+		}
+		return arg, nil
+	case *NumberNode, *IdentityNode, *StringNode, nil,
+		*ValueNode, *NullNode:
+		return nil, nil
+	case *IncludeNode:
+		return resolveInclude(ctx, n, depth+1)
+	}
+	return nil, fmt.Errorf("unrecognized node type %T", arg)
+}
+
+func resolveInclude(ctx Includer, inc *IncludeNode, depth int) (Node, error) {
+
+	if inc.inlineExpr == nil {
+		n, err := ctx.Include(inc.Identity.Text)
+		if err != nil {
+			// ErrNoIncluder is pretty common so don't log it
+			if err == ErrNoIncluder {
+				return nil, err
+			}
+			u.Debugf("Could not find include for filter:%s err=%v", inc.String(), err)
+			return nil, err
+		}
+		if n == nil {
+			u.Debugf("Includer %T returned a nil filter statement!", inc)
+			return nil, ErrIncludeNotFound
+		}
+		// Now inline, the inlines
+		n, err = InlineIncludes(ctx, n)
+		if err != nil {
+			return nil, err
+		}
+		if inc.Negated() {
+			inc.inlineExpr = NewUnary(lex.Token{T: lex.TokenNegate, V: "NOT"}, n)
+		} else {
+			inc.inlineExpr = n
+		}
+
+	}
+	return inc.inlineExpr, nil
+}

expr/include_test.go

@@ -0,0 +1,104 @@
+package expr_test
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/araddon/qlbridge/datasource"
+	"github.com/araddon/qlbridge/expr"
+	"github.com/araddon/qlbridge/rel"
+	//"github.com/araddon/qlbridge/vm"
+)
+
+type includectx struct {
+	expr.ContextReader
+	segs map[string]*rel.FilterStatement
+}
+
+func newIncluderCtx(cr expr.ContextReader, statements string) *includectx {
+	stmts := rel.MustParseFilters(statements)
+	segs := make(map[string]*rel.FilterStatement, len(stmts))
+	for _, stmt := range stmts {
+		segs[strings.ToLower(stmt.Alias)] = stmt
+	}
+	return &includectx{ContextReader: cr, segs: segs}
+}
+func (m *includectx) Include(name string) (expr.Node, error) {
+	if seg, ok := m.segs[strings.ToLower(name)]; ok {
+		return seg.Filter, nil
+	}
+	return nil, expr.ErrNoIncluder
+}
+
+type incTest struct {
+	in  string
+	out string
+}
+
+func TestInlineIncludes(t *testing.T) {
+
+	t1 := time.Now()
+
+	readers := []expr.ContextReader{
+		datasource.NewContextMap(map[string]interface{}{
+			"name":       "bob",
+			"city":       "Peoria, IL",
+			"zip":        5,
+			"signedup":   t1,
+			"lastevent":  map[string]time.Time{"signedup": t1},
+			"last.event": map[string]time.Time{"has.period": t1},
+		}, true),
+	}
+
+	nc := datasource.NewNestedContextReader(readers, time.Now())
+	includerCtx := newIncluderCtx(nc, `
+		FILTER name == "Yoda" ALIAS is_yoda_true;
+		FILTER AND (
+			planet == "Dagobah"
+			INCLUDE is_yoda_true
+		) ALIAS nested_includes_yoda;
+	`)
+
+	tests := []incTest{
+		{
+			in:  `lastvisit_ts < "now-1d"`,
+			out: `lastvisit_ts < "now-1d"`,
+		},
+		{
+			in:  `AND ( lastvisit_ts < "now-1d", INCLUDE is_yoda_true )`,
+			out: `AND ( lastvisit_ts < "now-1d", name == "Yoda" )`,
+		},
+		{
+			in:  `AND ( lastvisit_ts < "now-1d", NOT INCLUDE is_yoda_true )`,
+			out: `AND ( lastvisit_ts < "now-1d", NOT (name == "Yoda") )`,
+		},
+		{
+			in:  `AND ( lastvisit_ts < "now-1d", NOT INCLUDE nested_includes_yoda )`,
+			out: `AND ( lastvisit_ts < "now-1d", NOT AND ( planet == "Dagobah", (name == "Yoda") ) )`,
+		},
+	}
+	for _, tc := range tests {
+		n := expr.MustParse(tc.in)
+		out, err := expr.InlineIncludes(includerCtx, n)
+		assert.Equal(t, nil, err)
+		assert.NotEqual(t, nil, out)
+		if out != nil {
+			assert.Equal(t, tc.out, out.String())
+		}
+	}
+
+	testsErr := []incTest{
+		{
+			in:  `AND ( lastvisit_ts < "now-1d", INCLUDE not_gonna_be_found )`,
+			out: `AND ( lastvisit_ts < "now-1d", name == "Yoda" )`,
+		},
+	}
+	for _, tc := range testsErr {
+		n := expr.MustParse(tc.in)
+		_, err := expr.InlineIncludes(includerCtx, n)
+		assert.NotEqual(t, nil, err)
+	}
+}

expr/node.go

@@ -57,6 +57,14 @@ var (
 
 	// Ensure we implement interface
 	_ Includer = (*IncludeContext)(nil)
+
+	// Ensure some of our nodes implement NodeArgs
+	_ NodeArgs = (*BooleanNode)(nil)
+	_ NodeArgs = (*TriNode)(nil)
+	_ NodeArgs = (*BinaryNode)(nil)
+	_ NodeArgs = (*FuncNode)(nil)
+	_ NodeArgs = (*UnaryNode)(nil)
+	_ NodeArgs = (*ArrayNode)(nil)
 )
 
 type (
@@ -93,6 +101,11 @@ type (
 		NodeType() string
 	}
 
+	// NodeArgs is an interface for nodes which have child arguments
+	NodeArgs interface {
+		ChildrenArgs() []Node
+	}
+
 	// A negateable node requires a special type of String() function due to
 	// an enclosing urnary NOT being inserted into middle of string syntax
 	//
@@ -281,10 +294,11 @@ type (
 	//   (  ! INCLUDE <identity>  |  INCLUDE <identity> | NOT INCLUDE <identity> )
 	//
 	IncludeNode struct {
-		negated  bool
-		ExprNode Node
-		Identity *IdentityNode
-		Operator lex.Token
+		negated    bool
+		inlineExpr Node
+		ExprNode   Node
+		Identity   *IdentityNode
+		Operator   lex.Token
 	}
 
 	// Array Node for holding multiple similar elements
@@ -590,6 +604,9 @@ func (m *FuncNode) Validate() error {
 	}
 	return nil
 }
+func (m *FuncNode) ChildrenArgs() []Node {
+	return m.Args
+}
 func (m *FuncNode) NodePb() *NodePb {
 	n := &FuncNodePb{}
 	n.Name = m.Name
@@ -1339,6 +1356,9 @@ func (m *BinaryNode) Validate() error {
 	}
 	return nil
 }
+func (m *BinaryNode) ChildrenArgs() []Node {
+	return m.Args
+}
 func (m *BinaryNode) NodePb() *NodePb {
 	n := &BinaryNodePb{}
 	n.Paren = m.Paren
@@ -1484,6 +1504,9 @@ func (m *BooleanNode) Validate() error {
 	}
 	return nil
 }
+func (m *BooleanNode) ChildrenArgs() []Node {
+	return m.Args
+}
 func (m *BooleanNode) NodePb() *NodePb {
 	n := &BooleanNodePb{}
 	n.Op = int32(m.Operator.T)
@@ -1607,6 +1630,9 @@ func (m *TriNode) Validate() error {
 	}
 	return nil
 }
+func (m *TriNode) ChildrenArgs() []Node {
+	return m.Args
+}
 func (m *TriNode) NodePb() *NodePb {
 	n := &TriNodePb{Args: make([]NodePb, len(m.Args))}
 	n.Op = int32(m.Operator.T)
@@ -1738,6 +1764,9 @@ func (m *UnaryNode) WriteDialect(w DialectWriter) {
 func (m *UnaryNode) Validate() error {
 	return m.Arg.Validate()
 }
+func (m *UnaryNode) ChildrenArgs() []Node {
+	return []Node{m.Arg}
+}
 func (m *UnaryNode) Node() Node { return m }
 func (m *UnaryNode) NodePb() *NodePb {
 	n := &UnaryNodePb{}
@@ -1954,6 +1983,9 @@ func (m *ArrayNode) Validate() error {
 	}
 	return nil
 }
+func (m *ArrayNode) ChildrenArgs() []Node {
+	return m.Args
+}
 func (m *ArrayNode) Append(n Node) { m.Args = append(m.Args, n) }
 func (m *ArrayNode) NodePb() *NodePb {
 	n := &ArrayNodePb{Args: make([]NodePb, len(m.Args))}

rel/parse_filterql.go

@@ -13,7 +13,9 @@ import (
 
 // FilterQLParser
 type FilterQLParser struct {
-	statement string //can be a FilterStatement, FilterStatements, filterSelect, filterSelects, etc.  Which one is determined by which Parser func you call.
+	// can be a FilterStatement, FilterStatements, filterSelect, filterSelects, etc.
+	// Which one is determined by which Parser func you call.
+	statement string
 	fs        *FilterStatement
 	l         *lex.Lexer
 	comment   string