Dynamic fields in rename

Add the ability to rename fields dynamic for the rename op.

compiler/ast/dag/op.go

@@ -95,9 +95,9 @@ type (
 		Args []Assignment `json:"args"`
 	}
 	Rename struct {
-		Kind string  `json:"kind" unpack:""`
-		Dsts []*This `json:"dsts"`
-		Srcs []*This `json:"srcs"`
+		Kind string `json:"kind" unpack:""`
+		Dsts []Path `json:"dsts"`
+		Srcs []Path `json:"srcs"`
 	}
 	Scatter struct {
 		Kind  string `json:"kind" unpack:""`

compiler/kernel/op.go

@@ -177,10 +177,18 @@ func (b *Builder) compileLeaf(o dag.Op, parent zbuf.Puller) (zbuf.Puller, error)
 		putter := expr.NewPutter(b.octx.Zctx, clauses)
 		return op.NewApplier(b.octx, parent, putter), nil
 	case *dag.Rename:
-		var srcs, dsts field.List
+		var srcs, dsts []*expr.Path
 		for k := range v.Dsts {
-			srcs = append(srcs, v.Srcs[k].Path)
-			dsts = append(dsts, v.Dsts[k].Path)
+			src, err := b.compilePath(v.Srcs[k])
+			if err != nil {
+				return nil, err
+			}
+			dst, err := b.compilePath(v.Dsts[k])
+			if err != nil {
+				return nil, err
+			}
+			srcs = append(srcs, src)
+			dsts = append(dsts, dst)
 		}
 		renamer := expr.NewRenamer(b.octx.Zctx, srcs, dsts)
 		return op.NewApplier(b.octx, parent, renamer), nil

compiler/optimizer/op.go

@@ -53,8 +53,8 @@ func (o *Optimizer) analyzeSortKey(op dag.Op, in order.SortKey) (order.SortKey,
 	case *dag.Rename:
 		out := in
 		for k := range op.Dsts {
-			if fieldOf(op.Srcs[k]).Equal(key) {
-				lhs := fieldOf(op.Dsts[k])
+			if fieldOf(&op.Srcs[k]).Equal(key) {
+				lhs := fieldOf(&op.Dsts[k])
 				out = order.NewSortKey(in.Order, field.List{lhs})
 			}
 		}

compiler/semantic/op.go

@@ -650,28 +650,33 @@ func (a *analyzer) semOp(o ast.Op, seq dag.Seq) (dag.Seq, error) {
 		}
 		return append(seq, converted), nil
 	case *ast.Rename:
-		var dsts, srcs []*dag.This
+		var dsts, srcs []dag.Path
 		for _, arg := range o.Args {
-			dst, err := a.semStaticField(arg.LHS)
+			dst, err := a.semPath(arg.LHS)
 			if err != nil {
 				return nil, errors.New("rename: requires explicit field references")
 			}
-			src, err := a.semStaticField(arg.RHS)
+			src, err := a.semPath(arg.RHS)
 			if err != nil {
 				return nil, errors.New("rename: requires explicit field references")
 			}
-			if len(dst.Path) != len(src.Path) {
-				return nil, fmt.Errorf("rename: cannot rename %s to %s", src, dst)
-			}
-			// Check that the prefixes match and, if not, report first place
-			// that they don't.
-			for i := 0; i <= len(src.Path)-2; i++ {
-				if src.Path[i] != dst.Path[i] {
-					return nil, fmt.Errorf("rename: cannot rename %s to %s (differ in %s vs %s)", src, dst, src.Path[i], dst.Path[i])
+			// If both paths are static validate rename paths, otherwise this
+			// will be done at runtime.
+			if src, dst := src.StaticPath(), dst.StaticPath(); src != nil && dst != nil {
+				s, d := field.Path(src.Path), field.Path(dst.Path)
+				if len(dst.Path) != len(src.Path) {
+					return nil, fmt.Errorf("rename: cannot rename %s to %s", s, d)
+				}
+				// Check that the prefixes match and, if not, report first place
+				// that they don't.
+				for i := 0; i <= len(src.Path)-2; i++ {
+					if src.Path[i] != dst.Path[i] {
+						return nil, fmt.Errorf("rename: cannot rename %s to %s (differ in %s vs %s)", s, d, s[i], d[i])
+					}
 				}
 			}
-			dsts = append(dsts, dst)
-			srcs = append(srcs, src)
+			dsts = append(dsts, *dst)
+			srcs = append(srcs, *src)
 		}
 		return append(seq, &dag.Rename{
 			Kind: "Rename",

compiler/ztests/dynamic-field-rename.yaml

@@ -0,0 +1,22 @@
+script: |
+  echo '{target:"foo",src:"bar"} {target:"fool",src:"baz"}' | zq -z 'rename this[target] := src' -
+  echo '// ==='
+  echo '{target:"a",a:"bar"} {target:"b",b:"baz"}' | zq -z 'rename dst := this[target]' -
+  # runtime error cases
+  echo '// ==='
+  echo '{foo:"a",bar:"b"}' | zq -z 'rename this[foo]["c"] := this[bar]["d"]' -
+  echo '// ==='
+  echo '{foo:"a"}' | zq -z 'rename this[foo]["c"] := this[foo]["a"]["b"]' -
+
+outputs:
+  - name: stdout
+    data: |
+      {target:"foo",foo:"bar"}
+      {target:"fool",fool:"baz"}
+      // ===
+      {target:"a",dst:"bar"}
+      {target:"b",dst:"baz"}
+      // ===
+      error({message:"rename: cannot rename b.d to a.c (differ in b vs a)",on:{foo:"a",bar:"b"}})
+      // ===
+      error({message:"rename: cannot rename a.a.b to a.c",on:{foo:"a"}})

pkg/field/field.go

@@ -15,6 +15,20 @@ func (p Path) String() string {
 	return strings.Join(p, ".")
 }
 
+// AppendTo appends the string representation of the path to byte slice b.
+func (p Path) AppendTo(b []byte) []byte {
+	if len(p) == 0 {
+		return append(b, "this"...)
+	}
+	for i, s := range p {
+		if i > 0 {
+			b = append(b, '.')
+		}
+		b = append(b, s...)
+	}
+	return b
+}
+
 func (p Path) Leaf() string {
 	return p[len(p)-1]
 }
@@ -65,6 +79,17 @@ func (l List) String() string {
 	return strings.Join(paths, ",")
 }
 
+// AppendTo appends the string representation of the list to byte slice b.
+func (l List) AppendTo(b []byte) []byte {
+	for i, p := range l {
+		if i > 0 {
+			b = append(b, ',')
+		}
+		b = p.AppendTo(b)
+	}
+	return b
+}
+
 func (l List) Has(in Path) bool {
 	return slices.ContainsFunc(l, in.Equal)
 }

runtime/expr/renamer.go

@@ -17,13 +17,84 @@ type Renamer struct {
 	zctx *zed.Context
 	// For the dst field name, we just store the leaf name since the
 	// src path and the dst path are the same and only differ in the leaf name.
-	srcs    field.List
-	dsts    field.List
-	typeMap map[int]*zed.TypeRecord
+	srcs    []*Path
+	dsts    []*Path
+	typeMap map[int]map[string]*zed.TypeRecord
+	// fieldsStr is used to reduce allocations when computing the fields id.
+	fieldsStr []byte
 }
 
-func NewRenamer(zctx *zed.Context, srcs, dsts field.List) *Renamer {
-	return &Renamer{zctx, srcs, dsts, make(map[int]*zed.TypeRecord)}
+func NewRenamer(zctx *zed.Context, srcs, dsts []*Path) *Renamer {
+	return &Renamer{zctx, srcs, dsts, make(map[int]map[string]*zed.TypeRecord), nil}
+}
+
+func (r *Renamer) Eval(ectx Context, this *zed.Value) *zed.Value {
+	if !zed.IsRecordType(this.Type) {
+		return this
+	}
+	srcs, dsts, verr := r.evalFields(ectx, this)
+	if verr != nil {
+		if !verr.IsMissing() && !verr.IsQuiet() {
+			s := fmt.Sprintf("rename: %s", zed.DecodeError(verr.Bytes()))
+			verr = ectx.CopyValue(*r.zctx.WrapError(s, this))
+		}
+		return verr
+	}
+	id := this.Type.ID()
+	m, ok := r.typeMap[id]
+	if !ok {
+		m = make(map[string]*zed.TypeRecord)
+		r.typeMap[id] = m
+	}
+	r.fieldsStr = srcs.AppendTo(r.fieldsStr[:0])
+	r.fieldsStr = dsts.AppendTo(r.fieldsStr)
+	typ, ok := m[string(r.fieldsStr)]
+	if !ok {
+		var err error
+		typ, err = r.computeType(zed.TypeRecordOf(this.Type), srcs, dsts)
+		if err != nil {
+			return ectx.CopyValue(*r.zctx.WrapError(fmt.Sprintf("rename: %s", err), this))
+		}
+		m[string(r.fieldsStr)] = typ
+	}
+	out := this.Copy()
+	return ectx.NewValue(typ, out.Bytes())
+}
+
+func (r *Renamer) evalFields(ectx Context, this *zed.Value) (field.List, field.List, *zed.Value) {
+	var srcs, dsts field.List
+	for i := range r.srcs {
+		src, verr := r.srcs[i].Eval(ectx, this)
+		if verr != nil {
+			return nil, nil, verr
+		}
+		dst, verr := r.dsts[i].Eval(ectx, this)
+		if verr != nil {
+			return nil, nil, verr
+		}
+		if len(src) != len(dst) {
+			return nil, nil, ectx.CopyValue(*r.zctx.NewErrorf("cannot rename %s to %s", src, dst))
+		}
+		for i := 0; i <= len(src)-2; i++ {
+			if src[i] != dst[i] {
+				return nil, nil, ectx.CopyValue(*r.zctx.NewErrorf("cannot rename %s to %s (differ in %s vs %s)", src, dst, src[i], dst[i]))
+			}
+		}
+		srcs = append(srcs, src)
+		dsts = append(dsts, dst)
+	}
+	return srcs, dsts, nil
+}
+
+func (r *Renamer) computeType(typ *zed.TypeRecord, srcs, dsts field.List) (*zed.TypeRecord, error) {
+	for k, dst := range dsts {
+		var err error
+		typ, err = r.dstType(typ, srcs[k], dst)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return typ, nil
 }
 
 func (r *Renamer) dstType(typ *zed.TypeRecord, src, dst field.Path) (*zed.TypeRecord, error) {
@@ -57,32 +128,3 @@ func (r *Renamer) dstType(typ *zed.TypeRecord, src, dst field.Path) (*zed.TypeRe
 	}
 	return typ, nil
 }
-
-func (r *Renamer) computeType(typ *zed.TypeRecord) (*zed.TypeRecord, error) {
-	for k, dst := range r.dsts {
-		var err error
-		typ, err = r.dstType(typ, r.srcs[k], dst)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return typ, nil
-}
-
-func (r *Renamer) Eval(ectx Context, this *zed.Value) *zed.Value {
-	if !zed.IsRecordType(this.Type) {
-		return this
-	}
-	id := this.Type.ID()
-	typ, ok := r.typeMap[id]
-	if !ok {
-		var err error
-		typ, err = r.computeType(zed.TypeRecordOf(this.Type))
-		if err != nil {
-			return r.zctx.WrapError(fmt.Sprintf("rename: %s", err), this)
-		}
-		r.typeMap[id] = typ
-	}
-	out := this.Copy()
-	return ectx.NewValue(typ, out.Bytes())
-}

zfmt/dag.go

@@ -439,9 +439,9 @@ func (c *canonDAG) op(p dag.Op) {
 		c.next()
 		c.write("rename ")
 		for k := range p.Dsts {
-			c.fieldpath(p.Dsts[k].Path)
+			c.expr(&p.Dsts[k], "")
 			c.write(":=")
-			c.fieldpath(p.Srcs[k].Path)
+			c.expr(&p.Srcs[k], "")
 		}
 		// XXX
 		// c.assignments(p.Args)