From 2d108a744da6a4ffd5941e7c98328327b0904e3b Mon Sep 17 00:00:00 2001 From: Kazuma Watanabe Date: Mon, 29 Apr 2024 18:38:59 +0000 Subject: [PATCH] Add support for provider-contributed functions Follow up of https://github.com/hashicorp/terraform/pull/34394 --- terraform/lang/eval.go | 37 +++++-- terraform/lang/eval_test.go | 7 +- terraform/lang/function_calls.go | 109 +++++++++++++++++++ terraform/lang/functions.go | 38 ++++++- terraform/lang/functions_test.go | 14 +++ terraform/tfhcl/expressions_hclext.go | 150 ++++++++++++++++++++++++++ 6 files changed, 343 insertions(+), 12 deletions(-) create mode 100644 terraform/lang/function_calls.go create mode 100644 terraform/tfhcl/expressions_hclext.go diff --git a/terraform/lang/eval.go b/terraform/lang/eval.go index 44a0ad982..a9772eb99 100644 --- a/terraform/lang/eval.go +++ b/terraform/lang/eval.go @@ -16,13 +16,21 @@ import ( // Note that Terraform only expands dynamic blocks, but TFLint also expands // count/for_each here. // -// Expressions in expanded blocks are evaluated immediately, so all variables -// contained in attributes specified in the body schema are gathered. +// Expressions in expanded blocks are evaluated immediately, so all variables and +// function calls contained in attributes specified in the body schema are gathered. func (s *Scope) ExpandBlock(body hcl.Body, schema *hclext.BodySchema) (hcl.Body, hcl.Diagnostics) { traversals := tfhcl.ExpandVariablesHCLExt(body, schema) refs, diags := References(traversals) - ctx, ctxDiags := s.EvalContext(refs) + exprs := tfhcl.ExpandExpressionsHCLExt(body, schema) + funcCalls := []*FunctionCall{} + for _, expr := range exprs { + calls, funcDiags := FunctionCallsInExpr(expr) + diags = diags.Extend(funcDiags) + funcCalls = append(funcCalls, calls...) + } + + ctx, ctxDiags := s.EvalContext(refs, funcCalls) diags = diags.Extend(ctxDiags) return tfhcl.Expand(body, ctx), diags @@ -40,8 +48,10 @@ func (s *Scope) ExpandBlock(body hcl.Body, schema *hclext.BodySchema) (hcl.Body, // incomplete, but will always be of the requested type. func (s *Scope) EvalExpr(expr hcl.Expression, wantType cty.Type) (cty.Value, hcl.Diagnostics) { refs, diags := ReferencesInExpr(expr) + funcCalls, funcDiags := FunctionCallsInExpr(expr) + diags = diags.Extend(funcDiags) - ctx, ctxDiags := s.EvalContext(refs) + ctx, ctxDiags := s.EvalContext(refs, funcCalls) diags = diags.Extend(ctxDiags) if diags.HasErrors() { // We'll stop early if we found problems in the references, because @@ -72,16 +82,17 @@ func (s *Scope) EvalExpr(expr hcl.Expression, wantType cty.Type) (cty.Value, hcl } // EvalContext constructs a HCL expression evaluation context whose variable -// scope contains sufficient values to satisfy the given set of references. +// scope contains sufficient values to satisfy the given set of references +// and function calls. // // Most callers should prefer to use the evaluation helper methods that // this type offers, but this is here for less common situations where the // caller will handle the evaluation calls itself. -func (s *Scope) EvalContext(refs []*addrs.Reference) (*hcl.EvalContext, hcl.Diagnostics) { - return s.evalContext(refs, s.SelfAddr) +func (s *Scope) EvalContext(refs []*addrs.Reference, funcCalls []*FunctionCall) (*hcl.EvalContext, hcl.Diagnostics) { + return s.evalContext(refs, s.SelfAddr, funcCalls) } -func (s *Scope) evalContext(refs []*addrs.Reference, selfAddr addrs.Referenceable) (*hcl.EvalContext, hcl.Diagnostics) { +func (s *Scope) evalContext(refs []*addrs.Reference, selfAddr addrs.Referenceable, funcCalls []*FunctionCall) (*hcl.EvalContext, hcl.Diagnostics) { if s == nil { panic("attempt to construct EvalContext for nil Scope") } @@ -89,6 +100,16 @@ func (s *Scope) evalContext(refs []*addrs.Reference, selfAddr addrs.Referenceabl var diags hcl.Diagnostics vals := make(map[string]cty.Value) funcs := s.Functions() + // Provider-contributed functions introduced in Terraform v1.8 cannot be + // evaluated statically in many cases. Here, we avoid the error by dynamically + // generating an evaluation context in which the provider-contributed functions + // in the given expression are replaced with mock functions. + for _, call := range funcCalls { + if !call.IsProviderContributed() { + continue + } + funcs[call.Name] = NewMockFunction(call) + } ctx := &hcl.EvalContext{ Variables: vals, Functions: funcs, diff --git a/terraform/lang/eval_test.go b/terraform/lang/eval_test.go index 569fde1f9..0a6f928c4 100644 --- a/terraform/lang/eval_test.go +++ b/terraform/lang/eval_test.go @@ -215,10 +215,15 @@ func TestScopeEvalContext(t *testing.T) { t.Fatal(refsDiags) } + funcCalls, funcDiags := FunctionCallsInExpr(expr) + if funcDiags.HasErrors() { + t.Fatal(funcDiags) + } + scope := &Scope{ Data: data, } - ctx, ctxDiags := scope.EvalContext(refs) + ctx, ctxDiags := scope.EvalContext(refs, funcCalls) if ctxDiags.HasErrors() { t.Fatal(ctxDiags) } diff --git a/terraform/lang/function_calls.go b/terraform/lang/function_calls.go new file mode 100644 index 000000000..2586990b8 --- /dev/null +++ b/terraform/lang/function_calls.go @@ -0,0 +1,109 @@ +package lang + +import ( + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/hcl/v2/json" + "github.com/zclconf/go-cty/cty" +) + +// FunctionCall represents a function call in an HCL expression. +// The difference with hclsyntax.FunctionCallExpr is that +// function calls are also available in JSON syntax. +type FunctionCall struct { + Name string + ArgsCount int +} + +// FunctionCallsInExpr finds all of the function calls in the given expression. +func FunctionCallsInExpr(expr hcl.Expression) ([]*FunctionCall, hcl.Diagnostics) { + if expr == nil { + return nil, nil + } + + // For JSON syntax, walker is not implemented, + // so extract the hclsyntax.Node that we can walk on. + // See https://github.com/hashicorp/hcl/issues/543 + nodes, diags := walkableNodesInExpr(expr) + ret := []*FunctionCall{} + + for _, node := range nodes { + visitDiags := hclsyntax.VisitAll(node, func(n hclsyntax.Node) hcl.Diagnostics { + if funcCallExpr, ok := n.(*hclsyntax.FunctionCallExpr); ok { + ret = append(ret, &FunctionCall{ + Name: funcCallExpr.Name, + ArgsCount: len(funcCallExpr.Args), + }) + } + return nil + }) + diags = diags.Extend(visitDiags) + } + return ret, diags +} + +// IsProviderContributed returns true if the function is provider-contributed. +func (f *FunctionCall) IsProviderContributed() bool { + return strings.HasPrefix(f.Name, "provider::") +} + +// walkableNodesInExpr returns hclsyntax.Node from the given expression. +// If the expression is an hclsyntax expression, it is returned as is. +// If the expression is a JSON expression, it is parsed and +// hclsyntax.Node it contains is returned. +func walkableNodesInExpr(expr hcl.Expression) ([]hclsyntax.Node, hcl.Diagnostics) { + nodes := []hclsyntax.Node{} + + expr = hcl.UnwrapExpressionUntil(expr, func(expr hcl.Expression) bool { + _, native := expr.(hclsyntax.Expression) + return native || json.IsJSONExpression(expr) + }) + + if json.IsJSONExpression(expr) { + // HACK: For JSON expressions, we can get the JSON value as a literal + // without any prior HCL parsing by evaluating it in a nil context. + // We can take advantage of this property to walk through cty.Value + // that may contain HCL expressions instead of walking through + // expression nodes directly. + // See https://github.com/hashicorp/hcl/issues/642 + val, diags := expr.Value(nil) + if diags.HasErrors() { + return nodes, diags + } + + err := cty.Walk(val, func(path cty.Path, v cty.Value) (bool, error) { + if v.Type() != cty.String || v.IsNull() || !v.IsKnown() { + return true, nil + } + + node, parseDiags := hclsyntax.ParseTemplate([]byte(v.AsString()), expr.Range().Filename, expr.Range().Start) + if diags.HasErrors() { + diags = diags.Extend(parseDiags) + return true, nil + } + + nodes = append(nodes, node) + return true, nil + }) + if err != nil { + return nodes, hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to walk the expression value", + Detail: err.Error(), + Subject: expr.Range().Ptr(), + }} + } + + return nodes, diags + } + + // Usually only a native expression is here since a JSON expression is already + // processed, but we should check the type to avoid typed-nil. + if node, ok := expr.(hclsyntax.Expression); ok { + nodes = append(nodes, node) + } + + return nodes, nil +} diff --git a/terraform/lang/functions.go b/terraform/lang/functions.go index 3923166bb..df79f916b 100644 --- a/terraform/lang/functions.go +++ b/terraform/lang/functions.go @@ -31,7 +31,7 @@ func (s *Scope) Functions() map[string]function.Function { // later if the functionality seems to be something domain-agnostic // that would be useful to all applications using cty functions. - s.funcs = map[string]function.Function{ + coreFuncs := map[string]function.Function{ "abs": stdlib.AbsoluteFunc, "abspath": funcs.AbsPathFunc, "alltrue": funcs.AllTrueFunc, @@ -151,7 +151,7 @@ func (s *Scope) Functions() map[string]function.Function { "zipmap": stdlib.ZipmapFunc, } - s.funcs["templatefile"] = funcs.MakeTemplateFileFunc(s.BaseDir, func() map[string]function.Function { + coreFuncs["templatefile"] = funcs.MakeTemplateFileFunc(s.BaseDir, func() map[string]function.Function { // The templatefile function prevents recursive calls to itself // by copying this map and overwriting the "templatefile" entry. return s.funcs @@ -161,11 +161,43 @@ func (s *Scope) Functions() map[string]function.Function { // Force our few impure functions to return unknown so that we // can defer evaluating them until a later pass. for _, name := range impureFunctions { - s.funcs[name] = function.Unpredictable(s.funcs[name]) + coreFuncs[name] = function.Unpredictable(s.funcs[name]) } } + + // All of the built-in functions are also available under the "core::" + // namespace, to distinguish from the "provider::" and "module::" + // namespaces that can serve as external extension points. + s.funcs = make(map[string]function.Function, len(coreFuncs)*2) + for name, fn := range coreFuncs { + s.funcs[name] = fn + s.funcs["core::"+name] = fn + } } s.funcsLock.Unlock() return s.funcs } + +// NewMockFunction creates a mock function that returns a dynamic value. +// This is primarily used to replace provider-contributed functions. +func NewMockFunction(call *FunctionCall) function.Function { + params := make([]function.Parameter, call.ArgsCount) + for idx := 0; idx < call.ArgsCount; idx++ { + params[idx] = function.Parameter{ + Type: cty.DynamicPseudoType, + AllowNull: true, + AllowUnknown: true, + AllowDynamicType: true, + AllowMarked: true, + } + } + + return function.New(&function.Spec{ + Params: params, + Type: function.StaticReturnType(cty.DynamicPseudoType), + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + return cty.DynamicVal, nil + }, + }) +} diff --git a/terraform/lang/functions_test.go b/terraform/lang/functions_test.go index 18db46428..5144e74f8 100644 --- a/terraform/lang/functions_test.go +++ b/terraform/lang/functions_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "github.com/hashicorp/hcl/v2" @@ -905,6 +906,10 @@ func TestFunctions(t *testing.T) { `templatefile("hello.tmpl", {name = "Jodie"})`, cty.StringVal("Hello, Jodie!"), }, + { + `core::templatefile("hello.tmpl", {name = "Namespaced Jodie"})`, + cty.StringVal("Hello, Namespaced Jodie!"), + }, }, "timeadd": { @@ -1045,6 +1050,10 @@ func TestFunctions(t *testing.T) { `upper("hello")`, cty.StringVal("HELLO"), }, + { + `core::upper("hello")`, + cty.StringVal("HELLO"), + }, }, "urlencode": { @@ -1145,6 +1154,11 @@ func TestFunctions(t *testing.T) { delete(allFunctions, impureFunc) } for f := range scope.Functions() { + if strings.Contains(f, "::") { + // Only non-namespaced functions are absolutely required to + // have at least one test. (Others _may_ have tests.) + continue + } if _, ok := tests[f]; !ok { t.Errorf("Missing test for function %s\n", f) } diff --git a/terraform/tfhcl/expressions_hclext.go b/terraform/tfhcl/expressions_hclext.go new file mode 100644 index 000000000..5e0a8699e --- /dev/null +++ b/terraform/tfhcl/expressions_hclext.go @@ -0,0 +1,150 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package tfhcl + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/terraform-linters/tflint-plugin-sdk/hclext" +) + +// ExpandExpressionsHCLExt is ExpandVariablesHCLExt which returns +// []hcl.Expression instead of []hcl.Traversal. +func ExpandExpressionsHCLExt(body hcl.Body, schema *hclext.BodySchema) []hcl.Expression { + rootNode := WalkExpandExpressions(body) + return walkExpressionsWithHCLExt(rootNode, schema) +} + +func walkExpressionsWithHCLExt(node WalkExpressionsNode, schema *hclext.BodySchema) []hcl.Expression { + exprs, children := node.Visit(extendSchema(asHCLSchema(schema))) + + if len(children) > 0 { + childSchemas := childBlockTypes(schema) + for _, child := range children { + if childSchema, exists := childSchemas[child.BlockTypeName]; exists { + exprs = append(exprs, walkExpressionsWithHCLExt(child.Node, childSchema.Body)...) + } + } + } + + return exprs +} + +// WalkExpandExpressions is dynblock.WalkExpandVariables for expressions. +func WalkExpandExpressions(body hcl.Body) WalkExpressionsNode { + return WalkExpressionsNode{body: body} +} + +type WalkExpressionsNode struct { + body hcl.Body +} + +type WalkExpressionsChild struct { + BlockTypeName string + Node WalkExpressionsNode +} + +// Visit returns the expressions required for any "dynamic" blocks +// directly in the body associated with this node, and also returns any child +// nodes that must be visited in order to continue the walk. +// +// Each child node has its associated block type name given in its BlockTypeName +// field, which the calling application should use to determine the appropriate +// schema for the content of each child node and pass it to the child node's +// own Visit method to continue the walk recursively. +func (n WalkExpressionsNode) Visit(schema *hcl.BodySchema) (exprs []hcl.Expression, children []WalkExpressionsChild) { + extSchema := n.extendSchema(schema) + container, _, _ := n.body.PartialContent(extSchema) + if container == nil { + return exprs, children + } + + children = make([]WalkExpressionsChild, 0, len(container.Blocks)) + + for _, attr := range container.Attributes { + exprs = append(exprs, attr.Expr) + } + + for _, block := range container.Blocks { + switch block.Type { + + case "dynamic": + blockTypeName := block.Labels[0] + inner, _, _ := block.Body.PartialContent(variableDetectionInnerSchema) + if inner == nil { + continue + } + + if attr, exists := inner.Attributes["for_each"]; exists { + exprs = append(exprs, attr.Expr) + } + if attr, exists := inner.Attributes["labels"]; exists { + exprs = append(exprs, attr.Expr) + } + + for _, contentBlock := range inner.Blocks { + // We only request "content" blocks in our schema, so we know + // any blocks we find here will be content blocks. We require + // exactly one content block for actual expansion, but we'll + // be more liberal here so that callers can still collect + // expressions from erroneous "dynamic" blocks. + children = append(children, WalkExpressionsChild{ + BlockTypeName: blockTypeName, + Node: WalkExpressionsNode{ + body: contentBlock.Body, + }, + }) + } + + default: + children = append(children, WalkExpressionsChild{ + BlockTypeName: block.Type, + Node: WalkExpressionsNode{ + body: block.Body, + }, + }) + + } + } + + return exprs, children +} + +func (c WalkExpressionsNode) extendSchema(schema *hcl.BodySchema) *hcl.BodySchema { + // We augment the requested schema to also include our special "dynamic" + // block type, since then we'll get instances of it interleaved with + // all of the literal child blocks we must also include. + extSchema := &hcl.BodySchema{ + Attributes: schema.Attributes, + Blocks: make([]hcl.BlockHeaderSchema, len(schema.Blocks), len(schema.Blocks)+1), + } + copy(extSchema.Blocks, schema.Blocks) + extSchema.Blocks = append(extSchema.Blocks, dynamicBlockHeaderSchema) + + return extSchema +} + +// This is a more relaxed schema than what's in schema.go, since we +// want to maximize the amount of variables we can find even if there +// are erroneous blocks. +var variableDetectionInnerSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "for_each", + Required: false, + }, + { + Name: "labels", + Required: false, + }, + { + Name: "iterator", + Required: false, + }, + }, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "content", + }, + }, +}