Merge pull request #34394 from hashicorp/jbardin/provider-functions-lang

enable use of provider functions in the Terraform language

go.mod

@@ -52,7 +52,7 @@ require (
 	github.com/hashicorp/go-uuid v1.0.3
 	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/hcl v1.0.0
-	github.com/hashicorp/hcl/v2 v2.19.1
+	github.com/hashicorp/hcl/v2 v2.19.2-0.20231109190535-c964a71ca320
 	github.com/hashicorp/jsonapi v1.2.0
 	github.com/hashicorp/terraform-registry-address v0.2.0
 	github.com/hashicorp/terraform-svchost v0.1.1

go.sum

@@ -706,8 +706,8 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/hcl/v2 v2.19.1 h1://i05Jqznmb2EXqa39Nsvyan2o5XyMowW5fnCKW5RPI=
-github.com/hashicorp/hcl/v2 v2.19.1/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE=
+github.com/hashicorp/hcl/v2 v2.19.2-0.20231109190535-c964a71ca320 h1:XCxc/uVhiBd2uKHRCiOItsuH8RbpwvPC5Pi+LAzZDn8=
+github.com/hashicorp/hcl/v2 v2.19.2-0.20231109190535-c964a71ca320/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE=
 github.com/hashicorp/jsonapi v1.2.0 h1:ezDCzOFsKTL+KxVQuA1rNxkIGTvZph1rNu8kT5A8trI=
 github.com/hashicorp/jsonapi v1.2.0/go.mod h1:Yog5+CPEM3c99L1CL2CFCYoSzgWm5vTU58idbRUaLik=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=

internal/command/jsonfunction/function.go

@@ -55,9 +55,9 @@ func Marshal(f map[string]function.Function) ([]byte, tfdiags.Diagnostics) {
 	signatures := newFunctions()
 
 	for name, v := range f {
-		if name == "can" {
+		if name == "can" || name == "core::can" {
 			signatures.Signatures[name] = marshalCan(v)
-		} else if name == "try" {
+		} else if name == "try" || name == "core::try" {
 			signatures.Signatures[name] = marshalTry(v)
 		} else {
 			signature, err := marshalFunction(v)

internal/command/metadata_functions.go

@@ -12,7 +12,7 @@ import (
 )
 
 var (
-	ignoredFunctions = []string{"map", "list"}
+	ignoredFunctions = []string{"map", "list", "core::map", "core::list"}
 )
 
 // MetadataFunctionsCommand is a Command implementation that prints out information

internal/lang/funcs/descriptions.go

@@ -3,7 +3,9 @@
 
 package funcs
 
-import "github.com/zclconf/go-cty/cty/function"
+import (
+	"github.com/zclconf/go-cty/cty/function"
+)
 
 type descriptionEntry struct {
 	// Description is a description for the function.

internal/lang/funcs/filesystem.go

@@ -135,7 +135,7 @@ func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Fun
 		givenFuncs := funcsCb() // this callback indirection is to avoid chicken/egg problems
 		funcs := make(map[string]function.Function, len(givenFuncs))
 		for name, fn := range givenFuncs {
-			if name == "templatefile" {
+			if name == "templatefile" || name == "core::templatefile" {
 				// We stub this one out to prevent recursive calls.
 				funcs[name] = function.New(&function.Spec{
 					Params: params,

internal/lang/funcs/filesystem_test.go

@@ -151,6 +151,12 @@ func TestTemplateFile(t *testing.T) {
 			cty.NilVal,
 			`testdata/recursive.tmpl:1,3-16: Error in function call; Call to function "templatefile" failed: cannot recursively call templatefile from inside templatefile call.`,
 		},
+		{
+			cty.StringVal("testdata/recursive_namespaced.tmpl"),
+			cty.MapValEmpty(cty.String),
+			cty.NilVal,
+			`testdata/recursive_namespaced.tmpl:1,3-22: Error in function call; Call to function "core::templatefile" failed: cannot recursively call templatefile from inside templatefile call.`,
+		},
 		{
 			cty.StringVal("testdata/list.tmpl"),
 			cty.ObjectVal(map[string]cty.Value{
@@ -183,8 +189,10 @@ func TestTemplateFile(t *testing.T) {
 
 	templateFileFn := MakeTemplateFileFunc(".", func() map[string]function.Function {
 		return map[string]function.Function{
-			"join":         stdlib.JoinFunc,
-			"templatefile": MakeFileFunc(".", false), // just a placeholder, since templatefile itself overrides this
+			"join":               stdlib.JoinFunc,
+			"core::join":         stdlib.JoinFunc,
+			"templatefile":       MakeFileFunc(".", false), // just a placeholder, since templatefile itself overrides this
+			"core::templatefile": MakeFileFunc(".", false), // just a placeholder, since templatefile itself overrides this
 		}
 	})
 

internal/lang/funcs/testdata/recursive_namespaced.tmpl

@@ -0,0 +1 @@
+${core::templatefile("recursive_namespaced.tmpl", {})}

internal/lang/functions.go

@@ -29,34 +29,169 @@ func (s *Scope) Functions() map[string]function.Function {
 	if s.funcs == nil {
 		s.funcs = baseFunctions(s.BaseDir)
 
-		// Then we add some functions that are only relevant when being accessed
-		// from inside a specific scope.
+		coreFuncs := map[string]function.Function{
+			"abs":              stdlib.AbsoluteFunc,
+			"abspath":          funcs.AbsPathFunc,
+			"alltrue":          funcs.AllTrueFunc,
+			"anytrue":          funcs.AnyTrueFunc,
+			"basename":         funcs.BasenameFunc,
+			"base64decode":     funcs.Base64DecodeFunc,
+			"base64encode":     funcs.Base64EncodeFunc,
+			"base64gzip":       funcs.Base64GzipFunc,
+			"base64sha256":     funcs.Base64Sha256Func,
+			"base64sha512":     funcs.Base64Sha512Func,
+			"bcrypt":           funcs.BcryptFunc,
+			"can":              tryfunc.CanFunc,
+			"ceil":             stdlib.CeilFunc,
+			"chomp":            stdlib.ChompFunc,
+			"cidrhost":         funcs.CidrHostFunc,
+			"cidrnetmask":      funcs.CidrNetmaskFunc,
+			"cidrsubnet":       funcs.CidrSubnetFunc,
+			"cidrsubnets":      funcs.CidrSubnetsFunc,
+			"coalesce":         funcs.CoalesceFunc,
+			"coalescelist":     stdlib.CoalesceListFunc,
+			"compact":          stdlib.CompactFunc,
+			"concat":           stdlib.ConcatFunc,
+			"contains":         stdlib.ContainsFunc,
+			"csvdecode":        stdlib.CSVDecodeFunc,
+			"dirname":          funcs.DirnameFunc,
+			"distinct":         stdlib.DistinctFunc,
+			"element":          stdlib.ElementFunc,
+			"endswith":         funcs.EndsWithFunc,
+			"chunklist":        stdlib.ChunklistFunc,
+			"file":             funcs.MakeFileFunc(s.BaseDir, false),
+			"fileexists":       funcs.MakeFileExistsFunc(s.BaseDir),
+			"fileset":          funcs.MakeFileSetFunc(s.BaseDir),
+			"filebase64":       funcs.MakeFileFunc(s.BaseDir, true),
+			"filebase64sha256": funcs.MakeFileBase64Sha256Func(s.BaseDir),
+			"filebase64sha512": funcs.MakeFileBase64Sha512Func(s.BaseDir),
+			"filemd5":          funcs.MakeFileMd5Func(s.BaseDir),
+			"filesha1":         funcs.MakeFileSha1Func(s.BaseDir),
+			"filesha256":       funcs.MakeFileSha256Func(s.BaseDir),
+			"filesha512":       funcs.MakeFileSha512Func(s.BaseDir),
+			"flatten":          stdlib.FlattenFunc,
+			"floor":            stdlib.FloorFunc,
+			"format":           stdlib.FormatFunc,
+			"formatdate":       stdlib.FormatDateFunc,
+			"formatlist":       stdlib.FormatListFunc,
+			"indent":           stdlib.IndentFunc,
+			"index":            funcs.IndexFunc, // stdlib.IndexFunc is not compatible
+			"join":             stdlib.JoinFunc,
+			"jsondecode":       stdlib.JSONDecodeFunc,
+			"jsonencode":       stdlib.JSONEncodeFunc,
+			"keys":             stdlib.KeysFunc,
+			"length":           funcs.LengthFunc,
+			"list":             funcs.ListFunc,
+			"log":              stdlib.LogFunc,
+			"lookup":           funcs.LookupFunc,
+			"lower":            stdlib.LowerFunc,
+			"map":              funcs.MapFunc,
+			"matchkeys":        funcs.MatchkeysFunc,
+			"max":              stdlib.MaxFunc,
+			"md5":              funcs.Md5Func,
+			"merge":            stdlib.MergeFunc,
+			"min":              stdlib.MinFunc,
+			"one":              funcs.OneFunc,
+			"parseint":         stdlib.ParseIntFunc,
+			"pathexpand":       funcs.PathExpandFunc,
+			"pow":              stdlib.PowFunc,
+			"range":            stdlib.RangeFunc,
+			"regex":            stdlib.RegexFunc,
+			"regexall":         stdlib.RegexAllFunc,
+			"replace":          funcs.ReplaceFunc,
+			"reverse":          stdlib.ReverseListFunc,
+			"rsadecrypt":       funcs.RsaDecryptFunc,
+			"sensitive":        funcs.SensitiveFunc,
+			"nonsensitive":     funcs.NonsensitiveFunc,
+			"setintersection":  stdlib.SetIntersectionFunc,
+			"setproduct":       stdlib.SetProductFunc,
+			"setsubtract":      stdlib.SetSubtractFunc,
+			"setunion":         stdlib.SetUnionFunc,
+			"sha1":             funcs.Sha1Func,
+			"sha256":           funcs.Sha256Func,
+			"sha512":           funcs.Sha512Func,
+			"signum":           stdlib.SignumFunc,
+			"slice":            stdlib.SliceFunc,
+			"sort":             stdlib.SortFunc,
+			"split":            stdlib.SplitFunc,
+			"startswith":       funcs.StartsWithFunc,
+			"strcontains":      funcs.StrContainsFunc,
+			"strrev":           stdlib.ReverseFunc,
+			"substr":           stdlib.SubstrFunc,
+			"sum":              funcs.SumFunc,
+			"textdecodebase64": funcs.TextDecodeBase64Func,
+			"textencodebase64": funcs.TextEncodeBase64Func,
+			"timestamp":        funcs.TimestampFunc,
+			"timeadd":          stdlib.TimeAddFunc,
+			"timecmp":          funcs.TimeCmpFunc,
+			"title":            stdlib.TitleFunc,
+			"tostring":         funcs.MakeToFunc(cty.String),
+			"tonumber":         funcs.MakeToFunc(cty.Number),
+			"tobool":           funcs.MakeToFunc(cty.Bool),
+			"toset":            funcs.MakeToFunc(cty.Set(cty.DynamicPseudoType)),
+			"tolist":           funcs.MakeToFunc(cty.List(cty.DynamicPseudoType)),
+			"tomap":            funcs.MakeToFunc(cty.Map(cty.DynamicPseudoType)),
+			"transpose":        funcs.TransposeFunc,
+			"trim":             stdlib.TrimFunc,
+			"trimprefix":       stdlib.TrimPrefixFunc,
+			"trimspace":        stdlib.TrimSpaceFunc,
+			"trimsuffix":       stdlib.TrimSuffixFunc,
+			"try":              tryfunc.TryFunc,
+			"upper":            stdlib.UpperFunc,
+			"urlencode":        funcs.URLEncodeFunc,
+			"uuid":             funcs.UUIDFunc,
+			"uuidv5":           funcs.UUIDV5Func,
+			"values":           stdlib.ValuesFunc,
+			"yamldecode":       ctyyaml.YAMLDecodeFunc,
+			"yamlencode":       ctyyaml.YAMLEncodeFunc,
+			"zipmap":           stdlib.ZipmapFunc,
+		}
+
+		coreFuncs["templatefile"] = funcs.MakeTemplateFileFunc(s.BaseDir, func() map[string]function.Function {
+			// The templatefile function prevents recursive calls to itself
+			// by copying this map and overwriting the "templatefile" and
+			// "core:templatefile" entries.
+			return s.funcs
+		})
 
 		if s.ConsoleMode {
 			// The type function is only available in terraform console.
-			s.funcs["type"] = funcs.TypeFunc
+			coreFuncs["type"] = funcs.TypeFunc
 		}
 
 		if !s.ConsoleMode {
 			// The plantimestamp function doesn't make sense in the terraform
 			// console.
-			s.funcs["plantimestamp"] = funcs.MakeStaticTimestampFunc(s.PlanTimestamp)
+			coreFuncs["plantimestamp"] = funcs.MakeStaticTimestampFunc(s.PlanTimestamp)
 		}
 
 		if s.PureOnly {
 			// Force our few impure functions to return unknown so that we
 			// can defer evaluating them until a later pass.
 			for _, name := range impureFunctions {
-				s.funcs[name] = function.Unpredictable(s.funcs[name])
+				coreFuncs[name] = function.Unpredictable(coreFuncs[name])
 			}
 		}
 
-		// Add a description to each function and parameter based on the
-		// contents of descriptionList.
-		// One must create a matching description entry whenever a new
-		// function is introduced.
-		for name, f := range s.funcs {
-			s.funcs[name] = funcs.WithDescription(name, f)
+		// All of the built-in functions are also available under the "core::"
+		// namespace, to distinguish from the "provider::" and "module::"
+		// namespaces that can serve as external extension points.
+		s.funcs = make(map[string]function.Function, len(coreFuncs)*2)
+		for name, fn := range coreFuncs {
+			fn = funcs.WithDescription(name, fn)
+			s.funcs[name] = fn
+			s.funcs["core::"+name] = fn
+		}
+
+		// We'll also bring in any external functions that the caller provided
+		// when constructing this scope. For now, that's just
+		// provider-contributed functions, under a "provider::NAME::" namespace
+		// where NAME is the local name of the provider in the current module.
+		for providerLocalName, funcs := range s.ExternalFuncs.Provider {
+			for funcName, fn := range funcs {
+				name := fmt.Sprintf("provider::%s::%s", providerLocalName, funcName)
+				s.funcs[name] = fn
+			}
 		}
 	}
 	s.funcsLock.Unlock()
@@ -249,3 +384,15 @@ func (s *Scope) experimentalFunction(experiment experiments.Experiment, fn funct
 		},
 	})
 }
+
+// ExternalFuncs represents functions defined by extension components outside
+// of Terraform Core.
+//
+// This package expects the caller to provide ready-to-use function.Function
+// instances for each function, which themselves perform whatever adaptations
+// are necessary to translate a call into a form suitable for the external
+// component that's contributing the function, and to translate the results
+// to conform to the expected function return value conventions.
+type ExternalFuncs struct {
+	Provider map[string]map[string]function.Function
+}

internal/lang/functions_descriptions_test.go

@@ -5,28 +5,14 @@ package lang
 
 import (
 	"testing"
-
-	"github.com/hashicorp/terraform/internal/lang/funcs"
 )
 
 func TestFunctionDescriptions(t *testing.T) {
 	scope := &Scope{
 		ConsoleMode: true,
 	}
-	// This will implicitly test the parameter description count since
-	// WithNewDescriptions will panic if the number doesn't match.
-	allFunctions := scope.Functions()
-
-	// plantimestamp isn't available with ConsoleMode: true
-	expectedFunctionCount := len(funcs.DescriptionList) - 1
-
-	if len(allFunctions) != expectedFunctionCount {
-		t.Errorf("DescriptionList length expected: %d, got %d", len(allFunctions), expectedFunctionCount)
-	}
-
-	for name := range allFunctions {
-		_, ok := funcs.DescriptionList[name]
-		if !ok {
+	for name, fn := range scope.Functions() {
+		if fn.Description() == "" {
 			t.Errorf("missing DescriptionList entry for function %q", name)
 		}
 	}