Add the deferred changes into the plan

internal/addrs/instance_key.go

@@ -53,6 +53,11 @@ func ParseInstanceKey(key cty.Value) (InstanceKey, error) {
 // of a configuration object that does not use "count" or "for_each" at all.
 var NoKey InstanceKey
 
+// WildcardKey represents the "unknown" value of an InstanceKey. This is used
+// within the deferral logic to express absolute module and resource addresses
+// that are not known at the time of planning.
+var WildcardKey InstanceKey = StringKey("*")
+
 // IntKey is the InstanceKey representation representing integer indices, as
 // used when the "count" argument is specified or if for_each is used with
 // a sequence type.

internal/addrs/partial_expanded.go

@@ -36,6 +36,19 @@ func (m ModuleInstance) UnexpandedChild(call ModuleCall) PartialExpandedModule {
 	}
 }
 
+// UnknownModuleInstance expands the receiver to a full ModuleInstance by
+// replacing the unknown instance keys with a wildcard value.
+func (pem PartialExpandedModule) UnknownModuleInstance() ModuleInstance {
+	base := pem.expandedPrefix
+	for _, call := range pem.unexpandedSuffix {
+		base = append(base, ModuleInstanceStep{
+			Name:        call,
+			InstanceKey: WildcardKey,
+		})
+	}
+	return base
+}
+
 // LevelsKnown returns the number of module path segments of the address that
 // have known instance keys.
 //
@@ -195,8 +208,8 @@ func (pem PartialExpandedModule) String() string {
 	return buf.String()
 }
 
-func (per PartialExpandedModule) UniqueKey() UniqueKey {
-	return partialExpandedModuleKey(per.String())
+func (pem PartialExpandedModule) UniqueKey() UniqueKey {
+	return partialExpandedModuleKey(pem.String())
 }
 
 type partialExpandedModuleKey string
@@ -261,6 +274,16 @@ func (r AbsResource) UnexpandedResource() PartialExpandedResource {
 	}
 }
 
+// UnknownResourceInstance returns an [AbsResourceInstance] that represents the
+// same resource as the receiver but with all instance keys replaced with a
+// wildcard value.
+func (per PartialExpandedResource) UnknownResourceInstance() AbsResourceInstance {
+	return AbsResourceInstance{
+		Module:   per.module.UnknownModuleInstance(),
+		Resource: per.resource.Instance(WildcardKey),
+	}
+}
+
 // MatchesInstance returns true if and only if the given resource instance
 // belongs to the recieving partially-expanded resource address pattern.
 func (per PartialExpandedResource) MatchesInstance(inst AbsResourceInstance) bool {

internal/plans/deferring.go

@@ -0,0 +1,80 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package plans
+
+import "github.com/zclconf/go-cty/cty"
+
+type DeferredReason string
+
+const (
+	// DeferredReasonInvalid is used when the reason for deferring is
+	// unknown or irrelevant.
+	DeferredReasonInvalid DeferredReason = "invalid"
+
+	// DeferredReasonInstanceCountUnknown is used when the reason for deferring
+	// is that the count or for_each meta-attribute was unknown.
+	DeferredReasonInstanceCountUnknown DeferredReason = "instance_count_unknown"
+
+	// DeferredReasonResourceConfigUnknown is used when the reason for deferring
+	// is that the resource configuration was unknown.
+	DeferredReasonResourceConfigUnknown DeferredReason = "resource_config_unknown"
+
+	// DeferredReasonProviderConfigUnknown is used when the reason for deferring
+	// is that the provider configuration was unknown.
+	DeferredReasonProviderConfigUnknown DeferredReason = "provider_config_unknown"
+
+	// DeferredReasonAbsentPrereq is used when the reason for deferring is that
+	// a required prerequisite resource was absent.
+	DeferredReasonAbsentPrereq DeferredReason = "absent_prereq"
+
+	// DeferredReasonDeferredPrereq is used when the reason for deferring is
+	// that a required prerequisite resource was itself deferred.
+	DeferredReasonDeferredPrereq DeferredReason = "deferred_prereq"
+)
+
+// DeferredResourceInstanceChangeSrc tracks information about a resource that
+// has been deferred for some reason.
+type DeferredResourceInstanceChangeSrc struct {
+	// DeferredReason is the reason why this resource instance was deferred.
+	DeferredReason DeferredReason
+
+	// ChangeSrc contains any information we have about the deferred change.
+	// This could be incomplete so must be parsed with care.
+	ChangeSrc *ResourceInstanceChangeSrc
+}
+
+func (rcs *DeferredResourceInstanceChangeSrc) Decode(ty cty.Type) (*DeferredResourceInstanceChange, error) {
+	change, err := rcs.ChangeSrc.Decode(ty)
+	if err != nil {
+		return nil, err
+	}
+
+	return &DeferredResourceInstanceChange{
+		DeferredReason: rcs.DeferredReason,
+		Change:         change,
+	}, nil
+}
+
+// DeferredResourceInstanceChange tracks information about a resource that
+// has been deferred for some reason.
+type DeferredResourceInstanceChange struct {
+	// DeferredReason is the reason why this resource instance was deferred.
+	DeferredReason DeferredReason
+
+	// Change contains any information we have about the deferred change. This
+	// could be incomplete so must be parsed with care.
+	Change *ResourceInstanceChange
+}
+
+func (rcs *DeferredResourceInstanceChange) Encode(ty cty.Type) (*DeferredResourceInstanceChangeSrc, error) {
+	change, err := rcs.Change.Encode(ty)
+	if err != nil {
+		return nil, err
+	}
+
+	return &DeferredResourceInstanceChangeSrc{
+		DeferredReason: rcs.DeferredReason,
+		ChangeSrc:      change,
+	}, nil
+}

internal/plans/deferring/deferred.go

@@ -7,8 +7,6 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/zclconf/go-cty/cty"
-
 	"github.com/hashicorp/terraform/internal/addrs"
 	"github.com/hashicorp/terraform/internal/plans"
 )
@@ -65,7 +63,7 @@ type Deferred struct {
 	// configuration block at different amounts of instance expansion under
 	// different prefixes, and so some queries require us to search across
 	// all of those options to decide if each instance is relevant.
-	resourceInstancesDeferred addrs.Map[addrs.ConfigResource, addrs.Map[addrs.AbsResourceInstance, deferredResourceInstance]]
+	resourceInstancesDeferred addrs.Map[addrs.ConfigResource, addrs.Map[addrs.AbsResourceInstance, *plans.DeferredResourceInstanceChange]]
 
 	// partialExpandedResourcesDeferred tracks placeholders that cover an
 	// unbounded set of potential resource instances in situations where we
@@ -77,7 +75,17 @@ type Deferred struct {
 	// configuration block at different amounts of instance expansion under
 	// different prefixes, and so some queries require us to search across
 	// all of those options to find the one that matches most closely.
-	partialExpandedResourcesDeferred addrs.Map[addrs.ConfigResource, addrs.Map[addrs.PartialExpandedResource, deferredPartialExpandedResource]]
+	partialExpandedResourcesDeferred addrs.Map[addrs.ConfigResource, addrs.Map[addrs.PartialExpandedResource, *plans.DeferredResourceInstanceChange]]
+
+	// partialExpandedDataSourcesDeferred tracks placeholders that cover an
+	// unbounded set of potential data sources in situations where we don't yet
+	// even have enough information to predict which instances of a data source
+	// will exist.
+	//
+	// Data sources are never written into the plan, even when deferred, so we
+	// are tracking these for purely internal reasons. If a resource depends on
+	// a deferred data source, then that resource should be deferred as well.
+	partialExpandedDataSourcesDeferred addrs.Map[addrs.ConfigResource, addrs.Set[addrs.PartialExpandedResource]]
 
 	// partialExpandedModulesDeferred tracks all of the partial-expanded module
 	// prefixes we were notified about.
@@ -103,12 +111,30 @@ type Deferred struct {
 // calling this function.
 func NewDeferred(resourceGraph addrs.DirectedGraph[addrs.ConfigResource], enabled bool) *Deferred {
 	return &Deferred{
-		resourceGraph:                    resourceGraph,
-		deferralAllowed:                  enabled,
-		resourceInstancesDeferred:        addrs.MakeMap[addrs.ConfigResource, addrs.Map[addrs.AbsResourceInstance, deferredResourceInstance]](),
-		partialExpandedResourcesDeferred: addrs.MakeMap[addrs.ConfigResource, addrs.Map[addrs.PartialExpandedResource, deferredPartialExpandedResource]](),
-		partialExpandedModulesDeferred:   addrs.MakeSet[addrs.PartialExpandedModule](),
+		resourceGraph:                      resourceGraph,
+		deferralAllowed:                    enabled,
+		resourceInstancesDeferred:          addrs.MakeMap[addrs.ConfigResource, addrs.Map[addrs.AbsResourceInstance, *plans.DeferredResourceInstanceChange]](),
+		partialExpandedResourcesDeferred:   addrs.MakeMap[addrs.ConfigResource, addrs.Map[addrs.PartialExpandedResource, *plans.DeferredResourceInstanceChange]](),
+		partialExpandedDataSourcesDeferred: addrs.MakeMap[addrs.ConfigResource, addrs.Set[addrs.PartialExpandedResource]](),
+		partialExpandedModulesDeferred:     addrs.MakeSet[addrs.PartialExpandedModule](),
+	}
+}
+
+// GetDeferredChanges returns a slice of all the deferred changes that have
+// been reported to the receiver.
+func (d *Deferred) GetDeferredChanges() []*plans.DeferredResourceInstanceChange {
+	var changes []*plans.DeferredResourceInstanceChange
+	for _, configMapElem := range d.resourceInstancesDeferred.Elems {
+		for _, changeElem := range configMapElem.Value.Elems {
+			changes = append(changes, changeElem.Value)
+		}
+	}
+	for _, configMapElem := range d.partialExpandedResourcesDeferred.Elems {
+		for _, changeElem := range configMapElem.Value.Elems {
+			changes = append(changes, changeElem.Value)
+		}
 	}
+	return changes
 }
 
 // SetExternalDependencyDeferred modifies a freshly-constructed [Deferred]
@@ -146,12 +172,13 @@ func (d *Deferred) HaveAnyDeferrals() bool {
 		(d.externalDependencyDeferred ||
 			d.resourceInstancesDeferred.Len() != 0 ||
 			d.partialExpandedResourcesDeferred.Len() != 0 ||
+			d.partialExpandedDataSourcesDeferred.Len() != 0 ||
 			len(d.partialExpandedModulesDeferred) != 0)
 }
 
-// ShouldDeferResourceChanges returns true if the receiver knows some reason
-// why the resource instance with the given address should have its planned
-// action deferred for a future plan/apply round.
+// ShouldDeferResourceInstanceChanges returns true if the receiver knows some
+// reason why the resource instance with the given address should have its
+// planned action deferred for a future plan/apply round.
 //
 // This method is specifically for resource instances whose full address is
 // known and thus it would be possible in principle to plan changes, but we
@@ -181,7 +208,9 @@ func (d *Deferred) ShouldDeferResourceInstanceChanges(addr addrs.AbsResourceInst
 	// when the deferred-actions-related experiments are inactive, so we can
 	// minimize the risk of impacting non-participants.
 	// (Maybe we'll remove this check once this stuff is non-experimental.)
-	if d.resourceInstancesDeferred.Len() == 0 && d.partialExpandedResourcesDeferred.Len() == 0 {
+	if d.resourceInstancesDeferred.Len() == 0 &&
+		d.partialExpandedResourcesDeferred.Len() == 0 &&
+		d.partialExpandedDataSourcesDeferred.Len() == 0 {
 		return false
 	}
 
@@ -238,6 +267,9 @@ func (d *Deferred) ShouldDeferResourceInstanceChanges(addr addrs.AbsResourceInst
 			// at least one is enough.
 			return true
 		}
+		if d.partialExpandedDataSourcesDeferred.Has(configDep) {
+			return true
+		}
 
 		// We don't check d.partialExpandedModulesDeferred here because
 		// we expect that the graph nodes representing any resource under
@@ -256,18 +288,18 @@ func (d *Deferred) ShouldDeferResourceInstanceChanges(addr addrs.AbsResourceInst
 // ReportResourceExpansionDeferred reports that we cannot even predict which
 // instances of a resource will be declared and thus we must defer all planning
 // for that resource.
-//
-// Use the most precise partial-expanded resource address possible, and provide
-// a valuePlaceholder that has known values only for attributes/elements that
-// we can guarantee will be equal across all potential resource instances
-// under the partial-expanded prefix.
-func (d *Deferred) ReportResourceExpansionDeferred(addr addrs.PartialExpandedResource, valuePlaceholder cty.Value) {
+func (d *Deferred) ReportResourceExpansionDeferred(addr addrs.PartialExpandedResource, change *plans.ResourceInstanceChange) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
 
+	if addr.Resource().Mode != addrs.ManagedResourceMode {
+		// Use ReportDataSourceExpansionDeferred for data sources.
+		panic(fmt.Sprintf("unexpected resource mode %q for %s", addr.Resource().Mode, addr))
+	}
+
 	configAddr := addr.ConfigResource()
 	if !d.partialExpandedResourcesDeferred.Has(configAddr) {
-		d.partialExpandedResourcesDeferred.Put(configAddr, addrs.MakeMap[addrs.PartialExpandedResource, deferredPartialExpandedResource]())
+		d.partialExpandedResourcesDeferred.Put(configAddr, addrs.MakeMap[addrs.PartialExpandedResource, *plans.DeferredResourceInstanceChange]())
 	}
 
 	configMap := d.partialExpandedResourcesDeferred.Get(configAddr)
@@ -277,39 +309,49 @@ func (d *Deferred) ReportResourceExpansionDeferred(addr addrs.PartialExpandedRes
 		// prefix only once.
 		panic(fmt.Sprintf("duplicate deferral report for %s", addr))
 	}
-	configMap.Put(addr, deferredPartialExpandedResource{
-		valuePlaceholder: valuePlaceholder,
+	configMap.Put(addr, &plans.DeferredResourceInstanceChange{
+		DeferredReason: plans.DeferredReasonInstanceCountUnknown,
+		Change:         change,
 	})
 }
 
+// ReportDataSourceExpansionDeferred reports that we cannot even predict which
+// instances of a data source will be declared and thus we must defer all
+// planning for that data source.
+func (d *Deferred) ReportDataSourceExpansionDeferred(addr addrs.PartialExpandedResource) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	if addr.Resource().Mode != addrs.DataResourceMode {
+		// Use ReportResourceExpansionDeferred for resources.
+		panic(fmt.Sprintf("unexpected resource mode %q for %s", addr.Resource().Mode, addr))
+	}
+
+	configAddr := addr.ConfigResource()
+	if !d.partialExpandedDataSourcesDeferred.Has(configAddr) {
+		d.partialExpandedDataSourcesDeferred.Put(configAddr, addrs.MakeSet[addrs.PartialExpandedResource]())
+	}
+
+	configSet := d.partialExpandedDataSourcesDeferred.Get(configAddr)
+	if configSet.Has(addr) {
+		// This indicates a bug in the caller, since our graph walk should
+		// ensure that we visit and evaluate each distinct partial-expanded
+		// prefix only once.
+		panic(fmt.Sprintf("duplicate deferral report for %s", addr))
+	}
+	configSet.Add(addr)
+}
+
 // ReportResourceInstanceDeferred records that a fully-expanded resource
 // instance has had its planned action deferred to a future round for a reason
 // other than its address being only partially-decided.
-//
-// For example, this is the method to use if the reason for deferral is
-// that [Deferred.ShouldDeferResourceInstanceChanges] returns true for the
-// same address, or if the responsible provider indicated in its planning
-// response that it does not have enough information to produce a final
-// plan.
-//
-// expectedAction and expectedValue together provide an approximation of
-// what Terraform is expecting to plan in a future round. expectedAction may
-// be [plans.Undecided] if there isn't even enough information to decide on
-// an action. expectedValue should use unknown values to stand in for values
-// that cannot be predicted while being as precise as is practical; in the
-// worst case it's okay to provide a totally-unknown value, but better to
-// provide a known object with unknown values inside it when possible.
-//
-// TODO: Allow the caller to pass something representing the reason for the
-// deferral, so we can distinguish between the different variations in the
-// plan reported to the operator.
-func (d *Deferred) ReportResourceInstanceDeferred(addr addrs.AbsResourceInstance, expectedAction plans.Action, expectedValue cty.Value) {
+func (d *Deferred) ReportResourceInstanceDeferred(addr addrs.AbsResourceInstance, reason plans.DeferredReason, change *plans.ResourceInstanceChange) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
 
 	configAddr := addr.ConfigResource()
 	if !d.resourceInstancesDeferred.Has(configAddr) {
-		d.resourceInstancesDeferred.Put(configAddr, addrs.MakeMap[addrs.AbsResourceInstance, deferredResourceInstance]())
+		d.resourceInstancesDeferred.Put(configAddr, addrs.MakeMap[addrs.AbsResourceInstance, *plans.DeferredResourceInstanceChange]())
 	}
 
 	configMap := d.resourceInstancesDeferred.Get(configAddr)
@@ -318,9 +360,9 @@ func (d *Deferred) ReportResourceInstanceDeferred(addr addrs.AbsResourceInstance
 		// ensure that we visit and evaluate each resource instance only once.
 		panic(fmt.Sprintf("duplicate deferral report for %s", addr))
 	}
-	configMap.Put(addr, deferredResourceInstance{
-		plannedAction: expectedAction,
-		plannedValue:  expectedValue,
+	configMap.Put(addr, &plans.DeferredResourceInstanceChange{
+		DeferredReason: reason,
+		Change:         change,
 	})
 }