feat: add provider version to metadata generation (#2496)

Co-authored-by: Zheng Qin <zhengqin@google.com>
Co-authored-by: Andrew Peabody <andrewpeabody@google.com>
Co-authored-by: Bharath KKB <bharathkrishnakb@gmail.com>

cli/bpmetadata/cmd.go

@@ -249,7 +249,8 @@ func CreateBlueprintMetadata(bpPath string, bpMetadataObj *BlueprintMetadata) (*
  // get blueprint requirements
  rolesCfgPath := path.Join(repoDetails.Source.BlueprintRootPath, tfRolesFileName)
  svcsCfgPath := path.Join(repoDetails.Source.BlueprintRootPath, tfServicesFileName)
- requirements, err := getBlueprintRequirements(rolesCfgPath, svcsCfgPath)
+ versionsCfgPath := path.Join(repoDetails.Source.BlueprintRootPath, tfVersionsFileName)
+ requirements, err := getBlueprintRequirements(rolesCfgPath, svcsCfgPath, versionsCfgPath)
  if err != nil {
  Log.Info("skipping blueprint requirements since roles and/or services configurations were not found as per https://tinyurl.com/tf-iam and https://tinyurl.com/tf-services")
  } else {

cli/bpmetadata/int-test/goldens/golden-metadata.yaml

@@ -249,3 +249,8 @@ spec:
  - cloudresourcemanager.googleapis.com
  - compute.googleapis.com
  - serviceusage.googleapis.com
+ providerVersions:
+ - source: hashicorp/google
+ version: ">= 4.42, < 5.0"
+ - source: hashicorp/random
+ version: ">= 2.1"

cli/bpmetadata/proto/bpmetadata.proto

@@ -195,7 +195,7 @@ message BlueprintRequirements {
 
  // Required provider versions.
  // Gen: auto-generated from required providers block.
- repeated ProviderVersion provider_versions = 3; // @gotags: json:"versions,omitempty" yaml:"providerVersions,omitempty"
+ repeated ProviderVersion provider_versions = 3; // @gotags: json:"providerVersions,omitempty" yaml:"providerVersions,omitempty"
 }
 
 // ProviderVersion defines the required version for a provider.

cli/bpmetadata/schema/gcp-blueprint-metadata.json

@@ -440,7 +440,7 @@
  },
  "type": "array"
  },
- "versions": {
+ "providerVersions": {
  "items": {
  "$ref": "#/$defs/ProviderVersion"
  },

cli/bpmetadata/tfconfig.go

@@ -6,6 +6,7 @@ import (
  "path/filepath"
  "regexp"
  "sort"
+ "strings"
 
  hcl "github.com/hashicorp/hcl/v2"
  "github.com/hashicorp/hcl/v2/gohcl"
@@ -188,6 +189,36 @@ func parseBlueprintVersion(versionsFile *hcl.File, diags hcl.Diagnostics) (strin
  return "", nil
 }
 
+// parseBlueprintProviderVersions gets the blueprint provider_versions from the provided config
+// from the required_providers block.
+func parseBlueprintProviderVersions(versionsFile *hcl.File) ([]*ProviderVersion, error) {
+ var v []*ProviderVersion
+ // parse out the required providers from the config
+ var hclModule tfconfig.Module
+ hclModule.RequiredProviders = make(map[string]*tfconfig.ProviderRequirement)
+ diags := tfconfig.LoadModuleFromFile(versionsFile, &hclModule)
+ err := hasHclErrors(diags)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, providerData := range hclModule.RequiredProviders {
+ if providerData.Source == "" {
+ Log.Info("Not found source in provider settings\n")
+ continue
+ }
+ if len(providerData.VersionConstraints) == 0 {
+ Log.Info("Not found version in provider settings\n")
+ continue
+ }
+ v = append(v, &ProviderVersion{
+ Source: providerData.Source,
+ Version: strings.Join(providerData.VersionConstraints, ", "),
+ })
+ }
+ return v, nil
+}
+
 // getBlueprintInterfaces gets the variables and outputs associated
 // with the blueprint
 func getBlueprintInterfaces(configPath string) (*BlueprintInterface, error) {
@@ -253,7 +284,7 @@ func getBlueprintOutput(modOut *tfconfig.Output) *BlueprintOutput {
 
 // getBlueprintRequirements gets the services and roles associated
 // with the blueprint
-func getBlueprintRequirements(rolesConfigPath, servicesConfigPath string) (*BlueprintRequirements, error) {
+func getBlueprintRequirements(rolesConfigPath, servicesConfigPath, versionsConfigPath string) (*BlueprintRequirements, error) {
  //parse blueprint roles
  p := hclparse.NewParser()
  rolesFile, diags := p.ParseHCLFile(rolesConfigPath)
@@ -279,10 +310,33 @@ func getBlueprintRequirements(rolesConfigPath, servicesConfigPath string) (*Blue
  return nil, err
  }
 
+ versionCfgFileExists, _ := fileExists(versionsConfigPath)
+
+ if !versionCfgFileExists {
+ return &BlueprintRequirements{
+ Roles: r,
+ Services: s,
+ }, nil
+ }
+
+ //parse blueprint provider versions
+ versionsFile, diags := p.ParseHCLFile(versionsConfigPath)
+ err = hasHclErrors(diags)
+ if err != nil {
+ return nil, err
+ }
+
+ v, err := parseBlueprintProviderVersions(versionsFile)
+ if err != nil {
+ return nil, err
+ }
+
  return &BlueprintRequirements{
- Roles: r,
- Services: s,
+ Roles: r,
+ Services: s,
+ ProviderVersions: v,
  }, nil
+
 }
 
 // parseBlueprintRoles gets the roles required for the blueprint to be provisioned
@@ -399,15 +453,15 @@ func hasTfconfigErrors(diags tfconfig.Diagnostics) error {
 // MergeExistingConnections merges existing connections from an old BlueprintInterface into a new one,
 // preserving manually authored connections.
 func mergeExistingConnections(newInterfaces, existingInterfaces *BlueprintInterface) {
- if existingInterfaces == nil {
- return // Nothing to merge if existingInterfaces is nil
- }
-
- for i, variable := range newInterfaces.Variables {
- for _, existingVariable := range existingInterfaces.Variables {
- if variable.Name == existingVariable.Name && existingVariable.Connections != nil {
- newInterfaces.Variables[i].Connections = existingVariable.Connections
- }
- }
- }
+ if existingInterfaces == nil {
+ return // Nothing to merge if existingInterfaces is nil
+ }
+
+ for i, variable := range newInterfaces.Variables {
+ for _, existingVariable := range existingInterfaces.Variables {
+ if variable.Name == existingVariable.Name && existingVariable.Connections != nil {
+ newInterfaces.Variables[i].Connections = existingVariable.Connections
+ }
+ }
+ }
 }

cli/bpmetadata/tfconfig_test.go

@@ -11,9 +11,9 @@ import (
 )
 
 const (
- tfTestdataPath = "../testdata/bpmetadata/tf"
+ tfTestdataPath  = "../testdata/bpmetadata/tf"
  metadataTestdataPath = "../testdata/bpmetadata/metadata"
- interfaces = "sample-module"
+ interfaces  = "sample-module"
 )
 
 func TestTFInterfaces(t *testing.T) {
@@ -268,25 +268,58 @@ func TestTFRoles(t *testing.T) {
  }
 }
 
+func TestTFProviderVersions(t *testing.T) {
+ tests := []struct {
+ name string
+ configName string
+ wantProviderVersions []*ProviderVersion
+ }{
+ {
+ name: "Simple list of provider versions",
+ configName: "versions-beta.tf",
+ wantProviderVersions: []*ProviderVersion{
+ {
+ Source: "hashicorp/google",
+ Version: ">= 4.4.0, < 7",
+ },
+ {
+ Source: "hashicorp/google-beta",
+ Version: ">= 4.4.0, < 7",
+ },
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ p := hclparse.NewParser()
+ content, _ := p.ParseHCLFile(path.Join(tfTestdataPath, tt.configName))
+ got, err := parseBlueprintProviderVersions(content)
+ require.NoError(t, err)
+ assert.Equal(t, got, tt.wantProviderVersions)
+ })
+ }
+}
+
 func TestMergeExistingConnections(t *testing.T) {
  tests := []struct {
- name string
- newInterfacesFile string
+ name  string
+ newInterfacesFile  string
  existingInterfacesFile string
  }{
  {
- name: "No existing connections",
- newInterfacesFile: "new_interfaces_no_connections_metadata.yaml",
+ name:  "No existing connections",
+ newInterfacesFile:  "new_interfaces_no_connections_metadata.yaml",
  existingInterfacesFile: "existing_interfaces_without_connections_metadata.yaml",
  },
  {
- name: "One existing connection is preserved",
- newInterfacesFile: "new_interfaces_no_connections_metadata.yaml",
+ name:  "One existing connection is preserved",
+ newInterfacesFile:  "new_interfaces_no_connections_metadata.yaml",
  existingInterfacesFile: "existing_interfaces_with_one_connection_metadata.yaml",
  },
  {
- name: "Multiple existing connections are preserved",
- newInterfacesFile: "new_interfaces_no_connections_metadata.yaml",
+ name:  "Multiple existing connections are preserved",
+ newInterfacesFile:  "new_interfaces_no_connections_metadata.yaml",
  existingInterfacesFile: "existing_interfaces_with_some_connections_metadata.yaml",
  },
  }
@@ -309,3 +342,29 @@ func TestMergeExistingConnections(t *testing.T) {
  })
  }
 }
+
+func TestTFIncompleteProviderVersions(t *testing.T) {
+ tests := []struct {
+ name string
+ configName string
+ }{
+ {
+ name: "Empty list of provider versions",
+ configName: "provider-versions-empty.tf",
+ },
+ {
+ name: "Missing ProviderVersion field",
+ configName: "provider-versions-bad.tf",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ p := hclparse.NewParser()
+ content, _ := p.ParseHCLFile(path.Join(tfTestdataPath, tt.configName))
+ got, err := parseBlueprintProviderVersions(content)
+ require.NoError(t, err)
+ assert.Nil(t, got)
+ })
+ }
+}

cli/testdata/bpmetadata/tf/provider-versions-bad.tf

@@ -0,0 +1,19 @@
+terraform {
+ required_version = ">= 0.13.0"
+
+ required_providers {
+ google = {
+ version = ">= 4.4.0, < 6"
+ }
+ google-beta = {
+ source = "hashicorp/google-beta"
+ }
+ }
+
+ provider_meta "google" {
+ module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v23.1.0"
+ }
+ provider_meta "google-beta" {
+ module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v23.1.0"
+ }
+}

cli/testdata/bpmetadata/tf/provider-versions-empty.tf

@@ -0,0 +1,13 @@
+terraform {
+ required_version = ">= 0.13.0"
+
+ required_providers {
+ }
+
+ provider_meta "google" {
+ module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v23.1.0"
+ }
+ provider_meta "google-beta" {
+ module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v23.1.0"
+ }
+}