Create sdk package (#221)

Description of changes:
Creates a new `sdk` package that strictly serves the purpose of loading the operations and shapes from the AWS SDK. This is useful to isolate creating the custom fields that need to be injected during this stage.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: RedbackThomson

cmd/ack-generate/command/common.go

@@ -30,6 +30,7 @@ import (
 	ackgenerate "github.com/aws-controllers-k8s/code-generator/pkg/generate/ack"
 	ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/generate/config"
 	ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model"
+	acksdk "github.com/aws-controllers-k8s/code-generator/pkg/sdk"
 	"github.com/aws-controllers-k8s/code-generator/pkg/util"
 	k8sversion "k8s.io/apimachinery/pkg/version"
 )
@@ -236,7 +237,7 @@ func loadModel(svcAlias string, apiVersion string) (*ackmodel.Model, error) {
 		modelName = svcAlias
 	}
 
-	sdkHelper := ackmodel.NewSDKHelper(sdkDir)
+	sdkHelper := acksdk.NewHelper(sdkDir)
 	sdkAPI, err := sdkHelper.API(modelName)
 	if err != nil {
 		retryModelName, err := FallBackFindServiceID(sdkDir, svcAlias)

cmd/ack-generate/command/crossplane.go

@@ -28,8 +28,8 @@ import (
 	ackgenerate "github.com/aws-controllers-k8s/code-generator/pkg/generate/ack"
 	ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/generate/config"
 	cpgenerate "github.com/aws-controllers-k8s/code-generator/pkg/generate/crossplane"
-	"github.com/aws-controllers-k8s/code-generator/pkg/model"
 	ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model"
+	acksdk "github.com/aws-controllers-k8s/code-generator/pkg/sdk"
 )
 
 // crossplaneCmd is the command that generates Crossplane API types
@@ -70,7 +70,7 @@ func generateCrossplane(_ *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	sdkHelper := model.NewSDKHelper(sdkDir)
+	sdkHelper := acksdk.NewHelper(sdkDir)
 	sdkHelper.APIGroupSuffix = "aws.crossplane.io"
 	sdkAPI, err := sdkHelper.API(svcAlias)
 	if err != nil {

pkg/generate/code/check.go

@@ -192,5 +192,3 @@ func checkRequiredFieldsMissingFromShapeReadMany(
 	result = fmt.Sprintf("%s == nil", resVarPath)
 	return fmt.Sprintf("%sreturn %s\n", indent, result)
 }
-
-

pkg/generate/code/check_test.go

@@ -135,4 +135,4 @@ func TestCheckRequiredFields_StatusField_ReadMany(t *testing.T) {
 		strings.TrimSpace(expRequiredFieldsCode),
 		strings.TrimSpace(gotCode),
 	)
-}
+}

pkg/generate/code/set_sdk_test.go

@@ -1799,4 +1799,4 @@ func TestSetSDK_EC2_VPC_ReadMany(t *testing.T) {
 		expected,
 		code.SetSDK(crd.Config(), crd, model.OpTypeList, "r.ko", "res", 1),
 	)
-}
+}

pkg/model/crd.go

@@ -472,7 +472,7 @@ func (r *CRD) GetOutputShape(
 		wrapperOutputShape, err := r.getWrapperOutputShape(outputShape,
 			*wrapperFieldPath)
 		if err != nil {
-			msg := fmt.Sprintf("Unable to unwrap the output shape: %s " +
+			msg := fmt.Sprintf("Unable to unwrap the output shape: %s "+
 				"with field path override: %s. error: %v",
 				outputShape.OrigShapeName, *wrapperFieldPath, err)
 			panic(msg)
@@ -506,7 +506,7 @@ func (r *CRD) getWrapperOutputShape(
 	// wrapper field must be structure; otherwise cannot unpack
 	if memberRef.Shape.Type != "structure" {
 		return nil, fmt.Errorf(
-			"output wrapper overrides can only contain fields of type" +
+			"output wrapper overrides can only contain fields of type"+
 				" 'structure'. Found wrapper override field %s of type '%s'",
 			wrapperField, memberRef.Shape.Type)
 	}

pkg/model/field.go

@@ -96,7 +96,7 @@ func NewField(
 	}
 
 	if shape != nil {
-		gte, gt, gtwp = cleanGoType(crd.sdkAPI, crd.cfg, shape, cfg)
+		gte, gt, gtwp = CleanGoType(crd.sdkAPI, crd.cfg, shape, cfg)
 	} else {
 		gte = "string"
 		gt = "*string"

pkg/model/model.go

@@ -695,8 +695,8 @@ func (m *Model) GetConfig() *ackgenconfig.Config {
 // e.g. "sns.services.k8s.aws"
 func (m *Model) APIGroup() string {
 	suffix := "services.k8s.aws"
-	if m.SDKAPI.apiGroupSuffix != "" {
-		suffix = m.SDKAPI.apiGroupSuffix
+	if m.SDKAPI.APIGroupSuffix != "" {
+		suffix = m.SDKAPI.APIGroupSuffix
 	}
 	return fmt.Sprintf("%s.%s", m.servicePackageName, suffix)
 }

pkg/model/multiversion/manager.go

@@ -23,6 +23,7 @@ import (
 	ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/generate/config"
 	ackmetadata "github.com/aws-controllers-k8s/code-generator/pkg/metadata"
 	ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model"
+	acksdk "github.com/aws-controllers-k8s/code-generator/pkg/sdk"
 	"github.com/aws-controllers-k8s/code-generator/pkg/util"
 )
 
@@ -71,7 +72,7 @@ func NewAPIVersionManager(
 		return nil, fmt.Errorf("cannot read sdk git repository: %v", err)
 	}
 
-	SDKAPIHelper := ackmodel.NewSDKHelper(sdkCacheDir)
+	SDKAPIHelper := acksdk.NewHelper(sdkCacheDir)
 
 	// create model for each non-deprecated api version
 	models := map[string]*ackmodel.Model{}

pkg/model/sdk_helper.go renamed to pkg/model/sdk_api.go

@@ -14,177 +14,33 @@
 package model
 
 import (
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"sort"
 	"strings"
 
-	"gopkg.in/src-d/go-git.v4"
+	awssdkmodel "github.com/aws/aws-sdk-go/private/model/api"
 
 	ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/generate/config"
 	"github.com/aws-controllers-k8s/code-generator/pkg/names"
 	"github.com/aws-controllers-k8s/code-generator/pkg/util"
-
-	awssdkmodel "github.com/aws/aws-sdk-go/private/model/api"
 )
 
-var (
-	ErrInvalidVersionDirectory = errors.New(
-		"expected to find only directories in api model directory but found non-directory",
-	)
-	ErrNoValidVersionDirectory = errors.New(
-		"no valid version directories found",
-	)
-	ErrServiceNotFound = errors.New(
-		"no such service",
-	)
-	ErrAPIVersionNotFound = errors.New(
-		"no such api version",
-	)
+const (
+	// ConflictingNameSuffix is appended to type names when they overlap with
+	// well-known common struct names for things like a CRD itself, or its
+	// Spec/Status subfield struct type name.
+	ConflictingNameSuffix = "_SDK"
 )
 
-// SDKHelper is a helper struct that helps work with the aws-sdk-go models and
-// API model loader
-type SDKHelper struct {
-	gitRepository *git.Repository
-	basePath      string
-	loader        *awssdkmodel.Loader
-	// Default is set by `FirstAPIVersion`
-	apiVersion string
-	// Default is "services.k8s.aws"
-	APIGroupSuffix string
-}
-
-// NewSDKHelper returns a new SDKHelper object
-func NewSDKHelper(basePath string) *SDKHelper {
-	return &SDKHelper{
-		basePath: basePath,
-		loader: &awssdkmodel.Loader{
-			BaseImport:            basePath,
-			IgnoreUnsupportedAPIs: true,
-		},
-	}
-}
-
-// WithSDKVersion checks out the sdk git repository to the provided version. To use
-// this function h.basePath should point to a git repository.
-func (h *SDKHelper) WithSDKVersion(version string) error {
-	if h.gitRepository == nil {
-		gitRepository, err := util.LoadRepository(h.basePath)
-		if err != nil {
-			return fmt.Errorf("error loading repository from %s: %v", h.basePath, err)
-		}
-		h.gitRepository = gitRepository
-	}
-
-	err := util.CheckoutRepositoryTag(h.gitRepository, version)
-	if err != nil {
-		return fmt.Errorf("cannot checkout tag %s: %v", version, err)
-	}
-	return nil
-}
-
-// WithAPIVersion sets the `apiVersion` field.
-func (h *SDKHelper) WithAPIVersion(apiVersion string) {
-	h.apiVersion = apiVersion
-}
-
-// API returns the aws-sdk-go API model for a supplied service model name.
-func (h *SDKHelper) API(serviceModelName string) (*SDKAPI, error) {
-	modelPath, _, err := h.ModelAndDocsPath(serviceModelName)
-	if err != nil {
-		return nil, err
-	}
-	apis, err := h.loader.Load([]string{modelPath})
-	if err != nil {
-		return nil, err
-	}
-	// apis is a map, keyed by the service alias, of pointers to aws-sdk-go
-	// model API objects
-	for _, api := range apis {
-		// If we don't do this, we can end up with panic()'s like this:
-		// panic: assignment to entry in nil map
-		// when trying to execute Shape.GoType().
-		//
-		// Calling API.ServicePackageDoc() ends up resetting the API.imports
-		// unexported map variable...
-		_ = api.ServicePackageDoc()
-		return &SDKAPI{api, nil, nil, h.APIGroupSuffix}, nil
-	}
-	return nil, ErrServiceNotFound
-}
-
-// ModelAndDocsPath returns two string paths to the supplied service's API and
-// doc JSON files
-func (h *SDKHelper) ModelAndDocsPath(
-	serviceModelName string,
-) (string, string, error) {
-	if h.apiVersion == "" {
-		apiVersion, err := h.FirstAPIVersion(serviceModelName)
-		if err != nil {
-			return "", "", err
-		}
-		h.apiVersion = apiVersion
-	}
-	versionPath := filepath.Join(
-		h.basePath, "models", "apis", serviceModelName, h.apiVersion,
-	)
-	modelPath := filepath.Join(versionPath, "api-2.json")
-	docsPath := filepath.Join(versionPath, "docs-2.json")
-	return modelPath, docsPath, nil
-}
-
-// FirstAPIVersion returns the first found API version for a service API.
-// (e.h. "2012-10-03")
-func (h *SDKHelper) FirstAPIVersion(serviceModelName string) (string, error) {
-	versions, err := h.GetAPIVersions(serviceModelName)
-	if err != nil {
-		return "", err
-	}
-	sort.Strings(versions)
-	return versions[0], nil
-}
-
-// GetAPIVersions returns the list of API Versions found in a service directory.
-func (h *SDKHelper) GetAPIVersions(serviceModelName string) ([]string, error) {
-	apiPath := filepath.Join(h.basePath, "models", "apis", serviceModelName)
-	versionDirs, err := ioutil.ReadDir(apiPath)
-	if err != nil {
-		return nil, err
-	}
-	versions := []string{}
-	for _, f := range versionDirs {
-		version := f.Name()
-		fp := filepath.Join(apiPath, version)
-		fi, err := os.Lstat(fp)
-		if err != nil {
-			return nil, err
-		}
-		if !fi.IsDir() {
-			return nil, fmt.Errorf("found %s: %v", version, ErrInvalidVersionDirectory)
-		}
-		versions = append(versions, version)
-	}
-	if len(versions) == 0 {
-		return nil, ErrNoValidVersionDirectory
-	}
-	return versions, nil
-}
-
 // SDKAPI contains an API model for a single AWS service API
 type SDKAPI struct {
-	API *awssdkmodel.API
+	API            *awssdkmodel.API
+	APIGroupSuffix string
 	// A map of operation type and resource name to
 	// aws-sdk-go/private/model/api.Operation structs
 	opMap *OperationMap
 	// Map, keyed by original Shape GoTypeElem(), with the values being a
 	// renamed type name (due to conflicting names)
 	typeRenames map[string]string
 	// Default is "services.k8s.aws"
-	apiGroupSuffix string
 }
 
 // GetPayloads returns a slice of strings of Shape names representing input and
@@ -245,41 +101,6 @@ func (a *SDKAPI) GetOutputShapeRef(
 	return getMemberByPath(op.OutputRef.Shape, path)
 }
 
-// getMemberByPath returns a ShapeRef given a root Shape and a dot-notation
-// object search path. Given the explicit type check for list type members
-// both ".." and "." notations work currently.
-// TODO: Add support for other types such as map.
-func getMemberByPath(
-	shape *awssdkmodel.Shape,
-	path string,
-) (*awssdkmodel.ShapeRef, bool) {
-	elements := strings.Split(path, ".")
-	last := len(elements) - 1
-	for x, elem := range elements {
-		if elem == "" {
-			continue
-		}
-		if shape == nil {
-			return nil, false
-		}
-		shapeRef, ok := shape.MemberRefs[elem]
-		if !ok {
-			return nil, false
-		}
-		if x == last {
-			return shapeRef, true
-		}
-		elemType := shapeRef.Shape.Type
-		switch elemType {
-		case "list":
-			shape = shapeRef.Shape.MemberRef.Shape
-		default:
-			shape = shapeRef.Shape
-		}
-	}
-	return nil, false
-}
-
 // CRDNames returns a slice of names structs for all top-level resources in the
 // API
 func (a *SDKAPI) CRDNames(cfg *ackgenconfig.Config) []names.Names {
@@ -380,6 +201,17 @@ func (a *SDKAPI) APIInterfaceTypeName() string {
 	return a.API.StructName()
 }
 
+// NewSDKAPI returns a pointer to a new `ackmodel.SDKAPI` struct that describes
+// the AWS SDK API and its respective groupings, mappings and renamings.
+func NewSDKAPI(api *awssdkmodel.API, apiGroupSuffix string) *SDKAPI {
+	return &SDKAPI{
+		API:            api,
+		APIGroupSuffix: apiGroupSuffix,
+		opMap:          nil,
+		typeRenames:    nil,
+	}
+}
+
 // Override the operation type and/or resource name if specified in config
 func getOpTypeAndResourceName(opID string, cfg *ackgenconfig.Config) ([]OpType, string) {
 	opType, resName := GetOpTypeAndResourceNameFromOpID(opID, cfg)
@@ -400,3 +232,38 @@ func getOpTypeAndResourceName(opID string, cfg *ackgenconfig.Config) ([]OpType,
 	}
 	return opTypes, resName
 }
+
+// getMemberByPath returns a ShapeRef given a root Shape and a dot-notation
+// object search path. Given the explicit type check for list type members
+// both ".." and "." notations work currently.
+// TODO: Add support for other types such as map.
+func getMemberByPath(
+	shape *awssdkmodel.Shape,
+	path string,
+) (*awssdkmodel.ShapeRef, bool) {
+	elements := strings.Split(path, ".")
+	last := len(elements) - 1
+	for x, elem := range elements {
+		if elem == "" {
+			continue
+		}
+		if shape == nil {
+			return nil, false
+		}
+		shapeRef, ok := shape.MemberRefs[elem]
+		if !ok {
+			return nil, false
+		}
+		if x == last {
+			return shapeRef, true
+		}
+		elemType := shapeRef.Shape.Type
+		switch elemType {
+		case "list":
+			shape = shapeRef.Shape.MemberRef.Shape
+		default:
+			shape = shapeRef.Shape
+		}
+	}
+	return nil, false
+}