Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cmd/ack-generate/command/apis.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/spf13/cobra"

ackgenerate "github.com/aws-controllers-k8s/code-generator/pkg/generate/ack"
ackmetadata "github.com/aws-controllers-k8s/code-generator/pkg/metadata"
ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model"
"github.com/aws-controllers-k8s/code-generator/pkg/util"
)
Expand Down Expand Up @@ -59,10 +60,10 @@ func init() {
// saveGeneratedMetadata saves the parameters used to generate APIs and checksum
// of the generated code.
func saveGeneratedMetadata(cmd *cobra.Command, args []string) error {
err := ackgenerate.CreateGenerationMetadata(
err := ackmetadata.CreateGenerationMetadata(
optGenVersion,
filepath.Join(optOutputPath, "apis"),
ackgenerate.UpdateReasonAPIGeneration,
ackmetadata.UpdateReasonAPIGeneration,
optAWSSDKGoVersion,
optGeneratorConfigPath,
)
Expand Down
4 changes: 4 additions & 0 deletions cmd/ack-generate/command/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ var (
optDryRun bool
sdkDir string
optGeneratorConfigPath string
optMetadataConfigPath string
optOutputPath string
)

Expand Down Expand Up @@ -111,6 +112,9 @@ func init() {
rootCmd.PersistentFlags().StringVar(
&optGeneratorConfigPath, "generator-config-path", "", "Path to file containing instructions for code generation to use",
)
rootCmd.PersistentFlags().StringVar(
&optMetadataConfigPath, "metadata-config-path", "", "Path to file containing service metadata to use",
)
rootCmd.PersistentFlags().StringVarP(
&optOutputPath, "output", "o", "", "Path to directory to output generated files.",
)
Expand Down
8 changes: 6 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws-controllers-k8s/runtime v0.4.0 h1:qsCp1OKXoSMtjS+6zlH65PN3/UkVJNbI/ybwYjPmhrY=
github.com/aws-controllers-k8s/runtime v0.4.0/go.mod h1:xA2F18PJerBHaqrS4de1lpP7skeSMeStkmh+3x5sWvw=
github.com/aws-controllers-k8s/runtime v0.5.0 h1:lV2QxuEHHKS+nbJ7lYgPjwVBY6Aqw+RHJF8eDZZeswM=
github.com/aws-controllers-k8s/runtime v0.5.0/go.mod h1:xA2F18PJerBHaqrS4de1lpP7skeSMeStkmh+3x5sWvw=
github.com/aws/aws-sdk-go v1.37.4 h1:tWxrpMK/oRSXVnjUzhGeCWLR00fW0WF4V4sycYPPrJ8=
github.com/aws/aws-sdk-go v1.37.4/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
Expand Down Expand Up @@ -475,6 +475,7 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
Expand Down Expand Up @@ -584,6 +585,7 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPI
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0 h1:sfUMP1Gu8qASkorDVjnMuvgJzwFbTZSeXFiGBYAVdl4=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
Expand Down Expand Up @@ -622,6 +624,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
Expand Down Expand Up @@ -688,6 +691,7 @@ golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPj
golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,12 @@
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package multiversion

// TODO(a-hilaly) move this file outside of pkg/model/multiversion. Idealy we
// Should be able to access APIStatus and APIInfo to prevent regenerating removed or
// deprecated APIs.
package metadata

type APIStatus string

const (
APIStatusUnknown APIStatus = "unknown"
APIStatusAvailable = "available"
APIStatusAvailable APIStatus = "available"
APIStatusRemoved = "removed"
APIStatusDeprecated = "deprecated"
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package ack
package metadata

import (
"crypto/sha1"
Expand Down
109 changes: 109 additions & 0 deletions pkg/metadata/service_metadata.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
// http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package metadata

import (
"errors"
"io/ioutil"

"github.com/ghodss/yaml"
)

var (
ErrNoServiceMetadataFile = errors.New("expected metadata file path, none provided")
ErrNoAvailableVersions = errors.New("service metadata contains no available versions")
)

// ServiceMetadata consists of information about the service and relative links as well
// as a list of supported/deprecated versions
type ServiceMetadata struct {
Service ServiceDetails `json:"service"`
// A list of all generated API versions of the service
APIVersions []ServiceVersion `json:"api_versions"`
}

// ServiceDetails contains string identifiers and relevant links for the
// service
type ServiceDetails struct {
// The full display name for the service. eg. Amazon Elastic Kubernetes
// Service
FullName string `json:"full_name"`
// The short name (abbreviation) for the service. eg. S3
ShortName string `json:"short_name"`
// The URL of the service's homepage
Link string `json:"link"`
// The URL of the service's main documentation/user guide
Documentation string `json:"documentation"`
}

// ServiceVersion describes the status of all existing version of the controller
type ServiceVersion struct {
APIVersion string `json:"api_version"`
Status APIStatus `json:"status"`
}
Comment on lines +51 to +54
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could also add a boolean field called IsHub, to be able to override the hub version. In most of the cases the latest Available version is the hub (prefered storage version). However, according to k8s deprecation policy sometimes the hub can be different from the latest version.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am quite averse to the idea of the hub being anything other than the latest generated version - we rely on it being the latest for the common runtime elements to match.


func (m *ServiceMetadata) GetLatestAPIVersion() (string, error) {
availableVersions := m.GetAvailableAPIVersions()

if len(availableVersions) == 0 {
return "", ErrNoAvailableVersions
}

return availableVersions[len(availableVersions)-1], nil
}

func (m *ServiceMetadata) GetDeprecatedAPIVersions() []string {
return m.getVersionsByStatus(APIStatusDeprecated)
}

func (m *ServiceMetadata) GetRemovedAPIVersions() []string {
return m.getVersionsByStatus(APIStatusRemoved)
}

func (m *ServiceMetadata) GetAvailableAPIVersions() []string {
return m.getVersionsByStatus(APIStatusAvailable)
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

half nit:please add godoc comment, at least for the exported functions


func (m *ServiceMetadata) getVersionsByStatus(status APIStatus) []string {
if len(m.APIVersions) == 0 {
return []string{}
}

versions := []string{}
for _, v := range m.APIVersions {
if v.Status == status {
versions = append(versions, v.APIVersion)
}
}
return versions
}

// NewServiceMetadata returns a new Metadata object given a supplied
// path to a metadata file
func NewServiceMetadata(
metadataPath string,
) (ServiceMetadata, error) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: you can return a *ServiceMetadata to avoid allocating a new ServiceMetadata{} when you return an error

if metadataPath == "" {
return ServiceMetadata{}, ErrNoServiceMetadataFile
}
content, err := ioutil.ReadFile(metadataPath)
if err != nil {
return ServiceMetadata{}, err
}
gc := ServiceMetadata{}
if err = yaml.Unmarshal(content, &gc); err != nil {
return ServiceMetadata{}, err
}
return gc, nil
}
51 changes: 33 additions & 18 deletions pkg/model/multiversion/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"gopkg.in/src-d/go-git.v4"

ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/generate/config"
ackmetadata "github.com/aws-controllers-k8s/code-generator/pkg/metadata"
ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model"
"github.com/aws-controllers-k8s/code-generator/pkg/util"
)
Expand All @@ -35,31 +36,36 @@ var (
// of each non-deprecated version with their correspending ackmodel.Model
// and APIInfos.
type APIVersionManager struct {
gitRepo *git.Repository
gitRepo *git.Repository
metadata ackmetadata.ServiceMetadata

hubVersion string
spokeVersions []string

deprecatedVersions []string
removedVersions []string

apiInfos map[string]APIInfo
apiInfos map[string]ackmetadata.APIInfo
models map[string]*ackmodel.Model
}

// NewAPIVersionManager initialises and returns a new APIVersionManager.
func NewAPIVersionManager(
sdkCacheDir string,
metadataPath string,
serviceAlias string,
hubVersion string,
apisInfo map[string]APIInfo,
apisInfo map[string]ackmetadata.APIInfo,
defaultConfig ackgenconfig.Config,
) (*APIVersionManager, error) {
if len(apisInfo) == 0 {
return nil, fmt.Errorf("empty apisInfo")
}

spokeVersions := make([]string, 0, len(apisInfo)-1)
metadata, err := ackmetadata.NewServiceMetadata(metadataPath)
if err != nil {
return nil, err
}

spokeVersions := []string{}

gitRepo, err := util.LoadRepository(sdkCacheDir)
if err != nil {
return nil, fmt.Errorf("cannot read sdk git repository: %v", err)
Expand All @@ -68,11 +74,19 @@ func NewAPIVersionManager(
SDKAPIHelper := ackmodel.NewSDKHelper(sdkCacheDir)

// create model for each non-deprecated api version
models := make(map[string]*ackmodel.Model, len(apisInfo))
for apiVersion, apiInfo := range apisInfo {
// TODO(a-hilaly) filter deprecated and removed api versions
if apiVersion != hubVersion {
spokeVersions = append(spokeVersions, apiVersion)
models := map[string]*ackmodel.Model{}
for _, version := range metadata.APIVersions {
if version.Status == ackmetadata.APIStatusDeprecated || version.Status == ackmetadata.APIStatusRemoved {
continue
}

if version.APIVersion != hubVersion {
spokeVersions = append(spokeVersions, version.APIVersion)
}

apiInfo, ok := apisInfo[version.APIVersion]
if !ok {
return nil, fmt.Errorf("could not find API info for API version %s", version.APIVersion)
}

err = SDKAPIHelper.WithSDKVersion(apiInfo.AWSSDKVersion)
Expand All @@ -87,25 +101,26 @@ func NewAPIVersionManager(

i, err := ackmodel.New(
SDKAPI,
apiVersion,
version.APIVersion,
apiInfo.GeneratorConfigPath,
defaultConfig,
)
if err != nil {
return nil, fmt.Errorf("cannot create model for API version %s: %v", apiVersion, err)
return nil, fmt.Errorf("cannot create model for API version %s: %v", version.APIVersion, err)
}
models[apiVersion] = i
models[version.APIVersion] = i
}

sort.Strings(spokeVersions)
model := &APIVersionManager{
gitRepo: gitRepo,
metadata: metadata,
hubVersion: hubVersion,
spokeVersions: spokeVersions,
apiInfos: apisInfo,
models: models,
}
// TODO(hilalymh): Audit deprecated and removed versions
Copy link
Member

@a-hilaly a-hilaly Jul 14, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm still going to send a PR for API versions examination. Mainly to verify that service controller maintainers didn't violate one of the deprecation/removal policies.


return model, nil
}

Expand Down Expand Up @@ -140,10 +155,10 @@ func (m *APIVersionManager) VerifyAPIVersions(apiVersions ...string) error {
if !ok {
return fmt.Errorf("%v: %s", ErrAPIVersionNotFound, apiVersion)
}
if apiInfo.Status == APIStatusDeprecated {
if apiInfo.Status == ackmetadata.APIStatusDeprecated {
return fmt.Errorf("%v: %s", ErrAPIVersionDeprecated, apiVersion)
}
if apiInfo.Status == APIStatusRemoved {
if apiInfo.Status == ackmetadata.APIStatusRemoved {
return fmt.Errorf("%v: %s", ErrAPIVersionRemoved, apiVersion)
}
}
Expand Down
8 changes: 8 additions & 0 deletions pkg/testdata/models/metadata.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
service:
full_name: Amazon Test Service
short_name: ATS
link: https://example.com/
documentation: https://docs.example.com/
versions:
- api_version: v1alpha1
status: development
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might be useful to have a helper script in the code-generator that will examine a new controller repository and construct a default metadata file given a service name...

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm curious to know what would happen if we don't have an APIStatusDevelopment

19 changes: 18 additions & 1 deletion scripts/build-controller.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ DEFAULT_ACK_GENERATE_BIN_PATH="$ROOT_DIR/bin/ack-generate"
ACK_GENERATE_BIN_PATH=${ACK_GENERATE_BIN_PATH:-$DEFAULT_ACK_GENERATE_BIN_PATH}
ACK_GENERATE_API_VERSION=${ACK_GENERATE_API_VERSION:-"v1alpha1"}
ACK_GENERATE_CONFIG_PATH=${ACK_GENERATE_CONFIG_PATH:-""}
ACK_METADATA_CONFIG_PATH=${ACK_METADATA_CONFIG_PATH:-""}
AWS_SDK_GO_VERSION=${AWS_SDK_GO_VERSION:-""}
DEFAULT_RUNTIME_CRD_DIR="$ROOT_DIR/../../aws-controllers-k8s/runtime/config"
RUNTIME_CRD_DIR=${RUNTIME_CRD_DIR:-$DEFAULT_RUNTIME_CRD_DIR}
Expand All @@ -51,7 +52,10 @@ Environment variables:
to be generated, 'v1alpha1' is used.
ACK_GENERATE_CONFIG_PATH: Specify a path to the generator config YAML file to
instruct the code generator for the service.
Default: services/{SERVICE}/generator.yaml
Default: generator.yaml
ACK_METADATA_CONFIG_PATH: Specify a path to the metadata config YAML file to
instruct the code generator for the service.
Default: metadata.yaml
AWS_SDK_GO_VERSION: Overrides the version of github.com/aws/aws-sdk-go used
by 'ack-generate' to fetch the service API Specifications.
TEMPLATES_DIR: Overrides the directory containg ack-generate templates
Expand Down Expand Up @@ -133,6 +137,14 @@ if [ -z "$ACK_GENERATE_CONFIG_PATH" ]; then
fi
fi

# If there's a metadata.yaml in the service's directory and the caller hasn't
# specified an override, use that.
if [ -z "$ACK_METADATA_CONFIG_PATH" ]; then
if [ -f "$SERVICE_CONTROLLER_SOURCE_PATH/metadata.yaml" ]; then
ACK_METADATA_CONFIG_PATH="$SERVICE_CONTROLLER_SOURCE_PATH/metadata.yaml"
fi
fi

ag_args="$SERVICE -o $SERVICE_CONTROLLER_SOURCE_PATH --template-dirs $TEMPLATE_DIRS"
if [ -n "$ACK_GENERATE_CACHE_DIR" ]; then
ag_args="$ag_args --cache-dir $ACK_GENERATE_CACHE_DIR"
Expand All @@ -148,6 +160,11 @@ if [ -n "$ACK_GENERATE_CONFIG_PATH" ]; then
apis_args="$apis_args --generator-config-path $ACK_GENERATE_CONFIG_PATH"
fi

if [ -n "$ACK_METADATA_CONFIG_PATH" ]; then
ag_args="$ag_args --metadata-config-path $ACK_METADATA_CONFIG_PATH"
apis_args="$apis_args --metadata-config-path $ACK_METADATA_CONFIG_PATH"
fi

if [ -n "$AWS_SDK_GO_VERSION" ]; then
ag_args="$ag_args --aws-sdk-go-version $AWS_SDK_GO_VERSION"
apis_args="$apis_args --aws-sdk-go-version $AWS_SDK_GO_VERSION"
Expand Down
Loading