Add AWS gp3 support

cmd/drone-autoscaler/main.go

@@ -318,6 +318,7 @@ func setupProvider(c config.Config) (autoscaler.Provider, error) {
 			amazon.WithVolumeSize(c.Amazon.VolumeSize),
 			amazon.WithVolumeType(c.Amazon.VolumeType),
 			amazon.WithVolumeIops(c.Amazon.VolumeIops),
+			amazon.WithVolumeThroughput(c.Amazon.VolumeThroughput),
 			amazon.WithIamProfileArn(c.Amazon.IamProfileArn),
 			amazon.WithMarketType(c.Amazon.MarketType),
 		), nil

config/config.go

@@ -123,24 +123,25 @@ type (
 		}
 
 		Amazon struct {
-			DeviceName    string `envconfig:"DRONE_AMAZON_DEVICE_NAME"`
-			Image         string `envconfig:"DRONE_AMAZON_IMAGE"`
-			Instance      string `envconfig:"DRONE_AMAZON_INSTANCE"`
-			InstanceAlt   string `envconfig:"DRONE_AMAZON_INSTANCE_ALT"`
-			PrivateIP     bool   `split_words:"true"`
-			Region        string
-			Retries       int
-			SSHKey        string
-			SubnetID      string   `split_words:"true"`
-			SecurityGroup []string `split_words:"true"`
-			Tags          map[string]string
-			UserData      string `envconfig:"DRONE_AMAZON_USERDATA"`
-			UserDataFile  string `envconfig:"DRONE_AMAZON_USERDATA_FILE"`
-			VolumeSize    int64  `envconfig:"DRONE_AMAZON_VOLUME_SIZE"`
-			VolumeType    string `envconfig:"DRONE_AMAZON_VOLUME_TYPE"`
-			VolumeIops    int64  `envconfig:"DRONE_AMAZON_VOLUME_IOPS"`
-			IamProfileArn string `envconfig:"DRONE_AMAZON_IAM_PROFILE_ARN"`
-			MarketType    string `envconfig:"DRONE_AMAZON_MARKET_TYPE"`
+			DeviceName       string `envconfig:"DRONE_AMAZON_DEVICE_NAME"`
+			Image            string `envconfig:"DRONE_AMAZON_IMAGE"`
+			Instance         string `envconfig:"DRONE_AMAZON_INSTANCE"`
+			InstanceAlt      string `envconfig:"DRONE_AMAZON_INSTANCE_ALT"`
+			PrivateIP        bool   `split_words:"true"`
+			Region           string
+			Retries          int
+			SSHKey           string
+			SubnetID         string   `split_words:"true"`
+			SecurityGroup    []string `split_words:"true"`
+			Tags             map[string]string
+			UserData         string `envconfig:"DRONE_AMAZON_USERDATA"`
+			UserDataFile     string `envconfig:"DRONE_AMAZON_USERDATA_FILE"`
+			VolumeSize       int64  `envconfig:"DRONE_AMAZON_VOLUME_SIZE"`
+			VolumeType       string `envconfig:"DRONE_AMAZON_VOLUME_TYPE"`
+			VolumeIops       int64  `envconfig:"DRONE_AMAZON_VOLUME_IOPS"`
+			VolumeThroughput int64  `envconfig:"DRONE_AMAZON_VOLUME_THROUGHPUT"`
+			IamProfileArn    string `envconfig:"DRONE_AMAZON_IAM_PROFILE_ARN"`
+			MarketType       string `envconfig:"DRONE_AMAZON_MARKET_TYPE"`
 		}
 
 		DigitalOcean struct {

drivers/amazon/create.go

@@ -108,9 +108,19 @@ func (p *provider) create(ctx context.Context, opts autoscaler.InstanceCreateOpt
 		},
 	}
 
-	if p.volumeType == "io1" {
+	if p.volumeType == "io1" || p.volumeType == "io2" || p.volumeType == "gp3" {
 		for _, blockDeviceMapping := range in.BlockDeviceMappings {
-			blockDeviceMapping.Ebs.Iops = aws.Int64(p.volumeIops)
+			if p.volumeIops > 0 {
+				blockDeviceMapping.Ebs.Iops = aws.Int64(p.volumeIops)
+			}
+		}
+	}
+
+	if p.volumeType == "gp3" {
+		for _, blockDeviceMapping := range in.BlockDeviceMappings {
+			if p.volumeThroughput > 0 {
+				blockDeviceMapping.Ebs.Throughput = aws.Int64(p.volumeThroughput)
+			}
 		}
 	}
 

drivers/amazon/option.go

@@ -138,6 +138,13 @@ func WithVolumeIops(i int64) Option {
 	}
 }
 
+// WithVolumeThroughput returns an option to set the volume throughput.
+func WithVolumeThroughput(i int64) Option {
+	return func(p *provider) {
+		p.volumeThroughput = i
+	}
+}
+
 // WithIamProfileArn returns an option to set the iam profile arn.
 func WithIamProfileArn(t string) Option {
 	return func(p *provider) {

drivers/amazon/provider.go

@@ -19,23 +19,24 @@ import (
 type provider struct {
 	init sync.Once
 
-	deviceName    string
-	volumeSize    int64
-	volumeType    string
-	volumeIops    int64
-	retries       int
-	key           string
-	region        string
-	image         string
-	privateIP     bool
-	userdata      *template.Template
-	size          string
-	sizeAlt       string
-	subnet        string
-	groups        []string
-	tags          map[string]string
-	iamProfileArn string
-	spotInstance  bool
+	deviceName       string
+	volumeSize       int64
+	volumeType       string
+	volumeIops       int64
+	volumeThroughput int64
+	retries          int
+	key              string
+	region           string
+	image            string
+	privateIP        bool
+	userdata         *template.Template
+	size             string
+	sizeAlt          string
+	subnet           string
+	groups           []string
+	tags             map[string]string
+	iamProfileArn    string
+	spotInstance     bool
 }
 
 func (p *provider) getClient() *ec2.EC2 {
@@ -72,9 +73,15 @@ func New(opts ...Option) autoscaler.Provider {
 	if p.volumeType == "" {
 		p.volumeType = "gp2"
 	}
-	if p.volumeType == "io1" && p.volumeIops == 0 {
+	if (p.volumeType == "io1" || p.volumeType == "io2") && p.volumeIops == 0 {
 		p.volumeIops = 100
 	}
+	if p.volumeType == "gp3" && p.volumeIops == 0 {
+		p.volumeIops = 3000 // 3000 is the minimum for gp3
+	}
+	if p.volumeType == "gp3" && p.volumeThroughput == 0 {
+		p.volumeThroughput = 125 // 125 is the minimum for gp3
+	}
 	if p.userdata == nil {
 		p.userdata = userdata.T
 	}

drivers/google/create_test.go

@@ -19,18 +19,18 @@ import (
 func TestCreate(t *testing.T) {
 	defer gock.Off()
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Post("/compute/v1/projects/my-project/zones/us-central1-a/instances").
 		JSON(insertInstanceMock).
 		Reply(200).
 		BodyString(`{ "name": "operation-name" }`)
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Get("/compute/v1/projects/my-project/zones/us-central1-a/instances/agent-807jvfwj").
 		Reply(200).
 		BodyString(`{ "networkInterfaces": [ { "accessConfigs": [ { "natIP": "1.2.3.4" } ] } ] }`)
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Get("/compute/v1/projects/my-project/zones/us-central1-a/operations/operation-name").
 		Reply(200).
 		BodyString(`{ "status": "DONE" }`)
@@ -82,18 +82,18 @@ func TestCreate(t *testing.T) {
 func TestCreateWithMultiZones(t *testing.T) {
 	defer gock.Off()
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Post("/compute/v1/projects/my-project/zones/us-central1-b/instances").
 		JSON(insertInstanceMockB).
 		Reply(200).
 		BodyString(`{ "name": "operation-name" }`)
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Get("/compute/v1/projects/my-project/zones/us-central1-b/instances/agent-807jvfwj").
 		Reply(200).
 		BodyString(`{ "networkInterfaces": [ { "accessConfigs": [ { "natIP": "1.2.3.4" } ] } ] }`)
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Get("/compute/v1/projects/my-project/zones/us-central1-b/operations/operation-name").
 		Reply(200).
 		BodyString(`{ "status": "DONE" }`)

drivers/google/destroy_test.go

@@ -16,12 +16,12 @@ import (
 func TestDestroy(t *testing.T) {
 	defer gock.Off()
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Delete("/compute/v1/projects/my-project/zones/us-central1-a/instances/my-instance").
 		Reply(200).
 		BodyString(`{ "name": "operation-name" }`)
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Get("/compute/v1/projects/my-project/zones/us-central1-a/operations/operation-name").
 		Reply(200).
 		BodyString(`{ "status": "DONE" }`)
@@ -51,7 +51,7 @@ func TestDestroy(t *testing.T) {
 func TestDestroy_Error(t *testing.T) {
 	defer gock.Off()
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Delete("/compute/v1/projects/my-project/zones/us-central1-a/instances/my-instance").
 		Reply(404)
 

drivers/google/setup_test.go

@@ -16,17 +16,17 @@ import (
 func TestSetupFirewall(t *testing.T) {
 	defer gock.Off()
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Get("/compute/v1/projects/my-project/global/firewalls/default-allow-docker").
 		Reply(404)
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Post("/compute/v1/projects/my-project/global/firewalls").
 		JSON(createFirewallMock).
 		Reply(200).
 		BodyString(`{ "name": "operation-name" }`)
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Get("/compute/v1/projects/my-project/global/operations/operation-name").
 		Reply(200).
 		BodyString(`{ "status": "DONE" }`)
@@ -50,7 +50,7 @@ func TestSetupFirewall(t *testing.T) {
 func TestSetupFirewall_Exists(t *testing.T) {
 	defer gock.Off()
 
-	gock.New("https://www.googleapis.com").
+	gock.New("https://compute.googleapis.com").
 		Get("/compute/v1/projects/my-project/global/firewalls/default-allow-docker").
 		Reply(200).
 		BodyString(findFirewallRes)

go.mod

@@ -1,81 +1,84 @@
 module github.com/drone/autoscaler
 
-go 1.12
+go 1.19
 
 replace github.com/docker/docker => github.com/docker/engine v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
 
 require (
-	cloud.google.com/go v0.28.0 // indirect
 	github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d
-	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
-	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/semver v1.5.0 // indirect
-	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
-	github.com/Microsoft/go-winio v0.4.7 // indirect
-	github.com/avast/retry-go v3.0.0+incompatible // indirect
-	github.com/aws/aws-sdk-go v1.13.5
-	github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a // indirect
+	github.com/avast/retry-go v3.0.0+incompatible
+	github.com/aws/aws-sdk-go v1.44.186
 	github.com/bluele/slack v0.0.0-20171128075526-307046097ee9
-	github.com/containerd/containerd v1.3.4 // indirect
 	github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9
 	github.com/digitalocean/godo v1.1.1
-	github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff // indirect
 	github.com/docker/docker v0.0.0-00010101000000-000000000000
-	github.com/docker/go-connections v0.3.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
+	github.com/docker/go-connections v0.3.0
 	github.com/drone/drone-go v1.0.5-0.20190504210458-4d6116b897ba
 	github.com/drone/envconfig v1.4.1
-	github.com/drone/funcmap v0.0.0-20210903193859-704120d6923c
+	github.com/drone/funcmap v0.0.0-20220929084810-72602997d16f
 	github.com/drone/signal v0.0.0-20170915013802-ac5d07ef1315
 	github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4
 	github.com/go-chi/chi v3.3.2+incompatible
-	github.com/go-ini/ini v1.32.0 // indirect
 	github.com/go-sql-driver/mysql v1.3.0
-	github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506 // indirect
-	github.com/golang/mock v1.3.1
-	github.com/google/go-cmp v0.4.0
-	github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/golang/mock v1.4.4
+	github.com/google/go-cmp v0.5.8
 	github.com/gophercloud/gophercloud v0.0.0-20181014043407-c8947f7d1c51
-	github.com/gorilla/mux v1.7.4 // indirect
 	github.com/h2non/gock v1.0.7
 	github.com/hetznercloud/hcloud-go v1.4.0
-	github.com/huandu/xstrings v1.3.2 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 // indirect
 	github.com/jmoiron/sqlx v0.0.0-20180228184624-cf35089a1979
 	github.com/joho/godotenv v1.2.0
-	github.com/kr/pretty v0.0.0-20160823170715-cfb55aafdaf3
-	github.com/kr/text v0.0.0-20160504234017-7cafcd837844 // indirect
+	github.com/kr/pretty v0.1.0
 	github.com/lib/pq v1.10.4
 	github.com/mattn/go-sqlite3 v1.6.0
-	github.com/matttproud/golang_protobuf_extensions v1.0.0 // indirect
-	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/packethost/packngo v0.1.0
+	github.com/prometheus/client_golang v1.14.0
+	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.3
+	github.com/sirupsen/logrus v1.6.0
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
+	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
+	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
+	google.golang.org/api v0.30.0
+)
+
+require (
+	cloud.google.com/go v0.65.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
+	github.com/Microsoft/go-winio v0.4.7 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/containerd/containerd v1.3.4 // indirect
+	github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/gogo/protobuf v1.1.1 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/gorilla/mux v1.7.4 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
+	github.com/kr/text v0.1.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 // indirect
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/packethost/packngo v0.1.0
-	github.com/pkg/errors v0.8.1 // indirect
-	github.com/prometheus/client_golang v0.8.0
-	github.com/prometheus/common v0.0.0-20180110214958-89604d197083 // indirect
-	github.com/prometheus/procfs v0.0.0-20180212145926-282c8707aa21 // indirect
-	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.3
-	github.com/sirupsen/logrus v1.4.2
-	github.com/smartystreets/goconvey v1.6.4 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/tent/http-link-go v0.0.0-20130702225549-ac974c61c2f9 // indirect
-	golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4
-	golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 // indirect
-	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect
-	golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 // indirect
-	google.golang.org/api v0.0.0-20180921000521-920bb1beccf7
-	google.golang.org/appengine v1.4.0 // indirect
-	google.golang.org/grpc v1.30.0 // indirect
-	gopkg.in/ini.v1 v1.51.0 // indirect
+	go.opencensus.io v0.22.4 // indirect
+	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	google.golang.org/appengine v1.6.6 // indirect
+	google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 // indirect
+	google.golang.org/grpc v1.31.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gotest.tools v2.2.0+incompatible // indirect
-	honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc // indirect
+	launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
 )
-
-replace github.com/drone/funcmap => github.com/iainlane/funcmap v0.0.0-20211116113722-13f662008062