diff --git a/.idea/runConfigurations/Docker_Build_And_Push.xml b/.idea/runConfigurations/Docker_Build_And_Push.xml new file mode 100644 index 000000000..59d1c0d0d --- /dev/null +++ b/.idea/runConfigurations/Docker_Build_And_Push.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/internal/app/cloudinfo/api/handlers.go b/internal/app/cloudinfo/api/handlers.go index fa17bd8b7..321ee5beb 100644 --- a/internal/app/cloudinfo/api/handlers.go +++ b/internal/app/cloudinfo/api/handlers.go @@ -390,6 +390,42 @@ func (r *RouteHandler) getProducts() gin.HandlerFunc { } } +func (r *RouteHandler) getSeries() gin.HandlerFunc { + return func(c *gin.Context) { + pathParams := GetRegionPathParams{} + if err := mapstructure.Decode(getPathParamMap(c), &pathParams); err != nil { + r.errorResponder.Respond(c, errors.WithDetails(err, "validation")) + return + } + + if ve := ValidatePathData(pathParams); ve != nil { + r.errorResponder.Respond(c, errors.WithDetails(ve, "validation")) + return + } + + logger := log.WithFieldsForHandlers(c, r.log, + map[string]interface{}{"provider": pathParams.Provider, "service": pathParams.Service, "region": pathParams.Region}) + logger.Info("getting instance series details") + + scrapingTime, err := r.prod.GetStatus(pathParams.Provider) + if err != nil { + r.errorResponder.Respond(c, errors.WrapIfWithDetails(err, "failed to retrieve status", + "provider", pathParams.Provider)) + return + } + categorySeriesMap, seriesDetails, err := r.prod.GetSeriesDetails(pathParams.Provider, pathParams.Service, pathParams.Region) + if err != nil { + r.errorResponder.Respond(c, errors.WrapIfWithDetails(err, + "failed to retrieve instance series details", + "provider", pathParams.Provider, "service", pathParams.Service, "region", pathParams.Region)) + return + } + + logger.Debug("successfully retrieved instance series details") + c.JSON(http.StatusOK, SeriesDetailsResponse{categorySeriesMap, seriesDetails, scrapingTime}) + } +} + // swagger:route GET /providers/{provider}/services/{service}/regions/{region}/images images getImages // // Provides a list of available images on a given provider in a specific region for a service. diff --git a/internal/app/cloudinfo/api/routes.go b/internal/app/cloudinfo/api/routes.go index 20f06ab29..da952e9c2 100644 --- a/internal/app/cloudinfo/api/routes.go +++ b/internal/app/cloudinfo/api/routes.go @@ -116,6 +116,7 @@ func (r *RouteHandler) ConfigureRoutes(router *gin.Engine, basePath string) { providerGroup.GET("/:provider/services/:service/regions/:region/images", r.getImages()) providerGroup.GET("/:provider/services/:service/regions/:region/versions", r.getVersions()) providerGroup.GET("/:provider/services/:service/regions/:region/products", r.getProducts()) + providerGroup.GET("/:provider/services/:service/regions/:region/series", r.getSeries()) } base.POST("/graphql", r.query()) diff --git a/internal/app/cloudinfo/api/types.go b/internal/app/cloudinfo/api/types.go index bb4ff4d43..1e5156237 100644 --- a/internal/app/cloudinfo/api/types.go +++ b/internal/app/cloudinfo/api/types.go @@ -73,6 +73,12 @@ type ProductDetailsResponse struct { ScrapingTime string `json:"scrapingTime"` } +type SeriesDetailsResponse struct { + CategoryDetails map[string]map[string][]string `json:"categoryDetails"` + SeriesDetails []types.SeriesDetails `json:"seriesDetails"` + ScrapingTime string `json:"scrapingTime"` +} + // RegionsResponse holds the list of available regions of a cloud provider // swagger:model RegionsResponse type RegionsResponse []types.Region diff --git a/internal/cloudinfo/cloudinfo.go b/internal/cloudinfo/cloudinfo.go index b643ab176..554bd6de3 100644 --- a/internal/cloudinfo/cloudinfo.go +++ b/internal/cloudinfo/cloudinfo.go @@ -152,6 +152,38 @@ func (cpi *cloudInfo) GetProductDetails(provider, service, region string) ([]typ return details, nil } +func (cpi *cloudInfo) GetSeriesDetails(provider, service, region string) (map[string]map[string][]string, []types.SeriesDetails, error) { + vms, ok := cpi.cloudInfoStore.GetVm(provider, service, region) + if !ok { + cpi.log.Debug("VMs Information not yet cached") + return nil, nil, errors.NewWithDetails("VMs Information not yet cached", "provider", provider, "service", service, "region", region) + } + + categorySeriesMap := map[string]map[string][]string{} + for _, vm := range vms { + if _, ok := categorySeriesMap[vm.Category]; !ok { + categorySeriesMap[vm.Category] = map[string][]string{} + } + if _, ok := categorySeriesMap[vm.Category][vm.Series]; !ok { + categorySeriesMap[vm.Category][vm.Series] = make([]string, 0) + } + categorySeriesMap[vm.Category][vm.Series] = append(categorySeriesMap[vm.Category][vm.Series], vm.Type) + } + + var seriesDetails []types.SeriesDetails + for category, seriesMap := range categorySeriesMap { + for series, instanceTypeList := range seriesMap { + seriesDetails = append(seriesDetails, types.SeriesDetails{ + Series: series, + Category: category, + InstanceTypes: instanceTypeList, + }) + } + } + + return categorySeriesMap, seriesDetails, nil +} + // GetStatus retrieves status form the given provider func (cpi *cloudInfo) GetStatus(provider string) (string, error) { if cachedStatus, ok := cpi.cloudInfoStore.GetStatus(provider); ok { diff --git a/internal/cloudinfo/cloudinfo_test.go b/internal/cloudinfo/cloudinfo_test.go index eae1e9720..2ddb31806 100644 --- a/internal/cloudinfo/cloudinfo_test.go +++ b/internal/cloudinfo/cloudinfo_test.go @@ -48,6 +48,23 @@ func (dcis *DummyCloudInfoStore) GetRegions(provider, service string) (map[strin } } +func (dcis *DummyCloudInfoStore) GetVm(provider, service, region string) ([]types.VMInfo, bool) { + switch dcis.TcId { + case notCached: + return nil, false + default: + return []types.VMInfo{ + {Category: types.CategoryGeneral, Series: "series0", Type: "instanceType00"}, + {Category: types.CategoryCompute, Series: "series1", Type: "instanceType10"}, + {Category: types.CategoryCompute, Series: "series1", Type: "instanceType11"}, + {Category: types.CategoryMemory, Series: "series2", Type: "instanceType20"}, + {Category: types.CategoryMemory, Series: "series2", Type: "instanceType21"}, + {Category: types.CategoryMemory, Series: "series2", Type: "instanceType22"}, + }, + true + } +} + func (dcis *DummyCloudInfoStore) GetZones(provider, service, region string) ([]string, bool) { switch dcis.TcId { case notCached: @@ -193,6 +210,55 @@ func TestCachingCloudInfo_GetRegions(t *testing.T) { } } +func TestCloudInfo_GetSeries(t *testing.T) { + tests := []struct { + name string + ciStore CloudInfoStore + checker func(categorySeriesMap map[string]map[string][]string, seriesDetails []types.SeriesDetails, err error) + }{ + { + name: "successfully retrieved the series", + ciStore: &DummyCloudInfoStore{}, + checker: func(categorySeriesMap map[string]map[string][]string, seriesDetails []types.SeriesDetails, err error) { + assert.Nil(t, err, "the error should be nil") + + assert.Equal(t, categorySeriesMap, map[string]map[string][]string{ + types.CategoryGeneral: { + "series0": []string{"instanceType00"}, + }, + types.CategoryCompute: { + "series1": []string{"instanceType10", "instanceType11"}, + }, + types.CategoryMemory: { + "series2": []string{"instanceType20", "instanceType21", "instanceType22"}, + }, + }) + + assert.ElementsMatch(t, seriesDetails, []types.SeriesDetails{ + {Series: "series0", Category: types.CategoryGeneral, InstanceTypes: []string{"instanceType00"}}, + {Series: "series1", Category: types.CategoryCompute, InstanceTypes: []string{"instanceType10", "instanceType11"}}, + {Series: "series2", Category: types.CategoryMemory, InstanceTypes: []string{"instanceType20", "instanceType21", "instanceType22"}}, + }) + }, + }, + { + name: "failed to retrieve series", + ciStore: &DummyCloudInfoStore{TcId: notCached}, + checker: func(categorySeriesMap map[string]map[string][]string, seriesDetails []types.SeriesDetails, err error) { + assert.Nil(t, seriesDetails, "the seriesDetails should be nil") + assert.EqualError(t, err, "VMs Information not yet cached") + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + info, _ := NewCloudInfo([]string{}, &DummyCloudInfoStore{}, cloudinfoLogger) + info.cloudInfoStore = test.ciStore + test.checker(info.GetSeriesDetails("dummyProvider", "dummyService", "dummyRegion")) + }) + } +} + func TestCachingCloudInfo_GetVersions(t *testing.T) { tests := []struct { name string diff --git a/internal/cloudinfo/instance_type.go b/internal/cloudinfo/instance_type.go index 65faae808..f06bd543e 100644 --- a/internal/cloudinfo/instance_type.go +++ b/internal/cloudinfo/instance_type.go @@ -60,6 +60,7 @@ type InstanceType struct { Gpu float64 NetworkCategory NetworkCategory Category InstanceTypeCategory + Series string } // InstanceTypeQuery represents the input parameters if an instance type query. @@ -413,5 +414,6 @@ func transform(details types.ProductDetails, region string, zone string) Instanc Gpu: details.Gpus, NetworkCategory: NetworkCategory(strings.ToUpper(details.NtwPerfCat)), Category: instanceTypeCategoryReverseMap[details.Category], + Series: details.Series, } } diff --git a/internal/cloudinfo/providers/amazon/cloudinfo.go b/internal/cloudinfo/providers/amazon/cloudinfo.go index 2e1b5280a..11de3ab89 100644 --- a/internal/cloudinfo/providers/amazon/cloudinfo.go +++ b/internal/cloudinfo/providers/amazon/cloudinfo.go @@ -193,6 +193,7 @@ func (e *Ec2Infoer) GetVirtualMachines(region string) ([]types.VMInfo, error) { gpus, _ := strconv.ParseFloat(gpu, 64) vm := types.VMInfo{ Category: instanceFamily, + Series: e.mapSeries(instanceType), Type: instanceType, OnDemandPrice: onDemandPrice, Cpus: cpus, diff --git a/internal/cloudinfo/providers/amazon/instancefamily_mapper.go b/internal/cloudinfo/providers/amazon/instancefamily_mapper.go new file mode 100644 index 000000000..66d23ce96 --- /dev/null +++ b/internal/cloudinfo/providers/amazon/instancefamily_mapper.go @@ -0,0 +1,33 @@ +// Copyright © 2021 Banzai Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package amazon + +import ( + "strings" +) + +// mapSeries get instance series associated with the instanceType +func (e *Ec2Infoer) mapSeries(instanceType string) string { + series := strings.Split(instanceType, ".") + + if len(series) != 2 { + e.log.Warn("error parsing instance series from instanceType", map[string]interface{}{"instanceType": instanceType}) + + // return instanceType itself when there is a parsing error, to speedup debugging + return instanceType + } + + return series[0] +} diff --git a/internal/cloudinfo/providers/amazon/instancefamily_mapper_test.go b/internal/cloudinfo/providers/amazon/instancefamily_mapper_test.go new file mode 100644 index 000000000..0796c1016 --- /dev/null +++ b/internal/cloudinfo/providers/amazon/instancefamily_mapper_test.go @@ -0,0 +1,40 @@ +// Copyright © 2021 Banzai Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package amazon + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "logur.dev/logur" + + "github.com/banzaicloud/cloudinfo/internal/cloudinfo/cloudinfoadapter" +) + +func TestAmazonInfoer_mapSeries(t *testing.T) { + familySeriesMap := map[string]string{ + "r5.large": "r5", + "r5.metal": "r5", + "r5d.24xlarge": "r5d", + } + + ec2Infoer := Ec2Infoer{log: cloudinfoadapter.NewLogger(&logur.TestLogger{})} + + for family, series := range familySeriesMap { + t.Run("test parsing "+family, func(t *testing.T) { + assert.Equal(t, ec2Infoer.mapSeries(family), series, "unexpected series") + }) + } +} diff --git a/internal/cloudinfo/providers/azure/cloudinfo.go b/internal/cloudinfo/providers/azure/cloudinfo.go index 87766cd27..0db9b9244 100644 --- a/internal/cloudinfo/providers/azure/cloudinfo.go +++ b/internal/cloudinfo/providers/azure/cloudinfo.go @@ -388,6 +388,7 @@ func (a *AzureInfoer) GetVirtualMachines(region string) ([]types.VMInfo, error) virtualMachines = append(virtualMachines, types.VMInfo{ Category: category, + Series: a.mapSeries(*sku.Family), Type: *sku.Name, Mem: memory, Cpus: cpu, diff --git a/internal/cloudinfo/providers/azure/category_mapper.go b/internal/cloudinfo/providers/azure/instancefamily_mapper.go similarity index 67% rename from internal/cloudinfo/providers/azure/category_mapper.go rename to internal/cloudinfo/providers/azure/instancefamily_mapper.go index 04ef26ca8..7c5dffa62 100644 --- a/internal/cloudinfo/providers/azure/category_mapper.go +++ b/internal/cloudinfo/providers/azure/instancefamily_mapper.go @@ -27,19 +27,26 @@ import ( var ( categoryMap = map[string][]string{ types.CategoryGeneral: {"Dv2", "Av2", "Dv3", "DSv2", "DSv3", "BS", "DS", "D", "A0_A7", "A", "A8_A11", "DCS"}, - types.CategoryCompute: {"H", "FSv2", "FS", "", "HCS", "HBS"}, - types.CategoryMemory: {"Ev3", "ESv3", "MS", "G", "GS", "EIv3", "EISv3", "PBS", "MSv2"}, + types.CategoryCompute: {"H", "FSv2", "FS", "F", "", "HCS", "HBS"}, + types.CategoryMemory: {"Ev3", "ESv3", "MS", "G", "GS", "EIv3", "EISv3", "PBS", "MSv2", "MDSv2", "EDSv4", "ESv4"}, types.CategoryStorage: {"LS", "LSv2"}, - types.CategoryGpu: {"NC", "NV", "NCSv3", "NCSv2", "NDS", "NVSv2", "NVSv3"}, + types.CategoryGpu: {"NC", "NV", "NCSv3", "NCSv2", "NDS", "NVSv2", "NVSv3", "ND"}, + } + + customMap = map[string]string{ + "MDSMediumMemoryv2": "MDSv2", + "MIDSMediumMemoryv2": "MDSv2", + "MISMediumMemoryv2": "MSv2", + "MSMediumMemoryv2": "MSv2", + "NDASv4_A100": "ND", + "XEIDSv4": "EDSv4", + "XEISv4": "ESv4", } ) // mapCategory maps the family of the azure instance to category func (a *AzureInfoer) mapCategory(name string) (string, error) { - family := strings.TrimRight(name, "Family") - family = strings.TrimLeft(family, "standard") // nolint: staticcheck - family = strings.TrimRight(family, "Promo") // nolint: staticcheck - family = strings.TrimLeft(family, "basic") + family := GetFamily(name) for category, strVals := range categoryMap { if cloudinfo.Contains(strVals, family) { @@ -48,3 +55,24 @@ func (a *AzureInfoer) mapCategory(name string) (string, error) { } return "", emperror.Wrap(errors.New(family), "could not determine the category") } + +// mapSeries get instance series associated with the instanceType +func (a *AzureInfoer) mapSeries(name string) string { + return GetFamily(name) +} + +func GetFamily(name string) string { + family := strings.TrimRight(name, "Family") + family = strings.TrimLeft(family, "standard") // nolint: staticcheck + family = strings.TrimLeft(family, "Standard") // nolint: staticcheck + family = strings.TrimRight(family, "Promo") // nolint: staticcheck + family = strings.TrimLeft(family, "basic") + + for _, key := range customMap { + if key == family { + return customMap[family] + } + } + + return family +} diff --git a/internal/cloudinfo/providers/azure/instancefamily_mapper_test.go b/internal/cloudinfo/providers/azure/instancefamily_mapper_test.go new file mode 100644 index 000000000..399f77800 --- /dev/null +++ b/internal/cloudinfo/providers/azure/instancefamily_mapper_test.go @@ -0,0 +1,38 @@ +// Copyright © 2021 Banzai Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "logur.dev/logur" + + "github.com/banzaicloud/cloudinfo/internal/cloudinfo/cloudinfoadapter" +) + +func TestAzureInfoer_mapSeries(t *testing.T) { + familySeriesMap := map[string]string{ + "standardDADSv5Family": "DADSv5", + } + + azureInfoer := AzureInfoer{log: cloudinfoadapter.NewLogger(&logur.TestLogger{})} + + for family, series := range familySeriesMap { + t.Run("test parsing "+family, func(t *testing.T) { + assert.Equal(t, azureInfoer.mapSeries(family), series, "unexpected series") + }) + } +} diff --git a/internal/cloudinfo/providers/google/cloudinfo.go b/internal/cloudinfo/providers/google/cloudinfo.go index 34bd892b0..a6529c882 100644 --- a/internal/cloudinfo/providers/google/cloudinfo.go +++ b/internal/cloudinfo/providers/google/cloudinfo.go @@ -343,6 +343,7 @@ func (g *GceInfoer) GetVirtualMachines(region string) ([]types.VMInfo, error) { } vmsMap[mt.Name] = types.VMInfo{ Category: g.getCategory(mt.Name), + Series: g.mapSeries(mt.Name), Type: mt.Name, Cpus: float64(mt.GuestCpus), Mem: float64(mt.MemoryMb) / 1024, diff --git a/internal/cloudinfo/providers/google/instancefamily_mapper.go b/internal/cloudinfo/providers/google/instancefamily_mapper.go new file mode 100644 index 000000000..0bb357464 --- /dev/null +++ b/internal/cloudinfo/providers/google/instancefamily_mapper.go @@ -0,0 +1,36 @@ +// Copyright © 2021 Banzai Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import ( + "strings" +) + +// Instance series known to us +// a2 , c2 , e2 , f1, g1, m1, n1, n2, n2d, t2d + +// mapSeries get instance series associated with the instanceType +func (g *GceInfoer) mapSeries(instanceType string) string { + instanceTypeParts := strings.Split(instanceType, "-") + + if len(instanceTypeParts) == 2 || len(instanceTypeParts) == 3 { + return instanceTypeParts[0] + } + + g.log.Warn("error parsing instance series from instanceType", map[string]interface{}{"instanceType": instanceType}) + + // return instanceType as fallback so that it can be easily debugged from the caller of the APIs + return instanceType +} diff --git a/internal/cloudinfo/providers/google/instancefamily_mapper_test.go b/internal/cloudinfo/providers/google/instancefamily_mapper_test.go new file mode 100644 index 000000000..a302617ee --- /dev/null +++ b/internal/cloudinfo/providers/google/instancefamily_mapper_test.go @@ -0,0 +1,41 @@ +// Copyright © 2021 Banzai Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "logur.dev/logur" + + "github.com/banzaicloud/cloudinfo/internal/cloudinfo/cloudinfoadapter" +) + +func TestGoogleInfoer_mapSeries(t *testing.T) { + familySeriesMap := map[string]string{ + "n2-highmem-2": "n2", + "e2-micro": "e2", + "n2d-standard-8": "n2d", + "unidentifiedType": "unidentifiedType", + } + + gceInfoer := GceInfoer{log: cloudinfoadapter.NewLogger(&logur.TestLogger{})} + + for family, series := range familySeriesMap { + t.Run("test parsing "+family, func(t *testing.T) { + assert.Equal(t, gceInfoer.mapSeries(family), series, "unexpected series") + }) + } +} diff --git a/internal/cloudinfo/types/types.go b/internal/cloudinfo/types/types.go index 0eb28712e..5609607d7 100644 --- a/internal/cloudinfo/types/types.go +++ b/internal/cloudinfo/types/types.go @@ -48,6 +48,8 @@ type CloudInfo interface { GetProductDetails(provider, service, region string) ([]ProductDetails, error) + GetSeriesDetails(provider, service, region string) (map[string]map[string][]string, []SeriesDetails, error) + GetServiceImages(provider, service, region string) ([]Image, error) GetVersions(provider, service, region string) ([]LocationVersion, error) @@ -132,6 +134,12 @@ type ProductDetails struct { Burst bool `json:"burst,omitempty"` } +type SeriesDetails struct { + Series string `json:"series"` + Category string `json:"category"` + InstanceTypes []string `json:"instanceTypes"` +} + // ProductDetailSource product details related set of operations type ProductDetailSource interface { // GetProductDetails gathers the product details information known by telescope @@ -235,6 +243,7 @@ type Price struct { // VMInfo representation of a virtual machine type VMInfo struct { Category string `json:"category"` + Series string `json:"series"` Type string `json:"type"` OnDemandPrice float64 `json:"onDemandPrice"` SpotPrice []ZonePrice `json:"spotPrice"`