[CCM-5210]: support for instance family endpoint

Makefile

@@ -12,7 +12,7 @@ BUILD_DIR ?= build
 VERSION ?= $(shell git describe --tags --exact-match 2>/dev/null || git symbolic-ref -q --short HEAD)
 COMMIT_HASH ?= $(shell git rev-parse --short HEAD 2>/dev/null)
 BUILD_DATE ?= $(shell date +%FT%T%z)
-LDFLAGS += -X main.version=${VERSION} -X main.commitHash=${COMMIT_HASH} -X main.buildDate=${BUILD_DATE}
+LDFLAGS := -X main.version=${VERSION} -X main.commitHash=${COMMIT_HASH} -X main.buildDate=${BUILD_DATE}
 export CGO_ENABLED ?= 0
 ifeq (${VERBOSE}, 1)
 ifeq ($(filter -v,${GOARGS}),)

internal/app/cloudinfo/api/handlers.go

@@ -390,6 +390,42 @@ func (r *RouteHandler) getProducts() gin.HandlerFunc {
 	}
 }
 
+func (r *RouteHandler) getSeries() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		pathParams := GetRegionPathParams{}
+		if err := mapstructure.Decode(getPathParamMap(c), &pathParams); err != nil {
+			r.errorResponder.Respond(c, errors.WithDetails(err, "validation"))
+			return
+		}
+
+		if ve := ValidatePathData(pathParams); ve != nil {
+			r.errorResponder.Respond(c, errors.WithDetails(ve, "validation"))
+			return
+		}
+
+		logger := log.WithFieldsForHandlers(c, r.log,
+			map[string]interface{}{"provider": pathParams.Provider, "service": pathParams.Service, "region": pathParams.Region})
+		logger.Info("getting instance series details")
+
+		scrapingTime, err := r.prod.GetStatus(pathParams.Provider)
+		if err != nil {
+			r.errorResponder.Respond(c, errors.WrapIfWithDetails(err, "failed to retrieve status",
+				"provider", pathParams.Provider))
+			return
+		}
+		categorySeriesMap, seriesDetails, err := r.prod.GetSeriesDetails(pathParams.Provider, pathParams.Service, pathParams.Region)
+		if err != nil {
+			r.errorResponder.Respond(c, errors.WrapIfWithDetails(err,
+				"failed to retrieve instance series details",
+				"provider", pathParams.Provider, "service", pathParams.Service, "region", pathParams.Region))
+			return
+		}
+
+		logger.Debug("successfully retrieved instance series details")
+		c.JSON(http.StatusOK, SeriesDetailsResponse{categorySeriesMap, seriesDetails, scrapingTime})
+	}
+}
+
 // swagger:route GET /providers/{provider}/services/{service}/regions/{region}/images images getImages
 //
 // Provides a list of available images on a given provider in a specific region for a service.

internal/app/cloudinfo/api/routes.go

@@ -116,6 +116,7 @@ func (r *RouteHandler) ConfigureRoutes(router *gin.Engine, basePath string) {
 		providerGroup.GET("/:provider/services/:service/regions/:region/images", r.getImages())
 		providerGroup.GET("/:provider/services/:service/regions/:region/versions", r.getVersions())
 		providerGroup.GET("/:provider/services/:service/regions/:region/products", r.getProducts())
+		providerGroup.GET("/:provider/services/:service/regions/:region/series", r.getSeries())
 	}
 
 	base.POST("/graphql", r.query())

internal/app/cloudinfo/api/types.go

@@ -73,6 +73,12 @@ type ProductDetailsResponse struct {
 	ScrapingTime string `json:"scrapingTime"`
 }
 
+type SeriesDetailsResponse struct {
+	CategoryDetails map[string]map[string][]string `json:"categoryDetails"`
+	SeriesDetails   []types.SeriesDetails          `json:"seriesDetails"`
+	ScrapingTime    string                         `json:"scrapingTime"`
+}
+
 // RegionsResponse holds the list of available regions of a cloud provider
 // swagger:model RegionsResponse
 type RegionsResponse []types.Region

internal/cloudinfo/cloudinfo.go

@@ -152,6 +152,38 @@ func (cpi *cloudInfo) GetProductDetails(provider, service, region string) ([]typ
 	return details, nil
 }
 
+func (cpi *cloudInfo) GetSeriesDetails(provider, service, region string) (map[string]map[string][]string, []types.SeriesDetails, error) {
+	vms, ok := cpi.cloudInfoStore.GetVm(provider, service, region)
+	if !ok {
+		cpi.log.Debug("VMs Information not yet cached")
+		return nil, nil, errors.NewWithDetails("VMs Information not yet cached", "provider", provider, "service", service, "region", region)
+	}
+
+	categorySeriesMap := map[string]map[string][]string{}
+	for _, vm := range vms {
+		if _, ok := categorySeriesMap[vm.Category]; !ok {
+			categorySeriesMap[vm.Category] = map[string][]string{}
+		}
+		if _, ok := categorySeriesMap[vm.Category][vm.Series]; !ok {
+			categorySeriesMap[vm.Category][vm.Series] = make([]string, 0)
+		}
+		categorySeriesMap[vm.Category][vm.Series] = append(categorySeriesMap[vm.Category][vm.Series], vm.Type)
+	}
+
+	var seriesDetails []types.SeriesDetails
+	for category, seriesMap := range categorySeriesMap {
+		for series, instanceTypeList := range seriesMap {
+			seriesDetails = append(seriesDetails, types.SeriesDetails{
+				Series:        series,
+				Category:      category,
+				InstanceTypes: instanceTypeList,
+			})
+		}
+	}
+
+	return categorySeriesMap, seriesDetails, nil
+}
+
 // GetStatus retrieves status form the given provider
 func (cpi *cloudInfo) GetStatus(provider string) (string, error) {
 	if cachedStatus, ok := cpi.cloudInfoStore.GetStatus(provider); ok {

internal/cloudinfo/cloudinfo_test.go

@@ -48,6 +48,23 @@ func (dcis *DummyCloudInfoStore) GetRegions(provider, service string) (map[strin
 	}
 }
 
+func (dcis *DummyCloudInfoStore) GetVm(provider, service, region string) ([]types.VMInfo, bool) {
+	switch dcis.TcId {
+	case notCached:
+		return nil, false
+	default:
+		return []types.VMInfo{
+				{Category: types.CategoryGeneral, Series: "series0", Type: "instanceType00"},
+				{Category: types.CategoryCompute, Series: "series1", Type: "instanceType10"},
+				{Category: types.CategoryCompute, Series: "series1", Type: "instanceType11"},
+				{Category: types.CategoryMemory, Series: "series2", Type: "instanceType20"},
+				{Category: types.CategoryMemory, Series: "series2", Type: "instanceType21"},
+				{Category: types.CategoryMemory, Series: "series2", Type: "instanceType22"},
+			},
+			true
+	}
+}
+
 func (dcis *DummyCloudInfoStore) GetZones(provider, service, region string) ([]string, bool) {
 	switch dcis.TcId {
 	case notCached:
@@ -193,6 +210,55 @@ func TestCachingCloudInfo_GetRegions(t *testing.T) {
 	}
 }
 
+func TestCloudInfo_GetSeries(t *testing.T) {
+	tests := []struct {
+		name    string
+		ciStore CloudInfoStore
+		checker func(categorySeriesMap map[string]map[string][]string, seriesDetails []types.SeriesDetails, err error)
+	}{
+		{
+			name:    "successfully retrieved the series",
+			ciStore: &DummyCloudInfoStore{},
+			checker: func(categorySeriesMap map[string]map[string][]string, seriesDetails []types.SeriesDetails, err error) {
+				assert.Nil(t, err, "the error should be nil")
+
+				assert.Equal(t, categorySeriesMap, map[string]map[string][]string{
+					types.CategoryGeneral: {
+						"series0": []string{"instanceType00"},
+					},
+					types.CategoryCompute: {
+						"series1": []string{"instanceType10", "instanceType11"},
+					},
+					types.CategoryMemory: {
+						"series2": []string{"instanceType20", "instanceType21", "instanceType22"},
+					},
+				})
+
+				assert.EqualValues(t, seriesDetails, []types.SeriesDetails{
+					{Series: "series0", Category: types.CategoryGeneral, InstanceTypes: []string{"instanceType00"}},
+					{Series: "series1", Category: types.CategoryCompute, InstanceTypes: []string{"instanceType10", "instanceType11"}},
+					{Series: "series2", Category: types.CategoryMemory, InstanceTypes: []string{"instanceType20", "instanceType21", "instanceType22"}},
+				})
+			},
+		},
+		{
+			name:    "failed to retrieve series",
+			ciStore: &DummyCloudInfoStore{TcId: notCached},
+			checker: func(categorySeriesMap map[string]map[string][]string, seriesDetails []types.SeriesDetails, err error) {
+				assert.Nil(t, seriesDetails, "the seriesDetails should be nil")
+				assert.EqualError(t, err, "VMs Information not yet cached")
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			info, _ := NewCloudInfo([]string{}, &DummyCloudInfoStore{}, cloudinfoLogger)
+			info.cloudInfoStore = test.ciStore
+			test.checker(info.GetSeriesDetails("dummyProvider", "dummyService", "dummyRegion"))
+		})
+	}
+}
+
 func TestCachingCloudInfo_GetVersions(t *testing.T) {
 	tests := []struct {
 		name    string

internal/cloudinfo/instance_type.go

@@ -60,6 +60,7 @@ type InstanceType struct {
 	Gpu             float64
 	NetworkCategory NetworkCategory
 	Category        InstanceTypeCategory
+	Series          string
 }
 
 // InstanceTypeQuery represents the input parameters if an instance type query.
@@ -413,5 +414,6 @@ func transform(details types.ProductDetails, region string, zone string) Instanc
 		Gpu:             details.Gpus,
 		NetworkCategory: NetworkCategory(strings.ToUpper(details.NtwPerfCat)),
 		Category:        instanceTypeCategoryReverseMap[details.Category],
+		Series:          details.Series,
 	}
 }

internal/cloudinfo/providers/amazon/cloudinfo.go

@@ -193,6 +193,7 @@ func (e *Ec2Infoer) GetVirtualMachines(region string) ([]types.VMInfo, error) {
 		gpus, _ := strconv.ParseFloat(gpu, 64)
 		vm := types.VMInfo{
 			Category:      instanceFamily,
+			Series:        e.mapSeries(instanceType),
 			Type:          instanceType,
 			OnDemandPrice: onDemandPrice,
 			Cpus:          cpus,

internal/cloudinfo/providers/amazon/instancefamily_mapper.go

@@ -0,0 +1,33 @@
+// Copyright © 2021 Banzai Cloud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package amazon
+
+import (
+	"strings"
+)
+
+// mapSeries get instance series associated with the instanceType
+func (e *Ec2Infoer) mapSeries(instanceType string) string {
+	series := strings.Split(instanceType, ".")
+
+	if len(series) != 2 {
+		e.log.Warn("error parsing instance series from instanceType", map[string]interface{}{"instanceType": instanceType})
+
+		// return instanceType itself when there is a parsing error, to speedup debugging
+		return instanceType
+	}
+
+	return series[0]
+}

internal/cloudinfo/providers/amazon/instancefamily_mapper_test.go

@@ -0,0 +1,40 @@
+// Copyright © 2021 Banzai Cloud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package amazon
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"logur.dev/logur"
+
+	"github.com/banzaicloud/cloudinfo/internal/cloudinfo/cloudinfoadapter"
+)
+
+func TestAmazonInfoer_mapSeries(t *testing.T) {
+	familySeriesMap := map[string]string{
+		"r5.large":     "r5",
+		"r5.metal":     "r5",
+		"r5d.24xlarge": "r5d",
+	}
+
+	ec2Infoer := Ec2Infoer{log: cloudinfoadapter.NewLogger(&logur.TestLogger{})}
+
+	for family, series := range familySeriesMap {
+		t.Run("test parsing "+family, func(t *testing.T) {
+			assert.Equal(t, ec2Infoer.mapSeries(family), series, "unexpected series")
+		})
+	}
+}

internal/cloudinfo/providers/azure/cloudinfo.go

@@ -388,6 +388,7 @@ func (a *AzureInfoer) GetVirtualMachines(region string) ([]types.VMInfo, error)
 
 					virtualMachines = append(virtualMachines, types.VMInfo{
 						Category:   category,
+						Series:     a.mapSeries(*sku.Family),
 						Type:       *sku.Name,
 						Mem:        memory,
 						Cpus:       cpu,

internal/cloudinfo/providers/azure/category_mapper.go → internal/cloudinfo/providers/azure/instancefamily_mapper.go

@@ -28,18 +28,25 @@ var (
 	categoryMap = map[string][]string{
 		types.CategoryGeneral: {"Dv2", "Av2", "Dv3", "DSv2", "DSv3", "BS", "DS", "D", "A0_A7", "A", "A8_A11", "DCS"},
 		types.CategoryCompute: {"H", "FSv2", "FS", "", "HCS", "HBS"},
-		types.CategoryMemory:  {"Ev3", "ESv3", "MS", "G", "GS", "EIv3", "EISv3", "PBS", "MSv2"},
+		types.CategoryMemory:  {"Ev3", "ESv3", "MS", "G", "GS", "EIv3", "EISv3", "PBS", "MSv2", "MDSv2", "EDSv4", "ESv4"},
 		types.CategoryStorage: {"LS", "LSv2"},
-		types.CategoryGpu:     {"NC", "NV", "NCSv3", "NCSv2", "NDS", "NVSv2", "NVSv3"},
+		types.CategoryGpu:     {"NC", "NV", "NCSv3", "NCSv2", "NDS", "NVSv2", "NVSv3", "ND"},
+	}
+
+	customMap = map[string]string{
+		"MDSMediumMemoryv2":  "MDSv2",
+		"MIDSMediumMemoryv2": "MDSv2",
+		"MISMediumMemoryv2":  "MSv2",
+		"MSMediumMemoryv2":   "MSv2",
+		"NDASv4_A100":        "ND",
+		"XEIDSv4":            "EDSv4",
+		"XEISv4":             "ESv4",
 	}
 )
 
 // mapCategory maps the family of the azure instance to category
 func (a *AzureInfoer) mapCategory(name string) (string, error) {
-	family := strings.TrimRight(name, "Family")
-	family = strings.TrimLeft(family, "standard") // nolint: staticcheck
-	family = strings.TrimRight(family, "Promo")   // nolint: staticcheck
-	family = strings.TrimLeft(family, "basic")
+	family := GetFamily(name)
 
 	for category, strVals := range categoryMap {
 		if cloudinfo.Contains(strVals, family) {
@@ -48,3 +55,24 @@ func (a *AzureInfoer) mapCategory(name string) (string, error) {
 	}
 	return "", emperror.Wrap(errors.New(family), "could not determine the category")
 }
+
+// mapSeries get instance series associated with the instanceType
+func (a *AzureInfoer) mapSeries(name string) string {
+	return GetFamily(name)
+}
+
+func GetFamily(name string) string {
+	family := strings.TrimRight(name, "Family")
+	family = strings.TrimLeft(family, "standard") // nolint: staticcheck
+	family = strings.TrimLeft(family, "Standard") // nolint: staticcheck
+	family = strings.TrimRight(family, "Promo")   // nolint: staticcheck
+	family = strings.TrimLeft(family, "basic")
+
+	for _, key := range customMap {
+		if key == family {
+			return customMap[family]
+		}
+	}
+
+	return family
+}

internal/cloudinfo/providers/azure/instancefamily_mapper_test.go

@@ -0,0 +1,38 @@
+// Copyright © 2021 Banzai Cloud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package azure
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"logur.dev/logur"
+
+	"github.com/banzaicloud/cloudinfo/internal/cloudinfo/cloudinfoadapter"
+)
+
+func TestAzureInfoer_mapSeries(t *testing.T) {
+	familySeriesMap := map[string]string{
+		"standardDADSv5Family": "DADSv5",
+	}
+
+	azureInfoer := AzureInfoer{log: cloudinfoadapter.NewLogger(&logur.TestLogger{})}
+
+	for family, series := range familySeriesMap {
+		t.Run("test parsing "+family, func(t *testing.T) {
+			assert.Equal(t, azureInfoer.mapSeries(family), series, "unexpected series")
+		})
+	}
+}