Revert usageDetails api version to 2019-01-01

CHANGELOG.next.asciidoc

@@ -180,6 +180,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Relax time parsing and capture group and session type in Cisco ASA module {issue}24710[24710] {pull}28325[28325]
 - Correctly track bytes read when max_bytes is exceeded. {issue}28317[28317] {pull}28352[28352]
 - Upgrade azure-eventhub sdk reference, contains potential checkpoint fixes. {pull}28919[28919]
+- Revert usageDetails api version to 2019-01-01. {pull}28995[28995]
 - Fix in `aws-s3` input regarding provider discovery through endpoint {pull}28963[28963]
 
 *Heartbeat*

NOTICE.txt

@@ -863,15 +863,15 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-event-hubs
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/azure-sdk-for-go
-Version: v57.0.0+incompatible
+Version: v59.0.0+incompatible
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go@v57.0.0+incompatible/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go@v59.0.0+incompatible/LICENSE.txt:
 
 The MIT License (MIT)
 
-Copyright (c) 2021 Microsoft
+Copyright (c) Microsoft Corporation.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -891,6 +891,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
+
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/azure-storage-blob-go
 Version: v0.8.0

go.mod

@@ -10,7 +10,7 @@ require (
 	code.cloudfoundry.org/go-loggregator v7.4.0+incompatible
 	code.cloudfoundry.org/rfc5424 v0.0.0-20180905210152-236a6d29298a // indirect
 	github.com/Azure/azure-event-hubs-go/v3 v3.3.15
-	github.com/Azure/azure-sdk-for-go v57.0.0+incompatible
+	github.com/Azure/azure-sdk-for-go v59.0.0+incompatible
 	github.com/Azure/azure-storage-blob-go v0.8.0
 	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
 	github.com/Azure/go-autorest/autorest v0.11.19

go.sum

@@ -67,8 +67,8 @@ github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9mo
 github.com/Azure/azure-sdk-for-go v41.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go v51.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go v55.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/azure-sdk-for-go v57.0.0+incompatible h1:isVki3PbIFrwKvKdVP1byxo73/pt+Nn174YxW1k4PNw=
-github.com/Azure/azure-sdk-for-go v57.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-sdk-for-go v59.0.0+incompatible h1:I1ULJqny1qQhUBFy11yDXHhW3pLvbhwV0PTn7mjp9V0=
+github.com/Azure/azure-sdk-for-go v59.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
 github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o=
 github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=

x-pack/metricbeat/module/azure/billing/billing.go

@@ -5,8 +5,6 @@
 package billing
 
 import (
-	"time"
-
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/x-pack/metricbeat/module/azure"
@@ -60,13 +58,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 // It publishes the event which is then forwarded to the output. In case
 // of an error set the Error field of mb.Event or simply call report.Error().
 func (m *MetricSet) Fetch(report mb.ReporterV2) error {
-	startTime := time.Now().UTC().Truncate(24 * time.Hour).Add((-48) * time.Hour)
-	endTime := startTime.Add(time.Hour * 24).Add(time.Second * (-1))
-	results, err := m.client.GetMetrics(startTime, endTime)
+	results, err := m.client.GetMetrics()
 	if err != nil {
 		return errors.Wrap(err, "error retrieving usage information")
 	}
-	events := EventsMapping(results, startTime, endTime, m.client.Config.SubscriptionId)
+	events := EventsMapping(m.client.Config.SubscriptionId, results)
 	for _, event := range events {
 		isOpen := report.Event(event)
 		if !isOpen {

x-pack/metricbeat/module/azure/billing/client.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/pkg/errors"
 
+	prevConsumption "github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption"
 	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-10-01/consumption"
 
 	"github.com/elastic/beats/v7/libbeat/logp"
@@ -25,7 +26,7 @@ type Client struct {
 }
 
 type Usage struct {
-	UsageDetails  []consumption.BasicUsageDetail
+	UsageDetails  []prevConsumption.UsageDetail
 	ActualCosts   []consumption.Forecast
 	ForecastCosts []consumption.Forecast
 }
@@ -45,17 +46,20 @@ func NewClient(config azure.Config) (*Client, error) {
 }
 
 // GetMetrics returns the usage detail and forecast values.
-func (client *Client) GetMetrics(startTime time.Time, endTime time.Time) (Usage, error) {
+func (client *Client) GetMetrics() (Usage, error) {
+
 	var usage Usage
 	scope := fmt.Sprintf("subscriptions/%s", client.Config.SubscriptionId)
 	if client.Config.BillingScopeDepartment != "" {
 		scope = fmt.Sprintf("/providers/Microsoft.Billing/departments/%s", client.Config.BillingScopeDepartment)
 	} else if client.Config.BillingScopeAccountId != "" {
 		scope = fmt.Sprintf("/providers/Microsoft.Billing/billingAccounts/%s", client.Config.BillingScopeAccountId)
 	}
-
-	filter := fmt.Sprintf("properties/usageStart eq '%s' and properties/usageEnd eq '%s'", startTime.Format(time.RFC3339Nano), endTime.Format(time.RFC3339Nano))
-	usageDetails, err := client.BillingService.GetUsageDetails(scope, "properties/meterDetails", filter, "", nil, consumption.MetrictypeActualCostMetricType)
+	startTime := time.Now().UTC().Truncate(24 * time.Hour).Add((-24) * time.Hour)
+	endTime := startTime.Add(time.Hour * 24).Add(time.Second * (-1))
+	usageDetails, err := client.BillingService.GetUsageDetails(scope, "properties/meterDetails",
+		fmt.Sprintf("properties/usageStart eq '%s' and properties/usageEnd eq '%s'", startTime.Format(time.RFC3339Nano), endTime.Format(time.RFC3339Nano)),
+		"", nil, "properties/instanceLocation")
 	if err != nil {
 		return usage, errors.Wrap(err, "Retrieving usage details failed in client")
 	}

x-pack/metricbeat/module/azure/billing/client_test.go

@@ -7,8 +7,8 @@ package billing
 import (
 	"errors"
 	"testing"
-	"time"
 
+	prevConsumption "github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption"
 	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-10-01/consumption"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -26,11 +26,9 @@ func TestClient(t *testing.T) {
 		client.Config = config
 		m := &MockService{}
 		m.On("GetForcast", mock.Anything).Return(consumption.ForecastsListResult{}, errors.New("invalid query"))
-		m.On("GetUsageDetails", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(consumption.UsageDetailsListResultPage{}, nil)
+		m.On("GetUsageDetails", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(prevConsumption.UsageDetailsListResultPage{}, nil)
 		client.BillingService = m
-		startTime := time.Now().UTC().Truncate(24 * time.Hour).Add((-48) * time.Hour)
-		endTime := startTime.Add(time.Hour * 24).Add(time.Second * (-1))
-		results, err := client.GetMetrics(startTime, endTime)
+		results, err := client.GetMetrics()
 		assert.Error(t, err)
 		assert.Equal(t, len(results.ActualCosts), 0)
 		m.AssertExpectations(t)
@@ -41,11 +39,9 @@ func TestClient(t *testing.T) {
 		m := &MockService{}
 		forecasts := []consumption.Forecast{{}, {}}
 		m.On("GetForcast", mock.Anything).Return(consumption.ForecastsListResult{Value: &forecasts}, nil)
-		m.On("GetUsageDetails", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(consumption.UsageDetailsListResultPage{}, nil)
+		m.On("GetUsageDetails", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(prevConsumption.UsageDetailsListResultPage{}, nil)
 		client.BillingService = m
-		startTime := time.Now().UTC().Truncate(24 * time.Hour).Add((-48) * time.Hour)
-		endTime := startTime.Add(time.Hour * 24).Add(time.Second * (-1))
-		results, err := client.GetMetrics(startTime, endTime)
+		results, err := client.GetMetrics()
 		assert.NoError(t, err)
 		assert.Equal(t, len(results.ActualCosts), 2)
 		assert.Equal(t, len(results.ForecastCosts), 2)

x-pack/metricbeat/module/azure/billing/data.go

@@ -5,6 +5,8 @@
 package billing
 
 import (
+	"fmt"
+	"strings"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-10-01/consumption"
@@ -15,71 +17,36 @@ import (
 	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
-func EventsMapping(results Usage, startTime time.Time, endTime time.Time, subscriptionId string) []mb.Event {
+func EventsMapping(subscriptionId string, results Usage) []mb.Event {
 	var events []mb.Event
-	// usage details come in different forms, most common for this api call is LegacyUsageDetail
 	if len(results.UsageDetails) > 0 {
-		for _, ud := range results.UsageDetails {
-			event := mb.Event{Timestamp: time.Now().UTC()}
-			if legacyUsageDetail, err := ud.AsLegacyUsageDetail(); err == true {
-				event.ModuleFields = common.MapStr{
+		for _, usageDetail := range results.UsageDetails {
+			event := mb.Event{
+				ModuleFields: common.MapStr{
 					"resource": common.MapStr{
-						"type":  legacyUsageDetail.ConsumedService,
-						"group": legacyUsageDetail.ResourceGroup,
-						"name":  legacyUsageDetail.ResourceName,
+						"type":  usageDetail.ConsumedService,
+						"group": getResourceGroupFromId(*usageDetail.InstanceID),
+						"name":  usageDetail.InstanceName,
 					},
-					"subscription_id": legacyUsageDetail.SubscriptionID,
-				}
-				event.MetricSetFields = common.MapStr{
-					"pretax_cost":          legacyUsageDetail.Cost,
-					"department_name":      legacyUsageDetail.InvoiceSection,
-					"product":              legacyUsageDetail.Product,
-					"usage_start":          startTime,
-					"usage_end":            endTime,
-					"billing_period_start": legacyUsageDetail.BillingPeriodStartDate.ToTime(),
-					"billing_period_end":   legacyUsageDetail.BillingPeriodEndDate.ToTime(),
-					"currency":             legacyUsageDetail.BillingCurrency,
-					"effective_price":      legacyUsageDetail.EffectivePrice,
-					"account_name":         legacyUsageDetail.BillingAccountName,
-					"account_id":           legacyUsageDetail.BillingAccountID,
-					"subscription_name":    legacyUsageDetail.SubscriptionName,
-					"unit_price":           legacyUsageDetail.UnitPrice,
-					"quantity":             legacyUsageDetail.Quantity,
-				}
-				event.RootFields = common.MapStr{}
-				event.RootFields.Put("cloud.provider", "azure")
-				event.RootFields.Put("cloud.region", legacyUsageDetail.ResourceLocation)
-				event.RootFields.Put("cloud.instance.name", legacyUsageDetail.ResourceName)
-				event.RootFields.Put("cloud.instance.id", legacyUsageDetail.ResourceID)
-			}
-			if modernUsageDetail, err := ud.AsModernUsageDetail(); err == true {
-				event.ModuleFields = common.MapStr{
-					"resource": common.MapStr{
-						"type":  modernUsageDetail.ConsumedService,
-						"group": modernUsageDetail.ResourceGroup,
-						"name":  modernUsageDetail.InstanceName,
-					},
-					"subscription_id": modernUsageDetail.SubscriptionGUID,
-				}
-				event.MetricSetFields = common.MapStr{
-					"product":              modernUsageDetail.Product,
-					"usage_start":          startTime,
-					"usage_end":            endTime,
-					"billing_period_start": modernUsageDetail.BillingPeriodStartDate.ToTime(),
-					"billing_period_end":   modernUsageDetail.BillingPeriodEndDate.ToTime(),
-					"currency":             modernUsageDetail.BillingCurrencyCode,
-					"account_id":           modernUsageDetail.BillingAccountID,
-					"billing_account_name": modernUsageDetail.BillingAccountName,
-					"subscription_name":    modernUsageDetail.SubscriptionName,
-					"unit_price":           modernUsageDetail.UnitPrice,
-				}
-				event.RootFields = common.MapStr{}
-				event.RootFields.Put("cloud.provider", "azure")
-				event.RootFields.Put("cloud.region", modernUsageDetail.ResourceLocation)
-			}
-			if _, err := ud.AsUsageDetail(); err == true {
-				continue
+					"subscription_id": usageDetail.SubscriptionGUID,
+				},
+				MetricSetFields: common.MapStr{
+					"pretax_cost":       usageDetail.PretaxCost,
+					"department_name":   usageDetail.DepartmentName,
+					"product":           usageDetail.Product,
+					"usage_start":       usageDetail.UsageStart.ToTime(),
+					"usage_end":         usageDetail.UsageEnd.ToTime(),
+					"currency":          usageDetail.Currency,
+					"billing_period_id": usageDetail.BillingPeriodID,
+					"account_name":      usageDetail.AccountName,
+				},
+				Timestamp: time.Now().UTC(),
 			}
+			event.RootFields = common.MapStr{}
+			event.RootFields.Put("cloud.provider", "azure")
+			event.RootFields.Put("cloud.region", usageDetail.InstanceLocation)
+			event.RootFields.Put("cloud.instance.name", usageDetail.InstanceName)
+			event.RootFields.Put("cloud.instance.id", usageDetail.InstanceID)
 			events = append(events, event)
 		}
 	}
@@ -125,3 +92,14 @@ func EventsMapping(results Usage, startTime time.Time, endTime time.Time, subscr
 	}
 	return events
 }
+
+// getResourceGroupFromId maps resource group from resource ID
+func getResourceGroupFromId(path string) string {
+	params := strings.Split(path, "/")
+	for i, param := range params {
+		if param == "resourceGroups" {
+			return fmt.Sprintf("%s", params[i+1])
+		}
+	}
+	return ""
+}

x-pack/metricbeat/module/azure/billing/data_test.go

@@ -8,17 +8,16 @@ import (
 	"testing"
 	"time"
 
+	prevConsumption "github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption"
+	consumption "github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-10-01/consumption"
 	"github.com/Azure/go-autorest/autorest/date"
-
-	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-10-01/consumption"
 	"github.com/shopspring/decimal"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestEventMapping(t *testing.T) {
 	usageDate := "2020-08-08"
 	name := "test"
-	billingAccountId := "123"
 	startDate := date.Time{}
 
 	var charge decimal.Decimal = decimal.NewFromFloat(8.123456)
@@ -38,37 +37,44 @@ func TestEventMapping(t *testing.T) {
 		ChargeType:       "Actual",
 		ConfidenceLevels: nil,
 	}
-	var pros = consumption.LegacyUsageDetailProperties{
-		BillingAccountID:       &billingAccountId,
-		BillingAccountName:     &name,
-		BillingPeriodStartDate: &startDate,
-		BillingPeriodEndDate:   &startDate,
-		Cost:                   &charge,
-		InvoiceSection:         &name,
-		Product:                &name,
-	}
-	var legacy = consumption.LegacyUsageDetail{
-		LegacyUsageDetailProperties: &pros,
+	var prop1 = prevConsumption.UsageDetailProperties{
+		InstanceName:     &name,
+		SubscriptionName: &name,
+		AccountName:      &name,
+		DepartmentName:   &name,
+		Product:          &name,
+		InstanceID:       &name,
+		UsageStart:       &startDate,
+		UsageEnd:         &startDate,
 	}
-	var usage = Usage{UsageDetails: []consumption.BasicUsageDetail{legacy},
+	usage := Usage{
+		UsageDetails: []prevConsumption.UsageDetail{
+			{
+				UsageDetailProperties: &prop1,
+				ID:                    nil,
+				Name:                  nil,
+				Type:                  nil,
+				Tags:                  nil,
+			},
+		},
 		ActualCosts: []consumption.Forecast{
 			{
 				ForecastProperties: &prop2,
 				ID:                 nil,
 				Name:               nil,
 				Type:               nil,
 				Tags:               nil,
-			}}, ForecastCosts: []consumption.Forecast{
+			}},
+		ForecastCosts: []consumption.Forecast{
 			{
 				ForecastProperties: &prop,
 				ID:                 nil,
 				Name:               nil,
 				Type:               nil,
 				Tags:               nil,
-			}}}
-	startTime := time.Now().UTC().Truncate(24 * time.Hour).Add((-48) * time.Hour)
-	endTime := startTime.Add(time.Hour * 24).Add(time.Second * (-1))
-	events := EventsMapping(usage, startTime, endTime, "sub")
+			}},
+	}
+	events := EventsMapping("sub", usage)
 	assert.Equal(t, len(events), 2)
 	for _, event := range events {
 

x-pack/metricbeat/module/azure/billing/mock_service.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/elastic/beats/v7/libbeat/logp"
 
+	prevConsumption "github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption"
 	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-10-01/consumption"
 )
 
@@ -35,19 +36,7 @@ func (service *MockService) GetForcast(filter string) (consumption.ForecastsList
 }
 
 // GetUsageDetails is a mock function for the billing service
-func (service *MockService) GetUsageDetails(scope string, expand string, filter string, skiptoken string, top *int32, apply consumption.Metrictype) (consumption.UsageDetailsListResultPage, error) {
+func (service *MockService) GetUsageDetails(scope string, expand string, filter string, skiptoken string, top *int32, apply string) (prevConsumption.UsageDetailsListResultPage, error) {
 	args := service.Called(scope, expand, filter, skiptoken, top, apply)
-	return args.Get(0).(consumption.UsageDetailsListResultPage), args.Error(1)
-}
-
-// GetMarketplaceUsage
-func (service *MockService) GetMarketplaceUsage(scope string, filter string, skiptoken string, top *int32) (consumption.MarketplacesListResultPage, error) {
-	args := service.Called(scope, filter, skiptoken, top)
-	return args.Get(0).(consumption.MarketplacesListResultPage), args.Error(1)
-}
-
-// GetCharges
-func (service *MockService) GetCharges(scope string, startDate string, endDate string, filter string, apply string) (consumption.ChargesListResult, error) {
-	args := service.Called(scope, startDate, endDate, filter, apply)
-	return args.Get(0).(consumption.ChargesListResult), args.Error(1)
+	return args.Get(0).(prevConsumption.UsageDetailsListResultPage), args.Error(1)
 }