Add support for latency metrics

ecs-agent/acs/session/session.go

@@ -4,7 +4,7 @@
 // not use this file except in compliance with the License. A copy of the
 // License is located at
 //
-//	http://aws.amazon.com/apache2.0/
+//  http://aws.amazon.com/apache2.0/
 //
 // or in the "license" file accompanying this file. This file is distributed
 // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
@@ -251,7 +251,6 @@ func (s *session) startSessionOnce(ctx context.Context) error {
 		})
 		return err
 	}
-	s.metricsFactory.New(metrics.DiscoverPollEndpointDurationName).WithGauge(s.ecsClient.GetDiscoverPollEndpointDuration()).Done(nil)
 
 	client := s.clientFactory.New(
 		s.acsURL(acsEndpoint),
@@ -263,17 +262,18 @@ func (s *session) startSessionOnce(ctx context.Context) error {
 
 	// Invoke Connect method as soon as we create client. This will ensure all the
 	// request handlers to be associated with this client have a valid connection.
-	acsConnectionMetric := s.metricsFactory.New(metrics.ACSConnectionMetricDurationName)
+	acsConnectionStartTime := time.Now()
 	disconnectTimer, err := client.Connect(metrics.ACSDisconnectTimeoutMetricName, s.disconnectTimeout,
 		s.disconnectJitter)
+	s.metricsFactory.New(metrics.ACSSessionCallName).Done(err)
 	if err != nil {
 		logger.Error("Failed to connect to ACS", logger.Fields{
 			"containerInstanceARN": s.containerInstanceARN,
 			field.Error:            err,
 		})
 		return err
 	}
-	acsConnectionMetric.Done(err)
+	s.metricsFactory.New(metrics.ACSSessionCallDurationName).WithGauge(time.Since(acsConnectionStartTime)).Done(nil)
 	defer disconnectTimer.Stop()
 
 	if s.GetFirstACSConnectionTime().IsZero() {

ecs-agent/api/ecs/client/ecs_client.go

@@ -39,6 +39,7 @@ import (
 	"github.com/aws/amazon-ecs-agent/ecs-agent/httpclient"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/logger"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/logger/field"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/metrics"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/utils"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/utils/retry"
 )
@@ -77,7 +78,7 @@ type ecsClient struct {
 	shouldExcludeIPv6PortBinding     bool
 	sascCustomRetryBackoff           func(func() error) error
 	stscAttachmentCustomRetryBackoff func(func() error) error
-	discoverPollEndpointDuration     time.Duration
+	metricsFactory                   metrics.EntryFactory
 }
 
 // NewECSClient creates a new ECSClient interface object.
@@ -113,6 +114,9 @@ func NewECSClient(
 	if client.submitStateChangeClient == nil {
 		client.submitStateChangeClient = newSubmitStateChangeClient(&ecsConfig)
 	}
+	if client.metricsFactory == nil {
+		client.metricsFactory = metrics.NewNopEntryFactory()
+	}
 
 	return client, nil
 }
@@ -759,6 +763,7 @@ func (client *ecsClient) discoverPollEndpoint(containerInstanceArn string,
 		Cluster:           aws.String(client.configAccessor.Cluster()),
 		ZoneId:            aws.String(availabilityZone),
 	})
+	client.metricsFactory.New(metrics.DiscoverPollEndpointCallName).Done(err)
 	if err != nil {
 		// If we got an error calling the API, fallback to an expired cached endpoint if
 		// we have it.
@@ -777,8 +782,7 @@ func (client *ecsClient) discoverPollEndpoint(containerInstanceArn string,
 		}
 		return nil, err
 	}
-
-	client.discoverPollEndpointDuration = time.Since(discoverPollEndpointStartTime)
+	client.metricsFactory.New(metrics.DiscoverPollEndpointDurationName).WithGauge(time.Since(discoverPollEndpointStartTime)).Done(nil)
 	// Cache the response from ECS.
 	client.pollEndpointCache.Set(containerInstanceArn, output)
 	return output, nil
@@ -872,7 +876,3 @@ func trimString(inputString string, maxLen int) string {
 		return inputString
 	}
 }
-
-func (client *ecsClient) GetDiscoverPollEndpointDuration() time.Duration {
-	return client.discoverPollEndpointDuration
-}

ecs-agent/api/ecs/client/ecs_client_option.go

@@ -16,6 +16,7 @@ package ecsclient
 import (
 	"github.com/aws/amazon-ecs-agent/ecs-agent/api/ecs"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/async"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/metrics"
 )
 
 // ECSClientOption allows for configuration of an ecsClient.
@@ -87,3 +88,12 @@ func WithSubmitStateChangeClient(s ecs.ECSSubmitStateSDK) ECSClientOption {
 		client.submitStateChangeClient = s
 	}
 }
+
+// WithMetricsFactory is an ECSClientOption that configures
+// ecsClient.metricsFactory with the value passed as a parameter.
+// This is especially useful for emitting metrics in the ECS Client
+func WithMetricsFactory(metricsFactory metrics.EntryFactory) ECSClientOption {
+	return func(client *ecsClient) {
+		client.metricsFactory = metricsFactory
+	}
+}

ecs-agent/api/ecs/client/ecs_client_option_test.go

@@ -22,6 +22,7 @@ import (
 
 	mock_ecs "github.com/aws/amazon-ecs-agent/ecs-agent/api/ecs/mocks"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/async"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/metrics"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -102,3 +103,10 @@ func TestWithSubmitStateChangeClient(t *testing.T) {
 	option(client)
 	assert.Equal(t, newSubmitStateChangeClient, client.submitStateChangeClient)
 }
+
+func TestWithMetricsFactory(t *testing.T) {
+	client := &ecsClient{} // client.metricsFactory is nil by default
+	option := WithMetricsFactory(metrics.NewNopEntryFactory())
+	option(client)
+	assert.NotNil(t, client.metricsFactory)
+}

ecs-agent/api/ecs/interface.go

@@ -14,8 +14,6 @@
 package ecs
 
 import (
-	"time"
-
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/request"
 
@@ -64,8 +62,6 @@ type ECSClient interface {
 	UpdateContainerInstancesState(instanceARN, status string) error
 	// GetHostResources retrieves a map that map the resource name to the corresponding resource
 	GetHostResources() (map[string]*ecs.Resource, error)
-	// GetDiscoverPollEndpointDuration retrieves the time it takes for the DiscoverPollEndpoint call
-	GetDiscoverPollEndpointDuration() time.Duration
 }
 
 // ECSSDK is an interface that specifies the subset of the AWS Go SDK's ECS

ecs-agent/metrics/constants.go

@@ -47,9 +47,14 @@ const (
 	TCSDisconnectTimeoutMetricName = agentAvailabilityNamespace + ".TCSDisconnectTimeout"
 
 	// ACS Session Metrics
-	acsStartSessionNamespace         = "ACSStartSession"
-	DiscoverPollEndpointDurationName = acsStartSessionNamespace + ".DiscoverPollEndpointDuration"
-	ACSConnectionMetricDurationName  = acsStartSessionNamespace + ".ACSConnectEndpointDuration"
+	acsSessionNamespace        = "ACSSession"
+	ACSSessionCallName         = acsSessionNamespace + ".ACSConnect"
+	ACSSessionCallDurationName = acsSessionNamespace + ".ACSConnectDuration"
+
+	// ECS Client Metrics
+	ecsClientNamespace               = "ECSClient"
+	DiscoverPollEndpointCallName     = ecsClientNamespace + ".DiscoverPollEndpoint"
+	DiscoverPollEndpointDurationName = ecsClientNamespace + ".DiscoverPollEndpointDuration"
 
 	dbClientMetricNamespace                 = "Data"
 	GetNetworkConfigurationByTaskMetricName = dbClientMetricNamespace + ".GetNetworkConfigurationByTask"