Add RateLimit config to serviceDefaults (#2844)

.changelog/2844.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+helm: (Consul Enterprise) Adds rate limiting config to serviceDefaults CRD
+```

acceptance/go.mod

@@ -5,8 +5,8 @@ go 1.20
 require (
 	github.com/gruntwork-io/terratest v0.31.2
 	github.com/hashicorp/consul-k8s/control-plane v0.0.0-20230609143603-198c4433d892
-	github.com/hashicorp/consul/api v1.22.0-rc1
-	github.com/hashicorp/consul/sdk v0.14.0-rc1
+	github.com/hashicorp/consul/api v1.10.1-0.20230825164720-ecdcde430924
+	github.com/hashicorp/consul/sdk v0.14.1
 	github.com/hashicorp/go-uuid v1.0.3
 	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/hcp-sdk-go v0.50.0

acceptance/go.sum

@@ -399,10 +399,10 @@ github.com/gruntwork-io/terratest v0.31.2 h1:xvYHA80MUq5kx670dM18HInewOrrQrAN+Xb
 github.com/gruntwork-io/terratest v0.31.2/go.mod h1:EEgJie28gX/4AD71IFqgMj6e99KP5mi81hEtzmDjxTo=
 github.com/hashicorp/consul-k8s/control-plane v0.0.0-20230609143603-198c4433d892 h1:4iI0ztWbVPTSDax+m1/XDs4jIRorxY4kSMyuM0fX+Dc=
 github.com/hashicorp/consul-k8s/control-plane v0.0.0-20230609143603-198c4433d892/go.mod h1:iZ8BJGSnY52wnxJTo2VIfGX63CPjqiNzbuqdOtJCKnI=
-github.com/hashicorp/consul/api v1.22.0-rc1 h1:ePmGqndeMgaI38KUbSA/CqTzeEAIogXyWnfNJzglo70=
-github.com/hashicorp/consul/api v1.22.0-rc1/go.mod h1:wtduXtbAqSGtBdi3tyA5SSAYGAG51rBejV9SEUBciMY=
-github.com/hashicorp/consul/sdk v0.14.0-rc1 h1:PuETOfN0uxl28i0Pq6rK7TBCrIl7psMbL0YTSje4KvM=
-github.com/hashicorp/consul/sdk v0.14.0-rc1/go.mod h1:gHYeuDa0+0qRAD6Wwr6yznMBvBwHKoxSBoW5l73+saE=
+github.com/hashicorp/consul/api v1.10.1-0.20230825164720-ecdcde430924 h1:NPhzdwDho2r8pQv31oeGLlco7fnJ1i0WLYjtSXqWEck=
+github.com/hashicorp/consul/api v1.10.1-0.20230825164720-ecdcde430924/go.mod h1:NZJGRFYruc/80wYowkPFCp1LbGmJC9L8izrwfyVx/Wg=
+github.com/hashicorp/consul/sdk v0.14.1 h1:ZiwE2bKb+zro68sWzZ1SgHF3kRMBZ94TwOCFRF4ylPs=
+github.com/hashicorp/consul/sdk v0.14.1/go.mod h1:vFt03juSzocLRFo59NkeQHHmQa6+g7oU0pfzdI1mUhg=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=

acceptance/tests/config-entries/config_entries_test.go

@@ -9,17 +9,18 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/consul/sdk/testutil/retry"
+	"github.com/hashicorp/go-uuid"
+	"github.com/stretchr/testify/require"
+
 	"github.com/hashicorp/consul-k8s/acceptance/framework/config"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/environment"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/vault"
-	"github.com/hashicorp/consul/api"
-	"github.com/hashicorp/consul/sdk/testutil/retry"
-	"github.com/hashicorp/go-uuid"
-	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -104,6 +105,7 @@ func TestController(t *testing.T) {
 					svcDefaultEntry, ok := entry.(*api.ServiceConfigEntry)
 					require.True(r, ok, "could not cast to ServiceConfigEntry")
 					require.Equal(r, "http", svcDefaultEntry.Protocol)
+					require.Equal(r, 1234, svcDefaultEntry.RateLimits.InstanceLevel.RequestsPerSecond)
 
 					// service-resolver
 					entry, _, err = consulClient.ConfigEntries().Get(api.ServiceResolver, "resolver", nil)
@@ -232,8 +234,6 @@ func TestController(t *testing.T) {
 					require.Equal(r, 100.0, rateLimitIPConfigEntry.KV.WriteRate)
 					require.Equal(r, 100.0, rateLimitIPConfigEntry.Tenancy.ReadRate)
 					require.Equal(r, 100.0, rateLimitIPConfigEntry.Tenancy.WriteRate)
-					//require.Equal(r, 100.0, rateLimitIPConfigEntry.PreparedQuery.ReadRate)
-					//require.Equal(r, 100.0, rateLimitIPConfigEntry.PreparedQuery.WriteRate)
 					require.Equal(r, 100.0, rateLimitIPConfigEntry.Session.ReadRate)
 					require.Equal(r, 100.0, rateLimitIPConfigEntry.Session.WriteRate)
 					require.Equal(r, 100.0, rateLimitIPConfigEntry.Txn.ReadRate)

acceptance/tests/fixtures/bases/crds-oss/servicedefaults.yaml

@@ -31,6 +31,18 @@ spec:
         interval: 10s
         maxFailures: 2
   balanceInboundConnections: "exact_balance"
+  rateLimits:
+    instanceLevel:
+      requestsPerSecond: 1234
+      requestsMaxBurst: 2345
+      routes:
+        - pathExact: "/exact"
+          requestsPerSecond: 222
+          requestsMaxBurst: 333
+        - pathPrefix: "/prefix"
+          requestsPerSecond: 444
+        - pathRegex: "/regex"
+          requestsPerSecond: 555
   envoyExtensions:
     - name: builtin/aws/lambda
       required: false

charts/consul/templates/crd-servicedefaults.yaml

@@ -192,6 +192,69 @@ spec:
                   unlock usage of the service-splitter and service-router config entries
                   for a service.
                 type: string
+              rateLimits:
+                description: RateLimits is rate limiting configuration that is applied
+                  to inbound traffic for a service. Rate limiting is a Consul enterprise
+                  feature.
+                properties:
+                  instanceLevel:
+                    description: InstanceLevel represents rate limit configuration
+                      that is applied per service instance.
+                    properties:
+                      requestsMaxBurst:
+                        description: "RequestsMaxBurst is the maximum number of requests
+                          that can be sent in a burst. Should be equal to or greater
+                          than RequestsPerSecond. If unset, defaults to RequestsPerSecond.
+                          \n Internally, this is the maximum size of the token bucket
+                          used for rate limiting."
+                        type: integer
+                      requestsPerSecond:
+                        description: "RequestsPerSecond is the average number of requests
+                          per second that can be made without being throttled. This
+                          field is required if RequestsMaxBurst is set. The allowed
+                          number of requests may exceed RequestsPerSecond up to the
+                          value specified in RequestsMaxBurst. \n Internally, this
+                          is the refill rate of the token bucket used for rate limiting."
+                        type: integer
+                      routes:
+                        description: Routes is a list of rate limits applied to specific
+                          routes. For a given request, the first matching route will
+                          be applied, if any. Overrides any top-level configuration.
+                        items:
+                          properties:
+                            pathExact:
+                              description: Exact path to match. Exactly one of PathExact,
+                                PathPrefix, or PathRegex must be specified.
+                              type: string
+                            pathPrefix:
+                              description: Prefix to match. Exactly one of PathExact,
+                                PathPrefix, or PathRegex must be specified.
+                              type: string
+                            pathRegex:
+                              description: Regex to match. Exactly one of PathExact,
+                                PathPrefix, or PathRegex must be specified.
+                              type: string
+                            requestsMaxBurst:
+                              description: RequestsMaxBurst is the maximum number
+                                of requests that can be sent in a burst. Should be
+                                equal to or greater than RequestsPerSecond. If unset,
+                                defaults to RequestsPerSecond. Internally, this is
+                                the maximum size of the token bucket used for rate
+                                limiting.
+                              type: integer
+                            requestsPerSecond:
+                              description: RequestsPerSecond is the average number
+                                of requests per second that can be made without being
+                                throttled. This field is required if RequestsMaxBurst
+                                is set. The allowed number of requests may exceed
+                                RequestsPerSecond up to the value specified in RequestsMaxBurst.
+                                Internally, this is the refill rate of the token bucket
+                                used for rate limiting.
+                              type: integer
+                          type: object
+                        type: array
+                    type: object
+                type: object
               transparentProxy:
                 description: 'TransparentProxy controls configuration specific to
                   proxies in transparent mode. Note: This cannot be set using the

control-plane/api/v1alpha1/servicedefaults_types.go

@@ -18,8 +18,9 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 
-	"github.com/hashicorp/consul-k8s/control-plane/api/common"
 	capi "github.com/hashicorp/consul/api"
+
+	"github.com/hashicorp/consul-k8s/control-plane/api/common"
 )
 
 const (
@@ -115,6 +116,9 @@ type ServiceDefaultsSpec struct {
 	// proxy threads. The only supported value is exact_balance. By default, no connection balancing is used.
 	// Refer to the Envoy Connection Balance config for details.
 	BalanceInboundConnections string `json:"balanceInboundConnections,omitempty"`
+	// RateLimits is rate limiting configuration that is applied to
+	// inbound traffic for a service. Rate limiting is a Consul enterprise feature.
+	RateLimits *RateLimits `json:"rateLimits,omitempty"`
 	// EnvoyExtensions are a list of extensions to modify Envoy proxy configuration.
 	EnvoyExtensions EnvoyExtensions `json:"envoyExtensions,omitempty"`
 }
@@ -216,6 +220,150 @@ type ServiceDefaultsDestination struct {
 	Port uint32 `json:"port,omitempty"`
 }
 
+// RateLimits is rate limiting configuration that is applied to
+// inbound traffic for a service.
+// Rate limiting is a Consul Enterprise feature.
+type RateLimits struct {
+	// InstanceLevel represents rate limit configuration
+	// that is applied per service instance.
+	InstanceLevel InstanceLevelRateLimits `json:"instanceLevel,omitempty"`
+}
+
+func (rl *RateLimits) toConsul() *capi.RateLimits {
+	if rl == nil {
+		return nil
+	}
+	routes := make([]capi.InstanceLevelRouteRateLimits, len(rl.InstanceLevel.Routes))
+	for i, r := range rl.InstanceLevel.Routes {
+		routes[i] = capi.InstanceLevelRouteRateLimits{
+			PathExact:         r.PathExact,
+			PathPrefix:        r.PathPrefix,
+			PathRegex:         r.PathRegex,
+			RequestsPerSecond: r.RequestsPerSecond,
+			RequestsMaxBurst:  r.RequestsMaxBurst,
+		}
+	}
+	return &capi.RateLimits{
+		InstanceLevel: capi.InstanceLevelRateLimits{
+			RequestsPerSecond: rl.InstanceLevel.RequestsPerSecond,
+			RequestsMaxBurst:  rl.InstanceLevel.RequestsMaxBurst,
+			Routes:            routes,
+		},
+	}
+}
+
+func (rl *RateLimits) validate(path *field.Path) field.ErrorList {
+	if rl == nil {
+		return nil
+	}
+
+	return rl.InstanceLevel.validate(path.Child("instanceLevel"))
+}
+
+type InstanceLevelRateLimits struct {
+	// RequestsPerSecond is the average number of requests per second that can be
+	// made without being throttled. This field is required if RequestsMaxBurst
+	// is set. The allowed number of requests may exceed RequestsPerSecond up to
+	// the value specified in RequestsMaxBurst.
+	//
+	// Internally, this is the refill rate of the token bucket used for rate limiting.
+	RequestsPerSecond int `json:"requestsPerSecond,omitempty"`
+
+	// RequestsMaxBurst is the maximum number of requests that can be sent
+	// in a burst. Should be equal to or greater than RequestsPerSecond.
+	// If unset, defaults to RequestsPerSecond.
+	//
+	// Internally, this is the maximum size of the token bucket used for rate limiting.
+	RequestsMaxBurst int `json:"requestsMaxBurst,omitempty"`
+
+	// Routes is a list of rate limits applied to specific routes.
+	// For a given request, the first matching route will be applied, if any.
+	// Overrides any top-level configuration.
+	Routes []InstanceLevelRouteRateLimits `json:"routes,omitempty"`
+}
+
+func (irl InstanceLevelRateLimits) validate(path *field.Path) field.ErrorList {
+	var allErrs field.ErrorList
+
+	// Track if RequestsPerSecond is set in at least one place in the config
+	isRateLimitSet := irl.RequestsPerSecond > 0
+
+	// Top-level RequestsPerSecond can be 0 (unset) or a positive number.
+	if irl.RequestsPerSecond < 0 {
+		allErrs = append(allErrs,
+			field.Invalid(path.Child("requestsPerSecond"),
+				irl.RequestsPerSecond,
+				"RequestsPerSecond must be positive"))
+	}
+
+	if irl.RequestsPerSecond == 0 && irl.RequestsMaxBurst > 0 {
+		allErrs = append(allErrs,
+			field.Invalid(path.Child("requestsPerSecond"),
+				irl.RequestsPerSecond,
+				"RequestsPerSecond must be greater than 0 if RequestsMaxBurst is set"))
+	}
+
+	if irl.RequestsMaxBurst < 0 {
+		allErrs = append(allErrs,
+			field.Invalid(path.Child("requestsMaxBurst"),
+				irl.RequestsMaxBurst,
+				"RequestsMaxBurst must be positive"))
+	}
+
+	for i, route := range irl.Routes {
+		if exact, prefix, regex := route.PathExact != "", route.PathPrefix != "", route.PathRegex != ""; (!exact && !prefix && !regex) ||
+			(exact && prefix) || (exact && regex) || (prefix && regex) {
+			allErrs = append(allErrs, field.Required(
+				path.Child("routes").Index(i),
+				"Route must define exactly one of PathExact, PathPrefix, or PathRegex"))
+		}
+
+		isRateLimitSet = isRateLimitSet || route.RequestsPerSecond > 0
+
+		// Unlike top-level RequestsPerSecond, any route MUST have a RequestsPerSecond defined.
+		if route.RequestsPerSecond <= 0 {
+			allErrs = append(allErrs, field.Invalid(
+				path.Child("routes").Index(i).Child("requestsPerSecond"),
+				route.RequestsPerSecond, "RequestsPerSecond must be greater than 0"))
+		}
+
+		if route.RequestsMaxBurst < 0 {
+			allErrs = append(allErrs, field.Invalid(
+				path.Child("routes").Index(i).Child("requestsMaxBurst"),
+				route.RequestsMaxBurst, "RequestsMaxBurst must be positive"))
+		}
+	}
+
+	if !isRateLimitSet {
+		allErrs = append(allErrs, field.Invalid(
+			path.Child("requestsPerSecond"),
+			irl.RequestsPerSecond, "At least one of top-level or route-level RequestsPerSecond must be set"))
+	}
+	return allErrs
+}
+
+type InstanceLevelRouteRateLimits struct {
+	// Exact path to match. Exactly one of PathExact, PathPrefix, or PathRegex must be specified.
+	PathExact string `json:"pathExact,omitempty"`
+	// Prefix to match. Exactly one of PathExact, PathPrefix, or PathRegex must be specified.
+	PathPrefix string `json:"pathPrefix,omitempty"`
+	// Regex to match. Exactly one of PathExact, PathPrefix, or PathRegex must be specified.
+	PathRegex string `json:"pathRegex,omitempty"`
+
+	// RequestsPerSecond is the average number of requests per
+	// second that can be made without being throttled. This field is required
+	// if RequestsMaxBurst is set. The allowed number of requests may exceed
+	// RequestsPerSecond up to the value specified in RequestsMaxBurst.
+	// Internally, this is the refill rate of the token bucket used for rate limiting.
+	RequestsPerSecond int `json:"requestsPerSecond,omitempty"`
+
+	// RequestsMaxBurst is the maximum number of requests that can be sent
+	// in a burst. Should be equal to or greater than RequestsPerSecond. If unset,
+	// defaults to RequestsPerSecond. Internally, this is the maximum size of the token
+	// bucket used for rate limiting.
+	RequestsMaxBurst int `json:"requestsMaxBurst,omitempty"`
+}
+
 func (in *ServiceDefaults) ConsulKind() string {
 	return capi.ServiceDefaults
 }
@@ -308,6 +456,7 @@ func (in *ServiceDefaults) ToConsul(datacenter string) capi.ConfigEntry {
 		LocalConnectTimeoutMs:     in.Spec.LocalConnectTimeoutMs,
 		LocalRequestTimeoutMs:     in.Spec.LocalRequestTimeoutMs,
 		BalanceInboundConnections: in.Spec.BalanceInboundConnections,
+		RateLimits:                in.Spec.RateLimits.toConsul(),
 		EnvoyExtensions:           in.Spec.EnvoyExtensions.toConsul(),
 	}
 }
@@ -356,6 +505,7 @@ func (in *ServiceDefaults) Validate(consulMeta common.ConsulMeta) error {
 
 	allErrs = append(allErrs, in.Spec.UpstreamConfig.validate(path.Child("upstreamConfig"), consulMeta.PartitionsEnabled)...)
 	allErrs = append(allErrs, in.Spec.Expose.validate(path.Child("expose"))...)
+	allErrs = append(allErrs, in.Spec.RateLimits.validate(path.Child("rateLimits"))...)
 	allErrs = append(allErrs, in.Spec.EnvoyExtensions.validate(path.Child("envoyExtensions"))...)
 
 	if len(allErrs) > 0 {