Skip to content
This repository has been archived by the owner on Jul 11, 2023. It is now read-only.

Commit

Permalink
envoy: wait indefinitely on TLS secrets
Browse files Browse the repository at this point in the history 
When configuring transport sockets for clusters and
listeners, it is possible for the SDS secret fetch
to timeout (defaults to 15s). This timeout poses problems
because after timeout, the secret is never fetched,
leaving the proxy with broken TLS connections. This breaks
the eventual consistency model. The side effect of using
a timeout is that clusters and listeners that are warming
become active after the timeout, leading to connection
resets and rejections.

This change makes the proxy indefinitely wait till
it can retrieve the SDS secrets referenced in the
transport socket configurations of clusters and
listeners.

Potentially addresses #2749

Signed-off-by: Shashank Ram <shashr2204@gmail.com>
  • Loading branch information
@shashankram @nojnhuh
shashankram authored and nojnhuh committed Mar 12, 2021
1 parent f1ab537 commit 34d0046
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 18 deletions.
12 changes: 10 additions & 2 deletions pkg/envoy/xdsutil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"strings"
"time"

xds_accesslog_filter "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3"
xds_core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
Expand Down Expand Up @@ -197,18 +198,25 @@ func pbStringValue(v string) *structpb.Value {
// 'tlsSDSCert' determines the SDS Secret config used to present the TLS certificate.
// 'peerValidationSDSCert' determines the SDS Secret configs used to validate the peer TLS certificate.
func getCommonTLSContext(tlsSDSCert, peerValidationSDSCert SDSCert) *xds_auth.CommonTlsContext {
sdsConfigSource := GetADSConfigSource()
// SDS secret fetch should never be skipped for TLS transport sockets.
// Disable the fetch timeout so that Envoy doesn't ignore fetching the TLS secrets after
// the default timeout interval. These secrets should always be available for the proxy
// to fetch from the controller.
sdsConfigSource.InitialFetchTimeout = ptypes.DurationProto(0 * time.Second)

return &xds_auth.CommonTlsContext{
TlsParams: GetTLSParams(),
TlsCertificateSdsSecretConfigs: []*xds_auth.SdsSecretConfig{{
// Example ==> Name: "service-cert:NameSpaceHere/ServiceNameHere"
Name: tlsSDSCert.String(),
SdsConfig: GetADSConfigSource(),
SdsConfig: sdsConfigSource,
}},
ValidationContextType: &xds_auth.CommonTlsContext_ValidationContextSdsSecretConfig{
ValidationContextSdsSecretConfig: &xds_auth.SdsSecretConfig{
// Example ==> Name: "root-cert<type>:NameSpaceHere/ServiceNameHere"
Name: peerValidationSDSCert.String(),
SdsConfig: GetADSConfigSource(),
SdsConfig: sdsConfigSource,
},
},
}
Expand Down
75 changes: 59 additions & 16 deletions pkg/envoy/xdsutil_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
package envoy

import (
"time"

core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
auth "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
"github.com/golang/protobuf/ptypes"
"github.com/golang/protobuf/ptypes/wrappers"

. "github.com/onsi/ginkgo"
Expand Down Expand Up @@ -124,7 +127,8 @@ var _ = Describe("Test Envoy tools", func() {
ConfigSourceSpecifier: &core.ConfigSource_Ads{
Ads: &core.AggregatedConfigSource{},
},
ResourceApiVersion: core.ApiVersion_V3,
ResourceApiVersion: core.ApiVersion_V3,
InitialFetchTimeout: ptypes.DurationProto(0 * time.Second),
},
}},
ValidationContextType: &auth.CommonTlsContext_ValidationContextSdsSecretConfig{
Expand All @@ -137,7 +141,8 @@ var _ = Describe("Test Envoy tools", func() {
ConfigSourceSpecifier: &core.ConfigSource_Ads{
Ads: &core.AggregatedConfigSource{},
},
ResourceApiVersion: core.ApiVersion_V3,
ResourceApiVersion: core.ApiVersion_V3,
InitialFetchTimeout: ptypes.DurationProto(0 * time.Second),
},
},
},
Expand Down Expand Up @@ -185,7 +190,8 @@ var _ = Describe("Test Envoy tools", func() {
ConfigSourceSpecifier: &core.ConfigSource_Ads{
Ads: &core.AggregatedConfigSource{},
},
ResourceApiVersion: core.ApiVersion_V3,
ResourceApiVersion: core.ApiVersion_V3,
InitialFetchTimeout: ptypes.DurationProto(0 * time.Second),
},
}},
ValidationContextType: &auth.CommonTlsContext_ValidationContextSdsSecretConfig{
Expand All @@ -195,7 +201,8 @@ var _ = Describe("Test Envoy tools", func() {
ConfigSourceSpecifier: &core.ConfigSource_Ads{
Ads: &core.AggregatedConfigSource{},
},
ResourceApiVersion: core.ApiVersion_V3,
ResourceApiVersion: core.ApiVersion_V3,
InitialFetchTimeout: ptypes.DurationProto(0 * time.Second),
},
},
},
Expand Down Expand Up @@ -241,13 +248,25 @@ var _ = Describe("Test Envoy tools", func() {
expected := &auth.CommonTlsContext{
TlsParams: GetTLSParams(),
TlsCertificateSdsSecretConfigs: []*auth.SdsSecretConfig{{
Name: "service-cert:default/bookbuyer",
SdsConfig: GetADSConfigSource(),
Name: "service-cert:default/bookbuyer",
SdsConfig: &core.ConfigSource{
ConfigSourceSpecifier: &core.ConfigSource_Ads{
Ads: &core.AggregatedConfigSource{},
},
ResourceApiVersion: core.ApiVersion_V3,
InitialFetchTimeout: ptypes.DurationProto(0 * time.Second),
},
}},
ValidationContextType: &auth.CommonTlsContext_ValidationContextSdsSecretConfig{
ValidationContextSdsSecretConfig: &auth.SdsSecretConfig{
Name: "root-cert-for-mtls-outbound:default/bookstore-v1",
SdsConfig: GetADSConfigSource(),
Name: "root-cert-for-mtls-outbound:default/bookstore-v1",
SdsConfig: &core.ConfigSource{
ConfigSourceSpecifier: &core.ConfigSource_Ads{
Ads: &core.AggregatedConfigSource{},
},
ResourceApiVersion: core.ApiVersion_V3,
InitialFetchTimeout: ptypes.DurationProto(0 * time.Second),
},
},
},
AlpnProtocols: nil,
Expand All @@ -271,13 +290,25 @@ var _ = Describe("Test Envoy tools", func() {
expected := &auth.CommonTlsContext{
TlsParams: GetTLSParams(),
TlsCertificateSdsSecretConfigs: []*auth.SdsSecretConfig{{
Name: "service-cert:default/bookstore-v1",
SdsConfig: GetADSConfigSource(),
Name: "service-cert:default/bookstore-v1",
SdsConfig: &core.ConfigSource{
ConfigSourceSpecifier: &core.ConfigSource_Ads{
Ads: &core.AggregatedConfigSource{},
},
ResourceApiVersion: core.ApiVersion_V3,
InitialFetchTimeout: ptypes.DurationProto(0 * time.Second),
},
}},
ValidationContextType: &auth.CommonTlsContext_ValidationContextSdsSecretConfig{
ValidationContextSdsSecretConfig: &auth.SdsSecretConfig{
Name: "root-cert-for-mtls-inbound:default/bookstore-v1",
SdsConfig: GetADSConfigSource(),
Name: "root-cert-for-mtls-inbound:default/bookstore-v1",
SdsConfig: &core.ConfigSource{
ConfigSourceSpecifier: &core.ConfigSource_Ads{
Ads: &core.AggregatedConfigSource{},
},
ResourceApiVersion: core.ApiVersion_V3,
InitialFetchTimeout: ptypes.DurationProto(0 * time.Second),
},
},
},
AlpnProtocols: nil,
Expand All @@ -301,13 +332,25 @@ var _ = Describe("Test Envoy tools", func() {
expected := &auth.CommonTlsContext{
TlsParams: GetTLSParams(),
TlsCertificateSdsSecretConfigs: []*auth.SdsSecretConfig{{
Name: "service-cert:default/bookstore-v1",
SdsConfig: GetADSConfigSource(),
Name: "service-cert:default/bookstore-v1",
SdsConfig: &core.ConfigSource{
ConfigSourceSpecifier: &core.ConfigSource_Ads{
Ads: &core.AggregatedConfigSource{},
},
ResourceApiVersion: core.ApiVersion_V3,
InitialFetchTimeout: ptypes.DurationProto(0 * time.Second),
},
}},
ValidationContextType: &auth.CommonTlsContext_ValidationContextSdsSecretConfig{
ValidationContextSdsSecretConfig: &auth.SdsSecretConfig{
Name: "root-cert-https:default/bookstore-v1",
SdsConfig: GetADSConfigSource(),
Name: "root-cert-https:default/bookstore-v1",
SdsConfig: &core.ConfigSource{
ConfigSourceSpecifier: &core.ConfigSource_Ads{
Ads: &core.AggregatedConfigSource{},
},
ResourceApiVersion: core.ApiVersion_V3,
InitialFetchTimeout: ptypes.DurationProto(0 * time.Second),
},
},
},
AlpnProtocols: nil,
Expand Down

0 comments on commit 34d0046

Please sign in to comment.