diff --git a/CHANGELOG.md b/CHANGELOG.md index 13ca27ad27b..9d8192df584 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio Here is an overview of all **stable** additions: +- **General**: Add support to register custom CAs globally in KEDA operator ([#4168](https://github.com/kedacore/keda/issues/4168)) - **General**: Introduce admission webhooks to automatically validate resource changes to prevent misconfiguration and enforce best practices ([#3755](https://github.com/kedacore/keda/issues/3755)) - **General**: Introduce new ArangoDB Scaler ([#4000](https://github.com/kedacore/keda/issues/4000)) - **Prometheus Metrics**: Introduce scaler activity in Prometheus metrics ([#4114](https://github.com/kedacore/keda/issues/4114)) diff --git a/Makefile b/Makefile index bc48f92a581..ee0d43580f1 100644 --- a/Makefile +++ b/Makefile @@ -236,13 +236,13 @@ set-version: ##@ Deployment -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. +install: kustomize manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/crd | kubectl apply -f - -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. +uninstall: kustomize manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/crd | kubectl delete -f - -deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. +deploy: kustomize manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && \ $(KUSTOMIZE) edit set image ghcr.io/kedacore/keda=${IMAGE_CONTROLLER} && \ if [ "$(AZURE_RUN_AAD_POD_IDENTITY_TESTS)" = true ]; then \ @@ -274,10 +274,10 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in # until this issue is solved: https://github.com/kubernetes-sigs/kustomize/issues/1009 @sed -i".out" -e 's@version:[ ].*@version: $(VERSION)@g' config/default/kustomize-config/metadataLabelTransformer.yaml rm -rf config/default/kustomize-config/metadataLabelTransformer.yaml.out - $(KUSTOMIZE) build config/default | kubectl apply -f - + $(KUSTOMIZE) build config/e2e | kubectl apply -f - -undeploy: e2e-test-clean-crds ## Undeploy controller from the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/default | kubectl delete -f - +undeploy: kustomize e2e-test-clean-crds ## Undeploy controller from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/e2e | kubectl delete -f - ## Location to install dependencies to LOCALBIN ?= $(shell pwd)/bin diff --git a/config/e2e/create_cas_volume.yml b/config/e2e/create_cas_volume.yml new file mode 100644 index 00000000000..fb596f92554 --- /dev/null +++ b/config/e2e/create_cas_volume.yml @@ -0,0 +1,15 @@ +- op: add + path: /spec/template/spec/containers/0/volumeMounts/1 + value: + name: custom-cas + mountPath: /custom/ca + readOnly: true + +- op: add + path: /spec/template/spec/volumes/1 + value: + name: custom-cas + secret: + defaultMode: 420 + secretName: custom-cas + optional: true diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml new file mode 100644 index 00000000000..48cb86a4f30 --- /dev/null +++ b/config/e2e/kustomization.yaml @@ -0,0 +1,10 @@ +patchesJson6902: +- target: + group: apps + version: v1 + kind: Deployment + name: keda-operator + path: create_cas_volume.yml + +bases: +- ../default diff --git a/pkg/scalers/arangodb_scaler.go b/pkg/scalers/arangodb_scaler.go index 9154bd73827..29b343f971c 100644 --- a/pkg/scalers/arangodb_scaler.go +++ b/pkg/scalers/arangodb_scaler.go @@ -2,7 +2,6 @@ package scalers import ( "context" - "crypto/tls" "fmt" "strconv" "strings" @@ -15,6 +14,7 @@ import ( "k8s.io/metrics/pkg/apis/external_metrics" "github.com/kedacore/keda/v2/pkg/scalers/authentication" + "github.com/kedacore/keda/v2/pkg/util" ) type arangoDBScaler struct { @@ -97,10 +97,7 @@ func getNewArangoDBClient(meta *arangoDBMetadata) (driver.Client, error) { conn, err := http.NewConnection(http.ConnectionConfig{ Endpoints: strings.Split(meta.endpoints, ","), - TLSConfig: &tls.Config{ - MinVersion: tls.VersionTLS13, - InsecureSkipVerify: meta.unsafeSsl, - }, + TLSConfig: util.CreateTLSClientConfig(meta.unsafeSsl), }) if err != nil { return nil, fmt.Errorf("failed to create a new http connection, %w", err) diff --git a/pkg/scalers/authentication/authentication_helpers.go b/pkg/scalers/authentication/authentication_helpers.go index db5c9d8a49f..a10b6e702df 100644 --- a/pkg/scalers/authentication/authentication_helpers.go +++ b/pkg/scalers/authentication/authentication_helpers.go @@ -85,21 +85,20 @@ func GetBearerToken(auth *AuthMeta) string { return fmt.Sprintf("Bearer %s", auth.BearerToken) } -func NewTLSConfig(auth *AuthMeta) (*tls.Config, error) { +func NewTLSConfig(auth *AuthMeta, unsafeSsl bool) (*tls.Config, error) { return kedautil.NewTLSConfig( auth.Cert, auth.Key, auth.CA, + unsafeSsl, ) } func CreateHTTPRoundTripper(roundTripperType TransportType, auth *AuthMeta, conf ...*HTTPTransport) (rt http.RoundTripper, err error) { - tlsConfig := &tls.Config{ - MinVersion: kedautil.GetMinTLSVersion(), - InsecureSkipVerify: false, - } + unsafeSsl := false + tlsConfig := kedautil.CreateTLSClientConfig(unsafeSsl) if auth != nil && (auth.CA != "" || auth.EnableTLS) { - tlsConfig, err = NewTLSConfig(auth) + tlsConfig, err = NewTLSConfig(auth, unsafeSsl) if err != nil || tlsConfig == nil { return nil, fmt.Errorf("error creating the TLS config: %w", err) } diff --git a/pkg/scalers/elasticsearch_scaler.go b/pkg/scalers/elasticsearch_scaler.go index 73c3cc22788..932a187207f 100644 --- a/pkg/scalers/elasticsearch_scaler.go +++ b/pkg/scalers/elasticsearch_scaler.go @@ -3,12 +3,10 @@ package scalers import ( "bytes" "context" - "crypto/tls" "encoding/json" "errors" "fmt" "io" - "net/http" "strconv" "strings" @@ -18,7 +16,7 @@ import ( v2 "k8s.io/api/autoscaling/v2" "k8s.io/metrics/pkg/apis/external_metrics" - kedautil "github.com/kedacore/keda/v2/pkg/util" + "github.com/kedacore/keda/v2/pkg/util" ) type elasticsearchScaler struct { @@ -218,7 +216,7 @@ func parseElasticsearchMetadata(config *ScalerConfig) (*elasticsearchMetadata, e meta.activationTargetValue = activationTargetValue } - meta.metricName = GenerateMetricNameWithIndex(config.ScalerIndex, kedautil.NormalizeString(fmt.Sprintf("elasticsearch-%s", meta.searchTemplateName))) + meta.metricName = GenerateMetricNameWithIndex(config.ScalerIndex, util.NormalizeString(fmt.Sprintf("elasticsearch-%s", meta.searchTemplateName))) return &meta, nil } @@ -243,13 +241,7 @@ func newElasticsearchClient(meta *elasticsearchMetadata, logger logr.Logger) (*e } } - transport := http.DefaultTransport.(*http.Transport) - transport.TLSClientConfig = &tls.Config{ - MinVersion: kedautil.GetMinTLSVersion(), - InsecureSkipVerify: meta.unsafeSsl, - } - config.Transport = transport - + config.Transport = util.CreateHTTPTransport(meta.unsafeSsl) esClient, err := elasticsearch.NewClient(config) if err != nil { logger.Error(err, fmt.Sprintf("Found error when creating client: %s", err)) diff --git a/pkg/scalers/etcd_scaler.go b/pkg/scalers/etcd_scaler.go index e72e2cde761..2c4ad59107a 100644 --- a/pkg/scalers/etcd_scaler.go +++ b/pkg/scalers/etcd_scaler.go @@ -154,7 +154,7 @@ func getEtcdClients(metadata *etcdMetadata) (*clientv3.Client, error) { var tlsConfig *tls.Config var err error if metadata.enableTLS { - tlsConfig, err = kedautil.NewTLSConfigWithPassword(metadata.cert, metadata.key, metadata.keyPassword, metadata.ca) + tlsConfig, err = kedautil.NewTLSConfigWithPassword(metadata.cert, metadata.key, metadata.keyPassword, metadata.ca, false) if err != nil { return nil, err } diff --git a/pkg/scalers/ibmmq_scaler.go b/pkg/scalers/ibmmq_scaler.go index 700a69e2161..15591524d37 100644 --- a/pkg/scalers/ibmmq_scaler.go +++ b/pkg/scalers/ibmmq_scaler.go @@ -3,7 +3,6 @@ package scalers import ( "bytes" "context" - "crypto/tls" "encoding/json" "fmt" "io" @@ -178,14 +177,7 @@ func (s *IBMMQScaler) getQueueDepthViaHTTP(ctx context.Context) (int64, error) { req.Header.Set("Content-Type", "application/json") req.SetBasicAuth(s.metadata.username, s.metadata.password) - tr := &http.Transport{ - TLSClientConfig: &tls.Config{ - MinVersion: kedautil.GetMinTLSVersion(), - InsecureSkipVerify: s.metadata.tlsDisabled, - }, - } - client := kedautil.CreateHTTPClient(s.defaultHTTPTimeout, false) - client.Transport = tr + client := kedautil.CreateHTTPClient(s.defaultHTTPTimeout, s.metadata.tlsDisabled) resp, err := client.Do(req) if err != nil { diff --git a/pkg/scalers/influxdb_scaler.go b/pkg/scalers/influxdb_scaler.go index 6a3d605c948..bc59ac3ee9e 100644 --- a/pkg/scalers/influxdb_scaler.go +++ b/pkg/scalers/influxdb_scaler.go @@ -2,7 +2,6 @@ package scalers import ( "context" - "crypto/tls" "fmt" "strconv" @@ -12,7 +11,7 @@ import ( v2 "k8s.io/api/autoscaling/v2" "k8s.io/metrics/pkg/apis/external_metrics" - kedautil "github.com/kedacore/keda/v2/pkg/util" + "github.com/kedacore/keda/v2/pkg/util" ) type influxDBScaler struct { @@ -52,10 +51,7 @@ func NewInfluxDBScaler(config *ScalerConfig) (Scaler, error) { client := influxdb2.NewClientWithOptions( meta.serverURL, meta.authToken, - influxdb2.DefaultOptions().SetTLSConfig(&tls.Config{ - MinVersion: kedautil.GetMinTLSVersion(), - InsecureSkipVerify: meta.unsafeSsl, - })) + influxdb2.DefaultOptions().SetTLSConfig(util.CreateTLSClientConfig(meta.unsafeSsl))) return &influxDBScaler{ client: client, @@ -123,9 +119,9 @@ func parseInfluxDBMetadata(config *ScalerConfig) (*influxDBMetadata, error) { } if val, ok := config.TriggerMetadata["metricName"]; ok { - metricName = kedautil.NormalizeString(fmt.Sprintf("influxdb-%s", val)) + metricName = util.NormalizeString(fmt.Sprintf("influxdb-%s", val)) } else { - metricName = kedautil.NormalizeString(fmt.Sprintf("influxdb-%s", organizationName)) + metricName = util.NormalizeString(fmt.Sprintf("influxdb-%s", organizationName)) } if val, ok := config.TriggerMetadata["activationThresholdValue"]; ok { diff --git a/pkg/scalers/kafka_scaler.go b/pkg/scalers/kafka_scaler.go index 5e1cea26174..f7001c2cd39 100644 --- a/pkg/scalers/kafka_scaler.go +++ b/pkg/scalers/kafka_scaler.go @@ -318,7 +318,7 @@ func getKafkaClients(metadata kafkaMetadata) (sarama.Client, sarama.ClusterAdmin if metadata.enableTLS { config.Net.TLS.Enable = true - tlsConfig, err := kedautil.NewTLSConfigWithPassword(metadata.cert, metadata.key, metadata.keyPassword, metadata.ca) + tlsConfig, err := kedautil.NewTLSConfigWithPassword(metadata.cert, metadata.key, metadata.keyPassword, metadata.ca, false) if err != nil { return nil, nil, err } diff --git a/pkg/scalers/liiklus/LiiklusService.pb.go b/pkg/scalers/liiklus/LiiklusService.pb.go index a0027a6fff3..8d34aea93d9 100644 --- a/pkg/scalers/liiklus/LiiklusService.pb.go +++ b/pkg/scalers/liiklus/LiiklusService.pb.go @@ -7,10 +7,10 @@ package liiklus import ( - empty "github.com/golang/protobuf/ptypes/empty" - timestamp "github.com/golang/protobuf/ptypes/timestamp" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" ) @@ -807,11 +807,11 @@ type ReceiveReply_Record struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Offset uint64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"` - Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` - Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` - Timestamp *timestamp.Timestamp `protobuf:"bytes,4,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - Replay bool `protobuf:"varint,5,opt,name=replay,proto3" json:"replay,omitempty"` + Offset uint64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"` + Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + Replay bool `protobuf:"varint,5,opt,name=replay,proto3" json:"replay,omitempty"` } func (x *ReceiveReply_Record) Reset() { @@ -867,7 +867,7 @@ func (x *ReceiveReply_Record) GetValue() []byte { return nil } -func (x *ReceiveReply_Record) GetTimestamp() *timestamp.Timestamp { +func (x *ReceiveReply_Record) GetTimestamp() *timestamppb.Timestamp { if x != nil { return x.Timestamp } @@ -1073,8 +1073,8 @@ var file_LiiklusService_proto_goTypes = []interface{}{ (*ReceiveReply_Record)(nil), // 13: com.github.bsideup.liiklus.ReceiveReply.Record nil, // 14: com.github.bsideup.liiklus.GetOffsetsReply.OffsetsEntry nil, // 15: com.github.bsideup.liiklus.GetEndOffsetsReply.OffsetsEntry - (*timestamp.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*empty.Empty)(nil), // 17: google.protobuf.Empty + (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 17: google.protobuf.Empty } var file_LiiklusService_proto_depIdxs = []int32{ 0, // 0: com.github.bsideup.liiklus.SubscribeRequest.autoOffsetReset:type_name -> com.github.bsideup.liiklus.SubscribeRequest.AutoOffsetReset diff --git a/pkg/scalers/liiklus/LiiklusService_grpc.pb.go b/pkg/scalers/liiklus/LiiklusService_grpc.pb.go index 3ad3383d914..4ff6ae6c44f 100644 --- a/pkg/scalers/liiklus/LiiklusService_grpc.pb.go +++ b/pkg/scalers/liiklus/LiiklusService_grpc.pb.go @@ -8,10 +8,10 @@ package liiklus import ( context "context" - empty "github.com/golang/protobuf/ptypes/empty" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" + emptypb "google.golang.org/protobuf/types/known/emptypb" ) // This is a compile-time assertion to ensure that this generated file @@ -26,7 +26,7 @@ type LiiklusServiceClient interface { Publish(ctx context.Context, in *PublishRequest, opts ...grpc.CallOption) (*PublishReply, error) Subscribe(ctx context.Context, in *SubscribeRequest, opts ...grpc.CallOption) (LiiklusService_SubscribeClient, error) Receive(ctx context.Context, in *ReceiveRequest, opts ...grpc.CallOption) (LiiklusService_ReceiveClient, error) - Ack(ctx context.Context, in *AckRequest, opts ...grpc.CallOption) (*empty.Empty, error) + Ack(ctx context.Context, in *AckRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) GetOffsets(ctx context.Context, in *GetOffsetsRequest, opts ...grpc.CallOption) (*GetOffsetsReply, error) GetEndOffsets(ctx context.Context, in *GetEndOffsetsRequest, opts ...grpc.CallOption) (*GetEndOffsetsReply, error) } @@ -112,8 +112,8 @@ func (x *liiklusServiceReceiveClient) Recv() (*ReceiveReply, error) { return m, nil } -func (c *liiklusServiceClient) Ack(ctx context.Context, in *AckRequest, opts ...grpc.CallOption) (*empty.Empty, error) { - out := new(empty.Empty) +func (c *liiklusServiceClient) Ack(ctx context.Context, in *AckRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + out := new(emptypb.Empty) err := c.cc.Invoke(ctx, "/com.github.bsideup.liiklus.LiiklusService/Ack", in, out, opts...) if err != nil { return nil, err @@ -146,7 +146,7 @@ type LiiklusServiceServer interface { Publish(context.Context, *PublishRequest) (*PublishReply, error) Subscribe(*SubscribeRequest, LiiklusService_SubscribeServer) error Receive(*ReceiveRequest, LiiklusService_ReceiveServer) error - Ack(context.Context, *AckRequest) (*empty.Empty, error) + Ack(context.Context, *AckRequest) (*emptypb.Empty, error) GetOffsets(context.Context, *GetOffsetsRequest) (*GetOffsetsReply, error) GetEndOffsets(context.Context, *GetEndOffsetsRequest) (*GetEndOffsetsReply, error) mustEmbedUnimplementedLiiklusServiceServer() @@ -165,7 +165,7 @@ func (UnimplementedLiiklusServiceServer) Subscribe(*SubscribeRequest, LiiklusSer func (UnimplementedLiiklusServiceServer) Receive(*ReceiveRequest, LiiklusService_ReceiveServer) error { return status.Errorf(codes.Unimplemented, "method Receive not implemented") } -func (UnimplementedLiiklusServiceServer) Ack(context.Context, *AckRequest) (*empty.Empty, error) { +func (UnimplementedLiiklusServiceServer) Ack(context.Context, *AckRequest) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method Ack not implemented") } func (UnimplementedLiiklusServiceServer) GetOffsets(context.Context, *GetOffsetsRequest) (*GetOffsetsReply, error) { diff --git a/pkg/scalers/metrics_api_scaler.go b/pkg/scalers/metrics_api_scaler.go index c819d057b9e..668288e797c 100644 --- a/pkg/scalers/metrics_api_scaler.go +++ b/pkg/scalers/metrics_api_scaler.go @@ -79,11 +79,11 @@ func NewMetricsAPIScaler(config *ScalerConfig) (Scaler, error) { httpClient := kedautil.CreateHTTPClient(config.GlobalHTTPTimeout, meta.unsafeSsl) if meta.enableTLS || len(meta.ca) > 0 { - config, err := kedautil.NewTLSConfig(meta.cert, meta.key, meta.ca) + config, err := kedautil.NewTLSConfig(meta.cert, meta.key, meta.ca, meta.unsafeSsl) if err != nil { return nil, err } - httpClient.Transport = &http.Transport{TLSClientConfig: config} + httpClient.Transport = kedautil.CreateHTTPTransportWithTLSConfig(config) } return &metricsAPIScaler{ diff --git a/pkg/scalers/pulsar_scaler.go b/pkg/scalers/pulsar_scaler.go index 63b36b88869..13af199763c 100644 --- a/pkg/scalers/pulsar_scaler.go +++ b/pkg/scalers/pulsar_scaler.go @@ -104,11 +104,11 @@ func NewPulsarScaler(config *ScalerConfig) (Scaler, error) { if pulsarMetadata.pulsarAuth != nil { if pulsarMetadata.pulsarAuth.CA != "" || pulsarMetadata.pulsarAuth.EnableTLS { - config, err := authentication.NewTLSConfig(pulsarMetadata.pulsarAuth) + config, err := authentication.NewTLSConfig(pulsarMetadata.pulsarAuth, false) if err != nil { return nil, err } - client.Transport = &http.Transport{TLSClientConfig: config} + client.Transport = kedautil.CreateHTTPTransportWithTLSConfig(config) } if pulsarMetadata.pulsarAuth.EnableBearerAuth || pulsarMetadata.pulsarAuth.EnableBasicAuth { diff --git a/pkg/scalers/rabbitmq_scaler.go b/pkg/scalers/rabbitmq_scaler.go index 2f4b059f26e..4def62952a7 100644 --- a/pkg/scalers/rabbitmq_scaler.go +++ b/pkg/scalers/rabbitmq_scaler.go @@ -394,7 +394,7 @@ func getConnectionAndChannel(host string, meta *rabbitMQMetadata) (*amqp.Connect var conn *amqp.Connection var err error if meta.enableTLS { - tlsConfig, configErr := kedautil.NewTLSConfigWithPassword(meta.cert, meta.key, meta.keyPassword, meta.ca) + tlsConfig, configErr := kedautil.NewTLSConfigWithPassword(meta.cert, meta.key, meta.keyPassword, meta.ca, false) if configErr == nil { conn, err = amqp.DialTLS(host, tlsConfig) } diff --git a/pkg/scalers/redis_scaler.go b/pkg/scalers/redis_scaler.go index 03327e02416..f80a53baeed 100644 --- a/pkg/scalers/redis_scaler.go +++ b/pkg/scalers/redis_scaler.go @@ -2,7 +2,6 @@ package scalers import ( "context" - "crypto/tls" "errors" "fmt" "net" @@ -14,7 +13,7 @@ import ( v2 "k8s.io/api/autoscaling/v2" "k8s.io/metrics/pkg/apis/external_metrics" - kedautil "github.com/kedacore/keda/v2/pkg/util" + "github.com/kedacore/keda/v2/pkg/util" ) const ( @@ -258,7 +257,7 @@ func (s *redisScaler) Close(context.Context) error { // GetMetricSpecForScaling returns the metric spec for the HPA func (s *redisScaler) GetMetricSpecForScaling(context.Context) []v2.MetricSpec { - metricName := kedautil.NormalizeString(fmt.Sprintf("redis-%s", s.metadata.listName)) + metricName := util.NormalizeString(fmt.Sprintf("redis-%s", s.metadata.listName)) externalMetric := &v2.ExternalMetricSource{ Metric: v2.MetricIdentifier{ Name: GenerateMetricNameWithIndex(s.metadata.scalerIndex, metricName), @@ -464,10 +463,7 @@ func getRedisClusterClient(ctx context.Context, info redisConnectionInfo) (*redi Password: info.password, } if info.enableTLS { - options.TLSConfig = &tls.Config{ - MinVersion: kedautil.GetMinTLSVersion(), - InsecureSkipVerify: info.unsafeSsl, - } + options.TLSConfig = util.CreateTLSClientConfig(info.unsafeSsl) } // confirm if connected @@ -489,10 +485,7 @@ func getRedisSentinelClient(ctx context.Context, info redisConnectionInfo, dbInd MasterName: info.sentinelMaster, } if info.enableTLS { - options.TLSConfig = &tls.Config{ - MinVersion: kedautil.GetMinTLSVersion(), - InsecureSkipVerify: info.unsafeSsl, - } + options.TLSConfig = util.CreateTLSClientConfig(info.unsafeSsl) } // confirm if connected @@ -511,10 +504,7 @@ func getRedisClient(ctx context.Context, info redisConnectionInfo, dbIndex int) DB: dbIndex, } if info.enableTLS { - options.TLSConfig = &tls.Config{ - MinVersion: kedautil.GetMinTLSVersion(), - InsecureSkipVerify: info.unsafeSsl, - } + options.TLSConfig = util.CreateTLSClientConfig(info.unsafeSsl) } // confirm if connected diff --git a/pkg/util/certificates.go b/pkg/util/certificates.go new file mode 100644 index 00000000000..ed5be75ecaa --- /dev/null +++ b/pkg/util/certificates.go @@ -0,0 +1,74 @@ +/* +Copyright 2023 The KEDA Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "crypto/x509" + "errors" + "fmt" + "io/fs" + "os" + "path" + + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +const customCAPath = "/custom/ca" + +var logger = logf.Log.WithName("certificates") + +var rootCAs *x509.CertPool + +func getRootCAs() *x509.CertPool { + if rootCAs != nil { + return rootCAs.Clone() + } + + rootCAs, _ = x509.SystemCertPool() + if rootCAs == nil { + rootCAs = x509.NewCertPool() + } + + if _, err := os.Stat(customCAPath); errors.Is(err, fs.ErrNotExist) { + logger.V(1).Info(fmt.Sprintf("the path %s doesn't exist, skipping custom CA registrations", customCAPath)) + return rootCAs.Clone() + } + + files, err := os.ReadDir(customCAPath) + if err != nil { + logger.Error(err, fmt.Sprintf("unable to read %s", customCAPath)) + return rootCAs.Clone() + } + + for _, file := range files { + if file.IsDir() { + continue + } + + certs, err := os.ReadFile(path.Join(customCAPath, file.Name())) + if err != nil { + logger.Error(err, fmt.Sprintf("Failed to append %q to certPool", file.Name())) + } + + if ok := rootCAs.AppendCertsFromPEM(certs); !ok { + logger.Error(fmt.Errorf("no certs appended"), fmt.Sprintf("the certificate %s hasn't been added to the pool", file.Name())) + } + logger.V(1).Info(fmt.Sprintf("the certificate %s has been added to the pool", file.Name())) + } + + return rootCAs.Clone() +} diff --git a/pkg/util/certificates_test.go b/pkg/util/certificates_test.go new file mode 100644 index 00000000000..221edaa65d8 --- /dev/null +++ b/pkg/util/certificates_test.go @@ -0,0 +1,98 @@ +/* +Copyright 2023 The KEDA Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/pem" + "math/big" + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + caCrtPath = path.Join(customCAPath, "ca.crt") + certCommonName = "test-cert" +) + +func TestCustomCAsAreRegistered(t *testing.T) { + defer os.Remove(caCrtPath) + generateCA(t) + + rootCAs := getRootCAs() + //nolint:staticcheck // func (s *CertPool) Subjects was deprecated if s was returned by SystemCertPool, Subjects + subjects := rootCAs.Subjects() + var rdnSequence pkix.RDNSequence + _, err := asn1.Unmarshal(subjects[len(subjects)-1], &rdnSequence) + if err != nil { + t.Fatal("could not unmarshal der formatted subject") + } + var name pkix.Name + name.FillFromRDNSequence(&rdnSequence) + + assert.Equal(t, certCommonName, name.CommonName, "certificate not found") +} + +func generateCA(t *testing.T) { + err := os.MkdirAll(customCAPath, os.ModePerm) + require.NoErrorf(t, err, "error generating the custom ca folder - %s", err) + + ca := &x509.Certificate{ + SerialNumber: big.NewInt(2019), + Subject: pkix.Name{ + Organization: []string{"Company, INC."}, + Country: []string{"US"}, + Province: []string{""}, + Locality: []string{"San Francisco"}, + StreetAddress: []string{"Golden Gate Bridge"}, + PostalCode: []string{"94016"}, + CommonName: certCommonName, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + // create our private and public key + caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) + require.NoErrorf(t, err, "error generating custom CA key - %s", err) + + // create the CA + caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey) + require.NoErrorf(t, err, "error generating custom CA - %s", err) + + // pem encode + crtFile, err := os.OpenFile(caCrtPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + require.NoErrorf(t, err, "error opening custom CA file - %s", err) + err = pem.Encode(crtFile, &pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) + require.NoErrorf(t, err, "error opening custom CA file - %s", err) +} diff --git a/pkg/util/http.go b/pkg/util/http.go index 6655bec437a..d0d12878201 100644 --- a/pkg/util/http.go +++ b/pkg/util/http.go @@ -18,20 +18,13 @@ package util import ( "crypto/tls" - "fmt" "net/http" - "os" "time" - - "github.com/go-logr/logr" - ctrl "sigs.k8s.io/controller-runtime" ) var disableKeepAlives bool -var minTLSVersion uint16 func init() { - setupLog := ctrl.Log.WithName("http_setup") var err error // This code will be removed in https://github.com/kedacore/keda/pull/4191 // nosemgrep: trailofbits.go.invalid-usage-of-modified-variable.invalid-usage-of-modified-variable @@ -39,29 +32,17 @@ func init() { if err != nil { disableKeepAlives = false } +} - minTLSVersion = initMinTLSVersion(setupLog) +func init() { + disableKeepAlives = getKeepAliveValue() } -func initMinTLSVersion(logger logr.Logger) uint16 { - version, found := os.LookupEnv("KEDA_HTTP_MIN_TLS_VERSION") - minVersion := tls.VersionTLS12 - if found { - switch version { - case "TLS13": - minVersion = tls.VersionTLS13 - case "TLS12": - minVersion = tls.VersionTLS12 - case "TLS11": - minVersion = tls.VersionTLS11 - case "TLS10": - minVersion = tls.VersionTLS10 - default: - logger.Info(fmt.Sprintf("%s is not a valid value, using `TLS12`. Allowed values are: `TLS13`,`TLS12`,`TLS11`,`TLS10`", version)) - minVersion = tls.VersionTLS12 - } +func getKeepAliveValue() bool { + if val, err := ResolveOsEnvBool("KEDA_HTTP_DISABLE_KEEP_ALIVE", false); err == nil { + return val } - return uint16(minVersion) + return false } // HTTPDoer is an interface that matches the Do method on @@ -79,18 +60,7 @@ func CreateHTTPClient(timeout time.Duration, unsafeSsl bool) *http.Client { if timeout <= 0 { timeout = 300 * time.Millisecond } - transport := &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: unsafeSsl, - MinVersion: GetMinTLSVersion(), - }, - Proxy: http.ProxyFromEnvironment, - } - if disableKeepAlives { - // disable keep http connection alive - transport.DisableKeepAlives = true - transport.IdleConnTimeout = 100 * time.Second - } + transport := CreateHTTPTransport(unsafeSsl) httpClient := &http.Client{ Timeout: timeout, Transport: transport, @@ -98,6 +68,23 @@ func CreateHTTPClient(timeout time.Duration, unsafeSsl bool) *http.Client { return httpClient } -func GetMinTLSVersion() uint16 { - return minTLSVersion +// CreateHTTPTransport returns a new HTTP Transport with Proxy, Keep alives +// unsafeSsl parameter allows to avoid tls cert validation if it's required +func CreateHTTPTransport(unsafeSsl bool) *http.Transport { + return CreateHTTPTransportWithTLSConfig(CreateTLSClientConfig(unsafeSsl)) +} + +// CreateHTTPTransportWithTLSConfig returns a new HTTP Transport with Proxy, Keep alives +// using given tls.Config +func CreateHTTPTransportWithTLSConfig(config *tls.Config) *http.Transport { + transport := &http.Transport{ + TLSClientConfig: config, + Proxy: http.ProxyFromEnvironment, + } + if disableKeepAlives { + // disable keep http connection alive + transport.DisableKeepAlives = true + transport.IdleConnTimeout = 100 * time.Second + } + return transport } diff --git a/pkg/util/http_test.go b/pkg/util/http_test.go index 970a6c76dcf..f33da81192d 100644 --- a/pkg/util/http_test.go +++ b/pkg/util/http_test.go @@ -17,57 +17,20 @@ limitations under the License. package util import ( - "crypto/tls" - "os" "testing" + "time" - "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" ) -type minTLSVersionTestData struct { - envSet bool - envValue string - expectedVersion uint16 -} +func TestCreateHTTPClientWhenNegativeTimeout(t *testing.T) { + client := CreateHTTPClient(-1*time.Minute, false) -var minTLSVersionTestDatas = []minTLSVersionTestData{ - { - envSet: true, - envValue: "TLS10", - expectedVersion: tls.VersionTLS10, - }, - { - envSet: true, - envValue: "TLS11", - expectedVersion: tls.VersionTLS11, - }, - { - envSet: true, - envValue: "TLS12", - expectedVersion: tls.VersionTLS12, - }, - { - envSet: true, - envValue: "TLS13", - expectedVersion: tls.VersionTLS13, - }, - { - envSet: false, - expectedVersion: tls.VersionTLS12, - }, + assert.Equal(t, 300*time.Millisecond, client.Timeout) } -func TestResolveMinTLSVersion(t *testing.T) { - defer os.Unsetenv("KEDA_HTTP_MIN_TLS_VERSION") - for _, testData := range minTLSVersionTestDatas { - os.Unsetenv("KEDA_HTTP_MIN_TLS_VERSION") - if testData.envSet { - os.Setenv("KEDA_HTTP_MIN_TLS_VERSION", testData.envValue) - } - minVersion := initMinTLSVersion(logr.Discard()) +func TestCreateHTTPClientWhenValidTimeout(t *testing.T) { + client := CreateHTTPClient(1*time.Minute, false) - if testData.expectedVersion != minVersion { - t.Error("Failed to resolve minTLSVersion correctly", "wants", testData.expectedVersion, "got", minVersion) - } - } + assert.Equal(t, 1*time.Minute, client.Timeout) } diff --git a/pkg/util/tls_config.go b/pkg/util/tls_config.go index 3a64f1ed752..078b602a579 100644 --- a/pkg/util/tls_config.go +++ b/pkg/util/tls_config.go @@ -21,42 +21,25 @@ import ( "crypto/x509" "encoding/pem" "fmt" + "os" + "github.com/go-logr/logr" "github.com/youmark/pkcs8" + ctrl "sigs.k8s.io/controller-runtime" ) -func decryptClientKey(clientKey, clientKeyPassword string) ([]byte, error) { - block, _ := pem.Decode([]byte(clientKey)) - - key, err := pkcs8.ParsePKCS8PrivateKey(block.Bytes, []byte(clientKeyPassword)) - if err != nil { - return nil, err - } - - pemData, err := x509.MarshalPKCS8PrivateKey(key) - if err != nil { - return nil, err - } - - var pemPrivateBlock = &pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: pemData, - } +var minTLSVersion uint16 - encodedData := pem.EncodeToMemory(pemPrivateBlock) - - return encodedData, nil +func init() { + setupLog := ctrl.Log.WithName("tls_setup") + minTLSVersion = initMinTLSVersion(setupLog) } // NewTLSConfigWithPassword returns a *tls.Config using the given ceClient cert, ceClient key, // and CA certificate. If clientKeyPassword is not empty the provided password will be used to // decrypt the given key. If none are appropriate, a nil *tls.Config is returned. -func NewTLSConfigWithPassword(clientCert, clientKey, clientKeyPassword, caCert string) (*tls.Config, error) { - valid := false - - config := &tls.Config{ - MinVersion: GetMinTLSVersion(), - } +func NewTLSConfigWithPassword(clientCert, clientKey, clientKeyPassword, caCert string, unsafeSsl bool) (*tls.Config, error) { + config := CreateTLSClientConfig(unsafeSsl) if clientCert != "" && clientKey != "" { key := []byte(clientKey) @@ -73,18 +56,10 @@ func NewTLSConfigWithPassword(clientCert, clientKey, clientKeyPassword, caCert s return nil, fmt.Errorf("error parse X509KeyPair: %w", err) } config.Certificates = []tls.Certificate{cert} - valid = true } if caCert != "" { - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM([]byte(caCert)) - config.RootCAs = caCertPool - valid = true - } - - if !valid { - config = nil + config.RootCAs.AppendCertsFromPEM([]byte(caCert)) } return config, nil @@ -92,6 +67,65 @@ func NewTLSConfigWithPassword(clientCert, clientKey, clientKeyPassword, caCert s // NewTLSConfig returns a *tls.Config using the given ceClient cert, ceClient key, // and CA certificate. If none are appropriate, a nil *tls.Config is returned. -func NewTLSConfig(clientCert, clientKey, caCert string) (*tls.Config, error) { - return NewTLSConfigWithPassword(clientCert, clientKey, "", caCert) +func NewTLSConfig(clientCert, clientKey, caCert string, unsafeSsl bool) (*tls.Config, error) { + return NewTLSConfigWithPassword(clientCert, clientKey, "", caCert, unsafeSsl) +} + +// CreateTLSClientConfig returns a new TLS Config +// unsafeSsl parameter allows to avoid tls cert validation if it's required +func CreateTLSClientConfig(unsafeSsl bool) *tls.Config { + return &tls.Config{ + InsecureSkipVerify: unsafeSsl, + RootCAs: getRootCAs(), + MinVersion: GetMinTLSVersion(), + } +} + +// GetMinTLSVersion return the minTLSVersion based on configurations +func GetMinTLSVersion() uint16 { + return minTLSVersion +} + +func initMinTLSVersion(logger logr.Logger) uint16 { + version, found := os.LookupEnv("KEDA_HTTP_MIN_TLS_VERSION") + minVersion := tls.VersionTLS12 + if found { + switch version { + case "TLS13": + minVersion = tls.VersionTLS13 + case "TLS12": + minVersion = tls.VersionTLS12 + case "TLS11": + minVersion = tls.VersionTLS11 + case "TLS10": + minVersion = tls.VersionTLS10 + default: + logger.Info(fmt.Sprintf("%s is not a valid value, using `TLS12`. Allowed values are: `TLS13`,`TLS12`,`TLS11`,`TLS10`", version)) + minVersion = tls.VersionTLS12 + } + } + return uint16(minVersion) +} + +func decryptClientKey(clientKey, clientKeyPassword string) ([]byte, error) { + block, _ := pem.Decode([]byte(clientKey)) + + key, err := pkcs8.ParsePKCS8PrivateKey(block.Bytes, []byte(clientKeyPassword)) + if err != nil { + return nil, err + } + + pemData, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, err + } + + var pemPrivateBlock = &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: pemData, + } + + encodedData := pem.EncodeToMemory(pemPrivateBlock) + + return encodedData, nil } diff --git a/pkg/util/tls_config_test.go b/pkg/util/tls_config_test.go index b75bf776b8b..18b8513ef1a 100644 --- a/pkg/util/tls_config_test.go +++ b/pkg/util/tls_config_test.go @@ -1,9 +1,29 @@ +/* +Copyright 2023 The KEDA Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package util import ( + "crypto/tls" "crypto/x509" + "os" "strings" "testing" + + "github.com/go-logr/logr" ) var randomCACert = `-----BEGIN CERTIFICATE----- @@ -136,7 +156,7 @@ func TestNewTLSConfig_WithoutPassword(t *testing.T) { } for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - config, err := NewTLSConfig(test.cert, test.key, test.CACert) + config, err := NewTLSConfig(test.cert, test.key, test.CACert, false) if err != nil { t.Errorf("Should have no error %s", err) } @@ -146,7 +166,7 @@ func TestNewTLSConfig_WithoutPassword(t *testing.T) { } if test.CACert != "" { - caCertPool := x509.NewCertPool() + caCertPool := getRootCAs() caCertPool.AppendCertsFromPEM([]byte(randomCACert)) if !config.RootCAs.Equal(caCertPool) { t.Errorf("TLS config return different CA cert") @@ -203,7 +223,7 @@ func TestNewTLSConfig_WithPassword(t *testing.T) { } for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - config, err := NewTLSConfigWithPassword(test.cert, test.key, test.password, test.CACert) + config, err := NewTLSConfigWithPassword(test.cert, test.key, test.password, test.CACert, false) if err != nil { t.Errorf("Should have no error: %s", err) } @@ -213,7 +233,7 @@ func TestNewTLSConfig_WithPassword(t *testing.T) { } if test.CACert != "" { - caCertPool := x509.NewCertPool() + caCertPool := getRootCAs() caCertPool.AppendCertsFromPEM([]byte(randomCACert)) if !config.RootCAs.Equal(caCertPool) { t.Errorf("TLS config return different CA cert") @@ -225,3 +245,51 @@ func TestNewTLSConfig_WithPassword(t *testing.T) { }) } } + +type minTLSVersionTestData struct { + envSet bool + envValue string + expectedVersion uint16 +} + +var minTLSVersionTestDatas = []minTLSVersionTestData{ + { + envSet: true, + envValue: "TLS10", + expectedVersion: tls.VersionTLS10, + }, + { + envSet: true, + envValue: "TLS11", + expectedVersion: tls.VersionTLS11, + }, + { + envSet: true, + envValue: "TLS12", + expectedVersion: tls.VersionTLS12, + }, + { + envSet: true, + envValue: "TLS13", + expectedVersion: tls.VersionTLS13, + }, + { + envSet: false, + expectedVersion: tls.VersionTLS12, + }, +} + +func TestResolveMinTLSVersion(t *testing.T) { + defer os.Unsetenv("KEDA_HTTP_MIN_TLS_VERSION") + for _, testData := range minTLSVersionTestDatas { + os.Unsetenv("KEDA_HTTP_MIN_TLS_VERSION") + if testData.envSet { + os.Setenv("KEDA_HTTP_MIN_TLS_VERSION", testData.envValue) + } + minVersion := initMinTLSVersion(logr.Discard()) + + if testData.expectedVersion != minVersion { + t.Error("Failed to resolve minTLSVersion correctly", "wants", testData.expectedVersion, "got", minVersion) + } + } +} diff --git a/tests/helper/helper.go b/tests/helper/helper.go index 0b23f647923..35774d2df5b 100644 --- a/tests/helper/helper.go +++ b/tests/helper/helper.go @@ -6,9 +6,16 @@ package helper import ( "bytes" "context" + cryptoRand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "fmt" "io" + "math/big" "math/rand" + "net" "os" "os/exec" "regexp" @@ -19,6 +26,7 @@ import ( "github.com/joho/godotenv" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -49,6 +57,11 @@ const ( StringTrue = "true" ) +const ( + caCrtPath = "/tmp/keda-e2e-ca.crt" + caKeyPath = "/tmp/keda-e2e-ca.key" +) + var _ = godotenv.Load() var random = rand.New(rand.NewSource(time.Now().UnixNano())) @@ -619,3 +632,132 @@ func WaitForPodsTerminated(t *testing.T, kc *kubernetes.Clientset, selector, nam return false } + +func GetTestCA(t *testing.T) ([]byte, []byte) { + generateCA(t) + caCrt, err := os.ReadFile(caCrtPath) + require.NoErrorf(t, err, "error reading custom CA crt - %s", err) + caKey, err := os.ReadFile(caKeyPath) + require.NoErrorf(t, err, "error reading custom CA key - %s", err) + return caCrt, caKey +} + +func GenerateServerCert(t *testing.T, domain string) (string, string) { + cert := &x509.Certificate{ + SerialNumber: big.NewInt(2019), + Subject: pkix.Name{ + Organization: []string{"Company, INC."}, + Country: []string{"US"}, + Province: []string{""}, + Locality: []string{"San Francisco"}, + StreetAddress: []string{"Golden Gate Bridge"}, + PostalCode: []string{"94016"}, + }, + DNSNames: []string{ + domain, + }, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + SubjectKeyId: []byte{1, 2, 3, 4, 6}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + + certPrivKey, err := rsa.GenerateKey(cryptoRand.Reader, 4096) + require.NoErrorf(t, err, "error generating tls key - %s", err) + + caCrtBytes, caKeyBytes := GetTestCA(t) + block, _ := pem.Decode(caCrtBytes) + if block == nil { + t.Fail() + return "", "" + } + ca, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fail() + return "", "" + } + blockKey, _ := pem.Decode(caKeyBytes) + if blockKey == nil { + t.Fail() + return "", "" + } + caKey, err := x509.ParsePKCS1PrivateKey(blockKey.Bytes) + require.NoErrorf(t, err, "error reading custom CA key - %s", err) + certBytes, err := x509.CreateCertificate(cryptoRand.Reader, cert, ca, &certPrivKey.PublicKey, caKey) + require.NoErrorf(t, err, "error creating tls cert - %s", err) + + certPEM := new(bytes.Buffer) + err = pem.Encode(certPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + require.NoErrorf(t, err, "error encoding cert - %s", err) + + certPrivKeyPEM := new(bytes.Buffer) + err = pem.Encode(certPrivKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), + }) + require.NoErrorf(t, err, "error encoding key - %s", err) + + return certPEM.String(), certPrivKeyPEM.String() +} + +func generateCA(t *testing.T) { + _, err := os.Stat(caCrtPath) + if err == nil { + return + } + ca := &x509.Certificate{ + SerialNumber: big.NewInt(2019), + Subject: pkix.Name{ + Organization: []string{"Company, INC."}, + Country: []string{"US"}, + Province: []string{""}, + Locality: []string{"San Francisco"}, + StreetAddress: []string{"Golden Gate Bridge"}, + PostalCode: []string{"94016"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + // create our private and public key + caPrivKey, err := rsa.GenerateKey(cryptoRand.Reader, 4096) + require.NoErrorf(t, err, "error generating custom CA key - %s", err) + + // create the CA + caBytes, err := x509.CreateCertificate(cryptoRand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey) + require.NoErrorf(t, err, "error generating custom CA - %s", err) + + // pem encode + crtFile, err := os.OpenFile(caCrtPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + require.NoErrorf(t, err, "error opening custom CA file - %s", err) + err = pem.Encode(crtFile, &pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) + require.NoErrorf(t, err, "error encoding ca - %s", err) + if err := crtFile.Close(); err != nil { + require.NoErrorf(t, err, "error closing custom CA file - %s", err) + } + + keyFile, err := os.OpenFile(caKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + require.NoErrorf(t, err, "error opening custom CA key file- %s", err) + } + err = pem.Encode(keyFile, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(caPrivKey), + }) + require.NoErrorf(t, err, "error encoding CA key - %s", err) + if err := keyFile.Close(); err != nil { + require.NoErrorf(t, err, "error closing custom CA key file- %s", err) + } +} diff --git a/tests/internals/global_custom_ca/global_custom_ca_test.go b/tests/internals/global_custom_ca/global_custom_ca_test.go new file mode 100644 index 00000000000..a48c6f1d7b3 --- /dev/null +++ b/tests/internals/global_custom_ca/global_custom_ca_test.go @@ -0,0 +1,284 @@ +//go:build e2e +// +build e2e + +package global_custom_ca_test + +import ( + "encoding/base64" + "fmt" + "testing" + + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/keda/v2/tests/helper" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "global-custom-ca-test" +) + +var ( + testNamespace = fmt.Sprintf("%s-ns", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + metricsServerDeploymentName = fmt.Sprintf("%s-metrics-server", testName) + servciceName = fmt.Sprintf("%s-service", testName) + triggerAuthName = fmt.Sprintf("%s-ta", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + secretName = fmt.Sprintf("%s-secret", testName) + metricsServerEndpoint = fmt.Sprintf("http://%s.%s.svc.cluster.local:8080/api/value", servciceName, testNamespace) + metricsServerHTTPSEndpoint = fmt.Sprintf("https://%s.%s.svc.cluster.local:4333/api/value", servciceName, testNamespace) + minReplicaCount = 0 + maxReplicaCount = 2 +) + +type templateData struct { + TestNamespace string + DeploymentName string + MetricsServerDeploymentName string + MetricsServerEndpoint string + MetricsServerHTTPSEndpoint string + ServciceName string + ScaledObjectName string + TriggerAuthName string + TLSCertificate string + TLSKey string + SecretName string + MetricValue int + MinReplicaCount string + MaxReplicaCount string +} + +const ( + secretTemplate = `apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +data: + AUTH_PASSWORD: U0VDUkVUCg== + AUTH_USERNAME: VVNFUgo= +` + + tlsSecretTemplate = `apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}}-tls + namespace: {{.TestNamespace}} +data: + tls.crt: {{.TLSCertificate}} + tls.key: {{.TLSKey}} +` + + triggerAuthenticationTemplate = `apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthName}} + namespace: {{.TestNamespace}} +spec: + secretTargetRef: + - parameter: username + name: {{.SecretName}} + key: AUTH_USERNAME + - parameter: password + name: {{.SecretName}} + key: AUTH_PASSWORD +` + + metricsServerdeploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.MetricsServerDeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.MetricsServerDeploymentName}} +spec: + replicas: 1 + selector: + matchLabels: + app: {{.MetricsServerDeploymentName}} + template: + metadata: + labels: + app: {{.MetricsServerDeploymentName}} + spec: + volumes: + - name: certificates + secret: + defaultMode: 420 + secretName: {{.SecretName}}-tls + containers: + - name: metrics + image: ghcr.io/kedacore/tests-metrics-api + ports: + - containerPort: 8080 + name: http + - containerPort: 4333 + name: https + envFrom: + - secretRef: + name: {{.SecretName}} + env: + - name: USE_TLS + value: "true" + volumeMounts: + - mountPath: /certs + name: certificates + readOnly: true + imagePullPolicy: Always +` + + serviceTemplate = ` +apiVersion: v1 +kind: Service +metadata: + name: {{.ServciceName}} + namespace: {{.TestNamespace}} +spec: + selector: + app: {{.MetricsServerDeploymentName}} + ports: + - port: 8080 + targetPort: 8080 + name: http + - port: 4333 + targetPort: 4333 + name: https +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: {{.DeploymentName}} + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} +spec: + selector: + matchLabels: + app: {{.DeploymentName}} + replicas: 0 + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +` + + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + minReplicaCount: {{.MinReplicaCount}} + maxReplicaCount: {{.MaxReplicaCount}} + cooldownPeriod: 1 + triggers: + - type: metrics-api + metadata: + targetValue: "5" + url: "{{.MetricsServerHTTPSEndpoint}}" + valueLocation: 'value' + authMode: "basic" + method: "query" + authenticationRef: + name: {{.TriggerAuthName}} +` + updateMetricTemplate = `apiVersion: batch/v1 +kind: Job +metadata: + name: update-metric-value + namespace: {{.TestNamespace}} +spec: + ttlSecondsAfterFinished: 0 + template: + spec: + containers: + - name: curl-client + image: curlimages/curl + imagePullPolicy: Always + command: ["curl", "-X", "POST", "{{.MetricsServerEndpoint}}/{{.MetricValue}}"] + restartPolicy: Never` +) + +func TestCustomCa(t *testing.T) { + // setup + t.Log("--- setting up ---") + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData(t) + CreateKubernetesResources(t, kc, testNamespace, data, templates) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 180, 3), + "replica count should be %d after 3 minutes", minReplicaCount) + + // test scaling + testScaleOut(t, kc, data) + testScaleIn(t, kc, data) + + // cleanup + DeleteKubernetesResources(t, kc, testNamespace, data, templates) +} + +func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing scale out ---") + data.MetricValue = 50 + KubectlApplyWithTemplate(t, data, "updateMetricTemplate", updateMetricTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 60, 3), + "replica count should be %d after 3 minutes", maxReplicaCount) +} + +func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing scale in ---") + data.MetricValue = 0 + KubectlApplyWithTemplate(t, data, "updateMetricTemplate", updateMetricTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3), + "replica count should be %d after 3 minutes", minReplicaCount) +} + +func getTemplateData(t *testing.T) (templateData, []Template) { + tlsCrt, TLSKey := GenerateServerCert(t, fmt.Sprintf("%s.%s.svc.cluster.local", servciceName, testNamespace)) + return templateData{ + TestNamespace: testNamespace, + DeploymentName: deploymentName, + MetricsServerDeploymentName: metricsServerDeploymentName, + ServciceName: servciceName, + TriggerAuthName: triggerAuthName, + ScaledObjectName: scaledObjectName, + SecretName: secretName, + MetricsServerEndpoint: metricsServerEndpoint, + MetricsServerHTTPSEndpoint: metricsServerHTTPSEndpoint, + MinReplicaCount: fmt.Sprintf("%v", minReplicaCount), + MaxReplicaCount: fmt.Sprintf("%v", maxReplicaCount), + TLSCertificate: base64.StdEncoding.EncodeToString([]byte(tlsCrt)), + TLSKey: base64.StdEncoding.EncodeToString([]byte(TLSKey)), + MetricValue: 0, + }, []Template{ + {Name: "secretTemplate", Config: secretTemplate}, + {Name: "metricsServerdeploymentTemplate", Config: metricsServerdeploymentTemplate}, + {Name: "serviceTemplate", Config: serviceTemplate}, + {Name: "tlsSecretTemplate", Config: tlsSecretTemplate}, + {Name: "triggerAuthenticationTemplate", Config: triggerAuthenticationTemplate}, + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, + } +} diff --git a/tests/utils/setup_test.go b/tests/utils/setup_test.go index 718c8ca26ec..1d4da933756 100644 --- a/tests/utils/setup_test.go +++ b/tests/utils/setup_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/kedacore/keda/v2/tests/helper" @@ -166,6 +167,23 @@ func TestSetupGcpIdentityComponents(t *testing.T) { } func TestDeployKEDA(t *testing.T) { + KubeClient = GetKubernetesClient(t) + CreateNamespace(t, KubeClient, KEDANamespace) + + caCtr, _ := GetTestCA(t) + secret := &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: "custom-cas", + Namespace: KEDANamespace, + }, + StringData: map[string]string{ + "test-ca.crt": string(caCtr), + }, + } + + _, err := KubeClient.CoreV1().Secrets(KEDANamespace).Create(context.Background(), secret, v1.CreateOptions{}) + require.NoErrorf(t, err, "error deploying custom CA - %s", err) + out, err := ExecuteCommandWithDir("make deploy", "../..") require.NoErrorf(t, err, "error deploying KEDA - %s", err)