From 4f10d863e4fa37abbc382199791bf14052883760 Mon Sep 17 00:00:00 2001 From: keegancwinchester Date: Fri, 4 Nov 2022 08:26:58 -0500 Subject: [PATCH] trying push again --- CHANGELOG.md | 123 ++- apis/keda/v1alpha1/scaledjob_types.go | 29 +- config/crd/bases/keda.sh_scaledjobs.yaml | 1106 +++++++++++----------- controllers/keda/hpa.go | 35 +- controllers/keda/scaledjob_controller.go | 155 ++- controllers/keda/util/predicate.go | 8 + pkg/scaling/executor/scale_jobs.go | 25 +- 7 files changed, 904 insertions(+), 577 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8900ec5ed5b..89f7e354463 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,13 @@ To learn more about our roadmap, we recommend reading [this document](ROADMAP.md ## Deprecations +To learn more about active deprecations, we recommend checking [GitHub Discussions](https://github.com/kedacore/keda/discussions/categories/deprecations). + ## History - [Unreleased](#unreleased) +- [v2.8.1](#v281) +- [v2.8.0](#v280) - [v2.7.1](#v271) - [v2.7.0](#v270) - [v2.6.1](#v261) @@ -32,51 +36,134 @@ To learn more about our roadmap, we recommend reading [this document](ROADMAP.md - [v1.1.0](#v110) - [v1.0.0](#v100) -## Unreleased +### New +- **General:** Introduce autoscaling is paused annotation for ScaledJobs ([#3303](https://github.com/kedacore/keda/issues/3303)) +- **General**: Expand Prometheus metric with label "ScalerName" to distinguish different triggers. The scaleName is defined per Trigger.Name ([#3588](https://github.com/kedacore/keda/issues/3588) +- **General:** Introduce new Loki Scaler ([#3699](https://github.com/kedacore/keda/issues/3699)) +- **General**: Add ratelimitting parameters to keda manager to allow override of client defaults ([#3730](https://github.com/kedacore/keda/issues/2920)) +- **General**: Provide Prometheus metric with indication of total number of triggers per trigger type in `ScaledJob`/`ScaledObject`. ([#3663](https://github.com/kedacore/keda/issues/3663)) +- **AWS Scalers**: Add setting AWS endpoint url. ([#3337](https://github.com/kedacore/keda/issues/3337)) +- **Azure Service Bus Scaler**: Add support for Shared Access Signature (SAS) tokens for authentication. ([#2920](https://github.com/kedacore/keda/issues/2920)) +- **Azure Service Bus Scaler:** Support regex usage in queueName / subscriptionName parameters. ([#1624](https://github.com/kedacore/keda/issues/1624)) +- **Selenium Grid Scaler:** Allow setting url trigger parameter from TriggerAuthentication/ClusterTriggerAuthentication ([#3752](https://github.com/kedacore/keda/pull/3752)) + +### Improvements + +- **General:** Add explicit seccompProfile type to securityContext config ([#3561](https://github.com/kedacore/keda/issues/3561)) +- **General:** Add `Min` column to ScaledJob visualization ([#3689](https://github.com/kedacore/keda/issues/3689)) +- **Apache Kafka Scaler:** SASL/OAuthbearer Implementation ([#3681](https://github.com/kedacore/keda/issues/3681)) +- **Azure AD Pod Identity Authentication:** Improve error messages to emphasize problems around the integration with aad-pod-identity itself ([#3610](https://github.com/kedacore/keda/issues/3610)) +- **Azure Pipelines Scaler:** Improved speed of profiling large set of Job Requests from Azure Pipelines ([#3702](https://github.com/kedacore/keda/issues/3702)) +- **Prometheus Scaler:** Introduce skipping of certificate check for unsigned certs ([#2310](https://github.com/kedacore/keda/issues/2310)) + +### Fixes + +- **General:** Provide patch for CVE-2022-3172 vulnerability ([#3690](https://github.com/kedacore/keda/issues/3690)) +- **General:** Respect optional parameter inside envs for ScaledJobs ([#3568](https://github.com/kedacore/keda/issues/3568)) + +### Deprecations + +- TODO ([#XXX](https://github.com/kedacore/keda/issue/XXX)) + +### Breaking Changes + +- **General:** Change API version of HPA from `autoscaling/v2beta2` to `autoscaling/v2` ([#2462](https://github.com/kedacore/keda/issues/2462)) + +### Other + +- **General**: Bump Golang to 1.18.6 ([#3205](https://github.com/kedacore/keda/issues/3205)) +- **General**: Bump `github.com/Azure/azure-event-hubs-go/v3` ([#2986](https://github.com/kedacore/keda/issues/2986)) +- **General**: Migrate from `azure-service-bus-go` to `azservicebus` ([#3394](https://github.com/kedacore/keda/issues/3394)) +- **Azure EventHub**: Add e2e tests ([#2792](https://github.com/kedacore/keda/issues/2792)) + +## v2.8.1 + +### New + +None. + +### Improvements + +- **Datadog Scaler:** Support multi-query metrics, and aggregation ([#3423](https://github.com/kedacore/keda/issues/3423)) + +### Fixes + +- **General:** Metrics endpoint returns correct HPA values ([#3554](https://github.com/kedacore/keda/issues/3554)) +- **Datadog Scaler:** Fix: panic in datadog scaler ([#3448](https://github.com/kedacore/keda/issues/3448)) +- **RabbitMQ Scaler:** Parse vhost correctly if it's provided in the host url ([#3602](https://github.com/kedacore/keda/issues/3602)) + +### Deprecations + +None. + +### Breaking Changes + +None. + +### Other + +- **General:** Execute trivy scan (on PRs) only if there are changes in deps ([#3540](https://github.com/kedacore/keda/issues/3540)) +- **General:** Use re-usable workflows for GitHub Actions ([#2569](https://github.com/kedacore/keda/issues/2569)) + +## v2.8.0 ### New -- **General:** Add support to customize HPA name ([3057](https://github.com/kedacore/keda/issues/3057)) -- **General:** Basic setup for migrating e2e tests to Go. ([#2737](https://github.com/kedacore/keda/issues/2737)) - **General:** Introduce new AWS DynamoDB Streams Scaler ([#3124](https://github.com/kedacore/keda/issues/3124)) +- **General:** Introduce new NATS JetStream scaler ([#2391](https://github.com/kedacore/keda/issues/2391)) +- **General:** Introduce `activationThreshold`/`minMetricValue` for all scalers ([#2800](https://github.com/kedacore/keda/issues/2800)) +- **General:** Support for `minReplicaCount` in ScaledJob ([#3426](https://github.com/kedacore/keda/issues/3426)) +- **General:** Support to customize HPA name ([#3057](https://github.com/kedacore/keda/issues/3057)) +- **General:** Make propagation policy for ScaledJob rollout configurable ([#2910](https://github.com/kedacore/keda/issues/2910)) - **General:** Support for Azure AD Workload Identity as a pod identity provider. ([#2487](https://github.com/kedacore/keda/issues/2487)|[#2656](https://github.com/kedacore/keda/issues/2656)) - **General:** Support for permission segregation when using Azure AD Pod / Workload Identity. ([#2656](https://github.com/kedacore/keda/issues/2656)) +- **AWS SQS Queue Scaler:** Support for scaling to include in-flight messages. ([#3133](https://github.com/kedacore/keda/issues/3133)) +- **Azure Pipelines Scaler:** Support for Azure Pipelines to support demands (capabilities) ([#2328](https://github.com/kedacore/keda/issues/2328)) +- **CPU Scaler:** Support for targeting specific container in a pod ([#1378](https://github.com/kedacore/keda/issues/1378)) +- **GCP Stackdriver Scaler:** Added aggregation parameters ([#3008](https://github.com/kedacore/keda/issues/3008)) +- **Kafka Scaler:** Support of passphrase encrypted PKCS #\8 private key ([3449](https://github.com/kedacore/keda/issues/3449)) +- **Memory Scaler:** Support for targeting specific container in a pod ([#1378](https://github.com/kedacore/keda/issues/1378)) +- **Prometheus Scaler:** Add `ignoreNullValues` to return error when prometheus return null in values ([#3065](https://github.com/kedacore/keda/issues/3065)) ### Improvements +- **General:** Add settings for configuring leader election ([#2836](https://github.com/kedacore/keda/issues/2836)) - **General:** `external` extension reduces connection establishment with long links ([#3193](https://github.com/kedacore/keda/issues/3193)) -- **General:** Use `mili` scale for the returned metrics ([#3135](https://github.com/kedacore/keda/issue/3135)) -- **General:** Use more readable timestamps in KEDA Operator logs ([#3066](https://github.com/kedacore/keda/issue/3066)) -- **AWS SQS Queue Scaler:** Support for scaling to include in-flight messages. ([#3133](https://github.com/kedacore/keda/issues/3133)) -- **GCP Stackdriver Scaler:** Added aggregation parameters ([#3008](https://github.com/kedacore/keda/issues/3008)) -- **Prometheus Scaler:** Add ignoreNullValues to return error when prometheus return null in values ([#3065](https://github.com/kedacore/keda/issues/3065)) -- **Selenium Grid Scaler:** Edge active sessions not being properly counted ([#2709](https://github.com/kedacore/keda/issues/2709)) -- **Selenium Grid Scaler:** Max Sessions implementation issue ([#3061](https://github.com/kedacore/keda/issues/3061)) +- **General:** Reference ScaledObject's/ScaledJob's name in the scalers log ([3419](https://github.com/kedacore/keda/issues/3419)) +- **General:** Use `mili` scale for the returned metrics ([#3135](https://github.com/kedacore/keda/issues/3135)) +- **General:** Use more readable timestamps in KEDA Operator logs ([#3066](https://github.com/kedacore/keda/issues/3066)) +- **Kafka Scaler:** Handle Sarama errors properly ([#3056](https://github.com/kedacore/keda/issues/3056)) ### Fixes +- **General:** Provide patch for CVE-2022-27191 vulnerability ([#3378](https://github.com/kedacore/keda/issues/3378)) - **General:** Refactor adapter startup to ensure proper log initilization. ([2316](https://github.com/kedacore/keda/issues/2316)) -- **General:** Scaleobject ready condition 'False/Unknow' to 'True' requeue([#3096](https://github.com/kedacore/keda/issues/3096)) -- **General:** Use metricName from GetMetricsSpec in ScaledJobs instead of `queueLength` ([#3032](https://github.com/kedacore/keda/issue/3032)) +- **General:** Scaleobject ready condition 'False/Unknow' to 'True' requeue ([#3096](https://github.com/kedacore/keda/issues/3096)) +- **General:** Use `go install` in the Makefile for downloading dependencies ([#2916](https://github.com/kedacore/keda/issues/2916)) +- **General:** Use metricName from GetMetricsSpec in ScaledJobs instead of `queueLength` ([#3032](https://github.com/kedacore/keda/issues/3032)) - **ActiveMQ Scaler:** KEDA doesn't respect restAPITemplate ([#3188](https://github.com/kedacore/keda/issues/3188)) - **Azure Eventhub Scaler:** KEDA operator crashes on nil memory panic if the eventhub connectionstring for Azure Eventhub Scaler contains an invalid character ([#3082](https://github.com/kedacore/keda/issues/3082)) - **Azure Pipelines Scaler:** Fix issue with Azure Pipelines wrong PAT Auth. ([#3159](https://github.com/kedacore/keda/issues/3159)) - +- **Datadog Scaler:** Ensure that returns the same element that has been checked ([#3448](https://github.com/kedacore/keda/issues/3448)) +- **Kafka Scaler:** Check `lagThreshold` is a positive number ([#3366](https://github.com/kedacore/keda/issues/3366)) +- **Selenium Grid Scaler:** Fix bug where edge active sessions not being properly counted ([#2709](https://github.com/kedacore/keda/issues/2709)) +- **Selenium Grid Scaler:** Fix bug where Max Sessions was not working correctly ([#3061](https://github.com/kedacore/keda/issues/3061)) ### Deprecations -- TODO ([#XXX](https://github.com/kedacore/keda/issue/XXX)) +- **ScaledJob**: `rolloutStrategy` is deprecated in favor of `rollout.strategy` ([#2910](https://github.com/kedacore/keda/issues/2910)) ### Breaking Changes -- TODO ([#XXX](https://github.com/kedacore/keda/issue/XXX)) +None. ### Other +- **General:** Migrate e2e test to Go. ([2737](https://github.com/kedacore/keda/issues/2737)) +- **General**: Bump Golang to 1.17.13 and deps ([#3447](https://github.com/kedacore/keda/issues/3447)) - **General:** Fix devcontainer on ARM64 Arch. ([3084](https://github.com/kedacore/keda/issues/3084)) - **General:** Improve error message in resolving ServiceAccount for AWS EKS PodIdentity ([3142](https://github.com/kedacore/keda/issues/3142)) -- **General:** Improve e2e on PR process. ([3004](https://github.com/kedacore/keda/issues/3004)) -- **General:** Migrate e2e test to Go. ([2737](https://github.com/kedacore/keda/issues/2737)) +- **General:** Improve e2e on PR process through comments. ([3004](https://github.com/kedacore/keda/issues/3004)) - **General:** Split e2e test by functionality. ([#3270](https://github.com/kedacore/keda/issues/3270)) - **General:** Unify the used tooling on different workflows and arch. ([3092](https://github.com/kedacore/keda/issues/3092)) - **General:** Use Github's Checks API for e2e tests on PR. ([2567](https://github.com/kedacore/keda/issues/2567)) @@ -89,7 +176,7 @@ To learn more about our roadmap, we recommend reading [this document](ROADMAP.md ### Other -- **General**: Fix CVE-2022-21221 in `github.com/valyala/fasthttp` ([#2775](https://github.com/kedacore/keda/issue/2775)) +- **General**: Fix CVE-2022-21221 in `github.com/valyala/fasthttp` ([#2775](https://github.com/kedacore/keda/issues/2775)) - **General**: Bump Golang to 1.17.9 ([#3016](https://github.com/kedacore/keda/issues/3016)) - **General**: Fix autoscaling behaviour while paused. ([#3009](https://github.com/kedacore/keda/issues/3009)) diff --git a/apis/keda/v1alpha1/scaledjob_types.go b/apis/keda/v1alpha1/scaledjob_types.go index df867eb2e04..5ea00d4882b 100644 --- a/apis/keda/v1alpha1/scaledjob_types.go +++ b/apis/keda/v1alpha1/scaledjob_types.go @@ -25,6 +25,7 @@ import ( // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:path=scaledjobs,scope=Namespaced,shortName=sj +// +kubebuilder:printcolumn:name="Min",type="integer",JSONPath=".spec.minReplicaCount" // +kubebuilder:printcolumn:name="Max",type="integer",JSONPath=".spec.maxReplicaCount" // +kubebuilder:printcolumn:name="Triggers",type="string",JSONPath=".spec.triggers[*].type" // +kubebuilder:printcolumn:name="Authentication",type="string",JSONPath=".spec.triggers[*].authenticationRef.name" @@ -53,8 +54,12 @@ type ScaledJobSpec struct { // +optional RolloutStrategy string `json:"rolloutStrategy,omitempty"` // +optional + Rollout Rollout `json:"rollout,omitempty"` + // +optional EnvSourceContainerName string `json:"envSourceContainerName,omitempty"` // +optional + MinReplicaCount *int32 `json:"minReplicaCount,omitempty"` + // +optional MaxReplicaCount *int32 `json:"maxReplicaCount,omitempty"` // +optional ScalingStrategy ScalingStrategy `json:"scalingStrategy,omitempty"` @@ -68,6 +73,8 @@ type ScaledJobStatus struct { LastActiveTime *metav1.Time `json:"lastActiveTime,omitempty"` // +optional Conditions Conditions `json:"conditions,omitempty"` + // +optional + Paused string `json:"paused,omitempty"` } // ScaledJobList contains a list of ScaledJob @@ -93,6 +100,15 @@ type ScalingStrategy struct { MultipleScalersCalculation string `json:"multipleScalersCalculation,omitempty"` } +// Rollout defines the strategy for job rollouts +// +optional +type Rollout struct { + // +optional + Strategy string `json:"strategy,omitempty"` + // +optional + PropagationPolicy string `json:"propagationPolicy,omitempty"` +} + func init() { SchemeBuilder.Register(&ScaledJob{}, &ScaledJobList{}) } @@ -100,8 +116,19 @@ func init() { // MaxReplicaCount returns MaxReplicaCount func (s ScaledJob) MaxReplicaCount() int64 { if s.Spec.MaxReplicaCount != nil { - return int64(*s.Spec.MaxReplicaCount) + return int64(*s.Spec.MaxReplicaCount) - s.MinReplicaCount() } return 100 } + +// MinReplicaCount returns MinReplicaCount +func (s ScaledJob) MinReplicaCount() int64 { + if s.Spec.MinReplicaCount != nil { + if *s.Spec.MinReplicaCount > *s.Spec.MaxReplicaCount { + return int64(*s.Spec.MaxReplicaCount) + } + return int64(*s.Spec.MinReplicaCount) + } + return 0 +} diff --git a/config/crd/bases/keda.sh_scaledjobs.yaml b/config/crd/bases/keda.sh_scaledjobs.yaml index 2f73c3b175d..dcff8f19746 100644 --- a/config/crd/bases/keda.sh_scaledjobs.yaml +++ b/config/crd/bases/keda.sh_scaledjobs.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.6.1 + controller-gen.kubebuilder.io/version: v0.9.0 creationTimestamp: null name: scaledjobs.keda.sh spec: @@ -19,6 +18,9 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: + - jsonPath: .spec.minReplicaCount + name: Min + type: integer - jsonPath: .spec.maxReplicaCount name: Max type: integer @@ -91,10 +93,11 @@ spec: completed Pod for each index. When value is `Indexed`, .spec.completions must be specified and `.spec.parallelism` must be less than or equal to 10^5. In addition, The Pod name takes the form `$(job-name)-$(index)-$(random-string)`, - the Pod hostname takes the form `$(job-name)-$(index)`. \n This - field is beta-level. More completion modes can be added in the - future. If the Job controller observes a mode that it doesn't - recognize, the controller skips updates for the Job." + the Pod hostname takes the form `$(job-name)-$(index)`. \n More + completion modes can be added in the future. If the Job controller + observes a mode that it doesn't recognize, which is possible + during upgrades due to version skew, the controller skips updates + for the Job." type: string completions: description: 'Specifies the desired number of successfully finished @@ -172,7 +175,7 @@ spec: type: object type: object suspend: - description: "Suspend specifies whether the Job controller should + description: Suspend specifies whether the Job controller should create Pods or not. If a Job is created with suspend set to true, no Pods are created by the Job controller. If a Job is suspended after creation (i.e. the flag goes from false to true), @@ -180,8 +183,7 @@ spec: this Job. Users must design their workload to gracefully handle this. Suspending a Job will reset the StartTime field of the Job, effectively resetting the ActiveDeadlineSeconds timer too. - Defaults to false. \n This field is beta-level, gated by SuspendJob - feature flag (enabled by default)." + Defaults to false. type: boolean template: description: 'Describes the pod that will be created when executing @@ -531,10 +533,7 @@ spec: field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) - matches all namespaces. This field - is beta-level and is only honored - when PodAffinityNamespaceSelector - feature is enabled. + matches all namespaces. properties: matchExpressions: description: matchExpressions is @@ -600,7 +599,7 @@ spec: in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector - means "this pod's namespace" + means "this pod's namespace". items: type: string type: array @@ -716,9 +715,6 @@ spec: null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level and is only honored - when PodAffinityNamespaceSelector feature - is enabled. properties: matchExpressions: description: matchExpressions is a list @@ -778,7 +774,7 @@ spec: union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null - namespaceSelector means "this pod's namespace" + namespaceSelector means "this pod's namespace". items: type: string type: array @@ -896,10 +892,7 @@ spec: field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) - matches all namespaces. This field - is beta-level and is only honored - when PodAffinityNamespaceSelector - feature is enabled. + matches all namespaces. properties: matchExpressions: description: matchExpressions is @@ -965,7 +958,7 @@ spec: in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector - means "this pod's namespace" + means "this pod's namespace". items: type: string type: array @@ -1081,9 +1074,6 @@ spec: null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level and is only honored - when PodAffinityNamespaceSelector feature - is enabled. properties: matchExpressions: description: matchExpressions is a list @@ -1143,7 +1133,7 @@ spec: union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null - namespaceSelector means "this pod's namespace" + namespaceSelector means "this pod's namespace". items: type: string type: array @@ -1177,7 +1167,7 @@ spec: want to run within a pod. properties: args: - description: 'Arguments to the entrypoint. The docker + description: 'Arguments to the entrypoint. The container image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot @@ -1193,13 +1183,13 @@ spec: type: array command: description: 'Entrypoint array. Not executed within - a shell. The docker image''s ENTRYPOINT is used - if this is not provided. Variable references $(VAR_NAME) - are expanded using the container''s environment. - If a variable cannot be resolved, the reference - in the input string will be unchanged. Double - $$ are reduced to a single $, which allows for - escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + a shell. The container image''s ENTRYPOINT is + used if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, + the reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot @@ -1379,7 +1369,7 @@ spec: type: object type: array image: - description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' @@ -1646,7 +1636,7 @@ spec: type: integer grpc: description: GRPC specifies an action involving - a GRPC port. This is an alpha field and requires + a GRPC port. This is a beta field and requires enabling GRPCContainerProbe feature gate. properties: port: @@ -1866,7 +1856,7 @@ spec: type: integer grpc: description: GRPC specifies an action involving - a GRPC port. This is an alpha field and requires + a GRPC port. This is a beta field and requires enabling GRPCContainerProbe feature gate. properties: port: @@ -2263,7 +2253,7 @@ spec: type: integer grpc: description: GRPC specifies an action involving - a GRPC port. This is an alpha field and requires + a GRPC port. This is a beta field and requires enabling GRPCContainerProbe feature gate. properties: port: @@ -2596,10 +2586,10 @@ spec: haven't disabled the EphemeralContainers feature gate." properties: args: - description: 'Arguments to the entrypoint. The docker - image''s CMD is used if this is not provided. - Variable references $(VAR_NAME) are expanded using - the container''s environment. If a variable cannot + description: 'Arguments to the entrypoint. The image''s + CMD is used if this is not provided. Variable + references $(VAR_NAME) are expanded using the + container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) @@ -2612,8 +2602,8 @@ spec: type: array command: description: 'Entrypoint array. Not executed within - a shell. The docker image''s ENTRYPOINT is used - if this is not provided. Variable references $(VAR_NAME) + a shell. The image''s ENTRYPOINT is used if this + is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double @@ -2798,7 +2788,7 @@ spec: type: object type: array image: - description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images' + description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images' type: string imagePullPolicy: description: 'Image pull policy. One of Always, @@ -3060,7 +3050,7 @@ spec: type: integer grpc: description: GRPC specifies an action involving - a GRPC port. This is an alpha field and requires + a GRPC port. This is a beta field and requires enabling GRPCContainerProbe feature gate. properties: port: @@ -3273,7 +3263,7 @@ spec: type: integer grpc: description: GRPC specifies an action involving - a GRPC port. This is an alpha field and requires + a GRPC port. This is a beta field and requires enabling GRPCContainerProbe feature gate. properties: port: @@ -3663,7 +3653,7 @@ spec: type: integer grpc: description: GRPC specifies an action involving - a GRPC port. This is an alpha field and requires + a GRPC port. This is a beta field and requires enabling GRPCContainerProbe feature gate. properties: port: @@ -3975,9 +3965,7 @@ spec: references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. If specified, these secrets will be passed to individual puller implementations - for them to use. For example, in the case of docker, - only DockerConfig type secrets are honored. More info: - https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' + for them to use. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' items: description: LocalObjectReference contains enough information to let you locate the referenced object inside the @@ -4011,7 +3999,7 @@ spec: want to run within a pod. properties: args: - description: 'Arguments to the entrypoint. The docker + description: 'Arguments to the entrypoint. The container image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot @@ -4027,13 +4015,13 @@ spec: type: array command: description: 'Entrypoint array. Not executed within - a shell. The docker image''s ENTRYPOINT is used - if this is not provided. Variable references $(VAR_NAME) - are expanded using the container''s environment. - If a variable cannot be resolved, the reference - in the input string will be unchanged. Double - $$ are reduced to a single $, which allows for - escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + a shell. The container image''s ENTRYPOINT is + used if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, + the reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot @@ -4213,7 +4201,7 @@ spec: type: object type: array image: - description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + description: 'Container image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' @@ -4480,7 +4468,7 @@ spec: type: integer grpc: description: GRPC specifies an action involving - a GRPC port. This is an alpha field and requires + a GRPC port. This is a beta field and requires enabling GRPCContainerProbe feature gate. properties: port: @@ -4700,7 +4688,7 @@ spec: type: integer grpc: description: GRPC specifies an action involving - a GRPC port. This is an alpha field and requires + a GRPC port. This is a beta field and requires enabling GRPCContainerProbe feature gate. properties: port: @@ -5097,7 +5085,7 @@ spec: type: integer grpc: description: GRPC specifies an action involving - a GRPC port. This is an alpha field and requires + a GRPC port. This is a beta field and requires enabling GRPCContainerProbe feature gate. properties: port: @@ -5386,7 +5374,7 @@ spec: - spec.containers[*].securityContext.privileged - spec.containers[*].securityContext.allowPrivilegeEscalation - spec.containers[*].securityContext.procMount - spec.containers[*].securityContext.runAsUser - spec.containers[*].securityContext.runAsGroup This - is an alpha field and requires the IdentifyPodOS feature" + is a beta field and requires the IdentifyPodOS feature" properties: name: description: 'Name is the name of the operating system. @@ -5418,16 +5406,12 @@ spec: and selected in the PodSpec, Overhead will be set to the value defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. - More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md - This field is beta-level as of Kubernetes v1.18, and - is only honored by servers that enable the PodOverhead - feature.' + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md' type: object preemptionPolicy: description: PreemptionPolicy is the Policy for preempting pods with lower priority. One of Never, PreemptLowerPriority. - Defaults to PreemptLowerPriority if unset. This field - is beta-level, gated by the NonPreemptingPriority feature-gate. + Defaults to PreemptLowerPriority if unset. type: string priority: description: The priority value. Various system components @@ -5477,8 +5461,7 @@ spec: the named class, the pod will not be run. If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an empty definition that uses - the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class - This is a beta feature as of Kubernetes v1.14.' + the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class' type: string schedulerName: description: If specified, the pod will be dispatched @@ -5832,13 +5815,17 @@ spec: pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference between the number of matching pods in the target topology - and the global minimum. For example, in a 3-zone + and the global minimum. The global minimum is + the minimum number of matching pods in an eligible + domain or zero if the number of eligible domains + is less than MinDomains. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the - same labelSelector spread as 1/1/0: | zone1 | - zone2 | zone3 | | P | P | | - if - MaxSkew is 1, incoming pod can only be scheduled - to zone3 to become 1/1/1; scheduling it onto zone1(zone2) - would make the ActualSkew(2-0) on zone1(zone2) + same labelSelector spread as 2/2/1: In this case, + the global minimum is 1. | zone1 | zone2 | zone3 + | | P P | P P | P | - if MaxSkew is 1, + incoming pod can only be scheduled to zone3 to + become 2/2/2; scheduling it onto zone1(zone2) + would make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence to topologies @@ -5846,13 +5833,50 @@ spec: value is 1 and 0 is not allowed.' format: int32 type: integer + minDomains: + description: "MinDomains indicates a minimum number + of eligible domains. When the number of eligible + domains with matching topology keys is less than + minDomains, Pod Topology Spread treats \"global + minimum\" as 0, and then the calculation of Skew + is performed. And when the number of eligible + domains with matching topology keys equals or + greater than minDomains, this value has no effect + on scheduling. As a result, when the number of + eligible domains is less than minDomains, scheduler + won't schedule more than maxSkew Pods to those + domains. If value is nil, the constraint behaves + as if MinDomains is equal to 1. Valid values are + integers greater than 0. When value is not nil, + WhenUnsatisfiable must be DoNotSchedule. \n For + example, in a 3-zone cluster, MaxSkew is set to + 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: | zone1 | zone2 + | zone3 | | P P | P P | P P | The number + of domains is less than 5(MinDomains), so \"global + minimum\" is treated as 0. In this situation, + new pod with the same labelSelector cannot be + scheduled, because computed skew will be 3(3 - + 0) if new Pod is scheduled to any of the three + zones, it will violate MaxSkew. \n This is an + alpha field and requires enabling MinDomainsInPodTopologySpread + feature gate." + format: int32 + type: integer topologyKey: description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each as a "bucket", and try to put balanced number of pods into each bucket. - It's a required field. + We define a domain as a particular instance of + a topology. Also, we define an eligible domain + as a domain whose nodes match the node selector. + e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if + TopologyKey is "topology.kubernetes.io/zone", + each zone is a domain of that topology. It's a + required field. type: string whenUnsatisfiable: description: 'WhenUnsatisfiable indicates how to @@ -5860,8 +5884,8 @@ spec: constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any - location, but giving higher precedence to topologies - that would help reduce the skew. A constraint + location, but giving higher precedence to topologies + that would help reduce the skew. A constraint is considered "Unsatisfiable" for an incoming pod if and only if every possible node assignment for that pod would violate "MaxSkew" on some topology. @@ -5893,76 +5917,76 @@ spec: that may be accessed by any container in the pod. properties: awsElasticBlockStore: - description: 'AWSElasticBlockStore represents an + description: 'awsElasticBlockStore represents an AWS Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' properties: fsType: - description: 'Filesystem type of the volume - that you want to mount. Tip: Ensure that the - filesystem type is supported by the host operating - system. Examples: "ext4", "xfs", "ntfs". Implicitly - inferred to be "ext4" if unspecified. More - info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + description: 'fsType is the filesystem type + of the volume that you want to mount. Tip: + Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore TODO: how do we prevent errors in the filesystem from compromising the machine' type: string partition: - description: 'The partition in the volume that - you want to mount. If omitted, the default - is to mount by volume name. Examples: For - volume /dev/sda1, you specify the partition + description: 'partition is the partition in + the volume that you want to mount. If omitted, + the default is to mount by volume name. Examples: + For volume /dev/sda1, you specify the partition as "1". Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty).' format: int32 type: integer readOnly: - description: 'Specify "true" to force and set - the ReadOnly property in VolumeMounts to "true". - If omitted, the default is "false". More info: - https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + description: 'readOnly value true will force + the readOnly setting in VolumeMounts. More + info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' type: boolean volumeID: - description: 'Unique ID of the persistent disk - resource in AWS (Amazon EBS volume). More - info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + description: 'volumeID is unique ID of the persistent + disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' type: string required: - volumeID type: object azureDisk: - description: AzureDisk represents an Azure Data + description: azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. properties: cachingMode: - description: 'Host Caching mode: None, Read - Only, Read Write.' + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' type: string diskName: - description: The Name of the data disk in the - blob storage + description: diskName is the Name of the data + disk in the blob storage type: string diskURI: - description: The URI the data disk in the blob - storage + description: diskURI is the URI of data disk + in the blob storage type: string fsType: - description: Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - Implicitly inferred to be "ext4" if unspecified. + description: fsType is Filesystem type to mount. + Must be a filesystem type supported by the + host operating system. Ex. "ext4", "xfs", + "ntfs". Implicitly inferred to be "ext4" if + unspecified. type: string kind: - description: 'Expected values Shared: multiple - blob disks per storage account Dedicated: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' type: string readOnly: - description: Defaults to false (read/write). + description: readOnly Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean @@ -5971,55 +5995,58 @@ spec: - diskURI type: object azureFile: - description: AzureFile represents an Azure File + description: azureFile represents an Azure File Service mount on the host and bind mount to the pod. properties: readOnly: - description: Defaults to false (read/write). + description: readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean secretName: - description: the name of secret that contains - Azure Storage Account Name and Key + description: secretName is the name of secret + that contains Azure Storage Account Name and + Key type: string shareName: - description: Share Name + description: shareName is the azure share Name type: string required: - secretName - shareName type: object cephfs: - description: CephFS represents a Ceph FS mount on + description: cephFS represents a Ceph FS mount on the host that shares a pod's lifetime properties: monitors: - description: 'Required: Monitors is a collection - of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'monitors is Required: Monitors + is a collection of Ceph monitors More info: + https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' items: type: string type: array path: - description: 'Optional: Used as the mounted - root, rather than the full Ceph tree, default - is /' + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph tree, + default is /' type: string readOnly: - description: 'Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'readOnly is Optional: Defaults + to false (read/write). ReadOnly here will + force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: boolean secretFile: - description: 'Optional: SecretFile is the path - to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'secretFile is Optional: SecretFile + is the path to key ring for User, default + is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: string secretRef: - description: 'Optional: SecretRef is reference - to the authentication secret for User, default - is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'secretRef is Optional: SecretRef + is reference to the authentication secret + for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' properties: name: description: 'Name of the referent. More @@ -6029,32 +6056,34 @@ spec: type: string type: object user: - description: 'Optional: User is the rados user - name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'user is optional: User is the + rados user name, default is admin More info: + https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: string required: - monitors type: object cinder: - description: 'Cinder represents a cinder volume + description: 'cinder represents a cinder volume attached and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' properties: fsType: - description: 'Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Examples: "ext4", "xfs", - "ntfs". Implicitly inferred to be "ext4" if - unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + description: 'fsType is the filesystem type + to mount. Must be a filesystem type supported + by the host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: string readOnly: - description: 'Optional: Defaults to false (read/write). + description: 'readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: boolean secretRef: - description: 'Optional: points to a secret object - containing parameters used to connect to OpenStack.' + description: 'secretRef is optional: points + to a secret object containing parameters used + to connect to OpenStack.' properties: name: description: 'Name of the referent. More @@ -6064,32 +6093,32 @@ spec: type: string type: object volumeID: - description: 'volume id used to identify the + description: 'volumeID used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: string required: - volumeID type: object configMap: - description: ConfigMap represents a configMap that + description: configMap represents a configMap that should populate this volume properties: defaultMode: - description: 'Optional: mode bits used to set - permissions on created files by default. Must - be an octal value between 0000 and 0777 or - a decimal value between 0 and 511. YAML accepts - both octal and decimal values, JSON requires - decimal values for mode bits. Defaults to - 0644. Directories within the path are not - affected by this setting. This might be in - conflict with other options that affect the - file mode, like fsGroup, and the result can - be other mode bits set.' + description: 'defaultMode is optional: mode + bits used to set permissions on created files + by default. Must be an octal value between + 0000 and 0777 or a decimal value between 0 + and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within + the path are not affected by this setting. + This might be in conflict with other options + that affect the file mode, like fsGroup, and + the result can be other mode bits set.' format: int32 type: integer items: - description: If unspecified, each key-value + description: items if unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. @@ -6105,29 +6134,29 @@ spec: a volume. properties: key: - description: The key to project. + description: key is the key to project. type: string mode: - description: 'Optional: mode bits used - to set permissions on this file. Must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. - YAML accepts both octal and decimal - values, JSON requires decimal values - for mode bits. If not specified, the - volume defaultMode will be used. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits - set.' + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 + and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, like + fsGroup, and the result can be other + mode bits set.' format: int32 type: integer path: - description: The relative path of the - file to map the key to. May not be an - absolute path. May not contain the path - element '..'. May not start with the - string '..'. + description: path is the relative path + of the file to map the key to. May not + be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. type: string required: - key @@ -6141,30 +6170,29 @@ spec: kind, uid?' type: string optional: - description: Specify whether the ConfigMap or - its keys must be defined + description: optional specify whether the ConfigMap + or its keys must be defined type: boolean type: object csi: - description: CSI (Container Storage Interface) represents + description: csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers (Beta feature). properties: driver: - description: Driver is the name of the CSI driver + description: driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster. type: string fsType: - description: Filesystem type to mount. Ex. "ext4", - "xfs", "ntfs". If not provided, the empty - value is passed to the associated CSI driver - which will determine the default filesystem - to apply. + description: fsType to mount. Ex. "ext4", "xfs", + "ntfs". If not provided, the empty value is + passed to the associated CSI driver which + will determine the default filesystem to apply. type: string nodePublishSecretRef: - description: NodePublishSecretRef is a reference + description: nodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume @@ -6181,13 +6209,14 @@ spec: type: string type: object readOnly: - description: Specifies a read-only configuration - for the volume. Defaults to false (read/write). + description: readOnly specifies a read-only + configuration for the volume. Defaults to + false (read/write). type: boolean volumeAttributes: additionalProperties: type: string - description: VolumeAttributes stores driver-specific + description: volumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values. @@ -6196,7 +6225,7 @@ spec: - driver type: object downwardAPI: - description: DownwardAPI represents downward API + description: downwardAPI represents downward API about the pod that should populate this volume properties: defaultMode: @@ -6295,46 +6324,46 @@ spec: type: array type: object emptyDir: - description: 'EmptyDir represents a temporary directory + description: 'emptyDir represents a temporary directory that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' properties: medium: - description: 'What type of storage medium should - back this directory. The default is "" which - means to use the node''s default medium. Must - be an empty string (default) or Memory. More - info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + description: 'medium represents what type of + storage medium should back this directory. + The default is "" which means to use the node''s + default medium. Must be an empty string (default) + or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' type: string sizeLimit: anyOf: - type: integer - type: string - description: 'Total amount of local storage - required for this EmptyDir volume. The size - limit is also applicable for memory medium. - The maximum usage on memory medium EmptyDir - would be the minimum value between the SizeLimit - specified here and the sum of memory limits - of all containers in a pod. The default is - nil which means that the limit is undefined. - More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + description: 'sizeLimit is the total amount + of local storage required for this EmptyDir + volume. The size limit is also applicable + for memory medium. The maximum usage on memory + medium EmptyDir would be the minimum value + between the SizeLimit specified here and the + sum of memory limits of all containers in + a pod. The default is nil which means that + the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object ephemeral: - description: "Ephemeral represents a volume that + description: "ephemeral represents a volume that is handled by a cluster storage driver. The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed. \n Use this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from - snapshot or capacity tracking are needed, c) - the storage driver is specified through a storage + snapshot or capacity tracking are needed, c) the + storage driver is specified through a storage class, and d) the storage driver supports dynamic - volume provisioning through a PersistentVolumeClaim - (see EphemeralVolumeSource for more information - on the connection between this volume type and + volume provisioning through a PersistentVolumeClaim + (see EphemeralVolumeSource for more information + on the connection between this volume type and PersistentVolumeClaim). \n Use PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of @@ -6385,16 +6414,16 @@ spec: are also valid here. properties: accessModes: - description: 'AccessModes contains the + description: 'accessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' items: type: string type: array dataSource: - description: 'This field can be used - to specify either: * An existing VolumeSnapshot - object (snapshot.storage.k8s.io/VolumeSnapshot) + description: 'dataSource field can be + used to specify either: * An existing + VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified @@ -6427,18 +6456,18 @@ spec: - name type: object dataSourceRef: - description: 'Specifies the object from - which to populate the volume with - data, if a non-empty volume is desired. - This may be any local object from - a non-empty API group (non core object) - or a PersistentVolumeClaim object. - When this field is specified, volume - binding will only succeed if the type - of the specified object matches some - installed volume populator or dynamic - provisioner. This field will replace - the functionality of the DataSource + description: 'dataSourceRef specifies + the object from which to populate + the volume with data, if a non-empty + volume is desired. This may be any + local object from a non-empty API + group (non core object) or a PersistentVolumeClaim + object. When this field is specified, + volume binding will only succeed if + the type of the specified object matches + some installed volume populator or + dynamic provisioner. This field will + replace the functionality of the DataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, @@ -6449,14 +6478,14 @@ spec: differences between DataSource and DataSourceRef: * While DataSource only allows two specific types of - objects, DataSourceRef allows any + objects, DataSourceRef allows any non-core object, as well as PersistentVolumeClaim objects. * While DataSource ignores disallowed values (dropping them), - DataSourceRef preserves all values, + DataSourceRef preserves all values, and generates an error if a disallowed - value is specified. (Alpha) Using - this field requires the AnyVolumeDataSource + value is specified. (Beta) Using this + field requires the AnyVolumeDataSource feature gate to be enabled.' properties: apiGroup: @@ -6481,7 +6510,7 @@ spec: - name type: object resources: - description: 'Resources represents the + description: 'resources represents the minimum resources the volume should have. If RecoverVolumeExpansionFailure feature is enabled users are allowed @@ -6520,8 +6549,8 @@ spec: type: object type: object selector: - description: A label query over volumes - to consider for binding. + description: selector is a label query + over volumes to consider for binding. properties: matchExpressions: description: matchExpressions is @@ -6580,9 +6609,9 @@ spec: type: object type: object storageClassName: - description: 'Name of the StorageClass - required by the claim. More info: - https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + description: 'storageClassName is the + name of the StorageClass required + by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' type: string volumeMode: description: volumeMode defines what @@ -6591,7 +6620,7 @@ spec: when not included in claim spec. type: string volumeName: - description: VolumeName is the binding + description: volumeName is the binding reference to the PersistentVolume backing this claim. type: string @@ -6601,36 +6630,37 @@ spec: type: object type: object fc: - description: FC represents a Fibre Channel resource + description: fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. properties: fsType: - description: 'Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - Implicitly inferred to be "ext4" if unspecified. - TODO: how do we prevent errors in the filesystem - from compromising the machine' + description: 'fsType is the filesystem type + to mount. Must be a filesystem type supported + by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. TODO: how do we prevent errors + in the filesystem from compromising the machine' type: string lun: - description: 'Optional: FC target lun number' + description: 'lun is Optional: FC target lun + number' format: int32 type: integer readOnly: - description: 'Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts.' + description: 'readOnly is Optional: Defaults + to false (read/write). ReadOnly here will + force the ReadOnly setting in VolumeMounts.' type: boolean targetWWNs: - description: 'Optional: FC target worldwide - names (WWNs)' + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' items: type: string type: array wwids: - description: 'Optional: FC volume world wide - identifiers (wwids) Either wwids or combination + description: 'wwids Optional: FC volume world + wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.' items: @@ -6638,40 +6668,40 @@ spec: type: array type: object flexVolume: - description: FlexVolume represents a generic volume + description: flexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin. properties: driver: - description: Driver is the name of the driver + description: driver is the name of the driver to use for this volume. type: string fsType: - description: Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - The default filesystem depends on FlexVolume - script. + description: fsType is the filesystem type to + mount. Must be a filesystem type supported + by the host operating system. Ex. "ext4", + "xfs", "ntfs". The default filesystem depends + on FlexVolume script. type: string options: additionalProperties: type: string - description: 'Optional: Extra command options - if any.' + description: 'options is Optional: this field + holds extra command options if any.' type: object readOnly: - description: 'Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting - in VolumeMounts.' + description: 'readOnly is Optional: defaults + to false (read/write). ReadOnly here will + force the ReadOnly setting in VolumeMounts.' type: boolean secretRef: - description: 'Optional: SecretRef is reference - to the secret object containing sensitive - information to pass to the plugin scripts. - This may be empty if no secret object is specified. - If the secret object contains more than one - secret, all secrets are passed to the plugin - scripts.' + description: 'secretRef is Optional: secretRef + is reference to the secret object containing + sensitive information to pass to the plugin + scripts. This may be empty if no secret object + is specified. If the secret object contains + more than one secret, all secrets are passed + to the plugin scripts.' properties: name: description: 'Name of the referent. More @@ -6684,53 +6714,54 @@ spec: - driver type: object flocker: - description: Flocker represents a Flocker volume + description: flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running properties: datasetName: - description: Name of the dataset stored as metadata - -> name on the dataset for Flocker should - be considered as deprecated + description: datasetName is Name of the dataset + stored as metadata -> name on the dataset + for Flocker should be considered as deprecated type: string datasetUUID: - description: UUID of the dataset. This is unique - identifier of a Flocker dataset + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a Flocker + dataset type: string type: object gcePersistentDisk: - description: 'GCEPersistentDisk represents a GCE + description: 'gcePersistentDisk represents a GCE Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' properties: fsType: - description: 'Filesystem type of the volume - that you want to mount. Tip: Ensure that the - filesystem type is supported by the host operating - system. Examples: "ext4", "xfs", "ntfs". Implicitly - inferred to be "ext4" if unspecified. More - info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + description: 'fsType is filesystem type of the + volume that you want to mount. Tip: Ensure + that the filesystem type is supported by the + host operating system. Examples: "ext4", "xfs", + "ntfs". Implicitly inferred to be "ext4" if + unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk TODO: how do we prevent errors in the filesystem from compromising the machine' type: string partition: - description: 'The partition in the volume that - you want to mount. If omitted, the default - is to mount by volume name. Examples: For - volume /dev/sda1, you specify the partition + description: 'partition is the partition in + the volume that you want to mount. If omitted, + the default is to mount by volume name. Examples: + For volume /dev/sda1, you specify the partition as "1". Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' format: int32 type: integer pdName: - description: 'Unique name of the PD resource - in GCE. Used to identify the disk in GCE. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + description: 'pdName is unique name of the PD + resource in GCE. Used to identify the disk + in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' type: string readOnly: - description: 'ReadOnly here will force the ReadOnly + description: 'readOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' type: boolean @@ -6738,7 +6769,7 @@ spec: - pdName type: object gitRepo: - description: 'GitRepo represents a git repository + description: 'gitRepo represents a git repository at a particular revision. DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer @@ -6746,38 +6777,39 @@ spec: EmptyDir into the Pod''s container.' properties: directory: - description: Target directory name. Must not - contain or start with '..'. If '.' is supplied, - the volume directory will be the git repository. Otherwise, - if specified, the volume will contain the - git repository in the subdirectory with the - given name. + description: directory is the target directory + name. Must not contain or start with '..'. If + '.' is supplied, the volume directory will + be the git repository. Otherwise, if specified, + the volume will contain the git repository + in the subdirectory with the given name. type: string repository: - description: Repository URL + description: repository is the URL type: string revision: - description: Commit hash for the specified revision. + description: revision is the commit hash for + the specified revision. type: string required: - repository type: object glusterfs: - description: 'Glusterfs represents a Glusterfs mount + description: 'glusterfs represents a Glusterfs mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' properties: endpoints: - description: 'EndpointsName is the endpoint - name that details Glusterfs topology. More - info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + description: 'endpoints is the endpoint name + that details Glusterfs topology. More info: + https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: string path: - description: 'Path is the Glusterfs volume path. + description: 'path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: string readOnly: - description: 'ReadOnly here will force the Glusterfs + description: 'readOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: boolean @@ -6786,7 +6818,7 @@ spec: - path type: object hostPath: - description: 'HostPath represents a pre-existing + description: 'hostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other privileged things @@ -6797,74 +6829,76 @@ spec: mount host directories as read/write.' properties: path: - description: 'Path of the directory on the host. + description: 'path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' type: string type: - description: 'Type for HostPath Volume Defaults + description: 'type for HostPath Volume Defaults to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' type: string required: - path type: object iscsi: - description: 'ISCSI represents an ISCSI Disk resource + description: 'iscsi represents an ISCSI Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' properties: chapAuthDiscovery: - description: whether support iSCSI Discovery - CHAP authentication + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication type: boolean chapAuthSession: - description: whether support iSCSI Session CHAP - authentication + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication type: boolean fsType: - description: 'Filesystem type of the volume - that you want to mount. Tip: Ensure that the - filesystem type is supported by the host operating - system. Examples: "ext4", "xfs", "ntfs". Implicitly - inferred to be "ext4" if unspecified. More - info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + description: 'fsType is the filesystem type + of the volume that you want to mount. Tip: + Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi TODO: how do we prevent errors in the filesystem from compromising the machine' type: string initiatorName: - description: Custom iSCSI Initiator Name. If - initiatorName is specified with iscsiInterface - simultaneously, new iSCSI interface : will be created for - the connection. + description: initiatorName is the custom iSCSI + Initiator Name. If initiatorName is specified + with iscsiInterface simultaneously, new iSCSI + interface : will + be created for the connection. type: string iqn: - description: Target iSCSI Qualified Name. + description: iqn is the target iSCSI Qualified + Name. type: string iscsiInterface: - description: iSCSI Interface Name that uses - an iSCSI transport. Defaults to 'default' - (tcp). + description: iscsiInterface is the interface + Name that uses an iSCSI transport. Defaults + to 'default' (tcp). type: string lun: - description: iSCSI Target Lun number. + description: lun represents iSCSI Target Lun + number. format: int32 type: integer portals: - description: iSCSI Target Portal List. The portal - is either an IP or ip_addr:port if the port - is other than default (typically TCP ports - 860 and 3260). + description: portals is the iSCSI Target Portal + List. The portal is either an IP or ip_addr:port + if the port is other than default (typically + TCP ports 860 and 3260). items: type: string type: array readOnly: - description: ReadOnly here will force the ReadOnly + description: readOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. type: boolean secretRef: - description: CHAP Secret for iSCSI target and - initiator authentication + description: secretRef is the CHAP Secret for + iSCSI target and initiator authentication properties: name: description: 'Name of the referent. More @@ -6874,10 +6908,10 @@ spec: type: string type: object targetPortal: - description: iSCSI Target Portal. The Portal - is either an IP or ip_addr:port if the port - is other than default (typically TCP ports - 860 and 3260). + description: targetPortal is iSCSI Target Portal. + The Portal is either an IP or ip_addr:port + if the port is other than default (typically + TCP ports 860 and 3260). type: string required: - iqn @@ -6885,25 +6919,25 @@ spec: - targetPortal type: object name: - description: 'Volume''s name. Must be a DNS_LABEL + description: 'name of the volume. Must be a DNS_LABEL and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string nfs: - description: 'NFS represents an NFS mount on the + description: 'nfs represents an NFS mount on the host that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' properties: path: - description: 'Path that is exported by the NFS + description: 'path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: string readOnly: - description: 'ReadOnly here will force the NFS + description: 'readOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: boolean server: - description: 'Server is the hostname or IP address + description: 'server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: string required: @@ -6911,97 +6945,98 @@ spec: - server type: object persistentVolumeClaim: - description: 'PersistentVolumeClaimVolumeSource + description: 'persistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: claimName: - description: 'ClaimName is the name of a PersistentVolumeClaim + description: 'claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' type: string readOnly: - description: Will force the ReadOnly setting - in VolumeMounts. Default false. + description: readOnly Will force the ReadOnly + setting in VolumeMounts. Default false. type: boolean required: - claimName type: object photonPersistentDisk: - description: PhotonPersistentDisk represents a PhotonController + description: photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine properties: fsType: - description: Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - Implicitly inferred to be "ext4" if unspecified. + description: fsType is the filesystem type to + mount. Must be a filesystem type supported + by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. type: string pdID: - description: ID that identifies Photon Controller - persistent disk + description: pdID is the ID that identifies + Photon Controller persistent disk type: string required: - pdID type: object portworxVolume: - description: PortworxVolume represents a portworx + description: portworxVolume represents a portworx volume attached and mounted on kubelets host machine properties: fsType: - description: FSType represents the filesystem + description: fSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. type: string readOnly: - description: Defaults to false (read/write). + description: readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean volumeID: - description: VolumeID uniquely identifies a + description: volumeID uniquely identifies a Portworx volume type: string required: - volumeID type: object projected: - description: Items for all in one resources secrets, - configmaps, and downward API + description: projected items for all in one resources + secrets, configmaps, and downward API properties: defaultMode: - description: Mode bits used to set permissions - on created files by default. Must be an octal - value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal - and decimal values, JSON requires decimal - values for mode bits. Directories within the - path are not affected by this setting. This - might be in conflict with other options that - affect the file mode, like fsGroup, and the - result can be other mode bits set. + description: defaultMode are the mode bits used + to set permissions on created files by default. + Must be an octal value between 0000 and 0777 + or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON + requires decimal values for mode bits. Directories + within the path are not affected by this setting. + This might be in conflict with other options + that affect the file mode, like fsGroup, and + the result can be other mode bits set. format: int32 type: integer sources: - description: list of volume projections + description: sources is the list of volume projections items: description: Projection that may be projected along with other supported volume types properties: configMap: - description: information about the configMap - data to project + description: configMap information about + the configMap data to project properties: items: - description: If unspecified, each - key-value pair in the Data field - of the referenced ConfigMap will - be projected into the volume as - a file whose name is the key and - content is the value. If specified, + description: items if unspecified, + each key-value pair in the Data + field of the referenced ConfigMap + will be projected into the volume + as a file whose name is the key + and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key @@ -7016,11 +7051,12 @@ spec: a path within a volume. properties: key: - description: The key to project. + description: key is the key + to project. type: string mode: - description: 'Optional: mode - bits used to set permissions + description: 'mode is Optional: + mode bits used to set permissions on this file. Must be an octal value between 0000 and 0777 or a decimal value between @@ -7037,9 +7073,9 @@ spec: format: int32 type: integer path: - description: The relative path - of the file to map the key - to. May not be an absolute + description: path is the relative + path of the file to map the + key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. @@ -7056,13 +7092,14 @@ spec: kind, uid?' type: string optional: - description: Specify whether the ConfigMap - or its keys must be defined + description: optional specify whether + the ConfigMap or its keys must be + defined type: boolean type: object downwardAPI: - description: information about the downwardAPI - data to project + description: downwardAPI information about + the downwardAPI data to project properties: items: description: Items is a list of DownwardAPIVolume @@ -7156,18 +7193,18 @@ spec: type: array type: object secret: - description: information about the secret - data to project + description: secret information about + the secret data to project properties: items: - description: If unspecified, each - key-value pair in the Data field - of the referenced Secret will be - projected into the volume as a file - whose name is the key and content - is the value. If specified, the - listed keys will be projected into - the specified paths, and unlisted + description: items if unspecified, + each key-value pair in the Data + field of the referenced Secret will + be projected into the volume as + a file whose name is the key and + content is the value. If specified, + the listed keys will be projected + into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup @@ -7180,11 +7217,12 @@ spec: a path within a volume. properties: key: - description: The key to project. + description: key is the key + to project. type: string mode: - description: 'Optional: mode - bits used to set permissions + description: 'mode is Optional: + mode bits used to set permissions on this file. Must be an octal value between 0000 and 0777 or a decimal value between @@ -7201,9 +7239,9 @@ spec: format: int32 type: integer path: - description: The relative path - of the file to map the key - to. May not be an absolute + description: path is the relative + path of the file to map the + key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. @@ -7220,16 +7258,18 @@ spec: kind, uid?' type: string optional: - description: Specify whether the Secret - or its key must be defined + description: optional field specify + whether the Secret or its key must + be defined type: boolean type: object serviceAccountToken: - description: information about the serviceAccountToken - data to project + description: serviceAccountToken is information + about the serviceAccountToken data to + project properties: audience: - description: Audience is the intended + description: audience is the intended audience of the token. A recipient of a token must identify itself with an identifier specified in @@ -7239,7 +7279,7 @@ spec: apiserver. type: string expirationSeconds: - description: ExpirationSeconds is + description: expirationSeconds is the requested duration of validity of the service account token. As the token approaches expiration, @@ -7254,7 +7294,7 @@ spec: format: int64 type: integer path: - description: Path is the path relative + description: path is the path relative to the mount point of the file to project the token into. type: string @@ -7265,37 +7305,37 @@ spec: type: array type: object quobyte: - description: Quobyte represents a Quobyte mount + description: quobyte represents a Quobyte mount on the host that shares a pod's lifetime properties: group: - description: Group to map volume access to Default + description: group to map volume access to Default is no group type: string readOnly: - description: ReadOnly here will force the Quobyte + description: readOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false. type: boolean registry: - description: Registry represents a single or + description: registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with commas) which acts as the central registry for volumes type: string tenant: - description: Tenant owning the given Quobyte + description: tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin type: string user: - description: User to map volume access to Defaults + description: user to map volume access to Defaults to serivceaccount user type: string volume: - description: Volume is a string that references + description: volume is a string that references an already created Quobyte volume by name. type: string required: @@ -7303,46 +7343,46 @@ spec: - volume type: object rbd: - description: 'RBD represents a Rados Block Device + description: 'rbd represents a Rados Block Device mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' properties: fsType: - description: 'Filesystem type of the volume - that you want to mount. Tip: Ensure that the - filesystem type is supported by the host operating - system. Examples: "ext4", "xfs", "ntfs". Implicitly - inferred to be "ext4" if unspecified. More - info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + description: 'fsType is the filesystem type + of the volume that you want to mount. Tip: + Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd TODO: how do we prevent errors in the filesystem from compromising the machine' type: string image: - description: 'The rados image name. More info: - https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string keyring: - description: 'Keyring is the path to key ring + description: 'keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string monitors: - description: 'A collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'monitors is a collection of Ceph + monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' items: type: string type: array pool: - description: 'The rados pool name. Default is - rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'pool is the rados pool name. Default + is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string readOnly: - description: 'ReadOnly here will force the ReadOnly + description: 'readOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: boolean secretRef: - description: 'SecretRef is name of the authentication + description: 'secretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' properties: @@ -7354,38 +7394,39 @@ spec: type: string type: object user: - description: 'The rados user name. Default is - admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'user is the rados user name. Default + is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string required: - image - monitors type: object scaleIO: - description: ScaleIO represents a ScaleIO persistent + description: scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. properties: fsType: - description: Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - Default is "xfs". + description: fsType is the filesystem type to + mount. Must be a filesystem type supported + by the host operating system. Ex. "ext4", + "xfs", "ntfs". Default is "xfs". type: string gateway: - description: The host address of the ScaleIO - API Gateway. + description: gateway is the host address of + the ScaleIO API Gateway. type: string protectionDomain: - description: The name of the ScaleIO Protection - Domain for the configured storage. + description: protectionDomain is the name of + the ScaleIO Protection Domain for the configured + storage. type: string readOnly: - description: Defaults to false (read/write). + description: readOnly Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean secretRef: - description: SecretRef references to the secret + description: secretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail. @@ -7398,26 +7439,26 @@ spec: type: string type: object sslEnabled: - description: Flag to enable/disable SSL communication - with Gateway, default false + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default false type: boolean storageMode: - description: Indicates whether the storage for - a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. + description: storageMode indicates whether the + storage for a volume should be ThickProvisioned + or ThinProvisioned. Default is ThinProvisioned. type: string storagePool: - description: The ScaleIO Storage Pool associated - with the protection domain. + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. type: string system: - description: The name of the storage system - as configured in ScaleIO. + description: system is the name of the storage + system as configured in ScaleIO. type: string volumeName: - description: The name of a volume already created - in the ScaleIO system that is associated with - this volume source. + description: volumeName is the name of a volume + already created in the ScaleIO system that + is associated with this volume source. type: string required: - gateway @@ -7425,25 +7466,25 @@ spec: - system type: object secret: - description: 'Secret represents a secret that should + description: 'secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' properties: defaultMode: - description: 'Optional: mode bits used to set - permissions on created files by default. Must - be an octal value between 0000 and 0777 or - a decimal value between 0 and 511. YAML accepts - both octal and decimal values, JSON requires - decimal values for mode bits. Defaults to - 0644. Directories within the path are not - affected by this setting. This might be in - conflict with other options that affect the - file mode, like fsGroup, and the result can - be other mode bits set.' + description: 'defaultMode is Optional: mode + bits used to set permissions on created files + by default. Must be an octal value between + 0000 and 0777 or a decimal value between 0 + and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within + the path are not affected by this setting. + This might be in conflict with other options + that affect the file mode, like fsGroup, and + the result can be other mode bits set.' format: int32 type: integer items: - description: If unspecified, each key-value + description: items If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. @@ -7459,29 +7500,29 @@ spec: a volume. properties: key: - description: The key to project. + description: key is the key to project. type: string mode: - description: 'Optional: mode bits used - to set permissions on this file. Must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. - YAML accepts both octal and decimal - values, JSON requires decimal values - for mode bits. If not specified, the - volume defaultMode will be used. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits - set.' + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 + and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, like + fsGroup, and the result can be other + mode bits set.' format: int32 type: integer path: - description: The relative path of the - file to map the key to. May not be an - absolute path. May not contain the path - element '..'. May not start with the - string '..'. + description: path is the relative path + of the file to map the key to. May not + be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. type: string required: - key @@ -7489,31 +7530,33 @@ spec: type: object type: array optional: - description: Specify whether the Secret or its - keys must be defined + description: optional field specify whether + the Secret or its keys must be defined type: boolean secretName: - description: 'Name of the secret in the pod''s - namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + description: 'secretName is the name of the + secret in the pod''s namespace to use. More + info: https://kubernetes.io/docs/concepts/storage/volumes#secret' type: string type: object storageos: - description: StorageOS represents a StorageOS volume + description: storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. properties: fsType: - description: Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - Implicitly inferred to be "ext4" if unspecified. + description: fsType is the filesystem type to + mount. Must be a filesystem type supported + by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. type: string readOnly: - description: Defaults to false (read/write). + description: readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. type: boolean secretRef: - description: SecretRef specifies the secret + description: secretRef specifies the secret to use for obtaining the StorageOS API credentials. If not specified, default values will be attempted. properties: @@ -7525,12 +7568,12 @@ spec: type: string type: object volumeName: - description: VolumeName is the human-readable + description: volumeName is the human-readable name of the StorageOS volume. Volume names are only unique within a namespace. type: string volumeNamespace: - description: VolumeNamespace specifies the scope + description: volumeNamespace specifies the scope of the volume within StorageOS. If no namespace is specified then the Pod's namespace will be used. This allows the Kubernetes name @@ -7543,26 +7586,28 @@ spec: type: string type: object vsphereVolume: - description: VsphereVolume represents a vSphere + description: vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine properties: fsType: - description: Filesystem type to mount. Must - be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs", "ntfs". - Implicitly inferred to be "ext4" if unspecified. + description: fsType is filesystem type to mount. + Must be a filesystem type supported by the + host operating system. Ex. "ext4", "xfs", + "ntfs". Implicitly inferred to be "ext4" if + unspecified. type: string storagePolicyID: - description: Storage Policy Based Management - (SPBM) profile ID associated with the StoragePolicyName. + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. type: string storagePolicyName: - description: Storage Policy Based Management - (SPBM) profile name. + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. type: string volumePath: - description: Path that identifies vSphere volume - vmdk + description: volumePath is the path that identifies + vSphere volume vmdk type: string required: - volumePath @@ -7592,9 +7637,20 @@ spec: maxReplicaCount: format: int32 type: integer + minReplicaCount: + format: int32 + type: integer pollingInterval: format: int32 type: integer + rollout: + description: Rollout defines the strategy for job rollouts + properties: + propagationPolicy: + type: string + strategy: + type: string + type: object rolloutStrategy: type: string scalingStrategy: @@ -7687,15 +7743,11 @@ spec: lastActiveTime: format: date-time type: string + Paused: + type: string type: object type: object served: true storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/controllers/keda/hpa.go b/controllers/keda/hpa.go index 166eee55fd9..3832cca665e 100644 --- a/controllers/keda/hpa.go +++ b/controllers/keda/hpa.go @@ -24,7 +24,7 @@ import ( "unicode" "github.com/go-logr/logr" - autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta2" + autoscalingv2 "k8s.io/api/autoscaling/v2" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -70,14 +70,14 @@ func (r *ScaledObjectReconciler) createAndDeployNewHPA(ctx context.Context, logg } // newHPAForScaledObject returns HPA as it is specified in ScaledObject -func (r *ScaledObjectReconciler) newHPAForScaledObject(ctx context.Context, logger logr.Logger, scaledObject *kedav1alpha1.ScaledObject, gvkr *kedav1alpha1.GroupVersionKindResource) (*autoscalingv2beta2.HorizontalPodAutoscaler, error) { +func (r *ScaledObjectReconciler) newHPAForScaledObject(ctx context.Context, logger logr.Logger, scaledObject *kedav1alpha1.ScaledObject, gvkr *kedav1alpha1.GroupVersionKindResource) (*autoscalingv2.HorizontalPodAutoscaler, error) { scaledObjectMetricSpecs, err := r.getScaledObjectMetricSpecs(ctx, logger, scaledObject) if err != nil { return nil, err } - var behavior *autoscalingv2beta2.HorizontalPodAutoscalerBehavior - if r.kubeVersion.MinorVersion >= 18 && scaledObject.Spec.Advanced != nil && scaledObject.Spec.Advanced.HorizontalPodAutoscalerConfig != nil { + var behavior *autoscalingv2.HorizontalPodAutoscalerBehavior + if scaledObject.Spec.Advanced != nil && scaledObject.Spec.Advanced.HorizontalPodAutoscalerConfig != nil { behavior = scaledObject.Spec.Advanced.HorizontalPodAutoscalerConfig.Behavior } else { behavior = nil @@ -117,13 +117,13 @@ func (r *ScaledObjectReconciler) newHPAForScaledObject(ctx context.Context, logg maxReplicas = *pausedCount } - hpa := &autoscalingv2beta2.HorizontalPodAutoscaler{ - Spec: autoscalingv2beta2.HorizontalPodAutoscalerSpec{ + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ MinReplicas: minReplicas, MaxReplicas: maxReplicas, Metrics: scaledObjectMetricSpecs, Behavior: behavior, - ScaleTargetRef: autoscalingv2beta2.CrossVersionObjectReference{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ Name: scaledObject.Spec.ScaleTargetRef.Name, Kind: gvkr.Kind, APIVersion: gvkr.GroupVersion().String(), @@ -135,7 +135,7 @@ func (r *ScaledObjectReconciler) newHPAForScaledObject(ctx context.Context, logg Annotations: scaledObject.Annotations, }, TypeMeta: metav1.TypeMeta{ - APIVersion: "v2beta2", + APIVersion: "v2", }, } @@ -148,7 +148,7 @@ func (r *ScaledObjectReconciler) newHPAForScaledObject(ctx context.Context, logg } // updateHPAIfNeeded checks whether update of HPA is needed -func (r *ScaledObjectReconciler) updateHPAIfNeeded(ctx context.Context, logger logr.Logger, scaledObject *kedav1alpha1.ScaledObject, foundHpa *autoscalingv2beta2.HorizontalPodAutoscaler, gvkr *kedav1alpha1.GroupVersionKindResource) error { +func (r *ScaledObjectReconciler) updateHPAIfNeeded(ctx context.Context, logger logr.Logger, scaledObject *kedav1alpha1.ScaledObject, foundHpa *autoscalingv2.HorizontalPodAutoscaler, gvkr *kedav1alpha1.GroupVersionKindResource) error { hpa, err := r.newHPAForScaledObject(ctx, logger, scaledObject, gvkr) if err != nil { logger.Error(err, "Failed to create new HPA resource", "HPA.Namespace", scaledObject.Namespace, "HPA.Name", getHPAName(scaledObject)) @@ -163,8 +163,6 @@ func (r *ScaledObjectReconciler) updateHPAIfNeeded(ctx context.Context, logger l logger.Error(err, "Failed to update HPA", "HPA.Namespace", foundHpa.Namespace, "HPA.Name", foundHpa.Name) return err } - // check if scaledObject.spec.behavior was defined, because it is supported only on k8s >= 1.18 - r.checkMinK8sVersionforHPABehavior(logger, scaledObject) logger.Info("Updated HPA according to ScaledObject", "HPA.Namespace", foundHpa.Namespace, "HPA.Name", foundHpa.Name) } @@ -183,7 +181,7 @@ func (r *ScaledObjectReconciler) updateHPAIfNeeded(ctx context.Context, logger l } // deleteAndCreateHpa delete old HPA and create new one -func (r *ScaledObjectReconciler) renameHPA(ctx context.Context, logger logr.Logger, scaledObject *kedav1alpha1.ScaledObject, foundHpa *autoscalingv2beta2.HorizontalPodAutoscaler, gvkr *kedav1alpha1.GroupVersionKindResource) error { +func (r *ScaledObjectReconciler) renameHPA(ctx context.Context, logger logr.Logger, scaledObject *kedav1alpha1.ScaledObject, foundHpa *autoscalingv2.HorizontalPodAutoscaler, gvkr *kedav1alpha1.GroupVersionKindResource) error { logger.Info("Deleting old HPA", "HPA.Namespace", scaledObject.Namespace, "HPA.Name", foundHpa.Name) if err := r.Client.Delete(ctx, foundHpa); err != nil { logger.Error(err, "Failed to delete old HPA", "HPA.Namespace", foundHpa.Namespace, "HPA.Name", foundHpa.Name) @@ -194,8 +192,8 @@ func (r *ScaledObjectReconciler) renameHPA(ctx context.Context, logger logr.Logg } // getScaledObjectMetricSpecs returns MetricSpec for HPA, generater from Triggers defitinion in ScaledObject -func (r *ScaledObjectReconciler) getScaledObjectMetricSpecs(ctx context.Context, logger logr.Logger, scaledObject *kedav1alpha1.ScaledObject) ([]autoscalingv2beta2.MetricSpec, error) { - var scaledObjectMetricSpecs []autoscalingv2beta2.MetricSpec +func (r *ScaledObjectReconciler) getScaledObjectMetricSpecs(ctx context.Context, logger logr.Logger, scaledObject *kedav1alpha1.ScaledObject) ([]autoscalingv2.MetricSpec, error) { + var scaledObjectMetricSpecs []autoscalingv2.MetricSpec var externalMetricNames []string var resourceMetricNames []string @@ -260,15 +258,6 @@ func updateHealthStatus(scaledObject *kedav1alpha1.ScaledObject, externalMetricN status.Health = newHealth } -// checkMinK8sVersionforHPABehavior min version (k8s v1.18) for HPA Behavior -func (r *ScaledObjectReconciler) checkMinK8sVersionforHPABehavior(logger logr.Logger, scaledObject *kedav1alpha1.ScaledObject) { - if r.kubeVersion.MinorVersion < 18 { - if scaledObject.Spec.Advanced != nil && scaledObject.Spec.Advanced.HorizontalPodAutoscalerConfig != nil && scaledObject.Spec.Advanced.HorizontalPodAutoscalerConfig.Behavior != nil { - logger.Info("Warning: Ignoring scaledObject.spec.behavior, it is only supported on kubernetes version >= 1.18", "kubernetes.version", r.kubeVersion.PrettyVersion) - } - } -} - // getHPAName returns generated HPA name for ScaledObject specified in the parameter func getHPAName(scaledObject *kedav1alpha1.ScaledObject) string { if scaledObject.Spec.Advanced != nil && scaledObject.Spec.Advanced.HorizontalPodAutoscalerConfig != nil && scaledObject.Spec.Advanced.HorizontalPodAutoscalerConfig.Name != "" { diff --git a/controllers/keda/scaledjob_controller.go b/controllers/keda/scaledjob_controller.go index 1c05e196206..504e4fa7308 100644 --- a/controllers/keda/scaledjob_controller.go +++ b/controllers/keda/scaledjob_controller.go @@ -19,6 +19,7 @@ package keda import ( "context" "fmt" + "sync" "time" "github.com/go-logr/logr" @@ -27,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -38,6 +40,7 @@ import ( kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" kedacontrollerutil "github.com/kedacore/keda/v2/controllers/keda/util" "github.com/kedacore/keda/v2/pkg/eventreason" + "github.com/kedacore/keda/v2/pkg/metrics" "github.com/kedacore/keda/v2/pkg/scaling" ) @@ -51,18 +54,34 @@ type ScaledJobReconciler struct { GlobalHTTPTimeout time.Duration Recorder record.EventRecorder - scaleHandler scaling.ScaleHandler + scaledJobGenerations *sync.Map + scaleHandler scaling.ScaleHandler +} + +var ( + scaledJobTriggers map[string][]string + scaledJobTriggersLock *sync.Mutex +) + +func init() { + scaledJobTriggers = make(map[string][]string) + scaledJobTriggersLock = &sync.Mutex{} } // SetupWithManager initializes the ScaledJobReconciler instance and starts a new controller managed by the passed Manager instance. func (r *ScaledJobReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error { r.scaleHandler = scaling.NewScaleHandler(mgr.GetClient(), nil, mgr.GetScheme(), r.GlobalHTTPTimeout, mgr.GetEventRecorderFor("scale-handler")) + r.scaledJobGenerations = &sync.Map{} return ctrl.NewControllerManagedBy(mgr). WithOptions(options). // Ignore updates to ScaledJob Status (in this case metadata.Generation does not change) // so reconcile loop is not started on Status updates - For(&kedav1alpha1.ScaledJob{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). + For(&kedav1alpha1.ScaledJob{}, builder.WithPredicates( + predicate.Or( + kedacontrollerutil.PausedPredicate{}, + predicate.GenerationChangedPredicate{}, + ))). Complete(r) } @@ -90,8 +109,9 @@ func (r *ScaledJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // Check if the ScaledJob instance is marked to be deleted, which is // indicated by the deletion timestamp being set. if scaledJob.GetDeletionTimestamp() != nil { - return ctrl.Result{}, r.finalizeScaledJob(ctx, reqLogger, scaledJob) + return ctrl.Result{}, r.finalizeScaledJob(ctx, reqLogger, scaledJob, req.NamespacedName.String()) } + r.updateTriggerTotals(reqLogger, scaledJob, req.NamespacedName.String()) // ensure finalizer is set on this CR if err := r.ensureFinalizer(ctx, reqLogger, scaledJob); err != nil { @@ -105,6 +125,11 @@ func (r *ScaledJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, err } } + // check if scaledJob.Status.IsPaused constant is not an empty string, if not an empty string then stopScaleLoop + if scaledJob.Status.Paused != "" { + reqLogger.Info("ScaledJob is paused, stopping scale loop") + return ctrl.Result{}, r.Paused(ctx, reqLogger, scaledJob) + } // Check jobTargetRef is specified if scaledJob.Spec.JobTargetRef == nil { @@ -162,13 +187,28 @@ func (r *ScaledJobReconciler) reconcileScaledJob(ctx context.Context, logger log if err != nil { return "Failed to start a new scale loop with scaling logic", err } + // scaledJob.Status.Paused is set to true so pause the ScaleLoop + err = r.Paused(ctx, logger, scaledJob) + if err != nil { + return "Failed to pause the scale loop", err + } + logger.Info("Initializing Scaling logic according to ScaledJob Specification") return "ScaledJob is defined correctly and is ready to scaling", nil + } // Delete Jobs owned by the previous version of the scaledJob based on the rolloutStrategy given for this scaledJob, if any func (r *ScaledJobReconciler) deletePreviousVersionScaleJobs(ctx context.Context, logger logr.Logger, scaledJob *kedav1alpha1.ScaledJob) (string, error) { - switch scaledJob.Spec.RolloutStrategy { + var rolloutStrategy string + if len(scaledJob.Spec.RolloutStrategy) > 0 { + logger.Info("RolloutStrategy is deprecated, please us Rollout.Strategy in order to define the desired strategy for job rollouts") + rolloutStrategy = scaledJob.Spec.RolloutStrategy + } else { + rolloutStrategy = scaledJob.Spec.Rollout.Strategy + } + + switch rolloutStrategy { case "gradual": logger.Info("RolloutStrategy: gradual, Not deleting jobs owned by the previous version of the scaleJob") default: @@ -187,7 +227,12 @@ func (r *ScaledJobReconciler) deletePreviousVersionScaleJobs(ctx context.Context } for _, job := range jobs.Items { job := job - err = r.Client.Delete(ctx, &job, client.PropagationPolicy(metav1.DeletePropagationBackground)) + + propagationPolicy := metav1.DeletePropagationBackground + if scaledJob.Spec.Rollout.PropagationPolicy == "foreground" { + propagationPolicy = metav1.DeletePropagationForeground + } + err = r.Client.Delete(ctx, &job, client.PropagationPolicy(propagationPolicy)) if err != nil { return "Not able to delete job: " + job.Name, err } @@ -200,11 +245,107 @@ func (r *ScaledJobReconciler) deletePreviousVersionScaleJobs(ctx context.Context // requestScaleLoop request ScaleLoop handler for the respective ScaledJob func (r *ScaledJobReconciler) requestScaleLoop(ctx context.Context, logger logr.Logger, scaledJob *kedav1alpha1.ScaledJob) error { logger.V(1).Info("Starting a new ScaleLoop") - return r.scaleHandler.HandleScalableObject(ctx, scaledJob) + + key, err := cache.MetaNamespaceKeyFunc(scaledJob) + if err != nil { + logger.Error(err, "Error getting key for scaledJob") + return err + } + + if err = r.scaleHandler.HandleScalableObject(ctx, scaledJob); err != nil { + return err + } + + r.scaledJobGenerations.Store(key, scaledJob.Generation) + + return nil } // stopScaleLoop stops ScaleLoop handler for the respective ScaledJob func (r *ScaledJobReconciler) stopScaleLoop(ctx context.Context, logger logr.Logger, scaledJob *kedav1alpha1.ScaledJob) error { logger.V(1).Info("Stopping a ScaleLoop") - return r.scaleHandler.DeleteScalableObject(ctx, scaledJob) + + key, err := cache.MetaNamespaceKeyFunc(scaledJob) + if err != nil { + logger.Error(err, "Error getting key for scaledJob") + return err + } + + if err = r.scaleHandler.DeleteScalableObject(ctx, scaledJob); err != nil { + return err + } + + r.scaledJobGenerations.Delete(key) + return nil +} + +// scaledJobGenerationChanged returns true if ScaledJob's Generation was changed, ie. ScaledJob.Spec was changed +func (r *ScaledJobReconciler) scaledJobGenerationChanged(logger logr.Logger, scaledJob *kedav1alpha1.ScaledJob) (bool, error) { + key, err := cache.MetaNamespaceKeyFunc(scaledJob) + if err != nil { + logger.Error(err, "Error getting key for scaledJob") + return true, err + } + + value, loaded := r.scaledJobGenerations.Load(key) + if loaded { + generation := value.(int64) + if generation == scaledJob.Generation { + return false, nil + } + } + return true, nil +} + +func (r *ScaledJobReconciler) updateTriggerTotals(logger logr.Logger, scaledJob *kedav1alpha1.ScaledJob, namespacedName string) { + specChanged, err := r.scaledJobGenerationChanged(logger, scaledJob) + if err != nil { + logger.Error(err, "failed to update trigger totals") + return + } + + if !specChanged { + return + } + + scaledJobTriggersLock.Lock() + defer scaledJobTriggersLock.Unlock() + + if triggerTypes, ok := scaledJobTriggers[namespacedName]; ok { + for _, triggerType := range triggerTypes { + metrics.DecrementTriggerTotal(triggerType) + } + } + + triggerTypes := make([]string, len(scaledJob.Spec.Triggers)) + for _, trigger := range scaledJob.Spec.Triggers { + metrics.IncrementTriggerTotal(trigger.Type) + triggerTypes = append(triggerTypes, trigger.Type) + } + + scaledJobTriggers[namespacedName] = triggerTypes +} + +func (r *ScaledJobReconciler) updateTriggerTotalsOnDelete(namespacedName string) { + scaledJobTriggersLock.Lock() + defer scaledJobTriggersLock.Unlock() + + if triggerTypes, ok := scaledJobTriggers[namespacedName]; ok { + for _, triggerType := range triggerTypes { + metrics.DecrementTriggerTotal(triggerType) + } + } + + delete(scaledJobTriggers, namespacedName) +} + +// stopScaleLoops when scaledJob.Status.IsPaused() is set +func (r *ScaledJobReconciler) Paused(ctx context.Context, logger logr.Logger, scaledJob *kedav1alpha1.ScaledJob) error { + if scaledJob.Annotations["scaledjob.keda.sh/paused"] == "true" { + logger.V(1).Info("Stopping a ScaleLoop when scaledJob is paused") + // stopScaleLoop + return r.stopScaleLoop(ctx, logger, scaledJob) + // return r.scaleHandler.DeleteScalableObject(ctx, scaledJob) + } + return nil } diff --git a/controllers/keda/util/predicate.go b/controllers/keda/util/predicate.go index 03aeb16c0bf..17c0a9cc389 100644 --- a/controllers/keda/util/predicate.go +++ b/controllers/keda/util/predicate.go @@ -13,6 +13,14 @@ type PausedReplicasPredicate struct { predicate.Funcs } +// define annotation as a constant "Paused = "autoscaling.keda.sh/paused"" +// define a predicate that checks if the annotation is present and if it is, return true +const Paused = "autoscaling.keda.sh/paused" + +type PausedPredicate struct { + predicate.Funcs +} + func (PausedReplicasPredicate) Update(e event.UpdateEvent) bool { if e.ObjectOld == nil || e.ObjectNew == nil { return false diff --git a/pkg/scaling/executor/scale_jobs.go b/pkg/scaling/executor/scale_jobs.go index 582ea82a8b9..a86c88918e3 100644 --- a/pkg/scaling/executor/scale_jobs.go +++ b/pkg/scaling/executor/scale_jobs.go @@ -46,7 +46,7 @@ func (e *scaleExecutor) RequestJobScale(ctx context.Context, scaledJob *kedav1al logger.Info("Scaling Jobs", "Number of running Jobs", runningJobCount) logger.Info("Scaling Jobs", "Number of pending Jobs ", pendingJobCount) - effectiveMaxScale := NewScalingStrategy(logger, scaledJob).GetEffectiveMaxScale(maxScale, runningJobCount, pendingJobCount, scaledJob.MaxReplicaCount()) + effectiveMaxScale, scaleTo := e.getScalingDecision(scaledJob, runningJobCount, scaleTo, maxScale, pendingJobCount, logger) if effectiveMaxScale < 0 { effectiveMaxScale = 0 @@ -86,6 +86,20 @@ func (e *scaleExecutor) RequestJobScale(ctx context.Context, scaledJob *kedav1al } } +func (e *scaleExecutor) getScalingDecision(scaledJob *kedav1alpha1.ScaledJob, runningJobCount int64, scaleTo int64, maxScale int64, pendingJobCount int64, logger logr.Logger) (int64, int64) { + var effectiveMaxScale int64 + minReplicaCount := scaledJob.MinReplicaCount() + + if runningJobCount < minReplicaCount { + scaleToMinReplica := minReplicaCount - runningJobCount + scaleTo = scaleToMinReplica + effectiveMaxScale = scaleToMinReplica + } else { + effectiveMaxScale = NewScalingStrategy(logger, scaledJob).GetEffectiveMaxScale(maxScale, runningJobCount-minReplicaCount, pendingJobCount, scaledJob.MaxReplicaCount()) + } + return effectiveMaxScale, scaleTo +} + func (e *scaleExecutor) createJobs(ctx context.Context, logger logr.Logger, scaledJob *kedav1alpha1.ScaledJob, scaleTo int64, maxScale int64) { scaledJob.Spec.JobTargetRef.Template.GenerateName = scaledJob.GetName() + "-" if scaledJob.Spec.JobTargetRef.Template.Labels == nil { @@ -414,3 +428,12 @@ func min(x, y int64) int64 { } return x } + +// check if scaledJob has Paused annotation, if it does, pause the scaled job +func (e *scaleExecutor) Paused(ctx context.Context, logger logr.Logger, scaledJob *kedav1alpha1.ScaledJob) error { + if scaledJob.Annotations["scaledjob.keda.sh/paused"] == "true" { + logger.Info("ScaledJob is paused, stop scale loop") + return e.client.Status().Update(ctx, scaledJob) + } + return nil +}