diff --git a/README.md b/README.md index 5135247..f605f46 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Pod-Reaper is configurable through environment variables. The pod-reaper specifi - `GRACE_PERIOD` duration that pods should be given to shut down before hard killing the pod - `SCHEDULE` schedule for when pod-reaper should look for pods to reap - `RUN_DURATION` how long pod-reaper should run before exiting +- `EVICT` try to evict pods instead of deleting them - `EXCLUDE_LABEL_KEY` pod metadata label (of key-value pair) that pod-reaper should exclude - `EXCLUDE_LABEL_VALUES` comma-separated list of metadata label values (of key-value pair) that pod-reaper should exclude - `REQUIRE_LABEL_KEY` pod metadata label (of key-value pair) that pod-reaper should require @@ -80,6 +81,10 @@ Sustained running: - do not use `RUN_DURATION` - manage the pod reaper via a deployment +### `EVICT` + +Use the [Eviction API](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/#eviction-api) instead of pod deletion when reaping pods. The Eviction API will honor the [disruption budget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) assigned to pods, and can for example be useful when reaping pods by duration to ensure that you don't reap all the pods of a specific deployment simultaneously, interrupting a published service. When a pod cannot be reaped due to a disruption budget, the reason will be logged as a warning. + ### `EXCLUDE_LABEL_KEY` and `EXCLUDE_LABEL_VALUES` These environment variables are used to build a label selector to exclude pods from reaping. The key must be a properly formed kubernetes label key. Values are a comma-separated (without whitespace) list of kubernetes label values. Setting exactly one of the key or values environment variables will result in an error. diff --git a/chart/pod-reaper/templates/rbac.yaml b/chart/pod-reaper/templates/rbac.yaml index a35d777..e1f4c50 100644 --- a/chart/pod-reaper/templates/rbac.yaml +++ b/chart/pod-reaper/templates/rbac.yaml @@ -16,6 +16,9 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["list", "delete"] +- apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] --- # binding the above cluster role (permissions) to the above service account diff --git a/reaper/options.go b/reaper/options.go index a105175..ca4ab68 100644 --- a/reaper/options.go +++ b/reaper/options.go @@ -25,6 +25,7 @@ const envRequireLabelValues = "REQUIRE_LABEL_VALUES" const envRequireAnnotationKey = "REQUIRE_ANNOTATION_KEY" const envRequireAnnotationValues = "REQUIRE_ANNOTATION_VALUES" const envDryRun = "DRY_RUN" +const envEvict = "EVICT" type options struct { namespace string @@ -36,6 +37,7 @@ type options struct { annotationRequirement *labels.Requirement dryRun bool rules rules.Rules + evict bool } func namespace() string { @@ -141,6 +143,14 @@ func dryRun() (bool, error) { return strconv.ParseBool(value) } +func evict() (bool, error) { + value, exists := os.LookupEnv(envEvict) + if !exists { + return false, nil + } + return strconv.ParseBool(value) +} + func loadOptions() (options options, err error) { options.namespace = namespace() options.gracePeriod, err = gracePeriod() @@ -169,6 +179,11 @@ func loadOptions() (options options, err error) { return options, err } + options.evict, err = evict() + if err != nil { + return options, err + } + // rules options.rules, err = rules.LoadRules() if err != nil { diff --git a/reaper/reaper.go b/reaper/reaper.go index 4afd444..0b8f873 100644 --- a/reaper/reaper.go +++ b/reaper/reaper.go @@ -6,6 +6,7 @@ import ( "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" @@ -92,7 +93,15 @@ func (reaper reaper) reapPod(pod v1.Pod, reasons []string) { "pod": pod.Name, "reasons": reasons, }).Info("reaping pod") - err := reaper.clientSet.CoreV1().Pods(pod.Namespace).Delete(pod.Name, deleteOptions) + var err error + if reaper.options.evict { + err = reaper.clientSet.CoreV1().Pods(pod.Namespace).Evict(&policyv1.Eviction{ + ObjectMeta: metav1.ObjectMeta{Namespace: pod.Namespace, Name: pod.Name}, + DeleteOptions: deleteOptions, + }) + } else { + err = reaper.clientSet.CoreV1().Pods(pod.Namespace).Delete(pod.Name, deleteOptions) + } if err != nil { // log the error, but continue on logrus.WithFields(logrus.Fields{