Merge pull request #68 from bewing/evict

Allow use of Eviction API to reap pods

README.md

@@ -14,6 +14,7 @@ Pod-Reaper is configurable through environment variables. The pod-reaper specifi
 - `GRACE_PERIOD` duration that pods should be given to shut down before hard killing the pod
 - `SCHEDULE` schedule for when pod-reaper should look for pods to reap
 - `RUN_DURATION` how long pod-reaper should run before exiting
+- `EVICT` try to evict pods instead of deleting them
 - `EXCLUDE_LABEL_KEY` pod metadata label (of key-value pair) that pod-reaper should exclude
 - `EXCLUDE_LABEL_VALUES` comma-separated list of metadata label values (of key-value pair) that pod-reaper should exclude
 - `REQUIRE_LABEL_KEY` pod metadata label (of key-value pair) that pod-reaper should require
@@ -80,6 +81,10 @@ Sustained running:
 - do not use `RUN_DURATION`
 - manage the pod reaper via a deployment
 
+### `EVICT`
+
+Use the [Eviction API](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/#eviction-api) instead of pod deletion when reaping pods.  The Eviction API will honor the [disruption budget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) assigned to pods, and can for example be useful when reaping pods by duration to ensure that you don't reap all the pods of a specific deployment simultaneously, interrupting a published service.  When a pod cannot be reaped due to a disruption budget, the reason will be logged as a warning.
+
 ### `EXCLUDE_LABEL_KEY` and `EXCLUDE_LABEL_VALUES`
 
 These environment variables are used to build a label selector to exclude pods from reaping. The key must be a properly formed kubernetes label key. Values are a comma-separated (without whitespace) list of kubernetes label values. Setting exactly one of the key or values environment variables will result in an error.

chart/pod-reaper/templates/rbac.yaml

@@ -16,6 +16,9 @@ rules:
 - apiGroups: [""]
   resources: ["pods"]
   verbs: ["list", "delete"]
+- apiGroups: [""]
+  resources: ["pods/eviction"]
+  verbs: ["create"]
 
 ---
 # binding the above cluster role (permissions) to the above service account

reaper/options.go

@@ -25,6 +25,7 @@ const envRequireLabelValues = "REQUIRE_LABEL_VALUES"
 const envRequireAnnotationKey = "REQUIRE_ANNOTATION_KEY"
 const envRequireAnnotationValues = "REQUIRE_ANNOTATION_VALUES"
 const envDryRun = "DRY_RUN"
+const envEvict = "EVICT"
 
 type options struct {
 	namespace             string
@@ -36,6 +37,7 @@ type options struct {
 	annotationRequirement *labels.Requirement
 	dryRun                bool
 	rules                 rules.Rules
+	evict                 bool
 }
 
 func namespace() string {
@@ -141,6 +143,14 @@ func dryRun() (bool, error) {
 	return strconv.ParseBool(value)
 }
 
+func evict() (bool, error) {
+	value, exists := os.LookupEnv(envEvict)
+	if !exists {
+		return false, nil
+	}
+	return strconv.ParseBool(value)
+}
+
 func loadOptions() (options options, err error) {
 	options.namespace = namespace()
 	options.gracePeriod, err = gracePeriod()
@@ -169,6 +179,11 @@ func loadOptions() (options options, err error) {
 		return options, err
 	}
 
+	options.evict, err = evict()
+	if err != nil {
+		return options, err
+	}
+
 	// rules
 	options.rules, err = rules.LoadRules()
 	if err != nil {

reaper/reaper.go

@@ -6,6 +6,7 @@ import (
 	"github.com/robfig/cron/v3"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
+	policyv1 "k8s.io/api/policy/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
@@ -92,7 +93,15 @@ func (reaper reaper) reapPod(pod v1.Pod, reasons []string) {
 			"pod":     pod.Name,
 			"reasons": reasons,
 		}).Info("reaping pod")
-		err := reaper.clientSet.CoreV1().Pods(pod.Namespace).Delete(pod.Name, deleteOptions)
+		var err error
+		if reaper.options.evict {
+			err = reaper.clientSet.CoreV1().Pods(pod.Namespace).Evict(&policyv1.Eviction{
+				ObjectMeta:    metav1.ObjectMeta{Namespace: pod.Namespace, Name: pod.Name},
+				DeleteOptions: deleteOptions,
+			})
+		} else {
+			err = reaper.clientSet.CoreV1().Pods(pod.Namespace).Delete(pod.Name, deleteOptions)
+		}
 		if err != nil {
 			// log the error, but continue on
 			logrus.WithFields(logrus.Fields{