From 34f7a68c8fa4d5b804aec9bad2e997b8e66f28e0 Mon Sep 17 00:00:00 2001 From: Louis Garman Date: Fri, 22 Jan 2021 07:50:02 +0000 Subject: [PATCH] Record exit code in run status --- api/etok.dev/v1alpha1/run_types.go | 3 ++ cmd/launcher/launcher_test.go | 9 ++++- config/crd/bases/etok.dev_runs.yaml | 3 ++ pkg/controllers/run_controller.go | 45 +++++++++++++++++++++---- pkg/controllers/run_controller_test.go | 11 ++++++ pkg/controllers/workspace_controller.go | 5 +-- pkg/testobj/k8s.go | 5 +-- 7 files changed, 68 insertions(+), 13 deletions(-) diff --git a/api/etok.dev/v1alpha1/run_types.go b/api/etok.dev/v1alpha1/run_types.go index 9b9c8ec4..ee01cf18 100644 --- a/api/etok.dev/v1alpha1/run_types.go +++ b/api/etok.dev/v1alpha1/run_types.go @@ -119,6 +119,9 @@ type RunStatus struct { Phase RunPhase `json:"phase,omitempty"` Conditions []metav1.Condition `json:"conditions,omitempty"` + + // Exit code of run pod's runner container + ExitCode *int `json:"exitCode,omitempty"` } func (r *Run) IsReconciled() bool { diff --git a/cmd/launcher/launcher_test.go b/cmd/launcher/launcher_test.go index 0e51e477..860a9399 100644 --- a/cmd/launcher/launcher_test.go +++ b/cmd/launcher/launcher_test.go @@ -352,7 +352,14 @@ func mockControllers(t *testutil.T, f *cmdutil.Factory, o *launcherOptions, phas createPodAction := func(action testcore.Action) (bool, runtime.Object, error) { run := action.(testcore.CreateAction).GetObject().(*v1alpha1.Run) - pod := testobj.RunPod(run.Namespace, run.Name, testobj.WithPhase(phase), testobj.WithRunnerExitCode(exitCode)) + var pod *corev1.Pod + // Only set phase if non-empty + if phase != "" { + pod = testobj.RunPod(run.Namespace, run.Name, testobj.WithPhase(phase), testobj.WithRunnerExitCode(exitCode)) + } else { + pod = testobj.RunPod(run.Namespace, run.Name, testobj.WithRunnerExitCode(exitCode)) + } + _, err := o.PodsClient(run.Namespace).Create(context.Background(), pod, metav1.CreateOptions{}) require.NoError(t, err) diff --git a/config/crd/bases/etok.dev_runs.yaml b/config/crd/bases/etok.dev_runs.yaml index 85621067..75973acb 100644 --- a/config/crd/bases/etok.dev_runs.yaml +++ b/config/crd/bases/etok.dev_runs.yaml @@ -184,6 +184,9 @@ spec: - type type: object type: array + exitCode: + description: Exit code of run pod's runner container + type: integer phase: description: Current phase of the run's lifecycle. type: string diff --git a/pkg/controllers/run_controller.go b/pkg/controllers/run_controller.go index 36366c43..ac0ab397 100644 --- a/pkg/controllers/run_controller.go +++ b/pkg/controllers/run_controller.go @@ -2,10 +2,13 @@ package controllers import ( "context" + "errors" "time" v1alpha1 "github.com/leg100/etok/api/etok.dev/v1alpha1" "github.com/leg100/etok/cmd/launcher" + "github.com/leg100/etok/pkg/globals" + "github.com/leg100/etok/pkg/k8s" "github.com/leg100/etok/pkg/scheme" "github.com/leg100/etok/pkg/util/slice" corev1 "k8s.io/api/core/v1" @@ -258,18 +261,48 @@ func (r *RunReconciler) managePod(ctx context.Context, run *v1alpha1.Run, ws v1a return nil, err } - switch pod.Status.Phase { + var isCompleted = metav1.ConditionFalse + + if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + // Record exit code in run status + code, err := getExitCode(&pod) + if err != nil { + return nil, errors.New("unable to retrieve container status") + } + run.RunStatus.ExitCode = &code + + isCompleted = metav1.ConditionTrue + } + + return &metav1.Condition{ + Type: v1alpha1.RunCompleteCondition, + Status: isCompleted, + Reason: getReasonFromPodPhase(pod.Status.Phase), + }, nil +} + +// Translate pod phase to a reason string for the run completed condition +func getReasonFromPodPhase(phase corev1.PodPhase) string { + switch phase { case corev1.PodSucceeded: - return runComplete(v1alpha1.PodSucceededReason, ""), nil + return v1alpha1.PodSucceededReason case corev1.PodFailed: - return runComplete(v1alpha1.PodFailedReason, ""), nil + return v1alpha1.PodFailedReason case corev1.PodRunning: - return runIncomplete(v1alpha1.PodRunningReason, ""), nil + return v1alpha1.PodRunningReason case corev1.PodPending: - return runIncomplete(v1alpha1.PodPendingReason, ""), nil + return v1alpha1.PodPendingReason default: - return runIncomplete(v1alpha1.PodUnknownReason, ""), nil + return v1alpha1.PodUnknownReason + } +} + +func getExitCode(pod *corev1.Pod) (int, error) { + status := k8s.ContainerStatusByName(pod, globals.RunnerContainerName) + if status == nil { + return 0, errors.New("unable to retrieve container status") } + return int(status.State.Terminated.ExitCode), nil } func (r *RunReconciler) setOwnerOfArchive(ctx context.Context, run *v1alpha1.Run) error { diff --git a/pkg/controllers/run_controller_test.go b/pkg/controllers/run_controller_test.go index afd558b2..a94d80e4 100644 --- a/pkg/controllers/run_controller_test.go +++ b/pkg/controllers/run_controller_test.go @@ -159,6 +159,17 @@ func TestRunReconciler(t *testing.T) { assert.Equal(t, "plan-1", archive.OwnerReferences[0].Name) }, }, + { + name: "Exit code recorded in status", + run: testobj.Run("operator-test", "plan-1", "plan", testobj.WithWorkspace("workspace-1")), + objs: []runtime.Object{ + testobj.Workspace("operator-test", "workspace-1", testobj.WithSecret("secret-1")), + testobj.RunPod("operator-test", "plan-1", testobj.WithPhase(corev1.PodSucceeded), testobj.WithRunnerExitCode(5)), + }, + runAssertions: func(t *testutil.T, run *v1alpha1.Run) { + assert.Equal(t, 5, *run.RunStatus.ExitCode) + }, + }, } for _, tt := range tests { testutil.Run(t, tt.name, func(t *testutil.T) { diff --git a/pkg/controllers/workspace_controller.go b/pkg/controllers/workspace_controller.go index c02dd04b..a6bbfc79 100644 --- a/pkg/controllers/workspace_controller.go +++ b/pkg/controllers/workspace_controller.go @@ -459,8 +459,9 @@ func (r *WorkspaceReconciler) restore(ctx context.Context, ws *v1alpha1.Workspac return nil, err } - // Blank out resource version to avoid error upon create + // Blank out certain fields to avoid errors upon create secret.ResourceVersion = "" + secret.OwnerReferences = nil if err := r.Create(ctx, &secret); err != nil { r.recorder.Eventf(ws, "Warning", "RestoreError", "Error received when trying to restore state: %w", err) @@ -673,7 +674,7 @@ func (r *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error { blder = blder.Owns(&corev1.ConfigMap{}) // Watch terraform state files - blder = blder.Watches(&source.Kind{Type: &corev1.ConfigMap{}}, handler.EnqueueRequestsFromMapFunc(func(o client.Object) []ctrl.Request { + blder = blder.Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(func(o client.Object) []ctrl.Request { var isStateFile bool for k, v := range o.GetLabels() { if k == "tfstate" && v == "true" { diff --git a/pkg/testobj/k8s.go b/pkg/testobj/k8s.go index d9a1669c..fb781f53 100644 --- a/pkg/testobj/k8s.go +++ b/pkg/testobj/k8s.go @@ -201,10 +201,7 @@ func WorkspacePod(namespace, name string, opts ...func(*corev1.Pod)) *corev1.Pod func WithPhase(phase corev1.PodPhase) func(*corev1.Pod) { return func(pod *corev1.Pod) { - // Only set a phase if non-empty - if phase != "" { - pod.Status.Phase = phase - } + pod.Status.Phase = phase } }