feat #214: Force terminate if shutdown.timeout_seconds is defined

issues/issue_214/process-compose.yaml

@@ -0,0 +1,15 @@
+version: "0.5"
+log_level: info
+log_length: 300
+
+processes:
+  pc_log:
+    command: "tail -f -n100 process-compose-${USER}.log"
+    working_dir: "/tmp"
+    shutdown:
+      timeout_seconds: 5
+
+  sigterm_resistant:
+    command: "trap '' SIGTERM && sleep 60"
+    shutdown:
+      timeout_seconds: 5

src/app/process.go

@@ -53,6 +53,9 @@ type Process struct {
 	runCancelFn         context.CancelFunc
 	waitForPassCtx      context.Context
 	waitForPassCancelFn context.CancelFunc
+	mtxStopFn           sync.Mutex
+	waitForStoppedCtx   context.Context
+	waitForStoppedFn    context.CancelFunc
 	procColor           func(a ...interface{}) string
 	noColor             func(a ...interface{}) string
 	redColor            func(a ...interface{}) string
@@ -362,7 +365,9 @@ func (p *Process) stopProcess(cancelReadinessFuncs bool) error {
 	if !p.isRunning() {
 		log.Debug().Msgf("process %s is in state %s not shutting down", p.getName(), p.getStatusName())
 		// prevent pending process from running
-		p.onProcessEnd(types.ProcessStateTerminating)
+		if p.isOneOfStates(types.ProcessStatePending) {
+			p.onProcessEnd(types.ProcessStateTerminating)
+		}
 		return nil
 	}
 	p.setState(types.ProcessStateTerminating)
@@ -376,8 +381,26 @@ func (p *Process) stopProcess(cancelReadinessFuncs bool) error {
 	if isStringDefined(p.procConf.ShutDownParams.ShutDownCommand) {
 		return p.doConfiguredStop(p.procConf.ShutDownParams)
 	}
-
-	return p.command.Stop(p.procConf.ShutDownParams.Signal, p.procConf.ShutDownParams.ParentOnly)
+	err := p.command.Stop(p.procConf.ShutDownParams.Signal, p.procConf.ShutDownParams.ParentOnly)
+	if p.procConf.ShutDownParams.ShutDownTimeout == UndefinedShutdownTimeoutSec {
+		return err
+	}
+	p.mtxStopFn.Lock()
+	p.waitForStoppedCtx, p.waitForStoppedFn = context.WithTimeout(context.Background(), time.Duration(p.procConf.ShutDownParams.ShutDownTimeout)*time.Second)
+	p.mtxStopFn.Unlock()
+	select {
+	case <-p.waitForStoppedCtx.Done():
+		err = p.waitForStoppedCtx.Err()
+		switch {
+		case errors.Is(err, context.Canceled):
+			return nil
+		case errors.Is(err, context.DeadlineExceeded):
+			return p.command.Stop(int(syscall.SIGKILL), p.procConf.ShutDownParams.ParentOnly)
+		default:
+			log.Error().Err(err).Msgf("terminating %s with timeout %d failed", p.getName(), p.procConf.ShutDownParams.ShutDownTimeout)
+			return err
+		}
+	}
 }
 
 func (p *Process) doConfiguredStop(params types.ShutDownParams) error {
@@ -428,6 +451,12 @@ func (p *Process) onProcessEnd(state string) {
 	if isStringDefined(p.procConf.LogLocation) {
 		p.logger.Close()
 	}
+	p.mtxStopFn.Lock()
+	if p.waitForStoppedFn != nil {
+		p.waitForStoppedFn()
+		p.waitForStoppedFn = nil
+	}
+	p.mtxStopFn.Unlock()
 	p.stopProbes()
 	if p.readyProber != nil {
 		p.readyCancelFn()

src/app/system_test.go

@@ -10,6 +10,7 @@ import (
 	"reflect"
 	"slices"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 )
@@ -809,3 +810,87 @@ func TestUpdateProject(t *testing.T) {
 		t.Errorf("Process 'process1' status is %s want %s", updatedStatus, types.ProcessUpdateAdded)
 	}
 }
+
+func assertProcessStatus(t *testing.T, proc *Process, procName string, wantStatus string) {
+	t.Helper()
+	status := proc.getStatusName()
+	if status != wantStatus {
+		t.Fatalf("process %s status want %s got %s", procName, wantStatus, status)
+	}
+}
+
+func TestSystem_TestProcShutDownWithConfiguredTimeOut(t *testing.T) {
+	ignoresSigTerm := "IgnoresSIGTERM"
+	shell := command.DefaultShellConfig()
+	timeout := 3
+
+	project := &types.Project{
+		Processes: map[string]types.ProcessConfig{
+			ignoresSigTerm: {
+				Name:        ignoresSigTerm,
+				ReplicaName: ignoresSigTerm,
+				Executable:  shell.ShellCommand,
+				Args:        []string{shell.ShellArgument, ""},
+				ShutDownParams: types.ShutDownParams{
+					ShutDownTimeout: timeout,
+					Signal:          int(syscall.SIGTERM),
+				},
+			},
+		},
+		ShellConfig: shell,
+	}
+	t.Run("with timeout sigterm fail", func(t *testing.T) {
+		procConf := project.Processes[ignoresSigTerm]
+		procConf.Args[1] = "trap '' SIGTERM && sleep 60"
+		project.Processes[ignoresSigTerm] = procConf
+		runner, err := NewProjectRunner(&ProjectOpts{project: project})
+		if err != nil {
+			t.Fatalf("%s", err)
+		}
+		go runner.Run()
+		time.Sleep(100 * time.Millisecond)
+		proc := runner.getRunningProcess(ignoresSigTerm)
+		assertProcessStatus(t, proc, ignoresSigTerm, types.ProcessStateRunning)
+
+		// If the test fails, cleanup after ourselves
+		defer proc.command.Stop(int(syscall.SIGKILL), true)
+
+		go func() {
+			err = runner.StopProcess(ignoresSigTerm)
+			if err != nil {
+				t.Fatalf("%s", err)
+			}
+		}()
+
+		for i := 0; i < timeout-1; i++ {
+			time.Sleep(time.Second)
+			assertProcessStatus(t, proc, ignoresSigTerm, types.ProcessStateTerminating)
+		}
+
+		time.Sleep(2 * time.Second)
+		assertProcessStatus(t, proc, ignoresSigTerm, types.ProcessStateCompleted)
+	})
+
+	t.Run("with timeout sigterm success", func(t *testing.T) {
+		procConf := project.Processes[ignoresSigTerm]
+		procConf.Args[1] = "sleep 60"
+		project.Processes[ignoresSigTerm] = procConf
+		runner, err := NewProjectRunner(&ProjectOpts{project: project})
+		if err != nil {
+			t.Fatalf("%s", err)
+		}
+		go runner.Run()
+		time.Sleep(100 * time.Millisecond)
+		proc := runner.getRunningProcess(ignoresSigTerm)
+		assertProcessStatus(t, proc, ignoresSigTerm, types.ProcessStateRunning)
+		go func() {
+			err = runner.StopProcess(ignoresSigTerm)
+			if err != nil {
+				t.Fatalf("%s", err)
+			}
+		}()
+		time.Sleep(200 * time.Millisecond)
+		assertProcessStatus(t, proc, ignoresSigTerm, types.ProcessStateCompleted)
+	})
+
+}

src/tui/proc-table.go

@@ -183,7 +183,7 @@ func (pv *pcView) createProcTable() *tview.Table {
 		switch event.Key() {
 		case pv.shortcuts.ShortCutKeys[ActionProcessStop].key:
 			name := pv.getSelectedProcName()
-			pv.project.StopProcess(name)
+			go pv.handleProcessStopped(name)
 		case pv.shortcuts.ShortCutKeys[ActionProcessStart].key:
 			pv.startProcess()
 			pv.showPassIfNeeded()
@@ -458,3 +458,13 @@ func (pv *pcView) selectFirstEnabledProcess() {
 		}
 	}
 }
+
+func (pv *pcView) handleProcessStopped(name string) {
+	ctx, cancel := context.WithCancel(context.Background())
+	pv.showAutoProgress(ctx, time.Second*1)
+	err := pv.project.StopProcess(name)
+	cancel()
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to stop process")
+	}
+}

src/tui/stat-table.go

@@ -34,9 +34,11 @@ func (pv *pcView) createStatTable() *tview.Table {
 	table.SetCell(2, 1, pv.procCountCell)
 	table.SetCell(0, 2, tview.NewTableCell("").
 		SetSelectable(false).
+		SetAlign(tview.AlignCenter).
 		SetExpansion(0))
 	table.SetCell(1, 2, tview.NewTableCell("").
 		SetSelectable(false).
+		SetAlign(tview.AlignCenter).
 		SetExpansion(0))
 
 	table.SetCell(0, 3, tview.NewTableCell(pv.getPcTitle()).
@@ -102,7 +104,10 @@ func (pv *pcView) showAutoProgress(ctx context.Context, duration time.Duration)
 	go func() {
 		ticker := time.NewTicker(duration / time.Duration(full))
 		defer ticker.Stop()
-		defer pv.statTable.SetCell(1, 2, tview.NewTableCell(""))
+		defer pv.statTable.SetCell(1, 2, tview.NewTableCell("").
+			SetSelectable(false).
+			SetAlign(tview.AlignCenter).
+			SetExpansion(0))
 		for {
 			select {
 			case <-ctx.Done():
@@ -132,5 +137,8 @@ func (pv *pcView) showAutoProgress(ctx context.Context, duration time.Duration)
 }
 
 func (pv *pcView) hideAttentionMessage() {
-	pv.statTable.SetCell(0, 2, tview.NewTableCell(""))
+	pv.statTable.SetCell(0, 2, tview.NewTableCell("").
+		SetSelectable(false).
+		SetAlign(tview.AlignCenter).
+		SetExpansion(0))
 }

src/tui/styles.go

@@ -28,7 +28,6 @@ func (pv *pcView) setStatTableStyles(s *config.Styles) {
 		pv.statTable.GetCell(r, 1).SetTextColor(s.Style.StatTable.ValueFgColor.Color())
 	}
 	pv.statTable.GetCell(0, 3).SetTextColor(s.StatTable().LogoColor.Color())
-	pv.statTable.GetCell(1, 2).SetTextColor(s.Style.StatTable.KeyFgColor.Color())
 }
 
 func (pv *pcView) setProcTableStyles(s *config.Styles) {

www/docs/launcher.md

@@ -175,6 +175,8 @@ In case the `shutdown.command` is defined:
 2. Wait for `shutdown.timeout_seconds` for its completion (if not defined wait for 10 seconds)
 3. In case of timeout, the process group will receive the `SIGKILL` signal (irrespective of the `shutdown.parent_only` option).
 
+In case the `shutdown.timeout_seconds` is defined (without `shutdown.command`) and the process will fail to terminate within that time, the process group will receive the `SIGKILL` signal.
+
 ## Background (detached) Processes
 
 ```yaml hl_lines="4"