From 3c97bc5c53affcd0cbf1fa43d27f8683d3f8458f Mon Sep 17 00:00:00 2001 From: Saw-jan <saw.jan.grg3e@gmail.com> Date: Mon, 21 Aug 2023 09:55:49 +0545 Subject: [PATCH 1/6] ociswrapper: retry restarting ocis if it dies --- tests/ociswrapper/log/log.go | 6 +++++- tests/ociswrapper/ocis/ocis.go | 36 ++++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/tests/ociswrapper/log/log.go b/tests/ociswrapper/log/log.go index 6ecf3355ffd..1c2ce3da628 100644 --- a/tests/ociswrapper/log/log.go +++ b/tests/ociswrapper/log/log.go @@ -7,5 +7,9 @@ func Println(message string) { } func Panic(err error) { - log.Panic("[ociswrapper] ", err.Error()) + log.Panic("[ociswrapper]", err.Error()) +} + +func Fatalln(err error) { + log.Fatalln("[ociswrapper]", err.Error()) } diff --git a/tests/ociswrapper/ocis/ocis.go b/tests/ociswrapper/ocis/ocis.go index 785027d4c29..05f0bea5e96 100644 --- a/tests/ociswrapper/ocis/ocis.go +++ b/tests/ociswrapper/ocis/ocis.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "strconv" + "syscall" "time" "ociswrapper/common" @@ -58,23 +59,38 @@ func Start(envMap map[string]any) { for outputScanner.Scan() { m := outputScanner.Text() fmt.Println(m) - retryCount++ - - maxRetry, _ := strconv.Atoi(config.Get("retry")) - if retryCount <= maxRetry { - log.Println(fmt.Sprintf("Retry starting oCIS server... (retry %v)", retryCount)) - // Stop and start again - Stop() - Start(envMap) + } + + if err := cmd.Wait(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + status := exitErr.Sys().(syscall.WaitStatus) + // retry only if oCIS server exited with code > 0 + // -1 exit code means that the process was killed by a signal (cmd.Process.Kill()) + if status.ExitStatus() > 0 { + log.Println(fmt.Sprintf("oCIS server exited with code %v", status.ExitStatus())) + + // retry to start oCIS server + retryCount++ + retryCount++ + + retryCount++ + + maxRetry, _ := strconv.Atoi(config.Get("retry")) + if retryCount <= maxRetry { + log.Println(fmt.Sprintf("Retry starting oCIS server... (retry %v)", retryCount)) + // Stop and start again + Stop() + Start(envMap) + } + } } } - cmd.Wait() } func Stop() { err := cmd.Process.Kill() if err != nil { - log.Panic(err) + log.Fatalln(err) } cmd.Wait() } From 2b67d0c3823efb8689055364c17c09895733da2e Mon Sep 17 00:00:00 2001 From: Saw-jan <saw.jan.grg3e@gmail.com> Date: Thu, 21 Sep 2023 12:32:27 +0545 Subject: [PATCH 2/6] ociswrapper: do not panic if the process has already finished --- tests/ociswrapper/ocis/ocis.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/ociswrapper/ocis/ocis.go b/tests/ociswrapper/ocis/ocis.go index 05f0bea5e96..8f3648616b8 100644 --- a/tests/ociswrapper/ocis/ocis.go +++ b/tests/ociswrapper/ocis/ocis.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "strconv" + "strings" "syscall" "time" @@ -90,7 +91,9 @@ func Start(envMap map[string]any) { func Stop() { err := cmd.Process.Kill() if err != nil { - log.Fatalln(err) + if !strings.HasSuffix(err.Error(), "process already finished") { + log.Fatalln(err) + } } cmd.Wait() } From 6e6ae389638dff364007f9319c10fe7c12451e10 Mon Sep 17 00:00:00 2001 From: Saw-jan <saw.jan.grg3e@gmail.com> Date: Thu, 21 Sep 2023 12:33:12 +0545 Subject: [PATCH 3/6] ociswrapper: mark server ready if the server sends any response --- tests/ociswrapper/ocis/ocis.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/ociswrapper/ocis/ocis.go b/tests/ociswrapper/ocis/ocis.go index 8f3648616b8..459445abf53 100644 --- a/tests/ociswrapper/ocis/ocis.go +++ b/tests/ociswrapper/ocis/ocis.go @@ -124,7 +124,11 @@ func WaitForConnection() bool { req.Header.Set("X-Request-ID", "ociswrapper-"+strconv.Itoa(int(time.Now().UnixMilli()))) res, err := client.Do(req) - if err != nil || res.StatusCode != 200 { + if err != nil || res.StatusCode >= 400 { + if res != nil { + log.Println(fmt.Sprintf("oCIS server up but request failed: %v", res.StatusCode)) + return true + } // 500 milliseconds poll interval time.Sleep(500 * time.Millisecond) continue From 5d64f169b366bbff032d2b565b512b690d67e825 Mon Sep 17 00:00:00 2001 From: Saw-jan <saw.jan.grg3e@gmail.com> Date: Thu, 21 Sep 2023 12:39:06 +0545 Subject: [PATCH 4/6] cleanup mesh --- tests/ociswrapper/ocis/ocis.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/ociswrapper/ocis/ocis.go b/tests/ociswrapper/ocis/ocis.go index 459445abf53..a38f4b8168d 100644 --- a/tests/ociswrapper/ocis/ocis.go +++ b/tests/ociswrapper/ocis/ocis.go @@ -72,10 +72,6 @@ func Start(envMap map[string]any) { // retry to start oCIS server retryCount++ - retryCount++ - - retryCount++ - maxRetry, _ := strconv.Atoi(config.Get("retry")) if retryCount <= maxRetry { log.Println(fmt.Sprintf("Retry starting oCIS server... (retry %v)", retryCount)) From 3c364742c83a862918bcdf74085f3aeb55f9a5c2 Mon Sep 17 00:00:00 2001 From: Saw-jan <saw.jan.grg3e@gmail.com> Date: Thu, 21 Sep 2023 13:17:21 +0545 Subject: [PATCH 5/6] ociswrapper: meaningful error message --- tests/ociswrapper/wrapper/handlers/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ociswrapper/wrapper/handlers/handler.go b/tests/ociswrapper/wrapper/handlers/handler.go index e331a24062c..09e4c64e054 100644 --- a/tests/ociswrapper/wrapper/handlers/handler.go +++ b/tests/ociswrapper/wrapper/handlers/handler.go @@ -31,7 +31,7 @@ func sendResponse(res http.ResponseWriter, ocisStatus bool) { } else { res.WriteHeader(http.StatusInternalServerError) resBody["status"] = "ERROR" - resBody["message"] = "oCIS server error" + resBody["message"] = "Unable to start oCIS server" } res.Header().Set("Content-Type", "application/json") From d4be6a83a7c4e2fa094f6f7fcbfac865aa8c7845 Mon Sep 17 00:00:00 2001 From: Saw-jan <saw.jan.grg3e@gmail.com> Date: Thu, 21 Sep 2023 17:00:29 +0545 Subject: [PATCH 6/6] ociswrapper: break wait if server responds with 200 --- tests/ociswrapper/ocis/ocis.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/ociswrapper/ocis/ocis.go b/tests/ociswrapper/ocis/ocis.go index a38f4b8168d..0d9ff319b2c 100644 --- a/tests/ociswrapper/ocis/ocis.go +++ b/tests/ociswrapper/ocis/ocis.go @@ -120,11 +120,7 @@ func WaitForConnection() bool { req.Header.Set("X-Request-ID", "ociswrapper-"+strconv.Itoa(int(time.Now().UnixMilli()))) res, err := client.Do(req) - if err != nil || res.StatusCode >= 400 { - if res != nil { - log.Println(fmt.Sprintf("oCIS server up but request failed: %v", res.StatusCode)) - return true - } + if err != nil || res.StatusCode != 200 { // 500 milliseconds poll interval time.Sleep(500 * time.Millisecond) continue