Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: implement graceful shutdown for api server #15938

Closed
wants to merge 5 commits into from
Closed

fix: implement graceful shutdown for api server #15938

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
697207d
fix: implement graceful shutdown for api server
leoluz Oct 13, 2023
ca76022
Shutdown servers in parallel
leoluz Oct 13, 2023
6ec6efd
add noop shutdown function on api server constructor
leoluz Oct 13, 2023
7fde5eb
fix waitgroup
leoluz Oct 13, 2023
3eb12d7
Graceful api server shutdown tweaks (#4)
crenshaw-dev Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cmd/argocd-server/commands/argocd_server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,8 @@ func NewCommand() *cobra.Command {
if closer != nil {
closer()
}
log.Info("API Server successfully shut down.")
break
Comment on lines +212 to +213
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess this loop can go away completely now.

}
},
}
Expand Down
117 changes: 98 additions & 19 deletions server/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@ import (
"net/url"
"os"
"os/exec"
"os/signal"
"path"
"path/filepath"
"reflect"
"regexp"
go_runtime "runtime"
"strings"
gosync "sync"
"syscall"
"time"

// nolint:staticcheck
Expand Down Expand Up @@ -181,8 +183,9 @@ type ArgoCDServer struct {
appsetLister applisters.ApplicationSetNamespaceLister
db db.ArgoDB

// stopCh is the channel which when closed, will shutdown the Argo CD server
stopCh chan struct{}
// stopCh is the channel which when closed, will gracefully
//shutdown the Argo CD server
stopCh chan os.Signal
userStateStorage util_session.UserStateStorage
indexDataInit gosync.Once
indexData []byte
Expand All @@ -193,6 +196,7 @@ type ArgoCDServer struct {
configMapInformer cache.SharedIndexInformer
serviceSet *ArgoCDServiceSet
extensionManager *extension.Manager
Shutdown func()
}

type ArgoCDServerOpts struct {
Expand Down Expand Up @@ -298,6 +302,10 @@ func NewServer(ctx context.Context, opts ArgoCDServerOpts) *ArgoCDServer {
pg := extension.NewDefaultProjectGetter(projLister, dbInstance)
em := extension.NewManager(logger, sg, ag, pg, enf)

noopShutdown := func() {
log.Error("API Server Shutdown function called but server is not started yet.")
}

a := &ArgoCDServer{
ArgoCDServerOpts: opts,
log: logger,
Expand All @@ -319,6 +327,7 @@ func NewServer(ctx context.Context, opts ArgoCDServerOpts) *ArgoCDServer {
secretInformer: secretInformer,
configMapInformer: configMapInformer,
extensionManager: em,
Shutdown: noopShutdown,
}

err = a.logInClusterWarnings()
Expand Down Expand Up @@ -560,8 +569,89 @@ func (a *ArgoCDServer) Run(ctx context.Context, listeners *Listeners) {
log.Fatal("Timed out waiting for project cache to sync")
}

a.stopCh = make(chan struct{})
<-a.stopCh
shutdownFunc := func() {
log.Info("API Server shutdown initiated. Shutting down servers...")
sCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
var wg gosync.WaitGroup

// Shutdown http server
wg.Add(1)
go func() {
defer wg.Done()
err := httpS.Shutdown(sCtx)
if err != nil {
log.Errorf("Error shutting down http server: %s", err)
}
}()

if httpsS != nil {
// Shutdown https server
wg.Add(1)
go func() {
defer wg.Done()
err := httpsS.Shutdown(sCtx)
if err != nil {
log.Errorf("Error shutting down https server: %s", err)
}
}()
}

// Shutdown gRPC server
wg.Add(1)
go func() {
defer wg.Done()
grpcS.GracefulStop()
}()

// Shutdown metrics server
wg.Add(1)
go func() {
defer wg.Done()
err := metricsServ.Shutdown(sCtx)
if err != nil {
log.Errorf("Error shutting down metrics server: %s", err)
}
}()

if tlsm != nil {
// Shutdown tls server
wg.Add(1)
go func() {
defer wg.Done()
tlsm.Close()
}()
}

// Shutdown tcp server
wg.Add(1)
go func() {
defer wg.Done()
tcpm.Close()
}()

c := make(chan struct{})
// This goroutine will wait for all servers to conclude the shutdown
// process
go func() {
defer close(c)
wg.Wait()
}()

select {
case <-c:
log.Info("All servers were gracefully shutdown. Exiting...")
case <-sCtx.Done():
log.Warn("Graceful shutdown timeout. Exiting...")
}
}
a.Shutdown = shutdownFunc

a.stopCh = make(chan os.Signal, 1)
signal.Notify(a.stopCh, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
signal := <-a.stopCh
log.Infof("API Server received signal: %s", signal.String())
a.Shutdown()
}

func (a *ArgoCDServer) Initialized() bool {
Expand All @@ -571,24 +661,13 @@ func (a *ArgoCDServer) Initialized() bool {
// checkServeErr checks the error from a .Serve() call to decide if it was a graceful shutdown
func (a *ArgoCDServer) checkServeErr(name string, err error) {
if err != nil {
if a.stopCh == nil {
// a nil stopCh indicates a graceful shutdown
log.Infof("graceful shutdown %s: %v", name, err)
if err == http.ErrServerClosed {
log.Infof("Graceful shutdown of %s initiated", name)
} else {
log.Fatalf("%s: %v", name, err)
log.Errorf("Error received from server %s: %v", name, err)
}
} else {
log.Infof("graceful shutdown %s", name)
}
}

// Shutdown stops the Argo CD server
func (a *ArgoCDServer) Shutdown() {
log.Info("Shut down requested")
stopCh := a.stopCh
a.stopCh = nil
if stopCh != nil {
close(stopCh)
log.Infof("Graceful shutdown of %s initiated", name)
}
}

Expand Down
Loading