-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Backport of proxy-lifecycle: add HTTP Server with endpoints for proxy…
… lifecycle shutdown into release/1.1.x (#181) * backport of commit 937d893 * backport of commit 722f263 * backport of commit 2af6219 * backport of commit a1c21c9 * backport of commit 68f206d * backport of commit 892392d * backport of commit bb0f87a * backport of commit cde897a * backport of commit 471a087 * backport of commit 5b54f12 * backport of commit 2852040 * backport of commit bbb3785 * backport of commit c7e8f86 * backport of commit ae041fc * backport of commit 52e5fd5 * backport of commit 095aaf0 * backport of commit 2b0f0ee * backport of commit bf9acdb * backport of commit 9833553 * backport of commit f0dfd78 * backport of commit 7f9b0f0 * backport of commit 8c8141c * backport of commit f98ce24 * backport of commit 91a5b81 * backport of commit bfea751 * backport of commit aadfeed * backport of commit 496d196 * backport of commit 4340c2f * backport of commit 52b4557 * backport of commit 21595f0 * backport of commit b5e3aea * backport of commit bf8f0c8 * backport of commit 790881e * fix missing method --------- Co-authored-by: Mike Morris <mikemorris@users.noreply.github.com> Co-authored-by: Curt Bushko <cbushko@gmail.com>
- Loading branch information
3 people
authored
Jun 27, 2023
1 parent
38bb3f0
commit 954f060
Showing
11 changed files
with
595 additions
and
38 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| ```release-note:feature | ||
| Add HTTP server with configurable port and endpoint path for initiating graceful shutdown. | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,195 @@ | ||
| // Copyright (c) HashiCorp, Inc. | ||
| // SPDX-License-Identifier: MPL-2.0 | ||
|
|
||
| package consuldp | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "net/http" | ||
| "strconv" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/hashicorp/go-hclog" | ||
|
|
||
| "github.com/hashicorp/consul-dataplane/pkg/envoy" | ||
| ) | ||
|
|
||
| const ( | ||
| // defaultLifecycleBindPort is the port which will serve the proxy lifecycle HTTP | ||
| // endpoints on the loopback interface. | ||
| defaultLifecycleBindPort = "20300" | ||
| cdpLifecycleBindAddr = "127.0.0.1" | ||
| cdpLifecycleUrl = "http://" + cdpLifecycleBindAddr | ||
|
|
||
| defaultLifecycleShutdownPath = "/graceful_shutdown" | ||
| ) | ||
|
|
||
| // lifecycleConfig handles all configuration related to managing the Envoy proxy | ||
| // lifecycle, including exposing management controls via an HTTP server. | ||
| type lifecycleConfig struct { | ||
| logger hclog.Logger | ||
|
|
||
| // consuldp proxy lifecycle management config | ||
| shutdownDrainListenersEnabled bool | ||
| shutdownGracePeriodSeconds int | ||
| gracefulPort int | ||
| gracefulShutdownPath string | ||
|
|
||
| // manager for controlling the Envoy proxy process | ||
| proxy envoy.ProxyManager | ||
|
|
||
| // consuldp proxy lifecycle management server | ||
| lifecycleServer *http.Server | ||
|
|
||
| // consuldp proxy lifecycle server control | ||
| errorExitCh chan struct{} | ||
| running bool | ||
| mu sync.Mutex | ||
| } | ||
|
|
||
| func NewLifecycleConfig(cfg *Config, proxy envoy.ProxyManager) *lifecycleConfig { | ||
| return &lifecycleConfig{ | ||
| shutdownDrainListenersEnabled: cfg.Envoy.ShutdownDrainListenersEnabled, | ||
| shutdownGracePeriodSeconds: cfg.Envoy.ShutdownGracePeriodSeconds, | ||
| gracefulPort: cfg.Envoy.GracefulPort, | ||
| gracefulShutdownPath: cfg.Envoy.GracefulShutdownPath, | ||
|
|
||
| proxy: proxy, | ||
|
|
||
| errorExitCh: make(chan struct{}, 1), | ||
| mu: sync.Mutex{}, | ||
| } | ||
| } | ||
|
|
||
| func (m *lifecycleConfig) startLifecycleManager(ctx context.Context) error { | ||
| m.mu.Lock() | ||
| defer m.mu.Unlock() | ||
| if m.running { | ||
| return nil | ||
| } | ||
|
|
||
| m.logger = hclog.FromContext(ctx).Named("lifecycle") | ||
| m.running = true | ||
| go func() { | ||
| <-ctx.Done() | ||
| m.stopLifecycleServer() | ||
| }() | ||
|
|
||
| // Start the server which will expose HTTP endpoints for proxy lifecycle | ||
| // management control | ||
| mux := http.NewServeMux() | ||
|
|
||
| // Determine what HTTP endpoint paths to configure for the proxy lifecycle | ||
| // management server. These can be set as flags. | ||
| cdpLifecycleShutdownPath := defaultLifecycleShutdownPath | ||
| if m.gracefulShutdownPath != "" { | ||
| cdpLifecycleShutdownPath = m.gracefulShutdownPath | ||
| } | ||
|
|
||
| // Set config to allow introspection of default path for testing | ||
| m.gracefulShutdownPath = cdpLifecycleShutdownPath | ||
|
|
||
| m.logger.Info(fmt.Sprintf("setting graceful shutdown path: %s\n", cdpLifecycleShutdownPath)) | ||
| mux.HandleFunc(cdpLifecycleShutdownPath, m.gracefulShutdown) | ||
|
|
||
| // Determine what the proxy lifecycle management server bind port is. It can be | ||
| // set as a flag. | ||
| cdpLifecycleBindPort := defaultLifecycleBindPort | ||
| if m.gracefulPort != 0 { | ||
| cdpLifecycleBindPort = strconv.Itoa(m.gracefulPort) | ||
| } | ||
| m.lifecycleServer = &http.Server{ | ||
| Addr: fmt.Sprintf("%s:%s", cdpLifecycleBindAddr, cdpLifecycleBindPort), | ||
| Handler: mux, | ||
| } | ||
|
|
||
| // Start the proxy lifecycle management server | ||
| go m.startLifecycleServer() | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // startLifecycleServer starts the main proxy lifecycle management server that | ||
| // exposes HTTP endpoints for proxy lifecycle control. | ||
| func (m *lifecycleConfig) startLifecycleServer() { | ||
| m.logger.Info("starting proxy lifecycle management server", "address", m.lifecycleServer.Addr) | ||
| err := m.lifecycleServer.ListenAndServe() | ||
| if err != nil && err != http.ErrServerClosed { | ||
| m.logger.Error("failed to serve proxy lifecycle management requests", "error", err) | ||
| close(m.errorExitCh) | ||
| } | ||
| } | ||
|
|
||
| // stopLifecycleServer stops the consul dataplane proxy lifecycle server | ||
| func (m *lifecycleConfig) stopLifecycleServer() { | ||
| m.mu.Lock() | ||
| defer m.mu.Unlock() | ||
| m.running = false | ||
|
|
||
| if m.lifecycleServer != nil { | ||
| m.logger.Info("stopping the lifecycle management server") | ||
| err := m.lifecycleServer.Close() | ||
| if err != nil { | ||
| m.logger.Warn("error while closing lifecycle server", "error", err) | ||
| close(m.errorExitCh) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // lifecycleServerExited is used to signal that the lifecycle server | ||
| // recieved a signal to initiate shutdown. | ||
| func (m *lifecycleConfig) lifecycleServerExited() <-chan struct{} { | ||
| return m.errorExitCh | ||
| } | ||
|
|
||
| // gracefulShutdown blocks until shutdownGracePeriodSeconds seconds have elapsed, and, if | ||
| // configured, will drain inbound connections to Envoy listeners during that time. | ||
| func (m *lifecycleConfig) gracefulShutdown(rw http.ResponseWriter, _ *http.Request) { | ||
| m.logger.Info("initiating shutdown") | ||
|
|
||
| // Create a context that will signal a cancel at the specified duration. | ||
| // TODO: should this use lifecycleManager ctx instead of context.Background? | ||
| timeout := time.Duration(m.shutdownGracePeriodSeconds) * time.Second | ||
| ctx, cancel := context.WithTimeout(context.Background(), timeout) | ||
| defer cancel() | ||
|
|
||
| m.logger.Info(fmt.Sprintf("waiting %d seconds before terminating dataplane proxy", m.shutdownGracePeriodSeconds)) | ||
|
|
||
| var wg sync.WaitGroup | ||
| wg.Add(1) | ||
|
|
||
| go func() { | ||
| defer wg.Done() | ||
|
|
||
| // If shutdownDrainListenersEnabled, initiatie graceful shutdown of Envoy. | ||
| // We want to start draining connections from inbound listeners if | ||
| // configured, but still allow outbound traffic until gracefulShutdownPeriod | ||
| // has elapsed to facilitate a graceful application shutdown. | ||
| if m.shutdownDrainListenersEnabled { | ||
| err := m.proxy.Drain() | ||
| if err != nil { | ||
| m.logger.Warn("error while draining Envoy listeners", "error", err) | ||
| close(m.errorExitCh) | ||
| } | ||
| } | ||
|
|
||
| // Block until context timeout has elapsed | ||
| <-ctx.Done() | ||
|
|
||
| // Finish graceful shutdown, quit Envoy proxy | ||
| m.logger.Info("shutdown grace period timeout reached") | ||
| err := m.proxy.Quit() | ||
| if err != nil { | ||
| m.logger.Warn("error while shutting down Envoy", "error", err) | ||
| close(m.errorExitCh) | ||
| } | ||
| }() | ||
|
|
||
| // Wait for context timeout to elapse | ||
| wg.Wait() | ||
|
|
||
| // Return HTTP 200 Success | ||
| rw.WriteHeader(http.StatusOK) | ||
| } |
Oops, something went wrong.