diff --git a/changelog/15156.txt b/changelog/15156.txt new file mode 100644 index 000000000000..8718f1b1a81c --- /dev/null +++ b/changelog/15156.txt @@ -0,0 +1,3 @@ +```release-note:bug +rafft: fix Raft TLS key rotation panic that occurs if active key is more than 24 hours old +``` diff --git a/vault/raft.go b/vault/raft.go index c3de8f5b2547..daa919ea3414 100644 --- a/vault/raft.go +++ b/vault/raft.go @@ -473,27 +473,33 @@ func (c *Core) raftTLSRotatePhased(ctx context.Context, logger hclog.Logger, raf return errors.New("no active raft TLS key found") } + getNextRotationTime := func(next time.Time) time.Time { + now := time.Now() + + // active key's CreatedTime + raftTLSRotationPeriod might be in + // the past (meaning it is ready to be rotated) which will cause + // NewTicker to panic when used with time.Until, prevent this by + // pushing out rotation time into very near future + if next.Before(now) { + return now.Add(1 * time.Minute) + } + + // push out to ensure proposed time does not elapse + return next.Add(10 * time.Second) + } + // Start the process in a go routine go func() { - nextRotationTime := activeKey.CreatedTime.Add(raftTLSRotationPeriod) + nextRotationTime := getNextRotationTime(activeKey.CreatedTime.Add(raftTLSRotationPeriod)) keyCheckInterval := time.NewTicker(1 * time.Minute) defer keyCheckInterval.Stop() - var backoff bool // ticker is used to prevent memory leak of using time.After in // for - select pattern. ticker := time.NewTicker(time.Until(nextRotationTime)) defer ticker.Stop() for { - // If we encountered and error we should try to create the key - // again. - if backoff { - nextRotationTime = time.Now().Add(10 * time.Second) - backoff = false - } - - ticker.Reset(time.Until(nextRotationTime)) select { case <-keyCheckInterval.C: err := checkCommitted() @@ -505,11 +511,12 @@ func (c *Core) raftTLSRotatePhased(ctx context.Context, logger hclog.Logger, raf next, err := rotateKeyring() if err != nil { logger.Error("failed to rotate TLS key", "error", err) - backoff = true - continue + nextRotationTime = time.Now().Add(10 * time.Second) + } else { + nextRotationTime = getNextRotationTime(next) } - nextRotationTime = next + ticker.Reset(time.Until(nextRotationTime)) case <-stopCh: return