Seal HA Improvements, CE side (#25171)

* add fully_wrapped to seal-backend-status, try to find in-common seals in all cases

* changelog

Author: sgmiller

changelog/25171.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core (enterprise): Improve seal unwrap performance when in degraded mode with one or more unhealthy seals.
+```

vault/logical_system.go

@@ -5420,6 +5420,7 @@ type SealBackendStatusResponse struct {
 	Healthy        bool                `json:"healthy"`
 	UnhealthySince string              `json:"unhealthy_since,omitempty"`
 	Backends       []SealBackendStatus `json:"backends"`
+	FullyWrapped   bool                `json:"fully_wrapped"`
 }
 
 func (core *Core) GetSealStatus(ctx context.Context, lock bool) (*SealStatusResponse, error) {
@@ -5545,6 +5546,13 @@ func (c *Core) GetSealBackendStatus(ctx context.Context) (*SealBackendStatusResp
 		}
 		r.Healthy = true
 	}
+
+	pps, err := GetPartiallySealWrappedPaths(ctx, c.physical)
+	if err != nil {
+		return nil, fmt.Errorf("could not list partially seal wrapped values: %w", err)
+	}
+	genInfo := c.seal.GetAccess().GetSealGenerationInfo()
+	r.FullyWrapped = genInfo.IsRewrapped() && len(pps) == 0
 	return &r, nil
 }
 

vault/seal/seal.go

@@ -724,26 +724,31 @@ func (a *access) Decrypt(ctx context.Context, ciphertext *MultiWrapValue, option
 	}
 
 	// Start goroutines to decrypt the value
-
 	first := wrappersByPriority[0]
-	// First, if we only have one slot, try matching by keyId
-	if len(blobInfoMap) == 1 {
-	outer:
-		for k := range blobInfoMap {
-			for _, sealWrapper := range wrappersByPriority {
-				keyId, err := sealWrapper.Wrapper.KeyId(ctx)
-				if err != nil {
-					resultWg.Add(1)
-					go reportResult(sealWrapper.Name, nil, false, err)
-					continue
-				}
-				if keyId == k {
-					first = sealWrapper
-					break outer
-				}
+	found := false
+outer:
+	// This loop finds the highest priority seal with a keyId in common with the blobInfoMap,
+	// and ensures we'll use it first.  This should equal the highest priority wrapper in the nominal
+	// case, but may not if a seal is unhealthy.  This ensures we try the highest priority healthy
+	// seal first if available, and warn if we don't think we have one in common.
+	for k := range blobInfoMap {
+		for _, sealWrapper := range wrappersByPriority {
+			keyId, err := sealWrapper.Wrapper.KeyId(ctx)
+			if err != nil {
+				resultWg.Add(1)
+				go reportResult(sealWrapper.Name, nil, false, err)
+				continue
+			}
+			if keyId == k {
+				found = true
+				first = sealWrapper
+				break outer
 			}
 		}
 	}
+	if !found {
+		a.logger.Warn("while unwrapping, value has no key-id in common with currently healthy seals.  Trying all healthy seals")
+	}
 
 	resultWg.Add(1)
 	go decrypt(first)