Fix snapshot creation issue. (#18783)

The renaming of files from oss -> ce caused incorrect snapshots
to be created due to ce writes now happening prior to ent writes.
When this happens various entities will attempt to be restored
from the snapshot prior to a partition existing and will cause a
panic to occur.

agent/consul/fsm/snapshot.go

@@ -18,6 +18,8 @@ import (
 	raftstorage "github.com/hashicorp/consul/internal/storage/raft"
 )
 
+var cePersister, entPersister persister
+
 var SnapshotSummaries = []prometheus.SummaryDefinition{
 	{
 		Name: []string{"fsm", "persist"},
@@ -44,15 +46,6 @@ type SnapshotHeader struct {
 // persister is a function used to help snapshot the FSM state.
 type persister func(s *snapshot, sink raft.SnapshotSink, encoder *codec.Encoder) error
 
-// persisters is a list of snapshot functions.
-var persisters []persister
-
-// registerPersister adds a new helper. This should be called at package
-// init() time.
-func registerPersister(fn persister) {
-	persisters = append(persisters, fn)
-}
-
 // restorer is a function used to load back a snapshot of the FSM state.
 type restorer func(header *SnapshotHeader, restore *state.Restore, decoder *codec.Decoder) error
 
@@ -86,7 +79,16 @@ func (s *snapshot) Persist(sink raft.SnapshotSink) error {
 	}
 
 	// Run all the persisters to write the FSM state.
-	for _, fn := range persisters {
+	for _, fn := range []persister{
+		// The enterprise version MUST be executed first, otherwise the snapshot will
+		// not properly function during restore due to missing tenancy objects.
+		entPersister,
+		cePersister,
+	} {
+		// Check for nil, since the enterprise version may not exist in CE.
+		if fn == nil {
+			continue
+		}
 		if err := fn(s, sink, encoder); err != nil {
 			sink.Cancel()
 			return err

agent/consul/fsm/snapshot_ce.go

@@ -18,8 +18,7 @@ import (
 )
 
 func init() {
-	registerPersister(persistCE)
-
+	cePersister = persistCE
 	registerRestorer(structs.RegisterRequestType, restoreRegistration)
 	registerRestorer(structs.KVSRequestType, restoreKV)
 	registerRestorer(structs.TombstoneRequestType, restoreTombstone)