From d9a99c9fcb35a0f4193725581bbb216573749c1a Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Thu, 2 May 2024 00:30:05 -0400
Subject: [PATCH 01/60] trigger epoch fallback mode manually in test

---
 integration/tests/epochs/base_suite.go        | 35 +++++++++++++++++++
 .../epochs/dynamic_epoch_transition_suite.go  | 28 ---------------
 .../recover_epoch/recover_epoch_efm_test.go   | 28 ++++++++++-----
 3 files changed, 55 insertions(+), 36 deletions(-)

diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index 35e5fd2f15e..0b9db8f26f6 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -117,6 +117,13 @@ func (s *BaseSuite) SetupTest() {
 	go lib.LogStatusPeriodically(s.T(), s.Ctx, s.log, s.Client, 5*time.Second)
 }
 
+func (s *BaseSuite) TearDownTest() {
+	s.log.Info().Msg("================> Start TearDownTest")
+	s.net.Remove()
+	s.cancel()
+	s.log.Info().Msg("================> Finish TearDownTest")
+}
+
 func (s *BaseSuite) Ghost() *client.GhostClient {
 	client, err := s.net.ContainerByID(s.ghostID).GhostClient()
 	require.NoError(s.T(), err, "could not get ghost Client")
@@ -155,3 +162,31 @@ func (s *BaseSuite) GetContainersByRole(role flow.Role) []*testnet.Container {
 	require.True(s.T(), len(nodes) > 0)
 	return nodes
 }
+
+// AwaitFinalizedView polls until it observes that the latest finalized block has a view
+// greater than or equal to the input view. This is used to wait until when an epoch
+// transition must have happened.
+func (s *BaseSuite) AwaitFinalizedView(ctx context.Context, view uint64, waitFor, tick time.Duration) {
+	require.Eventually(s.T(), func() bool {
+		sealed := s.getLatestFinalizedHeader(ctx)
+		return sealed.View >= view
+	}, waitFor, tick)
+}
+
+// getLatestFinalizedHeader retrieves the latest finalized block, as reported in LatestSnapshot.
+func (s *BaseSuite) getLatestFinalizedHeader(ctx context.Context) *flow.Header {
+	snapshot, err := s.Client.GetLatestProtocolSnapshot(ctx)
+	require.NoError(s.T(), err)
+	finalized, err := snapshot.Head()
+	require.NoError(s.T(), err)
+	return finalized
+}
+
+// AssertInEpoch requires actual epoch counter is equal to counter provided.
+func (s *BaseSuite) AssertInEpoch(ctx context.Context, expectedEpoch uint64) {
+	snapshot, err := s.Client.GetLatestProtocolSnapshot(ctx)
+	require.NoError(s.T(), err)
+	actualEpoch, err := snapshot.Epochs().Current().Counter()
+	require.NoError(s.T(), err)
+	require.Equalf(s.T(), expectedEpoch, actualEpoch, "expected to be in epoch %d got %d", expectedEpoch, actualEpoch)
+}
diff --git a/integration/tests/epochs/dynamic_epoch_transition_suite.go b/integration/tests/epochs/dynamic_epoch_transition_suite.go
index a8954b7ec42..440695f4cc1 100644
--- a/integration/tests/epochs/dynamic_epoch_transition_suite.go
+++ b/integration/tests/epochs/dynamic_epoch_transition_suite.go
@@ -341,25 +341,6 @@ func (s *DynamicEpochTransitionSuite) StakeNewNode(ctx context.Context, env temp
 	return info, testContainer
 }
 
-// AwaitFinalizedView polls until it observes that the latest finalized block has a view
-// greater than or equal to the input view. This is used to wait until when an epoch
-// transition must have happened.
-func (s *DynamicEpochTransitionSuite) AwaitFinalizedView(ctx context.Context, view uint64, waitFor, tick time.Duration) {
-	require.Eventually(s.T(), func() bool {
-		sealed := s.getLatestFinalizedHeader(ctx)
-		return sealed.View >= view
-	}, waitFor, tick)
-}
-
-// getLatestFinalizedHeader retrieves the latest finalized block, as reported in LatestSnapshot.
-func (s *DynamicEpochTransitionSuite) getLatestFinalizedHeader(ctx context.Context) *flow.Header {
-	snapshot, err := s.Client.GetLatestProtocolSnapshot(ctx)
-	require.NoError(s.T(), err)
-	finalized, err := snapshot.Head()
-	require.NoError(s.T(), err)
-	return finalized
-}
-
 // AssertInEpochPhase checks if we are in the phase of the given epoch.
 func (s *DynamicEpochTransitionSuite) AssertInEpochPhase(ctx context.Context, expectedEpoch uint64, expectedPhase flow.EpochPhase) {
 	snapshot, err := s.Client.GetLatestProtocolSnapshot(ctx)
@@ -376,15 +357,6 @@ func (s *DynamicEpochTransitionSuite) AssertInEpochPhase(ctx context.Context, ex
 	s.TimedLogf("asserted in epoch %d, phase %s, finalized height/view: %d/%d", expectedEpoch, expectedPhase, head.Height, head.View)
 }
 
-// AssertInEpoch requires actual epoch counter is equal to counter provided.
-func (s *DynamicEpochTransitionSuite) AssertInEpoch(ctx context.Context, expectedEpoch uint64) {
-	snapshot, err := s.Client.GetLatestProtocolSnapshot(ctx)
-	require.NoError(s.T(), err)
-	actualEpoch, err := snapshot.Epochs().Current().Counter()
-	require.NoError(s.T(), err)
-	require.Equalf(s.T(), expectedEpoch, actualEpoch, "expected to be in epoch %d got %d", expectedEpoch, actualEpoch)
-}
-
 // AssertNodeNotParticipantInEpoch asserts that the given node ID does not exist
 // in the epoch's identity table.
 func (s *DynamicEpochTransitionSuite) AssertNodeNotParticipantInEpoch(epoch protocol.Epoch, nodeID flow.Identifier) {
diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 6de2caaba21..5a6b866beed 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -1,13 +1,12 @@
 package recover_epoch
 
 import (
-	"context"
-	"fmt"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
-
+	
 	"github.com/onflow/flow-go/model/flow"
 )
 
@@ -24,11 +23,24 @@ type RecoverEpochSuite struct {
 // CLI command to generate transaction arguments to submit a recover_epoch transaction, after submitting the transaction the test will
 // ensure the network is healthy.
 func (s *RecoverEpochSuite) TestRecoverEpoch() {
-	s.AwaitEpochPhase(context.Background(), 0, flow.EpochPhaseSetup, 20*time.Second, time.Second)
-	fmt.Println("in epoch phase setup")
+	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, 30*time.Second, 4*time.Second)
+
+	// pausing execution node will force the network into EFM
+	_ = s.GetContainersByRole(flow.RoleExecution)[0].Pause()
+
+	// get the latest snapshot and start new container with it
+	rootSnapshot, err := s.Client.GetLatestProtocolSnapshot(s.Ctx)
+	require.NoError(s.T(), err)
+
+	epoch1FinalView, err := rootSnapshot.Epochs().Current().FinalView()
+	require.NoError(s.T(), err)
 
-	sns := s.GetContainersByRole(flow.RoleConsensus)
-	_ = sns[0].Pause()
+	// wait for at least the first block of the next epoch to be sealed before we pause our container to replace
+	s.TimedLogf("waiting for epoch transition (finalized view %d) before pausing container", epoch1FinalView+1)
+	s.AwaitFinalizedView(s.Ctx, epoch1FinalView+1, 2*time.Minute, 500*time.Millisecond)
+	s.TimedLogf("observed finalized view %d -> pausing container", epoch1FinalView+1)
 
-	// @TODO: trigger EFM manually
+	// assert transition to second epoch did not happen
+	// if counter is still 0, epoch emergency fallback was triggered as expected
+	s.AssertInEpoch(s.Ctx, 0)
 }

From ebe557a16848e8fa7800e461a3607a01ad3b3c3a Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Wed, 8 May 2024 10:27:04 -0400
Subject: [PATCH 02/60] add flags

- access-address: address of access node used to retrieve snapshot
- access-network-key: network key of the access node
- insecure: use insecure API endpoint
- out: output dir for json file
---
 cmd/util/cmd/epochs/cmd/recover.go | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index 1e24d3a5460..61da8a9c6aa 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -40,8 +40,10 @@ This recovery process has some constraints:
 		Run: generateRecoverEpochTxArgs(getSnapshot),
 	}
 
+	flagOutputDir                string
 	flagAnAddress                string
 	flagAnPubkey                 string
+	flagAnInsecure               bool
 	flagInternalNodePrivInfoDir  string
 	flagNodeConfigJson           string
 	flagCollectionClusters       int
@@ -59,6 +61,10 @@ func init() {
 }
 
 func addGenerateRecoverEpochTxArgsCmdFlags() error {
+	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagOutputDir, "out", "", "the path to the output dir")
+	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnAddress, "access-address", "", "the address of the access node used for client connections")
+	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnPubkey, "access-network-key", "", "the network key of the access node used for client connections in hex string format")
+	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagAnInsecure, "insecure", true, "set to true if the protocol snapshot should be retrieved from the secure AN endpoint")
 	generateRecoverEpochTxArgsCmd.Flags().IntVar(&flagCollectionClusters, "collection-clusters", 0,
 		"number of collection clusters")
 	// required parameters for network configuration and generation of root node identities
@@ -70,7 +76,19 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 0, "length of the epoch staking phase measured in views")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagEpochCounter, "epoch-counter", 0, "the epoch counter used to generate the root cluster block")
 
-	err := generateRecoverEpochTxArgsCmd.MarkFlagRequired("epoch-length")
+	err := generateRecoverEpochTxArgsCmd.MarkFlagRequired("access-address")
+	if err != nil {
+		return fmt.Errorf("failed to mark access-address flag as required")
+	}
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("access-network-key")
+	if err != nil {
+		return fmt.Errorf("failed to mark epoch-length flag as required")
+	}
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("insecure")
+	if err != nil {
+		return fmt.Errorf("failed to mark epoch-length flag as required")
+	}
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("epoch-length")
 	if err != nil {
 		return fmt.Errorf("failed to mark epoch-length flag as required")
 	}

From 0d5dcf507ac7bf789ceed85b5119856e13cd287c Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Wed, 8 May 2024 10:27:28 -0400
Subject: [PATCH 03/60] add execute cmd func

---
 .../tests/epochs/recover_epoch/suite.go       | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 49e5a3ace58..27cbf67a25f 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -1,7 +1,13 @@
 package recover_epoch
 
 import (
+	"fmt"
+	"github.com/onflow/flow-go/cmd/util/cmd/epochs/cmd"
+	"github.com/onflow/flow-go/integration/testnet"
 	"github.com/onflow/flow-go/integration/tests/epochs"
+	"github.com/onflow/flow-go/model/bootstrap"
+	"github.com/onflow/flow-go/model/flow"
+	"os"
 )
 
 // Suite encapsulates common functionality for epoch integration tests.
@@ -19,3 +25,47 @@ func (s *Suite) SetupTest() {
 	// run the generic setup, which starts up the network
 	s.BaseSuite.SetupTest()
 }
+
+// getNodeInfoDirs returns the internal node private info dir and the node config dir from a container with the specified role.
+func (s *Suite) getNodeInfoDirs(role flow.Role) (string, string) {
+	internalNodePrivInfoDir := fmt.Sprintf("%s/%s", s.GetContainersByRole(role)[0].BootstrapPath(), bootstrap.DirPrivateRoot)
+	nodeConfigJson := fmt.Sprintf("%s/%s", s.GetContainersByRole(role)[0].BootstrapPath(), bootstrap.PathNodeInfosPub)
+	return internalNodePrivInfoDir, nodeConfigJson
+}
+
+// executeEFMRecoverTXArgsCMD executes the efm-recover-tx-args CLI command to generate EpochRecover transaction arguments.
+// Args:
+//
+//	role: the container role that will be used to read internal node private info and the node config json.
+//	snapshot: the protocol state snapshot.
+//	collectionClusters: the number of collector clusters.
+//	numViewsInEpoch: the number of views in the recovery epoch.
+//	numViewsInStakingAuction: the number of views in the staking auction of the recovery epoch.
+//	epochCounter: the container role that will be used to read internal node private info and the node config json.
+func (s *Suite) executeEFMRecoverTXArgsCMD(role flow.Role, collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter uint64) {
+	// read internal node info from one of the consensus nodes
+	internalNodePrivInfoDir, nodeConfigJson := s.getNodeInfoDirs(role)
+
+	an1 := s.GetContainersByRole(flow.RoleAccess)[0]
+	anAddress := an1.Addr(testnet.GRPCPort)
+
+	// set command line arguments
+	os.Args = []string{
+		"epochs", "efm-recover-tx-args",
+		"--insecure=true",
+		fmt.Sprintf("--access-address=%s", anAddress),
+		fmt.Sprintf("--collection-clusters=%d", collectionClusters),
+		fmt.Sprintf("--config=%s", nodeConfigJson),
+		fmt.Sprintf("--internal-priv-dir=%s", internalNodePrivInfoDir),
+		fmt.Sprintf("--epoch-length=%d", numViewsInEpoch),
+		fmt.Sprintf("--epoch-staking-phase-length=%d", numViewsInStakingAuction),
+		fmt.Sprintf("--epoch-counter=%d", epochCounter),
+	}
+
+	// execute the root command
+	rootCmd := cmd.RootCmd
+	rootCmd.SetArgs(os.Args[1:])
+	if err := rootCmd.Execute(); err != nil {
+		s.T().Fatalf("Failed to execute epochs efm-recover-tx-args command: %v", err)
+	}
+}

From a8427601660b37afd3cce9f01ec61832a875a24f Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Wed, 8 May 2024 16:42:01 -0400
Subject: [PATCH 04/60] write node infos pub to dir

---
 integration/testnet/network.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/integration/testnet/network.go b/integration/testnet/network.go
index 1dfa55f6a79..882e2545ae5 100644
--- a/integration/testnet/network.go
+++ b/integration/testnet/network.go
@@ -568,7 +568,7 @@ func PrepareFlowNetwork(t *testing.T, networkConf NetworkConfig, chainID flow.Ch
 	t.Logf("BootstrapDir: %s \n", bootstrapDir)
 
 	bootstrapData, err := BootstrapNetwork(networkConf, bootstrapDir, chainID)
-	require.Nil(t, err)
+	require.NoError(t, err)
 
 	root := bootstrapData.Root
 	result := bootstrapData.Result
@@ -1084,6 +1084,7 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl
 
 	// write staking and machine account private key files
 	writeJSONFile := func(relativePath string, val interface{}) error {
+		fmt.Println("BOOTSTRAP DIR: ", filepath.Join(bootstrapDir, relativePath))
 		return WriteJSON(filepath.Join(bootstrapDir, relativePath), val)
 	}
 	writeFile := func(relativePath string, data []byte) error {
@@ -1104,6 +1105,11 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl
 		return nil, fmt.Errorf("failed to write machine account files: %w", err)
 	}
 
+	err = utils.WriteNodeInternalPubInfos(allNodeInfos, writeJSONFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to node pub info file: %w", err)
+	}
+
 	// define root block parameters
 	parentID := flow.ZeroID
 	height := uint64(0)

From c3a05be80aa39a886c5e193a5e8ee6a67473973b Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Wed, 8 May 2024 16:43:23 -0400
Subject: [PATCH 05/60] add flag output dir

---
 cmd/util/cmd/epochs/cmd/recover.go            | 33 +++++++++++--------
 .../epochs/dynamic_epoch_transition_suite.go  | 18 +++++-----
 .../recover_epoch/recover_epoch_efm_test.go   | 18 +++++++---
 .../tests/epochs/recover_epoch/suite.go       |  3 +-
 4 files changed, 42 insertions(+), 30 deletions(-)

diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index 61da8a9c6aa..eb62af26537 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"os"
 
 	"github.com/spf13/cobra"
 
@@ -17,6 +18,10 @@ import (
 	"github.com/onflow/flow-go/state/protocol/inmem"
 )
 
+const (
+	outputFileName = "recover-epoch-tx-args.json"
+)
+
 // generateRecoverEpochTxArgsCmd represents a command to generate the data needed to submit an epoch-recovery transaction
 // to the network when it is in EFM (epoch fallback mode).
 // EFM can be exited only by a special service event, EpochRecover, which initially originates from a manual service account transaction.
@@ -80,14 +85,6 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	if err != nil {
 		return fmt.Errorf("failed to mark access-address flag as required")
 	}
-	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("access-network-key")
-	if err != nil {
-		return fmt.Errorf("failed to mark epoch-length flag as required")
-	}
-	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("insecure")
-	if err != nil {
-		return fmt.Errorf("failed to mark epoch-length flag as required")
-	}
 	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("epoch-length")
 	if err != nil {
 		return fmt.Errorf("failed to mark epoch-length flag as required")
@@ -130,8 +127,6 @@ func getSnapshot() *inmem.Snapshot {
 // generateRecoverEpochTxArgs generates recover epoch transaction arguments from a root protocol state snapshot and writes it to a JSON file
 func generateRecoverEpochTxArgs(getSnapshot func() *inmem.Snapshot) func(cmd *cobra.Command, args []string) {
 	return func(cmd *cobra.Command, args []string) {
-		stdout := cmd.OutOrStdout()
-
 		// extract arguments from recover epoch tx from snapshot
 		txArgs := extractRecoverEpochArgs(getSnapshot())
 
@@ -141,10 +136,20 @@ func generateRecoverEpochTxArgs(getSnapshot func() *inmem.Snapshot) func(cmd *co
 			log.Fatal().Err(err).Msg("could not encode recover epoch transaction arguments")
 		}
 
-		// write JSON args to stdout
-		_, err = stdout.Write(encodedTxArgs)
-		if err != nil {
-			log.Fatal().Err(err).Msg("could not write jsoncdc encoded arguments")
+		if flagOutputDir == "" {
+			// write JSON args to stdout
+			_, err = cmd.OutOrStdout().Write(encodedTxArgs)
+			if err != nil {
+				log.Fatal().Err(err).Msg("could not write jsoncdc encoded arguments")
+			}
+		} else {
+			// write JSON args to stdout
+			out := fmt.Sprintf("%s/%s", flagOutputDir, outputFileName)
+			err := os.WriteFile(out, encodedTxArgs, 0644)
+			if err != nil {
+				log.Fatal().Err(err).Msg(fmt.Sprintf("could not write jsoncdc encoded arguments to file %s", out))
+			}
+			log.Info().Msgf("wrote transaction args to output file %s", out)
 		}
 	}
 }
diff --git a/integration/tests/epochs/dynamic_epoch_transition_suite.go b/integration/tests/epochs/dynamic_epoch_transition_suite.go
index 440695f4cc1..5a5a0979a2c 100644
--- a/integration/tests/epochs/dynamic_epoch_transition_suite.go
+++ b/integration/tests/epochs/dynamic_epoch_transition_suite.go
@@ -136,7 +136,7 @@ func (s *DynamicEpochTransitionSuite) StakeNode(ctx context.Context, env templat
 		machineAccountAddr = accounts[1]
 	}
 
-	result = s.SubmitSetApprovedListTx(ctx, env, append(s.net.Identities().NodeIDs(), nodeID)...)
+	result = s.SubmitSetApprovedListTx(ctx, env, append(s.Net.Identities().NodeIDs(), nodeID)...)
 	require.NoError(s.T(), result.Error)
 
 	// ensure we are still in staking auction
@@ -272,7 +272,7 @@ func (s *DynamicEpochTransitionSuite) ExecuteReadApprovedNodesScript(ctx context
 
 // getTestContainerName returns a name for a test container in the form of ${role}_${nodeID}_test
 func (s *DynamicEpochTransitionSuite) getTestContainerName(role flow.Role) string {
-	i := len(s.net.ContainersByRole(role, false)) + 1
+	i := len(s.Net.ContainersByRole(role, false)) + 1
 	return fmt.Sprintf("%s_test_%d", role, i)
 }
 
@@ -304,27 +304,27 @@ func (s *DynamicEpochTransitionSuite) newTestContainerOnNetwork(role flow.Role,
 
 	nodeConfig := testnet.NewNodeConfig(role, containerConfigs...)
 	testContainerConfig := testnet.NewContainerConfig(info.ContainerName, nodeConfig, info.NetworkingKey, info.StakingKey)
-	err := testContainerConfig.WriteKeyFiles(s.net.BootstrapDir, info.MachineAccountAddress, encodable.MachineAccountPrivKey{PrivateKey: info.MachineAccountKey}, role)
+	err := testContainerConfig.WriteKeyFiles(s.Net.BootstrapDir, info.MachineAccountAddress, encodable.MachineAccountPrivKey{PrivateKey: info.MachineAccountKey}, role)
 	require.NoError(s.T(), err)
 
 	//add our container to the network
-	err = s.net.AddNode(s.T(), s.net.BootstrapDir, testContainerConfig)
+	err = s.Net.AddNode(s.T(), s.Net.BootstrapDir, testContainerConfig)
 	require.NoError(s.T(), err, "failed to add container to network")
 
 	// if node is of LN/SN role type add additional flags to node container for secure GRPC connection
 	if role == flow.RoleConsensus || role == flow.RoleCollection {
 		// ghost containers don't participate in the network skip any SN/LN ghost containers
-		nodeContainer := s.net.ContainerByID(testContainerConfig.NodeID)
+		nodeContainer := s.Net.ContainerByID(testContainerConfig.NodeID)
 		nodeContainer.AddFlag("insecure-access-api", "false")
 
 		accessNodeIDS := make([]string, 0)
-		for _, c := range s.net.ContainersByRole(flow.RoleAccess, false) {
+		for _, c := range s.Net.ContainersByRole(flow.RoleAccess, false) {
 			accessNodeIDS = append(accessNodeIDS, c.Config.NodeID.String())
 		}
 		nodeContainer.AddFlag("access-node-ids", strings.Join(accessNodeIDS, ","))
 	}
 
-	return s.net.ContainerByID(info.NodeID)
+	return s.Net.ContainerByID(info.NodeID)
 }
 
 // StakeNewNode will stake a new node, and create the corresponding docker container for that node
@@ -425,7 +425,7 @@ func (s *DynamicEpochTransitionSuite) AssertNetworkHealthyAfterANChange(ctx cont
 
 	// get snapshot directly from new AN and compare head with head from the
 	// snapshot that was used to bootstrap the node
-	client, err := s.net.ContainerByName(info.ContainerName).TestnetClient()
+	client, err := s.Net.ContainerByName(info.ContainerName).TestnetClient()
 	require.NoError(s.T(), err)
 
 	// overwrite Client to point to the new AN (since we have stopped the initial AN at this point)
@@ -485,7 +485,7 @@ func (s *DynamicEpochTransitionSuite) RunTestEpochJoinAndLeave(role flow.Role, c
 
 	// replace access_2, avoid replacing access_1 the container used for Client connections
 	if role == flow.RoleAccess {
-		containerToReplace = s.net.ContainerByName("access_2")
+		containerToReplace = s.Net.ContainerByName("access_2")
 		require.NotNil(s.T(), containerToReplace)
 	} else {
 		// grab the first container of this node role type, this is the container we will replace
diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 5a6b866beed..4f94d0b75e2 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -1,13 +1,11 @@
 package recover_epoch
 
 import (
-	"testing"
-	"time"
-
+	"github.com/onflow/flow-go/model/flow"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
-	
-	"github.com/onflow/flow-go/model/flow"
+	"testing"
+	"time"
 )
 
 func TestRecoverEpoch(t *testing.T) {
@@ -23,6 +21,7 @@ type RecoverEpochSuite struct {
 // CLI command to generate transaction arguments to submit a recover_epoch transaction, after submitting the transaction the test will
 // ensure the network is healthy.
 func (s *RecoverEpochSuite) TestRecoverEpoch() {
+	// wait until the epoch setup phase to force network into EFM
 	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, 30*time.Second, 4*time.Second)
 
 	// pausing execution node will force the network into EFM
@@ -43,4 +42,13 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	// assert transition to second epoch did not happen
 	// if counter is still 0, epoch emergency fallback was triggered as expected
 	s.AssertInEpoch(s.Ctx, 0)
+
+	// generate epoch recover transaction args
+	collectionClusters := uint64(1)
+	numViewsInEpoch := uint64(4000)
+	numViewsInStakingAuction := uint64(100)
+	epochCounter := uint64(2)
+	s.executeEFMRecoverTXArgsCMD(flow.RoleConsensus, collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter)
+
+	// submit recover epoch transaction to recover the network
 }
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 27cbf67a25f..8f73c1f6416 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -45,14 +45,13 @@ func (s *Suite) getNodeInfoDirs(role flow.Role) (string, string) {
 func (s *Suite) executeEFMRecoverTXArgsCMD(role flow.Role, collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter uint64) {
 	// read internal node info from one of the consensus nodes
 	internalNodePrivInfoDir, nodeConfigJson := s.getNodeInfoDirs(role)
-
 	an1 := s.GetContainersByRole(flow.RoleAccess)[0]
 	anAddress := an1.Addr(testnet.GRPCPort)
-
 	// set command line arguments
 	os.Args = []string{
 		"epochs", "efm-recover-tx-args",
 		"--insecure=true",
+		fmt.Sprintf("--out=%s", s.Net.BootstrapDir),
 		fmt.Sprintf("--access-address=%s", anAddress),
 		fmt.Sprintf("--collection-clusters=%d", collectionClusters),
 		fmt.Sprintf("--config=%s", nodeConfigJson),

From 0d0378cf3f5c403974666943bb21207249a50977 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Thu, 9 May 2024 16:10:30 -0400
Subject: [PATCH 06/60] generate transaction args and submit recover epoch
 transaction

---
 cmd/util/cmd/epochs/cmd/recover.go            | 42 +++++++++++++----
 integration/tests/epochs/base_suite.go        | 14 +++---
 .../recover_epoch/recover_epoch_efm_test.go   | 36 +++++++++++++--
 .../tests/epochs/recover_epoch/suite.go       | 45 +++++++++++++++++--
 integration/utils/arguments.go                | 42 +++++++++++++++++
 integration/utils/templates/recover-epoch.cdc | 37 +++++++++++++++
 integration/utils/transactions.go             | 32 ++++++++++++-
 7 files changed, 223 insertions(+), 25 deletions(-)
 create mode 100644 integration/utils/arguments.go
 create mode 100644 integration/utils/templates/recover-epoch.cdc

diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index eb62af26537..0d10d4db4a0 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -45,7 +45,7 @@ This recovery process has some constraints:
 		Run: generateRecoverEpochTxArgs(getSnapshot),
 	}
 
-	flagOutputDir                string
+	flagOut                      string
 	flagAnAddress                string
 	flagAnPubkey                 string
 	flagAnInsecure               bool
@@ -55,6 +55,9 @@ This recovery process has some constraints:
 	flagNumViewsInEpoch          uint64
 	flagNumViewsInStakingAuction uint64
 	flagEpochCounter             uint64
+	flagRandomSource             string
+	flagTargetDuration           uint64
+	flagTargetEndTime            uint64
 )
 
 func init() {
@@ -66,7 +69,7 @@ func init() {
 }
 
 func addGenerateRecoverEpochTxArgsCmdFlags() error {
-	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagOutputDir, "out", "", "the path to the output dir")
+	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagOut, "out", "", "file to write tx args output")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnAddress, "access-address", "", "the address of the access node used for client connections")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnPubkey, "access-network-key", "", "the network key of the access node used for client connections in hex string format")
 	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagAnInsecure, "insecure", true, "set to true if the protocol snapshot should be retrieved from the secure AN endpoint")
@@ -80,6 +83,9 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagNumViewsInEpoch, "epoch-length", 0, "length of each epoch measured in views")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 0, "length of the epoch staking phase measured in views")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagEpochCounter, "epoch-counter", 0, "the epoch counter used to generate the root cluster block")
+	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagRandomSource, "random-source", "", "the random source for the epoch")
+	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagTargetDuration, "target-duration", 0, "the target duration of the epoch")
+	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagTargetEndTime, "target-end-time", 0, "the target end time for the epoch")
 
 	err := generateRecoverEpochTxArgsCmd.MarkFlagRequired("access-address")
 	if err != nil {
@@ -101,6 +107,18 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	if err != nil {
 		return fmt.Errorf("failed to mark collection-clusters flag as required")
 	}
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("random-source")
+	if err != nil {
+		return fmt.Errorf("failed to mark random-source flag as required")
+	}
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("target-duration")
+	if err != nil {
+		return fmt.Errorf("failed to mark target-duration flag as required")
+	}
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("target-end-time")
+	if err != nil {
+		return fmt.Errorf("failed to mark target-end-time flag as required")
+	}
 	return nil
 }
 
@@ -136,7 +154,7 @@ func generateRecoverEpochTxArgs(getSnapshot func() *inmem.Snapshot) func(cmd *co
 			log.Fatal().Err(err).Msg("could not encode recover epoch transaction arguments")
 		}
 
-		if flagOutputDir == "" {
+		if flagOut == "" {
 			// write JSON args to stdout
 			_, err = cmd.OutOrStdout().Write(encodedTxArgs)
 			if err != nil {
@@ -144,12 +162,11 @@ func generateRecoverEpochTxArgs(getSnapshot func() *inmem.Snapshot) func(cmd *co
 			}
 		} else {
 			// write JSON args to stdout
-			out := fmt.Sprintf("%s/%s", flagOutputDir, outputFileName)
-			err := os.WriteFile(out, encodedTxArgs, 0644)
+			err := os.WriteFile(flagOut, encodedTxArgs, 0644)
 			if err != nil {
-				log.Fatal().Err(err).Msg(fmt.Sprintf("could not write jsoncdc encoded arguments to file %s", out))
+				log.Fatal().Err(err).Msg(fmt.Sprintf("could not write jsoncdc encoded arguments to file %s", flagOut))
 			}
-			log.Info().Msgf("wrote transaction args to output file %s", out)
+			log.Info().Msgf("wrote transaction args to output file %s", flagOut)
 		}
 	}
 }
@@ -255,18 +272,25 @@ func extractRecoverEpochArgs(snapshot *inmem.Snapshot) []cadence.Value {
 	}
 
 	args := []cadence.Value{
+		// random source
+		cadence.String(flagRandomSource),
 		// epoch start view
 		cadence.NewUInt64(currEpochFinalView + 1),
 		// staking phase end view
 		cadence.NewUInt64(currEpochFinalView + flagNumViewsInStakingAuction),
 		// epoch end view
 		cadence.NewUInt64(currEpochFinalView + flagNumViewsInEpoch),
+		// target duration
+		cadence.NewUInt64(flagTargetDuration),
+		// target end time
+		cadence.NewUInt64(flagTargetEndTime),
+		// clusters,
+		common.ConvertClusterAssignmentsCdc(assignments),
+		// @TODO: cluster QC voter data
 		// dkg pub keys
 		cadence.NewArray(dkgPubKeys),
 		// node ids
 		cadence.NewArray(nodeIds),
-		// clusters,
-		common.ConvertClusterAssignmentsCdc(assignments),
 	}
 
 	return args
diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index 0b9db8f26f6..be46f2719c0 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -30,7 +30,7 @@ type BaseSuite struct {
 	lib.TestnetStateTracker
 	cancel  context.CancelFunc
 	log     zerolog.Logger
-	net     *testnet.FlowNetwork
+	Net     *testnet.FlowNetwork
 	ghostID flow.Identifier
 
 	Client *testnet.Client
@@ -99,16 +99,16 @@ func (s *BaseSuite) SetupTest() {
 	netConf := testnet.NewNetworkConfigWithEpochConfig("epochs-tests", confs, s.StakingAuctionLen, s.DKGPhaseLen, s.EpochLen, s.EpochCommitSafetyThreshold)
 
 	// initialize the network
-	s.net = testnet.PrepareFlowNetwork(s.T(), netConf, flow.Localnet)
+	s.Net = testnet.PrepareFlowNetwork(s.T(), netConf, flow.Localnet)
 
 	// start the network
-	s.net.Start(s.Ctx)
+	s.Net.Start(s.Ctx)
 
 	// start tracking blocks
 	s.Track(s.T(), s.Ctx, s.Ghost())
 
 	// use AN1 for test-related queries - the AN join/leave test will replace AN2
-	client, err := s.net.ContainerByName(testnet.PrimaryAN).TestnetClient()
+	client, err := s.Net.ContainerByName(testnet.PrimaryAN).TestnetClient()
 	require.NoError(s.T(), err)
 
 	s.Client = client
@@ -119,13 +119,13 @@ func (s *BaseSuite) SetupTest() {
 
 func (s *BaseSuite) TearDownTest() {
 	s.log.Info().Msg("================> Start TearDownTest")
-	s.net.Remove()
+	s.Net.Remove()
 	s.cancel()
 	s.log.Info().Msg("================> Finish TearDownTest")
 }
 
 func (s *BaseSuite) Ghost() *client.GhostClient {
-	client, err := s.net.ContainerByID(s.ghostID).GhostClient()
+	client, err := s.Net.ContainerByID(s.ghostID).GhostClient()
 	require.NoError(s.T(), err, "could not get ghost Client")
 	return client
 }
@@ -158,7 +158,7 @@ func (s *BaseSuite) AwaitEpochPhase(ctx context.Context, expectedEpoch uint64, e
 
 // GetContainersByRole returns all containers from the network for the specified role, making sure the containers are not ghost nodes.
 func (s *BaseSuite) GetContainersByRole(role flow.Role) []*testnet.Container {
-	nodes := s.net.ContainersByRole(role, false)
+	nodes := s.Net.ContainersByRole(role, false)
 	require.True(s.T(), len(nodes) > 0)
 	return nodes
 }
diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 4f94d0b75e2..05d772f9650 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -1,11 +1,16 @@
 package recover_epoch
 
 import (
-	"github.com/onflow/flow-go/model/flow"
-	"github.com/stretchr/testify/require"
-	"github.com/stretchr/testify/suite"
+	"fmt"
+	"os"
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/onflow/flow-go/integration/utils"
+	"github.com/onflow/flow-go/model/flow"
 )
 
 func TestRecoverEpoch(t *testing.T) {
@@ -48,7 +53,30 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	numViewsInEpoch := uint64(4000)
 	numViewsInStakingAuction := uint64(100)
 	epochCounter := uint64(2)
-	s.executeEFMRecoverTXArgsCMD(flow.RoleConsensus, collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter)
+	randomSource := "ohsorandom"
+	targetDuration := uint64(3000)
+	targetEndTime := uint64(4000)
+	out := fmt.Sprintf("%s/recover-epoch-tx-args.josn", s.Net.BootstrapDir)
+
+	s.executeEFMRecoverTXArgsCMD(
+		flow.RoleConsensus,
+		collectionClusters,
+		numViewsInEpoch,
+		numViewsInStakingAuction,
+		epochCounter,
+		targetDuration,
+		targetEndTime,
+		randomSource,
+		out,
+	)
+	b, err := os.ReadFile(out)
+	require.NoError(s.T(), err)
+
+	txArgs, err := utils.ParseJSON(b)
+	require.NoError(s.T(), err)
 
+	env := utils.LocalnetEnv()
+	result := s.recoverEpoch(env, txArgs)
+	fmt.Println("TX RESULT: ", result)
 	// submit recover epoch transaction to recover the network
 }
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 8f73c1f6416..419c5339a98 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -2,12 +2,18 @@ package recover_epoch
 
 import (
 	"fmt"
+	"os"
+
+	"github.com/onflow/cadence"
+	"github.com/onflow/flow-core-contracts/lib/go/templates"
+	sdk "github.com/onflow/flow-go-sdk"
 	"github.com/onflow/flow-go/cmd/util/cmd/epochs/cmd"
 	"github.com/onflow/flow-go/integration/testnet"
 	"github.com/onflow/flow-go/integration/tests/epochs"
+	"github.com/onflow/flow-go/integration/utils"
 	"github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/flow"
-	"os"
+	"github.com/stretchr/testify/require"
 )
 
 // Suite encapsulates common functionality for epoch integration tests.
@@ -41,8 +47,12 @@ func (s *Suite) getNodeInfoDirs(role flow.Role) (string, string) {
 //	collectionClusters: the number of collector clusters.
 //	numViewsInEpoch: the number of views in the recovery epoch.
 //	numViewsInStakingAuction: the number of views in the staking auction of the recovery epoch.
-//	epochCounter: the container role that will be used to read internal node private info and the node config json.
-func (s *Suite) executeEFMRecoverTXArgsCMD(role flow.Role, collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter uint64) {
+//	epochCounter: the epoch counter.
+//	targetDuration: the target duration for the recover epoch.
+//	targetEndTime: the target end time for the recover epoch.
+//	randomSource: the random source of the recover epoch.
+//	out: the tx args output file full path.
+func (s *Suite) executeEFMRecoverTXArgsCMD(role flow.Role, collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter, targetDuration, targetEndTime uint64, randomSource, out string) {
 	// read internal node info from one of the consensus nodes
 	internalNodePrivInfoDir, nodeConfigJson := s.getNodeInfoDirs(role)
 	an1 := s.GetContainersByRole(flow.RoleAccess)[0]
@@ -51,7 +61,7 @@ func (s *Suite) executeEFMRecoverTXArgsCMD(role flow.Role, collectionClusters, n
 	os.Args = []string{
 		"epochs", "efm-recover-tx-args",
 		"--insecure=true",
-		fmt.Sprintf("--out=%s", s.Net.BootstrapDir),
+		fmt.Sprintf("--out=%s", out),
 		fmt.Sprintf("--access-address=%s", anAddress),
 		fmt.Sprintf("--collection-clusters=%d", collectionClusters),
 		fmt.Sprintf("--config=%s", nodeConfigJson),
@@ -59,6 +69,9 @@ func (s *Suite) executeEFMRecoverTXArgsCMD(role flow.Role, collectionClusters, n
 		fmt.Sprintf("--epoch-length=%d", numViewsInEpoch),
 		fmt.Sprintf("--epoch-staking-phase-length=%d", numViewsInStakingAuction),
 		fmt.Sprintf("--epoch-counter=%d", epochCounter),
+		fmt.Sprintf("--random-source=%s", randomSource),
+		fmt.Sprintf("--target-duration=%d", targetDuration),
+		fmt.Sprintf("--target-end-time=%d", targetEndTime),
 	}
 
 	// execute the root command
@@ -68,3 +81,27 @@ func (s *Suite) executeEFMRecoverTXArgsCMD(role flow.Role, collectionClusters, n
 		s.T().Fatalf("Failed to execute epochs efm-recover-tx-args command: %v", err)
 	}
 }
+
+// recoverEpoch submits the recover epoch transaction to the network.
+func (s *Suite) recoverEpoch(env templates.Environment, args []cadence.Value) *sdk.TransactionResult {
+	latestBlockID, err := s.Client.GetLatestBlockID(s.Ctx)
+	require.NoError(s.T(), err)
+
+	// create and register node
+	tx, err := utils.MakeRecoverEpochTx(
+		env,
+		s.Client.Account(),
+		0,
+		sdk.Identifier(latestBlockID),
+		args,
+	)
+	require.NoError(s.T(), err)
+
+	err = s.Client.SignAndSendTransaction(s.Ctx, tx)
+	require.NoError(s.T(), err)
+	result, err := s.Client.WaitForSealed(s.Ctx, tx.ID())
+	require.NoError(s.T(), err)
+	s.Client.Account().Keys[0].SequenceNumber++
+	require.NoError(s.T(), result.Error)
+	return result
+}
diff --git a/integration/utils/arguments.go b/integration/utils/arguments.go
new file mode 100644
index 00000000000..d14d730265f
--- /dev/null
+++ b/integration/utils/arguments.go
@@ -0,0 +1,42 @@
+package utils
+
+import (
+	"encoding/json"
+
+	"github.com/onflow/cadence"
+	jsoncdc "github.com/onflow/cadence/encoding/json"
+)
+
+type cadenceArgument struct {
+	Value cadence.Value
+}
+
+func (v *cadenceArgument) MarshalJSON() ([]byte, error) {
+	return jsoncdc.Encode(v.Value)
+}
+
+func (v *cadenceArgument) UnmarshalJSON(b []byte) (err error) {
+	v.Value, err = jsoncdc.Decode(nil, b)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// ParseJSON parses string representing JSON array with Cadence arguments.
+//
+// Cadence arguments must be defined in the JSON-Cadence format https://developers.flow.com/cadence/json-cadence-spec
+func ParseJSON(args []byte) ([]cadence.Value, error) {
+	var arg []cadenceArgument
+	err := json.Unmarshal(args, &arg)
+
+	if err != nil {
+		return nil, err
+	}
+
+	cadenceArgs := make([]cadence.Value, len(arg))
+	for i, arg := range arg {
+		cadenceArgs[i] = arg.Value
+	}
+	return cadenceArgs, nil
+}
diff --git a/integration/utils/templates/recover-epoch.cdc b/integration/utils/templates/recover-epoch.cdc
new file mode 100644
index 00000000000..cbd2172793d
--- /dev/null
+++ b/integration/utils/templates/recover-epoch.cdc
@@ -0,0 +1,37 @@
+import FlowEpoch from "FlowEpoch"
+import FlowIDTableStaking from "FlowIDTableStaking"
+import FlowClusterQC from "FlowClusterQC"
+
+// The recoverEpoch transaction creates and starts a new epoch in the FlowEpoch smart contract
+// to which will force the network exit EFM. The recoverEpoch service event will be emitted
+// and processed by all protocol participants and each participant will update their protocol
+// state with the new Epoch data.
+// This transaction should only be used with the output of the bootstrap utility:
+//   util epoch efm-recover-tx-args
+transaction(randomSource: String,
+            startView: UInt64,
+            stakingEndView: UInt64,
+            endView: UInt64,
+            targetDuration: UInt64,
+            targetEndTime: UInt64,
+            collectorClusters: [[String]],
+            clusterQCVoteData: [FlowClusterQC.ClusterQCVoteData],
+            dkgPubKeys: [String],
+            nodeIDs: [String]) {
+
+    prepare(signer: auth(BorrowValue) &Account) {
+        let epochAdmin = signer.storage.borrow<&FlowEpoch.Admin>(from: FlowEpoch.adminStoragePath)
+            ?? panic("Could not borrow epoch admin from storage path")
+
+        epochAdmin.recoverEpoch(randomSource: randomSource,
+                            startView: startView,
+                            stakingEndView: stakingEndView,
+                            endView: endView,
+                            targetDuration: targetDuration,
+                            targetEndTime: targetEndTime,
+                            collectorClusters: collectorClusters,
+                            clusterQCVoteData: clusterQCVoteData,
+                            dkgPubKeys: dkgPubKeys,
+                            nodeIDs: nodeIDs)
+    }
+}
diff --git a/integration/utils/transactions.go b/integration/utils/transactions.go
index 1eaef320b68..686f9f7db9a 100644
--- a/integration/utils/transactions.go
+++ b/integration/utils/transactions.go
@@ -18,6 +18,9 @@ import (
 	"github.com/onflow/flow-go/utils/unittest"
 )
 
+//go:embed templates/recover-epoch.cdc
+var recoverEpochTxScript string
+
 //go:embed templates/create-and-setup-node.cdc
 var createAndSetupNodeTxScript string
 
@@ -214,7 +217,7 @@ func MakeSetProtocolStateVersionTx(
 	return tx, nil
 }
 
-// submitSmokeTestTransaction will submit a create account transaction to smoke test network
+// CreateFlowAccount will submit a create account transaction to smoke test network
 // This ensures a single transaction can be sealed by the network.
 func CreateFlowAccount(ctx context.Context, client *testnet.Client) (sdk.Address, error) {
 	fullAccountKey := sdk.NewAccountKey().
@@ -235,3 +238,30 @@ func CreateFlowAccount(ctx context.Context, client *testnet.Client) (sdk.Address
 
 	return addr, nil
 }
+
+// MakeRecoverEpochTx makes an admin transaction to recover the network when it is in EFM mode.
+func MakeRecoverEpochTx(
+	env templates.Environment,
+	adminAccount *sdk.Account,
+	adminAccountKeyID int,
+	latestBlockID sdk.Identifier,
+	args []cadence.Value,
+) (*sdk.Transaction, error) {
+	accountKey := adminAccount.Keys[adminAccountKeyID]
+	tx := sdk.NewTransaction().
+		SetScript([]byte(templates.ReplaceAddresses(removeNodeTxScript, env))).
+		SetComputeLimit(9999).
+		SetReferenceBlockID(latestBlockID).
+		SetProposalKey(adminAccount.Address, adminAccountKeyID, accountKey.SequenceNumber).
+		SetPayer(adminAccount.Address).
+		AddAuthorizer(adminAccount.Address)
+
+	for _, arg := range args {
+		err := tx.AddArgument(arg)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return tx, nil
+}

From f0670d00225e211ad8c201cf6420c3e4e3d39932 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Fri, 14 Jun 2024 17:41:21 -0400
Subject: [PATCH 07/60] Apply suggestions from code review

Co-authored-by: Jordan Schalm <jordan.schalm@flowfoundation.org>
---
 cmd/util/cmd/epochs/cmd/recover.go                         | 7 +------
 integration/testnet/network.go                             | 1 -
 integration/tests/epochs/base_suite.go                     | 4 ++--
 .../tests/epochs/recover_epoch/recover_epoch_efm_test.go   | 5 +----
 integration/tests/epochs/recover_epoch/suite.go            | 2 +-
 5 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index 0d10d4db4a0..eeb35cd97cf 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -72,7 +72,7 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagOut, "out", "", "file to write tx args output")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnAddress, "access-address", "", "the address of the access node used for client connections")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnPubkey, "access-network-key", "", "the network key of the access node used for client connections in hex string format")
-	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagAnInsecure, "insecure", true, "set to true if the protocol snapshot should be retrieved from the secure AN endpoint")
+	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagAnInsecure, "insecure", true, "set to true if the protocol snapshot should be retrieved from the insecure AN endpoint")
 	generateRecoverEpochTxArgsCmd.Flags().IntVar(&flagCollectionClusters, "collection-clusters", 0,
 		"number of collection clusters")
 	// required parameters for network configuration and generation of root node identities
@@ -83,7 +83,6 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagNumViewsInEpoch, "epoch-length", 0, "length of each epoch measured in views")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 0, "length of the epoch staking phase measured in views")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagEpochCounter, "epoch-counter", 0, "the epoch counter used to generate the root cluster block")
-	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagRandomSource, "random-source", "", "the random source for the epoch")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagTargetDuration, "target-duration", 0, "the target duration of the epoch")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagTargetEndTime, "target-end-time", 0, "the target end time for the epoch")
 
@@ -107,10 +106,6 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	if err != nil {
 		return fmt.Errorf("failed to mark collection-clusters flag as required")
 	}
-	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("random-source")
-	if err != nil {
-		return fmt.Errorf("failed to mark random-source flag as required")
-	}
 	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("target-duration")
 	if err != nil {
 		return fmt.Errorf("failed to mark target-duration flag as required")
diff --git a/integration/testnet/network.go b/integration/testnet/network.go
index 882e2545ae5..fd17476af5c 100644
--- a/integration/testnet/network.go
+++ b/integration/testnet/network.go
@@ -1084,7 +1084,6 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl
 
 	// write staking and machine account private key files
 	writeJSONFile := func(relativePath string, val interface{}) error {
-		fmt.Println("BOOTSTRAP DIR: ", filepath.Join(bootstrapDir, relativePath))
 		return WriteJSON(filepath.Join(bootstrapDir, relativePath), val)
 	}
 	writeFile := func(relativePath string, data []byte) error {
diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index be46f2719c0..0b73d7a53c9 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -168,8 +168,8 @@ func (s *BaseSuite) GetContainersByRole(role flow.Role) []*testnet.Container {
 // transition must have happened.
 func (s *BaseSuite) AwaitFinalizedView(ctx context.Context, view uint64, waitFor, tick time.Duration) {
 	require.Eventually(s.T(), func() bool {
-		sealed := s.getLatestFinalizedHeader(ctx)
-		return sealed.View >= view
+		finalized := s.getLatestFinalizedHeader(ctx)
+		return finalized.View >= view
 	}, waitFor, tick)
 }
 
diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 05d772f9650..9485f70a3c5 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -33,10 +33,7 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	_ = s.GetContainersByRole(flow.RoleExecution)[0].Pause()
 
 	// get the latest snapshot and start new container with it
-	rootSnapshot, err := s.Client.GetLatestProtocolSnapshot(s.Ctx)
-	require.NoError(s.T(), err)
-
-	epoch1FinalView, err := rootSnapshot.Epochs().Current().FinalView()
+	epoch1FinalView, err := s.Net.BootstrapSnapshot.Epochs().Current().FinalView()
 	require.NoError(s.T(), err)
 
 	// wait for at least the first block of the next epoch to be sealed before we pause our container to replace
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 419c5339a98..45d61b327ef 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -52,7 +52,7 @@ func (s *Suite) getNodeInfoDirs(role flow.Role) (string, string) {
 //	targetEndTime: the target end time for the recover epoch.
 //	randomSource: the random source of the recover epoch.
 //	out: the tx args output file full path.
-func (s *Suite) executeEFMRecoverTXArgsCMD(role flow.Role, collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter, targetDuration, targetEndTime uint64, randomSource, out string) {
+func (s *Suite) executeEFMRecoverTXArgsCMD(collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter, targetDuration, targetEndTime uint64, randomSource, out string) {
 	// read internal node info from one of the consensus nodes
 	internalNodePrivInfoDir, nodeConfigJson := s.getNodeInfoDirs(role)
 	an1 := s.GetContainersByRole(flow.RoleAccess)[0]

From ee7fa73c7be23714d2e4870027ee412ca4233ee8 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Mon, 17 Jun 2024 11:31:46 -0400
Subject: [PATCH 08/60] move warning log


From 09f29e77c9e49cadb8dc75af2ff7fd662e32f851 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Mon, 17 Jun 2024 11:34:17 -0400
Subject: [PATCH 09/60] update comments

---
 .../tests/epochs/recover_epoch/recover_epoch_efm_test.go       | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 9485f70a3c5..dbd84bab72e 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -36,7 +36,8 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	epoch1FinalView, err := s.Net.BootstrapSnapshot.Epochs().Current().FinalView()
 	require.NoError(s.T(), err)
 
-	// wait for at least the first block of the next epoch to be sealed before we pause our container to replace
+	// wait for at least the first block of the next epoch to be sealed so that we can
+	// ensure that we are still in the same epoch after the final view of that epoch indicating we are in EFM
 	s.TimedLogf("waiting for epoch transition (finalized view %d) before pausing container", epoch1FinalView+1)
 	s.AwaitFinalizedView(s.Ctx, epoch1FinalView+1, 2*time.Minute, 500*time.Millisecond)
 	s.TimedLogf("observed finalized view %d -> pausing container", epoch1FinalView+1)

From 96b9776bef8f4d58c563ee63189ffcc6d029d8ff Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Mon, 17 Jun 2024 11:34:47 -0400
Subject: [PATCH 10/60] Update recover_epoch_efm_test.go

---
 integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index dbd84bab72e..ac1e9b9b5b6 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -57,7 +57,6 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	out := fmt.Sprintf("%s/recover-epoch-tx-args.josn", s.Net.BootstrapDir)
 
 	s.executeEFMRecoverTXArgsCMD(
-		flow.RoleConsensus,
 		collectionClusters,
 		numViewsInEpoch,
 		numViewsInStakingAuction,

From d48d4380950c368e5c3b640b796582c78466c9d9 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Mon, 17 Jun 2024 12:14:24 -0400
Subject: [PATCH 11/60] Update suite.go

---
 integration/tests/epochs/recover_epoch/suite.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 45d61b327ef..d6691a84c96 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -54,7 +54,7 @@ func (s *Suite) getNodeInfoDirs(role flow.Role) (string, string) {
 //	out: the tx args output file full path.
 func (s *Suite) executeEFMRecoverTXArgsCMD(collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter, targetDuration, targetEndTime uint64, randomSource, out string) {
 	// read internal node info from one of the consensus nodes
-	internalNodePrivInfoDir, nodeConfigJson := s.getNodeInfoDirs(role)
+	internalNodePrivInfoDir, nodeConfigJson := s.getNodeInfoDirs(flow.RoleConsensus)
 	an1 := s.GetContainersByRole(flow.RoleAccess)[0]
 	anAddress := an1.Addr(testnet.GRPCPort)
 	// set command line arguments

From d75354bedc1e520845b3c11283746177c71e8a4e Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Fri, 21 Jun 2024 13:00:37 -0400
Subject: [PATCH 12/60] add ClusterQCVoteData conversion to cdc arg

---
 cmd/util/cmd/common/clusters.go    | 46 +++++++++++++++++++++++++++---
 cmd/util/cmd/epochs/cmd/recover.go | 13 +++------
 2 files changed, 46 insertions(+), 13 deletions(-)

diff --git a/cmd/util/cmd/common/clusters.go b/cmd/util/cmd/common/clusters.go
index 3d51d06519e..3c961cda5dc 100644
--- a/cmd/util/cmd/common/clusters.go
+++ b/cmd/util/cmd/common/clusters.go
@@ -7,6 +7,7 @@ import (
 	"github.com/rs/zerolog"
 
 	"github.com/onflow/cadence"
+	cdcCommon "github.com/onflow/cadence/runtime/common"
 
 	"github.com/onflow/flow-go/cmd/bootstrap/run"
 	"github.com/onflow/flow-go/model/bootstrap"
@@ -163,8 +164,9 @@ func ConvertClusterAssignmentsCdc(assignments flow.AssignmentList) cadence.Array
 }
 
 // ConvertClusterQcsCdc converts cluster QCs from `QuorumCertificate` type to `ClusterQCVoteData` type.
-func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.ClusterList) ([]*flow.ClusterQCVoteData, error) {
-	voteData := make([]*flow.ClusterQCVoteData, len(qcs))
+func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.ClusterList) ([]cadence.Value, error) {
+	voteDataType := newFlowClusterQCVoteDataStructType()
+	qcVoteData := make([]cadence.Value, len(qcs))
 	for i, qc := range qcs {
 		c, ok := clusterList.ByIndex(uint(i))
 		if !ok {
@@ -174,13 +176,49 @@ func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.Cluste
 		if err != nil {
 			return nil, fmt.Errorf("could not decode signer indices: %w", err)
 		}
-		voteData[i] = &flow.ClusterQCVoteData{
+		voteData := &flow.ClusterQCVoteData{
 			SigData:  qc.SigData,
 			VoterIDs: voterIds,
 		}
+
+		cdcVoterIds := make([]cadence.Value, len(voterIds))
+		for i, id := range voterIds {
+			cdcVoterIds[i] = cadence.String(id.String())
+		}
+
+		qcVoteData[i] = cadence.NewStruct([]cadence.Value{
+			// voteSignatures
+			cadence.String(voteData.SigData.String()),
+			// voterIDs
+			cadence.NewArray(cdcVoterIds).WithType(cadence.NewVariableSizedArrayType(cadence.StringType)),
+		}).WithType(voteDataType)
+
 	}
 
-	return voteData, nil
+	return qcVoteData, nil
+}
+
+func newFlowClusterQCVoteDataStructType() *cadence.StructType {
+
+	// A.01cf0e2f2f715450.FlowClusterQC.ClusterQCVoteData
+
+	address, _ := cdcCommon.HexToAddress("01cf0e2f2f715450")
+	location := cdcCommon.NewAddressLocation(nil, address, "FlowClusterQC")
+
+	return &cadence.StructType{
+		Location:            location,
+		QualifiedIdentifier: "FlowClusterQC.ClusterQCVoteData",
+		Fields: []cadence.Field{
+			{
+				Identifier: "voteSignatures",
+				Type:       cadence.StringType,
+			},
+			{
+				Identifier: "voterIDs",
+				Type:       cadence.NewVariableSizedArrayType(cadence.StringType),
+			},
+		},
+	}
 }
 
 // Filters a list of nodes to include only nodes that will sign the QC for the
diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index eeb35cd97cf..09267a2b12f 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -55,7 +55,6 @@ This recovery process has some constraints:
 	flagNumViewsInEpoch          uint64
 	flagNumViewsInStakingAuction uint64
 	flagEpochCounter             uint64
-	flagRandomSource             string
 	flagTargetDuration           uint64
 	flagTargetEndTime            uint64
 )
@@ -72,7 +71,7 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagOut, "out", "", "file to write tx args output")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnAddress, "access-address", "", "the address of the access node used for client connections")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnPubkey, "access-network-key", "", "the network key of the access node used for client connections in hex string format")
-	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagAnInsecure, "insecure", true, "set to true if the protocol snapshot should be retrieved from the insecure AN endpoint")
+	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagAnInsecure, "insecure", false, "set to true if the protocol snapshot should be retrieved from the insecure AN endpoint")
 	generateRecoverEpochTxArgsCmd.Flags().IntVar(&flagCollectionClusters, "collection-clusters", 0,
 		"number of collection clusters")
 	// required parameters for network configuration and generation of root node identities
@@ -253,10 +252,7 @@ func extractRecoverEpochArgs(snapshot *inmem.Snapshot) []cadence.Value {
 		nodeIds = append(nodeIds, nodeIdCdc)
 	}
 
-	// @TODO: cluster qcs are converted into flow.ClusterQCVoteData types,
-	// we need a corresponding type in cadence on the FlowClusterQC contract
-	// to store this struct.
-	_, err = common.ConvertClusterQcsCdc(clusterQCs, clusters)
+	qcVoteData, err := common.ConvertClusterQcsCdc(clusterQCs, clusters)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to convert cluster qcs to cadence type")
 	}
@@ -267,8 +263,6 @@ func extractRecoverEpochArgs(snapshot *inmem.Snapshot) []cadence.Value {
 	}
 
 	args := []cadence.Value{
-		// random source
-		cadence.String(flagRandomSource),
 		// epoch start view
 		cadence.NewUInt64(currEpochFinalView + 1),
 		// staking phase end view
@@ -281,7 +275,8 @@ func extractRecoverEpochArgs(snapshot *inmem.Snapshot) []cadence.Value {
 		cadence.NewUInt64(flagTargetEndTime),
 		// clusters,
 		common.ConvertClusterAssignmentsCdc(assignments),
-		// @TODO: cluster QC voter data
+		// qcVoteData
+		cadence.NewArray(qcVoteData),
 		// dkg pub keys
 		cadence.NewArray(dkgPubKeys),
 		// node ids

From b93e8ac399f4f2a8a47f48dcf4eea8705d83b0ae Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Fri, 21 Jun 2024 13:01:47 -0400
Subject: [PATCH 13/60] update test

- add epoch address to env template
- update recover transaction template
---
 .../recover_epoch/recover_epoch_efm_test.go   | 16 ++++----
 .../tests/epochs/recover_epoch/suite.go       |  7 ++--
 integration/utils/templates/recover-epoch.cdc | 39 ++++++++++++-------
 integration/utils/transactions.go             |  3 +-
 4 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index ac1e9b9b5b6..a5675e2d726 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -27,10 +27,11 @@ type RecoverEpochSuite struct {
 // ensure the network is healthy.
 func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	// wait until the epoch setup phase to force network into EFM
-	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, 30*time.Second, 4*time.Second)
+	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, 10*time.Second, 4*time.Second)
 
 	// pausing execution node will force the network into EFM
-	_ = s.GetContainersByRole(flow.RoleExecution)[0].Pause()
+	enContainer := s.GetContainersByRole(flow.RoleExecution)[0]
+	_ = enContainer.Pause()
 
 	// get the latest snapshot and start new container with it
 	epoch1FinalView, err := s.Net.BootstrapSnapshot.Epochs().Current().FinalView()
@@ -42,16 +43,18 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	s.AwaitFinalizedView(s.Ctx, epoch1FinalView+1, 2*time.Minute, 500*time.Millisecond)
 	s.TimedLogf("observed finalized view %d -> pausing container", epoch1FinalView+1)
 
-	// assert transition to second epoch did not happen
-	// if counter is still 0, epoch emergency fallback was triggered as expected
+	//assert transition to second epoch did not happen
+	//if counter is still 0, epoch emergency fallback was triggered as expected
 	s.AssertInEpoch(s.Ctx, 0)
 
+	// start the paused execution node now that we are in EFM
+	enContainer.Start()
+
 	// generate epoch recover transaction args
 	collectionClusters := uint64(1)
 	numViewsInEpoch := uint64(4000)
 	numViewsInStakingAuction := uint64(100)
 	epochCounter := uint64(2)
-	randomSource := "ohsorandom"
 	targetDuration := uint64(3000)
 	targetEndTime := uint64(4000)
 	out := fmt.Sprintf("%s/recover-epoch-tx-args.josn", s.Net.BootstrapDir)
@@ -63,7 +66,6 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 		epochCounter,
 		targetDuration,
 		targetEndTime,
-		randomSource,
 		out,
 	)
 	b, err := os.ReadFile(out)
@@ -74,6 +76,6 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 
 	env := utils.LocalnetEnv()
 	result := s.recoverEpoch(env, txArgs)
-	fmt.Println("TX RESULT: ", result)
+	fmt.Println("TX RESULT STATUS: ", result)
 	// submit recover epoch transaction to recover the network
 }
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index d6691a84c96..ae07e11cdbf 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -52,7 +52,7 @@ func (s *Suite) getNodeInfoDirs(role flow.Role) (string, string) {
 //	targetEndTime: the target end time for the recover epoch.
 //	randomSource: the random source of the recover epoch.
 //	out: the tx args output file full path.
-func (s *Suite) executeEFMRecoverTXArgsCMD(collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter, targetDuration, targetEndTime uint64, randomSource, out string) {
+func (s *Suite) executeEFMRecoverTXArgsCMD(collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter, targetDuration, targetEndTime uint64, out string) {
 	// read internal node info from one of the consensus nodes
 	internalNodePrivInfoDir, nodeConfigJson := s.getNodeInfoDirs(flow.RoleConsensus)
 	an1 := s.GetContainersByRole(flow.RoleAccess)[0]
@@ -69,7 +69,6 @@ func (s *Suite) executeEFMRecoverTXArgsCMD(collectionClusters, numViewsInEpoch,
 		fmt.Sprintf("--epoch-length=%d", numViewsInEpoch),
 		fmt.Sprintf("--epoch-staking-phase-length=%d", numViewsInStakingAuction),
 		fmt.Sprintf("--epoch-counter=%d", epochCounter),
-		fmt.Sprintf("--random-source=%s", randomSource),
 		fmt.Sprintf("--target-duration=%d", targetDuration),
 		fmt.Sprintf("--target-end-time=%d", targetEndTime),
 	}
@@ -87,7 +86,6 @@ func (s *Suite) recoverEpoch(env templates.Environment, args []cadence.Value) *s
 	latestBlockID, err := s.Client.GetLatestBlockID(s.Ctx)
 	require.NoError(s.T(), err)
 
-	// create and register node
 	tx, err := utils.MakeRecoverEpochTx(
 		env,
 		s.Client.Account(),
@@ -95,6 +93,8 @@ func (s *Suite) recoverEpoch(env templates.Environment, args []cadence.Value) *s
 		sdk.Identifier(latestBlockID),
 		args,
 	)
+
+	fmt.Println("TRANSACTION SCRIPT \n", string(tx.Script))
 	require.NoError(s.T(), err)
 
 	err = s.Client.SignAndSendTransaction(s.Ctx, tx)
@@ -103,5 +103,6 @@ func (s *Suite) recoverEpoch(env templates.Environment, args []cadence.Value) *s
 	require.NoError(s.T(), err)
 	s.Client.Account().Keys[0].SequenceNumber++
 	require.NoError(s.T(), result.Error)
+
 	return result
 }
diff --git a/integration/utils/templates/recover-epoch.cdc b/integration/utils/templates/recover-epoch.cdc
index cbd2172793d..5431599e727 100644
--- a/integration/utils/templates/recover-epoch.cdc
+++ b/integration/utils/templates/recover-epoch.cdc
@@ -8,30 +8,41 @@ import FlowClusterQC from "FlowClusterQC"
 // state with the new Epoch data.
 // This transaction should only be used with the output of the bootstrap utility:
 //   util epoch efm-recover-tx-args
-transaction(randomSource: String,
-            startView: UInt64,
+transaction(startView: UInt64,
             stakingEndView: UInt64,
             endView: UInt64,
             targetDuration: UInt64,
             targetEndTime: UInt64,
-            collectorClusters: [[String]],
+            clusterAssignments: [[String]],
             clusterQCVoteData: [FlowClusterQC.ClusterQCVoteData],
             dkgPubKeys: [String],
-            nodeIDs: [String]) {
+            nodeIDs: [String],
+            initNewEpoch: Bool) {
 
     prepare(signer: auth(BorrowValue) &Account) {
         let epochAdmin = signer.storage.borrow<&FlowEpoch.Admin>(from: FlowEpoch.adminStoragePath)
             ?? panic("Could not borrow epoch admin from storage path")
 
-        epochAdmin.recoverEpoch(randomSource: randomSource,
-                            startView: startView,
-                            stakingEndView: stakingEndView,
-                            endView: endView,
-                            targetDuration: targetDuration,
-                            targetEndTime: targetEndTime,
-                            collectorClusters: collectorClusters,
-                            clusterQCVoteData: clusterQCVoteData,
-                            dkgPubKeys: dkgPubKeys,
-                            nodeIDs: nodeIDs)
+        if initNewEpoch {
+            epochAdmin.recoverNewEpoch(startView: startView,
+                    stakingEndView: stakingEndView,
+                    endView: endView,
+                    targetDuration: targetDuration,
+                    targetEndTime: targetEndTime,
+                    clusterAssignments: clusterAssignments  ,
+                    clusterQCVoteData: clusterQCVoteData,
+                    dkgPubKeys: dkgPubKeys,
+                    nodeIDs: nodeIDs)
+        } else {
+            epochAdmin.recoverCurrentEpoch(startView: startView,
+                    stakingEndView: stakingEndView,
+                    endView: endView,
+                    targetDuration: targetDuration,
+                    targetEndTime: targetEndTime,
+                    clusterAssignments: clusterAssignments  ,
+                    clusterQCVoteData: clusterQCVoteData,
+                    dkgPubKeys: dkgPubKeys,
+                    nodeIDs: nodeIDs)
+        }
     }
 }
diff --git a/integration/utils/transactions.go b/integration/utils/transactions.go
index 686f9f7db9a..ba59bb40393 100644
--- a/integration/utils/transactions.go
+++ b/integration/utils/transactions.go
@@ -32,6 +32,7 @@ var setProtocolStateVersionScript string
 
 func LocalnetEnv() templates.Environment {
 	return templates.Environment{
+		EpochAddress:             "f8d6e0586b0a20c7",
 		IDTableAddress:           "f8d6e0586b0a20c7",
 		FungibleTokenAddress:     "ee82856bf20e2aa6",
 		FlowTokenAddress:         "0ae53cb6e3f42a79",
@@ -249,7 +250,7 @@ func MakeRecoverEpochTx(
 ) (*sdk.Transaction, error) {
 	accountKey := adminAccount.Keys[adminAccountKeyID]
 	tx := sdk.NewTransaction().
-		SetScript([]byte(templates.ReplaceAddresses(removeNodeTxScript, env))).
+		SetScript([]byte(templates.ReplaceAddresses(recoverEpochTxScript, env))).
 		SetComputeLimit(9999).
 		SetReferenceBlockID(latestBlockID).
 		SetProposalKey(adminAccount.Address, adminAccountKeyID, accountKey.SequenceNumber).

From ec964a85e854e0b6f8e16b300e3cce59a63cebe4 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 25 Jun 2024 10:29:07 -0400
Subject: [PATCH 14/60] update cluster qc vote data address, fix epoch
 configuration

---
 cmd/util/cmd/common/clusters.go               |  4 +--
 cmd/util/cmd/epochs/cmd/recover.go            |  7 ++++-
 integration/tests/epochs/base_suite.go        |  2 +-
 .../recover_epoch/recover_epoch_efm_test.go   | 26 +++++++++----------
 .../tests/epochs/recover_epoch/suite.go       |  7 +++--
 5 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/cmd/util/cmd/common/clusters.go b/cmd/util/cmd/common/clusters.go
index 3c961cda5dc..ab654c244db 100644
--- a/cmd/util/cmd/common/clusters.go
+++ b/cmd/util/cmd/common/clusters.go
@@ -200,9 +200,9 @@ func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.Cluste
 
 func newFlowClusterQCVoteDataStructType() *cadence.StructType {
 
-	// A.01cf0e2f2f715450.FlowClusterQC.ClusterQCVoteData
+	// A.0xf8d6e0586b0a20c7.FlowClusterQC.ClusterQCVoteData
 
-	address, _ := cdcCommon.HexToAddress("01cf0e2f2f715450")
+	address, _ := cdcCommon.HexToAddress("f8d6e0586b0a20c7")
 	location := cdcCommon.NewAddressLocation(nil, address, "FlowClusterQC")
 
 	return &cadence.StructType{
diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index 09267a2b12f..f8db065bf2d 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -57,6 +57,7 @@ This recovery process has some constraints:
 	flagEpochCounter             uint64
 	flagTargetDuration           uint64
 	flagTargetEndTime            uint64
+	flagInitNewEpoch             bool
 )
 
 func init() {
@@ -84,6 +85,7 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagEpochCounter, "epoch-counter", 0, "the epoch counter used to generate the root cluster block")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagTargetDuration, "target-duration", 0, "the target duration of the epoch")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagTargetEndTime, "target-end-time", 0, "the target end time for the epoch")
+	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagInitNewEpoch, "init", true, "set to false if the recover transaction should overwrite the current epoch rather than initialize a new recover epoch")
 
 	err := generateRecoverEpochTxArgsCmd.MarkFlagRequired("access-address")
 	if err != nil {
@@ -118,7 +120,7 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 
 func getSnapshot() *inmem.Snapshot {
 	// get flow client with secure client connection to download protocol snapshot from access node
-	config, err := grpcclient.NewFlowClientConfig(flagAnAddress, flagAnPubkey, flow.ZeroID, false)
+	config, err := grpcclient.NewFlowClientConfig(flagAnAddress, flagAnPubkey, flow.ZeroID, flagAnInsecure)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to create flow client config")
 	}
@@ -281,6 +283,9 @@ func extractRecoverEpochArgs(snapshot *inmem.Snapshot) []cadence.Value {
 		cadence.NewArray(dkgPubKeys),
 		// node ids
 		cadence.NewArray(nodeIds),
+		// recover the network by initializing a new recover epoch which will increment the smart contract epoch counter
+		// or overwrite the epoch metadata for the current epoch
+		cadence.NewBool(flagInitNewEpoch),
 	}
 
 	return args
diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index 0b73d7a53c9..4b56f33c3c0 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -90,7 +90,7 @@ func (s *BaseSuite) SetupTest() {
 		testnet.NewNodeConfig(flow.RoleCollection, collectionConfigs...),
 		testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...),
 		testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...),
-		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.WarnLevel), testnet.WithAdditionalFlag("--extensive-logging=true")),
+		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.InfoLevel), testnet.WithAdditionalFlag("--extensive-logging=true")),
 		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.WarnLevel)),
 		testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.WarnLevel)),
 		ghostNode,
diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index a5675e2d726..84046a753cb 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -27,12 +27,17 @@ type RecoverEpochSuite struct {
 // ensure the network is healthy.
 func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	// wait until the epoch setup phase to force network into EFM
-	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, 10*time.Second, 4*time.Second)
+	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, 10*time.Second, 500*time.Millisecond)
 
-	// pausing execution node will force the network into EFM
-	enContainer := s.GetContainersByRole(flow.RoleExecution)[0]
+	// pausing consensus node will force the network into EFM
+	enContainer := s.GetContainersByRole(flow.RoleCollection)[0]
 	_ = enContainer.Pause()
 
+	s.AwaitFinalizedView(s.Ctx, 32, 2*time.Minute, 500*time.Millisecond)
+
+	// start the paused execution node now that we are in EFM
+	require.NoError(s.T(), enContainer.Start())
+
 	// get the latest snapshot and start new container with it
 	epoch1FinalView, err := s.Net.BootstrapSnapshot.Epochs().Current().FinalView()
 	require.NoError(s.T(), err)
@@ -47,18 +52,14 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	//if counter is still 0, epoch emergency fallback was triggered as expected
 	s.AssertInEpoch(s.Ctx, 0)
 
-	// start the paused execution node now that we are in EFM
-	enContainer.Start()
-
 	// generate epoch recover transaction args
 	collectionClusters := uint64(1)
-	numViewsInEpoch := uint64(4000)
-	numViewsInStakingAuction := uint64(100)
-	epochCounter := uint64(2)
+	numViewsInEpoch := uint64(80)
+	numViewsInStakingAuction := uint64(2)
+	epochCounter := uint64(1)
 	targetDuration := uint64(3000)
 	targetEndTime := uint64(4000)
-	out := fmt.Sprintf("%s/recover-epoch-tx-args.josn", s.Net.BootstrapDir)
-
+	out := fmt.Sprintf("%s/recover-epoch-tx-args.json", s.Net.BootstrapDir)
 	s.executeEFMRecoverTXArgsCMD(
 		collectionClusters,
 		numViewsInEpoch,
@@ -73,9 +74,8 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 
 	txArgs, err := utils.ParseJSON(b)
 	require.NoError(s.T(), err)
-
 	env := utils.LocalnetEnv()
 	result := s.recoverEpoch(env, txArgs)
 	fmt.Println("TX RESULT STATUS: ", result)
-	// submit recover epoch transaction to recover the network
+	// wait until recover event is processed and we transition into new epoch
 }
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index ae07e11cdbf..2171f0a0a1a 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -24,8 +24,8 @@ type Suite struct {
 func (s *Suite) SetupTest() {
 	// use a shorter staking auction because we don't have staking operations in this case
 	s.StakingAuctionLen = 2
-	s.DKGPhaseLen = 50
-	s.EpochLen = 250
+	s.DKGPhaseLen = 10
+	s.EpochLen = 80
 	s.EpochCommitSafetyThreshold = 20
 
 	// run the generic setup, which starts up the network
@@ -93,12 +93,11 @@ func (s *Suite) recoverEpoch(env templates.Environment, args []cadence.Value) *s
 		sdk.Identifier(latestBlockID),
 		args,
 	)
-
-	fmt.Println("TRANSACTION SCRIPT \n", string(tx.Script))
 	require.NoError(s.T(), err)
 
 	err = s.Client.SignAndSendTransaction(s.Ctx, tx)
 	require.NoError(s.T(), err)
+	fmt.Println("")
 	result, err := s.Client.WaitForSealed(s.Ctx, tx.ID())
 	require.NoError(s.T(), err)
 	s.Client.Account().Keys[0].SequenceNumber++

From d0c40f2272bc817b6045ff114df61e580e6326fc Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Wed, 26 Jun 2024 11:30:54 -0400
Subject: [PATCH 15/60] fix service event decoding

---
 integration/tests/epochs/base_suite.go        | 34 +++++++++++-----
 .../recover_epoch/recover_epoch_efm_test.go   | 20 ++++++++--
 model/convert/service_event.go                | 40 +++++--------------
 3 files changed, 49 insertions(+), 45 deletions(-)

diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index 4b56f33c3c0..776a0bf3f6b 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -21,6 +21,7 @@ import (
 	"github.com/onflow/flow-go/integration/testnet"
 	"github.com/onflow/flow-go/integration/tests/lib"
 	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/state/protocol/inmem"
 	"github.com/onflow/flow-go/utils/unittest"
 )
 
@@ -74,7 +75,7 @@ func (s *BaseSuite) SetupTest() {
 		testnet.WithAdditionalFlag("--cruise-ctl-enabled=false"), // disable cruise control for integration tests
 		testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", s.RequiredSealApprovals)),
 		testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", s.RequiredSealApprovals)),
-		testnet.WithLogLevel(zerolog.InfoLevel)}
+		testnet.WithLogLevel(zerolog.ErrorLevel)}
 
 	// a ghost node masquerading as an access node
 	s.ghostID = unittest.IdentifierFixture()
@@ -90,8 +91,8 @@ func (s *BaseSuite) SetupTest() {
 		testnet.NewNodeConfig(flow.RoleCollection, collectionConfigs...),
 		testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...),
 		testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...),
-		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.InfoLevel), testnet.WithAdditionalFlag("--extensive-logging=true")),
-		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.WarnLevel)),
+		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.ErrorLevel), testnet.WithAdditionalFlag("--extensive-logging=true")),
+		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.ErrorLevel), testnet.WithAdditionalFlag("--extensive-logging=true")),
 		testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.WarnLevel)),
 		ghostNode,
 	}
@@ -168,15 +169,14 @@ func (s *BaseSuite) GetContainersByRole(role flow.Role) []*testnet.Container {
 // transition must have happened.
 func (s *BaseSuite) AwaitFinalizedView(ctx context.Context, view uint64, waitFor, tick time.Duration) {
 	require.Eventually(s.T(), func() bool {
-		finalized := s.getLatestFinalizedHeader(ctx)
+		finalized := s.GetLatestFinalizedHeader(ctx)
 		return finalized.View >= view
 	}, waitFor, tick)
 }
 
-// getLatestFinalizedHeader retrieves the latest finalized block, as reported in LatestSnapshot.
-func (s *BaseSuite) getLatestFinalizedHeader(ctx context.Context) *flow.Header {
-	snapshot, err := s.Client.GetLatestProtocolSnapshot(ctx)
-	require.NoError(s.T(), err)
+// GetLatestFinalizedHeader retrieves the latest finalized block, as reported in LatestSnapshot.
+func (s *BaseSuite) GetLatestFinalizedHeader(ctx context.Context) *flow.Header {
+	snapshot := s.GetLatestProtocolSnapshot(ctx)
 	finalized, err := snapshot.Head()
 	require.NoError(s.T(), err)
 	return finalized
@@ -184,9 +184,21 @@ func (s *BaseSuite) getLatestFinalizedHeader(ctx context.Context) *flow.Header {
 
 // AssertInEpoch requires actual epoch counter is equal to counter provided.
 func (s *BaseSuite) AssertInEpoch(ctx context.Context, expectedEpoch uint64) {
-	snapshot, err := s.Client.GetLatestProtocolSnapshot(ctx)
+	actualEpoch := s.CurrentEpoch(ctx)
+	require.Equalf(s.T(), expectedEpoch, actualEpoch, "expected to be in epoch %d got %d", expectedEpoch, actualEpoch)
+}
+
+// CurrentEpoch returns the current epoch.
+func (s *BaseSuite) CurrentEpoch(ctx context.Context) uint64 {
+	snapshot := s.GetLatestProtocolSnapshot(ctx)
+	counter, err := snapshot.Epochs().Current().Counter()
 	require.NoError(s.T(), err)
-	actualEpoch, err := snapshot.Epochs().Current().Counter()
+	return counter
+}
+
+// GetLatestProtocolSnapshot returns the latest protocol snapshot.
+func (s *BaseSuite) GetLatestProtocolSnapshot(ctx context.Context) *inmem.Snapshot {
+	snapshot, err := s.Client.GetLatestProtocolSnapshot(ctx)
 	require.NoError(s.T(), err)
-	require.Equalf(s.T(), expectedEpoch, actualEpoch, "expected to be in epoch %d got %d", expectedEpoch, actualEpoch)
+	return snapshot
 }
diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 84046a753cb..6cf7d1cafcb 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -54,7 +54,7 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 
 	// generate epoch recover transaction args
 	collectionClusters := uint64(1)
-	numViewsInEpoch := uint64(80)
+	numViewsInRecoveryEpoch := uint64(80)
 	numViewsInStakingAuction := uint64(2)
 	epochCounter := uint64(1)
 	targetDuration := uint64(3000)
@@ -62,7 +62,7 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	out := fmt.Sprintf("%s/recover-epoch-tx-args.json", s.Net.BootstrapDir)
 	s.executeEFMRecoverTXArgsCMD(
 		collectionClusters,
-		numViewsInEpoch,
+		numViewsInRecoveryEpoch,
 		numViewsInStakingAuction,
 		epochCounter,
 		targetDuration,
@@ -76,6 +76,20 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	require.NoError(s.T(), err)
 	env := utils.LocalnetEnv()
 	result := s.recoverEpoch(env, txArgs)
+
+	latestFinalizedHeader := s.GetLatestFinalizedHeader(s.Ctx)
+	// wait for at-least 2 epoch transitions to ensure successful recovery
+	waitForView := latestFinalizedHeader.View + numViewsInRecoveryEpoch
+	s.TimedLogf("waiting for at-least 2 epoch transitions (finalized view %d)", waitForView)
+	s.AwaitFinalizedView(s.Ctx, waitForView, 2*time.Minute, 500*time.Millisecond)
+	s.TimedLogf("observed finalized view %d", waitForView)
+
+	snap := s.GetLatestProtocolSnapshot(s.Ctx)
+	c, _ := snap.Epochs().Next().Counter()
+	fmt.Println("NEXT EPOCH COUNTER", c)
+	currentEpoch := s.CurrentEpoch(s.Ctx)
+	// assert we have transitioned out of Epoch 0 indicating a successful recovery.
+	require.Greater(s.T(), currentEpoch, uint64(0))
+
 	fmt.Println("TX RESULT STATUS: ", result)
-	// wait until recover event is processed and we transition into new epoch
 }
diff --git a/model/convert/service_event.go b/model/convert/service_event.go
index 5e55b021f9e..0ed91fa29bb 100644
--- a/model/convert/service_event.go
+++ b/model/convert/service_event.go
@@ -29,6 +29,7 @@ func ServiceEvent(chainID flow.ChainID, event flow.Event) (*flow.ServiceEvent, e
 	case events.EpochCommit.EventType():
 		return convertServiceEventEpochCommit(event)
 	case events.EpochRecover.EventType():
+		fmt.Println("RECEIVED EPOCH RECOVER SERVICE EVENT")
 		return convertServiceEventEpochRecover(event)
 	case events.VersionBeacon.EventType():
 		return convertServiceEventVersionBeacon(event)
@@ -733,11 +734,6 @@ func convertClusterQCVoteData(cdcClusterQCVoteData []cadence.Value) ([]flow.Clus
 			)
 		}
 
-		cdcRawVotes, err := getField[cadence.Array](fields, "voteSignatures")
-		if err != nil {
-			return nil, fmt.Errorf("failed to decode clusterQCVoteData struct: %w", err)
-		}
-
 		cdcVoterIDs, err := getField[cadence.Array](fields, "voterIDs")
 		if err != nil {
 			return nil, fmt.Errorf("failed to decode clusterQCVoteData struct: %w", err)
@@ -760,32 +756,14 @@ func convertClusterQCVoteData(cdcClusterQCVoteData []cadence.Value) ([]flow.Clus
 			voterIDs = append(voterIDs, voterID)
 		}
 
-		// gather all the vote signatures
-		signatures := make([]crypto.Signature, 0, len(cdcRawVotes.Values))
-		for _, cdcRawVote := range cdcRawVotes.Values {
-			rawVoteHex, ok := cdcRawVote.(cadence.String)
-			if !ok {
-				return nil, invalidCadenceTypeError(
-					"clusterQC[i].vote",
-					cdcRawVote,
-					cadence.String(""),
-				)
-			}
-			rawVoteBytes, err := hex.DecodeString(string(rawVoteHex))
-			if err != nil {
-				return nil, fmt.Errorf("could not convert raw vote from hex: %w", err)
-			}
-			signatures = append(signatures, rawVoteBytes)
+		cdcVoteSignatures, err := getField[cadence.String](fields, "voteSignatures")
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode clusterQCVoteData struct: %w", err)
 		}
-		// Aggregate BLS signatures
-		aggregatedSignature, err := crypto.AggregateBLSSignatures(signatures)
+
+		rawVoteBytes, err := hex.DecodeString(string(cdcVoteSignatures))
 		if err != nil {
-			// expected errors of the function are:
-			//  - empty list of signatures
-			//  - an input signature does not deserialize to a valid point
-			// Both are not expected at this stage because list is guaranteed not to be
-			// empty and individual signatures have been validated.
-			return nil, fmt.Errorf("cluster qc vote aggregation failed: %w", err)
+			return nil, fmt.Errorf("could not convert raw vote from hex: %w", err)
 		}
 
 		// check that aggregated signature is not identity, because an identity signature
@@ -804,13 +782,13 @@ func convertClusterQCVoteData(cdcClusterQCVoteData []cadence.Value) ([]flow.Clus
 		//  w.r.t their corresponding staking public key. It is therefore enough to check
 		//  the aggregated signature to conclude whether the aggregated public key is identity.
 		//  This check is therefore a sanity check to catch a potential issue early.
-		if crypto.IsBLSSignatureIdentity(aggregatedSignature) {
+		if crypto.IsBLSSignatureIdentity(rawVoteBytes) {
 			return nil, fmt.Errorf("cluster qc vote aggregation failed because resulting BLS signature is identity")
 		}
 
 		// set the fields on the QC vote data object
 		qcVoteDatas = append(qcVoteDatas, flow.ClusterQCVoteData{
-			SigData:  aggregatedSignature,
+			SigData:  rawVoteBytes,
 			VoterIDs: voterIDs,
 		})
 	}

From d3d7af6c5e732c7a4d3ad9df436ac91208f56749 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Wed, 8 May 2024 10:25:44 -0400
Subject: [PATCH 16/60] write node node pub infos file when test network is
 bootstrapped

---
 cmd/bootstrap/utils/key_generation.go | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/cmd/bootstrap/utils/key_generation.go b/cmd/bootstrap/utils/key_generation.go
index 627030a789f..cd58470ef25 100644
--- a/cmd/bootstrap/utils/key_generation.go
+++ b/cmd/bootstrap/utils/key_generation.go
@@ -4,14 +4,14 @@ import (
 	"crypto/rand"
 	"crypto/sha256"
 	"fmt"
-	gohash "hash"
-	"io"
-
 	"github.com/onflow/crypto"
 	"golang.org/x/crypto/hkdf"
+	gohash "hash"
+	"io"
 
 	sdk "github.com/onflow/flow-go-sdk"
 	sdkcrypto "github.com/onflow/flow-go-sdk/crypto"
+	model "github.com/onflow/flow-go/model/bootstrap"
 
 	"github.com/onflow/flow-go/fvm/systemcontracts"
 	"github.com/onflow/flow-go/model/bootstrap"
@@ -296,3 +296,20 @@ func WriteStakingNetworkingKeyFiles(nodeInfos []bootstrap.NodeInfo, write WriteJ
 
 	return nil
 }
+
+// WriteNodeInternalPubInfos writes the node-internal-infos.pub.json file.
+func WriteNodeInternalPubInfos(nodeInfos []bootstrap.NodeInfo, write WriteJSONFileFunc) error {
+	configs := make([]model.NodeConfig, len(nodeInfos))
+	for i, nodeInfo := range nodeInfos {
+		configs[i] = model.NodeConfig{
+			Role:    nodeInfo.Role,
+			Address: nodeInfo.Address,
+			Weight:  nodeInfo.Weight,
+		}
+	}
+	err := write(bootstrap.PathNodeInfosPub, configs)
+	if err != nil {
+		return err
+	}
+	return nil
+}

From 4be8e65ae3ee8aa7196c403c9b5e0965638939be Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Thu, 27 Jun 2024 19:56:47 -0400
Subject: [PATCH 17/60] update test godocs

---
 go.mod                                        |  4 +-
 go.sum                                        |  4 ++
 insecure/go.mod                               |  4 +-
 insecure/go.sum                               |  4 ++
 integration/go.mod                            |  4 +-
 integration/go.sum                            |  4 ++
 .../recover_epoch/recover_epoch_efm_test.go   | 50 +++++++++++--------
 7 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/go.mod b/go.mod
index 4478289e694..51e42d7f64f 100644
--- a/go.mod
+++ b/go.mod
@@ -50,8 +50,8 @@ require (
 	github.com/onflow/cadence v1.0.0-preview.34
 	github.com/onflow/crypto v0.25.1
 	github.com/onflow/flow v0.3.4
-	github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0
-	github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0
+	github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4
+	github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4
 	github.com/onflow/flow-go-sdk v1.0.0-preview.36
 	github.com/onflow/flow/protobuf/go/flow v0.4.4
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
diff --git a/go.sum b/go.sum
index 7beb1e4191c..6fabda02eb0 100644
--- a/go.sum
+++ b/go.sum
@@ -2180,8 +2180,12 @@ github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE=
 github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 h1:cq3RfBr9TnTSnsGlUHMjMGZib24Horfb1XJqMpkN5ew=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
+github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 h1:zLDxmCxa/jx86uwkkaFXpztjgrofIHk74Bj6d/raeXA=
+github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 h1:aMFJdB2CW+Dzm+AJ5QN6J1yWh+a0l2RxHN2/TtLaXUo=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
+github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 h1:W5dPPXV8cNG0nYxKe+g/8U2hkhZiIR6W8kFKqsJ5o9Y=
+github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A=
 github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs=
diff --git a/insecure/go.mod b/insecure/go.mod
index 653b5dddc0f..3273cb7ee2a 100644
--- a/insecure/go.mod
+++ b/insecure/go.mod
@@ -200,8 +200,8 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/onflow/atree v0.7.0-rc.2 // indirect
 	github.com/onflow/cadence v1.0.0-preview.34 // indirect
-	github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 // indirect
-	github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 // indirect
+	github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 // indirect
+	github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 // indirect
 	github.com/onflow/flow-ft/lib/go/contracts v1.0.0 // indirect
 	github.com/onflow/flow-ft/lib/go/templates v1.0.0 // indirect
 	github.com/onflow/flow-go-sdk v1.0.0-preview.36 // indirect
diff --git a/insecure/go.sum b/insecure/go.sum
index 030ffa3f4cb..b97cc739ed4 100644
--- a/insecure/go.sum
+++ b/insecure/go.sum
@@ -2167,8 +2167,12 @@ github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A=
 github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 h1:cq3RfBr9TnTSnsGlUHMjMGZib24Horfb1XJqMpkN5ew=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
+github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 h1:zLDxmCxa/jx86uwkkaFXpztjgrofIHk74Bj6d/raeXA=
+github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 h1:aMFJdB2CW+Dzm+AJ5QN6J1yWh+a0l2RxHN2/TtLaXUo=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
+github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 h1:W5dPPXV8cNG0nYxKe+g/8U2hkhZiIR6W8kFKqsJ5o9Y=
+github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A=
 github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs=
diff --git a/integration/go.mod b/integration/go.mod
index ce66485b5a2..d7ae0585a06 100644
--- a/integration/go.mod
+++ b/integration/go.mod
@@ -21,8 +21,8 @@ require (
 	github.com/libp2p/go-libp2p v0.32.2
 	github.com/onflow/cadence v1.0.0-preview.34
 	github.com/onflow/crypto v0.25.1
-	github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0
-	github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0
+	github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4
+	github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4
 	github.com/onflow/flow-emulator v1.0.0-preview.24
 	github.com/onflow/flow-go v0.35.5-0.20240517202625-55f862b45dfd
 	github.com/onflow/flow-go-sdk v1.0.0-preview.36
diff --git a/integration/go.sum b/integration/go.sum
index 821a5c27fd9..5dba5c144db 100644
--- a/integration/go.sum
+++ b/integration/go.sum
@@ -2157,8 +2157,12 @@ github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A=
 github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 h1:cq3RfBr9TnTSnsGlUHMjMGZib24Horfb1XJqMpkN5ew=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
+github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 h1:zLDxmCxa/jx86uwkkaFXpztjgrofIHk74Bj6d/raeXA=
+github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 h1:aMFJdB2CW+Dzm+AJ5QN6J1yWh+a0l2RxHN2/TtLaXUo=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
+github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 h1:W5dPPXV8cNG0nYxKe+g/8U2hkhZiIR6W8kFKqsJ5o9Y=
+github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-emulator v1.0.0-preview.24 h1:SonXMBeYxVwNn94M+OUmKIYScIMQG22wugh9n/tHY5k=
 github.com/onflow/flow-emulator v1.0.0-preview.24/go.mod h1:QprPouTWO3iv9VF/y4Ksltv2XIbzNMzjjr5zzq51i7Q=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0=
diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 6cf7d1cafcb..d7a307740bc 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -3,12 +3,14 @@ package recover_epoch
 import (
 	"fmt"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
+	sdk "github.com/onflow/flow-go-sdk"
 	"github.com/onflow/flow-go/integration/utils"
 	"github.com/onflow/flow-go/model/flow"
 )
@@ -21,20 +23,21 @@ type RecoverEpochSuite struct {
 	Suite
 }
 
-// TestRecoverEpoch ensures that the recover_epoch transaction flow works as expected. This test will simulate the network going
-// into EFM by taking a consensus node offline before completing the DKG. While in EFM mode the test will execute the efm-recover-tx-args
-// CLI command to generate transaction arguments to submit a recover_epoch transaction, after submitting the transaction the test will
-// ensure the network is healthy.
+// TestRecoverEpoch ensures that the recover epoch governance transaction flow works as expected and a network that
+// enters Epoch Fallback Mode can successfully recover. This test will do the following:
+// 1. Manually triggers EFM by turning off a collection node before the end of the DKG forcing the DKG to fail.
+// 2. Generates epoch recover transaction args using the epoch efm-recover-tx-args.
+// 3. Submit recover epoch transaction.
+// 4. Ensure expected EpochRecover event is emitted.
+// Currently, this test does not test the processing of the EpochRecover event see this issue: https://github.com/onflow/flow-go/issues/6164
 func (s *RecoverEpochSuite) TestRecoverEpoch() {
+	// 1. Manually trigger EFM
 	// wait until the epoch setup phase to force network into EFM
 	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, 10*time.Second, 500*time.Millisecond)
-
 	// pausing consensus node will force the network into EFM
 	enContainer := s.GetContainersByRole(flow.RoleCollection)[0]
 	_ = enContainer.Pause()
-
 	s.AwaitFinalizedView(s.Ctx, 32, 2*time.Minute, 500*time.Millisecond)
-
 	// start the paused execution node now that we are in EFM
 	require.NoError(s.T(), enContainer.Start())
 
@@ -52,6 +55,7 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	//if counter is still 0, epoch emergency fallback was triggered as expected
 	s.AssertInEpoch(s.Ctx, 0)
 
+	// 2. Generate transaction arguments for epoch recover transaction.
 	// generate epoch recover transaction args
 	collectionClusters := uint64(1)
 	numViewsInRecoveryEpoch := uint64(80)
@@ -72,24 +76,26 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	b, err := os.ReadFile(out)
 	require.NoError(s.T(), err)
 
+	// 3. Submit recover epoch transaction to the network.
+	// submit the recover epoch transaction
 	txArgs, err := utils.ParseJSON(b)
 	require.NoError(s.T(), err)
 	env := utils.LocalnetEnv()
 	result := s.recoverEpoch(env, txArgs)
+	require.NoError(s.T(), result.Error)
+	require.Equal(s.T(), result.Status, sdk.TransactionStatusSealed)
+
+	// 3. Ensure EpochRecover event was emitted.
+	eventType := ""
+	for _, evt := range result.Events {
+		if strings.Contains(evt.Type, "FlowEpoch.EpochRecover") {
+			eventType = evt.Type
+			break
+		}
+	}
+	events, err := s.Client.GetEventsForBlockIDs(s.Ctx, eventType, []sdk.Identifier{result.BlockID})
+	require.NoError(s.T(), err)
+	require.Equal(s.T(), events[0].Events[0].Type, eventType)
 
-	latestFinalizedHeader := s.GetLatestFinalizedHeader(s.Ctx)
-	// wait for at-least 2 epoch transitions to ensure successful recovery
-	waitForView := latestFinalizedHeader.View + numViewsInRecoveryEpoch
-	s.TimedLogf("waiting for at-least 2 epoch transitions (finalized view %d)", waitForView)
-	s.AwaitFinalizedView(s.Ctx, waitForView, 2*time.Minute, 500*time.Millisecond)
-	s.TimedLogf("observed finalized view %d", waitForView)
-
-	snap := s.GetLatestProtocolSnapshot(s.Ctx)
-	c, _ := snap.Epochs().Next().Counter()
-	fmt.Println("NEXT EPOCH COUNTER", c)
-	currentEpoch := s.CurrentEpoch(s.Ctx)
-	// assert we have transitioned out of Epoch 0 indicating a successful recovery.
-	require.Greater(s.T(), currentEpoch, uint64(0))
-
-	fmt.Println("TX RESULT STATUS: ", result)
+	// 4. @TODO ensure EpochRecover service event is processed by the fallback state machine and the network recovers.
 }

From 62028cd6d07197fb66428825bacc6eae15fdf4f8 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Thu, 27 Jun 2024 20:19:21 -0400
Subject: [PATCH 18/60] update test components

- remove script
- move cluster qc contract address to argument
- remove debug line
---
 cmd/util/cmd/common/clusters.go               | 12 ++---
 cmd/util/cmd/epochs/cmd/recover.go            |  8 +++-
 integration/tests/epochs/base_suite.go        |  8 ++--
 .../recover_epoch/recover_epoch_efm_test.go   |  6 +--
 .../tests/epochs/recover_epoch/suite.go       |  1 +
 integration/utils/templates/recover-epoch.cdc | 48 -------------------
 integration/utils/transactions.go             |  5 +-
 model/convert/service_event.go                |  1 -
 8 files changed, 22 insertions(+), 67 deletions(-)
 delete mode 100644 integration/utils/templates/recover-epoch.cdc

diff --git a/cmd/util/cmd/common/clusters.go b/cmd/util/cmd/common/clusters.go
index ab654c244db..f7d9b128a4a 100644
--- a/cmd/util/cmd/common/clusters.go
+++ b/cmd/util/cmd/common/clusters.go
@@ -164,8 +164,8 @@ func ConvertClusterAssignmentsCdc(assignments flow.AssignmentList) cadence.Array
 }
 
 // ConvertClusterQcsCdc converts cluster QCs from `QuorumCertificate` type to `ClusterQCVoteData` type.
-func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.ClusterList) ([]cadence.Value, error) {
-	voteDataType := newFlowClusterQCVoteDataStructType()
+func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.ClusterList, address string) ([]cadence.Value, error) {
+	voteDataType := newFlowClusterQCVoteDataStructType(address)
 	qcVoteData := make([]cadence.Value, len(qcs))
 	for i, qc := range qcs {
 		c, ok := clusterList.ByIndex(uint(i))
@@ -198,11 +198,11 @@ func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.Cluste
 	return qcVoteData, nil
 }
 
-func newFlowClusterQCVoteDataStructType() *cadence.StructType {
+// newFlowClusterQCVoteDataStructType returns the FlowClusterQC cadence struct type.
+func newFlowClusterQCVoteDataStructType(clusterQcAddress string) *cadence.StructType {
 
-	// A.0xf8d6e0586b0a20c7.FlowClusterQC.ClusterQCVoteData
-
-	address, _ := cdcCommon.HexToAddress("f8d6e0586b0a20c7")
+	// FlowClusterQC.ClusterQCVoteData
+	address, _ := cdcCommon.HexToAddress(clusterQcAddress)
 	location := cdcCommon.NewAddressLocation(nil, address, "FlowClusterQC")
 
 	return &cadence.StructType{
diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index f8db065bf2d..3001051b0a7 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -58,6 +58,7 @@ This recovery process has some constraints:
 	flagTargetDuration           uint64
 	flagTargetEndTime            uint64
 	flagInitNewEpoch             bool
+	flagClusterQCAddress         string
 )
 
 func init() {
@@ -71,6 +72,7 @@ func init() {
 func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagOut, "out", "", "file to write tx args output")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnAddress, "access-address", "", "the address of the access node used for client connections")
+	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagClusterQCAddress, "cluster-qc-contract-address", "", "the contract address for the FlowClusterQC smart contract")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnPubkey, "access-network-key", "", "the network key of the access node used for client connections in hex string format")
 	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagAnInsecure, "insecure", false, "set to true if the protocol snapshot should be retrieved from the insecure AN endpoint")
 	generateRecoverEpochTxArgsCmd.Flags().IntVar(&flagCollectionClusters, "collection-clusters", 0,
@@ -115,6 +117,10 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	if err != nil {
 		return fmt.Errorf("failed to mark target-end-time flag as required")
 	}
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("cluster-qc-contract-address")
+	if err != nil {
+		return fmt.Errorf("failed to mark cluster-qc-contract-address flag as required")
+	}
 	return nil
 }
 
@@ -254,7 +260,7 @@ func extractRecoverEpochArgs(snapshot *inmem.Snapshot) []cadence.Value {
 		nodeIds = append(nodeIds, nodeIdCdc)
 	}
 
-	qcVoteData, err := common.ConvertClusterQcsCdc(clusterQCs, clusters)
+	qcVoteData, err := common.ConvertClusterQcsCdc(clusterQCs, clusters, flagClusterQCAddress)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to convert cluster qcs to cadence type")
 	}
diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index 776a0bf3f6b..3299a865d2a 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -86,14 +86,14 @@ func (s *BaseSuite) SetupTest() {
 		testnet.AsGhost())
 
 	confs := []testnet.NodeConfig{
-		testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.WarnLevel)),
-		testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.WarnLevel)),
+		testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.ErrorLevel)),
+		testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.ErrorLevel)),
 		testnet.NewNodeConfig(flow.RoleCollection, collectionConfigs...),
 		testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...),
 		testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...),
 		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.ErrorLevel), testnet.WithAdditionalFlag("--extensive-logging=true")),
-		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.ErrorLevel), testnet.WithAdditionalFlag("--extensive-logging=true")),
-		testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.WarnLevel)),
+		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.ErrorLevel)),
+		testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.ErrorLevel)),
 		ghostNode,
 	}
 
diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index d7a307740bc..4f74175603a 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -35,11 +35,11 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	// wait until the epoch setup phase to force network into EFM
 	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, 10*time.Second, 500*time.Millisecond)
 	// pausing consensus node will force the network into EFM
-	enContainer := s.GetContainersByRole(flow.RoleCollection)[0]
-	_ = enContainer.Pause()
+	ln := s.GetContainersByRole(flow.RoleCollection)[0]
+	_ = ln.Pause()
 	s.AwaitFinalizedView(s.Ctx, 32, 2*time.Minute, 500*time.Millisecond)
 	// start the paused execution node now that we are in EFM
-	require.NoError(s.T(), enContainer.Start())
+	require.NoError(s.T(), ln.Start())
 
 	// get the latest snapshot and start new container with it
 	epoch1FinalView, err := s.Net.BootstrapSnapshot.Epochs().Current().FinalView()
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 2171f0a0a1a..c5f91677fcc 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -61,6 +61,7 @@ func (s *Suite) executeEFMRecoverTXArgsCMD(collectionClusters, numViewsInEpoch,
 	os.Args = []string{
 		"epochs", "efm-recover-tx-args",
 		"--insecure=true",
+		"--cluster-qc-contract-address=0xf8d6e0586b0a20c7",
 		fmt.Sprintf("--out=%s", out),
 		fmt.Sprintf("--access-address=%s", anAddress),
 		fmt.Sprintf("--collection-clusters=%d", collectionClusters),
diff --git a/integration/utils/templates/recover-epoch.cdc b/integration/utils/templates/recover-epoch.cdc
deleted file mode 100644
index 5431599e727..00000000000
--- a/integration/utils/templates/recover-epoch.cdc
+++ /dev/null
@@ -1,48 +0,0 @@
-import FlowEpoch from "FlowEpoch"
-import FlowIDTableStaking from "FlowIDTableStaking"
-import FlowClusterQC from "FlowClusterQC"
-
-// The recoverEpoch transaction creates and starts a new epoch in the FlowEpoch smart contract
-// to which will force the network exit EFM. The recoverEpoch service event will be emitted
-// and processed by all protocol participants and each participant will update their protocol
-// state with the new Epoch data.
-// This transaction should only be used with the output of the bootstrap utility:
-//   util epoch efm-recover-tx-args
-transaction(startView: UInt64,
-            stakingEndView: UInt64,
-            endView: UInt64,
-            targetDuration: UInt64,
-            targetEndTime: UInt64,
-            clusterAssignments: [[String]],
-            clusterQCVoteData: [FlowClusterQC.ClusterQCVoteData],
-            dkgPubKeys: [String],
-            nodeIDs: [String],
-            initNewEpoch: Bool) {
-
-    prepare(signer: auth(BorrowValue) &Account) {
-        let epochAdmin = signer.storage.borrow<&FlowEpoch.Admin>(from: FlowEpoch.adminStoragePath)
-            ?? panic("Could not borrow epoch admin from storage path")
-
-        if initNewEpoch {
-            epochAdmin.recoverNewEpoch(startView: startView,
-                    stakingEndView: stakingEndView,
-                    endView: endView,
-                    targetDuration: targetDuration,
-                    targetEndTime: targetEndTime,
-                    clusterAssignments: clusterAssignments  ,
-                    clusterQCVoteData: clusterQCVoteData,
-                    dkgPubKeys: dkgPubKeys,
-                    nodeIDs: nodeIDs)
-        } else {
-            epochAdmin.recoverCurrentEpoch(startView: startView,
-                    stakingEndView: stakingEndView,
-                    endView: endView,
-                    targetDuration: targetDuration,
-                    targetEndTime: targetEndTime,
-                    clusterAssignments: clusterAssignments  ,
-                    clusterQCVoteData: clusterQCVoteData,
-                    dkgPubKeys: dkgPubKeys,
-                    nodeIDs: nodeIDs)
-        }
-    }
-}
diff --git a/integration/utils/transactions.go b/integration/utils/transactions.go
index ba59bb40393..475d5ad3b6d 100644
--- a/integration/utils/transactions.go
+++ b/integration/utils/transactions.go
@@ -18,9 +18,6 @@ import (
 	"github.com/onflow/flow-go/utils/unittest"
 )
 
-//go:embed templates/recover-epoch.cdc
-var recoverEpochTxScript string
-
 //go:embed templates/create-and-setup-node.cdc
 var createAndSetupNodeTxScript string
 
@@ -250,7 +247,7 @@ func MakeRecoverEpochTx(
 ) (*sdk.Transaction, error) {
 	accountKey := adminAccount.Keys[adminAccountKeyID]
 	tx := sdk.NewTransaction().
-		SetScript([]byte(templates.ReplaceAddresses(recoverEpochTxScript, env))).
+		SetScript([]byte(templates.GenerateRecoverEpochScript(env))).
 		SetComputeLimit(9999).
 		SetReferenceBlockID(latestBlockID).
 		SetProposalKey(adminAccount.Address, adminAccountKeyID, accountKey.SequenceNumber).
diff --git a/model/convert/service_event.go b/model/convert/service_event.go
index 0ed91fa29bb..2ccfd104701 100644
--- a/model/convert/service_event.go
+++ b/model/convert/service_event.go
@@ -29,7 +29,6 @@ func ServiceEvent(chainID flow.ChainID, event flow.Event) (*flow.ServiceEvent, e
 	case events.EpochCommit.EventType():
 		return convertServiceEventEpochCommit(event)
 	case events.EpochRecover.EventType():
-		fmt.Println("RECEIVED EPOCH RECOVER SERVICE EVENT")
 		return convertServiceEventEpochRecover(event)
 	case events.VersionBeacon.EventType():
 		return convertServiceEventVersionBeacon(event)

From 7a3b6b92ab93c8a9ce58c87df257b19b78657b96 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Thu, 27 Jun 2024 20:26:26 -0400
Subject: [PATCH 19/60] Update clusters.go

---
 cmd/util/cmd/common/clusters.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/cmd/util/cmd/common/clusters.go b/cmd/util/cmd/common/clusters.go
index f7d9b128a4a..922464149fd 100644
--- a/cmd/util/cmd/common/clusters.go
+++ b/cmd/util/cmd/common/clusters.go
@@ -176,11 +176,6 @@ func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.Cluste
 		if err != nil {
 			return nil, fmt.Errorf("could not decode signer indices: %w", err)
 		}
-		voteData := &flow.ClusterQCVoteData{
-			SigData:  qc.SigData,
-			VoterIDs: voterIds,
-		}
-
 		cdcVoterIds := make([]cadence.Value, len(voterIds))
 		for i, id := range voterIds {
 			cdcVoterIds[i] = cadence.String(id.String())
@@ -188,7 +183,7 @@ func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.Cluste
 
 		qcVoteData[i] = cadence.NewStruct([]cadence.Value{
 			// voteSignatures
-			cadence.String(voteData.SigData.String()),
+			cadence.String(fmt.Sprintf("%#x", qc.SigData)),
 			// voterIDs
 			cadence.NewArray(cdcVoterIds).WithType(cadence.NewVariableSizedArrayType(cadence.StringType)),
 		}).WithType(voteDataType)

From 4ba9d6a411d5de7410b0bd0d081b475c09100091 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Fri, 28 Jun 2024 12:22:55 -0400
Subject: [PATCH 20/60] update go.mod(s)

---
 go.sum             | 4 ----
 insecure/go.sum    | 4 ----
 integration/go.sum | 4 ----
 3 files changed, 12 deletions(-)

diff --git a/go.sum b/go.sum
index 6fabda02eb0..ac8beff7acf 100644
--- a/go.sum
+++ b/go.sum
@@ -2178,12 +2178,8 @@ github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A=
 github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
 github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE=
 github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c=
-github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 h1:cq3RfBr9TnTSnsGlUHMjMGZib24Horfb1XJqMpkN5ew=
-github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 h1:zLDxmCxa/jx86uwkkaFXpztjgrofIHk74Bj6d/raeXA=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
-github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 h1:aMFJdB2CW+Dzm+AJ5QN6J1yWh+a0l2RxHN2/TtLaXUo=
-github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 h1:W5dPPXV8cNG0nYxKe+g/8U2hkhZiIR6W8kFKqsJ5o9Y=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0=
diff --git a/insecure/go.sum b/insecure/go.sum
index b97cc739ed4..7db187ea31f 100644
--- a/insecure/go.sum
+++ b/insecure/go.sum
@@ -2165,12 +2165,8 @@ github.com/onflow/cadence v1.0.0-preview.34/go.mod h1:jOwvPSSLTr9TvaKMs7KKiBYMmp
 github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
 github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A=
 github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
-github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 h1:cq3RfBr9TnTSnsGlUHMjMGZib24Horfb1XJqMpkN5ew=
-github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 h1:zLDxmCxa/jx86uwkkaFXpztjgrofIHk74Bj6d/raeXA=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
-github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 h1:aMFJdB2CW+Dzm+AJ5QN6J1yWh+a0l2RxHN2/TtLaXUo=
-github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 h1:W5dPPXV8cNG0nYxKe+g/8U2hkhZiIR6W8kFKqsJ5o9Y=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0=
diff --git a/integration/go.sum b/integration/go.sum
index 5dba5c144db..4df7f6ca6f4 100644
--- a/integration/go.sum
+++ b/integration/go.sum
@@ -2155,12 +2155,8 @@ github.com/onflow/cadence v1.0.0-preview.34/go.mod h1:jOwvPSSLTr9TvaKMs7KKiBYMmp
 github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
 github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A=
 github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
-github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0 h1:cq3RfBr9TnTSnsGlUHMjMGZib24Horfb1XJqMpkN5ew=
-github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.0/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 h1:zLDxmCxa/jx86uwkkaFXpztjgrofIHk74Bj6d/raeXA=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
-github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0 h1:aMFJdB2CW+Dzm+AJ5QN6J1yWh+a0l2RxHN2/TtLaXUo=
-github.com/onflow/flow-core-contracts/lib/go/templates v1.3.0/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 h1:W5dPPXV8cNG0nYxKe+g/8U2hkhZiIR6W8kFKqsJ5o9Y=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-emulator v1.0.0-preview.24 h1:SonXMBeYxVwNn94M+OUmKIYScIMQG22wugh9n/tHY5k=

From 1a023976cd8421cffca677d9e6535e4d1d5580d6 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Mon, 8 Jul 2024 22:58:15 -0400
Subject: [PATCH 21/60] Apply suggestions from code review

Co-authored-by: Jordan Schalm <jordan.schalm@flowfoundation.org>
Co-authored-by: Yurii Oleksyshyn <yuraolex@gmail.com>
---
 cmd/util/cmd/epochs/cmd/recover.go              | 12 ++++++++----
 integration/tests/epochs/recover_epoch/suite.go |  1 -
 model/convert/service_event.go                  |  4 ++--
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index 3001051b0a7..8d858ae6d4b 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -85,9 +85,13 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagNumViewsInEpoch, "epoch-length", 0, "length of each epoch measured in views")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 0, "length of the epoch staking phase measured in views")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagEpochCounter, "epoch-counter", 0, "the epoch counter used to generate the root cluster block")
-	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagTargetDuration, "target-duration", 0, "the target duration of the epoch")
-	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagTargetEndTime, "target-end-time", 0, "the target end time for the epoch")
-	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagInitNewEpoch, "init", true, "set to false if the recover transaction should overwrite the current epoch rather than initialize a new recover epoch")
+	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagTargetDuration, "epoch-timing-duration", 0, "the target duration of the epoch, in seconds")
+	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagTargetEndTime, "epoch-timing-end-time", 0, "the target end time for the epoch, specified in second-precision Unix time")
+	// The following option allows the RecoveryEpoch specified by this command to overwrite an epoch which already exists in the smart contract.
+	// This is needed only if a previous recoverEpoch transaction was submitted and a race condition occurred such that:
+	//   - the RecoveryEpoch in the admin transaction was accepted by the smart contract
+	//   - the RecoveryEpoch service event (after sealing latency) was rejected by the Protocol State
+	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagInitNewEpoch, "unsafe-overwrite-epoch-data", false, "set to true if the resulting transaction is allowed to overwrite an existing epoch data entry in the smart contract. ")
 
 	err := generateRecoverEpochTxArgsCmd.MarkFlagRequired("access-address")
 	if err != nil {
@@ -163,7 +167,7 @@ func generateRecoverEpochTxArgs(getSnapshot func() *inmem.Snapshot) func(cmd *co
 				log.Fatal().Err(err).Msg("could not write jsoncdc encoded arguments")
 			}
 		} else {
-			// write JSON args to stdout
+			// write JSON args to file specified by flag
 			err := os.WriteFile(flagOut, encodedTxArgs, 0644)
 			if err != nil {
 				log.Fatal().Err(err).Msg(fmt.Sprintf("could not write jsoncdc encoded arguments to file %s", flagOut))
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index c5f91677fcc..8caa29833e0 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -98,7 +98,6 @@ func (s *Suite) recoverEpoch(env templates.Environment, args []cadence.Value) *s
 
 	err = s.Client.SignAndSendTransaction(s.Ctx, tx)
 	require.NoError(s.T(), err)
-	fmt.Println("")
 	result, err := s.Client.WaitForSealed(s.Ctx, tx.ID())
 	require.NoError(s.T(), err)
 	s.Client.Account().Keys[0].SequenceNumber++
diff --git a/model/convert/service_event.go b/model/convert/service_event.go
index 2ccfd104701..0232bf9eeac 100644
--- a/model/convert/service_event.go
+++ b/model/convert/service_event.go
@@ -755,12 +755,12 @@ func convertClusterQCVoteData(cdcClusterQCVoteData []cadence.Value) ([]flow.Clus
 			voterIDs = append(voterIDs, voterID)
 		}
 
-		cdcVoteSignatures, err := getField[cadence.String](fields, "voteSignatures")
+		cdcAggSignature, err := getField[cadence.String](fields, "voteSignatures")
 		if err != nil {
 			return nil, fmt.Errorf("failed to decode clusterQCVoteData struct: %w", err)
 		}
 
-		rawVoteBytes, err := hex.DecodeString(string(cdcVoteSignatures))
+		aggregatedSignature, err := hex.DecodeString(string(cdcVoteSignatures))
 		if err != nil {
 			return nil, fmt.Errorf("could not convert raw vote from hex: %w", err)
 		}

From d5da0871f6033b0dd8f89b0091b9fd42b64aec2c Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 21:06:23 -0400
Subject: [PATCH 22/60] Update integration/tests/epochs/base_suite.go

Co-authored-by: Jordan Schalm <jordan.schalm@flowfoundation.org>
---
 integration/tests/epochs/base_suite.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index 3299a865d2a..47af0e8dc88 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -188,7 +188,7 @@ func (s *BaseSuite) AssertInEpoch(ctx context.Context, expectedEpoch uint64) {
 	require.Equalf(s.T(), expectedEpoch, actualEpoch, "expected to be in epoch %d got %d", expectedEpoch, actualEpoch)
 }
 
-// CurrentEpoch returns the current epoch.
+// CurrentEpoch returns the current epoch counter.
 func (s *BaseSuite) CurrentEpoch(ctx context.Context) uint64 {
 	snapshot := s.GetLatestProtocolSnapshot(ctx)
 	counter, err := snapshot.Epochs().Current().Counter()

From d70de6eda8b8ea5b84f174295f622d3ad5054f37 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 21:24:50 -0400
Subject: [PATCH 23/60] use epoch-timing-duration and epoch-timing-end-time

- use aggregatedSignature
-remove unused arg
---
 cmd/util/cmd/epochs/cmd/recover.go              | 8 ++------
 integration/tests/epochs/recover_epoch/suite.go | 4 ++--
 model/convert/service_event.go                  | 6 +++---
 3 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index 8d858ae6d4b..e67824c9358 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -18,10 +18,6 @@ import (
 	"github.com/onflow/flow-go/state/protocol/inmem"
 )
 
-const (
-	outputFileName = "recover-epoch-tx-args.json"
-)
-
 // generateRecoverEpochTxArgsCmd represents a command to generate the data needed to submit an epoch-recovery transaction
 // to the network when it is in EFM (epoch fallback mode).
 // EFM can be exited only by a special service event, EpochRecover, which initially originates from a manual service account transaction.
@@ -113,11 +109,11 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	if err != nil {
 		return fmt.Errorf("failed to mark collection-clusters flag as required")
 	}
-	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("target-duration")
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("epoch-timing-duration")
 	if err != nil {
 		return fmt.Errorf("failed to mark target-duration flag as required")
 	}
-	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("target-end-time")
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("epoch-timing-end-time")
 	if err != nil {
 		return fmt.Errorf("failed to mark target-end-time flag as required")
 	}
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 8caa29833e0..67f060f91f7 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -70,8 +70,8 @@ func (s *Suite) executeEFMRecoverTXArgsCMD(collectionClusters, numViewsInEpoch,
 		fmt.Sprintf("--epoch-length=%d", numViewsInEpoch),
 		fmt.Sprintf("--epoch-staking-phase-length=%d", numViewsInStakingAuction),
 		fmt.Sprintf("--epoch-counter=%d", epochCounter),
-		fmt.Sprintf("--target-duration=%d", targetDuration),
-		fmt.Sprintf("--target-end-time=%d", targetEndTime),
+		fmt.Sprintf("--epoch-timing-duration=%d", targetDuration),
+		fmt.Sprintf("--epoch-timing-end-time=%d", targetEndTime),
 	}
 
 	// execute the root command
diff --git a/model/convert/service_event.go b/model/convert/service_event.go
index 0232bf9eeac..2f95bbbfe41 100644
--- a/model/convert/service_event.go
+++ b/model/convert/service_event.go
@@ -760,7 +760,7 @@ func convertClusterQCVoteData(cdcClusterQCVoteData []cadence.Value) ([]flow.Clus
 			return nil, fmt.Errorf("failed to decode clusterQCVoteData struct: %w", err)
 		}
 
-		aggregatedSignature, err := hex.DecodeString(string(cdcVoteSignatures))
+		aggregatedSignature, err := hex.DecodeString(string(cdcAggSignature))
 		if err != nil {
 			return nil, fmt.Errorf("could not convert raw vote from hex: %w", err)
 		}
@@ -781,13 +781,13 @@ func convertClusterQCVoteData(cdcClusterQCVoteData []cadence.Value) ([]flow.Clus
 		//  w.r.t their corresponding staking public key. It is therefore enough to check
 		//  the aggregated signature to conclude whether the aggregated public key is identity.
 		//  This check is therefore a sanity check to catch a potential issue early.
-		if crypto.IsBLSSignatureIdentity(rawVoteBytes) {
+		if crypto.IsBLSSignatureIdentity(aggregatedSignature) {
 			return nil, fmt.Errorf("cluster qc vote aggregation failed because resulting BLS signature is identity")
 		}
 
 		// set the fields on the QC vote data object
 		qcVoteDatas = append(qcVoteDatas, flow.ClusterQCVoteData{
-			SigData:  rawVoteBytes,
+			SigData:  aggregatedSignature,
 			VoterIDs: voterIDs,
 		})
 	}

From b1c23ed804a42fabaac340e9c959994dcd8f45d4 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 22:03:17 -0400
Subject: [PATCH 24/60] use --root-chain-id

---
 cmd/util/cmd/epochs/cmd/recover.go              | 12 +++++++-----
 integration/tests/epochs/recover_epoch/suite.go |  2 +-
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index e67824c9358..280e8fe6276 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -12,6 +12,7 @@ import (
 	"github.com/onflow/flow-go/cmd/bootstrap/run"
 	"github.com/onflow/flow-go/cmd/util/cmd/common"
 	epochcmdutil "github.com/onflow/flow-go/cmd/util/cmd/epochs/utils"
+	"github.com/onflow/flow-go/fvm/systemcontracts"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/model/flow/filter"
 	"github.com/onflow/flow-go/module/grpcclient"
@@ -54,7 +55,7 @@ This recovery process has some constraints:
 	flagTargetDuration           uint64
 	flagTargetEndTime            uint64
 	flagInitNewEpoch             bool
-	flagClusterQCAddress         string
+	flagRootChainID              string
 )
 
 func init() {
@@ -68,7 +69,7 @@ func init() {
 func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagOut, "out", "", "file to write tx args output")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnAddress, "access-address", "", "the address of the access node used for client connections")
-	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagClusterQCAddress, "cluster-qc-contract-address", "", "the contract address for the FlowClusterQC smart contract")
+	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagRootChainID, "root-chain-id", "", "the root chain id")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnPubkey, "access-network-key", "", "the network key of the access node used for client connections in hex string format")
 	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagAnInsecure, "insecure", false, "set to true if the protocol snapshot should be retrieved from the insecure AN endpoint")
 	generateRecoverEpochTxArgsCmd.Flags().IntVar(&flagCollectionClusters, "collection-clusters", 0,
@@ -117,9 +118,9 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	if err != nil {
 		return fmt.Errorf("failed to mark target-end-time flag as required")
 	}
-	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("cluster-qc-contract-address")
+	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("root-chain-id")
 	if err != nil {
-		return fmt.Errorf("failed to mark cluster-qc-contract-address flag as required")
+		return fmt.Errorf("failed to mark root-chain-id flag as required")
 	}
 	return nil
 }
@@ -260,7 +261,8 @@ func extractRecoverEpochArgs(snapshot *inmem.Snapshot) []cadence.Value {
 		nodeIds = append(nodeIds, nodeIdCdc)
 	}
 
-	qcVoteData, err := common.ConvertClusterQcsCdc(clusterQCs, clusters, flagClusterQCAddress)
+	clusterQCAddress := systemcontracts.SystemContractsForChain(flow.ChainID(flagRootChainID)).ClusterQC.Address.String()
+	qcVoteData, err := common.ConvertClusterQcsCdc(clusterQCs, clusters, clusterQCAddress)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to convert cluster qcs to cadence type")
 	}
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 67f060f91f7..23be68dd90b 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -61,7 +61,7 @@ func (s *Suite) executeEFMRecoverTXArgsCMD(collectionClusters, numViewsInEpoch,
 	os.Args = []string{
 		"epochs", "efm-recover-tx-args",
 		"--insecure=true",
-		"--cluster-qc-contract-address=0xf8d6e0586b0a20c7",
+		fmt.Sprintf("--root-chain-id=%s", flow.Localnet),
 		fmt.Sprintf("--out=%s", out),
 		fmt.Sprintf("--access-address=%s", anAddress),
 		fmt.Sprintf("--collection-clusters=%d", collectionClusters),

From 53b2bbfcbc6802ae596c5652a26b515f88079b07 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 22:04:58 -0400
Subject: [PATCH 25/60] Update
 integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go

Co-authored-by: Jordan Schalm <jordan.schalm@flowfoundation.org>
---
 .../tests/epochs/recover_epoch/recover_epoch_efm_test.go        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 4f74175603a..356b089ddd6 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -29,7 +29,7 @@ type RecoverEpochSuite struct {
 // 2. Generates epoch recover transaction args using the epoch efm-recover-tx-args.
 // 3. Submit recover epoch transaction.
 // 4. Ensure expected EpochRecover event is emitted.
-// Currently, this test does not test the processing of the EpochRecover event see this issue: https://github.com/onflow/flow-go/issues/6164
+// TODO(EFM, #6164): Currently, this test does not test the processing of the EpochRecover event
 func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	// 1. Manually trigger EFM
 	// wait until the epoch setup phase to force network into EFM

From 04880e702dd6a20f1109199401b4fc4bf2910e37 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 22:07:08 -0400
Subject: [PATCH 26/60] Update
 integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go

Co-authored-by: Jordan Schalm <jordan.schalm@flowfoundation.org>
---
 .../tests/epochs/recover_epoch/recover_epoch_efm_test.go        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 356b089ddd6..2d1874e2275 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -38,7 +38,7 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	ln := s.GetContainersByRole(flow.RoleCollection)[0]
 	_ = ln.Pause()
 	s.AwaitFinalizedView(s.Ctx, 32, 2*time.Minute, 500*time.Millisecond)
-	// start the paused execution node now that we are in EFM
+	// start the paused collection node now that we are in EFM
 	require.NoError(s.T(), ln.Start())
 
 	// get the latest snapshot and start new container with it

From d0527303a9d8f15c037a845018729495e6c7d9d2 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 22:07:21 -0400
Subject: [PATCH 27/60] Update
 integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go

Co-authored-by: Yurii Oleksyshyn <yuraolex@gmail.com>
---
 .../tests/epochs/recover_epoch/recover_epoch_efm_test.go        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 2d1874e2275..da71fdd3246 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -34,7 +34,7 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	// 1. Manually trigger EFM
 	// wait until the epoch setup phase to force network into EFM
 	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, 10*time.Second, 500*time.Millisecond)
-	// pausing consensus node will force the network into EFM
+	// pausing collection node will force the network into EFM
 	ln := s.GetContainersByRole(flow.RoleCollection)[0]
 	_ = ln.Pause()
 	s.AwaitFinalizedView(s.Ctx, 32, 2*time.Minute, 500*time.Millisecond)

From 9cff7d5795e1c342994c27c5145d9b044bb4abbb Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 22:07:30 -0400
Subject: [PATCH 28/60] Update
 integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go

Co-authored-by: Yurii Oleksyshyn <yuraolex@gmail.com>
---
 .../tests/epochs/recover_epoch/recover_epoch_efm_test.go      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index da71fdd3246..993c0f89daa 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -51,8 +51,8 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	s.AwaitFinalizedView(s.Ctx, epoch1FinalView+1, 2*time.Minute, 500*time.Millisecond)
 	s.TimedLogf("observed finalized view %d -> pausing container", epoch1FinalView+1)
 
-	//assert transition to second epoch did not happen
-	//if counter is still 0, epoch emergency fallback was triggered as expected
+	// assert transition to second epoch did not happen
+	// if counter is still 0, epoch emergency fallback was triggered as expected
 	s.AssertInEpoch(s.Ctx, 0)
 
 	// 2. Generate transaction arguments for epoch recover transaction.

From fc2355dbd58a7ade1ae91cb23b93647be5056d93 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 22:31:36 -0400
Subject: [PATCH 29/60] remove dkg end view magic number

---
 integration/tests/epochs/base_suite.go                       | 5 +++++
 .../tests/epochs/recover_epoch/recover_epoch_efm_test.go     | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index 47af0e8dc88..e35426f91c1 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -202,3 +202,8 @@ func (s *BaseSuite) GetLatestProtocolSnapshot(ctx context.Context) *inmem.Snapsh
 	require.NoError(s.T(), err)
 	return snapshot
 }
+
+// GetDKGEndView returns the end view of the dkg.
+func (s *BaseSuite) GetDKGEndView() uint64 {
+	return s.StakingAuctionLen + (s.DKGPhaseLen * 3)
+}
diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 993c0f89daa..30745530136 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -37,7 +37,7 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	// pausing collection node will force the network into EFM
 	ln := s.GetContainersByRole(flow.RoleCollection)[0]
 	_ = ln.Pause()
-	s.AwaitFinalizedView(s.Ctx, 32, 2*time.Minute, 500*time.Millisecond)
+	s.AwaitFinalizedView(s.Ctx, s.GetDKGEndView(), 2*time.Minute, 500*time.Millisecond)
 	// start the paused collection node now that we are in EFM
 	require.NoError(s.T(), ln.Start())
 

From c549f36509713056dd44abed59b7e16471d84a13 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 22:32:30 -0400
Subject: [PATCH 30/60] outdated comment

---
 .../tests/epochs/recover_epoch/recover_epoch_efm_test.go        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 30745530136..36b7c22a203 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -41,7 +41,7 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	// start the paused collection node now that we are in EFM
 	require.NoError(s.T(), ln.Start())
 
-	// get the latest snapshot and start new container with it
+	// get final view form the latest snapshot
 	epoch1FinalView, err := s.Net.BootstrapSnapshot.Epochs().Current().FinalView()
 	require.NoError(s.T(), err)
 

From 9e062d18786e92a73c89da141cb6e3814bb8effa Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 23:15:58 -0400
Subject: [PATCH 31/60] use aggregatedSignature

---
 cmd/util/cmd/common/clusters.go           | 4 ++--
 go.mod                                    | 4 ++--
 go.sum                                    | 4 ++++
 insecure/go.mod                           | 4 ++--
 insecure/go.sum                           | 4 ++++
 integration/go.mod                        | 4 ++--
 integration/go.sum                        | 4 ++++
 model/convert/service_event.go            | 2 +-
 utils/unittest/service_events_fixtures.go | 2 +-
 9 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/cmd/util/cmd/common/clusters.go b/cmd/util/cmd/common/clusters.go
index 922464149fd..d728c30d795 100644
--- a/cmd/util/cmd/common/clusters.go
+++ b/cmd/util/cmd/common/clusters.go
@@ -182,7 +182,7 @@ func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.Cluste
 		}
 
 		qcVoteData[i] = cadence.NewStruct([]cadence.Value{
-			// voteSignatures
+			// aggregatedSignature
 			cadence.String(fmt.Sprintf("%#x", qc.SigData)),
 			// voterIDs
 			cadence.NewArray(cdcVoterIds).WithType(cadence.NewVariableSizedArrayType(cadence.StringType)),
@@ -205,7 +205,7 @@ func newFlowClusterQCVoteDataStructType(clusterQcAddress string) *cadence.Struct
 		QualifiedIdentifier: "FlowClusterQC.ClusterQCVoteData",
 		Fields: []cadence.Field{
 			{
-				Identifier: "voteSignatures",
+				Identifier: "aggregatedSignature",
 				Type:       cadence.StringType,
 			},
 			{
diff --git a/go.mod b/go.mod
index 51e42d7f64f..b2c26cc8ee2 100644
--- a/go.mod
+++ b/go.mod
@@ -50,8 +50,8 @@ require (
 	github.com/onflow/cadence v1.0.0-preview.34
 	github.com/onflow/crypto v0.25.1
 	github.com/onflow/flow v0.3.4
-	github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4
-	github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4
+	github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920
+	github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920
 	github.com/onflow/flow-go-sdk v1.0.0-preview.36
 	github.com/onflow/flow/protobuf/go/flow v0.4.4
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
diff --git a/go.sum b/go.sum
index ac8beff7acf..2fb20f8db14 100644
--- a/go.sum
+++ b/go.sum
@@ -2180,8 +2180,12 @@ github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE=
 github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 h1:zLDxmCxa/jx86uwkkaFXpztjgrofIHk74Bj6d/raeXA=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
+github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920 h1:dvpU5WG++j9s3XXbFnMv/eVdupEHh1Xb0NyLZYDEz7A=
+github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 h1:W5dPPXV8cNG0nYxKe+g/8U2hkhZiIR6W8kFKqsJ5o9Y=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
+github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920 h1:SBUd8cq9OThExucjUO3EFMuzA5Sp8NLGUjch2phxKfg=
+github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A=
 github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs=
diff --git a/insecure/go.mod b/insecure/go.mod
index 3273cb7ee2a..f61c3719c47 100644
--- a/insecure/go.mod
+++ b/insecure/go.mod
@@ -200,8 +200,8 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/onflow/atree v0.7.0-rc.2 // indirect
 	github.com/onflow/cadence v1.0.0-preview.34 // indirect
-	github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 // indirect
-	github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 // indirect
+	github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920 // indirect
+	github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920 // indirect
 	github.com/onflow/flow-ft/lib/go/contracts v1.0.0 // indirect
 	github.com/onflow/flow-ft/lib/go/templates v1.0.0 // indirect
 	github.com/onflow/flow-go-sdk v1.0.0-preview.36 // indirect
diff --git a/insecure/go.sum b/insecure/go.sum
index 7db187ea31f..72be2698bf7 100644
--- a/insecure/go.sum
+++ b/insecure/go.sum
@@ -2167,8 +2167,12 @@ github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A=
 github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 h1:zLDxmCxa/jx86uwkkaFXpztjgrofIHk74Bj6d/raeXA=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
+github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920 h1:dvpU5WG++j9s3XXbFnMv/eVdupEHh1Xb0NyLZYDEz7A=
+github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 h1:W5dPPXV8cNG0nYxKe+g/8U2hkhZiIR6W8kFKqsJ5o9Y=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
+github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920 h1:SBUd8cq9OThExucjUO3EFMuzA5Sp8NLGUjch2phxKfg=
+github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A=
 github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs=
diff --git a/integration/go.mod b/integration/go.mod
index d7ae0585a06..37939172b51 100644
--- a/integration/go.mod
+++ b/integration/go.mod
@@ -21,8 +21,8 @@ require (
 	github.com/libp2p/go-libp2p v0.32.2
 	github.com/onflow/cadence v1.0.0-preview.34
 	github.com/onflow/crypto v0.25.1
-	github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4
-	github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4
+	github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920
+	github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920
 	github.com/onflow/flow-emulator v1.0.0-preview.24
 	github.com/onflow/flow-go v0.35.5-0.20240517202625-55f862b45dfd
 	github.com/onflow/flow-go-sdk v1.0.0-preview.36
diff --git a/integration/go.sum b/integration/go.sum
index 4df7f6ca6f4..4632a19b4d2 100644
--- a/integration/go.sum
+++ b/integration/go.sum
@@ -2157,8 +2157,12 @@ github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A=
 github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 h1:zLDxmCxa/jx86uwkkaFXpztjgrofIHk74Bj6d/raeXA=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
+github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920 h1:dvpU5WG++j9s3XXbFnMv/eVdupEHh1Xb0NyLZYDEz7A=
+github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 h1:W5dPPXV8cNG0nYxKe+g/8U2hkhZiIR6W8kFKqsJ5o9Y=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
+github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920 h1:SBUd8cq9OThExucjUO3EFMuzA5Sp8NLGUjch2phxKfg=
+github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-emulator v1.0.0-preview.24 h1:SonXMBeYxVwNn94M+OUmKIYScIMQG22wugh9n/tHY5k=
 github.com/onflow/flow-emulator v1.0.0-preview.24/go.mod h1:QprPouTWO3iv9VF/y4Ksltv2XIbzNMzjjr5zzq51i7Q=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0=
diff --git a/model/convert/service_event.go b/model/convert/service_event.go
index 2f95bbbfe41..e5643957c18 100644
--- a/model/convert/service_event.go
+++ b/model/convert/service_event.go
@@ -755,7 +755,7 @@ func convertClusterQCVoteData(cdcClusterQCVoteData []cadence.Value) ([]flow.Clus
 			voterIDs = append(voterIDs, voterID)
 		}
 
-		cdcAggSignature, err := getField[cadence.String](fields, "voteSignatures")
+		cdcAggSignature, err := getField[cadence.String](fields, "aggregatedSignature")
 		if err != nil {
 			return nil, fmt.Errorf("failed to decode clusterQCVoteData struct: %w", err)
 		}
diff --git a/utils/unittest/service_events_fixtures.go b/utils/unittest/service_events_fixtures.go
index f5936e0b2b2..84e4873dddd 100644
--- a/utils/unittest/service_events_fixtures.go
+++ b/utils/unittest/service_events_fixtures.go
@@ -1240,7 +1240,7 @@ func newFlowClusterQCClusterQCVoteDataStructType() *cadence.StructType {
 		QualifiedIdentifier: "FlowClusterQC.ClusterQCVoteData",
 		Fields: []cadence.Field{
 			{
-				Identifier: "voteSignatures",
+				Identifier: "aggregatedSignature",
 				Type:       cadence.NewVariableSizedArrayType(cadence.StringType),
 			},
 			{

From 582ca2468de44b6211f050f526835e57e94525af Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 23:22:04 -0400
Subject: [PATCH 32/60] make sure eventType not empty

---
 integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 36b7c22a203..49f42604d22 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -93,6 +93,7 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 			break
 		}
 	}
+	require.NotEmpty(s.T(), eventType, "expected FlowEpoch.EpochRecover event type")
 	events, err := s.Client.GetEventsForBlockIDs(s.Ctx, eventType, []sdk.Identifier{result.BlockID})
 	require.NoError(s.T(), err)
 	require.Equal(s.T(), events[0].Events[0].Type, eventType)

From 777267fff4dedeebd234b180ddba59327e95c206 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 23:28:22 -0400
Subject: [PATCH 33/60] get recover epoch config from test suite

---
 integration/tests/epochs/base_suite.go                      | 1 +
 .../tests/epochs/recover_epoch/recover_epoch_efm_test.go    | 6 +++---
 integration/tests/epochs/recover_epoch/suite.go             | 1 +
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index e35426f91c1..786f1325374 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -42,6 +42,7 @@ type BaseSuite struct {
 	DKGPhaseLen                uint64
 	EpochLen                   uint64
 	EpochCommitSafetyThreshold uint64
+	NumOfCollectionClusters    uint64
 	// Whether approvals are required for sealing (we only enable for VN tests because
 	// requiring approvals requires a longer DKG period to avoid flakiness)
 	RequiredSealApprovals uint // defaults to 0 (no approvals required)
diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 49f42604d22..8d6e346ca45 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -57,9 +57,9 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 
 	// 2. Generate transaction arguments for epoch recover transaction.
 	// generate epoch recover transaction args
-	collectionClusters := uint64(1)
-	numViewsInRecoveryEpoch := uint64(80)
-	numViewsInStakingAuction := uint64(2)
+	collectionClusters := s.NumOfCollectionClusters
+	numViewsInRecoveryEpoch := s.EpochLen
+	numViewsInStakingAuction := s.StakingAuctionLen
 	epochCounter := uint64(1)
 	targetDuration := uint64(3000)
 	targetEndTime := uint64(4000)
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 23be68dd90b..054de0c4bb7 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -27,6 +27,7 @@ func (s *Suite) SetupTest() {
 	s.DKGPhaseLen = 10
 	s.EpochLen = 80
 	s.EpochCommitSafetyThreshold = 20
+	s.NumOfCollectionClusters = 1
 
 	// run the generic setup, which starts up the network
 	s.BaseSuite.SetupTest()

From 0d4138923af8792f8260aab72a6f64ebe54181da Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 23:31:42 -0400
Subject: [PATCH 34/60] add note that cruise ctl disabled in integration tests

---
 .../tests/epochs/recover_epoch/recover_epoch_efm_test.go | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 8d6e346ca45..6d113a94b16 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -61,16 +61,17 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	numViewsInRecoveryEpoch := s.EpochLen
 	numViewsInStakingAuction := s.StakingAuctionLen
 	epochCounter := uint64(1)
-	targetDuration := uint64(3000)
-	targetEndTime := uint64(4000)
+
 	out := fmt.Sprintf("%s/recover-epoch-tx-args.json", s.Net.BootstrapDir)
 	s.executeEFMRecoverTXArgsCMD(
 		collectionClusters,
 		numViewsInRecoveryEpoch,
 		numViewsInStakingAuction,
 		epochCounter,
-		targetDuration,
-		targetEndTime,
+		// cruise control is disabled for integration tests
+		// targetDuration and targetEndTime will be ignored
+		3000,
+		4000,
 		out,
 	)
 	b, err := os.ReadFile(out)

From b64a5ca8512cc818af1fbfda224912b6d8bc84ad Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 23:33:08 -0400
Subject: [PATCH 35/60] add short dkg len comment

---
 integration/tests/epochs/recover_epoch/suite.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 054de0c4bb7..b534b1fcd8b 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -24,6 +24,7 @@ type Suite struct {
 func (s *Suite) SetupTest() {
 	// use a shorter staking auction because we don't have staking operations in this case
 	s.StakingAuctionLen = 2
+	// to manually trigger EFM we assign very short dkg phase len ensuring the dkg will fail
 	s.DKGPhaseLen = 10
 	s.EpochLen = 80
 	s.EpochCommitSafetyThreshold = 20

From a46ae41a205ae1d511b720b9bf67130eafd0e01b Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 23:46:50 -0400
Subject: [PATCH 36/60] add godoc comment for GetContainersByRole

---
 integration/tests/epochs/base_suite.go          | 1 +
 integration/tests/epochs/recover_epoch/suite.go | 5 +++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index 786f1325374..c80fff69b15 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -159,6 +159,7 @@ func (s *BaseSuite) AwaitEpochPhase(ctx context.Context, expectedEpoch uint64, e
 }
 
 // GetContainersByRole returns all containers from the network for the specified role, making sure the containers are not ghost nodes.
+// Since go maps have random iteration order the list of containers returned will be in random order.
 func (s *BaseSuite) GetContainersByRole(role flow.Role) []*testnet.Container {
 	nodes := s.Net.ContainersByRole(role, false)
 	require.True(s.T(), len(nodes) > 0)
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index b534b1fcd8b..58cde04f622 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -36,8 +36,9 @@ func (s *Suite) SetupTest() {
 
 // getNodeInfoDirs returns the internal node private info dir and the node config dir from a container with the specified role.
 func (s *Suite) getNodeInfoDirs(role flow.Role) (string, string) {
-	internalNodePrivInfoDir := fmt.Sprintf("%s/%s", s.GetContainersByRole(role)[0].BootstrapPath(), bootstrap.DirPrivateRoot)
-	nodeConfigJson := fmt.Sprintf("%s/%s", s.GetContainersByRole(role)[0].BootstrapPath(), bootstrap.PathNodeInfosPub)
+	bootstrapPath := s.GetContainersByRole(role)[0].BootstrapPath()
+	internalNodePrivInfoDir := fmt.Sprintf("%s/%s", bootstrapPath, bootstrap.DirPrivateRoot)
+	nodeConfigJson := fmt.Sprintf("%s/%s", bootstrapPath, bootstrap.PathNodeInfosPub)
 	return internalNodePrivInfoDir, nodeConfigJson
 }
 

From 78fb541350443ecf695db8b3536eefceec076fbd Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 9 Jul 2024 23:47:54 -0400
Subject: [PATCH 37/60] uodate executeEFMRecoverTXArgsCMD godoc

---
 integration/tests/epochs/recover_epoch/suite.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 58cde04f622..3c6a621c2dd 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -45,15 +45,12 @@ func (s *Suite) getNodeInfoDirs(role flow.Role) (string, string) {
 // executeEFMRecoverTXArgsCMD executes the efm-recover-tx-args CLI command to generate EpochRecover transaction arguments.
 // Args:
 //
-//	role: the container role that will be used to read internal node private info and the node config json.
-//	snapshot: the protocol state snapshot.
 //	collectionClusters: the number of collector clusters.
 //	numViewsInEpoch: the number of views in the recovery epoch.
 //	numViewsInStakingAuction: the number of views in the staking auction of the recovery epoch.
 //	epochCounter: the epoch counter.
 //	targetDuration: the target duration for the recover epoch.
 //	targetEndTime: the target end time for the recover epoch.
-//	randomSource: the random source of the recover epoch.
 //	out: the tx args output file full path.
 func (s *Suite) executeEFMRecoverTXArgsCMD(collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter, targetDuration, targetEndTime uint64, out string) {
 	// read internal node info from one of the consensus nodes

From 3b6d7078caa1f0e14b4e3ea13c372366496c9fdb Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Wed, 10 Jul 2024 00:19:29 -0400
Subject: [PATCH 38/60] replace use of rootcmd.execute and move logic to run
 package

---
 cmd/bootstrap/cmd/rootblock.go                |   2 +-
 cmd/bootstrap/run/epochs.go                   | 200 ++++++++++++++++++
 cmd/util/cmd/common/clusters.go               |  52 -----
 cmd/util/cmd/epochs/cmd/recover.go            |  23 +-
 integration/tests/epochs/base_suite.go        |  20 +-
 .../recover_epoch/recover_epoch_efm_test.go   |  10 +-
 .../tests/epochs/recover_epoch/suite.go       |  54 ++---
 7 files changed, 254 insertions(+), 107 deletions(-)
 create mode 100644 cmd/bootstrap/run/epochs.go

diff --git a/cmd/bootstrap/cmd/rootblock.go b/cmd/bootstrap/cmd/rootblock.go
index 194e625000c..4e3e5ed813d 100644
--- a/cmd/bootstrap/cmd/rootblock.go
+++ b/cmd/bootstrap/cmd/rootblock.go
@@ -187,7 +187,7 @@ func rootBlock(cmd *cobra.Command, args []string) {
 	log.Info().Msg("")
 
 	log.Info().Msg("constructing root QCs for collection node clusters")
-	clusterQCs := common.ConstructRootQCsForClusters(log, clusters, internalNodes, clusterBlocks)
+	clusterQCs := run.ConstructRootQCsForClusters(log, clusters, internalNodes, clusterBlocks)
 	log.Info().Msg("")
 
 	log.Info().Msg("constructing root header")
diff --git a/cmd/bootstrap/run/epochs.go b/cmd/bootstrap/run/epochs.go
new file mode 100644
index 00000000000..d405a5b7296
--- /dev/null
+++ b/cmd/bootstrap/run/epochs.go
@@ -0,0 +1,200 @@
+package run
+
+import (
+	"fmt"
+
+	"github.com/rs/zerolog"
+
+	"github.com/onflow/cadence"
+	"github.com/onflow/flow-go/cmd/util/cmd/common"
+	"github.com/onflow/flow-go/fvm/systemcontracts"
+	"github.com/onflow/flow-go/model/bootstrap"
+	model "github.com/onflow/flow-go/model/bootstrap"
+	"github.com/onflow/flow-go/model/cluster"
+	"github.com/onflow/flow-go/model/flow"
+	"github.com/onflow/flow-go/model/flow/filter"
+	"github.com/onflow/flow-go/state/protocol/inmem"
+)
+
+// GenerateRecoverEpochTxArgs generates the required transaction arguments for the `recoverEpoch` transaction.
+func GenerateRecoverEpochTxArgs(log zerolog.Logger,
+	internalNodePrivInfoDir string,
+	nodeConfigJson string,
+	collectionClusters int,
+	epochCounter uint64,
+	rootChainID flow.ChainID,
+	numViewsInStakingAuction uint64,
+	numViewsInEpoch uint64,
+	targetDuration uint64,
+	targetEndTime uint64,
+	initNewEpoch bool,
+	snapshot *inmem.Snapshot) ([]cadence.Value, error) {
+	epoch := snapshot.Epochs().Current()
+
+	currentEpochIdentities, err := snapshot.Identities(filter.IsValidProtocolParticipant)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get  valid protocol participants from snapshot: %w", err)
+	}
+
+	// separate collector nodes by internal and partner nodes
+	collectors := currentEpochIdentities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))
+	internalCollectors := make(flow.IdentityList, 0)
+	partnerCollectors := make(flow.IdentityList, 0)
+
+	log.Info().Msg("collecting internal node network and staking keys")
+	internalNodes, err := common.ReadFullInternalNodeInfos(log, internalNodePrivInfoDir, nodeConfigJson)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read full internal node infos: %w", err)
+	}
+
+	internalNodesMap := make(map[flow.Identifier]struct{})
+	for _, node := range internalNodes {
+		if !currentEpochIdentities.Exists(node.Identity()) {
+			return nil, fmt.Errorf("node ID found in internal node infos missing from protocol snapshot identities %s: %w", node.NodeID, err)
+		}
+		internalNodesMap[node.NodeID] = struct{}{}
+	}
+	log.Info().Msg("")
+
+	for _, collector := range collectors {
+		if _, ok := internalNodesMap[collector.NodeID]; ok {
+			internalCollectors = append(internalCollectors, collector)
+		} else {
+			partnerCollectors = append(partnerCollectors, collector)
+		}
+	}
+
+	currentEpochDKG, err := epoch.DKG()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get DKG for current epoch: %w", err)
+	}
+
+	log.Info().Msg("computing collection node clusters")
+
+	assignments, clusters, err := common.ConstructClusterAssignment(log, partnerCollectors, internalCollectors, collectionClusters)
+	if err != nil {
+		log.Fatal().Err(err).Msg("unable to generate cluster assignment")
+	}
+	log.Info().Msg("")
+
+	log.Info().Msg("constructing root blocks for collection node clusters")
+	clusterBlocks := GenerateRootClusterBlocks(epochCounter, clusters)
+	log.Info().Msg("")
+
+	log.Info().Msg("constructing root QCs for collection node clusters")
+	clusterQCs := ConstructRootQCsForClusters(log, clusters, internalNodes, clusterBlocks)
+	log.Info().Msg("")
+
+	dkgPubKeys := make([]cadence.Value, 0)
+	nodeIds := make([]cadence.Value, 0)
+
+	// NOTE: The RecoveryEpoch will re-use the last successful DKG output. This means that the consensus
+	// committee in the RecoveryEpoch must be identical to the committee which participated in that DKG.
+	dkgGroupKeyCdc, cdcErr := cadence.NewString(currentEpochDKG.GroupKey().String())
+	if cdcErr != nil {
+		log.Fatal().Err(cdcErr).Msg("failed to get dkg group key cadence string")
+	}
+	dkgPubKeys = append(dkgPubKeys, dkgGroupKeyCdc)
+	for _, id := range currentEpochIdentities {
+		if id.GetRole() == flow.RoleConsensus {
+			dkgPubKey, keyShareErr := currentEpochDKG.KeyShare(id.GetNodeID())
+			if keyShareErr != nil {
+				log.Fatal().Err(keyShareErr).Msg(fmt.Sprintf("failed to get dkg pub key share for node: %s", id.GetNodeID()))
+			}
+			dkgPubKeyCdc, cdcErr := cadence.NewString(dkgPubKey.String())
+			if cdcErr != nil {
+				log.Fatal().Err(cdcErr).Msg(fmt.Sprintf("failed to get dkg pub key cadence string for node: %s", id.GetNodeID()))
+			}
+			dkgPubKeys = append(dkgPubKeys, dkgPubKeyCdc)
+		}
+		nodeIdCdc, err := cadence.NewString(id.GetNodeID().String())
+		if err != nil {
+			log.Fatal().Err(err).Msg(fmt.Sprintf("failed to convert node ID to cadence string: %s", id.GetNodeID()))
+		}
+		nodeIds = append(nodeIds, nodeIdCdc)
+	}
+
+	clusterQCAddress := systemcontracts.SystemContractsForChain(flow.ChainID(rootChainID)).ClusterQC.Address.String()
+	qcVoteData, err := common.ConvertClusterQcsCdc(clusterQCs, clusters, clusterQCAddress)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to convert cluster qcs to cadence type")
+	}
+
+	currEpochFinalView, err := epoch.FinalView()
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to get final view of current epoch")
+	}
+
+	args := []cadence.Value{
+		// epoch start view
+		cadence.NewUInt64(currEpochFinalView + 1),
+		// staking phase end view
+		cadence.NewUInt64(currEpochFinalView + numViewsInStakingAuction),
+		// epoch end view
+		cadence.NewUInt64(currEpochFinalView + numViewsInEpoch),
+		// target duration
+		cadence.NewUInt64(targetDuration),
+		// target end time
+		cadence.NewUInt64(targetEndTime),
+		// clusters,
+		common.ConvertClusterAssignmentsCdc(assignments),
+		// qcVoteData
+		cadence.NewArray(qcVoteData),
+		// dkg pub keys
+		cadence.NewArray(dkgPubKeys),
+		// node ids
+		cadence.NewArray(nodeIds),
+		// recover the network by initializing a new recover epoch which will increment the smart contract epoch counter
+		// or overwrite the epoch metadata for the current epoch
+		cadence.NewBool(initNewEpoch),
+	}
+
+	return args, nil
+}
+
+// ConstructRootQCsForClusters constructs a root QC for each cluster in the list.
+// Args:
+// - log: the logger instance.
+// - clusterList: list of clusters
+// - nodeInfos: list of NodeInfos (must contain all internal nodes)
+// - clusterBlocks: list of root blocks for each cluster
+// Returns:
+// - flow.AssignmentList: the generated assignment list.
+// - flow.ClusterList: the generate collection cluster list.
+func ConstructRootQCsForClusters(log zerolog.Logger, clusterList flow.ClusterList, nodeInfos []bootstrap.NodeInfo, clusterBlocks []*cluster.Block) []*flow.QuorumCertificate {
+
+	if len(clusterBlocks) != len(clusterList) {
+		log.Fatal().Int("len(clusterBlocks)", len(clusterBlocks)).Int("len(clusterList)", len(clusterList)).
+			Msg("number of clusters needs to equal number of cluster blocks")
+	}
+
+	qcs := make([]*flow.QuorumCertificate, len(clusterBlocks))
+	for i, cluster := range clusterList {
+		signers := filterClusterSigners(cluster, nodeInfos)
+
+		qc, err := GenerateClusterRootQC(signers, cluster, clusterBlocks[i])
+		if err != nil {
+			log.Fatal().Err(err).Int("cluster index", i).Msg("generating collector cluster root QC failed")
+		}
+		qcs[i] = qc
+	}
+
+	return qcs
+}
+
+// Filters a list of nodes to include only nodes that will sign the QC for the
+// given cluster. The resulting list of nodes is only nodes that are in the
+// given cluster AND are not partner nodes (ie. we have the private keys).
+func filterClusterSigners(cluster flow.IdentitySkeletonList, nodeInfos []model.NodeInfo) []model.NodeInfo {
+	var filtered []model.NodeInfo
+	for _, node := range nodeInfos {
+		_, isInCluster := cluster.ByNodeID(node.NodeID)
+		isNotPartner := node.Type() == model.NodeInfoTypePrivate
+
+		if isInCluster && isNotPartner {
+			filtered = append(filtered, node)
+		}
+	}
+
+	return filtered
+}
diff --git a/cmd/util/cmd/common/clusters.go b/cmd/util/cmd/common/clusters.go
index d728c30d795..65da023ba1b 100644
--- a/cmd/util/cmd/common/clusters.go
+++ b/cmd/util/cmd/common/clusters.go
@@ -9,10 +9,6 @@ import (
 	"github.com/onflow/cadence"
 	cdcCommon "github.com/onflow/cadence/runtime/common"
 
-	"github.com/onflow/flow-go/cmd/bootstrap/run"
-	"github.com/onflow/flow-go/model/bootstrap"
-	model "github.com/onflow/flow-go/model/bootstrap"
-	"github.com/onflow/flow-go/model/cluster"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/model/flow/assignment"
 	"github.com/onflow/flow-go/model/flow/factory"
@@ -115,36 +111,6 @@ func ConstructClusterAssignment(log zerolog.Logger, partnerNodes, internalNodes
 	return assignments, clusters, nil
 }
 
-// ConstructRootQCsForClusters constructs a root QC for each cluster in the list.
-// Args:
-// - log: the logger instance.
-// - clusterList: list of clusters
-// - nodeInfos: list of NodeInfos (must contain all internal nodes)
-// - clusterBlocks: list of root blocks for each cluster
-// Returns:
-// - flow.AssignmentList: the generated assignment list.
-// - flow.ClusterList: the generate collection cluster list.
-func ConstructRootQCsForClusters(log zerolog.Logger, clusterList flow.ClusterList, nodeInfos []bootstrap.NodeInfo, clusterBlocks []*cluster.Block) []*flow.QuorumCertificate {
-
-	if len(clusterBlocks) != len(clusterList) {
-		log.Fatal().Int("len(clusterBlocks)", len(clusterBlocks)).Int("len(clusterList)", len(clusterList)).
-			Msg("number of clusters needs to equal number of cluster blocks")
-	}
-
-	qcs := make([]*flow.QuorumCertificate, len(clusterBlocks))
-	for i, cluster := range clusterList {
-		signers := filterClusterSigners(cluster, nodeInfos)
-
-		qc, err := run.GenerateClusterRootQC(signers, cluster, clusterBlocks[i])
-		if err != nil {
-			log.Fatal().Err(err).Int("cluster index", i).Msg("generating collector cluster root QC failed")
-		}
-		qcs[i] = qc
-	}
-
-	return qcs
-}
-
 // ConvertClusterAssignmentsCdc converts golang cluster assignments type to Cadence type `[[String]]`.
 func ConvertClusterAssignmentsCdc(assignments flow.AssignmentList) cadence.Array {
 	stringArrayType := cadence.NewVariableSizedArrayType(cadence.StringType)
@@ -215,21 +181,3 @@ func newFlowClusterQCVoteDataStructType(clusterQcAddress string) *cadence.Struct
 		},
 	}
 }
-
-// Filters a list of nodes to include only nodes that will sign the QC for the
-// given cluster. The resulting list of nodes is only nodes that are in the
-// given cluster AND are not partner nodes (ie. we have the private keys).
-func filterClusterSigners(cluster flow.IdentitySkeletonList, nodeInfos []model.NodeInfo) []model.NodeInfo {
-
-	var filtered []model.NodeInfo
-	for _, node := range nodeInfos {
-		_, isInCluster := cluster.ByNodeID(node.NodeID)
-		isNotPartner := node.Type() == model.NodeInfoTypePrivate
-
-		if isInCluster && isNotPartner {
-			filtered = append(filtered, node)
-		}
-	}
-
-	return filtered
-}
diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index 280e8fe6276..500592c86dc 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -148,9 +148,24 @@ func getSnapshot() *inmem.Snapshot {
 // generateRecoverEpochTxArgs generates recover epoch transaction arguments from a root protocol state snapshot and writes it to a JSON file
 func generateRecoverEpochTxArgs(getSnapshot func() *inmem.Snapshot) func(cmd *cobra.Command, args []string) {
 	return func(cmd *cobra.Command, args []string) {
-		// extract arguments from recover epoch tx from snapshot
-		txArgs := extractRecoverEpochArgs(getSnapshot())
-
+		// generate transaction arguments
+		txArgs, err := run.GenerateRecoverEpochTxArgs(
+			log,
+			flagInternalNodePrivInfoDir,
+			flagNodeConfigJson,
+			flagCollectionClusters,
+			flagEpochCounter,
+			flow.ChainID(flagRootChainID),
+			flagNumViewsInStakingAuction,
+			flagNumViewsInEpoch,
+			flagTargetDuration,
+			flagTargetEndTime,
+			flagInitNewEpoch,
+			getSnapshot(),
+		)
+		if err != nil {
+			log.Fatal().Err(err).Msg("failed to generate recover epoch transaction arguments")
+		}
 		// encode to JSON
 		encodedTxArgs, err := epochcmdutil.EncodeArgs(txArgs)
 		if err != nil {
@@ -229,7 +244,7 @@ func extractRecoverEpochArgs(snapshot *inmem.Snapshot) []cadence.Value {
 	log.Info().Msg("")
 
 	log.Info().Msg("constructing root QCs for collection node clusters")
-	clusterQCs := common.ConstructRootQCsForClusters(log, clusters, internalNodes, clusterBlocks)
+	clusterQCs := run.ConstructRootQCsForClusters(log, clusters, internalNodes, clusterBlocks)
 	log.Info().Msg("")
 
 	dkgPubKeys := make([]cadence.Value, 0)
diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index c80fff69b15..a7d304689b3 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -30,7 +30,7 @@ type BaseSuite struct {
 	suite.Suite
 	lib.TestnetStateTracker
 	cancel  context.CancelFunc
-	log     zerolog.Logger
+	Log     zerolog.Logger
 	Net     *testnet.FlowNetwork
 	ghostID flow.Identifier
 
@@ -42,7 +42,7 @@ type BaseSuite struct {
 	DKGPhaseLen                uint64
 	EpochLen                   uint64
 	EpochCommitSafetyThreshold uint64
-	NumOfCollectionClusters    uint64
+	NumOfCollectionClusters    int
 	// Whether approvals are required for sealing (we only enable for VN tests because
 	// requiring approvals requires a longer DKG period to avoid flakiness)
 	RequiredSealApprovals uint // defaults to 0 (no approvals required)
@@ -61,10 +61,10 @@ func (s *BaseSuite) SetupTest() {
 	require.Greater(s.T(), s.EpochLen, minEpochLength+s.EpochCommitSafetyThreshold, "epoch too short")
 
 	s.Ctx, s.cancel = context.WithCancel(context.Background())
-	s.log = unittest.LoggerForTest(s.Suite.T(), zerolog.InfoLevel)
-	s.log.Info().Msg("================> SetupTest")
+	s.Log = unittest.LoggerForTest(s.Suite.T(), zerolog.InfoLevel)
+	s.Log.Info().Msg("================> SetupTest")
 	defer func() {
-		s.log.Info().Msg("================> Finish SetupTest")
+		s.Log.Info().Msg("================> Finish SetupTest")
 	}()
 
 	collectionConfigs := []func(*testnet.NodeConfig){
@@ -115,15 +115,15 @@ func (s *BaseSuite) SetupTest() {
 
 	s.Client = client
 
-	// log network info periodically to aid in debugging future flaky tests
-	go lib.LogStatusPeriodically(s.T(), s.Ctx, s.log, s.Client, 5*time.Second)
+	// Log network info periodically to aid in debugging future flaky tests
+	go lib.LogStatusPeriodically(s.T(), s.Ctx, s.Log, s.Client, 5*time.Second)
 }
 
 func (s *BaseSuite) TearDownTest() {
-	s.log.Info().Msg("================> Start TearDownTest")
+	s.Log.Info().Msg("================> Start TearDownTest")
 	s.Net.Remove()
 	s.cancel()
-	s.log.Info().Msg("================> Finish TearDownTest")
+	s.Log.Info().Msg("================> Finish TearDownTest")
 }
 
 func (s *BaseSuite) Ghost() *client.GhostClient {
@@ -135,7 +135,7 @@ func (s *BaseSuite) Ghost() *client.GhostClient {
 // TimedLogf logs the message using t.Log and the suite logger, but prefixes the current time.
 // This enables viewing logs inline with Docker logs as well as other test logs.
 func (s *BaseSuite) TimedLogf(msg string, args ...interface{}) {
-	s.log.Info().Msgf(msg, args...)
+	s.Log.Info().Msgf(msg, args...)
 	args = append([]interface{}{time.Now().String()}, args...)
 	s.T().Logf("%s - "+msg, args...)
 }
diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 6d113a94b16..72b0c5ccc2d 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -1,8 +1,6 @@
 package recover_epoch
 
 import (
-	"fmt"
-	"os"
 	"strings"
 	"testing"
 	"time"
@@ -62,8 +60,7 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	numViewsInStakingAuction := s.StakingAuctionLen
 	epochCounter := uint64(1)
 
-	out := fmt.Sprintf("%s/recover-epoch-tx-args.json", s.Net.BootstrapDir)
-	s.executeEFMRecoverTXArgsCMD(
+	txArgs := s.executeEFMRecoverTXArgsCMD(
 		collectionClusters,
 		numViewsInRecoveryEpoch,
 		numViewsInStakingAuction,
@@ -72,15 +69,10 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 		// targetDuration and targetEndTime will be ignored
 		3000,
 		4000,
-		out,
 	)
-	b, err := os.ReadFile(out)
-	require.NoError(s.T(), err)
 
 	// 3. Submit recover epoch transaction to the network.
 	// submit the recover epoch transaction
-	txArgs, err := utils.ParseJSON(b)
-	require.NoError(s.T(), err)
 	env := utils.LocalnetEnv()
 	result := s.recoverEpoch(env, txArgs)
 	require.NoError(s.T(), result.Error)
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 3c6a621c2dd..c836bf1de3a 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -2,13 +2,10 @@ package recover_epoch
 
 import (
 	"fmt"
-	"os"
-
 	"github.com/onflow/cadence"
 	"github.com/onflow/flow-core-contracts/lib/go/templates"
 	sdk "github.com/onflow/flow-go-sdk"
-	"github.com/onflow/flow-go/cmd/util/cmd/epochs/cmd"
-	"github.com/onflow/flow-go/integration/testnet"
+	"github.com/onflow/flow-go/cmd/bootstrap/run"
 	"github.com/onflow/flow-go/integration/tests/epochs"
 	"github.com/onflow/flow-go/integration/utils"
 	"github.com/onflow/flow-go/model/bootstrap"
@@ -51,35 +48,30 @@ func (s *Suite) getNodeInfoDirs(role flow.Role) (string, string) {
 //	epochCounter: the epoch counter.
 //	targetDuration: the target duration for the recover epoch.
 //	targetEndTime: the target end time for the recover epoch.
-//	out: the tx args output file full path.
-func (s *Suite) executeEFMRecoverTXArgsCMD(collectionClusters, numViewsInEpoch, numViewsInStakingAuction, epochCounter, targetDuration, targetEndTime uint64, out string) {
+//
+// Returns:
+//
+//	[]cadence.Value: the transaction arguments.
+func (s *Suite) executeEFMRecoverTXArgsCMD(collectionClusters int, numViewsInEpoch, numViewsInStakingAuction, epochCounter, targetDuration, targetEndTime uint64) []cadence.Value {
 	// read internal node info from one of the consensus nodes
 	internalNodePrivInfoDir, nodeConfigJson := s.getNodeInfoDirs(flow.RoleConsensus)
-	an1 := s.GetContainersByRole(flow.RoleAccess)[0]
-	anAddress := an1.Addr(testnet.GRPCPort)
-	// set command line arguments
-	os.Args = []string{
-		"epochs", "efm-recover-tx-args",
-		"--insecure=true",
-		fmt.Sprintf("--root-chain-id=%s", flow.Localnet),
-		fmt.Sprintf("--out=%s", out),
-		fmt.Sprintf("--access-address=%s", anAddress),
-		fmt.Sprintf("--collection-clusters=%d", collectionClusters),
-		fmt.Sprintf("--config=%s", nodeConfigJson),
-		fmt.Sprintf("--internal-priv-dir=%s", internalNodePrivInfoDir),
-		fmt.Sprintf("--epoch-length=%d", numViewsInEpoch),
-		fmt.Sprintf("--epoch-staking-phase-length=%d", numViewsInStakingAuction),
-		fmt.Sprintf("--epoch-counter=%d", epochCounter),
-		fmt.Sprintf("--epoch-timing-duration=%d", targetDuration),
-		fmt.Sprintf("--epoch-timing-end-time=%d", targetEndTime),
-	}
-
-	// execute the root command
-	rootCmd := cmd.RootCmd
-	rootCmd.SetArgs(os.Args[1:])
-	if err := rootCmd.Execute(); err != nil {
-		s.T().Fatalf("Failed to execute epochs efm-recover-tx-args command: %v", err)
-	}
+	snapshot := s.GetLatestProtocolSnapshot(s.Ctx)
+	txArgs, err := run.GenerateRecoverEpochTxArgs(
+		s.Log,
+		internalNodePrivInfoDir,
+		nodeConfigJson,
+		collectionClusters,
+		epochCounter,
+		flow.Localnet,
+		numViewsInStakingAuction,
+		numViewsInEpoch,
+		targetDuration,
+		targetEndTime,
+		false,
+		snapshot,
+	)
+	require.NoError(s.T(), err)
+	return txArgs
 }
 
 // recoverEpoch submits the recover epoch transaction to the network.

From 4db4afc46a603a6aaffba032da0027344780c641 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Mon, 15 Jul 2024 12:33:34 -0400
Subject: [PATCH 39/60] Apply suggestions from code review

Co-authored-by: Alexander Hentschel <alex.hentschel@flowfoundation.org>
Co-authored-by: Jordan Schalm <jordan.schalm@flowfoundation.org>
---
 cmd/bootstrap/run/epochs.go                          | 12 ++++++++----
 cmd/bootstrap/utils/key_generation.go                |  3 ++-
 cmd/util/cmd/common/clusters.go                      |  2 +-
 cmd/util/cmd/epochs/cmd/recover.go                   |  4 ++--
 integration/tests/epochs/base_suite.go               |  6 +++---
 .../epochs/recover_epoch/recover_epoch_efm_test.go   |  6 +++---
 6 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/cmd/bootstrap/run/epochs.go b/cmd/bootstrap/run/epochs.go
index d405a5b7296..51a11929d49 100644
--- a/cmd/bootstrap/run/epochs.go
+++ b/cmd/bootstrap/run/epochs.go
@@ -28,13 +28,18 @@ func GenerateRecoverEpochTxArgs(log zerolog.Logger,
 	targetDuration uint64,
 	targetEndTime uint64,
 	initNewEpoch bool,
-	snapshot *inmem.Snapshot) ([]cadence.Value, error) {
+	snapshot *inmem.Snapshot,
+) ([]cadence.Value, error) {
 	epoch := snapshot.Epochs().Current()
 
 	currentEpochIdentities, err := snapshot.Identities(filter.IsValidProtocolParticipant)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get  valid protocol participants from snapshot: %w", err)
 	}
+		// We need canonical ordering here; sanity check to enforce this:  
+	if !currentEpochIdentities.Sorted(flow.Canonical[flow.Identity]) {
+		return nil, fmt.Errorf("identies from snapshot not in canonical order")
+	}
 
 	// separate collector nodes by internal and partner nodes
 	collectors := currentEpochIdentities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))
@@ -157,12 +162,11 @@ func GenerateRecoverEpochTxArgs(log zerolog.Logger,
 // - log: the logger instance.
 // - clusterList: list of clusters
 // - nodeInfos: list of NodeInfos (must contain all internal nodes)
-// - clusterBlocks: list of root blocks for each cluster
+// - clusterBlocks: list of root blocks (one for each cluster)
 // Returns:
 // - flow.AssignmentList: the generated assignment list.
 // - flow.ClusterList: the generate collection cluster list.
 func ConstructRootQCsForClusters(log zerolog.Logger, clusterList flow.ClusterList, nodeInfos []bootstrap.NodeInfo, clusterBlocks []*cluster.Block) []*flow.QuorumCertificate {
-
 	if len(clusterBlocks) != len(clusterList) {
 		log.Fatal().Int("len(clusterBlocks)", len(clusterBlocks)).Int("len(clusterList)", len(clusterList)).
 			Msg("number of clusters needs to equal number of cluster blocks")
@@ -189,7 +193,7 @@ func filterClusterSigners(cluster flow.IdentitySkeletonList, nodeInfos []model.N
 	var filtered []model.NodeInfo
 	for _, node := range nodeInfos {
 		_, isInCluster := cluster.ByNodeID(node.NodeID)
-		isNotPartner := node.Type() == model.NodeInfoTypePrivate
+		isPrivateKeyAvailable := node.Type() == model.NodeInfoTypePrivate
 
 		if isInCluster && isNotPartner {
 			filtered = append(filtered, node)
diff --git a/cmd/bootstrap/utils/key_generation.go b/cmd/bootstrap/utils/key_generation.go
index cd58470ef25..0e2f64a233a 100644
--- a/cmd/bootstrap/utils/key_generation.go
+++ b/cmd/bootstrap/utils/key_generation.go
@@ -297,7 +297,8 @@ func WriteStakingNetworkingKeyFiles(nodeInfos []bootstrap.NodeInfo, write WriteJ
 	return nil
 }
 
-// WriteNodeInternalPubInfos writes the node-internal-infos.pub.json file.
+// WriteNodeInternalPubInfos writes the `node-internal-infos.pub.json` file.
+// In a nutshell, this file contains the Role, address and weight for all authorized nodes. 
 func WriteNodeInternalPubInfos(nodeInfos []bootstrap.NodeInfo, write WriteJSONFileFunc) error {
 	configs := make([]model.NodeConfig, len(nodeInfos))
 	for i, nodeInfo := range nodeInfos {
diff --git a/cmd/util/cmd/common/clusters.go b/cmd/util/cmd/common/clusters.go
index 65da023ba1b..f02a495b1cb 100644
--- a/cmd/util/cmd/common/clusters.go
+++ b/cmd/util/cmd/common/clusters.go
@@ -150,7 +150,7 @@ func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.Cluste
 		qcVoteData[i] = cadence.NewStruct([]cadence.Value{
 			// aggregatedSignature
 			cadence.String(fmt.Sprintf("%#x", qc.SigData)),
-			// voterIDs
+			// Node IDs of signers
 			cadence.NewArray(cdcVoterIds).WithType(cadence.NewVariableSizedArrayType(cadence.StringType)),
 		}).WithType(voteDataType)
 
diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index 500592c86dc..12dfc9f4cac 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -68,7 +68,7 @@ func init() {
 
 func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagOut, "out", "", "file to write tx args output")
-	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnAddress, "access-address", "", "the address of the access node used for client connections")
+	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnAddress, "access-address", "", "the address of the access node used to retrieve the information")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagRootChainID, "root-chain-id", "", "the root chain id")
 	generateRecoverEpochTxArgsCmd.Flags().StringVar(&flagAnPubkey, "access-network-key", "", "the network key of the access node used for client connections in hex string format")
 	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagAnInsecure, "insecure", false, "set to true if the protocol snapshot should be retrieved from the insecure AN endpoint")
@@ -88,7 +88,7 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	// This is needed only if a previous recoverEpoch transaction was submitted and a race condition occurred such that:
 	//   - the RecoveryEpoch in the admin transaction was accepted by the smart contract
 	//   - the RecoveryEpoch service event (after sealing latency) was rejected by the Protocol State
-	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagInitNewEpoch, "unsafe-overwrite-epoch-data", false, "set to true if the resulting transaction is allowed to overwrite an existing epoch data entry in the smart contract. ")
+	generateRecoverEpochTxArgsCmd.Flags().BoolVar(&flagInitNewEpoch, "unsafe-overwrite-epoch-data", false, "set to true if the resulting transaction is allowed to overwrite an already specified epoch in the smart contract.")
 
 	err := generateRecoverEpochTxArgsCmd.MarkFlagRequired("access-address")
 	if err != nil {
diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index a7d304689b3..d478672b9b9 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -184,13 +184,13 @@ func (s *BaseSuite) GetLatestFinalizedHeader(ctx context.Context) *flow.Header {
 	return finalized
 }
 
-// AssertInEpoch requires actual epoch counter is equal to counter provided.
+// AssertInEpoch requires that the current epoch's counter (as of the latest finalized block) is equal to the counter value provided.
 func (s *BaseSuite) AssertInEpoch(ctx context.Context, expectedEpoch uint64) {
 	actualEpoch := s.CurrentEpoch(ctx)
 	require.Equalf(s.T(), expectedEpoch, actualEpoch, "expected to be in epoch %d got %d", expectedEpoch, actualEpoch)
 }
 
-// CurrentEpoch returns the current epoch counter.
+// CurrentEpoch returns the current epoch counter (as of the latest finalized block).
 func (s *BaseSuite) CurrentEpoch(ctx context.Context) uint64 {
 	snapshot := s.GetLatestProtocolSnapshot(ctx)
 	counter, err := snapshot.Epochs().Current().Counter()
@@ -198,7 +198,7 @@ func (s *BaseSuite) CurrentEpoch(ctx context.Context) uint64 {
 	return counter
 }
 
-// GetLatestProtocolSnapshot returns the latest protocol snapshot.
+// GetLatestProtocolSnapshot returns the protocol snapshot as of the latest finalized block.
 func (s *BaseSuite) GetLatestProtocolSnapshot(ctx context.Context) *inmem.Snapshot {
 	snapshot, err := s.Client.GetLatestProtocolSnapshot(ctx)
 	require.NoError(s.T(), err)
diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 72b0c5ccc2d..f4d82392512 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -23,7 +23,7 @@ type RecoverEpochSuite struct {
 
 // TestRecoverEpoch ensures that the recover epoch governance transaction flow works as expected and a network that
 // enters Epoch Fallback Mode can successfully recover. This test will do the following:
-// 1. Manually triggers EFM by turning off a collection node before the end of the DKG forcing the DKG to fail.
+// 1. Triggers EFM by turning off the sole collection node before the end of the DKG forcing the DKG to fail.
 // 2. Generates epoch recover transaction args using the epoch efm-recover-tx-args.
 // 3. Submit recover epoch transaction.
 // 4. Ensure expected EpochRecover event is emitted.
@@ -34,7 +34,7 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, 10*time.Second, 500*time.Millisecond)
 	// pausing collection node will force the network into EFM
 	ln := s.GetContainersByRole(flow.RoleCollection)[0]
-	_ = ln.Pause()
+	require.NoError(s.T(), ln.Pause())
 	s.AwaitFinalizedView(s.Ctx, s.GetDKGEndView(), 2*time.Minute, 500*time.Millisecond)
 	// start the paused collection node now that we are in EFM
 	require.NoError(s.T(), ln.Start())
@@ -91,5 +91,5 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	require.NoError(s.T(), err)
 	require.Equal(s.T(), events[0].Events[0].Type, eventType)
 
-	// 4. @TODO ensure EpochRecover service event is processed by the fallback state machine and the network recovers.
+	// 4. TODO(EFM, #6164) ensure EpochRecover service event is processed by the fallback state machine and the network recovers.
 }

From d2fb6bc715cc459fdc4b94c7eb737d4293f10525 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Mon, 15 Jul 2024 12:41:46 -0400
Subject: [PATCH 40/60] add godoc to ConvertClusterQcsCdc

---
 cmd/util/cmd/common/clusters.go | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/cmd/util/cmd/common/clusters.go b/cmd/util/cmd/common/clusters.go
index 65da023ba1b..7be67dc0cfe 100644
--- a/cmd/util/cmd/common/clusters.go
+++ b/cmd/util/cmd/common/clusters.go
@@ -130,8 +130,17 @@ func ConvertClusterAssignmentsCdc(assignments flow.AssignmentList) cadence.Array
 }
 
 // ConvertClusterQcsCdc converts cluster QCs from `QuorumCertificate` type to `ClusterQCVoteData` type.
-func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.ClusterList, address string) ([]cadence.Value, error) {
-	voteDataType := newFlowClusterQCVoteDataStructType(address)
+// Args:
+//   - qcs: list of quorum certificates.
+//   - clusterList: the list of cluster lists each used to generate one of the quorum certificates in qcs.
+//   - flowClusterQCAddress: the FlowClusterQC contract address where the ClusterQCVoteData type is defined.
+//
+// Returns:
+//   - []cadence.Value: cadence representation of the list of cluster qcs.
+//   - error: error if the cluster qcs and cluster lists don't match in size or
+//     signature indices decoding fails.
+func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.ClusterList, flowClusterQCAddress string) ([]cadence.Value, error) {
+	voteDataType := newFlowClusterQCVoteDataStructType(flowClusterQCAddress)
 	qcVoteData := make([]cadence.Value, len(qcs))
 	for i, qc := range qcs {
 		c, ok := clusterList.ByIndex(uint(i))

From 0d225b7d5a689c7e1f0ddf2b4a7eecff8b30b87f Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Mon, 15 Jul 2024 12:42:38 -0400
Subject: [PATCH 41/60] rename newFlowClusterQCVoteDataStructType ->
 newClusterQCVoteDataCdcType

---
 cmd/util/cmd/common/clusters.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/util/cmd/common/clusters.go b/cmd/util/cmd/common/clusters.go
index 7be67dc0cfe..17c441d104b 100644
--- a/cmd/util/cmd/common/clusters.go
+++ b/cmd/util/cmd/common/clusters.go
@@ -140,7 +140,7 @@ func ConvertClusterAssignmentsCdc(assignments flow.AssignmentList) cadence.Array
 //   - error: error if the cluster qcs and cluster lists don't match in size or
 //     signature indices decoding fails.
 func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.ClusterList, flowClusterQCAddress string) ([]cadence.Value, error) {
-	voteDataType := newFlowClusterQCVoteDataStructType(flowClusterQCAddress)
+	voteDataType := newClusterQCVoteDataCdcType(flowClusterQCAddress)
 	qcVoteData := make([]cadence.Value, len(qcs))
 	for i, qc := range qcs {
 		c, ok := clusterList.ByIndex(uint(i))
@@ -168,8 +168,8 @@ func ConvertClusterQcsCdc(qcs []*flow.QuorumCertificate, clusterList flow.Cluste
 	return qcVoteData, nil
 }
 
-// newFlowClusterQCVoteDataStructType returns the FlowClusterQC cadence struct type.
-func newFlowClusterQCVoteDataStructType(clusterQcAddress string) *cadence.StructType {
+// newClusterQCVoteDataCdcType returns the FlowClusterQC cadence struct type.
+func newClusterQCVoteDataCdcType(clusterQcAddress string) *cadence.StructType {
 
 	// FlowClusterQC.ClusterQCVoteData
 	address, _ := cdcCommon.HexToAddress(clusterQcAddress)

From 216ca7e63e62cb0eb9318c713ddca8543184b26f Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 16 Jul 2024 12:30:33 -0400
Subject: [PATCH 42/60] Update epochs.go

---
 cmd/bootstrap/run/epochs.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cmd/bootstrap/run/epochs.go b/cmd/bootstrap/run/epochs.go
index d405a5b7296..bf34be7ab86 100644
--- a/cmd/bootstrap/run/epochs.go
+++ b/cmd/bootstrap/run/epochs.go
@@ -113,8 +113,7 @@ func GenerateRecoverEpochTxArgs(log zerolog.Logger,
 		}
 		nodeIds = append(nodeIds, nodeIdCdc)
 	}
-
-	clusterQCAddress := systemcontracts.SystemContractsForChain(flow.ChainID(rootChainID)).ClusterQC.Address.String()
+	clusterQCAddress := systemcontracts.SystemContractsForChain(rootChainID).ClusterQC.Address.String()
 	qcVoteData, err := common.ConvertClusterQcsCdc(clusterQCs, clusters, clusterQCAddress)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to convert cluster qcs to cadence type")

From e9532efa02c31dbf5eadc888eeb3079dbe5a7c60 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 16 Jul 2024 14:09:37 -0400
Subject: [PATCH 43/60] remove duplicate code update test

---
 cmd/bootstrap/run/epochs.go               |   4 +-
 cmd/util/cmd/epochs/cmd/recover.go        | 129 ----------------------
 cmd/util/cmd/epochs/cmd/recover_test.go   |  87 ++++++++++++---
 utils/unittest/service_events_fixtures.go |  19 ++++
 4 files changed, 94 insertions(+), 145 deletions(-)

diff --git a/cmd/bootstrap/run/epochs.go b/cmd/bootstrap/run/epochs.go
index 7b7bcc0bc57..ea137d68c94 100644
--- a/cmd/bootstrap/run/epochs.go
+++ b/cmd/bootstrap/run/epochs.go
@@ -36,7 +36,7 @@ func GenerateRecoverEpochTxArgs(log zerolog.Logger,
 	if err != nil {
 		return nil, fmt.Errorf("failed to get  valid protocol participants from snapshot: %w", err)
 	}
-		// We need canonical ordering here; sanity check to enforce this:  
+	// We need canonical ordering here; sanity check to enforce this:
 	if !currentEpochIdentities.Sorted(flow.Canonical[flow.Identity]) {
 		return nil, fmt.Errorf("identies from snapshot not in canonical order")
 	}
@@ -194,7 +194,7 @@ func filterClusterSigners(cluster flow.IdentitySkeletonList, nodeInfos []model.N
 		_, isInCluster := cluster.ByNodeID(node.NodeID)
 		isPrivateKeyAvailable := node.Type() == model.NodeInfoTypePrivate
 
-		if isInCluster && isNotPartner {
+		if isInCluster && isPrivateKeyAvailable {
 			filtered = append(filtered, node)
 		}
 	}
diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index 12dfc9f4cac..a0d26ea635b 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -7,14 +7,10 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/onflow/cadence"
-
 	"github.com/onflow/flow-go/cmd/bootstrap/run"
 	"github.com/onflow/flow-go/cmd/util/cmd/common"
 	epochcmdutil "github.com/onflow/flow-go/cmd/util/cmd/epochs/utils"
-	"github.com/onflow/flow-go/fvm/systemcontracts"
 	"github.com/onflow/flow-go/model/flow"
-	"github.com/onflow/flow-go/model/flow/filter"
 	"github.com/onflow/flow-go/module/grpcclient"
 	"github.com/onflow/flow-go/state/protocol/inmem"
 )
@@ -188,128 +184,3 @@ func generateRecoverEpochTxArgs(getSnapshot func() *inmem.Snapshot) func(cmd *co
 		}
 	}
 }
-
-// extractRecoverEpochArgs extracts the required transaction arguments for the `recoverEpoch` transaction.
-func extractRecoverEpochArgs(snapshot *inmem.Snapshot) []cadence.Value {
-	epoch := snapshot.Epochs().Current()
-
-	currentEpochIdentities, err := snapshot.Identities(filter.IsValidProtocolParticipant)
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed to get  valid protocol participants from snapshot")
-	}
-
-	// separate collector nodes by internal and partner nodes
-	collectors := currentEpochIdentities.Filter(filter.HasRole[flow.Identity](flow.RoleCollection))
-	internalCollectors := make(flow.IdentityList, 0)
-	partnerCollectors := make(flow.IdentityList, 0)
-
-	log.Info().Msg("collecting internal node network and staking keys")
-	internalNodes, err := common.ReadFullInternalNodeInfos(log, flagInternalNodePrivInfoDir, flagNodeConfigJson)
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed to read full internal node infos")
-	}
-
-	internalNodesMap := make(map[flow.Identifier]struct{})
-	for _, node := range internalNodes {
-		if !currentEpochIdentities.Exists(node.Identity()) {
-			log.Fatal().Msg(fmt.Sprintf("node ID found in internal node infos missing from protocol snapshot identities: %s", node.NodeID))
-		}
-		internalNodesMap[node.NodeID] = struct{}{}
-	}
-	log.Info().Msg("")
-
-	for _, collector := range collectors {
-		if _, ok := internalNodesMap[collector.NodeID]; ok {
-			internalCollectors = append(internalCollectors, collector)
-		} else {
-			partnerCollectors = append(partnerCollectors, collector)
-		}
-	}
-
-	currentEpochDKG, err := epoch.DKG()
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed to get DKG for current epoch")
-	}
-
-	log.Info().Msg("computing collection node clusters")
-
-	assignments, clusters, err := common.ConstructClusterAssignment(log, partnerCollectors, internalCollectors, flagCollectionClusters)
-	if err != nil {
-		log.Fatal().Err(err).Msg("unable to generate cluster assignment")
-	}
-	log.Info().Msg("")
-
-	log.Info().Msg("constructing root blocks for collection node clusters")
-	clusterBlocks := run.GenerateRootClusterBlocks(flagEpochCounter, clusters)
-	log.Info().Msg("")
-
-	log.Info().Msg("constructing root QCs for collection node clusters")
-	clusterQCs := run.ConstructRootQCsForClusters(log, clusters, internalNodes, clusterBlocks)
-	log.Info().Msg("")
-
-	dkgPubKeys := make([]cadence.Value, 0)
-	nodeIds := make([]cadence.Value, 0)
-
-	// NOTE: The RecoveryEpoch will re-use the last successful DKG output. This means that the consensus
-	// committee in the RecoveryEpoch must be identical to the committee which participated in that DKG.
-	dkgGroupKeyCdc, cdcErr := cadence.NewString(currentEpochDKG.GroupKey().String())
-	if cdcErr != nil {
-		log.Fatal().Err(cdcErr).Msg("failed to get dkg group key cadence string")
-	}
-	dkgPubKeys = append(dkgPubKeys, dkgGroupKeyCdc)
-	for _, id := range currentEpochIdentities {
-		if id.GetRole() == flow.RoleConsensus {
-			dkgPubKey, keyShareErr := currentEpochDKG.KeyShare(id.GetNodeID())
-			if keyShareErr != nil {
-				log.Fatal().Err(keyShareErr).Msg(fmt.Sprintf("failed to get dkg pub key share for node: %s", id.GetNodeID()))
-			}
-			dkgPubKeyCdc, cdcErr := cadence.NewString(dkgPubKey.String())
-			if cdcErr != nil {
-				log.Fatal().Err(cdcErr).Msg(fmt.Sprintf("failed to get dkg pub key cadence string for node: %s", id.GetNodeID()))
-			}
-			dkgPubKeys = append(dkgPubKeys, dkgPubKeyCdc)
-		}
-		nodeIdCdc, err := cadence.NewString(id.GetNodeID().String())
-		if err != nil {
-			log.Fatal().Err(err).Msg(fmt.Sprintf("failed to convert node ID to cadence string: %s", id.GetNodeID()))
-		}
-		nodeIds = append(nodeIds, nodeIdCdc)
-	}
-
-	clusterQCAddress := systemcontracts.SystemContractsForChain(flow.ChainID(flagRootChainID)).ClusterQC.Address.String()
-	qcVoteData, err := common.ConvertClusterQcsCdc(clusterQCs, clusters, clusterQCAddress)
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed to convert cluster qcs to cadence type")
-	}
-
-	currEpochFinalView, err := epoch.FinalView()
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed to get final view of current epoch")
-	}
-
-	args := []cadence.Value{
-		// epoch start view
-		cadence.NewUInt64(currEpochFinalView + 1),
-		// staking phase end view
-		cadence.NewUInt64(currEpochFinalView + flagNumViewsInStakingAuction),
-		// epoch end view
-		cadence.NewUInt64(currEpochFinalView + flagNumViewsInEpoch),
-		// target duration
-		cadence.NewUInt64(flagTargetDuration),
-		// target end time
-		cadence.NewUInt64(flagTargetEndTime),
-		// clusters,
-		common.ConvertClusterAssignmentsCdc(assignments),
-		// qcVoteData
-		cadence.NewArray(qcVoteData),
-		// dkg pub keys
-		cadence.NewArray(dkgPubKeys),
-		// node ids
-		cadence.NewArray(nodeIds),
-		// recover the network by initializing a new recover epoch which will increment the smart contract epoch counter
-		// or overwrite the epoch metadata for the current epoch
-		cadence.NewBool(flagInitNewEpoch),
-	}
-
-	return args
-}
diff --git a/cmd/util/cmd/epochs/cmd/recover_test.go b/cmd/util/cmd/epochs/cmd/recover_test.go
index 980a9788a55..8ca87e36fb0 100644
--- a/cmd/util/cmd/epochs/cmd/recover_test.go
+++ b/cmd/util/cmd/epochs/cmd/recover_test.go
@@ -3,14 +3,15 @@ package cmd
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"testing"
 
-	"github.com/onflow/flow-go/cmd/util/cmd/common"
-	"github.com/onflow/flow-go/model/flow"
-
 	"github.com/stretchr/testify/require"
 
+	"github.com/onflow/cadence"
 	"github.com/onflow/flow-go/cmd/bootstrap/utils"
+	"github.com/onflow/flow-go/cmd/util/cmd/common"
+	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/state/protocol/inmem"
 	"github.com/onflow/flow-go/utils/unittest"
 )
@@ -26,18 +27,31 @@ func TestRecoverEpochHappyPath(t *testing.T) {
 		require.NoError(t, err)
 
 		allNodeIds := make(flow.IdentityList, 0)
-		for _, node := range internalNodes {
-			allNodeIds = append(allNodeIds, node.Identity())
-		}
-		for _, node := range partnerNodes {
+		allNodeIdsCdc := make(map[cadence.String]*flow.Identity)
+		for _, node := range append(internalNodes, partnerNodes...) {
 			allNodeIds = append(allNodeIds, node.Identity())
+			allNodeIdsCdc[cadence.String(node.Identity().NodeID.String())] = node.Identity()
 		}
 
 		// create a root snapshot
 		rootSnapshot := unittest.RootSnapshotFixture(allNodeIds)
-
 		snapshotFn := func() *inmem.Snapshot { return rootSnapshot }
 
+		// get expected dkg information
+		currentEpochDKG, err := rootSnapshot.Epochs().Current().DKG()
+		require.NoError(t, err)
+		expectedDKGPubKeys := make(map[cadence.String]struct{})
+		expectedDKGGroupKey := cadence.String(currentEpochDKG.GroupKey().String())
+		for _, id := range allNodeIds {
+			if id.GetRole() == flow.RoleConsensus {
+				dkgPubKey, keyShareErr := currentEpochDKG.KeyShare(id.GetNodeID())
+				if keyShareErr != nil {
+					log.Fatal().Err(keyShareErr).Msg(fmt.Sprintf("failed to get dkg pub key share for node: %s", id.GetNodeID()))
+				}
+				expectedDKGPubKeys[cadence.String(dkgPubKey.String())] = struct{}{}
+			}
+		}
+
 		// run command with overwritten stdout
 		stdout := bytes.NewBuffer(nil)
 		generateRecoverEpochTxArgsCmd.SetOut(stdout)
@@ -45,9 +59,10 @@ func TestRecoverEpochHappyPath(t *testing.T) {
 		flagInternalNodePrivInfoDir = internalPrivDir
 		flagNodeConfigJson = configPath
 		flagCollectionClusters = 2
-		flagNumViewsInEpoch = 4000
-		flagNumViewsInStakingAuction = 100
 		flagEpochCounter = 2
+		flagRootChainID = flow.Localnet.String()
+		flagNumViewsInStakingAuction = 100
+		flagNumViewsInEpoch = 4000
 
 		generateRecoverEpochTxArgs(snapshotFn)(generateRecoverEpochTxArgsCmd, nil)
 
@@ -55,9 +70,53 @@ func TestRecoverEpochHappyPath(t *testing.T) {
 		var outputTxArgs []interface{}
 		err = json.NewDecoder(stdout).Decode(&outputTxArgs)
 		require.NoError(t, err)
-		// compare to expected values
-		expectedArgs := extractRecoverEpochArgs(rootSnapshot)
-		unittest.VerifyCdcArguments(t, expectedArgs[:len(expectedArgs)-1], outputTxArgs[:len(expectedArgs)-1])
-		// @TODO validate cadence values for generated cluster assignments and clusters
+
+		// verify each argument
+		decodedValues := unittest.InterfafceToCdcValues(t, outputTxArgs)
+		currEpoch := rootSnapshot.Epochs().Current()
+		finalView, err := currEpoch.FinalView()
+		require.NoError(t, err)
+
+		// epoch start view
+		require.Equal(t, decodedValues[0], cadence.NewUInt64(finalView+1))
+		// staking phase end view
+		require.Equal(t, decodedValues[1], cadence.NewUInt64(finalView+flagNumViewsInStakingAuction))
+		// epoch end view
+		require.Equal(t, decodedValues[2], cadence.NewUInt64(finalView+flagNumViewsInEpoch))
+		// target duration
+		require.Equal(t, decodedValues[3], cadence.NewUInt64(flagTargetDuration))
+		// target end time
+		require.Equal(t, decodedValues[4], cadence.NewUInt64(flagTargetEndTime))
+		// clusters: we cannot guarantee order of the cluster when we generate the test fixtures
+		// so, we ensure each cluster member is part of the full set of node ids
+		for _, cluster := range decodedValues[5].(cadence.Array).Values {
+			for _, nodeId := range cluster.(cadence.Array).Values {
+				_, ok := allNodeIdsCdc[nodeId.(cadence.String)]
+				require.True(t, ok)
+			}
+		}
+		// qcVoteData: we cannot guarantee order of the cluster when we generate the test fixtures
+		// so, we ensure each voter id that participated in a qc vote exists and is a collection node
+		for _, voteData := range decodedValues[6].(cadence.Array).Values {
+			fields := cadence.FieldsMappedByName(voteData.(cadence.Struct))
+			for _, voterId := range fields["voterIDs"].(cadence.Array).Values {
+				id, ok := allNodeIdsCdc[voterId.(cadence.String)]
+				require.True(t, ok)
+				require.Equal(t, flow.RoleCollection, id.Role)
+			}
+		}
+		// dkg pub keys
+		require.Equal(t, expectedDKGGroupKey, decodedValues[7].(cadence.Array).Values[0])
+		for _, dkgPubKey := range decodedValues[7].(cadence.Array).Values[1:] {
+			_, ok := expectedDKGPubKeys[dkgPubKey.(cadence.String)]
+			require.True(t, ok)
+		}
+		// node ids
+		for _, nodeId := range decodedValues[8].(cadence.Array).Values {
+			_, ok := allNodeIdsCdc[nodeId.(cadence.String)]
+			require.True(t, ok)
+		}
+		// initNewEpoch
+		require.Equal(t, decodedValues[9], cadence.NewBool(false))
 	})
 }
diff --git a/utils/unittest/service_events_fixtures.go b/utils/unittest/service_events_fixtures.go
index 84e4873dddd..675bd575a1d 100644
--- a/utils/unittest/service_events_fixtures.go
+++ b/utils/unittest/service_events_fixtures.go
@@ -1457,6 +1457,25 @@ func VerifyCdcArguments(t *testing.T, expected []cadence.Value, actual []interfa
 	}
 }
 
+// InterfafceToCdcValues decodes jsoncdc encoded values from interface -> cadence value.
+func InterfafceToCdcValues(t *testing.T, vals []interface{}) []cadence.Value {
+	decoded := make([]cadence.Value, len(vals))
+	for index, val := range vals {
+
+		// marshal to bytes
+		bz, err := json.Marshal(val)
+		require.NoError(t, err)
+
+		// parse cadence value
+		cdcVal, err := jsoncdc.Decode(nil, bz)
+		require.NoError(t, err)
+
+		decoded[index] = cdcVal
+	}
+
+	return decoded
+}
+
 func NewFlowClusterQCClusterStructType() *cadence.StructType {
 
 	// A.01cf0e2f2f715450.FlowClusterQC.Cluster

From a0df4a205e1680c1e6146b652ec8d6701754351d Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 16 Jul 2024 14:21:35 -0400
Subject: [PATCH 44/60] get current epoch target time from snapshot

---
 cmd/bootstrap/run/epochs.go             |  8 ++++++--
 cmd/util/cmd/epochs/cmd/recover.go      | 10 ++--------
 cmd/util/cmd/epochs/cmd/recover_test.go |  4 +++-
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/cmd/bootstrap/run/epochs.go b/cmd/bootstrap/run/epochs.go
index ea137d68c94..5d041d5cf00 100644
--- a/cmd/bootstrap/run/epochs.go
+++ b/cmd/bootstrap/run/epochs.go
@@ -26,7 +26,6 @@ func GenerateRecoverEpochTxArgs(log zerolog.Logger,
 	numViewsInStakingAuction uint64,
 	numViewsInEpoch uint64,
 	targetDuration uint64,
-	targetEndTime uint64,
 	initNewEpoch bool,
 	snapshot *inmem.Snapshot,
 ) ([]cadence.Value, error) {
@@ -129,6 +128,11 @@ func GenerateRecoverEpochTxArgs(log zerolog.Logger,
 		log.Fatal().Err(err).Msg("failed to get final view of current epoch")
 	}
 
+	currEpochTargetEndTime, err := epoch.TargetEndTime()
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to get target end time of current epoch")
+	}
+
 	args := []cadence.Value{
 		// epoch start view
 		cadence.NewUInt64(currEpochFinalView + 1),
@@ -139,7 +143,7 @@ func GenerateRecoverEpochTxArgs(log zerolog.Logger,
 		// target duration
 		cadence.NewUInt64(targetDuration),
 		// target end time
-		cadence.NewUInt64(targetEndTime),
+		cadence.NewUInt64(currEpochTargetEndTime),
 		// clusters,
 		common.ConvertClusterAssignmentsCdc(assignments),
 		// qcVoteData
diff --git a/cmd/util/cmd/epochs/cmd/recover.go b/cmd/util/cmd/epochs/cmd/recover.go
index a0d26ea635b..cb56779d3f4 100644
--- a/cmd/util/cmd/epochs/cmd/recover.go
+++ b/cmd/util/cmd/epochs/cmd/recover.go
@@ -49,7 +49,6 @@ This recovery process has some constraints:
 	flagNumViewsInStakingAuction uint64
 	flagEpochCounter             uint64
 	flagTargetDuration           uint64
-	flagTargetEndTime            uint64
 	flagInitNewEpoch             bool
 	flagRootChainID              string
 )
@@ -79,7 +78,6 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 0, "length of the epoch staking phase measured in views")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagEpochCounter, "epoch-counter", 0, "the epoch counter used to generate the root cluster block")
 	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagTargetDuration, "epoch-timing-duration", 0, "the target duration of the epoch, in seconds")
-	generateRecoverEpochTxArgsCmd.Flags().Uint64Var(&flagTargetEndTime, "epoch-timing-end-time", 0, "the target end time for the epoch, specified in second-precision Unix time")
 	// The following option allows the RecoveryEpoch specified by this command to overwrite an epoch which already exists in the smart contract.
 	// This is needed only if a previous recoverEpoch transaction was submitted and a race condition occurred such that:
 	//   - the RecoveryEpoch in the admin transaction was accepted by the smart contract
@@ -108,12 +106,9 @@ func addGenerateRecoverEpochTxArgsCmdFlags() error {
 	}
 	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("epoch-timing-duration")
 	if err != nil {
-		return fmt.Errorf("failed to mark target-duration flag as required")
-	}
-	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("epoch-timing-end-time")
-	if err != nil {
-		return fmt.Errorf("failed to mark target-end-time flag as required")
+		return fmt.Errorf("failed to mark epoch-timing-duration flag as required")
 	}
+
 	err = generateRecoverEpochTxArgsCmd.MarkFlagRequired("root-chain-id")
 	if err != nil {
 		return fmt.Errorf("failed to mark root-chain-id flag as required")
@@ -155,7 +150,6 @@ func generateRecoverEpochTxArgs(getSnapshot func() *inmem.Snapshot) func(cmd *co
 			flagNumViewsInStakingAuction,
 			flagNumViewsInEpoch,
 			flagTargetDuration,
-			flagTargetEndTime,
 			flagInitNewEpoch,
 			getSnapshot(),
 		)
diff --git a/cmd/util/cmd/epochs/cmd/recover_test.go b/cmd/util/cmd/epochs/cmd/recover_test.go
index 8ca87e36fb0..5ee87b8dfd6 100644
--- a/cmd/util/cmd/epochs/cmd/recover_test.go
+++ b/cmd/util/cmd/epochs/cmd/recover_test.go
@@ -86,7 +86,9 @@ func TestRecoverEpochHappyPath(t *testing.T) {
 		// target duration
 		require.Equal(t, decodedValues[3], cadence.NewUInt64(flagTargetDuration))
 		// target end time
-		require.Equal(t, decodedValues[4], cadence.NewUInt64(flagTargetEndTime))
+		expectedTargetEndTime, err := rootSnapshot.Epochs().Current().TargetEndTime()
+		require.NoError(t, err)
+		require.Equal(t, decodedValues[4], cadence.NewUInt64(expectedTargetEndTime))
 		// clusters: we cannot guarantee order of the cluster when we generate the test fixtures
 		// so, we ensure each cluster member is part of the full set of node ids
 		for _, cluster := range decodedValues[5].(cadence.Array).Values {

From c331ab265f630402a00344d4f701166fa85a3c6c Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 16 Jul 2024 14:23:02 -0400
Subject: [PATCH 45/60] remove outdated comments "pausing container"

---
 .../tests/epochs/recover_epoch/recover_epoch_efm_test.go      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index f4d82392512..00a9a399fc0 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -45,9 +45,9 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 
 	// wait for at least the first block of the next epoch to be sealed so that we can
 	// ensure that we are still in the same epoch after the final view of that epoch indicating we are in EFM
-	s.TimedLogf("waiting for epoch transition (finalized view %d) before pausing container", epoch1FinalView+1)
+	s.TimedLogf("waiting for epoch transition (finalized view %d)", epoch1FinalView+1)
 	s.AwaitFinalizedView(s.Ctx, epoch1FinalView+1, 2*time.Minute, 500*time.Millisecond)
-	s.TimedLogf("observed finalized view %d -> pausing container", epoch1FinalView+1)
+	s.TimedLogf("observed finalized view %d", epoch1FinalView+1)
 
 	// assert transition to second epoch did not happen
 	// if counter is still 0, epoch emergency fallback was triggered as expected

From 01bfe0d049bca22eee1c1b3fcbd520b466b38a8f Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 16 Jul 2024 14:28:55 -0400
Subject: [PATCH 46/60] add EFM TODO to investigate short DKG phase len config
 not triggering EFM without pausing LN container

---
 .../tests/epochs/recover_epoch/recover_epoch_efm_test.go     | 5 ++++-
 integration/tests/epochs/recover_epoch/suite.go              | 1 -
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
index 00a9a399fc0..853768d6a41 100644
--- a/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
+++ b/integration/tests/epochs/recover_epoch/recover_epoch_efm_test.go
@@ -32,7 +32,10 @@ func (s *RecoverEpochSuite) TestRecoverEpoch() {
 	// 1. Manually trigger EFM
 	// wait until the epoch setup phase to force network into EFM
 	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, 10*time.Second, 500*time.Millisecond)
-	// pausing collection node will force the network into EFM
+
+	// We set the DKG phase view len to 10 which is very short and should cause the network to go into EFM
+	// without pausing the collection node. This is not the case, we still need to pause the collection node.
+	//TODO(EFM, #6164): Why short DKG phase len of 10 views does not trigger EFM without pausing container ; see https://github.com/onflow/flow-go/issues/6164
 	ln := s.GetContainersByRole(flow.RoleCollection)[0]
 	require.NoError(s.T(), ln.Pause())
 	s.AwaitFinalizedView(s.Ctx, s.GetDKGEndView(), 2*time.Minute, 500*time.Millisecond)
diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index c836bf1de3a..66166ba31b7 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -66,7 +66,6 @@ func (s *Suite) executeEFMRecoverTXArgsCMD(collectionClusters int, numViewsInEpo
 		numViewsInStakingAuction,
 		numViewsInEpoch,
 		targetDuration,
-		targetEndTime,
 		false,
 		snapshot,
 	)

From 4a2a888e76b7d3a7982627e51f5353a4d5f9f114 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 16 Jul 2024 23:11:24 -0400
Subject: [PATCH 47/60] added root go.sum

---
 go.sum | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/go.sum b/go.sum
index 2fb20f8db14..7efdc702141 100644
--- a/go.sum
+++ b/go.sum
@@ -2178,12 +2178,8 @@ github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A=
 github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
 github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE=
 github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c=
-github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 h1:zLDxmCxa/jx86uwkkaFXpztjgrofIHk74Bj6d/raeXA=
-github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920 h1:dvpU5WG++j9s3XXbFnMv/eVdupEHh1Xb0NyLZYDEz7A=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
-github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 h1:W5dPPXV8cNG0nYxKe+g/8U2hkhZiIR6W8kFKqsJ5o9Y=
-github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920 h1:SBUd8cq9OThExucjUO3EFMuzA5Sp8NLGUjch2phxKfg=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0=

From a982d13fe65d73de14f9db5f59559c8ba51830a7 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 16 Jul 2024 23:12:50 -0400
Subject: [PATCH 48/60] added insecure and integration go.sum

---
 insecure/go.sum    | 4 ----
 integration/go.sum | 4 ----
 2 files changed, 8 deletions(-)

diff --git a/insecure/go.sum b/insecure/go.sum
index 72be2698bf7..5feb6732c31 100644
--- a/insecure/go.sum
+++ b/insecure/go.sum
@@ -2165,12 +2165,8 @@ github.com/onflow/cadence v1.0.0-preview.34/go.mod h1:jOwvPSSLTr9TvaKMs7KKiBYMmp
 github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
 github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A=
 github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
-github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 h1:zLDxmCxa/jx86uwkkaFXpztjgrofIHk74Bj6d/raeXA=
-github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920 h1:dvpU5WG++j9s3XXbFnMv/eVdupEHh1Xb0NyLZYDEz7A=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
-github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 h1:W5dPPXV8cNG0nYxKe+g/8U2hkhZiIR6W8kFKqsJ5o9Y=
-github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920 h1:SBUd8cq9OThExucjUO3EFMuzA5Sp8NLGUjch2phxKfg=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-ft/lib/go/contracts v1.0.0 h1:mToacZ5NWqtlWwk/7RgIl/jeKB/Sy/tIXdw90yKHcV0=
diff --git a/integration/go.sum b/integration/go.sum
index 4632a19b4d2..760917ecf47 100644
--- a/integration/go.sum
+++ b/integration/go.sum
@@ -2155,12 +2155,8 @@ github.com/onflow/cadence v1.0.0-preview.34/go.mod h1:jOwvPSSLTr9TvaKMs7KKiBYMmp
 github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
 github.com/onflow/crypto v0.25.1 h1:0txy2PKPMM873JbpxQNbJmuOJtD56bfs48RQfm0ts5A=
 github.com/onflow/crypto v0.25.1/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI=
-github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4 h1:zLDxmCxa/jx86uwkkaFXpztjgrofIHk74Bj6d/raeXA=
-github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920 h1:dvpU5WG++j9s3XXbFnMv/eVdupEHh1Xb0NyLZYDEz7A=
 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1-0.20240710025346-d8135ebb9920/go.mod h1:u/mkP/B+PbV33tEG3qfkhhBlydSvAKxfLZSfB4lsJHg=
-github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4 h1:W5dPPXV8cNG0nYxKe+g/8U2hkhZiIR6W8kFKqsJ5o9Y=
-github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240620231448-967aae8f8ff4/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920 h1:SBUd8cq9OThExucjUO3EFMuzA5Sp8NLGUjch2phxKfg=
 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1-0.20240710025346-d8135ebb9920/go.mod h1:NgbMOYnMh0GN48VsNKZuiwK7uyk38Wyo8jN9+C9QE30=
 github.com/onflow/flow-emulator v1.0.0-preview.24 h1:SonXMBeYxVwNn94M+OUmKIYScIMQG22wugh9n/tHY5k=

From 3aaf62ffd1078475c80c0093c52ce2d6ce911ee9 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 16 Jul 2024 23:15:52 -0400
Subject: [PATCH 49/60] fix lint

---
 cmd/bootstrap/run/epochs.go             |  1 +
 cmd/bootstrap/utils/key_generation.go   | 10 +++++-----
 cmd/util/cmd/epochs/cmd/recover_test.go |  1 +
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/cmd/bootstrap/run/epochs.go b/cmd/bootstrap/run/epochs.go
index 5d041d5cf00..bd742ebd8bf 100644
--- a/cmd/bootstrap/run/epochs.go
+++ b/cmd/bootstrap/run/epochs.go
@@ -6,6 +6,7 @@ import (
 	"github.com/rs/zerolog"
 
 	"github.com/onflow/cadence"
+
 	"github.com/onflow/flow-go/cmd/util/cmd/common"
 	"github.com/onflow/flow-go/fvm/systemcontracts"
 	"github.com/onflow/flow-go/model/bootstrap"
diff --git a/cmd/bootstrap/utils/key_generation.go b/cmd/bootstrap/utils/key_generation.go
index 0e2f64a233a..3a6138edb15 100644
--- a/cmd/bootstrap/utils/key_generation.go
+++ b/cmd/bootstrap/utils/key_generation.go
@@ -4,17 +4,17 @@ import (
 	"crypto/rand"
 	"crypto/sha256"
 	"fmt"
-	"github.com/onflow/crypto"
-	"golang.org/x/crypto/hkdf"
 	gohash "hash"
 	"io"
 
+	"golang.org/x/crypto/hkdf"
+
+	"github.com/onflow/crypto"
 	sdk "github.com/onflow/flow-go-sdk"
 	sdkcrypto "github.com/onflow/flow-go-sdk/crypto"
-	model "github.com/onflow/flow-go/model/bootstrap"
-
 	"github.com/onflow/flow-go/fvm/systemcontracts"
 	"github.com/onflow/flow-go/model/bootstrap"
+	model "github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/encodable"
 	"github.com/onflow/flow-go/model/flow"
 )
@@ -298,7 +298,7 @@ func WriteStakingNetworkingKeyFiles(nodeInfos []bootstrap.NodeInfo, write WriteJ
 }
 
 // WriteNodeInternalPubInfos writes the `node-internal-infos.pub.json` file.
-// In a nutshell, this file contains the Role, address and weight for all authorized nodes. 
+// In a nutshell, this file contains the Role, address and weight for all authorized nodes.
 func WriteNodeInternalPubInfos(nodeInfos []bootstrap.NodeInfo, write WriteJSONFileFunc) error {
 	configs := make([]model.NodeConfig, len(nodeInfos))
 	for i, nodeInfo := range nodeInfos {
diff --git a/cmd/util/cmd/epochs/cmd/recover_test.go b/cmd/util/cmd/epochs/cmd/recover_test.go
index 5ee87b8dfd6..0877b1bcd23 100644
--- a/cmd/util/cmd/epochs/cmd/recover_test.go
+++ b/cmd/util/cmd/epochs/cmd/recover_test.go
@@ -9,6 +9,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/onflow/cadence"
+
 	"github.com/onflow/flow-go/cmd/bootstrap/utils"
 	"github.com/onflow/flow-go/cmd/util/cmd/common"
 	"github.com/onflow/flow-go/model/flow"

From 3b5c953f38c911e0a459bd4af92415492589fdd2 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 16 Jul 2024 23:16:34 -0400
Subject: [PATCH 50/60] Update key_generation.go

---
 cmd/bootstrap/utils/key_generation.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmd/bootstrap/utils/key_generation.go b/cmd/bootstrap/utils/key_generation.go
index 3a6138edb15..fd4c8c53444 100644
--- a/cmd/bootstrap/utils/key_generation.go
+++ b/cmd/bootstrap/utils/key_generation.go
@@ -10,6 +10,7 @@ import (
 	"golang.org/x/crypto/hkdf"
 
 	"github.com/onflow/crypto"
+
 	sdk "github.com/onflow/flow-go-sdk"
 	sdkcrypto "github.com/onflow/flow-go-sdk/crypto"
 	"github.com/onflow/flow-go/fvm/systemcontracts"

From f1db0343da292711aaee5fed5107d8396a183809 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Tue, 16 Jul 2024 23:42:48 -0400
Subject: [PATCH 51/60] fix suite.go imports

---
 integration/tests/epochs/recover_epoch/suite.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 66166ba31b7..573acc8ea95 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -2,6 +2,9 @@ package recover_epoch
 
 import (
 	"fmt"
+
+	"github.com/stretchr/testify/require"
+	
 	"github.com/onflow/cadence"
 	"github.com/onflow/flow-core-contracts/lib/go/templates"
 	sdk "github.com/onflow/flow-go-sdk"
@@ -10,7 +13,6 @@ import (
 	"github.com/onflow/flow-go/integration/utils"
 	"github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/flow"
-	"github.com/stretchr/testify/require"
 )
 
 // Suite encapsulates common functionality for epoch integration tests.

From 00e2f40d345e600affa2f480352757221955f873 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Wed, 17 Jul 2024 17:23:12 -0400
Subject: [PATCH 52/60] fix recover event fixture

- update state comittment const
---
 .../state/bootstrap/bootstrap_test.go         |  2 +-
 utils/unittest/execution_state.go             |  6 ++---
 utils/unittest/service_events_fixtures.go     | 24 +++++++------------
 3 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go
index cd3edb6f2cb..e716d9f6ec2 100644
--- a/engine/execution/state/bootstrap/bootstrap_test.go
+++ b/engine/execution/state/bootstrap/bootstrap_test.go
@@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) {
 }
 
 func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) {
-	expectedStateCommitmentBytes, _ := hex.DecodeString("7cb3d30faaaab3cb402338023b3a068bc856fb788086b1212aa0f1950f24d854")
+	expectedStateCommitmentBytes, _ := hex.DecodeString("a11fe9eb514a20f849649c9d903dc32f93ce216226e49d568757707a9995aec6")
 	expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes)
 	require.NoError(t, err)
 
diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go
index 48c86d5b1ea..c65d392e18e 100644
--- a/utils/unittest/execution_state.go
+++ b/utils/unittest/execution_state.go
@@ -23,7 +23,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256
 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256
 
 // Pre-calculated state commitment with root account with the above private key
-const GenesisStateCommitmentHex = "ede86048a53464cdd3cef060e5e2e1603d91c2d2a0568417501af5ca9455f430"
+const GenesisStateCommitmentHex = "d4bcde7560ab8aff284d6d4bfeafad435fa9ad50b1934733a8aeb316636981b2"
 
 var GenesisStateCommitment flow.StateCommitment
 
@@ -87,10 +87,10 @@ func genesisCommitHexByChainID(chainID flow.ChainID) string {
 		return GenesisStateCommitmentHex
 	}
 	if chainID == flow.Testnet {
-		return "ac9887d488bc90e33ce97478ca3f59c69795ed66061cf68d60d518be2fa0b658"
+		return "5caa5d4461ca577fffa09d2d97aba96672098b8126cbefffb4f2cd92e83d94f5"
 	}
 	if chainID == flow.Sandboxnet {
 		return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1"
 	}
-	return "eb6a5749ba1a4bd9ef5b3aa804b123e0d257f68735867e661b1a1418b26f9229"
+	return "da1c27803b64703dbdb3ce0fbbf90368f7961bc9b0e6703a67da59be2b4d0aa0"
 }
diff --git a/utils/unittest/service_events_fixtures.go b/utils/unittest/service_events_fixtures.go
index 675bd575a1d..b5538c4a727 100644
--- a/utils/unittest/service_events_fixtures.go
+++ b/utils/unittest/service_events_fixtures.go
@@ -804,13 +804,9 @@ func createEpochRecoverEvent(randomSourceHex string) cadence.Event {
 	clusterQCVoteDataType := newFlowClusterQCClusterQCVoteDataStructType()
 
 	cluster1 := cadence.NewStruct([]cadence.Value{
-		// voteSignatures
-		cadence.NewArray([]cadence.Value{
-			cadence.String("a39cd1e1bf7e2fb0609b7388ce5215a6a4c01eef2aee86e1a007faa28a6b2a3dc876e11bb97cdb26c3846231d2d01e4d"),
-			cadence.String("91673ad9c717d396c9a0953617733c128049ac1a639653d4002ab245b121df1939430e313bcbfd06948f6a281f6bf853"),
-		}).WithType(cadence.NewVariableSizedArrayType(cadence.StringType)),
-
-		// voterIDs
+		// aggregatedSignature
+		cadence.String("b072ed22ed305acd44818a6c836e09b4e844eebde6a4fdbf5cec983e2872b86c8b0f6c34c0777bf52e385ab7c45dc55d"),
+		// Node IDs of signers
 		cadence.NewArray([]cadence.Value{
 			cadence.String("0000000000000000000000000000000000000000000000000000000000000001"),
 			cadence.String("0000000000000000000000000000000000000000000000000000000000000002"),
@@ -818,13 +814,9 @@ func createEpochRecoverEvent(randomSourceHex string) cadence.Event {
 	}).WithType(clusterQCVoteDataType)
 
 	cluster2 := cadence.NewStruct([]cadence.Value{
-		// voteSignatures
-		cadence.NewArray([]cadence.Value{
-			cadence.String("b2bff159971852ed63e72c37991e62c94822e52d4fdcd7bf29aaf9fb178b1c5b4ce20dd9594e029f3574cb29533b857a"),
-			cadence.String("9931562f0248c9195758da3de4fb92f24fa734cbc20c0cb80280163560e0e0348f843ac89ecbd3732e335940c1e8dccb"),
-		}).WithType(cadence.NewVariableSizedArrayType(cadence.StringType)),
-
-		// voterIDs
+		// aggregatedSignature
+		cadence.String("899e266a543e1b3a564f68b22f7be571f2e944ec30fadc4b39e2d5f526ba044c0f3cb2648f8334fc216fa3360a0418b2"),
+		// Node IDs of signers
 		cadence.NewArray([]cadence.Value{
 			cadence.String("0000000000000000000000000000000000000000000000000000000000000003"),
 			cadence.String("0000000000000000000000000000000000000000000000000000000000000004"),
@@ -878,7 +870,9 @@ func createEpochRecoverEvent(randomSourceHex string) cadence.Event {
 
 		// clusterQCs
 		cadence.NewArray([]cadence.Value{
+			// cluster 1
 			cluster1,
+			// cluster 2
 			cluster2,
 		}).WithType(cadence.NewVariableSizedArrayType(clusterQCVoteDataType)),
 
@@ -1241,7 +1235,7 @@ func newFlowClusterQCClusterQCVoteDataStructType() *cadence.StructType {
 		Fields: []cadence.Field{
 			{
 				Identifier: "aggregatedSignature",
-				Type:       cadence.NewVariableSizedArrayType(cadence.StringType),
+				Type:       cadence.StringType,
 			},
 			{
 				Identifier: "voterIDs",

From 5a4cd701ebd897f6cde315dd5fc661f671115f52 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Wed, 17 Jul 2024 17:59:47 -0400
Subject: [PATCH 53/60] Update suite.go

---
 integration/tests/epochs/recover_epoch/suite.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 573acc8ea95..21b2226d3e4 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -4,10 +4,11 @@ import (
 	"fmt"
 
 	"github.com/stretchr/testify/require"
-	
+
 	"github.com/onflow/cadence"
 	"github.com/onflow/flow-core-contracts/lib/go/templates"
 	sdk "github.com/onflow/flow-go-sdk"
+
 	"github.com/onflow/flow-go/cmd/bootstrap/run"
 	"github.com/onflow/flow-go/integration/tests/epochs"
 	"github.com/onflow/flow-go/integration/utils"

From 1eb81e8252640b309e95611ba63960723ed4261d Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Wed, 17 Jul 2024 18:13:04 -0400
Subject: [PATCH 54/60] Update suite.go

---
 integration/tests/epochs/recover_epoch/suite.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 21b2226d3e4..4bf0d1dda19 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -3,11 +3,9 @@ package recover_epoch
 import (
 	"fmt"
 
-	"github.com/stretchr/testify/require"
-
 	"github.com/onflow/cadence"
-	"github.com/onflow/flow-core-contracts/lib/go/templates"
 	sdk "github.com/onflow/flow-go-sdk"
+	"github.com/stretchr/testify/require"
 
 	"github.com/onflow/flow-go/cmd/bootstrap/run"
 	"github.com/onflow/flow-go/integration/tests/epochs"

From c60abe33d109d2a4161d32b32ea34af9f96cb3ae Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Wed, 17 Jul 2024 19:10:13 -0400
Subject: [PATCH 55/60] Update suite.go

---
 integration/tests/epochs/recover_epoch/suite.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 4bf0d1dda19..1606efaaa3f 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/onflow/cadence"
+	"github.com/onflow/flow-core-contracts/lib/go/templates"
 	sdk "github.com/onflow/flow-go-sdk"
 	"github.com/stretchr/testify/require"
 

From b51a73f8af08e1ee13b949aef21e0fc6c60d6a8c Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Thu, 18 Jul 2024 07:20:43 -0400
Subject: [PATCH 56/60] Update suite.go

---
 integration/tests/epochs/recover_epoch/suite.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 1606efaaa3f..21b2226d3e4 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -3,10 +3,11 @@ package recover_epoch
 import (
 	"fmt"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/onflow/cadence"
 	"github.com/onflow/flow-core-contracts/lib/go/templates"
 	sdk "github.com/onflow/flow-go-sdk"
-	"github.com/stretchr/testify/require"
 
 	"github.com/onflow/flow-go/cmd/bootstrap/run"
 	"github.com/onflow/flow-go/integration/tests/epochs"

From 2ae7bd7beca1e967beb5445ab22e26df8a657280 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Thu, 18 Jul 2024 07:22:08 -0400
Subject: [PATCH 57/60] Update suite.go

---
 integration/tests/epochs/recover_epoch/suite.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 21b2226d3e4..a2d2c858550 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/onflow/cadence"
 	"github.com/onflow/flow-core-contracts/lib/go/templates"
+	
 	sdk "github.com/onflow/flow-go-sdk"
 
 	"github.com/onflow/flow-go/cmd/bootstrap/run"

From 5b92090a3be3f214e1d5bc174a77160b2e3ddb34 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Thu, 18 Jul 2024 07:28:10 -0400
Subject: [PATCH 58/60] Update suite.go

---
 integration/tests/epochs/recover_epoch/suite.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index a2d2c858550..4bf0d1dda19 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -3,12 +3,9 @@ package recover_epoch
 import (
 	"fmt"
 
-	"github.com/stretchr/testify/require"
-
 	"github.com/onflow/cadence"
-	"github.com/onflow/flow-core-contracts/lib/go/templates"
-	
 	sdk "github.com/onflow/flow-go-sdk"
+	"github.com/stretchr/testify/require"
 
 	"github.com/onflow/flow-go/cmd/bootstrap/run"
 	"github.com/onflow/flow-go/integration/tests/epochs"

From 34a49bfb51448de9e0e26f49f7bd8845ff4094af Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Thu, 18 Jul 2024 07:34:36 -0400
Subject: [PATCH 59/60] Update suite.go

---
 integration/tests/epochs/recover_epoch/suite.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/integration/tests/epochs/recover_epoch/suite.go b/integration/tests/epochs/recover_epoch/suite.go
index 4bf0d1dda19..9b00b0345e4 100644
--- a/integration/tests/epochs/recover_epoch/suite.go
+++ b/integration/tests/epochs/recover_epoch/suite.go
@@ -4,7 +4,10 @@ import (
 	"fmt"
 
 	"github.com/onflow/cadence"
+	"github.com/onflow/flow-core-contracts/lib/go/templates"
+
 	sdk "github.com/onflow/flow-go-sdk"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/onflow/flow-go/cmd/bootstrap/run"

From 1d2e03447e5b7b9806e39cbea30e55a96befc2c7 Mon Sep 17 00:00:00 2001
From: Khalil Claybon <khalil.claybon@dapperlabs.com>
Date: Fri, 19 Jul 2024 10:35:46 -0400
Subject: [PATCH 60/60] Update base_suite.go

---
 integration/tests/epochs/base_suite.go | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/integration/tests/epochs/base_suite.go b/integration/tests/epochs/base_suite.go
index d478672b9b9..fc6e3c13710 100644
--- a/integration/tests/epochs/base_suite.go
+++ b/integration/tests/epochs/base_suite.go
@@ -67,6 +67,11 @@ func (s *BaseSuite) SetupTest() {
 		s.Log.Info().Msg("================> Finish SetupTest")
 	}()
 
+	accessConfig := []func(*testnet.NodeConfig){
+		testnet.WithLogLevel(zerolog.WarnLevel),
+		testnet.WithAdditionalFlag("--supports-observer=true"),
+	}
+
 	collectionConfigs := []func(*testnet.NodeConfig){
 		testnet.WithAdditionalFlag("--hotstuff-proposal-duration=100ms"),
 		testnet.WithLogLevel(zerolog.WarnLevel)}
@@ -76,7 +81,7 @@ func (s *BaseSuite) SetupTest() {
 		testnet.WithAdditionalFlag("--cruise-ctl-enabled=false"), // disable cruise control for integration tests
 		testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", s.RequiredSealApprovals)),
 		testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", s.RequiredSealApprovals)),
-		testnet.WithLogLevel(zerolog.ErrorLevel)}
+		testnet.WithLogLevel(zerolog.WarnLevel)}
 
 	// a ghost node masquerading as an access node
 	s.ghostID = unittest.IdentifierFixture()
@@ -87,14 +92,14 @@ func (s *BaseSuite) SetupTest() {
 		testnet.AsGhost())
 
 	confs := []testnet.NodeConfig{
-		testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.ErrorLevel)),
-		testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.ErrorLevel)),
+		testnet.NewNodeConfig(flow.RoleAccess, accessConfig...),
+		testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.WarnLevel)),
 		testnet.NewNodeConfig(flow.RoleCollection, collectionConfigs...),
 		testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...),
 		testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...),
-		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.ErrorLevel), testnet.WithAdditionalFlag("--extensive-logging=true")),
-		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.ErrorLevel)),
-		testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.ErrorLevel)),
+		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.WarnLevel), testnet.WithAdditionalFlag("--extensive-logging=true")),
+		testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.WarnLevel)),
+		testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.WarnLevel)),
 		ghostNode,
 	}
 
@@ -115,7 +120,7 @@ func (s *BaseSuite) SetupTest() {
 
 	s.Client = client
 
-	// Log network info periodically to aid in debugging future flaky tests
+	// log network info periodically to aid in debugging future flaky tests
 	go lib.LogStatusPeriodically(s.T(), s.Ctx, s.Log, s.Client, 5*time.Second)
 }