Atree storage migration

cmd/util/cmd/execution-state-extract/cmd.go

@@ -15,15 +15,17 @@ import (
 )
 
 var (
-	flagExecutionStateDir string
-	flagOutputDir         string
-	flagBlockHash         string
-	flagStateCommitment   string
-	flagDatadir           string
-	flagChain             string
-	flagNoMigration       bool
-	flagNoReport          bool
-	flagNWorker           int
+	flagExecutionStateDir         string
+	flagOutputDir                 string
+	flagBlockHash                 string
+	flagStateCommitment           string
+	flagDatadir                   string
+	flagChain                     string
+	flagNWorker                   int
+	flagNoMigration               bool
+	flagNoReport                  bool
+	flagValidateMigration         bool
+	flagLogVerboseValidationError bool
 )
 
 var Cmd = &cobra.Command{
@@ -59,6 +61,13 @@ func init() {
 		"don't report the state")
 
 	Cmd.Flags().IntVar(&flagNWorker, "n-migrate-worker", 10, "number of workers to migrate payload concurrently")
+
+	Cmd.Flags().BoolVar(&flagValidateMigration, "validate", false,
+		"validate migrated Cadence values (atree migration)")
+
+	Cmd.Flags().BoolVar(&flagLogVerboseValidationError, "log-verbose-validation-error", false,
+		"log entire Cadence values on validation error (atree migration)")
+
 }
 
 func run(*cobra.Command, []string) {
@@ -135,6 +144,14 @@ func run(*cobra.Command, []string) {
 		log.Warn().Msgf("--no-migration flag is deprecated")
 	}
 
+	if flagValidateMigration {
+		log.Warn().Msgf("atree migration validation flag is enabled and will increase duration of migration")
+	}
+
+	if flagLogVerboseValidationError {
+		log.Warn().Msgf("atree migration has verbose validation error logging enabled which may increase size of log")
+	}
+
 	err := extractExecutionState(
 		flagExecutionStateDir,
 		stateCommitment,

cmd/util/cmd/execution-state-extract/execution_state_extract.go

@@ -9,6 +9,7 @@ import (
 	"github.com/rs/zerolog"
 	"go.uber.org/atomic"
 
+	migrators "github.com/onflow/flow-go/cmd/util/ledger/migrations"
 	"github.com/onflow/flow-go/cmd/util/ledger/reporters"
 	"github.com/onflow/flow-go/ledger"
 	"github.com/onflow/flow-go/ledger/common/hash"
@@ -82,7 +83,26 @@ func extractExecutionState(
 		<-compactor.Done()
 	}()
 
-	var migrations []ledger.Migration
+	rwf := reporters.NewReportFileWriterFactory(dir, log)
+
+	var migrations = []ledger.Migration{
+		migrators.CreateAccountBasedMigration(
+			log,
+			nWorker,
+			[]migrators.AccountBasedMigration{
+				migrators.NewAtreeRegisterMigrator(
+					rwf,
+					flagValidateMigration,
+					flagLogVerboseValidationError,
+				),
+
+				&migrators.DeduplicateContractNamesMigration{},
+
+				// This will fix storage used discrepancies caused by the
+				// DeduplicateContractNamesMigration.
+				&migrators.AccountUsageMigrator{},
+			}),
+	}
 	newState := ledger.State(targetHash)
 
 	// migrate the trie if there are migrations