Merge pull request #4633 from onflow/janez/atree-migration

Atree storage migration

cmd/util/cmd/execution-state-extract/cmd.go

@@ -15,15 +15,17 @@ import (
 )
 
 var (
-	flagExecutionStateDir string
-	flagOutputDir         string
-	flagBlockHash         string
-	flagStateCommitment   string
-	flagDatadir           string
-	flagChain             string
-	flagNoMigration       bool
-	flagNoReport          bool
-	flagNWorker           int
+	flagExecutionStateDir         string
+	flagOutputDir                 string
+	flagBlockHash                 string
+	flagStateCommitment           string
+	flagDatadir                   string
+	flagChain                     string
+	flagNWorker                   int
+	flagNoMigration               bool
+	flagNoReport                  bool
+	flagValidateMigration         bool
+	flagLogVerboseValidationError bool
 )
 
 var Cmd = &cobra.Command{
@@ -59,6 +61,13 @@ func init() {
 		"don't report the state")
 
 	Cmd.Flags().IntVar(&flagNWorker, "n-migrate-worker", 10, "number of workers to migrate payload concurrently")
+
+	Cmd.Flags().BoolVar(&flagValidateMigration, "validate", false,
+		"validate migrated Cadence values (atree migration)")
+
+	Cmd.Flags().BoolVar(&flagLogVerboseValidationError, "log-verbose-validation-error", false,
+		"log entire Cadence values on validation error (atree migration)")
+
 }
 
 func run(*cobra.Command, []string) {
@@ -131,16 +140,21 @@ func run(*cobra.Command, []string) {
 		log.Warn().Msgf("--no-report flag is deprecated")
 	}
 
-	if flagNoMigration {
-		log.Warn().Msgf("--no-migration flag is deprecated")
+	if flagValidateMigration {
+		log.Warn().Msgf("atree migration validation flag is enabled and will increase duration of migration")
+	}
+
+	if flagLogVerboseValidationError {
+		log.Warn().Msgf("atree migration has verbose validation error logging enabled which may increase size of log")
 	}
 
 	err := extractExecutionState(
+		log.Logger,
 		flagExecutionStateDir,
 		stateCommitment,
 		flagOutputDir,
-		log.Logger,
 		flagNWorker,
+		!flagNoMigration,
 	)
 
 	if err != nil {

cmd/util/cmd/execution-state-extract/execution_state_extract.go

@@ -9,6 +9,7 @@ import (
 	"github.com/rs/zerolog"
 	"go.uber.org/atomic"
 
+	migrators "github.com/onflow/flow-go/cmd/util/ledger/migrations"
 	"github.com/onflow/flow-go/cmd/util/ledger/reporters"
 	"github.com/onflow/flow-go/ledger"
 	"github.com/onflow/flow-go/ledger/common/hash"
@@ -27,11 +28,12 @@ func getStateCommitment(commits storage.Commits, blockHash flow.Identifier) (flo
 }
 
 func extractExecutionState(
+	log zerolog.Logger,
 	dir string,
 	targetHash flow.StateCommitment,
 	outputDir string,
-	log zerolog.Logger,
 	nWorker int, // number of concurrent worker to migation payloads
+	runMigrations bool,
 ) error {
 
 	log.Info().Msg("init WAL")
@@ -83,6 +85,30 @@ func extractExecutionState(
 	}()
 
 	var migrations []ledger.Migration
+
+	if runMigrations {
+		rwf := reporters.NewReportFileWriterFactory(dir, log)
+
+		migrations = []ledger.Migration{
+			migrators.CreateAccountBasedMigration(
+				log,
+				nWorker,
+				[]migrators.AccountBasedMigration{
+					migrators.NewAtreeRegisterMigrator(
+						rwf,
+						flagValidateMigration,
+						flagLogVerboseValidationError,
+					),
+
+					&migrators.DeduplicateContractNamesMigration{},
+
+					// This will fix storage used discrepancies caused by the
+					// DeduplicateContractNamesMigration.
+					&migrators.AccountUsageMigrator{},
+				}),
+		}
+	}
+
 	newState := ledger.State(targetHash)
 
 	// migrate the trie if there are migrations

cmd/util/cmd/execution-state-extract/execution_state_extract_test.go

@@ -60,11 +60,12 @@ func TestExtractExecutionState(t *testing.T) {
 	t.Run("empty WAL doesn't find anything", func(t *testing.T) {
 		withDirs(t, func(datadir, execdir, outdir string) {
 			err := extractExecutionState(
+				zerolog.Nop(),
 				execdir,
 				unittest.StateCommitmentFixture(),
 				outdir,
-				zerolog.Nop(),
 				10,
+				false,
 			)
 			require.Error(t, err)
 		})