Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add version, partial state indicator, and checksum to payload file (intermediate migration file) #5438

Merged
merged 5 commits into from
Feb 26, 2024
Merged

Add version, partial state indicator, and checksum to payload file (intermediate migration file) #5438

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e3401d2
Add version, flag, and checksum to payload file
fxamacker Feb 22, 2024
c5a937b
Merge branch 'master' into fxamacker/improve-payload-file
fxamacker Feb 22, 2024
847cd6e
Add more logging and minor refactoring
fxamacker Feb 23, 2024
fc709b9
Merge branch 'master' into fxamacker/improve-payload-file
fxamacker Feb 23, 2024
583caba
Merge branch 'master' into fxamacker/improve-payload-file
fxamacker Feb 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 42 additions & 14 deletions cmd/util/cmd/execution-state-extract/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,27 +13,29 @@ import (
runtimeCommon "github.com/onflow/cadence/runtime/common"

"github.com/onflow/flow-go/cmd/util/cmd/common"
"github.com/onflow/flow-go/cmd/util/ledger/util"
"github.com/onflow/flow-go/model/bootstrap"
"github.com/onflow/flow-go/model/flow"
"github.com/onflow/flow-go/module/metrics"
"github.com/onflow/flow-go/storage/badger"
)

var (
flagExecutionStateDir string
flagOutputDir string
flagBlockHash string
flagStateCommitment string
flagDatadir string
flagChain string
flagNWorker int
flagNoMigration bool
flagNoReport bool
flagValidateMigration bool
flagLogVerboseValidationError bool
flagInputPayloadFileName string
flagOutputPayloadFileName string
flagOutputPayloadByAddresses string
flagExecutionStateDir string
flagOutputDir string
flagBlockHash string
flagStateCommitment string
flagDatadir string
flagChain string
flagNWorker int
flagNoMigration bool
flagNoReport bool
flagValidateMigration bool
flagLogVerboseValidationError bool
flagAllowPartialStateFromPayloads bool
flagInputPayloadFileName string
flagOutputPayloadFileName string
flagOutputPayloadByAddresses string
)

var Cmd = &cobra.Command{
Expand Down Expand Up @@ -76,6 +78,9 @@ func init() {
Cmd.Flags().BoolVar(&flagLogVerboseValidationError, "log-verbose-validation-error", false,
"log entire Cadence values on validation error (atree migration)")

Cmd.Flags().BoolVar(&flagAllowPartialStateFromPayloads, "allow-partial-state-from-payload-file", false,
"allow input payload file containing partial state (e.g. not all accounts)")

// If specified, the state will consist of payloads from the given input payload file.
// If not specified, then the state will be extracted from the latest checkpoint file.
// This flag can be used to reduce total duration of migrations when state extraction involves
Expand Down Expand Up @@ -166,6 +171,29 @@ func run(*cobra.Command, []string) {
if _, err := os.Stat(flagInputPayloadFileName); os.IsNotExist(err) {
log.Fatal().Msgf("payload input file %s doesn't exist", flagInputPayloadFileName)
}

partialState, err := util.IsPayloadFilePartialState(flagInputPayloadFileName)
if err != nil {
log.Fatal().Err(err).Msgf("cannot get flag from payload input file %s", flagInputPayloadFileName)
}

// Check if payload file contains partial state and is allowed by --allow-partial-state-from-payload-file.
if !flagAllowPartialStateFromPayloads && partialState {
fxamacker marked this conversation as resolved.
Show resolved Hide resolved
log.Fatal().Msgf("payload input file %s contains partial state, please specify --allow-partial-state-from-payload-file", flagInputPayloadFileName)
}

msg := "input payloads represent "
if partialState {
msg += "partial state"
} else {
msg += "complete state"
}
if flagAllowPartialStateFromPayloads {
msg += ", and --allow-partial-state-from-payload-file is specified"
} else {
msg += ", and --allow-partial-state-from-payload-file is NOT specified"
}
log.Info().Msg(msg)
}

if len(flagOutputPayloadFileName) > 0 {
Expand Down
25 changes: 24 additions & 1 deletion cmd/util/cmd/execution-state-extract/execution_state_extract.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,11 +120,22 @@ func extractExecutionState(
if exportPayloads {
payloads := newTrie.AllPayloads()

log.Info().Msgf("sorting %d payloads", len(payloads))

// Sort payloads to produce deterministic payload file with
// same sequence of payloads inside.
payloads = util.SortPayloadsByAddress(payloads, nWorker)

log.Info().Msgf("sorted %d payloads", len(payloads))

log.Info().Msgf("creating payloads file %s", outputPayloadFile)

exportedPayloadCount, err := util.CreatePayloadFile(
log,
outputPayloadFile,
payloads,
exportPayloadsByAddresses,
false, // payloads represents entire state.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we create another CreatePayloadFileWithPartialStateFlag and make CreatePayloadFile to call CreatePayloadFileWithPartialStateFlag with partialStateFlag: false, so that we don't have to add this default value if not needed.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe not in this case because most of the non-test calls of CreatePayloadFile() uses variable returned from another function call (not hard-coded) to indicate if input payloads are for partial or full state. So If we split it into two functions, we need to check this boolean and decide which function to call at higher level.

)
if err != nil {
return fmt.Errorf("cannot generate payloads file: %w", err)
Expand Down Expand Up @@ -204,7 +215,7 @@ func extractExecutionStateFromPayloads(
exportPayloadsByAddresses []common.Address,
) error {

payloads, err := util.ReadPayloadFile(log, inputPayloadFile)
inputPayloadsFromPartialState, payloads, err := util.ReadPayloadFile(log, inputPayloadFile)
if err != nil {
return err
}
Expand All @@ -220,11 +231,23 @@ func extractExecutionStateFromPayloads(

exportPayloads := len(outputPayloadFile) > 0
if exportPayloads {

log.Info().Msgf("sorting %d payloads", len(payloads))

// Sort payloads to produce deterministic payload file with
// same sequence of payloads inside.
payloads = util.SortPayloadsByAddress(payloads, nWorker)
fxamacker marked this conversation as resolved.
Show resolved Hide resolved

log.Info().Msgf("sorted %d payloads", len(payloads))

log.Info().Msgf("creating payloads file %s", outputPayloadFile)

exportedPayloadCount, err := util.CreatePayloadFile(
log,
outputPayloadFile,
payloads,
exportPayloadsByAddresses,
inputPayloadsFromPartialState,
)
if err != nil {
return fmt.Errorf("cannot generate payloads file: %w", err)
Expand Down
80 changes: 77 additions & 3 deletions cmd/util/cmd/execution-state-extract/execution_state_extract_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,9 +286,10 @@ func TestExtractPayloadsFromExecutionState(t *testing.T) {
require.NoError(t, err)

// Verify exported payloads.
payloadsFromFile, err := util.ReadPayloadFile(zerolog.Nop(), outputPayloadFileName)
partialState, payloadsFromFile, err := util.ReadPayloadFile(zerolog.Nop(), outputPayloadFileName)
require.NoError(t, err)
require.Equal(t, len(keysValues), len(payloadsFromFile))
require.False(t, partialState)

for _, payloadFromFile := range payloadsFromFile {
k, err := payloadFromFile.Key()
Expand Down Expand Up @@ -392,9 +393,10 @@ func TestExtractPayloadsFromExecutionState(t *testing.T) {
require.NoError(t, err)

// Verify exported payloads.
payloadsFromFile, err := util.ReadPayloadFile(zerolog.Nop(), outputPayloadFileName)
partialState, payloadsFromFile, err := util.ReadPayloadFile(zerolog.Nop(), outputPayloadFileName)
require.NoError(t, err)
require.Equal(t, len(selectedKeysValues), len(payloadsFromFile))
require.True(t, partialState)

for _, payloadFromFile := range payloadsFromFile {
k, err := payloadFromFile.Key()
Expand Down Expand Up @@ -441,6 +443,7 @@ func TestExtractStateFromPayloads(t *testing.T) {
inputPayloadFileName,
payloads,
nil,
false,
)
require.NoError(t, err)
require.Equal(t, len(payloads), numOfPayloadWritten)
Expand Down Expand Up @@ -511,6 +514,7 @@ func TestExtractStateFromPayloads(t *testing.T) {
inputPayloadFileName,
payloads,
nil,
false,
)
require.NoError(t, err)
require.Equal(t, len(payloads), numOfPayloadWritten)
Expand All @@ -531,9 +535,79 @@ func TestExtractStateFromPayloads(t *testing.T) {
require.NoError(t, err)

// Verify exported payloads.
payloadsFromFile, err := util.ReadPayloadFile(zerolog.Nop(), outputPayloadFileName)
partialState, payloadsFromFile, err := util.ReadPayloadFile(zerolog.Nop(), outputPayloadFileName)
require.NoError(t, err)
require.Equal(t, len(keysValues), len(payloadsFromFile))
require.False(t, partialState)

for _, payloadFromFile := range payloadsFromFile {
k, err := payloadFromFile.Key()
require.NoError(t, err)

kv, exist := keysValues[k.String()]
require.True(t, exist)

require.Equal(t, kv.value, payloadFromFile.Value())
}
})
})

t.Run("input is partial state", func(t *testing.T) {
withDirs(t, func(_, execdir, outdir string) {
size := 10

inputPayloadFileName := filepath.Join(execdir, payloadFileName)
outputPayloadFileName := filepath.Join(outdir, "selected.payload")

// Generate some data
keysValues := make(map[string]keyPair)
var payloads []*ledger.Payload

for i := 0; i < size; i++ {
keys, values := getSampleKeyValues(i)

for j, key := range keys {
keysValues[key.String()] = keyPair{
key: key,
value: values[j],
}

payloads = append(payloads, ledger.NewPayload(key, values[j]))
}
}

// Create input payload file that represents partial state
numOfPayloadWritten, err := util.CreatePayloadFile(
zerolog.Nop(),
inputPayloadFileName,
payloads,
nil,
true,
)
require.NoError(t, err)
require.Equal(t, len(payloads), numOfPayloadWritten)

// Since input payload file is partial state, --allow-partial-state-from-payload-file must be specified.
Cmd.SetArgs([]string{
"--execution-state-dir", execdir,
"--output-dir", outdir,
"--no-migration",
"--no-report",
"--state-commitment", "",
"--input-payload-filename", inputPayloadFileName,
"--output-payload-filename", outputPayloadFileName,
"--extract-payloads-by-address", "",
"--allow-partial-state-from-payload-file",
"--chain", flow.Emulator.Chain().String()})

err = Cmd.Execute()
require.NoError(t, err)

// Verify exported payloads.
partialState, payloadsFromFile, err := util.ReadPayloadFile(zerolog.Nop(), outputPayloadFileName)
require.NoError(t, err)
require.Equal(t, len(keysValues), len(payloadsFromFile))
require.True(t, partialState)

for _, payloadFromFile := range payloadsFromFile {
k, err := payloadFromFile.Key()
Expand Down
Loading
Loading