-
Notifications
You must be signed in to change notification settings - Fork 535
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Update HasPending logic to account for out-of-order errors (#762)
- Loading branch information
Showing
8 changed files
with
429 additions
and
199 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| // Package gooseutil provides utility functions we want to keep internal to the package. It's | ||
| // intended to be a collection of well-tested helper functions. | ||
| package gooseutil | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "math" | ||
| "sort" | ||
| "strconv" | ||
| "strings" | ||
| ) | ||
|
|
||
| // UpVersions returns a list of migrations to apply based on the versions in the filesystem and the | ||
| // versions in the database. The target version can be used to specify a target version. In most | ||
| // cases this will be math.MaxInt64. | ||
| // | ||
| // The allowMissing flag can be used to allow missing migrations as part of the list of migrations | ||
| // to apply. Otherwise, an error will be returned if there are missing migrations in the database. | ||
| func UpVersions( | ||
| fsysVersions []int64, | ||
| dbVersions []int64, | ||
| target int64, | ||
| allowMissing bool, | ||
| ) ([]int64, error) { | ||
| // Sort the list of versions in the filesystem. This should already be sorted, but we do this | ||
| // just in case. | ||
| sortAscending(fsysVersions) | ||
|
|
||
| // dbAppliedVersions is a map of all applied migrations in the database. | ||
| dbAppliedVersions := make(map[int64]bool, len(dbVersions)) | ||
| var dbMaxVersion int64 | ||
| for _, v := range dbVersions { | ||
| dbAppliedVersions[v] = true | ||
| if v > dbMaxVersion { | ||
| dbMaxVersion = v | ||
| } | ||
| } | ||
|
|
||
| // Get a list of migrations that are missing from the database. A missing migration is one that | ||
| // has a version less than the max version in the database and has not been applied. | ||
| // | ||
| // In most cases the target version is math.MaxInt64, but it can be used to specify a target | ||
| // version. In which case we respect the target version and only surface migrations up to and | ||
| // including that target. | ||
| var missing []int64 | ||
| for _, v := range fsysVersions { | ||
| if dbAppliedVersions[v] { | ||
| continue | ||
| } | ||
| if v < dbMaxVersion && v <= target { | ||
| missing = append(missing, v) | ||
| } | ||
| } | ||
|
|
||
| // feat(mf): It is very possible someone may want to apply ONLY new migrations and skip missing | ||
| // migrations entirely. At the moment this is not supported, but leaving this comment because | ||
| // that's where that logic would be handled. | ||
| // | ||
| // For example, if database has 1,4 already applied and 2,3,5 are new, we would apply only 5 and | ||
| // skip 2,3. Not sure if this is a common use case, but it's possible someone may want to do | ||
| // this. | ||
| if len(missing) > 0 && !allowMissing { | ||
| return nil, newMissingError(missing, dbMaxVersion, target) | ||
| } | ||
|
|
||
| var out []int64 | ||
|
|
||
| // 1. Add missing migrations to the list of migrations to apply, if any. | ||
| out = append(out, missing...) | ||
|
|
||
| // 2. Add new migrations to the list of migrations to apply, if any. | ||
| for _, v := range fsysVersions { | ||
| if dbAppliedVersions[v] { | ||
| continue | ||
| } | ||
| if v > dbMaxVersion && v <= target { | ||
| out = append(out, v) | ||
| } | ||
| } | ||
| // 3. Sort the list of migrations to apply. | ||
| sortAscending(out) | ||
|
|
||
| return out, nil | ||
| } | ||
|
|
||
| func newMissingError( | ||
| missing []int64, | ||
| dbMaxVersion int64, | ||
| target int64, | ||
| ) error { | ||
| sortAscending(missing) | ||
|
|
||
| collected := make([]string, 0, len(missing)) | ||
| for _, v := range missing { | ||
| collected = append(collected, strconv.FormatInt(v, 10)) | ||
| } | ||
|
|
||
| msg := "migration" | ||
| if len(collected) > 1 { | ||
| msg += "s" | ||
| } | ||
|
|
||
| var versionsMsg string | ||
| if len(collected) > 1 { | ||
| versionsMsg = "versions " + strings.Join(collected, ",") | ||
| } else { | ||
| versionsMsg = "version " + collected[0] | ||
| } | ||
|
|
||
| desiredMsg := fmt.Sprintf("database version (%d)", dbMaxVersion) | ||
| if target != math.MaxInt64 { | ||
| desiredMsg += fmt.Sprintf(", with target version (%d)", target) | ||
| } | ||
|
|
||
| return fmt.Errorf("detected %d missing (out-of-order) %s lower than %s: %s", | ||
| len(missing), msg, desiredMsg, versionsMsg, | ||
| ) | ||
| } | ||
|
|
||
| func sortAscending(versions []int64) { | ||
| sort.Slice(versions, func(i, j int) bool { | ||
| return versions[i] < versions[j] | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,204 @@ | ||
| package gooseutil | ||
|
|
||
| import ( | ||
| "math" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestResolveVersions(t *testing.T) { | ||
| t.Run("not_allow_missing", func(t *testing.T) { | ||
| // Nothing to apply nil | ||
| got, err := UpVersions(nil, nil, math.MaxInt64, false) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, len(got)) | ||
| // Nothing to apply empty | ||
| got, err = UpVersions([]int64{}, []int64{}, math.MaxInt64, false) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, len(got)) | ||
|
|
||
| // Nothing new | ||
| got, err = UpVersions([]int64{1, 2, 3}, []int64{1, 2, 3}, math.MaxInt64, false) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, len(got)) | ||
|
|
||
| // All new | ||
| got, err = UpVersions([]int64{1, 2, 3}, []int64{}, math.MaxInt64, false) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 3, len(got)) | ||
| require.Equal(t, int64(1), got[0]) | ||
| require.Equal(t, int64(2), got[1]) | ||
| require.Equal(t, int64(3), got[2]) | ||
|
|
||
| // Squashed, no new | ||
| got, err = UpVersions([]int64{3}, []int64{3}, math.MaxInt64, false) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, len(got)) | ||
| // Squashed, 1 new | ||
| got, err = UpVersions([]int64{3, 4}, []int64{3}, math.MaxInt64, false) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 1, len(got)) | ||
| require.Equal(t, int64(4), got[0]) | ||
|
|
||
| // Some new with target | ||
| got, err = UpVersions([]int64{1, 2, 3, 4, 5}, []int64{1, 2}, 4, false) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 2, len(got)) | ||
| require.Equal(t, int64(3), got[0]) | ||
| require.Equal(t, int64(4), got[1]) // up to and including target | ||
| // Some new with zero target | ||
| got, err = UpVersions([]int64{1, 2, 3, 4, 5}, []int64{1, 2}, 0, false) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, len(got)) | ||
|
|
||
| // Error: one missing migrations with max target | ||
| _, err = UpVersions([]int64{1, 2, 3, 4}, []int64{1 /* 2*/, 3}, math.MaxInt64, false) | ||
| require.Error(t, err) | ||
| require.Equal(t, | ||
| "detected 1 missing (out-of-order) migration lower than database version (3): version 2", | ||
| err.Error(), | ||
| ) | ||
|
|
||
| // Error: multiple missing migrations with max target | ||
| _, err = UpVersions([]int64{1, 2, 3, 4, 5}, []int64{ /* 1 */ 2 /* 3 */, 4, 5}, math.MaxInt64, false) | ||
| require.Error(t, err) | ||
| require.Equal(t, | ||
| "detected 2 missing (out-of-order) migrations lower than database version (5): versions 1,3", | ||
| err.Error(), | ||
| ) | ||
|
|
||
| t.Run("target_lower_than_max", func(t *testing.T) { | ||
|
|
||
| // These tests are a bit of an edge case but an important one worth documenting. There | ||
| // can be missing migrations above and/or below the target version which itself can be | ||
| // lower than the max db version. For example, migrations 1,2,3,4 in the filesystem, and | ||
| // migrations 1,2,4 applied to the database and the user requested target 2. Technically | ||
| // there are no missing migrations based on the target version since 1,2 have been | ||
| // applied, but there is 1 missing migration (3) based on the max db version. Should | ||
| // this return an error, or report no pending migrations? | ||
| // | ||
| // We've taken the stance that this SHOULD respect the target version and surface an | ||
| // error if there are missing migrations below the target version. This is because the | ||
| // user has explicitly requested a target version and we should respect that. | ||
|
|
||
| got, err = UpVersions([]int64{1, 2, 3}, []int64{1 /* 2 */, 3}, 1, false) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, len(got)) | ||
| got, err = UpVersions([]int64{1, 2, 3}, []int64{1 /* 2 */, 3}, 2, false) | ||
| require.Error(t, err) | ||
| require.Equal(t, | ||
| "detected 1 missing (out-of-order) migration lower than database version (3), with target version (2): version 2", | ||
| err.Error(), | ||
| ) | ||
| got, err = UpVersions([]int64{1, 2, 3}, []int64{1 /* 2 */, 3}, 3, false) | ||
| require.Error(t, err) | ||
| require.Equal(t, | ||
| "detected 1 missing (out-of-order) migration lower than database version (3), with target version (3): version 2", | ||
| err.Error(), | ||
| ) | ||
|
|
||
| _, err = UpVersions([]int64{1, 2, 3, 4, 5, 6}, []int64{1 /* 2 */, 3, 4 /* 5*/, 6}, 4, false) | ||
| require.Error(t, err) | ||
| require.Equal(t, | ||
| "detected 1 missing (out-of-order) migration lower than database version (6), with target version (4): version 2", | ||
| err.Error(), | ||
| ) | ||
| _, err = UpVersions([]int64{1, 2, 3, 4, 5, 6}, []int64{1 /* 2 */, 3, 4 /* 5*/, 6}, 6, false) | ||
| require.Error(t, err) | ||
| require.Equal(t, | ||
| "detected 2 missing (out-of-order) migrations lower than database version (6), with target version (6): versions 2,5", | ||
| err.Error(), | ||
| ) | ||
| }) | ||
| }) | ||
|
|
||
| t.Run("allow_missing", func(t *testing.T) { | ||
| // Nothing to apply nil | ||
| got, err := UpVersions(nil, nil, math.MaxInt64, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, len(got)) | ||
| // Nothing to apply empty | ||
| got, err = UpVersions([]int64{}, []int64{}, math.MaxInt64, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, len(got)) | ||
|
|
||
| // Nothing new | ||
| got, err = UpVersions([]int64{1, 2, 3}, []int64{1, 2, 3}, math.MaxInt64, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, len(got)) | ||
|
|
||
| // All new | ||
| got, err = UpVersions([]int64{1, 2, 3}, []int64{}, math.MaxInt64, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 3, len(got)) | ||
| require.Equal(t, int64(1), got[0]) | ||
| require.Equal(t, int64(2), got[1]) | ||
| require.Equal(t, int64(3), got[2]) | ||
|
|
||
| // Squashed, no new | ||
| got, err = UpVersions([]int64{3}, []int64{3}, math.MaxInt64, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, len(got)) | ||
| // Squashed, 1 new | ||
| got, err = UpVersions([]int64{3, 4}, []int64{3}, math.MaxInt64, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 1, len(got)) | ||
| require.Equal(t, int64(4), got[0]) | ||
|
|
||
| // Some new with target | ||
| got, err = UpVersions([]int64{1, 2, 3, 4, 5}, []int64{1, 2}, 4, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 2, len(got)) | ||
| require.Equal(t, int64(3), got[0]) | ||
| require.Equal(t, int64(4), got[1]) // up to and including target | ||
| // Some new with zero target | ||
| got, err = UpVersions([]int64{1, 2, 3, 4, 5}, []int64{1, 2}, 0, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, len(got)) | ||
|
|
||
| // No error: one missing | ||
| got, err = UpVersions([]int64{1, 2, 3}, []int64{1 /* 2*/, 3}, math.MaxInt64, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 1, len(got)) | ||
| require.Equal(t, int64(2), got[0]) // missing | ||
|
|
||
| // No error: multiple missing and new with max target | ||
| got, err = UpVersions([]int64{1, 2, 3, 4, 5}, []int64{ /* 1 */ 2 /* 3 */, 4}, math.MaxInt64, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 3, len(got)) | ||
| require.Equal(t, int64(1), got[0]) // missing | ||
| require.Equal(t, int64(3), got[1]) // missing | ||
| require.Equal(t, int64(5), got[2]) | ||
|
|
||
| t.Run("target_lower_than_max", func(t *testing.T) { | ||
| got, err = UpVersions([]int64{1, 2, 3}, []int64{1 /* 2 */, 3}, 1, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 0, len(got)) | ||
| got, err = UpVersions([]int64{1, 2, 3}, []int64{1 /* 2 */, 3}, 2, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 1, len(got)) | ||
| require.Equal(t, int64(2), got[0]) // missing | ||
| got, err = UpVersions([]int64{1, 2, 3}, []int64{1 /* 2 */, 3}, 3, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 1, len(got)) | ||
| require.Equal(t, int64(2), got[0]) // missing | ||
|
|
||
| got, err = UpVersions([]int64{1, 2, 3, 4, 5, 6}, []int64{1 /* 2 */, 3, 4 /* 5*/, 6}, 4, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 1, len(got)) | ||
| require.Equal(t, int64(2), got[0]) // missing | ||
| got, err = UpVersions([]int64{1, 2, 3, 4, 5, 6}, []int64{1 /* 2 */, 3, 4 /* 5*/, 6}, 6, true) | ||
| require.NoError(t, err) | ||
| require.Equal(t, 2, len(got)) | ||
| require.Equal(t, int64(2), got[0]) // missing | ||
| require.Equal(t, int64(5), got[1]) // missing | ||
| }) | ||
| }) | ||
|
|
||
| t.Run("sort_ascending", func(t *testing.T) { | ||
| got := []int64{5, 3, 4, 2, 1} | ||
| sortAscending(got) | ||
| require.Equal(t, []int64{1, 2, 3, 4, 5}, got) | ||
| }) | ||
| } |
Oops, something went wrong.