From d9720bd25145c6e2b1b1a867b4eb49cdd779803d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 12 Jul 2018 00:17:33 +0200 Subject: [PATCH 1/3] repack: point out a bug handling stale shallow info A `git fetch --prune` can turn previously-reachable objects unreachable, even commits that are in the `shallow` list. A subsequent `git repack -ad` will then unceremoniously drop those unreachable commits, and the `shallow` list will become stale. This means that when we try to fetch with a larger `--depth` the next time, we may end up with: fatal: error in object: unshallow Reported by Alejandro Pauly. Signed-off-by: Johannes Schindelin --- t/t5537-fetch-shallow.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/t/t5537-fetch-shallow.sh b/t/t5537-fetch-shallow.sh index 7045685e2d3942..686fdc9280532c 100755 --- a/t/t5537-fetch-shallow.sh +++ b/t/t5537-fetch-shallow.sh @@ -186,6 +186,33 @@ EOF test_cmp expect actual ' +test_expect_failure '.git/shallow is edited by repack' ' + git init shallow-server && + test_commit -C shallow-server A && + test_commit -C shallow-server B && + git -C shallow-server checkout -b branch && + test_commit -C shallow-server C && + test_commit -C shallow-server E && + test_commit -C shallow-server D && + d="$(git -C shallow-server rev-parse --verify D^0)" && + git -C shallow-server checkout master && + + git clone --depth=1 --no-tags --no-single-branch \ + "file://$PWD/shallow-server" shallow-client && + + : now remove the branch and fetch with prune && + git -C shallow-server branch -D branch && + git -C shallow-client fetch --prune --depth=1 \ + origin "+refs/heads/*:refs/remotes/origin/*" && + git -C shallow-client repack -adfl && + test_must_fail git -C shallow-client rev-parse --verify $d^0 && + ! grep $d shallow-client/.git/shallow && + + git -C shallow-server branch branch-orig $d && + git -C shallow-client fetch --prune --depth=2 \ + origin "+refs/heads/*:refs/remotes/origin/*" +' + . "$TEST_DIRECTORY"/lib-httpd.sh start_httpd From 18308c13eee4568dbda39fbf6567666bf4b98eeb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 22 Oct 2018 16:22:12 +0200 Subject: [PATCH 2/3] shallow: offer to prune only non-existing entries The `prune_shallow()` function wants a full reachability check to be completed before it goes to work, to ensure that all unreachable entries are removed from the shallow file. However, in the upcoming patch we do not even want to go that far. We really only need to remove entries corresponding to pruned commits, i.e. to commits that no longer exist. Let's support that use case. Rather than extending the signature of `prune_shallow()` to accept another Boolean, let's turn it into a bit field and declare constants, for readability. Signed-off-by: Johannes Schindelin --- builtin/prune.c | 2 +- commit.h | 4 +++- shallow.c | 23 +++++++++++++++++------ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/builtin/prune.c b/builtin/prune.c index 41230f82157e48..1ec9ddd751df66 100644 --- a/builtin/prune.c +++ b/builtin/prune.c @@ -161,7 +161,7 @@ int cmd_prune(int argc, const char **argv, const char *prefix) free(s); if (is_repository_shallow(the_repository)) - prune_shallow(show_only); + prune_shallow(show_only ? PRUNE_SHOW_ONLY : 0); return 0; } diff --git a/commit.h b/commit.h index 1d260d62f57a24..b80d6fdb5cd65e 100644 --- a/commit.h +++ b/commit.h @@ -249,7 +249,9 @@ extern void assign_shallow_commits_to_refs(struct shallow_info *info, uint32_t **used, int *ref_status); extern int delayed_reachability_test(struct shallow_info *si, int c); -extern void prune_shallow(int show_only); +#define PRUNE_SHOW_ONLY 1 +#define PRUNE_QUICK 2 +extern void prune_shallow(unsigned options); extern struct trace_key trace_shallow; extern int interactive_add(int argc, const char **argv, const char *prefix, int patch); diff --git a/shallow.c b/shallow.c index 732e18d54f3b88..02fdbfc554c462 100644 --- a/shallow.c +++ b/shallow.c @@ -247,6 +247,7 @@ static void check_shallow_file_for_update(struct repository *r) #define SEEN_ONLY 1 #define VERBOSE 2 +#define QUICK 4 struct write_shallow_data { struct strbuf *out; @@ -261,7 +262,10 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data) const char *hex = oid_to_hex(&graft->oid); if (graft->nr_parent != -1) return 0; - if (data->flags & SEEN_ONLY) { + if (data->flags & QUICK) { + if (!has_object_file(&graft->oid)) + return 0; + } else if (data->flags & SEEN_ONLY) { struct commit *c = lookup_commit(the_repository, &graft->oid); if (!c || !(c->object.flags & SEEN)) { if (data->flags & VERBOSE) @@ -371,16 +375,23 @@ void advertise_shallow_grafts(int fd) /* * mark_reachable_objects() should have been run prior to this and all - * reachable commits marked as "SEEN". + * reachable commits marked as "SEEN", except when quick_prune is non-zero, + * in which case lines are excised from the shallow file if they refer to + * commits that do not exist (any longer). */ -void prune_shallow(int show_only) +void prune_shallow(unsigned options) { struct lock_file shallow_lock = LOCK_INIT; struct strbuf sb = STRBUF_INIT; + unsigned flags = SEEN_ONLY; int fd; - if (show_only) { - write_shallow_commits_1(&sb, 0, NULL, SEEN_ONLY | VERBOSE); + if (options & PRUNE_QUICK) + flags |= QUICK; + + if (options & PRUNE_SHOW_ONLY) { + flags |= VERBOSE; + write_shallow_commits_1(&sb, 0, NULL, flags); strbuf_release(&sb); return; } @@ -388,7 +399,7 @@ void prune_shallow(int show_only) git_path_shallow(the_repository), LOCK_DIE_ON_ERROR); check_shallow_file_for_update(the_repository); - if (write_shallow_commits_1(&sb, 0, NULL, SEEN_ONLY)) { + if (write_shallow_commits_1(&sb, 0, NULL, flags)) { if (write_in_full(fd, sb.buf, sb.len) < 0) die_errno("failed to write to %s", get_lock_file_path(&shallow_lock)); From 2858bc886ce2c687092f4b93b211ebcdf6281a22 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 12 Jul 2018 00:23:28 +0200 Subject: [PATCH 3/3] repack -ad: prune the list of shallow commits `git repack` can drop unreachable commits without further warning, making the corresponding entries in `.git/shallow` invalid, which causes serious problems when deepening the branches. One scenario where unreachable commits are dropped by `git repack` is when a `git fetch --prune` (or even a `git fetch` when a ref was force-pushed in the meantime) can make a commit unreachable that was reachable before. Therefore it is not safe to assume that a `git repack -adlf` will keep unreachable commits alone (under the assumption that they had not been packed in the first place, which is an assumption at least some of Git's code seems to make). This is particularly important to keep in mind when looking at the `.git/shallow` file: if any commits listed in that file become unreachable, it is not a problem, but if they go missing, it *is* a problem. One symptom of this problem is that a deepening fetch may now fail with fatal: error in object: unshallow To avoid this problem, let's prune the shallow list in `git repack` when the `-d` option is passed, unless `-A` is passed, too (which would force the now-unreachable objects to be turned into loose objects instead of being deleted). Additionally, we also need to take `--keep-reachable` and `--unpack-unreachable=` into account. Note: an alternative solution discussed during the review of this patch was to teach `git fetch` to simply ignore entries in .git/shallow if the corresponding commits do not exist locally. A quick test, however, revealed that the .git/shallow file is written during a shallow *clone*, in which case the commits do not exist, either, but the "shallow" line *does* need to be sent. Therefore, this approach would be a lot more finicky than the approach presented by the this patch. Signed-off-by: Johannes Schindelin --- builtin/repack.c | 6 ++++++ t/t5537-fetch-shallow.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/builtin/repack.c b/builtin/repack.c index c6a7943d5cb108..acd43c75d75a35 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -549,6 +549,12 @@ int cmd_repack(int argc, const char **argv, const char *prefix) if (!po_args.quiet && isatty(2)) opts |= PRUNE_PACKED_VERBOSE; prune_packed_objects(opts); + + if (!keep_unreachable && + (!(pack_everything & LOOSEN_UNREACHABLE) || + unpack_unreachable) && + is_repository_shallow(the_repository)) + prune_shallow(PRUNE_QUICK); } if (!no_update_server_info) diff --git a/t/t5537-fetch-shallow.sh b/t/t5537-fetch-shallow.sh index 686fdc9280532c..6faf17e17a133f 100755 --- a/t/t5537-fetch-shallow.sh +++ b/t/t5537-fetch-shallow.sh @@ -186,7 +186,7 @@ EOF test_cmp expect actual ' -test_expect_failure '.git/shallow is edited by repack' ' +test_expect_success '.git/shallow is edited by repack' ' git init shallow-server && test_commit -C shallow-server A && test_commit -C shallow-server B &&