From d5ff2d24aeb074b0e255c9b294994d16a18636ae Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 5 Apr 2017 10:58:09 -0600 Subject: [PATCH 01/13] reset --stdin: trim carriage return from the paths While using the reset --stdin feature on windows path added may have a \r at the end of the path that wasn't getting removed so didn't match the path in the index and wasn't reset. Signed-off-by: Kevin Willford --- t/t7108-reset-stdin.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/t/t7108-reset-stdin.sh b/t/t7108-reset-stdin.sh index b7cbcbf869296c..db5483b8f10052 100755 --- a/t/t7108-reset-stdin.sh +++ b/t/t7108-reset-stdin.sh @@ -29,4 +29,13 @@ test_expect_success '--stdin requires --mixed' ' git reset --mixed --stdin list && + git reset --stdin Date: Tue, 23 Apr 2024 12:21:01 +0200 Subject: [PATCH 02/13] Identify microsoft/git via a distinct version suffix It has been a long-standing practice in Git for Windows to append `.windows.`, and in microsoft/git to append `.vfs.0.0`. Let's keep doing that. Signed-off-by: Johannes Schindelin --- GIT-VERSION-GEN | 3 +++ 1 file changed, 3 insertions(+) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index 55850912d13034..7ca2fcf019bf80 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -3,6 +3,9 @@ GVF=GIT-VERSION-FILE DEF_VER=v2.46.1 +# Identify microsoft/git via a distinct version suffix +DEF_VER=$DEF_VER.vfs.0.0 + LF=' ' From 81e5a4c6546efc8ed1628ae27d511db1f9e090b0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 4 Apr 2017 12:04:11 +0200 Subject: [PATCH 03/13] gvfs: ensure that the version is based on a GVFS tag Since we really want to be based on a `.vfs.*` tag, let's make sure that there was a new-enough one, i.e. one that agrees with the first three version numbers of the recorded default version. This prevents e.g. v2.22.0.vfs.0.. from being used when the current release train was not yet tagged. It is important to get the first three numbers of the version right because e.g. Scalar makes decisions depending on those (such as assuming that the `git maintenance` built-in is not available, even though it actually _is_ available). Signed-off-by: Johannes Schindelin --- GIT-VERSION-GEN | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index 7ca2fcf019bf80..a7081ebdcc989f 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -15,10 +15,15 @@ if test -f version then VN=$(cat version) || VN="$DEF_VER" elif { test -d "${GIT_DIR:-.git}" || test -f .git; } && - VN=$(git describe --match "v[0-9]*" HEAD 2>/dev/null) && + VN=$(git describe --match "v[0-9]*vfs*" HEAD 2>/dev/null) && case "$VN" in *$LF*) (exit 1) ;; v[0-9]*) + if test "${VN%%.vfs.*}" != "${DEF_VER%%.vfs.*}" + then + echo "Found version $VN, which is not based on $DEF_VER" >&2 + exit 1 + fi git update-index -q --refresh test -z "$(git diff-index --name-only HEAD --)" || VN="$VN-dirty" ;; From a441c1ff33eb609954b441e5864201f523f6de30 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:30:59 +0100 Subject: [PATCH 04/13] gvfs: add a GVFS-specific header file This header file will accumulate GVFS-specific definitions. Signed-off-by: Kevin Willford --- gvfs.h | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 gvfs.h diff --git a/gvfs.h b/gvfs.h new file mode 100644 index 00000000000000..b6dbe85eae4071 --- /dev/null +++ b/gvfs.h @@ -0,0 +1,9 @@ +#ifndef GVFS_H +#define GVFS_H + +/* + * This file is for the specific settings and methods + * used for GVFS functionality + */ + +#endif /* GVFS_H */ From 4cfd73a381d750e0fb31e8d71dc318d648f728cb Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:34:12 +0100 Subject: [PATCH 05/13] gvfs: add the core.gvfs config setting This does not do anything yet. The next patches will add various values for that config setting that correspond to the various features offered/required by GVFS. Signed-off-by: Kevin Willford gvfs: refactor loading the core.gvfs config value This code change makes sure that the config value for core_gvfs is always loaded before checking it. Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 3 +++ Makefile | 1 + config.c | 6 +++++ environment.c | 1 + environment.h | 1 + gvfs.c | 45 +++++++++++++++++++++++++++++++++++ gvfs.h | 4 ++++ 7 files changed, 61 insertions(+) create mode 100644 gvfs.c diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 458a3f4f7f5d80..3f3343c7cbd982 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -743,6 +743,9 @@ core.multiPackIndex:: single index. See linkgit:git-multi-pack-index[1] for more information. Defaults to true. +core.gvfs:: + Enable the features needed for GVFS. + core.sparseCheckout:: Enable "sparse checkout" feature. See linkgit:git-sparse-checkout[1] for more information. diff --git a/Makefile b/Makefile index 4ed38a08d0b2a2..24811a49aa107d 100644 --- a/Makefile +++ b/Makefile @@ -1033,6 +1033,7 @@ LIB_OBJS += git-zlib.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o LIB_OBJS += grep.o +LIB_OBJS += gvfs.o LIB_OBJS += hash-lookup.o LIB_OBJS += hashmap.o LIB_OBJS += help.o diff --git a/config.c b/config.c index 0a8d6901c83608..9f8144c65a72b9 100644 --- a/config.c +++ b/config.c @@ -12,6 +12,7 @@ #include "abspath.h" #include "advice.h" #include "date.h" +#include "gvfs.h" #include "branch.h" #include "config.h" #include "parse.h" @@ -1665,6 +1666,11 @@ int git_default_core_config(const char *var, const char *value, return 0; } + if (!strcmp(var, "core.gvfs")) { + gvfs_load_config_value(value); + return 0; + } + if (!strcmp(var, "core.sparsecheckout")) { core_apply_sparse_checkout = git_config_bool(var, value); return 0; diff --git a/environment.c b/environment.c index 5cea2c9f5473e3..54b9d182ee6339 100644 --- a/environment.c +++ b/environment.c @@ -80,6 +80,7 @@ int grafts_keep_true_parents; int core_apply_sparse_checkout; int core_sparse_checkout_cone; int sparse_expect_files_outside_of_patterns; +int core_gvfs; int merge_log_config = -1; int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ unsigned long pack_size_limit_cfg; diff --git a/environment.h b/environment.h index e9f01d4d11c010..0e607ead0c8236 100644 --- a/environment.h +++ b/environment.h @@ -153,6 +153,7 @@ int get_shared_repository(void); void reset_shared_repository(void); extern int core_preload_index; +extern int core_gvfs; extern int precomposed_unicode; extern int protect_hfs; extern int protect_ntfs; diff --git a/gvfs.c b/gvfs.c new file mode 100644 index 00000000000000..c65f15ff02584a --- /dev/null +++ b/gvfs.c @@ -0,0 +1,45 @@ +#define USE_THE_REPOSITORY_VARIABLE +#include "git-compat-util.h" +#include "environment.h" +#include "gvfs.h" +#include "setup.h" +#include "config.h" + +static int gvfs_config_loaded; +static int core_gvfs_is_bool; + +static int early_core_gvfs_config(const char *var, const char *value, + const struct config_context *ctx, void *cb) +{ + if (!strcmp(var, "core.gvfs")) + core_gvfs = git_config_bool_or_int("core.gvfs", value, ctx->kvi, + &core_gvfs_is_bool); + return 0; +} + +void gvfs_load_config_value(const char *value) +{ + if (gvfs_config_loaded) + return; + + if (value) { + struct key_value_info default_kvi = KVI_INIT; + core_gvfs = git_config_bool_or_int("core.gvfs", value, &default_kvi, &core_gvfs_is_bool); + } else if (startup_info->have_repository == 0) + read_early_config(early_core_gvfs_config, NULL); + else + repo_config_get_bool_or_int(the_repository, "core.gvfs", + &core_gvfs_is_bool, &core_gvfs); + + /* Turn on all bits if a bool was set in the settings */ + if (core_gvfs_is_bool && core_gvfs) + core_gvfs = -1; + + gvfs_config_loaded = 1; +} + +int gvfs_config_is_set(int mask) +{ + gvfs_load_config_value(NULL); + return (core_gvfs & mask) == mask; +} diff --git a/gvfs.h b/gvfs.h index b6dbe85eae4071..011185dea93734 100644 --- a/gvfs.h +++ b/gvfs.h @@ -1,9 +1,13 @@ #ifndef GVFS_H #define GVFS_H + /* * This file is for the specific settings and methods * used for GVFS functionality */ +void gvfs_load_config_value(const char *value); +int gvfs_config_is_set(int mask); + #endif /* GVFS_H */ From d94ca7d241d94edb0f7fec70a28572b74252d8f4 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:38:59 +0100 Subject: [PATCH 06/13] gvfs: add the feature to skip writing the index' SHA-1 This takes a substantial amount of time, and if the user is reasonably sure that the files' integrity is not compromised, that time can be saved. Git no longer verifies the SHA-1 by default, anyway. Signed-off-by: Kevin Willford Update for 2023-02-27: This feature was upstreamed as the index.skipHash config option. This resulted in some changes to the struct and some of the setup code. In particular, the config reading was moved to prepare_repo_settings(), so the core.gvfs bit check was moved there, too. Signed-off-by: Derrick Stolee --- Documentation/config/core.txt | 10 +++++++++- gvfs.h | 6 ++++++ repo-settings.c | 8 ++++++++ t/t1017-read-tree-skip-sha-on-read.sh | 22 ++++++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100755 t/t1017-read-tree-skip-sha-on-read.sh diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 3f3343c7cbd982..925f164842ff00 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -744,7 +744,15 @@ core.multiPackIndex:: information. Defaults to true. core.gvfs:: - Enable the features needed for GVFS. + Enable the features needed for GVFS. This value can be set to true + to indicate all features should be turned on or the bit values listed + below can be used to turn on specific features. ++ +-- + GVFS_SKIP_SHA_ON_INDEX:: + Bit value 1 + Disables the calculation of the sha when writing the index +-- core.sparseCheckout:: Enable "sparse checkout" feature. See linkgit:git-sparse-checkout[1] diff --git a/gvfs.h b/gvfs.h index 011185dea93734..c75991530fa1fa 100644 --- a/gvfs.h +++ b/gvfs.h @@ -7,6 +7,12 @@ * used for GVFS functionality */ + +/* + * The list of bits in the core_gvfs setting + */ +#define GVFS_SKIP_SHA_ON_INDEX (1 << 0) + void gvfs_load_config_value(const char *value); int gvfs_config_is_set(int mask); diff --git a/repo-settings.c b/repo-settings.c index f104e6fc73c38f..efb131e5a01501 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -4,6 +4,7 @@ #include "midx.h" #include "fsmonitor-ipc.h" #include "fsmonitor-settings.h" +#include "gvfs.h" static void repo_cfg_bool(struct repository *r, const char *key, int *dest, int def) @@ -99,6 +100,13 @@ void prepare_repo_settings(struct repository *r) r->settings.pack_use_bitmap_boundary_traversal); repo_cfg_bool(r, "core.usereplacerefs", &r->settings.read_replace_refs, 1); + /* + * For historical compatibility reasons, enable index.skipHash based + * on a bit in core.gvfs. + */ + if (gvfs_config_is_set(GVFS_SKIP_SHA_ON_INDEX)) + r->settings.index_skip_hash = 1; + /* * The GIT_TEST_MULTI_PACK_INDEX variable is special in that * either it *or* the config sets diff --git a/t/t1017-read-tree-skip-sha-on-read.sh b/t/t1017-read-tree-skip-sha-on-read.sh new file mode 100755 index 00000000000000..5b76a80a0020dc --- /dev/null +++ b/t/t1017-read-tree-skip-sha-on-read.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +test_description='check that read-tree works with core.gvfs config value' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-read-tree.sh + +test_expect_success setup ' + echo one >a && + git add a && + git commit -m initial +' +test_expect_success 'read-tree without core.gvsf' ' + read_tree_u_must_succeed -m -u HEAD +' + +test_expect_success 'read-tree with core.gvfs set to 1' ' + git config core.gvfs 1 && + read_tree_u_must_succeed -m -u HEAD +' + +test_done From 363f882ab3a32530576dc9f8a127154382f56d14 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:54:55 +0100 Subject: [PATCH 07/13] gvfs: add the feature that blobs may be missing Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 4 ++++ cache-tree.c | 4 +++- commit.c | 9 +++++++-- gvfs.h | 1 + t/t0000-basic.sh | 5 +++++ 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 925f164842ff00..7c640bbf36bb68 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -752,6 +752,10 @@ core.gvfs:: GVFS_SKIP_SHA_ON_INDEX:: Bit value 1 Disables the calculation of the sha when writing the index + GVFS_MISSING_OK:: + Bit value 4 + Normally git write-tree ensures that the objects referenced by the + directory exist in the object database. This option disables this check. -- core.sparseCheckout:: diff --git a/cache-tree.c b/cache-tree.c index 50610c3f3cb8e7..02808209115ec0 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "environment.h" #include "hex.h" +#include "gvfs.h" #include "lockfile.h" #include "tree.h" #include "tree-walk.h" @@ -259,7 +260,8 @@ static int update_one(struct cache_tree *it, int flags) { struct strbuf buffer; - int missing_ok = flags & WRITE_TREE_MISSING_OK; + int missing_ok = gvfs_config_is_set(GVFS_MISSING_OK) ? + WRITE_TREE_MISSING_OK : (flags & WRITE_TREE_MISSING_OK); int dryrun = flags & WRITE_TREE_DRY_RUN; int repair = flags & WRITE_TREE_REPAIR; int to_invalidate = 0; diff --git a/commit.c b/commit.c index 087cb19f4f84f3..27eb39758b2275 100644 --- a/commit.c +++ b/commit.c @@ -1,6 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "gvfs.h" #include "tag.h" #include "commit.h" #include "commit-graph.h" @@ -562,13 +563,17 @@ int repo_parse_commit_internal(struct repository *r, .sizep = &size, .contentp = &buffer, }; + int ret; /* * Git does not support partial clones that exclude commits, so set * OBJECT_INFO_SKIP_FETCH_OBJECT to fail fast when an object is missing. */ int flags = OBJECT_INFO_LOOKUP_REPLACE | OBJECT_INFO_SKIP_FETCH_OBJECT | - OBJECT_INFO_DIE_IF_CORRUPT; - int ret; + OBJECT_INFO_DIE_IF_CORRUPT; + + /* But the GVFS Protocol _does_ support missing commits! */ + if (gvfs_config_is_set(GVFS_MISSING_OK)) + flags ^= OBJECT_INFO_SKIP_FETCH_OBJECT; if (!item) return -1; diff --git a/gvfs.h b/gvfs.h index c75991530fa1fa..7bedfaacf6d684 100644 --- a/gvfs.h +++ b/gvfs.h @@ -12,6 +12,7 @@ * The list of bits in the core_gvfs setting */ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) +#define GVFS_MISSING_OK (1 << 2) void gvfs_load_config_value(const char *value); int gvfs_config_is_set(int mask); diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh index 98b81e4d63fa4c..7baa03212bf3ba 100755 --- a/t/t0000-basic.sh +++ b/t/t0000-basic.sh @@ -1106,6 +1106,11 @@ test_expect_success 'writing this tree with --missing-ok' ' git write-tree --missing-ok ' +test_expect_success 'writing this tree with missing ok config value' ' + git config core.gvfs 4 && + git write-tree +' + ################################################################ test_expect_success 'git read-tree followed by write-tree should be idempotent' ' From 4bd94264510aee55d8e3f341d1529cd0f2116d9a Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 18 May 2016 13:40:39 +0000 Subject: [PATCH 08/13] gvfs: prevent files to be deleted outside the sparse checkout Prevent the sparse checkout to delete files that were marked with skip-worktree bit and are not in the sparse-checkout file. This is because everything with the skip-worktree bit turned on is being virtualized and will be removed with the change of HEAD. There was only one failing test when running with these changes that was checking to make sure the worktree narrows on checkout which was expected since we would no longer be narrowing the worktree. Update 2022-04-05: temporarily set 'sparse.expectfilesoutsideofpatterns' in test (until we start disabling the "remove present-despite-SKIP_WORKTREE" behavior with 'core.virtualfilesystem' in a later commit). Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 9 +++++++++ gvfs.h | 1 + t/t1090-sparse-checkout-scope.sh | 20 ++++++++++++++++++++ unpack-trees.c | 22 ++++++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 7c640bbf36bb68..cb37bca6d7ac08 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -756,6 +756,15 @@ core.gvfs:: Bit value 4 Normally git write-tree ensures that the objects referenced by the directory exist in the object database. This option disables this check. + GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT:: + Bit value 8 + When marking entries to remove from the index and the working + directory this option will take into account what the + skip-worktree bit was set to so that if the entry has the + skip-worktree bit set it will not be removed from the working + directory. This will allow virtualized working directories to + detect the change to HEAD and use the new commit tree to show + the files that are in the working directory. -- core.sparseCheckout:: diff --git a/gvfs.h b/gvfs.h index 7bedfaacf6d684..44131625828cfa 100644 --- a/gvfs.h +++ b/gvfs.h @@ -13,6 +13,7 @@ */ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) #define GVFS_MISSING_OK (1 << 2) +#define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) void gvfs_load_config_value(const char *value); int gvfs_config_is_set(int mask); diff --git a/t/t1090-sparse-checkout-scope.sh b/t/t1090-sparse-checkout-scope.sh index b6298e7e7d8ab1..7af01dc00b75ee 100755 --- a/t/t1090-sparse-checkout-scope.sh +++ b/t/t1090-sparse-checkout-scope.sh @@ -107,6 +107,26 @@ test_expect_success 'in partial clone, sparse checkout only fetches needed blobs test_cmp expect actual ' +test_expect_success 'checkout does not delete items outside the sparse checkout file' ' + # The "sparse.expectfilesoutsideofpatterns" config will prevent the + # SKIP_WORKTREE flag from being dropped on files present on-disk. + test_config sparse.expectfilesoutsideofpatterns true && + + test_config core.gvfs 8 && + git checkout -b outside && + echo "new file1" >d && + git add --sparse d && + git commit -m "branch initial" && + echo "new file1" >e && + git add --sparse e && + git commit -m "skipped worktree" && + git update-index --skip-worktree e && + echo "/d" >.git/info/sparse-checkout && + git checkout HEAD^ && + test_path_is_file d && + test_path_is_file e +' + test_expect_success MINGW 'no unnecessary opendir() with fscache' ' git clone . fscache-test && ( diff --git a/unpack-trees.c b/unpack-trees.c index 1cff7ca762374f..7c564ce5566102 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -2,6 +2,7 @@ #include "git-compat-util.h" #include "advice.h" +#include "gvfs.h" #include "strvec.h" #include "repository.h" #include "parse.h" @@ -2677,6 +2678,27 @@ static int deleted_entry(const struct cache_entry *ce, if (!(old->ce_flags & CE_CONFLICTED) && verify_uptodate(old, o)) return -1; + + /* + * When marking entries to remove from the index and the working + * directory this option will take into account what the + * skip-worktree bit was set to so that if the entry has the + * skip-worktree bit set it will not be removed from the working + * directory. This will allow virtualized working directories to + * detect the change to HEAD and use the new commit tree to show + * the files that are in the working directory. + * + * old is the cache_entry that will have the skip-worktree bit set + * which will need to be preserved when the CE_REMOVE entry is added + */ + if (gvfs_config_is_set(GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT) && + old && + old->ce_flags & CE_SKIP_WORKTREE) { + add_entry(o, old, CE_REMOVE, 0); + invalidate_ce_path(old, o); + return 1; + } + add_entry(o, ce, CE_REMOVE, 0); invalidate_ce_path(ce, o); return 1; From 77d8f4a99beb1ab164ecea5ea90a86e230ec29d8 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 30 May 2016 10:55:53 -0400 Subject: [PATCH 09/13] gvfs: optionally skip reachability checks/upload pack during fetch While performing a fetch with a virtual file system we know that there will be missing objects and we don't want to download them just because of the reachability of the commits. We also don't want to download a pack file with commits, trees, and blobs since these will be downloaded on demand. This flag will skip the first connectivity check and by returning zero will skip the upload pack. It will also skip the second connectivity check but continue to update the branches to the latest commit ids. Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 9 +++++++++ connected.c | 19 +++++++++++++++++++ gvfs.h | 1 + t/t5584-vfs.sh | 24 ++++++++++++++++++++++++ 4 files changed, 53 insertions(+) create mode 100755 t/t5584-vfs.sh diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index cb37bca6d7ac08..c2063fcaf6e407 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -765,6 +765,15 @@ core.gvfs:: directory. This will allow virtualized working directories to detect the change to HEAD and use the new commit tree to show the files that are in the working directory. + GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK:: + Bit value 16 + While performing a fetch with a virtual file system we know + that there will be missing objects and we don't want to download + them just because of the reachability of the commits. We also + don't want to download a pack file with commits, trees, and blobs + since these will be downloaded on demand. This flag will skip the + checks on the reachability of objects during a fetch as well as + the upload pack so that extraneous objects don't get downloaded. -- core.sparseCheckout:: diff --git a/connected.c b/connected.c index 87cc4b57a17ce5..44e5f75eb063b4 100644 --- a/connected.c +++ b/connected.c @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "gettext.h" #include "hex.h" +#include "gvfs.h" #include "object-store-ll.h" #include "run-command.h" #include "sigchain.h" @@ -34,6 +35,24 @@ int check_connected(oid_iterate_fn fn, void *cb_data, struct transport *transport; size_t base_len; + /* + * Running a virtual file system there will be objects that are + * missing locally and we don't want to download a bunch of + * commits, trees, and blobs just to make sure everything is + * reachable locally so this option will skip reachablility + * checks below that use rev-list. This will stop the check + * before uploadpack runs to determine if there is anything to + * fetch. Returning zero for the first check will also prevent the + * uploadpack from happening. It will also skip the check after + * the fetch is finished to make sure all the objects where + * downloaded in the pack file. This will allow the fetch to + * run and get all the latest tip commit ids for all the branches + * in the fetch but not pull down commits, trees, or blobs via + * upload pack. + */ + if (gvfs_config_is_set(GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK)) + return 0; + if (!opt) opt = &defaults; transport = opt->transport; diff --git a/gvfs.h b/gvfs.h index 44131625828cfa..e69dd4b2c8f785 100644 --- a/gvfs.h +++ b/gvfs.h @@ -14,6 +14,7 @@ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) #define GVFS_MISSING_OK (1 << 2) #define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) +#define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) void gvfs_load_config_value(const char *value); int gvfs_config_is_set(int mask); diff --git a/t/t5584-vfs.sh b/t/t5584-vfs.sh new file mode 100755 index 00000000000000..8a703cbb640387 --- /dev/null +++ b/t/t5584-vfs.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +test_description='fetch using the flag to skip reachability and upload pack' + +. ./test-lib.sh + + +test_expect_success setup ' + echo inital >a && + git add a && + git commit -m initial && + git clone . one +' + +test_expect_success "fetch test" ' + cd one && + git config core.gvfs 16 && + rm -rf .git/objects/* && + git -C .. cat-file commit HEAD | git hash-object -w --stdin -t commit && + git fetch && + test_must_fail git rev-parse --verify HEAD^{tree} +' + +test_done \ No newline at end of file From 4c1c45a2b3f628b2a3fa714dfec1a6af90163ba3 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Wed, 15 Jun 2016 14:59:16 +0000 Subject: [PATCH 10/13] gvfs: ensure all filters and EOL conversions are blocked Ensure all filters and EOL conversions are blocked when running under GVFS so that our projected file sizes will match the actual file size when it is hydrated on the local machine. Signed-off-by: Ben Peart --- Documentation/config/core.txt | 9 +++++++++ convert.c | 22 +++++++++++++++++++++ gvfs.h | 1 + t/t0021-conversion.sh | 37 +++++++++++++++++++++++++++++++++++ t/t0027-auto-crlf.sh | 12 ++++++++++++ 5 files changed, 81 insertions(+) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index c2063fcaf6e407..2fd2a10f51ad2c 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -774,6 +774,15 @@ core.gvfs:: since these will be downloaded on demand. This flag will skip the checks on the reachability of objects during a fetch as well as the upload pack so that extraneous objects don't get downloaded. + GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS:: + Bit value 64 + With a virtual file system we only know the file size before any + CRLF or smudge/clean filters processing is done on the client. + To prevent file corruption due to truncation or expansion with + garbage at the end, these filters must not run when the file + is first accessed and brought down to the client. Git.exe can't + currently tell the first access vs subsequent accesses so this + flag just blocks them from occurring at all. -- core.sparseCheckout:: diff --git a/convert.c b/convert.c index c4ddc4de81b557..f050f98679daea 100644 --- a/convert.c +++ b/convert.c @@ -2,6 +2,7 @@ #include "git-compat-util.h" #include "advice.h" +#include "gvfs.h" #include "config.h" #include "convert.h" #include "copy.h" @@ -562,6 +563,9 @@ static int crlf_to_git(struct index_state *istate, if (!buf) return 1; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* only grow if not in place */ if (strbuf_avail(buf) + buf->len < len) strbuf_grow(buf, len - buf->len); @@ -601,6 +605,9 @@ static int crlf_to_worktree(const char *src, size_t len, struct strbuf *buf, if (!will_convert_lf_to_crlf(&stats, crlf_action)) return 0; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* are we "faking" in place editing ? */ if (src == buf->buf) to_free = strbuf_detach(buf, NULL); @@ -710,6 +717,9 @@ static int apply_single_file_filter(const char *path, const char *src, size_t le struct async async; struct filter_params params; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("Filter \"%s\" not supported when running under GVFS", cmd); + memset(&async, 0, sizeof(async)); async.proc = filter_buffer_or_fd; async.data = ¶ms; @@ -1123,6 +1133,9 @@ static int ident_to_git(const char *src, size_t len, if (!buf) return 1; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + /* only grow if not in place */ if (strbuf_avail(buf) + buf->len < len) strbuf_grow(buf, len - buf->len); @@ -1170,6 +1183,9 @@ static int ident_to_worktree(const char *src, size_t len, if (!cnt) return 0; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + /* are we "faking" in place editing ? */ if (src == buf->buf) to_free = strbuf_detach(buf, NULL); @@ -1619,6 +1635,9 @@ static int lf_to_crlf_filter_fn(struct stream_filter *filter, size_t count, o = 0; struct lf_to_crlf_filter *lf_to_crlf = (struct lf_to_crlf_filter *)filter; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* * We may be holding onto the CR to see if it is followed by a * LF, in which case we would need to go to the main loop. @@ -1863,6 +1882,9 @@ static int ident_filter_fn(struct stream_filter *filter, struct ident_filter *ident = (struct ident_filter *)filter; static const char head[] = "$Id"; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + if (!input) { /* drain upon eof */ switch (ident->state) { diff --git a/gvfs.h b/gvfs.h index e69dd4b2c8f785..7c9367866f502a 100644 --- a/gvfs.h +++ b/gvfs.h @@ -15,6 +15,7 @@ #define GVFS_MISSING_OK (1 << 2) #define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) #define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) +#define GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS (1 << 6) void gvfs_load_config_value(const char *value); int gvfs_config_is_set(int mask); diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh index 8ba840cdc138cd..e7eccfaccb9a36 100755 --- a/t/t0021-conversion.sh +++ b/t/t0021-conversion.sh @@ -335,6 +335,43 @@ test_expect_success "filter: smudge empty file" ' test_cmp expected filtered-empty-in-repo ' +test_expect_success "filter: clean filters blocked when under GVFS" ' + test_config filter.empty-in-repo.clean "cat >/dev/null" && + test_config filter.empty-in-repo.smudge "echo smudged && cat" && + test_config core.gvfs 64 && + + echo dead data walking >empty-in-repo && + test_must_fail git add empty-in-repo +' + +test_expect_success "filter: smudge filters blocked when under GVFS" ' + test_config filter.empty-in-repo.clean "cat >/dev/null" && + test_config filter.empty-in-repo.smudge "echo smudged && cat" && + test_config core.gvfs 64 && + + test_must_fail git checkout +' + +test_expect_success "ident blocked on add when under GVFS" ' + test_config core.gvfs 64 && + test_config core.autocrlf false && + + echo "*.i ident" >.gitattributes && + echo "\$Id\$" > ident.i && + + test_must_fail git add ident.i +' + +test_expect_success "ident blocked when under GVFS" ' + git add ident.i && + + git commit -m "added ident.i" && + test_config core.gvfs 64 && + rm ident.i && + + test_must_fail git checkout -- ident.i +' + test_expect_success 'disable filter with empty override' ' test_config_global filter.disable.smudge false && test_config_global filter.disable.clean false && diff --git a/t/t0027-auto-crlf.sh b/t/t0027-auto-crlf.sh index 2f57c8669cb5af..c42bf106e6f4ea 100755 --- a/t/t0027-auto-crlf.sh +++ b/t/t0027-auto-crlf.sh @@ -344,6 +344,18 @@ checkout_files () { " } +test_expect_success 'crlf conversions blocked when under GVFS' ' + git checkout -b gvfs && + test_commit initial && + rm initial.t && + test_config core.gvfs 64 && + test_config core.autocrlf true && + test_must_fail git read-tree --reset -u HEAD && + + git config core.autocrlf false && + git read-tree --reset -u HEAD +' + # Test control characters # NUL SOH CR EOF==^Z test_expect_success 'ls-files --eol -o Text/Binary' ' From 695e26367357d871d17fd6a70cb3e59f53de00d5 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Tue, 10 Jan 2017 18:47:14 +0000 Subject: [PATCH 11/13] gvfs: allow "virtualizing" objects The idea is to allow blob objects to be missing from the local repository, and to load them lazily on demand. After discussing this idea on the mailing list, we will rename the feature to "lazy clone" and work more on this. Signed-off-by: Ben Peart Signed-off-by: Johannes Schindelin --- config.c | 5 +++++ connected.c | 3 +++ environment.c | 1 + environment.h | 2 ++ object-file.c | 23 +++++++++++++++++++++++ 5 files changed, 34 insertions(+) diff --git a/config.c b/config.c index 9f8144c65a72b9..329b7384f35bbe 100644 --- a/config.c +++ b/config.c @@ -1701,6 +1701,11 @@ int git_default_core_config(const char *var, const char *value, return 0; } + if (!strcmp(var, "core.virtualizeobjects")) { + core_virtualize_objects = git_config_bool(var, value); + return 0; + } + /* Add other config variables here and to Documentation/config.txt. */ return platform_core_config(var, value, ctx, cb); } diff --git a/connected.c b/connected.c index 44e5f75eb063b4..a8cf3dc351fc54 100644 --- a/connected.c +++ b/connected.c @@ -1,6 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "environment.h" #include "gettext.h" #include "hex.h" #include "gvfs.h" @@ -52,6 +53,8 @@ int check_connected(oid_iterate_fn fn, void *cb_data, */ if (gvfs_config_is_set(GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK)) return 0; + if (core_virtualize_objects) + return 0; if (!opt) opt = &defaults; diff --git a/environment.c b/environment.c index 54b9d182ee6339..01c6963183da3f 100644 --- a/environment.c +++ b/environment.c @@ -84,6 +84,7 @@ int core_gvfs; int merge_log_config = -1; int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ unsigned long pack_size_limit_cfg; +int core_virtualize_objects; enum log_refs_config log_all_ref_updates = LOG_REFS_UNSET; int max_allowed_tree_depth = #ifdef _MSC_VER diff --git a/environment.h b/environment.h index 0e607ead0c8236..881f8947ba8e19 100644 --- a/environment.h +++ b/environment.h @@ -11,6 +11,8 @@ struct strvec; extern const char *comment_line_str; extern int auto_comment_line_char; +extern int core_virtualize_objects; + /* * Wrapper of getenv() that returns a strdup value. This value is kept * in argv to be freed later. diff --git a/object-file.c b/object-file.c index c672adc74ca791..8eb1a2749bd921 100644 --- a/object-file.c +++ b/object-file.c @@ -40,6 +40,8 @@ #include "fsck.h" #include "loose.h" #include "object-file-convert.h" +#include "trace.h" +#include "hook.h" /* The maximum size for an object header. */ #define MAX_HEADER_LEN 32 @@ -1545,6 +1547,20 @@ void disable_obj_read_lock(void) pthread_mutex_destroy(&obj_read_mutex); } +static int run_read_object_hook(const struct object_id *oid) +{ + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + int ret; + uint64_t start; + + start = getnanotime(); + strvec_push(&opt.args, oid_to_hex(oid)); + ret = run_hooks_opt("read-object", &opt); + trace_performance_since(start, "run_read_object_hook"); + + return ret; +} + int fetch_if_missing = 1; static int do_oid_object_info_extended(struct repository *r, @@ -1557,6 +1573,7 @@ static int do_oid_object_info_extended(struct repository *r, int rtype; const struct object_id *real = oid; int already_retried = 0; + int tried_hook = 0; if (flags & OBJECT_INFO_LOOKUP_REPLACE) @@ -1568,6 +1585,7 @@ static int do_oid_object_info_extended(struct repository *r, if (!oi) oi = &blank_oi; +retry: co = find_cached_object(real); if (co) { if (oi->typep) @@ -1599,6 +1617,11 @@ static int do_oid_object_info_extended(struct repository *r, reprepare_packed_git(r); if (find_pack_entry(r, real, &e)) break; + if (core_virtualize_objects && !tried_hook) { + tried_hook = 1; + if (!run_read_object_hook(oid)) + goto retry; + } } /* From 5bcdb2cddf3be7b227a8eec11d727179b8897d62 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Wed, 15 Mar 2017 18:43:05 +0000 Subject: [PATCH 12/13] Hydrate missing loose objects in check_and_freshen() Hydrate missing loose objects in check_and_freshen() when running virtualized. Add test cases to verify read-object hook works when running virtualized. This hook is called in check_and_freshen() rather than check_and_freshen_local() to make the hook work also with alternates. Helped-by: Kevin Willford Signed-off-by: Ben Peart --- .../technical/read-object-protocol.txt | 102 +++++++++++++ contrib/long-running-read-object/example.pl | 114 ++++++++++++++ object-file.c | 141 ++++++++++++++++-- t/t0410/read-object | 114 ++++++++++++++ t/t0499-read-object.sh | 30 ++++ 5 files changed, 485 insertions(+), 16 deletions(-) create mode 100644 Documentation/technical/read-object-protocol.txt create mode 100644 contrib/long-running-read-object/example.pl create mode 100755 t/t0410/read-object create mode 100755 t/t0499-read-object.sh diff --git a/Documentation/technical/read-object-protocol.txt b/Documentation/technical/read-object-protocol.txt new file mode 100644 index 00000000000000..a893b46e7c28a9 --- /dev/null +++ b/Documentation/technical/read-object-protocol.txt @@ -0,0 +1,102 @@ +Read Object Process +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The read-object process enables Git to read all missing blobs with a +single process invocation for the entire life of a single Git command. +This is achieved by using a packet format (pkt-line, see technical/ +protocol-common.txt) based protocol over standard input and standard +output as follows. All packets, except for the "*CONTENT" packets and +the "0000" flush packet, are considered text and therefore are +terminated by a LF. + +Git starts the process when it encounters the first missing object that +needs to be retrieved. After the process is started, Git sends a welcome +message ("git-read-object-client"), a list of supported protocol version +numbers, and a flush packet. Git expects to read a welcome response +message ("git-read-object-server"), exactly one protocol version number +from the previously sent list, and a flush packet. All further +communication will be based on the selected version. + +The remaining protocol description below documents "version=1". Please +note that "version=42" in the example below does not exist and is only +there to illustrate how the protocol would look with more than one +version. + +After the version negotiation Git sends a list of all capabilities that +it supports and a flush packet. Git expects to read a list of desired +capabilities, which must be a subset of the supported capabilities list, +and a flush packet as response: +------------------------ +packet: git> git-read-object-client +packet: git> version=1 +packet: git> version=42 +packet: git> 0000 +packet: git< git-read-object-server +packet: git< version=1 +packet: git< 0000 +packet: git> capability=get +packet: git> capability=have +packet: git> capability=put +packet: git> capability=not-yet-invented +packet: git> 0000 +packet: git< capability=get +packet: git< 0000 +------------------------ +The only supported capability in version 1 is "get". + +Afterwards Git sends a list of "key=value" pairs terminated with a flush +packet. The list will contain at least the command (based on the +supported capabilities) and the sha1 of the object to retrieve. Please +note, that the process must not send any response before it received the +final flush packet. + +When the process receives the "get" command, it should make the requested +object available in the git object store and then return success. Git will +then check the object store again and this time find it and proceed. +------------------------ +packet: git> command=get +packet: git> sha1=0a214a649e1b3d5011e14a3dc227753f2bd2be05 +packet: git> 0000 +------------------------ + +The process is expected to respond with a list of "key=value" pairs +terminated with a flush packet. If the process does not experience +problems then the list must contain a "success" status. +------------------------ +packet: git< status=success +packet: git< 0000 +------------------------ + +In case the process cannot or does not want to process the content, it +is expected to respond with an "error" status. +------------------------ +packet: git< status=error +packet: git< 0000 +------------------------ + +In case the process cannot or does not want to process the content as +well as any future content for the lifetime of the Git process, then it +is expected to respond with an "abort" status at any point in the +protocol. +------------------------ +packet: git< status=abort +packet: git< 0000 +------------------------ + +Git neither stops nor restarts the process in case the "error"/"abort" +status is set. + +If the process dies during the communication or does not adhere to the +protocol then Git will stop the process and restart it with the next +object that needs to be processed. + +After the read-object process has processed an object it is expected to +wait for the next "key=value" list containing a command. Git will close +the command pipe on exit. The process is expected to detect EOF and exit +gracefully on its own. Git will wait until the process has stopped. + +A long running read-object process demo implementation can be found in +`contrib/long-running-read-object/example.pl` located in the Git core +repository. If you develop your own long running process then the +`GIT_TRACE_PACKET` environment variables can be very helpful for +debugging (see linkgit:git[1]). diff --git a/contrib/long-running-read-object/example.pl b/contrib/long-running-read-object/example.pl new file mode 100644 index 00000000000000..b8f37f836a813c --- /dev/null +++ b/contrib/long-running-read-object/example.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl +# +# Example implementation for the Git read-object protocol version 1 +# See Documentation/technical/read-object-protocol.txt +# +# Allows you to test the ability for blobs to be pulled from a host git repo +# "on demand." Called when git needs a blob it couldn't find locally due to +# a lazy clone that only cloned the commits and trees. +# +# A lazy clone can be simulated via the following commands from the host repo +# you wish to create a lazy clone of: +# +# cd /host_repo +# git rev-parse HEAD +# git init /guest_repo +# git cat-file --batch-check --batch-all-objects | grep -v 'blob' | +# cut -d' ' -f1 | git pack-objects /guest_repo/.git/objects/pack/noblobs +# cd /guest_repo +# git config core.virtualizeobjects true +# git reset --hard +# +# Please note, this sample is a minimal skeleton. No proper error handling +# was implemented. +# + +use strict; +use warnings; + +# +# Point $DIR to the folder where your host git repo is located so we can pull +# missing objects from it +# +my $DIR = "/host_repo/.git/"; + +sub packet_bin_read { + my $buffer; + my $bytes_read = read STDIN, $buffer, 4; + if ( $bytes_read == 0 ) { + + # EOF - Git stopped talking to us! + exit(); + } + elsif ( $bytes_read != 4 ) { + die "invalid packet: '$buffer'"; + } + my $pkt_size = hex($buffer); + if ( $pkt_size == 0 ) { + return ( 1, "" ); + } + elsif ( $pkt_size > 4 ) { + my $content_size = $pkt_size - 4; + $bytes_read = read STDIN, $buffer, $content_size; + if ( $bytes_read != $content_size ) { + die "invalid packet ($content_size bytes expected; $bytes_read bytes read)"; + } + return ( 0, $buffer ); + } + else { + die "invalid packet size: $pkt_size"; + } +} + +sub packet_txt_read { + my ( $res, $buf ) = packet_bin_read(); + unless ( $buf =~ s/\n$// ) { + die "A non-binary line MUST be terminated by an LF."; + } + return ( $res, $buf ); +} + +sub packet_bin_write { + my $buf = shift; + print STDOUT sprintf( "%04x", length($buf) + 4 ); + print STDOUT $buf; + STDOUT->flush(); +} + +sub packet_txt_write { + packet_bin_write( $_[0] . "\n" ); +} + +sub packet_flush { + print STDOUT sprintf( "%04x", 0 ); + STDOUT->flush(); +} + +( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || die "bad initialize"; +( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad version end"; + +packet_txt_write("git-read-object-server"); +packet_txt_write("version=1"); +packet_flush(); + +( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end"; + +packet_txt_write("capability=get"); +packet_flush(); + +while (1) { + my ($command) = packet_txt_read() =~ /^command=([^=]+)$/; + + if ( $command eq "get" ) { + my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/; + packet_bin_read(); + + system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); + packet_txt_write(($?) ? "status=error" : "status=success"); + packet_flush(); + } else { + die "bad command '$command'"; + } +} diff --git a/object-file.c b/object-file.c index 8eb1a2749bd921..1b37072b239536 100644 --- a/object-file.c +++ b/object-file.c @@ -42,6 +42,9 @@ #include "object-file-convert.h" #include "trace.h" #include "hook.h" +#include "sigchain.h" +#include "sub-process.h" +#include "pkt-line.h" /* The maximum size for an object header. */ #define MAX_HEADER_LEN 32 @@ -952,6 +955,115 @@ int has_alt_odb(struct repository *r) return !!r->objects->odb->next; } +#define CAP_GET (1u<<0) + +static int subprocess_map_initialized; +static struct hashmap subprocess_map; + +struct read_object_process { + struct subprocess_entry subprocess; + unsigned int supported_capabilities; +}; + +static int start_read_object_fn(struct subprocess_entry *subprocess) +{ + struct read_object_process *entry = (struct read_object_process *)subprocess; + static int versions[] = {1, 0}; + static struct subprocess_capability capabilities[] = { + { "get", CAP_GET }, + { NULL, 0 } + }; + + return subprocess_handshake(subprocess, "git-read-object", versions, + NULL, capabilities, + &entry->supported_capabilities); +} + +static int read_object_process(const struct object_id *oid) +{ + int err; + struct read_object_process *entry; + struct child_process *process; + struct strbuf status = STRBUF_INIT; + const char *cmd = find_hook("read-object"); + uint64_t start; + + start = getnanotime(); + + if (!subprocess_map_initialized) { + subprocess_map_initialized = 1; + hashmap_init(&subprocess_map, (hashmap_cmp_fn)cmd2process_cmp, + NULL, 0); + entry = NULL; + } else { + entry = (struct read_object_process *) subprocess_find_entry(&subprocess_map, cmd); + } + + if (!entry) { + entry = xmalloc(sizeof(*entry)); + entry->supported_capabilities = 0; + + if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, + start_read_object_fn)) { + free(entry); + return -1; + } + } + process = &entry->subprocess.process; + + if (!(CAP_GET & entry->supported_capabilities)) + return -1; + + sigchain_push(SIGPIPE, SIG_IGN); + + err = packet_write_fmt_gently(process->in, "command=get\n"); + if (err) + goto done; + + err = packet_write_fmt_gently(process->in, "sha1=%s\n", oid_to_hex(oid)); + if (err) + goto done; + + err = packet_flush_gently(process->in); + if (err) + goto done; + + err = subprocess_read_status(process->out, &status); + err = err ? err : strcmp(status.buf, "success"); + +done: + sigchain_pop(SIGPIPE); + + if (err || errno == EPIPE) { + err = err ? err : errno; + if (!strcmp(status.buf, "error")) { + /* The process signaled a problem with the file. */ + } + else if (!strcmp(status.buf, "abort")) { + /* + * The process signaled a permanent problem. Don't try to read + * objects with the same command for the lifetime of the current + * Git process. + */ + entry->supported_capabilities &= ~CAP_GET; + } + else { + /* + * Something went wrong with the read-object process. + * Force shutdown and restart if needed. + */ + error("external process '%s' failed", cmd); + subprocess_stop(&subprocess_map, + (struct subprocess_entry *)entry); + free(entry); + } + } + + trace_performance_since(start, "read_object_process"); + + return err; +} + /* Returns 1 if we have successfully freshened the file, 0 otherwise. */ static int freshen_file(const char *fn) { @@ -1002,8 +1114,19 @@ static int check_and_freshen_nonlocal(const struct object_id *oid, int freshen) static int check_and_freshen(const struct object_id *oid, int freshen) { - return check_and_freshen_local(oid, freshen) || + int ret; + int tried_hook = 0; + +retry: + ret = check_and_freshen_local(oid, freshen) || check_and_freshen_nonlocal(oid, freshen); + if (!ret && core_virtualize_objects && !tried_hook) { + tried_hook = 1; + if (!read_object_process(oid)) + goto retry; + } + + return ret; } int has_loose_object_nonlocal(const struct object_id *oid) @@ -1547,20 +1670,6 @@ void disable_obj_read_lock(void) pthread_mutex_destroy(&obj_read_mutex); } -static int run_read_object_hook(const struct object_id *oid) -{ - struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; - int ret; - uint64_t start; - - start = getnanotime(); - strvec_push(&opt.args, oid_to_hex(oid)); - ret = run_hooks_opt("read-object", &opt); - trace_performance_since(start, "run_read_object_hook"); - - return ret; -} - int fetch_if_missing = 1; static int do_oid_object_info_extended(struct repository *r, @@ -1619,7 +1728,7 @@ static int do_oid_object_info_extended(struct repository *r, break; if (core_virtualize_objects && !tried_hook) { tried_hook = 1; - if (!run_read_object_hook(oid)) + if (!read_object_process(oid)) goto retry; } } diff --git a/t/t0410/read-object b/t/t0410/read-object new file mode 100755 index 00000000000000..2b8feacc78577f --- /dev/null +++ b/t/t0410/read-object @@ -0,0 +1,114 @@ +#!/usr/bin/perl +# +# Example implementation for the Git read-object protocol version 1 +# See Documentation/technical/read-object-protocol.txt +# +# Allows you to test the ability for blobs to be pulled from a host git repo +# "on demand." Called when git needs a blob it couldn't find locally due to +# a lazy clone that only cloned the commits and trees. +# +# A lazy clone can be simulated via the following commands from the host repo +# you wish to create a lazy clone of: +# +# cd /host_repo +# git rev-parse HEAD +# git init /guest_repo +# git cat-file --batch-check --batch-all-objects | grep -v 'blob' | +# cut -d' ' -f1 | git pack-objects /guest_repo/.git/objects/pack/noblobs +# cd /guest_repo +# git config core.virtualizeobjects true +# git reset --hard +# +# Please note, this sample is a minimal skeleton. No proper error handling +# was implemented. +# + +use strict; +use warnings; + +# +# Point $DIR to the folder where your host git repo is located so we can pull +# missing objects from it +# +my $DIR = "../.git/"; + +sub packet_bin_read { + my $buffer; + my $bytes_read = read STDIN, $buffer, 4; + if ( $bytes_read == 0 ) { + + # EOF - Git stopped talking to us! + exit(); + } + elsif ( $bytes_read != 4 ) { + die "invalid packet: '$buffer'"; + } + my $pkt_size = hex($buffer); + if ( $pkt_size == 0 ) { + return ( 1, "" ); + } + elsif ( $pkt_size > 4 ) { + my $content_size = $pkt_size - 4; + $bytes_read = read STDIN, $buffer, $content_size; + if ( $bytes_read != $content_size ) { + die "invalid packet ($content_size bytes expected; $bytes_read bytes read)"; + } + return ( 0, $buffer ); + } + else { + die "invalid packet size: $pkt_size"; + } +} + +sub packet_txt_read { + my ( $res, $buf ) = packet_bin_read(); + unless ( $buf =~ s/\n$// ) { + die "A non-binary line MUST be terminated by an LF."; + } + return ( $res, $buf ); +} + +sub packet_bin_write { + my $buf = shift; + print STDOUT sprintf( "%04x", length($buf) + 4 ); + print STDOUT $buf; + STDOUT->flush(); +} + +sub packet_txt_write { + packet_bin_write( $_[0] . "\n" ); +} + +sub packet_flush { + print STDOUT sprintf( "%04x", 0 ); + STDOUT->flush(); +} + +( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || die "bad initialize"; +( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad version end"; + +packet_txt_write("git-read-object-server"); +packet_txt_write("version=1"); +packet_flush(); + +( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end"; + +packet_txt_write("capability=get"); +packet_flush(); + +while (1) { + my ($command) = packet_txt_read() =~ /^command=([^=]+)$/; + + if ( $command eq "get" ) { + my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40,64})$/; + packet_bin_read(); + + system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); + packet_txt_write(($?) ? "status=error" : "status=success"); + packet_flush(); + } else { + die "bad command '$command'"; + } +} diff --git a/t/t0499-read-object.sh b/t/t0499-read-object.sh new file mode 100755 index 00000000000000..2e208bdb46add5 --- /dev/null +++ b/t/t0499-read-object.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +test_description='tests for long running read-object process' + +. ./test-lib.sh + +test_expect_success 'setup host repo with a root commit' ' + test_commit zero && + hash1=$(git ls-tree HEAD | grep zero.t | cut -f1 | cut -d\ -f3) +' + +test_expect_success 'blobs can be retrieved from the host repo' ' + git init guest-repo && + (cd guest-repo && + mkdir -p .git/hooks && + sed "1s|/usr/bin/perl|$PERL_PATH|" \ + <$TEST_DIRECTORY/t0410/read-object \ + >.git/hooks/read-object && + chmod +x .git/hooks/read-object && + git config core.virtualizeobjects true && + git cat-file blob "$hash1") +' + +test_expect_success 'invalid blobs generate errors' ' + (cd guest-repo && + test_must_fail git cat-file blob "invalid") +' + + +test_done From b1fd1dfc92fdb9d022afc51b917ee0cccfe1ea75 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 8 Sep 2017 11:32:43 +0200 Subject: [PATCH 13/13] sha1_file: when writing objects, skip the read_object_hook If we are going to write an object there is no use in calling the read object hook to get an object from a potentially remote source. We would rather just write out the object and avoid the potential round trip for an object that doesn't exist. This change adds a flag to the check_and_freshen() and freshen_loose_object() functions' signatures so that the hook is bypassed when the functions are called before writing loose objects. The check for a local object is still performed so we don't overwrite something that has already been written to one of the objects directories. Based on a patch by Kevin Willford. Signed-off-by: Johannes Schindelin --- object-file.c | 19 +++++++++++-------- t/t0410/read-object | 4 ++++ t/t0499-read-object.sh | 7 +++++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/object-file.c b/object-file.c index 1b37072b239536..0024936fd210f1 100644 --- a/object-file.c +++ b/object-file.c @@ -1112,7 +1112,8 @@ static int check_and_freshen_nonlocal(const struct object_id *oid, int freshen) return 0; } -static int check_and_freshen(const struct object_id *oid, int freshen) +static int check_and_freshen(const struct object_id *oid, int freshen, + int skip_virtualized_objects) { int ret; int tried_hook = 0; @@ -1120,7 +1121,8 @@ static int check_and_freshen(const struct object_id *oid, int freshen) retry: ret = check_and_freshen_local(oid, freshen) || check_and_freshen_nonlocal(oid, freshen); - if (!ret && core_virtualize_objects && !tried_hook) { + if (!ret && core_virtualize_objects && !skip_virtualized_objects && + !tried_hook) { tried_hook = 1; if (!read_object_process(oid)) goto retry; @@ -1136,7 +1138,7 @@ int has_loose_object_nonlocal(const struct object_id *oid) int has_loose_object(const struct object_id *oid) { - return check_and_freshen(oid, 0); + return check_and_freshen(oid, 0, 0); } static void mmap_limit_check(size_t length) @@ -2321,9 +2323,10 @@ static int write_loose_object(const struct object_id *oid, char *hdr, return finalize_object_file(tmp_file.buf, filename.buf); } -static int freshen_loose_object(const struct object_id *oid) +static int freshen_loose_object(const struct object_id *oid, + int skip_virtualized_objects) { - return check_and_freshen(oid, 1); + return check_and_freshen(oid, 1, skip_virtualized_objects); } static int freshen_packed_object(const struct object_id *oid) @@ -2419,7 +2422,7 @@ int stream_loose_object(struct input_stream *in_stream, size_t len, die(_("deflateEnd on stream object failed (%d)"), ret); close_loose_object(fd, tmp_file.buf); - if (freshen_packed_object(oid) || freshen_loose_object(oid)) { + if (freshen_packed_object(oid) || freshen_loose_object(oid, 1)) { unlink_or_warn(tmp_file.buf); goto cleanup; } @@ -2480,7 +2483,7 @@ int write_object_file_flags(const void *buf, size_t len, * it out into .git/objects/??/?{38} file. */ write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen); - if (freshen_packed_object(oid) || freshen_loose_object(oid)) + if (freshen_packed_object(oid) || freshen_loose_object(oid, 1)) return 0; if (write_loose_object(oid, hdr, hdrlen, buf, len, 0, flags)) return -1; @@ -2524,7 +2527,7 @@ int write_object_file_literally(const void *buf, size_t len, if (!(flags & HASH_WRITE_OBJECT)) goto cleanup; - if (freshen_packed_object(oid) || freshen_loose_object(oid)) + if (freshen_packed_object(oid) || freshen_loose_object(oid, 1)) goto cleanup; status = write_loose_object(oid, header, hdrlen, buf, len, 0, 0); if (compat_type != -1) diff --git a/t/t0410/read-object b/t/t0410/read-object index 2b8feacc78577f..02c799837f4057 100755 --- a/t/t0410/read-object +++ b/t/t0410/read-object @@ -108,6 +108,10 @@ while (1) { system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); packet_txt_write(($?) ? "status=error" : "status=success"); packet_flush(); + + open my $log, '>>.git/read-object-hook.log'; + print $log "Read object $sha1, exit code $?\n"; + close $log; } else { die "bad command '$command'"; } diff --git a/t/t0499-read-object.sh b/t/t0499-read-object.sh index 2e208bdb46add5..0cee1963cf091e 100755 --- a/t/t0499-read-object.sh +++ b/t/t0499-read-object.sh @@ -26,5 +26,12 @@ test_expect_success 'invalid blobs generate errors' ' test_must_fail git cat-file blob "invalid") ' +test_expect_success 'read-object-hook is bypassed when writing objects' ' + (cd guest-repo && + echo hello >hello.txt && + git add hello.txt && + hash="$(git rev-parse --verify :hello.txt)" && + ! grep "$hash" .git/read-object-hook.log) +' test_done