From 0082dc7955de151a124ff7ddfdeddda6ff0c7920 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 29 Apr 2024 08:55:03 -0400 Subject: [PATCH 001/189] survey: stub in new experimental `git-survey` command Start work on a new `git survey` command to scan the repository for monorepo performance and scaling problems. The goal is to measure the various known "dimensions of scale" and serve as a foundation for adding additional measurements as we learn more about Git monorepo scaling problems. Results will be logged to the console and to Trace2. The initial goal is to complement the scanning and analysis performed by the GO-based `git-sizer` (https://github.com/github/git-sizer) tool. It is hoped that by creating a builtin command, we may be able to take advantage of internal Git data structures and code that is not accessible from GO to gain further insight into potential scaling problems. Signed-off-by: Jeff Hostetler --- .gitignore | 1 + Documentation/git-survey.txt | 36 ++++++++++++++++++++++ Makefile | 1 + builtin.h | 1 + builtin/survey.c | 58 ++++++++++++++++++++++++++++++++++++ command-list.txt | 1 + git.c | 1 + 7 files changed, 99 insertions(+) create mode 100644 Documentation/git-survey.txt create mode 100644 builtin/survey.c diff --git a/.gitignore b/.gitignore index bf97276163b19b..8f82ba30d5a17a 100644 --- a/.gitignore +++ b/.gitignore @@ -164,6 +164,7 @@ /git-submodule /git-submodule--helper /git-subtree +/git-survey /git-svn /git-switch /git-symbolic-ref diff --git a/Documentation/git-survey.txt b/Documentation/git-survey.txt new file mode 100644 index 00000000000000..cdd1ec4358b8bb --- /dev/null +++ b/Documentation/git-survey.txt @@ -0,0 +1,36 @@ +git-survey(1) +============= + +NAME +---- +git-survey - EXPERIMENTAL: Measure various repository dimensions of scale + +SYNOPSIS +-------- +[verse] +(EXPERIMENTAL!) `git survey` + +DESCRIPTION +----------- + +Survey the repository and measure various dimensions of scale. + +As repositories grow to "monorepo" size, certain data shapes can cause +performance problems. `git-survey` attempts to measure and report on +known problem areas. + +OPTIONS +------- + +--progress:: + Show progress. This is automatically enabled when interactive. + +OUTPUT +------ + +By default, `git survey` will print information about the repository in a +human-readable format that includes overviews and tables. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Makefile b/Makefile index b72d0730199cbc..4ed38a08d0b2a2 100644 --- a/Makefile +++ b/Makefile @@ -1308,6 +1308,7 @@ BUILTIN_OBJS += builtin/sparse-checkout.o BUILTIN_OBJS += builtin/stash.o BUILTIN_OBJS += builtin/stripspace.o BUILTIN_OBJS += builtin/submodule--helper.o +BUILTIN_OBJS += builtin/survey.o BUILTIN_OBJS += builtin/symbolic-ref.o BUILTIN_OBJS += builtin/tag.o BUILTIN_OBJS += builtin/unpack-file.o diff --git a/builtin.h b/builtin.h index 14fa0171607b17..8ab1623b32679c 100644 --- a/builtin.h +++ b/builtin.h @@ -238,6 +238,7 @@ int cmd_status(int argc, const char **argv, const char *prefix); int cmd_stash(int argc, const char **argv, const char *prefix); int cmd_stripspace(int argc, const char **argv, const char *prefix); int cmd_submodule__helper(int argc, const char **argv, const char *prefix); +int cmd_survey(int argc, const char **argv, const char *prefix); int cmd_switch(int argc, const char **argv, const char *prefix); int cmd_symbolic_ref(int argc, const char **argv, const char *prefix); int cmd_tag(int argc, const char **argv, const char *prefix); diff --git a/builtin/survey.c b/builtin/survey.c new file mode 100644 index 00000000000000..730ad9e4552048 --- /dev/null +++ b/builtin/survey.c @@ -0,0 +1,58 @@ +#include "builtin.h" +#include "config.h" +#include "parse-options.h" + +static const char * const survey_usage[] = { + N_("(EXPERIMENTAL!) git survey "), + NULL, +}; + +struct survey_opts { + int verbose; + int show_progress; +}; + +static struct survey_opts survey_opts = { + .verbose = 0, + .show_progress = -1, /* defaults to isatty(2) */ +}; + +static struct option survey_options[] = { + OPT__VERBOSE(&survey_opts.verbose, N_("verbose output")), + OPT_BOOL(0, "progress", &survey_opts.show_progress, N_("show progress")), + OPT_END(), +}; + +static int survey_load_config_cb(const char *var, const char *value, + const struct config_context *ctx, void *pvoid) +{ + if (!strcmp(var, "survey.verbose")) { + survey_opts.verbose = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "survey.progress")) { + survey_opts.show_progress = git_config_bool(var, value); + return 0; + } + + return git_default_config(var, value, ctx, pvoid); +} + +static void survey_load_config(void) +{ + git_config(survey_load_config_cb, NULL); +} + +int cmd_survey(int argc, const char **argv, const char *prefix) +{ + survey_load_config(); + + argc = parse_options(argc, argv, prefix, survey_options, survey_usage, 0); + + prepare_repo_settings(the_repository); + + if (survey_opts.show_progress < 0) + survey_opts.show_progress = isatty(2); + + return 0; +} diff --git a/command-list.txt b/command-list.txt index e0bb87b3b5c278..d389561a5f1161 100644 --- a/command-list.txt +++ b/command-list.txt @@ -186,6 +186,7 @@ git-stash mainporcelain git-status mainporcelain info git-stripspace purehelpers git-submodule mainporcelain +git-survey mainporcelain git-svn foreignscminterface git-switch mainporcelain history git-symbolic-ref plumbingmanipulators diff --git a/git.c b/git.c index e35af9b0e5e976..7c70fff6218d90 100644 --- a/git.c +++ b/git.c @@ -622,6 +622,7 @@ static struct cmd_struct commands[] = { { "status", cmd_status, RUN_SETUP | NEED_WORK_TREE }, { "stripspace", cmd_stripspace }, { "submodule--helper", cmd_submodule__helper, RUN_SETUP }, + { "survey", cmd_survey, RUN_SETUP }, { "switch", cmd_switch, RUN_SETUP | NEED_WORK_TREE }, { "symbolic-ref", cmd_symbolic_ref, RUN_SETUP }, { "tag", cmd_tag, RUN_SETUP | DELAY_PAGER_CONFIG }, From 5a54e19154ae92d8e01d561e55fd8fb5d315378e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 29 Apr 2024 09:51:34 -0400 Subject: [PATCH 002/189] survey: add command line opts to select references By default we will scan all references in "refs/heads/", "refs/tags/" and "refs/remotes/". Add command line opts let the use ask for all refs or a subset of them and to include a detached HEAD. Signed-off-by: Jeff Hostetler --- Documentation/git-survey.txt | 34 +++++++++++++ builtin/survey.c | 99 ++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/Documentation/git-survey.txt b/Documentation/git-survey.txt index cdd1ec4358b8bb..c648ef704e3806 100644 --- a/Documentation/git-survey.txt +++ b/Documentation/git-survey.txt @@ -19,12 +19,46 @@ As repositories grow to "monorepo" size, certain data shapes can cause performance problems. `git-survey` attempts to measure and report on known problem areas. +Ref Selection and Reachable Objects +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this first analysis phase, `git survey` will iterate over the set of +requested branches, tags, and other refs and treewalk over all of the +reachable commits, trees, and blobs and generate various statistics. + OPTIONS ------- --progress:: Show progress. This is automatically enabled when interactive. +Ref Selection +~~~~~~~~~~~~~ + +The following options control the set of refs that `git survey` will examine. +By default, `git survey` will look at tags, local branches, and remote refs. +If any of the following options are given, the default set is cleared and +only refs for the given options are added. + +--all-refs:: + Use all refs. This includes local branches, tags, remote refs, + notes, and stashes. This option overrides all of the following. + +--branches:: + Add local branches (`refs/heads/`) to the set. + +--tags:: + Add tags (`refs/tags/`) to the set. + +--remotes:: + Add remote branches (`refs/remote/`) to the set. + +--detached:: + Add HEAD to the set. + +--other:: + Add notes (`refs/notes/`) and stashes (`refs/stash/`) to the set. + OUTPUT ------ diff --git a/builtin/survey.c b/builtin/survey.c index 730ad9e4552048..443af35954d590 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -7,19 +7,117 @@ static const char * const survey_usage[] = { NULL, }; +struct survey_refs_wanted { + int want_all_refs; /* special override */ + + int want_branches; + int want_tags; + int want_remotes; + int want_detached; + int want_other; /* see FILTER_REFS_OTHERS -- refs/notes/, refs/stash/ */ +}; + +/* + * The set of refs that we will search if the user doesn't select + * any on the command line. + */ +static struct survey_refs_wanted refs_if_unspecified = { + .want_all_refs = 0, + + .want_branches = 1, + .want_tags = 1, + .want_remotes = 1, + .want_detached = 0, + .want_other = 0, +}; + struct survey_opts { int verbose; int show_progress; + struct survey_refs_wanted refs; }; static struct survey_opts survey_opts = { .verbose = 0, .show_progress = -1, /* defaults to isatty(2) */ + + .refs.want_all_refs = -1, + + .refs.want_branches = -1, /* default these to undefined */ + .refs.want_tags = -1, + .refs.want_remotes = -1, + .refs.want_detached = -1, + .refs.want_other = -1, }; +/* + * After parsing the command line arguments, figure out which refs we + * should scan. + * + * If ANY were given in positive sense, then we ONLY include them and + * do not use the builtin values. + */ +static void fixup_refs_wanted(void) +{ + struct survey_refs_wanted *rw = &survey_opts.refs; + + /* + * `--all-refs` overrides and enables everything. + */ + if (rw->want_all_refs == 1) { + rw->want_branches = 1; + rw->want_tags = 1; + rw->want_remotes = 1; + rw->want_detached = 1; + rw->want_other = 1; + return; + } + + /* + * If none of the `--` were given, we assume all + * of the builtin unspecified values. + */ + if (rw->want_branches == -1 && + rw->want_tags == -1 && + rw->want_remotes == -1 && + rw->want_detached == -1 && + rw->want_other == -1) { + *rw = refs_if_unspecified; + return; + } + + /* + * Since we only allow positive boolean values on the command + * line, we will only have true values where they specified + * a `--`. + * + * So anything that still has an unspecified value should be + * set to false. + */ + if (rw->want_branches == -1) + rw->want_branches = 0; + if (rw->want_tags == -1) + rw->want_tags = 0; + if (rw->want_remotes == -1) + rw->want_remotes = 0; + if (rw->want_detached == -1) + rw->want_detached = 0; + if (rw->want_other == -1) + rw->want_other = 0; +} + static struct option survey_options[] = { OPT__VERBOSE(&survey_opts.verbose, N_("verbose output")), OPT_BOOL(0, "progress", &survey_opts.show_progress, N_("show progress")), + + OPT_BOOL_F(0, "all-refs", &survey_opts.refs.want_all_refs, N_("include all refs"), PARSE_OPT_NONEG), + + OPT_BOOL_F(0, "branches", &survey_opts.refs.want_branches, N_("include branches"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "tags", &survey_opts.refs.want_tags, N_("include tags"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "remotes", &survey_opts.refs.want_remotes, N_("include all remotes refs"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "detached", &survey_opts.refs.want_detached, N_("include detached HEAD"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "other", &survey_opts.refs.want_other, N_("include notes and stashes"), PARSE_OPT_NONEG), + OPT_END(), }; @@ -53,6 +151,7 @@ int cmd_survey(int argc, const char **argv, const char *prefix) if (survey_opts.show_progress < 0) survey_opts.show_progress = isatty(2); + fixup_refs_wanted(); return 0; } From ca6d3d8a5721921b07321082bc7e4234e4017e2e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 29 Apr 2024 10:38:17 -0400 Subject: [PATCH 003/189] survey: collect the set of requested refs Collect the set of requested branches, tags, and etc into a ref_array and collect the set of requested patterns into a strvec. Signed-off-by: Jeff Hostetler --- builtin/survey.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/builtin/survey.c b/builtin/survey.c index 443af35954d590..9d7faa71ebc75d 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -1,12 +1,19 @@ #include "builtin.h" #include "config.h" #include "parse-options.h" +#include "progress.h" +#include "ref-filter.h" +#include "strvec.h" +#include "trace2.h" static const char * const survey_usage[] = { N_("(EXPERIMENTAL!) git survey "), NULL, }; +static struct progress *survey_progress = NULL; +static uint64_t survey_progress_total = 0; + struct survey_refs_wanted { int want_all_refs; /* special override */ @@ -17,6 +24,8 @@ struct survey_refs_wanted { int want_other; /* see FILTER_REFS_OTHERS -- refs/notes/, refs/stash/ */ }; +static struct strvec survey_vec_refs_wanted = STRVEC_INIT; + /* * The set of refs that we will search if the user doesn't select * any on the command line. @@ -141,6 +150,83 @@ static void survey_load_config(void) git_config(survey_load_config_cb, NULL); } +static void do_load_refs(struct ref_array *ref_array) +{ + struct ref_filter filter = REF_FILTER_INIT; + struct ref_sorting *sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; + + string_list_append(&sorting_options, "objectname"); + sorting = ref_sorting_options(&sorting_options); + + if (survey_opts.refs.want_detached) + strvec_push(&survey_vec_refs_wanted, "HEAD"); + + if (survey_opts.refs.want_all_refs) { + strvec_push(&survey_vec_refs_wanted, "refs/"); + } else { + if (survey_opts.refs.want_branches) + strvec_push(&survey_vec_refs_wanted, "refs/heads/"); + if (survey_opts.refs.want_tags) + strvec_push(&survey_vec_refs_wanted, "refs/tags/"); + if (survey_opts.refs.want_remotes) + strvec_push(&survey_vec_refs_wanted, "refs/remotes/"); + if (survey_opts.refs.want_other) { + strvec_push(&survey_vec_refs_wanted, "refs/notes/"); + strvec_push(&survey_vec_refs_wanted, "refs/stash/"); + } + } + + filter.name_patterns = survey_vec_refs_wanted.v; + filter.ignore_case = 0; + filter.match_as_path = 1; + + if (survey_opts.show_progress) { + survey_progress_total = 0; + survey_progress = start_progress(_("Scanning refs..."), 0); + } + + filter_refs(ref_array, &filter, FILTER_REFS_KIND_MASK); + + if (survey_opts.show_progress) { + survey_progress_total = ref_array->nr; + display_progress(survey_progress, survey_progress_total); + } + + ref_array_sort(sorting, ref_array); + + if (survey_opts.show_progress) + stop_progress(&survey_progress); + + ref_filter_clear(&filter); + ref_sorting_release(sorting); +} + +/* + * The REFS phase: + * + * Load the set of requested refs and assess them for scalablity problems. + * Use that set to start a treewalk to all reachable objects and assess + * them. + * + * This data will give us insights into the repository itself (the number + * of refs, the size and shape of the DAG, the number and size of the + * objects). + * + * Theoretically, this data is independent of the on-disk representation + * (e.g. independent of packing concerns). + */ +static void survey_phase_refs(struct repository *r) +{ + struct ref_array ref_array = { 0 }; + + trace2_region_enter("survey", "phase/refs", the_repository); + do_load_refs(&ref_array); + trace2_region_leave("survey", "phase/refs", the_repository); + + ref_array_clear(&ref_array); +} + int cmd_survey(int argc, const char **argv, const char *prefix) { survey_load_config(); @@ -153,5 +239,9 @@ int cmd_survey(int argc, const char **argv, const char *prefix) survey_opts.show_progress = isatty(2); fixup_refs_wanted(); + survey_phase_refs(the_repository); + + strvec_clear(&survey_vec_refs_wanted); + return 0; } From 9c19694e5ce96c332ebdeda927e7a0e51732f8dd Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Tue, 19 Sep 2023 18:20:32 -0700 Subject: [PATCH 004/189] sparse-index.c: fix use of index hashes in expand_index In ac8acb4f2c7 (sparse-index: complete partial expansion, 2022-05-23), 'expand_index()' was updated to expand the index to a given pathspec. However, the 'path_matches_pattern_list()' method used to facilitate this has the side effect of initializing or updating the index hash variables ('name_hash', 'dir_hash', and 'name_hash_initialized'). This operation is performed on 'istate', though, not 'full'; as a result, the initialized hashes are later overwritten when copied from 'full'. To ensure the correct hashes are in 'istate' after the index expansion, change the arg used in 'path_matches_pattern_list()' from 'istate' to 'full'. Note that this does not fully solve the problem. If 'istate' does not have an initialized 'name_hash' when its contents are copied to 'full', initialized hashes will be copied back into 'istate' but 'name_hash_initialized' will be 0. Therefore, we also need to copy 'full->name_hash_initialized' back to 'istate' after the index expansion is complete. Signed-off-by: Victoria Dye --- sparse-index.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sparse-index.c b/sparse-index.c index 9958656ded1931..e1cc61f80eb404 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -399,7 +399,7 @@ void expand_index(struct index_state *istate, struct pattern_list *pl) if (pl && path_matches_pattern_list(ce->name, ce->ce_namelen, NULL, &dtype, - pl, istate) == NOT_MATCHED) { + pl, full) == NOT_MATCHED) { set_index_entry(full, full->cache_nr++, ce); continue; } @@ -427,6 +427,7 @@ void expand_index(struct index_state *istate, struct pattern_list *pl) } /* Copy back into original index. */ + istate->name_hash_initialized = full->name_hash_initialized; memcpy(&istate->name_hash, &full->name_hash, sizeof(full->name_hash)); memcpy(&istate->dir_hash, &full->dir_hash, sizeof(full->dir_hash)); istate->sparse_index = pl ? INDEX_PARTIALLY_SPARSE : INDEX_EXPANDED; From 297189c6a550d7fdfaeeb663dc9d8fc53fa60620 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Aug 2023 09:58:27 -0400 Subject: [PATCH 005/189] t5300: confirm failure of git index-pack when non-idx suffix requested Add test case to demonstrate that `git index-pack -o pack-path` fails if does not end in ".idx" when `--rev-index` is enabled. In e37d0b8730b (builtin/index-pack.c: write reverse indexes, 2021-01-25) we learned to create `.rev` reverse indexes in addition to `.idx` index files. The `.rev` file pathname is constructed by replacing the suffix on the `.idx` file. The code assumes a hard-coded "idx" suffix. In a8dd7e05b1c (config: enable `pack.writeReverseIndex` by default, 2023-04-12) reverse indexes were enabled by default. If the `-o ` argument is used, the index file may have a different suffix. This causes an error when it tries to create the reverse index pathname. The test here demonstrates the failure. (The test forces `--rev-index` to avoid interaction with `GIT_TEST_NO_WRITE_REV_INDEX` during CI runs.) Signed-off-by: Jeff Hostetler --- t/t5300-pack-object.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh index 3b9dae331a5ea9..1b0602216eae07 100755 --- a/t/t5300-pack-object.sh +++ b/t/t5300-pack-object.sh @@ -355,6 +355,31 @@ test_expect_success 'build pack index for an existing pack' ' : ' +# The `--rev-index` option of `git index-pack` is now the default, so +# a `foo.rev` REV file will be created when a `foo.idx` IDX file is +# created. Normally, these pathnames are based upon the `foo.pack` +# PACK file pathname. +# +# However, the `-o` option lets you set the pathname of the IDX file +# indepdent of the PACK file. +# +# Verify what happens if these suffixes are changed. +# +test_expect_success 'complain about index name' ' + # Normal case { .pack, .idx, .rev } + cat test-1-${packname_1}.pack >test-complain-0.pack && + git index-pack -o test-complain-0.idx --rev-index test-complain-0.pack && + test -f test-complain-0.idx && + test -f test-complain-0.rev && + + # Non .idx suffix + cat test-1-${packname_1}.pack >test-complain-1.pack && + test_must_fail git index-pack -o test-complain-1.idx-suffix --rev-index test-complain-1.pack 2>err && + grep "does not end" err && + ! test -f test-complain-1.idx-suffix && + ! test -f test-complain-1.rev +' + test_expect_success 'unpacking with --strict' ' for j in a b c d e f g From 4c9a2e269d51eef1489dbab5a09fcf48a6ede7a6 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 29 Apr 2024 12:47:27 -0400 Subject: [PATCH 006/189] survey: calculate stats on refs and print results Calculate stats on the set of refs. This includes the number of branches, plain and annotated tags, remotes, and etc. Calculate the number of packed vs loose refs. Calculate the size of the set of refnames. Print results on the console in JSON format. Add Trace2 logging of the results as a data_json event. Signed-off-by: Jeff Hostetler --- builtin/survey.c | 345 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 344 insertions(+), 1 deletion(-) diff --git a/builtin/survey.c b/builtin/survey.c index 9d7faa71ebc75d..8e1a0e4e871246 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -1,8 +1,13 @@ #include "builtin.h" #include "config.h" +#include "json-writer.h" +#include "object-store.h" #include "parse-options.h" #include "progress.h" #include "ref-filter.h" +#include "refs.h" +#include "strbuf.h" +#include "strmap.h" #include "strvec.h" #include "trace2.h" @@ -50,7 +55,7 @@ static struct survey_opts survey_opts = { .verbose = 0, .show_progress = -1, /* defaults to isatty(2) */ - .refs.want_all_refs = -1, + .refs.want_all_refs = 0, .refs.want_branches = -1, /* default these to undefined */ .refs.want_tags = -1, @@ -150,6 +155,43 @@ static void survey_load_config(void) git_config(survey_load_config_cb, NULL); } +/* + * Stats on the set of refs that we found. + */ +struct survey_stats_refs { + uint32_t cnt_total; + uint32_t cnt_lightweight_tags; + uint32_t cnt_annotated_tags; + uint32_t cnt_branches; + uint32_t cnt_remotes; + uint32_t cnt_detached; + uint32_t cnt_other; + + uint32_t cnt_symref; + + uint32_t cnt_packed; + uint32_t cnt_loose; + + /* + * Measure the length of the refnames. We can look for + * potential platform limits. The partial sums may help us + * estimate the size of a haves/wants conversation, since each + * refname and a SHA must be transmitted. + */ + size_t len_max_local_refname; + size_t len_sum_local_refnames; + size_t len_max_remote_refname; + size_t len_sum_remote_refnames; + + struct strintmap refsmap; +}; + +struct survey_stats { + struct survey_stats_refs refs; +}; + +static struct survey_stats survey_stats = { 0 }; + static void do_load_refs(struct ref_array *ref_array) { struct ref_filter filter = REF_FILTER_INIT; @@ -202,6 +244,178 @@ static void do_load_refs(struct ref_array *ref_array) ref_sorting_release(sorting); } +/* + * If we want this type of ref, increment counters and return 1. + */ +static int maybe_count_ref(struct repository *r, struct ref_array_item *p) +{ + struct survey_refs_wanted *rw = &survey_opts.refs; + struct survey_stats_refs *prs = &survey_stats.refs; + struct object_id peeled; + + /* + * Classify the ref using the `kind` value. Note that + * p->kind was populated by `ref_kind_from_refname()` + * based strictly on the refname. This only knows about + * the basic stock categories and returns FILTER_REFS_OTHERS + * for notes, stashes, and any custom namespaces (like + * "refs/prefetch/"). + */ + switch (p->kind) { + case FILTER_REFS_TAGS: + if (rw->want_all_refs || rw->want_tags) { + /* + * NEEDSWORK: Both types of tags have the same + * "refs/tags/" prefix. Do we want to count them + * in separate buckets in the refsmap? + */ + strintmap_incr(&prs->refsmap, "refs/tags/", 1); + + if (!peel_iterated_oid(r, &p->objectname, &peeled)) + prs->cnt_annotated_tags++; + else + prs->cnt_lightweight_tags++; + + return 1; + } + return 0; + + case FILTER_REFS_BRANCHES: + if (rw->want_all_refs || rw->want_branches) { + strintmap_incr(&prs->refsmap, "refs/heads/", 1); + + prs->cnt_branches++; + return 1; + } + return 0; + + case FILTER_REFS_REMOTES: + if (rw->want_all_refs || rw->want_remotes) { + /* + * For the refsmap, group them by the "refs/remotes//". + * For example: + * "refs/remotes/origin/..." + */ + if (starts_with(p->refname, "refs/remotes/")) { + struct strbuf buf = STRBUF_INIT; + int begin = strlen("refs/remotes/"); + size_t j; + + strbuf_addstr(&buf, p->refname); + for (j = begin; j < buf.len; j++) { + if (buf.buf[j] == '/') { + strbuf_setlen(&buf, j+1); + break; + } + } + strintmap_incr(&prs->refsmap, buf.buf, 1); + strbuf_release(&buf); + } + + prs->cnt_remotes++; + return 1; + } + return 0; + + case FILTER_REFS_OTHERS: + if (rw->want_all_refs || rw->want_other) { + /* + * For the refsmap, group them by their "refs//". + * For example: + * "refs/notes/..." + * "refs/stash/..." + * "refs//..." + */ + if (starts_with(p->refname, "refs/")) { + struct strbuf buf = STRBUF_INIT; + int begin = strlen("refs/"); + size_t j; + + strbuf_addstr(&buf, p->refname); + for (j = begin; j < buf.len; j++) { + if (buf.buf[j] == '/') { + strbuf_setlen(&buf, j+1); + break; + } + } + strintmap_incr(&prs->refsmap, buf.buf, 1); + strbuf_release(&buf); + } + + prs->cnt_other++; + return 1; + } + return 0; + + case FILTER_REFS_DETACHED_HEAD: + if (rw->want_all_refs || rw->want_detached) { + strintmap_incr(&prs->refsmap, p->refname, 1); + + prs->cnt_detached++; + return 1; + } + return 0; + + default: + if (rw->want_all_refs) { + strintmap_incr(&prs->refsmap, p->refname, 1); /* probably "HEAD" */ + + return 1; + } + return 0; + } +} + +/* + * Calculate stats on the set of refs that we found. + */ +static void do_calc_stats_refs(struct repository *r, struct ref_array *ref_array) +{ + struct survey_stats_refs *prs = &survey_stats.refs; + int k; + + strintmap_init(&prs->refsmap, 0); + + for (k = 0; k < ref_array->nr; k++) { + struct ref_array_item *p = ref_array->items[k]; + size_t len; + + if (!maybe_count_ref(r, p)) + continue; + + prs->cnt_total++; + + /* + * SymRefs are somewhat orthogonal to the above + * classification (e.g. "HEAD" --> detached + * and "refs/remotes/origin/HEAD" --> remote) so + * our totals will already include them. + */ + if (p->flag & REF_ISSYMREF) + prs->cnt_symref++; + + /* + * Where/how is the ref stored in GITDIR. + */ + if (p->flag & REF_ISPACKED) + prs->cnt_packed++; + else + prs->cnt_loose++; + + len = strlen(p->refname); + + if (p->kind == FILTER_REFS_REMOTES) { + prs->len_sum_remote_refnames += len; + if (len > prs->len_max_remote_refname) + prs->len_max_remote_refname = len; + } else { + prs->len_sum_local_refnames += len; + if (len > prs->len_max_local_refname) + prs->len_max_local_refname = len; + } + } +} + /* * The REFS phase: * @@ -224,9 +438,135 @@ static void survey_phase_refs(struct repository *r) do_load_refs(&ref_array); trace2_region_leave("survey", "phase/refs", the_repository); + trace2_region_enter("survey", "phase/calcstats", the_repository); + do_calc_stats_refs(r, &ref_array); + trace2_region_leave("survey", "phase/calcstats", the_repository); + ref_array_clear(&ref_array); } +static void json_refs_section(struct json_writer *jw_top, int pretty, int want_trace2) +{ + struct survey_stats_refs *prs = &survey_stats.refs; + struct json_writer jw_refs = JSON_WRITER_INIT; + int k; + + jw_object_begin(&jw_refs, pretty); + { + jw_object_intmax(&jw_refs, "count", prs->cnt_total); + + jw_object_inline_begin_object(&jw_refs, "count_by_type"); + { + if (survey_opts.refs.want_branches) + jw_object_intmax(&jw_refs, "branches", prs->cnt_branches); + if (survey_opts.refs.want_tags) { + jw_object_intmax(&jw_refs, "lightweight_tags", prs->cnt_lightweight_tags); + jw_object_intmax(&jw_refs, "annotated_tags", prs->cnt_annotated_tags); + } + if (survey_opts.refs.want_remotes) + jw_object_intmax(&jw_refs, "remotes", prs->cnt_remotes); + if (survey_opts.refs.want_detached) + jw_object_intmax(&jw_refs, "detached", prs->cnt_detached); + if (survey_opts.refs.want_other) + jw_object_intmax(&jw_refs, "other", prs->cnt_other); + + /* + * SymRefs are somewhat orthogonal to + * the above classification + * (e.g. "HEAD" --> detached and + * "refs/remotes/origin/HEAD" --> + * remote) so the above classified + * counts will already include them, + * but it is less confusing to display + * them here than to create a whole + * new section. + */ + if (prs->cnt_symref) + jw_object_intmax(&jw_refs, "symrefs", prs->cnt_symref); + } + jw_end(&jw_refs); + + jw_object_inline_begin_object(&jw_refs, "count_by_storage"); + { + jw_object_intmax(&jw_refs, "loose_refs", prs->cnt_loose); + jw_object_intmax(&jw_refs, "packed_refs", prs->cnt_packed); + } + jw_end(&jw_refs); + + jw_object_inline_begin_object(&jw_refs, "refname_length"); + { + if (prs->len_sum_local_refnames) { + jw_object_intmax(&jw_refs, "max_local", prs->len_max_local_refname); + jw_object_intmax(&jw_refs, "sum_local", prs->len_sum_local_refnames); + } + if (prs->len_sum_remote_refnames) { + jw_object_intmax(&jw_refs, "max_remote", prs->len_max_remote_refname); + jw_object_intmax(&jw_refs, "sum_remote", prs->len_sum_remote_refnames); + } + } + jw_end(&jw_refs); + + jw_object_inline_begin_array(&jw_refs, "requested"); + { + for (k = 0; k < survey_vec_refs_wanted.nr; k++) + jw_array_string(&jw_refs, survey_vec_refs_wanted.v[k]); + } + jw_end(&jw_refs); + + jw_object_inline_begin_array(&jw_refs, "count_by_class"); + { + struct hashmap_iter iter; + struct strmap_entry *entry; + + strintmap_for_each_entry(&prs->refsmap, &iter, entry) { + const char *key = entry->key; + intptr_t count = (intptr_t)entry->value; + int value = count; + jw_array_inline_begin_object(&jw_refs); + { + jw_object_string(&jw_refs, "class", key); + jw_object_intmax(&jw_refs, "count", value); + } + jw_end(&jw_refs); + } + } + jw_end(&jw_refs); + } + jw_end(&jw_refs); + + if (jw_top) + jw_object_sub_jw(jw_top, "refs", &jw_refs); + + if (want_trace2) + trace2_data_json("survey", the_repository, "refs", &jw_refs); + + jw_release(&jw_refs); +} + +static void survey_print_json(void) +{ + struct json_writer jw_top = JSON_WRITER_INIT; + int pretty = 1; + + jw_object_begin(&jw_top, pretty); + { + json_refs_section(&jw_top, pretty, 0); + } + jw_end(&jw_top); + + printf("%s\n", jw_top.json.buf); + + jw_release(&jw_top); +} + +static void survey_emit_trace2(void) +{ + if (!trace2_is_enabled()) + return; + + json_refs_section(NULL, 0, 1); +} + int cmd_survey(int argc, const char **argv, const char *prefix) { survey_load_config(); @@ -241,6 +581,9 @@ int cmd_survey(int argc, const char **argv, const char *prefix) survey_phase_refs(the_repository); + survey_emit_trace2(); + survey_print_json(); + strvec_clear(&survey_vec_refs_wanted); return 0; From 3c7548c44602ee57280515fd8330a9a671c94577 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 16 Jul 2024 09:17:15 -0400 Subject: [PATCH 007/189] t: remove advice from some tests These seem to be custom tests to microsoft/git as they break without these changes, but these changes are not needed upstream. Signed-off-by: Derrick Stolee --- t/t1091-sparse-checkout-builtin.sh | 1 + t/t7002-mv-sparse-checkout.sh | 3 +++ 2 files changed, 4 insertions(+) diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 8c5cd651b4b80b..1d87cf611c0b7a 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -702,6 +702,7 @@ test_expect_success 'pattern-checks: contained glob characters' ' test_expect_success BSLASHPSPEC 'pattern-checks: escaped characters' ' git clone repo escaped && + git -C escaped config advice.sparseIndexExpanded false && TREEOID=$(git -C escaped rev-parse HEAD:folder1) && NEWTREE=$(git -C escaped mktree <<-EOF $(git -C escaped ls-tree HEAD) diff --git a/t/t7002-mv-sparse-checkout.sh b/t/t7002-mv-sparse-checkout.sh index 57969ce805a548..044038ae40a92b 100755 --- a/t/t7002-mv-sparse-checkout.sh +++ b/t/t7002-mv-sparse-checkout.sh @@ -156,6 +156,9 @@ test_expect_success 'mv refuses to move sparse-to-non-sparse' ' test_expect_success 'recursive mv refuses to move (possible) sparse' ' test_when_finished rm -rf b c e sub2 && + + git config advice.sparseIndexExpanded false && + git reset --hard && # Without cone mode, "sub" and "sub2" do not match git sparse-checkout set sub/dir sub2/dir && From 0ffd36c5cf4e5c6523d84f7ee322248b6ba65a49 Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Wed, 20 Sep 2023 13:12:30 -0700 Subject: [PATCH 008/189] t1092: add test for untracked files and directories Add a test verifying that sparse-checkout (with and without sparse index enabled) treat untracked files & directories correctly when changing sparse patterns. Specifically, it ensures that 'git sparse-checkout set' * deletes empty directories outside the sparse cone * does _not_ delete untracked files outside the sparse cone Signed-off-by: Victoria Dye --- t/t1092-sparse-checkout-compatibility.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index a2c0e1b4dcc564..3e27d270f61ef4 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -313,6 +313,22 @@ test_expect_success 'root directory cannot be sparse' ' test_cmp expect actual ' +test_expect_success 'sparse-checkout with untracked files and dirs' ' + init_repos && + + # Empty directories outside sparse cone are deleted + run_on_sparse mkdir -p deep/empty && + test_sparse_match git sparse-checkout set folder1 && + test_must_be_empty sparse-checkout-err && + run_on_sparse test_path_is_missing deep && + + # Untracked files outside sparse cone are not deleted + run_on_sparse touch folder1/another && + test_sparse_match git sparse-checkout set folder2 && + grep "directory ${SQ}folder1/${SQ} contains untracked files" sparse-checkout-err && + run_on_sparse test_path_exists folder1/another +' + test_expect_success 'status with options' ' init_repos && test_sparse_match ls && From 22f52166515c4938a86279ab533db34b26850404 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Aug 2023 11:06:28 -0400 Subject: [PATCH 009/189] index-pack: disable rev-index if index file has non .idx suffix Teach index-pack to silently omit the reverse index if the index file does not have the standard ".idx" suffix. In e37d0b8730b (builtin/index-pack.c: write reverse indexes, 2021-01-25) we learned to create `.rev` reverse indexes in addition to `.idx` index files. The `.rev` file pathname is constructed by replacing the suffix on the `.idx` file. The code assumes a hard-coded "idx" suffix. In a8dd7e05b1c (config: enable `pack.writeReverseIndex` by default, 2023-04-12) reverse indexes were enabled by default. If the `-o ` argument is used, the index file may have a different suffix. This causes an error when it tries to create the reverse index pathname. Since we do not know why the user requested a non-standard suffix for the index, we cannot guess what the proper corresponding suffix should be for the reverse index. So we disable it. The t5300 test has been updated to verify that we no longer error out and that the .rev file is not created. TODO We could warn the user that we skipped it (perhaps only if they TODO explicitly requested `--rev-index` on the command line). TODO TODO Ideally, we should add an `--rev-index-path=` argument TODO or change `--rev-index` to take a pathname. TODO TODO I'll leave these questions for a future series. Signed-off-by: Jeff Hostetler --- builtin/index-pack.c | 4 ++++ t/t5300-pack-object.sh | 7 +++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 763b01372aade4..07b86c68f9b7c6 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -1734,6 +1734,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix) unsigned foreign_nr = 1; /* zero is a "good" value, assume bad */ int report_end_of_input = 0; int hash_algo = 0; + int dash_o = 0; /* * index-pack never needs to fetch missing objects except when @@ -1827,6 +1828,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix) if (index_name || (i+1) >= argc) usage(index_pack_usage); index_name = argv[++i]; + dash_o = 1; } else if (starts_with(arg, "--index-version=")) { char *c; opts.version = strtoul(arg + 16, &c, 10); @@ -1878,6 +1880,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix) repo_set_hash_algo(the_repository, GIT_HASH_SHA1); opts.flags &= ~(WRITE_REV | WRITE_REV_VERIFY); + if (rev_index && dash_o && !ends_with(index_name, ".idx")) + rev_index = 0; if (rev_index) { opts.flags |= verify ? WRITE_REV_VERIFY : WRITE_REV; if (index_name) diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh index 1b0602216eae07..e041481e71c29e 100755 --- a/t/t5300-pack-object.sh +++ b/t/t5300-pack-object.sh @@ -372,11 +372,10 @@ test_expect_success 'complain about index name' ' test -f test-complain-0.idx && test -f test-complain-0.rev && - # Non .idx suffix + # Non .idx suffix -- implicitly omits the .rev cat test-1-${packname_1}.pack >test-complain-1.pack && - test_must_fail git index-pack -o test-complain-1.idx-suffix --rev-index test-complain-1.pack 2>err && - grep "does not end" err && - ! test -f test-complain-1.idx-suffix && + git index-pack -o test-complain-1.idx-suffix --rev-index test-complain-1.pack && + test -f test-complain-1.idx-suffix && ! test -f test-complain-1.rev ' From d361c6d494e4530b9f7eb635d74892d41c438afb Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 26 Jun 2024 12:27:41 -0400 Subject: [PATCH 010/189] trace2: prefetch value of GIT_TRACE2_DST_DEBUG at startup Prefetch the value of GIT_TRACE2_DST_DEBUG during startup and before we try to open any Trace2 destination pathnames. Normally, Trace2 always silently fails if a destination target cannot be opened so that it doesn't affect the execution of a Git command. The command should run normally, but just not generate any trace data. This can make it difficult to debug a telemetry setup, since the user doesn't know why telemetry isn't being generated. If the environment variable GIT_TRACE2_DST_DEBUG is true, the Trace2 startup will print a warning message with the `errno` to make debugging easier. However, on Windows, looking up the env variable resets `errno` so the warning message always ends with `...tracing: No error` which is not very helpful. Prefetch the env variable at startup. This avoids the need to update each call-site to capture `errno` in the usual `saved-errno` variable. Signed-off-by: Jeff Hostetler --- trace2.c | 10 ++++++++++ trace2/tr2_dst.c | 2 +- trace2/tr2_dst.h | 12 ++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/trace2.c b/trace2.c index f894532d05331c..0a4ff698f607ab 100644 --- a/trace2.c +++ b/trace2.c @@ -225,6 +225,16 @@ void trace2_initialize_fl(const char *file, int line) if (!tr2_tgt_want_builtins()) return; trace2_enabled = 1; + + /* + * getenv() on Windows stomps on `errno` and the code in + * tr2_dst.c verifies that warnings are enabled before + * formatting the warning message (and calling strerror()). + * So prefetch the value from the environment before we need + * it. + */ + tr2_dst_want_warning(); + if (!git_env_bool("GIT_TRACE2_REDACT", 1)) trace2_redact = 0; diff --git a/trace2/tr2_dst.c b/trace2/tr2_dst.c index 5be892cd5cdefa..61579f24bdbde3 100644 --- a/trace2/tr2_dst.c +++ b/trace2/tr2_dst.c @@ -24,7 +24,7 @@ */ static int tr2env_max_files = 0; -static int tr2_dst_want_warning(void) +int tr2_dst_want_warning(void) { static int tr2env_dst_debug = -1; diff --git a/trace2/tr2_dst.h b/trace2/tr2_dst.h index b1a8c144e073ba..4166539eb9e100 100644 --- a/trace2/tr2_dst.h +++ b/trace2/tr2_dst.h @@ -35,4 +35,16 @@ int tr2_dst_trace_want(struct tr2_dst *dst); */ void tr2_dst_write_line(struct tr2_dst *dst, struct strbuf *buf_line); +/* + * Return true if we want warning messages when trying to open a + * destination. + * + * (Trace2 always silently fails if a target cannot be opened so that + * we don't affect the execution of the Git command, but it is helpful + * for debugging telemetry configuration if we log warning messages + * when trying to open a target. This is controlled by another config + * value.) + */ +int tr2_dst_want_warning(void); + #endif /* TR2_DST_H */ From 62d0c8cc787fdd4a8683ef910269507b036dc46f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 29 Apr 2024 13:21:49 -0400 Subject: [PATCH 011/189] survey: stub in treewalk of reachable commits and objects Add treewalk on the commits and objects reachable from the set of refs. This commit sets up the treewalk, but only stubs in the traverse callbacks. We'll actually look at the commit and object data in the next commit. Signed-off-by: Jeff Hostetler --- builtin/survey.c | 96 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/builtin/survey.c b/builtin/survey.c index 8e1a0e4e871246..2b5e21fa4786f7 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -1,15 +1,20 @@ #include "builtin.h" #include "config.h" +#include "environment.h" #include "json-writer.h" +#include "list-objects.h" +#include "object-name.h" #include "object-store.h" #include "parse-options.h" #include "progress.h" #include "ref-filter.h" #include "refs.h" +#include "revision.h" #include "strbuf.h" #include "strmap.h" #include "strvec.h" #include "trace2.h" +#include "tree-walk.h" static const char * const survey_usage[] = { N_("(EXPERIMENTAL!) git survey "), @@ -244,6 +249,93 @@ static void do_load_refs(struct ref_array *ref_array) ref_sorting_release(sorting); } +/* + * Populate a "rev_info" with the OIDs of the REFS of interest. + * The treewalk will start from all of those starting points + * and walk backwards in the DAG to get the set of all reachable + * objects from those starting points. + */ +static void load_rev_info(struct rev_info *rev_info, + struct ref_array *ref_array) +{ + unsigned int add_flags = 0; + int k; + + for (k = 0; k < ref_array->nr; k++) { + struct ref_array_item *p = ref_array->items[k]; + struct object_id peeled; + + switch (p->kind) { + case FILTER_REFS_TAGS: + if (!peel_iterated_oid(rev_info->repo, &p->objectname, &peeled)) + add_pending_oid(rev_info, NULL, &peeled, add_flags); + else + add_pending_oid(rev_info, NULL, &p->objectname, add_flags); + break; + case FILTER_REFS_BRANCHES: + add_pending_oid(rev_info, NULL, &p->objectname, add_flags); + break; + case FILTER_REFS_REMOTES: + add_pending_oid(rev_info, NULL, &p->objectname, add_flags); + break; + case FILTER_REFS_OTHERS: + /* + * This may be a note, stash, or custom namespace branch. + */ + add_pending_oid(rev_info, NULL, &p->objectname, add_flags); + break; + case FILTER_REFS_DETACHED_HEAD: + add_pending_oid(rev_info, NULL, &p->objectname, add_flags); + break; + default: + break; + } + } +} + +static void traverse_commit_cb(struct commit *commit, void *data) +{ + if ((++survey_progress_total % 1000) == 0) + display_progress(survey_progress, survey_progress_total); +} + +static void traverse_object_cb(struct object *obj, const char *name, void *data) +{ + if ((++survey_progress_total % 1000) == 0) + display_progress(survey_progress, survey_progress_total); +} + +/* + * Treewalk all of the commits and objects reachable from the + * set of refs. + */ +static void do_treewalk_reachable(struct ref_array *ref_array) +{ + struct rev_info rev_info = REV_INFO_INIT; + + repo_init_revisions(the_repository, &rev_info, NULL); + rev_info.tree_objects = 1; + rev_info.blob_objects = 1; + load_rev_info(&rev_info, ref_array); + if (prepare_revision_walk(&rev_info)) + die(_("revision walk setup failed")); + + if (survey_opts.show_progress) { + survey_progress_total = 0; + survey_progress = start_progress(_("Walking reachable objects..."), 0); + } + + traverse_commit_list(&rev_info, + traverse_commit_cb, + traverse_object_cb, + NULL); + + if (survey_opts.show_progress) + stop_progress(&survey_progress); + + release_revisions(&rev_info); +} + /* * If we want this type of ref, increment counters and return 1. */ @@ -438,6 +530,10 @@ static void survey_phase_refs(struct repository *r) do_load_refs(&ref_array); trace2_region_leave("survey", "phase/refs", the_repository); + trace2_region_enter("survey", "phase/treewalk", the_repository); + do_treewalk_reachable(&ref_array); + trace2_region_leave("survey", "phase/treewalk", the_repository); + trace2_region_enter("survey", "phase/calcstats", the_repository); do_calc_stats_refs(r, &ref_array); trace2_region_leave("survey", "phase/calcstats", the_repository); From b7da696501df1ade7906ce1652f8bb54ff9f734f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 29 Apr 2024 15:40:00 -0400 Subject: [PATCH 012/189] survey: add traverse callback for commits Add callback to handle commit objects during the treewalk. Count the number of commits and group them by the number of parents. Signed-off-by: Jeff Hostetler --- builtin/survey.c | 474 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 473 insertions(+), 1 deletion(-) diff --git a/builtin/survey.c b/builtin/survey.c index 2b5e21fa4786f7..fa3ae48821e563 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -1,6 +1,7 @@ #include "builtin.h" #include "config.h" #include "environment.h" +#include "hex.h" #include "json-writer.h" #include "list-objects.h" #include "object-name.h" @@ -14,6 +15,7 @@ #include "strmap.h" #include "strvec.h" #include "trace2.h" +#include "tree.h" #include "tree-walk.h" static const char * const survey_usage[] = { @@ -191,8 +193,162 @@ struct survey_stats_refs { struct strintmap refsmap; }; +/* + * HBIN -- hex binning (histogram bucketing). + * + * We create histograms for various counts and sums. Since we have a + * wide range of values (objects range in size from 1 to 4G bytes), a + * linear bucketing is not interesting. Instead, lets use a + * log16()-based bucketing. This gives us a better spread on the low + * and middle range and a coarse bucketing on the high end. + * + * The idea here is that it doesn't matter if you have n 1GB blobs or + * n/2 1GB blobs and n/2 1.5GB blobs -- either way you have a scaling + * problem that we want to report on. + */ +#define HBIN_LEN (sizeof(unsigned long) * 2) +#define HBIN_MASK (0xF) +#define HBIN_SHIFT (4) + +static int hbin(unsigned long value) +{ + int k; + + for (k = 0; k < HBIN_LEN; k++) { + if ((value & ~(HBIN_MASK)) == 0) + return k; + value >>= HBIN_SHIFT; + } + + return 0; /* should not happen */ +} + +/* + * QBIN -- base4 binning (histogram bucketing). + * + * This is the same idea as the above, but we want better granularity + * in the low end and don't expect as many large values. + */ +#define QBIN_LEN (sizeof(unsigned long) * 4) +#define QBIN_MASK (0x3) +#define QBIN_SHIFT (2) + +static int qbin(unsigned long value) +{ + int k; + + for (k = 0; k < QBIN_LEN; k++) { + if ((value & ~(QBIN_MASK)) == 0) + return k; + value >>= (QBIN_SHIFT); + } + + return 0; /* should not happen */ +} + +/* + * histogram bin for objects. + */ +struct obj_hist_bin { + uint64_t sum_size; /* sum(object_size) for all objects in this bin */ + uint64_t sum_disk_size; /* sum(on_disk_size) for all objects in this bin */ + uint32_t cnt_seen; /* number seen in this bin */ +}; + +static void incr_obj_hist_bin(struct obj_hist_bin *pbin, + unsigned long object_length, + off_t disk_sizep) +{ + pbin->sum_size += object_length; + pbin->sum_disk_size += disk_sizep; + pbin->cnt_seen++; +} + +/* + * Common fields for any type of object. + */ +struct survey_stats_base_object { + uint32_t cnt_seen; + + uint32_t cnt_missing; /* we may have a partial clone. */ + + /* + * Number of objects grouped by where they are stored on disk. + * This is a function of how the ODB is packed. + */ + uint32_t cnt_cached; /* see oi.whence */ + uint32_t cnt_loose; /* see oi.whence */ + uint32_t cnt_packed; /* see oi.whence */ + uint32_t cnt_dbcached; /* see oi.whence */ + + uint64_t sum_size; /* sum(object_size) */ + uint64_t sum_disk_size; /* sum(disk_size) */ + + /* + * A histogram of the count of objects, the observed size, and + * the on-disk size grouped by the observed size. + */ + struct obj_hist_bin size_hbin[HBIN_LEN]; +}; + +/* + * PBIN -- parent vector binning (histogram bucketing). + * + * We create a histogram based upon the number of parents + * in a commit. This is a simple linear vector. It starts + * at zero for "initial" commits. + * + * If a commit has more parents, just put it in the last bin. + */ +#define PBIN_VEC_LEN (17) + +struct survey_stats_commits { + struct survey_stats_base_object base; + + /* + * Count of commits with k parents. + */ + uint32_t parent_cnt_pbin[PBIN_VEC_LEN]; +}; + +/* + * Stats for reachable trees. + */ +struct survey_stats_trees { + struct survey_stats_base_object base; + + /* + * In the following, nr_entries refers to the number of files or + * subdirectories in a tree. We are interested in how wide the + * tree is and if the repo has gigantic directories. + */ + uint64_t max_entries; /* max(nr_entries) -- the width of the largest tree */ + + /* + * Computing the sum of the number of entries across all trees + * is probably not that interesting. + */ + uint64_t sum_entries; /* sum(nr_entries) -- sum across all trees */ + + /* + * A histogram of the count of trees, the observed size, and + * the on-disk size grouped by the number of entries in the tree. + */ + struct obj_hist_bin entry_qbin[QBIN_LEN]; +}; + +/* + * Stats for reachable blobs. + */ +struct survey_stats_blobs { + struct survey_stats_base_object base; +}; + struct survey_stats { - struct survey_stats_refs refs; + struct survey_stats_refs refs; + struct survey_stats_commits commits; + struct survey_stats_trees trees; + struct survey_stats_blobs blobs; }; static struct survey_stats survey_stats = { 0 }; @@ -293,16 +449,134 @@ static void load_rev_info(struct rev_info *rev_info, } } +static int fill_in_base_object(struct survey_stats_base_object *base, + struct object *object, + enum object_type type_expected, + unsigned long *p_object_length, + off_t *p_disk_sizep) +{ + struct object_info oi = OBJECT_INFO_INIT; + unsigned oi_flags = OBJECT_INFO_FOR_PREFETCH; + unsigned long object_length = 0; + off_t disk_sizep = 0; + enum object_type type; + int hb; + + base->cnt_seen++; + + oi.typep = &type; + oi.sizep = &object_length; + oi.disk_sizep = &disk_sizep; + + if (oid_object_info_extended(the_repository, &object->oid, &oi, oi_flags) < 0 || + type != type_expected) { + base->cnt_missing++; + return 1; + } + + switch (oi.whence) { + case OI_CACHED: + base->cnt_cached++; + break; + case OI_LOOSE: + base->cnt_loose++; + break; + case OI_PACKED: + base->cnt_packed++; + break; + case OI_DBCACHED: + base->cnt_dbcached++; + break; + default: + break; + } + + base->sum_size += object_length; + base->sum_disk_size += disk_sizep; + + hb = hbin(object_length); + incr_obj_hist_bin(&base->size_hbin[hb], object_length, disk_sizep); + + if (p_object_length) + *p_object_length = object_length; + if (p_disk_sizep) + *p_disk_sizep = disk_sizep; + + return 0; +} + static void traverse_commit_cb(struct commit *commit, void *data) { + struct survey_stats_commits *psc = &survey_stats.commits; + unsigned k; + if ((++survey_progress_total % 1000) == 0) display_progress(survey_progress, survey_progress_total); + + fill_in_base_object(&psc->base, &commit->object, OBJ_COMMIT, NULL, NULL); + + k = commit_list_count(commit->parents); + if (k >= PBIN_VEC_LEN) + k = PBIN_VEC_LEN - 1; + + psc->parent_cnt_pbin[k]++; +} + +static void traverse_object_cb_tree(struct object *obj) +{ + struct survey_stats_trees *pst = &survey_stats.trees; + unsigned long object_length; + off_t disk_sizep; + struct tree_desc desc; + struct name_entry entry; + struct tree *tree; + int nr_entries; + int qb; + + if (fill_in_base_object(&pst->base, obj, OBJ_TREE, &object_length, &disk_sizep)) + return; + + tree = lookup_tree(the_repository, &obj->oid); + if (!tree) + return; + init_tree_desc(&desc, &obj->oid, tree->buffer, tree->size); + nr_entries = 0; + while (tree_entry(&desc, &entry)) + nr_entries++; + + pst->sum_entries += nr_entries; + + if (nr_entries > pst->max_entries) + pst->max_entries = nr_entries; + + qb = qbin(nr_entries); + incr_obj_hist_bin(&pst->entry_qbin[qb], object_length, disk_sizep); +} + +static void traverse_object_cb_blob(struct object *obj) +{ + struct survey_stats_blobs *psb = &survey_stats.blobs; + + fill_in_base_object(&psb->base, obj, OBJ_BLOB, NULL, NULL); } static void traverse_object_cb(struct object *obj, const char *name, void *data) { if ((++survey_progress_total % 1000) == 0) display_progress(survey_progress, survey_progress_total); + + switch (obj->type) { + case OBJ_TREE: + traverse_object_cb_tree(obj); + return; + case OBJ_BLOB: + traverse_object_cb_blob(obj); + return; + case OBJ_TAG: /* ignore -- counted when loading REFS */ + case OBJ_COMMIT: /* ignore/bug -- seen in the other callback */ + default: /* ignore/bug -- unknown type */ + return; + } } /* @@ -639,6 +913,198 @@ static void json_refs_section(struct json_writer *jw_top, int pretty, int want_t jw_release(&jw_refs); } +#define JW_OBJ_INT_NZ(jw, key, value) do { if (value) jw_object_intmax((jw), (key), (value)); } while (0) + +static void write_qbin_json(struct json_writer *jw, const char *label, + struct obj_hist_bin qbin[QBIN_LEN]) +{ + struct strbuf buf = STRBUF_INIT; + uint32_t lower = 0; + uint32_t upper = QBIN_MASK; + int k; + + jw_object_inline_begin_object(jw, label); + { + for (k = 0; k < QBIN_LEN; k++) { + struct obj_hist_bin *p = &qbin[k]; + uint32_t lower_k = lower; + uint32_t upper_k = upper; + + lower = upper+1; + upper = (upper << QBIN_SHIFT) + QBIN_MASK; + + if (!p->cnt_seen) + continue; + + strbuf_reset(&buf); + strbuf_addf(&buf, "Q%02d", k); + jw_object_inline_begin_object(jw, buf.buf); + { + jw_object_intmax(jw, "count", p->cnt_seen); + jw_object_intmax(jw, "sum_size", p->sum_size); + jw_object_intmax(jw, "sum_disk_size", p->sum_disk_size); + + /* maybe only include these in verbose mode */ + jw_object_intmax(jw, "qbin_lower", lower_k); + jw_object_intmax(jw, "qbin_upper", upper_k); + } + jw_end(jw); + } + } + jw_end(jw); + + strbuf_release(&buf); +} + +static void write_hbin_json(struct json_writer *jw, const char *label, + struct obj_hist_bin hbin[HBIN_LEN]) +{ + struct strbuf buf = STRBUF_INIT; + uint32_t lower = 0; + uint32_t upper = HBIN_MASK; + int k; + + jw_object_inline_begin_object(jw, label); + { + for (k = 0; k < HBIN_LEN; k++) { + struct obj_hist_bin *p = &hbin[k]; + uint32_t lower_k = lower; + uint32_t upper_k = upper; + + lower = upper+1; + upper = (upper << HBIN_SHIFT) + HBIN_MASK; + + if (!p->cnt_seen) + continue; + + strbuf_reset(&buf); + strbuf_addf(&buf, "H%d", k); + jw_object_inline_begin_object(jw, buf.buf); + { + jw_object_intmax(jw, "count", p->cnt_seen); + jw_object_intmax(jw, "sum_size", p->sum_size); + jw_object_intmax(jw, "sum_disk_size", p->sum_disk_size); + + /* maybe only include these in verbose mode */ + jw_object_intmax(jw, "hbin_lower", lower_k); + jw_object_intmax(jw, "hbin_upper", upper_k); + } + jw_end(jw); + } + } + jw_end(jw); + + strbuf_release(&buf); +} + +static void write_base_object_json(struct json_writer *jw, + struct survey_stats_base_object *base) +{ + jw_object_intmax(jw, "count", base->cnt_seen); + + jw_object_intmax(jw, "sum_size", base->sum_size); + jw_object_intmax(jw, "sum_disk_size", base->sum_disk_size); + + jw_object_inline_begin_object(jw, "count_by_whence"); + { + /* + * Missing is not technically a "whence" value, but + * we don't need to clutter up the results with that + * distinction. + */ + JW_OBJ_INT_NZ(jw, "missing", base->cnt_missing); + + JW_OBJ_INT_NZ(jw, "cached", base->cnt_cached); + JW_OBJ_INT_NZ(jw, "loose", base->cnt_loose); + JW_OBJ_INT_NZ(jw, "packed", base->cnt_packed); + JW_OBJ_INT_NZ(jw, "dbcached", base->cnt_dbcached); + } + jw_end(jw); + + write_hbin_json(jw, "dist_by_size", base->size_hbin); +} + +static void json_commits_section(struct json_writer *jw_top, int pretty, int want_trace2) +{ + struct survey_stats_commits *psc = &survey_stats.commits; + struct json_writer jw_commits = JSON_WRITER_INIT; + + jw_object_begin(&jw_commits, pretty); + { + write_base_object_json(&jw_commits, &psc->base); + + jw_object_inline_begin_object(&jw_commits, "count_by_nr_parents"); + { + struct strbuf parent_key = STRBUF_INIT; + int k; + + for (k = 0; k < PBIN_VEC_LEN; k++) + if (psc->parent_cnt_pbin[k]) { + strbuf_reset(&parent_key); + strbuf_addf(&parent_key, "P%02d", k); + jw_object_intmax(&jw_commits, parent_key.buf, psc->parent_cnt_pbin[k]); + } + + strbuf_release(&parent_key); + } + jw_end(&jw_commits); + } + jw_end(&jw_commits); + + if (jw_top) + jw_object_sub_jw(jw_top, "commits", &jw_commits); + + if (want_trace2) + trace2_data_json("survey", the_repository, "commits", &jw_commits); + + jw_release(&jw_commits); +} + +static void json_trees_section(struct json_writer *jw_top, int pretty, int want_trace2) +{ + struct survey_stats_trees *pst = &survey_stats.trees; + struct json_writer jw_trees = JSON_WRITER_INIT; + + jw_object_begin(&jw_trees, pretty); + { + write_base_object_json(&jw_trees, &pst->base); + + jw_object_intmax(&jw_trees, "max_entries", pst->max_entries); + jw_object_intmax(&jw_trees, "sum_entries", pst->sum_entries); + + write_qbin_json(&jw_trees, "dist_by_nr_entries", pst->entry_qbin); + } + jw_end(&jw_trees); + + if (jw_top) + jw_object_sub_jw(jw_top, "trees", &jw_trees); + + if (want_trace2) + trace2_data_json("survey", the_repository, "trees", &jw_trees); + + jw_release(&jw_trees); +} + +static void json_blobs_section(struct json_writer *jw_top, int pretty, int want_trace2) +{ + struct survey_stats_blobs *psb = &survey_stats.blobs; + struct json_writer jw_blobs = JSON_WRITER_INIT; + + jw_object_begin(&jw_blobs, pretty); + { + write_base_object_json(&jw_blobs, &psb->base); + } + jw_end(&jw_blobs); + + if (jw_top) + jw_object_sub_jw(jw_top, "blobs", &jw_blobs); + + if (want_trace2) + trace2_data_json("survey", the_repository, "blobs", &jw_blobs); + + jw_release(&jw_blobs); +} + static void survey_print_json(void) { struct json_writer jw_top = JSON_WRITER_INIT; @@ -647,6 +1113,9 @@ static void survey_print_json(void) jw_object_begin(&jw_top, pretty); { json_refs_section(&jw_top, pretty, 0); + json_commits_section(&jw_top, pretty, 0); + json_trees_section(&jw_top, pretty, 0); + json_blobs_section(&jw_top, pretty, 0); } jw_end(&jw_top); @@ -661,6 +1130,9 @@ static void survey_emit_trace2(void) return; json_refs_section(NULL, 0, 1); + json_commits_section(NULL, 0, 1); + json_trees_section(NULL, 0, 1); + json_blobs_section(NULL, 0, 1); } int cmd_survey(int argc, const char **argv, const char *prefix) From e388414967069806b9fd837b26df1629879ce37b Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 1 May 2024 12:56:38 -0400 Subject: [PATCH 013/189] survey: add vector of largest objects for various scaling dimensions Create `struct large_item` and `struct large_item_vec` to capture the n largest commits, trees, and blobs under various scaling dimensions, such as size in bytes, number of commit parents, or number of entries in a tree. Each of these have a command line option to set them independently. Signed-off-by: Jeff Hostetler --- Documentation/config.txt | 2 + Documentation/config/survey.txt | 36 +++++ Documentation/git-survey.txt | 31 ++++ builtin/survey.c | 245 ++++++++++++++++++++++++++++++-- 4 files changed, 304 insertions(+), 10 deletions(-) create mode 100644 Documentation/config/survey.txt diff --git a/Documentation/config.txt b/Documentation/config.txt index fedfaf30cd0d8b..939cc1387992f8 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -536,6 +536,8 @@ include::config/status.txt[] include::config/submodule.txt[] +include::config/survey.txt[] + include::config/tag.txt[] include::config/tar.txt[] diff --git a/Documentation/config/survey.txt b/Documentation/config/survey.txt new file mode 100644 index 00000000000000..672e7890ed2f79 --- /dev/null +++ b/Documentation/config/survey.txt @@ -0,0 +1,36 @@ +survey.progress:: + Boolean to show/hide progress information. Defaults to + true when interactive (stderr is bound to a TTY). + +survey.showBlobSizes:: + A non-negative integer value. Requests details on the + largest file blobs by size in bytes. Provides a default + value for `--blob-sizes=` in linkgit:git-survey[1]. + +survey.showCommitParents:: + A non-negative integer value. Requests details on the + commits with the most number of parents. Provides a default + value for `--commit-parents=` in linkgit:git-survey[1]. + +survey.showCommitSizes:: + A non-negative integer value. Requests details on the + largest commits by size in bytes. Generally, these are the + commits with the largest commit messages. Provides a default + value for `--commit-sizes=` in linkgit:git-survey[1]. + +survey.showTreeEntries:: + A non-negative integer value. Requests details on the + trees (directories) with the most number of entries (files + and subdirectories). Provides a default value for + `--tree-entries=` in linkgit:git-survey[1]. + +survey.showTreeSizes:: + A non-negative integer value. Requests details on the + largest trees (directories) by size in bytes. This will + set will usually be equal to the `survey.showTreeEntries` + set, but may be skewed by very long file or subdirectory + entry names. Provides a default value for + `--tree-sizes=` in linkgit:git-survey[1]. + +survey.verbose:: + Boolean to show/hide verbose output. Default to false. diff --git a/Documentation/git-survey.txt b/Documentation/git-survey.txt index c648ef704e3806..771a063efdc594 100644 --- a/Documentation/git-survey.txt +++ b/Documentation/git-survey.txt @@ -59,12 +59,43 @@ only refs for the given options are added. --other:: Add notes (`refs/notes/`) and stashes (`refs/stash/`) to the set. +Large Item Selection +~~~~~~~~~~~~~~~~~~~~ + +The following options control the optional display of large items under +various dimensions of scale. The OID of the largest `n` objects will be +displayed in reverse sorted order. For each, `n` defaults to 10. + +--commit-parents:: + Shows the OIDs of the commits with the most parent commits. + +--commit-sizes:: + Shows the OIDs of the largest commits by size in bytes. This is + usually the ones with the largest commit messages. + +--tree-entries:: + Shows the OIDs of the trees with the most number of entries. These + are the directories with the most number of files or subdirectories. + +--tree-sizes:: + Shows the OIDs of the largest trees by size in bytes. This set + will usually be the same as the vector of number of entries unless + skewed by very long entry names. + +--blob-sizes:: + Shows the OIDs of the largest blobs by size in bytes. + OUTPUT ------ By default, `git survey` will print information about the repository in a human-readable format that includes overviews and tables. +CONFIGURATION +------------- + +include::config/survey.txt[] + GIT --- Part of the linkgit:git[1] suite diff --git a/builtin/survey.c b/builtin/survey.c index fa3ae48821e563..cfface7948884d 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -55,13 +55,36 @@ static struct survey_refs_wanted refs_if_unspecified = { struct survey_opts { int verbose; int show_progress; + + int show_largest_commits_by_nr_parents; + int show_largest_commits_by_size_bytes; + + int show_largest_trees_by_nr_entries; + int show_largest_trees_by_size_bytes; + + int show_largest_blobs_by_size_bytes; + struct survey_refs_wanted refs; }; +#define DEFAULT_SHOW_LARGEST_VALUE (10) + static struct survey_opts survey_opts = { .verbose = 0, .show_progress = -1, /* defaults to isatty(2) */ + /* + * Show the largest `n` objects for some scaling dimension. + * We allow each to be requested independently. + */ + .show_largest_commits_by_nr_parents = DEFAULT_SHOW_LARGEST_VALUE, + .show_largest_commits_by_size_bytes = DEFAULT_SHOW_LARGEST_VALUE, + + .show_largest_trees_by_nr_entries = DEFAULT_SHOW_LARGEST_VALUE, + .show_largest_trees_by_size_bytes = DEFAULT_SHOW_LARGEST_VALUE, + + .show_largest_blobs_by_size_bytes = DEFAULT_SHOW_LARGEST_VALUE, + .refs.want_all_refs = 0, .refs.want_branches = -1, /* default these to undefined */ @@ -139,6 +162,14 @@ static struct option survey_options[] = { OPT_BOOL_F(0, "detached", &survey_opts.refs.want_detached, N_("include detached HEAD"), PARSE_OPT_NONEG), OPT_BOOL_F(0, "other", &survey_opts.refs.want_other, N_("include notes and stashes"), PARSE_OPT_NONEG), + OPT_INTEGER_F(0, "commit-parents", &survey_opts.show_largest_commits_by_nr_parents, N_("show N largest commits by parent count"), PARSE_OPT_NONEG), + OPT_INTEGER_F(0, "commit-sizes", &survey_opts.show_largest_commits_by_size_bytes, N_("show N largest commits by size in bytes"), PARSE_OPT_NONEG), + + OPT_INTEGER_F(0, "tree-entries", &survey_opts.show_largest_trees_by_nr_entries, N_("show N largest trees by entry count"), PARSE_OPT_NONEG), + OPT_INTEGER_F(0, "tree-sizes", &survey_opts.show_largest_trees_by_size_bytes, N_("show N largest trees by size in bytes"), PARSE_OPT_NONEG), + + OPT_INTEGER_F(0, "blob-sizes", &survey_opts.show_largest_blobs_by_size_bytes, N_("show N largest blobs by size in bytes"), PARSE_OPT_NONEG), + OPT_END(), }; @@ -154,6 +185,29 @@ static int survey_load_config_cb(const char *var, const char *value, return 0; } + if (!strcmp(var, "survey.showcommitparents")) { + survey_opts.show_largest_commits_by_nr_parents = git_config_ulong(var, value, ctx->kvi); + return 0; + } + if (!strcmp(var, "survey.showcommitsizes")) { + survey_opts.show_largest_commits_by_size_bytes = git_config_ulong(var, value, ctx->kvi); + return 0; + } + + if (!strcmp(var, "survey.showtreeentries")) { + survey_opts.show_largest_trees_by_nr_entries = git_config_ulong(var, value, ctx->kvi); + return 0; + } + if (!strcmp(var, "survey.showtreesizes")) { + survey_opts.show_largest_trees_by_size_bytes = git_config_ulong(var, value, ctx->kvi); + return 0; + } + + if (!strcmp(var, "survey.showblobsizes")) { + survey_opts.show_largest_blobs_by_size_bytes = git_config_ulong(var, value, ctx->kvi); + return 0; + } + return git_default_config(var, value, ctx, pvoid); } @@ -264,6 +318,84 @@ static void incr_obj_hist_bin(struct obj_hist_bin *pbin, pbin->cnt_seen++; } +/* + * Remember the largest n objects for some scaling dimension. This + * could be the observed object size or number of entries in a tree. + * We'll use this to generate a sorted vector in the output for that + * dimension. + */ +struct large_item { + uint64_t size; + struct object_id oid; +}; + +struct large_item_vec { + char *dimension_label; + char *item_label; + uint64_t nr_items; + struct large_item items[FLEX_ARRAY]; /* nr_items */ +}; + +static struct large_item_vec *alloc_large_item_vec(const char *dimension_label, + const char *item_label, + uint64_t nr_items) +{ + struct large_item_vec *vec; + size_t flex_len = nr_items * sizeof(struct large_item); + + if (!nr_items) + return NULL; + + vec = xcalloc(1, (sizeof(struct large_item_vec) + flex_len)); + vec->dimension_label = strdup(dimension_label); + vec->item_label = strdup(item_label); + vec->nr_items = nr_items; + + return vec; +} + +static void free_large_item_vec(struct large_item_vec *vec) +{ + free(vec->dimension_label); + free(vec->item_label); + free(vec); +} + +static void maybe_insert_large_item(struct large_item_vec *vec, + uint64_t size, + struct object_id *oid) +{ + size_t rest_len; + size_t k; + + if (!vec || !vec->nr_items) + return; + + /* + * Since the odds an object being among the largest n + * is small, shortcut and see if it is smaller than + * the smallest one in our set and quickly reject it. + */ + if (size < vec->items[vec->nr_items - 1].size) + return; + + for (k = 0; k < vec->nr_items; k++) { + if (size < vec->items[k].size) + continue; + + /* push items[k..] down one and insert it here */ + + rest_len = (vec->nr_items - k - 1) * sizeof(struct large_item); + if (rest_len) + memmove(&vec->items[k + 1], &vec->items[k], rest_len); + + memset(&vec->items[k], 0, sizeof(struct large_item)); + vec->items[k].size = size; + oidcpy(&vec->items[k].oid, oid); + return; + } +} + /* * Common fields for any type of object. */ @@ -309,6 +441,9 @@ struct survey_stats_commits { * Count of commits with k parents. */ uint32_t parent_cnt_pbin[PBIN_VEC_LEN]; + + struct large_item_vec *vec_largest_by_nr_parents; + struct large_item_vec *vec_largest_by_size_bytes; }; /* @@ -318,11 +453,18 @@ struct survey_stats_trees { struct survey_stats_base_object base; /* - * In the following, nr_entries refers to the number of files or - * subdirectories in a tree. We are interested in how wide the - * tree is and if the repo has gigantic directories. + * Keep a vector of the trees with the most number of entries. + * This gives us a feel for the width of a tree when there are + * gigantic directories. + */ + struct large_item_vec *vec_largest_by_nr_entries; + + /* + * Keep a vector of the trees with the largest size in bytes. + * The contents of this may or may not match items in the other + * vector, since entryname length can alter the results. */ - uint64_t max_entries; /* max(nr_entries) -- the width of the largest tree */ + struct large_item_vec *vec_largest_by_size_bytes; /* * Computing the sum of the number of entries across all trees @@ -342,6 +484,11 @@ struct survey_stats_trees { */ struct survey_stats_blobs { struct survey_stats_base_object base; + + /* + * Remember the OIDs of the largest n blobs. + */ + struct large_item_vec *vec_largest_by_size_bytes; }; struct survey_stats { @@ -508,17 +655,21 @@ static int fill_in_base_object(struct survey_stats_base_object *base, static void traverse_commit_cb(struct commit *commit, void *data) { struct survey_stats_commits *psc = &survey_stats.commits; + unsigned long object_length; unsigned k; if ((++survey_progress_total % 1000) == 0) display_progress(survey_progress, survey_progress_total); - fill_in_base_object(&psc->base, &commit->object, OBJ_COMMIT, NULL, NULL); + fill_in_base_object(&psc->base, &commit->object, OBJ_COMMIT, &object_length, NULL); k = commit_list_count(commit->parents); + + maybe_insert_large_item(psc->vec_largest_by_nr_parents, k, &commit->object.oid); + maybe_insert_large_item(psc->vec_largest_by_size_bytes, object_length, &commit->object.oid); + if (k >= PBIN_VEC_LEN) k = PBIN_VEC_LEN - 1; - psc->parent_cnt_pbin[k]++; } @@ -546,8 +697,8 @@ static void traverse_object_cb_tree(struct object *obj) pst->sum_entries += nr_entries; - if (nr_entries > pst->max_entries) - pst->max_entries = nr_entries; + maybe_insert_large_item(pst->vec_largest_by_nr_entries, nr_entries, &obj->oid); + maybe_insert_large_item(pst->vec_largest_by_size_bytes, object_length, &obj->oid); qb = qbin(nr_entries); incr_obj_hist_bin(&pst->entry_qbin[qb], object_length, disk_sizep); @@ -556,8 +707,11 @@ static void traverse_object_cb_tree(struct object *obj) static void traverse_object_cb_blob(struct object *obj) { struct survey_stats_blobs *psb = &survey_stats.blobs; + unsigned long object_length; - fill_in_base_object(&psb->base, obj, OBJ_BLOB, NULL, NULL); + fill_in_base_object(&psb->base, obj, OBJ_BLOB, &object_length, NULL); + + maybe_insert_large_item(psb->vec_largest_by_size_bytes, object_length, &obj->oid); } static void traverse_object_cb(struct object *obj, const char *name, void *data) @@ -1024,6 +1178,32 @@ static void write_base_object_json(struct json_writer *jw, write_hbin_json(jw, "dist_by_size", base->size_hbin); } +static void write_large_item_vec_json(struct json_writer *jw, + struct large_item_vec *vec) +{ + if (!vec || !vec->nr_items) + return; + + jw_object_inline_begin_array(jw, vec->dimension_label); + { + int k; + + for (k = 0; k < vec->nr_items; k++) { + struct large_item *pk = &vec->items[k]; + if (is_null_oid(&pk->oid)) + break; + + jw_array_inline_begin_object(jw); + { + jw_object_intmax(jw, vec->item_label, pk->size); + jw_object_string(jw, "oid", oid_to_hex(&pk->oid)); + } + jw_end(jw); + } + } + jw_end(jw); +} + static void json_commits_section(struct json_writer *jw_top, int pretty, int want_trace2) { struct survey_stats_commits *psc = &survey_stats.commits; @@ -1033,6 +1213,9 @@ static void json_commits_section(struct json_writer *jw_top, int pretty, int wan { write_base_object_json(&jw_commits, &psc->base); + write_large_item_vec_json(&jw_commits, psc->vec_largest_by_nr_parents); + write_large_item_vec_json(&jw_commits, psc->vec_largest_by_size_bytes); + jw_object_inline_begin_object(&jw_commits, "count_by_nr_parents"); { struct strbuf parent_key = STRBUF_INIT; @@ -1069,9 +1252,11 @@ static void json_trees_section(struct json_writer *jw_top, int pretty, int want_ { write_base_object_json(&jw_trees, &pst->base); - jw_object_intmax(&jw_trees, "max_entries", pst->max_entries); jw_object_intmax(&jw_trees, "sum_entries", pst->sum_entries); + write_large_item_vec_json(&jw_trees, pst->vec_largest_by_nr_entries); + write_large_item_vec_json(&jw_trees, pst->vec_largest_by_size_bytes); + write_qbin_json(&jw_trees, "dist_by_nr_entries", pst->entry_qbin); } jw_end(&jw_trees); @@ -1093,6 +1278,8 @@ static void json_blobs_section(struct json_writer *jw_top, int pretty, int want_ jw_object_begin(&jw_blobs, pretty); { write_base_object_json(&jw_blobs, &psb->base); + + write_large_item_vec_json(&jw_blobs, psb->vec_largest_by_size_bytes); } jw_end(&jw_blobs); @@ -1147,12 +1334,50 @@ int cmd_survey(int argc, const char **argv, const char *prefix) survey_opts.show_progress = isatty(2); fixup_refs_wanted(); + if (survey_opts.show_largest_commits_by_nr_parents) + survey_stats.commits.vec_largest_by_nr_parents = + alloc_large_item_vec( + "largest_commits_by_nr_parents", + "nr_parents", + survey_opts.show_largest_commits_by_nr_parents); + if (survey_opts.show_largest_commits_by_size_bytes) + survey_stats.commits.vec_largest_by_size_bytes = + alloc_large_item_vec( + "largest_commits_by_size_bytes", + "size", + survey_opts.show_largest_commits_by_size_bytes); + + if (survey_opts.show_largest_trees_by_nr_entries) + survey_stats.trees.vec_largest_by_nr_entries = + alloc_large_item_vec( + "largest_trees_by_nr_entries", + "nr_entries", + survey_opts.show_largest_trees_by_nr_entries); + if (survey_opts.show_largest_trees_by_size_bytes) + survey_stats.trees.vec_largest_by_size_bytes = + alloc_large_item_vec( + "largest_trees_by_size_bytes", + "size", + survey_opts.show_largest_trees_by_size_bytes); + + if (survey_opts.show_largest_blobs_by_size_bytes) + survey_stats.blobs.vec_largest_by_size_bytes = + alloc_large_item_vec( + "largest_blobs_by_size_bytes", + "size", + survey_opts.show_largest_blobs_by_size_bytes); + survey_phase_refs(the_repository); survey_emit_trace2(); survey_print_json(); strvec_clear(&survey_vec_refs_wanted); + free_large_item_vec(survey_stats.commits.vec_largest_by_nr_parents); + free_large_item_vec(survey_stats.commits.vec_largest_by_size_bytes); + free_large_item_vec(survey_stats.trees.vec_largest_by_nr_entries); + free_large_item_vec(survey_stats.trees.vec_largest_by_size_bytes); + free_large_item_vec(survey_stats.blobs.vec_largest_by_size_bytes); return 0; } From 7905a6eb208649200ab46a6367a1331ef0e5496e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 15 May 2024 15:56:36 -0400 Subject: [PATCH 014/189] survey: add pathname of blob or tree to large_item_vec Include the pathname of each blob or tree in the large_item_vec to help identify the file or directory associated with the OID and size information. This pathname is computed during the treewalk, so it reflects the first observed pathname seen for that OID during the traversal over all of the refs. Since the file or directory could have moved (without being modified), there may be multiple "correct" pathnames for a particular OID. Since we do not control the ref traversal order, we should consider it to be a "suggested pathname" for the OID. Signed-off-by: Jeff Hostetler --- builtin/survey.c | 54 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/builtin/survey.c b/builtin/survey.c index cfface7948884d..aa4a43e78ba503 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -327,6 +327,7 @@ static void incr_obj_hist_bin(struct obj_hist_bin *pbin, struct large_item { uint64_t size; struct object_id oid; + struct strbuf *name; }; struct large_item_vec { @@ -342,6 +343,7 @@ static struct large_item_vec *alloc_large_item_vec(const char *dimension_label, { struct large_item_vec *vec; size_t flex_len = nr_items * sizeof(struct large_item); + size_t k; if (!nr_items) return NULL; @@ -351,11 +353,24 @@ static struct large_item_vec *alloc_large_item_vec(const char *dimension_label, vec->item_label = strdup(item_label); vec->nr_items = nr_items; + for (k = 0; k < nr_items; k++) { + struct strbuf *p = xcalloc(1, sizeof(struct strbuf)); + strbuf_init(p, 0); + vec->items[k].name = p; + } + return vec; } static void free_large_item_vec(struct large_item_vec *vec) { + size_t k; + + for (k = 0; k < vec->nr_items; k++) { + strbuf_release(vec->items[k].name); + free(vec->items[k].name); + } + free(vec->dimension_label); free(vec->item_label); free(vec); @@ -363,8 +378,10 @@ static void free_large_item_vec(struct large_item_vec *vec) static void maybe_insert_large_item(struct large_item_vec *vec, uint64_t size, - struct object_id *oid) + struct object_id *oid, + const char *name) { + struct strbuf *pbuf_temp; size_t rest_len; size_t k; @@ -383,7 +400,17 @@ static void maybe_insert_large_item(struct large_item_vec *vec, if (size < vec->items[k].size) continue; - /* push items[k..] down one and insert it here */ + /* + * The last large_item in the vector is about to be + * overwritten by the previous one during the shift. + * Steal its allocated strbuf and reuse it. + */ + pbuf_temp = vec->items[vec->nr_items - 1].name; + strbuf_reset(pbuf_temp); + if (name && *name) + strbuf_addstr(pbuf_temp, name); + + /* push items[k..] down one and insert data for this item here */ rest_len = (vec->nr_items - k - 1) * sizeof(struct large_item); if (rest_len) @@ -392,6 +419,9 @@ static void maybe_insert_large_item(struct large_item_vec *vec, memset(&vec->items[k], 0, sizeof(struct large_item)); vec->items[k].size = size; oidcpy(&vec->items[k].oid, oid); + + vec->items[k].name = pbuf_temp; + return; } } @@ -665,15 +695,15 @@ static void traverse_commit_cb(struct commit *commit, void *data) k = commit_list_count(commit->parents); - maybe_insert_large_item(psc->vec_largest_by_nr_parents, k, &commit->object.oid); - maybe_insert_large_item(psc->vec_largest_by_size_bytes, object_length, &commit->object.oid); + maybe_insert_large_item(psc->vec_largest_by_nr_parents, k, &commit->object.oid, NULL); + maybe_insert_large_item(psc->vec_largest_by_size_bytes, object_length, &commit->object.oid, NULL); if (k >= PBIN_VEC_LEN) k = PBIN_VEC_LEN - 1; psc->parent_cnt_pbin[k]++; } -static void traverse_object_cb_tree(struct object *obj) +static void traverse_object_cb_tree(struct object *obj, const char *name) { struct survey_stats_trees *pst = &survey_stats.trees; unsigned long object_length; @@ -697,21 +727,21 @@ static void traverse_object_cb_tree(struct object *obj) pst->sum_entries += nr_entries; - maybe_insert_large_item(pst->vec_largest_by_nr_entries, nr_entries, &obj->oid); - maybe_insert_large_item(pst->vec_largest_by_size_bytes, object_length, &obj->oid); + maybe_insert_large_item(pst->vec_largest_by_nr_entries, nr_entries, &obj->oid, name); + maybe_insert_large_item(pst->vec_largest_by_size_bytes, object_length, &obj->oid, name); qb = qbin(nr_entries); incr_obj_hist_bin(&pst->entry_qbin[qb], object_length, disk_sizep); } -static void traverse_object_cb_blob(struct object *obj) +static void traverse_object_cb_blob(struct object *obj, const char *name) { struct survey_stats_blobs *psb = &survey_stats.blobs; unsigned long object_length; fill_in_base_object(&psb->base, obj, OBJ_BLOB, &object_length, NULL); - maybe_insert_large_item(psb->vec_largest_by_size_bytes, object_length, &obj->oid); + maybe_insert_large_item(psb->vec_largest_by_size_bytes, object_length, &obj->oid, name); } static void traverse_object_cb(struct object *obj, const char *name, void *data) @@ -721,10 +751,10 @@ static void traverse_object_cb(struct object *obj, const char *name, void *data) switch (obj->type) { case OBJ_TREE: - traverse_object_cb_tree(obj); + traverse_object_cb_tree(obj, name); return; case OBJ_BLOB: - traverse_object_cb_blob(obj); + traverse_object_cb_blob(obj, name); return; case OBJ_TAG: /* ignore -- counted when loading REFS */ case OBJ_COMMIT: /* ignore/bug -- seen in the other callback */ @@ -1197,6 +1227,8 @@ static void write_large_item_vec_json(struct json_writer *jw, { jw_object_intmax(jw, vec->item_label, pk->size); jw_object_string(jw, "oid", oid_to_hex(&pk->oid)); + if (pk->name->len) + jw_object_string(jw, "name", pk->name->buf); } jw_end(jw); } From dd0c573ef41a29ee90b5e6bca3df4e1ef43bf2fe Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 15 May 2024 17:44:41 -0400 Subject: [PATCH 015/189] survey: add commit-oid to large_item detail Signed-off-by: Jeff Hostetler --- builtin/survey.c | 59 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/builtin/survey.c b/builtin/survey.c index aa4a43e78ba503..d403e039104ca5 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -327,7 +327,21 @@ static void incr_obj_hist_bin(struct obj_hist_bin *pbin, struct large_item { uint64_t size; struct object_id oid; + + /* + * For blobs and trees the name field is the pathname of the + * file or directory. Root trees will have a zero-length + * name. The name field is not currenly used for commits. + */ struct strbuf *name; + + /* + * For blobs and trees remember the transient commit from + * the treewalk so that we can say that this large item + * first appeared in this commit (relative to the treewalk + * order). + */ + struct object_id containing_commit_oid; }; struct large_item_vec { @@ -379,7 +393,8 @@ static void free_large_item_vec(struct large_item_vec *vec) static void maybe_insert_large_item(struct large_item_vec *vec, uint64_t size, struct object_id *oid, - const char *name) + const char *name, + const struct object_id *containing_commit_oid) { struct strbuf *pbuf_temp; size_t rest_len; @@ -419,6 +434,7 @@ static void maybe_insert_large_item(struct large_item_vec *vec, memset(&vec->items[k], 0, sizeof(struct large_item)); vec->items[k].size = size; oidcpy(&vec->items[k].oid, oid); + oidcpy(&vec->items[k].containing_commit_oid, containing_commit_oid); vec->items[k].name = pbuf_temp; @@ -682,6 +698,14 @@ static int fill_in_base_object(struct survey_stats_base_object *base, return 0; } +/* + * Transient OID of the commit currently being visited + * during the treewalk. We can use this to create the + * : pair when a notable large file was + * created, for example. + */ +static struct object_id treewalk_transient_commit_oid; + static void traverse_commit_cb(struct commit *commit, void *data) { struct survey_stats_commits *psc = &survey_stats.commits; @@ -691,12 +715,23 @@ static void traverse_commit_cb(struct commit *commit, void *data) if ((++survey_progress_total % 1000) == 0) display_progress(survey_progress, survey_progress_total); + oidcpy(&treewalk_transient_commit_oid, &commit->object.oid); + fill_in_base_object(&psc->base, &commit->object, OBJ_COMMIT, &object_length, NULL); k = commit_list_count(commit->parents); - maybe_insert_large_item(psc->vec_largest_by_nr_parents, k, &commit->object.oid, NULL); - maybe_insert_large_item(psc->vec_largest_by_size_bytes, object_length, &commit->object.oid, NULL); + /* + * Send the commit-oid as both the OID and the CONTAINING-COMMIT-OID. + * This is somewhat redundant, but lets us later do `git name-rev` + * using the containing-oid in a consistent fashion. + */ + maybe_insert_large_item(psc->vec_largest_by_nr_parents, k, + &commit->object.oid, NULL, + &commit->object.oid); + maybe_insert_large_item(psc->vec_largest_by_size_bytes, object_length, + &commit->object.oid, NULL, + &commit->object.oid); if (k >= PBIN_VEC_LEN) k = PBIN_VEC_LEN - 1; @@ -727,8 +762,12 @@ static void traverse_object_cb_tree(struct object *obj, const char *name) pst->sum_entries += nr_entries; - maybe_insert_large_item(pst->vec_largest_by_nr_entries, nr_entries, &obj->oid, name); - maybe_insert_large_item(pst->vec_largest_by_size_bytes, object_length, &obj->oid, name); + maybe_insert_large_item(pst->vec_largest_by_nr_entries, nr_entries, + &obj->oid, name, + &treewalk_transient_commit_oid); + maybe_insert_large_item(pst->vec_largest_by_size_bytes, object_length, + &obj->oid, name, + &treewalk_transient_commit_oid); qb = qbin(nr_entries); incr_obj_hist_bin(&pst->entry_qbin[qb], object_length, disk_sizep); @@ -741,7 +780,9 @@ static void traverse_object_cb_blob(struct object *obj, const char *name) fill_in_base_object(&psb->base, obj, OBJ_BLOB, &object_length, NULL); - maybe_insert_large_item(psb->vec_largest_by_size_bytes, object_length, &obj->oid, name); + maybe_insert_large_item(psb->vec_largest_by_size_bytes, object_length, + &obj->oid, name, + &treewalk_transient_commit_oid); } static void traverse_object_cb(struct object *obj, const char *name, void *data) @@ -774,6 +815,7 @@ static void do_treewalk_reachable(struct ref_array *ref_array) repo_init_revisions(the_repository, &rev_info, NULL); rev_info.tree_objects = 1; rev_info.blob_objects = 1; + rev_info.tree_blobs_in_commit_order = 1; load_rev_info(&rev_info, ref_array); if (prepare_revision_walk(&rev_info)) die(_("revision walk setup failed")); @@ -783,10 +825,12 @@ static void do_treewalk_reachable(struct ref_array *ref_array) survey_progress = start_progress(_("Walking reachable objects..."), 0); } + oidcpy(&treewalk_transient_commit_oid, null_oid()); traverse_commit_list(&rev_info, traverse_commit_cb, traverse_object_cb, NULL); + oidcpy(&treewalk_transient_commit_oid, null_oid()); if (survey_opts.show_progress) stop_progress(&survey_progress); @@ -1229,6 +1273,9 @@ static void write_large_item_vec_json(struct json_writer *jw, jw_object_string(jw, "oid", oid_to_hex(&pk->oid)); if (pk->name->len) jw_object_string(jw, "name", pk->name->buf); + if (!is_null_oid(&pk->containing_commit_oid)) + jw_object_string(jw, "commit_oid", + oid_to_hex(&pk->containing_commit_oid)); } jw_end(jw); } From 6c6b28c3602b88c753f641ce34d561f5adb947d4 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 20 May 2024 17:23:39 -0400 Subject: [PATCH 016/189] survey: add commit name-rev lookup to each large_item Signed-off-by: Jeff Hostetler --- builtin/survey.c | 89 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/builtin/survey.c b/builtin/survey.c index d403e039104ca5..9cc1d0e23f0d00 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -11,6 +11,7 @@ #include "ref-filter.h" #include "refs.h" #include "revision.h" +#include "run-command.h" #include "strbuf.h" #include "strmap.h" #include "strvec.h" @@ -342,6 +343,12 @@ struct large_item { * order). */ struct object_id containing_commit_oid; + + /* + * Lookup `containing_commit_oid` using `git name-rev`. + * Lazy allocate this post-treewalk. + */ + struct strbuf *name_rev; }; struct large_item_vec { @@ -383,6 +390,11 @@ static void free_large_item_vec(struct large_item_vec *vec) for (k = 0; k < vec->nr_items; k++) { strbuf_release(vec->items[k].name); free(vec->items[k].name); + + if (vec->items[k].name_rev) { + strbuf_release(vec->items[k].name_rev); + free(vec->items[k].name_rev); + } } free(vec->dimension_label); @@ -419,6 +431,9 @@ static void maybe_insert_large_item(struct large_item_vec *vec, * The last large_item in the vector is about to be * overwritten by the previous one during the shift. * Steal its allocated strbuf and reuse it. + * + * We can ignore .name_rev because it will not be + * allocated until after the treewalk. */ pbuf_temp = vec->items[vec->nr_items - 1].name; strbuf_reset(pbuf_temp); @@ -442,6 +457,54 @@ static void maybe_insert_large_item(struct large_item_vec *vec, } } +/* + * Try to run `git name-rev` on each of the containing-commit-oid's + * in this large-item-vec to get a pretty name for each OID. Silently + * ignore errors if it fails because this info is nice to have but not + * essential. + */ +static void large_item_vec_lookup_name_rev(struct large_item_vec *vec) +{ + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf in = STRBUF_INIT; + struct strbuf out = STRBUF_INIT; + const char *line; + size_t k; + + if (!vec || !vec->nr_items) + return; + + survey_progress_total += vec->nr_items; + display_progress(survey_progress, survey_progress_total); + + for (k = 0; k < vec->nr_items; k++) + strbuf_addf(&in, "%s\n", oid_to_hex(&vec->items[k].containing_commit_oid)); + + cp.git_cmd = 1; + strvec_pushl(&cp.args, "name-rev", "--name-only", "--annotate-stdin", NULL); + if (pipe_command(&cp, in.buf, in.len, &out, 0, NULL, 0)) { + strbuf_release(&in); + strbuf_release(&out); + return; + } + + line = out.buf; + k = 0; + while (*line) { + const char *eol = strchrnul(line, '\n'); + + vec->items[k].name_rev = xcalloc(1, sizeof(struct strbuf)); + strbuf_init(vec->items[k].name_rev, 0); + strbuf_add(vec->items[k].name_rev, line, (eol - line)); + + line = eol + 1; + k++; + } + + strbuf_release(&in); + strbuf_release(&out); +} + /* * Common fields for any type of object. */ @@ -1010,6 +1073,25 @@ static void do_calc_stats_refs(struct repository *r, struct ref_array *ref_array } } +static void do_lookup_name_rev(void) +{ + if (survey_opts.show_progress) { + survey_progress_total = 0; + survey_progress = start_progress(_("Resolving name-revs..."), 0); + } + + large_item_vec_lookup_name_rev(survey_stats.commits.vec_largest_by_nr_parents); + large_item_vec_lookup_name_rev(survey_stats.commits.vec_largest_by_size_bytes); + + large_item_vec_lookup_name_rev(survey_stats.trees.vec_largest_by_nr_entries); + large_item_vec_lookup_name_rev(survey_stats.trees.vec_largest_by_size_bytes); + + large_item_vec_lookup_name_rev(survey_stats.blobs.vec_largest_by_size_bytes); + + if (survey_opts.show_progress) + stop_progress(&survey_progress); +} + /* * The REFS phase: * @@ -1040,6 +1122,10 @@ static void survey_phase_refs(struct repository *r) do_calc_stats_refs(r, &ref_array); trace2_region_leave("survey", "phase/calcstats", the_repository); + trace2_region_enter("survey", "phase/namerev", the_repository); + do_lookup_name_rev(); + trace2_region_enter("survey", "phase/namerev", the_repository); + ref_array_clear(&ref_array); } @@ -1276,6 +1362,9 @@ static void write_large_item_vec_json(struct json_writer *jw, if (!is_null_oid(&pk->containing_commit_oid)) jw_object_string(jw, "commit_oid", oid_to_hex(&pk->containing_commit_oid)); + if (pk->name_rev->len) + jw_object_string(jw, "name_rev", + pk->name_rev->buf); } jw_end(jw); } From 5a6a80bb0192838c6209a514bda66971cbc3146e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 21 May 2024 13:44:07 -0400 Subject: [PATCH 017/189] survey: add --json option and setup for pretty output Signed-off-by: Jeff Hostetler --- Documentation/git-survey.txt | 3 +++ builtin/survey.c | 20 +++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Documentation/git-survey.txt b/Documentation/git-survey.txt index 771a063efdc594..0e42fb32ac2964 100644 --- a/Documentation/git-survey.txt +++ b/Documentation/git-survey.txt @@ -32,6 +32,9 @@ OPTIONS --progress:: Show progress. This is automatically enabled when interactive. +--json:: + Print results in JSON rather than in a human-friendly format. + Ref Selection ~~~~~~~~~~~~~ diff --git a/builtin/survey.c b/builtin/survey.c index 9cc1d0e23f0d00..8632c974ebc3b1 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -56,6 +56,7 @@ static struct survey_refs_wanted refs_if_unspecified = { struct survey_opts { int verbose; int show_progress; + int show_json; int show_largest_commits_by_nr_parents; int show_largest_commits_by_size_bytes; @@ -73,6 +74,7 @@ struct survey_opts { static struct survey_opts survey_opts = { .verbose = 0, .show_progress = -1, /* defaults to isatty(2) */ + .show_json = 0, /* defaults to pretty */ /* * Show the largest `n` objects for some scaling dimension. @@ -154,6 +156,7 @@ static void fixup_refs_wanted(void) static struct option survey_options[] = { OPT__VERBOSE(&survey_opts.verbose, N_("verbose output")), OPT_BOOL(0, "progress", &survey_opts.show_progress, N_("show progress")), + OPT_BOOL(0, "json", &survey_opts.show_json, N_("report stats in JSON")), OPT_BOOL_F(0, "all-refs", &survey_opts.refs.want_all_refs, N_("include all refs"), PARSE_OPT_NONEG), @@ -185,6 +188,10 @@ static int survey_load_config_cb(const char *var, const char *value, survey_opts.show_progress = git_config_bool(var, value); return 0; } + if (!strcmp(var, "survey.json")) { + survey_opts.show_json = git_config_bool(var, value); + return 0; + } if (!strcmp(var, "survey.showcommitparents")) { survey_opts.show_largest_commits_by_nr_parents = git_config_ulong(var, value, ctx->kvi); @@ -1490,6 +1497,14 @@ static void survey_emit_trace2(void) json_blobs_section(NULL, 0, 1); } +/* + * Print all of the stats that we have collected in a more pretty format. + */ +static void survey_print_results_pretty(void) +{ + printf("TODO....\n"); +} + int cmd_survey(int argc, const char **argv, const char *prefix) { survey_load_config(); @@ -1538,7 +1553,10 @@ int cmd_survey(int argc, const char **argv, const char *prefix) survey_phase_refs(the_repository); survey_emit_trace2(); - survey_print_json(); + if (survey_opts.show_json) + survey_print_json(); + else + survey_print_results_pretty(); strvec_clear(&survey_vec_refs_wanted); free_large_item_vec(survey_stats.commits.vec_largest_by_nr_parents); From c84296d68e01eb7bdad9ea7a72d603012d3c1bc9 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 21 May 2024 15:47:58 -0400 Subject: [PATCH 018/189] survey: add pretty printing of stats Signed-off-by: Jeff Hostetler --- builtin/survey.c | 788 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 742 insertions(+), 46 deletions(-) diff --git a/builtin/survey.c b/builtin/survey.c index 8632c974ebc3b1..d2e53885fd5820 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -338,8 +338,9 @@ struct large_item { /* * For blobs and trees the name field is the pathname of the - * file or directory. Root trees will have a zero-length - * name. The name field is not currenly used for commits. + * file or directory (as reported by the treewalk). Root trees + * are reported with a zero-length name, but we'll fix them up. + * The name field is not currenly used for commits. */ struct strbuf *name; @@ -358,16 +359,24 @@ struct large_item { struct strbuf *name_rev; }; +struct large_item_vec_labels { + const char *dimension; + const char *item; +}; + struct large_item_vec { - char *dimension_label; - char *item_label; + const struct large_item_vec_labels *labels_json; + const struct large_item_vec_labels *labels_pretty; uint64_t nr_items; + enum object_type type; struct large_item items[FLEX_ARRAY]; /* nr_items */ }; -static struct large_item_vec *alloc_large_item_vec(const char *dimension_label, - const char *item_label, - uint64_t nr_items) +static struct large_item_vec *alloc_large_item_vec( + const struct large_item_vec_labels *labels_json, + const struct large_item_vec_labels *labels_pretty, + uint64_t nr_items, + enum object_type type) { struct large_item_vec *vec; size_t flex_len = nr_items * sizeof(struct large_item); @@ -377,9 +386,10 @@ static struct large_item_vec *alloc_large_item_vec(const char *dimension_label, return NULL; vec = xcalloc(1, (sizeof(struct large_item_vec) + flex_len)); - vec->dimension_label = strdup(dimension_label); - vec->item_label = strdup(item_label); + vec->labels_json = labels_json; + vec->labels_pretty = labels_pretty; vec->nr_items = nr_items; + vec->type = type; for (k = 0; k < nr_items; k++) { struct strbuf *p = xcalloc(1, sizeof(struct strbuf)); @@ -404,8 +414,6 @@ static void free_large_item_vec(struct large_item_vec *vec) } } - free(vec->dimension_label); - free(vec->item_label); free(vec); } @@ -446,6 +454,15 @@ static void maybe_insert_large_item(struct large_item_vec *vec, strbuf_reset(pbuf_temp); if (name && *name) strbuf_addstr(pbuf_temp, name); + else if (vec->type == OBJ_TREE) { + /* + * NEEDSWORK: Would it be better to wait and create + * a name of the form "^{tree}" after the + * treewalk is finished? + */ + strbuf_addf(pbuf_temp, "%s^{tree}", + oid_to_hex(containing_commit_oid)); + } /* push items[k..] down one and insert data for this item here */ @@ -616,6 +633,95 @@ struct survey_stats { static struct survey_stats survey_stats = { 0 }; +static void alloc_commit_by_parents(void) +{ + static struct large_item_vec_labels json = { + .dimension = "largest_commits_by_nr_parents", + .item = "nr_parents", + }; + static struct large_item_vec_labels pretty = { + .dimension = "Largest Commits by Number of Parents", + .item = "Parents", + }; + + if (survey_opts.show_largest_commits_by_nr_parents) + survey_stats.commits.vec_largest_by_nr_parents = + alloc_large_item_vec(&json, &pretty, + survey_opts.show_largest_commits_by_nr_parents, + OBJ_COMMIT); +} + +static void alloc_commit_by_size(void) { + static struct large_item_vec_labels json = { + .dimension = "largest_commits_by_size_bytes", + .item = "size", + }; + static struct large_item_vec_labels pretty = { + .dimension = "Largest Commits by Size in Bytes", + .item = "Size", + }; + + if (survey_opts.show_largest_commits_by_size_bytes) + survey_stats.commits.vec_largest_by_size_bytes = + alloc_large_item_vec(&json, &pretty, + survey_opts.show_largest_commits_by_size_bytes, + OBJ_COMMIT); +} + +static void alloc_tree_by_entries(void) +{ + static struct large_item_vec_labels json = { + .dimension = "largest_trees_by_nr_entries", + .item = "nr_entries", + }; + static struct large_item_vec_labels pretty = { + .dimension = "Largest Trees by Number of Entries", + .item = "Entries", + }; + + if (survey_opts.show_largest_trees_by_nr_entries) + survey_stats.trees.vec_largest_by_nr_entries = + alloc_large_item_vec(&json, &pretty, + survey_opts.show_largest_trees_by_nr_entries, + OBJ_TREE); +} + +static void alloc_tree_by_size(void) +{ + static struct large_item_vec_labels json = { + .dimension = "largest_trees_by_size_bytes", + .item = "size", + }; + static struct large_item_vec_labels pretty = { + .dimension = "Largest Trees by Size in Bytes", + .item = "Size", + }; + + if (survey_opts.show_largest_trees_by_size_bytes) + survey_stats.trees.vec_largest_by_size_bytes = + alloc_large_item_vec(&json, &pretty, + survey_opts.show_largest_trees_by_size_bytes, + OBJ_TREE); +} + +static void alloc_blob_by_size(void) +{ + static struct large_item_vec_labels json = { + .dimension = "largest_blobs_by_size_bytes", + .item = "size", + }; + static struct large_item_vec_labels pretty = { + .dimension = "Largest Blobs by Size in Bytes", + .item = "Size", + }; + + if (survey_opts.show_largest_blobs_by_size_bytes) + survey_stats.blobs.vec_largest_by_size_bytes = + alloc_large_item_vec(&json, &pretty, + survey_opts.show_largest_blobs_by_size_bytes, + OBJ_BLOB); +} + static void do_load_refs(struct ref_array *ref_array) { struct ref_filter filter = REF_FILTER_INIT; @@ -1351,7 +1457,7 @@ static void write_large_item_vec_json(struct json_writer *jw, if (!vec || !vec->nr_items) return; - jw_object_inline_begin_array(jw, vec->dimension_label); + jw_object_inline_begin_array(jw, vec->labels_json->dimension); { int k; @@ -1362,7 +1468,7 @@ static void write_large_item_vec_json(struct json_writer *jw, jw_array_inline_begin_object(jw); { - jw_object_intmax(jw, vec->item_label, pk->size); + jw_object_intmax(jw, vec->labels_json->item, pk->size); jw_object_string(jw, "oid", oid_to_hex(&pk->oid)); if (pk->name->len) jw_object_string(jw, "name", pk->name->buf); @@ -1497,12 +1603,629 @@ static void survey_emit_trace2(void) json_blobs_section(NULL, 0, 1); } +static void fmt_txt_line(struct strbuf *buf, int indent, const char *txt) +{ + if (indent) + strbuf_addchars(buf, ' ', indent); + + strbuf_addstr(buf, txt); + + strbuf_addch(buf, '\n'); +} + +static void fmt_txt_pair_ui64(struct strbuf *buf, + int indent, + const char *label, + uint64_t value) +{ + int column0 = 62; + + if (indent) + strbuf_addchars(buf, ' ', indent); + + strbuf_addf(buf, "%-*s : %14"PRIu64, + column0 - indent, label, + value); + + strbuf_addch(buf, '\n'); +} + +static void fmt_size_tbl_caption(struct strbuf *buf, + int indent, + const char *caption) +{ + strbuf_addch(buf, '\n'); + fmt_txt_line(buf, indent, caption); +} + +static void fmt_size_tbl_hdr(struct strbuf *buf, + int indent, + const char *bucket_hdr, + const char *count_hdr, + const char *size_hdr, + const char *disk_size_hdr) +{ + int column0 = 28; + + if (indent) + strbuf_addchars(buf, ' ', indent); + + strbuf_addf(buf, "%-*s | %14s | %14s | %14s", + column0 - indent, bucket_hdr, + count_hdr, size_hdr, disk_size_hdr); + + strbuf_addch(buf, '\n'); +} + +static void fmt_size_tbl_hr(struct strbuf *buf, + int indent) +{ + int column0 = 28; + + if (indent) + strbuf_addchars(buf, ' ', indent); + + strbuf_addchars(buf, '-', column0 - indent); + strbuf_addstr(buf, "-+-"); + strbuf_addchars(buf, '-', 14); + strbuf_addstr(buf, "-+-"); + strbuf_addchars(buf, '-', 14); + strbuf_addstr(buf, "-+-"); + strbuf_addchars(buf, '-', 14); + + strbuf_addch(buf, '\n'); +} + +static void fmt_size_tbl_row(struct strbuf *buf, + int indent, + const char *bucket, + uint64_t count, + uint64_t size, + uint64_t disk_size) +{ + int column0 = 28; + + if (indent) + strbuf_addchars(buf, ' ', indent); + + strbuf_addf(buf, "%-*s | %14"PRIu64" | %14"PRIu64" | %14"PRIu64, + column0 - indent, bucket, count, size, disk_size); + + strbuf_addch(buf, '\n'); +} + +static void fmt_qbin(struct strbuf *buf, + int indent, const char *title_caption, + const char *bucket_hdr, + struct obj_hist_bin qbin[QBIN_LEN]) +{ + struct strbuf bucket = STRBUF_INIT; + uint64_t lower = 0; + uint64_t upper = QBIN_MASK; + int k; + + fmt_size_tbl_caption(buf, indent, title_caption); + fmt_size_tbl_hr(buf, indent); + fmt_size_tbl_hdr(buf, indent, bucket_hdr, "Count", "Size", "Disk Size"); + fmt_size_tbl_hr(buf, indent); + + for (k = 0; k < QBIN_LEN; k++) { + struct obj_hist_bin *p = &qbin[k]; + uint64_t lower_k = lower; + uint64_t upper_k = upper; + + lower = upper+1; + upper = (upper << QBIN_SHIFT) + QBIN_MASK; + + if (!p->cnt_seen) + continue; + + strbuf_reset(&bucket); + strbuf_addf(&bucket, "%"PRIu64"..%"PRIu64, lower_k, upper_k); + + fmt_size_tbl_row(buf, indent, bucket.buf, + p->cnt_seen, p->sum_size, p->sum_disk_size); + } + fmt_size_tbl_hr(buf, indent); + + strbuf_release(&bucket); +} + +static void fmt_hbin(struct strbuf *buf, + int indent, const char *title_caption, + const char *bucket_hdr, + struct obj_hist_bin hbin[HBIN_LEN]) +{ + struct strbuf bucket = STRBUF_INIT; + uint64_t lower = 0; + uint64_t upper = HBIN_MASK; + int k; + + fmt_size_tbl_caption(buf, indent, title_caption); + fmt_size_tbl_hr(buf, indent); + fmt_size_tbl_hdr(buf, indent, bucket_hdr, "Count", "Size", "Disk Size"); + fmt_size_tbl_hr(buf, indent); + + for (k = 0; k < HBIN_LEN; k++) { + struct obj_hist_bin *p = &hbin[k]; + uint64_t lower_k = lower; + uint64_t upper_k = upper; + + lower = upper+1; + upper = (upper << HBIN_SHIFT) + HBIN_MASK; + + if (!p->cnt_seen) + continue; + + strbuf_reset(&bucket); + strbuf_addf(&bucket, "%"PRIu64"..%"PRIu64, lower_k, upper_k); + + fmt_size_tbl_row(buf, indent, bucket.buf, + p->cnt_seen, p->sum_size, p->sum_disk_size); + } + fmt_size_tbl_hr(buf, indent); + + strbuf_release(&bucket); +} + +static void fmt_pbin_hdr(struct strbuf *buf, + int indent, + const char *bucket, + const char *count) +{ + int column0 = 28; + + if (indent) + strbuf_addchars(buf, ' ', indent); + + strbuf_addf(buf, "%-*s | %14s", + column0 - indent, bucket, + count); + + strbuf_addch(buf, '\n'); +} + +static void fmt_pbin_hr(struct strbuf *buf, + int indent) +{ + int column0 = 28; + + if (indent) + strbuf_addchars(buf, ' ', indent); + + strbuf_addchars(buf, '-', column0 - indent); + strbuf_addstr(buf, "-+-"); + strbuf_addchars(buf, '-', 14); + + strbuf_addch(buf, '\n'); +} + +static void fmt_pbin_row(struct strbuf *buf, + int indent, + int nr, + int count) +{ + struct strbuf bucket = STRBUF_INIT; + int column0 = 28; + + if (indent) + strbuf_addchars(buf, ' ', indent); + + strbuf_addf(&bucket, "%2d", nr); + strbuf_addf(buf, "%-*s | %14d", + column0 - indent, bucket.buf, + count); + + strbuf_addch(buf, '\n'); + strbuf_release(&bucket); +} + +static void fmt_base_object(struct strbuf *buf, + int indent, + struct survey_stats_base_object *base) +{ + int indent1 = indent + 4; + + fmt_txt_pair_ui64(buf, indent, "Total Count", base->cnt_seen); + + strbuf_addch(buf, '\n'); + fmt_txt_line(buf, indent, "Count by Storage Location"); + if (base->cnt_missing) + fmt_txt_pair_ui64(buf, indent1, "Missing", base->cnt_missing); + if (base->cnt_cached) + fmt_txt_pair_ui64(buf, indent1, "Cached", base->cnt_cached); + if (base->cnt_loose) + fmt_txt_pair_ui64(buf, indent1, "Loose", base->cnt_loose); + if (base->cnt_packed) + fmt_txt_pair_ui64(buf, indent1, "Packed", base->cnt_packed); + if (base->cnt_dbcached) + fmt_txt_pair_ui64(buf, indent1, "DBCached", base->cnt_dbcached); + + strbuf_addch(buf, '\n'); + fmt_txt_pair_ui64(buf, indent, "Total Size in Bytes", base->sum_size); + fmt_txt_pair_ui64(buf, indent, "Total Disk Size in Bytes", base->sum_disk_size); + + fmt_hbin(buf, indent, "Histogram by Size in Bytes", "Byte Range", base->size_hbin); +} + +static void fmt_large_item_hdr(struct strbuf *buf, + int indent, + int name_length, + int name_rev_length, + const char *item_hdr_label) +{ + int column0 = the_hash_algo->hexsz; + + if (indent) + strbuf_addchars(buf, ' ', indent); + + strbuf_addf(buf, "%-*s | %14s", column0, "OID", item_hdr_label); + if (name_length) + strbuf_addf(buf, " | %-*s", name_length, "Name"); + strbuf_addf(buf, " | %-*s", name_rev_length, "Name Rev"); + + strbuf_addch(buf, '\n'); +} + +static void fmt_large_item_hr(struct strbuf *buf, + int indent, + int name_length, + int name_rev_length) +{ + int column0 = the_hash_algo->hexsz; + + if (indent) + strbuf_addchars(buf, ' ', indent); + + strbuf_addchars(buf, '-', column0); + strbuf_addstr(buf, "-+-"); + strbuf_addchars(buf, '-', 14); + if (name_length) { + strbuf_addstr(buf, "-+-"); + strbuf_addchars(buf, '-', name_length); + } + strbuf_addstr(buf, "-+-"); + strbuf_addchars(buf, '-', name_rev_length); + + strbuf_addch(buf, '\n'); +} + +static void fmt_large_item_row(struct strbuf *buf, + int indent, + int name_length, + int name_rev_length, + struct large_item *pitem) +{ + int column0 = the_hash_algo->hexsz; + + if (indent) + strbuf_addchars(buf, ' ', indent); + + strbuf_addf(buf, "%-*s | %14"PRIu64, + column0, oid_to_hex(&pitem->oid), + pitem->size); + if (name_length) + strbuf_addf(buf, " | %-*s", name_length, + (pitem->name ? pitem->name->buf: "")); + strbuf_addf(buf, " | %-*s", name_rev_length, pitem->name_rev->buf); + + strbuf_addch(buf, '\n'); +} + +static void fmt_large_item_vec(struct strbuf *buf, + int indent, + struct large_item_vec *pvec) +{ + int name_length = 0; + int name_rev_length = 10; + int k; + + if (pvec->type != OBJ_COMMIT) { + /* Add "Name" column for trees and blobs. */ + for (k = 0; k < pvec->nr_items; k++) + if (pvec->items[k].name && pvec->items[k].name->len > name_length) + name_length = pvec->items[k].name->len; + if (name_length) + if (name_length < 4) /* strlen("Name") */ + name_length = 4; + } + + for (k = 0; k < pvec->nr_items; k++) { + struct large_item *pk = &pvec->items[k]; + if (pk->name_rev->len > name_rev_length) + name_rev_length = pk->name_rev->len; + } + + strbuf_addch(buf, '\n'); + fmt_txt_line(buf, indent, pvec->labels_pretty->dimension); + fmt_large_item_hr(buf, indent, name_length, name_rev_length); + fmt_large_item_hdr(buf, indent, name_length, name_rev_length, pvec->labels_pretty->item); + fmt_large_item_hr(buf, indent, name_length, name_rev_length); + + for (k = 0; k < pvec->nr_items; k++) { + struct large_item *pk = &pvec->items[k]; + if (is_null_oid(&pk->oid)) + break; + + fmt_large_item_row(buf, indent, name_length, name_rev_length, pk); + } + + fmt_large_item_hr(buf, indent, name_length, name_rev_length); +} + +static void pretty_print_survey_hdr(void) +{ + struct strbuf buf = STRBUF_INIT; + int indent = 0; + int k; + + const char *intro[] = { + "", + "===============================================================================", + "Git Survey Results", + "===============================================================================", + "", + NULL + }; + + k = 0; + while (intro[k]) + fmt_txt_line(&buf, indent, intro[k++]); + + /* + * NEEDSWORK: Consider adding information about the repo pathname, + * the date, command line args, git version, etc. + */ + + fwrite(buf.buf, 1, buf.len, stdout); + strbuf_release(&buf); +} + +static void pretty_print_overview(int indent) +{ + struct survey_stats_refs *prs = &survey_stats.refs; + struct survey_stats_commits *psc = &survey_stats.commits; + struct survey_stats_trees *pst = &survey_stats.trees; + struct survey_stats_blobs *psb = &survey_stats.blobs; + struct strbuf buf = STRBUF_INIT; + int indent1 = indent + 4; + int indent2 = indent + 8; + int k; + + const char *intro[] = { + "", + "OVERVIEW", + "-------------------------------------------------------------------------------", + "", + NULL + }; + + k = 0; + while (intro[k]) + fmt_txt_line(&buf, indent, intro[k++]); + + fmt_txt_pair_ui64(&buf, indent1, "Total Number of Refs", prs->cnt_total); + + fmt_size_tbl_caption(&buf, indent1, "Overview by Object Type"); + + fmt_size_tbl_hr(&buf, indent1); + fmt_size_tbl_hdr(&buf, indent1, "Type", "Count", "Size", "Disk Size"); + fmt_size_tbl_hr(&buf, indent1); + + fmt_size_tbl_row(&buf, indent2, "Commits", psc->base.cnt_seen, psc->base.sum_size, psc->base.sum_disk_size); + fmt_size_tbl_row(&buf, indent2, "Trees", pst->base.cnt_seen, pst->base.sum_size, pst->base.sum_disk_size); + fmt_size_tbl_row(&buf, indent2, "Blobs", psb->base.cnt_seen, psb->base.sum_size, psb->base.sum_disk_size); + + fmt_size_tbl_hr(&buf, indent1); + fmt_size_tbl_row(&buf, indent1, "Total", + psc->base.cnt_seen + pst->base.cnt_seen + psb->base.cnt_seen, + psc->base.sum_size + pst->base.sum_size + psb->base.sum_size, + psc->base.sum_disk_size + pst->base.sum_disk_size + psb->base.sum_disk_size); + fmt_size_tbl_hr(&buf, indent1); + + strbuf_addch(&buf, '\n'); + fwrite(buf.buf, 1, buf.len, stdout); + strbuf_release(&buf); +} + +/* + * Pretty print information on the set of REFS that we examined. + */ +static void pretty_print_refs(int indent) +{ + struct survey_refs_wanted *prw = &survey_opts.refs; + struct survey_stats_refs *prs = &survey_stats.refs; + struct strbuf buf = STRBUF_INIT; + int indent1 = indent + 4; + int indent2 = indent + 8; + int indent3 = indent + 12; + int k; + + const char *intro[] = { + "", + "REFS", + "-------------------------------------------------------------------------------", + "", + NULL + }; + + k = 0; + while (intro[k]) + fmt_txt_line(&buf, indent, intro[k++]); + + fmt_txt_pair_ui64(&buf, indent1, "Total Number of Refs", prs->cnt_total); + + strbuf_addch(&buf, '\n'); + fmt_txt_line(&buf, indent1, "Reference Count by Type"); + + if (prw->want_remotes && prs->cnt_remotes) + fmt_txt_pair_ui64(&buf, indent2, "Remote Tracking Branches", prs->cnt_remotes); + + if (prw->want_branches && prs->cnt_branches) + fmt_txt_pair_ui64(&buf, indent2, "Branches", prs->cnt_branches); + if (prw->want_tags && prs->cnt_lightweight_tags) + fmt_txt_pair_ui64(&buf, indent2, "Tags (Lightweight)", prs->cnt_lightweight_tags); + if (prw->want_tags && prs->cnt_annotated_tags) + fmt_txt_pair_ui64(&buf, indent2, "Tags (Annotated)", prs->cnt_annotated_tags); + if (prw->want_detached && prs->cnt_detached) + fmt_txt_pair_ui64(&buf, indent2, "Detached", prs->cnt_detached); + if (prw->want_other && prs->cnt_other) + fmt_txt_pair_ui64(&buf, indent2, "Other (Notes and Stashes)", prs->cnt_other); + + if (prs->cnt_symref) + fmt_txt_pair_ui64(&buf, indent2, "Symbolic Refs (like 'HEAD')", prs->cnt_symref); + + strbuf_addch(&buf, '\n'); + fmt_txt_pair_ui64(&buf, indent1, "Reference Count by Class", strintmap_get_size(&prs->refsmap)); + { + struct hashmap_iter iter; + struct strmap_entry *entry; + + strintmap_for_each_entry(&prs->refsmap, &iter, entry) { + const char *key = entry->key; + intptr_t count = (intptr_t)entry->value; + int value = count; + + fmt_txt_pair_ui64(&buf, indent2, key, value); + } + } + + strbuf_addch(&buf, '\n'); + fmt_txt_line(&buf, indent1, "Reference Count by Storage Location"); + fmt_txt_pair_ui64(&buf, indent2, "Loose", prs->cnt_loose); + fmt_txt_pair_ui64(&buf, indent2, "Packed", prs->cnt_packed); + + strbuf_addch(&buf, '\n'); + fmt_txt_line(&buf, indent1, "String Length of Refnames"); + if (prs->len_sum_remote_refnames) { + fmt_txt_line(&buf, indent2, "Remote Refs"); + fmt_txt_pair_ui64(&buf, indent3, "Max", prs->len_max_remote_refname); + fmt_txt_pair_ui64(&buf, indent3, "Sum", prs->len_sum_remote_refnames); + } + if (prs->len_sum_local_refnames) { + fmt_txt_line(&buf, indent2, "Local Refs"); + fmt_txt_pair_ui64(&buf, indent3, "Max", prs->len_max_local_refname); + fmt_txt_pair_ui64(&buf, indent3, "Sum", prs->len_sum_local_refnames); + } + + strbuf_addch(&buf, '\n'); + fwrite(buf.buf, 1, buf.len, stdout); + strbuf_release(&buf); +} + +static void pretty_print_commits(int indent) +{ + struct survey_stats_commits *psc = &survey_stats.commits; + struct survey_stats_base_object *base = &psc->base; + struct strbuf buf = STRBUF_INIT; + int indent1 = indent + 4; + int k; + + const char *intro[] = { + "", + "COMMITS", + "-------------------------------------------------------------------------------", + "", + NULL + }; + + k = 0; + while (intro[k]) + fmt_txt_line(&buf, indent, intro[k++]); + + fmt_base_object(&buf, indent1, base); + + fmt_large_item_vec(&buf, indent1, psc->vec_largest_by_size_bytes); + + strbuf_addch(&buf, '\n'); + fmt_txt_line(&buf, indent1, "Histogram by Number of Parents"); + fmt_pbin_hr(&buf, indent1); + fmt_pbin_hdr(&buf, indent1, "Parents", "Count"); + fmt_pbin_hr(&buf, indent1); + for (k = 0; k < PBIN_VEC_LEN; k++) + if (psc->parent_cnt_pbin[k]) + fmt_pbin_row(&buf, indent1, k, psc->parent_cnt_pbin[k]); + fmt_pbin_hr(&buf, indent1); + + fmt_large_item_vec(&buf, indent1, psc->vec_largest_by_nr_parents); + + strbuf_addch(&buf, '\n'); + fwrite(buf.buf, 1, buf.len, stdout); + strbuf_release(&buf); +} + +static void pretty_print_trees(int indent) +{ + struct survey_stats_trees *pst = &survey_stats.trees; + struct survey_stats_base_object *base = &pst->base; + struct strbuf buf = STRBUF_INIT; + int indent1 = indent + 4; + int k; + + const char *intro[] = { + "", + "TREES", + "-------------------------------------------------------------------------------", + "", + NULL + }; + + k = 0; + while (intro[k]) + fmt_txt_line(&buf, indent, intro[k++]); + + fmt_base_object(&buf, indent1, base); + + fmt_large_item_vec(&buf, indent1, pst->vec_largest_by_size_bytes); + + fmt_qbin(&buf, indent1, "Tree Histogram by Number of Entries", "Entry Range", pst->entry_qbin); + fmt_large_item_vec(&buf, indent1, pst->vec_largest_by_nr_entries); + + strbuf_addch(&buf, '\n'); + fwrite(buf.buf, 1, buf.len, stdout); + strbuf_release(&buf); +} + +static void pretty_print_blobs(int indent) +{ + struct survey_stats_blobs *psb = &survey_stats.blobs; + struct survey_stats_base_object *base = &psb->base; + struct strbuf buf = STRBUF_INIT; + int indent1 = indent + 4; + int k; + + const char *intro[] = { + "", + "BLOBS", + "-------------------------------------------------------------------------------", + "", + NULL + }; + + k = 0; + while (intro[k]) + fmt_txt_line(&buf, indent, intro[k++]); + + fmt_base_object(&buf, indent1, base); + + fmt_large_item_vec(&buf, indent1, psb->vec_largest_by_size_bytes); + + strbuf_addch(&buf, '\n'); + fwrite(buf.buf, 1, buf.len, stdout); + strbuf_release(&buf); +} + /* * Print all of the stats that we have collected in a more pretty format. */ static void survey_print_results_pretty(void) { - printf("TODO....\n"); + pretty_print_survey_hdr(); + pretty_print_overview(0); + pretty_print_refs(0); + pretty_print_commits(0); + pretty_print_trees(0); + pretty_print_blobs(0); } int cmd_survey(int argc, const char **argv, const char *prefix) @@ -1517,38 +2240,11 @@ int cmd_survey(int argc, const char **argv, const char *prefix) survey_opts.show_progress = isatty(2); fixup_refs_wanted(); - if (survey_opts.show_largest_commits_by_nr_parents) - survey_stats.commits.vec_largest_by_nr_parents = - alloc_large_item_vec( - "largest_commits_by_nr_parents", - "nr_parents", - survey_opts.show_largest_commits_by_nr_parents); - if (survey_opts.show_largest_commits_by_size_bytes) - survey_stats.commits.vec_largest_by_size_bytes = - alloc_large_item_vec( - "largest_commits_by_size_bytes", - "size", - survey_opts.show_largest_commits_by_size_bytes); - - if (survey_opts.show_largest_trees_by_nr_entries) - survey_stats.trees.vec_largest_by_nr_entries = - alloc_large_item_vec( - "largest_trees_by_nr_entries", - "nr_entries", - survey_opts.show_largest_trees_by_nr_entries); - if (survey_opts.show_largest_trees_by_size_bytes) - survey_stats.trees.vec_largest_by_size_bytes = - alloc_large_item_vec( - "largest_trees_by_size_bytes", - "size", - survey_opts.show_largest_trees_by_size_bytes); - - if (survey_opts.show_largest_blobs_by_size_bytes) - survey_stats.blobs.vec_largest_by_size_bytes = - alloc_large_item_vec( - "largest_blobs_by_size_bytes", - "size", - survey_opts.show_largest_blobs_by_size_bytes); + alloc_commit_by_parents(); + alloc_commit_by_size(); + alloc_tree_by_entries(); + alloc_tree_by_size(); + alloc_blob_by_size(); survey_phase_refs(the_repository); From 592b09f415389153818e57c3d5bdcfd249d79eb9 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 29 May 2024 11:12:24 -0400 Subject: [PATCH 019/189] t8100: create test for git-survey Signed-off-by: Jeff Hostetler --- t/t8100-git-survey.sh | 98 +++++++ t/t8100/survey_parse_json.perl | 520 +++++++++++++++++++++++++++++++++ 2 files changed, 618 insertions(+) create mode 100755 t/t8100-git-survey.sh create mode 100644 t/t8100/survey_parse_json.perl diff --git a/t/t8100-git-survey.sh b/t/t8100-git-survey.sh new file mode 100755 index 00000000000000..f74e8743b6b746 --- /dev/null +++ b/t/t8100-git-survey.sh @@ -0,0 +1,98 @@ +#!/bin/sh + +test_description='measure repository and report on scaling dimensions' + +. ./test-lib.sh + +perl -MJSON::PP -e 0 >/dev/null 2>&1 && test_set_prereq JSON_PP + +test_expect_success 'verify zero counts before initial commit' ' + test_when_finished "rm -rf data.json actual* expect*" && + + git survey --json >data.json && + + # Verify that there are no refs and no objects of any kind. + # + perl "$TEST_DIRECTORY/t8100/survey_parse_json.perl" count actual.count && + cat >expect.count <<-\EOF && + refs.count:0 + commits.count:0 + trees.count:0 + blobs.count:0 + EOF + test_cmp expect.count actual.count && + + # Verify that each of the histograms and large-item arrays are empty. + # This is mainly to test the perl script, since `git survey` will generate + # JSON with empty objects and arrays and will get parsed into empty hashes + # and arrays which behave differently in perl. + # + perl "$TEST_DIRECTORY/t8100/survey_parse_json.perl" \ + commits.mostparents \ + commits.histparents \ + trees.histentries \ + trees.mostentries \ + blobs.histsize \ + blobs.largest \ + actual.empty && + cat >expect.empty <<-\EOF && + EOF + test_cmp expect.empty actual.empty +' + +test_expect_success 'initial commit' ' + test_when_finished "rm -rf data.json actual* expect*" && + + touch file0 && + git add file* && + git commit -m "initial" && + + git survey --json >data.json && + + perl "$TEST_DIRECTORY/t8100/survey_parse_json.perl" count actual.count && + cat >expect.count <<-\EOF && + refs.count:1 + commits.count:1 + trees.count:1 + blobs.count:1 + EOF + test_cmp expect.count actual.count && + + perl "$TEST_DIRECTORY/t8100/survey_parse_json.perl" commits.mostparents actual-mp && + cat >expect-mp <<-\EOF && + commits.mostparents[0].nr_parents:0 + EOF + test_cmp expect-mp actual-mp && + + perl "$TEST_DIRECTORY/t8100/survey_parse_json.perl" commits.histparents actual-hp && + cat >expect-hp <<-\EOF && + commits.histparents[P00].count:1 + EOF + test_cmp expect-hp actual-hp && + + perl "$TEST_DIRECTORY/t8100/survey_parse_json.perl" trees.histentries actual-he && + cat >expect-he <<-\EOF && + trees.histentries.Q00.count:1 + EOF + test_cmp expect-he actual-he && + + perl "$TEST_DIRECTORY/t8100/survey_parse_json.perl" trees.mostentries actual-me && + cat >expect-me <<-\EOF && + trees.mostentries[0].nr_entries:1 + EOF + test_cmp expect-me actual-me && + + perl "$TEST_DIRECTORY/t8100/survey_parse_json.perl" blobs.histsize actual-hs && + cat >expect-hs <<-\EOF && + blobs.histsize.H0.count:1 + EOF + test_cmp expect-hs actual-hs && + + perl "$TEST_DIRECTORY/t8100/survey_parse_json.perl" blobs.largest actual-lb && + cat >expect-lb <<-\EOF && + blobs.largest[0].size:0 + EOF + test_cmp expect-lb actual-lb +' + +test_done diff --git a/t/t8100/survey_parse_json.perl b/t/t8100/survey_parse_json.perl new file mode 100644 index 00000000000000..5ac1d4dc732536 --- /dev/null +++ b/t/t8100/survey_parse_json.perl @@ -0,0 +1,520 @@ +#!/usr/bin/perl +# +# Parse the JSON output generated by `git survey --json` to +# support the actual unit tests in the shell script. + +use strict; +use warnings; +use JSON::PP; +use Data::Dumper; + +$Data::Dumper::Sortkeys = 1; +$Data::Dumper::Indent = 1; +$Data::Dumper::Purity = 1; +$Data::Dumper::Pair = ':'; + +my $stdin = join("", ); +my $data = decode_json $stdin; + +#my $dump = Dumper($data); +#print $dump; + +# Create a series of functions / command line args to extract certain +# key values so that the shell script can verify them. +# +# (1) The full JSON data set contains too much data to sanely test in +# a shell script +# +# (2) Some JSON fields are fundamental/foundational, like the number +# of objects, the size of the largest item, or the pathname of the +# largest item. But others are transient, like whether an object or +# ref is packed or loose. And then there are some really transient +# values, like the SHAs of commits when we don't control for the +# data/time. So for simplicity our shell script test will verify the +# basics and not try to do an exact match on the entire data set. +# +# (3) Most of the functionality in `git survey` comes from the various +# existing iterators, for example to enumerate the desired set of refs +# and to treewalk the set of reachable commits, trees, and blobs and +# we are just using iterator callbacks to collect data on the repo. +# We do not need to verify the correctness of the iterator code; we +# just need to verify that we've used it properly when we collected +# our stats. + +# Print various '....count' values from the JSON data. +# +# We assume that the JSON looks like: +# +# { +# ... +# "refs": { +# "count": 3545, +# ... +# }, +# "commits": { +# "count": 197615, +# ... +# }, +# "trees": { +# "count": 331409, +# ... +# }, +# "blobs": { +# "count": 191847, +# ... +# }, +# ... +# } +# +# And we want to emit: +# +# refs.count:3545 +# commits.count:197615 +# trees.count:331409 +# blobs.count:191847 +# +sub Count { + print "refs.count:$data->{'refs'}->{'count'}\n"; + print "commits.count:$data->{'commits'}->{'count'}\n"; + print "trees.count:$data->{'trees'}->{'count'}\n"; + print "blobs.count:$data->{'blobs'}->{'count'}\n"; +} + +# We currently do not expose the "commits.dist_by_size.*" histogram +# for testing. The data is valid, but sensitive to the length of the +# SHAs of the parent commits and root tree and the length of the text +# of the commit message. This is not very interesting and we'll test +# the histogram construction for the other types of objects. +# +# { +# ... +# "commits": { +# "count": 197615, +# ... +# "dist_by_size": { +# "H1": { +# "count": 2268, +# "sum_size": 549925, +# "sum_disk_size": 388778, +# "hbin_lower": 16, +# "hbin_upper": 255 +# }, +# "H2": { +# "count": 194926, +# "sum_size": 138557614, +# "sum_disk_size": 76535965, +# "hbin_lower": 256, +# "hbin_upper": 4095 +# }, +# ... +# }, +# ... +# }, + +# We also do not expose the "commits.largest_commits_by_size_bytes" +# array for testing. This is also sensitive to the length of the SHAs +# and the commit message. We'll explore the histogram construction in +# other types of objects below. +# +# { +# ... +# "commits": { +# "count": 197615, +# ... +# "largest_commits_by_size_bytes": [ +# { +# "size": 78970, +# "oid": "0ab955aac3217bdc64a5df6dd747e8a2238f0473", +# ... +# }, +# { +# "size": 25831, +# "oid": "e74f1e05be5adb88b1d3b282fa500e15b3b04aa7", +# ... +# }, +# ... +# }, +# ... +# }, + + +# Print details for "Largest Commits by Number of Parents". This is +# an array sorted in descending order. For multiple commits with the +# same number of parents, the relative order is undefined. +# +# We assume that the JSON looks like: +# +# { +# ... +# "commits": { +# "count": 197615, +# ... +# "largest_commits_by_nr_parents": [ +# { +# "nr_parents": 10, +# "oid": "16d7601e176cd53f3c2f02367698d06b85e08879", +# ... +# }, +# { +# "nr_parents": 6, +# "oid": "d425142e2a045a9dd7879d028ec68bd748df48a3", +# ... +# }, +# ... +# ], +# ... +# }, +# +# And we want to emit: +# +# commits.mostparents[0].nr_parents:10 +# commits.mostparents[1].nr_parents:6 +# ... +# +sub CommitsMostParents { + my $nr_items = scalar @{ $data->{'commits'}->{'largest_commits_by_nr_parents'} }; + if ($nr_items == 0) { + return 0; + } + my @arr = @{ $data->{'commits'}->{'largest_commits_by_nr_parents'} }; + my $k; + for ($k=0; $k < $nr_items; $k++) { + print "commits.mostparents[$k].nr_parents:$arr[$k]->{'nr_parents'}\n"; + } +} + +# Print details of the "Histogram by Number of Parents" data. +# +# We assume that the JSON looks like: +# +# { +# ... +# "commits": { +# "count": 197615, +# ... +# "count_by_nr_parents": { +# "P00": 13, +# "P01": 148603, +# "P02": 48950, +# "P03": 37, +# "P04": 7, +# "P05": 3, +# "P06": 1, +# "P10": 1 +# } +# }, +# ... +# } +# +# And we want to emit: +# +# commits.histparents[P00].count:13 +# commits.histparents[P01].count:148603 +# ... +# +sub CommitsHistParents { + my $nr_buckets = keys %{ $data->{'commits'}->{'count_by_nr_parents'} }; + if ($nr_buckets == 0) { + return 0; + } + my %dist = %{ $data->{'commits'}->{'count_by_nr_parents'} }; + foreach my $key ( sort keys %dist ) { + my $value = $dist{$key}; + print "commits.histparents[$key].count:$value\n"; + } +} + +# We currently do not expose the "trees.dist_by_size" histogram for +# testing. The data is valid, but sensitive to the length of a SHA +# and the filenames in the tree. That makes it a little trickier to +# test and probably not worth the bother (since we'll test the +# histogram setup code with the "trees.dist_by_nr_entries" and the +# histogram size code in the "blobs.dist_by_size" cases. +# +# { +# ... +# "trees": { +# "count": 331409, +# ... +# "dist_by_size": { +# "H1": { +# "count": 13349, +# "sum_size": 1953155, +# "sum_disk_size": 912044, +# "hbin_lower": 16, +# "hbin_upper": 255 +# }, +# "H2": { +# "count": 52677, +# "sum_size": 101507410, +# "sum_disk_size": 6549425, +# "hbin_lower": 256, +# "hbin_upper": 4095 +# }, +# ... +# }, +# ... +# }, +# ... +# } + +# We also do not expose the "trees.largest_trees_by_size" array for +# testing (for the same SHA and filename reasons). We'll assume that +# the same code is used to build the array of largest blobs. +# +# { +# ... +# "trees": { +# "count": 331409, +# ... +# "largest_trees_by_size_bytes": [ +# { +# "size": 58487, +# "oid": "140160ee18ed56aeaf5e028c60e01874faa9c12d", +# "name": "t", +# ... +# }, +# { +# "size": 58487, +# "oid": "2d5af5733ab1061aae9a7babaabf9064783e3891", +# "name": "t", +# ... +# }, +# ... +# }, +# ... +# }, +# ... +# } + +# Print details for "Histogram by Number of Entries" for trees. For +# example, the bucket `Q00` contains the count of the trees that have +# between 0 and 3 files/subdirectories. +# +# We assume that the JSON looks like: +# +# { +# ... +# "trees": { +# "count": 331409, +# "sum_size": 5376309652, +# ... +# "dist_by_nr_entries": { +# "Q00": { +# "count": 5798, +# "sum_size": 480428, +# "sum_disk_size": 390478, +# "qbin_lower": 0, +# "qbin_upper": 3 +# }, +# "Q01": { +# "count": 15217, +# "sum_size": 4587357, +# "sum_disk_size": 1177431, +# "qbin_lower": 4, +# "qbin_upper": 15 +# }, +# ... +# "Q05": { +# "count": 12965, +# "sum_size": 714372748, +# "sum_disk_size": 11298665, +# "qbin_lower": 1024, +# "qbin_upper": 4095 +# }, +# ... +# } +# }, +# ... +# } +# +# And we want to emit: +# +# trees.histentries.Q00.count:5798 +# trees.histentries.Q01.count:15217 +# ... +# trees.histentries.Q05.count:12965 +# ... +# +sub TreesHistEntries { + my $nr_buckets = keys %{ $data->{'trees'}->{'dist_by_nr_entries'} }; + if ($nr_buckets == 0) { + return 0; + } + my %dist = %{ $data->{'trees'}->{'dist_by_nr_entries'} }; + foreach my $key ( sort keys %dist ) { + my $value = $dist{$key}; + print "trees.histentries.$key.count:$value->{'count'}\n"; + } +} + +# Print details for "Largest Trees by Number of Entries". This is an +# array sorted in descending order. For multiple trees with the same +# number of entries, the relative order is undefined. +# +# We assume that the JSON looks like: +# +# { +# ... +# "trees": { +# "count": 331409, +# ... +# "largest_trees_by_nr_entries": [ +# { +# "nr_entries": 1148, +# "oid": "140160ee18ed56aeaf5e028c60e01874faa9c12d", +# "name": "t", +# ... +# }, +# { +# "nr_entries": 942, +# "oid": "2d5af5733ab1061aae9a7babaabf9064783e3891", +# "name": "t", +# ... +# }, +# ... +# ], +# ... +# }, +# ... +# } +# +# And we want to emit: +# +# trees.mostentries[0].nr_entries:1148 +# trees.mostentries[1].nr_entries:942 +# ... +# +sub TreesMostEntries { + my $nr_items = scalar @{ $data->{'trees'}->{'largest_trees_by_nr_entries'} }; + if ($nr_items == 0) { + return 0; + } + my @arr = @{ $data->{'trees'}->{'largest_trees_by_nr_entries'} }; + my $k; + for ($k=0; $k < $nr_items; $k++) { + print "trees.mostentries[$k].nr_entries:$arr[$k]->{'nr_entries'}\n"; + } +} + +# Print details for the "Histogram by Size in Bytes" for blobs. +# +# We assume that the JSON looks like: +# +# { +# ... +# "blobs": { +# "count": 191847, +# ... +# "dist_by_size": { +# "H0": { +# "count": 47, +# "sum_size": 433, +# "sum_disk_size": 856, +# "hbin_lower": 0, +# "hbin_upper": 15 +# }, +# "H1": { +# "count": 2045, +# "sum_size": 224602, +# "sum_disk_size": 145658, +# "hbin_lower": 16, +# "hbin_upper": 255 +# }, +# ... +# } +# }, +# ... +# } +# +# And we want to emit: +# +# blobs.histsize.H0.count:47 +# blobs.histsize.H1.count:2045 +# ... +# +sub BlobsHistSize { + my $nr_buckets = keys %{ $data->{'blobs'}->{'dist_by_size'} }; + if ($nr_buckets == 0) { + return 0; + } + my %dist = %{ $data->{'blobs'}->{'dist_by_size'} }; + foreach my $key ( sort keys %dist ) { + my $value = $dist{$key}; + print "blobs.histsize.$key.count:$value->{'count'}\n"; + } +} + +# Print details for the "Largest Blobs by Size in Bytes" table. +# This is an array sorted in descending order. If there are multiple +# blobs with the same size, the relative order is undefined. +# +# We assume that the JSON looks like: +# +# { +# ... +# "blobs": { +# "count": 191847, +# ... +# "largest_blobs_by_size_bytes": [ +# { +# "size": 10577552, +# "oid": "667824451d9202e721b6d9413ce4c6b7ce58c36e", +# ... +# }, +# { +# "size": 6655520, +# "oid": "78bcd7f596df79b580e793957928be457a61c3f5", +# ... +# }, +# ... +# ], +# }, +# ... +# } +# +# And we want to emit: +# +# blobs.largest[0].size:10577552 +# blobs.largest[1].size:6655520 +# ... +# +sub BlobsLargest { + my $nr_items = scalar @{ $data->{'blobs'}->{'largest_blobs_by_size_bytes'} }; + if ($nr_items == 0) { + return 0; + } + my @arr = @{ $data->{'blobs'}->{'largest_blobs_by_size_bytes'} }; + my $k; + for ($k=0; $k < $nr_items; $k++) { + print "blobs.largest[$k].size:$arr[$k]->{'size'}\n"; + } +} + +foreach my $arg_k(@ARGV) { + if ($arg_k eq 'count') { + Count; + } + elsif ($arg_k eq 'commits.mostparents') { + CommitsMostParents; + } + elsif ($arg_k eq 'commits.histparents') { + CommitsHistParents; + } + elsif ($arg_k eq 'trees.histentries') { + TreesHistEntries; + } + elsif ($arg_k eq 'trees.mostentries') { + TreesMostEntries; + } + elsif ($arg_k eq 'blobs.histsize') { + BlobsHistSize; + } + elsif ($arg_k eq 'blobs.largest') { + BlobsLargest; + } + else { + print "ERROR: unknown command '$arg_k'\n"; + exit 1; + } +} From 494e9c565611ab2aef3bfefe47f855589188625f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 4 Jun 2024 10:37:23 -0400 Subject: [PATCH 020/189] survey: add --no-name-rev option Computing `git name-rev` on each commit, tree, and blob in each of the various large_item_vec can be very expensive if there are too many refs, especially if the user doesn't need the result. Lets make it optional. The `--no-name-rev` option can save 50 calls to `git name-rev` since we have 5 large_item_vec's and each defaults to 10 items. Signed-off-by: Jeff Hostetler --- Documentation/config/survey.txt | 5 +++ Documentation/git-survey.txt | 4 +++ builtin/survey.c | 59 +++++++++++++++++++++++++-------- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/Documentation/config/survey.txt b/Documentation/config/survey.txt index 672e7890ed2f79..857c1c3fff2d6a 100644 --- a/Documentation/config/survey.txt +++ b/Documentation/config/survey.txt @@ -1,3 +1,8 @@ +survey.namerev:: + Boolean to show/hide `git name-rev` information for + each reported commit and the containing commit of each + reported tree and blob. + survey.progress:: Boolean to show/hide progress information. Defaults to true when interactive (stderr is bound to a TTY). diff --git a/Documentation/git-survey.txt b/Documentation/git-survey.txt index 0e42fb32ac2964..7be11e4683822d 100644 --- a/Documentation/git-survey.txt +++ b/Documentation/git-survey.txt @@ -35,6 +35,10 @@ OPTIONS --json:: Print results in JSON rather than in a human-friendly format. +--[no-]name-rev:: + Print `git name-rev` output for each commit, tree, and blob. + Defaults to true. + Ref Selection ~~~~~~~~~~~~~ diff --git a/builtin/survey.c b/builtin/survey.c index d2e53885fd5820..81019a34d8616b 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -57,6 +57,7 @@ struct survey_opts { int verbose; int show_progress; int show_json; + int show_name_rev; int show_largest_commits_by_nr_parents; int show_largest_commits_by_size_bytes; @@ -75,6 +76,7 @@ static struct survey_opts survey_opts = { .verbose = 0, .show_progress = -1, /* defaults to isatty(2) */ .show_json = 0, /* defaults to pretty */ + .show_name_rev = 1, /* * Show the largest `n` objects for some scaling dimension. @@ -157,6 +159,7 @@ static struct option survey_options[] = { OPT__VERBOSE(&survey_opts.verbose, N_("verbose output")), OPT_BOOL(0, "progress", &survey_opts.show_progress, N_("show progress")), OPT_BOOL(0, "json", &survey_opts.show_json, N_("report stats in JSON")), + OPT_BOOL(0, "name-rev", &survey_opts.show_name_rev, N_("run name-rev on each reported commit")), OPT_BOOL_F(0, "all-refs", &survey_opts.refs.want_all_refs, N_("include all refs"), PARSE_OPT_NONEG), @@ -192,6 +195,10 @@ static int survey_load_config_cb(const char *var, const char *value, survey_opts.show_json = git_config_bool(var, value); return 0; } + if (!strcmp(var, "survey.namerev")) { + survey_opts.show_name_rev = git_config_bool(var, value); + return 0; + } if (!strcmp(var, "survey.showcommitparents")) { survey_opts.show_largest_commits_by_nr_parents = git_config_ulong(var, value, ctx->kvi); @@ -1188,6 +1195,13 @@ static void do_calc_stats_refs(struct repository *r, struct ref_array *ref_array static void do_lookup_name_rev(void) { + /* + * `git name-rev` can be very expensive when there are lots of + * refs, so make it optional. + */ + if (!survey_opts.show_name_rev) + return; + if (survey_opts.show_progress) { survey_progress_total = 0; survey_progress = start_progress(_("Resolving name-revs..."), 0); @@ -1235,9 +1249,11 @@ static void survey_phase_refs(struct repository *r) do_calc_stats_refs(r, &ref_array); trace2_region_leave("survey", "phase/calcstats", the_repository); - trace2_region_enter("survey", "phase/namerev", the_repository); - do_lookup_name_rev(); - trace2_region_enter("survey", "phase/namerev", the_repository); + if (survey_opts.show_name_rev) { + trace2_region_enter("survey", "phase/namerev", the_repository); + do_lookup_name_rev(); + trace2_region_enter("survey", "phase/namerev", the_repository); + } ref_array_clear(&ref_array); } @@ -1475,7 +1491,9 @@ static void write_large_item_vec_json(struct json_writer *jw, if (!is_null_oid(&pk->containing_commit_oid)) jw_object_string(jw, "commit_oid", oid_to_hex(&pk->containing_commit_oid)); - if (pk->name_rev->len) + if (survey_opts.show_name_rev && + pk->name_rev && + pk->name_rev->len) jw_object_string(jw, "name_rev", pk->name_rev->buf); } @@ -1862,7 +1880,8 @@ static void fmt_large_item_hdr(struct strbuf *buf, strbuf_addf(buf, "%-*s | %14s", column0, "OID", item_hdr_label); if (name_length) strbuf_addf(buf, " | %-*s", name_length, "Name"); - strbuf_addf(buf, " | %-*s", name_rev_length, "Name Rev"); + if (name_rev_length) + strbuf_addf(buf, " | %-*s", name_rev_length, "Commit / Name Rev"); strbuf_addch(buf, '\n'); } @@ -1884,8 +1903,10 @@ static void fmt_large_item_hr(struct strbuf *buf, strbuf_addstr(buf, "-+-"); strbuf_addchars(buf, '-', name_length); } - strbuf_addstr(buf, "-+-"); - strbuf_addchars(buf, '-', name_rev_length); + if (name_rev_length) { + strbuf_addstr(buf, "-+-"); + strbuf_addchars(buf, '-', name_rev_length); + } strbuf_addch(buf, '\n'); } @@ -1907,7 +1928,11 @@ static void fmt_large_item_row(struct strbuf *buf, if (name_length) strbuf_addf(buf, " | %-*s", name_length, (pitem->name ? pitem->name->buf: "")); - strbuf_addf(buf, " | %-*s", name_rev_length, pitem->name_rev->buf); + if (name_rev_length) + strbuf_addf(buf, " | %-*s", name_rev_length, + ((pitem->name_rev) + ? pitem->name_rev->buf + : oid_to_hex(&pitem->containing_commit_oid))); strbuf_addch(buf, '\n'); } @@ -1917,11 +1942,11 @@ static void fmt_large_item_vec(struct strbuf *buf, struct large_item_vec *pvec) { int name_length = 0; - int name_rev_length = 10; + int name_rev_length = 0; int k; if (pvec->type != OBJ_COMMIT) { - /* Add "Name" column for trees and blobs. */ + /* Add "Name" column for trees and blobs. This is relative pathname. */ for (k = 0; k < pvec->nr_items; k++) if (pvec->items[k].name && pvec->items[k].name->len > name_length) name_length = pvec->items[k].name->len; @@ -1930,10 +1955,16 @@ static void fmt_large_item_vec(struct strbuf *buf, name_length = 4; } - for (k = 0; k < pvec->nr_items; k++) { - struct large_item *pk = &pvec->items[k]; - if (pk->name_rev->len > name_rev_length) - name_rev_length = pk->name_rev->len; + if (survey_opts.show_name_rev) { + name_rev_length = 17; /* strlen("Commit / Name Rev") */ + for (k = 0; k < pvec->nr_items; k++) { + struct large_item *pk = &pvec->items[k]; + if (pk->name_rev && pk->name_rev->len > name_rev_length) + name_rev_length = pk->name_rev->len; + } + } else if (pvec->type != OBJ_COMMIT) { + /* for trees and blobs, just show containing commit OID */ + name_rev_length = the_hash_algo->hexsz; } strbuf_addch(buf, '\n'); From 2332438a78a959055afbe27c68436e7508de1b16 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 17 Jun 2024 15:20:05 -0400 Subject: [PATCH 021/189] survey: started TODO list at bottom of source file --- builtin/survey.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/builtin/survey.c b/builtin/survey.c index 81019a34d8616b..0aca935a8e95de 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -2294,3 +2294,49 @@ int cmd_survey(int argc, const char **argv, const char *prefix) return 0; } + +/* + * NEEDSWORK: The following is a bit of a laundry list of things + * that I'd like to add. + * + * [] Dump stats on all of the packfiles. The number and size of each. + * Whether each is in the .git directory or in an alternate. The state + * of the IDX or MIDX files and etc. Delta chain stats. All of this + * data is relative to the "lived-in" state of the repository. Stuff + * that may change after a GC or repack. + * + * [] Dump stats on each remote. When we fetch from a remote the size + * of the response is related to the set of haves on the server. You + * can see this in `GIT_TRACE_CURL=1 git fetch`. We get a `ls-refs` + * payload that lists all of the branches and tags on the server, so + * at a minimum the RefName and SHA for each. But for annotated tags + * we also get the peeled SHA. The size of this overhead on every + * fetch is proporational to the size of the `git ls-remote` response + * (roughly, although the latter repeats the RefName of the peeled + * tag). If, for example, you have 500K refs on a remote, you're + * going to have a long "haves" message, so every fetch will be slow + * just because of that overhead (not counting new objects to be + * downloaded). + * + * Note that the local set of tags in "refs/tags/" is a union over all + * remotes. However, since most people only have one remote, we can + * probaly estimate the overhead value directly from the size of the + * set of "refs/tags/" that we visited while building the `ref_info` + * and `ref_array` and not need to ask the remote. + * + * [] Dump info on the complexity of the DAG. Criss-cross merges. + * The number of edges that must be touched to compute merge bases. + * Edge length. The number of parallel lanes in the history that must + * be navigated to get to the merge base. What affects the cost of + * the Ahead/Behind computation? How often do criss-crosses occur and + * do they cause various operations to slow down? + * + * [] If there are primary branches (like "main" or "master") are they + * always on the left side of merges? Does the graph have a clean + * left edge? Or are there normal and "backwards" merges? Do these + * cause problems at scale? + * + * [] If we have a hierarchy of FI/RI branches like "L1", "L2, ..., + * can we learn anything about the shape of the repo around these FI + * and RI integrations? + */ From 4b9fd55567dc4c56126c5481973a3e235161f139 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 28 Jun 2024 15:22:46 -0400 Subject: [PATCH 022/189] survey: expanded TODO list at the bottom of the source file Signed-off-by: Jeff Hostetler --- builtin/survey.c | 148 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 116 insertions(+), 32 deletions(-) diff --git a/builtin/survey.c b/builtin/survey.c index 0aca935a8e95de..91ac0bd45dadf6 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -2296,47 +2296,131 @@ int cmd_survey(int argc, const char **argv, const char *prefix) } /* - * NEEDSWORK: The following is a bit of a laundry list of things - * that I'd like to add. + * NEEDSWORK: So far, I only have iteration on the requested set of + * refs and treewalk/reachable objects on that set of refs. The + * following is a bit of a laundry list of things that I'd like to + * add. * * [] Dump stats on all of the packfiles. The number and size of each. - * Whether each is in the .git directory or in an alternate. The state - * of the IDX or MIDX files and etc. Delta chain stats. All of this - * data is relative to the "lived-in" state of the repository. Stuff - * that may change after a GC or repack. + * Whether each is in the .git directory or in an alternate. The + * state of the IDX or MIDX files and etc. Delta chain stats. All + * of this data is relative to the "lived-in" state of the + * repository. Stuff that may change after a GC or repack. + * + * [] Clone and Index stats. partial, shallow, sparse-checkout, + * sparse-index, etc. Hydration stats. * * [] Dump stats on each remote. When we fetch from a remote the size - * of the response is related to the set of haves on the server. You - * can see this in `GIT_TRACE_CURL=1 git fetch`. We get a `ls-refs` - * payload that lists all of the branches and tags on the server, so - * at a minimum the RefName and SHA for each. But for annotated tags - * we also get the peeled SHA. The size of this overhead on every - * fetch is proporational to the size of the `git ls-remote` response - * (roughly, although the latter repeats the RefName of the peeled - * tag). If, for example, you have 500K refs on a remote, you're - * going to have a long "haves" message, so every fetch will be slow - * just because of that overhead (not counting new objects to be - * downloaded). + * of the response is related to the set of haves on the server. + * You can see this in `GIT_TRACE_CURL=1 git fetch`. We get a + * `ls-refs` payload that lists all of the branches and tags on the + * server, so at a minimum the RefName and SHA for each. But for + * annotated tags we also get the peeled SHA. The size of this + * overhead on every fetch is proporational to the size of the `git + * ls-remote` response (roughly, although the latter repeats the + * RefName of the peeled tag). If, for example, you have 500K refs + * on a remote, you're going to have a long "haves" message, so + * every fetch will be slow just because of that overhead (not + * counting new objects to be downloaded). * - * Note that the local set of tags in "refs/tags/" is a union over all - * remotes. However, since most people only have one remote, we can - * probaly estimate the overhead value directly from the size of the - * set of "refs/tags/" that we visited while building the `ref_info` - * and `ref_array` and not need to ask the remote. + * Note that the local set of tags in "refs/tags/" is a union over + * all remotes. However, since most people only have one remote, + * we can probaly estimate the overhead value directly from the + * size of the set of "refs/tags/" that we visited while building + * the `ref_info` and `ref_array` and not need to ask the remote. * * [] Dump info on the complexity of the DAG. Criss-cross merges. - * The number of edges that must be touched to compute merge bases. - * Edge length. The number of parallel lanes in the history that must - * be navigated to get to the merge base. What affects the cost of - * the Ahead/Behind computation? How often do criss-crosses occur and - * do they cause various operations to slow down? + * The number of edges that must be touched to compute merge bases. + * Edge length. The number of parallel lanes in the history that + * must be navigated to get to the merge base. What affects the + * cost of the Ahead/Behind computation? How often do + * criss-crosses occur and do they cause various operations to slow + * down? * * [] If there are primary branches (like "main" or "master") are they - * always on the left side of merges? Does the graph have a clean - * left edge? Or are there normal and "backwards" merges? Do these - * cause problems at scale? + * always on the left side of merges? Does the graph have a clean + * left edge? Or are there normal and "backwards" merges? Do + * these cause problems at scale? * * [] If we have a hierarchy of FI/RI branches like "L1", "L2, ..., - * can we learn anything about the shape of the repo around these FI - * and RI integrations? + * can we learn anything about the shape of the repo around these + * FI and RI integrations? + * + * [] Do we need a no-PII flag to omit pathnames or branch/tag names + * in the various histograms? (This would turn off --name-rev + * too.) + * + * [] I have so far avoided adding opinions about individual fields + * (such as the way `git-sizer` prints a row of stars or bangs in + * the last column). + * + * I'm wondering if that is a job of this executable or if it + * should be done in a post-processing step using the JSON output. + * + * My problem with the `git-sizer` approach is that it doesn't give + * the (casual) user any information on why it has stars or bangs. + * And there isn't a good way to print detailed information in the + * ASCII-art tables that would be easy to understand. + * + * [] For example, a large number of refs does not define a cliff. + * Performance will drop off (linearly, quadratically, ... ??). + * The tool should refer them to article(s) talking about the + * different problems that it could cause. So should `git + * survey` just print the number and (implicitly) refer them to + * the man page (chapter/verse) or to a tool that will interpret + * the number and explain it? + * + * [] Alternatively, should `git survey` do that analysis too and + * just print footnotes for each large number? + * + * [] The computation of the raw survey JSON data can take HOURS on + * a very large repo (like Windows), so I'm wondering if we + * want to keep the opinion portion separate. + * + * [] In addition to opinions based on the static data, I would like + * to dump the JSON results (or the Trace2 telemetry) into a DB and + * aggregate it with other users. + * + * Granted, they should all see the same DAG and the same set of + * reachable objects, but we could average across all datasets + * generated on a particular date and detect outlier users. + * + * [] Maybe someone cloned from the `_full` endpoint rather than + * the limited refs endpoint. + * + * [] Maybe that user is having problems with repacking / GC / + * maintenance without knowing it. + * + * [] I'd also like to dump use the DB to compare survey datasets over + * a time. How fast is their repository growing and in what ways? + * + * [] I'd rather have the delta analysis NOT be inside `git + * survey`, so it makes sense to consider having all of it in a + * post-process step. + * + * [] Another reason to put the opinion analysis in a post-process + * is that it would be easier to generate plots on the data tables. + * Granted, we can get plots from telemetry, but a stand-alone user + * could run the JSON thru python or jq or something and generate + * something nicer than ASCII-art and it could handle cross-referencing + * and hyperlinking to helpful information on each issue. + * + * [] I think there are several classes of data that we can report on: + * + * [] The "inherit repo properties", such as the shape and size of + * the DAG -- these should be universal in each enlistment. + * + * [] The "ODB lived in properties", such as the efficiency + * of the repack and things like partial and shallow clone. + * These will vary, but indicate health of the ODB. + * + * [] The "index related properties", such as sparse-checkout, + * sparse-index, cache-tree, untracked-cache, fsmonitor, and + * etc. These will also vary, but are more like knobs for + * the user to adjust. + * + * [] I want to compare these with Matt's "dimensions of scale" + * notes and see if there are other pieces of data that we + * could compute/consider. + * */ From 95bc9293c27f03460847637d7499018333507ac1 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 1 Jul 2024 12:07:01 -0400 Subject: [PATCH 023/189] survey: expanded TODO with more notes Signed-off-by: Jeff Hostetler --- builtin/survey.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/builtin/survey.c b/builtin/survey.c index 91ac0bd45dadf6..719e2e21e915d3 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -2329,6 +2329,16 @@ int cmd_survey(int argc, const char **argv, const char *prefix) * size of the set of "refs/tags/" that we visited while building * the `ref_info` and `ref_array` and not need to ask the remote. * + * [] Should the "string length of refnames / remote refs", for + * example, be sub-divided by remote so we can project the + * cost of the haves/wants overhead a fetch. + * + * [] Can we examine the merge commits and classify them as clean or + * dirty? (ie. ones with merge conflicts that needed to be + * addressed during the merge itself.) + * + * [] Do dirty merges affect performance of later operations? + * * [] Dump info on the complexity of the DAG. Criss-cross merges. * The number of edges that must be touched to compute merge bases. * Edge length. The number of parallel lanes in the history that From bd44fa946347efb7ec7213dcd5ad77e192dc835e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 1 Jul 2024 23:28:45 +0200 Subject: [PATCH 024/189] survey: clearly note the experimental nature in the output While this command is definitely something we _want_, chances are that upstreaming this will require substantial changes. We still want to be able to experiment with this before that, to focus on what we need out of this command: To assist with diagnosing issues with large repositories, as well as to help monitoring the growth and the associated painpoints of such repositories. To that end, we are about to integrate this command into `microsoft/git`, to get the tool into the hands of users who need it most, with the idea to iterate in close collaboration between these users and the developers familar with Git's internals. However, we will definitely want to avoid letting anybody have the impression that this command, its exact inner workings, as well as its output format, are anywhere close to stable. To make that fact utterly clear (and thereby protect the freedom to iterate and innovate freely before upstreaming the command), let's mark its output as experimental in all-caps, as the first thing we do. Signed-off-by: Johannes Schindelin --- builtin/survey.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/builtin/survey.c b/builtin/survey.c index 719e2e21e915d3..4d0f0f86c46649 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -18,6 +18,7 @@ #include "trace2.h" #include "tree.h" #include "tree-walk.h" +#include "color.h" static const char * const survey_usage[] = { N_("(EXPERIMENTAL!) git survey "), @@ -2265,6 +2266,10 @@ int cmd_survey(int argc, const char **argv, const char *prefix) argc = parse_options(argc, argv, prefix, survey_options, survey_usage, 0); + color_fprintf_ln(stderr, + want_color_fd(2, GIT_COLOR_AUTO) ? GIT_COLOR_YELLOW : "", + "(THIS IS EXPERIMENTAL, EXPECT THE OUTPUT FORMAT TO CHANGE!)"); + prepare_repo_settings(the_repository); if (survey_opts.show_progress < 0) From d5ff2d24aeb074b0e255c9b294994d16a18636ae Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 5 Apr 2017 10:58:09 -0600 Subject: [PATCH 025/189] reset --stdin: trim carriage return from the paths While using the reset --stdin feature on windows path added may have a \r at the end of the path that wasn't getting removed so didn't match the path in the index and wasn't reset. Signed-off-by: Kevin Willford --- t/t7108-reset-stdin.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/t/t7108-reset-stdin.sh b/t/t7108-reset-stdin.sh index b7cbcbf869296c..db5483b8f10052 100755 --- a/t/t7108-reset-stdin.sh +++ b/t/t7108-reset-stdin.sh @@ -29,4 +29,13 @@ test_expect_success '--stdin requires --mixed' ' git reset --mixed --stdin list && + git reset --stdin Date: Tue, 23 Apr 2024 12:21:01 +0200 Subject: [PATCH 026/189] Identify microsoft/git via a distinct version suffix It has been a long-standing practice in Git for Windows to append `.windows.`, and in microsoft/git to append `.vfs.0.0`. Let's keep doing that. Signed-off-by: Johannes Schindelin --- GIT-VERSION-GEN | 3 +++ 1 file changed, 3 insertions(+) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index 55850912d13034..7ca2fcf019bf80 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -3,6 +3,9 @@ GVF=GIT-VERSION-FILE DEF_VER=v2.46.1 +# Identify microsoft/git via a distinct version suffix +DEF_VER=$DEF_VER.vfs.0.0 + LF=' ' From 81e5a4c6546efc8ed1628ae27d511db1f9e090b0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 4 Apr 2017 12:04:11 +0200 Subject: [PATCH 027/189] gvfs: ensure that the version is based on a GVFS tag Since we really want to be based on a `.vfs.*` tag, let's make sure that there was a new-enough one, i.e. one that agrees with the first three version numbers of the recorded default version. This prevents e.g. v2.22.0.vfs.0.. from being used when the current release train was not yet tagged. It is important to get the first three numbers of the version right because e.g. Scalar makes decisions depending on those (such as assuming that the `git maintenance` built-in is not available, even though it actually _is_ available). Signed-off-by: Johannes Schindelin --- GIT-VERSION-GEN | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index 7ca2fcf019bf80..a7081ebdcc989f 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -15,10 +15,15 @@ if test -f version then VN=$(cat version) || VN="$DEF_VER" elif { test -d "${GIT_DIR:-.git}" || test -f .git; } && - VN=$(git describe --match "v[0-9]*" HEAD 2>/dev/null) && + VN=$(git describe --match "v[0-9]*vfs*" HEAD 2>/dev/null) && case "$VN" in *$LF*) (exit 1) ;; v[0-9]*) + if test "${VN%%.vfs.*}" != "${DEF_VER%%.vfs.*}" + then + echo "Found version $VN, which is not based on $DEF_VER" >&2 + exit 1 + fi git update-index -q --refresh test -z "$(git diff-index --name-only HEAD --)" || VN="$VN-dirty" ;; From a441c1ff33eb609954b441e5864201f523f6de30 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:30:59 +0100 Subject: [PATCH 028/189] gvfs: add a GVFS-specific header file This header file will accumulate GVFS-specific definitions. Signed-off-by: Kevin Willford --- gvfs.h | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 gvfs.h diff --git a/gvfs.h b/gvfs.h new file mode 100644 index 00000000000000..b6dbe85eae4071 --- /dev/null +++ b/gvfs.h @@ -0,0 +1,9 @@ +#ifndef GVFS_H +#define GVFS_H + +/* + * This file is for the specific settings and methods + * used for GVFS functionality + */ + +#endif /* GVFS_H */ From 4cfd73a381d750e0fb31e8d71dc318d648f728cb Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:34:12 +0100 Subject: [PATCH 029/189] gvfs: add the core.gvfs config setting This does not do anything yet. The next patches will add various values for that config setting that correspond to the various features offered/required by GVFS. Signed-off-by: Kevin Willford gvfs: refactor loading the core.gvfs config value This code change makes sure that the config value for core_gvfs is always loaded before checking it. Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 3 +++ Makefile | 1 + config.c | 6 +++++ environment.c | 1 + environment.h | 1 + gvfs.c | 45 +++++++++++++++++++++++++++++++++++ gvfs.h | 4 ++++ 7 files changed, 61 insertions(+) create mode 100644 gvfs.c diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 458a3f4f7f5d80..3f3343c7cbd982 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -743,6 +743,9 @@ core.multiPackIndex:: single index. See linkgit:git-multi-pack-index[1] for more information. Defaults to true. +core.gvfs:: + Enable the features needed for GVFS. + core.sparseCheckout:: Enable "sparse checkout" feature. See linkgit:git-sparse-checkout[1] for more information. diff --git a/Makefile b/Makefile index 4ed38a08d0b2a2..24811a49aa107d 100644 --- a/Makefile +++ b/Makefile @@ -1033,6 +1033,7 @@ LIB_OBJS += git-zlib.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o LIB_OBJS += grep.o +LIB_OBJS += gvfs.o LIB_OBJS += hash-lookup.o LIB_OBJS += hashmap.o LIB_OBJS += help.o diff --git a/config.c b/config.c index 0a8d6901c83608..9f8144c65a72b9 100644 --- a/config.c +++ b/config.c @@ -12,6 +12,7 @@ #include "abspath.h" #include "advice.h" #include "date.h" +#include "gvfs.h" #include "branch.h" #include "config.h" #include "parse.h" @@ -1665,6 +1666,11 @@ int git_default_core_config(const char *var, const char *value, return 0; } + if (!strcmp(var, "core.gvfs")) { + gvfs_load_config_value(value); + return 0; + } + if (!strcmp(var, "core.sparsecheckout")) { core_apply_sparse_checkout = git_config_bool(var, value); return 0; diff --git a/environment.c b/environment.c index 5cea2c9f5473e3..54b9d182ee6339 100644 --- a/environment.c +++ b/environment.c @@ -80,6 +80,7 @@ int grafts_keep_true_parents; int core_apply_sparse_checkout; int core_sparse_checkout_cone; int sparse_expect_files_outside_of_patterns; +int core_gvfs; int merge_log_config = -1; int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ unsigned long pack_size_limit_cfg; diff --git a/environment.h b/environment.h index e9f01d4d11c010..0e607ead0c8236 100644 --- a/environment.h +++ b/environment.h @@ -153,6 +153,7 @@ int get_shared_repository(void); void reset_shared_repository(void); extern int core_preload_index; +extern int core_gvfs; extern int precomposed_unicode; extern int protect_hfs; extern int protect_ntfs; diff --git a/gvfs.c b/gvfs.c new file mode 100644 index 00000000000000..c65f15ff02584a --- /dev/null +++ b/gvfs.c @@ -0,0 +1,45 @@ +#define USE_THE_REPOSITORY_VARIABLE +#include "git-compat-util.h" +#include "environment.h" +#include "gvfs.h" +#include "setup.h" +#include "config.h" + +static int gvfs_config_loaded; +static int core_gvfs_is_bool; + +static int early_core_gvfs_config(const char *var, const char *value, + const struct config_context *ctx, void *cb) +{ + if (!strcmp(var, "core.gvfs")) + core_gvfs = git_config_bool_or_int("core.gvfs", value, ctx->kvi, + &core_gvfs_is_bool); + return 0; +} + +void gvfs_load_config_value(const char *value) +{ + if (gvfs_config_loaded) + return; + + if (value) { + struct key_value_info default_kvi = KVI_INIT; + core_gvfs = git_config_bool_or_int("core.gvfs", value, &default_kvi, &core_gvfs_is_bool); + } else if (startup_info->have_repository == 0) + read_early_config(early_core_gvfs_config, NULL); + else + repo_config_get_bool_or_int(the_repository, "core.gvfs", + &core_gvfs_is_bool, &core_gvfs); + + /* Turn on all bits if a bool was set in the settings */ + if (core_gvfs_is_bool && core_gvfs) + core_gvfs = -1; + + gvfs_config_loaded = 1; +} + +int gvfs_config_is_set(int mask) +{ + gvfs_load_config_value(NULL); + return (core_gvfs & mask) == mask; +} diff --git a/gvfs.h b/gvfs.h index b6dbe85eae4071..011185dea93734 100644 --- a/gvfs.h +++ b/gvfs.h @@ -1,9 +1,13 @@ #ifndef GVFS_H #define GVFS_H + /* * This file is for the specific settings and methods * used for GVFS functionality */ +void gvfs_load_config_value(const char *value); +int gvfs_config_is_set(int mask); + #endif /* GVFS_H */ From d94ca7d241d94edb0f7fec70a28572b74252d8f4 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:38:59 +0100 Subject: [PATCH 030/189] gvfs: add the feature to skip writing the index' SHA-1 This takes a substantial amount of time, and if the user is reasonably sure that the files' integrity is not compromised, that time can be saved. Git no longer verifies the SHA-1 by default, anyway. Signed-off-by: Kevin Willford Update for 2023-02-27: This feature was upstreamed as the index.skipHash config option. This resulted in some changes to the struct and some of the setup code. In particular, the config reading was moved to prepare_repo_settings(), so the core.gvfs bit check was moved there, too. Signed-off-by: Derrick Stolee --- Documentation/config/core.txt | 10 +++++++++- gvfs.h | 6 ++++++ repo-settings.c | 8 ++++++++ t/t1017-read-tree-skip-sha-on-read.sh | 22 ++++++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100755 t/t1017-read-tree-skip-sha-on-read.sh diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 3f3343c7cbd982..925f164842ff00 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -744,7 +744,15 @@ core.multiPackIndex:: information. Defaults to true. core.gvfs:: - Enable the features needed for GVFS. + Enable the features needed for GVFS. This value can be set to true + to indicate all features should be turned on or the bit values listed + below can be used to turn on specific features. ++ +-- + GVFS_SKIP_SHA_ON_INDEX:: + Bit value 1 + Disables the calculation of the sha when writing the index +-- core.sparseCheckout:: Enable "sparse checkout" feature. See linkgit:git-sparse-checkout[1] diff --git a/gvfs.h b/gvfs.h index 011185dea93734..c75991530fa1fa 100644 --- a/gvfs.h +++ b/gvfs.h @@ -7,6 +7,12 @@ * used for GVFS functionality */ + +/* + * The list of bits in the core_gvfs setting + */ +#define GVFS_SKIP_SHA_ON_INDEX (1 << 0) + void gvfs_load_config_value(const char *value); int gvfs_config_is_set(int mask); diff --git a/repo-settings.c b/repo-settings.c index f104e6fc73c38f..efb131e5a01501 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -4,6 +4,7 @@ #include "midx.h" #include "fsmonitor-ipc.h" #include "fsmonitor-settings.h" +#include "gvfs.h" static void repo_cfg_bool(struct repository *r, const char *key, int *dest, int def) @@ -99,6 +100,13 @@ void prepare_repo_settings(struct repository *r) r->settings.pack_use_bitmap_boundary_traversal); repo_cfg_bool(r, "core.usereplacerefs", &r->settings.read_replace_refs, 1); + /* + * For historical compatibility reasons, enable index.skipHash based + * on a bit in core.gvfs. + */ + if (gvfs_config_is_set(GVFS_SKIP_SHA_ON_INDEX)) + r->settings.index_skip_hash = 1; + /* * The GIT_TEST_MULTI_PACK_INDEX variable is special in that * either it *or* the config sets diff --git a/t/t1017-read-tree-skip-sha-on-read.sh b/t/t1017-read-tree-skip-sha-on-read.sh new file mode 100755 index 00000000000000..5b76a80a0020dc --- /dev/null +++ b/t/t1017-read-tree-skip-sha-on-read.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +test_description='check that read-tree works with core.gvfs config value' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-read-tree.sh + +test_expect_success setup ' + echo one >a && + git add a && + git commit -m initial +' +test_expect_success 'read-tree without core.gvsf' ' + read_tree_u_must_succeed -m -u HEAD +' + +test_expect_success 'read-tree with core.gvfs set to 1' ' + git config core.gvfs 1 && + read_tree_u_must_succeed -m -u HEAD +' + +test_done From 363f882ab3a32530576dc9f8a127154382f56d14 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:54:55 +0100 Subject: [PATCH 031/189] gvfs: add the feature that blobs may be missing Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 4 ++++ cache-tree.c | 4 +++- commit.c | 9 +++++++-- gvfs.h | 1 + t/t0000-basic.sh | 5 +++++ 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 925f164842ff00..7c640bbf36bb68 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -752,6 +752,10 @@ core.gvfs:: GVFS_SKIP_SHA_ON_INDEX:: Bit value 1 Disables the calculation of the sha when writing the index + GVFS_MISSING_OK:: + Bit value 4 + Normally git write-tree ensures that the objects referenced by the + directory exist in the object database. This option disables this check. -- core.sparseCheckout:: diff --git a/cache-tree.c b/cache-tree.c index 50610c3f3cb8e7..02808209115ec0 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "environment.h" #include "hex.h" +#include "gvfs.h" #include "lockfile.h" #include "tree.h" #include "tree-walk.h" @@ -259,7 +260,8 @@ static int update_one(struct cache_tree *it, int flags) { struct strbuf buffer; - int missing_ok = flags & WRITE_TREE_MISSING_OK; + int missing_ok = gvfs_config_is_set(GVFS_MISSING_OK) ? + WRITE_TREE_MISSING_OK : (flags & WRITE_TREE_MISSING_OK); int dryrun = flags & WRITE_TREE_DRY_RUN; int repair = flags & WRITE_TREE_REPAIR; int to_invalidate = 0; diff --git a/commit.c b/commit.c index 087cb19f4f84f3..27eb39758b2275 100644 --- a/commit.c +++ b/commit.c @@ -1,6 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "gvfs.h" #include "tag.h" #include "commit.h" #include "commit-graph.h" @@ -562,13 +563,17 @@ int repo_parse_commit_internal(struct repository *r, .sizep = &size, .contentp = &buffer, }; + int ret; /* * Git does not support partial clones that exclude commits, so set * OBJECT_INFO_SKIP_FETCH_OBJECT to fail fast when an object is missing. */ int flags = OBJECT_INFO_LOOKUP_REPLACE | OBJECT_INFO_SKIP_FETCH_OBJECT | - OBJECT_INFO_DIE_IF_CORRUPT; - int ret; + OBJECT_INFO_DIE_IF_CORRUPT; + + /* But the GVFS Protocol _does_ support missing commits! */ + if (gvfs_config_is_set(GVFS_MISSING_OK)) + flags ^= OBJECT_INFO_SKIP_FETCH_OBJECT; if (!item) return -1; diff --git a/gvfs.h b/gvfs.h index c75991530fa1fa..7bedfaacf6d684 100644 --- a/gvfs.h +++ b/gvfs.h @@ -12,6 +12,7 @@ * The list of bits in the core_gvfs setting */ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) +#define GVFS_MISSING_OK (1 << 2) void gvfs_load_config_value(const char *value); int gvfs_config_is_set(int mask); diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh index 98b81e4d63fa4c..7baa03212bf3ba 100755 --- a/t/t0000-basic.sh +++ b/t/t0000-basic.sh @@ -1106,6 +1106,11 @@ test_expect_success 'writing this tree with --missing-ok' ' git write-tree --missing-ok ' +test_expect_success 'writing this tree with missing ok config value' ' + git config core.gvfs 4 && + git write-tree +' + ################################################################ test_expect_success 'git read-tree followed by write-tree should be idempotent' ' From 4bd94264510aee55d8e3f341d1529cd0f2116d9a Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 18 May 2016 13:40:39 +0000 Subject: [PATCH 032/189] gvfs: prevent files to be deleted outside the sparse checkout Prevent the sparse checkout to delete files that were marked with skip-worktree bit and are not in the sparse-checkout file. This is because everything with the skip-worktree bit turned on is being virtualized and will be removed with the change of HEAD. There was only one failing test when running with these changes that was checking to make sure the worktree narrows on checkout which was expected since we would no longer be narrowing the worktree. Update 2022-04-05: temporarily set 'sparse.expectfilesoutsideofpatterns' in test (until we start disabling the "remove present-despite-SKIP_WORKTREE" behavior with 'core.virtualfilesystem' in a later commit). Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 9 +++++++++ gvfs.h | 1 + t/t1090-sparse-checkout-scope.sh | 20 ++++++++++++++++++++ unpack-trees.c | 22 ++++++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 7c640bbf36bb68..cb37bca6d7ac08 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -756,6 +756,15 @@ core.gvfs:: Bit value 4 Normally git write-tree ensures that the objects referenced by the directory exist in the object database. This option disables this check. + GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT:: + Bit value 8 + When marking entries to remove from the index and the working + directory this option will take into account what the + skip-worktree bit was set to so that if the entry has the + skip-worktree bit set it will not be removed from the working + directory. This will allow virtualized working directories to + detect the change to HEAD and use the new commit tree to show + the files that are in the working directory. -- core.sparseCheckout:: diff --git a/gvfs.h b/gvfs.h index 7bedfaacf6d684..44131625828cfa 100644 --- a/gvfs.h +++ b/gvfs.h @@ -13,6 +13,7 @@ */ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) #define GVFS_MISSING_OK (1 << 2) +#define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) void gvfs_load_config_value(const char *value); int gvfs_config_is_set(int mask); diff --git a/t/t1090-sparse-checkout-scope.sh b/t/t1090-sparse-checkout-scope.sh index b6298e7e7d8ab1..7af01dc00b75ee 100755 --- a/t/t1090-sparse-checkout-scope.sh +++ b/t/t1090-sparse-checkout-scope.sh @@ -107,6 +107,26 @@ test_expect_success 'in partial clone, sparse checkout only fetches needed blobs test_cmp expect actual ' +test_expect_success 'checkout does not delete items outside the sparse checkout file' ' + # The "sparse.expectfilesoutsideofpatterns" config will prevent the + # SKIP_WORKTREE flag from being dropped on files present on-disk. + test_config sparse.expectfilesoutsideofpatterns true && + + test_config core.gvfs 8 && + git checkout -b outside && + echo "new file1" >d && + git add --sparse d && + git commit -m "branch initial" && + echo "new file1" >e && + git add --sparse e && + git commit -m "skipped worktree" && + git update-index --skip-worktree e && + echo "/d" >.git/info/sparse-checkout && + git checkout HEAD^ && + test_path_is_file d && + test_path_is_file e +' + test_expect_success MINGW 'no unnecessary opendir() with fscache' ' git clone . fscache-test && ( diff --git a/unpack-trees.c b/unpack-trees.c index 1cff7ca762374f..7c564ce5566102 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -2,6 +2,7 @@ #include "git-compat-util.h" #include "advice.h" +#include "gvfs.h" #include "strvec.h" #include "repository.h" #include "parse.h" @@ -2677,6 +2678,27 @@ static int deleted_entry(const struct cache_entry *ce, if (!(old->ce_flags & CE_CONFLICTED) && verify_uptodate(old, o)) return -1; + + /* + * When marking entries to remove from the index and the working + * directory this option will take into account what the + * skip-worktree bit was set to so that if the entry has the + * skip-worktree bit set it will not be removed from the working + * directory. This will allow virtualized working directories to + * detect the change to HEAD and use the new commit tree to show + * the files that are in the working directory. + * + * old is the cache_entry that will have the skip-worktree bit set + * which will need to be preserved when the CE_REMOVE entry is added + */ + if (gvfs_config_is_set(GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT) && + old && + old->ce_flags & CE_SKIP_WORKTREE) { + add_entry(o, old, CE_REMOVE, 0); + invalidate_ce_path(old, o); + return 1; + } + add_entry(o, ce, CE_REMOVE, 0); invalidate_ce_path(ce, o); return 1; From 77d8f4a99beb1ab164ecea5ea90a86e230ec29d8 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 30 May 2016 10:55:53 -0400 Subject: [PATCH 033/189] gvfs: optionally skip reachability checks/upload pack during fetch While performing a fetch with a virtual file system we know that there will be missing objects and we don't want to download them just because of the reachability of the commits. We also don't want to download a pack file with commits, trees, and blobs since these will be downloaded on demand. This flag will skip the first connectivity check and by returning zero will skip the upload pack. It will also skip the second connectivity check but continue to update the branches to the latest commit ids. Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 9 +++++++++ connected.c | 19 +++++++++++++++++++ gvfs.h | 1 + t/t5584-vfs.sh | 24 ++++++++++++++++++++++++ 4 files changed, 53 insertions(+) create mode 100755 t/t5584-vfs.sh diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index cb37bca6d7ac08..c2063fcaf6e407 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -765,6 +765,15 @@ core.gvfs:: directory. This will allow virtualized working directories to detect the change to HEAD and use the new commit tree to show the files that are in the working directory. + GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK:: + Bit value 16 + While performing a fetch with a virtual file system we know + that there will be missing objects and we don't want to download + them just because of the reachability of the commits. We also + don't want to download a pack file with commits, trees, and blobs + since these will be downloaded on demand. This flag will skip the + checks on the reachability of objects during a fetch as well as + the upload pack so that extraneous objects don't get downloaded. -- core.sparseCheckout:: diff --git a/connected.c b/connected.c index 87cc4b57a17ce5..44e5f75eb063b4 100644 --- a/connected.c +++ b/connected.c @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "gettext.h" #include "hex.h" +#include "gvfs.h" #include "object-store-ll.h" #include "run-command.h" #include "sigchain.h" @@ -34,6 +35,24 @@ int check_connected(oid_iterate_fn fn, void *cb_data, struct transport *transport; size_t base_len; + /* + * Running a virtual file system there will be objects that are + * missing locally and we don't want to download a bunch of + * commits, trees, and blobs just to make sure everything is + * reachable locally so this option will skip reachablility + * checks below that use rev-list. This will stop the check + * before uploadpack runs to determine if there is anything to + * fetch. Returning zero for the first check will also prevent the + * uploadpack from happening. It will also skip the check after + * the fetch is finished to make sure all the objects where + * downloaded in the pack file. This will allow the fetch to + * run and get all the latest tip commit ids for all the branches + * in the fetch but not pull down commits, trees, or blobs via + * upload pack. + */ + if (gvfs_config_is_set(GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK)) + return 0; + if (!opt) opt = &defaults; transport = opt->transport; diff --git a/gvfs.h b/gvfs.h index 44131625828cfa..e69dd4b2c8f785 100644 --- a/gvfs.h +++ b/gvfs.h @@ -14,6 +14,7 @@ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) #define GVFS_MISSING_OK (1 << 2) #define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) +#define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) void gvfs_load_config_value(const char *value); int gvfs_config_is_set(int mask); diff --git a/t/t5584-vfs.sh b/t/t5584-vfs.sh new file mode 100755 index 00000000000000..8a703cbb640387 --- /dev/null +++ b/t/t5584-vfs.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +test_description='fetch using the flag to skip reachability and upload pack' + +. ./test-lib.sh + + +test_expect_success setup ' + echo inital >a && + git add a && + git commit -m initial && + git clone . one +' + +test_expect_success "fetch test" ' + cd one && + git config core.gvfs 16 && + rm -rf .git/objects/* && + git -C .. cat-file commit HEAD | git hash-object -w --stdin -t commit && + git fetch && + test_must_fail git rev-parse --verify HEAD^{tree} +' + +test_done \ No newline at end of file From 4c1c45a2b3f628b2a3fa714dfec1a6af90163ba3 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Wed, 15 Jun 2016 14:59:16 +0000 Subject: [PATCH 034/189] gvfs: ensure all filters and EOL conversions are blocked Ensure all filters and EOL conversions are blocked when running under GVFS so that our projected file sizes will match the actual file size when it is hydrated on the local machine. Signed-off-by: Ben Peart --- Documentation/config/core.txt | 9 +++++++++ convert.c | 22 +++++++++++++++++++++ gvfs.h | 1 + t/t0021-conversion.sh | 37 +++++++++++++++++++++++++++++++++++ t/t0027-auto-crlf.sh | 12 ++++++++++++ 5 files changed, 81 insertions(+) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index c2063fcaf6e407..2fd2a10f51ad2c 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -774,6 +774,15 @@ core.gvfs:: since these will be downloaded on demand. This flag will skip the checks on the reachability of objects during a fetch as well as the upload pack so that extraneous objects don't get downloaded. + GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS:: + Bit value 64 + With a virtual file system we only know the file size before any + CRLF or smudge/clean filters processing is done on the client. + To prevent file corruption due to truncation or expansion with + garbage at the end, these filters must not run when the file + is first accessed and brought down to the client. Git.exe can't + currently tell the first access vs subsequent accesses so this + flag just blocks them from occurring at all. -- core.sparseCheckout:: diff --git a/convert.c b/convert.c index c4ddc4de81b557..f050f98679daea 100644 --- a/convert.c +++ b/convert.c @@ -2,6 +2,7 @@ #include "git-compat-util.h" #include "advice.h" +#include "gvfs.h" #include "config.h" #include "convert.h" #include "copy.h" @@ -562,6 +563,9 @@ static int crlf_to_git(struct index_state *istate, if (!buf) return 1; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* only grow if not in place */ if (strbuf_avail(buf) + buf->len < len) strbuf_grow(buf, len - buf->len); @@ -601,6 +605,9 @@ static int crlf_to_worktree(const char *src, size_t len, struct strbuf *buf, if (!will_convert_lf_to_crlf(&stats, crlf_action)) return 0; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* are we "faking" in place editing ? */ if (src == buf->buf) to_free = strbuf_detach(buf, NULL); @@ -710,6 +717,9 @@ static int apply_single_file_filter(const char *path, const char *src, size_t le struct async async; struct filter_params params; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("Filter \"%s\" not supported when running under GVFS", cmd); + memset(&async, 0, sizeof(async)); async.proc = filter_buffer_or_fd; async.data = ¶ms; @@ -1123,6 +1133,9 @@ static int ident_to_git(const char *src, size_t len, if (!buf) return 1; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + /* only grow if not in place */ if (strbuf_avail(buf) + buf->len < len) strbuf_grow(buf, len - buf->len); @@ -1170,6 +1183,9 @@ static int ident_to_worktree(const char *src, size_t len, if (!cnt) return 0; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + /* are we "faking" in place editing ? */ if (src == buf->buf) to_free = strbuf_detach(buf, NULL); @@ -1619,6 +1635,9 @@ static int lf_to_crlf_filter_fn(struct stream_filter *filter, size_t count, o = 0; struct lf_to_crlf_filter *lf_to_crlf = (struct lf_to_crlf_filter *)filter; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* * We may be holding onto the CR to see if it is followed by a * LF, in which case we would need to go to the main loop. @@ -1863,6 +1882,9 @@ static int ident_filter_fn(struct stream_filter *filter, struct ident_filter *ident = (struct ident_filter *)filter; static const char head[] = "$Id"; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + if (!input) { /* drain upon eof */ switch (ident->state) { diff --git a/gvfs.h b/gvfs.h index e69dd4b2c8f785..7c9367866f502a 100644 --- a/gvfs.h +++ b/gvfs.h @@ -15,6 +15,7 @@ #define GVFS_MISSING_OK (1 << 2) #define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) #define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) +#define GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS (1 << 6) void gvfs_load_config_value(const char *value); int gvfs_config_is_set(int mask); diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh index 8ba840cdc138cd..e7eccfaccb9a36 100755 --- a/t/t0021-conversion.sh +++ b/t/t0021-conversion.sh @@ -335,6 +335,43 @@ test_expect_success "filter: smudge empty file" ' test_cmp expected filtered-empty-in-repo ' +test_expect_success "filter: clean filters blocked when under GVFS" ' + test_config filter.empty-in-repo.clean "cat >/dev/null" && + test_config filter.empty-in-repo.smudge "echo smudged && cat" && + test_config core.gvfs 64 && + + echo dead data walking >empty-in-repo && + test_must_fail git add empty-in-repo +' + +test_expect_success "filter: smudge filters blocked when under GVFS" ' + test_config filter.empty-in-repo.clean "cat >/dev/null" && + test_config filter.empty-in-repo.smudge "echo smudged && cat" && + test_config core.gvfs 64 && + + test_must_fail git checkout +' + +test_expect_success "ident blocked on add when under GVFS" ' + test_config core.gvfs 64 && + test_config core.autocrlf false && + + echo "*.i ident" >.gitattributes && + echo "\$Id\$" > ident.i && + + test_must_fail git add ident.i +' + +test_expect_success "ident blocked when under GVFS" ' + git add ident.i && + + git commit -m "added ident.i" && + test_config core.gvfs 64 && + rm ident.i && + + test_must_fail git checkout -- ident.i +' + test_expect_success 'disable filter with empty override' ' test_config_global filter.disable.smudge false && test_config_global filter.disable.clean false && diff --git a/t/t0027-auto-crlf.sh b/t/t0027-auto-crlf.sh index 2f57c8669cb5af..c42bf106e6f4ea 100755 --- a/t/t0027-auto-crlf.sh +++ b/t/t0027-auto-crlf.sh @@ -344,6 +344,18 @@ checkout_files () { " } +test_expect_success 'crlf conversions blocked when under GVFS' ' + git checkout -b gvfs && + test_commit initial && + rm initial.t && + test_config core.gvfs 64 && + test_config core.autocrlf true && + test_must_fail git read-tree --reset -u HEAD && + + git config core.autocrlf false && + git read-tree --reset -u HEAD +' + # Test control characters # NUL SOH CR EOF==^Z test_expect_success 'ls-files --eol -o Text/Binary' ' From 695e26367357d871d17fd6a70cb3e59f53de00d5 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Tue, 10 Jan 2017 18:47:14 +0000 Subject: [PATCH 035/189] gvfs: allow "virtualizing" objects The idea is to allow blob objects to be missing from the local repository, and to load them lazily on demand. After discussing this idea on the mailing list, we will rename the feature to "lazy clone" and work more on this. Signed-off-by: Ben Peart Signed-off-by: Johannes Schindelin --- config.c | 5 +++++ connected.c | 3 +++ environment.c | 1 + environment.h | 2 ++ object-file.c | 23 +++++++++++++++++++++++ 5 files changed, 34 insertions(+) diff --git a/config.c b/config.c index 9f8144c65a72b9..329b7384f35bbe 100644 --- a/config.c +++ b/config.c @@ -1701,6 +1701,11 @@ int git_default_core_config(const char *var, const char *value, return 0; } + if (!strcmp(var, "core.virtualizeobjects")) { + core_virtualize_objects = git_config_bool(var, value); + return 0; + } + /* Add other config variables here and to Documentation/config.txt. */ return platform_core_config(var, value, ctx, cb); } diff --git a/connected.c b/connected.c index 44e5f75eb063b4..a8cf3dc351fc54 100644 --- a/connected.c +++ b/connected.c @@ -1,6 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" +#include "environment.h" #include "gettext.h" #include "hex.h" #include "gvfs.h" @@ -52,6 +53,8 @@ int check_connected(oid_iterate_fn fn, void *cb_data, */ if (gvfs_config_is_set(GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK)) return 0; + if (core_virtualize_objects) + return 0; if (!opt) opt = &defaults; diff --git a/environment.c b/environment.c index 54b9d182ee6339..01c6963183da3f 100644 --- a/environment.c +++ b/environment.c @@ -84,6 +84,7 @@ int core_gvfs; int merge_log_config = -1; int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ unsigned long pack_size_limit_cfg; +int core_virtualize_objects; enum log_refs_config log_all_ref_updates = LOG_REFS_UNSET; int max_allowed_tree_depth = #ifdef _MSC_VER diff --git a/environment.h b/environment.h index 0e607ead0c8236..881f8947ba8e19 100644 --- a/environment.h +++ b/environment.h @@ -11,6 +11,8 @@ struct strvec; extern const char *comment_line_str; extern int auto_comment_line_char; +extern int core_virtualize_objects; + /* * Wrapper of getenv() that returns a strdup value. This value is kept * in argv to be freed later. diff --git a/object-file.c b/object-file.c index c672adc74ca791..8eb1a2749bd921 100644 --- a/object-file.c +++ b/object-file.c @@ -40,6 +40,8 @@ #include "fsck.h" #include "loose.h" #include "object-file-convert.h" +#include "trace.h" +#include "hook.h" /* The maximum size for an object header. */ #define MAX_HEADER_LEN 32 @@ -1545,6 +1547,20 @@ void disable_obj_read_lock(void) pthread_mutex_destroy(&obj_read_mutex); } +static int run_read_object_hook(const struct object_id *oid) +{ + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + int ret; + uint64_t start; + + start = getnanotime(); + strvec_push(&opt.args, oid_to_hex(oid)); + ret = run_hooks_opt("read-object", &opt); + trace_performance_since(start, "run_read_object_hook"); + + return ret; +} + int fetch_if_missing = 1; static int do_oid_object_info_extended(struct repository *r, @@ -1557,6 +1573,7 @@ static int do_oid_object_info_extended(struct repository *r, int rtype; const struct object_id *real = oid; int already_retried = 0; + int tried_hook = 0; if (flags & OBJECT_INFO_LOOKUP_REPLACE) @@ -1568,6 +1585,7 @@ static int do_oid_object_info_extended(struct repository *r, if (!oi) oi = &blank_oi; +retry: co = find_cached_object(real); if (co) { if (oi->typep) @@ -1599,6 +1617,11 @@ static int do_oid_object_info_extended(struct repository *r, reprepare_packed_git(r); if (find_pack_entry(r, real, &e)) break; + if (core_virtualize_objects && !tried_hook) { + tried_hook = 1; + if (!run_read_object_hook(oid)) + goto retry; + } } /* From a1689192cd2f22cc58c459074cde3d3c43c9c64c Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Tue, 24 May 2016 00:32:38 +0000 Subject: [PATCH 036/189] gvfs: add global command pre and post hook procs This adds hard-coded call to GVFS.hooks.exe before and after each Git command runs. To make sure that this is only called on repositories cloned with GVFS, we test for the tell-tale .gvfs. 2021-10-30: Recent movement of find_hook() to hook.c required moving these changes out of run-command.c to hook.c. Signed-off-by: Ben Peart --- git.c | 84 ++++++++++++++++++++++++++++++++++-- hook.c | 55 ++++++++++++++++++++++- t/t0400-pre-command-hook.sh | 34 +++++++++++++++ t/t0401-post-command-hook.sh | 32 ++++++++++++++ 4 files changed, 201 insertions(+), 4 deletions(-) create mode 100755 t/t0400-pre-command-hook.sh create mode 100755 t/t0401-post-command-hook.sh diff --git a/git.c b/git.c index 7c70fff6218d90..aaff6eaf11909a 100644 --- a/git.c +++ b/git.c @@ -17,6 +17,8 @@ #include "shallow.h" #include "trace.h" #include "trace2.h" +#include "dir.h" +#include "hook.h" #define RUN_SETUP (1<<0) #define RUN_SETUP_GENTLY (1<<1) @@ -434,6 +436,67 @@ static int handle_alias(int *argcp, const char ***argv) return ret; } +/* Runs pre/post-command hook */ +static struct strvec sargv = STRVEC_INIT; +static int run_post_hook = 0; +static int exit_code = -1; + +static int run_pre_command_hook(const char **argv) +{ + char *lock; + int ret = 0; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + + /* + * Ensure the global pre/post command hook is only called for + * the outer command and not when git is called recursively + * or spawns multiple commands (like with the alias command) + */ + lock = getenv("COMMAND_HOOK_LOCK"); + if (lock && !strcmp(lock, "true")) + return 0; + setenv("COMMAND_HOOK_LOCK", "true", 1); + + /* call the hook proc */ + strvec_pushv(&sargv, argv); + strvec_pushv(&opt.args, sargv.v); + ret = run_hooks_opt("pre-command", &opt); + + if (!ret) + run_post_hook = 1; + return ret; +} + +static int run_post_command_hook(void) +{ + char *lock; + int ret = 0; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + + /* + * Only run post_command if pre_command succeeded in this process + */ + if (!run_post_hook) + return 0; + lock = getenv("COMMAND_HOOK_LOCK"); + if (!lock || strcmp(lock, "true")) + return 0; + + strvec_pushv(&opt.args, sargv.v); + strvec_pushf(&opt.args, "--exit_code=%u", exit_code); + ret = run_hooks_opt("post-command", &opt); + + run_post_hook = 0; + strvec_clear(&sargv); + setenv("COMMAND_HOOK_LOCK", "false", 1); + return ret; +} + +static void post_command_hook_atexit(void) +{ + run_post_command_hook(); +} + static int run_builtin(struct cmd_struct *p, int argc, const char **argv) { int status, help; @@ -469,16 +532,21 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv) if (!help && p->option & NEED_WORK_TREE) setup_work_tree(); + if (run_pre_command_hook(argv)) + die("pre-command hook aborted command"); + trace_argv_printf(argv, "trace: built-in: git"); trace2_cmd_name(p->cmd); validate_cache_entries(the_repository->index); - status = p->fn(argc, argv, prefix); + exit_code = status = p->fn(argc, argv, prefix); validate_cache_entries(the_repository->index); if (status) return status; + run_post_command_hook(); + /* Somebody closed stdout? */ if (fstat(fileno(stdout), &st)) return 0; @@ -758,13 +826,16 @@ static void execv_dashed_external(const char **argv) */ trace_argv_printf(cmd.args.v, "trace: exec:"); + if (run_pre_command_hook(cmd.args.v)) + die("pre-command hook aborted command"); + /* * If we fail because the command is not found, it is * OK to return. Otherwise, we just pass along the status code, * or our usual generic code if we were not even able to exec * the program. */ - status = run_command(&cmd); + exit_code = status = run_command(&cmd); /* * If the child process ran and we are now going to exit, emit a @@ -775,6 +846,8 @@ static void execv_dashed_external(const char **argv) exit(status); else if (errno != ENOENT) exit(128); + + run_post_command_hook(); } static int run_argv(int *argcp, const char ***argv) @@ -882,6 +955,7 @@ int cmd_main(int argc, const char **argv) } trace_command_performance(argv); + atexit(post_command_hook_atexit); /* * "git-xxxx" is the same as "git xxxx", but we obviously: @@ -907,10 +981,14 @@ int cmd_main(int argc, const char **argv) if (!argc) { /* The user didn't specify a command; give them help */ commit_pager_choice(); + if (run_pre_command_hook(argv)) + die("pre-command hook aborted command"); printf(_("usage: %s\n\n"), git_usage_string); list_common_cmds_help(); printf("\n%s\n", _(git_more_info_string)); - exit(1); + exit_code = 1; + run_post_command_hook(); + exit(exit_code); } if (!strcmp("--version", argv[0]) || !strcmp("-v", argv[0])) diff --git a/hook.c b/hook.c index 7e90787bca4f7c..c0494457412e23 100644 --- a/hook.c +++ b/hook.c @@ -1,5 +1,6 @@ #include "git-compat-util.h" #include "abspath.h" +#include "environment.h" #include "advice.h" #include "gettext.h" #include "hook.h" @@ -10,6 +11,54 @@ #include "environment.h" #include "setup.h" +static int early_hooks_path_config(const char *var, const char *value, + const struct config_context *ctx, void *cb) +{ + if (!strcmp(var, "core.hookspath")) + return git_config_pathname((char **)cb, var, value); + + return 0; +} + +/* Discover the hook before setup_git_directory() was called */ +static const char *hook_path_early(const char *name, struct strbuf *result) +{ + static struct strbuf hooks_dir = STRBUF_INIT; + static int initialized; + + if (initialized < 0) + return NULL; + + if (!initialized) { + struct strbuf gitdir = STRBUF_INIT, commondir = STRBUF_INIT; + char *early_hooks_dir = NULL; + + if (discover_git_directory(&commondir, &gitdir) < 0) { + strbuf_release(&gitdir); + strbuf_release(&commondir); + initialized = -1; + return NULL; + } + + read_early_config(early_hooks_path_config, &early_hooks_dir); + if (!early_hooks_dir) + strbuf_addf(&hooks_dir, "%s/hooks/", commondir.buf); + else { + strbuf_add_absolute_path(&hooks_dir, early_hooks_dir); + free(early_hooks_dir); + strbuf_addch(&hooks_dir, '/'); + } + + strbuf_release(&gitdir); + strbuf_release(&commondir); + + initialized = 1; + } + + strbuf_addf(result, "%s%s", hooks_dir.buf, name); + return result->buf; +} + const char *find_hook(const char *name) { static struct strbuf path = STRBUF_INIT; @@ -17,7 +66,11 @@ const char *find_hook(const char *name) int found_hook; strbuf_reset(&path); - strbuf_git_path(&path, "hooks/%s", name); + if (have_git_dir()) + strbuf_git_path(&path, "hooks/%s", name); + else if (!hook_path_early(name, &path)) + return NULL; + found_hook = access(path.buf, X_OK) >= 0; #ifdef STRIP_EXTENSION if (!found_hook) { diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh new file mode 100755 index 00000000000000..4f4f610b52b0a0 --- /dev/null +++ b/t/t0400-pre-command-hook.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +test_description='pre-command hook' + +. ./test-lib.sh + +test_expect_success 'with no hook' ' + echo "first" > file && + git add file && + git commit -m "first" +' + +test_expect_success 'with succeeding hook' ' + mkdir -p .git/hooks && + write_script .git/hooks/pre-command <<-EOF && + echo "\$*" >\$(git rev-parse --git-dir)/pre-command.out + EOF + echo "second" >> file && + git add file && + test "add file" = "$(cat .git/pre-command.out)" && + echo Hello | git hash-object --stdin && + test "hash-object --stdin" = "$(cat .git/pre-command.out)" +' + +test_expect_success 'with failing hook' ' + write_script .git/hooks/pre-command <<-EOF && + exit 1 + EOF + echo "third" >> file && + test_must_fail git add file && + test_path_is_missing "$(cat .git/pre-command.out)" +' + +test_done diff --git a/t/t0401-post-command-hook.sh b/t/t0401-post-command-hook.sh new file mode 100755 index 00000000000000..64646f7ad03b57 --- /dev/null +++ b/t/t0401-post-command-hook.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +test_description='post-command hook' + +. ./test-lib.sh + +test_expect_success 'with no hook' ' + echo "first" > file && + git add file && + git commit -m "first" +' + +test_expect_success 'with succeeding hook' ' + mkdir -p .git/hooks && + write_script .git/hooks/post-command <<-EOF && + echo "\$*" >\$(git rev-parse --git-dir)/post-command.out + EOF + echo "second" >> file && + git add file && + test "add file --exit_code=0" = "$(cat .git/post-command.out)" +' + +test_expect_success 'with failing pre-command hook' ' + write_script .git/hooks/pre-command <<-EOF && + exit 1 + EOF + echo "third" >> file && + test_must_fail git add file && + test_path_is_missing "$(cat .git/post-command.out)" +' + +test_done From 5bcdb2cddf3be7b227a8eec11d727179b8897d62 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Wed, 15 Mar 2017 18:43:05 +0000 Subject: [PATCH 037/189] Hydrate missing loose objects in check_and_freshen() Hydrate missing loose objects in check_and_freshen() when running virtualized. Add test cases to verify read-object hook works when running virtualized. This hook is called in check_and_freshen() rather than check_and_freshen_local() to make the hook work also with alternates. Helped-by: Kevin Willford Signed-off-by: Ben Peart --- .../technical/read-object-protocol.txt | 102 +++++++++++++ contrib/long-running-read-object/example.pl | 114 ++++++++++++++ object-file.c | 141 ++++++++++++++++-- t/t0410/read-object | 114 ++++++++++++++ t/t0499-read-object.sh | 30 ++++ 5 files changed, 485 insertions(+), 16 deletions(-) create mode 100644 Documentation/technical/read-object-protocol.txt create mode 100644 contrib/long-running-read-object/example.pl create mode 100755 t/t0410/read-object create mode 100755 t/t0499-read-object.sh diff --git a/Documentation/technical/read-object-protocol.txt b/Documentation/technical/read-object-protocol.txt new file mode 100644 index 00000000000000..a893b46e7c28a9 --- /dev/null +++ b/Documentation/technical/read-object-protocol.txt @@ -0,0 +1,102 @@ +Read Object Process +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The read-object process enables Git to read all missing blobs with a +single process invocation for the entire life of a single Git command. +This is achieved by using a packet format (pkt-line, see technical/ +protocol-common.txt) based protocol over standard input and standard +output as follows. All packets, except for the "*CONTENT" packets and +the "0000" flush packet, are considered text and therefore are +terminated by a LF. + +Git starts the process when it encounters the first missing object that +needs to be retrieved. After the process is started, Git sends a welcome +message ("git-read-object-client"), a list of supported protocol version +numbers, and a flush packet. Git expects to read a welcome response +message ("git-read-object-server"), exactly one protocol version number +from the previously sent list, and a flush packet. All further +communication will be based on the selected version. + +The remaining protocol description below documents "version=1". Please +note that "version=42" in the example below does not exist and is only +there to illustrate how the protocol would look with more than one +version. + +After the version negotiation Git sends a list of all capabilities that +it supports and a flush packet. Git expects to read a list of desired +capabilities, which must be a subset of the supported capabilities list, +and a flush packet as response: +------------------------ +packet: git> git-read-object-client +packet: git> version=1 +packet: git> version=42 +packet: git> 0000 +packet: git< git-read-object-server +packet: git< version=1 +packet: git< 0000 +packet: git> capability=get +packet: git> capability=have +packet: git> capability=put +packet: git> capability=not-yet-invented +packet: git> 0000 +packet: git< capability=get +packet: git< 0000 +------------------------ +The only supported capability in version 1 is "get". + +Afterwards Git sends a list of "key=value" pairs terminated with a flush +packet. The list will contain at least the command (based on the +supported capabilities) and the sha1 of the object to retrieve. Please +note, that the process must not send any response before it received the +final flush packet. + +When the process receives the "get" command, it should make the requested +object available in the git object store and then return success. Git will +then check the object store again and this time find it and proceed. +------------------------ +packet: git> command=get +packet: git> sha1=0a214a649e1b3d5011e14a3dc227753f2bd2be05 +packet: git> 0000 +------------------------ + +The process is expected to respond with a list of "key=value" pairs +terminated with a flush packet. If the process does not experience +problems then the list must contain a "success" status. +------------------------ +packet: git< status=success +packet: git< 0000 +------------------------ + +In case the process cannot or does not want to process the content, it +is expected to respond with an "error" status. +------------------------ +packet: git< status=error +packet: git< 0000 +------------------------ + +In case the process cannot or does not want to process the content as +well as any future content for the lifetime of the Git process, then it +is expected to respond with an "abort" status at any point in the +protocol. +------------------------ +packet: git< status=abort +packet: git< 0000 +------------------------ + +Git neither stops nor restarts the process in case the "error"/"abort" +status is set. + +If the process dies during the communication or does not adhere to the +protocol then Git will stop the process and restart it with the next +object that needs to be processed. + +After the read-object process has processed an object it is expected to +wait for the next "key=value" list containing a command. Git will close +the command pipe on exit. The process is expected to detect EOF and exit +gracefully on its own. Git will wait until the process has stopped. + +A long running read-object process demo implementation can be found in +`contrib/long-running-read-object/example.pl` located in the Git core +repository. If you develop your own long running process then the +`GIT_TRACE_PACKET` environment variables can be very helpful for +debugging (see linkgit:git[1]). diff --git a/contrib/long-running-read-object/example.pl b/contrib/long-running-read-object/example.pl new file mode 100644 index 00000000000000..b8f37f836a813c --- /dev/null +++ b/contrib/long-running-read-object/example.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl +# +# Example implementation for the Git read-object protocol version 1 +# See Documentation/technical/read-object-protocol.txt +# +# Allows you to test the ability for blobs to be pulled from a host git repo +# "on demand." Called when git needs a blob it couldn't find locally due to +# a lazy clone that only cloned the commits and trees. +# +# A lazy clone can be simulated via the following commands from the host repo +# you wish to create a lazy clone of: +# +# cd /host_repo +# git rev-parse HEAD +# git init /guest_repo +# git cat-file --batch-check --batch-all-objects | grep -v 'blob' | +# cut -d' ' -f1 | git pack-objects /guest_repo/.git/objects/pack/noblobs +# cd /guest_repo +# git config core.virtualizeobjects true +# git reset --hard +# +# Please note, this sample is a minimal skeleton. No proper error handling +# was implemented. +# + +use strict; +use warnings; + +# +# Point $DIR to the folder where your host git repo is located so we can pull +# missing objects from it +# +my $DIR = "/host_repo/.git/"; + +sub packet_bin_read { + my $buffer; + my $bytes_read = read STDIN, $buffer, 4; + if ( $bytes_read == 0 ) { + + # EOF - Git stopped talking to us! + exit(); + } + elsif ( $bytes_read != 4 ) { + die "invalid packet: '$buffer'"; + } + my $pkt_size = hex($buffer); + if ( $pkt_size == 0 ) { + return ( 1, "" ); + } + elsif ( $pkt_size > 4 ) { + my $content_size = $pkt_size - 4; + $bytes_read = read STDIN, $buffer, $content_size; + if ( $bytes_read != $content_size ) { + die "invalid packet ($content_size bytes expected; $bytes_read bytes read)"; + } + return ( 0, $buffer ); + } + else { + die "invalid packet size: $pkt_size"; + } +} + +sub packet_txt_read { + my ( $res, $buf ) = packet_bin_read(); + unless ( $buf =~ s/\n$// ) { + die "A non-binary line MUST be terminated by an LF."; + } + return ( $res, $buf ); +} + +sub packet_bin_write { + my $buf = shift; + print STDOUT sprintf( "%04x", length($buf) + 4 ); + print STDOUT $buf; + STDOUT->flush(); +} + +sub packet_txt_write { + packet_bin_write( $_[0] . "\n" ); +} + +sub packet_flush { + print STDOUT sprintf( "%04x", 0 ); + STDOUT->flush(); +} + +( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || die "bad initialize"; +( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad version end"; + +packet_txt_write("git-read-object-server"); +packet_txt_write("version=1"); +packet_flush(); + +( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end"; + +packet_txt_write("capability=get"); +packet_flush(); + +while (1) { + my ($command) = packet_txt_read() =~ /^command=([^=]+)$/; + + if ( $command eq "get" ) { + my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/; + packet_bin_read(); + + system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); + packet_txt_write(($?) ? "status=error" : "status=success"); + packet_flush(); + } else { + die "bad command '$command'"; + } +} diff --git a/object-file.c b/object-file.c index 8eb1a2749bd921..1b37072b239536 100644 --- a/object-file.c +++ b/object-file.c @@ -42,6 +42,9 @@ #include "object-file-convert.h" #include "trace.h" #include "hook.h" +#include "sigchain.h" +#include "sub-process.h" +#include "pkt-line.h" /* The maximum size for an object header. */ #define MAX_HEADER_LEN 32 @@ -952,6 +955,115 @@ int has_alt_odb(struct repository *r) return !!r->objects->odb->next; } +#define CAP_GET (1u<<0) + +static int subprocess_map_initialized; +static struct hashmap subprocess_map; + +struct read_object_process { + struct subprocess_entry subprocess; + unsigned int supported_capabilities; +}; + +static int start_read_object_fn(struct subprocess_entry *subprocess) +{ + struct read_object_process *entry = (struct read_object_process *)subprocess; + static int versions[] = {1, 0}; + static struct subprocess_capability capabilities[] = { + { "get", CAP_GET }, + { NULL, 0 } + }; + + return subprocess_handshake(subprocess, "git-read-object", versions, + NULL, capabilities, + &entry->supported_capabilities); +} + +static int read_object_process(const struct object_id *oid) +{ + int err; + struct read_object_process *entry; + struct child_process *process; + struct strbuf status = STRBUF_INIT; + const char *cmd = find_hook("read-object"); + uint64_t start; + + start = getnanotime(); + + if (!subprocess_map_initialized) { + subprocess_map_initialized = 1; + hashmap_init(&subprocess_map, (hashmap_cmp_fn)cmd2process_cmp, + NULL, 0); + entry = NULL; + } else { + entry = (struct read_object_process *) subprocess_find_entry(&subprocess_map, cmd); + } + + if (!entry) { + entry = xmalloc(sizeof(*entry)); + entry->supported_capabilities = 0; + + if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, + start_read_object_fn)) { + free(entry); + return -1; + } + } + process = &entry->subprocess.process; + + if (!(CAP_GET & entry->supported_capabilities)) + return -1; + + sigchain_push(SIGPIPE, SIG_IGN); + + err = packet_write_fmt_gently(process->in, "command=get\n"); + if (err) + goto done; + + err = packet_write_fmt_gently(process->in, "sha1=%s\n", oid_to_hex(oid)); + if (err) + goto done; + + err = packet_flush_gently(process->in); + if (err) + goto done; + + err = subprocess_read_status(process->out, &status); + err = err ? err : strcmp(status.buf, "success"); + +done: + sigchain_pop(SIGPIPE); + + if (err || errno == EPIPE) { + err = err ? err : errno; + if (!strcmp(status.buf, "error")) { + /* The process signaled a problem with the file. */ + } + else if (!strcmp(status.buf, "abort")) { + /* + * The process signaled a permanent problem. Don't try to read + * objects with the same command for the lifetime of the current + * Git process. + */ + entry->supported_capabilities &= ~CAP_GET; + } + else { + /* + * Something went wrong with the read-object process. + * Force shutdown and restart if needed. + */ + error("external process '%s' failed", cmd); + subprocess_stop(&subprocess_map, + (struct subprocess_entry *)entry); + free(entry); + } + } + + trace_performance_since(start, "read_object_process"); + + return err; +} + /* Returns 1 if we have successfully freshened the file, 0 otherwise. */ static int freshen_file(const char *fn) { @@ -1002,8 +1114,19 @@ static int check_and_freshen_nonlocal(const struct object_id *oid, int freshen) static int check_and_freshen(const struct object_id *oid, int freshen) { - return check_and_freshen_local(oid, freshen) || + int ret; + int tried_hook = 0; + +retry: + ret = check_and_freshen_local(oid, freshen) || check_and_freshen_nonlocal(oid, freshen); + if (!ret && core_virtualize_objects && !tried_hook) { + tried_hook = 1; + if (!read_object_process(oid)) + goto retry; + } + + return ret; } int has_loose_object_nonlocal(const struct object_id *oid) @@ -1547,20 +1670,6 @@ void disable_obj_read_lock(void) pthread_mutex_destroy(&obj_read_mutex); } -static int run_read_object_hook(const struct object_id *oid) -{ - struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; - int ret; - uint64_t start; - - start = getnanotime(); - strvec_push(&opt.args, oid_to_hex(oid)); - ret = run_hooks_opt("read-object", &opt); - trace_performance_since(start, "run_read_object_hook"); - - return ret; -} - int fetch_if_missing = 1; static int do_oid_object_info_extended(struct repository *r, @@ -1619,7 +1728,7 @@ static int do_oid_object_info_extended(struct repository *r, break; if (core_virtualize_objects && !tried_hook) { tried_hook = 1; - if (!run_read_object_hook(oid)) + if (!read_object_process(oid)) goto retry; } } diff --git a/t/t0410/read-object b/t/t0410/read-object new file mode 100755 index 00000000000000..2b8feacc78577f --- /dev/null +++ b/t/t0410/read-object @@ -0,0 +1,114 @@ +#!/usr/bin/perl +# +# Example implementation for the Git read-object protocol version 1 +# See Documentation/technical/read-object-protocol.txt +# +# Allows you to test the ability for blobs to be pulled from a host git repo +# "on demand." Called when git needs a blob it couldn't find locally due to +# a lazy clone that only cloned the commits and trees. +# +# A lazy clone can be simulated via the following commands from the host repo +# you wish to create a lazy clone of: +# +# cd /host_repo +# git rev-parse HEAD +# git init /guest_repo +# git cat-file --batch-check --batch-all-objects | grep -v 'blob' | +# cut -d' ' -f1 | git pack-objects /guest_repo/.git/objects/pack/noblobs +# cd /guest_repo +# git config core.virtualizeobjects true +# git reset --hard +# +# Please note, this sample is a minimal skeleton. No proper error handling +# was implemented. +# + +use strict; +use warnings; + +# +# Point $DIR to the folder where your host git repo is located so we can pull +# missing objects from it +# +my $DIR = "../.git/"; + +sub packet_bin_read { + my $buffer; + my $bytes_read = read STDIN, $buffer, 4; + if ( $bytes_read == 0 ) { + + # EOF - Git stopped talking to us! + exit(); + } + elsif ( $bytes_read != 4 ) { + die "invalid packet: '$buffer'"; + } + my $pkt_size = hex($buffer); + if ( $pkt_size == 0 ) { + return ( 1, "" ); + } + elsif ( $pkt_size > 4 ) { + my $content_size = $pkt_size - 4; + $bytes_read = read STDIN, $buffer, $content_size; + if ( $bytes_read != $content_size ) { + die "invalid packet ($content_size bytes expected; $bytes_read bytes read)"; + } + return ( 0, $buffer ); + } + else { + die "invalid packet size: $pkt_size"; + } +} + +sub packet_txt_read { + my ( $res, $buf ) = packet_bin_read(); + unless ( $buf =~ s/\n$// ) { + die "A non-binary line MUST be terminated by an LF."; + } + return ( $res, $buf ); +} + +sub packet_bin_write { + my $buf = shift; + print STDOUT sprintf( "%04x", length($buf) + 4 ); + print STDOUT $buf; + STDOUT->flush(); +} + +sub packet_txt_write { + packet_bin_write( $_[0] . "\n" ); +} + +sub packet_flush { + print STDOUT sprintf( "%04x", 0 ); + STDOUT->flush(); +} + +( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || die "bad initialize"; +( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad version end"; + +packet_txt_write("git-read-object-server"); +packet_txt_write("version=1"); +packet_flush(); + +( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end"; + +packet_txt_write("capability=get"); +packet_flush(); + +while (1) { + my ($command) = packet_txt_read() =~ /^command=([^=]+)$/; + + if ( $command eq "get" ) { + my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40,64})$/; + packet_bin_read(); + + system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); + packet_txt_write(($?) ? "status=error" : "status=success"); + packet_flush(); + } else { + die "bad command '$command'"; + } +} diff --git a/t/t0499-read-object.sh b/t/t0499-read-object.sh new file mode 100755 index 00000000000000..2e208bdb46add5 --- /dev/null +++ b/t/t0499-read-object.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +test_description='tests for long running read-object process' + +. ./test-lib.sh + +test_expect_success 'setup host repo with a root commit' ' + test_commit zero && + hash1=$(git ls-tree HEAD | grep zero.t | cut -f1 | cut -d\ -f3) +' + +test_expect_success 'blobs can be retrieved from the host repo' ' + git init guest-repo && + (cd guest-repo && + mkdir -p .git/hooks && + sed "1s|/usr/bin/perl|$PERL_PATH|" \ + <$TEST_DIRECTORY/t0410/read-object \ + >.git/hooks/read-object && + chmod +x .git/hooks/read-object && + git config core.virtualizeobjects true && + git cat-file blob "$hash1") +' + +test_expect_success 'invalid blobs generate errors' ' + (cd guest-repo && + test_must_fail git cat-file blob "invalid") +' + + +test_done From 8e7be74a0f9d0f4f87f20de8b7fbfcfe9117888e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 16 Mar 2017 21:07:54 +0100 Subject: [PATCH 038/189] t0400: verify that the hook is called correctly from a subdirectory Suggested by Ben Peart. Signed-off-by: Johannes Schindelin --- t/t0400-pre-command-hook.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh index 4f4f610b52b0a0..83c453c9643eae 100755 --- a/t/t0400-pre-command-hook.sh +++ b/t/t0400-pre-command-hook.sh @@ -31,4 +31,27 @@ test_expect_success 'with failing hook' ' test_path_is_missing "$(cat .git/pre-command.out)" ' +test_expect_success 'in a subdirectory' ' + echo touch i-was-here | write_script .git/hooks/pre-command && + mkdir sub && + ( + cd sub && + git version + ) && + test_path_is_file sub/i-was-here +' + +test_expect_success 'in a subdirectory, using an alias' ' + git reset --hard && + echo "echo \"\$@; \$(pwd)\" >>log" | + write_script .git/hooks/pre-command && + mkdir -p sub && + ( + cd sub && + git -c alias.v="version" v + ) && + test_path_is_missing log && + test_line_count = 2 sub/log +' + test_done From b1fd1dfc92fdb9d022afc51b917ee0cccfe1ea75 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 8 Sep 2017 11:32:43 +0200 Subject: [PATCH 039/189] sha1_file: when writing objects, skip the read_object_hook If we are going to write an object there is no use in calling the read object hook to get an object from a potentially remote source. We would rather just write out the object and avoid the potential round trip for an object that doesn't exist. This change adds a flag to the check_and_freshen() and freshen_loose_object() functions' signatures so that the hook is bypassed when the functions are called before writing loose objects. The check for a local object is still performed so we don't overwrite something that has already been written to one of the objects directories. Based on a patch by Kevin Willford. Signed-off-by: Johannes Schindelin --- object-file.c | 19 +++++++++++-------- t/t0410/read-object | 4 ++++ t/t0499-read-object.sh | 7 +++++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/object-file.c b/object-file.c index 1b37072b239536..0024936fd210f1 100644 --- a/object-file.c +++ b/object-file.c @@ -1112,7 +1112,8 @@ static int check_and_freshen_nonlocal(const struct object_id *oid, int freshen) return 0; } -static int check_and_freshen(const struct object_id *oid, int freshen) +static int check_and_freshen(const struct object_id *oid, int freshen, + int skip_virtualized_objects) { int ret; int tried_hook = 0; @@ -1120,7 +1121,8 @@ static int check_and_freshen(const struct object_id *oid, int freshen) retry: ret = check_and_freshen_local(oid, freshen) || check_and_freshen_nonlocal(oid, freshen); - if (!ret && core_virtualize_objects && !tried_hook) { + if (!ret && core_virtualize_objects && !skip_virtualized_objects && + !tried_hook) { tried_hook = 1; if (!read_object_process(oid)) goto retry; @@ -1136,7 +1138,7 @@ int has_loose_object_nonlocal(const struct object_id *oid) int has_loose_object(const struct object_id *oid) { - return check_and_freshen(oid, 0); + return check_and_freshen(oid, 0, 0); } static void mmap_limit_check(size_t length) @@ -2321,9 +2323,10 @@ static int write_loose_object(const struct object_id *oid, char *hdr, return finalize_object_file(tmp_file.buf, filename.buf); } -static int freshen_loose_object(const struct object_id *oid) +static int freshen_loose_object(const struct object_id *oid, + int skip_virtualized_objects) { - return check_and_freshen(oid, 1); + return check_and_freshen(oid, 1, skip_virtualized_objects); } static int freshen_packed_object(const struct object_id *oid) @@ -2419,7 +2422,7 @@ int stream_loose_object(struct input_stream *in_stream, size_t len, die(_("deflateEnd on stream object failed (%d)"), ret); close_loose_object(fd, tmp_file.buf); - if (freshen_packed_object(oid) || freshen_loose_object(oid)) { + if (freshen_packed_object(oid) || freshen_loose_object(oid, 1)) { unlink_or_warn(tmp_file.buf); goto cleanup; } @@ -2480,7 +2483,7 @@ int write_object_file_flags(const void *buf, size_t len, * it out into .git/objects/??/?{38} file. */ write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen); - if (freshen_packed_object(oid) || freshen_loose_object(oid)) + if (freshen_packed_object(oid) || freshen_loose_object(oid, 1)) return 0; if (write_loose_object(oid, hdr, hdrlen, buf, len, 0, flags)) return -1; @@ -2524,7 +2527,7 @@ int write_object_file_literally(const void *buf, size_t len, if (!(flags & HASH_WRITE_OBJECT)) goto cleanup; - if (freshen_packed_object(oid) || freshen_loose_object(oid)) + if (freshen_packed_object(oid) || freshen_loose_object(oid, 1)) goto cleanup; status = write_loose_object(oid, header, hdrlen, buf, len, 0, 0); if (compat_type != -1) diff --git a/t/t0410/read-object b/t/t0410/read-object index 2b8feacc78577f..02c799837f4057 100755 --- a/t/t0410/read-object +++ b/t/t0410/read-object @@ -108,6 +108,10 @@ while (1) { system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); packet_txt_write(($?) ? "status=error" : "status=success"); packet_flush(); + + open my $log, '>>.git/read-object-hook.log'; + print $log "Read object $sha1, exit code $?\n"; + close $log; } else { die "bad command '$command'"; } diff --git a/t/t0499-read-object.sh b/t/t0499-read-object.sh index 2e208bdb46add5..0cee1963cf091e 100755 --- a/t/t0499-read-object.sh +++ b/t/t0499-read-object.sh @@ -26,5 +26,12 @@ test_expect_success 'invalid blobs generate errors' ' test_must_fail git cat-file blob "invalid") ' +test_expect_success 'read-object-hook is bypassed when writing objects' ' + (cd guest-repo && + echo hello >hello.txt && + git add hello.txt && + hash="$(git rev-parse --verify :hello.txt)" && + ! grep "$hash" .git/read-object-hook.log) +' test_done From f27a1ffa6d0ed34e15b390812144395259449549 Mon Sep 17 00:00:00 2001 From: Alejandro Pauly Date: Mon, 10 Apr 2017 13:26:14 -0400 Subject: [PATCH 040/189] Pass PID of git process to hooks. Signed-off-by: Alejandro Pauly --- git.c | 1 + t/t0400-pre-command-hook.sh | 3 ++- t/t0401-post-command-hook.sh | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/git.c b/git.c index aaff6eaf11909a..6a9706a1f966d5 100644 --- a/git.c +++ b/git.c @@ -459,6 +459,7 @@ static int run_pre_command_hook(const char **argv) /* call the hook proc */ strvec_pushv(&sargv, argv); + strvec_pushf(&sargv, "--git-pid=%"PRIuMAX, (uintmax_t)getpid()); strvec_pushv(&opt.args, sargv.v); ret = run_hooks_opt("pre-command", &opt); diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh index 83c453c9643eae..f04a55a695bc97 100755 --- a/t/t0400-pre-command-hook.sh +++ b/t/t0400-pre-command-hook.sh @@ -13,7 +13,8 @@ test_expect_success 'with no hook' ' test_expect_success 'with succeeding hook' ' mkdir -p .git/hooks && write_script .git/hooks/pre-command <<-EOF && - echo "\$*" >\$(git rev-parse --git-dir)/pre-command.out + echo "\$*" | sed "s/ --git-pid=[0-9]*//" \ + >\$(git rev-parse --git-dir)/pre-command.out EOF echo "second" >> file && git add file && diff --git a/t/t0401-post-command-hook.sh b/t/t0401-post-command-hook.sh index 64646f7ad03b57..fcbfc4a0c79c1e 100755 --- a/t/t0401-post-command-hook.sh +++ b/t/t0401-post-command-hook.sh @@ -13,7 +13,8 @@ test_expect_success 'with no hook' ' test_expect_success 'with succeeding hook' ' mkdir -p .git/hooks && write_script .git/hooks/post-command <<-EOF && - echo "\$*" >\$(git rev-parse --git-dir)/post-command.out + echo "\$*" | sed "s/ --git-pid=[0-9]*//" \ + >\$(git rev-parse --git-dir)/post-command.out EOF echo "second" >> file && git add file && From a64248f13e152c0c32dbe75738d996981ba0dad1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 8 Aug 2017 00:27:50 +0200 Subject: [PATCH 041/189] pre-command: always respect core.hooksPath We need to respect that config setting even if we already know that we have a repository, but have not yet read the config. The regression test was written by Alejandro Pauly. 2021-10-30: Recent movement of find_hook() into hook.c required moving this change from run-command.c. Signed-off-by: Johannes Schindelin --- hook.c | 15 +++++++++++++-- t/t0400-pre-command-hook.sh | 11 +++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/hook.c b/hook.c index c0494457412e23..38749433eacfaa 100644 --- a/hook.c +++ b/hook.c @@ -66,9 +66,20 @@ const char *find_hook(const char *name) int found_hook; strbuf_reset(&path); - if (have_git_dir()) + if (have_git_dir()) { + static int forced_config; + + if (!forced_config) { + if (!git_hooks_path) { + git_config_get_pathname("core.hookspath", + &git_hooks_path); + UNLEAK(git_hooks_path); + } + forced_config = 1; + } + strbuf_git_path(&path, "hooks/%s", name); - else if (!hook_path_early(name, &path)) + } else if (!hook_path_early(name, &path)) return NULL; found_hook = access(path.buf, X_OK) >= 0; diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh index f04a55a695bc97..f2a9115e299385 100755 --- a/t/t0400-pre-command-hook.sh +++ b/t/t0400-pre-command-hook.sh @@ -55,4 +55,15 @@ test_expect_success 'in a subdirectory, using an alias' ' test_line_count = 2 sub/log ' +test_expect_success 'with core.hooksPath' ' + mkdir -p .git/alternateHooks && + write_script .git/alternateHooks/pre-command <<-EOF && + echo "alternate" >\$(git rev-parse --git-dir)/pre-command.out + EOF + write_script .git/hooks/pre-command <<-EOF && + echo "original" >\$(git rev-parse --git-dir)/pre-command.out + EOF + git -c core.hooksPath=.git/alternateHooks status && + test "alternate" = "$(cat .git/pre-command.out)" +' test_done From 7a255d4ecae72e6e5742d17998a5c7b8fe4c674e Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 22 Feb 2017 12:50:43 -0700 Subject: [PATCH 042/189] sparse-checkout: update files with a modify/delete conflict When using the sparse-checkout feature, the file might not be on disk because the skip-worktree bit is on. Signed-off-by: Kevin Willford --- merge-recursive.c | 2 +- t/t7615-merge-sparse-checkout.sh | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100755 t/t7615-merge-sparse-checkout.sh diff --git a/merge-recursive.c b/merge-recursive.c index f606324dee3bdc..4e535faad39e7f 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -1593,7 +1593,7 @@ static int handle_change_delete(struct merge_options *opt, * path. We could call update_file_flags() with update_cache=0 * and update_wd=0, but that's a no-op. */ - if (change_branch != opt->branch1 || alt_path) + if (change_branch != opt->branch1 || alt_path || !file_exists(update_path)) ret = update_file(opt, 0, changed, update_path); } free(alt_path); diff --git a/t/t7615-merge-sparse-checkout.sh b/t/t7615-merge-sparse-checkout.sh new file mode 100755 index 00000000000000..5ce12431f62ad1 --- /dev/null +++ b/t/t7615-merge-sparse-checkout.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +test_description='merge can handle sparse-checkout' + +. ./test-lib.sh + +# merges with conflicts + +test_expect_success 'setup' ' + git branch -M main && + test_commit a && + test_commit file && + git checkout -b delete-file && + git rm file.t && + test_tick && + git commit -m "remove file" && + git checkout main && + test_commit modify file.t changed +' + +test_expect_success 'merge conflict deleted file and modified' ' + echo "/a.t" >.git/info/sparse-checkout && + test_config core.sparsecheckout true && + git checkout -f && + test_path_is_missing file.t && + test_must_fail git merge delete-file && + test_path_is_file file.t && + test "changed" = "$(cat file.t)" +' + +test_done From 841750228647de995c6956ad729eafbf9222dd27 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 1 Mar 2017 15:17:12 -0800 Subject: [PATCH 043/189] sparse-checkout: avoid writing entries with the skip-worktree bit When using the sparse-checkout feature git should not write to the working directory for files with the skip-worktree bit on. With the skip-worktree bit on the file may or may not be in the working directory and if it is not we don't want or need to create it by calling checkout_entry. There are two callers of checkout_target. Both of which check that the file does not exist before calling checkout_target. load_current which make a call to lstat right before calling checkout_target and check_preimage which will only run checkout_taret it stat_ret is less than zero. It sets stat_ret to zero and only if !stat->cached will it lstat the file and set stat_ret to something other than zero. This patch checks if skip-worktree bit is on in checkout_target and just returns so that the entry doesn't not end up in the working directory. This is so that apply will not create a file in the working directory, then update the index but not keep the working directory up to date with the changes that happened in the index. Signed-off-by: Kevin Willford --- apply.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/apply.c b/apply.c index d2a34abafe480e..44d10339517375 100644 --- a/apply.c +++ b/apply.c @@ -3415,6 +3415,24 @@ static int checkout_target(struct index_state *istate, { struct checkout costate = CHECKOUT_INIT; + /* + * Do not checkout the entry if the skipworktree bit is set + * + * Both callers of this method (check_preimage and load_current) + * check for the existance of the file before calling this + * method so we know that the file doesn't exist at this point + * and we don't need to perform that check again here. + * We just need to check the skip-worktree and return. + * + * This is to prevent git from creating a file in the + * working directory that has the skip-worktree bit on, + * then updating the index from the patch and not keeping + * the working directory version up to date with what it + * changed the index version to be. + */ + if (ce_skip_worktree(ce)) + return 0; + costate.refresh_cache = 1; costate.istate = istate; if (checkout_entry(ce, &costate, NULL, NULL) || From 7be6acd09ce67594b09114a7dcfff06b572cfda5 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 5 Apr 2017 10:55:32 -0600 Subject: [PATCH 044/189] Do not remove files outside the sparse-checkout Signed-off-by: Kevin Willford --- unpack-trees.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unpack-trees.c b/unpack-trees.c index 7c564ce5566102..8babaac11728ef 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -571,7 +571,9 @@ static int apply_sparse_checkout(struct index_state *istate, ce->ce_flags &= ~CE_SKIP_WORKTREE; return -1; } - ce->ce_flags |= CE_WT_REMOVE; + if (!gvfs_config_is_set(GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT)) + ce->ce_flags |= CE_WT_REMOVE; + ce->ce_flags &= ~CE_UPDATE; } if (was_skip_worktree && !ce_skip_worktree(ce)) { From 783668aa222cdf2e477c535430099b102b02c2d3 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 16 Nov 2018 11:28:59 -0700 Subject: [PATCH 045/189] send-pack: do not check for sha1 file when GVFS_MISSING_OK set --- send-pack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/send-pack.c b/send-pack.c index f9e6255ee13328..9d7a3f50dcac4b 100644 --- a/send-pack.c +++ b/send-pack.c @@ -6,6 +6,7 @@ #include "date.h" #include "gettext.h" #include "hex.h" +#include "gvfs.h" #include "object-store-ll.h" #include "pkt-line.h" #include "sideband.h" @@ -46,7 +47,7 @@ int option_parse_push_signed(const struct option *opt, static void feed_object(const struct object_id *oid, FILE *fh, int negative) { - if (negative && + if (negative && !gvfs_config_is_set(GVFS_MISSING_OK) && !repo_has_object_file_with_flags(the_repository, oid, OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK)) From deee997e30b9c62322dfc1365bc9a0599a04fd1f Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 3 Jul 2017 13:39:45 -0600 Subject: [PATCH 046/189] cache-tree: remove use of strbuf_addf in update_one String formatting can be a performance issue when there are hundreds of thousands of trees. Change to stop using the strbuf_addf and just add the strings or characters individually. There are a limited number of modes so added a switch for the known ones and a default case if something comes through that are not a known one for git. In one scenario regarding a huge worktree, this reduces the time required for a `git checkout ` from 44 seconds to 38 seconds, i.e. it is a non-negligible performance improvement. Signed-off-by: Kevin Willford --- cache-tree.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/cache-tree.c b/cache-tree.c index 02808209115ec0..644a74021be0d6 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -430,7 +430,29 @@ static int update_one(struct cache_tree *it, continue; strbuf_grow(&buffer, entlen + 100); - strbuf_addf(&buffer, "%o %.*s%c", mode, entlen, path + baselen, '\0'); + + switch (mode) { + case 0100644: + strbuf_add(&buffer, "100644 ", 7); + break; + case 0100664: + strbuf_add(&buffer, "100664 ", 7); + break; + case 0100755: + strbuf_add(&buffer, "100755 ", 7); + break; + case 0120000: + strbuf_add(&buffer, "120000 ", 7); + break; + case 0160000: + strbuf_add(&buffer, "160000 ", 7); + break; + default: + strbuf_addf(&buffer, "%o ", mode); + break; + } + strbuf_add(&buffer, path + baselen, entlen); + strbuf_addch(&buffer, '\0'); strbuf_add(&buffer, oid->hash, the_hash_algo->rawsz); #if DEBUG_CACHE_TREE From 335dfe046d0e2df518fca091a3896c8537c9fdbb Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 6 Dec 2018 11:09:19 -0500 Subject: [PATCH 047/189] gvfs: block unsupported commands when running in a GVFS repo The following commands and options are not currently supported when working in a GVFS repo. Add code to detect and block these commands from executing. 1) fsck 2) gc 4) prune 5) repack 6) submodule 8) update-index --split-index 9) update-index --index-version (other than 4) 10) update-index --[no-]skip-worktree 11) worktree Signed-off-by: Ben Peart Signed-off-by: Johannes Schindelin --- builtin/gc.c | 4 ++++ builtin/update-index.c | 10 ++++++++ git.c | 15 ++++++++---- gvfs.h | 1 + t/t0402-block-command-on-gvfs.sh | 39 ++++++++++++++++++++++++++++++++ 5 files changed, 64 insertions(+), 5 deletions(-) create mode 100755 t/t0402-block-command-on-gvfs.sh diff --git a/builtin/gc.c b/builtin/gc.c index 72bac2554fc4b9..f1b1d13972d646 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -16,6 +16,7 @@ #include "environment.h" #include "hex.h" #include "repository.h" +#include "gvfs.h" #include "config.h" #include "tempfile.h" #include "lockfile.h" @@ -676,6 +677,9 @@ int cmd_gc(int argc, const char **argv, const char *prefix) if (quiet) strvec_push(&repack, "-q"); + if ((!opts.auto_flag || (opts.auto_flag && gc_auto_threshold > 0)) && gvfs_config_is_set(GVFS_BLOCK_COMMANDS)) + die(_("'git gc' is not supported on a GVFS repo")); + if (opts.auto_flag) { /* * Auto-gc should be least intrusive as possible. diff --git a/builtin/update-index.c b/builtin/update-index.c index d343416ae261ff..9e1eb6b0aead95 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -5,6 +5,7 @@ */ #include "builtin.h" +#include "gvfs.h" #include "bulk-checkin.h" #include "config.h" #include "environment.h" @@ -1110,7 +1111,13 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) argc = parse_options_end(&ctx); getline_fn = nul_term_line ? strbuf_getline_nul : strbuf_getline_lf; + if (mark_skip_worktree_only && gvfs_config_is_set(GVFS_BLOCK_COMMANDS)) + die(_("modifying the skip worktree bit is not supported on a GVFS repo")); + if (preferred_index_format) { + if (preferred_index_format != 4 && gvfs_config_is_set(GVFS_BLOCK_COMMANDS)) + die(_("changing the index version is not supported on a GVFS repo")); + if (preferred_index_format < 0) { printf(_("%d\n"), the_repository->index->version); } else if (preferred_index_format < INDEX_FORMAT_LB || @@ -1156,6 +1163,9 @@ int cmd_update_index(int argc, const char **argv, const char *prefix) end_odb_transaction(); if (split_index > 0) { + if (gvfs_config_is_set(GVFS_BLOCK_COMMANDS)) + die(_("split index is not supported on a GVFS repo")); + if (git_config_get_split_index() == 0) warning(_("core.splitIndex is set to false; " "remove or change it, if you really want to " diff --git a/git.c b/git.c index 6a9706a1f966d5..e97e74cb5dac9c 100644 --- a/git.c +++ b/git.c @@ -1,6 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "builtin.h" +#include "gvfs.h" #include "config.h" #include "environment.h" #include "exec-cmd.h" @@ -30,6 +31,7 @@ #define NEED_WORK_TREE (1<<3) #define DELAY_PAGER_CONFIG (1<<4) #define NO_PARSEOPT (1<<5) /* parse-options is not used */ +#define BLOCK_ON_GVFS_REPO (1<<6) /* command not allowed in GVFS repos */ struct cmd_struct { const char *cmd; @@ -533,6 +535,9 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv) if (!help && p->option & NEED_WORK_TREE) setup_work_tree(); + if (!help && p->option & BLOCK_ON_GVFS_REPO && gvfs_config_is_set(GVFS_BLOCK_COMMANDS)) + die("'git %s' is not supported on a GVFS repo", p->cmd); + if (run_pre_command_hook(argv)) die("pre-command hook aborted command"); @@ -615,7 +620,7 @@ static struct cmd_struct commands[] = { { "for-each-ref", cmd_for_each_ref, RUN_SETUP }, { "for-each-repo", cmd_for_each_repo, RUN_SETUP_GENTLY }, { "format-patch", cmd_format_patch, RUN_SETUP }, - { "fsck", cmd_fsck, RUN_SETUP }, + { "fsck", cmd_fsck, RUN_SETUP | BLOCK_ON_GVFS_REPO}, { "fsck-objects", cmd_fsck, RUN_SETUP }, { "fsmonitor--daemon", cmd_fsmonitor__daemon, RUN_SETUP }, { "gc", cmd_gc, RUN_SETUP }, @@ -656,7 +661,7 @@ static struct cmd_struct commands[] = { { "pack-refs", cmd_pack_refs, RUN_SETUP }, { "patch-id", cmd_patch_id, RUN_SETUP_GENTLY | NO_PARSEOPT }, { "pickaxe", cmd_blame, RUN_SETUP }, - { "prune", cmd_prune, RUN_SETUP }, + { "prune", cmd_prune, RUN_SETUP | BLOCK_ON_GVFS_REPO}, { "prune-packed", cmd_prune_packed, RUN_SETUP }, { "pull", cmd_pull, RUN_SETUP | NEED_WORK_TREE }, { "push", cmd_push, RUN_SETUP }, @@ -669,7 +674,7 @@ static struct cmd_struct commands[] = { { "remote", cmd_remote, RUN_SETUP }, { "remote-ext", cmd_remote_ext, NO_PARSEOPT }, { "remote-fd", cmd_remote_fd, NO_PARSEOPT }, - { "repack", cmd_repack, RUN_SETUP }, + { "repack", cmd_repack, RUN_SETUP | BLOCK_ON_GVFS_REPO }, { "replace", cmd_replace, RUN_SETUP }, { "replay", cmd_replay, RUN_SETUP }, { "rerere", cmd_rerere, RUN_SETUP }, @@ -690,7 +695,7 @@ static struct cmd_struct commands[] = { { "stash", cmd_stash, RUN_SETUP | NEED_WORK_TREE }, { "status", cmd_status, RUN_SETUP | NEED_WORK_TREE }, { "stripspace", cmd_stripspace }, - { "submodule--helper", cmd_submodule__helper, RUN_SETUP }, + { "submodule--helper", cmd_submodule__helper, RUN_SETUP | BLOCK_ON_GVFS_REPO }, { "survey", cmd_survey, RUN_SETUP }, { "switch", cmd_switch, RUN_SETUP | NEED_WORK_TREE }, { "symbolic-ref", cmd_symbolic_ref, RUN_SETUP }, @@ -709,7 +714,7 @@ static struct cmd_struct commands[] = { { "verify-tag", cmd_verify_tag, RUN_SETUP }, { "version", cmd_version }, { "whatchanged", cmd_whatchanged, RUN_SETUP }, - { "worktree", cmd_worktree, RUN_SETUP }, + { "worktree", cmd_worktree, RUN_SETUP | BLOCK_ON_GVFS_REPO }, { "write-tree", cmd_write_tree, RUN_SETUP }, }; diff --git a/gvfs.h b/gvfs.h index 7c9367866f502a..e193502151467a 100644 --- a/gvfs.h +++ b/gvfs.h @@ -12,6 +12,7 @@ * The list of bits in the core_gvfs setting */ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) +#define GVFS_BLOCK_COMMANDS (1 << 1) #define GVFS_MISSING_OK (1 << 2) #define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) #define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) diff --git a/t/t0402-block-command-on-gvfs.sh b/t/t0402-block-command-on-gvfs.sh new file mode 100755 index 00000000000000..3ec7620ce6194d --- /dev/null +++ b/t/t0402-block-command-on-gvfs.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +test_description='block commands in GVFS repo' + +. ./test-lib.sh + +not_with_gvfs () { + command=$1 && + shift && + test_expect_success "test $command $*" " + test_config alias.g4rbled $command && + test_config core.gvfs true && + test_must_fail git $command $* && + test_must_fail git g4rbled $* && + test_unconfig core.gvfs && + test_must_fail git -c core.gvfs=true $command $* && + test_must_fail git -c core.gvfs=true g4rbled $* + " +} + +not_with_gvfs fsck +not_with_gvfs gc +not_with_gvfs gc --auto +not_with_gvfs prune +not_with_gvfs repack +not_with_gvfs submodule status +not_with_gvfs update-index --index-version 2 +not_with_gvfs update-index --skip-worktree +not_with_gvfs update-index --no-skip-worktree +not_with_gvfs update-index --split-index +not_with_gvfs worktree list + +test_expect_success 'test gc --auto succeeds when disabled via config' ' + test_config core.gvfs true && + test_config gc.auto 0 && + git gc --auto +' + +test_done From 6a341075b09f22f1cce674cae7f6ac72e84773e8 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 30 Sep 2022 12:59:40 -0400 Subject: [PATCH 048/189] worktree: allow in Scalar repositories The 'git worktree' command was marked as BLOCK_ON_GVFS_REPO because it does not interact well with the virtual filesystem of VFS for Git. When a Scalar clone uses the GVFS protocol, it enables the GVFS_BLOCK_COMMANDS flag, since commands like 'git gc' do not work well with the GVFS protocol. However, 'git worktree' works just fine with the GVFS protocol since it isn't doing anything special. It copies the sparse-checkout from the current worktree, so it does not have performance issues. This is a highly requested option. The solution is to stop using the BLOCK_ON_GVFS_REPO option and instead add a special-case check in cmd_worktree() specifically for a particular bit of the 'core_gvfs' global variable (loaded by very early config reading) that corresponds to the virtual filesystem. The bit that most closely resembled this behavior was non-obviously named, but does provide a signal that we are in a Scalar clone and not a VFS for Git clone. The error message is copied from git.c, so it will have the same output as before if a user runs this in a VFS for Git clone. Signed-off-by: Derrick Stolee --- builtin/worktree.c | 8 ++++++++ git.c | 2 +- gvfs.h | 11 +++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/builtin/worktree.c b/builtin/worktree.c index 1d51e54fcdc189..49b8ca37c4f050 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -1,6 +1,7 @@ #include "builtin.h" #include "abspath.h" #include "advice.h" +#include "gvfs.h" #include "checkout.h" #include "config.h" #include "copy.h" @@ -1407,6 +1408,13 @@ int cmd_worktree(int ac, const char **av, const char *prefix) git_config(git_worktree_config, NULL); + /* + * git-worktree is special-cased to work in Scalar repositories + * even when they use the GVFS Protocol. + */ + if (core_gvfs & GVFS_USE_VIRTUAL_FILESYSTEM) + die("'git %s' is not supported on a GVFS repo", "worktree"); + if (!prefix) prefix = ""; diff --git a/git.c b/git.c index e97e74cb5dac9c..a7bce975d2310b 100644 --- a/git.c +++ b/git.c @@ -714,7 +714,7 @@ static struct cmd_struct commands[] = { { "verify-tag", cmd_verify_tag, RUN_SETUP }, { "version", cmd_version }, { "whatchanged", cmd_whatchanged, RUN_SETUP }, - { "worktree", cmd_worktree, RUN_SETUP | BLOCK_ON_GVFS_REPO }, + { "worktree", cmd_worktree, RUN_SETUP }, { "write-tree", cmd_write_tree, RUN_SETUP }, }; diff --git a/gvfs.h b/gvfs.h index e193502151467a..a8e58a6ebc88b8 100644 --- a/gvfs.h +++ b/gvfs.h @@ -14,7 +14,18 @@ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) #define GVFS_BLOCK_COMMANDS (1 << 1) #define GVFS_MISSING_OK (1 << 2) + +/* + * This behavior of not deleting outside of the sparse-checkout + * is specific to the virtual filesystem support. It is only + * enabled by VFS for Git, and so can be used as an indicator + * that we are in a virtualized filesystem environment and not + * in a Scalar environment. This bit has two names to reflect + * that. + */ #define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) +#define GVFS_USE_VIRTUAL_FILESYSTEM (1 << 3) + #define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) #define GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS (1 << 6) From fe61f9e3faaf0acc2276056960ca0bad49510786 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 15 Apr 2020 16:19:31 +0000 Subject: [PATCH 049/189] gvfs: allow overriding core.gvfs We found a user who had set "core.gvfs = false" in their global config. This should not have been necessary, but it also should not have caused a problem. However, it did. The reason is that gvfs_load_config_value() is called from config.c when reading config key/value pairs from all the config files. The local config should override the global config, and this is done by config.c reading the global config first then reading the local config. However, our logic only allowed writing the core_gvfs variable once. Put the guards against multiple assignments of core_gvfs into gvfs_config_is_set() instead, because that will fix the problem _and_ keep multiple calls to gvfs_config_is_set() from slowing down. Signed-off-by: Derrick Stolee --- gvfs.c | 10 ++++------ t/t0021-conversion.sh | 4 ++++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/gvfs.c b/gvfs.c index c65f15ff02584a..a017bf84c737dd 100644 --- a/gvfs.c +++ b/gvfs.c @@ -19,9 +19,6 @@ static int early_core_gvfs_config(const char *var, const char *value, void gvfs_load_config_value(const char *value) { - if (gvfs_config_loaded) - return; - if (value) { struct key_value_info default_kvi = KVI_INIT; core_gvfs = git_config_bool_or_int("core.gvfs", value, &default_kvi, &core_gvfs_is_bool); @@ -34,12 +31,13 @@ void gvfs_load_config_value(const char *value) /* Turn on all bits if a bool was set in the settings */ if (core_gvfs_is_bool && core_gvfs) core_gvfs = -1; - - gvfs_config_loaded = 1; } int gvfs_config_is_set(int mask) { - gvfs_load_config_value(NULL); + if (!gvfs_config_loaded) + gvfs_load_config_value(NULL); + + gvfs_config_loaded = 1; return (core_gvfs & mask) == mask; } diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh index e7eccfaccb9a36..110c741ba71935 100755 --- a/t/t0021-conversion.sh +++ b/t/t0021-conversion.sh @@ -349,6 +349,10 @@ test_expect_success "filter: smudge filters blocked when under GVFS" ' test_config filter.empty-in-repo.smudge "echo smudged && cat" && test_config core.gvfs 64 && + test_must_fail git checkout && + + # ensure the local core.gvfs setting overwrites the global setting + git config --global core.gvfs false && test_must_fail git checkout ' From a833057221075d961c1eb83d6d2f40d749e27559 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 11 Jan 2018 16:25:08 -0500 Subject: [PATCH 050/189] Add virtual file system settings and hook proc On index load, clear/set the skip worktree bits based on the virtual file system data. Use virtual file system data to update skip-worktree bit in unpack-trees. Use virtual file system data to exclude files and folders not explicitly requested. Update 2022-04-05: disable the "present-despite-SKIP_WORKTREE" file removal behavior when 'core.virtualfilesystem' is enabled. Signed-off-by: Ben Peart --- Documentation/config/core.txt | 8 + Documentation/githooks.txt | 20 ++ Makefile | 1 + config.c | 30 ++- config.h | 1 + dir.c | 36 ++- environment.c | 1 + environment.h | 1 + read-cache.c | 2 + sparse-index.c | 5 +- t/t1090-sparse-checkout-scope.sh | 4 +- t/t1093-virtualfilesystem.sh | 369 +++++++++++++++++++++++++++++++ unpack-trees.c | 14 +- virtualfilesystem.c | 312 ++++++++++++++++++++++++++ virtualfilesystem.h | 25 +++ wt-status.c | 2 + 16 files changed, 823 insertions(+), 8 deletions(-) create mode 100755 t/t1093-virtualfilesystem.sh create mode 100644 virtualfilesystem.c create mode 100644 virtualfilesystem.h diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 2fd2a10f51ad2c..50e4146af1d24e 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -111,6 +111,14 @@ Version 2 uses an opaque string so that the monitor can return something that can be used to determine what files have changed without race conditions. +core.virtualFilesystem:: + If set, the value of this variable is used as a command which + will identify all files and directories that are present in + the working directory. Git will only track and update files + listed in the virtual file system. Using the virtual file system + will supersede the sparse-checkout settings which will be ignored. + See the "virtual file system" section of linkgit:githooks[5]. + core.trustctime:: If false, the ctime differences between the index and the working tree are ignored; useful when the inode change time diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt index 0397dec64d7315..86c78b3b4825e2 100644 --- a/Documentation/githooks.txt +++ b/Documentation/githooks.txt @@ -758,6 +758,26 @@ and "0" meaning they were not. Only one parameter should be set to "1" when the hook runs. The hook running passing "1", "1" should not be possible. +virtualFilesystem +~~~~~~~~~~~~~~~~~~ + +"Virtual File System" allows populating the working directory sparsely. +The projection data is typically automatically generated by an external +process. Git will limit what files it checks for changes as well as which +directories are checked for untracked files based on the path names given. +Git will also only update those files listed in the projection. + +The hook is invoked when the configuration option core.virtualFilesystem +is set. It takes one argument, a version (currently 1). + +The hook should output to stdout the list of all files in the working +directory that git should track. The paths are relative to the root +of the working directory and are separated by a single NUL. Full paths +('dir1/a.txt') as well as directories are supported (ie 'dir1/'). + +The exit status determines whether git will use the data from the +hook. On error, git will abort the command with an error message. + SEE ALSO -------- linkgit:git-hook[1] diff --git a/Makefile b/Makefile index 24811a49aa107d..a18b18bd17e687 100644 --- a/Makefile +++ b/Makefile @@ -1191,6 +1191,7 @@ LIB_OBJS += utf8.o LIB_OBJS += varint.o LIB_OBJS += version.o LIB_OBJS += versioncmp.o +LIB_OBJS += virtualfilesystem.o LIB_OBJS += walker.o LIB_OBJS += wildmatch.o LIB_OBJS += worktree.o diff --git a/config.c b/config.c index 329b7384f35bbe..e69ea6f71323bb 100644 --- a/config.c +++ b/config.c @@ -1672,7 +1672,11 @@ int git_default_core_config(const char *var, const char *value, } if (!strcmp(var, "core.sparsecheckout")) { - core_apply_sparse_checkout = git_config_bool(var, value); + /* virtual file system relies on the sparse checkout logic so force it on */ + if (core_virtualfilesystem) + core_apply_sparse_checkout = 1; + else + core_apply_sparse_checkout = git_config_bool(var, value); return 0; } @@ -2837,6 +2841,30 @@ int git_config_get_max_percent_split_change(void) return -1; /* default value */ } +int git_config_get_virtualfilesystem(void) +{ + /* Run only once. */ + static int virtual_filesystem_result = -1; + if (virtual_filesystem_result >= 0) + return virtual_filesystem_result; + + if (git_config_get_pathname("core.virtualfilesystem", &core_virtualfilesystem)) + core_virtualfilesystem = getenv("GIT_VIRTUALFILESYSTEM_TEST"); + + if (core_virtualfilesystem && !*core_virtualfilesystem) + core_virtualfilesystem = NULL; + + /* virtual file system relies on the sparse checkout logic so force it on */ + if (core_virtualfilesystem) { + core_apply_sparse_checkout = 1; + virtual_filesystem_result = 1; + return 1; + } + + virtual_filesystem_result = 0; + return 0; +} + int git_config_get_index_threads(int *dest) { int is_bool, val; diff --git a/config.h b/config.h index 418384646b04d3..f3ad29834154b9 100644 --- a/config.h +++ b/config.h @@ -701,6 +701,7 @@ int git_config_get_pathname(const char *key, char **dest); int git_config_get_index_threads(int *dest); int git_config_get_split_index(void); int git_config_get_max_percent_split_change(void); +int git_config_get_virtualfilesystem(void); /* This dies if the configured or default date is in the future */ int git_config_get_expiry(const char *key, const char **output); diff --git a/dir.c b/dir.c index 837c812a149e08..0c2b2039791361 100644 --- a/dir.c +++ b/dir.c @@ -10,6 +10,7 @@ #include "git-compat-util.h" #include "abspath.h" +#include "virtualfilesystem.h" #include "config.h" #include "convert.h" #include "dir.h" @@ -1478,6 +1479,19 @@ enum pattern_match_result path_matches_pattern_list( int result = NOT_MATCHED; size_t slash_pos; + if (core_virtualfilesystem) { + /* + * The virtual file system data is used to prevent git from traversing + * any part of the tree that is not in the virtual file system. Return + * 1 to exclude the entry if it is not found in the virtual file system, + * else fall through to the regular excludes logic as it may further exclude. + */ + if (*dtype == DT_UNKNOWN) + *dtype = resolve_dtype(DT_UNKNOWN, istate, pathname, pathlen); + if (is_excluded_from_virtualfilesystem(pathname, pathlen, *dtype) > 0) + return 1; + } + if (!pl->use_cone_patterns) { pattern = last_matching_pattern_from_list(pathname, pathlen, basename, dtype, pl, istate); @@ -1822,8 +1836,22 @@ struct path_pattern *last_matching_pattern(struct dir_struct *dir, int is_excluded(struct dir_struct *dir, struct index_state *istate, const char *pathname, int *dtype_p) { - struct path_pattern *pattern = - last_matching_pattern(dir, istate, pathname, dtype_p); + struct path_pattern *pattern; + + if (core_virtualfilesystem) { + /* + * The virtual file system data is used to prevent git from traversing + * any part of the tree that is not in the virtual file system. Return + * 1 to exclude the entry if it is not found in the virtual file system, + * else fall through to the regular excludes logic as it may further exclude. + */ + if (*dtype_p == DT_UNKNOWN) + *dtype_p = resolve_dtype(DT_UNKNOWN, istate, pathname, strlen(pathname)); + if (is_excluded_from_virtualfilesystem(pathname, strlen(pathname), *dtype_p) > 0) + return 1; + } + + pattern = last_matching_pattern(dir, istate, pathname, dtype_p); if (pattern) return pattern->flags & PATTERN_FLAG_NEGATIVE ? 0 : 1; return 0; @@ -2443,6 +2471,8 @@ static enum path_treatment treat_path(struct dir_struct *dir, ignore_case); if (dtype != DT_DIR && has_path_in_index) return path_none; + if (is_excluded_from_virtualfilesystem(path->buf, path->len, dtype) > 0) + return path_excluded; /* * When we are looking at a directory P in the working tree, @@ -2647,6 +2677,8 @@ static void add_path_to_appropriate_result_list(struct dir_struct *dir, /* add the path to the appropriate result list */ switch (state) { case path_excluded: + if (is_excluded_from_virtualfilesystem(path->buf, path->len, DT_DIR) > 0) + break; if (dir->flags & DIR_SHOW_IGNORED) dir_add_name(dir, istate, path->buf, path->len); else if ((dir->flags & DIR_SHOW_IGNORED_TOO) || diff --git a/environment.c b/environment.c index 01c6963183da3f..3948d42e47d60c 100644 --- a/environment.c +++ b/environment.c @@ -81,6 +81,7 @@ int core_apply_sparse_checkout; int core_sparse_checkout_cone; int sparse_expect_files_outside_of_patterns; int core_gvfs; +char *core_virtualfilesystem; int merge_log_config = -1; int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ unsigned long pack_size_limit_cfg; diff --git a/environment.h b/environment.h index 881f8947ba8e19..df3f51e85dcc9e 100644 --- a/environment.h +++ b/environment.h @@ -155,6 +155,7 @@ int get_shared_repository(void); void reset_shared_repository(void); extern int core_preload_index; +extern char *core_virtualfilesystem; extern int core_gvfs; extern int precomposed_unicode; extern int protect_hfs; diff --git a/read-cache.c b/read-cache.c index 4699180724c0f2..4ab2df30bffbd8 100644 --- a/read-cache.c +++ b/read-cache.c @@ -8,6 +8,7 @@ #include "git-compat-util.h" #include "bulk-checkin.h" +#include "virtualfilesystem.h" #include "config.h" #include "date.h" #include "diff.h" @@ -1978,6 +1979,7 @@ static void post_read_index_from(struct index_state *istate) tweak_untracked_cache(istate); tweak_split_index(istate); tweak_fsmonitor(istate); + apply_virtualfilesystem(istate); } static size_t estimate_cache_size_from_compressed(unsigned int entries) diff --git a/sparse-index.c b/sparse-index.c index e1cc61f80eb404..fe369df3e53f76 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -264,7 +264,7 @@ static int add_path_to_index(const struct object_id *oid, size_t len = base->len; if (S_ISDIR(mode)) { - int dtype; + int dtype = DT_DIR; size_t baselen = base->len; if (!ctx->pl) return READ_TREE_RECURSIVE; @@ -388,7 +388,7 @@ void expand_index(struct index_state *istate, struct pattern_list *pl) struct cache_entry *ce = istate->cache[i]; struct tree *tree; struct pathspec ps; - int dtype; + int dtype = DT_UNKNOWN; if (!S_ISSPARSEDIR(ce->ce_mode)) { set_index_entry(full, full->cache_nr++, ce); @@ -663,6 +663,7 @@ static void clear_skip_worktree_from_present_files_full(struct index_state *ista void clear_skip_worktree_from_present_files(struct index_state *istate) { if (!core_apply_sparse_checkout || + core_virtualfilesystem || sparse_expect_files_outside_of_patterns) return; diff --git a/t/t1090-sparse-checkout-scope.sh b/t/t1090-sparse-checkout-scope.sh index 7af01dc00b75ee..1433e7e1fcbb58 100755 --- a/t/t1090-sparse-checkout-scope.sh +++ b/t/t1090-sparse-checkout-scope.sh @@ -108,9 +108,9 @@ test_expect_success 'in partial clone, sparse checkout only fetches needed blobs ' test_expect_success 'checkout does not delete items outside the sparse checkout file' ' - # The "sparse.expectfilesoutsideofpatterns" config will prevent the + # The "core.virtualfilesystem" config will prevent the # SKIP_WORKTREE flag from being dropped on files present on-disk. - test_config sparse.expectfilesoutsideofpatterns true && + test_config core.virtualfilesystem true && test_config core.gvfs 8 && git checkout -b outside && diff --git a/t/t1093-virtualfilesystem.sh b/t/t1093-virtualfilesystem.sh new file mode 100755 index 00000000000000..bd0c9f72ba3c4a --- /dev/null +++ b/t/t1093-virtualfilesystem.sh @@ -0,0 +1,369 @@ +#!/bin/sh + +test_description='virtual file system tests' + +. ./test-lib.sh + +clean_repo () { + rm .git/index && + git -c core.virtualfilesystem= reset --hard HEAD && + git -c core.virtualfilesystem= clean -fd && + touch untracked.txt && + touch dir1/untracked.txt && + touch dir2/untracked.txt +} + +test_expect_success 'setup' ' + git branch -M main && + mkdir -p .git/hooks/ && + cat > .gitignore <<-\EOF && + .gitignore + expect* + actual* + EOF + mkdir -p dir1 && + touch dir1/file1.txt && + touch dir1/file2.txt && + mkdir -p dir2 && + touch dir2/file1.txt && + touch dir2/file2.txt && + git add . && + git commit -m "initial" && + git config --local core.virtualfilesystem .git/hooks/virtualfilesystem +' + +test_expect_success 'test hook parameters and version' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + if test "$#" -ne 1 + then + echo "$0: Exactly 1 argument expected" >&2 + exit 2 + fi + + if test "$1" != 1 + then + echo "$0: Unsupported hook version." >&2 + exit 1 + fi + EOF + git status && + write_script .git/hooks/virtualfilesystem <<-\EOF && + exit 3 + EOF + test_must_fail git status +' + +test_expect_success 'verify status is clean' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir2/file1.txt\0" + EOF + rm -f .git/index && + git checkout -f && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir2/file1.txt\0" + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + EOF + git status > actual && + cat > expected <<-\EOF && + On branch main + nothing to commit, working tree clean + EOF + test_cmp expected actual +' + +test_expect_success 'verify skip-worktree bit is set for absolute path' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + EOF + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/file1.txt + S dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify skip-worktree bit is cleared for absolute path' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file2.txt\0" + EOF + git ls-files -v > actual && + cat > expected <<-\EOF && + S dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify folder wild cards' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + EOF + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify folders not included are ignored' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git add . && + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify including one file doesnt include the rest' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + printf "dir1/dir2/a\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git add . && + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1/dir2/a + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify files not listed are ignored by git clean -f -x' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "untracked.txt\0" + printf "dir1/\0" + EOF + mkdir -p dir3 && + touch dir3/untracked.txt && + git clean -f -x && + test ! -f untracked.txt && + test -d dir1 && + test -f dir1/file1.txt && + test -f dir1/file2.txt && + test ! -f dir1/untracked.txt && + test -f dir2/file1.txt && + test -f dir2/file2.txt && + test -f dir2/untracked.txt && + test -d dir3 && + test -f dir3/untracked.txt +' + +test_expect_success 'verify files not listed are ignored by git clean -f -d -x' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "untracked.txt\0" + printf "dir1/\0" + printf "dir3/\0" + EOF + mkdir -p dir3 && + touch dir3/untracked.txt && + git clean -f -d -x && + test ! -f untracked.txt && + test -d dir1 && + test -f dir1/file1.txt && + test -f dir1/file2.txt && + test ! -f dir1/untracked.txt && + test -f dir2/file1.txt && + test -f dir2/file2.txt && + test -f dir2/untracked.txt && + test ! -d dir3 && + test ! -f dir3/untracked.txt +' + +test_expect_success 'verify folder entries include all files' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git status -su > actual && + cat > expected <<-\EOF && + ?? dir1/a + ?? dir1/b + ?? dir1/untracked.txt + EOF + test_cmp expected actual +' + +test_expect_success 'verify case insensitivity of virtual file system entries' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/a\0" + printf "Dir1/Dir2/a\0" + printf "DIR2/\0" + EOF + mkdir -p dir1/dir2 && + touch dir1/a && + touch dir1/b && + touch dir1/dir2/a && + touch dir1/dir2/b && + git -c core.ignorecase=false status -su > actual && + cat > expected <<-\EOF && + ?? dir1/a + EOF + test_cmp expected actual && + git -c core.ignorecase=true status -su > actual && + cat > expected <<-\EOF && + ?? dir1/a + ?? dir1/dir2/a + ?? dir2/untracked.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file created' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file3.txt\0" + EOF + touch dir1/file3.txt && + git add . && + git ls-files -v > actual && + cat > expected <<-\EOF && + S dir1/file1.txt + S dir1/file2.txt + H dir1/file3.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file renamed' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + printf "dir1/file3.txt\0" + EOF + mv dir1/file1.txt dir1/file3.txt && + git status -su > actual && + cat > expected <<-\EOF && + D dir1/file1.txt + ?? dir1/file3.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file deleted' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + EOF + rm dir1/file1.txt && + git status -su > actual && + cat > expected <<-\EOF && + D dir1/file1.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on file overwritten' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/file1.txt\0" + EOF + echo "overwritten" > dir1/file1.txt && + git status -su > actual && + cat > expected <<-\EOF && + M dir1/file1.txt + EOF + test_cmp expected actual +' + +test_expect_success 'on folder created' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/dir1/\0" + EOF + mkdir -p dir1/dir1 && + git status -su > actual && + cat > expected <<-\EOF && + EOF + test_cmp expected actual && + git clean -fd && + test ! -d "/dir1/dir1" +' + +test_expect_success 'on folder renamed' ' + clean_repo && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir3/\0" + printf "dir1/file1.txt\0" + printf "dir1/file2.txt\0" + printf "dir3/file1.txt\0" + printf "dir3/file2.txt\0" + EOF + mv dir1 dir3 && + git status -su > actual && + cat > expected <<-\EOF && + D dir1/file1.txt + D dir1/file2.txt + ?? dir3/file1.txt + ?? dir3/file2.txt + ?? dir3/untracked.txt + EOF + test_cmp expected actual +' + +test_expect_success 'folder with same prefix as file' ' + clean_repo && + touch dir1.sln && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + printf "dir1.sln\0" + EOF + git add dir1.sln && + git ls-files -v > actual && + cat > expected <<-\EOF && + H dir1.sln + H dir1/file1.txt + H dir1/file2.txt + S dir2/file1.txt + S dir2/file2.txt + EOF + test_cmp expected actual +' + +test_done diff --git a/unpack-trees.c b/unpack-trees.c index 8babaac11728ef..883a5afbb49b3c 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "advice.h" #include "gvfs.h" +#include "virtualfilesystem.h" #include "strvec.h" #include "repository.h" #include "parse.h" @@ -1702,6 +1703,14 @@ static int clear_ce_flags_1(struct index_state *istate, continue; } + /* if it's not in the virtual file system, exit early */ + if (core_virtualfilesystem) { + if (is_included_in_virtualfilesystem(ce->name, ce->ce_namelen) > 0) + ce->ce_flags &= ~clear_mask; + cache++; + continue; + } + if (prefix->len && strncmp(ce->name, prefix->buf, prefix->len)) break; @@ -1928,7 +1937,10 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options if (!o->skip_sparse_checkout) { memset(&pl, 0, sizeof(pl)); free_pattern_list = 1; - populate_from_existing_patterns(o, &pl); + if (core_virtualfilesystem) + o->internal.pl = &pl; + else + populate_from_existing_patterns(o, &pl); } index_state_init(&o->internal.result, o->src_index->repo); diff --git a/virtualfilesystem.c b/virtualfilesystem.c new file mode 100644 index 00000000000000..6ff3a0ca2a340d --- /dev/null +++ b/virtualfilesystem.c @@ -0,0 +1,312 @@ +#include "git-compat-util.h" +#include "environment.h" +#include "gettext.h" +#include "config.h" +#include "dir.h" +#include "hashmap.h" +#include "run-command.h" +#include "name-hash.h" +#include "read-cache-ll.h" +#include "virtualfilesystem.h" + +#define HOOK_INTERFACE_VERSION (1) + +static struct strbuf virtual_filesystem_data = STRBUF_INIT; +static struct hashmap virtual_filesystem_hashmap; +static struct hashmap parent_directory_hashmap; + +struct virtualfilesystem { + struct hashmap_entry ent; /* must be the first member! */ + const char *pattern; + int patternlen; +}; + +static unsigned int(*vfshash)(const void *buf, size_t len); +static int(*vfscmp)(const char *a, const char *b, size_t len); + +static int vfs_hashmap_cmp(const void *unused_cmp_data, + const struct hashmap_entry *he1, + const struct hashmap_entry *he2, + const void *key) +{ + const struct virtualfilesystem *vfs1 = + container_of(he1, const struct virtualfilesystem, ent); + const struct virtualfilesystem *vfs2 = + container_of(he2, const struct virtualfilesystem, ent); + + return vfscmp(vfs1->pattern, vfs2->pattern, vfs1->patternlen); +} + +static void get_virtual_filesystem_data(struct strbuf *vfs_data) +{ + struct child_process cp = CHILD_PROCESS_INIT; + int err; + + strbuf_init(vfs_data, 0); + + strvec_push(&cp.args, core_virtualfilesystem); + strvec_pushf(&cp.args, "%d", HOOK_INTERFACE_VERSION); + cp.use_shell = 1; + cp.dir = get_git_work_tree(); + + err = capture_command(&cp, vfs_data, 1024); + if (err) + die("unable to load virtual file system"); +} + +static int check_includes_hashmap(struct hashmap *map, const char *pattern, int patternlen) +{ + struct strbuf sb = STRBUF_INIT; + struct virtualfilesystem vfs; + char *slash; + + /* Check straight mapping */ + strbuf_reset(&sb); + strbuf_add(&sb, pattern, patternlen); + vfs.pattern = sb.buf; + vfs.patternlen = sb.len; + hashmap_entry_init(&vfs.ent, vfshash(vfs.pattern, vfs.patternlen)); + if (hashmap_get_entry(map, &vfs, ent, NULL)) { + strbuf_release(&sb); + return 1; + } + + /* + * Check to see if it matches a directory or any path + * underneath it. In other words, 'a/b/foo.txt' will match + * '/', 'a/', and 'a/b/'. + */ + slash = strchr(sb.buf, '/'); + while (slash) { + vfs.pattern = sb.buf; + vfs.patternlen = slash - sb.buf + 1; + hashmap_entry_init(&vfs.ent, vfshash(vfs.pattern, vfs.patternlen)); + if (hashmap_get_entry(map, &vfs, ent, NULL)) { + strbuf_release(&sb); + return 1; + } + slash = strchr(slash + 1, '/'); + } + + strbuf_release(&sb); + return 0; +} + +static void includes_hashmap_add(struct hashmap *map, const char *pattern, const int patternlen) +{ + struct virtualfilesystem *vfs; + + vfs = xmalloc(sizeof(struct virtualfilesystem)); + vfs->pattern = pattern; + vfs->patternlen = patternlen; + hashmap_entry_init(&vfs->ent, vfshash(vfs->pattern, vfs->patternlen)); + hashmap_add(map, &vfs->ent); +} + +static void initialize_includes_hashmap(struct hashmap *map, struct strbuf *vfs_data) +{ + char *buf, *entry; + size_t len; + int i; + + /* + * Build a hashmap of the virtual file system data we can use to look + * for cache entry matches quickly + */ + vfshash = ignore_case ? memihash : memhash; + vfscmp = ignore_case ? strncasecmp : strncmp; + hashmap_init(map, vfs_hashmap_cmp, NULL, 0); + + entry = buf = vfs_data->buf; + len = vfs_data->len; + for (i = 0; i < len; i++) { + if (buf[i] == '\0') { + includes_hashmap_add(map, entry, buf + i - entry); + entry = buf + i + 1; + } + } +} + +/* + * Return 1 if the requested item is found in the virtual file system, + * 0 for not found and -1 for undecided. + */ +int is_included_in_virtualfilesystem(const char *pathname, int pathlen) +{ + if (!core_virtualfilesystem) + return -1; + + if (!virtual_filesystem_hashmap.tablesize && virtual_filesystem_data.len) + initialize_includes_hashmap(&virtual_filesystem_hashmap, &virtual_filesystem_data); + if (!virtual_filesystem_hashmap.tablesize) + return -1; + + return check_includes_hashmap(&virtual_filesystem_hashmap, pathname, pathlen); +} + +static void parent_directory_hashmap_add(struct hashmap *map, const char *pattern, const int patternlen) +{ + char *slash; + struct virtualfilesystem *vfs; + + /* + * Add any directories leading up to the file as the excludes logic + * needs to match directories leading up to the files as well. Detect + * and prevent unnecessary duplicate entries which will be common. + */ + if (patternlen > 1) { + slash = strchr(pattern + 1, '/'); + while (slash) { + vfs = xmalloc(sizeof(struct virtualfilesystem)); + vfs->pattern = pattern; + vfs->patternlen = slash - pattern + 1; + hashmap_entry_init(&vfs->ent, vfshash(vfs->pattern, vfs->patternlen)); + if (hashmap_get_entry(map, vfs, ent, NULL)) + free(vfs); + else + hashmap_add(map, &vfs->ent); + slash = strchr(slash + 1, '/'); + } + } +} + +static void initialize_parent_directory_hashmap(struct hashmap *map, struct strbuf *vfs_data) +{ + char *buf, *entry; + size_t len; + int i; + + /* + * Build a hashmap of the parent directories contained in the virtual + * file system data we can use to look for matches quickly + */ + vfshash = ignore_case ? memihash : memhash; + vfscmp = ignore_case ? strncasecmp : strncmp; + hashmap_init(map, vfs_hashmap_cmp, NULL, 0); + + entry = buf = vfs_data->buf; + len = vfs_data->len; + for (i = 0; i < len; i++) { + if (buf[i] == '\0') { + parent_directory_hashmap_add(map, entry, buf + i - entry); + entry = buf + i + 1; + } + } +} + +static int check_directory_hashmap(struct hashmap *map, const char *pathname, int pathlen) +{ + struct strbuf sb = STRBUF_INIT; + struct virtualfilesystem vfs; + + /* Check for directory */ + strbuf_reset(&sb); + strbuf_add(&sb, pathname, pathlen); + strbuf_addch(&sb, '/'); + vfs.pattern = sb.buf; + vfs.patternlen = sb.len; + hashmap_entry_init(&vfs.ent, vfshash(vfs.pattern, vfs.patternlen)); + if (hashmap_get_entry(map, &vfs, ent, NULL)) { + strbuf_release(&sb); + return 0; + } + + strbuf_release(&sb); + return 1; +} + +/* + * Return 1 for exclude, 0 for include and -1 for undecided. + */ +int is_excluded_from_virtualfilesystem(const char *pathname, int pathlen, int dtype) +{ + if (!core_virtualfilesystem) + return -1; + + if (dtype != DT_REG && dtype != DT_DIR && dtype != DT_LNK) + die(_("is_excluded_from_virtualfilesystem passed unhandled dtype")); + + if (dtype == DT_REG || dtype == DT_LNK) { + int ret = is_included_in_virtualfilesystem(pathname, pathlen); + if (ret > 0) + return 0; + if (ret == 0) + return 1; + return ret; + } + + if (dtype == DT_DIR) { + if (!parent_directory_hashmap.tablesize && virtual_filesystem_data.len) + initialize_parent_directory_hashmap(&parent_directory_hashmap, &virtual_filesystem_data); + if (!parent_directory_hashmap.tablesize) + return -1; + + return check_directory_hashmap(&parent_directory_hashmap, pathname, pathlen); + } + + return -1; +} + +/* + * Update the CE_SKIP_WORKTREE bits based on the virtual file system. + */ +void apply_virtualfilesystem(struct index_state *istate) +{ + char *buf, *entry; + int i; + + if (!git_config_get_virtualfilesystem()) + return; + + if (!virtual_filesystem_data.len) + get_virtual_filesystem_data(&virtual_filesystem_data); + + /* set CE_SKIP_WORKTREE bit on all entries */ + for (i = 0; i < istate->cache_nr; i++) + istate->cache[i]->ce_flags |= CE_SKIP_WORKTREE; + + /* clear CE_SKIP_WORKTREE bit for everything in the virtual file system */ + entry = buf = virtual_filesystem_data.buf; + for (i = 0; i < virtual_filesystem_data.len; i++) { + if (buf[i] == '\0') { + int pos, len; + + len = buf + i - entry; + + /* look for a directory wild card (ie "dir1/") */ + if (buf[i - 1] == '/') { + if (ignore_case) + adjust_dirname_case(istate, entry); + pos = index_name_pos(istate, entry, len); + if (pos < 0) { + pos = -pos - 1; + while (pos < istate->cache_nr && !fspathncmp(istate->cache[pos]->name, entry, len)) { + istate->cache[pos]->ce_flags &= ~CE_SKIP_WORKTREE; + pos++; + } + } + } else { + if (ignore_case) { + struct cache_entry *ce = index_file_exists(istate, entry, len, ignore_case); + if (ce) + ce->ce_flags &= ~CE_SKIP_WORKTREE; + } else { + int pos = index_name_pos(istate, entry, len); + if (pos >= 0) + istate->cache[pos]->ce_flags &= ~CE_SKIP_WORKTREE; + } + } + + entry += len + 1; + } + } +} + +/* + * Free the virtual file system data structures. + */ +void free_virtualfilesystem(void) { + hashmap_clear_and_free(&virtual_filesystem_hashmap, struct virtualfilesystem, ent); + hashmap_clear_and_free(&parent_directory_hashmap, struct virtualfilesystem, ent); + strbuf_release(&virtual_filesystem_data); +} diff --git a/virtualfilesystem.h b/virtualfilesystem.h new file mode 100644 index 00000000000000..5e8c5b096df09a --- /dev/null +++ b/virtualfilesystem.h @@ -0,0 +1,25 @@ +#ifndef VIRTUALFILESYSTEM_H +#define VIRTUALFILESYSTEM_H + +/* + * Update the CE_SKIP_WORKTREE bits based on the virtual file system. + */ +void apply_virtualfilesystem(struct index_state *istate); + +/* + * Return 1 if the requested item is found in the virtual file system, + * 0 for not found and -1 for undecided. + */ +int is_included_in_virtualfilesystem(const char *pathname, int pathlen); + +/* + * Return 1 for exclude, 0 for include and -1 for undecided. + */ +int is_excluded_from_virtualfilesystem(const char *pathname, int pathlen, int dtype); + +/* + * Free the virtual file system data structures. + */ +void free_virtualfilesystem(void); + +#endif diff --git a/wt-status.c b/wt-status.c index e9b09675822440..214ebcae18afb4 100644 --- a/wt-status.c +++ b/wt-status.c @@ -1606,6 +1606,8 @@ static void show_sparse_checkout_in_use(struct wt_status *s, { if (s->state.sparse_checkout_percentage == SPARSE_CHECKOUT_DISABLED) return; + if (core_virtualfilesystem) + return; if (s->state.sparse_checkout_percentage == SPARSE_CHECKOUT_SPARSE_INDEX) status_printf_ln(s, color, _("You are in a sparse checkout.")); From 234684e37a4e88f7ff2e74e655b44259d1cb9f89 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 27 Jul 2018 12:00:44 -0600 Subject: [PATCH 051/189] BRANCHES.md: Add explanation of branches and using forks --- BRANCHES.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 BRANCHES.md diff --git a/BRANCHES.md b/BRANCHES.md new file mode 100644 index 00000000000000..364158375e7d55 --- /dev/null +++ b/BRANCHES.md @@ -0,0 +1,59 @@ +Branches used in this repo +========================== + +The document explains the branching structure that we are using in the VFSForGit repository as well as the forking strategy that we have adopted for contributing. + +Repo Branches +------------- + +1. `vfs-#` + + These branches are used to track the specific version that match Git for Windows with the VFSForGit specific patches on top. When a new version of Git for Windows is released, the VFSForGit patches will be rebased on that windows version and a new gvfs-# branch created to create pull requests against. + + #### Examples + + ``` + vfs-2.27.0 + vfs-2.30.0 + ``` + + The versions of git for VFSForGit are based on the Git for Windows versions. v2.20.0.vfs.1 will correspond with the v2.20.0.windows.1 with the VFSForGit specific patches applied to the windows version. + +2. `vfs-#-exp` + + These branches are for releasing experimental features to early adopters. They + should contain everything within the corresponding `vfs-#` branch; if the base + branch updates, then merge into the `vfs-#-exp` branch as well. + +Tags +---- + +We are using annotated tags to build the version number for git. The build will look back through the commit history to find the first tag matching `v[0-9]*vfs*` and build the git version number using that tag. + +Full releases are of the form `v2.XX.Y.vfs.Z.W` where `v2.XX.Y` comes from the +upstream version and `Z.W` are custom updates within our fork. Specifically, +the `.Z` value represents the "compatibility level" with VFS for Git. Only +increase this version when making a breaking change with a released version +of VFS for Git. The `.W` version is used for minor updates between major +versions. + +Experimental releases are of the form `v2.XX.Y.vfs.Z.W.exp`. The `.exp` +suffix indicates that experimental features are available. The rest of the +version string comes from the full release tag. These versions will only +be made available as pre-releases on the releases page, never a full release. + +Forking +------- + +A personal fork of this repository and a branch in that repository should be used for development. + +These branches should be based on the latest vfs-# branch. If there are work in progress pull requests that you have based on a previous version branch when a new version branch is created, you will need to move your patches to the new branch to get them in that latest version. + +#### Example + +``` +git clone +git remote add ms https://github.com/Microsoft/git.git +git checkout -b my-changes ms/vfs-2.20.0 --no-track +git push -fu origin HEAD +``` From 487969df15fa4310cf836a7e2facd4f70daf9d80 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Wed, 1 Aug 2018 13:26:22 -0400 Subject: [PATCH 052/189] virtualfilesystem: don't run the virtual file system hook if the index has been redirected Fixes #13 Some git commands spawn helpers and redirect the index to a different location. These include "difftool -d" and the sequencer (i.e. `git rebase -i`, `git cherry-pick` and `git revert`) and others. In those instances we don't want to update their temporary index with our virtualization data. Helped-by: Johannes Schindelin Signed-off-by: Ben Peart --- config.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/config.c b/config.c index e69ea6f71323bb..408039761f31f0 100644 --- a/config.c +++ b/config.c @@ -2854,11 +2854,25 @@ int git_config_get_virtualfilesystem(void) if (core_virtualfilesystem && !*core_virtualfilesystem) core_virtualfilesystem = NULL; - /* virtual file system relies on the sparse checkout logic so force it on */ if (core_virtualfilesystem) { - core_apply_sparse_checkout = 1; - virtual_filesystem_result = 1; - return 1; + /* + * Some git commands spawn helpers and redirect the index to a different + * location. These include "difftool -d" and the sequencer + * (i.e. `git rebase -i`, `git cherry-pick` and `git revert`) and others. + * In those instances we don't want to update their temporary index with + * our virtualization data. + */ + char *default_index_file = xstrfmt("%s/%s", the_repository->gitdir, "index"); + int should_run_hook = !strcmp(default_index_file, the_repository->index_file); + + free(default_index_file); + if (should_run_hook) { + /* virtual file system relies on the sparse checkout logic so force it on */ + core_apply_sparse_checkout = 1; + virtual_filesystem_result = 1; + return 1; + } + core_virtualfilesystem = NULL; } virtual_filesystem_result = 0; From 5e9b79711eeefac0164e5d01f81b73466f1f586f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 22 Aug 2017 11:54:23 -0400 Subject: [PATCH 053/189] status: add status serialization mechanism Teach STATUS to optionally serialize the results of a status computation to a file. Teach STATUS to optionally read an existing serialization file and simply print the results, rather than actually scanning. This is intended for immediate status results on extremely large repos and assumes the use of a service/daemon to maintain a fresh current status snapshot. 2021-10-30: packet_read() changed its prototype in ec9a37d (pkt-line.[ch]: remove unused packet_read_line_buf(), 2021-10-14). 2021-10-30: sscanf() now does an extra check that "%d" goes into an "int" and complains about "uint32_t". Replacing with "%u" fixes the compile-time error. 2021-10-30: string_list_init() was removed by abf897b (string-list.[ch]: remove string_list_init() compatibility function, 2021-09-28), so we need to initialize manually. Signed-off-by: Jeff Hostetler Signed-off-by: Derrick Stolee --- Documentation/config/status.txt | 6 + Documentation/git-status.txt | 33 + .../technical/status-serialization-format.txt | 107 +++ Makefile | 2 + builtin/commit.c | 123 +++- contrib/completion/git-completion.bash | 2 +- pkt-line.c | 2 +- pkt-line.h | 1 + t/t7522-serialized-status.sh | 141 ++++ t/t7523-status-complete-untracked.sh | 39 ++ wt-status-deserialize.c | 609 ++++++++++++++++++ wt-status-serialize.c | 219 +++++++ wt-status.c | 6 + wt-status.h | 53 +- 14 files changed, 1339 insertions(+), 4 deletions(-) create mode 100644 Documentation/technical/status-serialization-format.txt create mode 100755 t/t7522-serialized-status.sh create mode 100755 t/t7523-status-complete-untracked.sh create mode 100644 wt-status-deserialize.c create mode 100644 wt-status-serialize.c diff --git a/Documentation/config/status.txt b/Documentation/config/status.txt index 8caf90f51c19a3..7302b066644e73 100644 --- a/Documentation/config/status.txt +++ b/Documentation/config/status.txt @@ -77,3 +77,9 @@ status.submoduleSummary:: the --ignore-submodules=dirty command-line option or the 'git submodule summary' command, which shows a similar output but does not honor these settings. + +status.deserializePath:: + EXPERIMENTAL, Pathname to a file containing cached status results + generated by `--serialize`. This will be overridden by + `--deserialize=` on the command line. If the cache file is + invalid or stale, git will fall-back and compute status normally. diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt index 9a376886a5867a..fedf86d32718eb 100644 --- a/Documentation/git-status.txt +++ b/Documentation/git-status.txt @@ -151,6 +151,19 @@ ignored, then the directory is not shown, but all contents are shown. threshold. See also linkgit:git-diff[1] `--find-renames`. +--serialize[=]:: + (EXPERIMENTAL) Serialize raw status results to stdout in a + format suitable for use by `--deserialize`. Valid values for + `` are "1" and "v1". + +--deserialize[=]:: + (EXPERIMENTAL) Deserialize raw status results from a file or + stdin rather than scanning the worktree. If `` is omitted + and `status.deserializePath` is unset, input is read from stdin. +--no-deserialize:: + (EXPERIMENTAL) Disable implicit deserialization of status results + from the value of `status.deserializePath`. + ...:: See the 'pathspec' entry in linkgit:gitglossary[7]. @@ -424,6 +437,26 @@ quoted as explained for the configuration variable `core.quotePath` (see linkgit:git-config[1]). +SERIALIZATION and DESERIALIZATION (EXPERIMENTAL) +------------------------------------------------ + +The `--serialize` option allows git to cache the result of a +possibly time-consuming status scan to a binary file. A local +service/daemon watching file system events could use this to +periodically pre-compute a fresh status result. + +Interactive users could then use `--deserialize` to simply +(and immediately) print the last-known-good result without +waiting for the status scan. + +The binary serialization file format includes some worktree state +information allowing `--deserialize` to reject the cached data +and force a normal status scan if, for example, the commit, branch, +or status modes/options change. The format cannot, however, indicate +when the cached data is otherwise stale -- that coordination belongs +to the task driving the serializations. + + CONFIGURATION ------------- diff --git a/Documentation/technical/status-serialization-format.txt b/Documentation/technical/status-serialization-format.txt new file mode 100644 index 00000000000000..475ae814495581 --- /dev/null +++ b/Documentation/technical/status-serialization-format.txt @@ -0,0 +1,107 @@ +Git status serialization format +=============================== + +Git status serialization enables git to dump the results of a status scan +to a binary file. This file can then be loaded by later status invocations +to print the cached status results. + +The file contains the essential fields from: +() the index +() the "struct wt_status" for the overall results +() the contents of "struct wt_status_change_data" for tracked changed files +() the list of untracked and ignored files + +Version 1 Format: +================= + +The V1 file begins with a required header section followed by optional +sections for each type of item (changed, untracked, ignored). Individual +item sections are only present if necessary. Each item section begins +with an item-type header with the number of items in the section. + +Each "line" in the format is encoded using pkt-line with a final LF. +Flush packets are used to terminate sections. + +----------------- +PKT-LINE("version" SP "1") + +[] +[] +[] +----------------- + + +V1 Header +--------- + +The v1-header-section fields are taken directly from "struct wt_status". +Each field is printed on a separate pkt-line. Lines for NULL string +values are omitted. All integers are printed with "%d". OIDs are +printed in hex. + +v1-header-section = + + PKT-LINE() + +v1-index-headers = PKT-LINE("index_mtime" SP SP LF) + +v1-wt-status-headers = PKT-LINE("is_initial" SP LF) + [ PKT-LINE("branch" SP LF) ] + [ PKT-LINE("reference" SP LF) ] + PKT-LINE("show_ignored_files" SP LF) + PKT-LINE("show_untracked_files" SP LF) + PKT-LINE("show_ignored_directory" SP LF) + [ PKT-LINE("ignore_submodule_arg" SP LF) ] + PKT-LINE("detect_rename" SP LF) + PKT-LINE("rename_score" SP LF) + PKT-LINE("rename_limit" SP LF) + PKT-LINE("detect_break" SP LF) + PKT-LINE("sha1_commit" SP LF) + PKT-LINE("committable" SP LF) + PKT-LINE("workdir_dirty" SP LF) + + +V1 Changed Items +---------------- + +The v1-changed-item-section lists all of the changed items with one +item per pkt-line. Each pkt-line contains: a binary block of data +from "struct wt_status_serialize_data_fixed" in a fixed header where +integers are in network byte order and OIDs are in raw (non-hex) form. +This is followed by one or two raw pathnames (not c-quoted) with NUL +terminators (both NULs are always present even if there is no rename). + +v1-changed-item-section = PKT-LINE("changed" SP LF) + [ PKT-LINE( LF) ]+ + PKT-LINE() + +changed_item = + + + + + + + + + + + + NUL + [ ] + NUL + + +V1 Untracked and Ignored Items +------------------------------ + +These sections are simple lists of pathnames. They ARE NOT +c-quoted. + +v1-untracked-item-section = PKT-LINE("untracked" SP LF) + [ PKT-LINE( LF) ]+ + PKT-LINE() + +v1-ignored-item-section = PKT-LINE("ignored" SP LF) + [ PKT-LINE( LF) ]+ + PKT-LINE() diff --git a/Makefile b/Makefile index a18b18bd17e687..895f799b2910d2 100644 --- a/Makefile +++ b/Makefile @@ -1199,6 +1199,8 @@ LIB_OBJS += wrapper.o LIB_OBJS += write-or-die.o LIB_OBJS += ws.o LIB_OBJS += wt-status.o +LIB_OBJS += wt-status-deserialize.o +LIB_OBJS += wt-status-serialize.o LIB_OBJS += xdiff-interface.o BUILTIN_OBJS += builtin/add.o diff --git a/builtin/commit.c b/builtin/commit.c index 6a5266075a8ede..e32514f03fa279 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -160,6 +160,70 @@ static int opt_parse_porcelain(const struct option *opt, const char *arg, int un return 0; } +static int do_serialize = 0; +static int do_implicit_deserialize = 0; +static int do_explicit_deserialize = 0; +static char *deserialize_path = NULL; + +/* + * --serialize | --serialize=1 | --serialize=v1 + * + * Request that we serialize our output rather than printing in + * any of the established formats. Optionally specify serialization + * version. + */ +static int opt_parse_serialize(const struct option *opt, const char *arg, int unset) +{ + enum wt_status_format *value = (enum wt_status_format *)opt->value; + if (unset || !arg) + *value = STATUS_FORMAT_SERIALIZE_V1; + else if (!strcmp(arg, "v1") || !strcmp(arg, "1")) + *value = STATUS_FORMAT_SERIALIZE_V1; + else + die("unsupported serialize version '%s'", arg); + + if (do_explicit_deserialize) + die("cannot mix --serialize and --deserialize"); + do_implicit_deserialize = 0; + + do_serialize = 1; + return 0; +} + +/* + * --deserialize | --deserialize= | + * --no-deserialize + * + * Request that we deserialize status data from some existing resource + * rather than performing a status scan. + * + * The input source can come from stdin or a path given here -- or be + * inherited from the config settings. + */ +static int opt_parse_deserialize(const struct option *opt, const char *arg, int unset) +{ + if (unset) { + do_implicit_deserialize = 0; + do_explicit_deserialize = 0; + } else { + if (do_serialize) + die("cannot mix --serialize and --deserialize"); + if (arg) { + /* override config or stdin */ + free(deserialize_path); + deserialize_path = xstrdup(arg); + } + if (deserialize_path && *deserialize_path + && (access(deserialize_path, R_OK) != 0)) + die("cannot find serialization file '%s'", + deserialize_path); + + do_explicit_deserialize = 1; + } + + return 0; +} + static int opt_parse_m(const struct option *opt, const char *arg, int unset) { struct strbuf *buf = opt->value; @@ -1172,6 +1236,8 @@ static enum untracked_status_type parse_untracked_setting_name(const char *u) return SHOW_NORMAL_UNTRACKED_FILES; else if (!strcmp(u, "all")) return SHOW_ALL_UNTRACKED_FILES; + else if (!strcmp(u,"complete")) + return SHOW_COMPLETE_UNTRACKED_FILES; else return SHOW_UNTRACKED_FILES_ERROR; } @@ -1469,6 +1535,19 @@ static int git_status_config(const char *k, const char *v, s->relative_paths = git_config_bool(k, v); return 0; } + if (!strcmp(k, "status.deserializepath")) { + /* + * Automatically assume deserialization if this is + * set in the config and the file exists. Do not + * complain if the file does not exist, because we + * silently fall back to normal mode. + */ + if (v && *v && access(v, R_OK) == 0) { + do_implicit_deserialize = 1; + deserialize_path = xstrdup(v); + } + return 0; + } if (!strcmp(k, "status.showuntrackedfiles")) { enum untracked_status_type u; @@ -1505,7 +1584,8 @@ int cmd_status(int argc, const char **argv, const char *prefix) static const char *rename_score_arg = (const char *)-1; static struct wt_status s; unsigned int progress_flag = 0; - int fd; + int try_deserialize; + int fd = -1; struct object_id oid; static struct option builtin_status_options[] = { OPT__VERBOSE(&verbose, N_("be verbose")), @@ -1520,6 +1600,12 @@ int cmd_status(int argc, const char **argv, const char *prefix) OPT_CALLBACK_F(0, "porcelain", &status_format, N_("version"), N_("machine-readable output"), PARSE_OPT_OPTARG, opt_parse_porcelain), + { OPTION_CALLBACK, 0, "serialize", &status_format, + N_("version"), N_("serialize raw status data to stdout"), + PARSE_OPT_OPTARG | PARSE_OPT_NONEG, opt_parse_serialize }, + { OPTION_CALLBACK, 0, "deserialize", NULL, + N_("path"), N_("deserialize raw status data from file"), + PARSE_OPT_OPTARG, opt_parse_deserialize }, OPT_SET_INT(0, "long", &status_format, N_("show status in long format (default)"), STATUS_FORMAT_LONG), @@ -1564,10 +1650,26 @@ int cmd_status(int argc, const char **argv, const char *prefix) s.show_untracked_files == SHOW_NO_UNTRACKED_FILES) die(_("Unsupported combination of ignored and untracked-files arguments")); + if (s.show_untracked_files == SHOW_COMPLETE_UNTRACKED_FILES && + s.show_ignored_mode == SHOW_NO_IGNORED) + die(_("Complete Untracked only supported with ignored files")); + parse_pathspec(&s.pathspec, 0, PATHSPEC_PREFER_FULL, prefix, argv); + /* + * If we want to try to deserialize status data from a cache file, + * we need to re-order the initialization code. The problem is that + * this makes for a very nasty diff and causes merge conflicts as we + * carry it forward. And it easy to mess up the merge, so we + * duplicate some code here to hopefully reduce conflicts. + */ + try_deserialize = (!do_serialize && + (do_implicit_deserialize || do_explicit_deserialize)); + if (try_deserialize) + goto skip_init; + enable_fscache(0); if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) @@ -1582,6 +1684,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) else fd = -1; +skip_init: s.is_initial = repo_get_oid(the_repository, s.reference, &oid) ? 1 : 0; if (!s.is_initial) oidcpy(&s.oid_commit, &oid); @@ -1598,6 +1701,24 @@ int cmd_status(int argc, const char **argv, const char *prefix) s.rename_score = parse_rename_score(&rename_score_arg); } + if (try_deserialize) { + if (s.relative_paths) + s.prefix = prefix; + + if (wt_status_deserialize(&s, deserialize_path) == DESERIALIZE_OK) + return 0; + + /* deserialize failed, so force the initialization we skipped above. */ + enable_fscache(1); + repo_read_index_preload(the_repository, &s.pathspec, 0); + refresh_index(the_repository->index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); + + if (use_optional_locks()) + fd = repo_hold_locked_index(the_repository, &index_lock, 0); + else + fd = -1; + } + wt_status_collect(&s); if (0 <= fd) diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index 60a22d619a85cc..94ff4f9363e1d8 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -1802,7 +1802,7 @@ _git_clone () esac } -__git_untracked_file_modes="all no normal" +__git_untracked_file_modes="all no normal complete" __git_trailer_tokens () { diff --git a/pkt-line.c b/pkt-line.c index 24479eae4dbe2a..b3ec9e5f0d636a 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -231,7 +231,7 @@ static int do_packet_write(const int fd_out, const char *buf, size_t size, return 0; } -static int packet_write_gently(const int fd_out, const char *buf, size_t size) +int packet_write_gently(const int fd_out, const char *buf, size_t size) { struct strbuf err = STRBUF_INIT; if (do_packet_write(fd_out, buf, size, &err)) { diff --git a/pkt-line.h b/pkt-line.h index 3b33cc64f34dcc..10fd9a812e1935 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -29,6 +29,7 @@ void packet_write(int fd_out, const char *buf, size_t size); void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((format (printf, 2, 3))); int packet_flush_gently(int fd); int packet_write_fmt_gently(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3))); +int packet_write_gently(const int fd_out, const char *buf, size_t size); int write_packetized_from_fd_no_flush(int fd_in, int fd_out); int write_packetized_from_buf_no_flush_count(const char *src_in, size_t len, int fd_out, int *packet_counter); diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh new file mode 100755 index 00000000000000..283a98bdf750e6 --- /dev/null +++ b/t/t7522-serialized-status.sh @@ -0,0 +1,141 @@ +#!/bin/sh + +test_description='git serialized status tests' + +. ./test-lib.sh + +# This file includes tests for serializing / deserializing +# status data. These tests cover two basic features: +# +# [1] Because users can request different types of untracked-file +# and ignored file reporting, the cache data generated by +# serialize must use either the same untracked and ignored +# parameters as the later deserialize invocation; otherwise, +# the deserialize invocation must disregard the cached data +# and run a full scan itself. +# +# To increase the number of cases where the cached status can +# be used, we have added a "--untracked-file=complete" option +# that reports a superset or union of the results from the +# "-u normal" and "-u all". We combine this with a filter in +# deserialize to filter the results. +# +# Ignored file reporting is simpler in that is an all or +# nothing; there are no subsets. +# +# The tests here (in addition to confirming that a cache +# file can be generated and used by a subsequent status +# command) need to test this untracked-file filtering. +# +# [2] ensuring the status calls are using data from the status +# cache as expected. This includes verifying cached data +# is used when appropriate as well as falling back to +# performing a new status scan when the data in the cache +# is insufficient/known stale. + +test_expect_success 'setup' ' + git branch -M main && + cat >.gitignore <<-\EOF && + *.ign + ignored_dir/ + EOF + + mkdir tracked ignored_dir && + touch tracked_1.txt tracked/tracked_1.txt && + git add . && + test_tick && + git commit -m"Adding original file." && + mkdir untracked && + touch ignored.ign ignored_dir/ignored_2.txt \ + untracked_1.txt untracked/untracked_2.txt untracked/untracked_3.txt +' + +test_expect_success 'verify untracked-files=complete with no conversion' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? serialized_status.dat + ? untracked/ + ? untracked/untracked_2.txt + ? untracked/untracked_3.txt + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --untracked-files=complete --ignored=matching --deserialize=serialized_status.dat >output && + test_cmp expect output +' + +test_expect_success 'verify untracked-files=complete to untracked-files=normal conversion' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --deserialize=serialized_status.dat >output && + test_cmp expect output +' + +test_expect_success 'verify untracked-files=complete to untracked-files=all conversion' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? serialized_status.dat + ? untracked/untracked_2.txt + ? untracked/untracked_3.txt + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --untracked-files=all --ignored=matching --deserialize=serialized_status.dat >output && + test_cmp expect output +' + +test_expect_success 'verify serialized status with non-convertible ignore mode does new scan' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? new_change.txt + ? output + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --ignored --deserialize=serialized_status.dat >output && + test_cmp expect output +' + +test_expect_success 'verify serialized status handles path scopes' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? untracked/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --deserialize=serialized_status.dat untracked >output && + test_cmp expect output +' + +test_done diff --git a/t/t7523-status-complete-untracked.sh b/t/t7523-status-complete-untracked.sh new file mode 100755 index 00000000000000..f79611fc024f48 --- /dev/null +++ b/t/t7523-status-complete-untracked.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +test_description='git status untracked complete tests' + +. ./test-lib.sh + +test_expect_success 'setup' ' + cat >.gitignore <<-\EOF && + *.ign + ignored_dir/ + EOF + + mkdir tracked ignored_dir && + touch tracked_1.txt tracked/tracked_1.txt && + git add . && + test_tick && + git commit -m"Adding original file." && + mkdir untracked && + touch ignored.ign ignored_dir/ignored_2.txt \ + untracked_1.txt untracked/untracked_2.txt untracked/untracked_3.txt +' + +test_expect_success 'verify untracked-files=complete' ' + cat >expect <<-\EOF && + ? expect + ? output + ? untracked/ + ? untracked/untracked_2.txt + ? untracked/untracked_3.txt + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --porcelain=v2 --untracked-files=complete --ignored >output && + test_cmp expect output +' + +test_done diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c new file mode 100644 index 00000000000000..a1f7d57670a131 --- /dev/null +++ b/wt-status-deserialize.c @@ -0,0 +1,609 @@ +#define USE_THE_REPOSITORY_VARIABLE +#include "git-compat-util.h" +#include "environment.h" +#include "hex.h" +#include "hash.h" +#include "wt-status.h" +#include "pkt-line.h" +#include "trace.h" +#include "statinfo.h" +#include "hex.h" + +static struct trace_key trace_deserialize = TRACE_KEY_INIT(DESERIALIZE); + +enum deserialize_parse_strategy { + DESERIALIZE_STRATEGY_AS_IS, + DESERIALIZE_STRATEGY_SKIP, + DESERIALIZE_STRATEGY_NORMAL, + DESERIALIZE_STRATEGY_ALL +}; + +static int check_path_contains(const char *out, int out_len, const char *in, int in_len) +{ + return (out_len > 0 && + out_len < in_len && + (out[out_len - 1] == '/') && + !memcmp(out, in, out_len)); +} + +static const char *my_packet_read_line(int fd, int *line_len) +{ + static char buf[LARGE_PACKET_MAX]; + + *line_len = packet_read(fd, buf, sizeof(buf), + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + return (*line_len > 0) ? buf : NULL; +} + +/* + * mtime_reported contains the mtime of the index when the + * serialization snapshot was computed. + * + * mtime_observed_on_disk contains the mtime of the index now. + * + * If these 2 times are different, then the .git/index has + * changed since the serialization cache was created and we + * must reject the cache because anything could have changed. + * + * If they are the same, we continue trying to use the cache. + */ +static int my_validate_index(const struct cache_time *mtime_reported) +{ + const char *path = get_index_file(); + struct stat st; + struct cache_time mtime_observed_on_disk; + + if (lstat(path, &st)) { + trace_printf_key(&trace_deserialize, "could not stat index"); + return DESERIALIZE_ERR; + } + mtime_observed_on_disk.sec = st.st_mtime; + mtime_observed_on_disk.nsec = ST_MTIME_NSEC(st); + if ((mtime_observed_on_disk.sec != mtime_reported->sec) || + (mtime_observed_on_disk.nsec != mtime_reported->nsec)) { + trace_printf_key(&trace_deserialize, "index mtime changed [des %d.%d][obs %d.%d]", + mtime_reported->sec, mtime_reported->nsec, + mtime_observed_on_disk.sec, mtime_observed_on_disk.nsec); + return DESERIALIZE_ERR; + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_v1_header(struct wt_status *s, int fd) +{ + struct cache_time index_mtime; + int line_len, nr_fields; + const char *line; + const char *arg; + + /* + * parse header lines up to the first flush packet. + */ + while ((line = my_packet_read_line(fd, &line_len))) { + + if (skip_prefix(line, "index_mtime ", &arg)) { + nr_fields = sscanf(arg, "%u %u", + &index_mtime.sec, + &index_mtime.nsec); + if (nr_fields != 2) { + trace_printf_key(&trace_deserialize, "invalid index_mtime (%d) '%s'", + nr_fields, line); + return DESERIALIZE_ERR; + } + continue; + } + + if (skip_prefix(line, "is_initial ", &arg)) { + s->is_initial = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "branch ", &arg)) { + s->branch = xstrdup(arg); + continue; + } + if (skip_prefix(line, "reference ", &arg)) { + s->reference = xstrdup(arg); + continue; + } + /* pathspec */ + /* verbose */ + /* amend */ + if (skip_prefix(line, "whence ", &arg)) { + s->whence = (int)strtol(arg, NULL, 10); + continue; + } + /* nowarn */ + /* use_color */ + /* no_gettext */ + /* display_comment_prefix */ + /* relative_paths */ + /* submodule_summary */ + if (skip_prefix(line, "show_ignored_mode ", &arg)) { + s->show_ignored_mode = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "show_untracked_files ", &arg)) { + s->show_untracked_files = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "ignore_submodule_arg ", &arg)) { + s->ignore_submodule_arg = xstrdup(arg); + continue; + } + /* color_palette */ + /* colopts */ + /* null_termination */ + /* commit_template */ + /* show_branch */ + /* show_stash */ + if (skip_prefix(line, "hints ", &arg)) { + s->hints = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "detect_rename ", &arg)) { + s->detect_rename = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "rename_score ", &arg)) { + s->rename_score = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "rename_limit ", &arg)) { + s->rename_limit = (int)strtol(arg, NULL, 10); + continue; + } + /* status_format */ + if (skip_prefix(line, "sha1_commit ", &arg)) { + if (get_oid_hex(arg, &s->oid_commit)) { + trace_printf_key(&trace_deserialize, "invalid sha1_commit"); + return DESERIALIZE_ERR; + } + continue; + } + if (skip_prefix(line, "committable ", &arg)) { + s->committable = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "workdir_dirty ", &arg)) { + s->workdir_dirty = (int)strtol(arg, NULL, 10); + continue; + } + /* prefix */ + + trace_printf_key(&trace_deserialize, "unexpected line '%s'", line); + return DESERIALIZE_ERR; + } + + return my_validate_index(&index_mtime); +} + +/* + * Build a string-list of (count) lines from the input. + */ +static int wt_deserialize_v1_changed_items(struct wt_status *s, int fd, int count) +{ + struct wt_status_serialize_data *sd; + char *p; + int line_len; + const char *line; + struct string_list_item *item; + + memset(&s->change, 0, sizeof(s->change)); + s->change.strdup_strings = 1; + + /* + * + + * + * + * NUL [] NUL + */ + while ((line = my_packet_read_line(fd, &line_len))) { + struct wt_status_change_data *d = xcalloc(1, sizeof(*d)); + sd = (struct wt_status_serialize_data *)line; + + d->worktree_status = ntohl(sd->fixed.worktree_status); + d->index_status = ntohl(sd->fixed.index_status); + d->stagemask = ntohl(sd->fixed.stagemask); + d->rename_status = ntohl(sd->fixed.rename_status); + d->rename_score = ntohl(sd->fixed.rename_score); + d->mode_head = ntohl(sd->fixed.mode_head); + d->mode_index = ntohl(sd->fixed.mode_index); + d->mode_worktree = ntohl(sd->fixed.mode_worktree); + d->dirty_submodule = ntohl(sd->fixed.dirty_submodule); + d->new_submodule_commits = ntohl(sd->fixed.new_submodule_commits); + oidcpy(&d->oid_head, &sd->fixed.oid_head); + oidcpy(&d->oid_index, &sd->fixed.oid_index); + + p = sd->variant; + item = string_list_append(&s->change, p); + p += strlen(p) + 1; + if (*p) + d->rename_source = xstrdup(p); + item->util = d; + + trace_printf_key( + &trace_deserialize, + "change: %d %d %d %d %d %o %o %o %d %d %s %s '%s' '%s'", + d->worktree_status, + d->index_status, + d->stagemask, + d->rename_status, + d->rename_score, + d->mode_head, + d->mode_index, + d->mode_worktree, + d->dirty_submodule, + d->new_submodule_commits, + oid_to_hex(&d->oid_head), + oid_to_hex(&d->oid_index), + item->string, + (d->rename_source ? d->rename_source : "")); + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_v1_untracked_items(struct wt_status *s, + int fd, + int count, + enum deserialize_parse_strategy strategy) +{ + int line_len; + const char *line; + char *out = NULL; + int out_len = 0; + + memset(&s->untracked, 0, sizeof(s->untracked)); + s->untracked.strdup_strings = 1; + + /* + * + + * + */ + while ((line = my_packet_read_line(fd, &line_len))) { + if (strategy == DESERIALIZE_STRATEGY_AS_IS) + string_list_append(&s->untracked, line); + if (strategy == DESERIALIZE_STRATEGY_SKIP) + continue; + if (strategy == DESERIALIZE_STRATEGY_NORMAL) { + + /* Only add "normal" entries to list */ + if (out && + check_path_contains(out, out_len, line, line_len)) { + continue; + } + else { + out = string_list_append(&s->untracked, line)->string; + out_len = line_len; + } + } + if (strategy == DESERIALIZE_STRATEGY_ALL) { + /* Only add "all" entries to list */ + if (line[line_len - 1] != '/') + string_list_append(&s->untracked, line); + } + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_v1_ignored_items(struct wt_status *s, + int fd, + int count, + enum deserialize_parse_strategy strategy) +{ + int line_len; + const char *line; + + memset(&s->ignored, 0, sizeof(s->ignored)); + s->ignored.strdup_strings = 1; + + /* + * + + * + */ + while ((line = my_packet_read_line(fd, &line_len))) { + if (strategy == DESERIALIZE_STRATEGY_AS_IS) + string_list_append(&s->ignored, line); + else + continue; + } + + return DESERIALIZE_OK; +} + +static int validate_untracked_files_arg(enum untracked_status_type cmd, + enum untracked_status_type des, + enum deserialize_parse_strategy *strategy) +{ + *strategy = DESERIALIZE_STRATEGY_AS_IS; + + if (cmd == des) { + *strategy = DESERIALIZE_STRATEGY_AS_IS; + } else if (cmd == SHOW_NO_UNTRACKED_FILES) { + *strategy = DESERIALIZE_STRATEGY_SKIP; + } else if (des == SHOW_COMPLETE_UNTRACKED_FILES) { + if (cmd == SHOW_ALL_UNTRACKED_FILES) + *strategy = DESERIALIZE_STRATEGY_ALL; + else if (cmd == SHOW_NORMAL_UNTRACKED_FILES) + *strategy = DESERIALIZE_STRATEGY_NORMAL; + } else { + return DESERIALIZE_ERR; + } + + return DESERIALIZE_OK; +} + +static int validate_ignored_files_arg(enum show_ignored_type cmd, + enum show_ignored_type des, + enum deserialize_parse_strategy *strategy) +{ + *strategy = DESERIALIZE_STRATEGY_AS_IS; + + if (cmd == SHOW_NO_IGNORED) { + *strategy = DESERIALIZE_STRATEGY_SKIP; + } + else if (cmd != des) { + return DESERIALIZE_ERR; + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, int fd) +{ + int line_len; + const char *line; + const char *arg; + int nr_changed = 0; + int nr_untracked = 0; + int nr_ignored = 0; + + enum deserialize_parse_strategy ignored_strategy = DESERIALIZE_STRATEGY_AS_IS, untracked_strategy = DESERIALIZE_STRATEGY_AS_IS; + + if (wt_deserialize_v1_header(s, fd) == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + + /* + * We now have the header parsed. Look at the command args (as passed in), and see how to parse + * the serialized data + */ + if (validate_untracked_files_arg(cmd_s->show_untracked_files, s->show_untracked_files, &untracked_strategy)) { + trace_printf_key(&trace_deserialize, "reject: show_untracked_file: command: %d, serialized : %d", + cmd_s->show_untracked_files, + s->show_untracked_files); + return DESERIALIZE_ERR; + } + + if (validate_ignored_files_arg(cmd_s->show_ignored_mode, s->show_ignored_mode, &ignored_strategy)) { + trace_printf_key(&trace_deserialize, "reject: show_ignored_mode: command: %d, serialized: %d", + cmd_s->show_ignored_mode, + s->show_ignored_mode); + return DESERIALIZE_ERR; + } + + /* + * [ [+] ] + * [ [+] ] + * [ [+] ] + */ + while ((line = my_packet_read_line(fd, &line_len))) { + if (skip_prefix(line, "changed ", &arg)) { + nr_changed = (int)strtol(arg, NULL, 10); + if (wt_deserialize_v1_changed_items(s, fd, nr_changed) + == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + continue; + } + if (skip_prefix(line, "untracked ", &arg)) { + nr_untracked = (int)strtol(arg, NULL, 10); + if (wt_deserialize_v1_untracked_items(s, fd, nr_untracked, untracked_strategy) + == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + continue; + } + if (skip_prefix(line, "ignored ", &arg)) { + nr_ignored = (int)strtol(arg, NULL, 10); + if (wt_deserialize_v1_ignored_items(s, fd, nr_ignored, ignored_strategy) + == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + continue; + } + trace_printf_key(&trace_deserialize, "unexpected line '%s'", line); + return DESERIALIZE_ERR; + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_parse(const struct wt_status *cmd_s, struct wt_status *s, int fd) +{ + int line_len; + const char *line; + const char *arg; + + memset(s, 0, sizeof(*s)); + + if ((line = my_packet_read_line(fd, &line_len)) && + (skip_prefix(line, "version ", &arg))) { + int version = (int)strtol(arg, NULL, 10); + if (version == 1) + return wt_deserialize_v1(cmd_s, s, fd); + } + trace_printf_key(&trace_deserialize, "missing/unsupported version"); + return DESERIALIZE_ERR; +} + +static inline int my_strcmp_null(const char *a, const char *b) +{ + const char *alt_a = (a) ? a : ""; + const char *alt_b = (b) ? b : ""; + + return strcmp(alt_a, alt_b); +} + +static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *des_s, int fd) +{ + /* + * Check the path spec on the current command + */ + if (cmd_s->pathspec.nr > 1) { + trace_printf_key(&trace_deserialize, "reject: multiple pathspecs"); + return DESERIALIZE_ERR; + } + + /* + * If we have a pathspec, but it maches the root (e.g. no filtering) + * then this is OK. + */ + if (cmd_s->pathspec.nr == 1 && + my_strcmp_null(cmd_s->pathspec.items[0].match, "")) { + trace_printf_key(&trace_deserialize, "reject: pathspec"); + return DESERIALIZE_ERR; + } + + /* + * Deserialize cached status + */ + if (wt_deserialize_parse(cmd_s, des_s, fd) == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + + /* + * Compare fields in cmd_s with those observed in des_s and + * complain if they are incompatible (such as different "-u" + * or "--ignored" settings). + */ + if (cmd_s->is_initial != des_s->is_initial) { + trace_printf_key(&trace_deserialize, "reject: is_initial"); + return DESERIALIZE_ERR; + } + if (my_strcmp_null(cmd_s->branch, des_s->branch)) { + trace_printf_key(&trace_deserialize, "reject: branch"); + return DESERIALIZE_ERR; + } + if (my_strcmp_null(cmd_s->reference, des_s->reference)) { + trace_printf_key(&trace_deserialize, "reject: reference"); + return DESERIALIZE_ERR; + } + /* verbose */ + /* amend */ + if (cmd_s->whence != des_s->whence) { + trace_printf_key(&trace_deserialize, "reject: whence"); + return DESERIALIZE_ERR; + } + /* nowarn */ + /* use_color */ + /* no_gettext */ + /* display_comment_prefix */ + /* relative_paths */ + /* submodule_summary */ + + /* show_ignored_files - already validated */ + /* show_untrackes_files - already validated */ + + /* + * Submodules are not supported by status serialization. + * The status will not be serialized if it contains submodules, + * and so this check is not needed. + * + * if (my_strcmp_null(cmd_s->ignore_submodule_arg, des_s->ignore_submodule_arg)) { + * trace_printf_key(&trace_deserialize, "reject: ignore_submodule_arg"); + * return DESERIALIZE_ERR; + * } + */ + + /* color_palette */ + /* colopts */ + /* null_termination */ + /* commit_template */ + /* show_branch */ + /* show_stash */ + /* hints */ + if (cmd_s->detect_rename != des_s->detect_rename) { + trace_printf_key(&trace_deserialize, "reject: detect_rename"); + return DESERIALIZE_ERR; + } + if (cmd_s->rename_score != des_s->rename_score) { + trace_printf_key(&trace_deserialize, "reject: rename_score"); + return DESERIALIZE_ERR; + } + if (cmd_s->rename_limit != des_s->rename_limit) { + trace_printf_key(&trace_deserialize, "reject: rename_limit"); + return DESERIALIZE_ERR; + } + /* status_format */ + if (!oideq(&cmd_s->oid_commit, &des_s->oid_commit)) { + trace_printf_key(&trace_deserialize, "reject: sha1_commit"); + return DESERIALIZE_ERR; + } + + /* + * Copy over display-related fields from the current command. + */ + des_s->repo = cmd_s->repo; + des_s->verbose = cmd_s->verbose; + /* amend */ + /* whence */ + des_s->nowarn = cmd_s->nowarn; + des_s->use_color = cmd_s->use_color; + des_s->no_gettext = cmd_s->no_gettext; + des_s->display_comment_prefix = cmd_s->display_comment_prefix; + des_s->relative_paths = cmd_s->relative_paths; + des_s->submodule_summary = cmd_s->submodule_summary; + memcpy(des_s->color_palette, cmd_s->color_palette, + sizeof(char)*WT_STATUS_MAXSLOT*COLOR_MAXLEN); + des_s->colopts = cmd_s->colopts; + des_s->null_termination = cmd_s->null_termination; + /* commit_template */ + des_s->show_branch = cmd_s->show_branch; + des_s->show_stash = cmd_s->show_stash; + /* hints */ + des_s->status_format = cmd_s->status_format; + des_s->fp = cmd_s->fp; + if (cmd_s->prefix && *cmd_s->prefix) + des_s->prefix = xstrdup(cmd_s->prefix); + + return DESERIALIZE_OK; +} + + +/* + * Read raw serialized status data from the given file + * + * Verify that the args specified in the current command + * are compatible with the deserialized data (such as "-uno"). + * + * Copy display-related fields from the current command + * into the deserialized data (so that the user can request + * long or short as they please). + */ +int wt_status_deserialize(const struct wt_status *cmd_s, + const char *path) +{ + struct wt_status des_s; + int result; + + if (path && *path && strcmp(path, "0")) { + int fd = xopen(path, O_RDONLY); + if (fd == -1) { + trace_printf_key(&trace_deserialize, "could not read '%s'", path); + return DESERIALIZE_ERR; + } + trace_printf_key(&trace_deserialize, "reading serialization file '%s'", path); + result = wt_deserialize_fd(cmd_s, &des_s, fd); + close(fd); + } else { + trace_printf_key(&trace_deserialize, "reading stdin"); + result = wt_deserialize_fd(cmd_s, &des_s, 0); + } + + if (result == DESERIALIZE_OK) { + wt_status_get_state(cmd_s->repo, &des_s.state, des_s.branch && + !strcmp(des_s.branch, "HEAD")); + wt_status_print(&des_s); + } + + return result; +} diff --git a/wt-status-serialize.c b/wt-status-serialize.c new file mode 100644 index 00000000000000..86175edfff85a3 --- /dev/null +++ b/wt-status-serialize.c @@ -0,0 +1,219 @@ +#include "git-compat-util.h" +#include "hex.h" +#include "repository.h" +#include "wt-status.h" +#include "pkt-line.h" +#include "trace.h" +#include "read-cache-ll.h" + +static struct trace_key trace_serialize = TRACE_KEY_INIT(SERIALIZE); + +/* + * Write V1 header fields. + */ +static void wt_serialize_v1_header(struct wt_status *s, int fd) +{ + /* + * Write select fields from the current index to help + * the deserializer recognize a stale data set. + */ + packet_write_fmt(fd, "index_mtime %d %d\n", + s->repo->index->timestamp.sec, + s->repo->index->timestamp.nsec); + + /* + * Write data from wt_status to qualify this status report. + * That is, if this run specified "-uno", the consumer of + * our serialization should know that. + */ + packet_write_fmt(fd, "is_initial %d\n", s->is_initial); + if (s->branch) + packet_write_fmt(fd, "branch %s\n", s->branch); + if (s->reference) + packet_write_fmt(fd, "reference %s\n", s->reference); + /* pathspec */ + /* verbose */ + /* amend */ + packet_write_fmt(fd, "whence %d\n", s->whence); + /* nowarn */ + /* use_color */ + /* no_gettext */ + /* display_comment_prefix */ + /* relative_paths */ + /* submodule_summary */ + packet_write_fmt(fd, "show_ignored_mode %d\n", s->show_ignored_mode); + packet_write_fmt(fd, "show_untracked_files %d\n", s->show_untracked_files); + if (s->ignore_submodule_arg) + packet_write_fmt(fd, "ignore_submodule_arg %s\n", s->ignore_submodule_arg); + /* color_palette */ + /* colopts */ + /* null_termination */ + /* commit_template */ + /* show_branch */ + /* show_stash */ + packet_write_fmt(fd, "hints %d\n", s->hints); + packet_write_fmt(fd, "detect_rename %d\n", s->detect_rename); + packet_write_fmt(fd, "rename_score %d\n", s->rename_score); + packet_write_fmt(fd, "rename_limit %d\n", s->rename_limit); + /* status_format */ + packet_write_fmt(fd, "sha1_commit %s\n", oid_to_hex(&s->oid_commit)); + packet_write_fmt(fd, "committable %d\n", s->committable); + packet_write_fmt(fd, "workdir_dirty %d\n", s->workdir_dirty); + /* prefix */ + packet_flush(fd); +} + +/* + * Print changed/unmerged items. + * We write raw (not c-quoted) pathname(s). The rename_source is only + * set when status computed a rename/copy. + * + * We ALWAYS write a final LF to the packet-line (for debugging) + * even though Linux pathnames allow LFs. + */ +static inline void wt_serialize_v1_changed(struct wt_status *s, int fd, + struct string_list_item *item) +{ + struct wt_status_change_data *d = item->util; + struct wt_status_serialize_data sd; + char *begin; + char *end; + char *p; + int len_path, len_rename_source; + + trace_printf_key(&trace_serialize, + "change: %d %d %d %d %d %o %o %o %d %d %s %s '%s' '%s'", + d->worktree_status, + d->index_status, + d->stagemask, + d->rename_status, + d->rename_score, + d->mode_head, + d->mode_index, + d->mode_worktree, + d->dirty_submodule, + d->new_submodule_commits, + oid_to_hex(&d->oid_head), + oid_to_hex(&d->oid_index), + item->string, + (d->rename_source ? d->rename_source : "")); + + sd.fixed.worktree_status = htonl(d->worktree_status); + sd.fixed.index_status = htonl(d->index_status); + sd.fixed.stagemask = htonl(d->stagemask); + sd.fixed.rename_status = htonl(d->rename_status); + sd.fixed.rename_score = htonl(d->rename_score); + sd.fixed.mode_head = htonl(d->mode_head); + sd.fixed.mode_index = htonl(d->mode_index); + sd.fixed.mode_worktree = htonl(d->mode_worktree); + sd.fixed.dirty_submodule = htonl(d->dirty_submodule); + sd.fixed.new_submodule_commits = htonl(d->new_submodule_commits); + oidcpy(&sd.fixed.oid_head, &d->oid_head); + oidcpy(&sd.fixed.oid_index, &d->oid_index); + + begin = (char *)&sd; + end = begin + sizeof(sd); + + p = sd.variant; + + /* + * Write NUL [] NUL LF at the end of the buffer. + */ + len_path = strlen(item->string); + len_rename_source = d->rename_source ? strlen(d->rename_source) : 0; + + /* + * This is a bit of a hack, but I don't want to split the + * status detail record across multiple pkt-lines. + */ + if (p + len_path + 1 + len_rename_source + 1 + 1 >= end) + BUG("path to long to serialize '%s'", item->string); + + memcpy(p, item->string, len_path); + p += len_path; + *p++ = '\0'; + + if (len_rename_source) { + memcpy(p, d->rename_source, len_rename_source); + p += len_rename_source; + } + *p++ = '\0'; + *p++ = '\n'; + + if (packet_write_gently(fd, begin, (p - begin))) + BUG("cannot serialize '%s'", item->string); +} + +/* + * Write raw (not c-quoted) pathname for an untracked item. + * We ALWAYS write a final LF to the packet-line (for debugging) + * even though Linux pathnames allows LFs. That is, deserialization + * should use the packet-line length and omit the final LF. + */ +static inline void wt_serialize_v1_untracked(struct wt_status *s, int fd, + struct string_list_item *item) +{ + packet_write_fmt(fd, "%s\n", item->string); +} + +/* + * Write raw (not c-quoted) pathname for an ignored item. + * We ALWAYS write a final LF to the packet-line (for debugging) + * even though Linux pathnames allows LFs. + */ +static inline void wt_serialize_v1_ignored(struct wt_status *s, int fd, + struct string_list_item *item) +{ + packet_write_fmt(fd, "%s\n", item->string); +} + +/* + * Serialize the list of changes to stdout. The goal of this + * is to just serialize the key fields in wt_status so that a + * later command can rebuilt it and do the printing. + * + * We DO NOT include the contents of wt_status_state NOR + * current branch info. This info easily gets stale and + * is relatively quick for the status consumer to compute + * as necessary. + */ +void wt_status_serialize_v1(struct wt_status *s) +{ + int fd = 1; /* we always write to stdout */ + struct string_list_item *iter; + int k; + + /* + * version header must be first line. + */ + packet_write_fmt(fd, "version 1\n"); + + wt_serialize_v1_header(s, fd); + + if (s->change.nr > 0) { + packet_write_fmt(fd, "changed %"PRIuMAX"\n", (uintmax_t)s->change.nr); + for (k = 0; k < s->change.nr; k++) { + iter = &(s->change.items[k]); + wt_serialize_v1_changed(s, fd, iter); + } + packet_flush(fd); + } + + if (s->untracked.nr > 0) { + packet_write_fmt(fd, "untracked %"PRIuMAX"\n", (uintmax_t)s->untracked.nr); + for (k = 0; k < s->untracked.nr; k++) { + iter = &(s->untracked.items[k]); + wt_serialize_v1_untracked(s, fd, iter); + } + packet_flush(fd); + } + + if (s->ignored.nr > 0) { + packet_write_fmt(fd, "ignored %"PRIuMAX"\n", (uintmax_t)s->ignored.nr); + for (k = 0; k < s->ignored.nr; k++) { + iter = &(s->ignored.items[k]); + wt_serialize_v1_ignored(s, fd, iter); + } + packet_flush(fd); + } +} diff --git a/wt-status.c b/wt-status.c index cf5483510e0db8..486ae6614801fb 100644 --- a/wt-status.c +++ b/wt-status.c @@ -786,6 +786,9 @@ static void wt_status_collect_untracked(struct wt_status *s) if (s->show_untracked_files != SHOW_ALL_UNTRACKED_FILES) dir.flags |= DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; + if (s->show_untracked_files == SHOW_COMPLETE_UNTRACKED_FILES) + dir.flags |= DIR_KEEP_UNTRACKED_CONTENTS; + if (s->show_ignored_mode) { dir.flags |= DIR_SHOW_IGNORED_TOO; @@ -2618,6 +2621,9 @@ void wt_status_print(struct wt_status *s) case STATUS_FORMAT_LONG: wt_longstatus_print(s); break; + case STATUS_FORMAT_SERIALIZE_V1: + wt_status_serialize_v1(s); + break; } trace2_region_leave("status", "print", s->repo); diff --git a/wt-status.h b/wt-status.h index 4e377ce62b8b28..afeaf5753a0494 100644 --- a/wt-status.h +++ b/wt-status.h @@ -4,6 +4,7 @@ #include "string-list.h" #include "color.h" #include "pathspec.h" +#include "pkt-line.h" #include "remote.h" struct repository; @@ -26,7 +27,8 @@ enum untracked_status_type { SHOW_UNTRACKED_FILES_ERROR = -1, SHOW_NO_UNTRACKED_FILES = 0, SHOW_NORMAL_UNTRACKED_FILES, - SHOW_ALL_UNTRACKED_FILES + SHOW_ALL_UNTRACKED_FILES, + SHOW_COMPLETE_UNTRACKED_FILES, }; enum show_ignored_type { @@ -74,6 +76,7 @@ enum wt_status_format { STATUS_FORMAT_SHORT, STATUS_FORMAT_PORCELAIN, STATUS_FORMAT_PORCELAIN_V2, + STATUS_FORMAT_SERIALIZE_V1, STATUS_FORMAT_UNSPECIFIED }; @@ -185,4 +188,52 @@ int require_clean_work_tree(struct repository *repo, int ignore_submodules, int gently); +#define DESERIALIZE_OK 0 +#define DESERIALIZE_ERR 1 + +struct wt_status_serialize_data_fixed +{ + uint32_t worktree_status; + uint32_t index_status; + uint32_t stagemask; + uint32_t rename_status; + uint32_t rename_score; + uint32_t mode_head; + uint32_t mode_index; + uint32_t mode_worktree; + uint32_t dirty_submodule; + uint32_t new_submodule_commits; + struct object_id oid_head; + struct object_id oid_index; +}; + +/* + * Consume the maximum amount of data possible in a + * packet-line record. This is overkill because we + * have at most 2 relative pathnames, but means we + * don't need to allocate a variable length structure. + */ +struct wt_status_serialize_data +{ + struct wt_status_serialize_data_fixed fixed; + char variant[LARGE_PACKET_DATA_MAX + - sizeof(struct wt_status_serialize_data_fixed)]; +}; + +/* + * Serialize computed status scan results using "version 1" format + * to the given file. + */ +void wt_status_serialize_v1(struct wt_status *s); + +/* + * Deserialize existing status results from the given file and + * populate a (new) "struct wt_status". Use the contents of "cmd_s" + * (computed from the command line arguments) to verify that the + * cached data is compatible and overlay various display-related + * fields. + */ +int wt_status_deserialize(const struct wt_status *cmd_s, + const char *path); + #endif /* STATUS_H */ From 4ddc91ec3676bf69955b389fa181d66632c89cec Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 9 Oct 2018 10:19:14 -0600 Subject: [PATCH 054/189] virtualfilesystem: check if directory is included Add check to see if a directory is included in the virtualfilesystem before checking the directory hashmap. This allows a directory entry like foo/ to find all untracked files in subdirectories. --- t/t1093-virtualfilesystem.sh | 2 ++ virtualfilesystem.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/t/t1093-virtualfilesystem.sh b/t/t1093-virtualfilesystem.sh index bd0c9f72ba3c4a..8ba9a2a75e093a 100755 --- a/t/t1093-virtualfilesystem.sh +++ b/t/t1093-virtualfilesystem.sh @@ -222,6 +222,8 @@ test_expect_success 'verify folder entries include all files' ' cat > expected <<-\EOF && ?? dir1/a ?? dir1/b + ?? dir1/dir2/a + ?? dir1/dir2/b ?? dir1/untracked.txt EOF test_cmp expected actual diff --git a/virtualfilesystem.c b/virtualfilesystem.c index 6ff3a0ca2a340d..99f5d24155a1bd 100644 --- a/virtualfilesystem.c +++ b/virtualfilesystem.c @@ -236,6 +236,10 @@ int is_excluded_from_virtualfilesystem(const char *pathname, int pathlen, int dt } if (dtype == DT_DIR) { + int ret = is_included_in_virtualfilesystem(pathname, pathlen); + if (ret > 0) + return 0; + if (!parent_directory_hashmap.tablesize && virtual_filesystem_data.len) initialize_parent_directory_hashmap(&parent_directory_hashmap, &virtual_filesystem_data); if (!parent_directory_hashmap.tablesize) From 601706a2d03dddd56dc15592d0a0f84cadc1d5af Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Wed, 10 Jan 2018 11:56:26 -0500 Subject: [PATCH 055/189] Teach ahead-behind and serialized status to play nicely together --- t/t7522-serialized-status.sh | 34 +++++++++++++++++++++++++++++++++- wt-status-deserialize.c | 3 ++- wt-status-serialize.c | 1 + 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index 283a98bdf750e6..0f5a33e2a23442 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -47,7 +47,13 @@ test_expect_success 'setup' ' git commit -m"Adding original file." && mkdir untracked && touch ignored.ign ignored_dir/ignored_2.txt \ - untracked_1.txt untracked/untracked_2.txt untracked/untracked_3.txt + untracked_1.txt untracked/untracked_2.txt untracked/untracked_3.txt && + + test_oid_cache <<-EOF + branch_oid sha1:68d4a437ea4c2de65800f48c053d4d543b55c410 + + branch_oid sha256:6b95e4b1ea911dad213f2020840f5e92d3066cf9e38cf35f79412ec58d409ce4 + EOF ' test_expect_success 'verify untracked-files=complete with no conversion' ' @@ -138,4 +144,30 @@ test_expect_success 'verify serialized status handles path scopes' ' test_cmp expect output ' +test_expect_success 'verify no-ahead-behind and serialized status integration' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-EOF && + # branch.oid $(test_oid branch_oid) + # branch.head alt_branch + # branch.upstream main + # branch.ab +1 -0 + ? expect + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + EOF + + git checkout -b alt_branch main --track >/dev/null && + touch alt_branch_changes.txt && + git add alt_branch_changes.txt && + test_tick && + git commit -m"New commit on alt branch" && + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git -c status.aheadBehind=false status --porcelain=v2 --branch --ahead-behind --deserialize=serialized_status.dat >output && + test_cmp expect output +' + test_done diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index a1f7d57670a131..186d9d9c614962 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -521,6 +521,7 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de /* show_branch */ /* show_stash */ /* hints */ + /* ahead_behind_flags */ if (cmd_s->detect_rename != des_s->detect_rename) { trace_printf_key(&trace_deserialize, "reject: detect_rename"); return DESERIALIZE_ERR; @@ -560,6 +561,7 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de des_s->show_branch = cmd_s->show_branch; des_s->show_stash = cmd_s->show_stash; /* hints */ + des_s->ahead_behind_flags = cmd_s->ahead_behind_flags; des_s->status_format = cmd_s->status_format; des_s->fp = cmd_s->fp; if (cmd_s->prefix && *cmd_s->prefix) @@ -568,7 +570,6 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de return DESERIALIZE_OK; } - /* * Read raw serialized status data from the given file * diff --git a/wt-status-serialize.c b/wt-status-serialize.c index 86175edfff85a3..9d5f970ea0aedd 100644 --- a/wt-status-serialize.c +++ b/wt-status-serialize.c @@ -52,6 +52,7 @@ static void wt_serialize_v1_header(struct wt_status *s, int fd) /* show_branch */ /* show_stash */ packet_write_fmt(fd, "hints %d\n", s->hints); + /* ahead_behind_flags */ packet_write_fmt(fd, "detect_rename %d\n", s->detect_rename); packet_write_fmt(fd, "rename_score %d\n", s->rename_score); packet_write_fmt(fd, "rename_limit %d\n", s->rename_limit); From 9d2dedbe4411a4c77fe6af5c88e90be4cf9011ec Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 28 May 2019 21:48:08 +0200 Subject: [PATCH 056/189] backwards-compatibility: support the post-indexchanged hook When our patches to support that hook were upstreamed, the hook's name was eliciting some reviewer suggestions, and it was renamed to `post-index-change`. These patches (with the new name) made it into v2.22.0. However, VFSforGit users may very well have checkouts with that hook installed under the original name. To support this, let's just introduce a hack where we look a bit more closely when we just failed to find the `post-index-change` hook, and allow any `post-indexchanged` hook to run instead (if it exists). --- hook.c | 14 +++++++++++++- t/t7113-post-index-change-hook.sh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/hook.c b/hook.c index 38749433eacfaa..1e8b0ad366a022 100644 --- a/hook.c +++ b/hook.c @@ -193,7 +193,7 @@ int run_hooks_opt(const char *hook_name, struct run_hooks_opt *options) .hook_name = hook_name, .options = options, }; - const char *const hook_path = find_hook(hook_name); + const char *hook_path = find_hook(hook_name); int ret = 0; const struct run_process_parallel_opts opts = { .tr2_category = "hook", @@ -209,6 +209,18 @@ int run_hooks_opt(const char *hook_name, struct run_hooks_opt *options) .data = &cb_data, }; + /* + * Backwards compatibility hack in VFS for Git: when originally + * introduced (and used!), it was called `post-indexchanged`, but this + * name was changed during the review on the Git mailing list. + * + * Therefore, when the `post-index-change` hook is not found, let's + * look for a hook with the old name (which would be found in case of + * already-existing checkouts). + */ + if (!hook_path && !strcmp(hook_name, "post-index-change")) + hook_path = find_hook("post-indexchanged"); + if (!options) BUG("a struct run_hooks_opt must be provided to run_hooks"); diff --git a/t/t7113-post-index-change-hook.sh b/t/t7113-post-index-change-hook.sh index 58e55a7c779161..455c61a92542ae 100755 --- a/t/t7113-post-index-change-hook.sh +++ b/t/t7113-post-index-change-hook.sh @@ -16,6 +16,36 @@ test_expect_success 'setup' ' git commit -m "initial" ' +test_expect_success 'post-indexchanged' ' + mkdir -p .git/hooks && + test_when_finished "rm -f .git/hooks/post-indexchanged marker" && + write_script .git/hooks/post-indexchanged <<-\EOF && + : >marker + EOF + + : make sure -changed is called if -change does not exist && + test_when_finished "echo testing >dir1/file2.txt && git status" && + echo changed >dir1/file2.txt && + : force index to be dirty && + test-tool chmtime -60 .git/index && + git status && + test_path_is_file marker && + + test_when_finished "rm -f .git/hooks/post-index-change marker2" && + write_script .git/hooks/post-index-change <<-\EOF && + : >marker2 + EOF + + : make sure -changed is not called if -change exists && + rm -f marker marker2 && + echo testing >dir1/file2.txt && + : force index to be dirty && + test-tool chmtime -60 .git/index && + git status && + test_path_is_missing marker && + test_path_is_file marker2 +' + test_expect_success 'test status, add, commit, others trigger hook without flags set' ' test_hook post-index-change <<-\EOF && if test "$1" -eq 1; then From 510ca5654b475a4012493f9a2d2d8440991e5610 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 2 Feb 2018 14:17:05 -0500 Subject: [PATCH 057/189] status: serialize to path Teach status serialization to take an optional pathname on the command line to direct that cache data be written there rather than to stdout. When used this way, normal status results will still be written to stdout. When no path is given, only binary serialization data is written to stdout. Usage: git status --serialize[=] Signed-off-by: Jeff Hostetler --- Documentation/git-status.txt | 10 ++++++---- builtin/commit.c | 36 +++++++++++++++++++++++++++--------- t/t7522-serialized-status.sh | 35 +++++++++++++++++++++++++++++++++++ wt-status-serialize.c | 5 ++--- wt-status.c | 2 +- wt-status.h | 2 +- 6 files changed, 72 insertions(+), 18 deletions(-) diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt index fedf86d32718eb..62254617dd7fc1 100644 --- a/Documentation/git-status.txt +++ b/Documentation/git-status.txt @@ -151,10 +151,12 @@ ignored, then the directory is not shown, but all contents are shown. threshold. See also linkgit:git-diff[1] `--find-renames`. ---serialize[=]:: - (EXPERIMENTAL) Serialize raw status results to stdout in a - format suitable for use by `--deserialize`. Valid values for - `` are "1" and "v1". +--serialize[=]:: + (EXPERIMENTAL) Serialize raw status results to a file or stdout + in a format suitable for use by `--deserialize`. If a path is + given, serialize data will be written to that path *and* normal + status output will be written to stdout. If path is omitted, + only binary serialization data will be written to stdout. --deserialize[=]:: (EXPERIMENTAL) Deserialize raw status results from a file or diff --git a/builtin/commit.c b/builtin/commit.c index e32514f03fa279..c84ccce3b29313 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -161,26 +161,34 @@ static int opt_parse_porcelain(const struct option *opt, const char *arg, int un } static int do_serialize = 0; +static char *serialize_path = NULL; + static int do_implicit_deserialize = 0; static int do_explicit_deserialize = 0; static char *deserialize_path = NULL; /* - * --serialize | --serialize=1 | --serialize=v1 + * --serialize | --serialize= + * + * Request that we serialize status output rather than or in addition to + * printing in any of the established formats. + * + * Without a path, we write binary serialization data to stdout (and omit + * the normal status output). * - * Request that we serialize our output rather than printing in - * any of the established formats. Optionally specify serialization - * version. + * With a path, we write binary serialization data to the and then + * write normal status output. */ static int opt_parse_serialize(const struct option *opt, const char *arg, int unset) { enum wt_status_format *value = (enum wt_status_format *)opt->value; if (unset || !arg) *value = STATUS_FORMAT_SERIALIZE_V1; - else if (!strcmp(arg, "v1") || !strcmp(arg, "1")) - *value = STATUS_FORMAT_SERIALIZE_V1; - else - die("unsupported serialize version '%s'", arg); + + if (arg) { + free(serialize_path); + serialize_path = xstrdup(arg); + } if (do_explicit_deserialize) die("cannot mix --serialize and --deserialize"); @@ -1601,7 +1609,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) N_("version"), N_("machine-readable output"), PARSE_OPT_OPTARG, opt_parse_porcelain), { OPTION_CALLBACK, 0, "serialize", &status_format, - N_("version"), N_("serialize raw status data to stdout"), + N_("path"), N_("serialize raw status data to path or stdout"), PARSE_OPT_OPTARG | PARSE_OPT_NONEG, opt_parse_serialize }, { OPTION_CALLBACK, 0, "deserialize", NULL, N_("path"), N_("deserialize raw status data from file"), @@ -1727,6 +1735,16 @@ int cmd_status(int argc, const char **argv, const char *prefix) if (s.relative_paths) s.prefix = prefix; + if (serialize_path) { + int fd_serialize = xopen(serialize_path, + O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (fd_serialize < 0) + die_errno(_("could not serialize to '%s'"), + serialize_path); + wt_status_serialize_v1(fd_serialize, &s); + close(fd_serialize); + } + wt_status_print(&s); wt_status_collect_free_buffers(&s); diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index 0f5a33e2a23442..2a81b4e625ee59 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -170,4 +170,39 @@ test_expect_success 'verify no-ahead-behind and serialized status integration' ' test_cmp expect output ' +test_expect_success 'verify new --serialize=path mode' ' + #test_when_finished "rm serialized_status.dat expect new_change.txt output.1 output.2" && + cat >expect <<-\EOF && + ? expect + ? output.1 + ? untracked/ + ? untracked_1.txt + EOF + + git checkout -b serialize_path_branch main --track >/dev/null && + touch alt_branch_changes.txt && + git add alt_branch_changes.txt && + test_tick && + git commit -m"New commit on serialize_path_branch" && + + git status --porcelain=v2 --serialize=serialized_status.dat >output.1 && + touch new_change.txt && + + git status --porcelain=v2 --deserialize=serialized_status.dat >output.2 && + test_cmp expect output.1 && + test_cmp expect output.2 +' + +test_expect_success 'renames' ' + git init -b main rename_test && + echo OLDNAME >rename_test/OLDNAME && + git -C rename_test add OLDNAME && + git -C rename_test commit -m OLDNAME && + git -C rename_test mv OLDNAME NEWNAME && + git -C rename_test status --serialize=renamed.dat >output.1 && + echo DIRT >rename_test/DIRT && + git -C rename_test status --deserialize=renamed.dat >output.2 && + test_cmp output.1 output.2 +' + test_done diff --git a/wt-status-serialize.c b/wt-status-serialize.c index 9d5f970ea0aedd..7af50341619ed0 100644 --- a/wt-status-serialize.c +++ b/wt-status-serialize.c @@ -169,7 +169,7 @@ static inline void wt_serialize_v1_ignored(struct wt_status *s, int fd, } /* - * Serialize the list of changes to stdout. The goal of this + * Serialize the list of changes to the given file. The goal of this * is to just serialize the key fields in wt_status so that a * later command can rebuilt it and do the printing. * @@ -178,9 +178,8 @@ static inline void wt_serialize_v1_ignored(struct wt_status *s, int fd, * is relatively quick for the status consumer to compute * as necessary. */ -void wt_status_serialize_v1(struct wt_status *s) +void wt_status_serialize_v1(int fd, struct wt_status *s) { - int fd = 1; /* we always write to stdout */ struct string_list_item *iter; int k; diff --git a/wt-status.c b/wt-status.c index 486ae6614801fb..764cdbe2af1d4d 100644 --- a/wt-status.c +++ b/wt-status.c @@ -2622,7 +2622,7 @@ void wt_status_print(struct wt_status *s) wt_longstatus_print(s); break; case STATUS_FORMAT_SERIALIZE_V1: - wt_status_serialize_v1(s); + wt_status_serialize_v1(1, s); break; } diff --git a/wt-status.h b/wt-status.h index afeaf5753a0494..314185cc1ae8f3 100644 --- a/wt-status.h +++ b/wt-status.h @@ -224,7 +224,7 @@ struct wt_status_serialize_data * Serialize computed status scan results using "version 1" format * to the given file. */ -void wt_status_serialize_v1(struct wt_status *s); +void wt_status_serialize_v1(int fd, struct wt_status *s); /* * Deserialize existing status results from the given file and From 5f83daedc05df8d5ecc160857c278d085e2b7eaf Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 18 Jun 2021 14:45:20 +0200 Subject: [PATCH 058/189] gvfs: verify that the built-in FSMonitor is disabled When using a virtual file system layer, the FSMonitor does not make sense. Signed-off-by: Johannes Schindelin --- t/t1093-virtualfilesystem.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/t/t1093-virtualfilesystem.sh b/t/t1093-virtualfilesystem.sh index 8ba9a2a75e093a..cad13d680cb199 100755 --- a/t/t1093-virtualfilesystem.sh +++ b/t/t1093-virtualfilesystem.sh @@ -368,4 +368,15 @@ test_expect_success 'folder with same prefix as file' ' test_cmp expected actual ' +test_expect_success MINGW,FSMONITOR_DAEMON 'virtualfilesystem hook disables built-in FSMonitor' ' + clean_repo && + test_config core.usebuiltinfsmonitor true && + write_script .git/hooks/virtualfilesystem <<-\EOF && + printf "dir1/\0" + EOF + git config core.virtualfilesystem .git/hooks/virtualfilesystem && + git status && + test_must_fail git fsmonitor--daemon status +' + test_done From 5ef9e22ba8ff11e81a7eb99ac74c5d085eedbcdb Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 7 Feb 2018 10:59:03 -0500 Subject: [PATCH 059/189] status: reject deserialize in V2 and conflicts Teach status deserialize code to reject status cache when printing in porcelain V2 and there are unresolved conflicts in the cache file. A follow-on task might extend the cache format to include this additiona data. See code for longer explanation. Signed-off-by: Jeff Hostetler --- t/t7522-serialized-status.sh | 90 +++++++++++++++++++++++++++++++++++- wt-status-deserialize.c | 28 ++++++++++- 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index 2a81b4e625ee59..361afca94835e0 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -51,8 +51,14 @@ test_expect_success 'setup' ' test_oid_cache <<-EOF branch_oid sha1:68d4a437ea4c2de65800f48c053d4d543b55c410 + x_base sha1:587be6b4c3f93f93c489c0111bba5596147a26cb + x_ours sha1:b68025345d5301abad4d9ec9166f455243a0d746 + x_theirs sha1:975fbec8256d3e8a3797e7a3611380f27c49f4ac branch_oid sha256:6b95e4b1ea911dad213f2020840f5e92d3066cf9e38cf35f79412ec58d409ce4 + x_base sha256:14f5162e2fe3d240d0d37aaab0f90e4af9a7cfa79639f3bab005b5bfb4174d9f + x_ours sha256:3a404ba030a4afa912155c476a48a253d4b3a43d0098431b6d6ca6e554bd78fb + x_theirs sha256:44dc634218adec09e34f37839b3840bad8c6103693e9216626b32d00e093fa35 EOF ' @@ -171,7 +177,7 @@ test_expect_success 'verify no-ahead-behind and serialized status integration' ' ' test_expect_success 'verify new --serialize=path mode' ' - #test_when_finished "rm serialized_status.dat expect new_change.txt output.1 output.2" && + test_when_finished "rm serialized_status.dat expect new_change.txt output.1 output.2" && cat >expect <<-\EOF && ? expect ? output.1 @@ -193,6 +199,88 @@ test_expect_success 'verify new --serialize=path mode' ' test_cmp expect output.2 ' +test_expect_success 'merge conflicts' ' + + # create a merge conflict. + + git init -b main conflicts && + echo x >conflicts/x.txt && + git -C conflicts add x.txt && + git -C conflicts commit -m x && + git -C conflicts branch a && + git -C conflicts branch b && + git -C conflicts checkout a && + echo y >conflicts/x.txt && + git -C conflicts add x.txt && + git -C conflicts commit -m a && + git -C conflicts checkout b && + echo z >conflicts/x.txt && + git -C conflicts add x.txt && + git -C conflicts commit -m b && + test_must_fail git -C conflicts merge --no-commit a && + + # verify that regular status correctly identifies it + # in each format. + + cat >expect.v2 <observed.v2 && + test_cmp expect.v2 observed.v2 && + + cat >expect.long <..." to mark resolution) + both modified: x.txt + +no changes added to commit (use "git add" and/or "git commit -a") +EOF + git -C conflicts status --long >observed.long && + test_cmp expect.long observed.long && + + cat >expect.short <observed.short && + test_cmp expect.short observed.short && + + # save status data in serialized cache. + + git -C conflicts status --serialize >serialized && + + # make some dirt in the worktree so we can tell whether subsequent + # status commands used the cached data or did a fresh status. + + echo dirt >conflicts/dirt.txt && + + # run status using the cached data. + + git -C conflicts status --long --deserialize=../serialized >observed.long && + test_cmp expect.long observed.long && + + git -C conflicts status --short --deserialize=../serialized >observed.short && + test_cmp expect.short observed.short && + + # currently, the cached data does not have enough information about + # merge conflicts for porcelain V2 format. (And V2 format looks at + # the index to get that data, but the whole point of the serialization + # is to avoid reading the index unnecessarily.) So V2 always rejects + # the cached data when there is an unresolved conflict. + + cat >expect.v2.dirty <observed.v2 && + test_cmp expect.v2.dirty observed.v2 + +' + test_expect_success 'renames' ' git init -b main rename_test && echo OLDNAME >rename_test/OLDNAME && diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index 186d9d9c614962..10e4e292739569 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -182,7 +182,8 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) /* * Build a string-list of (count) lines from the input. */ -static int wt_deserialize_v1_changed_items(struct wt_status *s, int fd, int count) +static int wt_deserialize_v1_changed_items(const struct wt_status *cmd_s, + struct wt_status *s, int fd, int count) { struct wt_status_serialize_data *sd; char *p; @@ -240,6 +241,29 @@ static int wt_deserialize_v1_changed_items(struct wt_status *s, int fd, int coun oid_to_hex(&d->oid_index), item->string, (d->rename_source ? d->rename_source : "")); + + if (d->stagemask && + cmd_s->status_format == STATUS_FORMAT_PORCELAIN_V2) { + /* + * We have an unresolved conflict and the user wants + * to see porcelain V2 output. The cached status data + * does not contain enough information for V2 (because + * the main status computation does not capture it). + * We only get a single change record for the file with + * a single SHA -- we don't get the stage [123] mode + * and SHA data. The V2 detail-line print code looks + * up this information directly from the index. The + * whole point of this serialization cache is to avoid + * reading the index, so the V2 print code gets zeros. + * So we reject the status cache and let the fallback + * code run. + */ + trace_printf_key( + &trace_deserialize, + "reject: V2 format and umerged file: %s", + item->string); + return DESERIALIZE_ERR; + } } return DESERIALIZE_OK; @@ -392,7 +416,7 @@ static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, while ((line = my_packet_read_line(fd, &line_len))) { if (skip_prefix(line, "changed ", &arg)) { nr_changed = (int)strtol(arg, NULL, 10); - if (wt_deserialize_v1_changed_items(s, fd, nr_changed) + if (wt_deserialize_v1_changed_items(cmd_s, s, fd, nr_changed) == DESERIALIZE_ERR) return DESERIALIZE_ERR; continue; From ba33c06df7db88ccad7d7e89632544d3b4362693 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 21 Jun 2024 17:18:59 -0400 Subject: [PATCH 060/189] wt-status: add trace2 data for sparse-checkout percentage When sparse-checkout is enabled, add the sparse-checkout percentage to the Trace2 data stream. This number was already computed and printed on the console in the "You are in a sparse checkout..." message. It would be helpful to log it too for performance monitoring. Signed-off-by: Jeff Hostetler --- wt-status.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/wt-status.c b/wt-status.c index 214ebcae18afb4..e8166bfee44ebb 100644 --- a/wt-status.c +++ b/wt-status.c @@ -2564,6 +2564,36 @@ void wt_status_print(struct wt_status *s) s->untracked.nr); trace2_data_intmax("status", s->repo, "count/ignored", s->ignored.nr); + switch (s->state.sparse_checkout_percentage) { + case SPARSE_CHECKOUT_DISABLED: + break; + case SPARSE_CHECKOUT_SPARSE_INDEX: + /* + * Log just the observed size of the sparse-index. + * + * When sparse-index is enabled we can have + * sparse-directory entries in addition to individual + * sparse-file entries, so we don't know the complete + * size of the index. And we do not want to force + * expand it just to emit some telemetry data. So we + * cannot report a percentage for the space savings. + * + * It is possible that if the telemetry data is + * aggregated, someone will have a good estimate for + * the size of a fully populated index and can compute + * a percentage after the fact. + */ + trace2_data_intmax("status", s->repo, + "sparse-index/size", + s->repo->index->cache_nr); + break; + default: + trace2_data_intmax("status", s->repo, + "sparse-checkout/percentage", + s->state.sparse_checkout_percentage); + break; + } + trace2_region_enter("status", "print", s->repo); switch (s->status_format) { From 1076a667eb3d10ff5c964cafbd2da66d8610a871 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 20 Jul 2018 12:08:50 -0400 Subject: [PATCH 061/189] serialize-status: serialize global and repo-local exclude file metadata Changes to the global or repo-local excludes files can change the results returned by "git status" for untracked files. Therefore, it is important that the exclude-file values used during serialization are still current at the time of deserialization. Teach "git status --serialize" to report metadata on the user's global exclude file (which defaults to "$XDG_HOME/git/ignore") and for the repo-local excludes file (which is in ".git/info/excludes"). Serialize will record the pathnames and mtimes for these files in the serialization header (next to the mtime data for the .git/index file). Teach "git status --deserialize" to validate this new metadata. If either exclude file has changed since the serialization-cache-file was written, then deserialize will reject the cache file and force a full/normal status run. Signed-off-by: Jeff Hostetler --- wt-status-deserialize.c | 85 ++++++++++++++++++++++++++++ wt-status-serialize.c | 120 ++++++++++++++++++++++++++++++++++++++++ wt-status.h | 8 +++ 3 files changed, 213 insertions(+) diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index 10e4e292739569..35d1b025a711fd 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -8,6 +8,7 @@ #include "trace.h" #include "statinfo.h" #include "hex.h" +#include "path.h" static struct trace_key trace_deserialize = TRACE_KEY_INIT(DESERIALIZE); @@ -71,12 +72,69 @@ static int my_validate_index(const struct cache_time *mtime_reported) return DESERIALIZE_OK; } +/* + * Use the given key and exclude pathname to compute a serialization header + * reflecting the current contents on disk. See if that matches the value + * computed for this key when the cache was written. Reject the cache if + * anything has changed. + */ +static int my_validate_excludes(const char *path, const char *key, const char *line) +{ + struct strbuf sb = STRBUF_INIT; + int r; + + wt_serialize_compute_exclude_header(&sb, key, path); + + r = (strcmp(line, sb.buf) ? DESERIALIZE_ERR : DESERIALIZE_OK); + + if (r == DESERIALIZE_ERR) + trace_printf_key(&trace_deserialize, + "%s changed [cached '%s'][observed '%s']", + key, line, sb.buf); + + strbuf_release(&sb); + return r; +} + +static int my_parse_core_excludes(const char *line) +{ + /* + * In dir.c:setup_standard_excludes() they use either the value of + * the "core.excludefile" variable (stored in the global "excludes_file" + * variable) -or- the default value "$XDG_HOME/git/ignore". This is done + * during wt_status_collect_untracked() which we are hoping to not call. + * + * Fake the setup here. + */ + + if (excludes_file) { + return my_validate_excludes(excludes_file, "core_excludes", line); + } else { + char *path = xdg_config_home("ignore"); + int r = my_validate_excludes(path, "core_excludes", line); + free(path); + return r; + } +} + +static int my_parse_repo_excludes(const char *line) +{ + char *path = git_pathdup("info/exclude"); + int r = my_validate_excludes(path, "repo_excludes", line); + free(path); + + return r; +} + static int wt_deserialize_v1_header(struct wt_status *s, int fd) { struct cache_time index_mtime; int line_len, nr_fields; const char *line; const char *arg; + int have_required_index_mtime = 0; + int have_required_core_excludes = 0; + int have_required_repo_excludes = 0; /* * parse header lines up to the first flush packet. @@ -92,6 +150,20 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) nr_fields, line); return DESERIALIZE_ERR; } + have_required_index_mtime = 1; + continue; + } + + if (skip_prefix(line, "core_excludes ", &arg)) { + if (my_parse_core_excludes(line) != DESERIALIZE_OK) + return DESERIALIZE_ERR; + have_required_core_excludes = 1; + continue; + } + if (skip_prefix(line, "repo_excludes ", &arg)) { + if (my_parse_repo_excludes(line) != DESERIALIZE_OK) + return DESERIALIZE_ERR; + have_required_repo_excludes = 1; continue; } @@ -176,6 +248,19 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) return DESERIALIZE_ERR; } + if (!have_required_index_mtime) { + trace_printf_key(&trace_deserialize, "missing '%s'", "index_mtime"); + return DESERIALIZE_ERR; + } + if (!have_required_core_excludes) { + trace_printf_key(&trace_deserialize, "missing '%s'", "core_excludes"); + return DESERIALIZE_ERR; + } + if (!have_required_repo_excludes) { + trace_printf_key(&trace_deserialize, "missing '%s'", "repo_excludes"); + return DESERIALIZE_ERR; + } + return my_validate_index(&index_mtime); } diff --git a/wt-status-serialize.c b/wt-status-serialize.c index 7af50341619ed0..036979f30ad026 100644 --- a/wt-status-serialize.c +++ b/wt-status-serialize.c @@ -1,13 +1,131 @@ #include "git-compat-util.h" +#include "environment.h" #include "hex.h" #include "repository.h" #include "wt-status.h" #include "pkt-line.h" #include "trace.h" #include "read-cache-ll.h" +#include "path.h" static struct trace_key trace_serialize = TRACE_KEY_INIT(SERIALIZE); +/* + * Compute header record for exclude file using format: + * SP SP LF + */ +void wt_serialize_compute_exclude_header(struct strbuf *sb, + const char *key, + const char *path) +{ + struct stat st; + struct stat_data sd; + + memset(&sd, 0, sizeof(sd)); + + strbuf_setlen(sb, 0); + + if (!path || !*path) { + strbuf_addf(sb, "%s U (unset)", key); + } else if (lstat(path, &st) == -1) { + if (is_missing_file_error(errno)) + strbuf_addf(sb, "%s E (not-found) %s", key, path); + else + strbuf_addf(sb, "%s E (other) %s", key, path); + } else { + fill_stat_data(&sd, &st); + strbuf_addf(sb, "%s F %d %d %s", + key, sd.sd_mtime.sec, sd.sd_mtime.nsec, path); + } +} + +static void append_exclude_info(int fd, const char *path, const char *key) +{ + struct strbuf sb = STRBUF_INIT; + + wt_serialize_compute_exclude_header(&sb, key, path); + + packet_write_fmt(fd, "%s\n", sb.buf); + + strbuf_release(&sb); +} + +static void append_core_excludes_file_info(int fd) +{ + /* + * Write pathname and mtime of the core/global excludes file to + * the status cache header. Since a change in the global excludes + * will/may change the results reported by status, the deserialize + * code should be able to reject the status cache if the excludes + * file changes since when the cache was written. + * + * The "core.excludefile" setting defaults to $XDG_HOME/git/ignore + * and uses a global variable which should have been set during + * wt_status_collect_untracked(). + * + * See dir.c:setup_standard_excludes() + */ + append_exclude_info(fd, excludes_file, "core_excludes"); +} + +static void append_repo_excludes_file_info(int fd) +{ + /* + * Likewise, there is a per-repo excludes file in .git/info/excludes + * that can change the results reported by status. And the deserialize + * code needs to be able to reject the status cache if this file + * changes. + * + * See dir.c:setup_standard_excludes() and git_path_info_excludes(). + * We replicate the pathname construction here because of the static + * variables/functions used in dir.c. + */ + char *path = git_pathdup("info/exclude"); + + append_exclude_info(fd, path, "repo_excludes"); + + free(path); +} + +/* + * WARNING: The status cache attempts to preserve the essential in-memory + * status data after a status scan into a "serialization" (aka "status cache") + * file. It allows later "git status --deserialize=" instances to + * just print the cached status results without scanning the workdir (and + * without reading the index). + * + * The status cache file is valid as long as: + * [1] the set of functional command line options are the same (think "-u"). + * [2] repo-local and user-global configuration settings are compatible. + * [3] nothing in the workdir has changed. + * + * We rely on: + * [1.a] We remember the relevant (functional, non-display) command line + * arguments in the status cache header. + * [2.a] We use the mtime of the .git/index to detect staging changes. + * [2.b] We use the mtimes of the excludes files to detect changes that + * might affect untracked file reporting. + * + * But we need external help to verify [3]. + * [] This includes changes to tracked files. + * [] This includes changes to tracked .gitignore files that might change + * untracked file reporting. + * [] This includes the creation of new, untracked per-directory .gitignore + * files that might change untracked file reporting. + * + * [3.a] On GVFS repos, we rely on the GVFS service (mount) daemon to + * watch the filesystem and invalidate (delete) the status cache + * when anything changes inside the workdir. + * + * [3.b] TODO This problem is not solved for non-GVFS repos. + * [] It is possible that the untracked-cache index extension + * could help with this but that requires status to read the + * index to load the extension. + * [] It is possible that the new fsmonitor facility could also + * provide this information, but that to requires reading the + * index. + */ + /* * Write V1 header fields. */ @@ -20,6 +138,8 @@ static void wt_serialize_v1_header(struct wt_status *s, int fd) packet_write_fmt(fd, "index_mtime %d %d\n", s->repo->index->timestamp.sec, s->repo->index->timestamp.nsec); + append_core_excludes_file_info(fd); + append_repo_excludes_file_info(fd); /* * Write data from wt_status to qualify this status report. diff --git a/wt-status.h b/wt-status.h index 314185cc1ae8f3..9728117f0e9217 100644 --- a/wt-status.h +++ b/wt-status.h @@ -236,4 +236,12 @@ void wt_status_serialize_v1(int fd, struct wt_status *s); int wt_status_deserialize(const struct wt_status *cmd_s, const char *path); +/* + * A helper routine for serialize and deserialize to compute + * metadata for the user-global and repo-local excludes files. + */ +void wt_serialize_compute_exclude_header(struct strbuf *sb, + const char *key, + const char *path); + #endif /* STATUS_H */ From 9995935ffeb05d0ff8c1ed264672812f007faf95 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 24 Jun 2024 11:24:20 -0400 Subject: [PATCH 062/189] wt-status: add VFS hydration percentage to normal `git status` output Add VFS checkout hydration percentage information to the default `git status` output. When VFS is enable, users will now see a "You are in a partially-hydrated checkout with of tracked files present." message. Upstream `git status` normally prints a "You are in a sparse checkout with of tracked files present." This message was hidden in `microsoft/git` when `core_virtualfilesystem` is set (because GVFS users are always (and secretly) in a sparse checkout) and it was thought that it would annoy users. However, we now believe that it may be helpful for users to always see the percentage and know when they are over-hyrdated, since over-hyrdation can occur by accident and may greatly impact their Git performance. Knowing this value may help with GVFS support. Helped-by: Johannes Schindelin Signed-off-by: Jeff Hostetler --- t/t1093-virtualfilesystem.sh | 2 ++ wt-status.c | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/t/t1093-virtualfilesystem.sh b/t/t1093-virtualfilesystem.sh index cad13d680cb199..7786735dffec06 100755 --- a/t/t1093-virtualfilesystem.sh +++ b/t/t1093-virtualfilesystem.sh @@ -69,6 +69,8 @@ test_expect_success 'verify status is clean' ' git status > actual && cat > expected <<-\EOF && On branch main + You are in a partially-hydrated checkout with 75% of tracked files present. + nothing to commit, working tree clean EOF test_cmp expected actual diff --git a/wt-status.c b/wt-status.c index e8166bfee44ebb..cf5483510e0db8 100644 --- a/wt-status.c +++ b/wt-status.c @@ -1606,10 +1606,15 @@ static void show_sparse_checkout_in_use(struct wt_status *s, { if (s->state.sparse_checkout_percentage == SPARSE_CHECKOUT_DISABLED) return; - if (core_virtualfilesystem) - return; - - if (s->state.sparse_checkout_percentage == SPARSE_CHECKOUT_SPARSE_INDEX) + if (core_virtualfilesystem) { + if (s->state.sparse_checkout_percentage == SPARSE_CHECKOUT_SPARSE_INDEX) + status_printf_ln(s, color, + _("You are in a partially-hydrated checkout with a sparse index.")); + else + status_printf_ln(s, color, + _("You are in a partially-hydrated checkout with %d%% of tracked files present."), + s->state.sparse_checkout_percentage); + } else if (s->state.sparse_checkout_percentage == SPARSE_CHECKOUT_SPARSE_INDEX) status_printf_ln(s, color, _("You are in a sparse checkout.")); else status_printf_ln(s, color, From 27e5fa9b1712521835173290e84e11e5ed4a3c8d Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 25 Jul 2018 14:49:37 -0400 Subject: [PATCH 063/189] status: deserialization wait Teach `git status --deserialize` to either wait indefintely or immediately fail if the status serialization cache file is stale. Signed-off-by: Jeff Hostetler --- Documentation/config/status.txt | 16 +++++ builtin/commit.c | 59 +++++++++++++++- t/t7522-serialized-status.sh | 52 ++++++++++++++ wt-status-deserialize.c | 119 +++++++++++++++++++++++++++++--- wt-status.h | 12 +++- 5 files changed, 245 insertions(+), 13 deletions(-) diff --git a/Documentation/config/status.txt b/Documentation/config/status.txt index 7302b066644e73..4d863fdaaec2eb 100644 --- a/Documentation/config/status.txt +++ b/Documentation/config/status.txt @@ -83,3 +83,19 @@ status.deserializePath:: generated by `--serialize`. This will be overridden by `--deserialize=` on the command line. If the cache file is invalid or stale, git will fall-back and compute status normally. + +status.deserializeWait:: + EXPERIMENTAL, Specifies what `git status --deserialize` should do + if the serialization cache file is stale and whether it should + fall-back and compute status normally. This will be overridden by + `--deserialize-wait=` on the command line. ++ +-- +* `fail` - cause git to exit with an error when the status cache file +is stale; this is intended for testing and debugging. +* `block` - cause git to spin and periodically retry the cache file +every 100 ms; this is intended to help coordinate with another git +instance concurrently computing the cache file. +* `no` - to immediately fall-back if cache file is stale. This is the default. +* `` - time (in tenths of a second) to spin and retry. +-- diff --git a/builtin/commit.c b/builtin/commit.c index c84ccce3b29313..54a2f47aa2edbc 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -167,6 +167,9 @@ static int do_implicit_deserialize = 0; static int do_explicit_deserialize = 0; static char *deserialize_path = NULL; +static enum wt_status_deserialize_wait implicit_deserialize_wait = DESERIALIZE_WAIT__UNSET; +static enum wt_status_deserialize_wait explicit_deserialize_wait = DESERIALIZE_WAIT__UNSET; + /* * --serialize | --serialize= * @@ -232,6 +235,40 @@ static int opt_parse_deserialize(const struct option *opt, const char *arg, int return 0; } +static enum wt_status_deserialize_wait parse_dw(const char *arg) +{ + int tenths; + + if (!strcmp(arg, "fail")) + return DESERIALIZE_WAIT__FAIL; + else if (!strcmp(arg, "block")) + return DESERIALIZE_WAIT__BLOCK; + else if (!strcmp(arg, "no")) + return DESERIALIZE_WAIT__NO; + + /* + * Otherwise, assume it is a timeout in tenths of a second. + * If it contains a bogus value, atol() will return zero + * which is OK. + */ + tenths = atol(arg); + if (tenths < 0) + tenths = DESERIALIZE_WAIT__NO; + return tenths; +} + +static int opt_parse_deserialize_wait(const struct option *opt, + const char *arg, + int unset) +{ + if (unset) + explicit_deserialize_wait = DESERIALIZE_WAIT__UNSET; + else + explicit_deserialize_wait = parse_dw(arg); + + return 0; +} + static int opt_parse_m(const struct option *opt, const char *arg, int unset) { struct strbuf *buf = opt->value; @@ -1556,6 +1593,13 @@ static int git_status_config(const char *k, const char *v, } return 0; } + if (!strcmp(k, "status.deserializewait")) { + if (!v || !*v) + implicit_deserialize_wait = DESERIALIZE_WAIT__UNSET; + else + implicit_deserialize_wait = parse_dw(v); + return 0; + } if (!strcmp(k, "status.showuntrackedfiles")) { enum untracked_status_type u; @@ -1614,6 +1658,9 @@ int cmd_status(int argc, const char **argv, const char *prefix) { OPTION_CALLBACK, 0, "deserialize", NULL, N_("path"), N_("deserialize raw status data from file"), PARSE_OPT_OPTARG, opt_parse_deserialize }, + { OPTION_CALLBACK, 0, "deserialize-wait", NULL, + N_("fail|block|no"), N_("how to wait if status cache file is invalid"), + PARSE_OPT_OPTARG, opt_parse_deserialize_wait }, OPT_SET_INT(0, "long", &status_format, N_("show status in long format (default)"), STATUS_FORMAT_LONG), @@ -1710,11 +1757,21 @@ int cmd_status(int argc, const char **argv, const char *prefix) } if (try_deserialize) { + int result; + enum wt_status_deserialize_wait dw = implicit_deserialize_wait; + if (explicit_deserialize_wait != DESERIALIZE_WAIT__UNSET) + dw = explicit_deserialize_wait; + if (dw == DESERIALIZE_WAIT__UNSET) + dw = DESERIALIZE_WAIT__NO; + if (s.relative_paths) s.prefix = prefix; - if (wt_status_deserialize(&s, deserialize_path) == DESERIALIZE_OK) + result = wt_status_deserialize(&s, deserialize_path, dw); + if (result == DESERIALIZE_OK) return 0; + if (dw == DESERIALIZE_WAIT__FAIL) + die(_("Rejected status serialization cache")); /* deserialize failed, so force the initialization we skipped above. */ enable_fscache(1); diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index 361afca94835e0..edf15d7af45489 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -199,6 +199,58 @@ test_expect_success 'verify new --serialize=path mode' ' test_cmp expect output.2 ' +test_expect_success 'try deserialize-wait feature' ' + test_when_finished "rm -f serialized_status.dat dirt expect.* output.* trace.*" && + + git status --serialize=serialized_status.dat >output.1 && + + # make status cache stale by updating the mtime on the index. confirm that + # deserialize fails when requested. + sleep 1 && + touch .git/index && + test_must_fail git status --deserialize=serialized_status.dat --deserialize-wait=fail && + test_must_fail git -c status.deserializeWait=fail status --deserialize=serialized_status.dat && + + cat >expect.1 <<-\EOF && + ? expect.1 + ? output.1 + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + EOF + + # refresh the status cache. + git status --porcelain=v2 --serialize=serialized_status.dat >output.1 && + test_cmp expect.1 output.1 && + + # create some dirt. confirm deserialize used the existing status cache. + echo x >dirt && + git status --porcelain=v2 --deserialize=serialized_status.dat >output.2 && + test_cmp output.1 output.2 && + + # make the cache stale and try the timeout feature and wait upto + # 2 tenths of a second. confirm deserialize timed out and rejected + # the status cache and did a normal scan. + + cat >expect.2 <<-\EOF && + ? dirt + ? expect.1 + ? expect.2 + ? output.1 + ? output.2 + ? serialized_status.dat + ? trace.2 + ? untracked/ + ? untracked_1.txt + EOF + + sleep 1 && + touch .git/index && + GIT_TRACE_DESERIALIZE=1 git status --porcelain=v2 --deserialize=serialized_status.dat --deserialize-wait=2 >output.2 2>trace.2 && + test_cmp expect.2 output.2 && + grep "wait polled=2 result=1" trace.2 >trace.2g +' + test_expect_success 'merge conflicts' ' # create a merge conflict. diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index 35d1b025a711fd..a7699a14672bab 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -63,7 +63,8 @@ static int my_validate_index(const struct cache_time *mtime_reported) mtime_observed_on_disk.nsec = ST_MTIME_NSEC(st); if ((mtime_observed_on_disk.sec != mtime_reported->sec) || (mtime_observed_on_disk.nsec != mtime_reported->nsec)) { - trace_printf_key(&trace_deserialize, "index mtime changed [des %d.%d][obs %d.%d]", + trace_printf_key(&trace_deserialize, + "index mtime changed [des %d %d][obs %d %d]", mtime_reported->sec, mtime_reported->nsec, mtime_observed_on_disk.sec, mtime_observed_on_disk.nsec); return DESERIALIZE_ERR; @@ -555,6 +556,8 @@ static inline int my_strcmp_null(const char *a, const char *b) static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *des_s, int fd) { + memset(des_s, 0, sizeof(*des_s)); + /* * Check the path spec on the current command */ @@ -679,8 +682,101 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de return DESERIALIZE_OK; } +static struct cache_time deserialize_prev_mtime = { 0, 0 }; + +static int try_deserialize_read_from_file_1(const struct wt_status *cmd_s, + const char *path, + struct wt_status *des_s) +{ + struct stat st; + int result; + int fd; + + /* + * If we are spinning waiting for the status cache to become + * valid, skip re-reading it if the mtime has not changed + * since the last time we read it. + */ + if (lstat(path, &st)) { + trace_printf_key(&trace_deserialize, + "could not lstat '%s'", path); + return DESERIALIZE_ERR; + } + if (st.st_mtime == deserialize_prev_mtime.sec && + ST_MTIME_NSEC(st) == deserialize_prev_mtime.nsec) { + trace_printf_key(&trace_deserialize, + "mtime has not changed '%s'", path); + return DESERIALIZE_ERR; + } + + fd = xopen(path, O_RDONLY); + if (fd == -1) { + trace_printf_key(&trace_deserialize, + "could not read '%s'", path); + return DESERIALIZE_ERR; + } + + deserialize_prev_mtime.sec = st.st_mtime; + deserialize_prev_mtime.nsec = ST_MTIME_NSEC(st); + + trace_printf_key(&trace_deserialize, + "reading serialization file (%d %d) '%s'", + deserialize_prev_mtime.sec, + deserialize_prev_mtime.nsec, + path); + + result = wt_deserialize_fd(cmd_s, des_s, fd); + close(fd); + + return result; +} + +static int try_deserialize_read_from_file(const struct wt_status *cmd_s, + const char *path, + enum wt_status_deserialize_wait dw, + struct wt_status *des_s) +{ + int k, limit; + int result = DESERIALIZE_ERR; + + /* + * For "fail" or "no", try exactly once to read the status cache. + * Return an error if the file is stale. + */ + if (dw == DESERIALIZE_WAIT__FAIL || dw == DESERIALIZE_WAIT__NO) + return try_deserialize_read_from_file_1(cmd_s, path, des_s); + + /* + * Wait for the status cache file to refresh. Wait duration can + * be in tenths of a second or unlimited. Poll every 100ms. + */ + if (dw == DESERIALIZE_WAIT__BLOCK) { + /* + * Convert "unlimited" to 1 day. + */ + limit = 10 * 60 * 60 * 24; + } else { + /* spin for dw tenths of a second */ + limit = dw; + } + for (k = 0; k < limit; k++) { + result = try_deserialize_read_from_file_1( + cmd_s, path, des_s); + + if (result == DESERIALIZE_OK) + break; + + sleep_millisec(100); + } + + trace_printf_key(&trace_deserialize, + "wait polled=%d result=%d '%s'", + k, result, path); + return result; +} + /* - * Read raw serialized status data from the given file + * Read raw serialized status data from the given file (or STDIN). * * Verify that the args specified in the current command * are compatible with the deserialized data (such as "-uno"). @@ -688,24 +784,25 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de * Copy display-related fields from the current command * into the deserialized data (so that the user can request * long or short as they please). + * + * Print status report using cached data. */ int wt_status_deserialize(const struct wt_status *cmd_s, - const char *path) + const char *path, + enum wt_status_deserialize_wait dw) { struct wt_status des_s; int result; if (path && *path && strcmp(path, "0")) { - int fd = xopen(path, O_RDONLY); - if (fd == -1) { - trace_printf_key(&trace_deserialize, "could not read '%s'", path); - return DESERIALIZE_ERR; - } - trace_printf_key(&trace_deserialize, "reading serialization file '%s'", path); - result = wt_deserialize_fd(cmd_s, &des_s, fd); - close(fd); + result = try_deserialize_read_from_file(cmd_s, path, dw, &des_s); } else { trace_printf_key(&trace_deserialize, "reading stdin"); + + /* + * Read status cache data from stdin. Ignore the deserialize-wait + * term, since we cannot read stdin multiple times. + */ result = wt_deserialize_fd(cmd_s, &des_s, 0); } diff --git a/wt-status.h b/wt-status.h index 9728117f0e9217..b6cf4531fe56a4 100644 --- a/wt-status.h +++ b/wt-status.h @@ -220,6 +220,15 @@ struct wt_status_serialize_data - sizeof(struct wt_status_serialize_data_fixed)]; }; +enum wt_status_deserialize_wait +{ + DESERIALIZE_WAIT__UNSET = -3, + DESERIALIZE_WAIT__FAIL = -2, /* return error, do not fallback */ + DESERIALIZE_WAIT__BLOCK = -1, /* unlimited timeout */ + DESERIALIZE_WAIT__NO = 0, /* immediately fallback */ + /* any positive value is a timeout in tenths of a second */ +}; + /* * Serialize computed status scan results using "version 1" format * to the given file. @@ -234,7 +243,8 @@ void wt_status_serialize_v1(int fd, struct wt_status *s); * fields. */ int wt_status_deserialize(const struct wt_status *cmd_s, - const char *path); + const char *path, + enum wt_status_deserialize_wait dw); /* * A helper routine for serialize and deserialize to compute From c2877969886c9eae2f03ba23dc6696e18854e126 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 21 May 2019 23:14:48 +0200 Subject: [PATCH 064/189] merge-recursive: avoid confusing logic in was_dirty() It took this developer more than a moment to verify that was_dirty() really returns 0 (i.e. "false") if the file was not even tracked. In other words, the `dirty` variable that was initialized to 1 (i.e. "true") and then negated to be returned was not helping readability. The same holds for the final return: rather than assigning the value to return to `dirty` and then *immediately* returning that, we can simplify it to a single statement. --- merge-recursive.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/merge-recursive.c b/merge-recursive.c index 4e535faad39e7f..28574a8e577669 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -874,15 +874,13 @@ static int would_lose_untracked(struct merge_options *opt, const char *path) static int was_dirty(struct merge_options *opt, const char *path) { struct cache_entry *ce; - int dirty = 1; if (opt->priv->call_depth || !was_tracked(opt, path)) - return !dirty; + return 0; ce = index_file_exists(opt->priv->unpack_opts.src_index, path, strlen(path), ignore_case); - dirty = verify_uptodate(ce, &opt->priv->unpack_opts) != 0; - return dirty; + return verify_uptodate(ce, &opt->priv->unpack_opts) != 0; } static int make_room_for_path(struct merge_options *opt, const char *path) From 757f5d7865882773493961535e2d549b6ff982a1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 21 May 2019 23:17:46 +0200 Subject: [PATCH 065/189] merge-recursive: add some defensive coding to was_dirty() It took this developer quite a good while to understand why the current code cannot get a `NULL` returned by `index_file_exists()`. To un-confuse readers (and future-proof the code), let's just be safe and check before we dereference the returned pointer. --- merge-recursive.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merge-recursive.c b/merge-recursive.c index 28574a8e577669..cf9d909d5c1f3a 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -880,7 +880,7 @@ static int was_dirty(struct merge_options *opt, const char *path) ce = index_file_exists(opt->priv->unpack_opts.src_index, path, strlen(path), ignore_case); - return verify_uptodate(ce, &opt->priv->unpack_opts) != 0; + return !ce || verify_uptodate(ce, &opt->priv->unpack_opts) != 0; } static int make_room_for_path(struct merge_options *opt, const char *path) From c29558401d88b4c6eebabf756dedbf81007c620d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 21 May 2019 23:20:16 +0200 Subject: [PATCH 066/189] merge-recursive: teach was_dirty() about the virtualfilesystem The idea of the virtual file system really is to tell Git to avoid accessing certain paths. This fixes the case where a given path is not yet included in the virtual file system and we are about to write a conflicted version of it. --- merge-recursive.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/merge-recursive.c b/merge-recursive.c index cf9d909d5c1f3a..0a00568064340f 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -8,6 +8,7 @@ #include "git-compat-util.h" #include "merge-recursive.h" +#include "virtualfilesystem.h" #include "alloc.h" #include "cache-tree.h" @@ -875,7 +876,8 @@ static int was_dirty(struct merge_options *opt, const char *path) { struct cache_entry *ce; - if (opt->priv->call_depth || !was_tracked(opt, path)) + if (opt->priv->call_depth || !was_tracked(opt, path) || + is_excluded_from_virtualfilesystem(path, strlen(path), DT_REG) == 1) return 0; ce = index_file_exists(opt->priv->unpack_opts.src_index, From c766116f5032158c8225f32ae9def537a4c189b9 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 25 Jun 2019 16:38:50 -0400 Subject: [PATCH 067/189] status: deserialize with -uno does not print correct hint With the "--untracked-files=complete" option status computes a superset of the untracked files. We use this when writing the status cache. If subsequent deserialize commands ask for either the complete set or one of the "no", "normal", or "all" subsets, it can still use the cache file because of filtering in the deserialize parser. When running status with the "-uno" option, the long format status would print a "(use -u to show untracked files)" hint. When deserializing with the "-uno" option and using a cache computed with "-ucomplete", the "nothing to commit, working tree clean" message would be printed instead of the hint. It was easy to miss because the correct hint message was printed if the cache was rejected for any reason (and status did the full fallback). The "struct wt_status des" structure was initialized with the content of the status cache (and thus defaulted to "complete"). This change sets "des.show_untracked_files" to the requested subset from the command-line or config. This allows the long format to print the hint. Signed-off-by: Jeff Hostetler --- t/t7522-serialized-status.sh | 55 ++++++++++++++++++++++++++++++++++++ wt-status-deserialize.c | 16 +++++++---- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index edf15d7af45489..b52a9b7fa2f520 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -345,4 +345,59 @@ test_expect_success 'renames' ' test_cmp output.1 output.2 ' +test_expect_success 'hint message when cached with u=complete' ' + git init -b main hint && + echo xxx >hint/xxx && + git -C hint add xxx && + git -C hint commit -m xxx && + + cat >expect.clean <expect.use_u <hint.output_normal && + test_cmp expect.clean hint.output_normal && + + git -C hint status --untracked-files=all >hint.output_all && + test_cmp expect.clean hint.output_all && + + git -C hint status --untracked-files=no >hint.output_no && + test_cmp expect.use_u hint.output_no && + + # Create long format output for "complete" and create status cache. + + git -C hint status --untracked-files=complete --ignored=matching --serialize=../hint.dat >hint.output_complete && + test_cmp expect.clean hint.output_complete && + + # Capture long format output using the status cache and verify + # that the output matches the non-cached version. There are 2 + # ways to specify untracked-files, so do them both. + + git -C hint status --deserialize=../hint.dat -unormal >hint.d1_normal && + test_cmp expect.clean hint.d1_normal && + git -C hint -c status.showuntrackedfiles=normal status --deserialize=../hint.dat >hint.d2_normal && + test_cmp expect.clean hint.d2_normal && + + git -C hint status --deserialize=../hint.dat -uall >hint.d1_all && + test_cmp expect.clean hint.d1_all && + git -C hint -c status.showuntrackedfiles=all status --deserialize=../hint.dat >hint.d2_all && + test_cmp expect.clean hint.d2_all && + + git -C hint status --deserialize=../hint.dat -uno >hint.d1_no && + test_cmp expect.use_u hint.d1_no && + git -C hint -c status.showuntrackedfiles=no status --deserialize=../hint.dat >hint.d2_no && + test_cmp expect.use_u hint.d2_no + +' + test_done diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index a7699a14672bab..d3a8415981d648 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -425,20 +425,24 @@ static int wt_deserialize_v1_ignored_items(struct wt_status *s, } static int validate_untracked_files_arg(enum untracked_status_type cmd, - enum untracked_status_type des, + enum untracked_status_type *des, enum deserialize_parse_strategy *strategy) { *strategy = DESERIALIZE_STRATEGY_AS_IS; - if (cmd == des) { + if (cmd == *des) { *strategy = DESERIALIZE_STRATEGY_AS_IS; } else if (cmd == SHOW_NO_UNTRACKED_FILES) { *strategy = DESERIALIZE_STRATEGY_SKIP; - } else if (des == SHOW_COMPLETE_UNTRACKED_FILES) { - if (cmd == SHOW_ALL_UNTRACKED_FILES) + *des = cmd; + } else if (*des == SHOW_COMPLETE_UNTRACKED_FILES) { + if (cmd == SHOW_ALL_UNTRACKED_FILES) { *strategy = DESERIALIZE_STRATEGY_ALL; - else if (cmd == SHOW_NORMAL_UNTRACKED_FILES) + *des = cmd; + } else if (cmd == SHOW_NORMAL_UNTRACKED_FILES) { *strategy = DESERIALIZE_STRATEGY_NORMAL; + *des = cmd; + } } else { return DESERIALIZE_ERR; } @@ -480,7 +484,7 @@ static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, * We now have the header parsed. Look at the command args (as passed in), and see how to parse * the serialized data */ - if (validate_untracked_files_arg(cmd_s->show_untracked_files, s->show_untracked_files, &untracked_strategy)) { + if (validate_untracked_files_arg(cmd_s->show_untracked_files, &s->show_untracked_files, &untracked_strategy)) { trace_printf_key(&trace_deserialize, "reject: show_untracked_file: command: %d, serialized : %d", cmd_s->show_untracked_files, s->show_untracked_files); From d9b2bce7ccedccc2ed95cc52580224f516401024 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 21 Nov 2019 12:01:04 -0700 Subject: [PATCH 068/189] fsmonitor: check CE_FSMONITOR_VALID in ce_uptodate When using fsmonitor the CE_FSMONITOR_VALID flag should be checked when wanting to know if the entry has been updated. If the flag is set the entry should be considered up to date and the same as if the CE_UPTODATE is set. In order to trust the CE_FSMONITOR_VALID flag, the fsmonitor data needs to be refreshed when the fsmonitor bitmap is applied to the index in tweak_fsmonitor. Since the fsmonitor data is kept up to date for every command, some tests needed to be updated to take that into account. istate->untracked->use_fsmonitor was set in tweak_fsmonitor when the fsmonitor bitmap data was loaded and is now in refresh_fsmonitor since that is being called in tweak_fsmonitor. refresh_fsmonitor will only be called once and any other callers should be setting it when refreshing the fsmonitor data so that code can use the fsmonitor data when checking untracked files. When writing the index, fsmonitor_last_update is used to determine if the fsmonitor bitmap should be created and the extension data written to the index. When running through unpack-trees this is not copied to the result index. This makes the next time a git command is ran do all the work of lstating all files to determine what is clean since all entries in the index are marked as dirty since there wasn't any fsmonitor data saved in the index extension. Copying the fsmonitor_last_update to the result index will cause the extension data for fsmonitor to be in the index for the next git command to use. Signed-off-by: Kevin Willford --- read-cache-ll.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/read-cache-ll.h b/read-cache-ll.h index e0e39607ef3231..a78d86cff54a75 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -118,7 +118,7 @@ static inline unsigned create_ce_flags(unsigned stage) #define ce_namelen(ce) ((ce)->ce_namelen) #define ce_size(ce) cache_entry_size(ce_namelen(ce)) #define ce_stage(ce) ((CE_STAGEMASK & (ce)->ce_flags) >> CE_STAGESHIFT) -#define ce_uptodate(ce) ((ce)->ce_flags & CE_UPTODATE) +#define ce_uptodate(ce) (((ce)->ce_flags & CE_UPTODATE) || ((ce)->ce_flags & CE_FSMONITOR_VALID)) #define ce_skip_worktree(ce) ((ce)->ce_flags & CE_SKIP_WORKTREE) #define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE) #define ce_intent_to_add(ce) ((ce)->ce_flags & CE_INTENT_TO_ADD) From a92b30465794dc45d181fe9ff6bf803560d82790 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 21 Nov 2019 09:24:36 -0700 Subject: [PATCH 069/189] fsmonitor: add script for debugging and update script for tests The fsmonitor script that can be used for running all the git tests using watchman was causing some of the tests to fail because it wrote to stderr and created some files for debugging purposes. Add a new debug script to use with debugging and modify the other script to remove the code that would cause tests to fail. Signed-off-by: Kevin Willford --- t/t7519/fsmonitor-watchman | 22 +----- t/t7519/fsmonitor-watchman-debug | 128 +++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 21 deletions(-) create mode 100755 t/t7519/fsmonitor-watchman-debug diff --git a/t/t7519/fsmonitor-watchman b/t/t7519/fsmonitor-watchman index 264b9daf834ec8..6461f625f64181 100755 --- a/t/t7519/fsmonitor-watchman +++ b/t/t7519/fsmonitor-watchman @@ -17,7 +17,6 @@ use IPC::Open2; # 'git config core.fsmonitor .git/hooks/query-watchman' # my ($version, $time) = @ARGV; -#print STDERR "$0 $version $time\n"; # Check the hook interface version @@ -44,7 +43,7 @@ launch_watchman(); sub launch_watchman { - my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j') + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty') or die "open2() failed: $!\n" . "Falling back to scanning...\n"; @@ -62,19 +61,11 @@ sub launch_watchman { "fields": ["name"] }] END - - open (my $fh, ">", ".git/watchman-query.json"); - print $fh $query; - close $fh; print CHLD_IN $query; close CHLD_IN; my $response = do {local $/; }; - open ($fh, ">", ".git/watchman-response.json"); - print $fh $response; - close $fh; - die "Watchman: command returned no output.\n" . "Falling back to scanning...\n" if $response eq ""; die "Watchman: command returned invalid output: $response\n" . @@ -93,7 +84,6 @@ sub launch_watchman { my $o = $json_pkg->new->utf8->decode($response); if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) { - print STDERR "Adding '$git_work_tree' to watchman's watch list.\n"; $retry--; qx/watchman watch "$git_work_tree"/; die "Failed to make watchman watch '$git_work_tree'.\n" . @@ -103,11 +93,6 @@ sub launch_watchman { # return the fast "everything is dirty" flag to git and do the # Watchman query just to get it over with now so we won't pay # the cost in git to look up each individual file. - - open ($fh, ">", ".git/watchman-output.out"); - print "/\0"; - close $fh; - print "/\0"; eval { launch_watchman() }; exit 0; @@ -116,11 +101,6 @@ sub launch_watchman { die "Watchman: $o->{error}.\n" . "Falling back to scanning...\n" if $o->{error}; - open ($fh, ">", ".git/watchman-output.out"); - binmode $fh, ":utf8"; - print $fh @{$o->{files}}; - close $fh; - binmode STDOUT, ":utf8"; local $, = "\0"; print @{$o->{files}}; diff --git a/t/t7519/fsmonitor-watchman-debug b/t/t7519/fsmonitor-watchman-debug new file mode 100755 index 00000000000000..d8e7a1e5ba85c0 --- /dev/null +++ b/t/t7519/fsmonitor-watchman-debug @@ -0,0 +1,128 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use IPC::Open2; + +# An example hook script to integrate Watchman +# (https://facebook.github.io/watchman/) with git to speed up detecting +# new and modified files. +# +# The hook is passed a version (currently 1) and a time in nanoseconds +# formatted as a string and outputs to stdout all files that have been +# modified since the given time. Paths must be relative to the root of +# the working tree and separated by a single NUL. +# +# To enable this hook, rename this file to "query-watchman" and set +# 'git config core.fsmonitor .git/hooks/query-watchman' +# +my ($version, $time) = @ARGV; +#print STDERR "$0 $version $time\n"; + +# Check the hook interface version + +if ($version == 1) { + # convert nanoseconds to seconds + # subtract one second to make sure watchman will return all changes + $time = int ($time / 1000000000) - 1; +} else { + die "Unsupported query-fsmonitor hook version '$version'.\n" . + "Falling back to scanning...\n"; +} + +my $git_work_tree; +if ($^O =~ 'msys' || $^O =~ 'cygwin') { + $git_work_tree = Win32::GetCwd(); + $git_work_tree =~ tr/\\/\//; +} else { + require Cwd; + $git_work_tree = Cwd::cwd(); +} + +my $retry = 1; + +launch_watchman(); + +sub launch_watchman { + + my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j') + or die "open2() failed: $!\n" . + "Falling back to scanning...\n"; + + # In the query expression below we're asking for names of files that + # changed since $time but were not transient (ie created after + # $time but no longer exist). + # + # To accomplish this, we're using the "since" generator to use the + # recency index to select candidate nodes and "fields" to limit the + # output to file names only. + + my $query = <<" END"; + ["query", "$git_work_tree", { + "since": $time, + "fields": ["name"] + }] + END + + open (my $fh, ">", ".git/watchman-query.json"); + print $fh $query; + close $fh; + + print CHLD_IN $query; + close CHLD_IN; + my $response = do {local $/; }; + + open ($fh, ">", ".git/watchman-response.json"); + print $fh $response; + close $fh; + + die "Watchman: command returned no output.\n" . + "Falling back to scanning...\n" if $response eq ""; + die "Watchman: command returned invalid output: $response\n" . + "Falling back to scanning...\n" unless $response =~ /^\{/; + + my $json_pkg; + eval { + require JSON::XS; + $json_pkg = "JSON::XS"; + 1; + } or do { + require JSON::PP; + $json_pkg = "JSON::PP"; + }; + + my $o = $json_pkg->new->utf8->decode($response); + + if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) { + print STDERR "Adding '$git_work_tree' to watchman's watch list.\n"; + $retry--; + qx/watchman watch "$git_work_tree"/; + die "Failed to make watchman watch '$git_work_tree'.\n" . + "Falling back to scanning...\n" if $? != 0; + + # Watchman will always return all files on the first query so + # return the fast "everything is dirty" flag to git and do the + # Watchman query just to get it over with now so we won't pay + # the cost in git to look up each individual file. + + open ($fh, ">", ".git/watchman-output.out"); + print "/\0"; + close $fh; + + print "/\0"; + eval { launch_watchman() }; + exit 0; + } + + die "Watchman: $o->{error}.\n" . + "Falling back to scanning...\n" if $o->{error}; + + open ($fh, ">", ".git/watchman-output.out"); + binmode $fh, ":utf8"; + print $fh @{$o->{files}}; + close $fh; + + binmode STDOUT, ":utf8"; + local $, = "\0"; + print @{$o->{files}}; +} From 863ebb57611f56b7467c480b532c294706507baa Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 26 Sep 2018 12:29:26 -0400 Subject: [PATCH 070/189] gvfs:trace2:data: add trace2 tracing around read_object_process Add trace2 region around read_object_process to collect time spent waiting for missing objects to be dynamically fetched. Signed-off-by: Jeff Hostetler --- object-file.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/object-file.c b/object-file.c index 0024936fd210f1..4b3eef527d0f92 100644 --- a/object-file.c +++ b/object-file.c @@ -41,6 +41,7 @@ #include "loose.h" #include "object-file-convert.h" #include "trace.h" +#include "trace2.h" #include "hook.h" #include "sigchain.h" #include "sub-process.h" @@ -990,6 +991,8 @@ static int read_object_process(const struct object_id *oid) start = getnanotime(); + trace2_region_enter("subprocess", "read_object", the_repository); + if (!subprocess_map_initialized) { subprocess_map_initialized = 1; hashmap_init(&subprocess_map, (hashmap_cmp_fn)cmd2process_cmp, @@ -1006,13 +1009,16 @@ static int read_object_process(const struct object_id *oid) if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, start_read_object_fn)) { free(entry); - return -1; + err = -1; + goto leave_region; } } process = &entry->subprocess.process; - if (!(CAP_GET & entry->supported_capabilities)) - return -1; + if (!(CAP_GET & entry->supported_capabilities)) { + err = -1; + goto leave_region; + } sigchain_push(SIGPIPE, SIG_IGN); @@ -1061,6 +1067,10 @@ static int read_object_process(const struct object_id *oid) trace_performance_since(start, "read_object_process"); +leave_region: + trace2_region_leave_printf("subprocess", "read_object", the_repository, + "result %d", err); + return err; } From 865bb627ea7c041599166bb7d9e8c7ac0ccb754d Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 26 Sep 2018 11:21:22 -0400 Subject: [PATCH 071/189] gvfs:trace2:data: status deserialization information Add trace2 region and data events describing attempts to deserialize status data using a status cache. A category:status, label:deserialize region is pushed around the deserialize code. Deserialization results when reading from a file are: category:status, path = category:status, polled = category:status, result = "ok" | "reject" When reading from STDIN are: category:status, path = "STDIN" category:status, result = "ok" | "reject" Status will fallback and run a normal status scan when a "reject" is reported (unless "--deserialize-wait=fail"). If "ok" is reported, status was able to use the status cache and avoid scanning the workdir. Additionally, a cmd_mode is emitted for each step: collection, deserialization, and serialization. For example, if deserialization is attempted and fails and status falls back to actually computing the status, a cmd_mode message containing "deserialize" is issued and then a cmd_mode for "collect" is issued. Also, if deserialization fails, a data message containing the rejection reason is emitted. Signed-off-by: Jeff Hostetler --- builtin/commit.c | 19 +++++++++++- wt-status-deserialize.c | 67 ++++++++++++++++++++++++++++++++++++++--- wt-status.h | 2 ++ 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index 81a7aa2a323dbd..6ec1f3bdd78186 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -164,6 +164,7 @@ static int opt_parse_porcelain(const struct option *opt, const char *arg, int un static int do_serialize = 0; static char *serialize_path = NULL; +static int reject_implicit = 0; static int do_implicit_deserialize = 0; static int do_explicit_deserialize = 0; static char *deserialize_path = NULL; @@ -227,7 +228,7 @@ static int opt_parse_deserialize(const struct option *opt, const char *arg, int } if (!deserialize_path || !*deserialize_path) do_explicit_deserialize = 1; /* read stdin */ - else if (access(deserialize_path, R_OK) == 0) + else if (wt_status_deserialize_access(deserialize_path, R_OK) == 0) do_explicit_deserialize = 1; /* can read from this file */ else { /* @@ -1597,6 +1598,8 @@ static int git_status_config(const char *k, const char *v, if (v && *v && access(v, R_OK) == 0) { do_implicit_deserialize = 1; deserialize_path = xstrdup(v); + } else { + reject_implicit = 1; } return 0; } @@ -1747,6 +1750,17 @@ int cmd_status(int argc, const char **argv, const char *prefix) if (try_deserialize) goto skip_init; + /* + * If we implicitly received a status cache pathname from the config + * and the file does not exist, we silently reject it and do the normal + * status "collect". Fake up some trace2 messages to reflect this and + * assist post-processors know this case is different. + */ + if (!do_serialize && reject_implicit) { + trace2_cmd_mode("implicit-deserialize"); + trace2_data_string("status", the_repository, "deserialize/reject", + "status-cache/access"); + } enable_fscache(0); if (status_format != STATUS_FORMAT_PORCELAIN && @@ -1790,6 +1804,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) if (s.relative_paths) s.prefix = prefix; + trace2_cmd_mode("deserialize"); result = wt_status_deserialize(&s, deserialize_path, dw); if (result == DESERIALIZE_OK) return 0; @@ -1807,6 +1822,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) fd = -1; } + trace2_cmd_mode("collect"); wt_status_collect(&s); if (0 <= fd) @@ -1821,6 +1837,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) if (fd_serialize < 0) die_errno(_("could not serialize to '%s'"), serialize_path); + trace2_cmd_mode("serialize"); wt_status_serialize_v1(fd_serialize, &s); close(fd_serialize); } diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index d3a8415981d648..67237027bafdb6 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -9,6 +9,23 @@ #include "statinfo.h" #include "hex.h" #include "path.h" +#include "trace2.h" + +static void set_deserialize_reject_reason(const char *reason) +{ + trace2_data_string("status", the_repository, "deserialize/reject", + reason); +} + +int wt_status_deserialize_access(const char *path, int mode) +{ + int a = access(path, mode); + + if (a != 0) + set_deserialize_reject_reason("status-cache/access"); + + return a; +} static struct trace_key trace_deserialize = TRACE_KEY_INIT(DESERIALIZE); @@ -56,6 +73,7 @@ static int my_validate_index(const struct cache_time *mtime_reported) struct cache_time mtime_observed_on_disk; if (lstat(path, &st)) { + set_deserialize_reject_reason("index/not-found"); trace_printf_key(&trace_deserialize, "could not stat index"); return DESERIALIZE_ERR; } @@ -63,6 +81,7 @@ static int my_validate_index(const struct cache_time *mtime_reported) mtime_observed_on_disk.nsec = ST_MTIME_NSEC(st); if ((mtime_observed_on_disk.sec != mtime_reported->sec) || (mtime_observed_on_disk.nsec != mtime_reported->nsec)) { + set_deserialize_reject_reason("index/mtime-changed"); trace_printf_key(&trace_deserialize, "index mtime changed [des %d %d][obs %d %d]", mtime_reported->sec, mtime_reported->nsec, @@ -88,10 +107,12 @@ static int my_validate_excludes(const char *path, const char *key, const char *l r = (strcmp(line, sb.buf) ? DESERIALIZE_ERR : DESERIALIZE_OK); - if (r == DESERIALIZE_ERR) + if (r == DESERIALIZE_ERR) { + set_deserialize_reject_reason("excludes/changed"); trace_printf_key(&trace_deserialize, "%s changed [cached '%s'][observed '%s']", key, line, sb.buf); + } strbuf_release(&sb); return r; @@ -147,6 +168,7 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) &index_mtime.sec, &index_mtime.nsec); if (nr_fields != 2) { + set_deserialize_reject_reason("v1-header/invalid-index-mtime"); trace_printf_key(&trace_deserialize, "invalid index_mtime (%d) '%s'", nr_fields, line); return DESERIALIZE_ERR; @@ -230,6 +252,7 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) /* status_format */ if (skip_prefix(line, "sha1_commit ", &arg)) { if (get_oid_hex(arg, &s->oid_commit)) { + set_deserialize_reject_reason("v1-header/invalid-commit-sha"); trace_printf_key(&trace_deserialize, "invalid sha1_commit"); return DESERIALIZE_ERR; } @@ -245,19 +268,23 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) } /* prefix */ + set_deserialize_reject_reason("v1-header/unexpected-line"); trace_printf_key(&trace_deserialize, "unexpected line '%s'", line); return DESERIALIZE_ERR; } if (!have_required_index_mtime) { + set_deserialize_reject_reason("v1-header/missing-index-mtime"); trace_printf_key(&trace_deserialize, "missing '%s'", "index_mtime"); return DESERIALIZE_ERR; } if (!have_required_core_excludes) { + set_deserialize_reject_reason("v1-header/missing-core-excludes"); trace_printf_key(&trace_deserialize, "missing '%s'", "core_excludes"); return DESERIALIZE_ERR; } if (!have_required_repo_excludes) { + set_deserialize_reject_reason("v1-header/missing-repo-excludes"); trace_printf_key(&trace_deserialize, "missing '%s'", "repo_excludes"); return DESERIALIZE_ERR; } @@ -344,6 +371,7 @@ static int wt_deserialize_v1_changed_items(const struct wt_status *cmd_s, * So we reject the status cache and let the fallback * code run. */ + set_deserialize_reject_reason("v1-data/unmerged"); trace_printf_key( &trace_deserialize, "reject: V2 format and umerged file: %s", @@ -485,6 +513,7 @@ static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, * the serialized data */ if (validate_untracked_files_arg(cmd_s->show_untracked_files, &s->show_untracked_files, &untracked_strategy)) { + set_deserialize_reject_reason("args/untracked-files"); trace_printf_key(&trace_deserialize, "reject: show_untracked_file: command: %d, serialized : %d", cmd_s->show_untracked_files, s->show_untracked_files); @@ -492,6 +521,7 @@ static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, } if (validate_ignored_files_arg(cmd_s->show_ignored_mode, s->show_ignored_mode, &ignored_strategy)) { + set_deserialize_reject_reason("args/ignored-mode"); trace_printf_key(&trace_deserialize, "reject: show_ignored_mode: command: %d, serialized: %d", cmd_s->show_ignored_mode, s->show_ignored_mode); @@ -525,6 +555,7 @@ static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, return DESERIALIZE_ERR; continue; } + set_deserialize_reject_reason("v1-data/unexpected-line"); trace_printf_key(&trace_deserialize, "unexpected line '%s'", line); return DESERIALIZE_ERR; } @@ -546,6 +577,7 @@ static int wt_deserialize_parse(const struct wt_status *cmd_s, struct wt_status if (version == 1) return wt_deserialize_v1(cmd_s, s, fd); } + set_deserialize_reject_reason("status-cache/unsupported-version"); trace_printf_key(&trace_deserialize, "missing/unsupported version"); return DESERIALIZE_ERR; } @@ -566,6 +598,7 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de * Check the path spec on the current command */ if (cmd_s->pathspec.nr > 1) { + set_deserialize_reject_reason("args/multiple-pathspecs"); trace_printf_key(&trace_deserialize, "reject: multiple pathspecs"); return DESERIALIZE_ERR; } @@ -576,6 +609,7 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de */ if (cmd_s->pathspec.nr == 1 && my_strcmp_null(cmd_s->pathspec.items[0].match, "")) { + set_deserialize_reject_reason("args/root-pathspec"); trace_printf_key(&trace_deserialize, "reject: pathspec"); return DESERIALIZE_ERR; } @@ -592,20 +626,24 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de * or "--ignored" settings). */ if (cmd_s->is_initial != des_s->is_initial) { + set_deserialize_reject_reason("args/is-initial-changed"); trace_printf_key(&trace_deserialize, "reject: is_initial"); return DESERIALIZE_ERR; } if (my_strcmp_null(cmd_s->branch, des_s->branch)) { + set_deserialize_reject_reason("args/branch-changed"); trace_printf_key(&trace_deserialize, "reject: branch"); return DESERIALIZE_ERR; } if (my_strcmp_null(cmd_s->reference, des_s->reference)) { + set_deserialize_reject_reason("args/reference-changed"); trace_printf_key(&trace_deserialize, "reject: reference"); return DESERIALIZE_ERR; } /* verbose */ /* amend */ if (cmd_s->whence != des_s->whence) { + set_deserialize_reject_reason("args/whence-changed"); trace_printf_key(&trace_deserialize, "reject: whence"); return DESERIALIZE_ERR; } @@ -639,19 +677,23 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de /* hints */ /* ahead_behind_flags */ if (cmd_s->detect_rename != des_s->detect_rename) { + set_deserialize_reject_reason("args/detect-rename-changed"); trace_printf_key(&trace_deserialize, "reject: detect_rename"); return DESERIALIZE_ERR; } if (cmd_s->rename_score != des_s->rename_score) { + set_deserialize_reject_reason("args/rename-score-changed"); trace_printf_key(&trace_deserialize, "reject: rename_score"); return DESERIALIZE_ERR; } if (cmd_s->rename_limit != des_s->rename_limit) { + set_deserialize_reject_reason("args/rename-limit-changed"); trace_printf_key(&trace_deserialize, "reject: rename_limit"); return DESERIALIZE_ERR; } /* status_format */ if (!oideq(&cmd_s->oid_commit, &des_s->oid_commit)) { + set_deserialize_reject_reason("args/commit-changed"); trace_printf_key(&trace_deserialize, "reject: sha1_commit"); return DESERIALIZE_ERR; } @@ -740,15 +782,18 @@ static int try_deserialize_read_from_file(const struct wt_status *cmd_s, enum wt_status_deserialize_wait dw, struct wt_status *des_s) { - int k, limit; + int k = 0; + int limit; int result = DESERIALIZE_ERR; /* * For "fail" or "no", try exactly once to read the status cache. * Return an error if the file is stale. */ - if (dw == DESERIALIZE_WAIT__FAIL || dw == DESERIALIZE_WAIT__NO) - return try_deserialize_read_from_file_1(cmd_s, path, des_s); + if (dw == DESERIALIZE_WAIT__FAIL || dw == DESERIALIZE_WAIT__NO) { + result = try_deserialize_read_from_file_1(cmd_s, path, des_s); + goto done; + } /* * Wait for the status cache file to refresh. Wait duration can @@ -773,6 +818,12 @@ static int try_deserialize_read_from_file(const struct wt_status *cmd_s, sleep_millisec(100); } +done: + trace2_data_string("status", the_repository, "deserialize/path", path); + trace2_data_intmax("status", the_repository, "deserialize/polled", k); + trace2_data_string("status", the_repository, "deserialize/result", + ((result == DESERIALIZE_OK) ? "ok" : "reject")); + trace_printf_key(&trace_deserialize, "wait polled=%d result=%d '%s'", k, result, path); @@ -798,6 +849,8 @@ int wt_status_deserialize(const struct wt_status *cmd_s, struct wt_status des_s; int result; + trace2_region_enter("status", "deserialize", the_repository); + if (path && *path && strcmp(path, "0")) { result = try_deserialize_read_from_file(cmd_s, path, dw, &des_s); } else { @@ -808,8 +861,14 @@ int wt_status_deserialize(const struct wt_status *cmd_s, * term, since we cannot read stdin multiple times. */ result = wt_deserialize_fd(cmd_s, &des_s, 0); + + trace2_data_string("status", the_repository, "deserialize/path", "STDIN"); + trace2_data_string("status", the_repository, "deserialize/result", + ((result == DESERIALIZE_OK) ? "ok" : "reject")); } + trace2_region_leave("status", "deserialize", the_repository); + if (result == DESERIALIZE_OK) { wt_status_get_state(cmd_s->repo, &des_s.state, des_s.branch && !strcmp(des_s.branch, "HEAD")); diff --git a/wt-status.h b/wt-status.h index b6cf4531fe56a4..03c25186491990 100644 --- a/wt-status.h +++ b/wt-status.h @@ -246,6 +246,8 @@ int wt_status_deserialize(const struct wt_status *cmd_s, const char *path, enum wt_status_deserialize_wait dw); +int wt_status_deserialize_access(const char *path, int mode); + /* * A helper routine for serialize and deserialize to compute * metadata for the user-global and repo-local excludes files. From a7f11502d905ffec4f0848c70b5ffedff83f8070 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 7 Jan 2019 12:45:48 -0500 Subject: [PATCH 072/189] gvfs:trace2:data: status serialization Add trace information around status serialization. Signed-off-by: Jeff Hostetler --- wt-status-serialize.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/wt-status-serialize.c b/wt-status-serialize.c index 036979f30ad026..6688020d58af56 100644 --- a/wt-status-serialize.c +++ b/wt-status-serialize.c @@ -1,3 +1,4 @@ +#define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" #include "environment.h" #include "hex.h" @@ -7,6 +8,7 @@ #include "trace.h" #include "read-cache-ll.h" #include "path.h" +#include "trace2.h" static struct trace_key trace_serialize = TRACE_KEY_INIT(SERIALIZE); @@ -303,6 +305,8 @@ void wt_status_serialize_v1(int fd, struct wt_status *s) struct string_list_item *iter; int k; + trace2_region_enter("status", "serialize", the_repository); + /* * version header must be first line. */ @@ -336,4 +340,6 @@ void wt_status_serialize_v1(int fd, struct wt_status *s) } packet_flush(fd); } + + trace2_region_leave("status", "serialize", the_repository); } From 237b0ddc73b2c448528facd0d436ea59b9a3242a Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 19 Nov 2018 16:26:37 -0500 Subject: [PATCH 073/189] gvfs:trace2:data: add vfs stats Report virtual filesystem summary data. Signed-off-by: Jeff Hostetler --- virtualfilesystem.c | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/virtualfilesystem.c b/virtualfilesystem.c index 99f5d24155a1bd..e6ec336ba3d605 100644 --- a/virtualfilesystem.c +++ b/virtualfilesystem.c @@ -1,6 +1,8 @@ +#define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" #include "environment.h" #include "gettext.h" +#include "trace2.h" #include "config.h" #include "dir.h" #include "hashmap.h" @@ -258,6 +260,11 @@ void apply_virtualfilesystem(struct index_state *istate) { char *buf, *entry; int i; + int nr_unknown = 0; + int nr_vfs_dirs = 0; + int nr_vfs_rows = 0; + int nr_bulk_skip = 0; + int nr_explicit_skip = 0; if (!git_config_get_virtualfilesystem()) return; @@ -275,16 +282,21 @@ void apply_virtualfilesystem(struct index_state *istate) if (buf[i] == '\0') { int pos, len; + nr_vfs_rows++; + len = buf + i - entry; /* look for a directory wild card (ie "dir1/") */ if (buf[i - 1] == '/') { + nr_vfs_dirs++; if (ignore_case) adjust_dirname_case(istate, entry); pos = index_name_pos(istate, entry, len); if (pos < 0) { pos = -pos - 1; while (pos < istate->cache_nr && !fspathncmp(istate->cache[pos]->name, entry, len)) { + if (istate->cache[pos]->ce_flags & CE_SKIP_WORKTREE) + nr_bulk_skip++; istate->cache[pos]->ce_flags &= ~CE_SKIP_WORKTREE; pos++; } @@ -292,18 +304,41 @@ void apply_virtualfilesystem(struct index_state *istate) } else { if (ignore_case) { struct cache_entry *ce = index_file_exists(istate, entry, len, ignore_case); - if (ce) + if (ce) { + if (ce->ce_flags & CE_SKIP_WORKTREE) + nr_explicit_skip++; ce->ce_flags &= ~CE_SKIP_WORKTREE; + } + else { + nr_unknown++; + } } else { int pos = index_name_pos(istate, entry, len); - if (pos >= 0) + if (pos >= 0) { + if (istate->cache[pos]->ce_flags & CE_SKIP_WORKTREE) + nr_explicit_skip++; istate->cache[pos]->ce_flags &= ~CE_SKIP_WORKTREE; + } + else { + nr_unknown++; + } } } entry += len + 1; } } + + if (nr_vfs_rows > 0) { + trace2_data_intmax("vfs", the_repository, "apply/tracked", nr_bulk_skip + nr_explicit_skip); + + trace2_data_intmax("vfs", the_repository, "apply/vfs_rows", nr_vfs_rows); + trace2_data_intmax("vfs", the_repository, "apply/vfs_dirs", nr_vfs_dirs); + + trace2_data_intmax("vfs", the_repository, "apply/nr_unknown", nr_unknown); + trace2_data_intmax("vfs", the_repository, "apply/nr_bulk_skip", nr_bulk_skip); + trace2_data_intmax("vfs", the_repository, "apply/nr_explicit_skip", nr_explicit_skip); + } } /* From 67070dfc3bc1f8b269a50845dfd5d4f66a585f8a Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 15 Apr 2019 13:39:43 -0700 Subject: [PATCH 074/189] trace2: refactor setting process starting time Create trace2_initialize_clock() and call from main() to capture process start time in isolation and before other sub-systems are ready. Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- compat/mingw.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index bb8760deec9433..4d49f31db29279 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -4088,6 +4088,8 @@ int wmain(int argc, const wchar_t **wargv) SetConsoleCtrlHandler(handle_ctrl_c, TRUE); + trace2_initialize_clock(); + maybe_redirect_std_handles(); adjust_symlink_flags(); fsync_object_files = 1; From 2e027f7ab01ed9c4b6d3ea23b3b71c942094b028 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 30 Apr 2019 14:12:51 -0400 Subject: [PATCH 075/189] trace2:gvfs:experiment: clear_ce_flags_1 Signed-off-by: Jeff Hostetler --- unpack-trees.c | 1 + 1 file changed, 1 insertion(+) diff --git a/unpack-trees.c b/unpack-trees.c index 883a5afbb49b3c..2c8e52edc7c930 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -1787,6 +1787,7 @@ static int clear_ce_flags(struct index_state *istate, xsnprintf(label, sizeof(label), "clear_ce_flags(0x%08lx,0x%08lx)", (unsigned long)select_mask, (unsigned long)clear_mask); trace2_region_enter("unpack_trees", label, the_repository); + rval = clear_ce_flags_1(istate, istate->cache, istate->cache_nr, From e9d77c0b53b1dc0d6725e262661a3f8e6e561700 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 30 Apr 2019 16:02:39 -0400 Subject: [PATCH 076/189] trace2:gvfs:experiment: report_tracking Signed-off-by: Jeff Hostetler --- builtin/checkout.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/builtin/checkout.c b/builtin/checkout.c index 44db2bae72a2f5..eab1bfa6bca938 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -1043,8 +1043,11 @@ static void update_refs_for_switch(const struct checkout_opts *opts, strbuf_release(&msg); if (!opts->quiet && !opts->force_detach && - (new_branch_info->path || !strcmp(new_branch_info->name, "HEAD"))) + (new_branch_info->path || !strcmp(new_branch_info->name, "HEAD"))) { + trace2_region_enter("exp", "report_tracking", the_repository); report_tracking(new_branch_info); + trace2_region_leave("exp", "report_tracking", the_repository); + } } static int add_pending_uninteresting_ref(const char *refname, From a0663345262b87dca8ed98fbe826d402bdd758ba Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 14 Jun 2019 12:38:31 -0400 Subject: [PATCH 077/189] trace2:gvfs:experiment: read_cache: annotate thread usage in read-cache Add trace2_thread_start() and trace2_thread_exit() events to the worker threads used to read the index. This gives per-thread perf data. These workers were introduced in: abb4bb83845 read-cache: load cache extensions on a worker thread 77ff1127a4c read-cache: load cache entries on worker threads Signed-off-by: Jeff Hostetler --- read-cache.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/read-cache.c b/read-cache.c index 4ab2df30bffbd8..941ad50879ee55 100644 --- a/read-cache.c +++ b/read-cache.c @@ -2052,6 +2052,17 @@ static void *load_index_extensions(void *_data) return NULL; } +static void *load_index_extensions_threadproc(void *_data) +{ + void *result; + + trace2_thread_start("load_index_extensions"); + result = load_index_extensions(_data); + trace2_thread_exit(); + + return result; +} + /* * A helper function that will load the specified range of cache entries * from the memory mapped file and add them to the given index. @@ -2128,12 +2139,17 @@ static void *load_cache_entries_thread(void *_data) struct load_cache_entries_thread_data *p = _data; int i; + trace2_thread_start("load_cache_entries"); + /* iterate across all ieot blocks assigned to this thread */ for (i = p->ieot_start; i < p->ieot_start + p->ieot_blocks; i++) { p->consumed += load_cache_entry_block(p->istate, p->ce_mem_pool, p->offset, p->ieot->entries[i].nr, p->mmap, p->ieot->entries[i].offset, NULL); p->offset += p->ieot->entries[i].nr; } + + trace2_thread_exit(); + return NULL; } @@ -2302,7 +2318,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) int err; p.src_offset = extension_offset; - err = pthread_create(&p.pthread, NULL, load_index_extensions, &p); + err = pthread_create(&p.pthread, NULL, load_index_extensions_threadproc, &p); if (err) die(_("unable to create load_index_extensions thread: %s"), strerror(err)); From 3cd3022f34c456ca13793917b269e3942d2dfa04 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 9 Jul 2019 14:43:47 -0400 Subject: [PATCH 078/189] trace2:gvfs:experiment: read-cache: time read/write of cache-tree extension Add regions around code to read and write the cache-tree extension when the index is read or written. This is an experiment and may be dropped in future releases if we don't need it anymore. This experiment demonstrates that it takes more time to parse and deserialize the cache-tree extension than it does to read the cache-entries. Commits [1] and [2] spreads cache-entry reading across N-1 cores and dedicates a single core to simultaneously read the index extensions. Local testing (on my machine) shows that reading the cache-tree extension takes ~0.28 seconds. The 11 cache-entry threads take ~0.08 seconds. The main thread is blocked for 0.15 to 0.20 seconds waiting for the extension thread to finish. Let's use this commit to gather some telemetry and confirm this. My point is that improvements, such as index V5 which makes the cache entries smaller, may improve performance, but the gains may be limited because of this extension. And that we may need to look inside the cache-tree extension to truly improve do_read_index() performance. [1] abb4bb83845 read-cache: load cache extensions on a worker thread [2] 77ff1127a4c read-cache: load cache entries on worker threads Signed-off-by: Jeff Hostetler --- read-cache.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/read-cache.c b/read-cache.c index 941ad50879ee55..401e5ee30eac4e 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1762,7 +1762,10 @@ static int read_index_extension(struct index_state *istate, { switch (CACHE_EXT(ext)) { case CACHE_EXT_TREE: + trace2_region_enter("index", "read/extension/cache_tree", NULL); istate->cache_tree = cache_tree_read(data, sz); + trace2_data_intmax("index", NULL, "read/extension/cache_tree/bytes", (intmax_t)sz); + trace2_region_leave("index", "read/extension/cache_tree", NULL); break; case CACHE_EXT_RESOLVE_UNDO: istate->resolve_undo = resolve_undo_read(data, sz); @@ -3058,9 +3061,13 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, !drop_cache_tree && istate->cache_tree) { struct strbuf sb = STRBUF_INIT; + trace2_region_enter("index", "write/extension/cache_tree", NULL); cache_tree_write(&sb, istate->cache_tree); err = write_index_ext_header(f, eoie_c, CACHE_EXT_TREE, sb.len) < 0; hashwrite(f, sb.buf, sb.len); + trace2_data_intmax("index", NULL, "write/extension/cache_tree/bytes", (intmax_t)sb.len); + trace2_region_leave("index", "write/extension/cache_tree", NULL); + strbuf_release(&sb); /* * NEEDSWORK: write_index_ext_header() never returns a failure, From a6dbfe403356fa9a3f4e5f5ccfe508ee6b20adcc Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 16 Jul 2019 09:09:53 -0400 Subject: [PATCH 079/189] trace2:gvfs:experiment: add region to apply_virtualfilesystem() Signed-off-by: Jeff Hostetler --- virtualfilesystem.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/virtualfilesystem.c b/virtualfilesystem.c index e6ec336ba3d605..fdb3f1300cc7b8 100644 --- a/virtualfilesystem.c +++ b/virtualfilesystem.c @@ -269,6 +269,8 @@ void apply_virtualfilesystem(struct index_state *istate) if (!git_config_get_virtualfilesystem()) return; + trace2_region_enter("vfs", "apply", the_repository); + if (!virtual_filesystem_data.len) get_virtual_filesystem_data(&virtual_filesystem_data); @@ -339,6 +341,8 @@ void apply_virtualfilesystem(struct index_state *istate) trace2_data_intmax("vfs", the_repository, "apply/nr_bulk_skip", nr_bulk_skip); trace2_data_intmax("vfs", the_repository, "apply/nr_explicit_skip", nr_explicit_skip); } + + trace2_region_leave("vfs", "apply", the_repository); } /* From 418ce1564072c91103a21af6ae22850d9d2d0a29 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 10 Apr 2020 21:14:44 -0400 Subject: [PATCH 080/189] status: disable deserialize when verbose output requested. Disable deserialization when verbose output requested. Verbose mode causes Git to print diffs for modified files. This requires the index to be loaded to have the currently staged OID values. Without loading the index, verbose output make it look like everything was deleted. Signed-off-by: Jeff Hostetler --- builtin/commit.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/builtin/commit.c b/builtin/commit.c index 54a2f47aa2edbc..1dd5e03118f463 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -38,6 +38,7 @@ #include "commit-graph.h" #include "pretty.h" #include "trailer.h" +#include "trace2.h" static const char * const builtin_commit_usage[] = { N_("git commit [-a | --interactive | --patch] [-s] [-v] [-u] [--amend]\n" @@ -1722,6 +1723,22 @@ int cmd_status(int argc, const char **argv, const char *prefix) */ try_deserialize = (!do_serialize && (do_implicit_deserialize || do_explicit_deserialize)); + + /* + * Disable deserialize when verbose is set because it causes us to + * print diffs for each modified file, but that requires us to have + * the index loaded and we don't want to do that (at least not now for + * this seldom used feature). My fear is that would further tangle + * the merge conflict with upstream. + * + * TODO Reconsider this in the future. + */ + if (try_deserialize && verbose) { + trace2_data_string("status", the_repository, "deserialize/reject", + "args/verbose"); + try_deserialize = 0; + } + if (try_deserialize) goto skip_init; From 619b945529ff20e688300b6a90802568a014978e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 16 Jul 2019 10:08:08 -0400 Subject: [PATCH 081/189] trace2:gvfs:experiment: add region around unpack_trees() Signed-off-by: Jeff Hostetler --- unpack-trees.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unpack-trees.c b/unpack-trees.c index 2c8e52edc7c930..8289b2419f9989 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -1910,6 +1910,8 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options if (o->df_conflict_entry) BUG("o->df_conflict_entry is an output only field"); + trace2_region_enter("exp", "unpack_trees", NULL); + trace_performance_enter(); trace2_region_enter("unpack_trees", "unpack_trees", the_repository); @@ -2114,6 +2116,7 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options } trace2_region_leave("unpack_trees", "unpack_trees", the_repository); trace_performance_leave("unpack_trees"); + trace2_region_leave("exp", "unpack_trees", NULL); return ret; return_failed: From 0c260d95bda6f85561b6ea4474cfb8d3396801a2 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 10 Apr 2020 21:18:41 -0400 Subject: [PATCH 082/189] t7524: add test for verbose status deserialzation Verify that `git status --deserialize=x -v` does not crash and generates the same output as a normal (scanning) status command. These issues are described in the previous 2 commits. Signed-off-by: Jeff Hostetler --- t/t7522-serialized-status.sh | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index b52a9b7fa2f520..6010fcd31635cd 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -400,4 +400,43 @@ EOF ' +test_expect_success 'ensure deserialize -v does not crash' ' + + git init -b main verbose_test && + touch verbose_test/a && + touch verbose_test/b && + touch verbose_test/c && + git -C verbose_test add a b c && + git -C verbose_test commit -m abc && + + echo green >>verbose_test/a && + git -C verbose_test add a && + echo red_1 >>verbose_test/b && + echo red_2 >verbose_test/dirt && + + git -C verbose_test status >output.ref && + git -C verbose_test status -v >output.ref_v && + + git -C verbose_test --no-optional-locks status --serialize=../verbose_test.dat >output.ser.long && + git -C verbose_test --no-optional-locks status --serialize=../verbose_test.dat_v -v >output.ser.long_v && + + # Verify that serialization does not affect the status output itself. + test_cmp output.ref output.ser.long && + test_cmp output.ref_v output.ser.long_v && + + GIT_TRACE2_PERF="$(pwd)"/verbose_test.log \ + git -C verbose_test status --deserialize=../verbose_test.dat >output.des.long && + + # Verify that normal deserialize was actually used and produces the same result. + test_cmp output.ser.long output.des.long && + grep -q "deserialize/result:ok" verbose_test.log && + + GIT_TRACE2_PERF="$(pwd)"/verbose_test.log_v \ + git -C verbose_test status --deserialize=../verbose_test.dat_v -v >output.des.long_v && + + # Verify that vebose mode produces the same result because verbose was rejected. + test_cmp output.ser.long_v output.des.long_v && + grep -q "deserialize/reject:args/verbose" verbose_test.log_v +' + test_done From e9ce59ed2cc38f1aa91ae4a0aacd11fe843c92cb Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 16 Jul 2019 10:16:37 -0400 Subject: [PATCH 083/189] trace2:gvfs:experiment: add region to cache_tree_fully_valid() Signed-off-by: Jeff Hostetler --- cache-tree.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cache-tree.c b/cache-tree.c index 644a74021be0d6..cec214bc3eeeb0 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -232,7 +232,7 @@ static void discard_unused_subtrees(struct cache_tree *it) } } -int cache_tree_fully_valid(struct cache_tree *it) +static int cache_tree_fully_valid_1(struct cache_tree *it) { int i; if (!it) @@ -240,7 +240,7 @@ int cache_tree_fully_valid(struct cache_tree *it) if (it->entry_count < 0 || !repo_has_object_file(the_repository, &it->oid)) return 0; for (i = 0; i < it->subtree_nr; i++) { - if (!cache_tree_fully_valid(it->down[i]->cache_tree)) + if (!cache_tree_fully_valid_1(it->down[i]->cache_tree)) return 0; } return 1; @@ -251,6 +251,17 @@ static int must_check_existence(const struct cache_entry *ce) return !(repo_has_promisor_remote(the_repository) && ce_skip_worktree(ce)); } +int cache_tree_fully_valid(struct cache_tree *it) +{ + int result; + + trace2_region_enter("cache_tree", "fully_valid", NULL); + result = cache_tree_fully_valid_1(it); + trace2_region_leave("cache_tree", "fully_valid", NULL); + + return result; +} + static int update_one(struct cache_tree *it, struct cache_entry **cache, int entries, From 52dc6c624f20bb833ccc92a28310fbbbd253f723 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 13 May 2020 17:38:50 -0400 Subject: [PATCH 084/189] deserialize-status: silently fallback if we cannot read cache file Teach Git to not throw a fatal error when an explicitly-specified status-cache file (`git status --deserialize=`) could not be found or opened for reading and silently fallback to a traditional scan. This matches the behavior when the status-cache file is implicitly given via a config setting. Note: the current version causes a test to start failing. Mark this as an expected result for now. Signed-off-by: Jeff Hostetler Signed-off-by: Derrick Stolee --- builtin/commit.c | 18 ++++++++++++------ t/t7522-serialized-status.sh | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/builtin/commit.c b/builtin/commit.c index 1dd5e03118f463..81a7aa2a323dbd 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -225,12 +225,18 @@ static int opt_parse_deserialize(const struct option *opt, const char *arg, int free(deserialize_path); deserialize_path = xstrdup(arg); } - if (deserialize_path && *deserialize_path - && (access(deserialize_path, R_OK) != 0)) - die("cannot find serialization file '%s'", - deserialize_path); - - do_explicit_deserialize = 1; + if (!deserialize_path || !*deserialize_path) + do_explicit_deserialize = 1; /* read stdin */ + else if (access(deserialize_path, R_OK) == 0) + do_explicit_deserialize = 1; /* can read from this file */ + else { + /* + * otherwise, silently fallback to the normal + * collection scan + */ + do_implicit_deserialize = 0; + do_explicit_deserialize = 0; + } } return 0; diff --git a/t/t7522-serialized-status.sh b/t/t7522-serialized-status.sh index 6010fcd31635cd..230e1e24cfc1c4 100755 --- a/t/t7522-serialized-status.sh +++ b/t/t7522-serialized-status.sh @@ -439,4 +439,20 @@ test_expect_success 'ensure deserialize -v does not crash' ' grep -q "deserialize/reject:args/verbose" verbose_test.log_v ' +test_expect_success 'fallback when implicit' ' + git init -b main implicit_fallback_test && + git -C implicit_fallback_test -c status.deserializepath=foobar status +' + +test_expect_success 'fallback when explicit' ' + git init -b main explicit_fallback_test && + git -C explicit_fallback_test status --deserialize=foobar +' + +test_expect_success 'deserialize from stdin' ' + git init -b main stdin_test && + git -C stdin_test status --serialize >serialized_status.dat && + cat serialize_status.dat | git -C stdin_test status --deserialize +' + test_done From 82bf4da920c5764b596a59726b38b374cfe79eae Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 3 Oct 2019 13:21:26 -0400 Subject: [PATCH 085/189] credential: set trace2_child_class for credential manager children Signed-off-by: Jeff Hostetler --- credential.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/credential.c b/credential.c index 4b1a2b94feff73..15b93913fa2b7a 100644 --- a/credential.c +++ b/credential.c @@ -417,6 +417,8 @@ static int run_credential_helper(struct credential *c, else helper.no_stdout = 1; + helper.trace2_child_class = "cred"; + if (start_command(&helper) < 0) return -1; From 3f3b8c0a4be0679d8e1bba4b82e0d5627ae242e6 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 18 Sep 2019 10:35:45 -0400 Subject: [PATCH 086/189] sub-process: do not borrow cmd pointer from caller Teach subprocess_start() to use a copy of the passed `cmd` string rather than borrowing the buffer from the caller. Some callers of subprocess_start() pass the value returned from find_hook() which points to a static buffer and therefore is only good until the next call to find_hook(). This could cause problems for the long-running background processes managed by sub-process.c where later calls to subprocess_find_entry() to get an existing process will fail. This could cause more than 1 long-running process to be created. TODO Need to confirm, but if only read_object_hook() uses TODO subprocess_start() in this manner, we could drop this TODO commit when we drop support for read_object_hook(). Signed-off-by: Jeff Hostetler --- sub-process.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sub-process.c b/sub-process.c index 1daf5a975254b9..3327b1e4396c0a 100644 --- a/sub-process.c +++ b/sub-process.c @@ -81,7 +81,12 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co int err; struct child_process *process; - entry->cmd = cmd; + // BUGBUG most callers to subprocess_start() pass in "cmd" the value + // BUGBUG of find_hook() which returns a static buffer (that's only + // BUGBUG good until the next call to find_hook()). + // BUGFIX Defer assignment until we copy the string in our argv. + // entry->cmd = cmd; + process = &entry->process; child_process_init(process); @@ -93,6 +98,8 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co process->clean_on_exit_handler = subprocess_exit_handler; process->trace2_child_class = "subprocess"; + entry->cmd = process->args.v[0]; + err = start_command(process); if (err) { error("cannot fork to run subprocess '%s'", cmd); From bf56ae8cf23867851a5aa4e3b6d6a3dbbd6d3d77 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 18 Sep 2019 10:45:58 -0400 Subject: [PATCH 087/189] sub-process: add subprocess_start_argv() Add function to start a subprocess with an argv. Signed-off-by: Jeff Hostetler --- sub-process.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ sub-process.h | 6 ++++++ 2 files changed, 53 insertions(+) diff --git a/sub-process.c b/sub-process.c index 3327b1e4396c0a..9a4951fdccf218 100644 --- a/sub-process.c +++ b/sub-process.c @@ -5,6 +5,7 @@ #include "sub-process.h" #include "sigchain.h" #include "pkt-line.h" +#include "quote.h" int cmd2process_cmp(const void *cmp_data UNUSED, const struct hashmap_entry *eptr, @@ -119,6 +120,52 @@ int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, co return 0; } +int subprocess_start_strvec(struct hashmap *hashmap, + struct subprocess_entry *entry, + int is_git_cmd, + const struct strvec *argv, + subprocess_start_fn startfn) +{ + int err; + int k; + struct child_process *process; + struct strbuf quoted = STRBUF_INIT; + + process = &entry->process; + + child_process_init(process); + for (k = 0; k < argv->nr; k++) + strvec_push(&process->args, argv->v[k]); + process->use_shell = 1; + process->in = -1; + process->out = -1; + process->git_cmd = is_git_cmd; + process->clean_on_exit = 1; + process->clean_on_exit_handler = subprocess_exit_handler; + process->trace2_child_class = "subprocess"; + + sq_quote_argv_pretty("ed, argv->v); + entry->cmd = strbuf_detach("ed, NULL); + + err = start_command(process); + if (err) { + error("cannot fork to run subprocess '%s'", entry->cmd); + return err; + } + + hashmap_entry_init(&entry->ent, strhash(entry->cmd)); + + err = startfn(entry); + if (err) { + error("initialization for subprocess '%s' failed", entry->cmd); + subprocess_stop(hashmap, entry); + return err; + } + + hashmap_add(hashmap, &entry->ent); + return 0; +} + static int handshake_version(struct child_process *process, const char *welcome_prefix, int *versions, int *chosen_version) diff --git a/sub-process.h b/sub-process.h index 6a61638a8ace0b..73cc536646df79 100644 --- a/sub-process.h +++ b/sub-process.h @@ -56,6 +56,12 @@ typedef int(*subprocess_start_fn)(struct subprocess_entry *entry); int subprocess_start(struct hashmap *hashmap, struct subprocess_entry *entry, const char *cmd, subprocess_start_fn startfn); +int subprocess_start_strvec(struct hashmap *hashmap, + struct subprocess_entry *entry, + int is_git_cmd, + const struct strvec *argv, + subprocess_start_fn startfn); + /* Kill a subprocess and remove it from the subprocess hashmap. */ void subprocess_stop(struct hashmap *hashmap, struct subprocess_entry *entry); From 9af167d8dc024491d96298a6d1f91a433153757c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 24 Sep 2019 14:31:10 -0400 Subject: [PATCH 088/189] sha1-file: add function to update existing loose object cache Create a function to add a new object to the loose object cache after the existing odb/xx/ directory was scanned. This will be used in a later commit to keep the loose object cache fresh after dynamically fetching an individual object and without requiring the odb/xx/ directory to be rescanned. Signed-off-by: Jeff Hostetler --- object-file.c | 7 +++++++ object-store-ll.h | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/object-file.c b/object-file.c index 4b3eef527d0f92..10855e5b2e95c5 100644 --- a/object-file.c +++ b/object-file.c @@ -3008,6 +3008,13 @@ struct oidtree *odb_loose_cache(struct object_directory *odb, return odb->loose_objects_cache; } +void odb_loose_cache_add_new_oid(struct object_directory *odb, + const struct object_id *oid) +{ + struct oidtree *cache = odb_loose_cache(odb, oid); + append_loose_object(oid, NULL, cache); +} + void odb_clear_loose_cache(struct object_directory *odb) { oidtree_clear(odb->loose_objects_cache); diff --git a/object-store-ll.h b/object-store-ll.h index ccade7117f8e37..103b535defbf19 100644 --- a/object-store-ll.h +++ b/object-store-ll.h @@ -94,6 +94,14 @@ void restore_primary_odb(struct object_directory *restore_odb, const char *old_p struct oidtree *odb_loose_cache(struct object_directory *odb, const struct object_id *oid); +/* + * Add a new object to the loose object cache (possibly after the + * cache was populated). This might be used after dynamically + * fetching a missing object. + */ +void odb_loose_cache_add_new_oid(struct object_directory *odb, + const struct object_id *oid); + /* Empty the loose object cache for the specified object directory. */ void odb_clear_loose_cache(struct object_directory *odb); From 69cf26ef6c7182985c6d8a560564a04ebeef89c4 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 16 Jul 2019 10:40:56 -0400 Subject: [PATCH 089/189] trace2:gvfs:experiment: add unpack_entry() counter to unpack_trees() and report_tracking() Signed-off-by: Jeff Hostetler --- builtin/checkout.c | 6 ++++++ packfile.c | 9 +++++++++ packfile.h | 5 +++++ unpack-trees.c | 5 +++++ 4 files changed, 25 insertions(+) diff --git a/builtin/checkout.c b/builtin/checkout.c index eab1bfa6bca938..3f1e9f7c577b60 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -17,6 +17,7 @@ #include "merge-recursive.h" #include "object-name.h" #include "object-store-ll.h" +#include "packfile.h" #include "parse-options.h" #include "path.h" #include "preload-index.h" @@ -1044,8 +1045,13 @@ static void update_refs_for_switch(const struct checkout_opts *opts, if (!opts->quiet && !opts->force_detach && (new_branch_info->path || !strcmp(new_branch_info->name, "HEAD"))) { + unsigned long nr_unpack_entry_at_start; + trace2_region_enter("exp", "report_tracking", the_repository); + nr_unpack_entry_at_start = get_nr_unpack_entry(); report_tracking(new_branch_info); + trace2_data_intmax("exp", NULL, "report_tracking/nr_unpack_entries", + (intmax_t)(get_nr_unpack_entry() - nr_unpack_entry_at_start)); trace2_region_leave("exp", "report_tracking", the_repository); } } diff --git a/packfile.c b/packfile.c index 813584646f762a..8adb82aaf11fcc 100644 --- a/packfile.c +++ b/packfile.c @@ -1667,6 +1667,13 @@ struct unpack_entry_stack_ent { unsigned long size; }; +static unsigned long g_nr_unpack_entry; + +unsigned long get_nr_unpack_entry(void) +{ + return g_nr_unpack_entry; +} + void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, enum object_type *final_type, unsigned long *final_size) { @@ -1680,6 +1687,8 @@ void *unpack_entry(struct repository *r, struct packed_git *p, off_t obj_offset, int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC; int base_from_cache = 0; + g_nr_unpack_entry++; + write_pack_access_log(p, obj_offset); /* PHASE 1: drill down to the innermost base object */ diff --git a/packfile.h b/packfile.h index eb18ec15dbf3bc..11ce71a4f97912 100644 --- a/packfile.h +++ b/packfile.h @@ -210,4 +210,9 @@ int is_promisor_object(const struct object_id *oid); int load_idx(const char *path, const unsigned int hashsz, void *idx_map, size_t idx_size, struct packed_git *p); +/* + * Return the number of objects fetched from a packfile. + */ +unsigned long get_nr_unpack_entry(void); + #endif diff --git a/unpack-trees.c b/unpack-trees.c index 8289b2419f9989..410e58edf3af4e 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -16,6 +16,7 @@ #include "tree-walk.h" #include "cache-tree.h" #include "unpack-trees.h" +#include "packfile.h" #include "progress.h" #include "refs.h" #include "attr.h" @@ -1897,6 +1898,7 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options struct pattern_list pl; int free_pattern_list = 0; struct dir_struct dir = DIR_INIT; + unsigned long nr_unpack_entry_at_start; if (o->reset == UNPACK_RESET_INVALID) BUG("o->reset had a value of 1; should be UNPACK_TREES_*_UNTRACKED"); @@ -1911,6 +1913,7 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options BUG("o->df_conflict_entry is an output only field"); trace2_region_enter("exp", "unpack_trees", NULL); + nr_unpack_entry_at_start = get_nr_unpack_entry(); trace_performance_enter(); trace2_region_enter("unpack_trees", "unpack_trees", the_repository); @@ -2116,6 +2119,8 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options } trace2_region_leave("unpack_trees", "unpack_trees", the_repository); trace_performance_leave("unpack_trees"); + trace2_data_intmax("unpack_trees", NULL, "unpack_trees/nr_unpack_entries", + (intmax_t)(get_nr_unpack_entry() - nr_unpack_entry_at_start)); trace2_region_leave("exp", "unpack_trees", NULL); return ret; From b576d358c963185a79ca8a925d89f8e7fb88582f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 25 Sep 2019 13:36:54 -0400 Subject: [PATCH 090/189] packfile: add install_packed_git_and_mru() Create a function to install a new packfile into the packed-git list and add it to the head of the MRU list. This function will be used in a later commit to install packfiles created by dynamic object fetching. Signed-off-by: Jeff Hostetler --- packfile.c | 6 ++++++ packfile.h | 1 + 2 files changed, 7 insertions(+) diff --git a/packfile.c b/packfile.c index 8adb82aaf11fcc..c0632ffdfb7d25 100644 --- a/packfile.c +++ b/packfile.c @@ -769,6 +769,12 @@ void install_packed_git(struct repository *r, struct packed_git *pack) hashmap_add(&r->objects->pack_map, &pack->packmap_ent); } +void install_packed_git_and_mru(struct repository *r, struct packed_git *pack) +{ + install_packed_git(r, pack); + list_add(&pack->mru, &r->objects->packed_git_mru); +} + void (*report_garbage)(unsigned seen_bits, const char *path); static void report_helper(const struct string_list *list, diff --git a/packfile.h b/packfile.h index 11ce71a4f97912..cfb0c9f50ffff6 100644 --- a/packfile.h +++ b/packfile.h @@ -67,6 +67,7 @@ extern void (*report_garbage)(unsigned seen_bits, const char *path); void reprepare_packed_git(struct repository *r); void install_packed_git(struct repository *r, struct packed_git *pack); +void install_packed_git_and_mru(struct repository *r, struct packed_git *pack); struct packed_git *get_packed_git(struct repository *r); struct list_head *get_packed_git_mru(struct repository *r); From d5d0685da68c279ac16a7d1098c9251eee54a7c3 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 25 Jul 2019 15:43:50 -0400 Subject: [PATCH 091/189] trace2:gvfs:experiment: increase default event depth for unpack-tree data --- trace2/tr2_tgt_event.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trace2/tr2_tgt_event.c b/trace2/tr2_tgt_event.c index 59910a1a4f7c0f..3bd80b7236daf9 100644 --- a/trace2/tr2_tgt_event.c +++ b/trace2/tr2_tgt_event.c @@ -37,7 +37,7 @@ static struct tr2_dst tr2dst_event = { * event target. Use the TR2_SYSENV_EVENT_NESTING setting to increase * region details in the event target. */ -static int tr2env_event_max_nesting_levels = 2; +static int tr2env_event_max_nesting_levels = 4; /* * Use the TR2_SYSENV_EVENT_BRIEF to omit the