diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 4017ed82ca4341..c19530b086311a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -1,8 +1,10 @@
- - [ ] I was not able to find an [open](https://github.com/git-for-windows/git/issues?q=is%3Aopen) or [closed](https://github.com/git-for-windows/git/issues?q=is%3Aclosed) issue matching what I'm seeing
+ - [ ] I was not able to find an [open](https://github.com/microsoft/git/issues?q=is%3Aopen)
+ or [closed](https://github.com/microsoft/git/issues?q=is%3Aclosed) issue matching
+ what I'm seeing, including in [the `git-for-windows/git` tracker](https://github.com/git-for-windows/git/issues).
### Setup
- - Which version of Git for Windows are you using? Is it 32-bit or 64-bit?
+ - Which version of `microsoft/git` are you using? Is it 32-bit or 64-bit?
```
$ git --version --build-options
@@ -10,24 +12,22 @@ $ git --version --build-options
** insert your machine's response here **
```
- - Which version of Windows are you running? Vista, 7, 8, 10? Is it 32-bit or 64-bit?
+Are you using Scalar or VFS for Git?
+
+** insert your answer here **
+
+If VFS for Git, then what version?
```
-$ cmd.exe /c ver
+$ gvfs version
** insert your machine's response here **
```
- - What options did you set as part of the installation? Or did you choose the
- defaults?
+ - Which version of Windows are you running? Vista, 7, 8, 10? Is it 32-bit or 64-bit?
```
-# One of the following:
-> type "C:\Program Files\Git\etc\install-options.txt"
-> type "C:\Program Files (x86)\Git\etc\install-options.txt"
-> type "%USERPROFILE%\AppData\Local\Programs\Git\etc\install-options.txt"
-> type "$env:USERPROFILE\AppData\Local\Programs\Git\etc\install-options.txt"
-$ cat /etc/install-options.txt
+$ cmd.exe /c ver
** insert your machine's response here **
```
@@ -58,7 +58,11 @@ $ cat /etc/install-options.txt
** insert here **
- - If the problem was occurring with a specific repository, can you provide the
- URL to that repository to help us with testing?
+ - If the problem was occurring with a specific repository, can you specify
+ the repository?
-** insert URL here **
+ * [ ] Public repo: **insert URL here**
+ * [ ] Windows monorepo
+ * [ ] Office monorepo
+ * [ ] Other Microsoft-internal repo: **insert name here**
+ * [ ] Other internal repo.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 7baf31f2c471ec..3cb48d8582f31c 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,22 +1,10 @@
Thanks for taking the time to contribute to Git!
-Those seeking to contribute to the Git for Windows fork should see
-http://gitforwindows.org/#contribute on how to contribute Windows specific
-enhancements.
-
-If your contribution is for the core Git functions and documentation
-please be aware that the Git community does not use the github.com issues
-or pull request mechanism for their contributions.
-
-Instead, we use the Git mailing list (git@vger.kernel.org) for code and
-documentation submissions, code reviews, and bug reports. The
-mailing list is plain text only (anything with HTML is sent directly
-to the spam folder).
-
-Nevertheless, you can use GitGitGadget (https://gitgitgadget.github.io/)
-to conveniently send your Pull Requests commits to our mailing list.
-
-For a single-commit pull request, please *leave the pull request description
-empty*: your commit message itself should describe your changes.
-
-Please read the "guidelines for contributing" linked above!
+This fork contains changes specific to monorepo scenarios. If you are an
+external contributor, then please detail your reason for submitting to
+this fork:
+
+* [ ] This is an early version of work already under review upstream.
+* [ ] This change only applies to interactions with Azure DevOps and the
+ GVFS Protocol.
+* [ ] This change only applies to the virtualization hook and VFS for Git.
diff --git a/.github/macos-installer/Makefile b/.github/macos-installer/Makefile
new file mode 100644
index 00000000000000..1a06f6200e62dc
--- /dev/null
+++ b/.github/macos-installer/Makefile
@@ -0,0 +1,157 @@
+SHELL := /bin/bash
+SUDO := sudo
+C_INCLUDE_PATH := /usr/include
+CPLUS_INCLUDE_PATH := /usr/include
+LD_LIBRARY_PATH := /usr/lib
+
+OSX_VERSION := $(shell sw_vers -productVersion)
+TARGET_FLAGS := -mmacosx-version-min=$(OSX_VERSION) -DMACOSX_DEPLOYMENT_TARGET=$(OSX_VERSION)
+
+uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not')
+
+ARCH_UNIV := universal
+ARCH_FLAGS := -arch x86_64 -arch arm64
+
+CFLAGS := $(TARGET_FLAGS) $(ARCH_FLAGS)
+LDFLAGS := $(TARGET_FLAGS) $(ARCH_FLAGS)
+
+PREFIX := /usr/local
+GIT_PREFIX := $(PREFIX)/git
+
+BUILD_DIR := $(GITHUB_WORKSPACE)/payload
+DESTDIR := $(PWD)/stage/git-$(ARCH_UNIV)-$(VERSION)
+ARTIFACTDIR := build-artifacts
+SUBMAKE := $(MAKE) C_INCLUDE_PATH="$(C_INCLUDE_PATH)" CPLUS_INCLUDE_PATH="$(CPLUS_INCLUDE_PATH)" LD_LIBRARY_PATH="$(LD_LIBRARY_PATH)" TARGET_FLAGS="$(TARGET_FLAGS)" CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" NO_GETTEXT=1 NO_DARWIN_PORTS=1 prefix=$(GIT_PREFIX) GIT_BUILT_FROM_COMMIT="$(GIT_BUILT_FROM_COMMIT)" DESTDIR=$(DESTDIR)
+CORES := $(shell bash -c "sysctl hw.ncpu | awk '{print \$$2}'")
+
+# Guard against environment variables
+APPLE_APP_IDENTITY =
+APPLE_INSTALLER_IDENTITY =
+APPLE_KEYCHAIN_PROFILE =
+
+.PHONY: image pkg payload codesign notarize
+
+.SECONDARY:
+
+$(DESTDIR)$(GIT_PREFIX)/VERSION-$(VERSION)-$(ARCH_UNIV):
+ rm -f $(BUILD_DIR)/git-$(VERSION)/osx-installed*
+ mkdir -p $(DESTDIR)$(GIT_PREFIX)
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-built-keychain:
+ cd $(BUILD_DIR)/git-$(VERSION)/contrib/credential/osxkeychain; $(SUBMAKE) CFLAGS="$(CFLAGS) -g -O2 -Wall"
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-built:
+ [ -d $(DESTDIR)$(GIT_PREFIX) ] && $(SUDO) rm -rf $(DESTDIR) || echo ok
+ cd $(BUILD_DIR)/git-$(VERSION); $(SUBMAKE) -j $(CORES) all strip
+ echo "================"
+ echo "Dumping Linkage"
+ cd $(BUILD_DIR)/git-$(VERSION); ./git version
+ echo "===="
+ cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git
+ echo "===="
+ cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git-http-fetch
+ echo "===="
+ cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git-http-push
+ echo "===="
+ cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git-remote-http
+ echo "===="
+ cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git-gvfs-helper
+ echo "================"
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-installed-bin: $(BUILD_DIR)/git-$(VERSION)/osx-built $(BUILD_DIR)/git-$(VERSION)/osx-built-keychain
+ cd $(BUILD_DIR)/git-$(VERSION); $(SUBMAKE) install
+ cp $(BUILD_DIR)/git-$(VERSION)/contrib/credential/osxkeychain/git-credential-osxkeychain $(DESTDIR)$(GIT_PREFIX)/bin/git-credential-osxkeychain
+ mkdir -p $(DESTDIR)$(GIT_PREFIX)/contrib/completion
+ cp $(BUILD_DIR)/git-$(VERSION)/contrib/completion/git-completion.bash $(DESTDIR)$(GIT_PREFIX)/contrib/completion/
+ cp $(BUILD_DIR)/git-$(VERSION)/contrib/completion/git-completion.zsh $(DESTDIR)$(GIT_PREFIX)/contrib/completion/
+ cp $(BUILD_DIR)/git-$(VERSION)/contrib/completion/git-prompt.sh $(DESTDIR)$(GIT_PREFIX)/contrib/completion/
+ # This is needed for Git-Gui, GitK
+ mkdir -p $(DESTDIR)$(GIT_PREFIX)/lib/perl5/site_perl
+ [ ! -f $(DESTDIR)$(GIT_PREFIX)/lib/perl5/site_perl/Error.pm ] && cp $(BUILD_DIR)/git-$(VERSION)/perl/private-Error.pm $(DESTDIR)$(GIT_PREFIX)/lib/perl5/site_perl/Error.pm || echo done
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-installed-man: $(BUILD_DIR)/git-$(VERSION)/osx-installed-bin
+ mkdir -p $(DESTDIR)$(GIT_PREFIX)/share/man
+ cp -R $(GITHUB_WORKSPACE)/manpages/ $(DESTDIR)$(GIT_PREFIX)/share/man
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-built-subtree:
+ cd $(BUILD_DIR)/git-$(VERSION)/contrib/subtree; $(SUBMAKE) XML_CATALOG_FILES="$(XML_CATALOG_FILES)" all git-subtree.1
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-installed-subtree: $(BUILD_DIR)/git-$(VERSION)/osx-built-subtree
+ mkdir -p $(DESTDIR)
+ cd $(BUILD_DIR)/git-$(VERSION)/contrib/subtree; $(SUBMAKE) XML_CATALOG_FILES="$(XML_CATALOG_FILES)" install install-man
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-installed-assets: $(BUILD_DIR)/git-$(VERSION)/osx-installed-bin
+ mkdir -p $(DESTDIR)$(GIT_PREFIX)/etc
+ cat assets/etc/gitconfig.osxkeychain >> $(DESTDIR)$(GIT_PREFIX)/etc/gitconfig
+ cp assets/uninstall.sh $(DESTDIR)$(GIT_PREFIX)/uninstall.sh
+ sh -c "echo .DS_Store >> $(DESTDIR)$(GIT_PREFIX)/share/git-core/templates/info/exclude"
+
+symlinks:
+ mkdir -p $(ARTIFACTDIR)$(PREFIX)/bin
+ cd $(ARTIFACTDIR)$(PREFIX)/bin; find ../git/bin -type f -exec ln -sf {} \;
+ for man in man1 man3 man5 man7; do mkdir -p $(ARTIFACTDIR)$(PREFIX)/share/man/$$man; (cd $(ARTIFACTDIR)$(PREFIX)/share/man/$$man; ln -sf ../../../git/share/man/$$man/* ./); done
+ ruby ../scripts/symlink-git-hardlinks.rb $(ARTIFACTDIR)
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-installed: $(DESTDIR)$(GIT_PREFIX)/VERSION-$(VERSION)-$(ARCH_UNIV) $(BUILD_DIR)/git-$(VERSION)/osx-installed-man $(BUILD_DIR)/git-$(VERSION)/osx-installed-assets $(BUILD_DIR)/git-$(VERSION)/osx-installed-subtree
+ find $(DESTDIR)$(GIT_PREFIX) -type d -exec chmod ugo+rx {} \;
+ find $(DESTDIR)$(GIT_PREFIX) -type f -exec chmod ugo+r {} \;
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-built-assert-$(ARCH_UNIV): $(BUILD_DIR)/git-$(VERSION)/osx-built
+ File $(BUILD_DIR)/git-$(VERSION)/git
+ File $(BUILD_DIR)/git-$(VERSION)/contrib/credential/osxkeychain/git-credential-osxkeychain
+ touch $@
+
+disk-image/VERSION-$(VERSION)-$(ARCH_UNIV):
+ rm -f disk-image/*.pkg disk-image/VERSION-* disk-image/.DS_Store
+ mkdir disk-image
+ touch "$@"
+
+pkg_cmd := pkgbuild --identifier com.git.pkg --version $(VERSION) \
+ --root $(ARTIFACTDIR)$(PREFIX) --scripts assets/scripts \
+ --install-location $(PREFIX) --component-plist ./assets/git-components.plist
+
+ifdef APPLE_INSTALLER_IDENTITY
+ pkg_cmd += --sign "$(APPLE_INSTALLER_IDENTITY)"
+endif
+
+pkg_cmd += disk-image/git-$(VERSION)-$(ARCH_UNIV).pkg
+disk-image/git-$(VERSION)-$(ARCH_UNIV).pkg: disk-image/VERSION-$(VERSION)-$(ARCH_UNIV) symlinks
+ $(pkg_cmd)
+
+git-%-$(ARCH_UNIV).dmg:
+ hdiutil create git-$(VERSION)-$(ARCH_UNIV).uncompressed.dmg -fs HFS+ -srcfolder disk-image -volname "Git $(VERSION) $(ARCH_UNIV)" -ov 2>&1 | tee err || { \
+ grep "Resource busy" err && \
+ sleep 5 && \
+ hdiutil create git-$(VERSION)-$(ARCH_UNIV).uncompressed.dmg -fs HFS+ -srcfolder disk-image -volname "Git $(VERSION) $(ARCH_UNIV)" -ov; }
+ hdiutil convert -format UDZO -o $@ git-$(VERSION)-$(ARCH_UNIV).uncompressed.dmg
+ rm -f git-$(VERSION)-$(ARCH_UNIV).uncompressed.dmg
+
+payload: $(BUILD_DIR)/git-$(VERSION)/osx-installed $(BUILD_DIR)/git-$(VERSION)/osx-built-assert-$(ARCH_UNIV)
+
+pkg: disk-image/git-$(VERSION)-$(ARCH_UNIV).pkg
+
+image: git-$(VERSION)-$(ARCH_UNIV).dmg
+
+ifdef APPLE_APP_IDENTITY
+codesign:
+ @$(CURDIR)/../scripts/codesign.sh --payload="build-artifacts/usr/local/git" \
+ --identity="$(APPLE_APP_IDENTITY)" \
+ --entitlements="$(CURDIR)/entitlements.xml"
+endif
+
+# Notarization can only happen if the package is fully signed
+ifdef APPLE_KEYCHAIN_PROFILE
+notarize:
+ @$(CURDIR)/../scripts/notarize.sh \
+ --package="disk-image/git-$(VERSION)-$(ARCH_UNIV).pkg" \
+ --keychain-profile="$(APPLE_KEYCHAIN_PROFILE)"
+endif
diff --git a/.github/macos-installer/assets/etc/gitconfig.osxkeychain b/.github/macos-installer/assets/etc/gitconfig.osxkeychain
new file mode 100644
index 00000000000000..788266b3a40a9d
--- /dev/null
+++ b/.github/macos-installer/assets/etc/gitconfig.osxkeychain
@@ -0,0 +1,2 @@
+[credential]
+ helper = osxkeychain
diff --git a/.github/macos-installer/assets/git-components.plist b/.github/macos-installer/assets/git-components.plist
new file mode 100644
index 00000000000000..78db36777df3ed
--- /dev/null
+++ b/.github/macos-installer/assets/git-components.plist
@@ -0,0 +1,18 @@
+
+
+
+
+
+ BundleHasStrictIdentifier
+
+ BundleIsRelocatable
+
+ BundleIsVersionChecked
+
+ BundleOverwriteAction
+ upgrade
+ RootRelativeBundlePath
+ git/share/git-gui/lib/Git Gui.app
+
+
+
diff --git a/.github/macos-installer/assets/scripts/postinstall b/.github/macos-installer/assets/scripts/postinstall
new file mode 100755
index 00000000000000..94056db9b7b864
--- /dev/null
+++ b/.github/macos-installer/assets/scripts/postinstall
@@ -0,0 +1,62 @@
+#!/bin/bash
+INSTALL_DST="$2"
+SCALAR_C_CMD="$INSTALL_DST/git/bin/scalar"
+SCALAR_DOTNET_CMD="/usr/local/scalar/scalar"
+SCALAR_UNINSTALL_SCRIPT="/usr/local/scalar/uninstall_scalar.sh"
+
+function cleanupScalar()
+{
+ echo "checking whether Scalar was installed"
+ if [ ! -f "$SCALAR_C_CMD" ]; then
+ echo "Scalar not installed; exiting..."
+ return 0
+ fi
+ echo "Scalar is installed!"
+
+ echo "looking for Scalar.NET"
+ if [ ! -f "$SCALAR_DOTNET_CMD" ]; then
+ echo "Scalar.NET not found; exiting..."
+ return 0
+ fi
+ echo "Scalar.NET found!"
+
+ currentUser=$(echo "show State:/Users/ConsoleUser" | scutil | awk '/Name :/ { print $3 }')
+
+ # Re-register Scalar.NET repositories with the newly-installed Scalar
+ for repo in $($SCALAR_DOTNET_CMD list); do
+ (
+ PATH="$INSTALL_DST/git/bin:$PATH"
+ sudo -u "$currentUser" scalar register $repo || \
+ echo "warning: skipping re-registration of $repo"
+ )
+ done
+
+ # Uninstall Scalar.NET
+ echo "removing Scalar.NET"
+
+ # Add /usr/local/bin to path - default install location of Homebrew
+ PATH="/usr/local/bin:$PATH"
+ if (sudo -u "$currentUser" brew list --cask scalar); then
+ # Remove from Homebrew
+ sudo -u "$currentUser" brew remove --cask scalar || echo "warning: Scalar.NET uninstall via Homebrew completed with code $?"
+ echo "Scalar.NET uninstalled via Homebrew!"
+ elif (sudo -u "$currentUser" brew list --cask scalar-azrepos); then
+ sudo -u "$currentUser" brew remove --cask scalar-azrepos || echo "warning: Scalar.NET with GVFS uninstall via Homebrew completed with code $?"
+ echo "Scalar.NET with GVFS uninstalled via Homebrew!"
+ elif [ -f $SCALAR_UNINSTALL_SCRIPT ]; then
+ # If not installed with Homebrew, manually remove package
+ sudo -S sh $SCALAR_UNINSTALL_SCRIPT || echo "warning: Scalar.NET uninstall completed with code $?"
+ echo "Scalar.NET uninstalled!"
+ else
+ echo "warning: Scalar.NET uninstall script not found"
+ fi
+
+ # Re-create the Scalar symlink, in case it was removed by the Scalar.NET uninstall operation
+ mkdir -p $INSTALL_DST/bin
+ /bin/ln -Fs "$SCALAR_C_CMD" "$INSTALL_DST/bin/scalar"
+}
+
+# Run Scalar cleanup (will exit if not applicable)
+cleanupScalar
+
+exit 0
\ No newline at end of file
diff --git a/.github/macos-installer/assets/uninstall.sh b/.github/macos-installer/assets/uninstall.sh
new file mode 100755
index 00000000000000..4fc79fbaa2e652
--- /dev/null
+++ b/.github/macos-installer/assets/uninstall.sh
@@ -0,0 +1,34 @@
+#!/bin/bash -e
+if [ ! -r "/usr/local/git" ]; then
+ echo "Git doesn't appear to be installed via this installer. Aborting"
+ exit 1
+fi
+
+if [ "$1" != "--yes" ]; then
+ echo "This will uninstall git by removing /usr/local/git/, and symlinks"
+ printf "Type 'yes' if you are sure you wish to continue: "
+ read response
+else
+ response="yes"
+fi
+
+if [ "$response" == "yes" ]; then
+ # remove all of the symlinks we've created
+ pkgutil --files com.git.pkg | grep bin | while read f; do
+ if [ -L /usr/local/$f ]; then
+ sudo rm /usr/local/$f
+ fi
+ done
+
+ # forget receipts.
+ pkgutil --packages | grep com.git.pkg | xargs -I {} sudo pkgutil --forget {}
+ echo "Uninstalled"
+
+ # The guts all go here.
+ sudo rm -rf /usr/local/git/
+else
+ echo "Aborted"
+ exit 1
+fi
+
+exit 0
diff --git a/.github/macos-installer/entitlements.xml b/.github/macos-installer/entitlements.xml
new file mode 100644
index 00000000000000..46f675661149b6
--- /dev/null
+++ b/.github/macos-installer/entitlements.xml
@@ -0,0 +1,12 @@
+
+
+
+
+ com.apple.security.cs.allow-jit
+
+ com.apple.security.cs.allow-unsigned-executable-memory
+
+ com.apple.security.cs.disable-library-validation
+
+
+
diff --git a/.github/scripts/codesign.sh b/.github/scripts/codesign.sh
new file mode 100755
index 00000000000000..076b29f93be45e
--- /dev/null
+++ b/.github/scripts/codesign.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+sign_directory () {
+ (
+ cd "$1"
+ for f in *
+ do
+ macho=$(file --mime $f | grep mach)
+ # Runtime sign dylibs and Mach-O binaries
+ if [[ $f == *.dylib ]] || [ ! -z "$macho" ];
+ then
+ echo "Runtime Signing $f"
+ codesign -s "$IDENTITY" $f --timestamp --force --options=runtime --entitlements $ENTITLEMENTS_FILE
+ elif [ -d "$f" ];
+ then
+ echo "Signing files in subdirectory $f"
+ sign_directory "$f"
+
+ else
+ echo "Signing $f"
+ codesign -s "$IDENTITY" $f --timestamp --force
+ fi
+ done
+ )
+}
+
+for i in "$@"
+do
+case "$i" in
+ --payload=*)
+ SIGN_DIR="${i#*=}"
+ shift # past argument=value
+ ;;
+ --identity=*)
+ IDENTITY="${i#*=}"
+ shift # past argument=value
+ ;;
+ --entitlements=*)
+ ENTITLEMENTS_FILE="${i#*=}"
+ shift # past argument=value
+ ;;
+ *)
+ die "unknown option '$i'"
+ ;;
+esac
+done
+
+if [ -z "$SIGN_DIR" ]; then
+ echo "error: missing directory argument"
+ exit 1
+elif [ -z "$IDENTITY" ]; then
+ echo "error: missing signing identity argument"
+ exit 1
+elif [ -z "$ENTITLEMENTS_FILE" ]; then
+ echo "error: missing entitlements file argument"
+ exit 1
+fi
+
+echo "======== INPUTS ========"
+echo "Directory: $SIGN_DIR"
+echo "Signing identity: $IDENTITY"
+echo "Entitlements: $ENTITLEMENTS_FILE"
+echo "======== END INPUTS ========"
+
+sign_directory "$SIGN_DIR"
diff --git a/.github/scripts/notarize.sh b/.github/scripts/notarize.sh
new file mode 100755
index 00000000000000..9315d688afbd49
--- /dev/null
+++ b/.github/scripts/notarize.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+for i in "$@"
+do
+case "$i" in
+ --package=*)
+ PACKAGE="${i#*=}"
+ shift # past argument=value
+ ;;
+ --keychain-profile=*)
+ KEYCHAIN_PROFILE="${i#*=}"
+ shift # past argument=value
+ ;;
+ *)
+ die "unknown option '$i'"
+ ;;
+esac
+done
+
+if [ -z "$PACKAGE" ]; then
+ echo "error: missing package argument"
+ exit 1
+elif [ -z "$KEYCHAIN_PROFILE" ]; then
+ echo "error: missing keychain profile argument"
+ exit 1
+fi
+
+# Exit as soon as any line fails
+set -e
+
+# Send the notarization request
+xcrun notarytool submit -v "$PACKAGE" -p "$KEYCHAIN_PROFILE" --wait
+
+# Staple the notarization ticket (to allow offline installation)
+xcrun stapler staple -v "$PACKAGE"
diff --git a/.github/scripts/symlink-git-hardlinks.rb b/.github/scripts/symlink-git-hardlinks.rb
new file mode 100644
index 00000000000000..174802ccc85d93
--- /dev/null
+++ b/.github/scripts/symlink-git-hardlinks.rb
@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+install_prefix = ARGV[0]
+puts install_prefix
+git_binary = File.join(install_prefix, '/usr/local/git/bin/git')
+
+[
+ ['git' , File.join(install_prefix, '/usr/local/git/bin')],
+ ['../../bin/git', File.join(install_prefix, '/usr/local/git/libexec/git-core')]
+].each do |link, path|
+ Dir.glob(File.join(path, '*')).each do |file|
+ next if file == git_binary
+ puts "#{file} #{File.size(file)} == #{File.size(git_binary)}"
+ next unless File.size(file) == File.size(git_binary)
+ puts "Symlinking #{file}"
+ puts `ln -sf #{link} #{file}`
+ exit $?.exitstatus if $?.exitstatus != 0
+ end
+end
\ No newline at end of file
diff --git a/.github/workflows/build-git-installers.yml b/.github/workflows/build-git-installers.yml
new file mode 100644
index 00000000000000..74180f14c7073a
--- /dev/null
+++ b/.github/workflows/build-git-installers.yml
@@ -0,0 +1,776 @@
+name: build-git-installers
+
+on:
+ push:
+ tags:
+ - 'v[0-9]*vfs*' # matches "vvfs"
+
+permissions:
+ id-token: write # required for Azure login via OIDC
+
+jobs:
+ # Check prerequisites for the workflow
+ prereqs:
+ runs-on: ubuntu-latest
+ environment: release
+ outputs:
+ tag_name: ${{ steps.tag.outputs.name }} # The full name of the tag, e.g. v2.32.0.vfs.0.0
+ tag_version: ${{ steps.tag.outputs.version }} # The version number (without preceding "v"), e.g. 2.32.0.vfs.0.0
+ steps:
+ - name: Validate tag
+ run: |
+ echo "$GITHUB_REF" |
+ grep -E '^refs/tags/v2\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.vfs\.0\.(0|[1-9][0-9]*)(\.rc[0-9])?$' || {
+ echo "::error::${GITHUB_REF#refs/tags/} is not of the form v2...vfs.0.[.rc]" >&2
+ exit 1
+ }
+ - name: Determine tag to build
+ run: |
+ echo "name=${GITHUB_REF#refs/tags/}" >>$GITHUB_OUTPUT
+ echo "version=${GITHUB_REF#refs/tags/v}" >>$GITHUB_OUTPUT
+ id: tag
+ - name: Clone git
+ uses: actions/checkout@v4
+ - name: Validate the tag identified with trigger
+ run: |
+ die () {
+ echo "::error::$*" >&2
+ exit 1
+ }
+
+ # `actions/checkout` only downloads the peeled tag (i.e. the commit)
+ git fetch origin +$GITHUB_REF:$GITHUB_REF
+
+ # Verify that the tag is annotated
+ test $(git cat-file -t "$GITHUB_REF") == "tag" || die "Tag ${{ steps.tag.outputs.name }} is not annotated"
+
+ # Verify tag follows rules in GIT-VERSION-GEN (i.e., matches the specified "DEF_VER" in
+ # GIT-VERSION-FILE) and matches tag determined from trigger
+ make GIT-VERSION-FILE
+ test "${{ steps.tag.outputs.version }}" == "$(sed -n 's/^GIT_VERSION = //p'< GIT-VERSION-FILE)" || die "GIT-VERSION-FILE tag does not match ${{ steps.tag.outputs.name }}"
+ # End check prerequisites for the workflow
+
+ # Build Windows installers (x86_64 installer & portable)
+ windows_pkg:
+ runs-on: windows-2019
+ environment: release
+ needs: prereqs
+ env:
+ GPG_OPTIONS: "--batch --yes --no-tty --list-options no-show-photos --verify-options no-show-photos --pinentry-mode loopback"
+ HOME: "${{github.workspace}}\\home"
+ USERPROFILE: "${{github.workspace}}\\home"
+ steps:
+ - name: Configure user
+ shell: bash
+ run:
+ USER_NAME="${{github.actor}}" &&
+ USER_EMAIL="${{github.actor}}@users.noreply.github.com" &&
+ mkdir -p "$HOME" &&
+ git config --global user.name "$USER_NAME" &&
+ git config --global user.email "$USER_EMAIL" &&
+ echo "PACKAGER=$USER_NAME <$USER_EMAIL>" >>$GITHUB_ENV
+ - uses: git-for-windows/setup-git-for-windows-sdk@v1
+ with:
+ flavor: build-installers
+ - name: Clone build-extra
+ shell: bash
+ run: |
+ git clone --filter=blob:none --single-branch -b main https://github.com/git-for-windows/build-extra /usr/src/build-extra
+ - name: Clone git
+ shell: bash
+ run: |
+ # Since we cannot directly clone a specified tag (as we would a branch with `git clone -b `),
+ # this clone has to be done manually (via init->fetch->reset).
+
+ tag_name="${{ needs.prereqs.outputs.tag_name }}" &&
+ git -c init.defaultBranch=main init &&
+ git remote add -f origin https://github.com/git-for-windows/git &&
+ git fetch "https://github.com/${{github.repository}}" refs/tags/${tag_name}:refs/tags/${tag_name} &&
+ git reset --hard ${tag_name}
+ - name: Prepare home directory for code-signing
+ env:
+ CODESIGN_P12: ${{secrets.CODESIGN_P12}}
+ CODESIGN_PASS: ${{secrets.CODESIGN_PASS}}
+ if: env.CODESIGN_P12 != '' && env.CODESIGN_PASS != ''
+ shell: bash
+ run: |
+ cd home &&
+ mkdir -p .sig &&
+ echo -n "$CODESIGN_P12" | tr % '\n' | base64 -d >.sig/codesign.p12 &&
+ echo -n "$CODESIGN_PASS" >.sig/codesign.pass
+ git config --global alias.signtool '!sh "/usr/src/build-extra/signtool.sh"'
+ - name: Prepare home directory for GPG signing
+ if: env.GPGKEY != ''
+ shell: bash
+ run: |
+ # This section ensures that the identity for the GPG key matches the git user identity, otherwise
+ # signing will fail
+
+ echo '${{secrets.PRIVGPGKEY}}' | tr % '\n' | gpg $GPG_OPTIONS --import &&
+ info="$(gpg --list-keys --with-colons "${GPGKEY%% *}" | cut -d : -f 1,10 | sed -n '/^uid/{s|uid:||p;q}')" &&
+ git config --global user.name "${info% <*}" &&
+ git config --global user.email "<${info#*<}"
+ env:
+ GPGKEY: ${{secrets.GPGKEY}}
+ - name: Build mingw-w64-x86_64-git
+ env:
+ GPGKEY: "${{secrets.GPGKEY}}"
+ shell: bash
+ run: |
+ set -x
+
+ # Make sure that there is a `/usr/bin/git` that can be used by `makepkg-mingw`
+ printf '#!/bin/sh\n\nexec /mingw64/bin/git.exe "$@"\n' >/usr/bin/git &&
+
+ # Restrict `PATH` to MSYS2 and to Visual Studio (to let `cv2pdb` find the relevant DLLs)
+ PATH="/mingw64/bin:/usr/bin:/C/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/amd64:/C/Windows/system32"
+
+ type -p mspdb140.dll || exit 1
+
+ sh -x /usr/src/build-extra/please.sh build-mingw-w64-git --only-64-bit --build-src-pkg -o artifacts HEAD &&
+ if test -n "$GPGKEY"
+ then
+ for tar in artifacts/*.tar*
+ do
+ /usr/src/build-extra/gnupg-with-gpgkey.sh --detach-sign --no-armor $tar
+ done
+ fi &&
+
+ b=$PWD/artifacts &&
+ version=${{ needs.prereqs.outputs.tag_name }} &&
+ (cd /usr/src/MINGW-packages/mingw-w64-git &&
+ cp PKGBUILD.$version PKGBUILD &&
+ git commit -s -m "mingw-w64-git: new version ($version)" PKGBUILD &&
+ git bundle create "$b"/MINGW-packages.bundle origin/main..main)
+ - name: Publish mingw-w64-x86_64-git
+ uses: actions/upload-artifact@v4
+ with:
+ name: pkg-x86_64
+ path: artifacts
+ windows_artifacts:
+ runs-on: windows-2019
+ environment: release
+ needs: [prereqs, windows_pkg]
+ env:
+ HOME: "${{github.workspace}}\\home"
+ strategy:
+ matrix:
+ artifact:
+ - name: installer
+ fileprefix: Git
+ - name: portable
+ fileprefix: PortableGit
+ fail-fast: false
+ steps:
+ - name: Download pkg-x86_64
+ uses: actions/download-artifact@v4
+ with:
+ name: pkg-x86_64
+ path: pkg-x86_64
+ - uses: git-for-windows/setup-git-for-windows-sdk@v1
+ with:
+ flavor: build-installers
+ - name: Clone build-extra
+ shell: bash
+ run: |
+ git clone --filter=blob:none --single-branch -b main https://github.com/git-for-windows/build-extra /usr/src/build-extra
+ - name: Prepare home directory for code-signing
+ env:
+ CODESIGN_P12: ${{secrets.CODESIGN_P12}}
+ CODESIGN_PASS: ${{secrets.CODESIGN_PASS}}
+ if: env.CODESIGN_P12 != '' && env.CODESIGN_PASS != ''
+ shell: bash
+ run: |
+ mkdir -p home/.sig &&
+ echo -n "$CODESIGN_P12" | tr % '\n' | base64 -d >home/.sig/codesign.p12 &&
+ echo -n "$CODESIGN_PASS" >home/.sig/codesign.pass &&
+ git config --global alias.signtool '!sh "/usr/src/build-extra/signtool.sh"'
+ - name: Retarget auto-update to microsoft/git
+ shell: bash
+ run: |
+ set -x
+
+ b=/usr/src/build-extra &&
+
+ filename=$b/git-update-git-for-windows.config
+ tr % '\t' >$filename <<-\EOF &&
+ [update]
+ %fromFork = microsoft/git
+ EOF
+
+ sed -i -e '/^#include "file-list.iss"/a\
+ Source: {#SourcePath}\\..\\git-update-git-for-windows.config; DestDir: {app}\\mingw64\\bin; Flags: replacesameversion; AfterInstall: DeleteFromVirtualStore' \
+ -e '/^Type: dirifempty; Name: {app}\\{#MINGW_BITNESS}$/i\
+ Type: files; Name: {app}\\{#MINGW_BITNESS}\\bin\\git-update-git-for-windows.config\
+ Type: dirifempty; Name: {app}\\{#MINGW_BITNESS}\\bin' \
+ $b/installer/install.iss
+ - name: Set alerts to continue until upgrade is taken
+ shell: bash
+ run: |
+ set -x
+
+ b=/mingw64/bin &&
+
+ sed -i -e '6 a use_recently_seen=no' \
+ $b/git-update-git-for-windows
+ - name: Set the installer Publisher to the Git Fundamentals team
+ shell: bash
+ run: |
+ b=/usr/src/build-extra &&
+ sed -i -e 's/^\(AppPublisher=\).*/\1The Git Fundamentals Team at GitHub/' $b/installer/install.iss
+ - name: Let the installer configure Visual Studio to use the installed Git
+ shell: bash
+ run: |
+ set -x
+
+ b=/usr/src/build-extra &&
+
+ sed -i -e '/^ *InstallAutoUpdater();$/a\
+ CustomPostInstall();' \
+ -e '/^ *UninstallAutoUpdater();$/a\
+ CustomPostUninstall();' \
+ $b/installer/install.iss &&
+
+ cat >>$b/installer/helpers.inc.iss <<\EOF
+
+ procedure CustomPostInstall();
+ begin
+ if not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\15.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or
+ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\16.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or
+ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\17.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or
+ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\18.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or
+ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\19.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or
+ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\20.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) then
+ LogError('Could not register TeamFoundation\GitSourceControl');
+ end;
+
+ procedure CustomPostUninstall();
+ begin
+ if not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\15.0\TeamFoundation\GitSourceControl','GitPath') or
+ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\16.0\TeamFoundation\GitSourceControl','GitPath') or
+ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\17.0\TeamFoundation\GitSourceControl','GitPath') or
+ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\18.0\TeamFoundation\GitSourceControl','GitPath') or
+ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\19.0\TeamFoundation\GitSourceControl','GitPath') or
+ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\20.0\TeamFoundation\GitSourceControl','GitPath') then
+ LogError('Could not register TeamFoundation\GitSourceControl');
+ end;
+ EOF
+ - name: Enable Scalar/C and the auto-updater in the installer by default
+ shell: bash
+ run: |
+ set -x
+
+ b=/usr/src/build-extra &&
+
+ sed -i -e "/ChosenOptions:=''/a\\
+ if (ExpandConstant('{param:components|/}')='/') then begin\n\
+ WizardSelectComponents('autoupdate');\n\
+ #ifdef WITH_SCALAR\n\
+ WizardSelectComponents('scalar');\n\
+ #endif\n\
+ end;" $b/installer/install.iss
+ - name: Build 64-bit ${{matrix.artifact.name}}
+ shell: bash
+ run: |
+ set -x
+
+ # Copy the PDB archive to the directory where `--include-pdbs` expects it
+ b=/usr/src/build-extra &&
+ mkdir -p $b/cached-source-packages &&
+ cp pkg-x86_64/*-pdb* $b/cached-source-packages/ &&
+
+ # Build the installer, embedding PDBs
+ eval $b/please.sh make_installers_from_mingw_w64_git --include-pdbs \
+ --version=${{ needs.prereqs.outputs.tag_version }} \
+ -o artifacts --${{matrix.artifact.name}} \
+ --pkg=pkg-x86_64/mingw-w64-x86_64-git-[0-9]*.tar.xz \
+ --pkg=pkg-x86_64/mingw-w64-x86_64-git-doc-html-[0-9]*.tar.xz &&
+
+ if test portable = '${{matrix.artifact.name}}' && test -n "$(git config alias.signtool)"
+ then
+ git signtool artifacts/PortableGit-*.exe
+ fi &&
+ openssl dgst -sha256 artifacts/${{matrix.artifact.fileprefix}}-*.exe | sed "s/.* //" >artifacts/sha-256.txt
+ - name: Verify that .exe files are code-signed
+ if: env.CODESIGN_P12 != '' && env.CODESIGN_PASS != ''
+ shell: bash
+ run: |
+ PATH=$PATH:"/c/Program Files (x86)/Windows Kits/10/App Certification Kit/" \
+ signtool verify //pa artifacts/${{matrix.artifact.fileprefix}}-*.exe
+ - name: Publish ${{matrix.artifact.name}}-x86_64
+ uses: actions/upload-artifact@v4
+ with:
+ name: win-${{matrix.artifact.name}}-x86_64
+ path: artifacts
+ # End build Windows installers
+
+ # Build and sign Mac OSX installers & upload artifacts
+ create-macos-artifacts:
+ strategy:
+ matrix:
+ arch:
+ - name: arm64
+ runner: macos-latest-xl-arm64
+ runs-on: ${{ matrix.arch.runner }}
+ needs: prereqs
+ env:
+ VERSION: "${{ needs.prereqs.outputs.tag_version }}"
+ environment: release
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@v4
+ with:
+ path: 'git'
+
+ - name: Install Git dependencies
+ run: |
+ set -ex
+
+ # Install x86_64 packages
+ arch -x86_64 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+ arch -x86_64 /usr/local/bin/brew install gettext
+
+ # Install arm64 packages
+ brew install automake asciidoc xmlto docbook
+ brew link --force gettext
+
+ # Make universal gettext library
+ lipo -create -output libintl.a /usr/local/opt/gettext/lib/libintl.a /opt/homebrew/opt/gettext/lib/libintl.a
+
+ - name: Set up signing/notarization infrastructure
+ env:
+ A1: ${{ secrets.APPLICATION_CERTIFICATE_BASE64 }}
+ A2: ${{ secrets.APPLICATION_CERTIFICATE_PASSWORD }}
+ I1: ${{ secrets.INSTALLER_CERTIFICATE_BASE64 }}
+ I2: ${{ secrets.INSTALLER_CERTIFICATE_PASSWORD }}
+ N1: ${{ secrets.APPLE_TEAM_ID }}
+ N2: ${{ secrets.APPLE_DEVELOPER_ID }}
+ N3: ${{ secrets.APPLE_DEVELOPER_PASSWORD }}
+ N4: ${{ secrets.APPLE_KEYCHAIN_PROFILE }}
+ run: |
+ echo "Setting up signing certificates"
+ security create-keychain -p pwd $RUNNER_TEMP/buildagent.keychain
+ security default-keychain -s $RUNNER_TEMP/buildagent.keychain
+ security unlock-keychain -p pwd $RUNNER_TEMP/buildagent.keychain
+ # Prevent re-locking
+ security set-keychain-settings $RUNNER_TEMP/buildagent.keychain
+
+ echo "$A1" | base64 -D > $RUNNER_TEMP/cert.p12
+ security import $RUNNER_TEMP/cert.p12 \
+ -k $RUNNER_TEMP/buildagent.keychain \
+ -P "$A2" \
+ -T /usr/bin/codesign
+ security set-key-partition-list \
+ -S apple-tool:,apple:,codesign: \
+ -s -k pwd \
+ $RUNNER_TEMP/buildagent.keychain
+
+ echo "$I1" | base64 -D > $RUNNER_TEMP/cert.p12
+ security import $RUNNER_TEMP/cert.p12 \
+ -k $RUNNER_TEMP/buildagent.keychain \
+ -P "$I2" \
+ -T /usr/bin/pkgbuild
+ security set-key-partition-list \
+ -S apple-tool:,apple:,pkgbuild: \
+ -s -k pwd \
+ $RUNNER_TEMP/buildagent.keychain
+
+ echo "Setting up notarytool"
+ xcrun notarytool store-credentials \
+ --team-id "$N1" \
+ --apple-id "$N2" \
+ --password "$N3" \
+ "$N4"
+
+ - name: Build, sign, and notarize artifacts
+ env:
+ A3: ${{ secrets.APPLE_APPLICATION_SIGNING_IDENTITY }}
+ I3: ${{ secrets.APPLE_INSTALLER_SIGNING_IDENTITY }}
+ N4: ${{ secrets.APPLE_KEYCHAIN_PROFILE }}
+ run: |
+ die () {
+ echo "$*" >&2
+ exit 1
+ }
+
+ # Trace execution, stop on error
+ set -ex
+
+ # Write to "version" file to force match with trigger payload version
+ echo "${{ needs.prereqs.outputs.tag_version }}" >>git/version
+
+ # Configure universal build
+ cat >git/config.mak <>git/config.mak <>git/config.mak <>git/config.mak
+
+ # To make use of the catalogs...
+ export XML_CATALOG_FILES=$homebrew_prefix/etc/xml/catalog
+
+ make -C git -j$(sysctl -n hw.physicalcpu) GIT-VERSION-FILE dist dist-doc
+
+ export GIT_BUILT_FROM_COMMIT=$(gunzip -c git/git-$VERSION.tar.gz | git get-tar-commit-id) ||
+ die "Could not determine commit for build"
+
+ # Extract tarballs
+ mkdir payload manpages
+ tar -xvf git/git-$VERSION.tar.gz -C payload
+ tar -xvf git/git-manpages-$VERSION.tar.gz -C manpages
+
+ # Lay out payload
+ cp git/config.mak payload/git-$VERSION/config.mak
+ make -C git/.github/macos-installer V=1 payload
+
+ # Codesign payload
+ cp -R stage/git-universal-$VERSION/ \
+ git/.github/macos-installer/build-artifacts
+ make -C git/.github/macos-installer V=1 codesign \
+ APPLE_APP_IDENTITY="$A3" || die "Creating signed payload failed"
+
+ # Build and sign pkg
+ make -C git/.github/macos-installer V=1 pkg \
+ APPLE_INSTALLER_IDENTITY="$I3" \
+ || die "Creating signed pkg failed"
+
+ # Notarize pkg
+ make -C git/.github/macos-installer V=1 notarize \
+ APPLE_INSTALLER_IDENTITY="$I3" APPLE_KEYCHAIN_PROFILE="$N4" \
+ || die "Creating signed and notarized pkg failed"
+
+ # Create DMG
+ make -C git/.github/macos-installer V=1 image || die "Creating DMG failed"
+
+ # Move all artifacts into top-level directory
+ mv git/.github/macos-installer/disk-image/*.pkg git/.github/macos-installer/
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: macos-artifacts
+ path: |
+ git/.github/macos-installer/*.dmg
+ git/.github/macos-installer/*.pkg
+ # End build and sign Mac OSX installers
+
+ # Build and sign Debian package
+ create-linux-artifacts:
+ runs-on: ubuntu-latest
+ needs: prereqs
+ environment: release
+ steps:
+ - name: Install git dependencies
+ run: |
+ set -ex
+ sudo apt-get update -q
+ sudo apt-get install -y -q --no-install-recommends gettext libcurl4-gnutls-dev libpcre3-dev asciidoc xmlto
+
+ - name: Clone git
+ uses: actions/checkout@v4
+ with:
+ path: git
+
+ - name: Build and create Debian package
+ run: |
+ set -ex
+
+ die () {
+ echo "$*" >&2
+ exit 1
+ }
+
+ echo "${{ needs.prereqs.outputs.tag_version }}" >>git/version
+ make -C git GIT-VERSION-FILE
+
+ VERSION="${{ needs.prereqs.outputs.tag_version }}"
+
+ ARCH="$(dpkg-architecture -q DEB_HOST_ARCH)"
+ if test -z "$ARCH"; then
+ die "Could not determine host architecture!"
+ fi
+
+ PKGNAME="microsoft-git_$VERSION"
+ PKGDIR="$(dirname $(pwd))/$PKGNAME"
+
+ rm -rf "$PKGDIR"
+ mkdir -p "$PKGDIR"
+
+ DESTDIR="$PKGDIR" make -C git -j5 V=1 DEVELOPER=1 \
+ USE_LIBPCRE=1 \
+ NO_CROSS_DIRECTORY_HARDLINKS=1 \
+ ASCIIDOC8=1 ASCIIDOC_NO_ROFF=1 \
+ ASCIIDOC='TZ=UTC asciidoc' \
+ prefix=/usr/local \
+ gitexecdir=/usr/local/lib/git-core \
+ libexecdir=/usr/local/lib/git-core \
+ htmldir=/usr/local/share/doc/git/html \
+ install install-doc install-html
+
+ cd ..
+ mkdir "$PKGNAME/DEBIAN"
+
+ # Based on https://packages.ubuntu.com/xenial/vcs/git
+ cat >"$PKGNAME/DEBIAN/control" <
+ Description: Git client built from the https://github.com/microsoft/git repository,
+ specialized in supporting monorepo scenarios. Includes the Scalar CLI.
+ EOF
+
+ dpkg-deb -Zxz --build "$PKGNAME"
+ # Move Debian package for later artifact upload
+ mv "$PKGNAME.deb" "$GITHUB_WORKSPACE"
+
+ - name: Log into Azure
+ uses: azure/login@v1
+ with:
+ client-id: ${{ secrets.AZURE_CLIENT_ID }}
+ tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+ subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+ - name: Prepare for GPG signing
+ env:
+ AZURE_VAULT: ${{ secrets.AZURE_VAULT }}
+ GPG_KEY_SECRET_NAME: ${{ secrets.GPG_KEY_SECRET_NAME }}
+ GPG_PASSPHRASE_SECRET_NAME: ${{ secrets.GPG_PASSPHRASE_SECRET_NAME }}
+ GPG_KEYGRIP_SECRET_NAME: ${{ secrets.GPG_KEYGRIP_SECRET_NAME }}
+ run: |
+ # Install debsigs
+ sudo apt install debsigs
+
+ # Download GPG key, passphrase, and keygrip from Azure Key Vault
+ key=$(az keyvault secret show --name $GPG_KEY_SECRET_NAME --vault-name $AZURE_VAULT --query "value")
+ passphrase=$(az keyvault secret show --name $GPG_PASSPHRASE_SECRET_NAME --vault-name $AZURE_VAULT --query "value")
+ keygrip=$(az keyvault secret show --name $GPG_KEYGRIP_SECRET_NAME --vault-name $AZURE_VAULT --query "value")
+
+ # Remove quotes from downloaded values
+ key=$(sed -e 's/^"//' -e 's/"$//' <<<"$key")
+ passphrase=$(sed -e 's/^"//' -e 's/"$//' <<<"$passphrase")
+ keygrip=$(sed -e 's/^"//' -e 's/"$//' <<<"$keygrip")
+
+ # Import GPG key
+ echo "$key" | base64 -d | gpg --import --no-tty --batch --yes
+
+ # Configure GPG
+ echo "allow-preset-passphrase" > ~/.gnupg/gpg-agent.conf
+ gpg-connect-agent RELOADAGENT /bye
+ /usr/lib/gnupg2/gpg-preset-passphrase --preset "$keygrip" <<<"$passphrase"
+
+ - name: Sign Debian package
+ run: |
+ # Sign Debian package
+ version="${{ needs.prereqs.outputs.tag_version }}"
+ debsigs --sign=origin --verify --check microsoft-git_"$version".deb
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: linux-artifacts
+ path: |
+ *.deb
+ # End build and sign Debian package
+
+ # Validate installers
+ validate-installers:
+ name: Validate installers
+ strategy:
+ matrix:
+ component:
+ - os: ubuntu-latest
+ artifact: linux-artifacts
+ command: git
+ - os: macos-latest-xl-arm64
+ artifact: macos-artifacts
+ command: git
+ - os: macos-latest
+ artifact: macos-artifacts
+ command: git
+ - os: windows-latest
+ artifact: win-installer-x86_64
+ command: $PROGRAMFILES\Git\cmd\git.exe
+ runs-on: ${{ matrix.component.os }}
+ needs: [prereqs, windows_artifacts, create-macos-artifacts, create-linux-artifacts]
+ steps:
+ - name: Download artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{ matrix.component.artifact }}
+
+ - name: Install Windows
+ if: contains(matrix.component.os, 'windows')
+ shell: pwsh
+ run: |
+ $exePath = Get-ChildItem -Path ./*.exe | %{$_.FullName}
+ Start-Process -Wait -FilePath "$exePath" -ArgumentList "/SILENT /VERYSILENT /NORESTART /SUPPRESSMSGBOXES /ALLOWDOWNGRADE=1"
+
+ - name: Install Linux
+ if: contains(matrix.component.os, 'ubuntu')
+ run: |
+ debpath=$(find ./*.deb)
+ sudo apt install $debpath
+
+ - name: Install macOS
+ if: contains(matrix.component.os, 'macos')
+ run: |
+ # avoid letting Homebrew's `git` in `/opt/homebrew/bin` override `/usr/local/bin/git`
+ arch="$(uname -m)"
+ test arm64 != "$arch" ||
+ brew uninstall git
+
+ pkgpath=$(find ./*universal*.pkg)
+ sudo installer -pkg $pkgpath -target /
+
+ - name: Validate
+ shell: bash
+ run: |
+ "${{ matrix.component.command }}" --version | sed 's/git version //' >actual
+ echo ${{ needs.prereqs.outputs.tag_version }} >expect
+ cmp expect actual || exit 1
+
+ - name: Validate universal binary CPU architecture
+ if: contains(matrix.component.os, 'macos')
+ shell: bash
+ run: |
+ set -ex
+ git version --build-options >actual
+ cat actual
+ grep "cpu: $(uname -m)" actual
+ # End validate installers
+
+ create-github-release:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ id-token: write # required for Azure login via OIDC
+ needs:
+ - validate-installers
+ - create-linux-artifacts
+ - create-macos-artifacts
+ - windows_artifacts
+ - prereqs
+ env:
+ AZURE_VAULT: ${{ secrets.AZURE_VAULT }}
+ GPG_PUBLIC_KEY_SECRET_NAME: ${{ secrets.GPG_PUBLIC_KEY_SECRET_NAME }}
+ environment: release
+ if: |
+ success() ||
+ (needs.create-linux-artifacts.result == 'skipped' &&
+ needs.create-macos-artifacts.result == 'success' &&
+ needs.windows_artifacts.result == 'success')
+ steps:
+ - name: Download Windows portable installer
+ uses: actions/download-artifact@v4
+ with:
+ name: win-portable-x86_64
+ path: win-portable-x86_64
+
+ - name: Download Windows x86_64 installer
+ uses: actions/download-artifact@v4
+ with:
+ name: win-installer-x86_64
+ path: win-installer-x86_64
+
+ - name: Download macOS artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: macos-artifacts
+ path: macos-artifacts
+
+ - name: Download Debian package
+ uses: actions/download-artifact@v4
+ with:
+ name: linux-artifacts
+ path: deb-package
+
+ - name: Log into Azure
+ uses: azure/login@v1
+ with:
+ client-id: ${{ secrets.AZURE_CLIENT_ID }}
+ tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+ subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+ - name: Download GPG public key signature file
+ run: |
+ az keyvault secret show --name "$GPG_PUBLIC_KEY_SECRET_NAME" \
+ --vault-name "$AZURE_VAULT" --query "value" \
+ | sed -e 's/^"//' -e 's/"$//' | base64 -d >msft-git-public.asc
+ mv msft-git-public.asc deb-package
+
+ - uses: actions/github-script@v6
+ with:
+ script: |
+ const fs = require('fs');
+ const path = require('path');
+
+ var releaseMetadata = {
+ owner: context.repo.owner,
+ repo: context.repo.repo
+ };
+
+ // Create the release
+ var tagName = "${{ needs.prereqs.outputs.tag_name }}";
+ var createdRelease = await github.rest.repos.createRelease({
+ ...releaseMetadata,
+ draft: true,
+ tag_name: tagName,
+ name: tagName
+ });
+ releaseMetadata.release_id = createdRelease.data.id;
+
+ // Uploads contents of directory to the release created above
+ async function uploadDirectoryToRelease(directory, includeExtensions=[]) {
+ return fs.promises.readdir(directory)
+ .then(async(files) => Promise.all(
+ files.filter(file => {
+ return includeExtensions.length==0 || includeExtensions.includes(path.extname(file).toLowerCase());
+ })
+ .map(async (file) => {
+ var filePath = path.join(directory, file);
+ github.rest.repos.uploadReleaseAsset({
+ ...releaseMetadata,
+ name: file,
+ headers: {
+ "content-length": (await fs.promises.stat(filePath)).size
+ },
+ data: fs.createReadStream(filePath)
+ });
+ }))
+ );
+ }
+
+ await Promise.all([
+ // Upload Windows artifacts
+ uploadDirectoryToRelease('win-installer-x86_64', ['.exe']),
+ uploadDirectoryToRelease('win-portable-x86_64', ['.exe']),
+
+ // Upload Mac artifacts
+ uploadDirectoryToRelease('macos-artifacts'),
+
+ // Upload Ubuntu artifacts
+ uploadDirectoryToRelease('deb-package')
+ ]);
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 82d45f4f42c351..43a9c56cff7b1a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -164,7 +164,7 @@ jobs:
vs-build:
name: win+VS build
needs: ci-config
- if: github.event.repository.owner.login == 'git-for-windows' && needs.ci-config.outputs.enabled == 'yes'
+ if: github.event.repository.owner.login == 'microsoft' && needs.ci-config.outputs.enabled == 'yes'
env:
NO_PERL: 1
GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'"
diff --git a/.github/workflows/monitor-components.yml b/.github/workflows/monitor-components.yml
deleted file mode 100644
index fedc6add69a9e0..00000000000000
--- a/.github/workflows/monitor-components.yml
+++ /dev/null
@@ -1,98 +0,0 @@
-name: Monitor component updates
-
-# Git for Windows is a slightly modified subset of MSYS2. Some of its
-# components are maintained by Git for Windows, others by MSYS2. To help
-# keeping the former up to date, this workflow monitors the Atom/RSS feeds
-# and opens new tickets for each new component version.
-
-on:
- schedule:
- - cron: "23 8,11,14,17 * * *"
- workflow_dispatch:
-
-env:
- CHARACTER_LIMIT: 5000
- MAX_AGE: 7d
-
-jobs:
- job:
- # Only run this in Git for Windows' fork
- if: github.event.repository.owner.login == 'git-for-windows'
- runs-on: ubuntu-latest
- permissions:
- issues: write
- strategy:
- matrix:
- component:
- - label: git
- feed: https://github.com/git/git/tags.atom
- - label: git-lfs
- feed: https://github.com/git-lfs/git-lfs/tags.atom
- - label: git-credential-manager
- feed: https://github.com/git-ecosystem/git-credential-manager/tags.atom
- - label: tig
- feed: https://github.com/jonas/tig/tags.atom
- - label: cygwin
- feed: https://github.com/cygwin/cygwin/releases.atom
- title-pattern: ^(?!.*newlib)
- - label: msys2-runtime-package
- feed: https://github.com/msys2/MSYS2-packages/commits/master/msys2-runtime.atom
- - label: msys2-runtime
- feed: https://github.com/msys2/msys2-runtime/commits/HEAD.atom
- aggregate: true
- - label: openssh
- feed: https://github.com/openssh/openssh-portable/tags.atom
- - label: libfido2
- feed: https://github.com/Yubico/libfido2/tags.atom
- - label: libcbor
- feed: https://github.com/PJK/libcbor/tags.atom
- - label: openssl
- feed: https://github.com/openssl/openssl/tags.atom
- title-pattern: ^(?!.*alpha)
- - label: gnutls
- feed: https://gnutls.org/news.atom
- - label: heimdal
- feed: https://github.com/heimdal/heimdal/tags.atom
- - label: git-sizer
- feed: https://github.com/github/git-sizer/tags.atom
- - label: gitflow
- feed: https://github.com/petervanderdoes/gitflow-avh/tags.atom
- - label: curl
- feed: https://github.com/curl/curl/tags.atom
- - label: libgpg-error
- feed: https://github.com/gpg/libgpg-error/releases.atom
- title-pattern: ^libgpg-error-[0-9\.]*$
- - label: libgcrypt
- feed: https://github.com/gpg/libgcrypt/releases.atom
- title-pattern: ^libgcrypt-[0-9\.]*$
- - label: gpg
- feed: https://github.com/gpg/gnupg/releases.atom
- - label: mintty
- feed: https://github.com/mintty/mintty/releases.atom
- - label: 7-zip
- feed: https://sourceforge.net/projects/sevenzip/rss?path=/7-Zip
- aggregate: true
- - label: bash
- feed: https://git.savannah.gnu.org/cgit/bash.git/atom/?h=master
- aggregate: true
- - label: perl
- feed: https://github.com/Perl/perl5/tags.atom
- title-pattern: ^(?!.*(5\.[0-9]+[13579]|RC))
- - label: pcre2
- feed: https://github.com/PCRE2Project/pcre2/tags.atom
- - label: mingw-w64-llvm
- feed: https://github.com/msys2/MINGW-packages/commits/master/mingw-w64-llvm.atom
- - label: innosetup
- feed: https://github.com/jrsoftware/issrc/tags.atom
- fail-fast: false
- steps:
- - uses: git-for-windows/rss-to-issues@v0
- with:
- feed: ${{matrix.component.feed}}
- prefix: "[New ${{matrix.component.label}} version]"
- labels: component-update
- github-token: ${{ secrets.GITHUB_TOKEN }}
- character-limit: ${{ env.CHARACTER_LIMIT }}
- max-age: ${{ env.MAX_AGE }}
- aggregate: ${{matrix.component.aggregate}}
- title-pattern: ${{matrix.component.title-pattern}}
diff --git a/.github/workflows/release-homebrew.yml b/.github/workflows/release-homebrew.yml
new file mode 100644
index 00000000000000..e2a2634ff60c97
--- /dev/null
+++ b/.github/workflows/release-homebrew.yml
@@ -0,0 +1,31 @@
+name: Update Homebrew Tap
+on:
+ release:
+ types: [released]
+
+jobs:
+ release:
+ runs-on: ubuntu-latest
+ environment: release
+ steps:
+ - id: version
+ name: Compute version number
+ run: |
+ echo "result=$(echo $GITHUB_REF | sed -e "s/^refs\/tags\/v//")" >>$GITHUB_OUTPUT
+ - id: hash
+ name: Compute release asset hash
+ uses: mjcheetham/asset-hash@v1.1
+ with:
+ asset: /git-(.*)\.pkg/
+ hash: sha256
+ token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Update scalar Cask
+ uses: mjcheetham/update-homebrew@v1.3
+ with:
+ token: ${{ secrets.HOMEBREW_TOKEN }}
+ tap: microsoft/git
+ name: microsoft-git
+ type: cask
+ version: ${{ steps.version.outputs.result }}
+ sha256: ${{ steps.hash.outputs.result }}
+ alwaysUsePullRequest: false
diff --git a/.github/workflows/release-winget.yml b/.github/workflows/release-winget.yml
new file mode 100644
index 00000000000000..61010a5ce65abb
--- /dev/null
+++ b/.github/workflows/release-winget.yml
@@ -0,0 +1,41 @@
+name: "release-winget"
+on:
+ release:
+ types: [released]
+
+ workflow_dispatch:
+ inputs:
+ release:
+ description: 'Release Id'
+ required: true
+ default: 'latest'
+
+jobs:
+ release:
+ runs-on: windows-latest
+ environment: release
+ steps:
+ - name: Publish manifest with winget-create
+ run: |
+ # Get correct release asset
+ $github = Get-Content '${{ github.event_path }}' | ConvertFrom-Json
+ $asset = $github.release.assets | Where-Object -Property name -match '64-bit.exe$'
+
+ # Remove 'v' and 'vfs' from the version
+ $github.release.tag_name -match '\d.*'
+ $version = $Matches[0] -replace ".vfs",""
+
+ # Download wingetcreate and create manifests
+ Invoke-WebRequest https://aka.ms/wingetcreate/latest -OutFile wingetcreate.exe
+ .\wingetcreate.exe update Microsoft.Git -u $asset.browser_download_url -v $version -o manifests
+
+ # Manually substitute the name of the default branch in the License
+ # and Copyright URLs since the tooling cannot do that for us.
+ $shortenedVersion = $version -replace ".{4}$"
+ $manifestPath = dir -Path ./manifests -Filter Microsoft.Git.locale.en-US.yaml -Recurse | %{$_.FullName}
+ sed -i "s/vfs-[.0-9]*/vfs-$shortenedVersion/g" "$manifestPath"
+
+ # Submit manifests
+ $manifestDirectory = Split-Path "$manifestPath"
+ .\wingetcreate.exe submit -t "${{ secrets.WINGET_TOKEN }}" $manifestDirectory
+ shell: powershell
diff --git a/.github/workflows/scalar-functional-tests.yml b/.github/workflows/scalar-functional-tests.yml
new file mode 100644
index 00000000000000..da9f3a0d14bb2f
--- /dev/null
+++ b/.github/workflows/scalar-functional-tests.yml
@@ -0,0 +1,220 @@
+name: Scalar Functional Tests
+
+env:
+ SCALAR_REPOSITORY: microsoft/scalar
+ SCALAR_REF: main
+ DEBUG_WITH_TMATE: false
+ SCALAR_TEST_SKIP_VSTS_INFO: true
+
+on:
+ push:
+ branches: [ vfs-*, tentative/vfs-* ]
+ pull_request:
+ branches: [ vfs-*, features/* ]
+
+jobs:
+ scalar:
+ name: "Scalar Functional Tests"
+
+ strategy:
+ fail-fast: false
+ matrix:
+ # Order by runtime (in descending order)
+ os: [windows-2019, macos-13, ubuntu-20.04, ubuntu-22.04]
+ # Scalar.NET used to be tested using `features: [false, experimental]`
+ # But currently, Scalar/C ignores `feature.scalar` altogether, so let's
+ # save some electrons and run only one of them...
+ features: [ignored]
+ exclude:
+ # The built-in FSMonitor is not (yet) supported on Linux
+ - os: ubuntu-20.04
+ features: experimental
+ - os: ubuntu-22.04
+ features: experimental
+ runs-on: ${{ matrix.os }}
+
+ env:
+ BUILD_FRAGMENT: bin/Release/netcoreapp3.1
+ GIT_FORCE_UNTRACKED_CACHE: 1
+
+ steps:
+ - name: Check out Git's source code
+ uses: actions/checkout@v3
+
+ - name: Setup build tools on Windows
+ if: runner.os == 'Windows'
+ uses: git-for-windows/setup-git-for-windows-sdk@v1
+
+ - name: Provide a minimal `install` on Windows
+ if: runner.os == 'Windows'
+ shell: bash
+ run: |
+ test -x /usr/bin/install ||
+ tr % '\t' >/usr/bin/install <<-\EOF
+ #!/bin/sh
+
+ cmd=cp
+ while test $# != 0
+ do
+ %case "$1" in
+ %-d) cmd="mkdir -p";;
+ %-m) shift;; # ignore mode
+ %*) break;;
+ %esac
+ %shift
+ done
+
+ exec $cmd "$@"
+ EOF
+
+ - name: Install build dependencies for Git (Linux)
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get update
+ sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev gettext
+
+ - name: Build and install Git
+ shell: bash
+ env:
+ NO_TCLTK: Yup
+ run: |
+ # We do require a VFS version
+ def_ver="$(sed -n 's/DEF_VER=\(.*vfs.*\)/\1/p' GIT-VERSION-GEN)"
+ test -n "$def_ver"
+
+ # Ensure that `git version` reflects DEF_VER
+ case "$(git describe --match "v[0-9]*vfs*" HEAD)" in
+ ${def_ver%%.vfs.*}.vfs.*) ;; # okay, we can use this
+ *) git -c user.name=ci -c user.email=ci@github tag -m for-testing ${def_ver}.NNN.g$(git rev-parse --short HEAD);;
+ esac
+
+ SUDO=
+ extra=
+ case "${{ runner.os }}" in
+ Windows)
+ extra=DESTDIR=/c/Progra~1/Git
+ cygpath -aw "/c/Program Files/Git/cmd" >>$GITHUB_PATH
+ ;;
+ Linux)
+ SUDO=sudo
+ extra=prefix=/usr
+ ;;
+ macOS)
+ SUDO=sudo
+ extra=prefix=/usr/local
+ ;;
+ esac
+
+ $SUDO make -j5 $extra install
+
+ - name: Ensure that we use the built Git and Scalar
+ shell: bash
+ run: |
+ type -p git
+ git version
+ case "$(git version)" in *.vfs.*) echo Good;; *) exit 1;; esac
+ type -p scalar
+ scalar version
+ case "$(scalar version 2>&1)" in *.vfs.*) echo Good;; *) exit 1;; esac
+
+ - name: Check out Scalar's source code
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0 # Indicate full history so Nerdbank.GitVersioning works.
+ path: scalar
+ repository: ${{ env.SCALAR_REPOSITORY }}
+ ref: ${{ env.SCALAR_REF }}
+
+ - name: Setup .NET Core
+ uses: actions/setup-dotnet@v3
+ with:
+ dotnet-version: '3.1.x'
+
+ - name: Install dependencies
+ run: dotnet restore
+ working-directory: scalar
+ env:
+ DOTNET_NOLOGO: 1
+
+ - name: Build
+ working-directory: scalar
+ run: dotnet build --configuration Release --no-restore -p:UseAppHost=true # Force generation of executable on macOS.
+
+ - name: Setup platform (Linux)
+ if: runner.os == 'Linux'
+ run: |
+ echo "BUILD_PLATFORM=${{ runner.os }}" >>$GITHUB_ENV
+ echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV
+
+ - name: Setup platform (Mac)
+ if: runner.os == 'macOS'
+ run: |
+ echo 'BUILD_PLATFORM=Mac' >>$GITHUB_ENV
+ echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV
+
+ - name: Setup platform (Windows)
+ if: runner.os == 'Windows'
+ run: |
+ echo "BUILD_PLATFORM=${{ runner.os }}" >>$env:GITHUB_ENV
+ echo 'BUILD_FILE_EXT=.exe' >>$env:GITHUB_ENV
+ echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$env:GITHUB_ENV
+
+ - name: Configure feature.scalar
+ run: git config --global feature.scalar ${{ matrix.features }}
+
+ - id: functional_test
+ name: Functional test
+ timeout-minutes: 60
+ working-directory: scalar
+ shell: bash
+ run: |
+ export GIT_TRACE2_EVENT="$PWD/$TRACE2_BASENAME/Event"
+ export GIT_TRACE2_PERF="$PWD/$TRACE2_BASENAME/Perf"
+ export GIT_TRACE2_EVENT_BRIEF=true
+ export GIT_TRACE2_PERF_BRIEF=true
+ mkdir -p "$TRACE2_BASENAME"
+ mkdir -p "$TRACE2_BASENAME/Event"
+ mkdir -p "$TRACE2_BASENAME/Perf"
+ git version --build-options
+ cd ../out
+ Scalar.FunctionalTests/$BUILD_FRAGMENT/Scalar.FunctionalTests$BUILD_FILE_EXT --test-scalar-on-path --test-git-on-path --timeout=300000 --full-suite
+
+ - name: Force-stop FSMonitor daemons and Git processes (Windows)
+ if: runner.os == 'Windows' && (success() || failure())
+ shell: bash
+ run: |
+ set -x
+ wmic process get CommandLine,ExecutablePath,HandleCount,Name,ParentProcessID,ProcessID
+ wmic process where "CommandLine Like '%fsmonitor--daemon %run'" delete
+ wmic process where "ExecutablePath Like '%git.exe'" delete
+
+ - id: trace2_zip_unix
+ if: runner.os != 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' )
+ name: Zip Trace2 Logs (Unix)
+ shell: bash
+ working-directory: scalar
+ run: zip -q -r $TRACE2_BASENAME.zip $TRACE2_BASENAME/
+
+ - id: trace2_zip_windows
+ if: runner.os == 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' )
+ name: Zip Trace2 Logs (Windows)
+ working-directory: scalar
+ run: Compress-Archive -DestinationPath ${{ env.TRACE2_BASENAME }}.zip -Path ${{ env.TRACE2_BASENAME }}
+
+ - name: Archive Trace2 Logs
+ if: ( success() || failure() ) && ( steps.trace2_zip_unix.conclusion == 'success' || steps.trace2_zip_windows.conclusion == 'success' )
+ uses: actions/upload-artifact@v3
+ with:
+ name: ${{ env.TRACE2_BASENAME }}.zip
+ path: scalar/${{ env.TRACE2_BASENAME }}.zip
+ retention-days: 3
+
+ # The GitHub Action `action-tmate` allows developers to connect to the running agent
+ # using SSH (it will be a `tmux` session; on Windows agents it will be inside the MSYS2
+ # environment in `C:\msys64`, therefore it can be slightly tricky to interact with
+ # Git for Windows, which runs a slightly incompatible MSYS2 runtime).
+ - name: action-tmate
+ if: env.DEBUG_WITH_TMATE == 'true' && failure()
+ uses: mxschmitt/action-tmate@v3
+ with:
+ limit-access-to-actor: true
diff --git a/.gitignore b/.gitignore
index bf97276163b19b..e1b01493765301 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
/fuzz_corpora
/GIT-BUILD-DIR
/GIT-BUILD-OPTIONS
+/GIT-BUILT-FROM-COMMIT
/GIT-CFLAGS
/GIT-LDFLAGS
/GIT-PREFIX
@@ -73,6 +74,7 @@
/git-gc
/git-get-tar-commit-id
/git-grep
+/git-gvfs-helper
/git-hash-object
/git-help
/git-hook
@@ -164,6 +166,7 @@
/git-submodule
/git-submodule--helper
/git-subtree
+/git-survey
/git-svn
/git-switch
/git-symbolic-ref
@@ -171,6 +174,7 @@
/git-unpack-file
/git-unpack-objects
/git-update-index
+/git-update-microsoft-git
/git-update-ref
/git-update-server-info
/git-upload-archive
diff --git a/BRANCHES.md b/BRANCHES.md
new file mode 100644
index 00000000000000..364158375e7d55
--- /dev/null
+++ b/BRANCHES.md
@@ -0,0 +1,59 @@
+Branches used in this repo
+==========================
+
+The document explains the branching structure that we are using in the VFSForGit repository as well as the forking strategy that we have adopted for contributing.
+
+Repo Branches
+-------------
+
+1. `vfs-#`
+
+ These branches are used to track the specific version that match Git for Windows with the VFSForGit specific patches on top. When a new version of Git for Windows is released, the VFSForGit patches will be rebased on that windows version and a new gvfs-# branch created to create pull requests against.
+
+ #### Examples
+
+ ```
+ vfs-2.27.0
+ vfs-2.30.0
+ ```
+
+ The versions of git for VFSForGit are based on the Git for Windows versions. v2.20.0.vfs.1 will correspond with the v2.20.0.windows.1 with the VFSForGit specific patches applied to the windows version.
+
+2. `vfs-#-exp`
+
+ These branches are for releasing experimental features to early adopters. They
+ should contain everything within the corresponding `vfs-#` branch; if the base
+ branch updates, then merge into the `vfs-#-exp` branch as well.
+
+Tags
+----
+
+We are using annotated tags to build the version number for git. The build will look back through the commit history to find the first tag matching `v[0-9]*vfs*` and build the git version number using that tag.
+
+Full releases are of the form `v2.XX.Y.vfs.Z.W` where `v2.XX.Y` comes from the
+upstream version and `Z.W` are custom updates within our fork. Specifically,
+the `.Z` value represents the "compatibility level" with VFS for Git. Only
+increase this version when making a breaking change with a released version
+of VFS for Git. The `.W` version is used for minor updates between major
+versions.
+
+Experimental releases are of the form `v2.XX.Y.vfs.Z.W.exp`. The `.exp`
+suffix indicates that experimental features are available. The rest of the
+version string comes from the full release tag. These versions will only
+be made available as pre-releases on the releases page, never a full release.
+
+Forking
+-------
+
+A personal fork of this repository and a branch in that repository should be used for development.
+
+These branches should be based on the latest vfs-# branch. If there are work in progress pull requests that you have based on a previous version branch when a new version branch is created, you will need to move your patches to the new branch to get them in that latest version.
+
+#### Example
+
+```
+git clone
+git remote add ms https://github.com/Microsoft/git.git
+git checkout -b my-changes ms/vfs-2.20.0 --no-track
+git push -fu origin HEAD
+```
diff --git a/Documentation/config.txt b/Documentation/config.txt
index fedfaf30cd0d8b..b87cb7a593b368 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -448,6 +448,8 @@ include::config/gui.txt[]
include::config/guitool.txt[]
+include::config/gvfs.txt[]
+
include::config/help.txt[]
include::config/http.txt[]
@@ -536,6 +538,8 @@ include::config/status.txt[]
include::config/submodule.txt[]
+include::config/survey.txt[]
+
include::config/tag.txt[]
include::config/tar.txt[]
diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt
index 458a3f4f7f5d80..2ac35ae97d24d3 100644
--- a/Documentation/config/core.txt
+++ b/Documentation/config/core.txt
@@ -111,6 +111,14 @@ Version 2 uses an opaque string so that the monitor can return
something that can be used to determine what files have changed
without race conditions.
+core.virtualFilesystem::
+ If set, the value of this variable is used as a command which
+ will identify all files and directories that are present in
+ the working directory. Git will only track and update files
+ listed in the virtual file system. Using the virtual file system
+ will supersede the sparse-checkout settings which will be ignored.
+ See the "virtual file system" section of linkgit:githooks[5].
+
core.trustctime::
If false, the ctime differences between the index and the
working tree are ignored; useful when the inode change time
@@ -743,6 +751,55 @@ core.multiPackIndex::
single index. See linkgit:git-multi-pack-index[1] for more
information. Defaults to true.
+core.gvfs::
+ Enable the features needed for GVFS. This value can be set to true
+ to indicate all features should be turned on or the bit values listed
+ below can be used to turn on specific features.
++
+--
+ GVFS_SKIP_SHA_ON_INDEX::
+ Bit value 1
+ Disables the calculation of the sha when writing the index
+ GVFS_MISSING_OK::
+ Bit value 4
+ Normally git write-tree ensures that the objects referenced by the
+ directory exist in the object database. This option disables this check.
+ GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT::
+ Bit value 8
+ When marking entries to remove from the index and the working
+ directory this option will take into account what the
+ skip-worktree bit was set to so that if the entry has the
+ skip-worktree bit set it will not be removed from the working
+ directory. This will allow virtualized working directories to
+ detect the change to HEAD and use the new commit tree to show
+ the files that are in the working directory.
+ GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK::
+ Bit value 16
+ While performing a fetch with a virtual file system we know
+ that there will be missing objects and we don't want to download
+ them just because of the reachability of the commits. We also
+ don't want to download a pack file with commits, trees, and blobs
+ since these will be downloaded on demand. This flag will skip the
+ checks on the reachability of objects during a fetch as well as
+ the upload pack so that extraneous objects don't get downloaded.
+ GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS::
+ Bit value 64
+ With a virtual file system we only know the file size before any
+ CRLF or smudge/clean filters processing is done on the client.
+ To prevent file corruption due to truncation or expansion with
+ garbage at the end, these filters must not run when the file
+ is first accessed and brought down to the client. Git.exe can't
+ currently tell the first access vs subsequent accesses so this
+ flag just blocks them from occurring at all.
+ GVFS_PREFETCH_DURING_FETCH::
+ Bit value 128
+ While performing a `git fetch` command, use the gvfs-helper to
+ perform a "prefetch" of commits and trees.
+--
+
+core.useGvfsHelper::
+ TODO
+
core.sparseCheckout::
Enable "sparse checkout" feature. See linkgit:git-sparse-checkout[1]
for more information.
@@ -777,3 +834,12 @@ core.WSLCompat::
The default value is false. When set to true, Git will set the mode
bits of the file in the way of wsl, so that the executable flag of
files can be set or read correctly.
+
+core.configWriteLockTimeoutMS::
+ When processes try to write to the config concurrently, it is likely
+ that one process "wins" and the other process(es) fail to lock the
+ config file. By configuring a timeout larger than zero, Git can be
+ told to try to lock the config again a couple times within the
+ specified timeout. If the timeout is configure to zero (which is the
+ default), Git will fail immediately when the config is already
+ locked.
diff --git a/Documentation/config/credential.txt b/Documentation/config/credential.txt
index 0221c3e620da89..470482ff4c2a38 100644
--- a/Documentation/config/credential.txt
+++ b/Documentation/config/credential.txt
@@ -9,6 +9,14 @@ credential.helper::
Note that multiple helpers may be defined. See linkgit:gitcredentials[7]
for details and examples.
+credential.interactive::
+ By default, Git and any configured credential helpers will ask for
+ user input when new credentials are required. Many of these helpers
+ will succeed based on stored credentials if those credentials are
+ still valid. To avoid the possibility of user interactivity from
+ Git, set `credential.interactive=false`. Some credential helpers
+ respect this option as well.
+
credential.useHttpPath::
When acquiring credentials, consider the "path" component of an http
or https URL to be important. Defaults to false. See
diff --git a/Documentation/config/gvfs.txt b/Documentation/config/gvfs.txt
new file mode 100644
index 00000000000000..7224939ac0b270
--- /dev/null
+++ b/Documentation/config/gvfs.txt
@@ -0,0 +1,10 @@
+gvfs.cache-server::
+ TODO
+
+gvfs.sharedcache::
+ TODO
+
+gvfs.fallback::
+ If set to `false`, then never fallback to the origin server when the cache
+ server fails to connect. This will alert users to failures with the cache
+ server, but avoid causing throttling on the origin server.
diff --git a/Documentation/config/index.txt b/Documentation/config/index.txt
index 3eff42036033ea..0d6d05b70ce03d 100644
--- a/Documentation/config/index.txt
+++ b/Documentation/config/index.txt
@@ -1,3 +1,9 @@
+index.deleteSparseDirectories::
+ When enabled, the cone mode sparse-checkout feature will delete
+ directories that are outside of the sparse-checkout cone, unless
+ such a directory contains an untracked, non-ignored file. Defaults
+ to true.
+
index.recordEndOfIndexEntries::
Specifies whether the index file should include an "End Of Index
Entry" section. This reduces index load time on multiprocessor
diff --git a/Documentation/config/status.txt b/Documentation/config/status.txt
index 8caf90f51c19a3..4d863fdaaec2eb 100644
--- a/Documentation/config/status.txt
+++ b/Documentation/config/status.txt
@@ -77,3 +77,25 @@ status.submoduleSummary::
the --ignore-submodules=dirty command-line option or the 'git
submodule summary' command, which shows a similar output but does
not honor these settings.
+
+status.deserializePath::
+ EXPERIMENTAL, Pathname to a file containing cached status results
+ generated by `--serialize`. This will be overridden by
+ `--deserialize=` on the command line. If the cache file is
+ invalid or stale, git will fall-back and compute status normally.
+
+status.deserializeWait::
+ EXPERIMENTAL, Specifies what `git status --deserialize` should do
+ if the serialization cache file is stale and whether it should
+ fall-back and compute status normally. This will be overridden by
+ `--deserialize-wait=` on the command line.
++
+--
+* `fail` - cause git to exit with an error when the status cache file
+is stale; this is intended for testing and debugging.
+* `block` - cause git to spin and periodically retry the cache file
+every 100 ms; this is intended to help coordinate with another git
+instance concurrently computing the cache file.
+* `no` - to immediately fall-back if cache file is stale. This is the default.
+* `` - time (in tenths of a second) to spin and retry.
+--
diff --git a/Documentation/config/survey.txt b/Documentation/config/survey.txt
new file mode 100644
index 00000000000000..857c1c3fff2d6a
--- /dev/null
+++ b/Documentation/config/survey.txt
@@ -0,0 +1,41 @@
+survey.namerev::
+ Boolean to show/hide `git name-rev` information for
+ each reported commit and the containing commit of each
+ reported tree and blob.
+
+survey.progress::
+ Boolean to show/hide progress information. Defaults to
+ true when interactive (stderr is bound to a TTY).
+
+survey.showBlobSizes::
+ A non-negative integer value. Requests details on the
+ largest file blobs by size in bytes. Provides a default
+ value for `--blob-sizes=` in linkgit:git-survey[1].
+
+survey.showCommitParents::
+ A non-negative integer value. Requests details on the
+ commits with the most number of parents. Provides a default
+ value for `--commit-parents=` in linkgit:git-survey[1].
+
+survey.showCommitSizes::
+ A non-negative integer value. Requests details on the
+ largest commits by size in bytes. Generally, these are the
+ commits with the largest commit messages. Provides a default
+ value for `--commit-sizes=` in linkgit:git-survey[1].
+
+survey.showTreeEntries::
+ A non-negative integer value. Requests details on the
+ trees (directories) with the most number of entries (files
+ and subdirectories). Provides a default value for
+ `--tree-entries=` in linkgit:git-survey[1].
+
+survey.showTreeSizes::
+ A non-negative integer value. Requests details on the
+ largest trees (directories) by size in bytes. This will
+ set will usually be equal to the `survey.showTreeEntries`
+ set, but may be skewed by very long file or subdirectory
+ entry names. Provides a default value for
+ `--tree-sizes=` in linkgit:git-survey[1].
+
+survey.verbose::
+ Boolean to show/hide verbose output. Default to false.
diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt
index 9a376886a5867a..62254617dd7fc1 100644
--- a/Documentation/git-status.txt
+++ b/Documentation/git-status.txt
@@ -151,6 +151,21 @@ ignored, then the directory is not shown, but all contents are shown.
threshold.
See also linkgit:git-diff[1] `--find-renames`.
+--serialize[=]::
+ (EXPERIMENTAL) Serialize raw status results to a file or stdout
+ in a format suitable for use by `--deserialize`. If a path is
+ given, serialize data will be written to that path *and* normal
+ status output will be written to stdout. If path is omitted,
+ only binary serialization data will be written to stdout.
+
+--deserialize[=]::
+ (EXPERIMENTAL) Deserialize raw status results from a file or
+ stdin rather than scanning the worktree. If `` is omitted
+ and `status.deserializePath` is unset, input is read from stdin.
+--no-deserialize::
+ (EXPERIMENTAL) Disable implicit deserialization of status results
+ from the value of `status.deserializePath`.
+
...::
See the 'pathspec' entry in linkgit:gitglossary[7].
@@ -424,6 +439,26 @@ quoted as explained for the configuration variable `core.quotePath`
(see linkgit:git-config[1]).
+SERIALIZATION and DESERIALIZATION (EXPERIMENTAL)
+------------------------------------------------
+
+The `--serialize` option allows git to cache the result of a
+possibly time-consuming status scan to a binary file. A local
+service/daemon watching file system events could use this to
+periodically pre-compute a fresh status result.
+
+Interactive users could then use `--deserialize` to simply
+(and immediately) print the last-known-good result without
+waiting for the status scan.
+
+The binary serialization file format includes some worktree state
+information allowing `--deserialize` to reject the cached data
+and force a normal status scan if, for example, the commit, branch,
+or status modes/options change. The format cannot, however, indicate
+when the cached data is otherwise stale -- that coordination belongs
+to the task driving the serializations.
+
+
CONFIGURATION
-------------
diff --git a/Documentation/git-survey.txt b/Documentation/git-survey.txt
new file mode 100644
index 00000000000000..7be11e4683822d
--- /dev/null
+++ b/Documentation/git-survey.txt
@@ -0,0 +1,108 @@
+git-survey(1)
+=============
+
+NAME
+----
+git-survey - EXPERIMENTAL: Measure various repository dimensions of scale
+
+SYNOPSIS
+--------
+[verse]
+(EXPERIMENTAL!) `git survey`
+
+DESCRIPTION
+-----------
+
+Survey the repository and measure various dimensions of scale.
+
+As repositories grow to "monorepo" size, certain data shapes can cause
+performance problems. `git-survey` attempts to measure and report on
+known problem areas.
+
+Ref Selection and Reachable Objects
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In this first analysis phase, `git survey` will iterate over the set of
+requested branches, tags, and other refs and treewalk over all of the
+reachable commits, trees, and blobs and generate various statistics.
+
+OPTIONS
+-------
+
+--progress::
+ Show progress. This is automatically enabled when interactive.
+
+--json::
+ Print results in JSON rather than in a human-friendly format.
+
+--[no-]name-rev::
+ Print `git name-rev` output for each commit, tree, and blob.
+ Defaults to true.
+
+Ref Selection
+~~~~~~~~~~~~~
+
+The following options control the set of refs that `git survey` will examine.
+By default, `git survey` will look at tags, local branches, and remote refs.
+If any of the following options are given, the default set is cleared and
+only refs for the given options are added.
+
+--all-refs::
+ Use all refs. This includes local branches, tags, remote refs,
+ notes, and stashes. This option overrides all of the following.
+
+--branches::
+ Add local branches (`refs/heads/`) to the set.
+
+--tags::
+ Add tags (`refs/tags/`) to the set.
+
+--remotes::
+ Add remote branches (`refs/remote/`) to the set.
+
+--detached::
+ Add HEAD to the set.
+
+--other::
+ Add notes (`refs/notes/`) and stashes (`refs/stash/`) to the set.
+
+Large Item Selection
+~~~~~~~~~~~~~~~~~~~~
+
+The following options control the optional display of large items under
+various dimensions of scale. The OID of the largest `n` objects will be
+displayed in reverse sorted order. For each, `n` defaults to 10.
+
+--commit-parents::
+ Shows the OIDs of the commits with the most parent commits.
+
+--commit-sizes::
+ Shows the OIDs of the largest commits by size in bytes. This is
+ usually the ones with the largest commit messages.
+
+--tree-entries::
+ Shows the OIDs of the trees with the most number of entries. These
+ are the directories with the most number of files or subdirectories.
+
+--tree-sizes::
+ Shows the OIDs of the largest trees by size in bytes. This set
+ will usually be the same as the vector of number of entries unless
+ skewed by very long entry names.
+
+--blob-sizes::
+ Shows the OIDs of the largest blobs by size in bytes.
+
+OUTPUT
+------
+
+By default, `git survey` will print information about the repository in a
+human-readable format that includes overviews and tables.
+
+CONFIGURATION
+-------------
+
+include::config/survey.txt[]
+
+GIT
+---
+Part of the linkgit:git[1] suite
diff --git a/Documentation/git-update-microsoft-git.txt b/Documentation/git-update-microsoft-git.txt
new file mode 100644
index 00000000000000..724bfc172f8ab7
--- /dev/null
+++ b/Documentation/git-update-microsoft-git.txt
@@ -0,0 +1,24 @@
+git-update-microsoft-git(1)
+===========================
+
+NAME
+----
+git-update-microsoft-git - Update the installed version of Git
+
+
+SYNOPSIS
+--------
+[verse]
+'git update-microsoft-git'
+
+DESCRIPTION
+-----------
+This version of Git is based on the Microsoft fork of Git, which
+has custom capabilities focused on supporting monorepos. This
+command checks for the latest release of that fork and installs
+it on your machine.
+
+
+GIT
+---
+Part of the linkgit:git[1] suite
diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt
index 0397dec64d7315..86c78b3b4825e2 100644
--- a/Documentation/githooks.txt
+++ b/Documentation/githooks.txt
@@ -758,6 +758,26 @@ and "0" meaning they were not.
Only one parameter should be set to "1" when the hook runs. The hook
running passing "1", "1" should not be possible.
+virtualFilesystem
+~~~~~~~~~~~~~~~~~~
+
+"Virtual File System" allows populating the working directory sparsely.
+The projection data is typically automatically generated by an external
+process. Git will limit what files it checks for changes as well as which
+directories are checked for untracked files based on the path names given.
+Git will also only update those files listed in the projection.
+
+The hook is invoked when the configuration option core.virtualFilesystem
+is set. It takes one argument, a version (currently 1).
+
+The hook should output to stdout the list of all files in the working
+directory that git should track. The paths are relative to the root
+of the working directory and are separated by a single NUL. Full paths
+('dir1/a.txt') as well as directories are supported (ie 'dir1/').
+
+The exit status determines whether git will use the data from the
+hook. On error, git will abort the command with an error message.
+
SEE ALSO
--------
linkgit:git-hook[1]
diff --git a/Documentation/lint-manpages.sh b/Documentation/lint-manpages.sh
index 92cfc0a15abd56..2622a493566950 100755
--- a/Documentation/lint-manpages.sh
+++ b/Documentation/lint-manpages.sh
@@ -27,6 +27,8 @@ check_missing_docs () (
git-init-db) continue;;
git-remote-*) continue;;
git-stage) continue;;
+ git-gvfs-helper) continue;;
+ git-update-microsoft-git) continue;;
git-legacy-*) continue;;
git-?*--?* ) continue ;;
esac
diff --git a/Documentation/scalar.txt b/Documentation/scalar.txt
index 361f51a64736fa..dab62be71fe496 100644
--- a/Documentation/scalar.txt
+++ b/Documentation/scalar.txt
@@ -9,7 +9,8 @@ SYNOPSIS
--------
[verse]
scalar clone [--single-branch] [--branch ] [--full-clone]
- [--[no-]src] []
+ [--[no-]src] [--local-cache-path ] [--cache-server-url ]
+ []
scalar list
scalar register []
scalar unregister []
@@ -17,6 +18,7 @@ scalar run ( all | config | commit-graph | fetch | loose-objects | pack-files )
scalar reconfigure [ --all | ]
scalar diagnose []
scalar delete
+scalar cache-server ( --get | --set | --list [] ) []
DESCRIPTION
-----------
@@ -90,6 +92,37 @@ cloning. If the HEAD at the remote did not point at any branch when
A sparse-checkout is initialized by default. This behavior can be
turned off via `--full-clone`.
+--local-cache-path ::
+ Override the path to the local cache root directory; Pre-fetched objects
+ are stored into a repository-dependent subdirectory of that path.
++
+The default is `:\.scalarCache` on Windows (on the same drive as the
+clone), and `~/.scalarCache` on macOS.
+
+--cache-server-url ::
+ Retrieve missing objects from the specified remote, which is expected to
+ understand the GVFS protocol.
+
+--[no-]gvfs-protocol::
+ When cloning from a `` with either `dev.azure.com` or
+ `visualstudio.com` in the name, `scalar clone` will attempt to use the GVFS
+ Protocol to access Git objects, specifically from a cache server when
+ available, and will fail to clone if there is an error over that protocol.
+
+ To enable the GVFS Protocol regardless of the origin ``, use
+ `--gvfs-protocol`. This will cause `scalar clone` to fail when the origin
+ server fails to provide a valid response to the `gvfs/config` endpoint.
+
+ To disable the GVFS Protocol, use `--no-gvfs-protocol` and `scalar clone`
+ will only use the Git protocol, starting with a partial clone. This can be
+ helpful if your `` points to Azure Repos but the repository does not
+ have GVFS cache servers enabled. It is likely more efficient to use its
+ partial clone functionality through the Git protocol.
+
+ Previous versions of `scalar clone` could fall back to a partial clone over
+ the Git protocol if there is any issue gathering GVFS configuration
+ information from the origin server.
+
List
~~~~
@@ -163,6 +196,27 @@ delete ::
This subcommand lets you delete an existing Scalar enlistment from your
local file system, unregistering the repository.
+Cache-server
+~~~~~~~~~~~~
+
+cache-server ( --get | --set | --list [] ) []::
+ This command lets you query or set the GVFS-enabled cache server used
+ to fetch missing objects.
+
+--get::
+ This is the default command mode: query the currently-configured cache
+ server URL, if any.
+
+--list::
+ Access the `gvfs/info` endpoint of the specified remote (default:
+ `origin`) to figure out which cache servers are available, if any.
++
+In contrast to the `--get` command mode (which only accesses the local
+repository), this command mode triggers a request via the network that
+potentially requires authentication. If authentication is required, the
+configured credential helper is employed (see linkgit:git-credential[1]
+for details).
+
SEE ALSO
--------
linkgit:git-clone[1], linkgit:git-maintenance[1].
diff --git a/Documentation/technical/read-object-protocol.txt b/Documentation/technical/read-object-protocol.txt
new file mode 100644
index 00000000000000..a893b46e7c28a9
--- /dev/null
+++ b/Documentation/technical/read-object-protocol.txt
@@ -0,0 +1,102 @@
+Read Object Process
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The read-object process enables Git to read all missing blobs with a
+single process invocation for the entire life of a single Git command.
+This is achieved by using a packet format (pkt-line, see technical/
+protocol-common.txt) based protocol over standard input and standard
+output as follows. All packets, except for the "*CONTENT" packets and
+the "0000" flush packet, are considered text and therefore are
+terminated by a LF.
+
+Git starts the process when it encounters the first missing object that
+needs to be retrieved. After the process is started, Git sends a welcome
+message ("git-read-object-client"), a list of supported protocol version
+numbers, and a flush packet. Git expects to read a welcome response
+message ("git-read-object-server"), exactly one protocol version number
+from the previously sent list, and a flush packet. All further
+communication will be based on the selected version.
+
+The remaining protocol description below documents "version=1". Please
+note that "version=42" in the example below does not exist and is only
+there to illustrate how the protocol would look with more than one
+version.
+
+After the version negotiation Git sends a list of all capabilities that
+it supports and a flush packet. Git expects to read a list of desired
+capabilities, which must be a subset of the supported capabilities list,
+and a flush packet as response:
+------------------------
+packet: git> git-read-object-client
+packet: git> version=1
+packet: git> version=42
+packet: git> 0000
+packet: git< git-read-object-server
+packet: git< version=1
+packet: git< 0000
+packet: git> capability=get
+packet: git> capability=have
+packet: git> capability=put
+packet: git> capability=not-yet-invented
+packet: git> 0000
+packet: git< capability=get
+packet: git< 0000
+------------------------
+The only supported capability in version 1 is "get".
+
+Afterwards Git sends a list of "key=value" pairs terminated with a flush
+packet. The list will contain at least the command (based on the
+supported capabilities) and the sha1 of the object to retrieve. Please
+note, that the process must not send any response before it received the
+final flush packet.
+
+When the process receives the "get" command, it should make the requested
+object available in the git object store and then return success. Git will
+then check the object store again and this time find it and proceed.
+------------------------
+packet: git> command=get
+packet: git> sha1=0a214a649e1b3d5011e14a3dc227753f2bd2be05
+packet: git> 0000
+------------------------
+
+The process is expected to respond with a list of "key=value" pairs
+terminated with a flush packet. If the process does not experience
+problems then the list must contain a "success" status.
+------------------------
+packet: git< status=success
+packet: git< 0000
+------------------------
+
+In case the process cannot or does not want to process the content, it
+is expected to respond with an "error" status.
+------------------------
+packet: git< status=error
+packet: git< 0000
+------------------------
+
+In case the process cannot or does not want to process the content as
+well as any future content for the lifetime of the Git process, then it
+is expected to respond with an "abort" status at any point in the
+protocol.
+------------------------
+packet: git< status=abort
+packet: git< 0000
+------------------------
+
+Git neither stops nor restarts the process in case the "error"/"abort"
+status is set.
+
+If the process dies during the communication or does not adhere to the
+protocol then Git will stop the process and restart it with the next
+object that needs to be processed.
+
+After the read-object process has processed an object it is expected to
+wait for the next "key=value" list containing a command. Git will close
+the command pipe on exit. The process is expected to detect EOF and exit
+gracefully on its own. Git will wait until the process has stopped.
+
+A long running read-object process demo implementation can be found in
+`contrib/long-running-read-object/example.pl` located in the Git core
+repository. If you develop your own long running process then the
+`GIT_TRACE_PACKET` environment variables can be very helpful for
+debugging (see linkgit:git[1]).
diff --git a/Documentation/technical/status-serialization-format.txt b/Documentation/technical/status-serialization-format.txt
new file mode 100644
index 00000000000000..475ae814495581
--- /dev/null
+++ b/Documentation/technical/status-serialization-format.txt
@@ -0,0 +1,107 @@
+Git status serialization format
+===============================
+
+Git status serialization enables git to dump the results of a status scan
+to a binary file. This file can then be loaded by later status invocations
+to print the cached status results.
+
+The file contains the essential fields from:
+() the index
+() the "struct wt_status" for the overall results
+() the contents of "struct wt_status_change_data" for tracked changed files
+() the list of untracked and ignored files
+
+Version 1 Format:
+=================
+
+The V1 file begins with a required header section followed by optional
+sections for each type of item (changed, untracked, ignored). Individual
+item sections are only present if necessary. Each item section begins
+with an item-type header with the number of items in the section.
+
+Each "line" in the format is encoded using pkt-line with a final LF.
+Flush packets are used to terminate sections.
+
+-----------------
+PKT-LINE("version" SP "1")
+
+[]
+[]
+[]
+-----------------
+
+
+V1 Header
+---------
+
+The v1-header-section fields are taken directly from "struct wt_status".
+Each field is printed on a separate pkt-line. Lines for NULL string
+values are omitted. All integers are printed with "%d". OIDs are
+printed in hex.
+
+v1-header-section =
+
+ PKT-LINE()
+
+v1-index-headers = PKT-LINE("index_mtime" SP SP LF)
+
+v1-wt-status-headers = PKT-LINE("is_initial" SP LF)
+ [ PKT-LINE("branch" SP LF) ]
+ [ PKT-LINE("reference" SP LF) ]
+ PKT-LINE("show_ignored_files" SP LF)
+ PKT-LINE("show_untracked_files" SP LF)
+ PKT-LINE("show_ignored_directory" SP LF)
+ [ PKT-LINE("ignore_submodule_arg" SP LF) ]
+ PKT-LINE("detect_rename" SP LF)
+ PKT-LINE("rename_score" SP LF)
+ PKT-LINE("rename_limit" SP LF)
+ PKT-LINE("detect_break" SP LF)
+ PKT-LINE("sha1_commit" SP LF)
+ PKT-LINE("committable" SP LF)
+ PKT-LINE("workdir_dirty" SP LF)
+
+
+V1 Changed Items
+----------------
+
+The v1-changed-item-section lists all of the changed items with one
+item per pkt-line. Each pkt-line contains: a binary block of data
+from "struct wt_status_serialize_data_fixed" in a fixed header where
+integers are in network byte order and OIDs are in raw (non-hex) form.
+This is followed by one or two raw pathnames (not c-quoted) with NUL
+terminators (both NULs are always present even if there is no rename).
+
+v1-changed-item-section = PKT-LINE("changed" SP LF)
+ [ PKT-LINE( LF) ]+
+ PKT-LINE()
+
+changed_item =
+
+
+
+
+
+
+
+
+
+
+
+ NUL
+ [ ]
+ NUL
+
+
+V1 Untracked and Ignored Items
+------------------------------
+
+These sections are simple lists of pathnames. They ARE NOT
+c-quoted.
+
+v1-untracked-item-section = PKT-LINE("untracked" SP LF)
+ [ PKT-LINE( LF) ]+
+ PKT-LINE()
+
+v1-ignored-item-section = PKT-LINE("ignored" SP LF)
+ [ PKT-LINE( LF) ]+
+ PKT-LINE()
diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN
index 55850912d13034..a7081ebdcc989f 100755
--- a/GIT-VERSION-GEN
+++ b/GIT-VERSION-GEN
@@ -3,6 +3,9 @@
GVF=GIT-VERSION-FILE
DEF_VER=v2.46.1
+# Identify microsoft/git via a distinct version suffix
+DEF_VER=$DEF_VER.vfs.0.0
+
LF='
'
@@ -12,10 +15,15 @@ if test -f version
then
VN=$(cat version) || VN="$DEF_VER"
elif { test -d "${GIT_DIR:-.git}" || test -f .git; } &&
- VN=$(git describe --match "v[0-9]*" HEAD 2>/dev/null) &&
+ VN=$(git describe --match "v[0-9]*vfs*" HEAD 2>/dev/null) &&
case "$VN" in
*$LF*) (exit 1) ;;
v[0-9]*)
+ if test "${VN%%.vfs.*}" != "${DEF_VER%%.vfs.*}"
+ then
+ echo "Found version $VN, which is not based on $DEF_VER" >&2
+ exit 1
+ fi
git update-index -q --refresh
test -z "$(git diff-index --name-only HEAD --)" ||
VN="$VN-dirty" ;;
diff --git a/Makefile b/Makefile
index b72d0730199cbc..6a063924daffdd 100644
--- a/Makefile
+++ b/Makefile
@@ -321,6 +321,10 @@ include shared.mak
# Define GIT_USER_AGENT if you want to change how git identifies itself during
# network interactions. The default is "git/$(GIT_VERSION)".
#
+# Define GIT_BUILT_FROM_COMMIT if you want to force the commit hash identified
+# in 'git version --build-options' to a specific value. The default is the
+# commit hash of the current HEAD.
+#
# Define DEFAULT_HELP_FORMAT to "man", "info" or "html"
# (defaults to "man") if you want to have a different default when
# "git help" is called without a parameter specifying the format.
@@ -1033,6 +1037,8 @@ LIB_OBJS += git-zlib.o
LIB_OBJS += gpg-interface.o
LIB_OBJS += graph.o
LIB_OBJS += grep.o
+LIB_OBJS += gvfs.o
+LIB_OBJS += gvfs-helper-client.o
LIB_OBJS += hash-lookup.o
LIB_OBJS += hashmap.o
LIB_OBJS += help.o
@@ -1190,6 +1196,7 @@ LIB_OBJS += utf8.o
LIB_OBJS += varint.o
LIB_OBJS += version.o
LIB_OBJS += versioncmp.o
+LIB_OBJS += virtualfilesystem.o
LIB_OBJS += walker.o
LIB_OBJS += wildmatch.o
LIB_OBJS += worktree.o
@@ -1197,6 +1204,8 @@ LIB_OBJS += wrapper.o
LIB_OBJS += write-or-die.o
LIB_OBJS += ws.o
LIB_OBJS += wt-status.o
+LIB_OBJS += wt-status-deserialize.o
+LIB_OBJS += wt-status-serialize.o
LIB_OBJS += xdiff-interface.o
BUILTIN_OBJS += builtin/add.o
@@ -1308,11 +1317,13 @@ BUILTIN_OBJS += builtin/sparse-checkout.o
BUILTIN_OBJS += builtin/stash.o
BUILTIN_OBJS += builtin/stripspace.o
BUILTIN_OBJS += builtin/submodule--helper.o
+BUILTIN_OBJS += builtin/survey.o
BUILTIN_OBJS += builtin/symbolic-ref.o
BUILTIN_OBJS += builtin/tag.o
BUILTIN_OBJS += builtin/unpack-file.o
BUILTIN_OBJS += builtin/unpack-objects.o
BUILTIN_OBJS += builtin/update-index.o
+BUILTIN_OBJS += builtin/update-microsoft-git.o
BUILTIN_OBJS += builtin/update-ref.o
BUILTIN_OBJS += builtin/update-server-info.o
BUILTIN_OBJS += builtin/upload-archive.o
@@ -1646,6 +1657,9 @@ endif
endif
BASIC_CFLAGS += $(CURL_CFLAGS)
+ PROGRAM_OBJS += gvfs-helper.o
+ TEST_PROGRAMS_NEED_X += test-gvfs-protocol
+
REMOTE_CURL_PRIMARY = git-remote-http$X
REMOTE_CURL_ALIASES = git-remote-https$X git-remote-ftp$X git-remote-ftps$X
REMOTE_CURL_NAMES = $(REMOTE_CURL_PRIMARY) $(REMOTE_CURL_ALIASES)
@@ -2376,6 +2390,15 @@ GIT-USER-AGENT: FORCE
echo '$(GIT_USER_AGENT_SQ)' >GIT-USER-AGENT; \
fi
+GIT_BUILT_FROM_COMMIT = $(eval GIT_BUILT_FROM_COMMIT := $$(shell \
+ GIT_CEILING_DIRECTORIES="$$(CURDIR)/.." \
+ git rev-parse -q --verify HEAD 2>/dev/null))$(GIT_BUILT_FROM_COMMIT)
+GIT-BUILT-FROM-COMMIT: FORCE
+ @if test x'$(GIT_BUILT_FROM_COMMIT)' != x"`cat GIT-BUILT-FROM-COMMIT 2>/dev/null`" ; then \
+ echo >&2 " * new built-from commit"; \
+ echo '$(GIT_BUILT_FROM_COMMIT)' >GIT-BUILT-FROM-COMMIT; \
+ fi
+
ifdef DEFAULT_HELP_FORMAT
BASIC_CFLAGS += -DDEFAULT_HELP_FORMAT='"$(DEFAULT_HELP_FORMAT)"'
endif
@@ -2513,13 +2536,11 @@ PAGER_ENV_CQ_SQ = $(subst ','\'',$(PAGER_ENV_CQ))
pager.sp pager.s pager.o: EXTRA_CPPFLAGS = \
-DPAGER_ENV='$(PAGER_ENV_CQ_SQ)'
-version.sp version.s version.o: GIT-VERSION-FILE GIT-USER-AGENT
+version.sp version.s version.o: GIT-VERSION-FILE GIT-USER-AGENT GIT-BUILT-FROM-COMMIT
version.sp version.s version.o: EXTRA_CPPFLAGS = \
'-DGIT_VERSION="$(GIT_VERSION)"' \
'-DGIT_USER_AGENT=$(GIT_USER_AGENT_CQ_SQ)' \
- '-DGIT_BUILT_FROM_COMMIT="$(shell \
- GIT_CEILING_DIRECTORIES="$(CURDIR)/.." \
- git rev-parse -q --verify HEAD 2>/dev/null)"'
+ '-DGIT_BUILT_FROM_COMMIT="$(GIT_BUILT_FROM_COMMIT)"'
$(BUILT_INS): git$X
$(QUIET_BUILT_IN)$(RM) $@ && \
@@ -2756,6 +2777,7 @@ GIT_OBJS += git.o
.PHONY: git-objs
git-objs: $(GIT_OBJS)
+SCALAR_OBJS := json-parser.o
SCALAR_OBJS += scalar.o
.PHONY: scalar-objs
scalar-objs: $(SCALAR_OBJS)
@@ -2854,7 +2876,7 @@ gettext.sp gettext.s gettext.o: GIT-PREFIX
gettext.sp gettext.s gettext.o: EXTRA_CPPFLAGS = \
-DGIT_LOCALE_PATH='"$(localedir_relative_SQ)"'
-http-push.sp http.sp http-walker.sp remote-curl.sp imap-send.sp: SP_EXTRA_FLAGS += \
+http-push.sp http.sp http-walker.sp remote-curl.sp imap-send.sp gvfs-helper.sp: SP_EXTRA_FLAGS += \
-DCURL_DISABLE_TYPECHECK
pack-revindex.sp: SP_EXTRA_FLAGS += -Wno-memcpy-max-count
@@ -2905,10 +2927,14 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o $(LAZYLOAD_LIBCURL_OB
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
$(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS)
-scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS)
+scalar$X: $(SCALAR_OBJS) GIT-LDFLAGS $(GITLIBS)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \
$(filter %.o,$^) $(LIBS)
+git-gvfs-helper$X: gvfs-helper.o http.o GIT-LDFLAGS $(GITLIBS) $(LAZYLOAD_LIBCURL_OBJ)
+ $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
+ $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS)
+
$(LIB_FILE): $(LIB_OBJS)
$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
@@ -3686,7 +3712,7 @@ dist: git-archive$(X) configure
@$(MAKE) -C git-gui TARDIR=../.dist-tmp-dir/git-gui dist-version
./git-archive --format=tar \
$(GIT_ARCHIVE_EXTRA_FILES) \
- --prefix=$(GIT_TARNAME)/ HEAD^{tree} > $(GIT_TARNAME).tar
+ --prefix=$(GIT_TARNAME)/ HEAD > $(GIT_TARNAME).tar
@$(RM) -r .dist-tmp-dir
gzip -f -9 $(GIT_TARNAME).tar
diff --git a/README.md b/README.md
index 4eabce53d89e1f..b39764e9ad1dcd 100644
--- a/README.md
+++ b/README.md
@@ -1,148 +1,225 @@
-Git for Windows
-===============
-
-[](CODE_OF_CONDUCT.md)
-[](https://open.vscode.dev/git-for-windows/git)
-[](https://github.com/git-for-windows/git/actions?query=branch%3Amain+event%3Apush)
-[](https://gitter.im/git-for-windows/git?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-
-This is [Git for Windows](http://git-for-windows.github.io/), the Windows port
-of [Git](http://git-scm.com/).
-
-The Git for Windows project is run using a [governance
-model](http://git-for-windows.github.io/governance-model.html). If you
-encounter problems, you can report them as [GitHub
-issues](https://github.com/git-for-windows/git/issues), discuss them on Git
-for Windows' [Google Group](http://groups.google.com/group/git-for-windows),
-and [contribute bug
-fixes](https://github.com/git-for-windows/git/wiki/How-to-participate).
-
-To build Git for Windows, please either install [Git for Windows'
-SDK](https://gitforwindows.org/#download-sdk), start its `git-bash.exe`, `cd`
-to your Git worktree and run `make`, or open the Git worktree as a folder in
-Visual Studio.
-
-To verify that your build works, use one of the following methods:
-
-- If you want to test the built executables within Git for Windows' SDK,
- prepend `/bin-wrappers` to the `PATH`.
-- Alternatively, run `make install` in the Git worktree.
-- If you need to test this in a full installer, run `sdk build
- git-and-installer`.
-- You can also "install" Git into an existing portable Git via `make install
- DESTDIR=` where `` refers to the top-level directory of the
- portable Git. In this instance, you will want to prepend that portable Git's
- `/cmd` directory to the `PATH`, or test by running that portable Git's
- `git-bash.exe` or `git-cmd.exe`.
-- If you built using a recent Visual Studio, you can use the menu item
- `Build>Install git` (you will want to click on `Project>CMake Settings for
- Git` first, then click on `Edit JSON` and then point `installRoot` to the
- `mingw64` directory of an already-unpacked portable Git).
-
- As in the previous bullet point, you will then prepend `/cmd` to the `PATH`
- or run using the portable Git's `git-bash.exe` or `git-cmd.exe`.
-- If you want to run the built executables in-place, but in a CMD instead of
- inside a Bash, you can run a snippet like this in the `git-bash.exe` window
- where Git was built (ensure that the `EOF` line has no leading spaces), and
- then paste into the CMD window what was put in the clipboard:
-
- ```sh
- clip.exe <
-including full documentation and Git related tools.
-
-See [Documentation/gittutorial.txt][] to get started, then see
-[Documentation/giteveryday.txt][] for a useful minimum set of commands, and
-`Documentation/git-.txt` for documentation of each command.
-If git has been correctly installed, then the tutorial can also be
-read with `man gittutorial` or `git help tutorial`, and the
-documentation of each command with `man git-` or `git help
-`.
-
-CVS users may also want to read [Documentation/gitcvs-migration.txt][]
-(`man gitcvs-migration` or `git help cvs-migration` if git is
-installed).
-
-The user discussion and development of core Git take place on the Git
-mailing list -- everyone is welcome to post bug reports, feature
-requests, comments and patches to git@vger.kernel.org (read
-[Documentation/SubmittingPatches][] for instructions on patch submission
-and [Documentation/CodingGuidelines][]).
-
-Those wishing to help with error message, usage and informational message
-string translations (localization l10) should see [po/README.md][]
-(a `po` file is a Portable Object file that holds the translations).
-
-To subscribe to the list, send an email to
-(see https://subspace.kernel.org/subscribing.html for details). The mailing
-list archives are available at ,
- and other archival sites.
-The core git mailing list is plain text (no HTML!).
-
-Issues which are security relevant should be disclosed privately to
-the Git Security mailing list .
-
-The maintainer frequently sends the "What's cooking" reports that
-list the current status of various development topics to the mailing
-list. The discussion following them give a good reference for
-project status, development direction and remaining tasks.
-
-The name "git" was given by Linus Torvalds when he wrote the very
-first version. He described the tool as "the stupid content tracker"
-and the name as (depending on your mood):
-
- - random three-letter combination that is pronounceable, and not
- actually used by any common UNIX command. The fact that it is a
- mispronunciation of "get" may or may not be relevant.
- - stupid. contemptible and despicable. simple. Take your pick from the
- dictionary of slang.
- - "global information tracker": you're in a good mood, and it actually
- works for you. Angels sing, and a light suddenly fills the room.
- - "goddamn idiotic truckload of sh*t": when it breaks
-
-[INSTALL]: INSTALL
-[Documentation/gittutorial.txt]: Documentation/gittutorial.txt
-[Documentation/giteveryday.txt]: Documentation/giteveryday.txt
-[Documentation/gitcvs-migration.txt]: Documentation/gitcvs-migration.txt
-[Documentation/SubmittingPatches]: Documentation/SubmittingPatches
-[Documentation/CodingGuidelines]: Documentation/CodingGuidelines
-[po/README.md]: po/README.md
+If you're working in a monorepo and want to take advantage of the performance boosts in
+`microsoft/git`, then you can download the latest version installer for your OS from the
+[Releases page](https://github.com/microsoft/git/releases). Alternatively, you can opt to install
+via the command line, using the below instructions for supported OSes:
+
+## Windows
+
+__Note:__ Winget is still in public preview, meaning you currently
+[need to take special installation steps](https://docs.microsoft.com/en-us/windows/package-manager/winget/#install-winget):
+Either manually install the `.appxbundle` available at the
+[preview version of App Installer](https://www.microsoft.com/p/app-installer/9nblggh4nns1?ocid=9nblggh4nns1_ORSEARCH_Bing&rtc=1&activetab=pivot:overviewtab),
+or participate in the
+[Windows Insider flight ring](https://insider.windows.com/https://insider.windows.com/)
+since `winget` is available by default on preview versions of Windows.
+
+To install with Winget, run
+
+```shell
+winget install --id microsoft.git
+```
+
+Double-check that you have the right version by running these commands,
+which should have the same output:
+
+```shell
+git version
+scalar version
+```
+
+To upgrade `microsoft/git`, use the following Git command, which will download and install the latest
+release.
+
+```shell
+git update-microsoft-git
+```
+
+You may also be alerted with a notification to upgrade, which presents a single-click process for
+running `git update-microsoft-git`.
+
+## macOS
+
+To install `microsoft/git` on macOS, first [be sure that Homebrew is installed](https://brew.sh/) then
+install the `microsoft-git` cask with these steps:
+
+```shell
+brew tap microsoft/git
+brew install --cask microsoft-git
+```
+
+Double-check that you have the right version by running these commands,
+which should have the same output:
+
+```shell
+git version
+scalar version
+```
+
+To upgrade microsoft/git, you can run the necessary `brew` commands:
+
+```shell
+brew update
+brew upgrade --cask microsoft-git
+```
+
+Or you can run the `git update-microsoft-git` command, which will run those brew commands for you.
+
+## Linux
+### Ubuntu/Debian distributions
+
+On newer distributions*, you can install using the most recent Debian package.
+To download and validate the signature of this package, run the following:
+
+```shell
+# Install needed packages
+sudo apt-get install -y curl debsig-verify
+
+# Download public key signature file
+curl -s https://api.github.com/repos/microsoft/git/releases/latest \
+| grep -E 'browser_download_url.*msft-git-public.asc' \
+| cut -d : -f 2,3 \
+| tr -d \" \
+| xargs -I 'url' curl -L -o msft-git-public.asc 'url'
+
+# De-armor public key signature file
+gpg --output msft-git-public.gpg --dearmor msft-git-public.asc
+
+# Note that the fingerprint of this key is "B8F12E25441124E1", which you can
+# determine by running:
+gpg --show-keys msft-git-public.asc | head -n 2 | tail -n 1 | tail -c 17
+
+# Copy de-armored public key to debsig keyring folder
+sudo mkdir /usr/share/debsig/keyrings/B8F12E25441124E1
+sudo mv msft-git-public.gpg /usr/share/debsig/keyrings/B8F12E25441124E1/
+
+# Create an appropriate policy file
+sudo mkdir /etc/debsig/policies/B8F12E25441124E1
+cat > generic.pol << EOL
+
+
+
+
+
+
+
+
+
+
+
+EOL
+
+sudo mv generic.pol /etc/debsig/policies/B8F12E25441124E1/generic.pol
+
+# Download Debian package
+curl -s https://api.github.com/repos/microsoft/git/releases/latest \
+| grep "browser_download_url.*deb" \
+| cut -d : -f 2,3 \
+| tr -d \" \
+| xargs -I 'url' curl -L -o msft-git.deb 'url'
+
+# Verify
+debsig-verify msft-git.deb
+
+# Install
+sudo dpkg -i msft-git.deb
+```
+
+Double-check that you have the right version by running these commands,
+which should have the same output:
+
+```shell
+git version
+scalar version
+```
+
+To upgrade, you will need to repeat these steps to reinstall.
+
+*Older distributions are missing some required dependencies. Even
+though the package may appear to install successfully, `microsoft/
+git` will not function as expected. If you are running `Ubuntu 20.04` or
+older, please follow the install from source instructions below
+instead of installing the debian package.
+
+### Installing From Source
+
+On older or other distros you will need to compile and install `microsoft/git` from source:
+
+```shell
+git clone https://github.com/microsoft/git microsoft-git
+cd microsoft-git
+make -j12 prefix=/usr/local
+sudo make -j12 prefix=/usr/local install
+```
+
+For more assistance building Git from source, see
+[the INSTALL file in the core Git project](https://github.com/git/git/blob/master/INSTALL).
+
+#### Common Debian based dependencies
+While the INSTALL file covers dependencies in detail, here is a shortlist of common required dependencies on older Debian/Ubuntu distros:
+
+```shell
+sudo apt-get update
+sudo apt-get install libz-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext cmake gcc
+```
+
+Contributing
+=========================================================
+
+This project welcomes contributions and suggestions. Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit .
+
+When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
diff --git a/abspath.c b/abspath.c
index 0c17e98654e4b0..e899f46d02097a 100644
--- a/abspath.c
+++ b/abspath.c
@@ -14,7 +14,7 @@ int is_directory(const char *path)
}
/* removes the last path component from 'path' except if 'path' is root */
-static void strip_last_component(struct strbuf *path)
+void strip_last_path_component(struct strbuf *path)
{
size_t offset = offset_1st_component(path->buf);
size_t len = path->len;
@@ -119,7 +119,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path,
continue; /* '.' component */
} else if (next.len == 2 && !strcmp(next.buf, "..")) {
/* '..' component; strip the last path component */
- strip_last_component(resolved);
+ strip_last_path_component(resolved);
continue;
}
@@ -171,7 +171,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path,
* strip off the last component since it will
* be replaced with the contents of the symlink
*/
- strip_last_component(resolved);
+ strip_last_path_component(resolved);
}
/*
diff --git a/abspath.h b/abspath.h
index 4653080d5e4b7a..06241ba13cf646 100644
--- a/abspath.h
+++ b/abspath.h
@@ -10,6 +10,11 @@ char *real_pathdup(const char *path, int die_on_error);
const char *absolute_path(const char *path);
char *absolute_pathdup(const char *path);
+/**
+ * Remove the last path component from 'path' except if 'path' is root.
+ */
+void strip_last_path_component(struct strbuf *path);
+
/*
* Concatenate "prefix" (if len is non-zero) and "path", with no
* connecting characters (so "prefix" should end with a "/").
diff --git a/apply.c b/apply.c
index d2a34abafe480e..44d10339517375 100644
--- a/apply.c
+++ b/apply.c
@@ -3415,6 +3415,24 @@ static int checkout_target(struct index_state *istate,
{
struct checkout costate = CHECKOUT_INIT;
+ /*
+ * Do not checkout the entry if the skipworktree bit is set
+ *
+ * Both callers of this method (check_preimage and load_current)
+ * check for the existance of the file before calling this
+ * method so we know that the file doesn't exist at this point
+ * and we don't need to perform that check again here.
+ * We just need to check the skip-worktree and return.
+ *
+ * This is to prevent git from creating a file in the
+ * working directory that has the skip-worktree bit on,
+ * then updating the index from the patch and not keeping
+ * the working directory version up to date with what it
+ * changed the index version to be.
+ */
+ if (ce_skip_worktree(ce))
+ return 0;
+
costate.refresh_cache = 1;
costate.istate = istate;
if (checkout_entry(ce, &costate, NULL, NULL) ||
diff --git a/builtin.h b/builtin.h
index 14fa0171607b17..6792775b20fdb0 100644
--- a/builtin.h
+++ b/builtin.h
@@ -238,12 +238,14 @@ int cmd_status(int argc, const char **argv, const char *prefix);
int cmd_stash(int argc, const char **argv, const char *prefix);
int cmd_stripspace(int argc, const char **argv, const char *prefix);
int cmd_submodule__helper(int argc, const char **argv, const char *prefix);
+int cmd_survey(int argc, const char **argv, const char *prefix);
int cmd_switch(int argc, const char **argv, const char *prefix);
int cmd_symbolic_ref(int argc, const char **argv, const char *prefix);
int cmd_tag(int argc, const char **argv, const char *prefix);
int cmd_unpack_file(int argc, const char **argv, const char *prefix);
int cmd_unpack_objects(int argc, const char **argv, const char *prefix);
int cmd_update_index(int argc, const char **argv, const char *prefix);
+int cmd_update_microsoft_git(int argc, const char **argv, const char *prefix);
int cmd_update_ref(int argc, const char **argv, const char *prefix);
int cmd_update_server_info(int argc, const char **argv, const char *prefix);
int cmd_upload_archive(int argc, const char **argv, const char *prefix);
diff --git a/builtin/add.c b/builtin/add.c
index 077f5c171a90b5..255b9c211a7f3d 100644
--- a/builtin/add.c
+++ b/builtin/add.c
@@ -5,6 +5,7 @@
*/
#include "builtin.h"
+#include "environment.h"
#include "advice.h"
#include "config.h"
#include "lockfile.h"
@@ -45,6 +46,7 @@ static int chmod_pathspec(struct pathspec *pathspec, char flip, int show_only)
int err;
if (!include_sparse &&
+ !core_virtualfilesystem &&
(ce_skip_worktree(ce) ||
!path_in_sparse_checkout(ce->name, the_repository->index)))
continue;
@@ -125,8 +127,9 @@ static int refresh(int verbose, const struct pathspec *pathspec)
if (!seen[i]) {
const char *path = pathspec->items[i].original;
- if (matches_skip_worktree(pathspec, i, &skip_worktree_seen) ||
- !path_in_sparse_checkout(path, the_repository->index)) {
+ if (!core_virtualfilesystem &&
+ (matches_skip_worktree(pathspec, i, &skip_worktree_seen) ||
+ !path_in_sparse_checkout(path, the_repository->index))) {
string_list_append(&only_match_skip_worktree,
pathspec->items[i].original);
} else {
@@ -136,7 +139,11 @@ static int refresh(int verbose, const struct pathspec *pathspec)
}
}
- if (only_match_skip_worktree.nr) {
+ /*
+ * When using a virtual filesystem, we might re-add a path
+ * that is currently virtual and we want that to succeed.
+ */
+ if (!core_virtualfilesystem && only_match_skip_worktree.nr) {
advise_on_updating_sparse_paths(&only_match_skip_worktree);
ret = 1;
}
@@ -512,7 +519,11 @@ int cmd_add(int argc, const char **argv, const char *prefix)
if (seen[i])
continue;
- if (!include_sparse &&
+ /*
+ * When using a virtual filesystem, we might re-add a path
+ * that is currently virtual and we want that to succeed.
+ */
+ if (!include_sparse && !core_virtualfilesystem &&
matches_skip_worktree(&pathspec, i, &skip_worktree_seen)) {
string_list_append(&only_match_skip_worktree,
pathspec.items[i].original);
@@ -536,7 +547,6 @@ int cmd_add(int argc, const char **argv, const char *prefix)
}
}
-
if (only_match_skip_worktree.nr) {
advise_on_updating_sparse_paths(&only_match_skip_worktree);
exit_status = 1;
diff --git a/builtin/checkout.c b/builtin/checkout.c
index 44db2bae72a2f5..59a582a400f230 100644
--- a/builtin/checkout.c
+++ b/builtin/checkout.c
@@ -17,6 +17,7 @@
#include "merge-recursive.h"
#include "object-name.h"
#include "object-store-ll.h"
+#include "packfile.h"
#include "parse-options.h"
#include "path.h"
#include "preload-index.h"
@@ -1043,8 +1044,16 @@ static void update_refs_for_switch(const struct checkout_opts *opts,
strbuf_release(&msg);
if (!opts->quiet &&
!opts->force_detach &&
- (new_branch_info->path || !strcmp(new_branch_info->name, "HEAD")))
+ (new_branch_info->path || !strcmp(new_branch_info->name, "HEAD"))) {
+ unsigned long nr_unpack_entry_at_start;
+
+ trace2_region_enter("tracking", "report_tracking", the_repository);
+ nr_unpack_entry_at_start = get_nr_unpack_entry();
report_tracking(new_branch_info);
+ trace2_data_intmax("tracking", NULL, "report_tracking/nr_unpack_entries",
+ (intmax_t)(get_nr_unpack_entry() - nr_unpack_entry_at_start));
+ trace2_region_leave("tracking", "report_tracking", the_repository);
+ }
}
static int add_pending_uninteresting_ref(const char *refname,
diff --git a/builtin/commit.c b/builtin/commit.c
index 6a5266075a8ede..6ec1f3bdd78186 100644
--- a/builtin/commit.c
+++ b/builtin/commit.c
@@ -38,6 +38,7 @@
#include "commit-graph.h"
#include "pretty.h"
#include "trailer.h"
+#include "trace2.h"
static const char * const builtin_commit_usage[] = {
N_("git commit [-a | --interactive | --patch] [-s] [-v] [-u] [--amend]\n"
@@ -160,6 +161,122 @@ static int opt_parse_porcelain(const struct option *opt, const char *arg, int un
return 0;
}
+static int do_serialize = 0;
+static char *serialize_path = NULL;
+
+static int reject_implicit = 0;
+static int do_implicit_deserialize = 0;
+static int do_explicit_deserialize = 0;
+static char *deserialize_path = NULL;
+
+static enum wt_status_deserialize_wait implicit_deserialize_wait = DESERIALIZE_WAIT__UNSET;
+static enum wt_status_deserialize_wait explicit_deserialize_wait = DESERIALIZE_WAIT__UNSET;
+
+/*
+ * --serialize | --serialize=
+ *
+ * Request that we serialize status output rather than or in addition to
+ * printing in any of the established formats.
+ *
+ * Without a path, we write binary serialization data to stdout (and omit
+ * the normal status output).
+ *
+ * With a path, we write binary serialization data to the and then
+ * write normal status output.
+ */
+static int opt_parse_serialize(const struct option *opt, const char *arg, int unset)
+{
+ enum wt_status_format *value = (enum wt_status_format *)opt->value;
+ if (unset || !arg)
+ *value = STATUS_FORMAT_SERIALIZE_V1;
+
+ if (arg) {
+ free(serialize_path);
+ serialize_path = xstrdup(arg);
+ }
+
+ if (do_explicit_deserialize)
+ die("cannot mix --serialize and --deserialize");
+ do_implicit_deserialize = 0;
+
+ do_serialize = 1;
+ return 0;
+}
+
+/*
+ * --deserialize | --deserialize= |
+ * --no-deserialize
+ *
+ * Request that we deserialize status data from some existing resource
+ * rather than performing a status scan.
+ *
+ * The input source can come from stdin or a path given here -- or be
+ * inherited from the config settings.
+ */
+static int opt_parse_deserialize(const struct option *opt, const char *arg, int unset)
+{
+ if (unset) {
+ do_implicit_deserialize = 0;
+ do_explicit_deserialize = 0;
+ } else {
+ if (do_serialize)
+ die("cannot mix --serialize and --deserialize");
+ if (arg) {
+ /* override config or stdin */
+ free(deserialize_path);
+ deserialize_path = xstrdup(arg);
+ }
+ if (!deserialize_path || !*deserialize_path)
+ do_explicit_deserialize = 1; /* read stdin */
+ else if (wt_status_deserialize_access(deserialize_path, R_OK) == 0)
+ do_explicit_deserialize = 1; /* can read from this file */
+ else {
+ /*
+ * otherwise, silently fallback to the normal
+ * collection scan
+ */
+ do_implicit_deserialize = 0;
+ do_explicit_deserialize = 0;
+ }
+ }
+
+ return 0;
+}
+
+static enum wt_status_deserialize_wait parse_dw(const char *arg)
+{
+ int tenths;
+
+ if (!strcmp(arg, "fail"))
+ return DESERIALIZE_WAIT__FAIL;
+ else if (!strcmp(arg, "block"))
+ return DESERIALIZE_WAIT__BLOCK;
+ else if (!strcmp(arg, "no"))
+ return DESERIALIZE_WAIT__NO;
+
+ /*
+ * Otherwise, assume it is a timeout in tenths of a second.
+ * If it contains a bogus value, atol() will return zero
+ * which is OK.
+ */
+ tenths = atol(arg);
+ if (tenths < 0)
+ tenths = DESERIALIZE_WAIT__NO;
+ return tenths;
+}
+
+static int opt_parse_deserialize_wait(const struct option *opt,
+ const char *arg,
+ int unset)
+{
+ if (unset)
+ explicit_deserialize_wait = DESERIALIZE_WAIT__UNSET;
+ else
+ explicit_deserialize_wait = parse_dw(arg);
+
+ return 0;
+}
+
static int opt_parse_m(const struct option *opt, const char *arg, int unset)
{
struct strbuf *buf = opt->value;
@@ -1172,6 +1289,8 @@ static enum untracked_status_type parse_untracked_setting_name(const char *u)
return SHOW_NORMAL_UNTRACKED_FILES;
else if (!strcmp(u, "all"))
return SHOW_ALL_UNTRACKED_FILES;
+ else if (!strcmp(u,"complete"))
+ return SHOW_COMPLETE_UNTRACKED_FILES;
else
return SHOW_UNTRACKED_FILES_ERROR;
}
@@ -1469,6 +1588,28 @@ static int git_status_config(const char *k, const char *v,
s->relative_paths = git_config_bool(k, v);
return 0;
}
+ if (!strcmp(k, "status.deserializepath")) {
+ /*
+ * Automatically assume deserialization if this is
+ * set in the config and the file exists. Do not
+ * complain if the file does not exist, because we
+ * silently fall back to normal mode.
+ */
+ if (v && *v && access(v, R_OK) == 0) {
+ do_implicit_deserialize = 1;
+ deserialize_path = xstrdup(v);
+ } else {
+ reject_implicit = 1;
+ }
+ return 0;
+ }
+ if (!strcmp(k, "status.deserializewait")) {
+ if (!v || !*v)
+ implicit_deserialize_wait = DESERIALIZE_WAIT__UNSET;
+ else
+ implicit_deserialize_wait = parse_dw(v);
+ return 0;
+ }
if (!strcmp(k, "status.showuntrackedfiles")) {
enum untracked_status_type u;
@@ -1505,7 +1646,8 @@ int cmd_status(int argc, const char **argv, const char *prefix)
static const char *rename_score_arg = (const char *)-1;
static struct wt_status s;
unsigned int progress_flag = 0;
- int fd;
+ int try_deserialize;
+ int fd = -1;
struct object_id oid;
static struct option builtin_status_options[] = {
OPT__VERBOSE(&verbose, N_("be verbose")),
@@ -1520,6 +1662,15 @@ int cmd_status(int argc, const char **argv, const char *prefix)
OPT_CALLBACK_F(0, "porcelain", &status_format,
N_("version"), N_("machine-readable output"),
PARSE_OPT_OPTARG, opt_parse_porcelain),
+ { OPTION_CALLBACK, 0, "serialize", &status_format,
+ N_("path"), N_("serialize raw status data to path or stdout"),
+ PARSE_OPT_OPTARG | PARSE_OPT_NONEG, opt_parse_serialize },
+ { OPTION_CALLBACK, 0, "deserialize", NULL,
+ N_("path"), N_("deserialize raw status data from file"),
+ PARSE_OPT_OPTARG, opt_parse_deserialize },
+ { OPTION_CALLBACK, 0, "deserialize-wait", NULL,
+ N_("fail|block|no"), N_("how to wait if status cache file is invalid"),
+ PARSE_OPT_OPTARG, opt_parse_deserialize_wait },
OPT_SET_INT(0, "long", &status_format,
N_("show status in long format (default)"),
STATUS_FORMAT_LONG),
@@ -1564,10 +1715,53 @@ int cmd_status(int argc, const char **argv, const char *prefix)
s.show_untracked_files == SHOW_NO_UNTRACKED_FILES)
die(_("Unsupported combination of ignored and untracked-files arguments"));
+ if (s.show_untracked_files == SHOW_COMPLETE_UNTRACKED_FILES &&
+ s.show_ignored_mode == SHOW_NO_IGNORED)
+ die(_("Complete Untracked only supported with ignored files"));
+
parse_pathspec(&s.pathspec, 0,
PATHSPEC_PREFER_FULL,
prefix, argv);
+ /*
+ * If we want to try to deserialize status data from a cache file,
+ * we need to re-order the initialization code. The problem is that
+ * this makes for a very nasty diff and causes merge conflicts as we
+ * carry it forward. And it easy to mess up the merge, so we
+ * duplicate some code here to hopefully reduce conflicts.
+ */
+ try_deserialize = (!do_serialize &&
+ (do_implicit_deserialize || do_explicit_deserialize));
+
+ /*
+ * Disable deserialize when verbose is set because it causes us to
+ * print diffs for each modified file, but that requires us to have
+ * the index loaded and we don't want to do that (at least not now for
+ * this seldom used feature). My fear is that would further tangle
+ * the merge conflict with upstream.
+ *
+ * TODO Reconsider this in the future.
+ */
+ if (try_deserialize && verbose) {
+ trace2_data_string("status", the_repository, "deserialize/reject",
+ "args/verbose");
+ try_deserialize = 0;
+ }
+
+ if (try_deserialize)
+ goto skip_init;
+ /*
+ * If we implicitly received a status cache pathname from the config
+ * and the file does not exist, we silently reject it and do the normal
+ * status "collect". Fake up some trace2 messages to reflect this and
+ * assist post-processors know this case is different.
+ */
+ if (!do_serialize && reject_implicit) {
+ trace2_cmd_mode("implicit-deserialize");
+ trace2_data_string("status", the_repository, "deserialize/reject",
+ "status-cache/access");
+ }
+
enable_fscache(0);
if (status_format != STATUS_FORMAT_PORCELAIN &&
status_format != STATUS_FORMAT_PORCELAIN_V2)
@@ -1582,6 +1776,7 @@ int cmd_status(int argc, const char **argv, const char *prefix)
else
fd = -1;
+skip_init:
s.is_initial = repo_get_oid(the_repository, s.reference, &oid) ? 1 : 0;
if (!s.is_initial)
oidcpy(&s.oid_commit, &oid);
@@ -1598,6 +1793,36 @@ int cmd_status(int argc, const char **argv, const char *prefix)
s.rename_score = parse_rename_score(&rename_score_arg);
}
+ if (try_deserialize) {
+ int result;
+ enum wt_status_deserialize_wait dw = implicit_deserialize_wait;
+ if (explicit_deserialize_wait != DESERIALIZE_WAIT__UNSET)
+ dw = explicit_deserialize_wait;
+ if (dw == DESERIALIZE_WAIT__UNSET)
+ dw = DESERIALIZE_WAIT__NO;
+
+ if (s.relative_paths)
+ s.prefix = prefix;
+
+ trace2_cmd_mode("deserialize");
+ result = wt_status_deserialize(&s, deserialize_path, dw);
+ if (result == DESERIALIZE_OK)
+ return 0;
+ if (dw == DESERIALIZE_WAIT__FAIL)
+ die(_("Rejected status serialization cache"));
+
+ /* deserialize failed, so force the initialization we skipped above. */
+ enable_fscache(1);
+ repo_read_index_preload(the_repository, &s.pathspec, 0);
+ refresh_index(the_repository->index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL);
+
+ if (use_optional_locks())
+ fd = repo_hold_locked_index(the_repository, &index_lock, 0);
+ else
+ fd = -1;
+ }
+
+ trace2_cmd_mode("collect");
wt_status_collect(&s);
if (0 <= fd)
@@ -1606,6 +1831,17 @@ int cmd_status(int argc, const char **argv, const char *prefix)
if (s.relative_paths)
s.prefix = prefix;
+ if (serialize_path) {
+ int fd_serialize = xopen(serialize_path,
+ O_WRONLY | O_CREAT | O_TRUNC, 0666);
+ if (fd_serialize < 0)
+ die_errno(_("could not serialize to '%s'"),
+ serialize_path);
+ trace2_cmd_mode("serialize");
+ wt_status_serialize_v1(fd_serialize, &s);
+ close(fd_serialize);
+ }
+
wt_status_print(&s);
wt_status_collect_free_buffers(&s);
diff --git a/builtin/fetch.c b/builtin/fetch.c
index 693f02b95802bc..c99f47dac4806c 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -18,6 +18,9 @@
#include "string-list.h"
#include "remote.h"
#include "transport.h"
+#include "gvfs.h"
+#include "gvfs-helper-client.h"
+#include "packfile.h"
#include "run-command.h"
#include "parse-options.h"
#include "sigchain.h"
@@ -1160,6 +1163,13 @@ static int store_updated_refs(struct display_state *display_state,
opt.exclude_hidden_refs_section = "fetch";
rm = ref_map;
+
+ /*
+ * Before checking connectivity, be really sure we have the
+ * latest pack-files loaded into memory.
+ */
+ reprepare_packed_git(the_repository);
+
if (check_connected(iterate_ref_map, &rm, &opt)) {
rc = error(_("%s did not send all necessary objects\n"),
display_state->url);
@@ -2402,6 +2412,9 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
}
string_list_remove_duplicates(&list, 0);
+ if (core_gvfs & GVFS_PREFETCH_DURING_FETCH)
+ gh_client__prefetch(0, NULL);
+
if (negotiate_only) {
struct oidset acked_commits = OIDSET_INIT;
struct oidset_iter iter;
diff --git a/builtin/gc.c b/builtin/gc.c
index 72bac2554fc4b9..c8ad72a59cb49e 100644
--- a/builtin/gc.c
+++ b/builtin/gc.c
@@ -16,6 +16,7 @@
#include "environment.h"
#include "hex.h"
#include "repository.h"
+#include "gvfs.h"
#include "config.h"
#include "tempfile.h"
#include "lockfile.h"
@@ -676,6 +677,9 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
if (quiet)
strvec_push(&repack, "-q");
+ if ((!opts.auto_flag || (opts.auto_flag && gc_auto_threshold > 0)) && gvfs_config_is_set(GVFS_BLOCK_COMMANDS))
+ die(_("'git gc' is not supported on a GVFS repo"));
+
if (opts.auto_flag) {
/*
* Auto-gc should be least intrusive as possible.
@@ -1056,6 +1060,8 @@ static int write_loose_object_to_stdin(const struct object_id *oid,
return ++(d->count) > d->batch_size;
}
+static const char *object_dir = NULL;
+
static int pack_loose(struct maintenance_run_opts *opts)
{
struct repository *r = the_repository;
@@ -1063,11 +1069,14 @@ static int pack_loose(struct maintenance_run_opts *opts)
struct write_loose_object_data data;
struct child_process pack_proc = CHILD_PROCESS_INIT;
+ if (!object_dir)
+ object_dir = r->objects->odb->path;
+
/*
* Do not start pack-objects process
* if there are no loose objects.
*/
- if (!for_each_loose_file_in_objdir(r->objects->odb->path,
+ if (!for_each_loose_file_in_objdir(object_dir,
bail_on_loose,
NULL, NULL, NULL))
return 0;
@@ -1077,7 +1086,7 @@ static int pack_loose(struct maintenance_run_opts *opts)
strvec_push(&pack_proc.args, "pack-objects");
if (opts->quiet)
strvec_push(&pack_proc.args, "--quiet");
- strvec_pushf(&pack_proc.args, "%s/pack/loose", r->objects->odb->path);
+ strvec_pushf(&pack_proc.args, "%s/pack/loose", object_dir);
pack_proc.in = -1;
@@ -1090,7 +1099,7 @@ static int pack_loose(struct maintenance_run_opts *opts)
data.count = 0;
data.batch_size = 50000;
- for_each_loose_file_in_objdir(r->objects->odb->path,
+ for_each_loose_file_in_objdir(object_dir,
write_loose_object_to_stdin,
NULL,
NULL,
@@ -1468,6 +1477,7 @@ static int maintenance_run(int argc, const char **argv, const char *prefix)
{
int i;
struct maintenance_run_opts opts;
+ const char *tmp_obj_dir = NULL;
struct option builtin_maintenance_run_options[] = {
OPT_BOOL(0, "auto", &opts.auto_flag,
N_("run tasks based on the state of the repository")),
@@ -1501,6 +1511,18 @@ static int maintenance_run(int argc, const char **argv, const char *prefix)
if (argc != 0)
usage_with_options(builtin_maintenance_run_usage,
builtin_maintenance_run_options);
+
+ /*
+ * To enable the VFS for Git/Scalar shared object cache, use
+ * the gvfs.sharedcache config option to redirect the
+ * maintenance to that location.
+ */
+ if (!git_config_get_value("gvfs.sharedcache", &tmp_obj_dir) &&
+ tmp_obj_dir) {
+ object_dir = xstrdup(tmp_obj_dir);
+ setenv(DB_ENVIRONMENT, object_dir, 1);
+ }
+
return maintenance_run_tasks(&opts);
}
@@ -1664,6 +1686,42 @@ static const char *get_frequency(enum schedule_priority schedule)
}
}
+static const char *extraconfig[] = {
+ "credential.interactive=false",
+ "core.askPass=true", /* 'true' returns success, but no output. */
+ NULL
+};
+
+static const char *get_extra_config_parameters(void) {
+ static const char *result = NULL;
+ struct strbuf builder = STRBUF_INIT;
+
+ if (result)
+ return result;
+
+ for (const char **s = extraconfig; s && *s; s++)
+ strbuf_addf(&builder, "-c %s ", *s);
+
+ result = strbuf_detach(&builder, NULL);
+ return result;
+}
+
+static const char *get_extra_launchctl_strings(void) {
+ static const char *result = NULL;
+ struct strbuf builder = STRBUF_INIT;
+
+ if (result)
+ return result;
+
+ for (const char **s = extraconfig; s && *s; s++) {
+ strbuf_addstr(&builder, "-c\n");
+ strbuf_addf(&builder, "%s\n", *s);
+ }
+
+ result = strbuf_detach(&builder, NULL);
+ return result;
+}
+
/*
* get_schedule_cmd` reads the GIT_TEST_MAINT_SCHEDULER environment variable
* to mock the schedulers that `git maintenance start` rely on.
@@ -1870,6 +1928,7 @@ static int launchctl_schedule_plist(const char *exec_path, enum schedule_priorit
"\n"
"%s/git\n"
"--exec-path=%s\n"
+ "%s" /* For extra config parameters. */
"for-each-repo\n"
"--keep-going\n"
"--config=maintenance.repo\n"
@@ -1879,7 +1938,8 @@ static int launchctl_schedule_plist(const char *exec_path, enum schedule_priorit
"\n"
"StartCalendarInterval\n"
"\n";
- strbuf_addf(&plist, preamble, name, exec_path, exec_path, frequency);
+ strbuf_addf(&plist, preamble, name, exec_path, exec_path,
+ get_extra_launchctl_strings(), frequency);
switch (schedule) {
case SCHEDULE_HOURLY:
@@ -2114,11 +2174,12 @@ static int schtasks_schedule_task(const char *exec_path, enum schedule_priority
"\n"
"\n"
"\"%s\\headless-git.exe\"\n"
- "--exec-path=\"%s\" for-each-repo --keep-going --config=maintenance.repo maintenance run --schedule=%s\n"
+ "--exec-path=\"%s\" %s for-each-repo --keep-going --config=maintenance.repo maintenance run --schedule=%s\n"
"\n"
"\n"
"\n";
- fprintf(tfile->fp, xml, exec_path, exec_path, frequency);
+ fprintf(tfile->fp, xml, exec_path, exec_path,
+ get_extra_config_parameters(), frequency);
strvec_split(&child.args, cmd);
strvec_pushl(&child.args, "/create", "/tn", name, "/f", "/xml",
get_tempfile_path(tfile), NULL);
@@ -2259,8 +2320,8 @@ static int crontab_update_schedule(int run_maintenance, int fd)
"# replaced in the future by a Git command.\n\n");
strbuf_addf(&line_format,
- "%%d %%s * * %%s \"%s/git\" --exec-path=\"%s\" for-each-repo --keep-going --config=maintenance.repo maintenance run --schedule=%%s\n",
- exec_path, exec_path);
+ "%%d %%s * * %%s \"%s/git\" --exec-path=\"%s\" %s for-each-repo --keep-going --config=maintenance.repo maintenance run --schedule=%%s\n",
+ exec_path, exec_path, get_extra_config_parameters());
fprintf(cron_in, line_format.buf, minute, "1-23", "*", "hourly");
fprintf(cron_in, line_format.buf, minute, "0", "1-6", "daily");
fprintf(cron_in, line_format.buf, minute, "0", "0", "weekly");
@@ -2460,7 +2521,7 @@ static int systemd_timer_write_service_template(const char *exec_path)
"\n"
"[Service]\n"
"Type=oneshot\n"
- "ExecStart=\"%s/git\" --exec-path=\"%s\" for-each-repo --keep-going --config=maintenance.repo maintenance run --schedule=%%i\n"
+ "ExecStart=\"%s/git\" --exec-path=\"%s\" %s for-each-repo --keep-going --config=maintenance.repo maintenance run --schedule=%%i\n"
"LockPersonality=yes\n"
"MemoryDenyWriteExecute=yes\n"
"NoNewPrivileges=yes\n"
@@ -2470,7 +2531,7 @@ static int systemd_timer_write_service_template(const char *exec_path)
"RestrictSUIDSGID=yes\n"
"SystemCallArchitectures=native\n"
"SystemCallFilter=@system-service\n";
- if (fprintf(file, unit, exec_path, exec_path) < 0) {
+ if (fprintf(file, unit, exec_path, exec_path, get_extra_config_parameters()) < 0) {
error(_("failed to write to '%s'"), filename);
fclose(file);
goto error;
diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index 763b01372aade4..5bbf86a996088d 100644
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -811,7 +811,7 @@ static void sha1_object(const void *data, struct object_entry *obj_entry,
read_lock();
collision_test_needed =
repo_has_object_file_with_flags(the_repository, oid,
- OBJECT_INFO_QUICK);
+ OBJECT_INFO_FOR_PREFETCH);
read_unlock();
}
@@ -1734,6 +1734,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
unsigned foreign_nr = 1; /* zero is a "good" value, assume bad */
int report_end_of_input = 0;
int hash_algo = 0;
+ int dash_o = 0;
/*
* index-pack never needs to fetch missing objects except when
@@ -1827,6 +1828,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
if (index_name || (i+1) >= argc)
usage(index_pack_usage);
index_name = argv[++i];
+ dash_o = 1;
} else if (starts_with(arg, "--index-version=")) {
char *c;
opts.version = strtoul(arg + 16, &c, 10);
@@ -1878,6 +1880,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
repo_set_hash_algo(the_repository, GIT_HASH_SHA1);
opts.flags &= ~(WRITE_REV | WRITE_REV_VERIFY);
+ if (rev_index && dash_o && !ends_with(index_name, ".idx"))
+ rev_index = 0;
if (rev_index) {
opts.flags |= verify ? WRITE_REV_VERIFY : WRITE_REV;
if (index_name)
diff --git a/builtin/reset.c b/builtin/reset.c
index 9cba244fb045d0..6e11fc88c0b512 100644
--- a/builtin/reset.c
+++ b/builtin/reset.c
@@ -37,6 +37,8 @@
#include "add-interactive.h"
#include "strbuf.h"
#include "quote.h"
+#include "dir.h"
+#include "entry.h"
#define REFRESH_INDEX_DELAY_WARNING_IN_MS (2 * 1000)
@@ -157,9 +159,48 @@ static void update_index_from_diff(struct diff_queue_struct *q,
for (i = 0; i < q->nr; i++) {
int pos;
+ int respect_skip_worktree = 1;
struct diff_filespec *one = q->queue[i]->one;
+ struct diff_filespec *two = q->queue[i]->two;
int is_in_reset_tree = one->mode && !is_null_oid(&one->oid);
+ int is_missing = !(one->mode && !is_null_oid(&one->oid));
+ int was_missing = !two->mode && is_null_oid(&two->oid);
struct cache_entry *ce;
+ struct cache_entry *ceBefore;
+ struct checkout state = CHECKOUT_INIT;
+
+ /*
+ * When using the virtual filesystem feature, the cache entries that are
+ * added here will not have the skip-worktree bit set.
+ *
+ * Without this code there is data that is lost because the files that
+ * would normally be in the working directory are not there and show as
+ * deleted for the next status or in the case of added files just disappear.
+ * We need to create the previous version of the files in the working
+ * directory so that they will have the right content and the next
+ * status call will show modified or untracked files correctly.
+ */
+ if (core_virtualfilesystem && !file_exists(two->path))
+ {
+ respect_skip_worktree = 0;
+ pos = index_name_pos(the_repository->index, two->path, strlen(two->path));
+
+ if ((pos >= 0 && ce_skip_worktree(the_repository->index->cache[pos])) &&
+ (is_missing || !was_missing))
+ {
+ state.force = 1;
+ state.refresh_cache = 1;
+ state.istate = the_repository->index;
+ ceBefore = make_cache_entry(the_repository->index, two->mode,
+ &two->oid, two->path,
+ 0, 0);
+ if (!ceBefore)
+ die(_("make_cache_entry failed for path '%s'"),
+ two->path);
+
+ checkout_entry(ceBefore, &state, NULL, NULL);
+ }
+ }
if (!is_in_reset_tree && !intent_to_add) {
remove_file_from_index(the_repository->index, one->path);
@@ -178,8 +219,14 @@ static void update_index_from_diff(struct diff_queue_struct *q,
* to properly construct the reset sparse directory.
*/
pos = index_name_pos(the_repository->index, one->path, strlen(one->path));
- if ((pos >= 0 && ce_skip_worktree(the_repository->index->cache[pos])) ||
- (pos < 0 && !path_in_sparse_checkout(one->path, the_repository->index)))
+
+ /*
+ * Do not add the SKIP_WORKTREE bit back if we populated the
+ * file on purpose in a virtual filesystem scenario.
+ */
+ if (respect_skip_worktree &&
+ ((pos >= 0 && ce_skip_worktree(the_repository->index->cache[pos])) ||
+ (pos < 0 && !path_in_sparse_checkout(one->path, the_repository->index))))
ce->ce_flags |= CE_SKIP_WORKTREE;
if (!ce)
diff --git a/builtin/rm.c b/builtin/rm.c
index 0e79cbab62b86a..ac048e4a47f9ae 100644
--- a/builtin/rm.c
+++ b/builtin/rm.c
@@ -5,6 +5,7 @@
*/
#include "builtin.h"
+#include "environment.h"
#include "advice.h"
#include "config.h"
#include "lockfile.h"
@@ -311,7 +312,7 @@ int cmd_rm(int argc, const char **argv, const char *prefix)
for (i = 0; i < the_repository->index->cache_nr; i++) {
const struct cache_entry *ce = the_repository->index->cache[i];
- if (!include_sparse &&
+ if (!include_sparse && !core_virtualfilesystem &&
(ce_skip_worktree(ce) ||
!path_in_sparse_checkout(ce->name, the_repository->index)))
continue;
@@ -348,7 +349,11 @@ int cmd_rm(int argc, const char **argv, const char *prefix)
*original ? original : ".");
}
- if (only_match_skip_worktree.nr) {
+ /*
+ * When using a virtual filesystem, we might re-add a path
+ * that is currently virtual and we want that to succeed.
+ */
+ if (!core_virtualfilesystem && only_match_skip_worktree.nr) {
advise_on_updating_sparse_paths(&only_match_skip_worktree);
ret = 1;
}
diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c
index 2604ab04df5aee..623a5b0c4d7379 100644
--- a/builtin/sparse-checkout.c
+++ b/builtin/sparse-checkout.c
@@ -108,7 +108,7 @@ static int sparse_checkout_list(int argc, const char **argv, const char *prefix)
static void clean_tracked_sparse_directories(struct repository *r)
{
- int i, was_full = 0;
+ int i, value, was_full = 0;
struct strbuf path = STRBUF_INIT;
size_t pathlen;
struct string_list_item *item;
@@ -124,6 +124,13 @@ static void clean_tracked_sparse_directories(struct repository *r)
!r->index->sparse_checkout_patterns->use_cone_patterns)
return;
+ /*
+ * Users can disable this behavior.
+ */
+ if (!repo_config_get_bool(r, "index.deletesparsedirectories", &value) &&
+ !value)
+ return;
+
/*
* Use the sparse index as a data structure to assist finding
* directories that are safe to delete. This conversion to a
diff --git a/builtin/survey.c b/builtin/survey.c
new file mode 100644
index 00000000000000..4d0f0f86c46649
--- /dev/null
+++ b/builtin/survey.c
@@ -0,0 +1,2441 @@
+#include "builtin.h"
+#include "config.h"
+#include "environment.h"
+#include "hex.h"
+#include "json-writer.h"
+#include "list-objects.h"
+#include "object-name.h"
+#include "object-store.h"
+#include "parse-options.h"
+#include "progress.h"
+#include "ref-filter.h"
+#include "refs.h"
+#include "revision.h"
+#include "run-command.h"
+#include "strbuf.h"
+#include "strmap.h"
+#include "strvec.h"
+#include "trace2.h"
+#include "tree.h"
+#include "tree-walk.h"
+#include "color.h"
+
+static const char * const survey_usage[] = {
+ N_("(EXPERIMENTAL!) git survey "),
+ NULL,
+};
+
+static struct progress *survey_progress = NULL;
+static uint64_t survey_progress_total = 0;
+
+struct survey_refs_wanted {
+ int want_all_refs; /* special override */
+
+ int want_branches;
+ int want_tags;
+ int want_remotes;
+ int want_detached;
+ int want_other; /* see FILTER_REFS_OTHERS -- refs/notes/, refs/stash/ */
+};
+
+static struct strvec survey_vec_refs_wanted = STRVEC_INIT;
+
+/*
+ * The set of refs that we will search if the user doesn't select
+ * any on the command line.
+ */
+static struct survey_refs_wanted refs_if_unspecified = {
+ .want_all_refs = 0,
+
+ .want_branches = 1,
+ .want_tags = 1,
+ .want_remotes = 1,
+ .want_detached = 0,
+ .want_other = 0,
+};
+
+struct survey_opts {
+ int verbose;
+ int show_progress;
+ int show_json;
+ int show_name_rev;
+
+ int show_largest_commits_by_nr_parents;
+ int show_largest_commits_by_size_bytes;
+
+ int show_largest_trees_by_nr_entries;
+ int show_largest_trees_by_size_bytes;
+
+ int show_largest_blobs_by_size_bytes;
+
+ struct survey_refs_wanted refs;
+};
+
+#define DEFAULT_SHOW_LARGEST_VALUE (10)
+
+static struct survey_opts survey_opts = {
+ .verbose = 0,
+ .show_progress = -1, /* defaults to isatty(2) */
+ .show_json = 0, /* defaults to pretty */
+ .show_name_rev = 1,
+
+ /*
+ * Show the largest `n` objects for some scaling dimension.
+ * We allow each to be requested independently.
+ */
+ .show_largest_commits_by_nr_parents = DEFAULT_SHOW_LARGEST_VALUE,
+ .show_largest_commits_by_size_bytes = DEFAULT_SHOW_LARGEST_VALUE,
+
+ .show_largest_trees_by_nr_entries = DEFAULT_SHOW_LARGEST_VALUE,
+ .show_largest_trees_by_size_bytes = DEFAULT_SHOW_LARGEST_VALUE,
+
+ .show_largest_blobs_by_size_bytes = DEFAULT_SHOW_LARGEST_VALUE,
+
+ .refs.want_all_refs = 0,
+
+ .refs.want_branches = -1, /* default these to undefined */
+ .refs.want_tags = -1,
+ .refs.want_remotes = -1,
+ .refs.want_detached = -1,
+ .refs.want_other = -1,
+};
+
+/*
+ * After parsing the command line arguments, figure out which refs we
+ * should scan.
+ *
+ * If ANY were given in positive sense, then we ONLY include them and
+ * do not use the builtin values.
+ */
+static void fixup_refs_wanted(void)
+{
+ struct survey_refs_wanted *rw = &survey_opts.refs;
+
+ /*
+ * `--all-refs` overrides and enables everything.
+ */
+ if (rw->want_all_refs == 1) {
+ rw->want_branches = 1;
+ rw->want_tags = 1;
+ rw->want_remotes = 1;
+ rw->want_detached = 1;
+ rw->want_other = 1;
+ return;
+ }
+
+ /*
+ * If none of the `--` were given, we assume all
+ * of the builtin unspecified values.
+ */
+ if (rw->want_branches == -1 &&
+ rw->want_tags == -1 &&
+ rw->want_remotes == -1 &&
+ rw->want_detached == -1 &&
+ rw->want_other == -1) {
+ *rw = refs_if_unspecified;
+ return;
+ }
+
+ /*
+ * Since we only allow positive boolean values on the command
+ * line, we will only have true values where they specified
+ * a `--`.
+ *
+ * So anything that still has an unspecified value should be
+ * set to false.
+ */
+ if (rw->want_branches == -1)
+ rw->want_branches = 0;
+ if (rw->want_tags == -1)
+ rw->want_tags = 0;
+ if (rw->want_remotes == -1)
+ rw->want_remotes = 0;
+ if (rw->want_detached == -1)
+ rw->want_detached = 0;
+ if (rw->want_other == -1)
+ rw->want_other = 0;
+}
+
+static struct option survey_options[] = {
+ OPT__VERBOSE(&survey_opts.verbose, N_("verbose output")),
+ OPT_BOOL(0, "progress", &survey_opts.show_progress, N_("show progress")),
+ OPT_BOOL(0, "json", &survey_opts.show_json, N_("report stats in JSON")),
+ OPT_BOOL(0, "name-rev", &survey_opts.show_name_rev, N_("run name-rev on each reported commit")),
+
+ OPT_BOOL_F(0, "all-refs", &survey_opts.refs.want_all_refs, N_("include all refs"), PARSE_OPT_NONEG),
+
+ OPT_BOOL_F(0, "branches", &survey_opts.refs.want_branches, N_("include branches"), PARSE_OPT_NONEG),
+ OPT_BOOL_F(0, "tags", &survey_opts.refs.want_tags, N_("include tags"), PARSE_OPT_NONEG),
+ OPT_BOOL_F(0, "remotes", &survey_opts.refs.want_remotes, N_("include all remotes refs"), PARSE_OPT_NONEG),
+ OPT_BOOL_F(0, "detached", &survey_opts.refs.want_detached, N_("include detached HEAD"), PARSE_OPT_NONEG),
+ OPT_BOOL_F(0, "other", &survey_opts.refs.want_other, N_("include notes and stashes"), PARSE_OPT_NONEG),
+
+ OPT_INTEGER_F(0, "commit-parents", &survey_opts.show_largest_commits_by_nr_parents, N_("show N largest commits by parent count"), PARSE_OPT_NONEG),
+ OPT_INTEGER_F(0, "commit-sizes", &survey_opts.show_largest_commits_by_size_bytes, N_("show N largest commits by size in bytes"), PARSE_OPT_NONEG),
+
+ OPT_INTEGER_F(0, "tree-entries", &survey_opts.show_largest_trees_by_nr_entries, N_("show N largest trees by entry count"), PARSE_OPT_NONEG),
+ OPT_INTEGER_F(0, "tree-sizes", &survey_opts.show_largest_trees_by_size_bytes, N_("show N largest trees by size in bytes"), PARSE_OPT_NONEG),
+
+ OPT_INTEGER_F(0, "blob-sizes", &survey_opts.show_largest_blobs_by_size_bytes, N_("show N largest blobs by size in bytes"), PARSE_OPT_NONEG),
+
+ OPT_END(),
+};
+
+static int survey_load_config_cb(const char *var, const char *value,
+ const struct config_context *ctx, void *pvoid)
+{
+ if (!strcmp(var, "survey.verbose")) {
+ survey_opts.verbose = git_config_bool(var, value);
+ return 0;
+ }
+ if (!strcmp(var, "survey.progress")) {
+ survey_opts.show_progress = git_config_bool(var, value);
+ return 0;
+ }
+ if (!strcmp(var, "survey.json")) {
+ survey_opts.show_json = git_config_bool(var, value);
+ return 0;
+ }
+ if (!strcmp(var, "survey.namerev")) {
+ survey_opts.show_name_rev = git_config_bool(var, value);
+ return 0;
+ }
+
+ if (!strcmp(var, "survey.showcommitparents")) {
+ survey_opts.show_largest_commits_by_nr_parents = git_config_ulong(var, value, ctx->kvi);
+ return 0;
+ }
+ if (!strcmp(var, "survey.showcommitsizes")) {
+ survey_opts.show_largest_commits_by_size_bytes = git_config_ulong(var, value, ctx->kvi);
+ return 0;
+ }
+
+ if (!strcmp(var, "survey.showtreeentries")) {
+ survey_opts.show_largest_trees_by_nr_entries = git_config_ulong(var, value, ctx->kvi);
+ return 0;
+ }
+ if (!strcmp(var, "survey.showtreesizes")) {
+ survey_opts.show_largest_trees_by_size_bytes = git_config_ulong(var, value, ctx->kvi);
+ return 0;
+ }
+
+ if (!strcmp(var, "survey.showblobsizes")) {
+ survey_opts.show_largest_blobs_by_size_bytes = git_config_ulong(var, value, ctx->kvi);
+ return 0;
+ }
+
+ return git_default_config(var, value, ctx, pvoid);
+}
+
+static void survey_load_config(void)
+{
+ git_config(survey_load_config_cb, NULL);
+}
+
+/*
+ * Stats on the set of refs that we found.
+ */
+struct survey_stats_refs {
+ uint32_t cnt_total;
+ uint32_t cnt_lightweight_tags;
+ uint32_t cnt_annotated_tags;
+ uint32_t cnt_branches;
+ uint32_t cnt_remotes;
+ uint32_t cnt_detached;
+ uint32_t cnt_other;
+
+ uint32_t cnt_symref;
+
+ uint32_t cnt_packed;
+ uint32_t cnt_loose;
+
+ /*
+ * Measure the length of the refnames. We can look for
+ * potential platform limits. The partial sums may help us
+ * estimate the size of a haves/wants conversation, since each
+ * refname and a SHA must be transmitted.
+ */
+ size_t len_max_local_refname;
+ size_t len_sum_local_refnames;
+ size_t len_max_remote_refname;
+ size_t len_sum_remote_refnames;
+
+ struct strintmap refsmap;
+};
+
+/*
+ * HBIN -- hex binning (histogram bucketing).
+ *
+ * We create histograms for various counts and sums. Since we have a
+ * wide range of values (objects range in size from 1 to 4G bytes), a
+ * linear bucketing is not interesting. Instead, lets use a
+ * log16()-based bucketing. This gives us a better spread on the low
+ * and middle range and a coarse bucketing on the high end.
+ *
+ * The idea here is that it doesn't matter if you have n 1GB blobs or
+ * n/2 1GB blobs and n/2 1.5GB blobs -- either way you have a scaling
+ * problem that we want to report on.
+ */
+#define HBIN_LEN (sizeof(unsigned long) * 2)
+#define HBIN_MASK (0xF)
+#define HBIN_SHIFT (4)
+
+static int hbin(unsigned long value)
+{
+ int k;
+
+ for (k = 0; k < HBIN_LEN; k++) {
+ if ((value & ~(HBIN_MASK)) == 0)
+ return k;
+ value >>= HBIN_SHIFT;
+ }
+
+ return 0; /* should not happen */
+}
+
+/*
+ * QBIN -- base4 binning (histogram bucketing).
+ *
+ * This is the same idea as the above, but we want better granularity
+ * in the low end and don't expect as many large values.
+ */
+#define QBIN_LEN (sizeof(unsigned long) * 4)
+#define QBIN_MASK (0x3)
+#define QBIN_SHIFT (2)
+
+static int qbin(unsigned long value)
+{
+ int k;
+
+ for (k = 0; k < QBIN_LEN; k++) {
+ if ((value & ~(QBIN_MASK)) == 0)
+ return k;
+ value >>= (QBIN_SHIFT);
+ }
+
+ return 0; /* should not happen */
+}
+
+/*
+ * histogram bin for objects.
+ */
+struct obj_hist_bin {
+ uint64_t sum_size; /* sum(object_size) for all objects in this bin */
+ uint64_t sum_disk_size; /* sum(on_disk_size) for all objects in this bin */
+ uint32_t cnt_seen; /* number seen in this bin */
+};
+
+static void incr_obj_hist_bin(struct obj_hist_bin *pbin,
+ unsigned long object_length,
+ off_t disk_sizep)
+{
+ pbin->sum_size += object_length;
+ pbin->sum_disk_size += disk_sizep;
+ pbin->cnt_seen++;
+}
+
+/*
+ * Remember the largest n objects for some scaling dimension. This
+ * could be the observed object size or number of entries in a tree.
+ * We'll use this to generate a sorted vector in the output for that
+ * dimension.
+ */
+struct large_item {
+ uint64_t size;
+ struct object_id oid;
+
+ /*
+ * For blobs and trees the name field is the pathname of the
+ * file or directory (as reported by the treewalk). Root trees
+ * are reported with a zero-length name, but we'll fix them up.
+ * The name field is not currenly used for commits.
+ */
+ struct strbuf *name;
+
+ /*
+ * For blobs and trees remember the transient commit from
+ * the treewalk so that we can say that this large item
+ * first appeared in this commit (relative to the treewalk
+ * order).
+ */
+ struct object_id containing_commit_oid;
+
+ /*
+ * Lookup `containing_commit_oid` using `git name-rev`.
+ * Lazy allocate this post-treewalk.
+ */
+ struct strbuf *name_rev;
+};
+
+struct large_item_vec_labels {
+ const char *dimension;
+ const char *item;
+};
+
+struct large_item_vec {
+ const struct large_item_vec_labels *labels_json;
+ const struct large_item_vec_labels *labels_pretty;
+ uint64_t nr_items;
+ enum object_type type;
+ struct large_item items[FLEX_ARRAY]; /* nr_items */
+};
+
+static struct large_item_vec *alloc_large_item_vec(
+ const struct large_item_vec_labels *labels_json,
+ const struct large_item_vec_labels *labels_pretty,
+ uint64_t nr_items,
+ enum object_type type)
+{
+ struct large_item_vec *vec;
+ size_t flex_len = nr_items * sizeof(struct large_item);
+ size_t k;
+
+ if (!nr_items)
+ return NULL;
+
+ vec = xcalloc(1, (sizeof(struct large_item_vec) + flex_len));
+ vec->labels_json = labels_json;
+ vec->labels_pretty = labels_pretty;
+ vec->nr_items = nr_items;
+ vec->type = type;
+
+ for (k = 0; k < nr_items; k++) {
+ struct strbuf *p = xcalloc(1, sizeof(struct strbuf));
+ strbuf_init(p, 0);
+ vec->items[k].name = p;
+ }
+
+ return vec;
+}
+
+static void free_large_item_vec(struct large_item_vec *vec)
+{
+ size_t k;
+
+ for (k = 0; k < vec->nr_items; k++) {
+ strbuf_release(vec->items[k].name);
+ free(vec->items[k].name);
+
+ if (vec->items[k].name_rev) {
+ strbuf_release(vec->items[k].name_rev);
+ free(vec->items[k].name_rev);
+ }
+ }
+
+ free(vec);
+}
+
+static void maybe_insert_large_item(struct large_item_vec *vec,
+ uint64_t size,
+ struct object_id *oid,
+ const char *name,
+ const struct object_id *containing_commit_oid)
+{
+ struct strbuf *pbuf_temp;
+ size_t rest_len;
+ size_t k;
+
+ if (!vec || !vec->nr_items)
+ return;
+
+ /*
+ * Since the odds an object being among the largest n
+ * is small, shortcut and see if it is smaller than
+ * the smallest one in our set and quickly reject it.
+ */
+ if (size < vec->items[vec->nr_items - 1].size)
+ return;
+
+ for (k = 0; k < vec->nr_items; k++) {
+ if (size < vec->items[k].size)
+ continue;
+
+ /*
+ * The last large_item in the vector is about to be
+ * overwritten by the previous one during the shift.
+ * Steal its allocated strbuf and reuse it.
+ *
+ * We can ignore .name_rev because it will not be
+ * allocated until after the treewalk.
+ */
+ pbuf_temp = vec->items[vec->nr_items - 1].name;
+ strbuf_reset(pbuf_temp);
+ if (name && *name)
+ strbuf_addstr(pbuf_temp, name);
+ else if (vec->type == OBJ_TREE) {
+ /*
+ * NEEDSWORK: Would it be better to wait and create
+ * a name of the form "^{tree}" after the
+ * treewalk is finished?
+ */
+ strbuf_addf(pbuf_temp, "%s^{tree}",
+ oid_to_hex(containing_commit_oid));
+ }
+
+ /* push items[k..] down one and insert data for this item here */
+
+ rest_len = (vec->nr_items - k - 1) * sizeof(struct large_item);
+ if (rest_len)
+ memmove(&vec->items[k + 1], &vec->items[k], rest_len);
+
+ memset(&vec->items[k], 0, sizeof(struct large_item));
+ vec->items[k].size = size;
+ oidcpy(&vec->items[k].oid, oid);
+ oidcpy(&vec->items[k].containing_commit_oid, containing_commit_oid);
+
+ vec->items[k].name = pbuf_temp;
+
+ return;
+ }
+}
+
+/*
+ * Try to run `git name-rev` on each of the containing-commit-oid's
+ * in this large-item-vec to get a pretty name for each OID. Silently
+ * ignore errors if it fails because this info is nice to have but not
+ * essential.
+ */
+static void large_item_vec_lookup_name_rev(struct large_item_vec *vec)
+{
+ struct child_process cp = CHILD_PROCESS_INIT;
+ struct strbuf in = STRBUF_INIT;
+ struct strbuf out = STRBUF_INIT;
+ const char *line;
+ size_t k;
+
+ if (!vec || !vec->nr_items)
+ return;
+
+ survey_progress_total += vec->nr_items;
+ display_progress(survey_progress, survey_progress_total);
+
+ for (k = 0; k < vec->nr_items; k++)
+ strbuf_addf(&in, "%s\n", oid_to_hex(&vec->items[k].containing_commit_oid));
+
+ cp.git_cmd = 1;
+ strvec_pushl(&cp.args, "name-rev", "--name-only", "--annotate-stdin", NULL);
+ if (pipe_command(&cp, in.buf, in.len, &out, 0, NULL, 0)) {
+ strbuf_release(&in);
+ strbuf_release(&out);
+ return;
+ }
+
+ line = out.buf;
+ k = 0;
+ while (*line) {
+ const char *eol = strchrnul(line, '\n');
+
+ vec->items[k].name_rev = xcalloc(1, sizeof(struct strbuf));
+ strbuf_init(vec->items[k].name_rev, 0);
+ strbuf_add(vec->items[k].name_rev, line, (eol - line));
+
+ line = eol + 1;
+ k++;
+ }
+
+ strbuf_release(&in);
+ strbuf_release(&out);
+}
+
+/*
+ * Common fields for any type of object.
+ */
+struct survey_stats_base_object {
+ uint32_t cnt_seen;
+
+ uint32_t cnt_missing; /* we may have a partial clone. */
+
+ /*
+ * Number of objects grouped by where they are stored on disk.
+ * This is a function of how the ODB is packed.
+ */
+ uint32_t cnt_cached; /* see oi.whence */
+ uint32_t cnt_loose; /* see oi.whence */
+ uint32_t cnt_packed; /* see oi.whence */
+ uint32_t cnt_dbcached; /* see oi.whence */
+
+ uint64_t sum_size; /* sum(object_size) */
+ uint64_t sum_disk_size; /* sum(disk_size) */
+
+ /*
+ * A histogram of the count of objects, the observed size, and
+ * the on-disk size grouped by the observed size.
+ */
+ struct obj_hist_bin size_hbin[HBIN_LEN];
+};
+
+/*
+ * PBIN -- parent vector binning (histogram bucketing).
+ *
+ * We create a histogram based upon the number of parents
+ * in a commit. This is a simple linear vector. It starts
+ * at zero for "initial" commits.
+ *
+ * If a commit has more parents, just put it in the last bin.
+ */
+#define PBIN_VEC_LEN (17)
+
+struct survey_stats_commits {
+ struct survey_stats_base_object base;
+
+ /*
+ * Count of commits with k parents.
+ */
+ uint32_t parent_cnt_pbin[PBIN_VEC_LEN];
+
+ struct large_item_vec *vec_largest_by_nr_parents;
+ struct large_item_vec *vec_largest_by_size_bytes;
+};
+
+/*
+ * Stats for reachable trees.
+ */
+struct survey_stats_trees {
+ struct survey_stats_base_object base;
+
+ /*
+ * Keep a vector of the trees with the most number of entries.
+ * This gives us a feel for the width of a tree when there are
+ * gigantic directories.
+ */
+ struct large_item_vec *vec_largest_by_nr_entries;
+
+ /*
+ * Keep a vector of the trees with the largest size in bytes.
+ * The contents of this may or may not match items in the other
+ * vector, since entryname length can alter the results.
+ */
+ struct large_item_vec *vec_largest_by_size_bytes;
+
+ /*
+ * Computing the sum of the number of entries across all trees
+ * is probably not that interesting.
+ */
+ uint64_t sum_entries; /* sum(nr_entries) -- sum across all trees */
+
+ /*
+ * A histogram of the count of trees, the observed size, and
+ * the on-disk size grouped by the number of entries in the tree.
+ */
+ struct obj_hist_bin entry_qbin[QBIN_LEN];
+};
+
+/*
+ * Stats for reachable blobs.
+ */
+struct survey_stats_blobs {
+ struct survey_stats_base_object base;
+
+ /*
+ * Remember the OIDs of the largest n blobs.
+ */
+ struct large_item_vec *vec_largest_by_size_bytes;
+};
+
+struct survey_stats {
+ struct survey_stats_refs refs;
+ struct survey_stats_commits commits;
+ struct survey_stats_trees trees;
+ struct survey_stats_blobs blobs;
+};
+
+static struct survey_stats survey_stats = { 0 };
+
+static void alloc_commit_by_parents(void)
+{
+ static struct large_item_vec_labels json = {
+ .dimension = "largest_commits_by_nr_parents",
+ .item = "nr_parents",
+ };
+ static struct large_item_vec_labels pretty = {
+ .dimension = "Largest Commits by Number of Parents",
+ .item = "Parents",
+ };
+
+ if (survey_opts.show_largest_commits_by_nr_parents)
+ survey_stats.commits.vec_largest_by_nr_parents =
+ alloc_large_item_vec(&json, &pretty,
+ survey_opts.show_largest_commits_by_nr_parents,
+ OBJ_COMMIT);
+}
+
+static void alloc_commit_by_size(void) {
+ static struct large_item_vec_labels json = {
+ .dimension = "largest_commits_by_size_bytes",
+ .item = "size",
+ };
+ static struct large_item_vec_labels pretty = {
+ .dimension = "Largest Commits by Size in Bytes",
+ .item = "Size",
+ };
+
+ if (survey_opts.show_largest_commits_by_size_bytes)
+ survey_stats.commits.vec_largest_by_size_bytes =
+ alloc_large_item_vec(&json, &pretty,
+ survey_opts.show_largest_commits_by_size_bytes,
+ OBJ_COMMIT);
+}
+
+static void alloc_tree_by_entries(void)
+{
+ static struct large_item_vec_labels json = {
+ .dimension = "largest_trees_by_nr_entries",
+ .item = "nr_entries",
+ };
+ static struct large_item_vec_labels pretty = {
+ .dimension = "Largest Trees by Number of Entries",
+ .item = "Entries",
+ };
+
+ if (survey_opts.show_largest_trees_by_nr_entries)
+ survey_stats.trees.vec_largest_by_nr_entries =
+ alloc_large_item_vec(&json, &pretty,
+ survey_opts.show_largest_trees_by_nr_entries,
+ OBJ_TREE);
+}
+
+static void alloc_tree_by_size(void)
+{
+ static struct large_item_vec_labels json = {
+ .dimension = "largest_trees_by_size_bytes",
+ .item = "size",
+ };
+ static struct large_item_vec_labels pretty = {
+ .dimension = "Largest Trees by Size in Bytes",
+ .item = "Size",
+ };
+
+ if (survey_opts.show_largest_trees_by_size_bytes)
+ survey_stats.trees.vec_largest_by_size_bytes =
+ alloc_large_item_vec(&json, &pretty,
+ survey_opts.show_largest_trees_by_size_bytes,
+ OBJ_TREE);
+}
+
+static void alloc_blob_by_size(void)
+{
+ static struct large_item_vec_labels json = {
+ .dimension = "largest_blobs_by_size_bytes",
+ .item = "size",
+ };
+ static struct large_item_vec_labels pretty = {
+ .dimension = "Largest Blobs by Size in Bytes",
+ .item = "Size",
+ };
+
+ if (survey_opts.show_largest_blobs_by_size_bytes)
+ survey_stats.blobs.vec_largest_by_size_bytes =
+ alloc_large_item_vec(&json, &pretty,
+ survey_opts.show_largest_blobs_by_size_bytes,
+ OBJ_BLOB);
+}
+
+static void do_load_refs(struct ref_array *ref_array)
+{
+ struct ref_filter filter = REF_FILTER_INIT;
+ struct ref_sorting *sorting;
+ struct string_list sorting_options = STRING_LIST_INIT_DUP;
+
+ string_list_append(&sorting_options, "objectname");
+ sorting = ref_sorting_options(&sorting_options);
+
+ if (survey_opts.refs.want_detached)
+ strvec_push(&survey_vec_refs_wanted, "HEAD");
+
+ if (survey_opts.refs.want_all_refs) {
+ strvec_push(&survey_vec_refs_wanted, "refs/");
+ } else {
+ if (survey_opts.refs.want_branches)
+ strvec_push(&survey_vec_refs_wanted, "refs/heads/");
+ if (survey_opts.refs.want_tags)
+ strvec_push(&survey_vec_refs_wanted, "refs/tags/");
+ if (survey_opts.refs.want_remotes)
+ strvec_push(&survey_vec_refs_wanted, "refs/remotes/");
+ if (survey_opts.refs.want_other) {
+ strvec_push(&survey_vec_refs_wanted, "refs/notes/");
+ strvec_push(&survey_vec_refs_wanted, "refs/stash/");
+ }
+ }
+
+ filter.name_patterns = survey_vec_refs_wanted.v;
+ filter.ignore_case = 0;
+ filter.match_as_path = 1;
+
+ if (survey_opts.show_progress) {
+ survey_progress_total = 0;
+ survey_progress = start_progress(_("Scanning refs..."), 0);
+ }
+
+ filter_refs(ref_array, &filter, FILTER_REFS_KIND_MASK);
+
+ if (survey_opts.show_progress) {
+ survey_progress_total = ref_array->nr;
+ display_progress(survey_progress, survey_progress_total);
+ }
+
+ ref_array_sort(sorting, ref_array);
+
+ if (survey_opts.show_progress)
+ stop_progress(&survey_progress);
+
+ ref_filter_clear(&filter);
+ ref_sorting_release(sorting);
+}
+
+/*
+ * Populate a "rev_info" with the OIDs of the REFS of interest.
+ * The treewalk will start from all of those starting points
+ * and walk backwards in the DAG to get the set of all reachable
+ * objects from those starting points.
+ */
+static void load_rev_info(struct rev_info *rev_info,
+ struct ref_array *ref_array)
+{
+ unsigned int add_flags = 0;
+ int k;
+
+ for (k = 0; k < ref_array->nr; k++) {
+ struct ref_array_item *p = ref_array->items[k];
+ struct object_id peeled;
+
+ switch (p->kind) {
+ case FILTER_REFS_TAGS:
+ if (!peel_iterated_oid(rev_info->repo, &p->objectname, &peeled))
+ add_pending_oid(rev_info, NULL, &peeled, add_flags);
+ else
+ add_pending_oid(rev_info, NULL, &p->objectname, add_flags);
+ break;
+ case FILTER_REFS_BRANCHES:
+ add_pending_oid(rev_info, NULL, &p->objectname, add_flags);
+ break;
+ case FILTER_REFS_REMOTES:
+ add_pending_oid(rev_info, NULL, &p->objectname, add_flags);
+ break;
+ case FILTER_REFS_OTHERS:
+ /*
+ * This may be a note, stash, or custom namespace branch.
+ */
+ add_pending_oid(rev_info, NULL, &p->objectname, add_flags);
+ break;
+ case FILTER_REFS_DETACHED_HEAD:
+ add_pending_oid(rev_info, NULL, &p->objectname, add_flags);
+ break;
+ default:
+ break;
+ }
+ }
+}
+
+static int fill_in_base_object(struct survey_stats_base_object *base,
+ struct object *object,
+ enum object_type type_expected,
+ unsigned long *p_object_length,
+ off_t *p_disk_sizep)
+{
+ struct object_info oi = OBJECT_INFO_INIT;
+ unsigned oi_flags = OBJECT_INFO_FOR_PREFETCH;
+ unsigned long object_length = 0;
+ off_t disk_sizep = 0;
+ enum object_type type;
+ int hb;
+
+ base->cnt_seen++;
+
+ oi.typep = &type;
+ oi.sizep = &object_length;
+ oi.disk_sizep = &disk_sizep;
+
+ if (oid_object_info_extended(the_repository, &object->oid, &oi, oi_flags) < 0 ||
+ type != type_expected) {
+ base->cnt_missing++;
+ return 1;
+ }
+
+ switch (oi.whence) {
+ case OI_CACHED:
+ base->cnt_cached++;
+ break;
+ case OI_LOOSE:
+ base->cnt_loose++;
+ break;
+ case OI_PACKED:
+ base->cnt_packed++;
+ break;
+ case OI_DBCACHED:
+ base->cnt_dbcached++;
+ break;
+ default:
+ break;
+ }
+
+ base->sum_size += object_length;
+ base->sum_disk_size += disk_sizep;
+
+ hb = hbin(object_length);
+ incr_obj_hist_bin(&base->size_hbin[hb], object_length, disk_sizep);
+
+ if (p_object_length)
+ *p_object_length = object_length;
+ if (p_disk_sizep)
+ *p_disk_sizep = disk_sizep;
+
+ return 0;
+}
+
+/*
+ * Transient OID of the commit currently being visited
+ * during the treewalk. We can use this to create the
+ * [: pair when a notable large file was
+ * created, for example.
+ */
+static struct object_id treewalk_transient_commit_oid;
+
+static void traverse_commit_cb(struct commit *commit, void *data)
+{
+ struct survey_stats_commits *psc = &survey_stats.commits;
+ unsigned long object_length;
+ unsigned k;
+
+ if ((++survey_progress_total % 1000) == 0)
+ display_progress(survey_progress, survey_progress_total);
+
+ oidcpy(&treewalk_transient_commit_oid, &commit->object.oid);
+
+ fill_in_base_object(&psc->base, &commit->object, OBJ_COMMIT, &object_length, NULL);
+
+ k = commit_list_count(commit->parents);
+
+ /*
+ * Send the commit-oid as both the OID and the CONTAINING-COMMIT-OID.
+ * This is somewhat redundant, but lets us later do `git name-rev`
+ * using the containing-oid in a consistent fashion.
+ */
+ maybe_insert_large_item(psc->vec_largest_by_nr_parents, k,
+ &commit->object.oid, NULL,
+ &commit->object.oid);
+ maybe_insert_large_item(psc->vec_largest_by_size_bytes, object_length,
+ &commit->object.oid, NULL,
+ &commit->object.oid);
+
+ if (k >= PBIN_VEC_LEN)
+ k = PBIN_VEC_LEN - 1;
+ psc->parent_cnt_pbin[k]++;
+}
+
+static void traverse_object_cb_tree(struct object *obj, const char *name)
+{
+ struct survey_stats_trees *pst = &survey_stats.trees;
+ unsigned long object_length;
+ off_t disk_sizep;
+ struct tree_desc desc;
+ struct name_entry entry;
+ struct tree *tree;
+ int nr_entries;
+ int qb;
+
+ if (fill_in_base_object(&pst->base, obj, OBJ_TREE, &object_length, &disk_sizep))
+ return;
+
+ tree = lookup_tree(the_repository, &obj->oid);
+ if (!tree)
+ return;
+ init_tree_desc(&desc, &obj->oid, tree->buffer, tree->size);
+ nr_entries = 0;
+ while (tree_entry(&desc, &entry))
+ nr_entries++;
+
+ pst->sum_entries += nr_entries;
+
+ maybe_insert_large_item(pst->vec_largest_by_nr_entries, nr_entries,
+ &obj->oid, name,
+ &treewalk_transient_commit_oid);
+ maybe_insert_large_item(pst->vec_largest_by_size_bytes, object_length,
+ &obj->oid, name,
+ &treewalk_transient_commit_oid);
+
+ qb = qbin(nr_entries);
+ incr_obj_hist_bin(&pst->entry_qbin[qb], object_length, disk_sizep);
+}
+
+static void traverse_object_cb_blob(struct object *obj, const char *name)
+{
+ struct survey_stats_blobs *psb = &survey_stats.blobs;
+ unsigned long object_length;
+
+ fill_in_base_object(&psb->base, obj, OBJ_BLOB, &object_length, NULL);
+
+ maybe_insert_large_item(psb->vec_largest_by_size_bytes, object_length,
+ &obj->oid, name,
+ &treewalk_transient_commit_oid);
+}
+
+static void traverse_object_cb(struct object *obj, const char *name, void *data)
+{
+ if ((++survey_progress_total % 1000) == 0)
+ display_progress(survey_progress, survey_progress_total);
+
+ switch (obj->type) {
+ case OBJ_TREE:
+ traverse_object_cb_tree(obj, name);
+ return;
+ case OBJ_BLOB:
+ traverse_object_cb_blob(obj, name);
+ return;
+ case OBJ_TAG: /* ignore -- counted when loading REFS */
+ case OBJ_COMMIT: /* ignore/bug -- seen in the other callback */
+ default: /* ignore/bug -- unknown type */
+ return;
+ }
+}
+
+/*
+ * Treewalk all of the commits and objects reachable from the
+ * set of refs.
+ */
+static void do_treewalk_reachable(struct ref_array *ref_array)
+{
+ struct rev_info rev_info = REV_INFO_INIT;
+
+ repo_init_revisions(the_repository, &rev_info, NULL);
+ rev_info.tree_objects = 1;
+ rev_info.blob_objects = 1;
+ rev_info.tree_blobs_in_commit_order = 1;
+ load_rev_info(&rev_info, ref_array);
+ if (prepare_revision_walk(&rev_info))
+ die(_("revision walk setup failed"));
+
+ if (survey_opts.show_progress) {
+ survey_progress_total = 0;
+ survey_progress = start_progress(_("Walking reachable objects..."), 0);
+ }
+
+ oidcpy(&treewalk_transient_commit_oid, null_oid());
+ traverse_commit_list(&rev_info,
+ traverse_commit_cb,
+ traverse_object_cb,
+ NULL);
+ oidcpy(&treewalk_transient_commit_oid, null_oid());
+
+ if (survey_opts.show_progress)
+ stop_progress(&survey_progress);
+
+ release_revisions(&rev_info);
+}
+
+/*
+ * If we want this type of ref, increment counters and return 1.
+ */
+static int maybe_count_ref(struct repository *r, struct ref_array_item *p)
+{
+ struct survey_refs_wanted *rw = &survey_opts.refs;
+ struct survey_stats_refs *prs = &survey_stats.refs;
+ struct object_id peeled;
+
+ /*
+ * Classify the ref using the `kind` value. Note that
+ * p->kind was populated by `ref_kind_from_refname()`
+ * based strictly on the refname. This only knows about
+ * the basic stock categories and returns FILTER_REFS_OTHERS
+ * for notes, stashes, and any custom namespaces (like
+ * "refs/prefetch/").
+ */
+ switch (p->kind) {
+ case FILTER_REFS_TAGS:
+ if (rw->want_all_refs || rw->want_tags) {
+ /*
+ * NEEDSWORK: Both types of tags have the same
+ * "refs/tags/" prefix. Do we want to count them
+ * in separate buckets in the refsmap?
+ */
+ strintmap_incr(&prs->refsmap, "refs/tags/", 1);
+
+ if (!peel_iterated_oid(r, &p->objectname, &peeled))
+ prs->cnt_annotated_tags++;
+ else
+ prs->cnt_lightweight_tags++;
+
+ return 1;
+ }
+ return 0;
+
+ case FILTER_REFS_BRANCHES:
+ if (rw->want_all_refs || rw->want_branches) {
+ strintmap_incr(&prs->refsmap, "refs/heads/", 1);
+
+ prs->cnt_branches++;
+ return 1;
+ }
+ return 0;
+
+ case FILTER_REFS_REMOTES:
+ if (rw->want_all_refs || rw->want_remotes) {
+ /*
+ * For the refsmap, group them by the "refs/remotes//".
+ * For example:
+ * "refs/remotes/origin/..."
+ */
+ if (starts_with(p->refname, "refs/remotes/")) {
+ struct strbuf buf = STRBUF_INIT;
+ int begin = strlen("refs/remotes/");
+ size_t j;
+
+ strbuf_addstr(&buf, p->refname);
+ for (j = begin; j < buf.len; j++) {
+ if (buf.buf[j] == '/') {
+ strbuf_setlen(&buf, j+1);
+ break;
+ }
+ }
+ strintmap_incr(&prs->refsmap, buf.buf, 1);
+ strbuf_release(&buf);
+ }
+
+ prs->cnt_remotes++;
+ return 1;
+ }
+ return 0;
+
+ case FILTER_REFS_OTHERS:
+ if (rw->want_all_refs || rw->want_other) {
+ /*
+ * For the refsmap, group them by their "refs//".
+ * For example:
+ * "refs/notes/..."
+ * "refs/stash/..."
+ * "refs//..."
+ */
+ if (starts_with(p->refname, "refs/")) {
+ struct strbuf buf = STRBUF_INIT;
+ int begin = strlen("refs/");
+ size_t j;
+
+ strbuf_addstr(&buf, p->refname);
+ for (j = begin; j < buf.len; j++) {
+ if (buf.buf[j] == '/') {
+ strbuf_setlen(&buf, j+1);
+ break;
+ }
+ }
+ strintmap_incr(&prs->refsmap, buf.buf, 1);
+ strbuf_release(&buf);
+ }
+
+ prs->cnt_other++;
+ return 1;
+ }
+ return 0;
+
+ case FILTER_REFS_DETACHED_HEAD:
+ if (rw->want_all_refs || rw->want_detached) {
+ strintmap_incr(&prs->refsmap, p->refname, 1);
+
+ prs->cnt_detached++;
+ return 1;
+ }
+ return 0;
+
+ default:
+ if (rw->want_all_refs) {
+ strintmap_incr(&prs->refsmap, p->refname, 1); /* probably "HEAD" */
+
+ return 1;
+ }
+ return 0;
+ }
+}
+
+/*
+ * Calculate stats on the set of refs that we found.
+ */
+static void do_calc_stats_refs(struct repository *r, struct ref_array *ref_array)
+{
+ struct survey_stats_refs *prs = &survey_stats.refs;
+ int k;
+
+ strintmap_init(&prs->refsmap, 0);
+
+ for (k = 0; k < ref_array->nr; k++) {
+ struct ref_array_item *p = ref_array->items[k];
+ size_t len;
+
+ if (!maybe_count_ref(r, p))
+ continue;
+
+ prs->cnt_total++;
+
+ /*
+ * SymRefs are somewhat orthogonal to the above
+ * classification (e.g. "HEAD" --> detached
+ * and "refs/remotes/origin/HEAD" --> remote) so
+ * our totals will already include them.
+ */
+ if (p->flag & REF_ISSYMREF)
+ prs->cnt_symref++;
+
+ /*
+ * Where/how is the ref stored in GITDIR.
+ */
+ if (p->flag & REF_ISPACKED)
+ prs->cnt_packed++;
+ else
+ prs->cnt_loose++;
+
+ len = strlen(p->refname);
+
+ if (p->kind == FILTER_REFS_REMOTES) {
+ prs->len_sum_remote_refnames += len;
+ if (len > prs->len_max_remote_refname)
+ prs->len_max_remote_refname = len;
+ } else {
+ prs->len_sum_local_refnames += len;
+ if (len > prs->len_max_local_refname)
+ prs->len_max_local_refname = len;
+ }
+ }
+}
+
+static void do_lookup_name_rev(void)
+{
+ /*
+ * `git name-rev` can be very expensive when there are lots of
+ * refs, so make it optional.
+ */
+ if (!survey_opts.show_name_rev)
+ return;
+
+ if (survey_opts.show_progress) {
+ survey_progress_total = 0;
+ survey_progress = start_progress(_("Resolving name-revs..."), 0);
+ }
+
+ large_item_vec_lookup_name_rev(survey_stats.commits.vec_largest_by_nr_parents);
+ large_item_vec_lookup_name_rev(survey_stats.commits.vec_largest_by_size_bytes);
+
+ large_item_vec_lookup_name_rev(survey_stats.trees.vec_largest_by_nr_entries);
+ large_item_vec_lookup_name_rev(survey_stats.trees.vec_largest_by_size_bytes);
+
+ large_item_vec_lookup_name_rev(survey_stats.blobs.vec_largest_by_size_bytes);
+
+ if (survey_opts.show_progress)
+ stop_progress(&survey_progress);
+}
+
+/*
+ * The REFS phase:
+ *
+ * Load the set of requested refs and assess them for scalablity problems.
+ * Use that set to start a treewalk to all reachable objects and assess
+ * them.
+ *
+ * This data will give us insights into the repository itself (the number
+ * of refs, the size and shape of the DAG, the number and size of the
+ * objects).
+ *
+ * Theoretically, this data is independent of the on-disk representation
+ * (e.g. independent of packing concerns).
+ */
+static void survey_phase_refs(struct repository *r)
+{
+ struct ref_array ref_array = { 0 };
+
+ trace2_region_enter("survey", "phase/refs", the_repository);
+ do_load_refs(&ref_array);
+ trace2_region_leave("survey", "phase/refs", the_repository);
+
+ trace2_region_enter("survey", "phase/treewalk", the_repository);
+ do_treewalk_reachable(&ref_array);
+ trace2_region_leave("survey", "phase/treewalk", the_repository);
+
+ trace2_region_enter("survey", "phase/calcstats", the_repository);
+ do_calc_stats_refs(r, &ref_array);
+ trace2_region_leave("survey", "phase/calcstats", the_repository);
+
+ if (survey_opts.show_name_rev) {
+ trace2_region_enter("survey", "phase/namerev", the_repository);
+ do_lookup_name_rev();
+ trace2_region_enter("survey", "phase/namerev", the_repository);
+ }
+
+ ref_array_clear(&ref_array);
+}
+
+static void json_refs_section(struct json_writer *jw_top, int pretty, int want_trace2)
+{
+ struct survey_stats_refs *prs = &survey_stats.refs;
+ struct json_writer jw_refs = JSON_WRITER_INIT;
+ int k;
+
+ jw_object_begin(&jw_refs, pretty);
+ {
+ jw_object_intmax(&jw_refs, "count", prs->cnt_total);
+
+ jw_object_inline_begin_object(&jw_refs, "count_by_type");
+ {
+ if (survey_opts.refs.want_branches)
+ jw_object_intmax(&jw_refs, "branches", prs->cnt_branches);
+ if (survey_opts.refs.want_tags) {
+ jw_object_intmax(&jw_refs, "lightweight_tags", prs->cnt_lightweight_tags);
+ jw_object_intmax(&jw_refs, "annotated_tags", prs->cnt_annotated_tags);
+ }
+ if (survey_opts.refs.want_remotes)
+ jw_object_intmax(&jw_refs, "remotes", prs->cnt_remotes);
+ if (survey_opts.refs.want_detached)
+ jw_object_intmax(&jw_refs, "detached", prs->cnt_detached);
+ if (survey_opts.refs.want_other)
+ jw_object_intmax(&jw_refs, "other", prs->cnt_other);
+
+ /*
+ * SymRefs are somewhat orthogonal to
+ * the above classification
+ * (e.g. "HEAD" --> detached and
+ * "refs/remotes/origin/HEAD" -->
+ * remote) so the above classified
+ * counts will already include them,
+ * but it is less confusing to display
+ * them here than to create a whole
+ * new section.
+ */
+ if (prs->cnt_symref)
+ jw_object_intmax(&jw_refs, "symrefs", prs->cnt_symref);
+ }
+ jw_end(&jw_refs);
+
+ jw_object_inline_begin_object(&jw_refs, "count_by_storage");
+ {
+ jw_object_intmax(&jw_refs, "loose_refs", prs->cnt_loose);
+ jw_object_intmax(&jw_refs, "packed_refs", prs->cnt_packed);
+ }
+ jw_end(&jw_refs);
+
+ jw_object_inline_begin_object(&jw_refs, "refname_length");
+ {
+ if (prs->len_sum_local_refnames) {
+ jw_object_intmax(&jw_refs, "max_local", prs->len_max_local_refname);
+ jw_object_intmax(&jw_refs, "sum_local", prs->len_sum_local_refnames);
+ }
+ if (prs->len_sum_remote_refnames) {
+ jw_object_intmax(&jw_refs, "max_remote", prs->len_max_remote_refname);
+ jw_object_intmax(&jw_refs, "sum_remote", prs->len_sum_remote_refnames);
+ }
+ }
+ jw_end(&jw_refs);
+
+ jw_object_inline_begin_array(&jw_refs, "requested");
+ {
+ for (k = 0; k < survey_vec_refs_wanted.nr; k++)
+ jw_array_string(&jw_refs, survey_vec_refs_wanted.v[k]);
+ }
+ jw_end(&jw_refs);
+
+ jw_object_inline_begin_array(&jw_refs, "count_by_class");
+ {
+ struct hashmap_iter iter;
+ struct strmap_entry *entry;
+
+ strintmap_for_each_entry(&prs->refsmap, &iter, entry) {
+ const char *key = entry->key;
+ intptr_t count = (intptr_t)entry->value;
+ int value = count;
+ jw_array_inline_begin_object(&jw_refs);
+ {
+ jw_object_string(&jw_refs, "class", key);
+ jw_object_intmax(&jw_refs, "count", value);
+ }
+ jw_end(&jw_refs);
+ }
+ }
+ jw_end(&jw_refs);
+ }
+ jw_end(&jw_refs);
+
+ if (jw_top)
+ jw_object_sub_jw(jw_top, "refs", &jw_refs);
+
+ if (want_trace2)
+ trace2_data_json("survey", the_repository, "refs", &jw_refs);
+
+ jw_release(&jw_refs);
+}
+
+#define JW_OBJ_INT_NZ(jw, key, value) do { if (value) jw_object_intmax((jw), (key), (value)); } while (0)
+
+static void write_qbin_json(struct json_writer *jw, const char *label,
+ struct obj_hist_bin qbin[QBIN_LEN])
+{
+ struct strbuf buf = STRBUF_INIT;
+ uint32_t lower = 0;
+ uint32_t upper = QBIN_MASK;
+ int k;
+
+ jw_object_inline_begin_object(jw, label);
+ {
+ for (k = 0; k < QBIN_LEN; k++) {
+ struct obj_hist_bin *p = &qbin[k];
+ uint32_t lower_k = lower;
+ uint32_t upper_k = upper;
+
+ lower = upper+1;
+ upper = (upper << QBIN_SHIFT) + QBIN_MASK;
+
+ if (!p->cnt_seen)
+ continue;
+
+ strbuf_reset(&buf);
+ strbuf_addf(&buf, "Q%02d", k);
+ jw_object_inline_begin_object(jw, buf.buf);
+ {
+ jw_object_intmax(jw, "count", p->cnt_seen);
+ jw_object_intmax(jw, "sum_size", p->sum_size);
+ jw_object_intmax(jw, "sum_disk_size", p->sum_disk_size);
+
+ /* maybe only include these in verbose mode */
+ jw_object_intmax(jw, "qbin_lower", lower_k);
+ jw_object_intmax(jw, "qbin_upper", upper_k);
+ }
+ jw_end(jw);
+ }
+ }
+ jw_end(jw);
+
+ strbuf_release(&buf);
+}
+
+static void write_hbin_json(struct json_writer *jw, const char *label,
+ struct obj_hist_bin hbin[HBIN_LEN])
+{
+ struct strbuf buf = STRBUF_INIT;
+ uint32_t lower = 0;
+ uint32_t upper = HBIN_MASK;
+ int k;
+
+ jw_object_inline_begin_object(jw, label);
+ {
+ for (k = 0; k < HBIN_LEN; k++) {
+ struct obj_hist_bin *p = &hbin[k];
+ uint32_t lower_k = lower;
+ uint32_t upper_k = upper;
+
+ lower = upper+1;
+ upper = (upper << HBIN_SHIFT) + HBIN_MASK;
+
+ if (!p->cnt_seen)
+ continue;
+
+ strbuf_reset(&buf);
+ strbuf_addf(&buf, "H%d", k);
+ jw_object_inline_begin_object(jw, buf.buf);
+ {
+ jw_object_intmax(jw, "count", p->cnt_seen);
+ jw_object_intmax(jw, "sum_size", p->sum_size);
+ jw_object_intmax(jw, "sum_disk_size", p->sum_disk_size);
+
+ /* maybe only include these in verbose mode */
+ jw_object_intmax(jw, "hbin_lower", lower_k);
+ jw_object_intmax(jw, "hbin_upper", upper_k);
+ }
+ jw_end(jw);
+ }
+ }
+ jw_end(jw);
+
+ strbuf_release(&buf);
+}
+
+static void write_base_object_json(struct json_writer *jw,
+ struct survey_stats_base_object *base)
+{
+ jw_object_intmax(jw, "count", base->cnt_seen);
+
+ jw_object_intmax(jw, "sum_size", base->sum_size);
+ jw_object_intmax(jw, "sum_disk_size", base->sum_disk_size);
+
+ jw_object_inline_begin_object(jw, "count_by_whence");
+ {
+ /*
+ * Missing is not technically a "whence" value, but
+ * we don't need to clutter up the results with that
+ * distinction.
+ */
+ JW_OBJ_INT_NZ(jw, "missing", base->cnt_missing);
+
+ JW_OBJ_INT_NZ(jw, "cached", base->cnt_cached);
+ JW_OBJ_INT_NZ(jw, "loose", base->cnt_loose);
+ JW_OBJ_INT_NZ(jw, "packed", base->cnt_packed);
+ JW_OBJ_INT_NZ(jw, "dbcached", base->cnt_dbcached);
+ }
+ jw_end(jw);
+
+ write_hbin_json(jw, "dist_by_size", base->size_hbin);
+}
+
+static void write_large_item_vec_json(struct json_writer *jw,
+ struct large_item_vec *vec)
+{
+ if (!vec || !vec->nr_items)
+ return;
+
+ jw_object_inline_begin_array(jw, vec->labels_json->dimension);
+ {
+ int k;
+
+ for (k = 0; k < vec->nr_items; k++) {
+ struct large_item *pk = &vec->items[k];
+ if (is_null_oid(&pk->oid))
+ break;
+
+ jw_array_inline_begin_object(jw);
+ {
+ jw_object_intmax(jw, vec->labels_json->item, pk->size);
+ jw_object_string(jw, "oid", oid_to_hex(&pk->oid));
+ if (pk->name->len)
+ jw_object_string(jw, "name", pk->name->buf);
+ if (!is_null_oid(&pk->containing_commit_oid))
+ jw_object_string(jw, "commit_oid",
+ oid_to_hex(&pk->containing_commit_oid));
+ if (survey_opts.show_name_rev &&
+ pk->name_rev &&
+ pk->name_rev->len)
+ jw_object_string(jw, "name_rev",
+ pk->name_rev->buf);
+ }
+ jw_end(jw);
+ }
+ }
+ jw_end(jw);
+}
+
+static void json_commits_section(struct json_writer *jw_top, int pretty, int want_trace2)
+{
+ struct survey_stats_commits *psc = &survey_stats.commits;
+ struct json_writer jw_commits = JSON_WRITER_INIT;
+
+ jw_object_begin(&jw_commits, pretty);
+ {
+ write_base_object_json(&jw_commits, &psc->base);
+
+ write_large_item_vec_json(&jw_commits, psc->vec_largest_by_nr_parents);
+ write_large_item_vec_json(&jw_commits, psc->vec_largest_by_size_bytes);
+
+ jw_object_inline_begin_object(&jw_commits, "count_by_nr_parents");
+ {
+ struct strbuf parent_key = STRBUF_INIT;
+ int k;
+
+ for (k = 0; k < PBIN_VEC_LEN; k++)
+ if (psc->parent_cnt_pbin[k]) {
+ strbuf_reset(&parent_key);
+ strbuf_addf(&parent_key, "P%02d", k);
+ jw_object_intmax(&jw_commits, parent_key.buf, psc->parent_cnt_pbin[k]);
+ }
+
+ strbuf_release(&parent_key);
+ }
+ jw_end(&jw_commits);
+ }
+ jw_end(&jw_commits);
+
+ if (jw_top)
+ jw_object_sub_jw(jw_top, "commits", &jw_commits);
+
+ if (want_trace2)
+ trace2_data_json("survey", the_repository, "commits", &jw_commits);
+
+ jw_release(&jw_commits);
+}
+
+static void json_trees_section(struct json_writer *jw_top, int pretty, int want_trace2)
+{
+ struct survey_stats_trees *pst = &survey_stats.trees;
+ struct json_writer jw_trees = JSON_WRITER_INIT;
+
+ jw_object_begin(&jw_trees, pretty);
+ {
+ write_base_object_json(&jw_trees, &pst->base);
+
+ jw_object_intmax(&jw_trees, "sum_entries", pst->sum_entries);
+
+ write_large_item_vec_json(&jw_trees, pst->vec_largest_by_nr_entries);
+ write_large_item_vec_json(&jw_trees, pst->vec_largest_by_size_bytes);
+
+ write_qbin_json(&jw_trees, "dist_by_nr_entries", pst->entry_qbin);
+ }
+ jw_end(&jw_trees);
+
+ if (jw_top)
+ jw_object_sub_jw(jw_top, "trees", &jw_trees);
+
+ if (want_trace2)
+ trace2_data_json("survey", the_repository, "trees", &jw_trees);
+
+ jw_release(&jw_trees);
+}
+
+static void json_blobs_section(struct json_writer *jw_top, int pretty, int want_trace2)
+{
+ struct survey_stats_blobs *psb = &survey_stats.blobs;
+ struct json_writer jw_blobs = JSON_WRITER_INIT;
+
+ jw_object_begin(&jw_blobs, pretty);
+ {
+ write_base_object_json(&jw_blobs, &psb->base);
+
+ write_large_item_vec_json(&jw_blobs, psb->vec_largest_by_size_bytes);
+ }
+ jw_end(&jw_blobs);
+
+ if (jw_top)
+ jw_object_sub_jw(jw_top, "blobs", &jw_blobs);
+
+ if (want_trace2)
+ trace2_data_json("survey", the_repository, "blobs", &jw_blobs);
+
+ jw_release(&jw_blobs);
+}
+
+static void survey_print_json(void)
+{
+ struct json_writer jw_top = JSON_WRITER_INIT;
+ int pretty = 1;
+
+ jw_object_begin(&jw_top, pretty);
+ {
+ json_refs_section(&jw_top, pretty, 0);
+ json_commits_section(&jw_top, pretty, 0);
+ json_trees_section(&jw_top, pretty, 0);
+ json_blobs_section(&jw_top, pretty, 0);
+ }
+ jw_end(&jw_top);
+
+ printf("%s\n", jw_top.json.buf);
+
+ jw_release(&jw_top);
+}
+
+static void survey_emit_trace2(void)
+{
+ if (!trace2_is_enabled())
+ return;
+
+ json_refs_section(NULL, 0, 1);
+ json_commits_section(NULL, 0, 1);
+ json_trees_section(NULL, 0, 1);
+ json_blobs_section(NULL, 0, 1);
+}
+
+static void fmt_txt_line(struct strbuf *buf, int indent, const char *txt)
+{
+ if (indent)
+ strbuf_addchars(buf, ' ', indent);
+
+ strbuf_addstr(buf, txt);
+
+ strbuf_addch(buf, '\n');
+}
+
+static void fmt_txt_pair_ui64(struct strbuf *buf,
+ int indent,
+ const char *label,
+ uint64_t value)
+{
+ int column0 = 62;
+
+ if (indent)
+ strbuf_addchars(buf, ' ', indent);
+
+ strbuf_addf(buf, "%-*s : %14"PRIu64,
+ column0 - indent, label,
+ value);
+
+ strbuf_addch(buf, '\n');
+}
+
+static void fmt_size_tbl_caption(struct strbuf *buf,
+ int indent,
+ const char *caption)
+{
+ strbuf_addch(buf, '\n');
+ fmt_txt_line(buf, indent, caption);
+}
+
+static void fmt_size_tbl_hdr(struct strbuf *buf,
+ int indent,
+ const char *bucket_hdr,
+ const char *count_hdr,
+ const char *size_hdr,
+ const char *disk_size_hdr)
+{
+ int column0 = 28;
+
+ if (indent)
+ strbuf_addchars(buf, ' ', indent);
+
+ strbuf_addf(buf, "%-*s | %14s | %14s | %14s",
+ column0 - indent, bucket_hdr,
+ count_hdr, size_hdr, disk_size_hdr);
+
+ strbuf_addch(buf, '\n');
+}
+
+static void fmt_size_tbl_hr(struct strbuf *buf,
+ int indent)
+{
+ int column0 = 28;
+
+ if (indent)
+ strbuf_addchars(buf, ' ', indent);
+
+ strbuf_addchars(buf, '-', column0 - indent);
+ strbuf_addstr(buf, "-+-");
+ strbuf_addchars(buf, '-', 14);
+ strbuf_addstr(buf, "-+-");
+ strbuf_addchars(buf, '-', 14);
+ strbuf_addstr(buf, "-+-");
+ strbuf_addchars(buf, '-', 14);
+
+ strbuf_addch(buf, '\n');
+}
+
+static void fmt_size_tbl_row(struct strbuf *buf,
+ int indent,
+ const char *bucket,
+ uint64_t count,
+ uint64_t size,
+ uint64_t disk_size)
+{
+ int column0 = 28;
+
+ if (indent)
+ strbuf_addchars(buf, ' ', indent);
+
+ strbuf_addf(buf, "%-*s | %14"PRIu64" | %14"PRIu64" | %14"PRIu64,
+ column0 - indent, bucket, count, size, disk_size);
+
+ strbuf_addch(buf, '\n');
+}
+
+static void fmt_qbin(struct strbuf *buf,
+ int indent, const char *title_caption,
+ const char *bucket_hdr,
+ struct obj_hist_bin qbin[QBIN_LEN])
+{
+ struct strbuf bucket = STRBUF_INIT;
+ uint64_t lower = 0;
+ uint64_t upper = QBIN_MASK;
+ int k;
+
+ fmt_size_tbl_caption(buf, indent, title_caption);
+ fmt_size_tbl_hr(buf, indent);
+ fmt_size_tbl_hdr(buf, indent, bucket_hdr, "Count", "Size", "Disk Size");
+ fmt_size_tbl_hr(buf, indent);
+
+ for (k = 0; k < QBIN_LEN; k++) {
+ struct obj_hist_bin *p = &qbin[k];
+ uint64_t lower_k = lower;
+ uint64_t upper_k = upper;
+
+ lower = upper+1;
+ upper = (upper << QBIN_SHIFT) + QBIN_MASK;
+
+ if (!p->cnt_seen)
+ continue;
+
+ strbuf_reset(&bucket);
+ strbuf_addf(&bucket, "%"PRIu64"..%"PRIu64, lower_k, upper_k);
+
+ fmt_size_tbl_row(buf, indent, bucket.buf,
+ p->cnt_seen, p->sum_size, p->sum_disk_size);
+ }
+ fmt_size_tbl_hr(buf, indent);
+
+ strbuf_release(&bucket);
+}
+
+static void fmt_hbin(struct strbuf *buf,
+ int indent, const char *title_caption,
+ const char *bucket_hdr,
+ struct obj_hist_bin hbin[HBIN_LEN])
+{
+ struct strbuf bucket = STRBUF_INIT;
+ uint64_t lower = 0;
+ uint64_t upper = HBIN_MASK;
+ int k;
+
+ fmt_size_tbl_caption(buf, indent, title_caption);
+ fmt_size_tbl_hr(buf, indent);
+ fmt_size_tbl_hdr(buf, indent, bucket_hdr, "Count", "Size", "Disk Size");
+ fmt_size_tbl_hr(buf, indent);
+
+ for (k = 0; k < HBIN_LEN; k++) {
+ struct obj_hist_bin *p = &hbin[k];
+ uint64_t lower_k = lower;
+ uint64_t upper_k = upper;
+
+ lower = upper+1;
+ upper = (upper << HBIN_SHIFT) + HBIN_MASK;
+
+ if (!p->cnt_seen)
+ continue;
+
+ strbuf_reset(&bucket);
+ strbuf_addf(&bucket, "%"PRIu64"..%"PRIu64, lower_k, upper_k);
+
+ fmt_size_tbl_row(buf, indent, bucket.buf,
+ p->cnt_seen, p->sum_size, p->sum_disk_size);
+ }
+ fmt_size_tbl_hr(buf, indent);
+
+ strbuf_release(&bucket);
+}
+
+static void fmt_pbin_hdr(struct strbuf *buf,
+ int indent,
+ const char *bucket,
+ const char *count)
+{
+ int column0 = 28;
+
+ if (indent)
+ strbuf_addchars(buf, ' ', indent);
+
+ strbuf_addf(buf, "%-*s | %14s",
+ column0 - indent, bucket,
+ count);
+
+ strbuf_addch(buf, '\n');
+}
+
+static void fmt_pbin_hr(struct strbuf *buf,
+ int indent)
+{
+ int column0 = 28;
+
+ if (indent)
+ strbuf_addchars(buf, ' ', indent);
+
+ strbuf_addchars(buf, '-', column0 - indent);
+ strbuf_addstr(buf, "-+-");
+ strbuf_addchars(buf, '-', 14);
+
+ strbuf_addch(buf, '\n');
+}
+
+static void fmt_pbin_row(struct strbuf *buf,
+ int indent,
+ int nr,
+ int count)
+{
+ struct strbuf bucket = STRBUF_INIT;
+ int column0 = 28;
+
+ if (indent)
+ strbuf_addchars(buf, ' ', indent);
+
+ strbuf_addf(&bucket, "%2d", nr);
+ strbuf_addf(buf, "%-*s | %14d",
+ column0 - indent, bucket.buf,
+ count);
+
+ strbuf_addch(buf, '\n');
+ strbuf_release(&bucket);
+}
+
+static void fmt_base_object(struct strbuf *buf,
+ int indent,
+ struct survey_stats_base_object *base)
+{
+ int indent1 = indent + 4;
+
+ fmt_txt_pair_ui64(buf, indent, "Total Count", base->cnt_seen);
+
+ strbuf_addch(buf, '\n');
+ fmt_txt_line(buf, indent, "Count by Storage Location");
+ if (base->cnt_missing)
+ fmt_txt_pair_ui64(buf, indent1, "Missing", base->cnt_missing);
+ if (base->cnt_cached)
+ fmt_txt_pair_ui64(buf, indent1, "Cached", base->cnt_cached);
+ if (base->cnt_loose)
+ fmt_txt_pair_ui64(buf, indent1, "Loose", base->cnt_loose);
+ if (base->cnt_packed)
+ fmt_txt_pair_ui64(buf, indent1, "Packed", base->cnt_packed);
+ if (base->cnt_dbcached)
+ fmt_txt_pair_ui64(buf, indent1, "DBCached", base->cnt_dbcached);
+
+ strbuf_addch(buf, '\n');
+ fmt_txt_pair_ui64(buf, indent, "Total Size in Bytes", base->sum_size);
+ fmt_txt_pair_ui64(buf, indent, "Total Disk Size in Bytes", base->sum_disk_size);
+
+ fmt_hbin(buf, indent, "Histogram by Size in Bytes", "Byte Range", base->size_hbin);
+}
+
+static void fmt_large_item_hdr(struct strbuf *buf,
+ int indent,
+ int name_length,
+ int name_rev_length,
+ const char *item_hdr_label)
+{
+ int column0 = the_hash_algo->hexsz;
+
+ if (indent)
+ strbuf_addchars(buf, ' ', indent);
+
+ strbuf_addf(buf, "%-*s | %14s", column0, "OID", item_hdr_label);
+ if (name_length)
+ strbuf_addf(buf, " | %-*s", name_length, "Name");
+ if (name_rev_length)
+ strbuf_addf(buf, " | %-*s", name_rev_length, "Commit / Name Rev");
+
+ strbuf_addch(buf, '\n');
+}
+
+static void fmt_large_item_hr(struct strbuf *buf,
+ int indent,
+ int name_length,
+ int name_rev_length)
+{
+ int column0 = the_hash_algo->hexsz;
+
+ if (indent)
+ strbuf_addchars(buf, ' ', indent);
+
+ strbuf_addchars(buf, '-', column0);
+ strbuf_addstr(buf, "-+-");
+ strbuf_addchars(buf, '-', 14);
+ if (name_length) {
+ strbuf_addstr(buf, "-+-");
+ strbuf_addchars(buf, '-', name_length);
+ }
+ if (name_rev_length) {
+ strbuf_addstr(buf, "-+-");
+ strbuf_addchars(buf, '-', name_rev_length);
+ }
+
+ strbuf_addch(buf, '\n');
+}
+
+static void fmt_large_item_row(struct strbuf *buf,
+ int indent,
+ int name_length,
+ int name_rev_length,
+ struct large_item *pitem)
+{
+ int column0 = the_hash_algo->hexsz;
+
+ if (indent)
+ strbuf_addchars(buf, ' ', indent);
+
+ strbuf_addf(buf, "%-*s | %14"PRIu64,
+ column0, oid_to_hex(&pitem->oid),
+ pitem->size);
+ if (name_length)
+ strbuf_addf(buf, " | %-*s", name_length,
+ (pitem->name ? pitem->name->buf: ""));
+ if (name_rev_length)
+ strbuf_addf(buf, " | %-*s", name_rev_length,
+ ((pitem->name_rev)
+ ? pitem->name_rev->buf
+ : oid_to_hex(&pitem->containing_commit_oid)));
+
+ strbuf_addch(buf, '\n');
+}
+
+static void fmt_large_item_vec(struct strbuf *buf,
+ int indent,
+ struct large_item_vec *pvec)
+{
+ int name_length = 0;
+ int name_rev_length = 0;
+ int k;
+
+ if (pvec->type != OBJ_COMMIT) {
+ /* Add "Name" column for trees and blobs. This is relative pathname. */
+ for (k = 0; k < pvec->nr_items; k++)
+ if (pvec->items[k].name && pvec->items[k].name->len > name_length)
+ name_length = pvec->items[k].name->len;
+ if (name_length)
+ if (name_length < 4) /* strlen("Name") */
+ name_length = 4;
+ }
+
+ if (survey_opts.show_name_rev) {
+ name_rev_length = 17; /* strlen("Commit / Name Rev") */
+ for (k = 0; k < pvec->nr_items; k++) {
+ struct large_item *pk = &pvec->items[k];
+ if (pk->name_rev && pk->name_rev->len > name_rev_length)
+ name_rev_length = pk->name_rev->len;
+ }
+ } else if (pvec->type != OBJ_COMMIT) {
+ /* for trees and blobs, just show containing commit OID */
+ name_rev_length = the_hash_algo->hexsz;
+ }
+
+ strbuf_addch(buf, '\n');
+ fmt_txt_line(buf, indent, pvec->labels_pretty->dimension);
+ fmt_large_item_hr(buf, indent, name_length, name_rev_length);
+ fmt_large_item_hdr(buf, indent, name_length, name_rev_length, pvec->labels_pretty->item);
+ fmt_large_item_hr(buf, indent, name_length, name_rev_length);
+
+ for (k = 0; k < pvec->nr_items; k++) {
+ struct large_item *pk = &pvec->items[k];
+ if (is_null_oid(&pk->oid))
+ break;
+
+ fmt_large_item_row(buf, indent, name_length, name_rev_length, pk);
+ }
+
+ fmt_large_item_hr(buf, indent, name_length, name_rev_length);
+}
+
+static void pretty_print_survey_hdr(void)
+{
+ struct strbuf buf = STRBUF_INIT;
+ int indent = 0;
+ int k;
+
+ const char *intro[] = {
+ "",
+ "===============================================================================",
+ "Git Survey Results",
+ "===============================================================================",
+ "",
+ NULL
+ };
+
+ k = 0;
+ while (intro[k])
+ fmt_txt_line(&buf, indent, intro[k++]);
+
+ /*
+ * NEEDSWORK: Consider adding information about the repo pathname,
+ * the date, command line args, git version, etc.
+ */
+
+ fwrite(buf.buf, 1, buf.len, stdout);
+ strbuf_release(&buf);
+}
+
+static void pretty_print_overview(int indent)
+{
+ struct survey_stats_refs *prs = &survey_stats.refs;
+ struct survey_stats_commits *psc = &survey_stats.commits;
+ struct survey_stats_trees *pst = &survey_stats.trees;
+ struct survey_stats_blobs *psb = &survey_stats.blobs;
+ struct strbuf buf = STRBUF_INIT;
+ int indent1 = indent + 4;
+ int indent2 = indent + 8;
+ int k;
+
+ const char *intro[] = {
+ "",
+ "OVERVIEW",
+ "-------------------------------------------------------------------------------",
+ "",
+ NULL
+ };
+
+ k = 0;
+ while (intro[k])
+ fmt_txt_line(&buf, indent, intro[k++]);
+
+ fmt_txt_pair_ui64(&buf, indent1, "Total Number of Refs", prs->cnt_total);
+
+ fmt_size_tbl_caption(&buf, indent1, "Overview by Object Type");
+
+ fmt_size_tbl_hr(&buf, indent1);
+ fmt_size_tbl_hdr(&buf, indent1, "Type", "Count", "Size", "Disk Size");
+ fmt_size_tbl_hr(&buf, indent1);
+
+ fmt_size_tbl_row(&buf, indent2, "Commits", psc->base.cnt_seen, psc->base.sum_size, psc->base.sum_disk_size);
+ fmt_size_tbl_row(&buf, indent2, "Trees", pst->base.cnt_seen, pst->base.sum_size, pst->base.sum_disk_size);
+ fmt_size_tbl_row(&buf, indent2, "Blobs", psb->base.cnt_seen, psb->base.sum_size, psb->base.sum_disk_size);
+
+ fmt_size_tbl_hr(&buf, indent1);
+ fmt_size_tbl_row(&buf, indent1, "Total",
+ psc->base.cnt_seen + pst->base.cnt_seen + psb->base.cnt_seen,
+ psc->base.sum_size + pst->base.sum_size + psb->base.sum_size,
+ psc->base.sum_disk_size + pst->base.sum_disk_size + psb->base.sum_disk_size);
+ fmt_size_tbl_hr(&buf, indent1);
+
+ strbuf_addch(&buf, '\n');
+ fwrite(buf.buf, 1, buf.len, stdout);
+ strbuf_release(&buf);
+}
+
+/*
+ * Pretty print information on the set of REFS that we examined.
+ */
+static void pretty_print_refs(int indent)
+{
+ struct survey_refs_wanted *prw = &survey_opts.refs;
+ struct survey_stats_refs *prs = &survey_stats.refs;
+ struct strbuf buf = STRBUF_INIT;
+ int indent1 = indent + 4;
+ int indent2 = indent + 8;
+ int indent3 = indent + 12;
+ int k;
+
+ const char *intro[] = {
+ "",
+ "REFS",
+ "-------------------------------------------------------------------------------",
+ "",
+ NULL
+ };
+
+ k = 0;
+ while (intro[k])
+ fmt_txt_line(&buf, indent, intro[k++]);
+
+ fmt_txt_pair_ui64(&buf, indent1, "Total Number of Refs", prs->cnt_total);
+
+ strbuf_addch(&buf, '\n');
+ fmt_txt_line(&buf, indent1, "Reference Count by Type");
+
+ if (prw->want_remotes && prs->cnt_remotes)
+ fmt_txt_pair_ui64(&buf, indent2, "Remote Tracking Branches", prs->cnt_remotes);
+
+ if (prw->want_branches && prs->cnt_branches)
+ fmt_txt_pair_ui64(&buf, indent2, "Branches", prs->cnt_branches);
+ if (prw->want_tags && prs->cnt_lightweight_tags)
+ fmt_txt_pair_ui64(&buf, indent2, "Tags (Lightweight)", prs->cnt_lightweight_tags);
+ if (prw->want_tags && prs->cnt_annotated_tags)
+ fmt_txt_pair_ui64(&buf, indent2, "Tags (Annotated)", prs->cnt_annotated_tags);
+ if (prw->want_detached && prs->cnt_detached)
+ fmt_txt_pair_ui64(&buf, indent2, "Detached", prs->cnt_detached);
+ if (prw->want_other && prs->cnt_other)
+ fmt_txt_pair_ui64(&buf, indent2, "Other (Notes and Stashes)", prs->cnt_other);
+
+ if (prs->cnt_symref)
+ fmt_txt_pair_ui64(&buf, indent2, "Symbolic Refs (like 'HEAD')", prs->cnt_symref);
+
+ strbuf_addch(&buf, '\n');
+ fmt_txt_pair_ui64(&buf, indent1, "Reference Count by Class", strintmap_get_size(&prs->refsmap));
+ {
+ struct hashmap_iter iter;
+ struct strmap_entry *entry;
+
+ strintmap_for_each_entry(&prs->refsmap, &iter, entry) {
+ const char *key = entry->key;
+ intptr_t count = (intptr_t)entry->value;
+ int value = count;
+
+ fmt_txt_pair_ui64(&buf, indent2, key, value);
+ }
+ }
+
+ strbuf_addch(&buf, '\n');
+ fmt_txt_line(&buf, indent1, "Reference Count by Storage Location");
+ fmt_txt_pair_ui64(&buf, indent2, "Loose", prs->cnt_loose);
+ fmt_txt_pair_ui64(&buf, indent2, "Packed", prs->cnt_packed);
+
+ strbuf_addch(&buf, '\n');
+ fmt_txt_line(&buf, indent1, "String Length of Refnames");
+ if (prs->len_sum_remote_refnames) {
+ fmt_txt_line(&buf, indent2, "Remote Refs");
+ fmt_txt_pair_ui64(&buf, indent3, "Max", prs->len_max_remote_refname);
+ fmt_txt_pair_ui64(&buf, indent3, "Sum", prs->len_sum_remote_refnames);
+ }
+ if (prs->len_sum_local_refnames) {
+ fmt_txt_line(&buf, indent2, "Local Refs");
+ fmt_txt_pair_ui64(&buf, indent3, "Max", prs->len_max_local_refname);
+ fmt_txt_pair_ui64(&buf, indent3, "Sum", prs->len_sum_local_refnames);
+ }
+
+ strbuf_addch(&buf, '\n');
+ fwrite(buf.buf, 1, buf.len, stdout);
+ strbuf_release(&buf);
+}
+
+static void pretty_print_commits(int indent)
+{
+ struct survey_stats_commits *psc = &survey_stats.commits;
+ struct survey_stats_base_object *base = &psc->base;
+ struct strbuf buf = STRBUF_INIT;
+ int indent1 = indent + 4;
+ int k;
+
+ const char *intro[] = {
+ "",
+ "COMMITS",
+ "-------------------------------------------------------------------------------",
+ "",
+ NULL
+ };
+
+ k = 0;
+ while (intro[k])
+ fmt_txt_line(&buf, indent, intro[k++]);
+
+ fmt_base_object(&buf, indent1, base);
+
+ fmt_large_item_vec(&buf, indent1, psc->vec_largest_by_size_bytes);
+
+ strbuf_addch(&buf, '\n');
+ fmt_txt_line(&buf, indent1, "Histogram by Number of Parents");
+ fmt_pbin_hr(&buf, indent1);
+ fmt_pbin_hdr(&buf, indent1, "Parents", "Count");
+ fmt_pbin_hr(&buf, indent1);
+ for (k = 0; k < PBIN_VEC_LEN; k++)
+ if (psc->parent_cnt_pbin[k])
+ fmt_pbin_row(&buf, indent1, k, psc->parent_cnt_pbin[k]);
+ fmt_pbin_hr(&buf, indent1);
+
+ fmt_large_item_vec(&buf, indent1, psc->vec_largest_by_nr_parents);
+
+ strbuf_addch(&buf, '\n');
+ fwrite(buf.buf, 1, buf.len, stdout);
+ strbuf_release(&buf);
+}
+
+static void pretty_print_trees(int indent)
+{
+ struct survey_stats_trees *pst = &survey_stats.trees;
+ struct survey_stats_base_object *base = &pst->base;
+ struct strbuf buf = STRBUF_INIT;
+ int indent1 = indent + 4;
+ int k;
+
+ const char *intro[] = {
+ "",
+ "TREES",
+ "-------------------------------------------------------------------------------",
+ "",
+ NULL
+ };
+
+ k = 0;
+ while (intro[k])
+ fmt_txt_line(&buf, indent, intro[k++]);
+
+ fmt_base_object(&buf, indent1, base);
+
+ fmt_large_item_vec(&buf, indent1, pst->vec_largest_by_size_bytes);
+
+ fmt_qbin(&buf, indent1, "Tree Histogram by Number of Entries", "Entry Range", pst->entry_qbin);
+ fmt_large_item_vec(&buf, indent1, pst->vec_largest_by_nr_entries);
+
+ strbuf_addch(&buf, '\n');
+ fwrite(buf.buf, 1, buf.len, stdout);
+ strbuf_release(&buf);
+}
+
+static void pretty_print_blobs(int indent)
+{
+ struct survey_stats_blobs *psb = &survey_stats.blobs;
+ struct survey_stats_base_object *base = &psb->base;
+ struct strbuf buf = STRBUF_INIT;
+ int indent1 = indent + 4;
+ int k;
+
+ const char *intro[] = {
+ "",
+ "BLOBS",
+ "-------------------------------------------------------------------------------",
+ "",
+ NULL
+ };
+
+ k = 0;
+ while (intro[k])
+ fmt_txt_line(&buf, indent, intro[k++]);
+
+ fmt_base_object(&buf, indent1, base);
+
+ fmt_large_item_vec(&buf, indent1, psb->vec_largest_by_size_bytes);
+
+ strbuf_addch(&buf, '\n');
+ fwrite(buf.buf, 1, buf.len, stdout);
+ strbuf_release(&buf);
+}
+
+/*
+ * Print all of the stats that we have collected in a more pretty format.
+ */
+static void survey_print_results_pretty(void)
+{
+ pretty_print_survey_hdr();
+ pretty_print_overview(0);
+ pretty_print_refs(0);
+ pretty_print_commits(0);
+ pretty_print_trees(0);
+ pretty_print_blobs(0);
+}
+
+int cmd_survey(int argc, const char **argv, const char *prefix)
+{
+ survey_load_config();
+
+ argc = parse_options(argc, argv, prefix, survey_options, survey_usage, 0);
+
+ color_fprintf_ln(stderr,
+ want_color_fd(2, GIT_COLOR_AUTO) ? GIT_COLOR_YELLOW : "",
+ "(THIS IS EXPERIMENTAL, EXPECT THE OUTPUT FORMAT TO CHANGE!)");
+
+ prepare_repo_settings(the_repository);
+
+ if (survey_opts.show_progress < 0)
+ survey_opts.show_progress = isatty(2);
+ fixup_refs_wanted();
+
+ alloc_commit_by_parents();
+ alloc_commit_by_size();
+ alloc_tree_by_entries();
+ alloc_tree_by_size();
+ alloc_blob_by_size();
+
+ survey_phase_refs(the_repository);
+
+ survey_emit_trace2();
+ if (survey_opts.show_json)
+ survey_print_json();
+ else
+ survey_print_results_pretty();
+
+ strvec_clear(&survey_vec_refs_wanted);
+ free_large_item_vec(survey_stats.commits.vec_largest_by_nr_parents);
+ free_large_item_vec(survey_stats.commits.vec_largest_by_size_bytes);
+ free_large_item_vec(survey_stats.trees.vec_largest_by_nr_entries);
+ free_large_item_vec(survey_stats.trees.vec_largest_by_size_bytes);
+ free_large_item_vec(survey_stats.blobs.vec_largest_by_size_bytes);
+
+ return 0;
+}
+
+/*
+ * NEEDSWORK: So far, I only have iteration on the requested set of
+ * refs and treewalk/reachable objects on that set of refs. The
+ * following is a bit of a laundry list of things that I'd like to
+ * add.
+ *
+ * [] Dump stats on all of the packfiles. The number and size of each.
+ * Whether each is in the .git directory or in an alternate. The
+ * state of the IDX or MIDX files and etc. Delta chain stats. All
+ * of this data is relative to the "lived-in" state of the
+ * repository. Stuff that may change after a GC or repack.
+ *
+ * [] Clone and Index stats. partial, shallow, sparse-checkout,
+ * sparse-index, etc. Hydration stats.
+ *
+ * [] Dump stats on each remote. When we fetch from a remote the size
+ * of the response is related to the set of haves on the server.
+ * You can see this in `GIT_TRACE_CURL=1 git fetch`. We get a
+ * `ls-refs` payload that lists all of the branches and tags on the
+ * server, so at a minimum the RefName and SHA for each. But for
+ * annotated tags we also get the peeled SHA. The size of this
+ * overhead on every fetch is proporational to the size of the `git
+ * ls-remote` response (roughly, although the latter repeats the
+ * RefName of the peeled tag). If, for example, you have 500K refs
+ * on a remote, you're going to have a long "haves" message, so
+ * every fetch will be slow just because of that overhead (not
+ * counting new objects to be downloaded).
+ *
+ * Note that the local set of tags in "refs/tags/" is a union over
+ * all remotes. However, since most people only have one remote,
+ * we can probaly estimate the overhead value directly from the
+ * size of the set of "refs/tags/" that we visited while building
+ * the `ref_info` and `ref_array` and not need to ask the remote.
+ *
+ * [] Should the "string length of refnames / remote refs", for
+ * example, be sub-divided by remote so we can project the
+ * cost of the haves/wants overhead a fetch.
+ *
+ * [] Can we examine the merge commits and classify them as clean or
+ * dirty? (ie. ones with merge conflicts that needed to be
+ * addressed during the merge itself.)
+ *
+ * [] Do dirty merges affect performance of later operations?
+ *
+ * [] Dump info on the complexity of the DAG. Criss-cross merges.
+ * The number of edges that must be touched to compute merge bases.
+ * Edge length. The number of parallel lanes in the history that
+ * must be navigated to get to the merge base. What affects the
+ * cost of the Ahead/Behind computation? How often do
+ * criss-crosses occur and do they cause various operations to slow
+ * down?
+ *
+ * [] If there are primary branches (like "main" or "master") are they
+ * always on the left side of merges? Does the graph have a clean
+ * left edge? Or are there normal and "backwards" merges? Do
+ * these cause problems at scale?
+ *
+ * [] If we have a hierarchy of FI/RI branches like "L1", "L2, ...,
+ * can we learn anything about the shape of the repo around these
+ * FI and RI integrations?
+ *
+ * [] Do we need a no-PII flag to omit pathnames or branch/tag names
+ * in the various histograms? (This would turn off --name-rev
+ * too.)
+ *
+ * [] I have so far avoided adding opinions about individual fields
+ * (such as the way `git-sizer` prints a row of stars or bangs in
+ * the last column).
+ *
+ * I'm wondering if that is a job of this executable or if it
+ * should be done in a post-processing step using the JSON output.
+ *
+ * My problem with the `git-sizer` approach is that it doesn't give
+ * the (casual) user any information on why it has stars or bangs.
+ * And there isn't a good way to print detailed information in the
+ * ASCII-art tables that would be easy to understand.
+ *
+ * [] For example, a large number of refs does not define a cliff.
+ * Performance will drop off (linearly, quadratically, ... ??).
+ * The tool should refer them to article(s) talking about the
+ * different problems that it could cause. So should `git
+ * survey` just print the number and (implicitly) refer them to
+ * the man page (chapter/verse) or to a tool that will interpret
+ * the number and explain it?
+ *
+ * [] Alternatively, should `git survey` do that analysis too and
+ * just print footnotes for each large number?
+ *
+ * [] The computation of the raw survey JSON data can take HOURS on
+ * a very large repo (like Windows), so I'm wondering if we
+ * want to keep the opinion portion separate.
+ *
+ * [] In addition to opinions based on the static data, I would like
+ * to dump the JSON results (or the Trace2 telemetry) into a DB and
+ * aggregate it with other users.
+ *
+ * Granted, they should all see the same DAG and the same set of
+ * reachable objects, but we could average across all datasets
+ * generated on a particular date and detect outlier users.
+ *
+ * [] Maybe someone cloned from the `_full` endpoint rather than
+ * the limited refs endpoint.
+ *
+ * [] Maybe that user is having problems with repacking / GC /
+ * maintenance without knowing it.
+ *
+ * [] I'd also like to dump use the DB to compare survey datasets over
+ * a time. How fast is their repository growing and in what ways?
+ *
+ * [] I'd rather have the delta analysis NOT be inside `git
+ * survey`, so it makes sense to consider having all of it in a
+ * post-process step.
+ *
+ * [] Another reason to put the opinion analysis in a post-process
+ * is that it would be easier to generate plots on the data tables.
+ * Granted, we can get plots from telemetry, but a stand-alone user
+ * could run the JSON thru python or jq or something and generate
+ * something nicer than ASCII-art and it could handle cross-referencing
+ * and hyperlinking to helpful information on each issue.
+ *
+ * [] I think there are several classes of data that we can report on:
+ *
+ * [] The "inherit repo properties", such as the shape and size of
+ * the DAG -- these should be universal in each enlistment.
+ *
+ * [] The "ODB lived in properties", such as the efficiency
+ * of the repack and things like partial and shallow clone.
+ * These will vary, but indicate health of the ODB.
+ *
+ * [] The "index related properties", such as sparse-checkout,
+ * sparse-index, cache-tree, untracked-cache, fsmonitor, and
+ * etc. These will also vary, but are more like knobs for
+ * the user to adjust.
+ *
+ * [] I want to compare these with Matt's "dimensions of scale"
+ * notes and see if there are other pieces of data that we
+ * could compute/consider.
+ *
+ */
diff --git a/builtin/update-index.c b/builtin/update-index.c
index d343416ae261ff..9e1eb6b0aead95 100644
--- a/builtin/update-index.c
+++ b/builtin/update-index.c
@@ -5,6 +5,7 @@
*/
#include "builtin.h"
+#include "gvfs.h"
#include "bulk-checkin.h"
#include "config.h"
#include "environment.h"
@@ -1110,7 +1111,13 @@ int cmd_update_index(int argc, const char **argv, const char *prefix)
argc = parse_options_end(&ctx);
getline_fn = nul_term_line ? strbuf_getline_nul : strbuf_getline_lf;
+ if (mark_skip_worktree_only && gvfs_config_is_set(GVFS_BLOCK_COMMANDS))
+ die(_("modifying the skip worktree bit is not supported on a GVFS repo"));
+
if (preferred_index_format) {
+ if (preferred_index_format != 4 && gvfs_config_is_set(GVFS_BLOCK_COMMANDS))
+ die(_("changing the index version is not supported on a GVFS repo"));
+
if (preferred_index_format < 0) {
printf(_("%d\n"), the_repository->index->version);
} else if (preferred_index_format < INDEX_FORMAT_LB ||
@@ -1156,6 +1163,9 @@ int cmd_update_index(int argc, const char **argv, const char *prefix)
end_odb_transaction();
if (split_index > 0) {
+ if (gvfs_config_is_set(GVFS_BLOCK_COMMANDS))
+ die(_("split index is not supported on a GVFS repo"));
+
if (git_config_get_split_index() == 0)
warning(_("core.splitIndex is set to false; "
"remove or change it, if you really want to "
diff --git a/builtin/update-microsoft-git.c b/builtin/update-microsoft-git.c
new file mode 100644
index 00000000000000..3152ee23c30096
--- /dev/null
+++ b/builtin/update-microsoft-git.c
@@ -0,0 +1,69 @@
+#include "builtin.h"
+#include "repository.h"
+#include "parse-options.h"
+#include "run-command.h"
+#include "strvec.h"
+
+#if defined(GIT_WINDOWS_NATIVE)
+/*
+ * On Windows, run 'git update-git-for-windows' which
+ * is installed by the installer, based on the script
+ * in git-for-windows/build-extra.
+ */
+static int platform_specific_upgrade(void)
+{
+ struct child_process cp = CHILD_PROCESS_INIT;
+
+ strvec_push(&cp.args, "git-update-git-for-windows");
+ return run_command(&cp);
+}
+#elif defined(__APPLE__)
+/*
+ * On macOS, we expect the user to have the microsoft-git
+ * cask installed via Homebrew. We check using these
+ * commands:
+ *
+ * 1. 'brew update' to get latest versions.
+ * 2. 'brew upgrade --cask microsoft-git' to get the
+ * latest version.
+ */
+static int platform_specific_upgrade(void)
+{
+ int res;
+ struct child_process update = CHILD_PROCESS_INIT;
+ struct child_process upgrade = CHILD_PROCESS_INIT;
+
+ printf("Updating Homebrew with 'brew update'\n");
+
+ strvec_pushl(&update.args, "brew", "update", NULL);
+ res = run_command(&update);
+
+ if (res) {
+ error(_("'brew update' failed; is brew installed?"));
+ return 1;
+ }
+
+ printf("Upgrading microsoft-git with 'brew upgrade --cask microsoft-git'\n");
+ strvec_pushl(&upgrade.args, "brew", "upgrade", "--cask", "microsoft-git", NULL);
+ res = run_command(&upgrade);
+
+ return res;
+}
+#else
+static int platform_specific_upgrade(void)
+{
+ error(_("update-microsoft-git is not supported on this platform"));
+ return 1;
+}
+#endif
+
+static const char builtin_update_microsoft_git_usage[] =
+ N_("git update-microsoft-git");
+
+int cmd_update_microsoft_git(int argc, const char **argv, const char *prefix)
+{
+ if (argc == 2 && !strcmp(argv[1], "-h"))
+ usage(builtin_update_microsoft_git_usage);
+
+ return platform_specific_upgrade();
+}
diff --git a/builtin/worktree.c b/builtin/worktree.c
index 1d51e54fcdc189..49b8ca37c4f050 100644
--- a/builtin/worktree.c
+++ b/builtin/worktree.c
@@ -1,6 +1,7 @@
#include "builtin.h"
#include "abspath.h"
#include "advice.h"
+#include "gvfs.h"
#include "checkout.h"
#include "config.h"
#include "copy.h"
@@ -1407,6 +1408,13 @@ int cmd_worktree(int ac, const char **av, const char *prefix)
git_config(git_worktree_config, NULL);
+ /*
+ * git-worktree is special-cased to work in Scalar repositories
+ * even when they use the GVFS Protocol.
+ */
+ if (core_gvfs & GVFS_USE_VIRTUAL_FILESYSTEM)
+ die("'git %s' is not supported on a GVFS repo", "worktree");
+
if (!prefix)
prefix = "";
diff --git a/cache-tree.c b/cache-tree.c
index 50610c3f3cb8e7..cec214bc3eeeb0 100644
--- a/cache-tree.c
+++ b/cache-tree.c
@@ -3,6 +3,7 @@
#include "git-compat-util.h"
#include "environment.h"
#include "hex.h"
+#include "gvfs.h"
#include "lockfile.h"
#include "tree.h"
#include "tree-walk.h"
@@ -231,7 +232,7 @@ static void discard_unused_subtrees(struct cache_tree *it)
}
}
-int cache_tree_fully_valid(struct cache_tree *it)
+static int cache_tree_fully_valid_1(struct cache_tree *it)
{
int i;
if (!it)
@@ -239,7 +240,7 @@ int cache_tree_fully_valid(struct cache_tree *it)
if (it->entry_count < 0 || !repo_has_object_file(the_repository, &it->oid))
return 0;
for (i = 0; i < it->subtree_nr; i++) {
- if (!cache_tree_fully_valid(it->down[i]->cache_tree))
+ if (!cache_tree_fully_valid_1(it->down[i]->cache_tree))
return 0;
}
return 1;
@@ -250,6 +251,17 @@ static int must_check_existence(const struct cache_entry *ce)
return !(repo_has_promisor_remote(the_repository) && ce_skip_worktree(ce));
}
+int cache_tree_fully_valid(struct cache_tree *it)
+{
+ int result;
+
+ trace2_region_enter("cache_tree", "fully_valid", NULL);
+ result = cache_tree_fully_valid_1(it);
+ trace2_region_leave("cache_tree", "fully_valid", NULL);
+
+ return result;
+}
+
static int update_one(struct cache_tree *it,
struct cache_entry **cache,
int entries,
@@ -259,7 +271,8 @@ static int update_one(struct cache_tree *it,
int flags)
{
struct strbuf buffer;
- int missing_ok = flags & WRITE_TREE_MISSING_OK;
+ int missing_ok = gvfs_config_is_set(GVFS_MISSING_OK) ?
+ WRITE_TREE_MISSING_OK : (flags & WRITE_TREE_MISSING_OK);
int dryrun = flags & WRITE_TREE_DRY_RUN;
int repair = flags & WRITE_TREE_REPAIR;
int to_invalidate = 0;
@@ -428,7 +441,29 @@ static int update_one(struct cache_tree *it,
continue;
strbuf_grow(&buffer, entlen + 100);
- strbuf_addf(&buffer, "%o %.*s%c", mode, entlen, path + baselen, '\0');
+
+ switch (mode) {
+ case 0100644:
+ strbuf_add(&buffer, "100644 ", 7);
+ break;
+ case 0100664:
+ strbuf_add(&buffer, "100664 ", 7);
+ break;
+ case 0100755:
+ strbuf_add(&buffer, "100755 ", 7);
+ break;
+ case 0120000:
+ strbuf_add(&buffer, "120000 ", 7);
+ break;
+ case 0160000:
+ strbuf_add(&buffer, "160000 ", 7);
+ break;
+ default:
+ strbuf_addf(&buffer, "%o ", mode);
+ break;
+ }
+ strbuf_add(&buffer, path + baselen, entlen);
+ strbuf_addch(&buffer, '\0');
strbuf_add(&buffer, oid->hash, the_hash_algo->rawsz);
#if DEBUG_CACHE_TREE
diff --git a/command-list.txt b/command-list.txt
index e0bb87b3b5c278..d389561a5f1161 100644
--- a/command-list.txt
+++ b/command-list.txt
@@ -186,6 +186,7 @@ git-stash mainporcelain
git-status mainporcelain info
git-stripspace purehelpers
git-submodule mainporcelain
+git-survey mainporcelain
git-svn foreignscminterface
git-switch mainporcelain history
git-symbolic-ref plumbingmanipulators
diff --git a/commit.c b/commit.c
index 087cb19f4f84f3..27eb39758b2275 100644
--- a/commit.c
+++ b/commit.c
@@ -1,6 +1,7 @@
#define USE_THE_REPOSITORY_VARIABLE
#include "git-compat-util.h"
+#include "gvfs.h"
#include "tag.h"
#include "commit.h"
#include "commit-graph.h"
@@ -562,13 +563,17 @@ int repo_parse_commit_internal(struct repository *r,
.sizep = &size,
.contentp = &buffer,
};
+ int ret;
/*
* Git does not support partial clones that exclude commits, so set
* OBJECT_INFO_SKIP_FETCH_OBJECT to fail fast when an object is missing.
*/
int flags = OBJECT_INFO_LOOKUP_REPLACE | OBJECT_INFO_SKIP_FETCH_OBJECT |
- OBJECT_INFO_DIE_IF_CORRUPT;
- int ret;
+ OBJECT_INFO_DIE_IF_CORRUPT;
+
+ /* But the GVFS Protocol _does_ support missing commits! */
+ if (gvfs_config_is_set(GVFS_MISSING_OK))
+ flags ^= OBJECT_INFO_SKIP_FETCH_OBJECT;
if (!item)
return -1;
diff --git a/compat/mingw.c b/compat/mingw.c
index bb8760deec9433..4d49f31db29279 100644
--- a/compat/mingw.c
+++ b/compat/mingw.c
@@ -4088,6 +4088,8 @@ int wmain(int argc, const wchar_t **wargv)
SetConsoleCtrlHandler(handle_ctrl_c, TRUE);
+ trace2_initialize_clock();
+
maybe_redirect_std_handles();
adjust_symlink_flags();
fsync_object_files = 1;
diff --git a/config.c b/config.c
index 0a8d6901c83608..b3eb022d8614b5 100644
--- a/config.c
+++ b/config.c
@@ -12,6 +12,7 @@
#include "abspath.h"
#include "advice.h"
#include "date.h"
+#include "gvfs.h"
#include "branch.h"
#include "config.h"
#include "parse.h"
@@ -41,6 +42,7 @@
#include "wildmatch.h"
#include "ws.h"
#include "write-or-die.h"
+#include "transport.h"
struct config_source {
struct config_source *prev;
@@ -1665,8 +1667,22 @@ int git_default_core_config(const char *var, const char *value,
return 0;
}
+ if (!strcmp(var, "core.gvfs")) {
+ gvfs_load_config_value(value);
+ return 0;
+ }
+
+ if (!strcmp(var, "core.usegvfshelper")) {
+ core_use_gvfs_helper = git_config_bool(var, value);
+ return 0;
+ }
+
if (!strcmp(var, "core.sparsecheckout")) {
- core_apply_sparse_checkout = git_config_bool(var, value);
+ /* virtual file system relies on the sparse checkout logic so force it on */
+ if (core_virtualfilesystem)
+ core_apply_sparse_checkout = 1;
+ else
+ core_apply_sparse_checkout = git_config_bool(var, value);
return 0;
}
@@ -1695,6 +1711,11 @@ int git_default_core_config(const char *var, const char *value,
return 0;
}
+ if (!strcmp(var, "core.virtualizeobjects")) {
+ core_virtualize_objects = git_config_bool(var, value);
+ return 0;
+ }
+
/* Add other config variables here and to Documentation/config.txt. */
return platform_core_config(var, value, ctx, cb);
}
@@ -1807,6 +1828,37 @@ static int git_default_mailmap_config(const char *var, const char *value)
return 0;
}
+static int git_default_gvfs_config(const char *var, const char *value)
+{
+ if (!strcmp(var, "gvfs.cache-server")) {
+ char *v2 = NULL;
+
+ if (!git_config_string(&v2, var, value) && v2 && *v2) {
+ free(gvfs_cache_server_url);
+ gvfs_cache_server_url = transport_anonymize_url(v2);
+ }
+ free(v2);
+ return 0;
+ }
+
+ if (!strcmp(var, "gvfs.sharedcache") && value && *value) {
+ strbuf_setlen(&gvfs_shared_cache_pathname, 0);
+ strbuf_addstr(&gvfs_shared_cache_pathname, value);
+ if (strbuf_normalize_path(&gvfs_shared_cache_pathname) < 0) {
+ /*
+ * Pretend it wasn't set. This will cause us to
+ * fallback to ".git/objects" effectively.
+ */
+ strbuf_release(&gvfs_shared_cache_pathname);
+ return 0;
+ }
+ strbuf_trim_trailing_dir_sep(&gvfs_shared_cache_pathname);
+ return 0;
+ }
+
+ return 0;
+}
+
static int git_default_attr_config(const char *var, const char *value)
{
if (!strcmp(var, "attr.tree")) {
@@ -1874,6 +1926,9 @@ int git_default_config(const char *var, const char *value,
if (starts_with(var, "sparse."))
return git_default_sparse_config(var, value);
+ if (starts_with(var, "gvfs."))
+ return git_default_gvfs_config(var, value);
+
/* Add other config variables here and to Documentation/config.txt. */
return 0;
}
@@ -2826,6 +2881,44 @@ int git_config_get_max_percent_split_change(void)
return -1; /* default value */
}
+int git_config_get_virtualfilesystem(void)
+{
+ /* Run only once. */
+ static int virtual_filesystem_result = -1;
+ if (virtual_filesystem_result >= 0)
+ return virtual_filesystem_result;
+
+ if (git_config_get_pathname("core.virtualfilesystem", &core_virtualfilesystem))
+ core_virtualfilesystem = getenv("GIT_VIRTUALFILESYSTEM_TEST");
+
+ if (core_virtualfilesystem && !*core_virtualfilesystem)
+ core_virtualfilesystem = NULL;
+
+ if (core_virtualfilesystem) {
+ /*
+ * Some git commands spawn helpers and redirect the index to a different
+ * location. These include "difftool -d" and the sequencer
+ * (i.e. `git rebase -i`, `git cherry-pick` and `git revert`) and others.
+ * In those instances we don't want to update their temporary index with
+ * our virtualization data.
+ */
+ char *default_index_file = xstrfmt("%s/%s", the_repository->gitdir, "index");
+ int should_run_hook = !strcmp(default_index_file, the_repository->index_file);
+
+ free(default_index_file);
+ if (should_run_hook) {
+ /* virtual file system relies on the sparse checkout logic so force it on */
+ core_apply_sparse_checkout = 1;
+ virtual_filesystem_result = 1;
+ return 1;
+ }
+ core_virtualfilesystem = NULL;
+ }
+
+ virtual_filesystem_result = 0;
+ return 0;
+}
+
int git_config_get_index_threads(int *dest)
{
int is_bool, val;
@@ -3299,6 +3392,7 @@ int git_config_set_multivar_in_file_gently(const char *config_filename,
const char *comment,
unsigned flags)
{
+ static unsigned long timeout_ms = ULONG_MAX;
int fd = -1, in_fd = -1;
int ret;
struct lock_file lock = LOCK_INIT;
@@ -3319,11 +3413,16 @@ int git_config_set_multivar_in_file_gently(const char *config_filename,
if (!config_filename)
config_filename = filename_buf = git_pathdup("config");
+ if ((long)timeout_ms < 0 &&
+ git_config_get_ulong("core.configWriteLockTimeoutMS", &timeout_ms))
+ timeout_ms = 0;
+
/*
* The lock serves a purpose in addition to locking: the new
* contents of .git/config will be written into it.
*/
- fd = hold_lock_file_for_update(&lock, config_filename, 0);
+ fd = hold_lock_file_for_update_timeout(&lock, config_filename, 0,
+ timeout_ms);
if (fd < 0) {
error_errno(_("could not lock config file %s"), config_filename);
ret = CONFIG_NO_LOCK;
diff --git a/config.h b/config.h
index 418384646b04d3..f3ad29834154b9 100644
--- a/config.h
+++ b/config.h
@@ -701,6 +701,7 @@ int git_config_get_pathname(const char *key, char **dest);
int git_config_get_index_threads(int *dest);
int git_config_get_split_index(void);
int git_config_get_max_percent_split_change(void);
+int git_config_get_virtualfilesystem(void);
/* This dies if the configured or default date is in the future */
int git_config_get_expiry(const char *key, const char **output);
diff --git a/connected.c b/connected.c
index 87cc4b57a17ce5..a8cf3dc351fc54 100644
--- a/connected.c
+++ b/connected.c
@@ -1,8 +1,10 @@
#define USE_THE_REPOSITORY_VARIABLE
#include "git-compat-util.h"
+#include "environment.h"
#include "gettext.h"
#include "hex.h"
+#include "gvfs.h"
#include "object-store-ll.h"
#include "run-command.h"
#include "sigchain.h"
@@ -34,6 +36,26 @@ int check_connected(oid_iterate_fn fn, void *cb_data,
struct transport *transport;
size_t base_len;
+ /*
+ * Running a virtual file system there will be objects that are
+ * missing locally and we don't want to download a bunch of
+ * commits, trees, and blobs just to make sure everything is
+ * reachable locally so this option will skip reachablility
+ * checks below that use rev-list. This will stop the check
+ * before uploadpack runs to determine if there is anything to
+ * fetch. Returning zero for the first check will also prevent the
+ * uploadpack from happening. It will also skip the check after
+ * the fetch is finished to make sure all the objects where
+ * downloaded in the pack file. This will allow the fetch to
+ * run and get all the latest tip commit ids for all the branches
+ * in the fetch but not pull down commits, trees, or blobs via
+ * upload pack.
+ */
+ if (gvfs_config_is_set(GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK))
+ return 0;
+ if (core_virtualize_objects)
+ return 0;
+
if (!opt)
opt = &defaults;
transport = opt->transport;
diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt
index 90553639179e04..30379e37549ae6 100644
--- a/contrib/buildsystems/CMakeLists.txt
+++ b/contrib/buildsystems/CMakeLists.txt
@@ -649,7 +649,7 @@ if(NOT CURL_FOUND)
add_compile_definitions(NO_CURL)
message(WARNING "git-http-push and git-http-fetch will not be built")
else()
- list(APPEND PROGRAMS_BUILT git-http-fetch git-http-push git-imap-send git-remote-http)
+ list(APPEND PROGRAMS_BUILT git-http-fetch git-http-push git-imap-send git-remote-http git-gvfs-helper)
if(CURL_VERSION_STRING VERSION_GREATER_EQUAL 7.34.0)
add_compile_definitions(USE_CURL_FOR_IMAP_SEND)
endif()
@@ -799,7 +799,7 @@ target_link_libraries(git-sh-i18n--envsubst common-main)
add_executable(git-shell ${CMAKE_SOURCE_DIR}/shell.c)
target_link_libraries(git-shell common-main)
-add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c)
+add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c ${CMAKE_SOURCE_DIR}/json-parser.c)
target_link_libraries(scalar common-main)
if(CURL_FOUND)
@@ -818,6 +818,9 @@ if(CURL_FOUND)
add_executable(git-http-push ${CMAKE_SOURCE_DIR}/http-push.c)
target_link_libraries(git-http-push http_obj common-main ${CURL_LIBRARIES} ${EXPAT_LIBRARIES})
endif()
+
+ add_executable(git-gvfs-helper ${CMAKE_SOURCE_DIR}/gvfs-helper.c)
+ target_link_libraries(git-gvfs-helper http_obj common-main ${CURL_LIBRARIES} )
endif()
parse_makefile_for_executables(git_builtin_extra "BUILT_INS")
@@ -1058,6 +1061,20 @@ set(wrapper_scripts
set(wrapper_test_scripts
test-fake-ssh test-tool)
+if(CURL_FOUND)
+ list(APPEND wrapper_test_scripts test-gvfs-protocol)
+
+ add_executable(test-gvfs-protocol ${CMAKE_SOURCE_DIR}/t/helper/test-gvfs-protocol.c)
+ target_link_libraries(test-gvfs-protocol common-main)
+
+ if(MSVC)
+ set_target_properties(test-gvfs-protocol
+ PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR}/t/helper)
+ set_target_properties(test-gvfs-protocol
+ PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}/t/helper)
+ endif()
+endif()
+
foreach(script ${wrapper_scripts})
file(STRINGS ${CMAKE_SOURCE_DIR}/wrap-for-bin.sh content NEWLINE_CONSUME)
diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash
index 60a22d619a85cc..94ff4f9363e1d8 100644
--- a/contrib/completion/git-completion.bash
+++ b/contrib/completion/git-completion.bash
@@ -1802,7 +1802,7 @@ _git_clone ()
esac
}
-__git_untracked_file_modes="all no normal"
+__git_untracked_file_modes="all no normal complete"
__git_trailer_tokens ()
{
diff --git a/contrib/long-running-read-object/example.pl b/contrib/long-running-read-object/example.pl
new file mode 100644
index 00000000000000..b8f37f836a813c
--- /dev/null
+++ b/contrib/long-running-read-object/example.pl
@@ -0,0 +1,114 @@
+#!/usr/bin/perl
+#
+# Example implementation for the Git read-object protocol version 1
+# See Documentation/technical/read-object-protocol.txt
+#
+# Allows you to test the ability for blobs to be pulled from a host git repo
+# "on demand." Called when git needs a blob it couldn't find locally due to
+# a lazy clone that only cloned the commits and trees.
+#
+# A lazy clone can be simulated via the following commands from the host repo
+# you wish to create a lazy clone of:
+#
+# cd /host_repo
+# git rev-parse HEAD
+# git init /guest_repo
+# git cat-file --batch-check --batch-all-objects | grep -v 'blob' |
+# cut -d' ' -f1 | git pack-objects /guest_repo/.git/objects/pack/noblobs
+# cd /guest_repo
+# git config core.virtualizeobjects true
+# git reset --hard
+#
+# Please note, this sample is a minimal skeleton. No proper error handling
+# was implemented.
+#
+
+use strict;
+use warnings;
+
+#
+# Point $DIR to the folder where your host git repo is located so we can pull
+# missing objects from it
+#
+my $DIR = "/host_repo/.git/";
+
+sub packet_bin_read {
+ my $buffer;
+ my $bytes_read = read STDIN, $buffer, 4;
+ if ( $bytes_read == 0 ) {
+
+ # EOF - Git stopped talking to us!
+ exit();
+ }
+ elsif ( $bytes_read != 4 ) {
+ die "invalid packet: '$buffer'";
+ }
+ my $pkt_size = hex($buffer);
+ if ( $pkt_size == 0 ) {
+ return ( 1, "" );
+ }
+ elsif ( $pkt_size > 4 ) {
+ my $content_size = $pkt_size - 4;
+ $bytes_read = read STDIN, $buffer, $content_size;
+ if ( $bytes_read != $content_size ) {
+ die "invalid packet ($content_size bytes expected; $bytes_read bytes read)";
+ }
+ return ( 0, $buffer );
+ }
+ else {
+ die "invalid packet size: $pkt_size";
+ }
+}
+
+sub packet_txt_read {
+ my ( $res, $buf ) = packet_bin_read();
+ unless ( $buf =~ s/\n$// ) {
+ die "A non-binary line MUST be terminated by an LF.";
+ }
+ return ( $res, $buf );
+}
+
+sub packet_bin_write {
+ my $buf = shift;
+ print STDOUT sprintf( "%04x", length($buf) + 4 );
+ print STDOUT $buf;
+ STDOUT->flush();
+}
+
+sub packet_txt_write {
+ packet_bin_write( $_[0] . "\n" );
+}
+
+sub packet_flush {
+ print STDOUT sprintf( "%04x", 0 );
+ STDOUT->flush();
+}
+
+( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || die "bad initialize";
+( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version";
+( packet_bin_read() eq ( 1, "" ) ) || die "bad version end";
+
+packet_txt_write("git-read-object-server");
+packet_txt_write("version=1");
+packet_flush();
+
+( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability";
+( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end";
+
+packet_txt_write("capability=get");
+packet_flush();
+
+while (1) {
+ my ($command) = packet_txt_read() =~ /^command=([^=]+)$/;
+
+ if ( $command eq "get" ) {
+ my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/;
+ packet_bin_read();
+
+ system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1');
+ packet_txt_write(($?) ? "status=error" : "status=success");
+ packet_flush();
+ } else {
+ die "bad command '$command'";
+ }
+}
diff --git a/contrib/scalar/docs/faq.md b/contrib/scalar/docs/faq.md
new file mode 100644
index 00000000000000..a14f78a996d5d5
--- /dev/null
+++ b/contrib/scalar/docs/faq.md
@@ -0,0 +1,51 @@
+Frequently Asked Questions
+==========================
+
+Using Scalar
+------------
+
+### I don't want a sparse clone, I want every file after I clone!
+
+Run `scalar clone --full-clone ` to initialize your repo to include
+every file. You can switch to a sparse-checkout later by running
+`git sparse-checkout init --cone`.
+
+### I already cloned without `--full-clone`. How do I get everything?
+
+Run `git sparse-checkout disable`.
+
+Scalar Design Decisions
+-----------------------
+
+There may be many design decisions within Scalar that are confusing at first
+glance. Some of them may cause friction when you use Scalar with your existing
+repos and existing habits.
+
+> Scalar has the most benefit when users design repositories
+> with efficient patterns.
+
+For example: Scalar uses the sparse-checkout feature to limit the size of the
+working directory within a large monorepo. It is designed to work efficiently
+with monorepos that are highly componentized, allowing most developers to
+need many fewer files in their daily work.
+
+### Why does `scalar clone` create a `/src` folder?
+
+Scalar uses a file system watcher to keep track of changes under this `src` folder.
+Any activity in this folder is assumed to be important to Git operations. By
+creating the `src` folder, we are making it easy for your build system to
+create output folders outside the `src` directory. We commonly see systems
+create folders for build outputs and package downloads. Scalar itself creates
+these folders during its builds.
+
+Your build system may create build artifacts such as `.obj` or `.lib` files
+next to your source code. These are commonly "hidden" from Git using
+`.gitignore` files. Having such artifacts in your source tree creates
+additional work for Git because it needs to look at these files and match them
+against the `.gitignore` patterns.
+
+By following the `src` pattern Scalar tries to establish and placing your build
+intermediates and outputs parallel with the `src` folder and not inside it,
+you can help optimize Git command performance for developers in the repository
+by limiting the number of files Git needs to consider for many common
+operations.
diff --git a/contrib/scalar/docs/getting-started.md b/contrib/scalar/docs/getting-started.md
new file mode 100644
index 00000000000000..d5125330320d2c
--- /dev/null
+++ b/contrib/scalar/docs/getting-started.md
@@ -0,0 +1,109 @@
+Getting Started
+===============
+
+Registering existing Git repos
+------------------------------
+
+To add a repository to the list of registered repos, run `scalar register []`.
+If `` is not provided, then the "current repository" is discovered from
+the working directory by scanning the parent paths for a path containing a `.git`
+folder, possibly inside a `src` folder.
+
+To see which repositories are currently tracked by the service, run
+`scalar list`.
+
+Run `scalar unregister []` to remove the repo from this list.
+
+Creating a new Scalar clone
+---------------------------------------------------
+
+The `clone` verb creates a local enlistment of a remote repository using the
+partial clone feature available e.g. on GitHub, or using the
+[GVFS protocol](https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md),
+such as Azure Repos.
+
+```
+scalar clone [options] []
+```
+
+Create a local copy of the repository at ``. If specified, create the ``
+directory and place the repository there. Otherwise, the last section of the ``
+will be used for ``.
+
+At the end, the repo is located at `/src`. By default, the sparse-checkout
+feature is enabled and the only files present are those in the root of your
+Git repository. Use `git sparse-checkout set` to expand the set of directories
+you want to see, or `git sparse-checkout disable` to expand to all files. You
+can explore the subdirectories outside your sparse-checkout specification using
+`git ls-tree HEAD`.
+
+### Sparse Repo Mode
+
+By default, Scalar reduces your working directory to only the files at the
+root of the repository. You need to add the folders you care about to build up
+to your working set.
+
+* `scalar clone `
+ * Please choose the **Clone with HTTPS** option in the `Clone Repository` dialog in Azure Repos, not **Clone with SSH**.
+* `cd \src`
+* At this point, your `src` directory only contains files that appear in your root
+ tree. No folders are populated.
+* Set the directory list for your sparse-checkout using:
+ 1. `git sparse-checkout set ...`
+ 2. `git sparse-checkout set --stdin < dir-list.txt`
+* Run git commands as you normally would.
+* To fully populate your working directory, run `git sparse-checkout disable`.
+
+If instead you want to start with all files on-disk, you can clone with the
+`--full-clone` option. To enable sparse-checkout after the fact, run
+`git sparse-checkout init --cone`. This will initialize your sparse-checkout
+patterns to only match the files at root.
+
+If you are unfamiliar with what directories are available in the repository,
+then you can run `git ls-tree -d --name-only HEAD` to discover the directories
+at root, or `git ls-tree -d --name-only HEAD ` to discover the directories
+in ``.
+
+### Options
+
+These options allow a user to customize their initial enlistment.
+
+* `--full-clone`: If specified, do not initialize the sparse-checkout feature.
+ All files will be present in your `src` directory. This behaves very similar
+ to a Git partial clone in that blobs are downloaded on demand. However, it
+ will use the GVFS protocol to download all Git objects.
+
+* `--cache-server-url=`: If specified, set the intended cache server to
+ the specified ``. All object queries will use the GVFS protocol to this
+ `` instead of the origin remote. If the remote supplies a list of
+ cache servers via the `/gvfs/config` endpoint, then the `clone` command
+ will select a nearby cache server from that list.
+
+* `--branch=][`: Specify the branch to checkout after clone.
+
+* `--local-cache-path=`: Use this option to override the path for the
+ local Scalar cache. If not specified, then Scalar will select a default
+ path to share objects with your other enlistments. On Windows, this path
+ is a subdirectory of `:\.scalarCache\`. On Mac, this path is a
+ subdirectory of `~/.scalarCache/`. The default cache path is recommended so
+ multiple enlistments of the same remote repository share objects on the
+ same device.
+
+### Advanced Options
+
+The options below are not intended for use by a typical user. These are
+usually used by build machines to create a temporary enlistment that
+operates on a single commit.
+
+* `--single-branch`: Use this option to only download metadata for the branch
+ that will be checked out. This is helpful for build machines that target
+ a remote with many branches. Any `git fetch` commands after the clone will
+ still ask for all branches.
+
+Removing a Scalar Clone
+-----------------------
+
+Since the `scalar clone` command sets up a file-system watcher (when available),
+that watcher could prevent deleting the enlistment. Run `scalar delete `
+from outside of your enlistment to unregister the enlistment from the filesystem
+watcher and delete the enlistment at ``.
diff --git a/contrib/scalar/docs/index.md b/contrib/scalar/docs/index.md
new file mode 100644
index 00000000000000..4f56e2b0ebbac6
--- /dev/null
+++ b/contrib/scalar/docs/index.md
@@ -0,0 +1,54 @@
+Scalar: Enabling Git at Scale
+=============================
+
+Scalar is a tool that helps Git scale to some of the largest Git repositories.
+It achieves this by enabling some advanced Git features, such as:
+
+* *Partial clone:* reduces time to get a working repository by not
+ downloading all Git objects right away.
+
+* *Background prefetch:* downloads Git object data from all remotes every
+ hour, reducing the amount of time for foreground `git fetch` calls.
+
+* *Sparse-checkout:* limits the size of your working directory.
+
+* *File system monitor:* tracks the recently modified files and eliminates
+ the need for Git to scan the entire worktree.
+
+* *Commit-graph:* accelerates commit walks and reachability calculations,
+ speeding up commands like `git log`.
+
+* *Multi-pack-index:* enables fast object lookups across many pack-files.
+
+* *Incremental repack:* Repacks the packed Git data into fewer pack-file
+ without disrupting concurrent commands by using the multi-pack-index.
+
+By running `scalar register` in any Git repo, Scalar will automatically enable
+these features for that repo (except partial clone) and start running suggested
+maintenance in the background using
+[the `git maintenance` feature](https://git-scm.com/docs/git-maintenance).
+
+Repos cloned with the `scalar clone` command use partial clone or the
+[GVFS protocol](https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md)
+to significantly reduce the amount of data required to get started
+using a repository. By delaying all blob downloads until they are required,
+Scalar allows you to work with very large repositories quickly. The GVFS
+protocol allows a network of _cache servers_ to serve objects with lower
+latency and higher throughput. The cache servers also reduce load on the
+central server.
+
+Documentation
+-------------
+
+* [Getting Started](getting-started.md): Get started with Scalar.
+ Includes `scalar register`, `scalar unregister`, `scalar clone`, and
+ `scalar delete`.
+
+* [Troubleshooting](troubleshooting.md):
+ Collect diagnostic information or update custom settings. Includes
+ `scalar diagnose` and `scalar cache-server`.
+
+* [The Philosophy of Scalar](philosophy.md): Why does Scalar work the way
+ it does, and how do we make decisions about its future?
+
+* [Frequently Asked Questions](faq.md)
diff --git a/contrib/scalar/docs/philosophy.md b/contrib/scalar/docs/philosophy.md
new file mode 100644
index 00000000000000..e3dfa025a2504c
--- /dev/null
+++ b/contrib/scalar/docs/philosophy.md
@@ -0,0 +1,71 @@
+The Philosophy of Scalar
+========================
+
+The team building Scalar has **opinions** about Git performance. Scalar
+takes out the guesswork by automatically configuring your Git repositories
+to take advantage of the latest and greatest features. It is difficult to
+say that these are the absolute best settings for every repository, but
+these settings do work for some of the largest repositories in the world.
+
+Scalar intends to do very little more than the standard Git client. We
+actively implement new features into Git instead of Scalar, then update
+Scalar only to configure those new settings. In particular, we ported
+features like background maintenance to Git to make Scalar simpler and
+make Git more powerful.
+
+Scalar ships inside [a custom version of Git][microsoft-git], but we are
+working to make it available in other forks of Git. The only feature
+that is not intended to ever reach the standard Git client is Scalar's use
+of [the GVFS Protocol][gvfs-protocol], which is essentially an older
+version of [Git's partial clone feature](https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/)
+that was available first in Azure Repos. Services such as GitHub support
+only partial clone instead of the GVFS protocol because that is the
+standard adopted by the Git project. If your hosting service supports
+partial clone, then we absolutely recommend it as a way to greatly speed
+up your clone and fetch times and to reduce how much disk space your Git
+repository requires. Scalar will help with this!
+
+If you don't use the GVFS Protocol, then most of the value of Scalar can
+be found in the core Git client. However, most of the advanced features
+that really optimize Git's performance are off by default for compatibility
+reasons. To really take advantage of Git's latest and greatest features,
+you either need to study the [`git config` documentation](https://git-scm.com/docs/git-config)
+and regularly read [the Git release notes](https://github.com/git/git/tree/master/Documentation/RelNotes).
+Even if you do all that work and customize your Git settings on your machines,
+you likely will want to share those settings with other team members.
+Or, you can just use Scalar!
+
+Using `scalar register` on an existing Git repository will give you these
+benefits:
+
+* Additional compression of your `.git/index` file.
+* Hourly background `git fetch` operations, keeping you in-sync with your
+ remotes.
+* Advanced data structures, such as the `commit-graph` and `multi-pack-index`
+ are updated automatically in the background.
+* If using macOS or Windows, then Scalar configures Git's builtin File System
+ Monitor, providing faster commands such as `git status` or `git add`.
+
+Additionally, if you use `scalar clone` to create a new repository, then
+you will automatically get these benefits:
+
+* Use Git's partial clone feature to only download the files you need for
+ your current checkout.
+* Use Git's [sparse-checkout feature][sparse-checkout] to minimize the
+ number of files required in your working directory.
+ [Read more about sparse-checkout here.][sparse-checkout-blog]
+* Create the Git repository inside `/src` to make it easy to
+ place build artifacts outside of the Git repository, such as in
+ `/bin` or `/packages`.
+
+We also admit that these **opinions** can always be improved! If you have
+an idea of how to improve our setup, consider
+[creating an issue](https://github.com/microsoft/scalar/issues/new) or
+contributing a pull request! Some [existing](https://github.com/microsoft/scalar/issues/382)
+[issues](https://github.com/microsoft/scalar/issues/388) have already
+improved our configuration settings and roadmap!
+
+[gvfs-protocol]: https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md
+[microsoft-git]: https://github.com/microsoft/git
+[sparse-checkout]: https://git-scm.com/docs/git-sparse-checkout
+[sparse-checkout-blog]: https://github.blog/2020-01-17-bring-your-monorepo-down-to-size-with-sparse-checkout/
diff --git a/contrib/scalar/docs/troubleshooting.md b/contrib/scalar/docs/troubleshooting.md
new file mode 100644
index 00000000000000..c54d2438f22523
--- /dev/null
+++ b/contrib/scalar/docs/troubleshooting.md
@@ -0,0 +1,40 @@
+Troubleshooting
+===============
+
+Diagnosing Issues
+-----------------
+
+The `scalar diagnose` command collects logs and config details for the current
+repository. The resulting zip file helps root-cause issues.
+
+When run inside your repository, creates a zip file containing several important
+files for that repository. This includes:
+
+* Configuration files from your `.git` folder, such as the `config` file,
+ `index`, `hooks`, and `refs`.
+
+* A summary of your Git object database, including the number of loose objects
+ and the names and sizes of pack-files.
+
+As the `diagnose` command completes, it provides the path of the resulting
+zip file. This zip can be attached to bug reports to make the analysis easier.
+
+Modifying Configuration Values
+------------------------------
+
+The Scalar-specific configuration is only available for repos using the
+GVFS protocol.
+
+### Cache Server URL
+
+When using an enlistment cloned with `scalar clone` and the GVFS protocol,
+you will have a value called the cache server URL. Cache servers are a feature
+of the GVFS protocol to provide low-latency access to the on-demand object
+requests. This modifies the `gvfs.cache-server` setting in your local Git config
+file.
+
+Run `scalar cache-server --get` to see the current cache server.
+
+Run `scalar cache-server --list` to see the available cache server URLs.
+
+Run `scalar cache-server --set=` to set your cache server to ``.
diff --git a/convert.c b/convert.c
index c4ddc4de81b557..f050f98679daea 100644
--- a/convert.c
+++ b/convert.c
@@ -2,6 +2,7 @@
#include "git-compat-util.h"
#include "advice.h"
+#include "gvfs.h"
#include "config.h"
#include "convert.h"
#include "copy.h"
@@ -562,6 +563,9 @@ static int crlf_to_git(struct index_state *istate,
if (!buf)
return 1;
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("CRLF conversions not supported when running under GVFS");
+
/* only grow if not in place */
if (strbuf_avail(buf) + buf->len < len)
strbuf_grow(buf, len - buf->len);
@@ -601,6 +605,9 @@ static int crlf_to_worktree(const char *src, size_t len, struct strbuf *buf,
if (!will_convert_lf_to_crlf(&stats, crlf_action))
return 0;
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("CRLF conversions not supported when running under GVFS");
+
/* are we "faking" in place editing ? */
if (src == buf->buf)
to_free = strbuf_detach(buf, NULL);
@@ -710,6 +717,9 @@ static int apply_single_file_filter(const char *path, const char *src, size_t le
struct async async;
struct filter_params params;
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("Filter \"%s\" not supported when running under GVFS", cmd);
+
memset(&async, 0, sizeof(async));
async.proc = filter_buffer_or_fd;
async.data = ¶ms;
@@ -1123,6 +1133,9 @@ static int ident_to_git(const char *src, size_t len,
if (!buf)
return 1;
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("ident conversions not supported when running under GVFS");
+
/* only grow if not in place */
if (strbuf_avail(buf) + buf->len < len)
strbuf_grow(buf, len - buf->len);
@@ -1170,6 +1183,9 @@ static int ident_to_worktree(const char *src, size_t len,
if (!cnt)
return 0;
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("ident conversions not supported when running under GVFS");
+
/* are we "faking" in place editing ? */
if (src == buf->buf)
to_free = strbuf_detach(buf, NULL);
@@ -1619,6 +1635,9 @@ static int lf_to_crlf_filter_fn(struct stream_filter *filter,
size_t count, o = 0;
struct lf_to_crlf_filter *lf_to_crlf = (struct lf_to_crlf_filter *)filter;
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("CRLF conversions not supported when running under GVFS");
+
/*
* We may be holding onto the CR to see if it is followed by a
* LF, in which case we would need to go to the main loop.
@@ -1863,6 +1882,9 @@ static int ident_filter_fn(struct stream_filter *filter,
struct ident_filter *ident = (struct ident_filter *)filter;
static const char head[] = "$Id";
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("ident conversions not supported when running under GVFS");
+
if (!input) {
/* drain upon eof */
switch (ident->state) {
diff --git a/credential.c b/credential.c
index 4b1a2b94feff73..941a4da0a57c96 100644
--- a/credential.c
+++ b/credential.c
@@ -1,3 +1,4 @@
+#define USE_THE_REPOSITORY_VARIABLE
#include "git-compat-util.h"
#include "abspath.h"
#include "config.h"
@@ -11,6 +12,8 @@
#include "strbuf.h"
#include "urlmatch.h"
#include "git-compat-util.h"
+#include "trace2.h"
+#include "repository.h"
void credential_init(struct credential *c)
{
@@ -249,14 +252,36 @@ static char *credential_ask_one(const char *what, struct credential *c,
return xstrdup(r);
}
-static void credential_getpass(struct credential *c)
+static int credential_getpass(struct credential *c)
{
+ int interactive;
+ char *value;
+ if (!git_config_get_maybe_bool("credential.interactive", &interactive) &&
+ !interactive) {
+ trace2_data_intmax("credential", the_repository,
+ "interactive/skipped", 1);
+ return -1;
+ }
+ if (!git_config_get_string("credential.interactive", &value)) {
+ int same = !strcmp(value, "never");
+ free(value);
+ if (same) {
+ trace2_data_intmax("credential", the_repository,
+ "interactive/skipped", 1);
+ return -1;
+ }
+ }
+
+ trace2_region_enter("credential", "interactive", the_repository);
if (!c->username)
c->username = credential_ask_one("Username", c,
PROMPT_ASKPASS|PROMPT_ECHO);
if (!c->password)
c->password = credential_ask_one("Password", c,
PROMPT_ASKPASS);
+ trace2_region_leave("credential", "interactive", the_repository);
+
+ return 0;
}
int credential_has_capability(const struct credential_capability *capa,
@@ -417,6 +442,8 @@ static int run_credential_helper(struct credential *c,
else
helper.no_stdout = 1;
+ helper.trace2_child_class = "cred";
+
if (start_command(&helper) < 0)
return -1;
@@ -499,8 +526,8 @@ void credential_fill(struct credential *c, int all_capabilities)
c->helpers.items[i].string);
}
- credential_getpass(c);
- if (!c->username && !c->password && !c->credential)
+ if (credential_getpass(c) ||
+ (!c->username && !c->password && !c->credential))
die("unable to get password from user");
}
diff --git a/diagnose.c b/diagnose.c
index cc2d535b60d7c8..6386e93aaab939 100644
--- a/diagnose.c
+++ b/diagnose.c
@@ -13,6 +13,7 @@
#include "packfile.h"
#include "parse-options.h"
#include "write-or-die.h"
+#include "config.h"
struct archive_dir {
const char *path;
@@ -73,6 +74,39 @@ static int dir_file_stats(struct object_directory *object_dir, void *data)
return 0;
}
+static void dir_stats(struct strbuf *buf, const char *path)
+{
+ DIR *dir = opendir(path);
+ struct dirent *e;
+ struct stat e_stat;
+ struct strbuf file_path = STRBUF_INIT;
+ size_t base_path_len;
+
+ if (!dir)
+ return;
+
+ strbuf_addstr(buf, "Contents of ");
+ strbuf_add_absolute_path(buf, path);
+ strbuf_addstr(buf, ":\n");
+
+ strbuf_add_absolute_path(&file_path, path);
+ strbuf_addch(&file_path, '/');
+ base_path_len = file_path.len;
+
+ while ((e = readdir(dir)) != NULL)
+ if (!is_dot_or_dotdot(e->d_name) && e->d_type == DT_REG) {
+ strbuf_setlen(&file_path, base_path_len);
+ strbuf_addstr(&file_path, e->d_name);
+ if (!stat(file_path.buf, &e_stat))
+ strbuf_addf(buf, "%-70s %16"PRIuMAX"\n",
+ e->d_name,
+ (uintmax_t)e_stat.st_size);
+ }
+
+ strbuf_release(&file_path);
+ closedir(dir);
+}
+
static int count_files(struct strbuf *path)
{
DIR *dir = opendir(path->buf);
@@ -185,7 +219,8 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode)
struct strvec archiver_args = STRVEC_INIT;
char **argv_copy = NULL;
int stdout_fd = -1, archiver_fd = -1;
- struct strbuf buf = STRBUF_INIT;
+ char *cache_server_url = NULL, *shared_cache = NULL;
+ struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT;
int res, i;
struct archive_dir archive_dirs[] = {
{ ".git", 0 },
@@ -220,6 +255,13 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode)
get_version_info(&buf, 1);
strbuf_addf(&buf, "Repository root: %s\n", the_repository->worktree);
+
+ git_config_get_string("gvfs.cache-server", &cache_server_url);
+ git_config_get_string("gvfs.sharedCache", &shared_cache);
+ strbuf_addf(&buf, "Cache Server: %s\nLocal Cache: %s\n\n",
+ cache_server_url ? cache_server_url : "None",
+ shared_cache ? shared_cache : "None");
+
get_disk_info(&buf);
write_or_die(stdout_fd, buf.buf, buf.len);
strvec_pushf(&archiver_args,
@@ -250,6 +292,52 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode)
}
}
+ if (shared_cache) {
+ size_t path_len;
+
+ strbuf_reset(&buf);
+ strbuf_addf(&path, "%s/pack", shared_cache);
+ strbuf_reset(&buf);
+ strbuf_addstr(&buf, "--add-virtual-file=packs-cached.txt:");
+ dir_stats(&buf, path.buf);
+ strvec_push(&archiver_args, buf.buf);
+
+ strbuf_reset(&buf);
+ strbuf_addstr(&buf, "--add-virtual-file=objects-cached.txt:");
+ loose_objs_stats(&buf, shared_cache);
+ strvec_push(&archiver_args, buf.buf);
+
+ strbuf_reset(&path);
+ strbuf_addf(&path, "%s/info", shared_cache);
+ path_len = path.len;
+
+ if (is_directory(path.buf)) {
+ DIR *dir = opendir(path.buf);
+ struct dirent *e;
+
+ while ((e = readdir(dir))) {
+ if (!strcmp(".", e->d_name) || !strcmp("..", e->d_name))
+ continue;
+ if (e->d_type == DT_DIR)
+ continue;
+
+ strbuf_reset(&buf);
+ strbuf_addf(&buf, "--add-virtual-file=info/%s:", e->d_name);
+
+ strbuf_setlen(&path, path_len);
+ strbuf_addch(&path, '/');
+ strbuf_addstr(&path, e->d_name);
+
+ if (strbuf_read_file(&buf, path.buf, 0) < 0) {
+ res = error_errno(_("could not read '%s'"), path.buf);
+ goto diagnose_cleanup;
+ }
+ strvec_push(&archiver_args, buf.buf);
+ }
+ closedir(dir);
+ }
+ }
+
strvec_pushl(&archiver_args, "--prefix=",
oid_to_hex(the_hash_algo->empty_tree), "--", NULL);
@@ -263,10 +351,13 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode)
goto diagnose_cleanup;
}
- fprintf(stderr, "\n"
- "Diagnostics complete.\n"
- "All of the gathered info is captured in '%s'\n",
- zip_path->buf);
+ strbuf_reset(&buf);
+ strbuf_addf(&buf, "\n"
+ "Diagnostics complete.\n"
+ "All of the gathered info is captured in '%s'\n",
+ zip_path->buf);
+ write_or_die(stdout_fd, buf.buf, buf.len);
+ write_or_die(2, buf.buf, buf.len);
diagnose_cleanup:
if (archiver_fd >= 0) {
@@ -277,6 +368,8 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode)
free(argv_copy);
strvec_clear(&archiver_args);
strbuf_release(&buf);
+ free(cache_server_url);
+ free(shared_cache);
return res;
}
diff --git a/diff.c b/diff.c
index ebb7538e040a71..0a86e8b53a96b9 100644
--- a/diff.c
+++ b/diff.c
@@ -4040,6 +4040,13 @@ static int reuse_worktree_file(struct index_state *istate,
if (!FAST_WORKING_DIRECTORY && !want_file && has_object_pack(oid))
return 0;
+ /*
+ * If this path does not match our sparse-checkout definition,
+ * then the file will not be in the working directory.
+ */
+ if (!path_in_sparse_checkout(name, istate))
+ return 0;
+
/*
* Similarly, if we'd have to convert the file contents anyway, that
* makes the optimization not worthwhile.
diff --git a/dir.c b/dir.c
index 837c812a149e08..6c7c85ec0d77f5 100644
--- a/dir.c
+++ b/dir.c
@@ -10,6 +10,7 @@
#include "git-compat-util.h"
#include "abspath.h"
+#include "virtualfilesystem.h"
#include "config.h"
#include "convert.h"
#include "dir.h"
@@ -1478,6 +1479,19 @@ enum pattern_match_result path_matches_pattern_list(
int result = NOT_MATCHED;
size_t slash_pos;
+ if (core_virtualfilesystem) {
+ /*
+ * The virtual file system data is used to prevent git from traversing
+ * any part of the tree that is not in the virtual file system. Return
+ * 1 to exclude the entry if it is not found in the virtual file system,
+ * else fall through to the regular excludes logic as it may further exclude.
+ */
+ if (*dtype == DT_UNKNOWN)
+ *dtype = resolve_dtype(DT_UNKNOWN, istate, pathname, pathlen);
+ if (is_excluded_from_virtualfilesystem(pathname, pathlen, *dtype) > 0)
+ return 1;
+ }
+
if (!pl->use_cone_patterns) {
pattern = last_matching_pattern_from_list(pathname, pathlen, basename,
dtype, pl, istate);
@@ -1567,6 +1581,13 @@ static int path_in_sparse_checkout_1(const char *path,
enum pattern_match_result match = UNDECIDED;
const char *end, *slash;
+ /*
+ * When using a virtual filesystem, there aren't really patterns
+ * to follow, but be extra careful to skip this check.
+ */
+ if (core_virtualfilesystem)
+ return 1;
+
/*
* We default to accepting a path if the path is empty, there are no
* patterns, or the patterns are of the wrong type.
@@ -1822,8 +1843,22 @@ struct path_pattern *last_matching_pattern(struct dir_struct *dir,
int is_excluded(struct dir_struct *dir, struct index_state *istate,
const char *pathname, int *dtype_p)
{
- struct path_pattern *pattern =
- last_matching_pattern(dir, istate, pathname, dtype_p);
+ struct path_pattern *pattern;
+
+ if (core_virtualfilesystem) {
+ /*
+ * The virtual file system data is used to prevent git from traversing
+ * any part of the tree that is not in the virtual file system. Return
+ * 1 to exclude the entry if it is not found in the virtual file system,
+ * else fall through to the regular excludes logic as it may further exclude.
+ */
+ if (*dtype_p == DT_UNKNOWN)
+ *dtype_p = resolve_dtype(DT_UNKNOWN, istate, pathname, strlen(pathname));
+ if (is_excluded_from_virtualfilesystem(pathname, strlen(pathname), *dtype_p) > 0)
+ return 1;
+ }
+
+ pattern = last_matching_pattern(dir, istate, pathname, dtype_p);
if (pattern)
return pattern->flags & PATTERN_FLAG_NEGATIVE ? 0 : 1;
return 0;
@@ -2443,6 +2478,8 @@ static enum path_treatment treat_path(struct dir_struct *dir,
ignore_case);
if (dtype != DT_DIR && has_path_in_index)
return path_none;
+ if (is_excluded_from_virtualfilesystem(path->buf, path->len, dtype) > 0)
+ return path_excluded;
/*
* When we are looking at a directory P in the working tree,
@@ -2647,6 +2684,8 @@ static void add_path_to_appropriate_result_list(struct dir_struct *dir,
/* add the path to the appropriate result list */
switch (state) {
case path_excluded:
+ if (is_excluded_from_virtualfilesystem(path->buf, path->len, DT_DIR) > 0)
+ break;
if (dir->flags & DIR_SHOW_IGNORED)
dir_add_name(dir, istate, path->buf, path->len);
else if ((dir->flags & DIR_SHOW_IGNORED_TOO) ||
@@ -3194,6 +3233,8 @@ static int cmp_icase(char a, char b)
{
if (a == b)
return 0;
+ if (is_dir_sep(a))
+ return is_dir_sep(b) ? 0 : -1;
if (ignore_case)
return toupper(a) - toupper(b);
return a - b;
diff --git a/environment.c b/environment.c
index 5cea2c9f5473e3..919e4ba2dd7be6 100644
--- a/environment.c
+++ b/environment.c
@@ -80,9 +80,12 @@ int grafts_keep_true_parents;
int core_apply_sparse_checkout;
int core_sparse_checkout_cone;
int sparse_expect_files_outside_of_patterns;
+int core_gvfs;
+char *core_virtualfilesystem;
int merge_log_config = -1;
int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */
unsigned long pack_size_limit_cfg;
+int core_virtualize_objects;
enum log_refs_config log_all_ref_updates = LOG_REFS_UNSET;
int max_allowed_tree_depth =
#ifdef _MSC_VER
@@ -108,6 +111,9 @@ int protect_hfs = PROTECT_HFS_DEFAULT;
#define PROTECT_NTFS_DEFAULT 1
#endif
int protect_ntfs = PROTECT_NTFS_DEFAULT;
+int core_use_gvfs_helper;
+char *gvfs_cache_server_url;
+struct strbuf gvfs_shared_cache_pathname = STRBUF_INIT;
/*
* The character that begins a commented line in user-editable file
diff --git a/environment.h b/environment.h
index e9f01d4d11c010..b514d4a40fa227 100644
--- a/environment.h
+++ b/environment.h
@@ -11,6 +11,8 @@ struct strvec;
extern const char *comment_line_str;
extern int auto_comment_line_char;
+extern int core_virtualize_objects;
+
/*
* Wrapper of getenv() that returns a strdup value. This value is kept
* in argv to be freed later.
@@ -153,9 +155,14 @@ int get_shared_repository(void);
void reset_shared_repository(void);
extern int core_preload_index;
+extern char *core_virtualfilesystem;
+extern int core_gvfs;
extern int precomposed_unicode;
extern int protect_hfs;
extern int protect_ntfs;
+extern int core_use_gvfs_helper;
+extern char *gvfs_cache_server_url;
+extern struct strbuf gvfs_shared_cache_pathname;
extern int core_apply_sparse_checkout;
extern int core_sparse_checkout_cone;
diff --git a/git.c b/git.c
index e35af9b0e5e976..b656bee3a534b6 100644
--- a/git.c
+++ b/git.c
@@ -1,6 +1,7 @@
#define USE_THE_REPOSITORY_VARIABLE
#include "builtin.h"
+#include "gvfs.h"
#include "config.h"
#include "environment.h"
#include "exec-cmd.h"
@@ -17,6 +18,8 @@
#include "shallow.h"
#include "trace.h"
#include "trace2.h"
+#include "dir.h"
+#include "hook.h"
#define RUN_SETUP (1<<0)
#define RUN_SETUP_GENTLY (1<<1)
@@ -28,6 +31,7 @@
#define NEED_WORK_TREE (1<<3)
#define DELAY_PAGER_CONFIG (1<<4)
#define NO_PARSEOPT (1<<5) /* parse-options is not used */
+#define BLOCK_ON_GVFS_REPO (1<<6) /* command not allowed in GVFS repos */
struct cmd_struct {
const char *cmd;
@@ -434,6 +438,68 @@ static int handle_alias(int *argcp, const char ***argv)
return ret;
}
+/* Runs pre/post-command hook */
+static struct strvec sargv = STRVEC_INIT;
+static int run_post_hook = 0;
+static int exit_code = -1;
+
+static int run_pre_command_hook(const char **argv)
+{
+ char *lock;
+ int ret = 0;
+ struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT;
+
+ /*
+ * Ensure the global pre/post command hook is only called for
+ * the outer command and not when git is called recursively
+ * or spawns multiple commands (like with the alias command)
+ */
+ lock = getenv("COMMAND_HOOK_LOCK");
+ if (lock && !strcmp(lock, "true"))
+ return 0;
+ setenv("COMMAND_HOOK_LOCK", "true", 1);
+
+ /* call the hook proc */
+ strvec_pushv(&sargv, argv);
+ strvec_pushf(&sargv, "--git-pid=%"PRIuMAX, (uintmax_t)getpid());
+ strvec_pushv(&opt.args, sargv.v);
+ ret = run_hooks_opt("pre-command", &opt);
+
+ if (!ret)
+ run_post_hook = 1;
+ return ret;
+}
+
+static int run_post_command_hook(void)
+{
+ char *lock;
+ int ret = 0;
+ struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT;
+
+ /*
+ * Only run post_command if pre_command succeeded in this process
+ */
+ if (!run_post_hook)
+ return 0;
+ lock = getenv("COMMAND_HOOK_LOCK");
+ if (!lock || strcmp(lock, "true"))
+ return 0;
+
+ strvec_pushv(&opt.args, sargv.v);
+ strvec_pushf(&opt.args, "--exit_code=%u", exit_code);
+ ret = run_hooks_opt("post-command", &opt);
+
+ run_post_hook = 0;
+ strvec_clear(&sargv);
+ setenv("COMMAND_HOOK_LOCK", "false", 1);
+ return ret;
+}
+
+static void post_command_hook_atexit(void)
+{
+ run_post_command_hook();
+}
+
static int run_builtin(struct cmd_struct *p, int argc, const char **argv)
{
int status, help;
@@ -469,16 +535,24 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv)
if (!help && p->option & NEED_WORK_TREE)
setup_work_tree();
+ if (!help && p->option & BLOCK_ON_GVFS_REPO && gvfs_config_is_set(GVFS_BLOCK_COMMANDS))
+ die("'git %s' is not supported on a GVFS repo", p->cmd);
+
+ if (run_pre_command_hook(argv))
+ die("pre-command hook aborted command");
+
trace_argv_printf(argv, "trace: built-in: git");
trace2_cmd_name(p->cmd);
validate_cache_entries(the_repository->index);
- status = p->fn(argc, argv, prefix);
+ exit_code = status = p->fn(argc, argv, prefix);
validate_cache_entries(the_repository->index);
if (status)
return status;
+ run_post_command_hook();
+
/* Somebody closed stdout? */
if (fstat(fileno(stdout), &st))
return 0;
@@ -546,7 +620,7 @@ static struct cmd_struct commands[] = {
{ "for-each-ref", cmd_for_each_ref, RUN_SETUP },
{ "for-each-repo", cmd_for_each_repo, RUN_SETUP_GENTLY },
{ "format-patch", cmd_format_patch, RUN_SETUP },
- { "fsck", cmd_fsck, RUN_SETUP },
+ { "fsck", cmd_fsck, RUN_SETUP | BLOCK_ON_GVFS_REPO},
{ "fsck-objects", cmd_fsck, RUN_SETUP },
{ "fsmonitor--daemon", cmd_fsmonitor__daemon, RUN_SETUP },
{ "gc", cmd_gc, RUN_SETUP },
@@ -587,7 +661,7 @@ static struct cmd_struct commands[] = {
{ "pack-refs", cmd_pack_refs, RUN_SETUP },
{ "patch-id", cmd_patch_id, RUN_SETUP_GENTLY | NO_PARSEOPT },
{ "pickaxe", cmd_blame, RUN_SETUP },
- { "prune", cmd_prune, RUN_SETUP },
+ { "prune", cmd_prune, RUN_SETUP | BLOCK_ON_GVFS_REPO},
{ "prune-packed", cmd_prune_packed, RUN_SETUP },
{ "pull", cmd_pull, RUN_SETUP | NEED_WORK_TREE },
{ "push", cmd_push, RUN_SETUP },
@@ -600,7 +674,7 @@ static struct cmd_struct commands[] = {
{ "remote", cmd_remote, RUN_SETUP },
{ "remote-ext", cmd_remote_ext, NO_PARSEOPT },
{ "remote-fd", cmd_remote_fd, NO_PARSEOPT },
- { "repack", cmd_repack, RUN_SETUP },
+ { "repack", cmd_repack, RUN_SETUP | BLOCK_ON_GVFS_REPO },
{ "replace", cmd_replace, RUN_SETUP },
{ "replay", cmd_replay, RUN_SETUP },
{ "rerere", cmd_rerere, RUN_SETUP },
@@ -621,13 +695,15 @@ static struct cmd_struct commands[] = {
{ "stash", cmd_stash, RUN_SETUP | NEED_WORK_TREE },
{ "status", cmd_status, RUN_SETUP | NEED_WORK_TREE },
{ "stripspace", cmd_stripspace },
- { "submodule--helper", cmd_submodule__helper, RUN_SETUP },
+ { "submodule--helper", cmd_submodule__helper, RUN_SETUP | BLOCK_ON_GVFS_REPO },
+ { "survey", cmd_survey, RUN_SETUP },
{ "switch", cmd_switch, RUN_SETUP | NEED_WORK_TREE },
{ "symbolic-ref", cmd_symbolic_ref, RUN_SETUP },
{ "tag", cmd_tag, RUN_SETUP | DELAY_PAGER_CONFIG },
{ "unpack-file", cmd_unpack_file, RUN_SETUP | NO_PARSEOPT },
{ "unpack-objects", cmd_unpack_objects, RUN_SETUP | NO_PARSEOPT },
{ "update-index", cmd_update_index, RUN_SETUP },
+ { "update-microsoft-git", cmd_update_microsoft_git },
{ "update-ref", cmd_update_ref, RUN_SETUP },
{ "update-server-info", cmd_update_server_info, RUN_SETUP },
{ "upload-archive", cmd_upload_archive, NO_PARSEOPT },
@@ -757,13 +833,16 @@ static void execv_dashed_external(const char **argv)
*/
trace_argv_printf(cmd.args.v, "trace: exec:");
+ if (run_pre_command_hook(cmd.args.v))
+ die("pre-command hook aborted command");
+
/*
* If we fail because the command is not found, it is
* OK to return. Otherwise, we just pass along the status code,
* or our usual generic code if we were not even able to exec
* the program.
*/
- status = run_command(&cmd);
+ exit_code = status = run_command(&cmd);
/*
* If the child process ran and we are now going to exit, emit a
@@ -774,6 +853,8 @@ static void execv_dashed_external(const char **argv)
exit(status);
else if (errno != ENOENT)
exit(128);
+
+ run_post_command_hook();
}
static int run_argv(int *argcp, const char ***argv)
@@ -881,6 +962,7 @@ int cmd_main(int argc, const char **argv)
}
trace_command_performance(argv);
+ atexit(post_command_hook_atexit);
/*
* "git-xxxx" is the same as "git xxxx", but we obviously:
@@ -906,10 +988,14 @@ int cmd_main(int argc, const char **argv)
if (!argc) {
/* The user didn't specify a command; give them help */
commit_pager_choice();
+ if (run_pre_command_hook(argv))
+ die("pre-command hook aborted command");
printf(_("usage: %s\n\n"), git_usage_string);
list_common_cmds_help();
printf("\n%s\n", _(git_more_info_string));
- exit(1);
+ exit_code = 1;
+ run_post_command_hook();
+ exit(exit_code);
}
if (!strcmp("--version", argv[0]) || !strcmp("-v", argv[0]))
diff --git a/gvfs-helper-client.c b/gvfs-helper-client.c
new file mode 100644
index 00000000000000..45d4d86783c73b
--- /dev/null
+++ b/gvfs-helper-client.c
@@ -0,0 +1,581 @@
+#define USE_THE_REPOSITORY_VARIABLE
+#include "git-compat-util.h"
+#include "environment.h"
+#include "hex.h"
+#include "strvec.h"
+#include "trace2.h"
+#include "oidset.h"
+#include "object.h"
+#include "object-store.h"
+#include "gvfs-helper-client.h"
+#include "sub-process.h"
+#include "sigchain.h"
+#include "pkt-line.h"
+#include "quote.h"
+#include "packfile.h"
+#include "hex.h"
+#include "config.h"
+
+static struct oidset gh_client__oidset_queued = OIDSET_INIT;
+static unsigned long gh_client__oidset_count;
+
+struct gh_server__process {
+ struct subprocess_entry subprocess; /* must be first */
+ unsigned int supported_capabilities;
+};
+
+static int gh_server__subprocess_map_initialized;
+static struct hashmap gh_server__subprocess_map;
+static struct object_directory *gh_client__chosen_odb;
+
+/*
+ * The "objects" capability has verbs: "get" and "post" and "prefetch".
+ */
+#define CAP_OBJECTS (1u<<1)
+#define CAP_OBJECTS_NAME "objects"
+
+#define CAP_OBJECTS__VERB_GET1_NAME "get"
+#define CAP_OBJECTS__VERB_POST_NAME "post"
+#define CAP_OBJECTS__VERB_PREFETCH_NAME "prefetch"
+
+static int gh_client__start_fn(struct subprocess_entry *subprocess)
+{
+ static int versions[] = {1, 0};
+ static struct subprocess_capability capabilities[] = {
+ { CAP_OBJECTS_NAME, CAP_OBJECTS },
+ { NULL, 0 }
+ };
+
+ struct gh_server__process *entry = (struct gh_server__process *)subprocess;
+
+ return subprocess_handshake(subprocess, "gvfs-helper", versions,
+ NULL, capabilities,
+ &entry->supported_capabilities);
+}
+
+/*
+ * Send the queued OIDs in the OIDSET to gvfs-helper for it to
+ * fetch from the cache-server or main Git server using "/gvfs/objects"
+ * POST semantics.
+ *
+ * objects.post LF
+ * ( LF)*
+ *
+ *
+ */
+static int gh_client__send__objects_post(struct child_process *process)
+{
+ struct oidset_iter iter;
+ struct object_id *oid;
+ int err;
+
+ /*
+ * We assume that all of the packet_ routines call error()
+ * so that we don't have to.
+ */
+
+ err = packet_write_fmt_gently(
+ process->in,
+ (CAP_OBJECTS_NAME "." CAP_OBJECTS__VERB_POST_NAME "\n"));
+ if (err)
+ return err;
+
+ oidset_iter_init(&gh_client__oidset_queued, &iter);
+ while ((oid = oidset_iter_next(&iter))) {
+ err = packet_write_fmt_gently(process->in, "%s\n",
+ oid_to_hex(oid));
+ if (err)
+ return err;
+ }
+
+ err = packet_flush_gently(process->in);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+/*
+ * Send the given OID to gvfs-helper for it to fetch from the
+ * cache-server or main Git server using "/gvfs/objects" GET
+ * semantics.
+ *
+ * This ignores any queued OIDs.
+ *
+ * objects.get LF
+ * LF
+ *
+ *
+ */
+static int gh_client__send__objects_get(struct child_process *process,
+ const struct object_id *oid)
+{
+ int err;
+
+ /*
+ * We assume that all of the packet_ routines call error()
+ * so that we don't have to.
+ */
+
+ err = packet_write_fmt_gently(
+ process->in,
+ (CAP_OBJECTS_NAME "." CAP_OBJECTS__VERB_GET1_NAME "\n"));
+ if (err)
+ return err;
+
+ err = packet_write_fmt_gently(process->in, "%s\n",
+ oid_to_hex(oid));
+ if (err)
+ return err;
+
+ err = packet_flush_gently(process->in);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+/*
+ * Send a request to gvfs-helper to prefetch packfiles from either the
+ * cache-server or the main Git server using "/gvfs/prefetch".
+ *
+ * objects.prefetch LF
+ * [ LF]
+ *
+ */
+static int gh_client__send__objects_prefetch(struct child_process *process,
+ timestamp_t seconds_since_epoch)
+{
+ int err;
+
+ /*
+ * We assume that all of the packet_ routines call error()
+ * so that we don't have to.
+ */
+
+ err = packet_write_fmt_gently(
+ process->in,
+ (CAP_OBJECTS_NAME "." CAP_OBJECTS__VERB_PREFETCH_NAME "\n"));
+ if (err)
+ return err;
+
+ if (seconds_since_epoch) {
+ err = packet_write_fmt_gently(process->in, "%" PRItime "\n",
+ seconds_since_epoch);
+ if (err)
+ return err;
+ }
+
+ err = packet_flush_gently(process->in);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+/*
+ * Update the loose object cache to include the newly created
+ * object.
+ */
+static void gh_client__update_loose_cache(const char *line)
+{
+ const char *v1_oid;
+ struct object_id oid;
+
+ if (!skip_prefix(line, "loose ", &v1_oid))
+ BUG("update_loose_cache: invalid line '%s'", line);
+
+ if (get_oid_hex(v1_oid, &oid))
+ BUG("update_loose_cache: invalid line '%s'", line);
+
+ odb_loose_cache_add_new_oid(gh_client__chosen_odb, &oid);
+}
+
+/*
+ * Update the packed-git list to include the newly created packfile.
+ */
+static void gh_client__update_packed_git(const char *line)
+{
+ struct strbuf path = STRBUF_INIT;
+ const char *v1_filename;
+ struct packed_git *p;
+ int is_local;
+
+ if (!skip_prefix(line, "packfile ", &v1_filename))
+ BUG("update_packed_git: invalid line '%s'", line);
+
+ /*
+ * ODB[0] is the local .git/objects. All others are alternates.
+ */
+ is_local = (gh_client__chosen_odb == the_repository->objects->odb);
+
+ strbuf_addf(&path, "%s/pack/%s",
+ gh_client__chosen_odb->path, v1_filename);
+ strbuf_strip_suffix(&path, ".pack");
+ strbuf_addstr(&path, ".idx");
+
+ p = add_packed_git(path.buf, path.len, is_local);
+ if (p)
+ install_packed_git_and_mru(the_repository, p);
+}
+
+/*
+ * CAP_OBJECTS verbs return the same format response:
+ *
+ *
+ * *
+ *
+ *
+ *
+ * Where:
+ *
+ * ::= odb SP LF
+ *
+ * ::= /
+ *
+ * ::= packfile SP LF
+ *
+ * ::= loose SP LF
+ *
+ * ::= ok LF
+ * / partial LF
+ * / error SP LF
+ *
+ * Note that `gvfs-helper` controls how/if it chunks the request when
+ * it talks to the cache-server and/or main Git server. So it is
+ * possible for us to receive many packfiles and/or loose objects *AND
+ * THEN* get a hard network error or a 404 on an individual object.
+ *
+ * If we get a partial result, we can let the caller try to continue
+ * -- for example, maybe an immediate request for a tree object was
+ * grouped with a queued request for a blob. The tree-walk *might* be
+ * able to continue and let the 404 blob be handled later.
+ */
+static int gh_client__objects__receive_response(
+ struct child_process *process,
+ enum gh_client__created *p_ghc,
+ int *p_nr_loose, int *p_nr_packfile)
+{
+ enum gh_client__created ghc = GHC__CREATED__NOTHING;
+ const char *v1;
+ char *line;
+ int len;
+ int nr_loose = 0;
+ int nr_packfile = 0;
+ int err = 0;
+
+ while (1) {
+ /*
+ * Warning: packet_read_line_gently() calls die()
+ * despite the _gently moniker.
+ */
+ len = packet_read_line_gently(process->out, NULL, &line);
+ if ((len < 0) || !line)
+ break;
+
+ if (starts_with(line, "odb")) {
+ /* trust that this matches what we expect */
+ }
+
+ else if (starts_with(line, "packfile")) {
+ gh_client__update_packed_git(line);
+ ghc |= GHC__CREATED__PACKFILE;
+ nr_packfile++;
+ }
+
+ else if (starts_with(line, "loose")) {
+ gh_client__update_loose_cache(line);
+ ghc |= GHC__CREATED__LOOSE;
+ nr_loose++;
+ }
+
+ else if (starts_with(line, "ok"))
+ ;
+ else if (starts_with(line, "partial"))
+ ;
+ else if (skip_prefix(line, "error ", &v1)) {
+ error("gvfs-helper error: '%s'", v1);
+ err = -1;
+ }
+ }
+
+ *p_ghc = ghc;
+ *p_nr_loose = nr_loose;
+ *p_nr_packfile = nr_packfile;
+
+ return err;
+}
+
+/*
+ * Select the preferred ODB for fetching missing objects.
+ * This should be the alternate with the same directory
+ * name as set in `gvfs.sharedCache`.
+ *
+ * Fallback to .git/objects if necessary.
+ */
+static void gh_client__choose_odb(void)
+{
+ struct object_directory *odb;
+
+ if (gh_client__chosen_odb)
+ return;
+
+ prepare_alt_odb(the_repository);
+ gh_client__chosen_odb = the_repository->objects->odb;
+
+ if (!gvfs_shared_cache_pathname.len)
+ return;
+
+ for (odb = the_repository->objects->odb->next; odb; odb = odb->next) {
+ if (!fspathcmp(odb->path, gvfs_shared_cache_pathname.buf)) {
+ gh_client__chosen_odb = odb;
+ return;
+ }
+ }
+}
+
+static struct gh_server__process *gh_client__find_long_running_process(
+ unsigned int cap_needed)
+{
+ struct gh_server__process *entry;
+ struct strvec argv = STRVEC_INIT;
+ struct strbuf quoted = STRBUF_INIT;
+ int fallback;
+
+ gh_client__choose_odb();
+
+ /*
+ * TODO decide what defaults we want.
+ */
+ strvec_push(&argv, "gvfs-helper");
+ strvec_push(&argv, "--cache-server=trust");
+ strvec_pushf(&argv, "--shared-cache=%s",
+ gh_client__chosen_odb->path);
+
+ /* If gvfs.fallback=false, then don't add --fallback. */
+ if (!git_config_get_bool("gvfs.fallback", &fallback) &&
+ !fallback)
+ strvec_push(&argv, "--no-fallback");
+ else
+ strvec_push(&argv, "--fallback");
+
+ strvec_push(&argv, "server");
+
+ sq_quote_argv_pretty("ed, argv.v);
+
+ /*
+ * Find an existing long-running process with the above command
+ * line -or- create a new long-running process for this and
+ * subsequent requests.
+ */
+ if (!gh_server__subprocess_map_initialized) {
+ gh_server__subprocess_map_initialized = 1;
+ hashmap_init(&gh_server__subprocess_map,
+ (hashmap_cmp_fn)cmd2process_cmp, NULL, 0);
+ entry = NULL;
+ } else
+ entry = (struct gh_server__process *)subprocess_find_entry(
+ &gh_server__subprocess_map, quoted.buf);
+
+ if (!entry) {
+ entry = xmalloc(sizeof(*entry));
+ entry->supported_capabilities = 0;
+
+ if (subprocess_start_strvec(&gh_server__subprocess_map,
+ &entry->subprocess, 1,
+ &argv, gh_client__start_fn))
+ FREE_AND_NULL(entry);
+ }
+
+ if (entry &&
+ (entry->supported_capabilities & cap_needed) != cap_needed) {
+ error("gvfs-helper: does not support needed capabilities");
+ subprocess_stop(&gh_server__subprocess_map,
+ (struct subprocess_entry *)entry);
+ FREE_AND_NULL(entry);
+ }
+
+ strvec_clear(&argv);
+ strbuf_release("ed);
+
+ return entry;
+}
+
+void gh_client__queue_oid(const struct object_id *oid)
+{
+ /*
+ * Keep this trace as a printf only, so that it goes to the
+ * perf log, but not the event log. It is useful for interactive
+ * debugging, but generates way too much (unuseful) noise for the
+ * database.
+ */
+ if (trace2_is_enabled())
+ trace2_printf("gh_client__queue_oid: %s", oid_to_hex(oid));
+
+ if (!oidset_insert(&gh_client__oidset_queued, oid))
+ gh_client__oidset_count++;
+}
+
+/*
+ * This routine should actually take a "const struct oid_array *"
+ * rather than the component parts, but fetch_objects() uses
+ * this model (because of the call in sha1-file.c).
+ */
+void gh_client__queue_oid_array(const struct object_id *oids, int oid_nr)
+{
+ int k;
+
+ for (k = 0; k < oid_nr; k++)
+ gh_client__queue_oid(&oids[k]);
+}
+
+/*
+ * Bulk fetch all of the queued OIDs in the OIDSET.
+ */
+int gh_client__drain_queue(enum gh_client__created *p_ghc)
+{
+ struct gh_server__process *entry;
+ struct child_process *process;
+ int nr_loose = 0;
+ int nr_packfile = 0;
+ int err = 0;
+
+ *p_ghc = GHC__CREATED__NOTHING;
+
+ if (!gh_client__oidset_count)
+ return 0;
+
+ entry = gh_client__find_long_running_process(CAP_OBJECTS);
+ if (!entry)
+ return -1;
+
+ trace2_region_enter("gh-client", "objects/post", the_repository);
+
+ process = &entry->subprocess.process;
+
+ sigchain_push(SIGPIPE, SIG_IGN);
+
+ err = gh_client__send__objects_post(process);
+ if (!err)
+ err = gh_client__objects__receive_response(
+ process, p_ghc, &nr_loose, &nr_packfile);
+
+ sigchain_pop(SIGPIPE);
+
+ if (err) {
+ subprocess_stop(&gh_server__subprocess_map,
+ (struct subprocess_entry *)entry);
+ FREE_AND_NULL(entry);
+ }
+
+ trace2_data_intmax("gh-client", the_repository,
+ "objects/post/nr_objects", gh_client__oidset_count);
+ trace2_region_leave("gh-client", "objects/post", the_repository);
+
+ oidset_clear(&gh_client__oidset_queued);
+ gh_client__oidset_count = 0;
+
+ return err;
+}
+
+/*
+ * Get exactly 1 object immediately.
+ * Ignore any queued objects.
+ */
+int gh_client__get_immediate(const struct object_id *oid,
+ enum gh_client__created *p_ghc)
+{
+ struct gh_server__process *entry;
+ struct child_process *process;
+ int nr_loose = 0;
+ int nr_packfile = 0;
+ int err = 0;
+
+ /*
+ * Keep this trace as a printf only, so that it goes to the
+ * perf log, but not the event log. It is useful for interactive
+ * debugging, but generates way too much (unuseful) noise for the
+ * database.
+ */
+ if (trace2_is_enabled())
+ trace2_printf("gh_client__get_immediate: %s", oid_to_hex(oid));
+
+ entry = gh_client__find_long_running_process(CAP_OBJECTS);
+ if (!entry)
+ return -1;
+
+ trace2_region_enter("gh-client", "objects/get", the_repository);
+
+ process = &entry->subprocess.process;
+
+ sigchain_push(SIGPIPE, SIG_IGN);
+
+ err = gh_client__send__objects_get(process, oid);
+ if (!err)
+ err = gh_client__objects__receive_response(
+ process, p_ghc, &nr_loose, &nr_packfile);
+
+ sigchain_pop(SIGPIPE);
+
+ if (err) {
+ subprocess_stop(&gh_server__subprocess_map,
+ (struct subprocess_entry *)entry);
+ FREE_AND_NULL(entry);
+ }
+
+ trace2_region_leave("gh-client", "objects/get", the_repository);
+
+ return err;
+}
+
+/*
+ * Ask gvfs-helper to prefetch commits-and-trees packfiles since a
+ * given timestamp.
+ *
+ * If seconds_since_epoch is zero, gvfs-helper will scan the ODB for
+ * the last received prefetch and ask for ones newer than that.
+ */
+int gh_client__prefetch(timestamp_t seconds_since_epoch,
+ int *nr_packfiles_received)
+{
+ struct gh_server__process *entry;
+ struct child_process *process;
+ enum gh_client__created ghc;
+ int nr_loose = 0;
+ int nr_packfile = 0;
+ int err = 0;
+
+ entry = gh_client__find_long_running_process(CAP_OBJECTS);
+ if (!entry)
+ return -1;
+
+ trace2_region_enter("gh-client", "objects/prefetch", the_repository);
+ trace2_data_intmax("gh-client", the_repository, "prefetch/since",
+ seconds_since_epoch);
+
+ process = &entry->subprocess.process;
+
+ sigchain_push(SIGPIPE, SIG_IGN);
+
+ err = gh_client__send__objects_prefetch(process, seconds_since_epoch);
+ if (!err)
+ err = gh_client__objects__receive_response(
+ process, &ghc, &nr_loose, &nr_packfile);
+
+ sigchain_pop(SIGPIPE);
+
+ if (err) {
+ subprocess_stop(&gh_server__subprocess_map,
+ (struct subprocess_entry *)entry);
+ FREE_AND_NULL(entry);
+ }
+
+ trace2_data_intmax("gh-client", the_repository,
+ "prefetch/packfile_count", nr_packfile);
+ trace2_region_leave("gh-client", "objects/prefetch", the_repository);
+
+ if (nr_packfiles_received)
+ *nr_packfiles_received = nr_packfile;
+
+ return err;
+}
diff --git a/gvfs-helper-client.h b/gvfs-helper-client.h
new file mode 100644
index 00000000000000..7692534ecda54c
--- /dev/null
+++ b/gvfs-helper-client.h
@@ -0,0 +1,87 @@
+#ifndef GVFS_HELPER_CLIENT_H
+#define GVFS_HELPER_CLIENT_H
+
+struct repository;
+struct commit;
+struct object_id;
+
+enum gh_client__created {
+ /*
+ * The _get_ operation did not create anything. If doesn't
+ * matter if `gvfs-helper` had errors or not -- just that
+ * nothing was created.
+ */
+ GHC__CREATED__NOTHING = 0,
+
+ /*
+ * The _get_ operation created one or more packfiles.
+ */
+ GHC__CREATED__PACKFILE = 1<<1,
+
+ /*
+ * The _get_ operation created one or more loose objects.
+ * (Not necessarily the for the individual OID you requested.)
+ */
+ GHC__CREATED__LOOSE = 1<<2,
+
+ /*
+ * The _get_ operation created one or more packfilea *and*
+ * one or more loose objects.
+ */
+ GHC__CREATED__PACKFILE_AND_LOOSE = (GHC__CREATED__PACKFILE |
+ GHC__CREATED__LOOSE),
+};
+
+/*
+ * Ask `gvfs-helper server` to immediately fetch a single object
+ * using "/gvfs/objects" GET semantics.
+ *
+ * A long-running background process is used to make subsequent
+ * requests more efficient.
+ *
+ * A loose object will be created in the shared-cache ODB and
+ * in-memory cache updated.
+ */
+int gh_client__get_immediate(const struct object_id *oid,
+ enum gh_client__created *p_ghc);
+
+/*
+ * Queue this OID for a future fetch using `gvfs-helper service`.
+ * It does not wait.
+ *
+ * Callers should not rely on the queued object being on disk until
+ * the queue has been drained.
+ */
+void gh_client__queue_oid(const struct object_id *oid);
+void gh_client__queue_oid_array(const struct object_id *oids, int oid_nr);
+
+/*
+ * Ask `gvfs-helper server` to fetch the set of queued OIDs using
+ * "/gvfs/objects" POST semantics.
+ *
+ * A long-running background process is used to subsequent requests
+ * more efficient.
+ *
+ * One or more packfiles will be created in the shared-cache ODB.
+ */
+int gh_client__drain_queue(enum gh_client__created *p_ghc);
+
+/*
+ * Ask `gvfs-helper server` to fetch any "prefetch packs"
+ * available on the server more recent than the requested time.
+ *
+ * If seconds_since_epoch is zero, gvfs-helper will scan the ODB for
+ * the last received prefetch and ask for ones newer than that.
+ *
+ * A long-running background process is used to subsequent requests
+ * (either prefetch or regular immediate/queued requests) more efficient.
+ *
+ * One or more packfiles will be created in the shared-cache ODB.
+ *
+ * Returns 0 on success, -1 on error. Optionally also returns the
+ * number of prefetch packs received.
+ */
+int gh_client__prefetch(timestamp_t seconds_since_epoch,
+ int *nr_packfiles_received);
+
+#endif /* GVFS_HELPER_CLIENT_H */
diff --git a/gvfs-helper.c b/gvfs-helper.c
new file mode 100644
index 00000000000000..19b819d9972d6a
--- /dev/null
+++ b/gvfs-helper.c
@@ -0,0 +1,4231 @@
+// TODO Write a man page. Here are some notes for dogfooding.
+// TODO
+//
+// Usage: git gvfs-helper [] []
+//
+// :
+//
+// --remote= // defaults to "origin"
+//
+// --fallback // boolean. defaults to off
+//
+// When a fetch from the cache-server fails, automatically
+// fallback to the main Git server. This option has no effect
+// if no cache-server is defined.
+//
+// --cache-server=]