From 49cb579583794cd7190953369009f78544ec6a65 Mon Sep 17 00:00:00 2001 From: Steffen Prohaska Date: Sun, 17 Sep 2017 18:41:30 +0200 Subject: [PATCH] Support standalonetransferagent based on API URL prefix match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-2429 has implemented a mechanism to unconditionally specify standalone custom transfer agents. This commit extends the mechanism to specify custom transfer agents based on an URL prefix match on the API URL. Together with the previous commit, which added the Git remote to the custom transfer stage 1 initiation message, standalone custom transfer can now be configured per remote. One remote can, for example, use Rsync transfer to an SSH server. Another remote can use standard LFS to GitHub. Example Git config: ``` remote.github.url=... remote.origin.url=ssh://gitssh.example.com/git/some/repo lfs.customtransfer.rsync.path=git-lfs-rsync-agent lfs.https://gitssh.example.com/git/.standalonetransferagent=rsync ``` The config assumes that `git-lfs-rsync-agent` determines the remote from the stage 1 init message and then inspecting `remote.origin.url` to infer the rsync host. , 2017-09-17, does not yet do that. [PR-2429] 09b7c5381bc3fa4ba7ca3564681a5ae5b7e6f3dc 'Allow using custom transfer agents directly', . CC: Alexandru Băluț Signed-off-by: Steffen Prohaska --- docs/custom-transfers.md | 20 +++++--- test/test-custom-transfers.sh | 86 +++++++++++++++++++++++++++++++++++ tq/manifest.go | 21 ++++++++- 3 files changed, 119 insertions(+), 8 deletions(-) diff --git a/docs/custom-transfers.md b/docs/custom-transfers.md index ff54b9d656..4984747d09 100644 --- a/docs/custom-transfers.md +++ b/docs/custom-transfers.md @@ -28,12 +28,16 @@ the transfers should be made, without having to query the API server. In this case it's possible to use the custom transfer agent directly, without querying the server, by using the following config option: -* `lfs.standalonetransferagent` +* `lfs.standalonetransferagent`, `lfs..standalonetransferagent` - Allows the specified custom transfer agent to be used directly - for transferring files, without asking the server how the transfers - should be made. The custom transfer agent has to be defined in a - `lfs.customtransfer.` settings group. + Specifies a custom transfer agent to be used if the API server URL matches as + in `git config --get-urlmatch lfs.standalonetransferagent `. + `git-lfs` will not contact the API server. It instead sets stage 2 transfer + actions to `null`. `lfs..standalonetransferagent` can be used to + configure a custom transfer agent for individual remotes. + `lfs.standalonetransferagent` unconditionally configures a custom transfer + agent for all remotes. The custom transfer agent must be specified in + a `lfs.customtransfer.` settings group. ## Defining a Custom Transfer Type @@ -171,7 +175,8 @@ like this: conventions, but can be interpreted however the custom transfer agent wishes (this is an NFS example, but it doesn't even have to be an URL). Generally, `href` will give the primary connection details, with `header` containing any - miscellaneous information needed. + miscellaneous information needed. `action` is `null` for standalone transfer + agents. The transfer process should post one or more [progress messages](#progress) and then a final completion message as follows: @@ -210,7 +215,8 @@ like this: conventions, but can be interpreted however the custom transfer agent wishes (this is an NFS example, but it doesn't even have to be an URL). Generally, `href` will give the primary connection details, with `header` containing any - miscellaneous information needed. + miscellaneous information needed. `action` is `null` for standalone transfer + agents. Note there is no file path included in the download request; the transfer process should create a file itself and return the path in the final response diff --git a/test/test-custom-transfers.sh b/test/test-custom-transfers.sh index b2bd1dd66a..edaad6a4a8 100755 --- a/test/test-custom-transfers.sh +++ b/test/test-custom-transfers.sh @@ -192,3 +192,89 @@ begin_test "custom-transfer-standalone" [ "$(echo "$objectlist" | wc -l)" -eq 12 ] ) end_test + +begin_test "custom-transfer-standalone-urlmatch" +( + set -e + + # setup a git repo to be used as a local repo, not remote + reponame="test-custom-transfer-standalone-urlmatch" + setup_remote_repo "$reponame" + + # clone directly, not through lfstest-gitserver + clone_repo_url "$REMOTEDIR/$reponame.git" $reponame + + # set up custom transfer adapter to use a specific transfer agent, using a URL prefix match + git config lfs.customtransfer.testcustom.path lfstest-standalonecustomadapter + git config lfs.customtransfer.testcustom.concurrent false + git config remote.origin.lfsurl https://git.example.com/example/path/to/repo + git config lfs.https://git.example.com/example/path/.standalonetransferagent testcustom + git config lfs.standalonetransferagent invalid-agent + + # git config lfs.standalonetransferagent testcustom + export TEST_STANDALONE_BACKUP_PATH="$(pwd)/test-custom-transfer-standalone-urlmatch-backup" + mkdir -p $TEST_STANDALONE_BACKUP_PATH + rm -rf $TEST_STANDALONE_BACKUP_PATH/* + + git lfs track "*.dat" 2>&1 | tee track.log + grep "Tracking \"\*.dat\"" track.log + git add .gitattributes + git commit -m "Tracking" + + # set up a decent amount of data so that there's work for multiple concurrent adapters + echo "[ + { + \"CommitDate\":\"$(get_date -10d)\", + \"Files\":[ + {\"Filename\":\"verify.dat\",\"Size\":18,\"Data\":\"send-verify-action\"}, + {\"Filename\":\"file1.dat\",\"Size\":1024}, + {\"Filename\":\"file2.dat\",\"Size\":750}] + }, + { + \"CommitDate\":\"$(get_date -7d)\", + \"Files\":[ + {\"Filename\":\"file1.dat\",\"Size\":1050}, + {\"Filename\":\"file3.dat\",\"Size\":660}, + {\"Filename\":\"file4.dat\",\"Size\":230}] + }, + { + \"CommitDate\":\"$(get_date -5d)\", + \"Files\":[ + {\"Filename\":\"file5.dat\",\"Size\":1200}, + {\"Filename\":\"file6.dat\",\"Size\":300}] + }, + { + \"CommitDate\":\"$(get_date -2d)\", + \"Files\":[ + {\"Filename\":\"file3.dat\",\"Size\":120}, + {\"Filename\":\"file5.dat\",\"Size\":450}, + {\"Filename\":\"file7.dat\",\"Size\":520}, + {\"Filename\":\"file8.dat\",\"Size\":2048}] + } + ]" | lfstest-testutils addcommits + + GIT_TRACE=1 GIT_TRANSFER_TRACE=1 git push origin master 2>&1 | tee pushcustom.log + # use PIPESTATUS otherwise we get exit code from tee + [ ${PIPESTATUS[0]} = "0" ] + + # Make sure the lock verification is not attempted. + grep "locks/verify$" pushcustom.log && false + + grep "xfer: started custom adapter process" pushcustom.log + grep "xfer\[lfstest-standalonecustomadapter\]:" pushcustom.log + grep "12 of 12 files" pushcustom.log + + rm -rf .git/lfs/objects + GIT_TRACE=1 GIT_TRANSFER_TRACE=1 git lfs fetch --all 2>&1 | tee fetchcustom.log + [ ${PIPESTATUS[0]} = "0" ] + + grep "xfer: started custom adapter process" fetchcustom.log + grep "xfer\[lfstest-standalonecustomadapter\]:" fetchcustom.log + grep "12 of 12 files" fetchcustom.log + + grep "Terminating test custom adapter gracefully" fetchcustom.log + + objectlist=`find .git/lfs/objects -type f` + [ "$(echo "$objectlist" | wc -l)" -eq 12 ] +) +end_test diff --git a/tq/manifest.go b/tq/manifest.go index bf86c9bb90..5942e8f3fa 100644 --- a/tq/manifest.go +++ b/tq/manifest.go @@ -3,6 +3,7 @@ package tq import ( "sync" + "github.com/git-lfs/git-lfs/config" "github.com/git-lfs/git-lfs/lfsapi" "github.com/rubyist/tracerx" ) @@ -83,7 +84,9 @@ func NewManifestClientOperationRemote( m.concurrentTransfers = v } m.basicTransfersOnly = git.Bool("lfs.basictransfersonly", false) - m.standaloneTransferAgent, _ = git.Get("lfs.standalonetransferagent") + m.standaloneTransferAgent = findStandaloneTransfer( + apiClient, operation, remote, + ) tusAllowed = git.Bool("lfs.tustransfers", false) configureCustomAdapters(git, m) } @@ -104,6 +107,22 @@ func NewManifestClientOperationRemote( return m } +func findStandaloneTransfer(client *lfsapi.Client, operation, remote string) string { + if operation == "" || remote == "" { + v, _ := client.GitEnv().Get("lfs.standalonetransferagent") + return v + } + + ep := client.Endpoints.RemoteEndpoint(operation, remote) + uc := config.NewURLConfig(client.GitEnv()) + v, ok := uc.Get("lfs", ep.Url, "standalonetransferagent") + if !ok { + return "" + } + + return v +} + // GetAdapterNames returns a list of the names of adapters available to be created func (m *Manifest) GetAdapterNames(dir Direction) []string { switch dir {