Support standalonetransferagent based on API URL prefix match

PR-2429 has implemented a mechanism to unconditionally specify
standalone custom transfer agents.

This commit extends the mechanism to specify custom transfer agents
based on an URL prefix match on the API URL.

Together with the previous commit, which added the Git remote to the
custom transfer stage 1 initiation message, standalone custom transfer
can now be configured per remote.  One remote can, for example, use
Rsync transfer to an SSH server.  Another remote can use standard LFS to
GitHub.  Example Git config:

```
remote.github.url=...
remote.origin.url=ssh://gitssh.example.com/git/some/repo
lfs.customtransfer.rsync.path=git-lfs-rsync-agent
lfs.https://gitssh.example.com/git/.standalonetransferagent=rsync
```

The config assumes that `git-lfs-rsync-agent` determines the remote from
the stage 1 init message and then inspecting `remote.origin.url` to
infer the rsync host.  <https://github.com/aleb/git-lfs-rsync-agent>,
2017-09-17, does not yet do that.

[PR-2429] 09b7c53 'Allow using custom transfer agents directly', <git-lfs#2429>.

CC: Alexandru Băluț <ab@daedalean.ai>
Signed-off-by: Steffen Prohaska <prohaska@zib.de>

docs/custom-transfers.md

@@ -28,12 +28,16 @@ the transfers should be made, without having to query the API server.
 In this case it's possible to use the custom transfer agent directly,
 without querying the server, by using the following config option:
 
-* `lfs.standalonetransferagent`
+* `lfs.standalonetransferagent`, `lfs.<url>.standalonetransferagent`
 
-  Allows the specified custom transfer agent to be used directly
-  for transferring files, without asking the server how the transfers
-  should be made. The custom transfer agent has to be defined in a
-  `lfs.customtransfer.<name>` settings group.
+  Specifies a custom transfer agent to be used if the API server URL matches as
+  in `git config --get-urlmatch lfs.standalonetransferagent <apiurl>`.
+  `git-lfs` will not contact the API server.  It instead sets stage 2 transfer
+  actions to `null`.  `lfs.<url>.standalonetransferagent` can be used to
+  configure a custom transfer agent for individual remotes.
+  `lfs.standalonetransferagent` unconditionally configures a custom transfer
+  agent for all remotes.  The custom transfer agent must be specified in
+  a `lfs.customtransfer.<name>` settings group.
 
 ## Defining a Custom Transfer Type
 
@@ -171,7 +175,8 @@ like this:
   conventions, but can be interpreted however the custom transfer agent wishes
   (this is an NFS example, but it doesn't even have to be an URL). Generally,
   `href` will give the primary connection details, with `header` containing any
-  miscellaneous information needed.
+  miscellaneous information needed.  `action` is `null` for standalone transfer
+  agents.
 
 The transfer process should post one or more [progress messages](#progress) and
 then a final completion message as follows:
@@ -210,7 +215,8 @@ like this:
   conventions, but can be interpreted however the custom transfer agent wishes
   (this is an NFS example, but it doesn't even have to be an URL). Generally,
   `href` will give the primary connection details, with `header` containing any
-  miscellaneous information needed.
+  miscellaneous information needed.  `action` is `null` for standalone transfer
+  agents.
 
 Note there is no file path included in the download request; the transfer
 process should create a file itself and return the path in the final response

test/test-custom-transfers.sh

@@ -192,3 +192,89 @@ begin_test "custom-transfer-standalone"
   [ "$(echo "$objectlist" | wc -l)" -eq 12 ]
 )
 end_test
+
+begin_test "custom-transfer-standalone-urlmatch"
+(
+  set -e
+
+  # setup a git repo to be used as a local repo, not remote
+  reponame="test-custom-transfer-standalone-urlmatch"
+  setup_remote_repo "$reponame"
+
+  # clone directly, not through lfstest-gitserver
+  clone_repo_url "$REMOTEDIR/$reponame.git" $reponame
+
+  # set up custom transfer adapter to use a specific transfer agent, using a URL prefix match
+  git config lfs.customtransfer.testcustom.path lfstest-standalonecustomadapter
+  git config lfs.customtransfer.testcustom.concurrent false
+  git config remote.origin.lfsurl https://git.example.com/example/path/to/repo
+  git config lfs.https://git.example.com/example/path/.standalonetransferagent testcustom
+  git config lfs.standalonetransferagent invalid-agent
+
+  # git config lfs.standalonetransferagent testcustom
+  export TEST_STANDALONE_BACKUP_PATH="$(pwd)/test-custom-transfer-standalone-urlmatch-backup"
+  mkdir -p $TEST_STANDALONE_BACKUP_PATH
+  rm -rf $TEST_STANDALONE_BACKUP_PATH/*
+
+  git lfs track "*.dat" 2>&1 | tee track.log
+  grep "Tracking \"\*.dat\"" track.log
+  git add .gitattributes
+  git commit -m "Tracking"
+
+  # set up a decent amount of data so that there's work for multiple concurrent adapters
+  echo "[
+  {
+    \"CommitDate\":\"$(get_date -10d)\",
+    \"Files\":[
+      {\"Filename\":\"verify.dat\",\"Size\":18,\"Data\":\"send-verify-action\"},
+      {\"Filename\":\"file1.dat\",\"Size\":1024},
+      {\"Filename\":\"file2.dat\",\"Size\":750}]
+  },
+  {
+    \"CommitDate\":\"$(get_date -7d)\",
+    \"Files\":[
+      {\"Filename\":\"file1.dat\",\"Size\":1050},
+      {\"Filename\":\"file3.dat\",\"Size\":660},
+      {\"Filename\":\"file4.dat\",\"Size\":230}]
+  },
+  {
+    \"CommitDate\":\"$(get_date -5d)\",
+    \"Files\":[
+      {\"Filename\":\"file5.dat\",\"Size\":1200},
+      {\"Filename\":\"file6.dat\",\"Size\":300}]
+  },
+  {
+    \"CommitDate\":\"$(get_date -2d)\",
+    \"Files\":[
+      {\"Filename\":\"file3.dat\",\"Size\":120},
+      {\"Filename\":\"file5.dat\",\"Size\":450},
+      {\"Filename\":\"file7.dat\",\"Size\":520},
+      {\"Filename\":\"file8.dat\",\"Size\":2048}]
+  }
+  ]" | lfstest-testutils addcommits
+
+  GIT_TRACE=1 GIT_TRANSFER_TRACE=1 git push origin master 2>&1 | tee pushcustom.log
+  # use PIPESTATUS otherwise we get exit code from tee
+  [ ${PIPESTATUS[0]} = "0" ]
+
+  # Make sure the lock verification is not attempted.
+  grep "locks/verify$" pushcustom.log && false
+
+  grep "xfer: started custom adapter process" pushcustom.log
+  grep "xfer\[lfstest-standalonecustomadapter\]:" pushcustom.log
+  grep "12 of 12 files" pushcustom.log
+
+  rm -rf .git/lfs/objects
+  GIT_TRACE=1 GIT_TRANSFER_TRACE=1 git lfs fetch --all  2>&1 | tee fetchcustom.log
+  [ ${PIPESTATUS[0]} = "0" ]
+
+  grep "xfer: started custom adapter process" fetchcustom.log
+  grep "xfer\[lfstest-standalonecustomadapter\]:" fetchcustom.log
+  grep "12 of 12 files" fetchcustom.log
+
+  grep "Terminating test custom adapter gracefully" fetchcustom.log
+
+  objectlist=`find .git/lfs/objects -type f`
+  [ "$(echo "$objectlist" | wc -l)" -eq 12 ]
+)
+end_test

tq/manifest.go

@@ -3,6 +3,7 @@ package tq
 import (
 	"sync"
 
+	"github.com/git-lfs/git-lfs/config"
 	"github.com/git-lfs/git-lfs/lfsapi"
 	"github.com/rubyist/tracerx"
 )
@@ -83,7 +84,9 @@ func NewManifestClientOperationRemote(
 			m.concurrentTransfers = v
 		}
 		m.basicTransfersOnly = git.Bool("lfs.basictransfersonly", false)
-		m.standaloneTransferAgent, _ = git.Get("lfs.standalonetransferagent")
+		m.standaloneTransferAgent = findStandaloneTransfer(
+			apiClient, operation, remote,
+		)
 		tusAllowed = git.Bool("lfs.tustransfers", false)
 		configureCustomAdapters(git, m)
 	}
@@ -104,6 +107,22 @@ func NewManifestClientOperationRemote(
 	return m
 }
 
+func findStandaloneTransfer(client *lfsapi.Client, operation, remote string) string {
+	if operation == "" || remote == "" {
+		v, _ := client.GitEnv().Get("lfs.standalonetransferagent")
+		return v
+	}
+
+	ep := client.Endpoints.RemoteEndpoint(operation, remote)
+	uc := config.NewURLConfig(client.GitEnv())
+	v, ok := uc.Get("lfs", ep.Url, "standalonetransferagent")
+	if !ok {
+		return ""
+	}
+
+	return v
+}
+
 // GetAdapterNames returns a list of the names of adapters available to be created
 func (m *Manifest) GetAdapterNames(dir Direction) []string {
 	switch dir {