feat(remote): support include git remote (#1652)

go.mod

@@ -9,6 +9,8 @@ require (
  github.com/davecgh/go-spew v1.1.1
  github.com/dominikbraun/graph v0.23.0
  github.com/fatih/color v1.17.0
+ github.com/go-git/go-billy/v5 v5.5.0
+ github.com/go-git/go-git/v5 v5.12.0
  github.com/go-task/slim-sprig/v3 v3.0.0
  github.com/go-task/template v0.1.0
  github.com/joho/godotenv v1.5.1
@@ -19,6 +21,7 @@ require (
  github.com/sajari/fuzzy v1.0.0
  github.com/spf13/pflag v1.0.5
  github.com/stretchr/testify v1.9.0
+ github.com/whilp/git-urls v1.0.0
  github.com/zeebo/xxh3 v1.0.2
  golang.org/x/sync v0.8.0
  golang.org/x/term v0.24.0
@@ -27,13 +30,31 @@ require (
 )
 
 require (
+ dario.cat/mergo v1.0.0 // indirect
+ github.com/Microsoft/go-winio v0.6.1 // indirect
+ github.com/ProtonMail/go-crypto v1.0.0 // indirect
+ github.com/cloudflare/circl v1.3.7 // indirect
+ github.com/cyphar/filepath-securejoin v0.2.4 // indirect
  github.com/dlclark/regexp2 v1.11.0 // indirect
- github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+ github.com/emirpasic/gods v1.18.1 // indirect
+ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+ github.com/kevinburke/ssh_config v1.2.0 // indirect
+ github.com/klauspost/cpuid/v2 v2.2.7 // indirect
  github.com/mattn/go-colorable v0.1.13 // indirect
  github.com/mattn/go-isatty v0.0.20 // indirect
  github.com/muesli/cancelreader v0.2.2 // indirect
+ github.com/pjbgf/sha1cd v0.3.0 // indirect
  github.com/pmezard/go-difflib v1.0.0 // indirect
+ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
+ github.com/skeema/knownhosts v1.2.2 // indirect
  github.com/stretchr/objx v0.5.2 // indirect
+ github.com/xanzy/ssh-agent v0.3.3 // indirect
+ golang.org/x/crypto v0.23.0 // indirect
+ golang.org/x/mod v0.17.0 // indirect
+ golang.org/x/net v0.25.0 // indirect
  golang.org/x/sys v0.25.0 // indirect
- gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+ golang.org/x/tools v0.21.0 // indirect
+ gopkg.in/warnings.v0 v0.1.2 // indirect
 )

taskfile/node.go

@@ -7,6 +7,8 @@ import (
  "strings"
  "time"
 
+ giturls "github.com/whilp/git-urls"
+
  "github.com/go-task/task/v3/errors"
  "github.com/go-task/task/v3/internal/experiments"
  "github.com/go-task/task/v3/internal/logger"
@@ -48,24 +50,39 @@ func NewNode(
 ) (Node, error) {
  var node Node
  var err error
- switch getScheme(entrypoint) {
+ scheme, err := getScheme(entrypoint)
+ if err != nil {
+ return nil, err
+ }
+ switch scheme {
+ case "git":
+ node, err = NewGitNode(entrypoint, dir, insecure, opts...)
  case "http", "https":
  node, err = NewHTTPNode(l, entrypoint, dir, insecure, timeout, opts...)
  default:
- // If no other scheme matches, we assume it's a file
  node, err = NewFileNode(l, entrypoint, dir, opts...)
+
  }
+
  if node.Remote() && !experiments.RemoteTaskfiles.Enabled {
  return nil, errors.New("task: Remote taskfiles are not enabled. You can read more about this experiment and how to enable it at https://taskfile.dev/experiments/remote-taskfiles")
  }
  return node, err
 }
 
-func getScheme(uri string) string {
+func getScheme(uri string) (string, error) {
+ u, err := giturls.Parse(uri)
+ if u == nil {
+ return "", err
+ }
+ if strings.HasSuffix(strings.Split(u.Path, "//")[0], ".git") && (u.Scheme == "git" || u.Scheme == "ssh" || u.Scheme == "https" || u.Scheme == "http") {
+ return "git", nil
+ }
+
  if i := strings.Index(uri, "://"); i != -1 {
- return uri[:i]
+ return uri[:i], nil
  }
- return ""
+ return "", nil
 }
 
 func getDefaultDir(entrypoint, dir string) string {

taskfile/node_file.go

@@ -81,6 +81,9 @@ func (node *FileNode) ResolveEntrypoint(entrypoint string) (string, error) {
  if strings.Contains(entrypoint, "://") {
  return entrypoint, nil
  }
+ if strings.HasPrefix(entrypoint, "git") {
+ return entrypoint, nil
+ }
 
  path, err := execext.Expand(entrypoint)
  if err != nil {

taskfile/node_git.go

@@ -0,0 +1,126 @@
+package taskfile
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/url"
+ "path/filepath"
+ "strings"
+
+ "github.com/go-git/go-billy/v5/memfs"
+ "github.com/go-git/go-git/v5"
+ "github.com/go-git/go-git/v5/plumbing"
+ "github.com/go-git/go-git/v5/storage/memory"
+ giturls "github.com/whilp/git-urls"
+
+ "github.com/go-task/task/v3/errors"
+ "github.com/go-task/task/v3/internal/execext"
+ "github.com/go-task/task/v3/internal/filepathext"
+)
+
+// An GitNode is a node that reads a Taskfile from a remote location via Git.
+type GitNode struct {
+ *BaseNode
+ URL *url.URL
+ rawUrl string
+ ref string
+ path string
+}
+
+func NewGitNode(
+ entrypoint string,
+ dir string,
+ insecure bool,
+ opts ...NodeOption,
+) (*GitNode, error) {
+ base := NewBaseNode(dir, opts...)
+ u, err := giturls.Parse(entrypoint)
+ if err != nil {
+ return nil, err
+ }
+
+ basePath, path := func() (string, string) {
+ x := strings.Split(u.Path, "//")
+ return x[0], x[1]
+ }()
+ ref := u.Query().Get("ref")
+
+ rawUrl := u.String()
+
+ u.RawQuery = ""
+ u.Path = basePath
+
+ if u.Scheme == "http" && !insecure {
+ return nil, &errors.TaskfileNotSecureError{URI: entrypoint}
+ }
+ return &GitNode{
+ BaseNode: base,
+ URL: u,
+ rawUrl: rawUrl,
+ ref: ref,
+ path: path,
+ }, nil
+}
+
+func (node *GitNode) Location() string {
+ return node.rawUrl
+}
+
+func (node *GitNode) Remote() bool {
+ return true
+}
+
+func (node *GitNode) Read(_ context.Context) ([]byte, error) {
+ fs := memfs.New()
+ storer := memory.NewStorage()
+ _, err := git.Clone(storer, fs, &git.CloneOptions{
+ URL: node.URL.String(),
+ ReferenceName: plumbing.ReferenceName(node.ref),
+ SingleBranch: true,
+ Depth: 1,
+ })
+ if err != nil {
+ return nil, err
+ }
+ file, err := fs.Open(node.path)
+ if err != nil {
+ return nil, err
+ }
+ // Read the entire response body
+ b, err := io.ReadAll(file)
+ if err != nil {
+ return nil, err
+ }
+
+ return b, nil
+}
+
+func (node *GitNode) ResolveEntrypoint(entrypoint string) (string, error) {
+ dir, _ := filepath.Split(node.path)
+ resolvedEntrypoint := fmt.Sprintf("%s//%s", node.URL, filepath.Join(dir, entrypoint))
+ if node.ref != "" {
+ return fmt.Sprintf("%s?ref=%s", resolvedEntrypoint, node.ref), nil
+ }
+ return resolvedEntrypoint, nil
+}
+
+func (node *GitNode) ResolveDir(dir string) (string, error) {
+ path, err := execext.Expand(dir)
+ if err != nil {
+ return "", err
+ }
+
+ if filepathext.IsAbs(path) {
+ return path, nil
+ }
+
+ // NOTE: Uses the directory of the entrypoint (Taskfile), not the current working directory
+ // This means that files are included relative to one another
+ entrypointDir := filepath.Dir(node.Dir())
+ return filepathext.SmartJoin(entrypointDir, path), nil
+}
+
+func (node *GitNode) FilenameAndLastDir() (string, string) {
+ return filepath.Base(node.path), filepath.Base(filepath.Dir(node.path))
+}

taskfile/node_git_test.go

@@ -0,0 +1,75 @@
+package taskfile
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGitNode_ssh(t *testing.T) {
+ node, err := NewGitNode("git@github.com:foo/bar.git//Taskfile.yml?ref=main", "", false)
+ assert.NoError(t, err)
+ assert.Equal(t, "main", node.ref)
+ assert.Equal(t, "Taskfile.yml", node.path)
+ assert.Equal(t, "ssh://git@github.com/foo/bar.git//Taskfile.yml?ref=main", node.rawUrl)
+ assert.Equal(t, "ssh://git@github.com/foo/bar.git", node.URL.String())
+ entrypoint, err := node.ResolveEntrypoint("common.yml")
+ assert.NoError(t, err)
+ assert.Equal(t, "ssh://git@github.com/foo/bar.git//common.yml?ref=main", entrypoint)
+}
+
+func TestGitNode_sshWithDir(t *testing.T) {
+ node, err := NewGitNode("git@github.com:foo/bar.git//directory/Taskfile.yml?ref=main", "", false)
+ assert.NoError(t, err)
+ assert.Equal(t, "main", node.ref)
+ assert.Equal(t, "directory/Taskfile.yml", node.path)
+ assert.Equal(t, "ssh://git@github.com/foo/bar.git//directory/Taskfile.yml?ref=main", node.rawUrl)
+ assert.Equal(t, "ssh://git@github.com/foo/bar.git", node.URL.String())
+ entrypoint, err := node.ResolveEntrypoint("common.yml")
+ assert.NoError(t, err)
+ assert.Equal(t, "ssh://git@github.com/foo/bar.git//directory/common.yml?ref=main", entrypoint)
+}
+
+func TestGitNode_https(t *testing.T) {
+ node, err := NewGitNode("https://github.com/foo/bar.git//Taskfile.yml?ref=main", "", false)
+ assert.NoError(t, err)
+ assert.Equal(t, "main", node.ref)
+ assert.Equal(t, "Taskfile.yml", node.path)
+ assert.Equal(t, "https://github.com/foo/bar.git//Taskfile.yml?ref=main", node.rawUrl)
+ assert.Equal(t, "https://github.com/foo/bar.git", node.URL.String())
+ entrypoint, err := node.ResolveEntrypoint("common.yml")
+ assert.NoError(t, err)
+ assert.Equal(t, "https://github.com/foo/bar.git//common.yml?ref=main", entrypoint)
+}
+
+func TestGitNode_httpsWithDir(t *testing.T) {
+ node, err := NewGitNode("https://github.com/foo/bar.git//directory/Taskfile.yml?ref=main", "", false)
+ assert.NoError(t, err)
+ assert.Equal(t, "main", node.ref)
+ assert.Equal(t, "directory/Taskfile.yml", node.path)
+ assert.Equal(t, "https://github.com/foo/bar.git//directory/Taskfile.yml?ref=main", node.rawUrl)
+ assert.Equal(t, "https://github.com/foo/bar.git", node.URL.String())
+ entrypoint, err := node.ResolveEntrypoint("common.yml")
+ assert.NoError(t, err)
+ assert.Equal(t, "https://github.com/foo/bar.git//directory/common.yml?ref=main", entrypoint)
+}
+
+func TestGitNode_FilenameAndDir(t *testing.T) {
+ node, err := NewGitNode("https://github.com/foo/bar.git//directory/Taskfile.yml?ref=main", "", false)
+ assert.NoError(t, err)
+ filename, dir := node.FilenameAndLastDir()
+ assert.Equal(t, "Taskfile.yml", filename)
+ assert.Equal(t, "directory", dir)
+
+ node, err = NewGitNode("https://github.com/foo/bar.git//Taskfile.yml?ref=main", "", false)
+ assert.NoError(t, err)
+ filename, dir = node.FilenameAndLastDir()
+ assert.Equal(t, "Taskfile.yml", filename)
+ assert.Equal(t, ".", dir)
+
+ node, err = NewGitNode("https://github.com/foo/bar.git//multiple/directory/Taskfile.yml?ref=main", "", false)
+ assert.NoError(t, err)
+ filename, dir = node.FilenameAndLastDir()
+ assert.Equal(t, "Taskfile.yml", filename)
+ assert.Equal(t, "directory", dir)
+}

taskfile/node_http.go

@@ -74,7 +74,6 @@ func (node *HTTPNode) Read(ctx context.Context) ([]byte, error) {
  return nil, errors.TaskfileFetchFailedError{URI: node.URL.String()}
  }
  defer resp.Body.Close()
-
  if resp.StatusCode != http.StatusOK {
  return nil, errors.TaskfileFetchFailedError{
  URI: node.URL.String(),

taskfile/node_test.go

@@ -0,0 +1,22 @@
+package taskfile
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestScheme(t *testing.T) {
+ scheme, err := getScheme("https://github.com/foo/bar.git")
+ assert.NoError(t, err)
+ assert.Equal(t, "git", scheme)
+ scheme, err = getScheme("https://github.com/foo/bar.git?ref=v1//taskfile/common.yml")
+ assert.NoError(t, err)
+ assert.Equal(t, "git", scheme)
+ scheme, err = getScheme("git@github.com:foo/bar.git?ref=main//Taskfile.yml")
+ assert.NoError(t, err)
+ assert.Equal(t, "git", scheme)
+ scheme, err = getScheme("https://github.com/foo/common.yml")
+ assert.NoError(t, err)
+ assert.Equal(t, "https", scheme)
+}

website/docs/experiments/remote_taskfiles.mdx

@@ -62,6 +62,16 @@ includes:
 `TOKEN=my-token task my-remote-namespace:hello` will be resolved by Task to
 `https://my-token@raw.githubusercontent.com/my-org/my-repo/main/Taskfile.yml`
 
+## Git nodes
+
+You can also include a Taskfile from a Git node. We currently support ssh-style and http / https addresses like `git@example.com/foo/bar.git//Taskfiles.yml?ref=v1` and `https://example.com/foo/bar.git//Taskfiles.yml?ref=v1`.
+
+You need to follow this pattern : `<baseUrl>.git//<path>?ref=<ref>`.
+The `ref` parameter, optional, can be a branch name or a tag, if not provided it'll pick up the default branch.
+The `path` is the path to the Taskfile in the repository.
+
+If you want to use the SSH protocol, you need to make sure that your ssh-agent has your private ssh keys added so that they can be used during authentication.
+
 ## Security
 
 Running commands from sources that you do not control is always a potential