internal/datasource: git supports cloning submodules recursively

The `recurse_submodules` configuration can be set to a non-zero value to
specify the maximum depth of the recursion.

.changelog/3351.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+core: git cloning now supports recursively cloning submodules
+```

internal/cli/project_apply.go

@@ -32,6 +32,7 @@ type ProjectApplyCommand struct {
  flagGitPassword string
  flagGitKeyPath string
  flagGitKeyPassword string
+ flagGitRecurseSubmodules int
  flagFromWaypointHcl string
  flagWaypointHcl string
  flagPoll *bool
@@ -208,6 +209,9 @@ func (c *ProjectApplyCommand) Run(args []string) int {
  if v := c.flagGitRef; v != "" {
  gitInfo.Ref = v
  }
+ if v := c.flagGitRecurseSubmodules; v > 0 {
+ gitInfo.RecurseSubmodules = uint32(v)
+ }
 
  switch strings.ToLower(c.flagGitAuthType) {
  case "basic":
@@ -485,6 +489,14 @@ func (c *ProjectApplyCommand) Flags() *flag.Sets {
  "the private key doesn't require a password.",
  })
 
+ f.IntVar(&flag.IntVar{
+ Name: "git-recurse-submodules",
+ Target: &c.flagGitRecurseSubmodules,
+ Default: 0,
+ Usage: "The maximum depth to recursively clone submodules. A value of " +
+ "zero disables cloning any submodules recursively.",
+ })
+
  f.BoolPtrVar(&flag.BoolPtrVar{
  Name: "poll",
  Target: &c.flagPoll,

internal/datasource/git.go

@@ -58,6 +58,7 @@ func (s *GitSource) ProjectSource(body hcl.Body, ctx *hcl.EvalContext) (*pb.Job_
  Path: cfg.Path,
  IgnoreChangesOutsidePath: cfg.IgnoreChangesOutsidePath,
  Ref: cfg.Ref,
+ RecurseSubmodules: cfg.RecurseSubmodules,
  }
  switch {
  case cfg.Username != "":
@@ -187,6 +188,11 @@ func (s *GitSource) Get(
  URL: source.Git.Url,
  Auth: auth,
  Progress: &output,
+
+ // Note: we don't set RecurseSubmodules here because if we're checking
+ // out a ref without submodules or with different submodules, we
+ // don't want to waste time recursing HEAD. We fetch submodules
+ // later.
  })
  if err != nil {
  closer()
@@ -236,6 +242,32 @@ func (s *GitSource) Get(
  }
  }
 
+ if depth := source.Git.RecurseSubmodules; depth > 0 {
+ wt, err := repo.Worktree()
+ if err != nil {
+ closer()
+ return "", nil, nil, status.Errorf(codes.Aborted,
+ "Failed to load Git working tree: %s", err)
+ }
+
+ sm, err := wt.Submodules()
+ if err != nil {
+ closer()
+ return "", nil, nil, status.Errorf(codes.Aborted,
+ "Failed to load submodules: %s", err)
+ }
+
+ if err := sm.UpdateContext(ctx, &git.SubmoduleUpdateOptions{
+ Init: true,
+ Auth: auth,
+ RecurseSubmodules: git.SubmoduleRescursivity(depth),
+ }); err != nil {
+ closer()
+ return "", nil, nil, status.Errorf(codes.Aborted,
+ "Failed to update submodules: %s", err)
+ }
+ }
+
  // Get our ref
  ref, err := repo.Head()
  if err != nil {
@@ -611,6 +643,7 @@ type gitConfig struct {
  SSHKeyPassword string `hcl:"key_password,optional"`
  Ref string `hcl:"ref,optional"`
  IgnoreChangesOutsidePath bool `hcl:"ignore_changes_outside_path,optional"`
+ RecurseSubmodules uint32 `hcl:"recurse_submodules,optional"`
 }
 
 var _ Sourcer = (*GitSource)(nil)

internal/datasource/git_test.go

@@ -277,6 +277,157 @@ func TestGitSourceGet(t *testing.T) {
  _, err = os.Stat(filepath.Join(dir, "two"))
  require.Error(err)
  })
+
+ t.Run("submodule: clone disabled (default)", func(t *testing.T) {
+ require := require.New(t)
+
+ var s GitSource
+ dir, _, closer, err := s.Get(
+ context.Background(),
+ hclog.L(),
+ terminal.ConsoleUI(context.Background()),
+ &pb.Job_DataSource{
+ Source: &pb.Job_DataSource_Git{
+ Git: &pb.Job_Git{
+ Url: testGitFixture(t, "git-submodule"),
+ },
+ },
+ },
+ "",
+ )
+ require.NoError(err)
+ if closer != nil {
+ defer closer()
+ }
+
+ // Verify files
+ _, err = os.Stat(filepath.Join(dir, "hello.txt"))
+ require.NoError(err)
+ _, err = os.Stat(filepath.Join(dir, "examples", "README.md"))
+ require.Error(err)
+ })
+
+ t.Run("submodule: recursion on clone", func(t *testing.T) {
+ require := require.New(t)
+
+ var s GitSource
+ dir, _, closer, err := s.Get(
+ context.Background(),
+ hclog.L(),
+ terminal.ConsoleUI(context.Background()),
+ &pb.Job_DataSource{
+ Source: &pb.Job_DataSource_Git{
+ Git: &pb.Job_Git{
+ Url: testGitFixture(t, "git-submodule"),
+ RecurseSubmodules: 10,
+ },
+ },
+ },
+ "",
+ )
+ require.NoError(err)
+ if closer != nil {
+ defer closer()
+ }
+
+ // Verify files
+ _, err = os.Stat(filepath.Join(dir, "hello.txt"))
+ require.NoError(err)
+ _, err = os.Stat(filepath.Join(dir, "examples", "README.md"))
+ require.NoError(err)
+ })
+
+ t.Run("submodule: ref with no submodules", func(t *testing.T) {
+ require := require.New(t)
+
+ var s GitSource
+ dir, _, closer, err := s.Get(
+ context.Background(),
+ hclog.L(),
+ terminal.ConsoleUI(context.Background()),
+ &pb.Job_DataSource{
+ Source: &pb.Job_DataSource_Git{
+ Git: &pb.Job_Git{
+ Url: testGitFixture(t, "git-submodule"),
+ Ref: "758c263",
+ RecurseSubmodules: 10,
+ },
+ },
+ },
+ "",
+ )
+ require.NoError(err)
+ if closer != nil {
+ defer closer()
+ }
+
+ // Verify files
+ _, err = os.Stat(filepath.Join(dir, "hello.txt"))
+ require.NoError(err)
+ _, err = os.Stat(filepath.Join(dir, "examples", "README.md"))
+ require.Error(err)
+ })
+
+ t.Run("submodule: removed HEAD", func(t *testing.T) {
+ require := require.New(t)
+
+ var s GitSource
+ dir, _, closer, err := s.Get(
+ context.Background(),
+ hclog.L(),
+ terminal.ConsoleUI(context.Background()),
+ &pb.Job_DataSource{
+ Source: &pb.Job_DataSource_Git{
+ Git: &pb.Job_Git{
+ Url: testGitFixture(t, "git-submodule-rm"),
+ RecurseSubmodules: 10,
+ },
+ },
+ },
+ "",
+ )
+ require.NoError(err)
+ if closer != nil {
+ defer closer()
+ }
+
+ // Verify files
+ _, err = os.Stat(filepath.Join(dir, "hello.txt"))
+ require.NoError(err)
+ _, err = os.Stat(filepath.Join(dir, "examples", "README.md"))
+ require.Error(err)
+ })
+
+ t.Run("submodule: ref with submodule", func(t *testing.T) {
+ require := require.New(t)
+
+ var s GitSource
+ dir, _, closer, err := s.Get(
+ context.Background(),
+ hclog.L(),
+ terminal.ConsoleUI(context.Background()),
+ &pb.Job_DataSource{
+ Source: &pb.Job_DataSource_Git{
+ Git: &pb.Job_Git{
+ Url: testGitFixture(t, "git-submodule-rm"),
+ Ref: "27e97ef4f312fe37588f84209e4d056825dee614",
+ RecurseSubmodules: 10,
+ },
+ },
+ },
+ "",
+ )
+ require.NoError(err)
+ if closer != nil {
+ defer closer()
+ }
+
+ // Verify files
+ _, err = os.Stat(filepath.Join(dir, "hello.txt"))
+ require.NoError(err)
+ _, err = os.Stat(filepath.Join(dir, "examples", "README.md"))
+ require.NoError(err)
+ })
 }
 
 func TestGitSourceChanges(t *testing.T) {

internal/datasource/testdata/.gitignore

@@ -0,0 +1 @@
+git-submodule/examples