Inherit submodules from template repository content

modules/git/submodule.go

@@ -6,12 +6,17 @@
 package git
 
 import (
+	"bufio"
+	"context"
 	"fmt"
+	"io"
 	"net"
 	"net/url"
 	"path"
 	"regexp"
 	"strings"
+
+	"code.gitea.io/gitea/modules/log"
 )
 
 var scpSyntax = regexp.MustCompile(`^([a-zA-Z0-9_]+@)?([a-zA-Z0-9._-]+):(.*)$`)
@@ -22,6 +27,12 @@ type SubModule struct {
 	URL  string
 }
 
+// SubModuleCommit submodule name and commit from a repository
+type SubModuleCommit struct {
+	Name   string
+	Commit string
+}
+
 // SubModuleFile represents a file with submodule type.
 type SubModuleFile struct {
 	*Commit
@@ -120,3 +131,90 @@ func (sf *SubModuleFile) RefURL(urlPrefix, repoFullName, sshDomain string) strin
 func (sf *SubModuleFile) RefID() string {
 	return sf.refID
 }
+
+// GetSubmoduleCommits Returns a list of active submodules in the repository
+func GetSubmoduleCommits(ctx context.Context, repoPath string) []SubModuleCommit {
+	stdoutReader, stdoutWriter := io.Pipe()
+	defer func() {
+		_ = stdoutReader.Close()
+		_ = stdoutWriter.Close()
+	}()
+
+	go func() {
+		stderrBuilder := &strings.Builder{}
+		err := NewCommand(ctx, "config", "-f", ".gitmodules", "--list", "--name-only").RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder)
+		if err != nil {
+			_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String()))
+		} else {
+			_ = stdoutWriter.Close()
+		}
+	}()
+
+	var submodules []SubModuleCommit
+	bufReader := bufio.NewReader(stdoutReader)
+
+	for {
+		line, err := bufReader.ReadString('\n')
+		if err != nil {
+			break
+		}
+
+		if len(line) < len("submodule.x.url\n") ||
+			!strings.HasPrefix(line, "submodule.") ||
+			!strings.HasSuffix(line, ".url\n") {
+
+			continue
+		}
+
+		name := line[len("submodule.") : len(line)-len(".url\n")]
+		name = strings.TrimSpace(name)
+
+		if len(name) == 0 {
+			log.Debug("Submodule skipped because it has no name")
+			continue
+		}
+
+		// If no commit was found for the module skip it
+		commit, err := NewCommand(ctx, "submodule", "status", name).
+			RunInDir(repoPath)
+		if err != nil {
+			log.Debug("Submodule %s skipped because it has no commit", name)
+			continue
+		}
+
+		if len(commit) > 0 {
+			commit = commit[1:]
+		}
+
+		fields := strings.Fields(commit)
+
+		if len(fields) == 0 {
+			log.Debug("Submodule %s skipped because it has no valid commit", name)
+			continue
+		}
+
+		commit = fields[0]
+
+		if len(commit) != 40 {
+			log.Debug("Submodule %s skipped due to malformed commit hash", name)
+			continue
+		}
+
+		submodules = append(submodules, SubModuleCommit{name, commit})
+	}
+
+	return submodules
+}
+
+// AddSubmoduleIndexes Adds the given submodules to the git index. Requires the .gitmodules file to be already present.
+func AddSubmoduleIndexes(ctx context.Context, repoPath string, submodules []SubModuleCommit) error {
+	for _, submodule := range submodules {
+		if stdout, err := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000", submodule.Commit, submodule.Name).
+			RunInDir(repoPath); err != nil {
+			log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Name, repoPath, stdout, err)
+			return err
+		}
+	}
+
+	return nil
+}

modules/git/submodule_test.go

@@ -5,8 +5,10 @@
 package git
 
 import (
+	"path/filepath"
 	"testing"
 
+	"code.gitea.io/gitea/modules/util"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -41,3 +43,20 @@ func TestGetRefURL(t *testing.T) {
 		assert.EqualValues(t, kase.expect, getRefURL(kase.refURL, kase.prefixURL, kase.parentPath, kase.SSHDomain))
 	}
 }
+
+func TestRepository_GetSubmoduleCommits(t *testing.T) {
+	bareRepo1Path := filepath.Join(testReposDir, "repo4_submodules")
+	clonedPath, err := cloneRepo(bareRepo1Path, "repo4_TestRepository_GetSubmoduleCommits")
+	assert.NoError(t, err)
+	defer util.RemoveAll(clonedPath)
+
+	submodules := GetSubmoduleCommits(DefaultContext, clonedPath)
+
+	assert.EqualValues(t, len(submodules), 2)
+
+	assert.EqualValues(t, submodules[0].Name, "libtest")
+	assert.EqualValues(t, submodules[0].Commit, "1234567890123456789012345678901234567890")
+
+	assert.EqualValues(t, submodules[1].Name, "<°)))><")
+	assert.EqualValues(t, submodules[1].Commit, "d2932de67963f23d43e1c7ecf20173e92ee6c43c")
+}

modules/git/tests/repos/repo4_submodules/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/master

modules/git/tests/repos/repo4_submodules/config

@@ -0,0 +1,4 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = true

modules/git/tests/repos/repo4_submodules/description

@@ -0,0 +1 @@
+Unnamed repository; edit this file 'description' to name the repository.

modules/git/tests/repos/repo4_submodules/hooks/applypatch-msg.sample

@@ -0,0 +1,15 @@
+#!/bin/sh
+#
+# An example hook script to check the commit log message taken by
+# applypatch from an e-mail message.
+#
+# The hook should exit with non-zero status after issuing an
+# appropriate message if it wants to stop the commit.  The hook is
+# allowed to edit the commit message file.
+#
+# To enable this hook, rename this file to "applypatch-msg".
+
+. git-sh-setup
+commitmsg="$(git rev-parse --git-path hooks/commit-msg)"
+test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"}
+:

modules/git/tests/repos/repo4_submodules/hooks/commit-msg.sample

@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# An example hook script to check the commit log message.
+# Called by "git commit" with one argument, the name of the file
+# that has the commit message.  The hook should exit with non-zero
+# status after issuing an appropriate message if it wants to stop the
+# commit.  The hook is allowed to edit the commit message file.
+#
+# To enable this hook, rename this file to "commit-msg".
+
+# Uncomment the below to add a Signed-off-by line to the message.
+# Doing this in a hook is a bad idea in general, but the prepare-commit-msg
+# hook is more suited to it.
+#
+# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p')
+# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1"
+
+# This example catches duplicate Signed-off-by lines.
+
+test "" = "$(grep '^Signed-off-by: ' "$1" |
+	 sort | uniq -c | sed -e '/^[ 	]*1[ 	]/d')" || {
+	echo >&2 Duplicate Signed-off-by lines.
+	exit 1
+}

modules/git/tests/repos/repo4_submodules/hooks/fsmonitor-watchman.sample

@@ -0,0 +1,173 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use IPC::Open2;
+
+# An example hook script to integrate Watchman
+# (https://facebook.github.io/watchman/) with git to speed up detecting
+# new and modified files.
+#
+# The hook is passed a version (currently 2) and last update token
+# formatted as a string and outputs to stdout a new update token and
+# all files that have been modified since the update token. Paths must
+# be relative to the root of the working tree and separated by a single NUL.
+#
+# To enable this hook, rename this file to "query-watchman" and set
+# 'git config core.fsmonitor .git/hooks/query-watchman'
+#
+my ($version, $last_update_token) = @ARGV;
+
+# Uncomment for debugging
+# print STDERR "$0 $version $last_update_token\n";
+
+# Check the hook interface version
+if ($version ne 2) {
+	die "Unsupported query-fsmonitor hook version '$version'.\n" .
+	    "Falling back to scanning...\n";
+}
+
+my $git_work_tree = get_working_dir();
+
+my $retry = 1;
+
+my $json_pkg;
+eval {
+	require JSON::XS;
+	$json_pkg = "JSON::XS";
+	1;
+} or do {
+	require JSON::PP;
+	$json_pkg = "JSON::PP";
+};
+
+launch_watchman();
+
+sub launch_watchman {
+	my $o = watchman_query();
+	if (is_work_tree_watched($o)) {
+		output_result($o->{clock}, @{$o->{files}});
+	}
+}
+
+sub output_result {
+	my ($clockid, @files) = @_;
+
+	# Uncomment for debugging watchman output
+	# open (my $fh, ">", ".git/watchman-output.out");
+	# binmode $fh, ":utf8";
+	# print $fh "$clockid\n@files\n";
+	# close $fh;
+
+	binmode STDOUT, ":utf8";
+	print $clockid;
+	print "\0";
+	local $, = "\0";
+	print @files;
+}
+
+sub watchman_clock {
+	my $response = qx/watchman clock "$git_work_tree"/;
+	die "Failed to get clock id on '$git_work_tree'.\n" .
+		"Falling back to scanning...\n" if $? != 0;
+
+	return $json_pkg->new->utf8->decode($response);
+}
+
+sub watchman_query {
+	my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty')
+	or die "open2() failed: $!\n" .
+	"Falling back to scanning...\n";
+
+	# In the query expression below we're asking for names of files that
+	# changed since $last_update_token but not from the .git folder.
+	#
+	# To accomplish this, we're using the "since" generator to use the
+	# recency index to select candidate nodes and "fields" to limit the
+	# output to file names only. Then we're using the "expression" term to
+	# further constrain the results.
+	if (substr($last_update_token, 0, 1) eq "c") {
+		$last_update_token = "\"$last_update_token\"";
+	}
+	my $query = <<"	END";
+		["query", "$git_work_tree", {
+			"since": $last_update_token,
+			"fields": ["name"],
+			"expression": ["not", ["dirname", ".git"]]
+		}]
+	END
+
+	# Uncomment for debugging the watchman query
+	# open (my $fh, ">", ".git/watchman-query.json");
+	# print $fh $query;
+	# close $fh;
+
+	print CHLD_IN $query;
+	close CHLD_IN;
+	my $response = do {local $/; <CHLD_OUT>};
+
+	# Uncomment for debugging the watch response
+	# open ($fh, ">", ".git/watchman-response.json");
+	# print $fh $response;
+	# close $fh;
+
+	die "Watchman: command returned no output.\n" .
+	"Falling back to scanning...\n" if $response eq "";
+	die "Watchman: command returned invalid output: $response\n" .
+	"Falling back to scanning...\n" unless $response =~ /^\{/;
+
+	return $json_pkg->new->utf8->decode($response);
+}
+
+sub is_work_tree_watched {
+	my ($output) = @_;
+	my $error = $output->{error};
+	if ($retry > 0 and $error and $error =~ m/unable to resolve root .* directory (.*) is not watched/) {
+		$retry--;
+		my $response = qx/watchman watch "$git_work_tree"/;
+		die "Failed to make watchman watch '$git_work_tree'.\n" .
+		    "Falling back to scanning...\n" if $? != 0;
+		$output = $json_pkg->new->utf8->decode($response);
+		$error = $output->{error};
+		die "Watchman: $error.\n" .
+		"Falling back to scanning...\n" if $error;
+
+		# Uncomment for debugging watchman output
+		# open (my $fh, ">", ".git/watchman-output.out");
+		# close $fh;
+
+		# Watchman will always return all files on the first query so
+		# return the fast "everything is dirty" flag to git and do the
+		# Watchman query just to get it over with now so we won't pay
+		# the cost in git to look up each individual file.
+		my $o = watchman_clock();
+		$error = $output->{error};
+
+		die "Watchman: $error.\n" .
+		"Falling back to scanning...\n" if $error;
+
+		output_result($o->{clock}, ("/"));
+		$last_update_token = $o->{clock};
+
+		eval { launch_watchman() };
+		return 0;
+	}
+
+	die "Watchman: $error.\n" .
+	"Falling back to scanning...\n" if $error;
+
+	return 1;
+}
+
+sub get_working_dir {
+	my $working_dir;
+	if ($^O =~ 'msys' || $^O =~ 'cygwin') {
+		$working_dir = Win32::GetCwd();
+		$working_dir =~ tr/\\/\//;
+	} else {
+		require Cwd;
+		$working_dir = Cwd::cwd();
+	}
+
+	return $working_dir;
+}

modules/git/tests/repos/repo4_submodules/hooks/post-update.sample

@@ -0,0 +1,8 @@
+#!/bin/sh
+#
+# An example hook script to prepare a packed repository for use over
+# dumb transports.
+#
+# To enable this hook, rename this file to "post-update".
+
+exec git update-server-info