Skip to content

Commit

Permalink
feat: add depth flag
Browse files Browse the repository at this point in the history
Signed-off-by: Yordis Prieto <yordis.prieto@gmail.com>
  • Loading branch information
yordis committed Jun 6, 2024
1 parent 92b22fd commit 095683d
Show file tree
Hide file tree
Showing 12 changed files with 783 additions and 732 deletions.
5 changes: 5 additions & 0 deletions assets/swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -8104,6 +8104,11 @@
"type": "string",
"title": "GCPServiceAccountKey specifies the service account key in JSON format to be used for getting credentials to Google Cloud Source repos"
},
"gitDepth": {
"type": "string",
"format": "uint64",
"title": "GitDepth specifies the depth of the git clone"
},
"githubAppEnterpriseBaseUrl": {
"type": "string",
"title": "GithubAppEnterpriseBaseURL specifies the base URL of GitHub Enterprise installation. If empty will default to https://api.github.com"
Expand Down
1,413 changes: 720 additions & 693 deletions pkg/apis/application/v1alpha1/generated.pb.go

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions pkg/apis/application/v1alpha1/generated.proto

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions pkg/apis/application/v1alpha1/openapi_generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions pkg/apis/application/v1alpha1/repository_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,8 @@ type Repository struct {
GCPServiceAccountKey string `json:"gcpServiceAccountKey,omitempty" protobuf:"bytes,21,opt,name=gcpServiceAccountKey"`
// ForceHttpBasicAuth specifies whether Argo CD should attempt to force basic auth for HTTP connections
ForceHttpBasicAuth bool `json:"forceHttpBasicAuth,omitempty" protobuf:"bytes,22,opt,name=forceHttpBasicAuth"`
// GitDepth specifies the depth of the git clone
GitDepth uint64 `json:"gitDepth,omitempty" protobuf:"bytes,23,opt,name=gitDepth"`
}

// IsInsecure returns true if the repository has been configured to skip server verification
Expand Down
22 changes: 11 additions & 11 deletions reposerver/repository/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ func (s *Service) ListApps(ctx context.Context, q *apiclient.ListAppsRequest) (*
defer s.metricsServer.DecPendingRepoRequest(q.Repo.Repo)

closer, err := s.repoLock.Lock(gitClient.Root(), commitSHA, true, func() (goio.Closer, error) {
return s.checkoutRevision(gitClient, commitSHA, s.initConstants.SubmoduleEnabled)
return s.checkoutRevision(gitClient, commitSHA, s.initConstants.SubmoduleEnabled, q.Repo.GitDepth)
})

if err != nil {
Expand Down Expand Up @@ -380,7 +380,7 @@ func (s *Service) runRepoOperation(
})
} else {
closer, err := s.repoLock.Lock(gitClient.Root(), revision, settings.allowConcurrent, func() (goio.Closer, error) {
return s.checkoutRevision(gitClient, revision, s.initConstants.SubmoduleEnabled)
return s.checkoutRevision(gitClient, revision, s.initConstants.SubmoduleEnabled, repo.GitDepth)
})

if err != nil {
Expand Down Expand Up @@ -757,7 +757,7 @@ func (s *Service) runManifestGenAsync(ctx context.Context, repoRoot, commitSHA,
return
}
closer, err := s.repoLock.Lock(gitClient.Root(), referencedCommitSHA, true, func() (goio.Closer, error) {
return s.checkoutRevision(gitClient, referencedCommitSHA, s.initConstants.SubmoduleEnabled)
return s.checkoutRevision(gitClient, referencedCommitSHA, s.initConstants.SubmoduleEnabled, q.Repo.GitDepth)
})
if err != nil {
log.Errorf("failed to acquire lock for referenced source %s", normalizedRepoURL)
Expand Down Expand Up @@ -2244,7 +2244,7 @@ func (s *Service) GetRevisionMetadata(ctx context.Context, q *apiclient.RepoServ
defer s.metricsServer.DecPendingRepoRequest(q.Repo.Repo)

closer, err := s.repoLock.Lock(gitClient.Root(), q.Revision, true, func() (goio.Closer, error) {
return s.checkoutRevision(gitClient, q.Revision, s.initConstants.SubmoduleEnabled)
return s.checkoutRevision(gitClient, q.Revision, s.initConstants.SubmoduleEnabled, q.Repo.GitDepth)
})

if err != nil {
Expand Down Expand Up @@ -2417,23 +2417,23 @@ func directoryPermissionInitializer(rootPath string) goio.Closer {
// checkoutRevision is a convenience function to initialize a repo, fetch, and checkout a revision
// Returns the 40 character commit SHA after the checkout has been performed
// nolint:unparam
func (s *Service) checkoutRevision(gitClient git.Client, revision string, submoduleEnabled bool) (goio.Closer, error) {
func (s *Service) checkoutRevision(gitClient git.Client, revision string, submoduleEnabled bool, depth uint64) (goio.Closer, error) {
closer := s.gitRepoInitializer(gitClient.Root())
err := checkoutRevision(gitClient, revision, submoduleEnabled)
err := checkoutRevision(gitClient, revision, submoduleEnabled, depth)
if err != nil {
s.metricsServer.IncGitFetchFail(gitClient.Root(), revision)
}
return closer, err
}

func checkoutRevision(gitClient git.Client, revision string, submoduleEnabled bool) error {
func checkoutRevision(gitClient git.Client, revision string, submoduleEnabled bool, depth uint64) error {
err := gitClient.Init()
if err != nil {
return status.Errorf(codes.Internal, "Failed to initialize git repo: %v", err)
}

// Fetching with no revision first. Fetching with an explicit version can cause repo bloat. https://github.com/argoproj/argo-cd/issues/8845
err = gitClient.Fetch("")
err = gitClient.Fetch("", depth)
if err != nil {
return status.Errorf(codes.Internal, "Failed to fetch default: %v", err)
}
Expand All @@ -2445,7 +2445,7 @@ func checkoutRevision(gitClient git.Client, revision string, submoduleEnabled bo
log.Infof("Failed to checkout revision %s: %v", revision, err)
log.Infof("Fallback to fetching specific revision %s. ref might not have been in the default refspec fetched.", revision)

err = gitClient.Fetch(revision)
err = gitClient.Fetch(revision, depth)
if err != nil {
return status.Errorf(codes.Internal, "Failed to checkout revision %s: %v", revision, err)
}
Expand Down Expand Up @@ -2576,7 +2576,7 @@ func (s *Service) GetGitFiles(_ context.Context, request *apiclient.GitFilesRequ

// cache miss, generate the results
closer, err := s.repoLock.Lock(gitClient.Root(), revision, true, func() (goio.Closer, error) {
return s.checkoutRevision(gitClient, revision, request.GetSubmoduleEnabled())
return s.checkoutRevision(gitClient, revision, request.GetSubmoduleEnabled(), request.Repo.GitDepth)
})
if err != nil {
return nil, status.Errorf(codes.Internal, "unable to checkout git repo %s with revision %s pattern %s: %v", repo.Repo, revision, gitPath, err)
Expand Down Expand Up @@ -2634,7 +2634,7 @@ func (s *Service) GetGitDirectories(_ context.Context, request *apiclient.GitDir

// cache miss, generate the results
closer, err := s.repoLock.Lock(gitClient.Root(), revision, true, func() (goio.Closer, error) {
return s.checkoutRevision(gitClient, revision, request.GetSubmoduleEnabled())
return s.checkoutRevision(gitClient, revision, request.GetSubmoduleEnabled(), request.Repo.GitDepth)
})
if err != nil {
return nil, status.Errorf(codes.Internal, "unable to checkout git repo %s with revision %s: %v", repo.Repo, revision, err)
Expand Down
12 changes: 6 additions & 6 deletions reposerver/repository/repository_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func newServiceWithMocks(t *testing.T, root string, signed bool) (*Service, *git
}
return newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) {
gitClient.On("Init").Return(nil)
gitClient.On("Fetch", mock.Anything).Return(nil)
gitClient.On("Fetch", mock.Anything, mock.Anything).Return(nil)
gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil)
gitClient.On("LsRemote", mock.Anything).Return(mock.Anything, nil)
gitClient.On("CommitSHA").Return(mock.Anything, nil)
Expand Down Expand Up @@ -177,7 +177,7 @@ func newServiceWithCommitSHA(t *testing.T, root, revision string) *Service {

service, gitClient, _ := newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) {
gitClient.On("Init").Return(nil)
gitClient.On("Fetch", mock.Anything).Return(nil)
gitClient.On("Fetch", mock.Anything, mock.Anything).Return(nil)
gitClient.On("Checkout", mock.Anything, mock.Anything).Return(nil)
gitClient.On("LsRemote", revision).Return(revision, revisionErr)
gitClient.On("CommitSHA").Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil)
Expand Down Expand Up @@ -2956,10 +2956,10 @@ func TestCheckoutRevisionCanGetNonstandardRefs(t *testing.T) {
pullSha, err := gitClient.LsRemote("refs/pull/123/head")
require.NoError(t, err)

err = checkoutRevision(gitClient, "does-not-exist", false)
err = checkoutRevision(gitClient, "does-not-exist", false, 0)
assert.Error(t, err)

err = checkoutRevision(gitClient, pullSha, false)
err = checkoutRevision(gitClient, pullSha, false, 0)
assert.NoError(t, err)
}

Expand Down Expand Up @@ -3312,7 +3312,7 @@ func TestGetGitDirectories(t *testing.T) {
root := "./testdata/git-files-dirs"
s, _, cacheMocks := newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) {
gitClient.On("Init").Return(nil)
gitClient.On("Fetch", mock.Anything).Return(nil)
gitClient.On("Fetch", mock.Anything, mock.Anything).Return(nil)
gitClient.On("Checkout", mock.Anything, mock.Anything).Once().Return(nil)
gitClient.On("LsRemote", "HEAD").Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil)
gitClient.On("Root").Return(root)
Expand Down Expand Up @@ -3435,7 +3435,7 @@ func TestGetGitFiles(t *testing.T) {
root := ""
s, _, cacheMocks := newServiceWithOpt(t, func(gitClient *gitmocks.Client, helmClient *helmmocks.Client, paths *iomocks.TempPaths) {
gitClient.On("Init").Return(nil)
gitClient.On("Fetch", mock.Anything).Return(nil)
gitClient.On("Fetch", mock.Anything, mock.Anything).Return(nil)
gitClient.On("Checkout", mock.Anything, mock.Anything).Once().Return(nil)
gitClient.On("LsRemote", "HEAD").Return("632039659e542ed7de0c170a4fcc1c571b288fc0", nil)
gitClient.On("Root").Return(root)
Expand Down
3 changes: 2 additions & 1 deletion test/e2e/repo_management_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,8 @@ func TestGetRepoWithInheritedCreds(t *testing.T) {

_, err = repoClient.UpdateRepository(context.Background(), &repositorypkg.RepoUpdateRequest{
Repo: &v1alpha1.Repository{
Repo: repoUrl,
Repo: repoUrl,
GitDepth: 2,
},
})
assert.NoError(t, err)
Expand Down
22 changes: 14 additions & 8 deletions util/git/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ type gitRefCache interface {
type Client interface {
Root() string
Init() error
Fetch(revision string) error
Fetch(revision string, depth uint64) error
Submodule() error
Checkout(revision string, submoduleEnabled bool) error
LsRefs() (*Refs, error)
Expand Down Expand Up @@ -350,24 +350,30 @@ func (m *nativeGitClient) IsLFSEnabled() bool {
return m.enableLfs
}

func (m *nativeGitClient) fetch(revision string) error {
var err error
func (m *nativeGitClient) fetch(revision string, depth uint64) error {
args := []string{"fetch", "--tags", "--force", "--prune"}

if depth > 0 {
args = append(args, "--depth", fmt.Sprintf("%d", depth))
}

if revision != "" {
err = m.runCredentialedCmd("fetch", "origin", revision, "--tags", "--force", "--prune")
args = append(args, "origin", revision)
} else {
err = m.runCredentialedCmd("fetch", "origin", "--tags", "--force", "--prune")
args = append(args, "origin")
}
return err

return m.runCredentialedCmd(args...)
}

// Fetch fetches latest updates from origin
func (m *nativeGitClient) Fetch(revision string) error {
func (m *nativeGitClient) Fetch(revision string, depth uint64) error {
if m.OnFetch != nil {
done := m.OnFetch(m.repoURL)
defer done()
}

err := m.fetch(revision)
err := m.fetch(revision, 0)

// When we have LFS support enabled, check for large files and fetch them too.
if err == nil && m.IsLFSEnabled() {
Expand Down
8 changes: 4 additions & 4 deletions util/git/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func Test_nativeGitClient_Fetch(t *testing.T) {
err = client.Init()
require.NoError(t, err)

err = client.Fetch("")
err = client.Fetch("", 0)
assert.NoError(t, err)
}

Expand All @@ -62,15 +62,15 @@ func Test_nativeGitClient_Fetch_Prune(t *testing.T) {
err = runCmd(tempDir, "git", "branch", "test/foo")
require.NoError(t, err)

err = client.Fetch("")
err = client.Fetch("", 0)
assert.NoError(t, err)

err = runCmd(tempDir, "git", "branch", "-d", "test/foo")
require.NoError(t, err)
err = runCmd(tempDir, "git", "branch", "test/foo/bar")
require.NoError(t, err)

err = client.Fetch("")
err = client.Fetch("", 0)
assert.NoError(t, err)
}

Expand Down Expand Up @@ -215,7 +215,7 @@ func Test_nativeGitClient_Submodule(t *testing.T) {
err = client.Init()
require.NoError(t, err)

err = client.Fetch("")
err = client.Fetch("", 0)
assert.NoError(t, err)

commitSHA, err := client.LsRemote("HEAD")
Expand Down
8 changes: 4 additions & 4 deletions util/git/git_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ func TestLFSClient(t *testing.T) {
err = client.Init()
assert.NoError(t, err)

err = client.Fetch("")
err = client.Fetch("", 0)
assert.NoError(t, err)

err = client.Checkout(commitSHA, true)
Expand Down Expand Up @@ -357,7 +357,7 @@ func TestVerifyCommitSignature(t *testing.T) {
err = client.Init()
assert.NoError(t, err)

err = client.Fetch("")
err = client.Fetch("", 0)
assert.NoError(t, err)

commitSHA, err := client.LsRemote("HEAD")
Expand Down Expand Up @@ -414,11 +414,11 @@ func TestNewFactory(t *testing.T) {
err = client.Init()
assert.NoError(t, err)

err = client.Fetch("")
err = client.Fetch("", 0)
assert.NoError(t, err)

// Do a second fetch to make sure we can treat `already up-to-date` error as not an error
err = client.Fetch("")
err = client.Fetch("", 0)
assert.NoError(t, err)

err = client.Checkout(commitSHA, true)
Expand Down
10 changes: 5 additions & 5 deletions util/git/mocks/Client.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 095683d

Please sign in to comment.