Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(repo-server): unify semver resolution in new versions subpackage #20216

Open
wants to merge 11 commits into
base: master
Choose a base branch
from
48 changes: 20 additions & 28 deletions reposerver/repository/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (
"strings"
"time"

"github.com/Masterminds/semver/v3"
"github.com/TomOnTime/utfutil"
"github.com/argoproj/gitops-engine/pkg/utils/kube"
textutils "github.com/argoproj/gitops-engine/pkg/utils/text"
Expand Down Expand Up @@ -61,6 +60,7 @@ import (
"github.com/argoproj/argo-cd/v2/util/kustomize"
"github.com/argoproj/argo-cd/v2/util/manifeststream"
"github.com/argoproj/argo-cd/v2/util/text"
"github.com/argoproj/argo-cd/v2/util/versions"
)

const (
Expand Down Expand Up @@ -2420,39 +2420,34 @@ func (s *Service) newClientResolveRevision(repo *v1alpha1.Repository, revision s
func (s *Service) newHelmClientResolveRevision(repo *v1alpha1.Repository, revision string, chart string, noRevisionCache bool) (helm.Client, string, error) {
enableOCI := repo.EnableOCI || helm.IsHelmOciRepo(repo.Repo)
helmClient := s.newHelmClient(repo.Repo, repo.GetHelmCreds(), enableOCI, repo.Proxy, repo.NoProxy, helm.WithIndexCache(s.cache), helm.WithChartPaths(s.chartPaths))
if helm.IsVersion(revision) {
if versions.IsVersion(revision) {
return helmClient, revision, nil
}
constraints, err := semver.NewConstraint(revision)
if err != nil {
return nil, "", fmt.Errorf("invalid revision '%s': %w", revision, err)
}

var tags []string
if enableOCI {
tags, err := helmClient.GetTags(chart, noRevisionCache)
var err error
tags, err = helmClient.GetTags(chart, noRevisionCache)
if err != nil {
return nil, "", fmt.Errorf("unable to get tags: %w", err)
}

version, err := tags.MaxVersion(constraints)
} else {
index, err := helmClient.GetIndex(noRevisionCache, s.initConstants.HelmRegistryMaxIndexSize)
if err != nil {
return nil, "", fmt.Errorf("no version for constraints: %w", err)
return nil, "", err
}
return helmClient, version.String(), nil
entries, err := index.GetEntries(chart)
if err != nil {
return nil, "", err
}
tags = entries.Tags()
}

index, err := helmClient.GetIndex(noRevisionCache, s.initConstants.HelmRegistryMaxIndexSize)
if err != nil {
return nil, "", err
}
entries, err := index.GetEntries(chart)
if err != nil {
return nil, "", err
}
version, err := entries.MaxVersion(constraints)
version, err := versions.MaxVersion(revision, tags)
if err != nil {
return nil, "", err
return nil, "", fmt.Errorf("invalid revision: %w", err)
}

return helmClient, version.String(), nil
}

Expand Down Expand Up @@ -2538,13 +2533,10 @@ func (s *Service) GetHelmCharts(ctx context.Context, q *apiclient.HelmChartsRequ
}
res := apiclient.HelmChartsResponse{}
for chartName, entries := range index.Entries {
chart := apiclient.HelmChart{
Name: chartName,
}
for _, entry := range entries {
chart.Versions = append(chart.Versions, entry.Version)
}
res.Items = append(res.Items, &chart)
res.Items = append(res.Items, &apiclient.HelmChart{
Name: chartName,
Versions: entries.Tags(),
})
}
return &res, nil
}
Expand Down
11 changes: 6 additions & 5 deletions reposerver/repository/repository_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ func newServiceWithMocks(t *testing.T, root string, signed bool) (*Service, *git
chart: {{Version: "1.0.0"}, {Version: version}},
oobChart: {{Version: "1.0.0"}, {Version: version}},
}}, nil)
helmClient.On("GetTags", mock.AnythingOfType("string"), mock.AnythingOfType("bool")).Return([]string{"1.0.0", version}, nil)
helmClient.On("ExtractChart", chart, version, "", false, int64(0), false).Return("./testdata/my-chart", io.NopCloser, nil)
helmClient.On("ExtractChart", oobChart, version, "", false, int64(0), false).Return("./testdata2/out-of-bounds-chart", io.NopCloser, nil)
helmClient.On("CleanChartCache", chart, version, "").Return(nil)
Expand Down Expand Up @@ -1834,12 +1835,12 @@ func TestService_newHelmClientResolveRevision(t *testing.T) {
service := newService(t, ".")

t.Run("EmptyRevision", func(t *testing.T) {
_, _, err := service.newHelmClientResolveRevision(&argoappv1.Repository{}, "", "", true)
assert.EqualError(t, err, "invalid revision '': improper constraint: ")
_, _, err := service.newHelmClientResolveRevision(&argoappv1.Repository{}, "", "my-chart", true)
assert.EqualError(t, err, "invalid revision: failed to determine semver constraint: improper constraint: ")
})
t.Run("InvalidRevision", func(t *testing.T) {
_, _, err := service.newHelmClientResolveRevision(&argoappv1.Repository{}, "???", "", true)
assert.EqualError(t, err, "invalid revision '???': improper constraint: ???", true)
_, _, err := service.newHelmClientResolveRevision(&argoappv1.Repository{}, "???", "my-chart", true)
assert.EqualError(t, err, "invalid revision: failed to determine semver constraint: improper constraint: ???", true)
})
}

Expand Down Expand Up @@ -4048,7 +4049,7 @@ func TestGetRefs_CacheLockTryLockGitRefCacheError(t *testing.T) {
}

func TestGetRevisionChartDetails(t *testing.T) {
t.Run("Test revision semvar", func(t *testing.T) {
t.Run("Test revision semver", func(t *testing.T) {
root := t.TempDir()
service := newService(t, root)
_, err := service.GetRevisionChartDetails(context.Background(), &apiclient.RepoServerRevisionChartDetailsRequest{
Expand Down
72 changes: 14 additions & 58 deletions util/git/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ import (
"syscall"
"time"

"github.com/Masterminds/semver/v3"

argoexec "github.com/argoproj/pkg/exec"
"github.com/bmatcuk/doublestar/v4"
"github.com/go-git/go-git/v5"
Expand All @@ -39,6 +37,7 @@ import (
"github.com/argoproj/argo-cd/v2/util/env"
executil "github.com/argoproj/argo-cd/v2/util/exec"
"github.com/argoproj/argo-cd/v2/util/proxy"
"github.com/argoproj/argo-cd/v2/util/versions"
)

var ErrInvalidRepoURL = fmt.Errorf("repo URL is invalid")
Expand Down Expand Up @@ -617,6 +616,16 @@ func (m *nativeGitClient) LsRemote(revision string) (res string, err error) {
return
}

func getGitTags(refs []*plumbing.Reference) []string {
var tags []string
for _, ref := range refs {
if ref.Name().IsTag() {
tags = append(tags, ref.Name().Short())
}
}
return tags
}

func (m *nativeGitClient) lsRemote(revision string) (string, error) {
if IsCommitSHA(revision) {
return revision, nil
Expand All @@ -631,9 +640,9 @@ func (m *nativeGitClient) lsRemote(revision string) (string, error) {
revision = "HEAD"
}

semverSha := m.resolveSemverRevision(revision, refs)
if semverSha != "" {
return semverSha, nil
version, err := versions.MaxVersion(revision, getGitTags(refs))
if err == nil {
revision = version.Original()
}

// refToHash keeps a maps of remote refs to their hash
Expand Down Expand Up @@ -682,59 +691,6 @@ func (m *nativeGitClient) lsRemote(revision string) (string, error) {
return "", fmt.Errorf("unable to resolve '%s' to a commit SHA", revision)
}

// resolveSemverRevision is a part of the lsRemote method workflow.
// When the user correctly configures the Git repository revision, and that revision is a valid semver constraint, we
// use this logic path rather than the standard lsRemote revision resolution loop.
// Some examples to illustrate the actual behavior - if the revision is:
// * "v0.1.2"/"0.1.2" or "v0.1"/"0.1", then this is not a constraint, it's a pinned version - so we fall back to the standard tag matching in the lsRemote loop.
// * "v0.1.*"/"0.1.*", and there's a tag matching that constraint, then we find the latest matching version and return its commit hash.
// * "v0.1.*"/"0.1.*", and there is *no* tag matching that constraint, then we fall back to the standard tag matching in the lsRemote loop.
// * "custom-tag", only the lsRemote loop will run - because that revision is an invalid semver;
// * "master-branch", only the lsRemote loop will run because that revision is an invalid semver;
func (m *nativeGitClient) resolveSemverRevision(revision string, refs []*plumbing.Reference) string {
if _, err := semver.NewVersion(revision); err == nil {
// If the revision is a valid version, then we know it isn't a constraint; it's just a pin.
// In which case, we should use standard tag resolution mechanisms.
return ""
}

constraint, err := semver.NewConstraint(revision)
if err != nil {
log.Debugf("Revision '%s' is not a valid semver constraint, skipping semver resolution.", revision)
return ""
}

maxVersion := semver.New(0, 0, 0, "", "")
maxVersionHash := plumbing.ZeroHash
for _, ref := range refs {
if !ref.Name().IsTag() {
continue
}

tag := ref.Name().Short()
version, err := semver.NewVersion(tag)
if err != nil {
log.Debugf("Error parsing version for tag: '%s': %v", tag, err)
// Skip this tag and continue to the next one
continue
}

if constraint.Check(version) {
if version.GreaterThan(maxVersion) {
maxVersion = version
maxVersionHash = ref.Hash()
}
}
}

if maxVersionHash.IsZero() {
return ""
}

log.Debugf("Semver constraint '%s' resolved to tag '%s', at reference '%s'", revision, maxVersion.Original(), maxVersionHash.String())
return maxVersionHash.String()
}

// CommitSHA returns current commit sha from `git rev-parse HEAD`
func (m *nativeGitClient) CommitSHA() (string, error) {
out, err := m.runCmd("rev-parse", "HEAD")
Expand Down
2 changes: 1 addition & 1 deletion util/git/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ func Test_SemverTags(t *testing.T) {
// However, if one specifies the minor/patch versions, semver constraints can be used to match non-semver tags.
// 2024-banana is considered as "2024.0.0-banana" in semver-ish, and banana > apple, so it's a match.
// Note: this is more for documentation and future reference than real testing, as it seems like quite odd behaviour.
name: "semver constraints on non-semver tags",
name: "semver constraints on semver tags",
ref: "> 2024.0.0-apple",
expected: mapTagRefs["2024-banana"],
}} {
Expand Down
19 changes: 11 additions & 8 deletions util/helm/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ import (
"strings"
"time"

executil "github.com/argoproj/argo-cd/v2/util/exec"

"github.com/argoproj/pkg/sync"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
Expand All @@ -28,6 +26,7 @@ import (
"oras.land/oras-go/v2/registry/remote/credentials"

"github.com/argoproj/argo-cd/v2/util/cache"
executil "github.com/argoproj/argo-cd/v2/util/exec"
argoio "github.com/argoproj/argo-cd/v2/util/io"
"github.com/argoproj/argo-cd/v2/util/io/files"
"github.com/argoproj/argo-cd/v2/util/proxy"
Expand Down Expand Up @@ -58,7 +57,7 @@ type Client interface {
CleanChartCache(chart string, version string, project string) error
ExtractChart(chart string, version string, project string, passCredentials bool, manifestMaxExtractedSize int64, disableManifestMaxExtractedSize bool) (string, argoio.Closer, error)
GetIndex(noCache bool, maxIndexSize int64) (*Index, error)
GetTags(chart string, noCache bool) (*TagsList, error)
GetTags(chart string, noCache bool) ([]string, error)
TestHelmOCI() (bool, error)
}

Expand Down Expand Up @@ -414,7 +413,7 @@ func getIndexURL(rawURL string) (string, error) {
return repoURL.String(), nil
}

func (c *nativeHelmChart) GetTags(chart string, noCache bool) (*TagsList, error) {
func (c *nativeHelmChart) GetTags(chart string, noCache bool) ([]string, error) {
if !c.enableOci {
return nil, OCINotEnabledErr
}
Expand All @@ -430,7 +429,11 @@ func (c *nativeHelmChart) GetTags(chart string, noCache bool) (*TagsList, error)
}
}

tags := &TagsList{}
type entriesStruct struct {
Tags []string
}

entries := &entriesStruct{}
if len(data) == 0 {
start := time.Now()
repo, err := remote.NewRepository(tagsURL)
Expand Down Expand Up @@ -472,7 +475,7 @@ func (c *nativeHelmChart) GetTags(chart string, noCache bool) (*TagsList, error)
for _, tag := range tagsResult {
// By convention: Change underscore (_) back to plus (+) to get valid SemVer
convertedTag := strings.ReplaceAll(tag, "_", "+")
tags.Tags = append(tags.Tags, convertedTag)
entries.Tags = append(entries.Tags, convertedTag)
}

return nil
Expand All @@ -490,11 +493,11 @@ func (c *nativeHelmChart) GetTags(chart string, noCache bool) (*TagsList, error)
}
}
} else {
err := json.Unmarshal(data, tags)
err := json.Unmarshal(data, entries)
if err != nil {
return nil, fmt.Errorf("failed to decode tags: %w", err)
}
}

return tags, nil
return entries.Tags, nil
}
36 changes: 22 additions & 14 deletions util/helm/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ type fakeIndexCache struct {
data []byte
}

type fakeTagsList struct {
Tags []string `json:"tags"`
}

func (f *fakeIndexCache) SetHelmIndex(_ string, indexData []byte) error {
f.data = indexData
return nil
Expand Down Expand Up @@ -169,19 +173,23 @@ func TestGetTagsFromUrl(t *testing.T) {
t.Run("should return tags correctly while following the link header", func(t *testing.T) {
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Logf("called %s", r.URL.Path)
responseTags := TagsList{}
var responseTags fakeTagsList
w.Header().Set("Content-Type", "application/json")
if !strings.Contains(r.URL.String(), "token") {
w.Header().Set("Link", fmt.Sprintf("<https://%s%s?token=next-token>; rel=next", r.Host, r.URL.Path))
responseTags.Tags = []string{"first"}
responseTags = fakeTagsList{
Tags: []string{"first"},
}
} else {
responseTags.Tags = []string{
"second",
"2.8.0",
"2.8.0-prerelease",
"2.8.0_build",
"2.8.0-prerelease_build",
"2.8.0-prerelease.1_build.1234",
responseTags = fakeTagsList{
Tags: []string{
"second",
"2.8.0",
"2.8.0-prerelease",
"2.8.0_build",
"2.8.0-prerelease_build",
"2.8.0-prerelease.1_build.1234",
},
}
}
w.WriteHeader(http.StatusOK)
Expand All @@ -195,7 +203,7 @@ func TestGetTagsFromUrl(t *testing.T) {

tags, err := client.GetTags("mychart", true)
require.NoError(t, err)
assert.ElementsMatch(t, tags.Tags, []string{
assert.ElementsMatch(t, tags, []string{
"first",
"second",
"2.8.0",
Expand Down Expand Up @@ -231,7 +239,7 @@ func TestGetTagsFromURLPrivateRepoAuthentication(t *testing.T) {

assert.Equal(t, expectedAuthorization, authorization)

responseTags := TagsList{
responseTags := fakeTagsList{
Tags: []string{
"2.8.0",
"2.8.0-prerelease",
Expand Down Expand Up @@ -286,7 +294,7 @@ func TestGetTagsFromURLPrivateRepoAuthentication(t *testing.T) {
tags, err := client.GetTags("mychart", true)

require.NoError(t, err)
assert.ElementsMatch(t, tags.Tags, []string{
assert.ElementsMatch(t, tags, []string{
"2.8.0",
"2.8.0-prerelease",
"2.8.0+build",
Expand All @@ -312,7 +320,7 @@ func TestGetTagsFromURLEnvironmentAuthentication(t *testing.T) {

assert.Equal(t, expectedAuthorization, authorization)

responseTags := TagsList{
responseTags := fakeTagsList{
Tags: []string{
"2.8.0",
"2.8.0-prerelease",
Expand Down Expand Up @@ -372,7 +380,7 @@ func TestGetTagsFromURLEnvironmentAuthentication(t *testing.T) {
tags, err := client.GetTags("mychart", true)

require.NoError(t, err)
assert.ElementsMatch(t, tags.Tags, []string{
assert.ElementsMatch(t, tags, []string{
"2.8.0",
"2.8.0-prerelease",
"2.8.0+build",
Expand Down
Loading