Skip to content

Commit

Permalink
fix: fixed an issue where dependencies could not be downloaded under …
Browse files Browse the repository at this point in the history 
…the new storage structure (#544)

* fix: fix missing download when running on new stroage

Signed-off-by: zongz <zongzhe1024@163.com>

* fix: Add tests under different features

Signed-off-by: zongz <zongzhe1024@163.com>

---------

Signed-off-by: zongz <zongzhe1024@163.com>
  • Loading branch information
@zong-zhe
zong-zhe authored Nov 15, 2024
1 parent c99c0f5 commit d591035
Show file tree
Hide file tree
Showing 4 changed files with 105 additions and 67 deletions.
31 changes: 4 additions & 27 deletions pkg/client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,12 @@ func TestWithGlobalLock(t *testing.T) {
test.RunTestWithGlobalLock(t, "TestAddWithDiffVersionNoSumCheck", testAddWithDiffVersionNoSumCheck)
test.RunTestWithGlobalLock(t, "TestAddWithDiffVersionWithSumCheck", testAddWithDiffVersionWithSumCheck)
test.RunTestWithGlobalLock(t, "TestDownloadOci", testDownloadOci)
test.RunTestWithGlobalLock(t, "TestRunWithOciDownloader", testRunWithOciDownloader)
test.RunTestWithGlobalLock(t, "TestAddWithOciDownloader", testAddWithOciDownloader)
test.RunTestWithGlobalLock(t, "TestAddDefaultRegistryDep", testAddDefaultRegistryDep)
test.RunTestWithGlobalLock(t, "TestUpdateDefaultRegistryDep", testUpdateDefaultRegistryDep)
test.RunTestWithGlobalLock(t, "TestRunDefaultRegistryDep", testRunDefaultRegistryDep)
test.RunTestWithGlobalLock(t, "TestAddWithNoSumCheck", testAddWithNoSumCheck)
test.RunTestWithGlobalLock(t, "TestAddWithGitCommit", testAddWithGitCommit)
test.RunTestWithGlobalLock(t, "TestDependenciesOrder", testDependenciesOrder)
test.RunTestWithGlobalLock(t, "TestRunInVendor", testRunInVendor)
test.RunTestWithGlobalLock(t, "TestPkgWithInVendorMode", testPkgWithInVendorMode)
test.RunTestWithGlobalLock(t, "TestResolveMetadataInJsonStrWithPackage", testResolveMetadataInJsonStrWithPackage)
test.RunTestWithGlobalLock(t, "TestResolveMetadataInJsonStr", testResolveMetadataInJsonStr)
Expand All @@ -63,17 +60,11 @@ func TestWithGlobalLock(t *testing.T) {
test.RunTestWithGlobalLock(t, "TestDownloadGitWithPackage", testDownloadGitWithPackage)
test.RunTestWithGlobalLock(t, "TestModandLockFilesWithGitPackageDownload", testModandLockFilesWithGitPackageDownload)
test.RunTestWithGlobalLock(t, "TestDependencyGraph", testDependencyGraph)
test.RunTestWithGlobalLock(t, "TestRunRemoteWithArgsInvalid", testRunRemoteWithArgsInvalid)
test.RunTestWithGlobalLock(t, "TestRunRemoteWithArgs", testRunRemoteWithArgs)
test.RunTestWithGlobalLock(t, "TestRunWithNoSumCheck", testRunWithGitPackage)
test.RunTestWithGlobalLock(t, "TestRunGit", testRunGit)
test.RunTestWithGlobalLock(t, "TestRunOciWithSettingsFile", testRunOciWithSettingsFile)
test.RunTestWithGlobalLock(t, "TestVendorWithGlobalLock", testVendorWithGlobalLock)
test.RunTestWithGlobalLock(t, "TestPull", testPull)
test.RunTestWithGlobalLock(t, "TestPullWithInsecureSkipTLSverify", testPullWithInsecureSkipTLSverify)
test.RunTestWithGlobalLock(t, "TestPullWithModSpec", testPullWithModSpec)
test.RunTestWithGlobalLock(t, "testPullWithOnlySpec", testPullWithOnlySpec)
test.RunTestWithGlobalLock(t, "TestRunWithModSpecVersion", testRunWithModSpecVersion)
test.RunTestWithGlobalLock(t, "TestGraph", testGraph)
}

Expand Down Expand Up @@ -1757,7 +1748,7 @@ func testUpdateDefaultRegistryDep(t *testing.T) {
}()
}

func testRunDefaultRegistryDep(t *testing.T) {
func testRunDefaultRegistryDep(t *testing.T, kpmcli *KpmClient) {
pkgPath := getTestDir("run_with_default_dep")

pkgWithSumCheckPathModBak := filepath.Join(pkgPath, "kcl.mod.bak")
Expand All @@ -1773,9 +1764,6 @@ func testRunDefaultRegistryDep(t *testing.T) {
err = copy.Copy(pkgWithSumCheckPathModLockBak, pkgWithSumCheckPathModLock)
assert.Equal(t, err, nil)

kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

kclPkg, err := kpmcli.LoadPkgFromPath(pkgPath)
assert.Equal(t, err, nil)

Expand Down Expand Up @@ -1996,10 +1984,7 @@ func TestRunLocalWithArgs(t *testing.T) {
}
}

func testRunRemoteWithArgsInvalid(t *testing.T) {
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

func testRunRemoteWithArgsInvalid(t *testing.T, kpmcli *KpmClient) {
logbuf := new(bytes.Buffer)
kpmcli.SetLogWriter(logbuf)

Expand All @@ -2025,10 +2010,8 @@ func testRunRemoteWithArgsInvalid(t *testing.T) {
}
}

func testRunRemoteWithArgs(t *testing.T) {
func testRunRemoteWithArgs(t *testing.T, kpmcli *KpmClient) {
pkgPath := getTestDir("test_run_options")
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

logbuf := new(bytes.Buffer)
kpmcli.SetLogWriter(logbuf)
Expand Down Expand Up @@ -2091,13 +2074,7 @@ func testRunRemoteWithArgs(t *testing.T) {
}
}

func testRunInVendor(t *testing.T) {
// Create a new kpm client.
kpmcli, err := NewKpmClient()
if err != nil {
t.Fatal(err)
}

func testRunInVendor(t *testing.T, kpmcli *KpmClient) {
pkgPath := getTestDir("test_run_in_vendor")
workdir := filepath.Join(pkgPath, "pkg")

Expand Down
28 changes: 7 additions & 21 deletions pkg/client/deperated_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,14 @@ import (
"kcl-lang.io/kpm/pkg/utils"
)

func TestRunWithNoSumCheck(t *testing.T) {
func testRunWithNoSumCheck(t *testing.T, kpmcli *KpmClient) {
pkgPath := getTestDir("test_run_no_sum_check")

kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

opts := opt.DefaultCompileOptions()
opts.SetNoSumCheck(true)
opts.SetPkgPath(pkgPath)

_, err = kpmcli.CompileWithOpts(opts)
_, err := kpmcli.CompileWithOpts(opts)
assert.Equal(t, err, nil)
assert.Equal(t, utils.DirExists(filepath.Join(pkgPath, "kcl.mod.lock")), false)

Expand All @@ -42,12 +39,9 @@ func TestRunWithNoSumCheck(t *testing.T) {
}()
}

func testRunWithGitPackage(t *testing.T) {
func testRunWithGitPackage(t *testing.T, kpmcli *KpmClient) {
pkgPath := getTestDir("test_run_git_package")

kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

opts := opt.DefaultCompileOptions()
opts.SetPkgPath(pkgPath)

Expand All @@ -63,11 +57,8 @@ func testRunWithGitPackage(t *testing.T) {
}()
}

func testRunWithOciDownloader(t *testing.T) {
kpmCli, err := NewKpmClient()
func testRunWithOciDownloader(t *testing.T, kpmCli *KpmClient) {
path := getTestDir("test_oci_downloader")
assert.Equal(t, err, nil)

kpmCli.DepDownloader = downloader.NewOciDownloader("linux/amd64")

var buf bytes.Buffer
Expand All @@ -84,10 +75,7 @@ func testRunWithOciDownloader(t *testing.T) {
assert.Equal(t, res.GetRawYamlResult(), "The_first_kcl_program: Hello World!")
}

func testRunGit(t *testing.T) {
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

func testRunGit(t *testing.T, kpmcli *KpmClient) {
testPath := getTestDir("test_run_git")

opts := opt.DefaultCompileOptions()
Expand All @@ -107,15 +95,13 @@ func testRunGit(t *testing.T) {
assert.Equal(t, utils.RmNewline(string(bytes)), utils.RmNewline(string(resultStr)))
}

func testRunOciWithSettingsFile(t *testing.T) {
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)
func testRunOciWithSettingsFile(t *testing.T, kpmcli *KpmClient) {
kpmcli.SetLogWriter(nil)
opts := opt.DefaultCompileOptions()
opts.SetEntries([]string{})
opts.Merge(kcl.WithSettings(filepath.Join(".", "test_data", "test_run_oci_with_settings", "kcl.yaml")))
opts.SetHasSettingsYaml(true)
_, err = kpmcli.CompileOciPkg("oci://ghcr.io/kcl-lang/helloworld", "", opts)
_, err := kpmcli.CompileOciPkg("oci://ghcr.io/kcl-lang/helloworld", "", opts)
assert.Equal(t, err, nil)
}

Expand Down
34 changes: 28 additions & 6 deletions pkg/client/run_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@ import (
"github.com/otiai10/copy"
"gotest.tools/v3/assert"
"kcl-lang.io/kpm/pkg/downloader"
"kcl-lang.io/kpm/pkg/features"
"kcl-lang.io/kpm/pkg/utils"
)

func testRunWithModSpecVersion(t *testing.T) {
func testRunWithModSpecVersion(t *testing.T, kpmcli *KpmClient) {
pkgPath := getTestDir("test_run_with_modspec_version")
modbkPath := filepath.Join(pkgPath, "kcl.mod.bk")
modPath := filepath.Join(pkgPath, "kcl.mod")
Expand Down Expand Up @@ -41,11 +42,6 @@ func testRunWithModSpecVersion(t *testing.T) {
}
}()

kpmcli, err := NewKpmClient()
if err != nil {
t.Errorf("Failed to create kpm client: %v", err)
}

res, err := kpmcli.Run(
WithRunSource(
&downloader.Source{
Expand Down Expand Up @@ -83,3 +79,29 @@ func testRunWithModSpecVersion(t *testing.T) {
assert.Equal(t, utils.RmNewline(string(expectedMod)), utils.RmNewline(string(gotMod)))
assert.Equal(t, utils.RmNewline(string(expectedLock)), utils.RmNewline(string(gotLock)))
}

func TestRun(t *testing.T) {
features.Enable(features.SupportNewStorage)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithOciDownloader", testRunWithOciDownloader)
RunTestWithGlobalLockAndKpmCli(t, "TestRunDefaultRegistryDep", testRunDefaultRegistryDep)
RunTestWithGlobalLockAndKpmCli(t, "TestRunInVendor", testRunInVendor)
RunTestWithGlobalLockAndKpmCli(t, "TestRunRemoteWithArgsInvalid", testRunRemoteWithArgsInvalid)
RunTestWithGlobalLockAndKpmCli(t, "TestRunRemoteWithArgs", testRunRemoteWithArgs)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithNoSumCheck", testRunWithNoSumCheck)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithGitPackage", testRunWithGitPackage)
RunTestWithGlobalLockAndKpmCli(t, "TestRunGit", testRunGit)
RunTestWithGlobalLockAndKpmCli(t, "TestRunOciWithSettingsFile", testRunOciWithSettingsFile)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithModSpecVersion", testRunWithModSpecVersion)

features.Disable(features.SupportNewStorage)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithOciDownloader", testRunWithOciDownloader)
RunTestWithGlobalLockAndKpmCli(t, "TestRunDefaultRegistryDep", testRunDefaultRegistryDep)
RunTestWithGlobalLockAndKpmCli(t, "TestRunInVendor", testRunInVendor)
RunTestWithGlobalLockAndKpmCli(t, "TestRunRemoteWithArgsInvalid", testRunRemoteWithArgsInvalid)
RunTestWithGlobalLockAndKpmCli(t, "TestRunRemoteWithArgs", testRunRemoteWithArgs)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithNoSumCheck", testRunWithNoSumCheck)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithGitPackage", testRunWithGitPackage)
RunTestWithGlobalLockAndKpmCli(t, "TestRunGit", testRunGit)
RunTestWithGlobalLockAndKpmCli(t, "TestRunOciWithSettingsFile", testRunOciWithSettingsFile)
RunTestWithGlobalLockAndKpmCli(t, "TestRunWithModSpecVersion", testRunWithModSpecVersion)
}
79 changes: 66 additions & 13 deletions pkg/downloader/downloader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,39 @@ func (d *OciDownloader) Download(opts *DownloadOptions) error {
return err
}
}
} else {
reporter.ReportMsgTo(
fmt.Sprintf(
"downloading '%s:%s' from '%s/%s:%s'",
ociSource.Repo, ociSource.Tag, ociSource.Reg, ociSource.Repo, ociSource.Tag,
),
opts.LogWriter,
)

err = ociCli.Pull(localPath, ociSource.Tag)
if err != nil {
return err
}
tarPath, err := utils.FindPkgArchive(localPath)
if err != nil {
return err
}
if utils.IsTar(tarPath) {
err = utils.UnTarDir(tarPath, localPath)
} else {
err = utils.ExtractTarball(tarPath, localPath)
}
if err != nil {
return fmt.Errorf("failed to untar the kcl package tar from '%s' into '%s'", tarPath, localPath)
}

// After untar the downloaded kcl package tar file, remove the tar file.
if utils.DirExists(tarPath) {
rmErr := os.Remove(tarPath)
if rmErr != nil {
return fmt.Errorf("failed to remove the downloaded kcl package tar file '%s'", tarPath)
}
}
}
} else {
reporter.ReportMsgTo(
Expand Down Expand Up @@ -498,6 +531,19 @@ func (d *GitDownloader) Download(opts *DownloadOptions) error {
git.WithTag(gitSource.Tag),
}

var msg string
if len(opts.Source.Git.Tag) != 0 {
msg = fmt.Sprintf("with tag '%s'", opts.Source.Git.Tag)
}

if len(opts.Source.Git.Commit) != 0 {
msg = fmt.Sprintf("with commit '%s'", opts.Source.Git.Commit)
}

if len(opts.Source.Git.Branch) != 0 {
msg = fmt.Sprintf("with branch '%s'", opts.Source.Git.Branch)
}

if ok, err := features.Enabled(features.SupportNewStorage); err == nil && ok {
if opts.EnableCache {
cacheFullPath := opts.CachePath
Expand Down Expand Up @@ -525,6 +571,10 @@ func (d *GitDownloader) Download(opts *DownloadOptions) error {
return err
}
} else {
reporter.ReportMsgTo(
fmt.Sprintf("cloning '%s' %s", opts.Source.Git.Url, msg),
opts.LogWriter,
)
// If not, clone the bare repository from the remote git repository, update the cache.
if utils.DirExists(cacheFullPath) {
err = os.Remove(cacheFullPath)
Expand Down Expand Up @@ -558,21 +608,24 @@ func (d *GitDownloader) Download(opts *DownloadOptions) error {
}
}
}
} else {
reporter.ReportMsgTo(
fmt.Sprintf("cloning '%s' %s", opts.Source.Git.Url, msg),
opts.LogWriter,
)
// If the cache is disabled, clone the repository from the remote git repository.
_, err := git.CloneWithOpts(
append(
cloneOpts,
git.WithRepoURL(gitSource.Url),
git.WithLocalPath(opts.LocalPath),
)...,
)
if err != nil {
return err
}
}
} else {
var msg string
if len(opts.Source.Git.Tag) != 0 {
msg = fmt.Sprintf("with tag '%s'", opts.Source.Git.Tag)
}

if len(opts.Source.Git.Commit) != 0 {
msg = fmt.Sprintf("with commit '%s'", opts.Source.Git.Commit)
}

if len(opts.Source.Git.Branch) != 0 {
msg = fmt.Sprintf("with branch '%s'", opts.Source.Git.Branch)
}

reporter.ReportMsgTo(
fmt.Sprintf("cloning '%s' %s", opts.Source.Git.Url, msg),
opts.LogWriter,
Expand Down

0 comments on commit d591035

Please sign in to comment.