Skip to content

Commit c7ff03e

Browse files
nirsnirs
committed
Test parallel downloads
Ad sub tests for parallel downloads with "with cache" and "cahing-only mode". The test run 10 concurrent downloads with the current test http server and ensure that downloads succeeded. To make it possible to test using the same process, we include now a counter in the perProcessTempfile. The file name is now: {filename}.pid.{count} The test revealed an issue with clonefile, not seen when testing 3 parallel `limactl create`. When the data file is replaced during a clonefile syscall, it may fail with: clonefile failed: no such file or directory This smells like a darwin bug, but we can avoid this issue by using the first download result. When a download finishes, we check if data file exists, and return success if it does. We take a lock for the very short time needed to check and rename the temporary data file to the target file. Tested using: % go test ./pkg/downloader -run TestDownloadRemote/with_cache/parallel -count=1000 -timeout=0 ok github.com/lima-vm/lima/pkg/downloader 487.422s Signed-off-by: Nir Soffer <nsoffer@redhat.com>
1 parent 11eb750 commit c7ff03e

File tree

2 files changed

+114
-32
lines changed

2 files changed

+114
-32
lines changed

‎pkg/downloader/downloader.go‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,15 @@ import (
1212
"os/exec"
1313
"path"
1414
"path/filepath"
15-
"strconv"
1615
"strings"
16+
"sync/atomic"
1717
"time"
1818

1919
"github.com/cheggaaa/pb/v3"
2020
"github.com/containerd/continuity/fs"
2121
"github.com/lima-vm/lima/pkg/httpclientutil"
2222
"github.com/lima-vm/lima/pkg/localpathutil"
23+
"github.com/lima-vm/lima/pkg/lockutil"
2324
"github.com/lima-vm/lima/pkg/progressbar"
2425
"github.com/opencontainers/go-digest"
2526
"github.com/sirupsen/logrus"
@@ -273,15 +274,15 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result,
273274
if err := downloadHTTP(ctx, shadData, shadTime, shadType, remote, o.description, o.expectedDigest); err != nil {
274275
return nil, err
275276
}
276-
// no need to pass the digest to copyLocal(), as we already verified the digest
277-
if err := copyLocal(ctx, localPath, shadData, ext, o.decompress, "", ""); err != nil {
278-
return nil, err
279-
}
280277
if shadDigest != "" && o.expectedDigest != "" {
281278
if err := atomicWrite(shadDigest, []byte(o.expectedDigest.String()), 0o644); err != nil {
282279
return nil, err
283280
}
284281
}
282+
// no need to pass the digest to copyLocal(), as we already verified the digest
283+
if err := copyLocal(ctx, localPath, shadData, ext, o.decompress, "", ""); err != nil {
284+
return nil, err
285+
}
285286
res := &Result{
286287
Status: StatusDownloaded,
287288
CachePath: shadData,
@@ -672,15 +673,29 @@ func downloadHTTP(ctx context.Context, localPath, lastModified, contentType, url
672673
return err
673674
}
674675

675-
return os.Rename(localPathTmp, localPath)
676+
// If localPath was created by a parallel download keep it. Replacing it
677+
// while another process is copying it to the destination may fail the
678+
// clonefile syscall. We use a lock to ensure that only one process updates
679+
// data, and when we return data file exists.
680+
681+
return lockutil.WithDirLock(filepath.Dir(localPath), func() error {
682+
if _, err := os.Stat(localPath); err == nil {
683+
return nil
684+
}
685+
return os.Rename(localPathTmp, localPath)
686+
})
676687
}
677688

689+
var tempfileCount atomic.Uint64
690+
678691
// To allow parallel download we use a per-process unique suffix for tempoary
679692
// files. Renaming the temporary file to the final file is safe without
680693
// synchronization on posix.
694+
// To make it easy to test we also include a counter ensuring that each
695+
// temporary file is unique in the same process.
681696
// https://github.com/lima-vm/lima/issues/2722
682697
func perProcessTempfile(path string) string {
683-
return path + ".tmp." + strconv.FormatInt(int64(os.Getpid()), 10)
698+
return fmt.Sprintf("%s.tmp.%d.%d", path, os.Getpid(), tempfileCount.Add(1))
684699
}
685700

686701
// atomicWrite writes data to path, creating a new file or replacing existing

‎pkg/downloader/downloader_test.go‎

Lines changed: 92 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"os/exec"
99
"path/filepath"
1010
"runtime"
11+
"slices"
1112
"strings"
1213
"testing"
1314
"time"
@@ -21,6 +22,20 @@ func TestMain(m *testing.M) {
2122
os.Exit(m.Run())
2223
}
2324

25+
type downloadResult struct {
26+
r *Result
27+
err error
28+
}
29+
30+
// We expect only few parallel downloads. Testing with larger number to find
31+
// races quicker. 20 parallel downloads take about 0.5 seocnds on M1 Pro.
32+
const parallelDownloads = 20
33+
34+
// When downloading in parallel usually all downloads completed with
35+
// StatusDownload, but some may be delayed and find the data file when they
36+
// start. Can be reproduced locally using 100 parallel downloads.
37+
var parallelStatus []Status = []Status{StatusDownloaded, StatusUsedCache}
38+
2439
func TestDownloadRemote(t *testing.T) {
2540
ts := httptest.NewServer(http.FileServer(http.Dir("testdata")))
2641
t.Cleanup(ts.Close)
@@ -57,38 +72,90 @@ func TestDownloadRemote(t *testing.T) {
5772
})
5873
})
5974
t.Run("with cache", func(t *testing.T) {
60-
cacheDir := filepath.Join(t.TempDir(), "cache")
61-
localPath := filepath.Join(t.TempDir(), t.Name())
62-
r, err := Download(context.Background(), localPath, dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
63-
assert.NilError(t, err)
64-
assert.Equal(t, StatusDownloaded, r.Status)
75+
t.Run("serial", func(t *testing.T) {
76+
cacheDir := filepath.Join(t.TempDir(), "cache")
77+
localPath := filepath.Join(t.TempDir(), t.Name())
78+
r, err := Download(context.Background(), localPath, dummyRemoteFileURL,
79+
WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
80+
assert.NilError(t, err)
81+
assert.Equal(t, StatusDownloaded, r.Status)
6582

66-
r, err = Download(context.Background(), localPath, dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
67-
assert.NilError(t, err)
68-
assert.Equal(t, StatusSkipped, r.Status)
83+
r, err = Download(context.Background(), localPath, dummyRemoteFileURL,
84+
WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
85+
assert.NilError(t, err)
86+
assert.Equal(t, StatusSkipped, r.Status)
6987

70-
localPath2 := localPath + "-2"
71-
r, err = Download(context.Background(), localPath2, dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
72-
assert.NilError(t, err)
73-
assert.Equal(t, StatusUsedCache, r.Status)
88+
localPath2 := localPath + "-2"
89+
r, err = Download(context.Background(), localPath2, dummyRemoteFileURL,
90+
WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
91+
assert.NilError(t, err)
92+
assert.Equal(t, StatusUsedCache, r.Status)
93+
})
94+
t.Run("parallel", func(t *testing.T) {
95+
cacheDir := filepath.Join(t.TempDir(), "cache")
96+
results := make(chan downloadResult, parallelDownloads)
97+
for i := 0; i < parallelDownloads; i++ {
98+
go func() {
99+
// Parallel download is supported only for different instances with unique localPath.
100+
localPath := filepath.Join(t.TempDir(), t.Name())
101+
r, err := Download(context.Background(), localPath, dummyRemoteFileURL,
102+
WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
103+
results <- downloadResult{r, err}
104+
}()
105+
}
106+
// We must process all results before cleanup.
107+
for i := 0; i < parallelDownloads; i++ {
108+
result := <-results
109+
if result.err != nil {
110+
t.Errorf("Download failed: %s", result.err)
111+
} else if !slices.Contains(parallelStatus, result.r.Status) {
112+
t.Errorf("Expected download status %s, got %s", parallelStatus, result.r.Status)
113+
}
114+
}
115+
})
74116
})
75117
t.Run("caching-only mode", func(t *testing.T) {
76-
_, err := Download(context.Background(), "", dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest))
77-
assert.ErrorContains(t, err, "cache directory to be specified")
118+
t.Run("serial", func(t *testing.T) {
119+
_, err := Download(context.Background(), "", dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest))
120+
assert.ErrorContains(t, err, "cache directory to be specified")
78121

79-
cacheDir := filepath.Join(t.TempDir(), "cache")
80-
r, err := Download(context.Background(), "", dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
81-
assert.NilError(t, err)
82-
assert.Equal(t, StatusDownloaded, r.Status)
122+
cacheDir := filepath.Join(t.TempDir(), "cache")
123+
r, err := Download(context.Background(), "", dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest),
124+
WithCacheDir(cacheDir))
125+
assert.NilError(t, err)
126+
assert.Equal(t, StatusDownloaded, r.Status)
83127

84-
r, err = Download(context.Background(), "", dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
85-
assert.NilError(t, err)
86-
assert.Equal(t, StatusUsedCache, r.Status)
128+
r, err = Download(context.Background(), "", dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest),
129+
WithCacheDir(cacheDir))
130+
assert.NilError(t, err)
131+
assert.Equal(t, StatusUsedCache, r.Status)
87132

88-
localPath := filepath.Join(t.TempDir(), t.Name())
89-
r, err = Download(context.Background(), localPath, dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
90-
assert.NilError(t, err)
91-
assert.Equal(t, StatusUsedCache, r.Status)
133+
localPath := filepath.Join(t.TempDir(), t.Name())
134+
r, err = Download(context.Background(), localPath, dummyRemoteFileURL,
135+
WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
136+
assert.NilError(t, err)
137+
assert.Equal(t, StatusUsedCache, r.Status)
138+
})
139+
t.Run("parallel", func(t *testing.T) {
140+
cacheDir := filepath.Join(t.TempDir(), "cache")
141+
results := make(chan downloadResult, parallelDownloads)
142+
for i := 0; i < parallelDownloads; i++ {
143+
go func() {
144+
r, err := Download(context.Background(), "", dummyRemoteFileURL,
145+
WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
146+
results <- downloadResult{r, err}
147+
}()
148+
}
149+
// We must process all results before cleanup.
150+
for i := 0; i < parallelDownloads; i++ {
151+
result := <-results
152+
if result.err != nil {
153+
t.Errorf("Download failed: %s", result.err)
154+
} else if !slices.Contains(parallelStatus, result.r.Status) {
155+
t.Errorf("Expected download status %s, got %s", parallelStatus, result.r.Status)
156+
}
157+
}
158+
})
92159
})
93160
t.Run("cached", func(t *testing.T) {
94161
_, err := Cached(dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest))

0 commit comments

Comments
 (0)