Skip to content

Commit cdc9e91

Browse files
fuxiaoheiwolfogrelunny
authored
add path prefix to ObjectStorage.Iterator (#23332)
Support to iterator subdirectory in ObjectStorage for ObjectStorage.Iterator method. It's required for #22738 to make artifact files cleanable. --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
1 parent 757b4c1 commit cdc9e91

File tree

9 files changed

+67
-15
lines changed

9 files changed

+67
-15
lines changed

‎cmd/dump.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ func runDump(ctx *cli.Context) error {
250250

251251
if ctx.IsSet("skip-lfs-data") && ctx.Bool("skip-lfs-data") {
252252
log.Info("Skip dumping LFS data")
253-
} else if err := storage.LFS.IterateObjects(func(objPath string, object storage.Object) error {
253+
} else if err := storage.LFS.IterateObjects("", func(objPath string, object storage.Object) error {
254254
info, err := object.Stat()
255255
if err != nil {
256256
return err
@@ -351,7 +351,7 @@ func runDump(ctx *cli.Context) error {
351351

352352
if ctx.IsSet("skip-attachment-data") && ctx.Bool("skip-attachment-data") {
353353
log.Info("Skip dumping attachment data")
354-
} else if err := storage.Attachments.IterateObjects(func(objPath string, object storage.Object) error {
354+
} else if err := storage.Attachments.IterateObjects("", func(objPath string, object storage.Object) error {
355355
info, err := object.Stat()
356356
if err != nil {
357357
return err
@@ -364,7 +364,7 @@ func runDump(ctx *cli.Context) error {
364364

365365
if ctx.IsSet("skip-package-data") && ctx.Bool("skip-package-data") {
366366
log.Info("Skip dumping package data")
367-
} else if err := storage.Packages.IterateObjects(func(objPath string, object storage.Object) error {
367+
} else if err := storage.Packages.IterateObjects("", func(objPath string, object storage.Object) error {
368368
info, err := object.Stat()
369369
if err != nil {
370370
return err

‎modules/doctor/storage.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ func commonCheckStorage(ctx context.Context, logger log.Logger, autofix bool, op
3131
totalSize, orphanedSize := int64(0), int64(0)
3232

3333
var pathsToDelete []string
34-
if err := opts.storer.IterateObjects(func(p string, obj storage.Object) error {
34+
if err := opts.storer.IterateObjects("", func(p string, obj storage.Object) error {
3535
defer obj.Close()
3636

3737
totalCount++

‎modules/storage/helper.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,6 @@ func (s discardStorage) URL(_, _ string) (*url.URL, error) {
9090
return nil, fmt.Errorf("%s", s)
9191
}
9292

93-
func (s discardStorage) IterateObjects(_ func(string, Object) error) error {
93+
func (s discardStorage) IterateObjects(_ string, _ func(string, Object) error) error {
9494
return fmt.Errorf("%s", s)
9595
}

‎modules/storage/helper_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ func Test_discardStorage(t *testing.T) {
4242
assert.Errorf(t, err, string(tt))
4343
}
4444
{
45-
err := tt.IterateObjects(func(_ string, _ Object) error { return nil })
45+
err := tt.IterateObjects("", func(_ string, _ Object) error { return nil })
4646
assert.Error(t, err, string(tt))
4747
}
4848
})

‎modules/storage/local.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -127,8 +127,12 @@ func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
127127
}
128128

129129
// IterateObjects iterates across the objects in the local storage
130-
func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) error {
131-
return filepath.WalkDir(l.dir, func(path string, d os.DirEntry, err error) error {
130+
func (l *LocalStorage) IterateObjects(prefix string, fn func(path string, obj Object) error) error {
131+
dir := l.dir
132+
if prefix != "" {
133+
dir = filepath.Join(l.dir, util.CleanPath(prefix))
134+
}
135+
return filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
132136
if err != nil {
133137
return err
134138
}

‎modules/storage/local_test.go

+42
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@
44
package storage
55

66
import (
7+
"bytes"
8+
"context"
9+
"os"
10+
"path/filepath"
711
"testing"
812

913
"github.com/stretchr/testify/assert"
@@ -50,3 +54,41 @@ func TestBuildLocalPath(t *testing.T) {
5054
})
5155
}
5256
}
57+
58+
func TestLocalStorageIterator(t *testing.T) {
59+
dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir")
60+
l, err := NewLocalStorage(context.Background(), LocalStorageConfig{Path: dir})
61+
assert.NoError(t, err)
62+
63+
testFiles := [][]string{
64+
{"a/1.txt", "a1"},
65+
{"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim
66+
{"b/1.txt", "b1"},
67+
{"b/2.txt", "b2"},
68+
{"b/3.txt", "b3"},
69+
{"b/x 4.txt", "bx4"},
70+
}
71+
for _, f := range testFiles {
72+
_, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1)
73+
assert.NoError(t, err)
74+
}
75+
76+
expectedList := map[string][]string{
77+
"a": {"a/1.txt"},
78+
"b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
79+
"": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
80+
"/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
81+
"a/b/../../a": {"a/1.txt"},
82+
}
83+
for dir, expected := range expectedList {
84+
count := 0
85+
err = l.IterateObjects(dir, func(path string, f Object) error {
86+
defer f.Close()
87+
assert.Contains(t, expected, path)
88+
count++
89+
return nil
90+
})
91+
assert.NoError(t, err)
92+
assert.Equal(t, count, len(expected))
93+
}
94+
}

‎modules/storage/minio.go

+9-3
Original file line numberDiff line numberDiff line change
@@ -209,12 +209,18 @@ func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
209209
}
210210

211211
// IterateObjects iterates across the objects in the miniostorage
212-
func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) error {
212+
func (m *MinioStorage) IterateObjects(prefix string, fn func(path string, obj Object) error) error {
213213
opts := minio.GetObjectOptions{}
214214
lobjectCtx, cancel := context.WithCancel(m.ctx)
215215
defer cancel()
216+
217+
basePath := m.basePath
218+
if prefix != "" {
219+
basePath = m.buildMinioPath(prefix)
220+
}
221+
216222
for mObjInfo := range m.client.ListObjects(lobjectCtx, m.bucket, minio.ListObjectsOptions{
217-
Prefix: m.basePath,
223+
Prefix: basePath,
218224
Recursive: true,
219225
}) {
220226
object, err := m.client.GetObject(lobjectCtx, m.bucket, mObjInfo.Key, opts)
@@ -223,7 +229,7 @@ func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) er
223229
}
224230
if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
225231
defer object.Close()
226-
return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object})
232+
return fn(strings.TrimPrefix(mObjInfo.Key, basePath), &minioObject{object})
227233
}(object, fn); err != nil {
228234
return convertMinioErr(err)
229235
}

‎modules/storage/storage.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ type ObjectStorage interface {
6565
Stat(path string) (os.FileInfo, error)
6666
Delete(path string) error
6767
URL(path, name string) (*url.URL, error)
68-
IterateObjects(func(path string, obj Object) error) error
68+
IterateObjects(path string, iterator func(path string, obj Object) error) error
6969
}
7070

7171
// Copy copies a file from source ObjectStorage to dest ObjectStorage
@@ -87,7 +87,7 @@ func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, sr
8787

8888
// Clean delete all the objects in this storage
8989
func Clean(storage ObjectStorage) error {
90-
return storage.IterateObjects(func(path string, obj Object) error {
90+
return storage.IterateObjects("", func(path string, obj Object) error {
9191
_ = obj.Close()
9292
return storage.Delete(path)
9393
})

‎tests/test_utils.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() {
201201
lfsFixtures, err := storage.NewStorage("", storage.LocalStorageConfig{Path: path.Join(filepath.Dir(setting.AppPath), "tests/gitea-lfs-meta")})
202202
assert.NoError(t, err)
203203
assert.NoError(t, storage.Clean(storage.LFS))
204-
assert.NoError(t, lfsFixtures.IterateObjects(func(path string, _ storage.Object) error {
204+
assert.NoError(t, lfsFixtures.IterateObjects("", func(path string, _ storage.Object) error {
205205
_, err := storage.Copy(storage.LFS, path, lfsFixtures, path)
206206
return err
207207
}))
@@ -258,7 +258,7 @@ func ResetFixtures(t *testing.T) {
258258
lfsFixtures, err := storage.NewStorage("", storage.LocalStorageConfig{Path: path.Join(filepath.Dir(setting.AppPath), "tests/gitea-lfs-meta")})
259259
assert.NoError(t, err)
260260
assert.NoError(t, storage.Clean(storage.LFS))
261-
assert.NoError(t, lfsFixtures.IterateObjects(func(path string, _ storage.Object) error {
261+
assert.NoError(t, lfsFixtures.IterateObjects("", func(path string, _ storage.Object) error {
262262
_, err := storage.Copy(storage.LFS, path, lfsFixtures, path)
263263
return err
264264
}))

0 commit comments

Comments
 (0)