Skip to content

Commit 7979b43

Browse files
Fix Git.Clone after 35bab8f (#3587)
* Fix Git.Clone after 35bab8f * Fix lint. \o/
1 parent 9cba1f3 commit 7979b43

File tree

4 files changed

+38
-20
lines changed

4 files changed

+38
-20
lines changed

internal/config/server/provider.go

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ type ProviderConfig struct {
2222
Git GitConfig `mapstructure:"git"`
2323
}
2424

25+
// GitConfig provides server-side configuration for Git operations like "clone"
2526
type GitConfig struct {
2627
MaxFiles int64 `mapstructure:"max_files" default:"10000"`
2728
MaxBytes int64 `mapstructure:"max_bytes" default:"100_000_000"`

internal/engine/ingester/git/git_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,17 @@ package git_test
1717
import (
1818
"bytes"
1919
"context"
20-
"github.com/stacklok/minder/internal/config/server"
21-
v1 "github.com/stacklok/minder/pkg/providers/v1"
2220
"testing"
2321

2422
"github.com/stretchr/testify/require"
2523

24+
"github.com/stacklok/minder/internal/config/server"
2625
engerrors "github.com/stacklok/minder/internal/engine/errors"
2726
gitengine "github.com/stacklok/minder/internal/engine/ingester/git"
2827
"github.com/stacklok/minder/internal/providers/credentials"
2928
gitclient "github.com/stacklok/minder/internal/providers/git"
3029
pb "github.com/stacklok/minder/pkg/api/protobuf/go/minder/v1"
30+
v1 "github.com/stacklok/minder/pkg/providers/v1"
3131
)
3232

3333
func TestGitIngestWithCloneURLFromRepo(t *testing.T) {

internal/providers/git/git.go

+19-8
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,14 @@ import (
1919
"context"
2020
"errors"
2121
"fmt"
22-
"io/fs"
2322

2423
"github.com/go-git/go-billy/v5/memfs"
25-
"github.com/go-git/go-git/v5/plumbing/cache"
26-
"github.com/go-git/go-git/v5/storage/filesystem"
27-
2824
"github.com/go-git/go-git/v5"
2925
"github.com/go-git/go-git/v5/plumbing"
26+
"github.com/go-git/go-git/v5/plumbing/cache"
3027
"github.com/go-git/go-git/v5/plumbing/transport"
28+
"github.com/go-git/go-git/v5/storage/filesystem"
29+
3130
"github.com/stacklok/minder/internal/config/server"
3231
"github.com/stacklok/minder/internal/providers/git/memboxfs"
3332
minderv1 "github.com/stacklok/minder/pkg/api/protobuf/go/minder/v1"
@@ -46,6 +45,7 @@ const maxCachedObjectSize = 100 * 1024 // 100KiB
4645
// Ensure that the Git client implements the Git interface
4746
var _ provifv1.Git = (*Git)(nil)
4847

48+
// Options implements the "functional options" pattern for Git
4949
type Options func(*Git)
5050

5151
// NewGit creates a new GitHub client
@@ -59,6 +59,7 @@ func NewGit(token provifv1.GitCredential, opts ...Options) *Git {
5959
return ret
6060
}
6161

62+
// WithConfig configures the Git implementation with server-side configuration options.
6263
func WithConfig(cfg server.GitConfig) Options {
6364
return func(g *Git) {
6465
g.maxFiles = cfg.MaxFiles
@@ -97,9 +98,17 @@ func (g *Git) Clone(ctx context.Context, url, branch string) (*git.Repository, e
9798
TotalFileSize: g.maxBytes,
9899
}
99100
}
101+
// go-git seems to want separate filesystems for the storer and the checked out files
102+
storerFs := memfs.New()
103+
if g.maxFiles != 0 && g.maxBytes != 0 {
104+
storerFs = &memboxfs.LimitedFs{
105+
Fs: storerFs,
106+
MaxFiles: g.maxFiles,
107+
TotalFileSize: g.maxBytes,
108+
}
109+
}
100110
storerCache := cache.NewObjectLRU(maxCachedObjectSize)
101-
// reuse same FS for storage and cloned files
102-
storer := filesystem.NewStorage(memFS, storerCache)
111+
storer := filesystem.NewStorage(storerFs, storerCache)
103112

104113
// We clone to the memfs go-billy filesystem driver, which doesn't
105114
// allow for direct access to the underlying filesystem. This is
@@ -112,8 +121,10 @@ func (g *Git) Clone(ctx context.Context, url, branch string) (*git.Repository, e
112121
return nil, provifv1.ErrProviderGitBranchNotFound
113122
} else if errors.Is(err, transport.ErrEmptyRemoteRepository) {
114123
return nil, provifv1.ErrRepositoryEmpty
115-
} else if errors.Is(err, fs.ErrPermission) {
116-
return nil, provifv1.ErrRepositoryTooLarge
124+
} else if errors.Is(err, memboxfs.ErrTooManyFiles) {
125+
return nil, fmt.Errorf("%w: %w", provifv1.ErrRepositoryTooLarge, err)
126+
} else if errors.Is(err, memboxfs.ErrTooBig) {
127+
return nil, fmt.Errorf("%w: %w", provifv1.ErrRepositoryTooLarge, err)
117128
}
118129
return nil, fmt.Errorf("could not clone repo: %w", err)
119130
}

internal/providers/git/memboxfs/fs.go

+16-10
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,23 @@ import (
2828
// LimitedFs provides a size-limited billy.Filesystem. This is a struct, there's
2929
// no constructor here. Note that LimitedFs is not thread-safe.
3030
type LimitedFs struct {
31-
Fs billy.Filesystem
31+
Fs billy.Filesystem
3232
MaxFiles int64
3333
TotalFileSize int64
3434

35-
currentFiles int64
36-
currentSize int64
35+
currentFiles int64
36+
currentSize int64
3737
}
3838

3939
// ErrNotImplemented is returned when a method is not implemented.
4040
var ErrNotImplemented = fmt.Errorf("not implemented")
4141

42+
// ErrTooBig is returned when a file is too big.
43+
var ErrTooBig = fmt.Errorf("file too big")
44+
45+
// ErrTooManyFiles is returned when there are too many files.
46+
var ErrTooManyFiles = fmt.Errorf("too many files")
47+
4248
var _ billy.Filesystem = (*LimitedFs)(nil)
4349

4450
// Chroot implements billy.Filesystem.
@@ -50,7 +56,7 @@ func (_ *LimitedFs) Chroot(_ string) (billy.Filesystem, error) {
5056
func (f *LimitedFs) Create(filename string) (billy.File, error) {
5157
f.currentFiles++
5258
if f.currentFiles >= f.MaxFiles {
53-
return nil, fs.ErrPermission
59+
return nil, fmt.Errorf("%w: %s", ErrTooManyFiles, filename)
5460
}
5561
file, err := f.Fs.Create(filename)
5662
return &fileWrapper{f: file, fs: f}, err
@@ -71,7 +77,7 @@ func (f *LimitedFs) MkdirAll(filename string, perm fs.FileMode) error {
7177
// TODO: account for path segments correctly
7278
f.currentFiles++
7379
if f.currentFiles >= f.MaxFiles {
74-
return fs.ErrPermission
80+
return fmt.Errorf("%w: %s", ErrTooManyFiles, filename)
7581
}
7682
return f.Fs.MkdirAll(filename, perm)
7783
}
@@ -86,7 +92,7 @@ func (f *LimitedFs) OpenFile(filename string, flag int, perm fs.FileMode) (billy
8692
if flag&os.O_CREATE != 0 {
8793
f.currentFiles++
8894
if f.currentFiles >= f.MaxFiles {
89-
return nil, fs.ErrPermission
95+
return nil, fmt.Errorf("%w: %s", ErrTooManyFiles, filename)
9096
}
9197
}
9298
file, err := f.Fs.OpenFile(filename, flag, perm)
@@ -129,7 +135,7 @@ func (f *LimitedFs) Stat(filename string) (fs.FileInfo, error) {
129135
func (f *LimitedFs) Symlink(target string, link string) error {
130136
f.currentFiles++
131137
if f.currentFiles >= f.MaxFiles {
132-
return fs.ErrPermission
138+
return fmt.Errorf("%w: %s", ErrTooManyFiles, link)
133139
}
134140
return f.Fs.Symlink(target, link)
135141
}
@@ -138,7 +144,7 @@ func (f *LimitedFs) Symlink(target string, link string) error {
138144
func (f *LimitedFs) TempFile(dir string, prefix string) (billy.File, error) {
139145
f.currentFiles++
140146
if f.currentFiles >= f.MaxFiles {
141-
return nil, fs.ErrPermission
147+
return nil, fmt.Errorf("%w: %s/%s", ErrTooManyFiles, dir, prefix)
142148
}
143149
file, err := f.Fs.TempFile(dir, prefix)
144150
return &fileWrapper{f: file, fs: f}, err
@@ -191,7 +197,7 @@ func (f *fileWrapper) Truncate(size int64) error {
191197

192198
growth := size - existingSize
193199
if growth+f.fs.currentSize > f.fs.TotalFileSize {
194-
return fs.ErrPermission
200+
return fmt.Errorf("%w: %s", ErrTooBig, f.Name())
195201
}
196202

197203
f.fs.currentSize += growth
@@ -219,7 +225,7 @@ func (f *fileWrapper) Write(p []byte) (n int, err error) {
219225
growth = 0
220226
}
221227
if growth+f.fs.currentSize > f.fs.TotalFileSize {
222-
return 0, fs.ErrPermission
228+
return 0, fmt.Errorf("%w: %s", ErrTooBig, f.Name())
223229
}
224230

225231
f.fs.currentSize += growth

0 commit comments

Comments
 (0)