diff --git a/internal/strategy/git/command.go b/internal/gitclone/command.go similarity index 72% rename from internal/strategy/git/command.go rename to internal/gitclone/command.go index bbdd6a6..31a3a17 100644 --- a/internal/strategy/git/command.go +++ b/internal/gitclone/command.go @@ -1,4 +1,6 @@ -package git +// Package gitclone provides reusable git clone management with lifecycle control, +// concurrency management, and large repository optimizations. +package gitclone import ( "bufio" @@ -9,8 +11,6 @@ import ( "github.com/alecthomas/errors" ) -// gitCommand creates a git command with insteadOf URL rewriting disabled for the given URL -// to prevent infinite loops where git config rules rewrite URLs to point back through the proxy. func gitCommand(ctx context.Context, url string, args ...string) (*exec.Cmd, error) { configArgs, err := getInsteadOfDisableArgsForURL(ctx, url) if err != nil { @@ -27,7 +27,6 @@ func gitCommand(ctx context.Context, url string, args ...string) (*exec.Cmd, err return cmd, nil } -// getInsteadOfDisableArgsForURL returns arguments to disable insteadOf rules that would affect the given URL. func getInsteadOfDisableArgsForURL(ctx context.Context, targetURL string) ([]string, error) { if targetURL == "" { return nil, nil @@ -36,7 +35,6 @@ func getInsteadOfDisableArgsForURL(ctx context.Context, targetURL string) ([]str cmd := exec.CommandContext(ctx, "git", "config", "--get-regexp", "^url\\..*\\.(insteadof|pushinsteadof)$") output, err := cmd.CombinedOutput() if err != nil { - // Exit code 1 when no insteadOf rules exist is expected, not an error return []string{}, nil //nolint:nilerr } @@ -60,3 +58,18 @@ func getInsteadOfDisableArgsForURL(ctx context.Context, targetURL string) ([]str return args, nil } + +func ParseGitRefs(output []byte) map[string]string { + refs := make(map[string]string) + scanner := bufio.NewScanner(strings.NewReader(string(output))) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + if len(parts) >= 2 { + sha := parts[0] + ref := parts[1] + refs[ref] = sha + } + } + return refs +} diff --git a/internal/strategy/git/command_test.go b/internal/gitclone/command_test.go similarity index 78% rename from internal/strategy/git/command_test.go rename to internal/gitclone/command_test.go index cf64b13..e700a21 100644 --- a/internal/strategy/git/command_test.go +++ b/internal/gitclone/command_test.go @@ -1,4 +1,4 @@ -package git //nolint:testpackage // Internal functions need to be tested +package gitclone //nolint:testpackage // Internal functions need to be tested import ( "context" @@ -13,9 +13,7 @@ func TestGetInsteadOfDisableArgsForURL(t *testing.T) { tests := []struct { name string targetURL string - // We can't easily test the actual git config reading in a unit test, - // but we can test the logic would work correctly - skipTest bool + skipTest bool }{ { name: "EmptyURL", @@ -47,12 +45,10 @@ func TestGetInsteadOfDisableArgsForURL(t *testing.T) { func TestGitCommand(t *testing.T) { ctx := context.Background() - // Test that gitCommand creates a valid command cmd, err := gitCommand(ctx, "https://github.com/user/repo", "version") assert.NoError(t, err) assert.NotZero(t, cmd) - // Should have at least "git" and "version" in args assert.True(t, len(cmd.Args) >= 2) // First arg should be git binary path assert.Equal(t, "git", cmd.Args[0]) @@ -63,7 +59,6 @@ func TestGitCommand(t *testing.T) { func TestGitCommandWithEmptyURL(t *testing.T) { ctx := context.Background() - // Test with empty URL (for commands that don't need URL filtering) cmd, err := gitCommand(ctx, "", "version") assert.NoError(t, err) diff --git a/internal/gitclone/manager.go b/internal/gitclone/manager.go new file mode 100644 index 0000000..79de399 --- /dev/null +++ b/internal/gitclone/manager.go @@ -0,0 +1,431 @@ +package gitclone + +import ( + "context" + "io/fs" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/alecthomas/errors" +) + +type State int + +const ( + StateEmpty State = iota // Not cloned yet + StateCloning // Clone in progress + StateReady // Ready to use +) + +func (s State) String() string { + switch s { + case StateEmpty: + return "empty" + case StateCloning: + return "cloning" + case StateReady: + return "ready" + default: + return "unknown" + } +} + +type GitTuningConfig struct { + PostBuffer int // http.postBuffer size in bytes + LowSpeedLimit int // http.lowSpeedLimit in bytes/sec + LowSpeedTime time.Duration // http.lowSpeedTime +} + +func DefaultGitTuningConfig() GitTuningConfig { + return GitTuningConfig{ + PostBuffer: 524288000, // 500MB buffer + LowSpeedLimit: 1000, // 1KB/s minimum speed + LowSpeedTime: 10 * time.Minute, + } +} + +type Config struct { + RootDir string + FetchInterval time.Duration + RefCheckInterval time.Duration + CloneDepth int + GitConfig GitTuningConfig +} + +type Repository struct { + mu sync.RWMutex + state State + path string + upstreamURL string + lastFetch time.Time + lastRefCheck time.Time + refCheckValid bool + fetchSem chan struct{} +} + +type Manager struct { + config Config + clones map[string]*Repository + clonesMu sync.RWMutex +} + +func NewManager(_ context.Context, config Config) (*Manager, error) { + if config.RootDir == "" { + return nil, errors.New("RootDir is required") + } + + if err := os.MkdirAll(config.RootDir, 0o750); err != nil { + return nil, errors.Wrap(err, "create root directory") + } + + return &Manager{ + config: config, + clones: make(map[string]*Repository), + }, nil +} + +func (m *Manager) GetOrCreate(_ context.Context, upstreamURL string) (*Repository, error) { + m.clonesMu.RLock() + repo, exists := m.clones[upstreamURL] + m.clonesMu.RUnlock() + + if exists { + return repo, nil + } + + m.clonesMu.Lock() + defer m.clonesMu.Unlock() + + if repo, exists = m.clones[upstreamURL]; exists { + return repo, nil + } + + clonePath := m.clonePathForURL(upstreamURL) + + repo = &Repository{ + state: StateEmpty, + path: clonePath, + upstreamURL: upstreamURL, + fetchSem: make(chan struct{}, 1), + } + + gitDir := filepath.Join(clonePath, ".git") + if _, err := os.Stat(gitDir); err == nil { + repo.state = StateReady + } + + repo.fetchSem <- struct{}{} + + m.clones[upstreamURL] = repo + return repo, nil +} + +func (m *Manager) Get(upstreamURL string) *Repository { + m.clonesMu.RLock() + defer m.clonesMu.RUnlock() + return m.clones[upstreamURL] +} + +func (m *Manager) DiscoverExisting(_ context.Context) error { + err := filepath.Walk(m.config.RootDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() { + return nil + } + + gitDir := filepath.Join(path, ".git") + headPath := filepath.Join(path, ".git", "HEAD") + if _, statErr := os.Stat(gitDir); statErr != nil { + if errors.Is(statErr, os.ErrNotExist) { + return nil + } + return errors.Wrap(statErr, "stat .git directory") + } + if _, statErr := os.Stat(headPath); statErr != nil { + if errors.Is(statErr, os.ErrNotExist) { + return nil + } + return errors.Wrap(statErr, "stat HEAD file") + } + + relPath, err := filepath.Rel(m.config.RootDir, path) + if err != nil { + return errors.Wrap(err, "get relative path") + } + + urlPath := filepath.ToSlash(relPath) + + idx := strings.Index(urlPath, "/") + if idx == -1 { + return nil + } + + host := urlPath[:idx] + repoPath := urlPath[idx+1:] + upstreamURL := "https://" + host + "/" + repoPath + + repo := &Repository{ + state: StateReady, + path: path, + upstreamURL: upstreamURL, + fetchSem: make(chan struct{}, 1), + } + repo.fetchSem <- struct{}{} + + m.clonesMu.Lock() + m.clones[upstreamURL] = repo + m.clonesMu.Unlock() + + return fs.SkipDir + }) + + if err != nil { + return errors.Wrap(err, "walk root directory") + } + + return nil +} + +func (m *Manager) clonePathForURL(upstreamURL string) string { + parsed, err := url.Parse(upstreamURL) + if err != nil { + return filepath.Join(m.config.RootDir, "unknown") + } + + repoPath := strings.TrimSuffix(parsed.Path, ".git") + return filepath.Join(m.config.RootDir, parsed.Host, repoPath) +} + +func (r *Repository) State() State { + r.mu.RLock() + defer r.mu.RUnlock() + return r.state +} + +func (r *Repository) Path() string { + return r.path +} + +func (r *Repository) UpstreamURL() string { + return r.upstreamURL +} + +func (r *Repository) LastFetch() time.Time { + r.mu.RLock() + defer r.mu.RUnlock() + return r.lastFetch +} + +func (r *Repository) NeedsFetch(fetchInterval time.Duration) bool { + r.mu.RLock() + defer r.mu.RUnlock() + return time.Since(r.lastFetch) >= fetchInterval +} + +func (r *Repository) WithReadLock(fn func()) { + r.mu.RLock() + defer r.mu.RUnlock() + fn() +} + +func (r *Repository) Clone(ctx context.Context, config Config) error { + r.mu.Lock() + if r.state != StateEmpty { + r.mu.Unlock() + return nil + } + r.state = StateCloning + + err := r.executeClone(ctx, config) + + if err != nil { + r.state = StateEmpty + r.mu.Unlock() + return err + } + + r.state = StateReady + r.lastFetch = time.Now() + r.mu.Unlock() + return nil +} + +func (r *Repository) executeClone(ctx context.Context, config Config) error { + if err := os.MkdirAll(filepath.Dir(r.path), 0o750); err != nil { + return errors.Wrap(err, "create clone directory") + } + + // #nosec G204 - r.upstreamURL and r.path are controlled by us + args := []string{"clone"} + if config.CloneDepth > 0 { + args = append(args, "--depth", strconv.Itoa(config.CloneDepth)) + } + args = append(args, + "-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer), + "-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit), + "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())), + r.upstreamURL, r.path) + + cmd, err := gitCommand(ctx, r.upstreamURL, args...) + if err != nil { + return errors.Wrap(err, "create git command") + } + output, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrapf(err, "git clone: %s", string(output)) + } + + // #nosec G204 - r.path is controlled by us + cmd = exec.CommandContext(ctx, "git", "-C", r.path, "config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*") + output, err = cmd.CombinedOutput() + if err != nil { + return errors.Wrapf(err, "configure fetch refspec: %s", string(output)) + } + + cmd, err = gitCommand(ctx, r.upstreamURL, "-C", r.path, + "-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer), + "-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit), + "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())), + "fetch", "--all") + if err != nil { + return errors.Wrap(err, "create git command for fetch") + } + output, err = cmd.CombinedOutput() + if err != nil { + return errors.Wrapf(err, "fetch all branches: %s", string(output)) + } + + return nil +} + +func (r *Repository) Fetch(ctx context.Context, config Config) error { + select { + case <-r.fetchSem: + defer func() { + r.fetchSem <- struct{}{} + }() + case <-ctx.Done(): + return errors.Wrap(ctx.Err(), "context cancelled before acquiring fetch semaphore") + default: + select { + case <-r.fetchSem: + r.fetchSem <- struct{}{} + return nil + case <-ctx.Done(): + return errors.Wrap(ctx.Err(), "context cancelled while waiting for fetch") + } + } + + r.mu.Lock() + + // #nosec G204 - r.path is controlled by us + cmd, err := gitCommand(ctx, r.upstreamURL, "-C", r.path, + "-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer), + "-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit), + "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())), + "remote", "update", "--prune") + if err != nil { + return errors.Wrap(err, "create git command") + } + output, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrapf(err, "git remote update: %s", string(output)) + } + + r.lastFetch = time.Now() + r.mu.Unlock() + + return nil +} + +func (r *Repository) EnsureRefsUpToDate(ctx context.Context, config Config) error { + r.mu.Lock() + if r.refCheckValid && time.Since(r.lastRefCheck) < config.RefCheckInterval { + r.mu.Unlock() + return nil + } + r.lastRefCheck = time.Now() + r.refCheckValid = true + r.mu.Unlock() + + localRefs, err := r.GetLocalRefs(ctx) + if err != nil { + return errors.Wrap(err, "get local refs") + } + + upstreamRefs, err := r.GetUpstreamRefs(ctx) + if err != nil { + return errors.Wrap(err, "get upstream refs") + } + + needsFetch := false + for ref, upstreamSHA := range upstreamRefs { + if strings.HasSuffix(ref, "^{}") { + continue + } + if !strings.HasPrefix(ref, "refs/heads/") { + continue + } + localRef := "refs/remotes/origin/" + strings.TrimPrefix(ref, "refs/heads/") + localSHA, exists := localRefs[localRef] + if !exists || localSHA != upstreamSHA { + needsFetch = true + break + } + } + + if !needsFetch { + r.mu.Lock() + r.refCheckValid = true + r.mu.Unlock() + return nil + } + + err = r.Fetch(ctx, config) + if err != nil { + r.mu.Lock() + r.refCheckValid = false + r.mu.Unlock() + } + return err +} + +func (r *Repository) GetLocalRefs(ctx context.Context) (map[string]string, error) { + var output []byte + var err error + + r.mu.RLock() + // #nosec G204 - r.path is controlled by us + cmd := exec.CommandContext(ctx, "git", "-C", r.path, "for-each-ref", "--format=%(objectname) %(refname)") + output, err = cmd.CombinedOutput() + r.mu.RUnlock() + + if err != nil { + return nil, errors.Wrap(err, "git for-each-ref") + } + + return ParseGitRefs(output), nil +} + +func (r *Repository) GetUpstreamRefs(ctx context.Context) (map[string]string, error) { + // #nosec G204 - r.upstreamURL is controlled by us + cmd, err := gitCommand(ctx, r.upstreamURL, "ls-remote", r.upstreamURL) + if err != nil { + return nil, errors.Wrap(err, "create git command") + } + output, err := cmd.CombinedOutput() + if err != nil { + return nil, errors.Wrap(err, "git ls-remote") + } + + return ParseGitRefs(output), nil +} diff --git a/internal/gitclone/manager_test.go b/internal/gitclone/manager_test.go new file mode 100644 index 0000000..6ad97df --- /dev/null +++ b/internal/gitclone/manager_test.go @@ -0,0 +1,216 @@ +package gitclone //nolint:testpackage // white-box testing required for unexported fields + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/alecthomas/assert/v2" +) + +func TestNewManager(t *testing.T) { + tmpDir := t.TempDir() + + config := Config{ + RootDir: tmpDir, + FetchInterval: 15 * time.Minute, + RefCheckInterval: 10 * time.Second, + CloneDepth: 0, + GitConfig: DefaultGitTuningConfig(), + } + + manager, err := NewManager(context.Background(), config) + assert.NoError(t, err) + assert.NotZero(t, manager) + assert.Equal(t, tmpDir, manager.config.RootDir) +} + +func TestNewManager_RequiresRootDir(t *testing.T) { + config := Config{ + FetchInterval: 15 * time.Minute, + RefCheckInterval: 10 * time.Second, + } + + _, err := NewManager(context.Background(), config) + assert.Error(t, err) + assert.Contains(t, err.Error(), "RootDir is required") +} + +func TestManager_GetOrCreate(t *testing.T) { + tmpDir := t.TempDir() + config := Config{ + RootDir: tmpDir, + FetchInterval: 15 * time.Minute, + RefCheckInterval: 10 * time.Second, + GitConfig: DefaultGitTuningConfig(), + } + + manager, err := NewManager(context.Background(), config) + assert.NoError(t, err) + + upstreamURL := "https://github.com/user/repo" + repo, err := manager.GetOrCreate(context.Background(), upstreamURL) + assert.NoError(t, err) + assert.NotZero(t, repo) + + assert.Equal(t, upstreamURL, repo.UpstreamURL()) + assert.Equal(t, StateEmpty, repo.State()) + assert.Equal(t, filepath.Join(tmpDir, "github.com", "user", "repo"), repo.Path()) + + repo2, err := manager.GetOrCreate(context.Background(), upstreamURL) + assert.NoError(t, err) + assert.True(t, repo == repo2, "expected same repository instance") +} + +func TestManager_GetOrCreate_ExistingClone(t *testing.T) { + tmpDir := t.TempDir() + config := Config{ + RootDir: tmpDir, + FetchInterval: 15 * time.Minute, + RefCheckInterval: 10 * time.Second, + GitConfig: DefaultGitTuningConfig(), + } + + manager, err := NewManager(context.Background(), config) + assert.NoError(t, err) + + repoPath := filepath.Join(tmpDir, "github.com", "user", "repo") + gitDir := filepath.Join(repoPath, ".git") + assert.NoError(t, os.MkdirAll(gitDir, 0o755)) + assert.NoError(t, os.WriteFile(filepath.Join(gitDir, "HEAD"), []byte("ref: refs/heads/main\n"), 0o644)) + + upstreamURL := "https://github.com/user/repo" + repo, err := manager.GetOrCreate(context.Background(), upstreamURL) + assert.NoError(t, err) + assert.NotZero(t, repo) + + assert.Equal(t, StateReady, repo.State()) +} + +func TestManager_Get(t *testing.T) { + tmpDir := t.TempDir() + config := Config{ + RootDir: tmpDir, + FetchInterval: 15 * time.Minute, + RefCheckInterval: 10 * time.Second, + GitConfig: DefaultGitTuningConfig(), + } + + manager, err := NewManager(context.Background(), config) + assert.NoError(t, err) + + upstreamURL := "https://github.com/user/repo" + + repo := manager.Get(upstreamURL) + assert.Zero(t, repo) + + _, err = manager.GetOrCreate(context.Background(), upstreamURL) + assert.NoError(t, err) + + repo = manager.Get(upstreamURL) + assert.NotZero(t, repo) + assert.Equal(t, upstreamURL, repo.UpstreamURL()) +} + +func TestManager_DiscoverExisting(t *testing.T) { + tmpDir := t.TempDir() + config := Config{ + RootDir: tmpDir, + FetchInterval: 15 * time.Minute, + RefCheckInterval: 10 * time.Second, + GitConfig: DefaultGitTuningConfig(), + } + + manager, err := NewManager(context.Background(), config) + assert.NoError(t, err) + + repos := []string{ + filepath.Join(tmpDir, "github.com", "user1", "repo1"), + filepath.Join(tmpDir, "github.com", "user2", "repo2"), + filepath.Join(tmpDir, "gitlab.com", "org", "project"), + } + + for _, repoPath := range repos { + gitDir := filepath.Join(repoPath, ".git") + assert.NoError(t, os.MkdirAll(gitDir, 0o755)) + assert.NoError(t, os.WriteFile(filepath.Join(gitDir, "HEAD"), []byte("ref: refs/heads/main\n"), 0o644)) + } + + err = manager.DiscoverExisting(context.Background()) + assert.NoError(t, err) + + repo1 := manager.Get("https://github.com/user1/repo1") + assert.NotZero(t, repo1) + assert.Equal(t, StateReady, repo1.State()) + + repo2 := manager.Get("https://github.com/user2/repo2") + assert.NotZero(t, repo2) + assert.Equal(t, StateReady, repo2.State()) + + repo3 := manager.Get("https://gitlab.com/org/project") + assert.NotZero(t, repo3) + assert.Equal(t, StateReady, repo3.State()) +} + +func TestRepository_StateTransitions(t *testing.T) { + repo := &Repository{ + state: StateEmpty, + path: "/tmp/test", + upstreamURL: "https://github.com/user/repo", + fetchSem: make(chan struct{}, 1), + } + repo.fetchSem <- struct{}{} + + assert.Equal(t, StateEmpty, repo.State()) + + repo.mu.Lock() + repo.state = StateCloning + repo.mu.Unlock() + assert.Equal(t, StateCloning, repo.State()) + + repo.mu.Lock() + repo.state = StateReady + repo.mu.Unlock() + assert.Equal(t, StateReady, repo.State()) +} + +func TestRepository_NeedsFetch(t *testing.T) { + repo := &Repository{ + state: StateEmpty, + lastFetch: time.Now().Add(-20 * time.Minute), + fetchSem: make(chan struct{}, 1), + } + repo.fetchSem <- struct{}{} + + assert.True(t, repo.NeedsFetch(15*time.Minute)) + + assert.False(t, repo.NeedsFetch(30*time.Minute)) + + repo.mu.Lock() + repo.lastFetch = time.Now() + repo.mu.Unlock() + + assert.False(t, repo.NeedsFetch(15*time.Minute)) +} + +func TestParseGitRefs(t *testing.T) { + output := []byte(` +abc123 refs/heads/main +def456 refs/heads/develop +789012 refs/tags/v1.0.0 + `) + + refs := ParseGitRefs(output) + + assert.Equal(t, "abc123", refs["refs/heads/main"]) + assert.Equal(t, "def456", refs["refs/heads/develop"]) + assert.Equal(t, "789012", refs["refs/tags/v1.0.0"]) +} + +func TestState_String(t *testing.T) { + assert.Equal(t, "empty", StateEmpty.String()) + assert.Equal(t, "cloning", StateCloning.String()) + assert.Equal(t, "ready", StateReady.String()) +} diff --git a/internal/strategy/git/backend.go b/internal/strategy/git/backend.go index 82325a0..677a155 100644 --- a/internal/strategy/git/backend.go +++ b/internal/strategy/git/backend.go @@ -1,7 +1,6 @@ package git import ( - "bufio" "bytes" "context" "log/slog" @@ -10,17 +9,16 @@ import ( "os" "os/exec" "path/filepath" - "strconv" "strings" - "time" "github.com/alecthomas/errors" + "github.com/block/cachew/internal/gitclone" "github.com/block/cachew/internal/httputil" "github.com/block/cachew/internal/logging" ) -func (s *Strategy) serveFromBackend(w http.ResponseWriter, r *http.Request, c *clone) { +func (s *Strategy) serveFromBackend(w http.ResponseWriter, r *http.Request, repo *gitclone.Repository) { ctx := r.Context() logger := logging.FromContext(ctx) @@ -57,245 +55,45 @@ func (s *Strategy) serveFromBackend(w http.ResponseWriter, r *http.Request, c *c logger.DebugContext(r.Context(), "Serving with git http-backend", slog.String("original_path", r.URL.Path), slog.String("backend_path", backendPath), - slog.String("clone_path", c.path)) - - var stderrBuf bytes.Buffer - - handler := &cgi.Handler{ - Path: gitPath, - Args: []string{"http-backend"}, - Stderr: &stderrBuf, - Env: []string{ - "GIT_PROJECT_ROOT=" + absRoot, - "GIT_HTTP_EXPORT_ALL=1", - "PATH=" + os.Getenv("PATH"), - }, - } - - r2 := r.Clone(r.Context()) - r2.URL.Path = backendPath - - handler.ServeHTTP(w, r2) - - if stderrBuf.Len() > 0 { - logger.ErrorContext(r.Context(), "git http-backend error", - slog.String("stderr", stderrBuf.String()), - slog.String("path", backendPath)) - } -} - -func (s *Strategy) executeClone(ctx context.Context, c *clone) error { - logger := logging.FromContext(ctx) - - if err := os.MkdirAll(filepath.Dir(c.path), 0o750); err != nil { - return errors.Wrap(err, "create clone directory") - } - - // Configure git for large repositories to avoid network buffer issues - // #nosec G204 - c.upstreamURL and c.path are controlled by us - args := []string{"clone"} - if s.config.CloneDepth > 0 { - args = append(args, "--depth", strconv.Itoa(s.config.CloneDepth)) - } - args = append(args, - "-c", "http.postBuffer=524288000", // 500MB buffer - "-c", "http.lowSpeedLimit=1000", // 1KB/s minimum speed - "-c", "http.lowSpeedTime=600", // 10 minute timeout at low speed - c.upstreamURL, c.path) - cmd, err := gitCommand(ctx, c.upstreamURL, args...) - if err != nil { - return errors.Wrap(err, "create git command") - } - output, err := cmd.CombinedOutput() - if err != nil { - logger.ErrorContext(ctx, "git clone failed", - slog.String("error", err.Error()), - slog.String("output", string(output))) - return errors.Wrap(err, "git clone") - } - - // git clone only sets up fetching for the default branch, change it to fetch all branches - // #nosec G204 - c.path is controlled by us - cmd = exec.CommandContext(ctx, "git", "-C", c.path, "config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*") - output, err = cmd.CombinedOutput() - if err != nil { - logger.ErrorContext(ctx, "git config failed", - slog.String("error", err.Error()), - slog.String("output", string(output))) - return errors.Wrap(err, "configure fetch refspec") - } - - cmd, err = gitCommand(ctx, c.upstreamURL, "-C", c.path, - "-c", "http.postBuffer=524288000", - "-c", "http.lowSpeedLimit=1000", - "-c", "http.lowSpeedTime=600", - "fetch", "--all") - if err != nil { - return errors.Wrap(err, "create git command for fetch") - } - output, err = cmd.CombinedOutput() - if err != nil { - logger.ErrorContext(ctx, "git fetch --all failed", - slog.String("error", err.Error()), - slog.String("output", string(output))) - return errors.Wrap(err, "fetch all branches") - } - - return nil -} - -func (s *Strategy) executeFetch(ctx context.Context, c *clone) error { - logger := logging.FromContext(ctx) - - select { - case <-c.fetchSem: - defer func() { - c.fetchSem <- struct{}{} - }() - case <-ctx.Done(): - return errors.Wrap(ctx.Err(), "context cancelled before acquiring fetch semaphore") - default: - logger.DebugContext(ctx, "Fetch already in progress, waiting") - select { - case <-c.fetchSem: - c.fetchSem <- struct{}{} - return nil - case <-ctx.Done(): - return errors.Wrap(ctx.Err(), "context cancelled while waiting for fetch") + slog.String("clone_path", repo.Path())) + + repo.WithReadLock(func() { + var stderrBuf bytes.Buffer + + handler := &cgi.Handler{ + Path: gitPath, + Args: []string{"http-backend"}, + Stderr: &stderrBuf, + Env: []string{ + "GIT_PROJECT_ROOT=" + absRoot, + "GIT_HTTP_EXPORT_ALL=1", + "PATH=" + os.Getenv("PATH"), + }, } - } - - // Configure git for large repositories to avoid network buffer issues - // #nosec G204 - c.path is controlled by us - cmd, err := gitCommand(ctx, c.upstreamURL, "-C", c.path, - "-c", "http.postBuffer=524288000", // 500MB buffer - "-c", "http.lowSpeedLimit=1000", // 1KB/s minimum speed - "-c", "http.lowSpeedTime=600", // 10 minute timeout at low speed - "remote", "update", "--prune") - if err != nil { - logger.ErrorContext(ctx, "Failed to create git command", - slog.String("upstream", c.upstreamURL), - slog.String("error", err.Error())) - return errors.Wrap(err, "create git command") - } - output, err := cmd.CombinedOutput() - if err != nil { - logger.ErrorContext(ctx, "git remote update failed", - slog.String("error", err.Error()), - slog.String("output", string(output))) - return errors.Wrap(err, "git remote update") - } - - logger.DebugContext(ctx, "git remote update succeeded", slog.String("output", string(output))) - return nil -} - -// ensureRefsUpToDate checks if upstream has refs we don't have and fetches if needed. -// Short-lived cache avoids excessive ls-remote calls. -func (s *Strategy) ensureRefsUpToDate(ctx context.Context, c *clone) error { - logger := logging.FromContext(ctx) - - c.mu.Lock() - if c.refCheckValid && time.Since(c.lastRefCheck) < s.config.RefCheckInterval { - c.mu.Unlock() - logger.DebugContext(ctx, "Skipping ref check, recently checked", - slog.Duration("since_last_check", time.Since(c.lastRefCheck))) - return nil - } - c.lastRefCheck = time.Now() - c.refCheckValid = true - c.mu.Unlock() - logger.DebugContext(ctx, "Checking upstream for new refs", - slog.String("upstream", c.upstreamURL)) + r2 := r.Clone(r.Context()) + r2.URL.Path = backendPath - localRefs, err := s.getLocalRefs(ctx, c) - if err != nil { - return errors.Wrap(err, "get local refs") - } - - upstreamRefs, err := s.getUpstreamRefs(ctx, c) - if err != nil { - return errors.Wrap(err, "get upstream refs") - } + handler.ServeHTTP(w, r2) - needsFetch := false - for ref, upstreamSHA := range upstreamRefs { - if strings.HasSuffix(ref, "^{}") { - continue + if stderrBuf.Len() > 0 { + logger.ErrorContext(r.Context(), "git http-backend error", + slog.String("stderr", stderrBuf.String()), + slog.String("path", backendPath)) } - // Only check refs/heads/* since GitHub exposes refs/pull/* and other refs we don't fetch - if !strings.HasPrefix(ref, "refs/heads/") { - continue - } - localRef := "refs/remotes/origin/" + strings.TrimPrefix(ref, "refs/heads/") - localSHA, exists := localRefs[localRef] - if !exists || localSHA != upstreamSHA { - logger.DebugContext(ctx, "Upstream ref differs from local", - slog.String("upstream_ref", ref), - slog.String("local_ref", localRef), - slog.String("upstream_sha", upstreamSHA), - slog.String("local_sha", localSHA)) - needsFetch = true - break - } - } - - if !needsFetch { - c.mu.Lock() - c.refCheckValid = true - c.mu.Unlock() - logger.DebugContext(ctx, "No upstream changes detected") - return nil - } - - logger.InfoContext(ctx, "Upstream has new or updated refs, fetching") - err = s.executeFetch(ctx, c) - if err != nil { - c.mu.Lock() - c.refCheckValid = false - c.mu.Unlock() - } - return err -} - -func (s *Strategy) getLocalRefs(ctx context.Context, c *clone) (map[string]string, error) { - // #nosec G204 - c.path is controlled by us - cmd := exec.CommandContext(ctx, "git", "-C", c.path, "for-each-ref", "--format=%(objectname) %(refname)") - output, err := cmd.CombinedOutput() - if err != nil { - return nil, errors.Wrap(err, "git for-each-ref") - } - - return ParseGitRefs(output), nil + }) } -func (s *Strategy) getUpstreamRefs(ctx context.Context, c *clone) (map[string]string, error) { - // #nosec G204 - c.upstreamURL is controlled by us - cmd, err := gitCommand(ctx, c.upstreamURL, "ls-remote", c.upstreamURL) - if err != nil { - return nil, errors.Wrap(err, "create git command") +func (s *Strategy) ensureRefsUpToDate(ctx context.Context, repo *gitclone.Repository) error { + gitcloneConfig := gitclone.Config{ + RootDir: s.config.MirrorRoot, + FetchInterval: s.config.FetchInterval, + RefCheckInterval: s.config.RefCheckInterval, + CloneDepth: s.config.CloneDepth, + GitConfig: gitclone.DefaultGitTuningConfig(), } - output, err := cmd.CombinedOutput() - if err != nil { - return nil, errors.Wrap(err, "git ls-remote") + if err := repo.EnsureRefsUpToDate(ctx, gitcloneConfig); err != nil { + return errors.Wrap(err, "ensure refs up to date") } - - return ParseGitRefs(output), nil -} - -// ParseGitRefs parses the output of git show-ref or git ls-remote (format: ). -func ParseGitRefs(output []byte) map[string]string { - refs := make(map[string]string) - scanner := bufio.NewScanner(strings.NewReader(string(output))) - for scanner.Scan() { - line := scanner.Text() - parts := strings.Fields(line) - if len(parts) >= 2 { - sha := parts[0] - ref := parts[1] - refs[ref] = sha - } - } - return refs + return nil } diff --git a/internal/strategy/git/bundle.go b/internal/strategy/git/bundle.go index 951b90d..972a7b6 100644 --- a/internal/strategy/git/bundle.go +++ b/internal/strategy/git/bundle.go @@ -5,20 +5,21 @@ import ( "io" "log/slog" "net/http" - "strings" + "os/exec" "time" "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/gitclone" "github.com/block/cachew/internal/logging" ) -func (s *Strategy) generateAndUploadBundle(ctx context.Context, c *clone) { +func (s *Strategy) generateAndUploadBundle(ctx context.Context, repo *gitclone.Repository) { logger := logging.FromContext(ctx) logger.InfoContext(ctx, "Generating bundle", - slog.String("upstream", c.upstreamURL)) + slog.String("upstream", repo.UpstreamURL())) - cacheKey := cache.NewKey(c.upstreamURL + ".bundle") + cacheKey := cache.NewKey(repo.UpstreamURL() + ".bundle") headers := http.Header{ "Content-Type": []string{"application/x-git-bundle"}, @@ -27,59 +28,54 @@ func (s *Strategy) generateAndUploadBundle(ctx context.Context, c *clone) { w, err := s.cache.Create(ctx, cacheKey, headers, ttl) if err != nil { logger.ErrorContext(ctx, "Failed to create cache entry", - slog.String("upstream", c.upstreamURL), + slog.String("upstream", repo.UpstreamURL()), slog.String("error", err.Error())) return } defer w.Close() - // Use --branches --remotes to include all branches but exclude tags (which can be massive) - // #nosec G204 - c.path is controlled by us - args := []string{"-C", c.path, "bundle", "create", "-", "--branches", "--remotes"} - cmd, err := gitCommand(ctx, "", args...) - if err != nil { - logger.ErrorContext(ctx, "Failed to create git command", - slog.String("upstream", c.upstreamURL), - slog.String("error", err.Error())) - return - } - cmd.Stdout = w + repo.WithReadLock(func() { + // Use --branches --remotes to include all branches but exclude tags (which can be massive) + // #nosec G204 - repo.Path() is controlled by us + cmd := exec.CommandContext(ctx, "git", "-C", repo.Path(), "bundle", "create", "-", "--branches", "--remotes") + cmd.Stdout = w - stderrPipe, err := cmd.StderrPipe() - if err != nil { - logger.ErrorContext(ctx, "Failed to create stderr pipe", - slog.String("upstream", c.upstreamURL), - slog.String("error", err.Error())) - return - } + stderrPipe, err := cmd.StderrPipe() + if err != nil { + logger.ErrorContext(ctx, "Failed to create stderr pipe", + slog.String("upstream", repo.UpstreamURL()), + slog.String("error", err.Error())) + return + } - logger.DebugContext(ctx, "Starting bundle generation", - slog.String("upstream", c.upstreamURL), - slog.String("command", "git "+strings.Join(args, " "))) + logger.DebugContext(ctx, "Starting bundle generation", + slog.String("upstream", repo.UpstreamURL()), + slog.String("path", repo.Path())) - if err := cmd.Start(); err != nil { - logger.ErrorContext(ctx, "Failed to start bundle generation", - slog.String("upstream", c.upstreamURL), - slog.String("error", err.Error())) - return - } + if err := cmd.Start(); err != nil { + logger.ErrorContext(ctx, "Failed to start bundle generation", + slog.String("upstream", repo.UpstreamURL()), + slog.String("error", err.Error())) + return + } - stderr, _ := io.ReadAll(stderrPipe) //nolint:errcheck + stderr, _ := io.ReadAll(stderrPipe) //nolint:errcheck - if err := cmd.Wait(); err != nil { - logger.ErrorContext(ctx, "Failed to generate bundle", - slog.String("upstream", c.upstreamURL), - slog.String("error", err.Error()), - slog.String("stderr", string(stderr))) - return - } + if err := cmd.Wait(); err != nil { + logger.ErrorContext(ctx, "Failed to generate bundle", + slog.String("upstream", repo.UpstreamURL()), + slog.String("error", err.Error()), + slog.String("stderr", string(stderr))) + return + } - if len(stderr) > 0 { - logger.DebugContext(ctx, "Bundle generation stderr", - slog.String("upstream", c.upstreamURL), - slog.String("stderr", string(stderr))) - } + if len(stderr) > 0 { + logger.DebugContext(ctx, "Bundle generation stderr", + slog.String("upstream", repo.UpstreamURL()), + slog.String("stderr", string(stderr))) + } - logger.InfoContext(ctx, "Bundle uploaded successfully", - slog.String("upstream", c.upstreamURL)) + logger.InfoContext(ctx, "Bundle uploaded successfully", + slog.String("upstream", repo.UpstreamURL())) + }) } diff --git a/internal/strategy/git/git.go b/internal/strategy/git/git.go index e2a1f3c..c1605d1 100644 --- a/internal/strategy/git/git.go +++ b/internal/strategy/git/git.go @@ -7,16 +7,14 @@ import ( "log/slog" "net/http" "net/http/httputil" - "net/url" "os" - "path/filepath" "strings" - "sync" "time" "github.com/alecthomas/errors" "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/gitclone" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy" @@ -34,34 +32,14 @@ type Config struct { CloneDepth int `hcl:"clone-depth,optional" help:"Depth for shallow clones. 0 means full clone." default:"0"` } -type cloneState int - -const ( - stateEmpty cloneState = iota - stateCloning - stateReady -) - -type clone struct { - mu sync.RWMutex - state cloneState - path string - upstreamURL string - lastFetch time.Time - lastRefCheck time.Time - refCheckValid bool - fetchSem chan struct{} -} - type Strategy struct { - config Config - cache cache.Cache - clones map[string]*clone - clonesMu sync.RWMutex - httpClient *http.Client - proxy *httputil.ReverseProxy - ctx context.Context - scheduler jobscheduler.Scheduler + config Config + cache cache.Cache + cloneManager *gitclone.Manager + httpClient *http.Client + proxy *httputil.ReverseProxy + ctx context.Context + scheduler jobscheduler.Scheduler } func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, cache cache.Cache, mux strategy.Mux) (*Strategy, error) { @@ -79,20 +57,27 @@ func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, c config.RefCheckInterval = 10 * time.Second } - if err := os.MkdirAll(config.MirrorRoot, 0o750); err != nil { - return nil, errors.Wrap(err, "create mirror root directory") + cloneManager, err := gitclone.NewManager(ctx, gitclone.Config{ + RootDir: config.MirrorRoot, + FetchInterval: config.FetchInterval, + RefCheckInterval: config.RefCheckInterval, + CloneDepth: config.CloneDepth, + GitConfig: gitclone.DefaultGitTuningConfig(), + }) + if err != nil { + return nil, errors.Wrap(err, "create clone manager") } s := &Strategy{ - config: config, - cache: cache, - clones: make(map[string]*clone), - httpClient: http.DefaultClient, - ctx: ctx, - scheduler: scheduler.WithQueuePrefix("git"), + config: config, + cache: cache, + cloneManager: cloneManager, + httpClient: http.DefaultClient, + ctx: ctx, + scheduler: scheduler.WithQueuePrefix("git"), } - if err := s.discoverExistingClones(ctx); err != nil { + if err := s.cloneManager.DiscoverExisting(ctx); err != nil { logger.WarnContext(ctx, "Failed to discover existing clones", slog.String("error", err.Error())) } @@ -156,33 +141,36 @@ func (s *Strategy) handleRequest(w http.ResponseWriter, r *http.Request) { repoPath := ExtractRepoPath(pathValue) upstreamURL := "https://" + host + "/" + repoPath - c := s.getOrCreateClone(ctx, upstreamURL) - - c.mu.RLock() - state := c.state - c.mu.RUnlock() + repo, err := s.cloneManager.GetOrCreate(ctx, upstreamURL) + if err != nil { + logger.ErrorContext(ctx, "Failed to get or create clone", + slog.String("error", err.Error())) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + state := repo.State() isInfoRefs := strings.HasSuffix(pathValue, "/info/refs") switch state { - case stateReady: + case gitclone.StateReady: if isInfoRefs { - if err := s.ensureRefsUpToDate(ctx, c); err != nil { + if err := s.ensureRefsUpToDate(ctx, repo); err != nil { logger.WarnContext(ctx, "Failed to ensure refs up to date", slog.String("error", err.Error())) } } - s.maybeBackgroundFetch(c) - s.serveFromBackend(w, r, c) + s.maybeBackgroundFetch(repo) + s.serveFromBackend(w, r, repo) - case stateCloning: + case gitclone.StateCloning: logger.DebugContext(ctx, "Clone in progress, forwarding to upstream") s.forwardToUpstream(w, r, host, pathValue) - case stateEmpty: + case gitclone.StateEmpty: logger.DebugContext(ctx, "Starting background clone, forwarding to upstream") - s.scheduler.Submit(c.upstreamURL, "clone", func(ctx context.Context) error { - s.startClone(ctx, c) + s.scheduler.Submit(repo.UpstreamURL(), "clone", func(ctx context.Context) error { + s.startClone(ctx, repo) return nil }) s.forwardToUpstream(w, r, host, pathValue) @@ -241,211 +229,79 @@ func (s *Strategy) handleBundleRequest(w http.ResponseWriter, r *http.Request, h } } -func (s *Strategy) getOrCreateClone(ctx context.Context, upstreamURL string) *clone { - s.clonesMu.RLock() - c, exists := s.clones[upstreamURL] - s.clonesMu.RUnlock() - - if exists { - return c - } - - s.clonesMu.Lock() - defer s.clonesMu.Unlock() - - if c, exists = s.clones[upstreamURL]; exists { - return c - } - - clonePath := s.clonePathForURL(upstreamURL) - - c = &clone{ - state: stateEmpty, - path: clonePath, - upstreamURL: upstreamURL, - fetchSem: make(chan struct{}, 1), - } - - gitDir := filepath.Join(clonePath, ".git") - if _, err := os.Stat(gitDir); err == nil { - c.state = stateReady - logging.FromContext(ctx).DebugContext(ctx, "Found existing clone on disk", - slog.String("path", clonePath)) - - if s.config.BundleInterval > 0 { - s.scheduleBundleJobs(c) - } - } - - c.fetchSem <- struct{}{} - - s.clones[upstreamURL] = c - return c -} - -func (s *Strategy) clonePathForURL(upstreamURL string) string { - parsed, err := url.Parse(upstreamURL) - if err != nil { - return filepath.Join(s.config.MirrorRoot, "unknown") - } - - repoPath := strings.TrimSuffix(parsed.Path, ".git") - return filepath.Join(s.config.MirrorRoot, parsed.Host, repoPath) -} - -func (s *Strategy) discoverExistingClones(ctx context.Context) error { - logger := logging.FromContext(ctx) - - err := filepath.Walk(s.config.MirrorRoot, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() { - return nil - } - - gitDir := filepath.Join(path, ".git") - headPath := filepath.Join(path, ".git", "HEAD") - if _, statErr := os.Stat(gitDir); statErr != nil { - if errors.Is(statErr, os.ErrNotExist) { - return nil - } - return errors.Wrap(statErr, "stat .git directory") - } - if _, statErr := os.Stat(headPath); statErr != nil { - if errors.Is(statErr, os.ErrNotExist) { - return nil - } - return errors.Wrap(statErr, "stat HEAD file") - } - - relPath, err := filepath.Rel(s.config.MirrorRoot, path) - if err != nil { - logger.WarnContext(ctx, "Failed to get relative path", - slog.String("path", path), - slog.String("error", err.Error())) - return nil - } - - parts := strings.Split(filepath.ToSlash(relPath), "/") - if len(parts) < 2 { - return nil - } - - host := parts[0] - repoPath := strings.Join(parts[1:], "/") - upstreamURL := "https://" + host + "/" + repoPath - - c := &clone{ - state: stateReady, - path: path, - upstreamURL: upstreamURL, - fetchSem: make(chan struct{}, 1), - } - c.fetchSem <- struct{}{} - - s.clonesMu.Lock() - s.clones[upstreamURL] = c - s.clonesMu.Unlock() - - logger.DebugContext(ctx, "Discovered existing clone", - slog.String("path", path), - slog.String("upstream", upstreamURL)) - - if s.config.BundleInterval > 0 { - s.scheduleBundleJobs(c) - } - - return nil - }) - - if err != nil { - return errors.Wrap(err, "walk mirror root") - } - - return nil -} - -func (s *Strategy) startClone(ctx context.Context, c *clone) { +func (s *Strategy) startClone(ctx context.Context, repo *gitclone.Repository) { logger := logging.FromContext(ctx) - c.mu.Lock() - if c.state != stateEmpty { - c.mu.Unlock() - return - } - c.state = stateCloning - c.mu.Unlock() - logger.InfoContext(ctx, "Starting clone", - slog.String("upstream", c.upstreamURL), - slog.String("path", c.path)) - - err := s.executeClone(ctx, c) + slog.String("upstream", repo.UpstreamURL()), + slog.String("path", repo.Path())) + + gitcloneConfig := gitclone.Config{ + RootDir: s.config.MirrorRoot, + FetchInterval: s.config.FetchInterval, + RefCheckInterval: s.config.RefCheckInterval, + CloneDepth: s.config.CloneDepth, + GitConfig: gitclone.DefaultGitTuningConfig(), + } - c.mu.Lock() - defer c.mu.Unlock() + err := repo.Clone(ctx, gitcloneConfig) if err != nil { logger.ErrorContext(ctx, "Clone failed", - slog.String("upstream", c.upstreamURL), + slog.String("upstream", repo.UpstreamURL()), slog.String("error", err.Error())) - c.state = stateEmpty return } - c.state = stateReady - c.lastFetch = time.Now() logger.InfoContext(ctx, "Clone completed", - slog.String("upstream", c.upstreamURL), - slog.String("path", c.path)) + slog.String("upstream", repo.UpstreamURL()), + slog.String("path", repo.Path())) if s.config.BundleInterval > 0 { - s.scheduleBundleJobs(c) + s.scheduleBundleJobs(repo) } } -func (s *Strategy) maybeBackgroundFetch(c *clone) { - c.mu.RLock() - lastFetch := c.lastFetch - c.mu.RUnlock() - - if time.Since(lastFetch) < s.config.FetchInterval { +func (s *Strategy) maybeBackgroundFetch(repo *gitclone.Repository) { + if !repo.NeedsFetch(s.config.FetchInterval) { return } - s.scheduler.Submit(c.upstreamURL, "fetch", func(ctx context.Context) error { - s.backgroundFetch(ctx, c) + s.scheduler.Submit(repo.UpstreamURL(), "fetch", func(ctx context.Context) error { + s.backgroundFetch(ctx, repo) return nil }) } -func (s *Strategy) backgroundFetch(ctx context.Context, c *clone) { +func (s *Strategy) backgroundFetch(ctx context.Context, repo *gitclone.Repository) { logger := logging.FromContext(ctx) - c.mu.Lock() - if time.Since(c.lastFetch) < s.config.FetchInterval { - c.mu.Unlock() + if !repo.NeedsFetch(s.config.FetchInterval) { return } - c.lastFetch = time.Now() - c.mu.Unlock() logger.DebugContext(ctx, "Fetching updates", - slog.String("upstream", c.upstreamURL), - slog.String("path", c.path)) + slog.String("upstream", repo.UpstreamURL()), + slog.String("path", repo.Path())) + + gitcloneConfig := gitclone.Config{ + RootDir: s.config.MirrorRoot, + FetchInterval: s.config.FetchInterval, + RefCheckInterval: s.config.RefCheckInterval, + CloneDepth: s.config.CloneDepth, + GitConfig: gitclone.DefaultGitTuningConfig(), + } - if err := s.executeFetch(ctx, c); err != nil { + if err := repo.Fetch(ctx, gitcloneConfig); err != nil { logger.ErrorContext(ctx, "Fetch failed", - slog.String("upstream", c.upstreamURL), + slog.String("upstream", repo.UpstreamURL()), slog.String("error", err.Error())) } } -func (s *Strategy) scheduleBundleJobs(c *clone) { - s.scheduler.SubmitPeriodicJob(c.upstreamURL, "bundle-periodic", s.config.BundleInterval, func(ctx context.Context) error { - s.generateAndUploadBundle(ctx, c) +func (s *Strategy) scheduleBundleJobs(repo *gitclone.Repository) { + s.scheduler.SubmitPeriodicJob(repo.UpstreamURL(), "bundle-periodic", s.config.BundleInterval, func(ctx context.Context) error { + s.generateAndUploadBundle(ctx, repo) return nil }) } diff --git a/internal/strategy/git/git_test.go b/internal/strategy/git/git_test.go index 22fda72..a56f118 100644 --- a/internal/strategy/git/git_test.go +++ b/internal/strategy/git/git_test.go @@ -10,6 +10,7 @@ import ( "github.com/alecthomas/assert/v2" + "github.com/block/cachew/internal/gitclone" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy/git" @@ -227,7 +228,7 @@ abc123def456 refs/heads/main`, for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := git.ParseGitRefs([]byte(tt.output)) + result := gitclone.ParseGitRefs([]byte(tt.output)) assert.Equal(t, tt.expected, result) }) }