feat: private repo support

cachew.hcl

@@ -26,3 +26,7 @@ disk {
   limit-mb = 250000
   max-ttl = "8h"
 }
+
+gomod {
+  proxy = "https://proxy.golang.org"
+}

go.mod

@@ -5,12 +5,14 @@ go 1.25.5
 require (
 	github.com/alecthomas/hcl/v2 v2.3.1
 	github.com/alecthomas/kong v1.13.0
+	github.com/goproxy/goproxy v0.25.0
 	github.com/lmittmann/tint v1.1.2
 	github.com/minio/minio-go/v7 v7.0.97
 	go.etcd.io/bbolt v1.4.3
 )
 
 require (
+	github.com/aofei/backoff v1.1.0 // indirect
 	github.com/dlclark/regexp2 v1.11.5 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
@@ -28,6 +30,7 @@ require (
 	github.com/stretchr/testify v1.11.1 // indirect
 	github.com/tinylib/msgp v1.3.0 // indirect
 	golang.org/x/crypto v0.44.0 // indirect
+	golang.org/x/mod v0.31.0 // indirect
 	golang.org/x/net v0.47.0 // indirect
 	golang.org/x/sys v0.38.0 // indirect
 	golang.org/x/text v0.32.0 // indirect

go.sum

@@ -12,6 +12,8 @@ github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WS
 github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI=
 github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
 github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/aofei/backoff v1.1.0 h1:7ey7Ydpx/eFIyyrBNKPbgvTzvIuUOHcwkR3gPjjY9ag=
+github.com/aofei/backoff v1.1.0/go.mod h1:IHCkMdd5vGP6dcDHD+uLn6lVuBw7+rKYaS7e7QIQwYA=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -23,6 +25,8 @@ github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
 github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/goproxy/goproxy v0.25.0 h1:TujZjUbKCwpFYrm+j04HACs1EAcBbFSGLwLMn8ynTys=
+github.com/goproxy/goproxy v0.25.0/go.mod h1:6RIssMPDpQ0IHZus17gPUyBtU62RoqblQDYWx2sz/qs=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
@@ -65,6 +69,8 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
 golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU=
 golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc=
+golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
+golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
 golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
 golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
 golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
@@ -73,6 +79,8 @@ golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
 golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
 golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
+golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

internal/strategy/api.go

@@ -4,6 +4,7 @@ package strategy
 import (
 	"context"
 	"net/http"
+	"sync"
 
 	"github.com/alecthomas/errors"
 	"github.com/alecthomas/hcl/v2"
@@ -79,3 +80,26 @@ func Create(
 type Strategy interface {
 	String() string
 }
+
+type GitStrategy interface {
+	Strategy
+	EnsureClone(ctx context.Context, gitURL string) (string, error)
+	GetClonePath(gitURL string) string
+}
+
+var (
+	strategyRegistry   = make(map[string]Strategy)
+	strategyRegistryMu sync.RWMutex
+)
+
+func RegisterStrategy(name string, strategy Strategy) {
+	strategyRegistryMu.Lock()
+	defer strategyRegistryMu.Unlock()
+	strategyRegistry[name] = strategy
+}
+
+func GetStrategy(name string) Strategy {
+	strategyRegistryMu.RLock()
+	defer strategyRegistryMu.RUnlock()
+	return strategyRegistry[name]
+}

internal/strategy/git/git.go

@@ -120,13 +120,79 @@ func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, c
 		"ref_check_interval", config.RefCheckInterval,
 		"bundle_interval", config.BundleInterval)
 
+	strategy.RegisterStrategy("git", s)
+
 	return s, nil
 }
 
 var _ strategy.Strategy = (*Strategy)(nil)
 
 func (s *Strategy) String() string { return "git" }
 
+func (s *Strategy) EnsureClone(ctx context.Context, gitURL string) (string, error) {
+	c := s.getOrCreateClone(ctx, gitURL)
+
+	c.mu.RLock()
+	state := c.state
+	c.mu.RUnlock()
+
+	switch state {
+	case stateEmpty:
+		s.startClone(ctx, c)
+		c.mu.RLock()
+		finalState := c.state
+		c.mu.RUnlock()
+
+		if finalState != stateReady {
+			return "", errors.New("clone failed")
+		}
+
+	case stateCloning:
+		for {
+			time.Sleep(100 * time.Millisecond)
+			c.mu.RLock()
+			currentState := c.state
+			c.mu.RUnlock()
+
+			if currentState == stateReady {
+				break
+			}
+			if currentState == stateEmpty {
+				return "", errors.New("clone failed")
+			}
+
+			select {
+			case <-ctx.Done():
+				return "", errors.Wrap(ctx.Err(), "context cancelled waiting for clone state")
+			default:
+			}
+		}
+
+	case stateReady:
+		c.mu.RLock()
+		lastFetch := c.lastFetch
+		c.mu.RUnlock()
+
+		if time.Since(lastFetch) > s.config.FetchInterval {
+			if err := s.executeFetch(ctx, c); err != nil {
+				logging.FromContext(ctx).WarnContext(ctx, "Failed to fetch updates",
+					slog.String("upstream", gitURL),
+					slog.String("error", err.Error()))
+			} else {
+				c.mu.Lock()
+				c.lastFetch = time.Now()
+				c.mu.Unlock()
+			}
+		}
+	}
+
+	return c.path, nil
+}
+
+func (s *Strategy) GetClonePath(gitURL string) string {
+	return s.clonePathForURL(gitURL)
+}
+
 func (s *Strategy) handleRequest(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	logger := logging.FromContext(ctx)

internal/strategy/gomod.go

@@ -6,55 +6,35 @@ import (
 	"log/slog"
 	"net/http"
 	"net/url"
-	"strings"
-	"time"
+
+	"github.com/alecthomas/errors"
+	"github.com/goproxy/goproxy"
 
 	"github.com/block/cachew/internal/cache"
-	"github.com/block/cachew/internal/httputil"
 	"github.com/block/cachew/internal/jobscheduler"
 	"github.com/block/cachew/internal/logging"
-	"github.com/block/cachew/internal/strategy/handler"
 )
 
 func init() {
 	Register("gomod", "Caches Go module proxy requests.", NewGoMod)
 }
 
-// GoModConfig represents the configuration for the Go module proxy strategy.
-//
-// In HCL it looks like:
-//
-//	gomod {
-//	  proxy = "https://proxy.golang.org"
-//	}
 type GoModConfig struct {
-	Proxy        string        `hcl:"proxy,optional" help:"Upstream Go module proxy URL (defaults to proxy.golang.org)" default:"https://proxy.golang.org"`
-	MutableTTL   time.Duration `hcl:"mutable-ttl,optional" help:"TTL for mutable Go module proxy endpoints (list, latest). Defaults to 5m." default:"5m"`
-	ImmutableTTL time.Duration `hcl:"immutable-ttl,optional" help:"TTL for immutable Go module proxy endpoints (versioned info, mod, zip). Defaults to 168h (7 days)." default:"168h"`
+	Proxy        string   `hcl:"proxy,optional" help:"Upstream Go module proxy URL (defaults to proxy.golang.org)" default:"https://proxy.golang.org"`
+	PrivatePaths []string `hcl:"private-paths,optional" help:"Module path patterns for private repositories"`
 }
 
-// The GoMod strategy implements a caching proxy for the Go module proxy protocol.
-//
-// It supports all standard GOPROXY endpoints:
-// - /$module/@v/list - Lists available versions
-// - /$module/@v/$version.info - Version metadata JSON
-// - /$module/@v/$version.mod - go.mod file
-// - /$module/@v/$version.zip - Module source code
-// - /$module/@latest - Latest version info
-//
-// The strategy uses differential caching: short TTL (5 minutes) for mutable
-// endpoints (list, latest) and long TTL (7 days) for immutable versioned content.
 type GoMod struct {
-	config GoModConfig
-	cache  cache.Cache
-	client *http.Client
-	logger *slog.Logger
-	proxy  *url.URL
+	config      GoModConfig
+	cache       cache.Cache
+	logger      *slog.Logger
+	proxy       *url.URL
+	goproxy     *goproxy.Goproxy
+	gitStrategy GitStrategy // Reference to git strategy for private repo access
 }
 
 var _ Strategy = (*GoMod)(nil)
 
-// NewGoMod creates a new Go module proxy strategy.
 func NewGoMod(ctx context.Context, config GoModConfig, _ jobscheduler.Scheduler, cache cache.Cache, mux Mux) (*GoMod, error) {
 	parsedURL, err := url.Parse(config.Proxy)
 	if err != nil {
@@ -64,105 +44,51 @@ func NewGoMod(ctx context.Context, config GoModConfig, _ jobscheduler.Scheduler,
 	g := &GoMod{
 		config: config,
 		cache:  cache,
-		client: http.DefaultClient,
 		logger: logging.FromContext(ctx),
 		proxy:  parsedURL,
 	}
 
-	g.logger.InfoContext(ctx, "Initialized Go module proxy strategy",
-		slog.String("proxy", g.proxy.String()))
-
-	// Create handler with caching configuration
-	h := handler.New(g.client, g.cache).
-		CacheKey(func(r *http.Request) string {
-			return g.buildUpstreamURL(r).String()
-		}).
-		Transform(g.transformRequest).
-		TTL(g.calculateTTL)
-
-	// Register a namespaced handler for Go module proxy patterns
-	mux.Handle("GET /gomod/{path...}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		path := r.URL.Path
-		// Check if this is a valid Go module proxy endpoint
-		if g.isGoModulePath(path) {
-			h.ServeHTTP(w, r)
-			return
-		}
-		http.NotFound(w, r)
-	}))
-
-	return g, nil
-}
-
-// isGoModulePath checks if the path matches a valid Go module proxy endpoint pattern.
-func (g *GoMod) isGoModulePath(path string) bool {
-	// Strip the /gomod prefix before checking the pattern
-	path = strings.TrimPrefix(path, "/gomod")
-
-	// Valid patterns:
-	// - /@v/list
-	// - /@v/{version}.info
-	// - /@v/{version}.mod
-	// - /@v/{version}.zip
-	// - /@latest
-	return strings.HasSuffix(path, "/@v/list") ||
-		strings.HasSuffix(path, "/@latest") ||
-		(strings.Contains(path, "/@v/") &&
-			(strings.HasSuffix(path, ".info") ||
-				strings.HasSuffix(path, ".mod") ||
-				strings.HasSuffix(path, ".zip")))
-}
-
-func (g *GoMod) String() string {
-	return "gomod:" + g.proxy.Host
-}
-
-// buildUpstreamURL constructs the full upstream URL from the incoming request.
-func (g *GoMod) buildUpstreamURL(r *http.Request) *url.URL {
-	// The full path includes the module path and the endpoint
-	// e.g., /gomod/github.com/user/repo/@v/v1.0.0.info
-	// We need to strip the /gomod prefix before forwarding to the upstream proxy
-	path := r.URL.Path
-	path = strings.TrimPrefix(path, "/gomod")
-	if !strings.HasPrefix(path, "/") {
-		path = "/" + path
+	publicFetcher := &goproxy.GoFetcher{
+		Env: []string{
+			"GOPROXY=" + config.Proxy,
+			"GOSUMDB=off", // Disable checksum database validation in fetcher, to prevent unneccessary double validation
+		},
 	}
 
-	targetURL := *g.proxy
-	targetURL.Path = g.proxy.Path + path
-	targetURL.RawQuery = r.URL.RawQuery
-
-	return &targetURL
-}
+	var fetcher goproxy.Fetcher = publicFetcher
 
-// transformRequest creates the upstream request to the Go module proxy.
-func (g *GoMod) transformRequest(r *http.Request) (*http.Request, error) {
-	targetURL := g.buildUpstreamURL(r)
+	if len(config.PrivatePaths) > 0 {
+		gitStrategy, ok := GetStrategy("git").(GitStrategy)
+		if !ok {
+			return nil, errors.New("git strategy not available or does not implement GitStrategy interface")
+		}
+		g.gitStrategy = gitStrategy
+		privateFetcher := newPrivateFetcher(g, gitStrategy)
+		fetcher = newCompositeFetcher(publicFetcher, privateFetcher, config.PrivatePaths)
 
-	g.logger.DebugContext(r.Context(), "Transforming Go module request",
-		slog.String("original_path", r.URL.Path),
-		slog.String("upstream_url", targetURL.String()))
+		g.logger.InfoContext(ctx, "Configured private module support",
+			slog.Any("private_paths", config.PrivatePaths))
+	}
 
-	req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, targetURL.String(), nil)
-	if err != nil {
-		return nil, httputil.Errorf(http.StatusInternalServerError, "create upstream request: %w", err)
+	g.goproxy = &goproxy.Goproxy{
+		Logger:  g.logger,
+		Fetcher: fetcher,
+		Cacher: &goproxyCacher{
+			cache: cache,
+		},
+		ProxiedSumDBs: []string{
+			"sum.golang.org https://sum.golang.org",
+		},
 	}
 
-	return req, nil
-}
+	g.logger.InfoContext(ctx, "Initialized Go module proxy strategy",
+		slog.String("proxy", g.proxy.String()))
 
-// calculateTTL returns the appropriate cache TTL based on the endpoint type.
-//
-// Mutable endpoints (list, latest) get short TTL (5 minutes).
-// Immutable versioned content (info, mod, zip) gets long TTL (7 days).
-func (g *GoMod) calculateTTL(r *http.Request) time.Duration {
-	path := r.URL.Path
+	mux.Handle("GET /gomod/{path...}", http.StripPrefix("/gomod", g.goproxy))
 
-	// Short TTL for mutable endpoints
-	if strings.HasSuffix(path, "/@v/list") || strings.HasSuffix(path, "/@latest") {
-		return g.config.MutableTTL
-	}
+	return g, nil
+}
 
-	// Long TTL for immutable versioned content (.info, .mod, .zip)
-	return g.config.ImmutableTTL
+func (g *GoMod) String() string {
+	return "gomod:" + g.proxy.Host
 }