feat: gomod private repo support #56
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| package gomod | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| package gomod | ||
|
|
||
| import ( | ||
| "context" | ||
| "io" | ||
| "time" | ||
|
|
||
| "github.com/alecthomas/errors" | ||
| "github.com/goproxy/goproxy" | ||
| ) | ||
|
|
||
| type compositeFetcher struct { | ||
| publicFetcher goproxy.Fetcher | ||
| privateFetcher goproxy.Fetcher | ||
| matcher *ModulePathMatcher | ||
| } | ||
|
|
||
| func newCompositeFetcher( | ||
| publicFetcher goproxy.Fetcher, | ||
| privateFetcher goproxy.Fetcher, | ||
| patterns []string, | ||
| ) *compositeFetcher { | ||
| return &compositeFetcher{ | ||
| publicFetcher: publicFetcher, | ||
| privateFetcher: privateFetcher, | ||
| matcher: NewModulePathMatcher(patterns), | ||
| } | ||
| } | ||
|
|
||
| func (c *compositeFetcher) Query(ctx context.Context, path, query string) (version string, t time.Time, err error) { | ||
| if c.matcher.IsPrivate(path) { | ||
| v, tm, err := c.privateFetcher.Query(ctx, path, query) | ||
| return v, tm, errors.Wrap(err, "private fetcher query") | ||
| } | ||
| v, tm, err := c.publicFetcher.Query(ctx, path, query) | ||
| return v, tm, errors.Wrap(err, "public fetcher query") | ||
| } | ||
|
|
||
| func (c *compositeFetcher) List(ctx context.Context, path string) (versions []string, err error) { | ||
| if c.matcher.IsPrivate(path) { | ||
| v, err := c.privateFetcher.List(ctx, path) | ||
| return v, errors.Wrap(err, "private fetcher list") | ||
| } | ||
| v, err := c.publicFetcher.List(ctx, path) | ||
| return v, errors.Wrap(err, "public fetcher list") | ||
| } | ||
|
|
||
| func (c *compositeFetcher) Download(ctx context.Context, path, version string) (info, mod, zip io.ReadSeekCloser, err error) { | ||
| if c.matcher.IsPrivate(path) { | ||
| i, m, z, err := c.privateFetcher.Download(ctx, path, version) | ||
| return i, m, z, errors.Wrap(err, "private fetcher download") | ||
| } | ||
| i, m, z, err := c.publicFetcher.Download(ctx, path, version) | ||
| return i, m, z, errors.Wrap(err, "public fetcher download") | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| package gomod | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "log/slog" | ||
| "net/http" | ||
| "net/url" | ||
| "os" | ||
| "path/filepath" | ||
| "time" | ||
|
|
||
| "github.com/alecthomas/errors" | ||
| "github.com/goproxy/goproxy" | ||
|
|
||
| "github.com/block/cachew/internal/cache" | ||
| "github.com/block/cachew/internal/gitclone" | ||
| "github.com/block/cachew/internal/jobscheduler" | ||
| "github.com/block/cachew/internal/logging" | ||
| "github.com/block/cachew/internal/strategy" | ||
| ) | ||
|
|
||
| func init() { | ||
| strategy.Register("gomod", "Caches Go module proxy requests.", New) | ||
| } | ||
|
|
||
| type Config struct { | ||
| Proxy string `hcl:"proxy,optional" help:"Upstream Go module proxy URL (defaults to proxy.golang.org)" default:"https://proxy.golang.org"` | ||
| PrivatePaths []string `hcl:"private-paths,optional" help:"Module path patterns for private repositories"` | ||
| MirrorRoot string `hcl:"mirror-root,optional" help:"Directory to store git clones for private repos." default:""` | ||
| FetchInterval time.Duration `hcl:"fetch-interval,optional" help:"How often to fetch from upstream for private repos." default:"15m"` | ||
| RefCheckInterval time.Duration `hcl:"ref-check-interval,optional" help:"How long to cache ref checks for private repos." default:"10s"` | ||
| CloneDepth int `hcl:"clone-depth,optional" help:"Depth for shallow clones of private repos. 0 means full clone." default:"0"` | ||
| } | ||
|
|
||
| type Strategy struct { | ||
| config Config | ||
| cache cache.Cache | ||
| logger *slog.Logger | ||
| proxy *url.URL | ||
| goproxy *goproxy.Goproxy | ||
| cloneManager *gitclone.Manager // Manager for cloning private repositories | ||
| } | ||
|
|
||
| var _ strategy.Strategy = (*Strategy)(nil) | ||
|
|
||
| func New(ctx context.Context, config Config, _ jobscheduler.Scheduler, cache cache.Cache, mux strategy.Mux) (*Strategy, error) { | ||
| parsedURL, err := url.Parse(config.Proxy) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("invalid proxy URL: %w", err) | ||
| } | ||
|
|
||
| s := &Strategy{ | ||
| config: config, | ||
| cache: cache, | ||
| logger: logging.FromContext(ctx), | ||
| proxy: parsedURL, | ||
| } | ||
|
|
||
| publicFetcher := &goproxy.GoFetcher{ | ||
| Env: []string{ | ||
| "GOPROXY=" + config.Proxy, | ||
| "GOSUMDB=off", // Disable checksum database validation in fetcher, to prevent unneccessary double validation | ||
| }, | ||
| } | ||
|
|
||
| var fetcher goproxy.Fetcher = publicFetcher | ||
|
|
||
| if len(config.PrivatePaths) > 0 { | ||
| // Set default mirror root if not specified | ||
| mirrorRoot := config.MirrorRoot | ||
| if mirrorRoot == "" { | ||
| homeDir, err := os.UserHomeDir() | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "get user home directory") | ||
| } | ||
| mirrorRoot = filepath.Join(homeDir, ".cache", "cachew", "gomod-git-mirrors") | ||
| } | ||
|
|
||
| // Create gitclone manager for private repositories | ||
| cloneManager, err := gitclone.NewManager(ctx, gitclone.Config{ | ||
|
Collaborator
There was a problem hiding this comment. We should construct the gitclone manager in main and inject it into the gomod strategy. This'll require some refactoring of the factory functions, so I'll do that post merge. |
||
| RootDir: mirrorRoot, | ||
| FetchInterval: config.FetchInterval, | ||
| RefCheckInterval: config.RefCheckInterval, | ||
| CloneDepth: config.CloneDepth, | ||
| GitConfig: gitclone.DefaultGitTuningConfig(), | ||
| }) | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "create clone manager for private repos") | ||
| } | ||
| s.cloneManager = cloneManager | ||
|
|
||
| // Discover existing clones | ||
| if err := cloneManager.DiscoverExisting(ctx); err != nil { | ||
| s.logger.WarnContext(ctx, "Failed to discover existing clones for private repos", | ||
| slog.String("error", err.Error())) | ||
| } | ||
|
|
||
| privateFetcher := newPrivateFetcher(s, cloneManager) | ||
| fetcher = newCompositeFetcher(publicFetcher, privateFetcher, config.PrivatePaths) | ||
|
|
||
| s.logger.InfoContext(ctx, "Configured private module support", | ||
| slog.Any("private_paths", config.PrivatePaths), | ||
| slog.String("mirror_root", mirrorRoot)) | ||
| } | ||
|
|
||
| s.goproxy = &goproxy.Goproxy{ | ||
| Logger: s.logger, | ||
| Fetcher: fetcher, | ||
| Cacher: &goproxyCacher{ | ||
| cache: cache, | ||
| }, | ||
| ProxiedSumDBs: []string{ | ||
| "sum.golang.org https://sum.golang.org", | ||
| }, | ||
| } | ||
|
|
||
| s.logger.InfoContext(ctx, "Initialized Go module proxy strategy", | ||
| slog.String("proxy", s.proxy.String())) | ||
|
|
||
| mux.Handle("GET /gomod/{path...}", http.StripPrefix("/gomod", s.goproxy)) | ||
|
|
||
| return s, nil | ||
| } | ||
|
|
||
| func (s *Strategy) String() string { | ||
| return "gomod:" + s.proxy.Host | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| package gomod | ||
|
|
||
| import ( | ||
| "path" | ||
| "strings" | ||
| ) | ||
|
|
||
| // ModulePathMatcher matches module paths against patterns. | ||
| type ModulePathMatcher struct { | ||
| patterns []string | ||
| } | ||
|
|
||
| // NewModulePathMatcher creates a new matcher with the given patterns. | ||
| func NewModulePathMatcher(patterns []string) *ModulePathMatcher { | ||
| return &ModulePathMatcher{patterns: patterns} | ||
| } | ||
|
|
||
| // IsPrivate checks if a module path matches any private pattern. | ||
| func (m *ModulePathMatcher) IsPrivate(modulePath string) bool { | ||
| for _, pattern := range m.patterns { | ||
| matched, err := path.Match(pattern, modulePath) | ||
| if err == nil && matched { | ||
| return true | ||
| } | ||
|
|
||
| if strings.HasPrefix(modulePath, pattern+"/") || modulePath == pattern { | ||
| return true | ||
| } | ||
| } | ||
|
|
||
| return false | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We should require a config path so we can put all state into a single directory, like we do with the other git mirrors.
Also remind me again why we can't reuse our existing git mirrors?