Upstream goproxy

cmd/proxy/actions/app_proxy.go

@@ -57,6 +57,11 @@ func addProxyRoutes(
 
 	lister := download.NewVCSLister(c.GoBinary, fs)
 
+	if c.Proxy != "" {
+		mf = module.WithProxy(c.Proxy, mf)
+		lister = download.WithProxy(c.Proxy, lister)
+	}
+
 	withSingleFlight, err := getSingleFlight(c, s)
 	if err != nil {
 		return err

go.mod

@@ -24,11 +24,11 @@ require (
 	github.com/jtolds/gls v4.2.1+incompatible // indirect
 	github.com/kelseyhightower/envconfig v1.3.0
 	github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
 	github.com/minio/minio-go v6.0.5+incompatible
 	github.com/mitchellh/go-homedir v1.0.0
 	github.com/philhofer/fwd v1.0.0 // indirect
 	github.com/prometheus/client_golang v0.9.1 // indirect
+	github.com/rogpeppe/go-internal v1.2.2
 	github.com/sirupsen/logrus v1.1.1
 	github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf // indirect
 	github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a // indirect
@@ -46,7 +46,6 @@ require (
 	google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf
 	google.golang.org/appengine v1.3.0 // indirect
 	gopkg.in/DataDog/dd-trace-go.v1 v1.3.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 	gopkg.in/go-playground/validator.v9 v9.20.2
 )

go.sum

@@ -138,6 +138,8 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q
 github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 h1:agujYaXJSxSo18YNX3jzl+4G6Bstwt+kqv47GS12uL0=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/rogpeppe/go-internal v1.2.2 h1:J7U/N7eRtzjhs26d6GqMh2HBuXP8/Z64Densiiieafo=
+github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.1.1 h1:VzGj7lhU7KEB9e9gMpAV/v5XT2NVSvLJhJLCWbnkgXg=
 github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A=
@@ -209,6 +211,7 @@ gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
 gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=

pkg/config/config.go

@@ -10,7 +10,7 @@ import (
 	"github.com/BurntSushi/toml"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/kelseyhightower/envconfig"
-	validator "gopkg.in/go-playground/validator.v9"
+	"gopkg.in/go-playground/validator.v9"
 )
 
 // Config provides configuration values for all components
@@ -40,6 +40,7 @@ type Config struct {
 	TLSCertFile      string `envconfig:"ATHENS_TLSCERT_FILE"`
 	TLSKeyFile       string `envconfig:"ATHENS_TLSKEY_FILE"`
 	SingleFlightType string `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"`
+	Proxy            string `envconfig:"ATHENS_PROXY"`
 	SingleFlight     *SingleFlight
 	Storage          *StorageConfig
 }

pkg/download/protocol.go

@@ -77,7 +77,7 @@ func (p *protocol) List(ctx context.Context, mod string) ([]string, error) {
 
 	go func() {
 		defer wg.Done()
-		_, goList, goErr = p.lister.List(ctx, mod)
+		goList, goErr = p.lister.List(ctx, mod)
 	}()
 
 	wg.Wait()
@@ -108,7 +108,7 @@ func (p *protocol) Latest(ctx context.Context, mod string) (*storage.RevInfo, er
 	const op errors.Op = "protocol.Latest"
 	ctx, span := observ.StartSpan(ctx, op.String())
 	defer span.End()
-	lr, _, err := p.lister.List(ctx, mod)
+	lr, err := p.lister.Latest(ctx, mod)
 	if err != nil {
 		return nil, errors.E(op, err)
 	}

pkg/download/proxy.go

@@ -0,0 +1,53 @@
+package download
+
+import (
+	"context"
+
+	"github.com/gomods/athens/pkg/goproxy"
+	"github.com/gomods/athens/pkg/storage"
+)
+
+type proxyLister struct {
+	baseURL string
+
+	delegate UpstreamLister
+}
+
+func (p *proxyLister) proxyList(ctx context.Context, mod string) ([]string, error) {
+	repo, err := goproxy.NewProxyRepo(p.baseURL, mod)
+	if err != nil {
+		return nil, err
+	}
+	return repo.Versions("")
+}
+
+func (p *proxyLister) proxyLatest(ctx context.Context, mod string) (*storage.RevInfo, error) {
+	repo, err := goproxy.NewProxyRepo(p.baseURL, mod)
+	if err != nil {
+		return nil, err
+	}
+	return repo.Latest()
+}
+
+func (p *proxyLister) List(ctx context.Context, mod string) ([]string, error) {
+	list, err := p.proxyList(ctx, mod)
+	if err != nil {
+		return p.delegate.List(ctx, mod)
+	}
+	return list, nil
+}
+
+func (p *proxyLister) Latest(ctx context.Context, mod string) (*storage.RevInfo, error) {
+	latest, err := p.proxyLatest(ctx, mod)
+	if err != nil {
+		return p.delegate.Latest(ctx, mod)
+	}
+	return latest, nil
+}
+
+func WithProxy(baseURL string, delegate UpstreamLister) UpstreamLister {
+	return &proxyLister{
+		baseURL:  baseURL,
+		delegate: delegate,
+	}
+}

pkg/download/upstream_lister.go

@@ -19,7 +19,9 @@ import (
 // UpstreamLister retrieves a list of available module versions from upstream
 // i.e. VCS, and a Storage backend.
 type UpstreamLister interface {
-	List(ctx context.Context, mod string) (*storage.RevInfo, []string, error)
+	List(ctx context.Context, mod string) ([]string, error)
+
+	Latest(ctx context.Context, mod string) (*storage.RevInfo, error)
 }
 
 type listResp struct {
@@ -34,7 +36,17 @@ type vcsLister struct {
 	fs        afero.Fs
 }
 
-func (l *vcsLister) List(ctx context.Context, mod string) (*storage.RevInfo, []string, error) {
+func (l *vcsLister) List(ctx context.Context, mod string) ([]string, error) {
+	_, versions, err := l.list(ctx, mod)
+	return versions, err
+}
+
+func (l *vcsLister) Latest(ctx context.Context, mod string) (*storage.RevInfo, error) {
+	revInfo, _, err := l.list(ctx, mod)
+	return revInfo, err
+}
+
+func (l *vcsLister) list(ctx context.Context, mod string) (*storage.RevInfo, []string, error) {
 	const op errors.Op = "vcsLister.List"
 	ctx, span := observ.StartSpan(ctx, op.String())
 	defer span.End()

pkg/goproxy/proxy.go

@@ -0,0 +1,139 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// see github.com/golang/go/src/cmd/go/internal/modfetch/proxy.go
+
+package goproxy
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/gomods/athens/pkg/storage"
+	"github.com/rogpeppe/go-internal/module"
+	"github.com/rogpeppe/go-internal/semver"
+)
+
+type ProxyRepo struct {
+	url string
+}
+
+func NewProxyRepo(baseURL, path string) (*ProxyRepo, error) {
+	enc, err := module.EncodePath(path)
+	if err != nil {
+		return nil, err
+	}
+	return &ProxyRepo{strings.TrimSuffix(baseURL, "/") + "/" + pathEscape(enc)}, nil
+}
+
+func (p *ProxyRepo) Versions(prefix string) ([]string, error) {
+	var data []byte
+	err := webGetBytes(p.url+"/@v/list", &data)
+	if err != nil {
+		return nil, err
+	}
+	var list []string
+	for _, line := range strings.Split(string(data), "\n") {
+		f := strings.Fields(line)
+		if len(f) >= 1 && semver.IsValid(f[0]) && strings.HasPrefix(f[0], prefix) {
+			list = append(list, f[0])
+		}
+	}
+	return list, nil
+}
+
+func (p *ProxyRepo) latest() (*storage.RevInfo, error) {
+	var data []byte
+	err := webGetBytes(p.url+"/@v/list", &data)
+	if err != nil {
+		return nil, err
+	}
+	var best time.Time
+	var bestVersion string
+	for _, line := range strings.Split(string(data), "\n") {
+		f := strings.Fields(line)
+		if len(f) >= 2 && semver.IsValid(f[0]) {
+			ft, err := time.Parse(time.RFC3339, f[1])
+			if err == nil && best.Before(ft) {
+				best = ft
+				bestVersion = f[0]
+			}
+		}
+	}
+	if bestVersion == "" {
+		return nil, fmt.Errorf("no commits")
+	}
+	info := &storage.RevInfo{
+		Version: bestVersion,
+		Time:    best,
+	}
+	return info, nil
+}
+
+func (p *ProxyRepo) Stat(rev string) ([]byte, error) {
+	var data []byte
+	encRev, err := module.EncodeVersion(rev)
+	if err != nil {
+		return nil, err
+	}
+	err = webGetBytes(p.url+"/@v/"+pathEscape(encRev)+".info", &data)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func (p *ProxyRepo) Latest() (*storage.RevInfo, error) {
+	var data []byte
+	u := p.url + "/@latest"
+	err := webGetBytes(u, &data)
+	if err != nil {
+		// TODO return err if not 404
+		return p.latest()
+	}
+	info := new(storage.RevInfo)
+	if err := json.Unmarshal(data, info); err != nil {
+		return nil, err
+	}
+	return info, nil
+}
+
+func (p *ProxyRepo) GoMod(version string) ([]byte, error) {
+	var data []byte
+	encVer, err := module.EncodeVersion(version)
+	if err != nil {
+		return nil, err
+	}
+	err = webGetBytes(p.url+"/@v/"+pathEscape(encVer)+".mod", &data)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func (p *ProxyRepo) Zip(version string) (io.ReadCloser, error) {
+	var body io.ReadCloser
+	encVer, err := module.EncodeVersion(version)
+	if err != nil {
+		return nil, err
+	}
+	err = webGetBody(p.url+"/@v/"+pathEscape(encVer)+".zip", &body)
+	if err != nil {
+		return nil, err
+	}
+	defer body.Close()
+
+	return body, nil
+}
+
+// pathEscape escapes s so it can be used in a path.
+// That is, it escapes things like ? and # (which really shouldn't appear anyway).
+// It does not escape / to %2F: our REST API is designed so that / can be left as is.
+func pathEscape(s string) string {
+	return strings.ReplaceAll(url.PathEscape(s), "%2F", "/")
+}

pkg/goproxy/web.go

@@ -0,0 +1,31 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// see github.com/golang/go/src/cmd/go/internal/modfetch/web.go
+
+package goproxy
+
+import (
+	"io"
+
+	"github.com/gomods/athens/pkg/goproxy/web"
+)
+
+// webGetGoGet fetches a go-get=1 URL and returns the body in *body.
+// It allows non-200 responses, as usual for these URLs.
+func webGetGoGet(url string, body *io.ReadCloser) error {
+	return web.Get(url, web.Non200OK(), web.Body(body))
+}
+
+// webGetBytes returns the body returned by an HTTP GET, as a []byte.
+// It insists on a 200 response.
+func webGetBytes(url string, body *[]byte) error {
+	return web.Get(url, web.ReadAllBody(body))
+}
+
+// webGetBody returns the body returned by an HTTP GET, as a io.ReadCloser.
+// It insists on a 200 response.
+func webGetBody(url string, body *io.ReadCloser) error {
+	return web.Get(url, web.Body(body))
+}