SDK selection refactor

Makefile

@@ -11,14 +11,18 @@ prepare_test:
 test:
 	go test -cover ./...
 
+.PHONY: test_race
+test_race:
+	go test -race -gcflags=all=-d=checkptr=0 ./...
+
 .PHONY: test_circleci
 test_circleci:
 	scripts/wait_for_wallet.sh
 	go get golang.org/x/tools/cmd/cover
 	go get github.com/mattn/goveralls
 	go run . db_migrate_up
 	go test -covermode=count -coverprofile=coverage.out ./...
-	goveralls -coverprofile=coverage.out -service=circle-ci -repotoken $(COVERALLS_TOKEN)
+	goveralls -coverprofile=coverage.out -service=circle-ci -ignore=models/ -repotoken $(COVERALLS_TOKEN)
 
 release:
 	GO111MODULE=on goreleaser --rm-dist

api/benchmarks_test.go

@@ -16,9 +16,8 @@ import (
 
 	"github.com/lbryio/lbrytv/app/proxy"
 	"github.com/lbryio/lbrytv/app/sdkrouter"
-	"github.com/lbryio/lbrytv/app/users"
+	"github.com/lbryio/lbrytv/app/wallet"
 	"github.com/lbryio/lbrytv/config"
-	"github.com/lbryio/lbrytv/internal/lbrynet"
 	"github.com/lbryio/lbrytv/internal/responses"
 	"github.com/lbryio/lbrytv/internal/storage"
 	"github.com/lbryio/lbrytv/models"
@@ -33,14 +32,14 @@ func launchAuthenticatingAPIServer() *httptest.Server {
 	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		t := r.PostFormValue("auth_token")
 
-		responses.PrepareJSONWriter(w)
+		responses.AddJSONContentType(w)
 
 		reply := fmt.Sprintf(`
 		{
 			"success": true,
 			"error": null,
 			"data": {
-				"id": %v,
+				"id": %s,
 				"language": "en",
 				"given_name": null,
 				"family_name": null,
@@ -87,31 +86,29 @@ func setupDBTables() {
 func BenchmarkWalletCommands(b *testing.B) {
 	setupDBTables()
 
+	wallet.DisableLogger()
+	sdkrouter.DisableLogger()
+	log.SetOutput(ioutil.Discard)
+
+	rand.Seed(time.Now().UnixNano())
+
+	rt := sdkrouter.New(config.GetLbrynetServers())
+
 	ts := launchAuthenticatingAPIServer()
 	defer ts.Close()
-	config.Override("InternalAPIHost", ts.URL)
-	defer config.RestoreOverridden()
 
 	walletsNum := 30
 	wallets := make([]*models.User, walletsNum)
-	rt := sdkrouter.New(config.GetLbrynetServers())
-	svc := users.NewWalletService(rt)
-
-	svc.Logger.Disable()
-	lbrynet.Logger.Disable()
-	log.SetOutput(ioutil.Discard)
-
-	rand.Seed(time.Now().UnixNano())
 
 	for i := 0; i < walletsNum; i++ {
 		uid := int(rand.Int31())
-		u, err := svc.Retrieve(users.Query{Token: fmt.Sprintf("%v", uid)})
+		u, err := wallet.GetUserWithWallet(rt, ts.URL, fmt.Sprintf("%d", uid), "")
 		require.NoError(b, err, errors.Unwrap(err))
 		require.NotNil(b, u)
 		wallets[i] = u
 	}
 
-	handler := proxy.NewRequestHandler(proxy.NewService(proxy.Opts{SDKRouter: rt}))
+	handler := sdkrouter.Middleware(rt)(http.HandlerFunc(proxy.Handle))
 
 	b.SetParallelism(30)
 	b.ResetTimer()
@@ -120,17 +117,20 @@ func BenchmarkWalletCommands(b *testing.B) {
 		for pb.Next() {
 			u := wallets[rand.Intn(len(wallets))]
 
-			var response jsonrpc.RPCResponse
 			q := jsonrpc.NewRequest("wallet_balance", map[string]string{"wallet_id": u.WalletID})
 
-			qBody, _ := json.Marshal(q)
-			r, _ := http.NewRequest("POST", proxySuffix, bytes.NewBuffer(qBody))
-			r.Header.Add("X-Lbry-Auth-Token", fmt.Sprintf("%v", u.ID))
+			qBody, err := json.Marshal(q)
+			require.NoError(b, err)
+			r, err := http.NewRequest("POST", proxySuffix, bytes.NewBuffer(qBody))
+			require.NoError(b, err)
+			r.Header.Add("X-Lbry-Auth-Token", fmt.Sprintf("%d", u.ID))
 
 			rr := httptest.NewRecorder()
-			handler.Handle(rr, r)
+			handler.ServeHTTP(rr, r)
 
 			require.Equal(b, http.StatusOK, rr.Code)
+
+			var response jsonrpc.RPCResponse
 			json.Unmarshal(rr.Body.Bytes(), &response)
 			require.Nil(b, response.Error)
 		}

api/routes.go

@@ -1,51 +1,69 @@
 package api
 
 import (
-	"context"
+	"encoding/json"
+	"fmt"
 	"net/http"
+	"runtime/debug"
 	"strings"
 	"time"
 
+	"github.com/lbryio/lbrytv/app/auth"
 	"github.com/lbryio/lbrytv/app/proxy"
 	"github.com/lbryio/lbrytv/app/publish"
 	"github.com/lbryio/lbrytv/app/sdkrouter"
-	"github.com/lbryio/lbrytv/app/users"
+	"github.com/lbryio/lbrytv/config"
 	"github.com/lbryio/lbrytv/internal/metrics"
+	"github.com/lbryio/lbrytv/internal/monitor"
+	"github.com/lbryio/lbrytv/internal/responses"
 	"github.com/lbryio/lbrytv/internal/status"
+	"github.com/ybbus/jsonrpc"
 
 	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
+var logger = monitor.NewModuleLogger("api")
+
 // InstallRoutes sets up global API handlers
-func InstallRoutes(proxyService *proxy.ProxyService, r *mux.Router) {
-	authenticator := users.NewAuthenticator(users.NewWalletService(proxyService.SDKRouter))
-	proxyHandler := proxy.NewRequestHandler(proxyService)
-	upHandler, err := publish.NewUploadHandler(publish.UploadOpts{ProxyService: proxyService})
-	if err != nil {
-		panic(err)
-	}
+func InstallRoutes(r *mux.Router, sdkRouter *sdkrouter.Router) {
+	upHandler := &publish.Handler{UploadPath: config.GetPublishSourceDir()}
 
+	r.Use(recoveryHandler)
 	r.Use(methodTimer)
 
-	r.HandleFunc("/", Index)
+	r.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
+		http.Redirect(w, req, config.GetProjectURL(), http.StatusSeeOther)
+	})
+
+	authProvider := auth.NewIAPIProvider(sdkRouter, config.GetInternalAPIHost())
+	middlewareStack := middlewares(
+		sdkrouter.Middleware(sdkRouter),
+		auth.Middleware(authProvider),
+	)
 
 	v1Router := r.PathPrefix("/api/v1").Subrouter()
-	v1Router.HandleFunc("/proxy", proxyHandler.HandleOptions).Methods(http.MethodOptions)
-	v1Router.HandleFunc("/proxy", authenticator.Wrap(upHandler.Handle)).MatcherFunc(upHandler.CanHandle)
-	v1Router.HandleFunc("/proxy", proxyHandler.Handle)
+	v1Router.Use(middlewareStack)
+	v1Router.HandleFunc("/proxy", proxy.HandleCORS).Methods(http.MethodOptions)
+	v1Router.HandleFunc("/proxy", upHandler.Handle).MatcherFunc(upHandler.CanHandle)
+	v1Router.HandleFunc("/proxy", proxy.Handle)
 	v1Router.HandleFunc("/metric/ui", metrics.TrackUIMetric).Methods(http.MethodPost)
 
 	internalRouter := r.PathPrefix("/internal").Subrouter()
 	internalRouter.Handle("/metrics", promhttp.Handler())
-	internalRouter.HandleFunc("/status", injectSDKRouter(proxyService.SDKRouter, status.GetStatus))
+	internalRouter.Handle("/status", middlewareStack(http.HandlerFunc(status.GetStatus)))
 	internalRouter.HandleFunc("/whoami", status.WhoAMI)
 }
 
-// i can't tell if this is really a best practice or a hack
-func injectSDKRouter(rt *sdkrouter.Router, fn http.HandlerFunc) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		fn(w, r.Clone(context.WithValue(r.Context(), status.SDKRouterContextKey, rt)))
+// applies several middleware in order
+func middlewares(mws ...mux.MiddlewareFunc) mux.MiddlewareFunc {
+	return func(next http.Handler) http.Handler {
+		for _, mw := range mws {
+			next = mw(next)
+		}
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			next.ServeHTTP(w, r)
+		})
 	}
 }
 
@@ -62,3 +80,37 @@ func methodTimer(next http.Handler) http.Handler {
 		metrics.LbrytvCallDurations.WithLabelValues(path).Observe(time.Since(start).Seconds())
 	})
 }
+
+func recoveryHandler(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		recovered, stack := func() (err error, stack []byte) {
+			defer func() {
+				if r := recover(); r != nil {
+					var ok bool
+					err, ok = r.(error)
+					if !ok {
+						err = fmt.Errorf("%v", r)
+					}
+					if !config.IsProduction() {
+						stack = debug.Stack()
+					}
+				}
+			}()
+			next.ServeHTTP(w, r)
+			return err, nil
+		}()
+		if recovered != nil {
+			logger.Log().Errorf("PANIC %v, trace %s", recovered, stack)
+			responses.AddJSONContentType(w)
+			rsp, _ := json.Marshal(jsonrpc.RPCResponse{
+				JSONRPC: "2.0",
+				Error: &jsonrpc.RPCError{
+					Code:    -1,
+					Message: recovered.Error(),
+					Data:    string(stack),
+				},
+			})
+			w.Write(rsp)
+		}
+	})
+}

api/routes_test.go

@@ -2,29 +2,29 @@ package api
 
 import (
 	"bytes"
+	"encoding/json"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
-	"github.com/lbryio/lbrytv/app/proxy"
+	"github.com/gorilla/mux"
 	"github.com/lbryio/lbrytv/app/publish"
 	"github.com/lbryio/lbrytv/app/sdkrouter"
 	"github.com/lbryio/lbrytv/config"
-
-	"github.com/gorilla/mux"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/ybbus/jsonrpc"
 )
 
 func TestRoutesProxy(t *testing.T) {
 	r := mux.NewRouter()
-	proxy := proxy.NewService(proxy.Opts{SDKRouter: sdkrouter.New(config.GetLbrynetServers())})
+	rt := sdkrouter.New(config.GetLbrynetServers())
 
 	req, err := http.NewRequest("POST", "/api/v1/proxy", bytes.NewBuffer([]byte(`{"method": "status"}`)))
 	require.NoError(t, err)
 	rr := httptest.NewRecorder()
 
-	InstallRoutes(proxy, r)
+	InstallRoutes(r, rt)
 	r.ServeHTTP(rr, req)
 
 	assert.Equal(t, http.StatusOK, rr.Code)
@@ -33,29 +33,29 @@ func TestRoutesProxy(t *testing.T) {
 
 func TestRoutesPublish(t *testing.T) {
 	r := mux.NewRouter()
-	proxy := proxy.NewService(proxy.Opts{SDKRouter: sdkrouter.New(config.GetLbrynetServers())})
+	rt := sdkrouter.New(config.GetLbrynetServers())
 
 	req := publish.CreatePublishRequest(t, []byte("test file"))
 	rr := httptest.NewRecorder()
 
-	InstallRoutes(proxy, r)
+	InstallRoutes(r, rt)
 	r.ServeHTTP(rr, req)
 
 	assert.Equal(t, http.StatusOK, rr.Code)
 	// Authentication Required error here is enough to see that the request
 	// has been dispatched through the publish handler
-	assert.Contains(t, rr.Body.String(), `"code": -32080`)
+	assert.Contains(t, rr.Body.String(), `"code": -32084`)
 }
 
 func TestRoutesOptions(t *testing.T) {
 	r := mux.NewRouter()
-	proxy := proxy.NewService(proxy.Opts{SDKRouter: sdkrouter.New(config.GetLbrynetServers())})
+	rt := sdkrouter.New(config.GetLbrynetServers())
 
 	req, err := http.NewRequest("OPTIONS", "/api/v1/proxy", nil)
 	require.NoError(t, err)
 	rr := httptest.NewRecorder()
 
-	InstallRoutes(proxy, r)
+	InstallRoutes(r, rt)
 	r.ServeHTTP(rr, req)
 	assert.Equal(t, http.StatusOK, rr.Code)
 	assert.Equal(t, "7200", rr.Result().Header.Get("Access-Control-Max-Age"))
@@ -66,3 +66,53 @@ func TestRoutesOptions(t *testing.T) {
 		rr.Result().Header.Get("Access-Control-Allow-Headers"),
 	)
 }
+
+func TestRecoveryHandler_Panic(t *testing.T) {
+	h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		panic("xoxox")
+	})
+	rr := httptest.NewRecorder()
+	r, err := http.NewRequest(http.MethodGet, "/", &bytes.Buffer{})
+	require.NoError(t, err)
+	logger.Disable()
+	assert.NotPanics(t, func() {
+		recoveryHandler(h).ServeHTTP(rr, r)
+	})
+	var res jsonrpc.RPCResponse
+	err = json.Unmarshal(rr.Body.Bytes(), &res)
+	require.NoError(t, err)
+	require.NotNil(t, res.Error)
+	assert.Contains(t, res.Error.Message, "xoxox")
+}
+
+func TestRecoveryHandler_NoPanic(t *testing.T) {
+	h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("no panic recovery"))
+	})
+	rr := httptest.NewRecorder()
+	r, err := http.NewRequest(http.MethodGet, "/", &bytes.Buffer{})
+	require.NoError(t, err)
+	assert.NotPanics(t, func() {
+		recoveryHandler(h).ServeHTTP(rr, r)
+	})
+	assert.Equal(t, rr.Body.String(), "no panic recovery")
+
+}
+
+func TestRecoveryHandler_RecoveredPanic(t *testing.T) {
+	h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if r := recover(); r != nil {
+				w.Write([]byte("panic recovered in here"))
+			}
+		}()
+		panic("xoxoxo")
+	})
+	rr := httptest.NewRecorder()
+	r, err := http.NewRequest(http.MethodGet, "/", &bytes.Buffer{})
+	require.NoError(t, err)
+	assert.NotPanics(t, func() {
+		recoveryHandler(h).ServeHTTP(rr, r)
+	})
+	assert.Equal(t, rr.Body.String(), "panic recovered in here")
+}