Add Beatconnect downloader, add go.mod and bump Go

README.md

@@ -1,6 +1,6 @@
 <p align="center"><img src="https://y.zxq.co/jobeei.png"></p>
 
-# CheeseGull [![Build Status](https://travis-ci.org/osuripple/cheesegull.svg?branch=master)](https://travis-ci.org/osuripple/cheesegull)
+# CheeseGull
 
 CheeseGull creates an unofficial "slave" database of the official osu! beatmap
 database, trying to keep every beatmap up to date as much as possible, as well

api/api.go

@@ -26,9 +26,10 @@ type Context struct {
 	DB       *sql.DB
 	SearchDB *sql.DB
 	House    *housekeeper.House
-	DLClient *downloader.Client
+	DLClient downloader.Client
 	writer   http.ResponseWriter
 	params   httprouter.Params
+	Options  Options
 }
 
 // Write writes content to the response body.
@@ -93,9 +94,13 @@ func POST(path string, f func(c *Context)) {
 	handlers = append(handlers, handlerPath{"POST", path, f})
 }
 
+type Options struct {
+	AllowUnranked bool
+}
+
 // CreateHandler creates a new http.Handler using the handlers registered
 // through GET and POST.
-func CreateHandler(db, searchDB *sql.DB, house *housekeeper.House, dlc *downloader.Client) http.Handler {
+func CreateHandler(db, searchDB *sql.DB, house *housekeeper.House, dlc downloader.Client, options Options) http.Handler {
 	r := httprouter.New()
 	for _, h := range handlers {
 		// Create local copy that we know won't change as the loop proceeds.
@@ -110,6 +115,7 @@ func CreateHandler(db, searchDB *sql.DB, house *housekeeper.House, dlc *download
 				DLClient: dlc,
 				writer:   w,
 				params:   p,
+				Options:  options,
 			}
 			defer func() {
 				err := recover()

api/download/download.go

@@ -45,7 +45,7 @@ func Download(c *api.Context) {
 		errorMessage(c, 404, "Set not found")
 		return
 	}
-	if set.RankedStatus <= 0 {
+	if set.RankedStatus <= 0 && !c.Options.AllowUnranked {
 		errorMessage(c, 406, "Unranked beatmap sets are currently not available for download, following a warning")
 		return
 	}
@@ -97,7 +97,7 @@ func Download(c *api.Context) {
 	}
 }
 
-func downloadBeatmap(c *downloader.Client, b *housekeeper.CachedBeatmap, house *housekeeper.House) error {
+func downloadBeatmap(c downloader.Client, b *housekeeper.CachedBeatmap, house *housekeeper.House) error {
 	log.Println("[⬇️]", b.String())
 
 	var fileSize uint64

cheesegull.go

@@ -30,9 +30,15 @@ const searchDSNDocs = `"DSN to use for fulltext searches. ` +
 	`behaviour and you should definetely bother to set it up (follow the README).`
 
 var (
-	osuAPIKey    = kingpin.Flag("api-key", "osu! API key").Short('k').Envar("OSU_API_KEY").String()
-	osuUsername  = kingpin.Flag("osu-username", "osu! username (for downloading and fetching whether a beatmap has a video)").Short('u').Envar("OSU_USERNAME").String()
-	osuPassword  = kingpin.Flag("osu-password", "osu! password (for downloading and fetching whether a beatmap has a video)").Short('p').Envar("OSU_PASSWORD").String()
+	osuAPIKey = kingpin.Flag("api-key", "osu! API key").Short('k').Envar("OSU_API_KEY").String()
+
+	osuUsername = kingpin.Flag("osu-username", "osu! username (for downloading and fetching whether a beatmap has a video)").Short('u').Envar("OSU_USERNAME").String()
+	osuPassword = kingpin.Flag("osu-password", "osu! password (for downloading and fetching whether a beatmap has a video)").Short('p').Envar("OSU_PASSWORD").String()
+
+	beatconnectToken = kingpin.Flag("beatconnect-token", "beatconnect token. if provided, will use beatconnect rather than osu! website for downloading beatmaps").Envar("BEATCONNECT_TOKEN").String()
+
+	allowUnranked = kingpin.Flag("allow-unranked", "Allow unranked beatmaps to be downloaded").Envar("ALLOW_UNRANKED").Default("false").Bool()
+
 	mysqlDSN     = kingpin.Flag("mysql-dsn", "DSN of MySQL").Short('m').Default("root@/cheesegull").Envar("MYSQL_DSN").String()
 	searchDSN    = kingpin.Flag("search-dsn", searchDSNDocs).Default("root@tcp(127.0.0.1:9306)/cheesegull").Envar("SEARCH_DSN").String()
 	httpAddr     = kingpin.Flag("http-addr", "Address on which to take HTTP requests.").Short('a').Default("127.0.0.1:62011").Envar("HTTP_ADDR").String()
@@ -75,20 +81,30 @@ func main() {
 	c := osuapi.NewClient(*osuAPIKey)
 
 	// set up downloader
-	var reqPreparer downloader.LogInRequestPreparer
-	if *fckcfAddr == "" {
-		// No fckck address provided, disable it.
-		reqPreparer = &downloader.EmptyLogInRequestPreparer{}
+	var downloaderClient downloader.Client
+	if *beatconnectToken != "" {
+		fmt.Println("Using beatconnect")
+		downloaderClient = downloader.NewBeatConnectClient(*beatconnectToken)
 	} else {
-		// Fckcf address provided, use it as a proxy
-		reqPreparer = &downloader.FckCf{Address: *fckcfAddr}
+		fmt.Println("Using osu! website")
+
+		var reqPreparer downloader.LogInRequestPreparer
+		if *fckcfAddr == "" {
+			// No fckck address provided, disable it.
+			reqPreparer = &downloader.EmptyLogInRequestPreparer{}
+		} else {
+			// Fckcf address provided, use it as a proxy
+			reqPreparer = &downloader.FckCf{Address: *fckcfAddr}
+		}
+
+		downloaderClient, err = downloader.NewOsuClient(*osuUsername, *osuPassword, reqPreparer)
+		if err != nil {
+			fmt.Println("Can't log in into osu!:", err)
+			os.Exit(1)
+		}
 	}
-	d, err := downloader.LogIn(*osuUsername, *osuPassword, reqPreparer)
-	if err != nil {
-		fmt.Println("Can't log in into osu!:", err)
-		os.Exit(1)
-	}
-	dbmirror.SetHasVideo(d.HasVideo)
+
+	d := downloader.NewDownloader(downloaderClient)
 
 	// set up mysql
 	db, err := sql.Open("mysql", addTimeParsing(*mysqlDSN))
@@ -115,5 +131,7 @@ func main() {
 	go dbmirror.DiscoverEvery(c, db, time.Hour*6, time.Second*20)
 
 	// create request handler
-	panic(http.ListenAndServe(*httpAddr, api.CreateHandler(db, db2, house, d)))
+	panic(http.ListenAndServe(*httpAddr, api.CreateHandler(db, db2, house, d, api.Options{
+		AllowUnranked: *allowUnranked,
+	})))
 }

dbmirror/dbmirror.go

@@ -25,17 +25,6 @@ const (
 	SetUpdaterWorkers = PerBatch / 20
 )
 
-// hasVideo checks whether a beatmap set has a video.
-var hasVideo func(set int) (bool, error)
-
-// SetHasVideo sets the hasVideo function to the one passed.
-func SetHasVideo(f func(int) (bool, error)) {
-	if f == nil {
-		return
-	}
-	hasVideo = f
-}
-
 func createChildrenBeatmaps(bms []osuapi.Beatmap) []models.Beatmap {
 	cgBms := make([]models.Beatmap, len(bms))
 	for idx, bm := range bms {
@@ -111,10 +100,7 @@ func updateSet(c *osuapi.Client, db *sql.DB, set models.Set) error {
 	if updated {
 		// if it has been updated, video might have been added or removed
 		// so we need to check for it
-		set.HasVideo, err = hasVideo(x.BeatmapSetID)
-		if err != nil {
-			return err
-		}
+		set.HasVideo = bool(x.Video)
 	}
 
 	return models.CreateSet(db, set)

dbmirror/discover.go

@@ -51,10 +51,7 @@ func Discover(c *osuapi.Client, db *sql.DB) error {
 
 		set := setFromOsuAPIBeatmap(bms[0])
 		set.ChildrenBeatmaps = createChildrenBeatmaps(bms)
-		set.HasVideo, err = hasVideo(bms[0].BeatmapSetID)
-		if err != nil {
-			return err
-		}
+		set.HasVideo = bool(bms[0].Video)
 
 		err = models.CreateSet(db, set)
 		if err != nil {

doc.go

@@ -4,4 +4,4 @@
 package main
 
 // Version is the version of cheesegull.
-const Version = "v2.1.2"
+const Version = "v2.2.0"

downloader/beatconnect.go

@@ -0,0 +1,88 @@
+package downloader
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+const BeatConnectAPIBase = "https://beatconnect.io"
+
+// BeatConnectClient is a client for the BeatConnect API.
+type BeatConnectClient struct {
+	httpClient *http.Client
+	apiToken   string
+}
+
+// NewBeatConnectClient returns a new BeatConnectClient.
+func NewBeatConnectClient(apiToken string) *BeatConnectClient {
+	cl := http.DefaultClient
+	// I really wish we had context.Context back in the day.
+	cl.Timeout = time.Minute * 2
+	return &BeatConnectClient{
+		httpClient: cl,
+		apiToken:   apiToken,
+	}
+}
+
+// newRequest creates a new http.Request with the given relativeURL.
+// If withToken is true, it adds the 'token' query parameter to the request.
+func (c *BeatConnectClient) newRequest(relativeURL string, withToken bool) (*http.Request, error) {
+	req, err := http.NewRequest(
+		http.MethodGet,
+		BeatConnectAPIBase+"/"+relativeURL,
+		nil,
+	)
+	if err != nil {
+		return nil, err
+	}
+	if withToken {
+		query := req.URL.Query()
+		query.Set("token", c.apiToken)
+		req.URL.RawQuery = query.Encode()
+	}
+	return req, nil
+}
+
+// HasVideo returns whether the beatmap set with the given ID has a video
+// according to the BeatConnect API.
+/* func (c *BeatConnectClient) HasVideo(setID int) (bool, error) {
+	req, err := c.newRequest(fmt.Sprintf("api/beatmaps/%d/", setID))
+	if err != nil {
+		return false, fmt.Errorf("new request: %w", err)
+	}
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return false, fmt.Errorf("do request: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return false, fmt.Errorf("bad status code for %q: %d", req.URL, resp.StatusCode)
+	}
+	var body struct {
+		Video bool `json:"video"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
+		return false, fmt.Errorf("decode preliminary response: %w", err)
+	}
+	return body.Video, nil
+} */
+
+// Download downloads the osz file from BeatConnect.
+// The noVideo flag is ignored, the video is always included.
+func (c *BeatConnectClient) Download(setID int, noVideo bool) (io.ReadCloser, error) {
+	oszReq, err := c.newRequest(fmt.Sprintf("b/%d", setID), false)
+	if err != nil {
+		return nil, fmt.Errorf("new request: %w", err)
+	}
+	resp, err := c.httpClient.Do(oszReq)
+	if err != nil {
+		return nil, fmt.Errorf("do request: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return nil, fmt.Errorf("bad status code: %d", resp.StatusCode)
+	}
+	return resp.Body, err
+}