支持并发

README.md

@@ -1,6 +1,9 @@
 # dl-talebook
 Downloading books from [talebook server](https://github.com/talebook/talebook) , inspired from gist https://gist.github.com/syhily/9feb936bcaebf2beec567733810f4666 .
 
+1. 并发任务
+2. 断点下载
+
 # Demo
 ![demo](demo.png)
 # Install
@@ -11,16 +14,18 @@ go install github.com/hellojukay/dl-talebook@latest
 ```
 hellojukay@local dl-talebook (main) $ ./dl-talebook -h
 Usage of ./dl-talebook:
+  -c int
+    	maximum number of concurrent download tasks allowed per second (default 5)
   -dir string
-        data dir (default "./")
+    	data dir (default "./")
   -password string
-        password
+    	password
   -site string
-        tabebook web site (default "https://book.codefine.site:6870/")
+    	tabebook web site (default "https://book.codefine.site:6870/")
   -timeout duration
-        http timeout (default 10s)
+    	http timeout (default 10s)
   -user-agent string
-        http userAgent (default "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36")
+    	http userAgent (default "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36")
   -username string
-        username
-```
+    	username
+```

go.mod

@@ -2,4 +2,7 @@ module github.com/hellojukay/dl-talebook
 
 go 1.18
 
-require github.com/dustin/go-humanize v1.0.0 // indirect
+require (
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect
+)

go.sum

@@ -1,2 +1,4 @@
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 h1:M73Iuj3xbbb9Uk1DYhzydthsj6oOd6l9bpuFcNoUvTs=
+golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

main.go

@@ -1,23 +1,28 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"flag"
 	"log"
 	"os"
 	"time"
+
+	"golang.org/x/time/rate"
 )
 
 var (
-	userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36"
-	site      = `https://book.codefine.site:6870/`
-	dir       = "./"
-	timeout   = time.Duration(10) * time.Second
-	username  = ""
-	password  = ""
+	userAgent  = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36"
+	site       = `https://book.codefine.site:6870/`
+	dir        = "./"
+	timeout    = time.Duration(10) * time.Second
+	username   = ""
+	password   = ""
+	concurrent = 5
 )
 
 func init() {
+	flag.IntVar(&concurrent, "c", concurrent, "maximum number of concurrent download tasks allowed per second")
 	flag.StringVar(&username, "username", username, "username")
 	flag.StringVar(&password, "password", password, "password")
 	flag.StringVar(&site, "site", site, "tabebook web site")
@@ -37,7 +42,10 @@ func main() {
 		log.Fatal(err)
 	}
 	log.Printf("%d books retrieved on server %s", tale.ServerInfo.Sys.Books, site)
+	l := rate.NewLimiter(rate.Limit(concurrent), concurrent)
 	for {
+		// 限制速度
+		l.Wait(context.Background())
 		book, err := tale.Next()
 		if err != nil {
 			log.Printf("%s [skiped]", err.Error())
@@ -47,8 +55,10 @@ func main() {
 			continue
 		}
 		log.Printf("downloading %s", book.String())
-		if err = tale.Download(book, dir); err != nil {
-			log.Printf("%s %s [skiped]", book.Book.Title, err)
-		}
+		go func() {
+			if err = tale.Download(book, dir); err != nil {
+				log.Printf("%s %s [skiped]", book.Book.Title, err)
+			}
+		}()
 	}
 }

talebook.go

@@ -118,9 +118,6 @@ func (tale *TaleBook) Next() (*Book, error) {
 		return nil, NO_MORE_BOOK_ERROR
 	}
 	var api = urlJoin(tale.api, "api", "book", fmt.Sprintf("%d", tale.index))
-	if err := tale.check(api); err != nil {
-		return nil, err
-	}
 	response, err := tale.client.Get(api)
 	if err != nil {
 		return nil, err
@@ -168,18 +165,6 @@ func (tale *TaleBook) Download(b *Book, dir string) error {
 	return nil
 }
 
-func (tale *TaleBook) check(api string) error {
-	response, err := tale.client.Get(api)
-	if err != nil {
-		return wrapperTimeOutError(err)
-	}
-	defer response.Body.Close()
-	if response.StatusCode != http.StatusOK {
-		return fmt.Errorf(" %s", response.Status)
-	}
-	return nil
-}
-
 func NewTableBook(site string, opstions ...func(*TaleBook)) (*TaleBook, error) {
 	var client http.Client = http.Client{
 		Timeout: time.Duration(30) * time.Second,