init

README.md

@@ -27,13 +27,14 @@ go run main.go
 
 小说资源说明：
 
-本项目利用了互联网上的一些小说资源作为检索目标进行资源提取，定义如下：
+本项目利用了互联网上的一些小说资源作为检索目标进行资源提取，定义如下，100以后的命名定位第三方检索：
 
-| 名称 | 编号     |
-| :--- | -------- |
-| 10   | 笔趣阁   |
-| 20   | 新笔趣阁 |
-| 30   | 笔下文学 |
+| 名称 | 编号     |搜索类型		|
+| :--- | -------- | ----------- |
+| 10   | 笔趣阁   | 站内		|
+| 100  | 新笔趣阁 |	百度第三方  |
+| 110  | 笔下文学 | 百度第三方  |
+| 120  | 顶点小说 | 百度第三方  |
 
 **搜索小说：**
 
@@ -42,7 +43,7 @@ go run main.go
 请求：
 
 ``` shell
-# 请求资源10下小说牧神记的检索结果
+# 请求资源为10下小说牧神记的检索结果
 curl http://0.0.0.0:8080/v1/novels/牧神记/10
 ```
 

apis/novels.go

@@ -23,13 +23,14 @@ func SearchNovels(c *gin.Context) {
 			resultData, err := common.FetchHtml(novelName, currentRule)
 			if err != nil {
 				log.Println("Request URL error", err)
-				c.JSON(http.StatusOK, gin.H{"statue": 0, "info": "Request error"})
+				c.JSON(http.StatusOK, gin.H{"statue": 0, "msg": "Request error"})
+			} else {
+				c.JSON(http.StatusOK, gin.H{"status": 1, "info": resultData})
 			}
-			c.JSON(http.StatusOK, gin.H{"status": 1, "info": resultData})
 		} else {
-			c.JSON(http.StatusOK, gin.H{"statue": 0, "info": "Parameter error"})
+			c.JSON(http.StatusOK, gin.H{"statue": 0, "msg": "Parameter error"})
 		}
 	} else {
-		c.JSON(http.StatusOK, gin.H{"statue": 0, "info": "Parameter name can't be empty"})
+		c.JSON(http.StatusOK, gin.H{"statue": 0, "msg": "Parameter name can't be empty"})
 	}
 }

common/utils.go

@@ -1,15 +1,14 @@
 package common
 
 import (
-	"io/ioutil"
 	"log"
-	"net/http"
 	"net/url"
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/axgle/mahonia"
 	"github.com/howie6879/owllook_api/config"
+	"github.com/levigross/grequests"
 	"github.com/saintfish/chardet"
 )
 
@@ -50,10 +49,8 @@ func FetchHtml(name string, rule config.NovelRule) ([]map[string]string, error)
 		log.Println("Request URL error", err)
 		return resultData, err
 	}
-	defer response.Body.Close()
 	if response.StatusCode == 200 {
-		body, _ := ioutil.ReadAll(response.Body)
-		raw_html := DetectBody(body)
+		raw_html := DetectBody(response.Bytes())
 		doc, _ := goquery.NewDocumentFromReader(strings.NewReader(raw_html))
 		doc.Find(rule.TargetItem).Each(func(i int, s *goquery.Selection) {
 			novelName := s.Find(rule.ItemRule.NovelName).Text()
@@ -85,14 +82,15 @@ func FetchHtml(name string, rule config.NovelRule) ([]map[string]string, error)
 }
 
 // RequestURL returns a search result
-func RequestURL(url string) (*http.Response, error) {
-	tr := &http.Transport{
-		MaxIdleConns:        20,
-		MaxIdleConnsPerHost: 20,
+func RequestURL(url string) (*grequests.Response, error) {
+	ro := &grequests.RequestOptions{
+		Headers: map[string]string{"User-Agent": config.GetUserAgent()},
 	}
-	client := &http.Client{Transport: tr}
-	req, _ := http.NewRequest("GET", url, nil)
-	req.Header.Set("User-Agent", config.GetUserAgent())
-	response, err := client.Do(req)
-	return response, err
+	resp, err := grequests.Get(url, ro)
+	if err != nil {
+		log.Println("Unable to make request: ", err)
+	}
+
+	// log.Println(resp.String())
+	return resp, err
 }

config/config.go

@@ -22,6 +22,8 @@ type NovelRule struct {
 	Name       string
 	HomeUrl    string
 	SearchUrl  string
+	Method     string
+	Params     map[string]string
 	TargetItem string
 	ItemRule   ItemRuleConfig
 }
@@ -70,34 +72,53 @@ var (
 			Name:       "笔趣阁01",
 			HomeUrl:    "https://www.bqg99.cc/",
 			SearchUrl:  "https://www.bqg99.cc/s.php?q=",
+			Method:     "Get",
+			Params:     make(map[string](string)),
 			TargetItem: ".bookbox",
 			ItemRule:   ItemRule01,
 		},
 		"11": NovelRule{
 			Name:       "笔趣阁02",
 			HomeUrl:    "http://www.cdzdgw.com/",
 			SearchUrl:  "http://www.cdzdgw.com/s.php?q=",
+			Method:     "Get",
+			Params:     make(map[string](string)),
 			TargetItem: ".bookbox",
 			ItemRule:   ItemRule01,
 		},
 		"12": NovelRule{
 			Name:       "笔趣阁03",
 			HomeUrl:    "http://www.biqugex.com/",
 			SearchUrl:  "http://www.biqugex.com/s.php?q=",
+			Method:     "Get",
+			Params:     make(map[string](string)),
 			TargetItem: ".bookbox",
 			ItemRule:   ItemRule01,
 		},
-		"20": NovelRule{
+		"100": NovelRule{
 			Name:       "新笔趣阁01",
 			HomeUrl:    "http://www.biqugetv.com/",
 			SearchUrl:  "http://zhannei.baidu.com/cse/search?click=1&s=16765504158186272814&q=",
+			Method:     "Get",
+			Params:     make(map[string](string)),
 			TargetItem: "div.result-list div.result-item",
 			ItemRule:   ItemRule02,
 		},
-		"30": NovelRule{
+		"110": NovelRule{
 			Name:       "笔下文学01",
 			HomeUrl:    "http://www.xbxwx.net/",
 			SearchUrl:  "http://so.xbxwx.net/cse/search?click=1&entry=1&s=10874778206555383279&q=",
+			Method:     "Get",
+			Params:     make(map[string](string)),
+			TargetItem: "div.result-list div.result-item",
+			ItemRule:   ItemRule02,
+		},
+		"120": NovelRule{
+			Name:       "顶点小说01",
+			HomeUrl:    "http://www.23wx.cc/du/99/99646/",
+			SearchUrl:  "http://zhannei.baidu.com/cse/search?s=17788970894453164958&q=",
+			Method:     "Get",
+			Params:     make(map[string](string)),
 			TargetItem: "div.result-list div.result-item",
 			ItemRule:   ItemRule02,
 		},

vendor/vendor.json

@@ -56,6 +56,18 @@
 			"revision": "5831880292e721c76b58a16ecc60adc27d8e6355",
 			"revisionTime": "2018-06-12T18:59:29Z"
 		},
+		{
+			"checksumSHA1": "p3IB18uJRs4dL2K5yx24MrLYE9A=",
+			"path": "github.com/google/go-querystring/query",
+			"revision": "53e6ce116135b80d037921a7fdd5138cf32d7a8a",
+			"revisionTime": "2017-01-11T10:11:55Z"
+		},
+		{
+			"checksumSHA1": "e3krTtxeGeDTyWRkZC7BOYfexCw=",
+			"path": "github.com/levigross/grequests",
+			"revision": "bf9788368aa04abf510962fd0fb0d8e4b74396cc",
+			"revisionTime": "2017-10-09T01:03:47Z"
+		},
 		{
 			"checksumSHA1": "w5RcOnfv5YDr3j2bd1YydkPiZx4=",
 			"path": "github.com/mattn/go-isatty",
@@ -78,6 +90,10 @@
 			"path": "golang.org/x/net/html",
 			"revision": ""
 		},
+		{
+			"path": "golang.org/x/net/publicsuffix",
+			"revision": ""
+		},
 		{
 			"path": "golang.org/x/sys/unix",
 			"revision": ""