Merge pull request #17 from wrfly/develop

URL Filter (blacklist, whitelist ...)

.gitignore

@@ -17,3 +17,5 @@ vendor/
 
 yasuser
 yasuser.db
+
+.vscode/*

Makefile

@@ -42,11 +42,10 @@ push-dev-img:
 
 .PHONY: tools
 tools:
-	go get github.com/jteeuwen/go-bindata/...
-	go get github.com/elazarl/go-bindata-assetfs/...
+	go get github.com/wrfly/bindata
 
 .PHONY: asset
 asset:
-	go-bindata-assetfs -nometadata -prefix routes/index -pkg routes routes/index/...
-	mv bindata_assetfs.go routes/asset.go
-	gofmt -w routes/asset.go
+	bindata -pkg asset \
+		-resource routes/index \
+		-target routes/asset

README.md

@@ -111,14 +111,13 @@ See [benchmark](benchmark/readme.md)
 - [x] customization
 - [x] TTL of URL
 - [x] rate limit
-- [ ] management(auth)
-  - [ ] remove(domain or keywords)
-  - [ ] blacklist(domain or keywords)
-  - [ ] whitelist(domain or keywords)
+- ~~[ ] management(auth)~~
+  - [x] domain filter
+  - [x] keyword filter
 - [ ] statistic
   - [ ] URL status
   - [ ] runtime metrics
 - [x] UI index
   - [x] google analytics
-  - [ ] prettify the index
+  - [x] prettify the index
 - [x] pprof

config.yml

@@ -1,11 +1,22 @@
 debug: false
-shortener:
-  store:
-    dbpath: ./yasuser.db
-    dbtype: bolt
-    redis: redis://localhost:6379
+store:
+  dbpath: ./yasuser.db
+  dbtype: bolt
+  redis: redis://localhost:6379
 server:
   domain: https://u.kfd.me
   port: 8084
+  limit: 10
   pprof: false
-  gaid: 62244864-8
+  gaid: 62244864-8
+filter:
+  domain:
+    whitelist:
+      - kfd.me
+    blacklist:
+      - t66y.com
+  keyword:
+    whitelist:
+      - yasuser
+    blacklist:
+      - gg

config/config.go

@@ -5,11 +5,11 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/sirupsen/logrus"
 	"github.com/wrfly/ecp"
+	yaml "gopkg.in/yaml.v2"
 
-	"github.com/sirupsen/logrus"
 	"github.com/wrfly/yasuser/utils"
-	yaml "gopkg.in/yaml.v2"
 )
 
 type SrvConfig struct {
@@ -26,24 +26,26 @@ type StoreConfig struct {
 	Redis  string `default:"redis://localhost:6379"`
 }
 
-type ShortenerConfig struct {
-	Store StoreConfig
+type list struct {
+	WhiteList []string
+	BlackList []string
+}
+
+type Filter struct {
+	Domain  list
+	Keyword list
 }
 
 type Config struct {
-	Debug     bool `default:"false"`
-	Shortener ShortenerConfig
-	Server    SrvConfig
+	Debug  bool   `default:"false"`
+	Auth   string `default:"password"`
+	Store  StoreConfig
+	Server SrvConfig
+	Filter Filter
 }
 
 func New() *Config {
-	conf := Config{
-		Server: SrvConfig{},
-		Shortener: ShortenerConfig{
-			Store: StoreConfig{},
-		},
-	}
-	return &conf
+	return &Config{}
 }
 
 func (c *Config) Parse(filePath string) {

filter/errors.go

@@ -0,0 +1,9 @@
+package filter
+
+import "errors"
+
+// errors
+var (
+	ErrBadDomain  = errors.New("domain in blacklist")
+	ErrBadKeyword = errors.New("url contains bad keyword")
+)

filter/filter.go

@@ -0,0 +1,71 @@
+package filter
+
+import (
+	"net/url"
+	"strings"
+
+	"github.com/wrfly/yasuser/config"
+)
+
+type Filter interface {
+	OK(*url.URL) error
+}
+
+type list struct {
+	blacklist map[string]bool
+	whitelist map[string]bool
+}
+
+type urlFilter struct {
+	domain  list
+	keyword list
+}
+
+func (f *urlFilter) OK(u *url.URL) error {
+	// bypass domain
+	if f.domain.whitelist[u.Hostname()] {
+		return nil
+	}
+
+	// bypass keyword
+	for x := range f.keyword.whitelist {
+		if strings.Contains(u.Path, x) {
+			return nil
+		}
+	}
+
+	// bad domain
+	if f.domain.blacklist[u.Hostname()] {
+		return ErrBadDomain
+	}
+
+	// bad keyword
+	for x := range f.keyword.blacklist {
+		if strings.Contains(u.Path, x) {
+			return ErrBadKeyword
+		}
+	}
+
+	return nil
+}
+
+func makeList(slice []string) map[string]bool {
+	x := make(map[string]bool, len(slice))
+	for _, s := range slice {
+		x[s] = true
+	}
+	return x
+}
+
+func New(conf config.Filter) Filter {
+	return &urlFilter{
+		domain: list{
+			whitelist: makeList(conf.Domain.WhiteList),
+			blacklist: makeList(conf.Domain.BlackList),
+		},
+		keyword: list{
+			whitelist: makeList(conf.Keyword.WhiteList),
+			blacklist: makeList(conf.Keyword.BlackList),
+		},
+	}
+}

main.go

@@ -10,6 +10,7 @@ import (
 	"gopkg.in/urfave/cli.v2"
 
 	"github.com/wrfly/yasuser/config"
+	"github.com/wrfly/yasuser/filter"
 	"github.com/wrfly/yasuser/routes"
 	"github.com/wrfly/yasuser/shortener"
 )
@@ -67,7 +68,11 @@ func main() {
 				gin.SetMode(gin.ReleaseMode)
 			}
 
-			err := routes.Serve(conf.Server, shortener.New(conf.Shortener))
+			err := routes.Serve(
+				conf.Server,
+				shortener.New(conf.Store),
+				filter.New(conf.Filter),
+			)
 			if err != nil {
 				logrus.Error(err)
 			}