From dda9fe79b27cd4d505ec6319a984889809bf9c62 Mon Sep 17 00:00:00 2001 From: Quan Tong Date: Sat, 23 Mar 2024 07:38:34 +0700 Subject: [PATCH] server-side tracking: add /stats --- .github/workflows/go.yml | 5 +- Dockerfile | 2 +- cmd/blog/main.go | 41 ++- config.go | 8 + event.go | 20 ++ go.mod | 39 ++- go.sum | 113 ++++++- http/event.go | 65 ++++ http/server.go | 67 +++- http/server_test.go | 6 +- http/stats.go | 48 +++ http/webhook.go | 2 +- kafka/event.go | 84 +++++ message_queue.go => queue.go | 2 +- rabbitmq/{message_queue.go => queue.go} | 8 +- sqlite/migrations/000001_init_schema.down.sql | 1 + sqlite/migrations/000001_init_schema.up.sql | 11 + .../000002_create_ip2location_table.down.sql | 1 + .../000002_create_ip2location_table.up.sql | 6 + sqlite/sqlite.go | 125 +++++++ sqlite/stats.go | 316 ++++++++++++++++++ stats.go | 41 +++ ui/html/stats.html | 99 ++++++ 23 files changed, 1056 insertions(+), 54 deletions(-) create mode 100644 event.go create mode 100644 http/event.go create mode 100644 http/stats.go create mode 100644 kafka/event.go rename message_queue.go => queue.go (62%) rename rabbitmq/{message_queue.go => queue.go} (76%) create mode 100644 sqlite/migrations/000001_init_schema.down.sql create mode 100644 sqlite/migrations/000001_init_schema.up.sql create mode 100644 sqlite/migrations/000002_create_ip2location_table.down.sql create mode 100644 sqlite/migrations/000002_create_ip2location_table.up.sql create mode 100644 sqlite/sqlite.go create mode 100644 sqlite/stats.go create mode 100644 stats.go create mode 100644 ui/html/stats.html diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 146f14c..302d175 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -28,8 +28,11 @@ jobs: - name: Test run: go test -v ./... + - name: Install AArch64 target toolchain + run: sudo apt-get update && sudo apt-get install gcc-aarch64-linux-gnu libc6-dev-arm64-cross + - name: Build - run: CGO_ENABLED=0 GOARCH=arm64 go build -o blog -v -ldflags="-s -w" cmd/blog/main.go + run: CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ CGO_ENABLED=1 GOARCH=arm64 go build -o blog -v -ldflags="-s -w -linkmode 'external' -extldflags '-static'" cmd/blog/main.go - name: Set up QEMU if: github.event_name == 'push' diff --git a/Dockerfile b/Dockerfile index d715620..44a6b70 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:3.13 WORKDIR /app -RUN apk add --no-cache ca-certificates git +RUN apk add --no-cache ca-certificates git sqlite RUN mkdir db COPY blog . EXPOSE 8009 diff --git a/cmd/blog/main.go b/cmd/blog/main.go index 875f578..b7dffe9 100644 --- a/cmd/blog/main.go +++ b/cmd/blog/main.go @@ -11,12 +11,15 @@ import ( "github.com/getsentry/sentry-go" "github.com/pkg/errors" + "github.com/rs/zerolog" "github.com/spf13/viper" "github.com/quantonganh/blog" "github.com/quantonganh/blog/http" + "github.com/quantonganh/blog/kafka" "github.com/quantonganh/blog/markdown" "github.com/quantonganh/blog/rabbitmq" + "github.com/quantonganh/blog/sqlite" ) func main() { @@ -48,9 +51,13 @@ func main() { log.Fatal(err) } - a, err := newApp(config, posts) + logger := zerolog.New(os.Stdout).With(). + Timestamp(). + Logger() + + a, err := newApp(logger, config, posts) if err != nil { - log.Fatal(err) + logger.Error().Err(err).Msg("error creating new app") } ctx, cancel := context.WithCancel(context.Background()) @@ -76,30 +83,46 @@ func main() { } type app struct { + db *sqlite.DB config *blog.Config httpServer *http.Server } -func newApp(config *blog.Config, posts []*blog.Post) (*app, error) { - mqService, err := rabbitmq.NewMessageQueueService(config.AMQP.URL) +func newApp(logger zerolog.Logger, config *blog.Config, posts []*blog.Post) (*app, error) { + httpServer, err := http.NewServer(logger, config, posts) if err != nil { + logger.Error().Err(err).Msg("error creating new HTTP server") return nil, err } - httpServer, err := http.NewServer(config, posts) + queueService, err := rabbitmq.NewQueueService(config.AMQP.URL) if err != nil { - log.Fatalf("%+v\n", err) return nil, err } - httpServer.MessageQueueService = mqService + httpServer.QueueService = queueService + + eventService, err := kafka.NewEventService(config.Kafka.Broker) + if err != nil { + return nil, err + } + httpServer.EventService = eventService + + db := sqlite.NewDB("db/stats.db") + statService := sqlite.NewStatService(logger, db) + httpServer.StatService = statService return &app{ + db: db, config: config, httpServer: httpServer, }, nil } func (a *app) Run(ctx context.Context) error { + if err := a.db.Open(); err != nil { + return err + } + a.httpServer.Addr = a.config.HTTP.Addr baseURL, err := url.Parse(a.config.Site.BaseURL) if err != nil { @@ -111,6 +134,10 @@ func (a *app) Run(ctx context.Context) error { return err } + if err := a.httpServer.ProcessActivityStream(ctx, a.config.IP2Location.Token); err != nil { + return err + } + return nil } diff --git a/config.go b/config.go index f3890c2..f3caba8 100644 --- a/config.go +++ b/config.go @@ -65,6 +65,14 @@ type Config struct { AMQP struct { URL string } + + Kafka struct { + Broker string + } + + IP2Location struct { + Token string + } } // Item represents a navbar item diff --git a/event.go b/event.go new file mode 100644 index 0000000..ff91c83 --- /dev/null +++ b/event.go @@ -0,0 +1,20 @@ +package blog + +import "context" + +type Event struct { + UserID string `json:"user_id"` + IP string `json:"ip"` + Country string `json:"country"` + UserAgent string `json:"user_agent"` + Browser string `json:"browser"` + OS string `json:"os"` + Referer string `json:"referer"` + URL string `json:"url"` + Time string `json:"time"` +} + +type EventService interface { + SendMessage(topic, key string, value []byte) error + Consume(ctx context.Context, topic string) (<-chan *Event, error) +} diff --git a/go.mod b/go.mod index 2e483a0..6c066a0 100644 --- a/go.mod +++ b/go.mod @@ -10,19 +10,40 @@ require ( github.com/blevesearch/bleve v1.0.14 github.com/getsentry/sentry-go v0.9.0 github.com/gorilla/feeds v1.1.1 - github.com/gorilla/mux v1.7.3 + github.com/gorilla/mux v1.7.4 github.com/pkg/errors v0.9.1 github.com/quantonganh/httperror v0.0.5 github.com/rs/zerolog v1.23.0 github.com/russross/blackfriday/v2 v2.1.0 github.com/spf13/viper v1.3.2 - github.com/stretchr/testify v1.8.0 - golang.org/x/net v0.17.0 - golang.org/x/sync v0.1.0 + github.com/stretchr/testify v1.8.4 + golang.org/x/net v0.21.0 + golang.org/x/sync v0.6.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/eapache/go-resiliency v1.6.0 // indirect + github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect + github.com/eapache/queue v1.1.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/jcmturner/aescts/v2 v2.0.0 // indirect + github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect + github.com/jcmturner/gofork v1.7.6 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect + github.com/jcmturner/rpc/v2 v2.0.3 // indirect + github.com/klauspost/compress v1.17.7 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + golang.org/x/crypto v0.19.0 // indirect +) + +require ( + github.com/IBM/sarama v1.43.0 github.com/RoaringBitmap/roaring v0.4.23 // indirect github.com/andybalholm/cascadia v1.0.0 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect @@ -41,10 +62,12 @@ require ( github.com/fsnotify/fsnotify v1.4.7 // indirect github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect github.com/go-errors/errors v1.5.1 // indirect - github.com/golang/protobuf v1.5.0 // indirect - github.com/golang/snappy v0.0.1 // indirect + github.com/golang-migrate/migrate/v4 v4.17.0 + github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/magiconair/properties v1.8.0 // indirect + github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/pelletier/go-toml v1.7.0 // indirect @@ -60,8 +83,8 @@ require ( github.com/tinylib/msgp v1.1.0 // indirect github.com/willf/bitset v1.1.10 // indirect go.etcd.io/bbolt v1.3.5 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v2 v2.2.8 // indirect diff --git a/go.sum b/go.sum index 657dc9a..7f05827 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3 github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/Depado/bfchroma v1.3.0 h1:zz14vpvySU6S0CL6yGPr1vkFevQecIt8dJdCsMS2JpM= github.com/Depado/bfchroma v1.3.0/go.mod h1:c0bFk0tFmT+clD3TIGurjWCfD/QV8/EebfM3JGr+98M= +github.com/IBM/sarama v1.43.0 h1:YFFDn8mMI2QL0wOrG0J2sFoVIAFl7hS9JQi2YZsXtJc= +github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP9BM= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= @@ -98,6 +100,12 @@ github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55k github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.6.0 h1:CqGDTLtpwuWKn6Nj3uNUdflaq+/kIPsg0gfNzHton30= +github.com/eapache/go-resiliency v1.6.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= +github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI= @@ -108,6 +116,8 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqL github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= @@ -137,6 +147,8 @@ github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang-migrate/migrate/v4 v4.17.0 h1:rd40H3QXU0AA4IoLllFcEAEo9dYKRHYND2gB4p7xcaU= +github.com/golang-migrate/migrate/v4 v4.17.0/go.mod h1:+Cp2mtLP4/aXDTKb9wmXYitdrNx2HGs45rbWAo6OsKM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -146,12 +158,14 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -166,9 +180,19 @@ github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKp github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/feeds v1.1.1 h1:HwKXxqzcRNg9to+BbvJog4+f3s/xzvtZXICcQGutYfY= github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -182,6 +206,18 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= +github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -198,12 +234,15 @@ github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7 github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -218,8 +257,10 @@ github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -241,7 +282,6 @@ github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5Vgl github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -256,6 +296,8 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv github.com/peterh/liner v1.0.1-0.20171122030339-3681c2a91233/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -280,7 +322,11 @@ github.com/quantonganh/httperror v0.0.5/go.mod h1:brtwPDwG4J80Xzva7ibcdKg/VnLubp github.com/rabbitmq/amqp091-go v1.9.0 h1:qrQtyzB4H8BQgEuJwhmVQqVHB9O4+MNDJCCAcpc3Aoo= github.com/rabbitmq/amqp091-go v1.9.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.23.0 h1:UskrK+saS9P9Y789yNNulYKdARjPZuS35B8gJF2x60g= @@ -319,13 +365,16 @@ github.com/steveyen/gtreap v0.1.0/go.mod h1:kl/5J7XbrOmlIbYIXdRHDDE5QxHqpk0cmkT7 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= @@ -357,9 +406,12 @@ github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FB github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20171031051903-609c9cd26973/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -369,8 +421,13 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -383,17 +440,23 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -413,14 +476,24 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -430,11 +503,13 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200117065230-39095c1d176c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -442,14 +517,16 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= diff --git a/http/event.go b/http/event.go new file mode 100644 index 0000000..5f84f4b --- /dev/null +++ b/http/event.go @@ -0,0 +1,65 @@ +package http + +import ( + "context" + "regexp" +) + +func (s *Server) ProcessActivityStream(ctx context.Context, token string) error { + events, err := s.EventService.Consume(ctx, "page-views") + if err != nil { + return err + } + + if err := s.StatService.ImportIP2LocationDB(token); err != nil { + return err + } + + for e := range events { + country, err := s.StatService.GetCountryFromIP(e.IP) + if err != nil { + e.Country = "Unknown" + } + e.Country = country + e.Browser = getBrowser(e.UserAgent) + e.OS = getOS(e.UserAgent) + if err := s.StatService.Insert(e); err != nil { + s.logger.Error().Err(err).Msg("failed to insert event") + } + } + + return nil +} + +func getBrowser(ua string) string { + var browserRegexp = []*regexp.Regexp{ + regexp.MustCompile(`(?i)(firefox|fxios)\/\S+`), + regexp.MustCompile(`(?i)(chrome|chromium|crios)\/\S+`), + regexp.MustCompile(`(?i)version\/\S+ (safari)\/\S+`), + regexp.MustCompile(`(?i)(opera|opr)\/\S+`), + } + for _, re := range browserRegexp { + matches := re.FindStringSubmatch(ua) + if len(matches) > 1 { + return matches[1] // Return the first matching group which should be the browser name + } + } + return "Unknown" +} + +func getOS(ua string) string { + var osRegexp = []*regexp.Regexp{ + regexp.MustCompile(`(?i)(Windows NT)`), + regexp.MustCompile(`(?i)(Mac OS X)`), + regexp.MustCompile(`(?i)(Linux)`), + regexp.MustCompile(`(?i)(Android)`), + regexp.MustCompile(`(?i)(iOS)`), + } + for _, re := range osRegexp { + matches := re.FindStringSubmatch(ua) + if len(matches) > 1 { + return matches[1] // Return the first matching group which should be the OS + } + } + return "Unknown" +} diff --git a/http/server.go b/http/server.go index fb474e5..fdb80e6 100644 --- a/http/server.go +++ b/http/server.go @@ -2,11 +2,13 @@ package http import ( "context" + "crypto/md5" + "encoding/base64" + "encoding/json" "flag" "fmt" "net" "net/http" - "os" "path" "strings" "time" @@ -30,6 +32,7 @@ const ( // Server represents HTTP server type Server struct { + logger zerolog.Logger ln net.Listener server *http.Server router *mux.Router @@ -37,15 +40,17 @@ type Server struct { Addr string Domain string - PostService blog.PostService - SearchService blog.SearchService - Renderer blog.Renderer - NewsletterService blog.NewsletterService - MessageQueueService blog.MessageQueueService + PostService blog.PostService + SearchService blog.SearchService + Renderer blog.Renderer + NewsletterService blog.NewsletterService + QueueService blog.QueueService + EventService blog.EventService + StatService blog.StatService } // NewServer create new HTTP server -func NewServer(config *blog.Config, posts []*blog.Post) (*Server, error) { +func NewServer(logger zerolog.Logger, config *blog.Config, posts []*blog.Post) (*Server, error) { postService := markdown.NewPostService(posts) indexPath := path.Join(path.Dir(config.Posts.Dir), path.Base(config.Posts.Dir)+".bleve") searchService, err := markdown.NewSearchService(indexPath, posts) @@ -54,6 +59,7 @@ func NewServer(config *blog.Config, posts []*blog.Post) (*Server, error) { } s := &Server{ + logger: logger, server: &http.Server{}, router: mux.NewRouter().StrictSlash(true), PostService: postService, @@ -62,12 +68,9 @@ func NewServer(config *blog.Config, posts []*blog.Post) (*Server, error) { NewsletterService: client.NewNewsletter(config.Newsletter.BaseURL), } - zlog := zerolog.New(os.Stdout).With(). - Timestamp(). - Logger() - s.router.Use(hlog.NewHandler(zlog)) + s.router.Use(hlog.NewHandler(logger)) s.router.Use(hlog.AccessHandler(func(r *http.Request, status, size int, duration time.Duration) { - if !strings.HasPrefix(r.URL.Path, "/static") { + if !strings.HasPrefix(r.URL.Path, "/static") && !hasSuffix(r.URL.Path, []string{"ico", "jpg", "jpeg", "png", "gif"}) { var event *zerolog.Event if 400 <= status && status <= 599 { event = hlog.FromRequest(r).Error() @@ -81,6 +84,36 @@ func NewServer(config *blog.Config, posts []*blog.Post) (*Server, error) { Int("size", size). Dur("duration", duration). Msg("") + + ua := r.Header.Get("User-Agent") + if !strings.Contains(strings.ToLower(ua), "bot") { + ip, err := httperror.GetIP(r) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get IP address") + return + } + userID := generateUserID(ip, ua) + + referer := r.Header.Get("Referer") + if referer == "" { + referer = "Unknown" + } + now := time.Now().Format("2006-01-02T15:04:05Z") + data := map[string]string{ + "ip": ip, + "user_agent": ua, + "url": r.URL.Path, + "referer": referer, + "time": now, + } + jsonData, err := json.Marshal(data) + if err != nil { + s.logger.Error().Err(err).Msg("failed to encode message value") + } + if err := s.EventService.SendMessage("page-views", userID, jsonData); err != nil { + s.logger.Error().Err(err).Msg("error sending message") + } + } } })) s.router.Use(httperror.RealIPHandler("ip")) @@ -120,6 +153,8 @@ func NewServer(config *blog.Config, posts []*blog.Post) (*Server, error) { subRouter.HandleFunc("/confirm", s.Error(s.confirmHandler)) s.newRoute("/unsubscribe", s.unsubscribeHandler) + s.newRoute("/stats", s.statsHandler) + return s, nil } @@ -127,6 +162,14 @@ func (s *Server) newRoute(path string, h appHandler) *mux.Route { return s.router.HandleFunc(path, s.Error(h)) } +func generateUserID(ip, ua string) string { + encodedStr := base64.StdEncoding.EncodeToString([]byte(ip + ua)) + + hash := md5.Sum([]byte(encodedStr)) + + return fmt.Sprintf("%x", hash) +} + // Scheme returns scheme func (s *Server) Scheme() string { if s.UseTLS() { diff --git a/http/server_test.go b/http/server_test.go index 1df6aad..250dfe8 100644 --- a/http/server_test.go +++ b/http/server_test.go @@ -15,6 +15,7 @@ import ( "testing" "github.com/PuerkitoBio/goquery" + "github.com/rs/zerolog" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -70,7 +71,10 @@ Test.`) } fmt.Printf("post: %+v", post) - s, err = NewServer(cfg, []*blog.Post{post}) + logger := zerolog.New(os.Stdout).With(). + Timestamp(). + Logger() + s, err = NewServer(logger, cfg, []*blog.Post{post}) if err != nil { log.Fatal(err) } diff --git a/http/stats.go b/http/stats.go new file mode 100644 index 0000000..b54863a --- /dev/null +++ b/http/stats.go @@ -0,0 +1,48 @@ +package http + +import ( + "net/http" + + "github.com/quantonganh/blog/ui/html" +) + +func (s *Server) statsHandler(w http.ResponseWriter, r *http.Request) error { + top10VisitedPages, err := s.StatService.Top10VisitedPages() + if err != nil { + return err + } + + top10Countries, err := s.StatService.Top10Countries() + if err != nil { + return err + } + + top10Referers, err := s.StatService.Top10Referers(s.Domain) + if err != nil { + return err + } + + top10Browsers, err := s.StatService.Top10Browsers() + if err != nil { + return err + } + + top10OperatingSystems, err := s.StatService.Top10OperatingSystems() + if err != nil { + return err + } + + tmpl := html.Parse(nil, "stats.html") + data := map[string]interface{}{ + "top10VisitedPages": top10VisitedPages, + "top10Countries": top10Countries, + "top10Referers": top10Referers, + "top10Browsers": top10Browsers, + "top10OperatingSystems": top10OperatingSystems, + } + if err := tmpl.ExecuteTemplate(w, "base", data); err != nil { + return err + } + + return nil +} diff --git a/http/webhook.go b/http/webhook.go index e3c8220..590f810 100644 --- a/http/webhook.go +++ b/http/webhook.go @@ -77,7 +77,7 @@ func (s *Server) webhookHandler(config *blog.Config) appHandler { return err } - if err := s.MessageQueueService.Publish("added-posts", data); err != nil { + if err := s.QueueService.Publish("added-posts", data); err != nil { return err } } diff --git a/kafka/event.go b/kafka/event.go new file mode 100644 index 0000000..106dc55 --- /dev/null +++ b/kafka/event.go @@ -0,0 +1,84 @@ +package kafka + +import ( + "context" + "encoding/json" + + "github.com/IBM/sarama" + "github.com/quantonganh/blog" + "github.com/rs/zerolog/log" +) + +type eventService struct { + producer sarama.SyncProducer + consumer sarama.Consumer +} + +func NewEventService(brokerAddr string) (*eventService, error) { + config := sarama.NewConfig() + config.Producer.RequiredAcks = sarama.WaitForAll + config.Producer.Retry.Max = 3 + config.Producer.Return.Successes = true + + producer, err := sarama.NewSyncProducer([]string{brokerAddr}, config) + if err != nil { + return nil, err + } + + consumer, err := sarama.NewConsumer([]string{brokerAddr}, config) + if err != nil { + return nil, err + } + + return &eventService{ + producer: producer, + consumer: consumer, + }, nil +} + +func (es *eventService) SendMessage(topic, key string, value []byte) error { + message := &sarama.ProducerMessage{ + Topic: topic, + Key: sarama.StringEncoder(key), + Value: sarama.ByteEncoder(value), + } + + _, _, err := es.producer.SendMessage(message) + if err != nil { + return err + } + + return nil +} + +func (es *eventService) Consume(ctx context.Context, topic string) (<-chan *blog.Event, error) { + partition := int32(0) + offset := int64(sarama.OffsetNewest) + pc, err := es.consumer.ConsumePartition(topic, partition, offset) + if err != nil { + return nil, err + } + + c := make(chan *blog.Event) + go func() { + defer close(c) + + for { + select { + case <-ctx.Done(): + return + case msg := <-pc.Messages(): + var e *blog.Event + if err := json.Unmarshal(msg.Value, &e); err != nil { + log.Error().Err(err).Msg("failed to unmarshal message value") + return + } + e.UserID = string(msg.Key) + + c <- e + } + } + }() + + return c, nil +} diff --git a/message_queue.go b/queue.go similarity index 62% rename from message_queue.go rename to queue.go index b5fa639..7314300 100644 --- a/message_queue.go +++ b/queue.go @@ -1,5 +1,5 @@ package blog -type MessageQueueService interface { +type QueueService interface { Publish(topic string, message []byte) error } diff --git a/rabbitmq/message_queue.go b/rabbitmq/queue.go similarity index 76% rename from rabbitmq/message_queue.go rename to rabbitmq/queue.go index 152ce74..054a96c 100644 --- a/rabbitmq/message_queue.go +++ b/rabbitmq/queue.go @@ -7,11 +7,11 @@ import ( amqp "github.com/rabbitmq/amqp091-go" ) -type MessageQueueService struct { +type queueService struct { ch *amqp.Channel } -func NewMessageQueueService(url string) (*MessageQueueService, error) { +func NewQueueService(url string) (*queueService, error) { conn, err := amqp.Dial(url) if err != nil { return nil, err @@ -22,12 +22,12 @@ func NewMessageQueueService(url string) (*MessageQueueService, error) { return nil, err } - return &MessageQueueService{ + return &queueService{ ch: ch, }, nil } -func (s *MessageQueueService) Publish(topic string, message []byte) error { +func (s *queueService) Publish(topic string, message []byte) error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() diff --git a/sqlite/migrations/000001_init_schema.down.sql b/sqlite/migrations/000001_init_schema.down.sql new file mode 100644 index 0000000..57b5c0f --- /dev/null +++ b/sqlite/migrations/000001_init_schema.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS activities; \ No newline at end of file diff --git a/sqlite/migrations/000001_init_schema.up.sql b/sqlite/migrations/000001_init_schema.up.sql new file mode 100644 index 0000000..a11a69f --- /dev/null +++ b/sqlite/migrations/000001_init_schema.up.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS activities ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT NOT NULL, + ip_address TEXT NOT NULL, + country TEXT NOT NULL, + browser TEXT NOT NULL, + os TEXT NOT NULL, + referer TEXT NOT NULL, + url TEXT NOT NULL, + time TIMESTAMP +); diff --git a/sqlite/migrations/000002_create_ip2location_table.down.sql b/sqlite/migrations/000002_create_ip2location_table.down.sql new file mode 100644 index 0000000..353239f --- /dev/null +++ b/sqlite/migrations/000002_create_ip2location_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS ip2location; \ No newline at end of file diff --git a/sqlite/migrations/000002_create_ip2location_table.up.sql b/sqlite/migrations/000002_create_ip2location_table.up.sql new file mode 100644 index 0000000..1b5f110 --- /dev/null +++ b/sqlite/migrations/000002_create_ip2location_table.up.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS ip2location ( + start_ip TEXT, + end_ip TEXT, + iso2 TEXT, + country TEXT +); diff --git a/sqlite/sqlite.go b/sqlite/sqlite.go new file mode 100644 index 0000000..7b7c61b --- /dev/null +++ b/sqlite/sqlite.go @@ -0,0 +1,125 @@ +package sqlite + +import ( + "context" + "database/sql" + "embed" + "errors" + "fmt" + "io/fs" + "log" + "sort" + + _ "github.com/mattn/go-sqlite3" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +// DB represents the database connection. +type DB struct { + sqlDB *sql.DB + ctx context.Context + cancel func() + + path string +} + +// NewDB returns new database +func NewDB(path string) *DB { + db := &DB{ + path: path, + } + + db.ctx, db.cancel = context.WithCancel(context.Background()) + + return db +} + +// Open opens new database connection +func (db *DB) Open() (err error) { + if db.path == "" { + return errors.New("path required") + } + + if db.sqlDB != nil { + return nil + } + + if db.sqlDB, err = sql.Open("sqlite3", db.path); err != nil { + return err + } + + if err := db.migrate(); err != nil { + return fmt.Errorf("migrate: %w", err) + } + + return nil +} + +func (db *DB) migrate() error { + if _, err := db.sqlDB.Exec(`CREATE TABLE IF NOT EXISTS migrations (name TEXT PRIMARY KEY);`); err != nil { + return fmt.Errorf("cannot create migrations table: %w", err) + } + + names, err := fs.Glob(migrationsFS, "migrations/*.sql") + if err != nil { + return err + } + sort.Strings(names) + + for _, name := range names { + if err := db.migrateFile(name); err != nil { + return fmt.Errorf("migration error: name=%q, err=%w", name, err) + } + } + + return nil +} + +func (db *DB) migrateFile(name string) error { + tx, err := db.sqlDB.Begin() + if err != nil { + return err + } + defer func() { + _ = tx.Rollback() + }() + + var n int + if err := tx.QueryRow(`SELECT COUNT(*) FROM migrations WHERE name = ?`, name).Scan(&n); err != nil { + return err + } + if n != 0 { + return nil + } + + buf, err := fs.ReadFile(migrationsFS, name) + if err != nil { + return err + } + if _, err := tx.Exec(string(buf)); err != nil { + return err + } + + if _, err := tx.Exec(`INSERT INTO migrations (name) VALUES (?)`, name); err != nil { + return err + } + + return tx.Commit() +} + +// Close closes database connection +func (db *DB) Close() error { + if db.sqlDB == nil { + return nil + } + + db.cancel() + + if err := db.sqlDB.Close(); err != nil { + log.Println("Error closing database:", err) + } + + return nil +} diff --git a/sqlite/stats.go b/sqlite/stats.go new file mode 100644 index 0000000..bbebb8b --- /dev/null +++ b/sqlite/stats.go @@ -0,0 +1,316 @@ +package sqlite + +import ( + "archive/zip" + "database/sql" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + + "github.com/quantonganh/blog" + "github.com/rs/zerolog" +) + +const ( + ip2LocationFileName = "IP2LOCATION-LITE-DB1.CSV" + ip2LocationZipFileName = ip2LocationFileName + ".zip" +) + +type statService struct { + logger zerolog.Logger + db *DB +} + +func NewStatService(logger zerolog.Logger, db *DB) blog.StatService { + return &statService{ + logger: logger, + db: db, + } +} + +// Insert inserts new activity into SQLite +func (s *statService) Insert(e *blog.Event) error { + tx, err := s.db.sqlDB.Begin() + if err != nil { + return fmt.Errorf("failed to start a transaction: %w", err) + } + defer func() { + if err != nil { + if err := tx.Rollback(); err != nil { + s.logger.Error().Err(err).Msg("error rollbacking") + } + return + } + if err := tx.Commit(); err != nil { + s.logger.Error().Err(err).Msg("error committing") + } + }() + + _, err = tx.Exec("INSERT INTO activities (user_id, ip_address, country, browser, os, referer, url, time) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + e.UserID, e.IP, e.Country, e.Browser, e.OS, e.Referer, e.URL, e.Time) + if err != nil { + return fmt.Errorf("failed to insert into activities table: %w", err) + } + + return nil +} + +func (s *statService) ImportIP2LocationDB(token string) error { + imported, err := checkIP2LocationData(s.db.sqlDB) + if err != nil { + s.logger.Error().Err(err).Msg("error checking IP2Location data") + } + if !imported { + if err := downloadIP2LocationDB(token); err != nil { + return err + } + + cmd := exec.Command("sqlite3", "db/stats.db", "-cmd", fmt.Sprintf(".import --csv --skip 1 %s ip2location", ip2LocationFileName)) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error importing CSV data into ip2location table: %s: %w", string(output), err) + } + defer os.Remove(ip2LocationFileName) + + _, err = s.db.sqlDB.Exec("INSERT INTO migrations (name) VALUES ('ip2location')") + if err != nil { + return fmt.Errorf("error marking migration as applied: %w", err) + } + } + + return nil +} + +func checkIP2LocationData(db *sql.DB) (bool, error) { + query := `SELECT EXISTS(SELECT 1 FROM ip2location LIMIT 1);` + + var exists int + err := db.QueryRow(query).Scan(&exists) + if err != nil { + if err == sql.ErrNoRows { + return false, nil + } + return false, err + } + + return exists == 1, nil +} + +func downloadIP2LocationDB(token string) error { + resp, err := http.Get(fmt.Sprintf("https://www.ip2location.com/download/?token=%s&file=DB1LITE", token)) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + file, err := os.Create(ip2LocationZipFileName) + if err != nil { + return fmt.Errorf("error creating ip2Location file: %w", err) + } + defer file.Close() + + _, err = io.Copy(file, resp.Body) + if err != nil { + return err + } + + r, err := zip.OpenReader(ip2LocationZipFileName) + if err != nil { + return err + } + defer r.Close() + + for _, file := range r.File { + if file.Name != ip2LocationFileName { + continue + } + + outFile, err := os.Create(ip2LocationFileName) + if err != nil { + return err + } + defer outFile.Close() + + rc, err := file.Open() + if err != nil { + return err + } + defer rc.Close() + + _, err = io.Copy(outFile, rc) + if err != nil { + return err + } + } + + if err := os.Remove(ip2LocationZipFileName); err != nil { + return err + } + + return nil +} + +func (s *statService) GetCountryFromIP(ip string) (string, error) { + ipInteger, err := ipToInteger(ip) + if err != nil { + return "", err + } + + var country string + if err := s.db.sqlDB.QueryRow(` + SELECT country FROM ip2location WHERE ? BETWEEN start_ip AND end_ip ORDER BY end_ip LIMIT 1 + `, ipInteger).Scan(&country); err != nil { + return "", err + } + if country == "-" { + country = "Unknown" + } + + return country, nil +} + +func (s *statService) Top10VisitedPages() ([]blog.PageStats, error) { + rows, err := s.db.sqlDB.Query("SELECT url, count(id) AS visits FROM activities GROUP BY url ORDER BY count(id) DESC LIMIT 10;") + if err != nil { + return nil, err + } + + var pages []blog.PageStats + for rows.Next() { + var p blog.PageStats + if err := rows.Scan(&p.URL, &p.Visits); err != nil { + return nil, err + } + pages = append(pages, p) + } + + return pages, nil +} + +func (s *statService) Top10Referers(domain string) ([]blog.RefererStats, error) { + rows, err := s.db.sqlDB.Query(` +SELECT + CAST((COUNT(id) * 100.0 / (SELECT COUNT(id) FROM activities)) AS int) AS Share, + referer, + count(id) AS visits +FROM activities +WHERE referer NOT LIKE '%' || ? || '%' +GROUP BY referer +ORDER BY count(id) DESC +LIMIT 10;`, domain) + if err != nil { + return nil, err + } + + var referers []blog.RefererStats + for rows.Next() { + var r blog.RefererStats + if err := rows.Scan(&r.Share, &r.Referer, &r.Visits); err != nil { + return nil, err + } + referers = append(referers, r) + } + + return referers, nil +} + +func (s *statService) Top10Countries() ([]blog.CountryStats, error) { + rows, err := s.db.sqlDB.Query(` +SELECT + CAST((COUNT(id) * 100.0 / (SELECT COUNT(id) FROM activities)) AS int) AS Share, + country, + COUNT(id) AS visits +FROM activities +GROUP BY country +ORDER BY visits DESC +LIMIT 10;`) + if err != nil { + return nil, err + } + + var countries []blog.CountryStats + for rows.Next() { + var c blog.CountryStats + if err := rows.Scan(&c.Share, &c.Country, &c.Visits); err != nil { + return nil, err + } + countries = append(countries, c) + } + + return countries, nil +} + +func (s *statService) Top10Browsers() ([]blog.BrowserStats, error) { + rows, err := s.db.sqlDB.Query(` +SELECT + CAST((COUNT(id) * 100.0 / (SELECT COUNT(id) FROM activities)) AS int) AS Share, + browser, + COUNT(id) AS visits +FROM activities +GROUP BY browser +ORDER BY visits DESC +LIMIT 10;`) + if err != nil { + return nil, err + } + + var browsers []blog.BrowserStats + for rows.Next() { + var b blog.BrowserStats + if err := rows.Scan(&b.Share, &b.Browser, &b.Visits); err != nil { + return nil, err + } + browsers = append(browsers, b) + } + + return browsers, nil +} + +func (s *statService) Top10OperatingSystems() ([]blog.OSStats, error) { + rows, err := s.db.sqlDB.Query(` +SELECT + ROUND((COUNT(id) * 100.0 / (SELECT COUNT(id) FROM activities)), 0) AS Share, + os, + COUNT(id) AS visits +FROM activities +GROUP BY os +ORDER BY visits DESC +LIMIT 10;`) + if err != nil { + return nil, err + } + + var operatingSystems []blog.OSStats + for rows.Next() { + var os blog.OSStats + if err := rows.Scan(&os.Share, &os.OS, &os.Visits); err != nil { + return nil, err + } + operatingSystems = append(operatingSystems, os) + } + + return operatingSystems, nil +} + +func ipToInteger(ipAddr string) (uint32, error) { + parsedIP := net.ParseIP(ipAddr) + if parsedIP == nil { + return 0, fmt.Errorf("invalid IP address: %s", ipAddr) + } + + ipBytes := parsedIP.To4() + if ipBytes == nil { + return 0, fmt.Errorf("not an IPv4 address: %s", ipAddr) + } + + ipInteger := uint32(ipBytes[0])<<24 | uint32(ipBytes[1])<<16 | uint32(ipBytes[2])<<8 | uint32(ipBytes[3]) + + return ipInteger, nil +} diff --git a/stats.go b/stats.go new file mode 100644 index 0000000..d62302f --- /dev/null +++ b/stats.go @@ -0,0 +1,41 @@ +package blog + +type PageStats struct { + URL string `json:"url"` + Visits int `json:"visits"` +} + +type RefererStats struct { + Share string `json:"share"` + Referer string `json:"referrer"` + Visits int `json:"visits"` +} + +type CountryStats struct { + Share string `json:"share"` + Country string `json:"country"` + Visits int `json:"visits"` +} + +type BrowserStats struct { + Share string `json:"share"` + Browser string `json:"browser"` + Visits int `json:"visits"` +} + +type OSStats struct { + Share string `json:"share"` + OS string `json:"os"` + Visits int `json:"visits"` +} + +type StatService interface { + Insert(e *Event) error + ImportIP2LocationDB(token string) error + GetCountryFromIP(ip string) (string, error) + Top10VisitedPages() ([]PageStats, error) + Top10Referers(domain string) ([]RefererStats, error) + Top10Countries() ([]CountryStats, error) + Top10Browsers() ([]BrowserStats, error) + Top10OperatingSystems() ([]OSStats, error) +} diff --git a/ui/html/stats.html b/ui/html/stats.html new file mode 100644 index 0000000..9570d56 --- /dev/null +++ b/ui/html/stats.html @@ -0,0 +1,99 @@ +{{ define "content" }} +

Top 10 Visited Pages

+ + + + + + + + + {{ range $_, $p := .top10VisitedPages }} + + + + + {{ end }} + +
URLVisits
{{ $p.URL }}{{ $p.Visits }}
+ +

Top 10 Referers

+ + + + + + + + + + {{ range $_, $r := .top10Referers }} + + + + + + {{ end }} + +
ShareRefererVisits
{{ $r.Share }}%{{ $r.Referer }}{{ $r.Visits }}
+ +

Top 10 Countries

+ + + + + + + + + + {{ range $_, $c := .top10Countries }} + + + + + + {{ end }} + +
ShareCountryVisits
{{ $c.Share }}%{{ $c.Country }}{{ $c.Visits }}
+ +

Top 10 Browsers

+ + + + + + + + + + {{ range $_, $b := .top10Browsers }} + + + + + + {{ end }} + +
ShareBrowserVisits
{{ $b.Share }}%{{ $b.Browser }}{{ $b.Visits }}
+ +

Top 10 Operating Systems

+ + + + + + + + + + {{ range $_, $os := .top10OperatingSystems }} + + + + + + {{ end }} + +
ShareOperating SystemVisits
{{ $os.Share }}%{{ $os.OS }}{{ $os.Visits }}
+{{ end }} \ No newline at end of file