From fdec9373b601d199bf711fb8b86d6969dca7247a Mon Sep 17 00:00:00 2001 From: Dawid Ciepiela <71898979-sarumaj@users.noreply.github.com> Date: Mon, 20 Jan 2025 00:41:21 +0100 Subject: [PATCH] finalize wildcard domain proxying --- .github/workflows/deploy.yml | 1 + .vscode/settings.json | 4 + cmd/kagi/main.go | 18 +- go.mod | 7 +- go.sum | 24 ++- pkg/api/proxy.go | 360 +++++++++++++++++++++++++++++++---- pkg/api/rate.go | 3 +- pkg/api/template.go | 31 ++- pkg/api/templates/error.html | 90 +++++++++ pkg/api/templates/proxy.js | 102 ++++++++++ 10 files changed, 586 insertions(+), 54 deletions(-) create mode 100644 pkg/api/templates/error.html create mode 100644 pkg/api/templates/proxy.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3010b7b..74f31bc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -77,6 +77,7 @@ jobs: HD_KAGI_SESSION_SECRET: ${{ secrets.KAGI_SESSION_SECRET }} HD_KAGI_SESSION_USER: ${{ secrets.KAGI_SESSION_USER }} HD_KAGI_SESSION_PASS: ${{ secrets.KAGI_SESSION_PASS }} + HD_PROXY_HOST: ${{ secrets.KAGI_PROXY_HOST }} - name: Update code documentation run: curl -fsSL https://proxy.golang.org/${{ steps.export_module_name.outputs.MODULE_NAME }}/@v/${{ github.ref_name }}.info diff --git a/.vscode/settings.json b/.vscode/settings.json index 4e7994b..eab4a13 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,7 @@ { "cSpell.words": [ "akhileshns", + "andybalholm", "buildvcs", "bytedance", "CCPA", @@ -22,6 +23,7 @@ "gopls", "healthcheck", "HITPOINT", + "icholy", "isatty", "justlogin", "kagi", @@ -30,8 +32,10 @@ "leodido", "mattn", "Menlo", + "MIMEHTML", "multierr", "netgo", + "NXDOMAIN", "osusergo", "Roboto", "rollbackonhealthcheckfailed", diff --git a/cmd/kagi/main.go b/cmd/kagi/main.go index 3cda706..aa1c0bc 100644 --- a/cmd/kagi/main.go +++ b/cmd/kagi/main.go @@ -12,10 +12,11 @@ import ( "github.com/gin-contrib/sessions/cookie" ginzap "github.com/gin-contrib/zap" "github.com/gin-gonic/gin" - "github.com/sarumaj/kagi/pkg/api" - "github.com/sarumaj/kagi/pkg/common" "go.uber.org/zap" "go.uber.org/zap/zapcore" + + "github.com/sarumaj/kagi/pkg/api" + "github.com/sarumaj/kagi/pkg/common" ) var ( @@ -28,6 +29,7 @@ var ( sessionSecret = flag.String("session-secret", common.Getenv[string]("KAGI_SESSION_SECRET", "test"), "test") sessionUser = flag.String("session-user", common.Getenv[string]("KAGI_SESSION_USER", "user"), "user") sessionPass = flag.String("session-pass", common.Getenv[string]("KAGI_SESSION_PASS", "pass"), "pass") + proxyHost = flag.String("proxy-host", common.Getenv[string]("KAGI_PROXY_HOST", "kagi.com"), "kagi.com") ) func main() { @@ -55,11 +57,12 @@ func main() { // Create a new cookie store. store := cookie.NewStore([]byte(*sessionSecret)) store.Options(sessions.Options{ + Domain: *proxyHost, // Required to support wildcard subdomains Path: "/", MaxAge: 3600 * 24, // 1 day Secure: true, HttpOnly: true, - SameSite: http.SameSiteStrictMode, + SameSite: http.SameSiteLaxMode, }) e.Use(sessions.Sessions("proxy_session", store)) @@ -77,7 +80,7 @@ func main() { })) // Set the HTML templates. - e.SetHTMLTemplate(api.Templates()) + e.SetHTMLTemplate(api.HTMLTemplates()) // Use the rate limiting middleware. e.Use(api.Rate(*limitRPS, int(*limitBurst))) @@ -100,8 +103,13 @@ func main() { router.GET("/settings", api.HandleLogout()) // Add a proxy route. - router.NoRoute(api.BasicAuth([]string{"/favicon.ico"}), api.ProxyPass(*sessionToken)) + router.NoRoute(api.BasicAuth([]string{"/favicon.ico"}), api.ProxyPass(map[string]string{ + *proxyHost: "kagi.com", + "translate." + *proxyHost: "translate.kagi.com", + "assets." + *proxyHost: "assets.kagi.com", + }, *sessionToken)) + // Start the server. if err := router.Run(fmt.Sprintf(":%d", *port)); err != nil { common.Logger.Fatal("Unexpected server error", zap.Error(err)) } diff --git a/go.mod b/go.mod index 1792f38..f052d39 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,14 @@ module github.com/sarumaj/kagi go 1.23.5 require ( + github.com/andybalholm/brotli v1.1.1 github.com/gin-contrib/sessions v1.0.2 github.com/gin-contrib/zap v1.1.4 github.com/gin-gonic/gin v1.10.0 + github.com/klauspost/compress v1.17.7 + github.com/utrack/gin-csrf v0.0.0-20190424104817-40fb8d2c8fca go.uber.org/zap v1.27.0 + golang.org/x/net v0.34.0 golang.org/x/time v0.9.0 ) @@ -24,6 +28,7 @@ require ( github.com/gorilla/context v1.1.2 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/sessions v1.2.2 // indirect + github.com/icholy/replace v0.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -33,11 +38,9 @@ require ( github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - github.com/utrack/gin-csrf v0.0.0-20190424104817-40fb8d2c8fca // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.13.0 // indirect golang.org/x/crypto v0.32.0 // indirect - golang.org/x/net v0.34.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.36.3 // indirect diff --git a/go.sum b/go.sum index aaffbfe..925ff16 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff/go.mod h1:+RTT1BOk5P97fT2CiHkbFQwkK3mjsFAP6zCYV2aXtjw= github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/bradleypeabody/gorilla-sessions-memcache v0.0.0-20181103040241-659414f458e1/go.mod h1:dkChI7Tbtx7H1Tj7TqGSZMOeGpMP5gLHtjroHd4agiI= @@ -41,6 +43,7 @@ github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -56,17 +59,24 @@ github.com/gorilla/sessions v1.1.1/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= github.com/gorilla/sessions v1.2.2 h1:lqzMYz6bOfvn2WriPUjNByzeXIlVzURcPmgMczkmTjY= github.com/gorilla/sessions v1.2.2/go.mod h1:ePLdVu+jbEgHH+KWw8I1z2wqd0BAdAQh/8LRvBeoNcQ= +github.com/icholy/replace v0.6.0 h1:EBiD2pGqZIOJAbEaf/5GVRaD/Pmbb4n+K3LrBdXd4dw= +github.com/icholy/replace v0.6.0/go.mod h1:zzi8pxElj2t/5wHHHYmH45D+KxytX/t4w3ClY5nlK+g= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kidstuff/mongostore v0.0.0-20181113001930-e650cd85ee4b/go.mod h1:g2nVr8KZVXJSS97Jo8pJ0jgq29P6H7dG0oplUA86MQw= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -81,9 +91,11 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quasoft/memstore v0.0.0-20180925164028-84a050167438/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -104,6 +116,8 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/utrack/gin-csrf v0.0.0-20190424104817-40fb8d2c8fca h1:lpvAjPK+PcxnbcB8H7axIb4fMNwjX9bE4DzwPjGg8aE= github.com/utrack/gin-csrf v0.0.0-20190424104817-40fb8d2c8fca/go.mod h1:XXKxNbpoLihvvT7orUZbs/iZayg1n4ip7iJakJPAwA8= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -112,25 +126,32 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/arch v0.13.0 h1:KCkqVVV1kGg0X87TFysjCJ8MxtZEIU4Ja/yXGeoECdA= golang.org/x/arch v0.13.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -141,4 +162,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/pkg/api/proxy.go b/pkg/api/proxy.go index 9809f58..e47bee9 100644 --- a/pkg/api/proxy.go +++ b/pkg/api/proxy.go @@ -1,69 +1,353 @@ package api import ( + "bytes" + "compress/flate" + "compress/gzip" + "context" + "crypto/sha256" "crypto/tls" "crypto/x509" + "encoding/base64" + "fmt" + "io" + "log" "net/http" "net/http/httputil" + "strconv" + "strings" "time" + "golang.org/x/net/html" + + "github.com/andybalholm/brotli" "github.com/gin-gonic/gin" - "github.com/sarumaj/kagi/pkg/common" + "github.com/icholy/replace" + "github.com/klauspost/compress/zstd" "go.uber.org/zap" + + "github.com/sarumaj/kagi/pkg/common" ) -// ProxyPass is a middleware that proxies requests to the kagi.com server. -func ProxyPass(sessionToken string) gin.HandlerFunc { - rootCAs, err := x509.SystemCertPool() - if err != nil { - common.Logger.Warn("Failed to load system root CAs", zap.Error(err)) - rootCAs = x509.NewCertPool() +// sessionProxy is a reverse proxy that injects a session token into requests +// and modifies the response to include a script that proxies requests to the +// target hosts. +type sessionProxy struct { + SessionToken string + TargetHosts targetHostConfig + *httputil.ReverseProxy +} + +// Director is a function that modifies the request before it is sent. +// It injects a session token into the request if it is not already present. +// It also modifies the request to use the target host specified in the +// targetHostConfig. +func (p sessionProxy) Director(req *http.Request) { + req.URL.Scheme = "https" + targetHost := p.TargetHosts.Get(req.Host, "kagi.com") + req.URL.Host, req.Host = targetHost, targetHost + + common.Logger.Debug("Proxying request", zap.String("url", req.URL.String())) + + switch cookie, err := req.Cookie("kagi_session"); { + + case len(req.URL.Query().Get("token")) > 0: + common.Logger.Debug("Session token found in query string", zap.String("token", req.URL.Query().Get("token"))) + return // Skip session cookie injection if token is present in query string + + case err == nil && + cookie != nil && + len(cookie.Value) > 0 && + (cookie.Domain == targetHost || cookie.Domain == "."+targetHost || len(cookie.Domain) == 0): + + common.Logger.Debug("Session cookie found in request", zap.Reflect("cookie", cookie)) + return // Skip session cookie injection if session cookie is already present + + default: + common.Logger.Debug("Session cookie not found in request") + + } + + req.AddCookie(&http.Cookie{ + Name: "kagi_session", + Value: p.SessionToken, + Expires: time.Now().Add(time.Hour), + Path: "/", + Domain: targetHost, + Secure: true, + HttpOnly: true, + SameSite: http.SameSiteLaxMode, + }) + + common.Logger.Debug("Session token added to request", zap.String("sessionToken", p.SessionToken), zap.Reflect("cookies", req.Cookies())) +} + +// ErrorHandler is a function that handles errors that occur during the proxying process. +func (p sessionProxy) ErrorHandler(w http.ResponseWriter, r *http.Request, err error) { + if err == nil { + return + } + + // Check if the client has already disconnected + if err == context.Canceled { + common.Logger.Warn("Client disconnected", zap.String("url", r.URL.String()), zap.Error(err)) + return + } + + common.Logger.Error("Proxy error", zap.Error(err), zap.String("url", r.URL.String())) + + if len(w.Header().Get("Content-Type")) > 0 { + common.Logger.Warn("Headers already sent, cannot modify response") + return + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Retry-After", "30") + w.WriteHeader(http.StatusServiceUnavailable) + + if err := HTMLTemplates().ExecuteTemplate(w, "error.html", map[string]any{ + "error": html.EscapeString(err.Error()), + }); err != nil { + common.Logger.Error("Failed to execute error template", zap.Error(err)) + } +} + +// modifyCSP modifies the Content-Security-Policy header to allow the proxy script. +// Furthermore, it whitelists the target hosts specified in the targetHostConfig. +func (p sessionProxy) modifyCSP(csp string, scripts ...[]byte) string { + // Generate hashes for the inline scripts + hashes := make([]string, 0, len(scripts)) + for _, script := range scripts { + hasher := sha256.New() + _, _ = hasher.Write(script) + hashes = append(hashes, "'sha256-"+base64.StdEncoding.EncodeToString(hasher.Sum(nil))+"'") + } + + directives := strings.Split(csp, ";") + modified := make([]string, 0, len(directives)) + + // Helper to extend the directive values with the proxy domains + extendDirective := func(directiveValues []string) (values []string) { + for _, value := range directiveValues { + values = append(values, value) + + // If value matches any of our target hosts, add corresponding proxy domain + for proxyDomain, targetHost := range p.TargetHosts { + if strings.Contains(value, targetHost) { + if newValue := strings.ReplaceAll(value, targetHost, proxyDomain); newValue != value { + values = append(values, newValue) + } + } + } + } + + return + } + + // Check if the script-src directive is present + scriptSrcFound := false + for _, directive := range directives { + directive = strings.TrimSpace(directive) + if directive == "" { + continue + } + + parts := strings.Fields(directive) + if len(parts) < 1 { + continue + } + + directiveName := parts[0] + directiveValues := parts[1:] + + switch directiveName { + case "script-src", "script-src-elem": + scriptSrcFound = true + newValues := append(hashes, extendDirective(directiveValues)...) + if !strings.Contains(strings.Join(newValues, " "), "'unsafe-inline'") { + newValues = append(newValues, "'unsafe-inline'") + } + directiveValues = newValues + + case "default-src", "style-src", "img-src", "connect-src", "font-src", + "frame-src", "media-src", "object-src", "manifest-src", "frame-ancestors": + directiveValues = extendDirective(directiveValues) + + } + + // Rebuild the directive + if len(directiveValues) > 0 { + modified = append(modified, directiveName+" "+strings.Join(directiveValues, " ")) + } else { + modified = append(modified, directiveName) + } + } + + if !scriptSrcFound { + modified = append(modified, fmt.Sprintf("script-src %s 'unsafe-inline'", strings.Join(hashes, " "))) + } + + return strings.Join(modified, "; ") +} + +// ModifyResponse is a function that modifies the response before it is sent. +// It injects a script that proxies requests to the target hosts specified in +// the targetHostConfig. +func (p sessionProxy) ModifyResponse(resp *http.Response) error { + var script bytes.Buffer + if err := TextTemplates().ExecuteTemplate(&script, "proxy.js", map[string]any{ + "host_map": p.TargetHosts, + }); err != nil { + return err + } + + if csp := resp.Header.Get("Content-Security-Policy"); len(csp) > 0 { + resp.Header.Set("Content-Security-Policy", p.modifyCSP(csp, script.Bytes())) } - transport := &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: rootCAs, - ServerName: "kagi.com", - }, + if contentType := resp.Header.Get("Content-Type"); resp.Body == nil || !strings.Contains(contentType, gin.MIMEHTML) { + return nil } - director := func(req *http.Request) { - req.URL.Scheme = "https" - req.URL.Host = "kagi.com" - req.Host = "kagi.com" + contentEncoding := strings.ToLower(resp.Header.Get("Content-Encoding")) - common.Logger.Debug("Proxying request", zap.String("url", req.URL.String())) + // Setup decompression + var reader io.Reader = resp.Body + switch contentEncoding { + case "gzip": + common.Logger.Debug("Decoding gzip response") + gzReader, err := gzip.NewReader(resp.Body) + if err != nil { + common.Logger.Error("Failed to create gzip reader", zap.Error(err)) + return err + } + defer gzReader.Close() + reader = gzReader + + case "deflate": + common.Logger.Debug("Decoding deflate response") + reader = flate.NewReader(resp.Body) + defer reader.(io.Closer).Close() + + case "br": + common.Logger.Debug("Decoding brotli response") + reader = brotli.NewReader(resp.Body) - sessionEstablished := len(req.URL.Query().Get("token")) > 0 - if !sessionEstablished { - cookie, err := req.Cookie("kagi_session") - sessionEstablished = err == nil && cookie != nil && len(cookie.Value) > 0 && cookie.Expires.After(time.Now()) + case "zstd": + common.Logger.Debug("Decoding zstd response") + zstdReader, err := zstd.NewReader(resp.Body) + if err != nil { + common.Logger.Error("Failed to create zstd reader", zap.Error(err)) + return err } + defer zstdReader.Close() + reader = zstdReader + + case "", "identity": + // No transformation needed + + default: + common.Logger.Warn("Unknown content encoding", zap.String("encoding", contentEncoding)) + return fmt.Errorf("unknown content encoding: %s", contentEncoding) + } - common.Logger.Debug("Session established", zap.Bool("sessionEstablished", sessionEstablished)) + // Inject the proxy script into the head tag + reader = replace.Chain(reader, replace.String(``, "\n\t\t")) - if sessionEstablished { - return + // Compress the modified content + var compressedContent bytes.Buffer + switch contentEncoding { + case "gzip": + gzWriter := gzip.NewWriter(&compressedContent) + if _, err := io.Copy(gzWriter, reader); err != nil { + return err } + _ = gzWriter.Close() - req.AddCookie(&http.Cookie{ - Name: "kagi_session", - Value: sessionToken, - Expires: time.Now().Add(time.Hour), - Path: "/", - Domain: "kagi.com", - Secure: true, - HttpOnly: true, - SameSite: http.SameSiteLaxMode, - }) + case "deflate": + flateWriter, err := flate.NewWriter(&compressedContent, flate.BestSpeed) + if err != nil { + return err + } + if _, err := io.Copy(flateWriter, reader); err != nil { + return err + } + _ = flateWriter.Close() + + case "br": + brWriter := brotli.NewWriter(&compressedContent) + if _, err := io.Copy(brWriter, reader); err != nil { + return err + } + _ = brWriter.Close() - common.Logger.Debug("Session token added to request", zap.String("sessionToken", sessionToken), zap.Reflect("cookies", req.Cookies())) + case "zstd": + zstdWriter, err := zstd.NewWriter(&compressedContent) + if err != nil { + return err + } + if _, err := io.Copy(zstdWriter, reader); err != nil { + return err + } + _ = zstdWriter.Close() + + default: + if _, err := io.Copy(&compressedContent, reader); err != nil { + return err + } + + } + + resp.Body = io.NopCloser(&compressedContent) + resp.ContentLength = int64(compressedContent.Len()) + resp.Header.Set("Content-Length", strconv.Itoa(compressedContent.Len())) + resp.TransferEncoding = nil // Remove chunked encoding since we know the content length + + return nil +} + +// targetHostConfig is a map that maps proxy host names to target hosts. +type targetHostConfig map[string]string + +// Get returns the target host for the given proxy host. +func (t targetHostConfig) Get(host, def string) string { + if targetHost, ok := t[host]; ok { + return targetHost + } + + for hostPort, targetHost := range t { + if strings.Split(hostPort, ":")[0] == host { + return targetHost + } + } + + common.Logger.Warn("Target host not found", zap.String("host", host), zap.Reflect("targetHosts", t)) + + if len(def) > 0 { + return def + } + + return "NXDOMAIN" +} + +// ProxyPass is a middleware that proxies requests to the kagi.com and *.kagi.com servers. +func ProxyPass(targetHostConfig map[string]string, sessionToken string) gin.HandlerFunc { + rootCAs, err := x509.SystemCertPool() + if err != nil { + common.Logger.Warn("Failed to load system root CAs", zap.Error(err)) + rootCAs = x509.NewCertPool() } - proxy := &httputil.ReverseProxy{ - Transport: transport, - Director: director, + proxy := &sessionProxy{ + SessionToken: sessionToken, + ReverseProxy: &httputil.ReverseProxy{}, + TargetHosts: targetHostConfig, } + proxy.Transport = &http.Transport{TLSClientConfig: &tls.Config{RootCAs: rootCAs}} + proxy.ErrorLog = log.New(io.Discard, "", 0) + proxy.ReverseProxy.Director = proxy.Director + proxy.ReverseProxy.ModifyResponse = proxy.ModifyResponse + proxy.ReverseProxy.ErrorHandler = proxy.ErrorHandler return func(ctx *gin.Context) { proxy.ServeHTTP(ctx.Writer, ctx.Request) } } diff --git a/pkg/api/rate.go b/pkg/api/rate.go index 6dc30a5..1dbdbaa 100644 --- a/pkg/api/rate.go +++ b/pkg/api/rate.go @@ -4,9 +4,10 @@ import ( "net/http" "github.com/gin-gonic/gin" - "github.com/sarumaj/kagi/pkg/common" "go.uber.org/zap" "golang.org/x/time/rate" + + "github.com/sarumaj/kagi/pkg/common" ) // Rate is a middleware that limits the request rate. diff --git a/pkg/api/template.go b/pkg/api/template.go index 91582fb..f02589f 100644 --- a/pkg/api/template.go +++ b/pkg/api/template.go @@ -1,14 +1,31 @@ package api import ( - _ "embed" - "html/template" + "embed" + "encoding/json" + "io/fs" + + htmlTemplate "html/template" + textTemplate "text/template" ) -//go:embed templates/login.html -var loginTemplate string +//go:embed templates/*.html templates/*.js +var templatesFS embed.FS + +// HTMLTemplates returns the HTML templates. +func HTMLTemplates() *htmlTemplate.Template { + subFS, _ := fs.Sub(templatesFS, "templates") + + return htmlTemplate.Must(htmlTemplate.New("").ParseFS(subFS, "*.html")) +} -// Templates returns the HTML templates. -func Templates() *template.Template { - return template.Must(template.New("login.html").Parse(loginTemplate)) +// TextTemplates returns the text templates. +func TextTemplates() *textTemplate.Template { + subFS, _ := fs.Sub(templatesFS, "templates") + return textTemplate.Must(textTemplate.New("").Funcs(textTemplate.FuncMap{ + "json": func(v any) string { + out, _ := json.Marshal(v) + return string(out) + }, + }).ParseFS(subFS, "*.js")) } diff --git a/pkg/api/templates/error.html b/pkg/api/templates/error.html new file mode 100644 index 0000000..9ab9ebd --- /dev/null +++ b/pkg/api/templates/error.html @@ -0,0 +1,90 @@ + + + + + + 503 Service Unavailable + + + +
+

Service Temporarily Unavailable

+

+ We're sorry, but the service you're trying to reach is currently + unavailable. +

+

Please try again in a few moments.

+
{{ .error }}
+ Retry +
Error 503
+
+ + diff --git a/pkg/api/templates/proxy.js b/pkg/api/templates/proxy.js new file mode 100644 index 0000000..74234ad --- /dev/null +++ b/pkg/api/templates/proxy.js @@ -0,0 +1,102 @@ +(function () { + const originalMap = JSON.parse(`{{ json .host_map}}`); + const hostMap = Object.fromEntries( + Object.entries(originalMap).map(([proxy, target]) => [target, proxy]) + ); + + function replaceHost(url) { + if (!url) return url; + try { + const urlObj = new URL(url, window.location.href); + for (const [targetHost, proxyDomain] of Object.entries(hostMap)) { + if ( + urlObj.host === targetHost || + urlObj.host.endsWith("." + targetHost) + ) { + urlObj.host = urlObj.host.replace(targetHost, proxyDomain); + return urlObj.toString(); + } + } + } catch (e) { + console.debug("URL parsing failed:", e); + } + return url; + } + + function processNode(node) { + // Handle attributes + if (node.nodeType === Node.ELEMENT_NODE) { + ["href", "src", "action", "data-url"].forEach((attr) => { + if (node.hasAttribute(attr)) { + const newValue = replaceHost(node.getAttribute(attr)); + if (newValue !== node.getAttribute(attr)) { + node.setAttribute(attr, newValue); + } + } + }); + } + + // Handle inline scripts + if (node.tagName === "SCRIPT" && !node.src) { + const originalText = node.textContent; + let modifiedText = originalText; + for (const [targetHost, proxyDomain] of Object.entries(hostMap)) { + const regex = new RegExp( + targetHost.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), + "g" + ); + modifiedText = modifiedText.replace(regex, proxyDomain); + } + if (modifiedText !== originalText) { + const newScript = document.createElement("script"); + newScript.textContent = modifiedText; + node.parentNode.replaceChild(newScript, node); + } + } + } + + // Create a MutationObserver to handle dynamically added content + const observer = new MutationObserver((mutations) => { + mutations.forEach((mutation) => { + mutation.addedNodes.forEach((node) => { + if (node.nodeType === Node.ELEMENT_NODE) { + processNode(node); + node.querySelectorAll("*").forEach(processNode); + } + }); + }); + }); + + // Process existing content + document.querySelectorAll("*").forEach(processNode); + + // Observe future changes + observer.observe(document.documentElement, { + childList: true, + subtree: true, + }); + + // Handle dynamic XHR/Fetch requests + const originalFetch = window.fetch; + window.fetch = function (input, init) { + if (typeof input === "string") { + input = replaceHost(input); + } else if (input instanceof Request) { + input = new Request(replaceHost(input.url), input); + } + return originalFetch.call(this, input, init); + }; + + const originalXHROpen = XMLHttpRequest.prototype.open; + XMLHttpRequest.prototype.open = function (method, url, ...args) { + url = replaceHost(url); + return originalXHROpen.call(this, method, url, ...args); + }; + + // Handle WebSocket connections + const originalWebSocket = window.WebSocket; + window.WebSocket = function (url, protocols) { + url = replaceHost(url); + return new originalWebSocket(url, protocols); + }; +})();