add auto resolve captcha using headless browser locally

go.mod

@@ -10,6 +10,8 @@ require (
 	github.com/go-chi/chi/v5 v5.0.10
 	github.com/go-chi/cors v1.2.1
 	github.com/go-resty/resty/v2 v2.11.0
+	github.com/go-rod/rod v0.114.7
+	github.com/go-rod/stealth v0.4.9
 	github.com/google/uuid v1.5.0
 	github.com/hashicorp/go-version v1.6.0
 	github.com/huandu/go-clone/generic v1.7.2
@@ -69,6 +71,11 @@ require (
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/wailsapp/go-webview2 v1.0.10 // indirect
 	github.com/wailsapp/mimetype v1.4.1 // indirect
+	github.com/ysmood/fetchup v0.2.4 // indirect
+	github.com/ysmood/goob v0.4.0 // indirect
+	github.com/ysmood/got v0.39.4 // indirect
+	github.com/ysmood/gson v0.7.3 // indirect
+	github.com/ysmood/leakless v0.8.0 // indirect
 	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect
 	golang.org/x/image v0.15.0 // indirect

go.sum

@@ -33,6 +33,11 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
 github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
 github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
 github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
+github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
+github.com/go-rod/rod v0.114.7 h1:h4pimzSOUnw7Eo41zdJA788XsawzHjJMyzCE3BrBww0=
+github.com/go-rod/rod v0.114.7/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
+github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4=
+github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
@@ -151,6 +156,23 @@ github.com/wailsapp/mimetype v1.4.1 h1:pQN9ycO7uo4vsUUuPeHEYoUkLVkaRntMnHJxVwYhw
 github.com/wailsapp/mimetype v1.4.1/go.mod h1:9aV5k31bBOv5z6u+QP8TltzvNGJPmNJD4XlAL3U+j3o=
 github.com/wailsapp/wails/v2 v2.7.1 h1:HAzp2c5ODOzsLC6ZMDVtNOB72ozM7/SJecJPB2Ur+UU=
 github.com/wailsapp/wails/v2 v2.7.1/go.mod h1:oIJVwwso5fdOgprBYWXBBqtx6PaSvxg8/KTQHNGkadc=
+github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
+github.com/ysmood/fetchup v0.2.4 h1:2kfWr/UrdiHg4KYRrxL2Jcrqx4DZYD+OtWu7WPBZl5o=
+github.com/ysmood/fetchup v0.2.4/go.mod h1:hbysoq65PXL0NQeNzUczNYIKpwpkwFL4LXMDEvIQq9A=
+github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
+github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
+github.com/ysmood/gop v0.0.2/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
+github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg=
+github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
+github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM=
+github.com/ysmood/got v0.39.4 h1:8ru7J25Zmf/sMTNYOF2172xVkjQrPMJ3R5d6uymoqL8=
+github.com/ysmood/got v0.39.4/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=
+github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
+github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
+github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
+github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
+github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak=
+github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
 go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=

sydney/captcha.go

@@ -6,23 +6,89 @@ import (
 	"errors"
 	"fmt"
 	"github.com/go-resty/resty/v2"
+	"github.com/go-rod/rod"
+	"github.com/go-rod/rod/lib/launcher"
+	"github.com/go-rod/rod/lib/proto"
+	"github.com/go-rod/stealth"
 	"github.com/google/uuid"
 	"log/slog"
+	"os"
+	"path/filepath"
 	"strings"
 	"sydneyqt/sydney/internal/hex"
 	"sydneyqt/util"
 	"time"
 )
 
-func (o *Sydney) BypassCaptcha(
-	stopCtx context.Context, conversationID string, messageID string,
-) (cookies map[string]string, err error) {
+func (o *Sydney) ResolveCaptcha(stopCtx context.Context) (err error) {
+	defer func() {
+		if err0 := recover(); err0 != nil {
+			slog.Warn("Error resolving captcha", "err", err0)
+			err = err0.(error)
+		}
+	}()
+	iframeID := uuid.New().String()
+	u := launcher.NewUserMode().Context(stopCtx).
+		Leakless(true).
+		UserDataDir(filepath.Join(os.TempDir(), "rod-user-data-"+uuid.New().String())).
+		Set("disable-default-apps").
+		Set("no-first-run").Headless(false).MustLaunch()
+	browser := rod.New().Context(stopCtx).NoDefaultDevice().ControlURL(u).MustConnect()
+	defer browser.MustClose()
+	var cookies []*proto.NetworkCookie
+	for k, v := range o.cookies {
+		cookies = append(cookies, &proto.NetworkCookie{
+			Name:    k,
+			Value:   v,
+			Domain:  ".bing.com",
+			Path:    "/",
+			Expires: proto.TimeSinceEpoch(time.Now().Add(1 * time.Hour).Unix()),
+		})
+	}
+	browser.MustSetCookies(cookies...)
+	page := stealth.MustPage(browser)
+	page.MustNavigate("https://www.bing.com/turing/captcha/challenge?" +
+		"q=&iframeid=local-gen-" + iframeID)
+	router := page.HijackRequests()
+	waitCh := make(chan struct{}, 16)
+	defer close(waitCh)
+	var resCookies map[string]string
+	router.MustAdd("https://www.bing.com/challenge/verify*", func(hijack *rod.Hijack) {
+		hijack.MustLoadResponse()
+		for key, values := range hijack.Response.Headers() {
+			if strings.ToLower(key) != "set-cookie" {
+				continue
+			}
+			var arr []string
+			for _, v := range values {
+				arr = append(arr, strings.Split(v, ";")[0])
+			}
+			resCookies = util.ParseCookiesFromString(strings.Join(arr, "; "))
+		}
+		waitCh <- struct{}{}
+	})
+	go router.Run()
+	defer router.Stop()
+	select {
+	case <-time.Tick(60 * time.Second):
+		return errors.New("timeout verifying challenge token")
+	case <-stopCtx.Done():
+		return stopCtx.Err()
+	case <-waitCh:
+	}
+	slog.Info("Captcha resCookies", "v", resCookies)
+	if err := o.postprocessCaptchaCookies(resCookies); err != nil {
+		return err
+	}
+	return nil
+}
+func (o *Sydney) BypassCaptcha(stopCtx context.Context, conversationID string, messageID string) error {
 	if o.bypassServer == "" {
-		return nil, errors.New("no bypass server specified")
+		return errors.New("no bypass server specified")
 	}
 	hClient, err := util.MakeHTTPClient(o.proxy, 0)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	client := resty.New().SetTransport(hClient.Transport).SetTimeout(60 * time.Second)
 	req := BypassCaptchaRequest{
@@ -35,23 +101,35 @@ func (o *Sydney) BypassCaptcha(
 	slog.Debug("Bypass CAPTCHA request", "v", req)
 	resp, err := client.R().SetContext(stopCtx).SetBody(req).Post(o.bypassServer)
 	if err != nil {
-		return nil, fmt.Errorf("cannot communicate with captcha bypass server: %w", err)
+		return fmt.Errorf("cannot communicate with captcha bypass server: %w", err)
 	}
 	slog.Debug("Bypass captcha response body", "v", string(resp.Body()))
 	var response BypassCaptchaResponse
 	err = json.Unmarshal(resp.Body(), &response)
 	if err != nil {
-		return nil, fmt.Errorf("cannot unmarshal json from captcha bypass server: %w", err)
+		return fmt.Errorf("cannot unmarshal json from captcha bypass server: %w", err)
 	}
 	if response.Error != "" {
-		return nil, errors.New("bypass captcha error: " + response.Error)
+		return errors.New("bypass captcha error: " + response.Error)
+	}
+	cookies := util.ParseCookiesFromString(response.Result.Cookies)
+	if err := o.postprocessCaptchaCookies(cookies); err != nil {
+		return fmt.Errorf("%w; screenshot: "+
+			strings.TrimSuffix(o.bypassServer, "/")+
+			response.Result.ScreenShot, err)
 	}
-	cookies = util.ParseCookiesFromString(response.Result.Cookies)
+	return nil
+}
+func (o *Sydney) postprocessCaptchaCookies(cookies map[string]string) error {
 	if _, ok := cookies["cct"]; !ok {
-		return nil, errors.New("bypass captcha error: no cookie named cct found; screenshot: " +
-			strings.TrimSuffix(o.bypassServer, "/") +
-			response.Result.ScreenShot)
+		return errors.New("captcha cookies not valid: no cookie named cct found")
+	}
+	for k, v := range cookies { // keep the map pointer
+		o.cookies[k] = v
+	}
+	err := util.UpdateCookiesFile(o.cookies)
+	if err != nil {
+		slog.Warn("Cannot update cookies file: ", "err", err)
 	}
-	// new cookies: cct, GC, _C_ETH=1, _C_Auth=
-	return cookies, nil
+	return nil
 }

sydney/stream.go

@@ -50,37 +50,31 @@ func (o *Sydney) AskStream(options AskStreamOptions) (<-chan Message, error) {
 			if msg.Error != nil {
 				slog.Error("Ask stream message", "error", msg.Error)
 				if strings.Contains(msg.Error.Error(), "CAPTCHA") {
-					if o.bypassServer == "" {
-						err0 := errors.New("encountered CAPTCHA challenge; " +
-							"please resolve it manually on Bing's website or configure a Bypass Server")
-						out <- Message{
-							Type:  MessageTypeError,
-							Text:  err0.Error(),
-							Error: err0,
-						}
-						return
-					}
 					if options.disableCaptchaBypass {
 						err0 := errors.New("infinite CAPTCHA detected; " +
-							"please resolve it manually on Bing's website")
+							"please resolve it manually on Bing's website or mobile client")
 						out <- Message{
 							Type:  MessageTypeError,
 							Text:  err0.Error(),
 							Error: err0,
 						}
 						return
 					}
-					slog.Info("Start to bypass the captcha", "server", o.bypassServer)
+					slog.Info("Start to resolve the captcha", "server", o.bypassServer)
 					out <- Message{
 						Type: MessageTypeSolvingCaptcha,
-						Text: "Please wait patiently while we are automatically resolving the CAPTCHA...",
+						Text: "Please wait patiently while we are resolving the CAPTCHA...",
+					}
+					if o.bypassServer == "" {
+						err = o.ResolveCaptcha(options.StopCtx)
+					} else {
+						err = o.BypassCaptcha(options.StopCtx, conversation.ConversationId,
+							options.messageID)
 					}
-					cookies, err := o.BypassCaptcha(options.StopCtx, conversation.ConversationId,
-						options.messageID)
 					if err != nil {
 						if !errors.Is(err, context.Canceled) {
 							err = fmt.Errorf("cannot resolve CAPTCHA automatically; "+
-								"please resolve it manually on Bing's website: %w", err)
+								"please resolve it manually on Bing's website or mobile client: %w", err)
 							out <- Message{
 								Type:  MessageTypeError,
 								Text:  err.Error(),
@@ -89,14 +83,6 @@ func (o *Sydney) AskStream(options AskStreamOptions) (<-chan Message, error) {
 						}
 						return
 					}
-					slog.Info("New Cookie", "v", cookies)
-					for k, v := range cookies { // keep the map pointer
-						o.cookies[k] = v
-					}
-					err = util.UpdateCookiesFile(cookies)
-					if err != nil {
-						slog.Warn("Cannot update cookies file: ", "err", err)
-					}
 					newOptions := options
 					newOptions.disableCaptchaBypass = true
 					newOptions.messageID = ""