feat: webview bypass cloudflare (#172)

* feat: webview bypass cloudflare * feat: desktop webview bypass cloudflare * ua settings * fix: android ua settings * upload
miru-project · Jan 13, 2024 · 5eb25d2 · 5eb25d2
1 parent f85b967
commit 5eb25d2
Show file tree

Hide file tree

Showing 22 changed files with 162 additions and 75 deletions.
diff --git a/assets/i18n/en.json b/assets/i18n/en.json
@@ -102,6 +102,8 @@
     "theme-black": "Black",
     "nsfw": "NSFW",
     "nsfw-subtitle": "Show NSFW content",
+    "webview-ua": "Webview User-Agent",
+    "webview-ua-subtitle": "Modify the User-Agent in the request header for Webviews and extensions.",
     "external-player": "Preferred video player",
     "external-player-subtitle": "Currently, the preferred player is {player}",
     "external-player-builtin": "Built-in",

diff --git a/assets/i18n/zh.json b/assets/i18n/zh.json
@@ -90,6 +90,8 @@
     "theme-black": "黑色",
     "nsfw": "NSFW",
     "nsfw-subtitle": "显示 NSFW 内容",
+    "webview-ua": "Webview User-Agent",
+    "webview-ua-subtitle": "修改 Webview 和扩展的请求头的 User-Agent",
     "external-player": "优先使用的视频播放器",
     "external-player-subtitle": "当前优先使用的是 {player}",
     "external-player-builtin": "内置播放器",

diff --git a/lib/controllers/detail_controller.dart b/lib/controllers/detail_controller.dart
@@ -3,10 +3,10 @@
 import 'dart:convert';
 import 'dart:io';
 
-import 'package:desktop_webview_window/desktop_webview_window.dart';
 import 'package:fluent_ui/fluent_ui.dart' as fluent;
 import 'package:flutter/material.dart';
 import 'package:flutter_i18n/flutter_i18n.dart';
+import 'package:flutter_windows_webview/flutter_windows_webview.dart';
 import 'package:get/get.dart';
 import 'package:miru_app/data/providers/tmdb_provider.dart';
 import 'package:miru_app/models/index.dart';
@@ -20,7 +20,6 @@ import 'package:miru_app/utils/extension.dart';
 import 'package:miru_app/data/services/extension_service.dart';
 import 'package:miru_app/utils/external_player.dart';
 import 'package:miru_app/utils/i18n.dart';
-import 'package:miru_app/utils/miru_directory.dart';
 import 'package:miru_app/utils/miru_storage.dart';
 import 'package:miru_app/views/widgets/messenger.dart';
 
@@ -78,23 +77,31 @@ class DetailPageController extends GetxController {
       fluent.IconButton(
         icon: const Icon(fluent.FluentIcons.pop_expand),
         onPressed: () async {
-          final webview = await WebviewWindow.create(
-            configuration: CreateConfiguration(
-              userDataFolderWindows: await MiruDirectory.getDirectory,
-              title: detail!.title,
+          final webview = FlutterWindowsWebview();
+          await webview.setUA(MiruStorage.getUASetting());
+          webview.launchWebview(
+            extension!.webSite + url,
+            WebviewOptions(
+              onNavigation: (url) {
+                if (Uri.parse(url).host != Uri.parse(extension!.webSite).host) {
+                  return false;
+                }
+                webview.getCookies(url).then((value) async {
+                  if (value.containsKey("cf_clearance")) {
+                    debugPrint("验证通过");
+                  }
+                  runtime.value!.setCookie(
+                    value.entries
+                        .map((e) => '${e.key}=${e.value}')
+                        .toList()
+                        .join(';'),
+                  );
+                });
+
+                return false;
+              },
             ),
           );
-          webview
-            ..addOnUrlRequestCallback((url) async {
-              if (Uri.parse(url).host != Uri.parse(extension!.webSite).host) {
-                return;
-              }
-              final cookies = await webview.evaluateJavaScript(
-                'document.cookie',
-              );
-              runtime.value?.setCookie(cookies!.split("\"")[1]);
-            })
-            ..launch(extension!.webSite + url);
         },
       ),
       fluent.FlyoutTarget(

diff --git a/lib/data/services/extension_service.dart b/lib/data/services/extension_service.dart
@@ -2,6 +2,7 @@ import 'dart:convert';
 import 'package:flutter/services.dart';
 import 'package:html/dom.dart';
 import 'package:html/parser.dart';
+import 'package:miru_app/utils/miru_storage.dart';
 import 'package:xpath_selector_html_parser/xpath_selector_html_parser.dart';
 import 'dart:io';
 
@@ -58,12 +59,16 @@ class ExtensionService {
         "Request: ${args[0]} , ${args[1]}",
       );
       _cuurentRequestUrl = args[0];
+      final headers = args[1]['headers'] ?? {};
+      if (headers['User-Agent'] == null) {
+        headers['User-Agent'] = MiruStorage.getUASetting();
+      }
       return (await _dio.request<String>(
         args[0],
         data: args[1]['data'],
         queryParameters: args[1]['queryParameters'] ?? {},
         options: Options(
-          headers: args[1]['headers'] ?? {},
+          headers: headers,
           method: args[1]['method'] ?? 'get',
         ),
       ))
@@ -386,11 +391,11 @@ class ExtensionService {
   }
 
   // 列出所有的 cookie
-  // Future<String> listCookie() async {
-  //   final cookies =
-  //       await _cookieJar.loadForRequest(Uri.parse(extension.webSite));
-  //   return cookies.map((e) => e.toString()).join(';');
-  // }
+  Future<String> listCookie() async {
+    final cookies =
+        await _cookieJar.loadForRequest(Uri.parse(extension.webSite));
+    return cookies.map((e) => e.toString()).join(';');
+  }
 
   Future<T> _runExtension<T>(Future<T> Function() fun) async {
     try {
@@ -405,6 +410,14 @@ class ExtensionService {
     }
   }
 
+  Future<Map<String, String>> get _defaultHeaders async {
+    return {
+      "Referer": _cuurentRequestUrl,
+      "User-Agent": MiruStorage.getUASetting(),
+      "Cookie": await listCookie(),
+    };
+  }
+
   Future<List<ExtensionListItem>> latest(int page) async {
     return _runExtension(() async {
       final jsResult = await runtime.handlePromise(
@@ -415,9 +428,7 @@ class ExtensionService {
         return ExtensionListItem.fromJson(e);
       }).toList();
       for (var element in result) {
-        element.headers ??= {
-          "Referer": _cuurentRequestUrl,
-        };
+        element.headers ??= await _defaultHeaders;
       }
       return result;
     });
@@ -438,9 +449,7 @@ class ExtensionService {
         return ExtensionListItem.fromJson(e);
       }).toList();
       for (var element in result) {
-        element.headers ??= {
-          "Referer": _cuurentRequestUrl,
-        };
+        element.headers ??= await _defaultHeaders;
       }
       return result;
     });
@@ -477,9 +486,7 @@ class ExtensionService {
       );
       final result =
           ExtensionDetail.fromJson(jsonDecode(jsResult.stringResult));
-      result.headers ??= {
-        "Referer": _cuurentRequestUrl,
-      };
+      result.headers ??= await _defaultHeaders;
       return result;
     });
   }
@@ -494,15 +501,11 @@ class ExtensionService {
       switch (extension.type) {
         case ExtensionType.bangumi:
           final result = ExtensionBangumiWatch.fromJson(data);
-          result.headers ??= {
-            "Referer": _cuurentRequestUrl,
-          };
+          result.headers ??= await _defaultHeaders;
           return result;
         case ExtensionType.manga:
           final result = ExtensionMangaWatch.fromJson(data);
-          result.headers ??= {
-            "Referer": _cuurentRequestUrl,
-          };
+          result.headers ??= await _defaultHeaders;
           return result;
         default:
           return ExtensionFikushonWatch.fromJson(data);

diff --git a/lib/main.dart b/lib/main.dart
@@ -1,11 +1,9 @@
 import 'dart:convert';
 import 'dart:io';
 import 'package:desktop_multi_window/desktop_multi_window.dart';
-import 'package:desktop_webview_window/desktop_webview_window.dart';
 import 'package:fluent_ui/fluent_ui.dart' as fluent;
 import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
-import 'package:flutter_inappwebview/flutter_inappwebview.dart';
 import 'package:get/get.dart';
 import 'package:media_kit/media_kit.dart';
 import 'package:miru_app/controllers/application_controller.dart';
@@ -22,10 +20,6 @@ import 'package:window_manager/window_manager.dart';
 void main(List<String> args) async {
   WidgetsFlutterBinding.ensureInitialized();
 
-  if (runWebViewTitleBarWidget(args)) {
-    return;
-  }
-
   // 多窗口
   if (args.firstOrNull == 'multi_window') {
     final windowId = int.parse(args[1]);
@@ -82,7 +76,6 @@ void main(List<String> args) async {
       statusBarIconBrightness: Brightness.dark,
     );
     SystemChrome.setSystemUIOverlayStyle(style);
-    await AndroidInAppWebViewController.setWebContentsDebuggingEnabled(true);
   }
   runApp(const MainApp());
 }

diff --git a/lib/utils/miru_storage.dart b/lib/utils/miru_storage.dart
@@ -119,6 +119,10 @@ class MiruStorage {
     await _initSetting(SettingKey.arrowRight, 2.0);
     await _initSetting(SettingKey.readingMode, "standard");
     await _initSetting(SettingKey.windowSize, "1280,720");
+    await _initSetting(SettingKey.androidWebviewUA,
+        "Mozilla/5.0 (Linux; Android 13; Android) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.43 Mobile Safari/537.36");
+    await _initSetting(SettingKey.windowsWebviewUA,
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0");
   }
 
   static _initSetting(String key, dynamic value) async {
@@ -134,6 +138,21 @@ class MiruStorage {
   static getSetting(String key) {
     return settings.get(key);
   }
+
+  static getUASetting() {
+    if (Platform.isAndroid) {
+      return settings.get(SettingKey.androidWebviewUA);
+    }
+    return settings.get(SettingKey.windowsWebviewUA);
+  }
+
+  static setUASetting(String value) async {
+    if (Platform.isAndroid) {
+      setSetting(SettingKey.androidWebviewUA, value);
+    } else {
+      setSetting(SettingKey.windowsWebviewUA, value);
+    }
+  }
 }
 
 class SettingKey {
@@ -154,4 +173,6 @@ class SettingKey {
   static String readingMode = 'ReadingMode';
   static String windowSize = 'WindowsSize';
   static String windowPosition = 'WindowsPosition';
+  static String androidWebviewUA = "AndroidWebviewUA";
+  static String windowsWebviewUA = "WindowsWebviewUA";
 }
diff --git a/lib/views/pages/detail_page.dart b/lib/views/pages/detail_page.dart
@@ -255,6 +255,7 @@ class _DetailPageState extends State<DetailPage> {
               alt: c.detail?.title ?? '',
               url: c.backgorund,
               noText: true,
+              headers: c.detail?.headers,
             ),
           ).blur(
             begin: const Offset(10, 10),

diff --git a/lib/views/pages/search/extension_searcher_page.dart b/lib/views/pages/search/extension_searcher_page.dart
@@ -237,6 +237,7 @@ class _ExtensionSearcherPageState extends fluent.State<ExtensionSearcherPage> {
                 package: widget.package,
                 cover: item.cover,
                 update: item.update,
+                headers: item.headers,
               );
             },
           ),
@@ -316,6 +317,7 @@ class _ExtensionSearcherPageState extends fluent.State<ExtensionSearcherPage> {
                         package: widget.package,
                         cover: item.cover,
                         update: item.update,
+                        headers: item.headers,
                       );
                     },
                   )),

diff --git a/lib/views/pages/settings/settings_page.dart b/lib/views/pages/settings/settings_page.dart
@@ -147,6 +147,20 @@ class _SettingsPageState extends State<SettingsPage> {
                 MiruStorage.setSetting(SettingKey.enableNSFW, value);
               },
             ),
+            // UA
+            SettingsIntpuTile(
+              title: 'settings.webview-ua'.i18n,
+              buildSubtitle: () {
+                if (!Platform.isAndroid) {
+                  return 'settings.webview-ua-subtitle'.i18n;
+                }
+                return MiruStorage.getUASetting();
+              },
+              onChanged: (value) {
+                MiruStorage.setUASetting(value);
+              },
+              text: MiruStorage.getUASetting(),
+            ),
           ],
         ),
       ),

diff --git a/lib/views/pages/webview_page.dart b/lib/views/pages/webview_page.dart
@@ -1,6 +1,8 @@
 import 'package:flutter/material.dart';
 import 'package:flutter_inappwebview/flutter_inappwebview.dart';
 import 'package:miru_app/data/services/extension_service.dart';
+import 'package:miru_app/utils/miru_storage.dart';
+import 'package:webview_cookie_manager/webview_cookie_manager.dart';
 
 class WebViewPage extends StatefulWidget {
   const WebViewPage({
@@ -17,26 +19,47 @@ class WebViewPage extends StatefulWidget {
 
 class _WebViewPageState extends State<WebViewPage> {
   late String url = widget.extensionRuntime.extension.webSite + widget.url;
+  final cookieManager = WebviewCookieManager();
+  late Uri loadUrl = Uri.parse(url);
+
+  _setCookie() async {
+    if (loadUrl.host != Uri.parse(url).host) {
+      return;
+    }
+    final cookies = await cookieManager.getCookies(loadUrl.toString());
+    final cookieString =
+        cookies.map((e) => '${e.name}=${e.value}').toList().join(';');
+    debugPrint('$url $cookieString');
+    widget.extensionRuntime.setCookie(
+      cookieString,
+    );
+  }
+
+  @override
+  void dispose() {
+    _setCookie();
+    super.dispose();
+  }
 
   @override
   Widget build(BuildContext context) {
     return Scaffold(
       appBar: AppBar(
-        title: Text(url),
+        title: Text(loadUrl.toString()),
       ),
       body: InAppWebView(
         initialUrlRequest: URLRequest(
           url: Uri.parse(url),
         ),
-        onLoadStop: (controller, url) async {
-          if (url!.host !=
-              Uri.parse(widget.extensionRuntime.extension.webSite).host) {
-            return;
-          }
-          final cookies = await controller.evaluateJavascript(
-            source: 'document.cookie',
-          );
-          widget.extensionRuntime.setCookie(cookies);
+        initialOptions: InAppWebViewGroupOptions(
+          crossPlatform: InAppWebViewOptions(
+            userAgent: MiruStorage.getUASetting(),
+          ),
+        ),
+        onLoadStart: (controller, url) {
+          setState(() {
+            loadUrl = url!;
+          });
         },
       ),
     );

diff --git a/lib/views/widgets/cover.dart b/lib/views/widgets/cover.dart
@@ -8,10 +8,12 @@ class Cover extends StatelessWidget {
     required this.alt,
     this.url,
     this.noText = false,
+    required this.headers,
   });
   final String? url;
   final String alt;
   final bool noText;
+  final Map<String, String>? headers;
 
   @override
   Widget build(BuildContext context) {
@@ -20,6 +22,7 @@ class Cover extends StatelessWidget {
         url!,
         width: double.infinity,
         height: double.infinity,
+        headers: headers,
       );
     }