From e9f5f25908ec440cf495f04bcc314446d9d7fbfa Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 21 May 2025 11:37:55 +0000 Subject: [PATCH 1/2] set `WindowsSelectorEventLoopPolicy` only for curl-impersonate without `playwright` --- .../{{cookiecutter.__package_name}}/__main__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/crawlee/project_template/{{cookiecutter.project_name}}/{{cookiecutter.__package_name}}/__main__.py b/src/crawlee/project_template/{{cookiecutter.project_name}}/{{cookiecutter.__package_name}}/__main__.py index 692f2fdd47..32363b1c5f 100644 --- a/src/crawlee/project_template/{{cookiecutter.project_name}}/{{cookiecutter.__package_name}}/__main__.py +++ b/src/crawlee/project_template/{{cookiecutter.project_name}}/{{cookiecutter.__package_name}}/__main__.py @@ -1,12 +1,15 @@ import asyncio +# % if cookiecutter.http_client == 'curl-impersonate' and 'playwright' not in cookiecutter.crawler_type import platform - +# % endif +{{ '' }} from .main import main - if __name__ == '__main__': + # % if cookiecutter.http_client == 'curl-impersonate' and 'playwright' not in cookiecutter.crawler_type if platform.system() == 'Windows': - # This mitigates a warning raised by curl-cffi. If you do not need to use curl-impersonate, you may remove this. + # This mitigates a warning raised by curl-cffi. asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - + # % endif +{{ '' }} asyncio.run(main()) From 56e1e34ba9025f17f2bbff5570a57579240d8a48 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 21 May 2025 12:20:46 +0000 Subject: [PATCH 2/2] add warning --- .../{{cookiecutter.__package_name}}/__main__.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/crawlee/project_template/{{cookiecutter.project_name}}/{{cookiecutter.__package_name}}/__main__.py b/src/crawlee/project_template/{{cookiecutter.project_name}}/{{cookiecutter.__package_name}}/__main__.py index 32363b1c5f..52217c2624 100644 --- a/src/crawlee/project_template/{{cookiecutter.project_name}}/{{cookiecutter.__package_name}}/__main__.py +++ b/src/crawlee/project_template/{{cookiecutter.project_name}}/{{cookiecutter.__package_name}}/__main__.py @@ -1,15 +1,27 @@ import asyncio -# % if cookiecutter.http_client == 'curl-impersonate' and 'playwright' not in cookiecutter.crawler_type +# % if cookiecutter.http_client == 'curl-impersonate' import platform +# % if 'playwright' in cookiecutter.crawler_type +import warnings +# % endif # % endif {{ '' }} from .main import main if __name__ == '__main__': - # % if cookiecutter.http_client == 'curl-impersonate' and 'playwright' not in cookiecutter.crawler_type + # % if cookiecutter.http_client == 'curl-impersonate' if platform.system() == 'Windows': # This mitigates a warning raised by curl-cffi. + # % if 'playwright' in cookiecutter.crawler_type + warnings.warn( + message=('curl-cffi suggests using WindowsSelectorEventLoopPolicy, but this conflicts with Playwright. ' + 'Ignore the curl-cffi warning.'), + category=UserWarning, + stacklevel=2, + ) + # % else asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + # % endif # % endif {{ '' }} asyncio.run(main())