diff --git a/src/crawlee/_cli.py b/src/crawlee/_cli.py index 395a086ff9..6a71478c83 100644 --- a/src/crawlee/_cli.py +++ b/src/crawlee/_cli.py @@ -226,10 +226,10 @@ def create( f'activate the virtual environment in ".venv" ("source .venv/bin/activate") ' f'and run your project using "python -m {package_name}".' ) - elif package_manager == 'poetry': + elif package_manager in {'poetry', 'uv'}: typer.echo( f'To run it, navigate to the directory: "cd {project_name}", ' - f'and run it using "poetry run python -m {package_name}".' + f'and run it using "{package_manager} run python -m {package_name}".' ) typer.echo(f'See the "{project_name}/README.md" for more information.') diff --git a/src/crawlee/project_template/cookiecutter.json b/src/crawlee/project_template/cookiecutter.json index 8afc215207..a32f376736 100644 --- a/src/crawlee/project_template/cookiecutter.json +++ b/src/crawlee/project_template/cookiecutter.json @@ -4,7 +4,7 @@ "crawler_type": ["beautifulsoup", "parsel", "playwright", "playwright-camoufox"], "__crawler_type": "{{ cookiecutter.crawler_type|lower|replace('-', '_') }}", "http_client": ["httpx", "curl-impersonate"], - "package_manager": ["poetry", "pip", "manual"], + "package_manager": ["poetry", "pip", "uv", "manual"], "enable_apify_integration": false, "start_url": "https://crawlee.dev", "_jinja2_env_vars": { diff --git a/src/crawlee/project_template/hooks/post_gen_project.py b/src/crawlee/project_template/hooks/post_gen_project.py index a62f99031b..877f234b6e 100644 --- a/src/crawlee/project_template/hooks/post_gen_project.py +++ b/src/crawlee/project_template/hooks/post_gen_project.py @@ -4,13 +4,21 @@ Path('_pyproject.toml').rename('pyproject.toml') -# % if cookiecutter.package_manager == 'poetry' +# % if cookiecutter.package_manager in ['poetry', 'uv'] Path('requirements.txt').unlink() +# % if cookiecutter.package_manager == 'poetry' subprocess.check_call(['poetry', 'install']) +# % elif cookiecutter.package_manager == 'uv' +subprocess.check_call(['uv', 'sync']) +# % endif + # % if cookiecutter.crawler_type == 'playwright' -subprocess.check_call(['poetry', 'run', 'playwright', 'install']) +manager = "{{ cookiecutter.package_manager }}" +subprocess.check_call([manager, 'run', 'playwright', 'install']) # % endif + + # % elif cookiecutter.package_manager == 'pip' import venv # noqa: E402 diff --git a/src/crawlee/project_template/hooks/pre_gen_project.py b/src/crawlee/project_template/hooks/pre_gen_project.py index 9830ee93c6..b884e16c6f 100644 --- a/src/crawlee/project_template/hooks/pre_gen_project.py +++ b/src/crawlee/project_template/hooks/pre_gen_project.py @@ -1,12 +1,21 @@ -# % if cookiecutter.package_manager == 'poetry' +# % if cookiecutter.package_manager in ['poetry', 'uv'] import subprocess import re +manager = "{{cookiecutter.package_manager}}" +manager_text = manager.title() +# % if cookiecutter.package_manager == 'poetry' +version_regex = r'Poetry \(version 2\..*\)' +r_version = '2.x' +# % elif cookiecutter.package_manager == 'uv' +version_regex = r'uv (0\..*)' +r_version = '0.x' +# % endif + try: - version = subprocess.check_output(['poetry', '--version']).decode().strip() + version = subprocess.check_output([manager, '--version']).decode().strip() except OSError as exc: - raise RuntimeError('You chose to use the Poetry package manager, but it does not seem to be installed') from exc - -if not re.match(r'Poetry \(version 2\..*\)', version): - raise RuntimeError(f'Poetry 2.x is required, but "{version}" is installed') + raise RuntimeError(f'You chose to use the {manager_text} package manager, but it does not seem to be installed') from exc +if not re.match(version_regex, version): + raise RuntimeError(f'{manager_text} {r_version} is required, but "{version}" is installed') # % endif diff --git a/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile b/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile index f42adfb43b..65383fa740 100644 --- a/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile +++ b/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile @@ -35,6 +35,28 @@ RUN echo "Python version:" \ pip install -r /dev/stdin --no-dependencies \ && echo "All installed Python packages:" \ && pip freeze +# % elif cookiecutter.package_manager == 'uv' +RUN pip install -U pip setuptools \ + && pip install 'uv<1' + +ENV UV_PROJECT_ENVIRONMENT="/usr/local" + +COPY pyproject.toml uv.lock ./ + +RUN echo "Python version:" \ + && python --version \ + && echo "Installing dependencies:" \ + # Check if playwright is already installed + && PLAYWRIGHT_INSTALLED=$(pip freeze | grep -q playwright && echo "true" || echo "false") \ + && if [ "$PLAYWRIGHT_INSTALLED" = "true" ]; then \ + echo "Playwright already installed, excluding from uv sync" \ + && uv sync --frozen --no-install-project --no-editable -q --no-dev --inexact --no-install-package playwright; \ + else \ + echo "Playwright not found, installing all dependencies" \ + && uv sync --frozen --no-install-project --no-editable -q --no-dev --inexact; \ + fi \ + && echo "All installed Python packages:" \ + && pip freeze # % elif cookiecutter.package_manager == 'pip' RUN pip install -U pip setuptools diff --git a/src/crawlee/project_template/{{cookiecutter.project_name}}/README.md b/src/crawlee/project_template/{{cookiecutter.project_name}}/README.md index 34583553cf..b4e4d159e5 100644 --- a/src/crawlee/project_template/{{cookiecutter.project_name}}/README.md +++ b/src/crawlee/project_template/{{cookiecutter.project_name}}/README.md @@ -31,6 +31,37 @@ python -m pip install . When the dependencies are installed, you may launch the crawler with: +```sh +python -m {{cookiecutter.__package_name}} +``` + +{% elif cookiecutter.package_manager == 'uv' -%} +To get started, ensure you have [UV](https://docs.astral.sh/uv/), a package and dependency management system, installed on your machine. We recommend installing it with the following command: + +```sh +pipx install uv +``` + +Next, install the project dependencies: + +```sh +uv sync +``` + +Finally, launch the crawler with: + +```sh +uv run python -m {{cookiecutter.__package_name}} +``` +{% elif cookiecutter.package_manager == 'pip' -%} +To install dependencies, your can run the following command: + +```sh +python -m pip install . +``` + +When the dependencies are installed, you may launch the crawler with: + ```sh python -m {{cookiecutter.__package_name}} ``` diff --git a/uv.lock b/uv.lock index b55a7192fc..72d45efadc 100644 --- a/uv.lock +++ b/uv.lock @@ -578,7 +578,7 @@ toml = [ [[package]] name = "crawlee" -version = "0.6.1" +version = "0.6.3" source = { editable = "." } dependencies = [ { name = "browserforge" },