diff --git a/Cargo.lock b/Cargo.lock index e133bd2..92191de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,17 +2,71 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "autocfg" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "djc-core" version = "1.1.0" dependencies = [ "djc-html-transformer", + "djc-template-parser", "pyo3", "quick-xml", ] @@ -21,8 +75,30 @@ dependencies = [ name = "djc-html-transformer" version = "1.0.3" dependencies = [ + "quick-xml", +] + +[[package]] +name = "djc-template-parser" +version = "1.0.0" +dependencies = [ + "lazy_static", + "pest", + "pest_derive", "pyo3", "quick-xml", + "regex", + "thiserror", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", ] [[package]] @@ -40,6 +116,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.169" @@ -67,6 +149,49 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "pest" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "187da9a3030dbafabbbfb20cb323b976dc7b7ce91fcd84f2f74d6e31d378e2de" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b401d98f5757ebe97a26085998d6c0eecec4995cad6ab7fc30ffdf4b052843" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f27a2cfee9f9039c4d86faa5af122a0ac3851441a34865b8a043b46be0065a" +dependencies = [ + "pest", + "sha2", +] + [[package]] name = "portable-atomic" version = "1.11.1" @@ -84,9 +209,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8e48c12afdeb26aa4be4e5c49fb5e11c3efa0878db783a960eea2b9ac6dd19" +checksum = "37a6df7eab65fc7bee654a421404947e10a0f7085b6951bf2ea395f4659fb0cf" dependencies = [ "indoc", "libc", @@ -101,18 +226,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1989dbf2b60852e0782c7487ebf0b4c7f43161ffe820849b56cf05f945cee1" +checksum = "f77d387774f6f6eec64a004eac0ed525aab7fa1966d94b42f743797b3e395afb" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c808286da7500385148930152e54fb6883452033085bf1f857d85d4e82ca905c" +checksum = "2dd13844a4242793e02df3e2ec093f540d948299a6a77ea9ce7afd8623f542be" dependencies = [ "libc", "pyo3-build-config", @@ -120,9 +245,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0543c16be0d86cf0dbf2e2b636ece9fd38f20406bb43c255e0bc368095f92" +checksum = "eaf8f9f1108270b90d3676b8679586385430e5c0bb78bb5f043f95499c821a71" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -132,9 +257,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a00da2ce064dcd582448ea24a5a26fa9527e0483103019b741ebcbe632dcd29" +checksum = "70a3b2274450ba5288bc9b8c1b69ff569d1d61189d4bff38f8d22e03d17f932b" dependencies = [ "heck", "proc-macro2", @@ -161,12 +286,52 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + [[package]] name = "rustversion" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "syn" version = "2.0.107" @@ -184,6 +349,38 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "unicode-ident" version = "1.0.14" @@ -195,3 +392,9 @@ name = "unindent" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" diff --git a/Cargo.toml b/Cargo.toml index 4f41e57..f2bbf77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,12 +2,18 @@ members = [ "crates/djc-core", "crates/djc-html-transformer", + "crates/djc-template-parser", ] resolver = "2" [workspace.dependencies] pyo3 = { version = "0.27.1", features = ["extension-module"] } quick-xml = "0.38.3" +pest = "2.8.3" +pest_derive = "2.8.3" +thiserror = "2.0.17" +regex = "1.12.2" +lazy_static = "1.5.0" # https://ohadravid.github.io/posts/2023-03-rusty-python [profile.release] diff --git a/crates/djc-core/Cargo.toml b/crates/djc-core/Cargo.toml index c8ed537..d3d4a91 100644 --- a/crates/djc-core/Cargo.toml +++ b/crates/djc-core/Cargo.toml @@ -11,5 +11,6 @@ crate-type = ["cdylib"] [dependencies] djc-html-transformer = { path = "../djc-html-transformer" } +djc-template-parser = { path = "../djc-template-parser" } pyo3 = { workspace = true } quick-xml = { workspace = true } diff --git a/crates/djc-core/src/lib.rs b/crates/djc-core/src/lib.rs index dcc7135..f6bcfb4 100644 --- a/crates/djc-core/src/lib.rs +++ b/crates/djc-core/src/lib.rs @@ -1,18 +1,46 @@ -use djc_html_transformer::{ - set_html_attributes as set_html_attributes_rust, HtmlTransformerConfig, +use djc_html_transformer::{set_html_attributes as set_html_attributes_rust, HtmlTransformerConfig}; +use djc_template_parser::{ + compile_ast_to_string as compile_ast_to_string_rust, parse_tag as parse_tag_rust, Tag, TagAttr, + TagSyntax, TagToken, TagValue, TagValueFilter, ValueKind, }; -use pyo3::exceptions::{PyValueError}; +use pyo3::exceptions::{PySyntaxError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyTuple}; +use pyo3::types::{PyList, PyDict, PyTuple}; +use std::collections::HashSet; /// Singular Python API that brings togther all the other Rust crates. #[pymodule] fn djc_core(m: &Bound<'_, PyModule>) -> PyResult<()> { // HTML transformer m.add_function(wrap_pyfunction!(set_html_attributes, m)?)?; + + // Template parser + m.add_function(wrap_pyfunction!(parse_tag, m)?)?; + m.add_function(wrap_pyfunction!(compile_ast_to_string, m)?)?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + Ok(()) } +#[pyfunction] +#[pyo3(signature = (input, flags=None))] +fn parse_tag(input: &str, flags: Option>) -> PyResult { + parse_tag_rust(input, flags).map_err(|e| PySyntaxError::new_err(e.to_string())) +} + +#[pyfunction] +fn compile_ast_to_string(py: Python, attributes: &Bound) -> PyResult { + let attrs: Vec = attributes.extract()?; + let result = py.detach(|| compile_ast_to_string_rust(&attrs)); + result.map_err(|e| PySyntaxError::new_err(e.to_string())) +} + /// Transform HTML by adding attributes to the elements. /// /// Args: diff --git a/crates/djc-template-parser/Cargo.toml b/crates/djc-template-parser/Cargo.toml new file mode 100644 index 0000000..ce2d5a2 --- /dev/null +++ b/crates/djc-template-parser/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "djc-template-parser" +description = "Parse Django template tags into an AST and compile it into a Python function" +version = "1.0.0" +edition = "2021" + +[dependencies] +pyo3 = { workspace = true } +quick-xml = { workspace = true } +pest = { workspace = true } +pest_derive = { workspace = true } +thiserror = { workspace = true } +regex = { workspace = true } +lazy_static = { workspace = true } diff --git a/crates/djc-template-parser/README.md b/crates/djc-template-parser/README.md new file mode 100644 index 0000000..c15396e --- /dev/null +++ b/crates/djc-template-parser/README.md @@ -0,0 +1,453 @@ +# Django Components Template Parser + +A high-performance Rust-based template parser for [`django-components`](https://github.com/django-components/django-components), designed to parse Django template syntax into an Abstract Syntax Tree (AST) and compile it into callable Python functions. + +## Overview + +This package provides a fast, Rust-implemented parser for Django template syntax using the [Pest](https://pest.rs/) parsing library. This library has follow parts: + +1. **tag_parser** - Turn `{% ... %}` or `<... />` into an AST using the grammar defined in `grammar.pest` +2. **tag_compiler** - Compile the AST into optimized, callable Python functions + +The parser supports: + +- Tag name (must be the first token, e.g. `{% my_tag ... %}`) +- Key-value pairs (e.g. `key=value`) +- Standalone values (e.g. `1`, `"my string"`, `val`) +- Spread operators (e.g. `...value`, `**value`, `*value`) +- Filters (e.g. `value|filter:arg`) +- Lists and dictionaries (e.g. `[1, 2, 3]`, `{"key": "value"}`) +- String literals (single/double quoted) (e.g. `"my string"`, `'my string'`) +- Numbers (e.g. `1`, `1.23`, `1e-10`) +- Variables (e.g. `val`, `key`) +- Translation strings (e.g. `_("text")`) +- Comments (e.g. `{# comment #}`) +- Self-closing tags (e.g. `{% my_tag / %}`) + +## Development + +### Prerequisites + +- Rust (latest stable version) +- Python 3.8+ +- [Maturin](https://github.com/PyO3/maturin) for building Python extensions + +### Setup + +1. Install Maturin: + + ```bash + pip install maturin + ``` + +2. Build and install the package in development mode: + ```bash + maturin develop + ``` + +This will compile the Rust code and install the Python package in your current environment. + +### Running tests + +```bash +# Run Rust tests +cargo test + +# Run specific Rust test +cargo test tag_parser::tests::test_list_spread_comments + +# Run Python tests +python -m pytest tests/ +``` + +## Developing django-components + +If you're making changes to the parser, you should test that the updated parser still works with `django-components`. + +To do that, you need to build the parser package and install it in your local fork of `django-components`. + +1. Build the parser package: + + ```bash + cd djc_core_template_parser + maturin develop + ``` + +2. Install `djc_core_template_parser` in django-components: + ```bash + cd ../django_components + pip install -e ../djc_core_template_parser + ``` + +## Publishing + +There is a Github workflow to release the package. It runs when a new tag is pushed (AKA a new release is created). + +The CI workflow compiles the package in many different environments, ensuring that this package can run across all major platforms. + +Steps: + +1. Bump version in `pyproject.toml` +2. Make a release on GitHub +3. The package will be automatically compiled and published to PyPI. + +## Type definitions + +### `djc_core_template_parser/__init__.pyi` + +This file contains the **public interface** that will be used by other packages (like the main Django Components package) in VSCode and other IDEs. It defines: + +- All public functions and classes +- Type hints for parameters and return values +- Documentation for the API + +This is the interface that external consumers of the package will see. + +### Root `__init__.pyi` + +The root `__init__.pyi` file is for **local development** and should be a copy of `djc_core_template_parser/__init__.pyi`. + +**Important**: Keep these files in sync. When updating the interface, update both files. + +## Test compatibility + +To ensure that the parser works correctly with `django-components`, we define tests in `test_tag_parser.py`. + +This test file exists in two locations: + +- `djc_core_template_parser/tests/test_tag_parser.py` - Tests for the parser package itself +- `django_components/tests/test_tag_parser.py` - Integration tests for Django Components (copy) + +When updating the `test_tag_parser.py`, you should also update the copy in `django_components/tests/test_tag_parser.py`. + +## Architecture + +``` +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ Django │ │ Rust Parser │ │ Python │ +│ Template │───▶│ (grammar.pest) │───▶│ Function │ +│ Syntax │ │ │ │ │ +└─────────────────┘ └──────────────────┘ └─────────────────┘ +``` + +The parser uses Pest's declarative grammar to define Django template syntax rules, then compiles the parsed AST into optimized Python functions that can be called directly from Django Components. + +## Contributing + +1. Make changes to the Rust code in `src/` +2. Update the grammar in `grammar.pest` if needed +3. Update both `__init__.pyi` files if the interface changes +4. Add tests to both test files if needed +5. Run `maturin develop` to test your changes +6. Ensure all tests pass before submitting a PR + +## On template tag parser + +The template syntax parsing was implemented using [Pest](https://pest.rs/). Pest works in 3 parts: + +1. "grammar rules" - definition of patterns that are supported in the.. language? I'm not sure about the correct terminology. + + Pest defines it's own language for defining these rules, see `djc-template-parser/src/grammar.pest`. + + This is similar to [Backus–Naur Form](https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form), e.g. + + ``` + ::= + ::= | + ::= + ::= "," + ``` + + Or the MDN's formal syntax, e.g. [here](https://developer.mozilla.org/en-US/docs/Web/CSS/border-left-width#formal_syntax): + ``` + border-left-width = + + + = + [](https://developer.mozilla.org/en-US/docs/Web/CSS/length) [|](https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_values_and_units/Value_definition_syntax#single_bar) + thin [|](https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_values_and_units/Value_definition_syntax#single_bar) + medium [|](https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_values_and_units/Value_definition_syntax#single_bar) + thick + ``` + + Well and this Pest grammar is where all the permissible patterns are defined. E.g. here's a high-level example for a `{% ... %}` template tag (NOTE: outdated version): + + ``` + // The full tag is a sequence of attributes + // E.g. `{% slot key=val key2=val2 %}` + tag_wrapper = { SOI ~ django_tag ~ EOI } + + django_tag = { "{%" ~ tag_content ~ "%}" } + + // The contents of a tag, without the delimiters + tag_content = ${ + spacing* // Optional leading whitespace/comments + ~ tag_name // The tag name must come first, MAY be preceded by whitespace + ~ (spacing+ ~ attribute)* // Then zero or more attributes, MUST be separated by whitespace/comments + ~ spacing* // Optional trailing whitespace/comments + ~ self_closing_slash? // Optional self-closing slash + ~ spacing* // More optional trailing whitespace + } + ``` + +2. Parsing and handling of the matched grammar rules. + + So each defined rule has its own name, e.g. `django_tag`. + + When a text is parsed with Pest in Rust, we get a list of parsed rules (or a single rule?). + + Since the grammar definition specifies the entire `{% .. %}` template tag, and we pass in a string starting and ending in `{% ... %}`, we should match exactly the top-level `tag_wrapper` rule. + + If we match anything else in its place, we raise an error. + + Once we have `tag_wrapper`, we walk down it, rule by rule, constructing the AST from the patterns we come across. + +3. Constructing the AST. + + The AST consists of these nodes - Tag, TagAttr, TagToken, TagValue, TagValueFilter + + - `Tag` - the entire `{% ... %}`, e.g `{% my_tag x ...[1, 2, 3] key=val / %}` + + - The first word inside a `Tag` is the `tag_name`, e.g. `my_tag`. + - After the tag name, there are zero or more `TagAttrs`. This is ALL inputs, both positional and keyword + - Tag attrs are `x`, `...[1, 2, 3]`, `key=val` + - If a tag attribute has a key, that's stored on `TagAttrs`. + - But ALL `TagAttrs` MUST have a value. + - TagValue holds a single value, may have a filter, e.g. `"cool"|upper` + - TagValue may be of different kinds, e.g. string, int, float, literal list, literal dict, variable, translation `_('mystr')`, etc. The specific kind is identified by what rules we parse, and the resulting TagValue nodes are distinguished by the `ValueKind`, an enum with values like `"string"`, `"float"`, etc. + - Since TagValue can be also e.g. literal lists, TagValues may contain other TagValues. This implies that: + 1. Lists and dicts themselves can have filters applied to them, e.g. `[1, 2, 3]|append:4` + 2. items inside lists and dicts can too have filters applied to them. e.g. `[1|add:1, 2|add:2]` + - Any TagValue can have 0 or more filters applied to it. Filters have a name and an optional argument, e.g. `3|add:2` - filter name `add`, arg `2`. These filters are held by `TagValueFilter`. + - While the filter name is a plain identifier, the argument can be yet another TagValue. so even using literal lists and dicts at the position of filter argument is permitted, e.g. `[1]|extend:[2, 3]` + + - Lastly, `TagToken` is a secondary object used by the nodes above. It contains info about the original raw string, and the line / col where the string was found. + +The final AST can look like this: + +INPUT: +```django +{% my_tag value|lower %} +``` + +AST: +```rs +Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + arg: None, + token: TagToken { + token: "lower".to_string(), + start_index: 16, + end_index: 21, + line_col: (1, 17), + }, + start_index: 15, + end_index: 21, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 21, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 21, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 24, + line_col: (1, 4), +} +``` + + +## On template tag compilation + +Another important part is the "tag compiler". This turns the parsed AST into an executable Python function. When this function is called with the `Context` object, it resolves the inputs to a tag into Python args and kwargs. + +```py +from djc_core import parse_tag, compile_tag + +ast = parse_tag('{% my_tag var1 ...[2, 3] key=val ...{"other": "x"} / %}') +tag_fn = compile_tag(ast) + +args, kwargs = tag_fn({"var1": "hello", "val": "abc"}) + +assert args == ["hello", 2, 3] +assert kwargs == {"key": "abc", "other": "x"} +``` + +How it works is: + +1. We start with the AST of the template tag. +2. TagAttrs with keys become function's kwargs, and TagAttrs without keys are functions args. +3. For each TagAttr, we walk down it's value, and handle each ValueKind differently + - Literals - 1, 1.5, "abc", etc - These are compiled as literal Python values + - Variables - e.g. `my_var` - we replace that with function call `variable(context, "my_var")` + - Filters - `my_var|add:"txt"` - replaced with function call `filter(context, "add", my_var, "txt")` + - Translation `_("abc")` - function call `translation(context, "abc")` + - String with nested template tags, e.g. `"Hello {{ first_name }}"` - function call `template_string(context, "Hello {{ first_name }}")` + - Literal lists and dicts - structure preserved, and we walk down and convert each item, key, value. + + Input: + + ```django + {% component my_var|add:"txt" / %} + ``` + + Generated function: + + ```py + def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + args.append(filter(context, 'add', variable(context, 'my_var'), "txt")) + return args, kwargs + ``` + +4. Apply Django-specific logic + + As you can see, the generated function accepts the definitions for the functions `variable()`, `filter()`, etc. + + This means that the implementation for these is defined in Python. So we can still easily change how individual features are handled. These definitions of `variable()`, etc are NOT exposed to the users of django-components. + + The implementation is defined in django-components, and it looks something like below. + + There you can see e.g. that when the Rust compiler came across a variable `my_var`, it generated `variable(..)` call. And the implementation for `variable(...)` calls Django's `Variable(var).resolve(ctx)`. + + So at the end of the day we're still using the same Django logic to actually resolve variables into actual values. + + ```py + def resolve_template_string(ctx: Context, expr: str) -> Any: + return TemplateStringExpression( + expr_str=expr, + filters=filters, + tags=tags, + ).resolve(ctx) + + def resolve_filter(_ctx: Context, name: str, value: Any, arg: Any) -> Any: + if name not in filters: + raise TemplateSyntaxError(f"Invalid filter: '{name}'") + + filter_func = filters[name] + if arg is None: + return filter_func(value) + else: + return filter_func(value, arg) + + def resolve_variable(ctx: Context, var: str) -> Any: + try: + return Variable(var).resolve(ctx) + except VariableDoesNotExist: + return "" + + def resolve_translation(ctx: Context, var: str) -> Any: + # The compiler gives us the variable stripped of `_(")` and `"), + # so we put it back for Django's Variable class to interpret it as a translation. + translation_var = "_('" + var + "')" + return Variable(translation_var).resolve(ctx) + + args, kwargs = compiled_tag( + context=context, + template_string=template_string, + variable=resolve_variable, + translation=resolve_translation, + filter=resolve_filter, + ) + ``` + +5. Call the component with the args and kwargs + + The compiled function returned a list of args and a dict of kwargs. We then simply pass these further to the implementation of the `{% component %}` node. + + So a template tag like this: + + ```django + {% component "my_table" var1 ...[2, 3] key=val ...{"other": "x"} / %} + ``` + + Eventually gets resolved to something like so: + + ```py + ComponentNode.render("my_table", var1, 2, 3, key=val, other="x") + ``` + +**Validation** + +The template tag inputs respect Python's convetion of not allowing args after kwargs. + +When compiling AST into a Python function, we're able to detect obvious cases and raise an error early, like: + +```django +{% component key=val my_var / %} {# Error! #} +``` +However, some cases can be figured out only at render time. Becasue the spread syntax `...my_var` can be used with both a list of args or a dict of kwargs. + +So we need to wait for the Context object to figure out whether this: +```django +{% component ...items my_var / %} +``` +Resolves to lists (OK): +```django +{% component ...[1, 2, 3] my_var / %} +``` +Or to dict (Error): +```django +{% component ...{"key": "x"} my_var / %} +``` + +So when we detect that there is a spread within the template tag, we add a render-time function that checks whether the spread resolves to list or a dict, and raises if it's not permitted: + +INPUT: +```django +{% component ...options1 key1="value1" ...options2 key1="value1" / %} +``` + +Generated function: +```py +def compiled_func(context, *, template_string, translation, variable, filter): + def _handle_spread(value, raw_token_str, args, kwargs, kwarg_seen): + if hasattr(value, "keys"): + kwargs.extend(value.items()) + return True + else: + if kwarg_seen: + raise SyntaxError("positional argument follows keyword argument") + try: + args.extend(value) + except TypeError: + raise TypeError( + f"Value of '...{raw_token_str}' must be a mapping or an iterable, " + f"not {type(value).__name__}." + ) + return False + + args = [] + kwargs = [] + kwargs.append(('key1', "value1")) + kwarg_seen = True + kwarg_seen = _handle_spread(variable(context, 'options1'), """options1""", args, kwargs, kwarg_seen) + kwargs.append(('key2', "value2")) + kwarg_seen = _handle_spread(variable(context, 'options2'), """options2""", args, kwargs, kwarg_seen) + return args, kwargs +``` diff --git a/crates/djc-template-parser/src/ast.rs b/crates/djc-template-parser/src/ast.rs new file mode 100644 index 0000000..5fd0d08 --- /dev/null +++ b/crates/djc-template-parser/src/ast.rs @@ -0,0 +1,466 @@ +//! # Abstract Syntax Tree (AST) for Django Template Tags +//! +//! This module defines the core data structures that represent parsed Django template tags +//! as an Abstract Syntax Tree (AST). These structures are used throughout the template +//! parsing and compilation pipeline. +//! +//! ## Overview +//! +//! The AST represents Django template tags in a structured format that captures: +//! - Tag names and attributes +//! - Values with their types (strings, numbers, variables, template_strings, etc.) +//! - Filter chains and filter arguments +//! - Position information (line/column, start/end indices) +//! - Syntax type (Django `{% %}` vs HTML `< />` tags) +//! +//! ## Core types +//! +//! - **`Tag`**: Represents a complete template tag with name, attributes, and metadata - `{% my_tag ... %}` or `` +//! - **`TagAttr`**: Represents a single attribute (key-value pair or flag) - `key=value` or `flag` +//! - **`TagValue`**: Represents a value with type information and optional filters - `'some_val'|upper` +//! - **`TagToken`**: Represents a token with position information +//! - **`TagValueFilter`**: Represents a filter applied to a value +//! - **`ValueKind`**: Enum of supported value types (list, dict, int, float, variable, template_string, translation, string) +//! - **`TagSyntax`**: Enum of supported tag syntaxes (Django vs HTML) +//! +//! All AST types are exposed to Python via PyO3 bindings. +//! +//! ## Example +//! +//! ```rust +//! use crate::djc_template_parser::ast::*; +//! +//! // A Django tag: {% my_tag key=val %} +//! let tag = Tag { +//! name: TagToken { +//! token: "my_tag".to_string(), +//! start_index: 3, +//! end_index: 9, +//! line_col: (1, 4), +//! }, +//! attrs: vec![TagAttr { +//! key: Some(TagToken { +//! token: "key".to_string(), +//! start_index: 10, +//! end_index: 13, +//! line_col: (1, 11), +//! }), +//! value: TagValue { +//! token: TagToken { +//! token: "val".to_string(), +//! start_index: 14, +//! end_index: 17, +//! line_col: (1, 15), +//! }, +//! children: vec![], +//! spread: None, +//! filters: vec![], +//! kind: ValueKind::Variable, +//! start_index: 14, +//! end_index: 17, +//! line_col: (1, 15), +//! }, +//! is_flag: false, +//! start_index: 10, +//! end_index: 17, +//! line_col: (1, 11), +//! }], +//! is_self_closing: false, +//! syntax: TagSyntax::Django, +//! start_index: 0, +//! end_index: 20, +//! line_col: (1, 4), +//! }; +//! ``` + +use pyo3::prelude::*; + +/// Top-level tag attribute, e.g. `key=my_var` or without key like `my_var|filter` +#[pyclass] +#[derive(Debug, PartialEq, Clone)] +pub struct TagAttr { + #[pyo3(get)] + pub key: Option, + #[pyo3(get)] + pub value: TagValue, + #[pyo3(get)] + pub is_flag: bool, + + /// Start index (incl. filters) + #[pyo3(get)] + pub start_index: usize, + /// End index (incl. filters) + #[pyo3(get)] + pub end_index: usize, + /// Line and column (incl. filters) + #[pyo3(get)] + pub line_col: (usize, usize), +} + +#[pymethods] +impl TagAttr { + // These methods with `[new]` will become constructors (`__init__()`) + #[new] + #[pyo3(signature = (key, value, is_flag, start_index, end_index, line_col))] + fn new( + key: Option, + value: TagValue, + is_flag: bool, + start_index: usize, + end_index: usize, + line_col: (usize, usize), + ) -> Self { + Self { + key, + value, + is_flag, + start_index, + end_index, + line_col, + } + } + + // Allow to compare objects with `==` + fn __eq__(&self, other: &TagAttr) -> bool { + self.key == other.key + && self.value == other.value + && self.is_flag == other.is_flag + && self.start_index == other.start_index + && self.end_index == other.end_index + && self.line_col == other.line_col + } + + fn __repr__(&self) -> String { + format!("TagAttr(key={:?}, value={:?}, is_flag={}, start_index={}, end_index={}, line_col={:?})", + self.key, self.value, self.is_flag, self.start_index, self.end_index, self.line_col) + } +} + +#[pyclass(eq, eq_int)] +#[derive(Debug, PartialEq, Clone)] +pub enum ValueKind { + List, + Dict, + Int, + Float, + Variable, + TemplateString, // A string that contains a Django template tags, e.g. `"{{ my_var }}"` + Translation, + String, +} + +#[pymethods] +impl ValueKind { + #[new] + fn new(kind: &str) -> PyResult { + match kind { + "list" => Ok(ValueKind::List), + "dict" => Ok(ValueKind::Dict), + "int" => Ok(ValueKind::Int), + "float" => Ok(ValueKind::Float), + "variable" => Ok(ValueKind::Variable), + "template_string" => Ok(ValueKind::TemplateString), + "translation" => Ok(ValueKind::Translation), + "string" => Ok(ValueKind::String), + _ => Err(pyo3::exceptions::PyValueError::new_err(format!( + "Invalid ValueKind: {}", + kind + ))), + } + } + + fn __str__(&self) -> String { + match self { + ValueKind::List => "list".to_string(), + ValueKind::Dict => "dict".to_string(), + ValueKind::Int => "int".to_string(), + ValueKind::Float => "float".to_string(), + ValueKind::Variable => "variable".to_string(), + ValueKind::TemplateString => "template_string".to_string(), + ValueKind::Translation => "translation".to_string(), + ValueKind::String => "string".to_string(), + } + } +} + +/// Metadata of the matched token +#[pyclass] +#[derive(Debug, PartialEq, Clone)] +pub struct TagToken { + /// String value of the token (excl. filters and spread) + #[pyo3(get)] + pub token: String, + /// Start index (excl. filters and spread) + #[pyo3(get)] + pub start_index: usize, + /// End index (excl. filters and spread) + #[pyo3(get)] + pub end_index: usize, + /// Line and column (excl. filters and spread) + #[pyo3(get)] + pub line_col: (usize, usize), +} + +#[pymethods] +impl TagToken { + #[new] + fn new(token: String, start_index: usize, end_index: usize, line_col: (usize, usize)) -> Self { + Self { + token, + start_index, + end_index, + line_col, + } + } + + fn __eq__(&self, other: &TagToken) -> bool { + self.token == other.token + && self.start_index == other.start_index + && self.end_index == other.end_index + && self.line_col == other.line_col + } + + fn __repr__(&self) -> String { + format!( + "TagToken(token='{}', start_index={}, end_index={}, line_col={:?})", + self.token, self.start_index, self.end_index, self.line_col + ) + } +} + +#[pyclass] +#[derive(Debug, PartialEq, Clone)] +pub struct TagValue { + /// Position and string value of the value (excl. filters and spread) + /// + /// NOTE: If this TagValue has NO filters, position and index in `token` are the same + /// as `start_index`, `end_index` and `line_col` defined directly on `TagValue`. + #[pyo3(get)] + pub token: TagToken, + /// Children of this TagValue - e.g. list items like `[1, 2, 3]` or dict key-value entries like `{"key": "value"}` + #[pyo3(get)] + pub children: Vec, + + #[pyo3(get)] + pub kind: ValueKind, + #[pyo3(get)] + pub spread: Option, + #[pyo3(get)] + pub filters: Vec, + + /// Start index (incl. filters and spread) + #[pyo3(get)] + pub start_index: usize, + /// End index (incl. filters and spread) + #[pyo3(get)] + pub end_index: usize, + /// Line and column (incl. filters and spread) + #[pyo3(get)] + pub line_col: (usize, usize), +} + +#[pymethods] +impl TagValue { + #[new] + #[pyo3(signature = (token, children, kind, spread, filters, start_index, end_index, line_col))] + fn new( + token: TagToken, + children: Vec, + kind: ValueKind, + spread: Option, + filters: Vec, + start_index: usize, + end_index: usize, + line_col: (usize, usize), + ) -> Self { + Self { + token, + children, + kind, + spread, + filters, + start_index, + end_index, + line_col, + } + } + + fn __eq__(&self, other: &TagValue) -> bool { + self.token == other.token + && self.children == other.children + && self.kind == other.kind + && self.spread == other.spread + && self.filters == other.filters + && self.start_index == other.start_index + && self.end_index == other.end_index + && self.line_col == other.line_col + } + + fn __repr__(&self) -> String { + format!("TagValue(token={:?}, children={:?}, kind={:?}, spread={:?}, filters={:?}, start_index={}, end_index={}, line_col={:?})", + self.token, self.children, self.kind, self.spread, self.filters, self.start_index, self.end_index, self.line_col) + } +} + +#[pyclass] +#[derive(Debug, PartialEq, Clone)] +pub struct TagValueFilter { + /// Token of the filter, e.g. `filter` + #[pyo3(get)] + pub token: TagToken, + /// Argument of the filter, e.g. `my_var` + #[pyo3(get)] + pub arg: Option, + + /// Start index (incl. `|`) + #[pyo3(get)] + pub start_index: usize, + /// End index (incl. `|`) + #[pyo3(get)] + pub end_index: usize, + /// Line and column (incl. `|`) + #[pyo3(get)] + pub line_col: (usize, usize), +} + +#[pymethods] +impl TagValueFilter { + #[new] + #[pyo3(signature = (token, arg, start_index, end_index, line_col))] + fn new( + token: TagToken, + arg: Option, + start_index: usize, + end_index: usize, + line_col: (usize, usize), + ) -> Self { + Self { + token, + arg, + start_index, + end_index, + line_col, + } + } + + fn __eq__(&self, other: &TagValueFilter) -> bool { + self.token == other.token + && self.arg == other.arg + && self.start_index == other.start_index + && self.end_index == other.end_index + && self.line_col == other.line_col + } + + fn __repr__(&self) -> String { + format!( + "TagValueFilter(token={:?}, arg={:?}, start_index={}, end_index={}, line_col={:?})", + self.token, self.arg, self.start_index, self.end_index, self.line_col + ) + } +} + +#[pyclass(eq, eq_int)] +#[derive(Debug, PartialEq, Clone)] +pub enum TagSyntax { + Django, // For tags like {% my_tag ... %} + Html, // For tags like +} + +#[pymethods] +impl TagSyntax { + #[new] + fn new(syntax: &str) -> PyResult { + match syntax { + "django" => Ok(TagSyntax::Django), + "html" => Ok(TagSyntax::Html), + _ => Err(pyo3::exceptions::PyValueError::new_err(format!( + "Invalid TagSyntax: {}", + syntax + ))), + } + } + + fn __str__(&self) -> String { + match self { + TagSyntax::Django => "django".to_string(), + TagSyntax::Html => "html".to_string(), + } + } +} + +/// Represents a full template tag, including its name, attributes, and other metadata. +/// E.g. `{% slot key=val key2=val2 %}` or `` +#[pyclass] +#[derive(Debug, PartialEq, Clone)] +pub struct Tag { + /// The name of the tag, e.g., 'slot' in `{% slot ... %}`. + /// This is a `TagToken` to include positional data. + #[pyo3(get)] + pub name: TagToken, + + /// A list of attributes passed to the tag. + #[pyo3(get)] + pub attrs: Vec, + + /// Whether the tag is self-closing. + /// E.g. `{% my_tag / %}` or ``. + #[pyo3(get)] + pub is_self_closing: bool, + + /// The syntax of the tag: + /// - `django` for `{% my_tag / %}` + /// - `html` for ``. + #[pyo3(get)] + pub syntax: TagSyntax, + + /// Start index of the tag in the original input string. + #[pyo3(get)] + pub start_index: usize, + + /// End index of the tag in the original input string. + #[pyo3(get)] + pub end_index: usize, + + /// Line and column number of the start of the tag. + #[pyo3(get)] + pub line_col: (usize, usize), +} + +#[pymethods] +impl Tag { + #[new] + fn new( + name: TagToken, + attrs: Vec, + is_self_closing: bool, + syntax: TagSyntax, + start_index: usize, + end_index: usize, + line_col: (usize, usize), + ) -> Self { + Self { + name, + attrs, + is_self_closing, + syntax, + start_index, + end_index, + line_col, + } + } + + fn __eq__(&self, other: &Tag) -> bool { + self.name == other.name + && self.attrs == other.attrs + && self.is_self_closing == other.is_self_closing + && self.syntax == other.syntax + && self.start_index == other.start_index + && self.end_index == other.end_index + && self.line_col == other.line_col + } + + fn __repr__(&self) -> String { + format!("Tag(name={:?}, attrs={:?}, is_self_closing={}, syntax={:?}, start_index={}, end_index={}, line_col={:?})", + self.name, self.attrs, self.is_self_closing, self.syntax, self.start_index, self.end_index, self.line_col) + } +} diff --git a/crates/djc-template-parser/src/error.rs b/crates/djc-template-parser/src/error.rs new file mode 100644 index 0000000..02cd065 --- /dev/null +++ b/crates/djc-template-parser/src/error.rs @@ -0,0 +1,19 @@ +use thiserror::Error; + +#[derive(Debug, Error, PartialEq)] +pub enum CompileError { + #[error("{0}")] + Generic(String), +} + +impl From for CompileError { + fn from(error: String) -> Self { + CompileError::Generic(error) + } +} + +impl From<&str> for CompileError { + fn from(error: &str) -> Self { + CompileError::Generic(error.to_string()) + } +} diff --git a/crates/djc-template-parser/src/grammar.pest b/crates/djc-template-parser/src/grammar.pest new file mode 100644 index 0000000..6a2b380 --- /dev/null +++ b/crates/djc-template-parser/src/grammar.pest @@ -0,0 +1,317 @@ +// Legend: +// See https://docs.rs/pest/latest/pest/#pest-the-elegant-parser +// - { ... }: None - Accept whitespace or comments within their expressions. +// Create token pairs during parsing, error-reported. +// - _{ ... }: Silent - Do not create token pairs during parsing, nor are they error-reported. +// - @{ ... }: Atomic - Do not accept whitespace or comments within their expressions. +// Any rules called by atomic rules do not generate token pairs. +// - ${ ... }: Compound-atomic - Same as atomic, plus not forbidden from generating token pairs. + + +///////////////////////////// +// TAG +///////////////////////////// + +// The full tag is a sequence of attributes +// E.g. `{% slot key=val key2=val2 %}` +// NOTE: tag_wrapper is used when parsing exclusively a single Django template tag. +tag_wrapper = { SOI ~ django_tag ~ EOI } +django_tag = @{ "{%" ~ spacing_with_whitespace ~ tag_content ~ spacing_with_whitespace ~ "%}" } + +// The contents of a tag, without the delimiters +tag_content = ${ + tag_name // The tag name must come first + ~ (spacing_with_whitespace ~ attribute)* // Then zero or more attributes, MUST be separated by whitespace + ~ (spacing_with_whitespace ~ self_closing_slash)? // Optional self-closing slash at the end + // ^^^^^^^^^^^^^^^^^^^^^^^ This space is REQUIRED in Django tags + // There MUST be space between last attribute and self-closing slash +} + +// NOTE: For supporting HTML tags, we could add rules like this: +// tag_wrapper = { SOI ~ (django_tag | html_tag) ~ EOI } +// html_tag = { "<" ~ html_tag_content ~ ">" } +// html_tag_content = ${ +// tag_name // The tag name must come first +// ~ (spacing_with_whitespace ~ attribute)* // Then zero or more attributes, MUST be separated by whitespace +// ~ (spacing* ~ self_closing_slash)? // Optional self-closing slash at the end +// // ^^^^^^^^^ This space is OPTIONAL in HTML +// // There MAY be space between last attribute and self-closing slash +// // and NO space between `/` and closing `>` +// } + +// Tag name SHOULD be a valid Python identifier, but this syntax leaves us space +// to also support kebab-case, snake_case, PascalCase, and tag namespacing (for sharing components) +// with either `.` or `:` +tag_name = @{ (ASCII_ALPHA | "_") ~ (ASCII_ALPHANUMERIC | "_" | "-" | ":" | ".")* } + +self_closing_slash = { "/" } + +// An attribute can be either a key-value pair or just a value +// E.g. `key=val`, `key2=val2`, `"val3"`, `...[1, 2, 3]` +attribute = ${ + (key ~ "=" ~ filtered_value) // key=value form with NO whitespace allowed around = + | spread_value // spread operator form (e.g. `...[1, 2, 3]`) + | filtered_value // value-only form (e.g. `"val3"`) +} + +// Spread operator followed by a value, e.g. `...[1, 2, 3]` +spread_value = { + "..." ~ filtered_value +} + +///////////////////////////// +// KEY +///////////////////////////// + +// Full key definition +// NOTE: @ in front of a rule means they should not have whitespace in between +key = @{ key_start ~ key_char* } + +// A key must not start with certain characters +key_start = @{ + !((":" | "'" | "\"" | "_(" | "[" | "{" | "*" | "..." | WHITESPACE) ~ ANY) ~ + key_char +} + +// Characters that can appear in a key +// +// Keys are preferably alphanumeric + `_-`. But practically we allow much more, +// as these may be HTML attributes. +// +// Rules to keys: +// 1. Keys do not contain whitespaces +// 2. Must NOT start with `:` or `'`, `"`, `_("`, `_('`, `[`, `{`, `*`, `**`, `...` +// 3. Key is terminated when it comes across: +// - Any of `=`, `'`, `"`, `_("`, `_('`, `[`, `{`, `*`, `**`, `...` +// - Any whitespace +// +// Examples: +// - `key=val` - valid key +// - `:key=val` - invalid +// - `attr:key=val` - valid, because `:` is not first +// - `...key=val` - invalid, key cannot follow `...` +// - `_("hello")` - not a key. It is a valid value, but not a key +// - `"key"=val` - invalid, keys must not contain quotes +// - `key[0]=val` - invalid. we don't allow setting indices. So the key would end at `[`, so it +// would be as if having two attributes: `key` and `[0]=val`. +// However, then there's a missing space between the two, so that's an error too. +// And `[0]` would also be an invalid key. +key_char = @{ + ASCII_ALPHANUMERIC | + "_" | "-" | "@" | "#" | "." | ":" | // Common special chars + // Add other allowed special chars here + !(("=" | "'" | "\"" | "_(" | "[" | "{" | "*" | "..." | WHITESPACE) ~ ANY) ~ ANY +} + +///////////////////////////// +// FILTER +///////////////////////////// + +// Tag value can be a simple value followed by optional filters +// +// Value syntax supports Django filters, e.g. `"HELLO"|lower` or `my_list|select:1`, +// generally with format `value|filter:filter_arg`. Multiple filters can be chained +// up one after another. +// +// For context on how Django parses filters, see: +// https://github.com/django/django/blob/862b7f98a02b7973848db578ff6d24ec8500fdb4/django/template/base.py#L621 +// +// NOTE: Filter pipe CAN be separated by whitespace +filtered_value = { + value ~ filter_chain? +} + +// A chain of filters separated by pipes +filter_chain = { + spacing* ~ filter ~ (spacing* ~ filter)* +} +// In the position of a dictionary key we don't allow filter arguments +// because filter args also use colon `:`, which conflicts with dict keys. +// So something like `{"key"|lower:arg: value}` would be ambiguous. +filter_chain_noarg = { + spacing* ~ filter_noarg ~ (spacing* ~ filter_noarg)* +} + +// A single filter with optional argument +// NOTE: Filter pipe `|` CAN be surrounded by whitespace +filter = { + "|" ~ spacing* ~ filter_name ~ filter_arg_part? +} +filter_noarg = { + "|" ~ spacing* ~ filter_name +} + +// Filter name must be alphanumeric + `_` +filter_name = @{ + !(":" | "|") ~ (ASCII_ALPHANUMERIC | "_")+ // Ensure filter name doesn't start with : or | +} + +// NOTE: Filter arg `:` CAN be surrounded by whitespace +// The argument part of a filter, including the colon +filter_arg_part = { + spacing* ~ ":" ~ spacing* ~ filter_arg +} + +filter_arg = { + value +} + +///////////////////////////// +// VALUE +///////////////////////////// + +// The actual value +// +// - A number (e.g. -1, +1.5, .5, 1e-10) +// - A variable name (alphanumeric + dots) +// - A string literal (single or double quoted) +// - An i18n string (string literal wrapped in _()) +// - List of values +// - Dictionary of values +// +// NOTE: Order matters here - We need to first check for `_("...")` because `_` +// is also a valid variable name. +value = { + dict | + list | + i18n_string | + variable | + number | + string_literal +} + +///////////////////////////// +// LIST +///////////////////////////// + +// List of values, e.g. [1, "a"|upper, [2, 3],] +// List items MAY be spread with `*` +list = { + "[" ~ spacing* + ~ ( + list_item ~ spacing* // First value + ~ ("," ~ spacing* ~ list_item ~ spacing*)* // Additional values + ~ ("," ~ spacing*)? // Optional trailing comma + )? + ~ "]" +} + + +// A single list item, which can be spread +// NOTE: Spread operator CAN be surrounded by whitespace +list_item = { + spacing* ~ "*"? ~ spacing* ~ filtered_value +} + + +///////////////////////////// +// DICT +///////////////////////////// + +// Dictionary rules +dict = { + "{" + ~ spacing* // Optional leading whitespace + ~ ( + dict_item ~ (spacing* ~ "," ~ spacing* ~ dict_item)* // 0 or more dict items, separated by commas + )? + ~ spacing* ~ ","? // Optional trailing comma + ~ spacing* ~ "}" +} +dict_item = _{ (dict_item_pair | dict_item_spread) } +dict_item_pair = { dict_key ~ spacing* ~ ":" ~ spacing* ~ filtered_value } // `key: value` pair +dict_item_spread = { "**" ~ spacing* ~ filtered_value } // `**value` spread + +// A filtered key can have filters but not filter arguments +// because filter args also use colon `:`, which conflicts with dict keys. +// So something like `{"key"|lower:arg: value}` would be ambiguous. +dict_key = { + dict_key_inner ~ filter_chain_noarg? +} + +// NOTE: Order matters here - We need to first check for `_("...")` because `_` +// is also a valid variable name. +dict_key_inner = _{ + i18n_string + | variable + | number + | string_literal +} + +///////////////////////////// +// SCALARS AND UTILS +// +// Common value types used in multiple places +///////////////////////////// + +number = _{ + float | int +} + +// Float pattern: Matches numbers with decimal point or scientific notation +float = @{ + ("-" | "+")? ~ + ( + // .42, .42e-10 + ("." ~ ASCII_DIGIT+ ~ ("e" ~ ("-" | "+")? ~ ASCII_DIGIT+)?) | + // 42.42, 42.42e-10 + (ASCII_DIGIT+ ~ "." ~ ASCII_DIGIT* ~ ("e" ~ ("-" | "+")? ~ ASCII_DIGIT+)?) | + // 42e-10 + (ASCII_DIGIT+ ~ "e" ~ ("-" | "+")? ~ ASCII_DIGIT+) + ) +} + +// Int pattern: Matches whole numbers +int = @{ + ("-" | "+")? ~ ASCII_DIGIT+ +} + +// Variable pattern: \w\. +variable = @{ + (ASCII_ALPHA | "_") ~ + (ASCII_ALPHANUMERIC | "_" | ".")* +} + +// String literals (single or double quoted) +string_literal = @{ + double_quoted_string | + single_quoted_string +} + +// Double quoted strings that may contain escaped quote +double_quoted_string = @{ + "\"" ~ (!"\"" ~ ("\\\"" | ANY))* ~ "\"" +} + +// Single quoted strings that may contain escaped quote +single_quoted_string = @{ + "'" ~ (!"'" ~ ("\\'" | ANY))* ~ "'" +} + +// i18n strings: _("string") or _('string') +i18n_string = @{ + "_(" + ~ spacing* + ~ (double_quoted_string | single_quoted_string) + ~ spacing* + ~ ")" +} + +// Spacing includes both whitespace and comments +spacing = _{ WHITESPACE | COMMENT } + +// Spacing that requires at least one WHITESPACE character +// This ensures there's at least one whitespace between tag_name and attributes, +// but allows any number of comments +spacing_with_whitespace = { + (COMMENT* ~ WHITESPACE ~ COMMENT*)+ +} + +// Comments are wrapped in {# ... #} and can contain anything except the closing #} +// Comments may be between attributes, e.g. `key1=val1 {# comment #} key2=val2` +COMMENT = @{ "{#" ~ (!"#}" ~ ANY)* ~ "#}" } + +// NOTE: `_{` in front of a rule means that the characters will be excluded from the AST. +// So in other words, the AST won't contain a "whitespace" node. It will be simply +// blind to whitespace. +WHITESPACE = _{ " " | "\t" | "\n" | "\r" } diff --git a/crates/djc-template-parser/src/lib.rs b/crates/djc-template-parser/src/lib.rs new file mode 100644 index 0000000..0581b41 --- /dev/null +++ b/crates/djc-template-parser/src/lib.rs @@ -0,0 +1,21 @@ +use std::collections::HashSet; +use tag_parser::{ParseError, TagParser}; + +pub mod ast; +pub mod error; +pub mod tag_compiler; +pub mod tag_parser; + +// Re-export the types that users need +pub use ast::{Tag, TagAttr, TagSyntax, TagToken, TagValue, TagValueFilter, ValueKind}; + +/// Parse a template tag string into a Tag AST +pub fn parse_tag(input: &str, flags: Option>) -> Result { + let flags_set = flags.unwrap_or_else(HashSet::new); + TagParser::parse_tag(input, &flags_set) +} + +/// Compile a list of TagAttr to a string +pub fn compile_ast_to_string(attributes: &[TagAttr]) -> Result { + tag_compiler::compile_ast_to_string(attributes) +} diff --git a/crates/djc-template-parser/src/tag_compiler.rs b/crates/djc-template-parser/src/tag_compiler.rs new file mode 100644 index 0000000..6ff2c54 --- /dev/null +++ b/crates/djc-template-parser/src/tag_compiler.rs @@ -0,0 +1,749 @@ +//! # Django Template Tag Compiler +//! +//! This module translates parsed AST representations of Django template tags (e.g. `{% component %}`) +//! into a code definition of a callable Python function (e.g. `def func(context, ...):\n ...`) +//! +//! The generated function takes a `context` object and returns a tuple of arguments and keyword arguments. +//! +//! ## Features +//! +//! - **Argument ordering**: Maintains Python-like behavior with positional args before keyword args +//! - **Spread operators**: Handles `...list` and `**dict` spread syntax +//! - **Filter compilation**: Converts Django filter chains to Python expressions +//! - **Value type handling**: Properly handles strings, numbers, variables, template_strings, etc. +//! - **Error detection**: Compile-time detection of invalid argument ordering +//! - **Indentation**: Properly indents generated code for readability +//! +//! ## Error handling +//! +//! The compiler returns `Result` where errors include: +//! - Invalid argument ordering (positional after keyword) +//! - Compilation failures for complex values +//! - Filter chain compilation errors +//! + +pub use crate::ast::{TagAttr, TagValue, ValueKind}; +use crate::error::CompileError; + +pub fn compile_ast_to_string(attributes: &[TagAttr]) -> Result { + let mut body = String::new(); + // We want to keep Python-like behaviour with args having to come before kwargs. + // When we have only args and kwargs, we can check at compile-time whether + // there are any args after kwargs, and raise an error if so. + // But if there is a spread (`...var`) then this has to be handled at runtime, + // because we don't know if `var` is a mapping or an iterable. + // + // So what we do is that we preferentially raise an error at compile-time. + // And once we come across a spread (`...var`), then we set `kwarg_seen` also in Python, + // and run the checks in both Python and Rust. + let mut has_spread = false; + let mut kwarg_seen = false; + + for attr in attributes { + if attr.is_flag { + continue; + } + + if let Some(key) = &attr.key { + // It's a kwarg: key=value + let value_str = compile_value(&attr.value)?; + body.push_str(&format!( + "kwargs.append(('{}', {}))\n", + key.token, value_str + )); + // Spreads have to be handled at runtime. But before we come across a spread, + // we can check solely at compile-time whether there are any args after kwargs, + // which should raise an error. + if !kwarg_seen { + if has_spread { + body.push_str("kwarg_seen = True\n"); + } + kwarg_seen = true; + } + } else if attr.value.spread.is_some() { + // It's a spread: ...value + if !has_spread { + has_spread = true; + // First time we come across a spread, + // start tracking arg/kwarg orders at run time, + // because we need the Context to know if this spread is a dict or an iterable. + body.push_str(&format!( + "kwarg_seen = {}\n", + if kwarg_seen { "True" } else { "False" } + )); + } + let value_str = compile_value(&attr.value)?; + let raw_token_str = attr.value.token.token.replace("\"", "\\\""); + body.push_str(&format!( + // NOTE: We wrap the raw token in triple quotes because it may contain newlines + "kwarg_seen = _handle_spread({}, \"\"\"{}\"\"\", args, kwargs, kwarg_seen)\n", + value_str, raw_token_str + )); + } else { + // This is a positional arg: value + // Capture args after kwargs at compile time + if kwarg_seen { + return Err(CompileError::from( + "positional argument follows keyword argument", + )); + } + // Capture args after kwargs at run time + if has_spread { + body.push_str("if kwarg_seen:\n"); + body.push_str( + " raise SyntaxError(\"positional argument follows keyword argument\")\n", + ); + } + let value_str = compile_value(&attr.value)?; + body.push_str(&format!("args.append({})\n", value_str)); + } + } + + let mut final_code = String::new(); + let signature = "def compiled_func(context, *, template_string, translation, variable, filter):"; + final_code.push_str(signature); + final_code.push_str("\n"); + + // The top-level `...var` spread may be handled either as a list of args or a dict of kwargs + // depending on the value of `var`. + // So we check if `var` is a mapping by checking for `.keys()` or if it's an iterable. + // We use this helper function to handle this. + if has_spread { + let helper_func = r#"def _handle_spread(value, raw_token_str, args, kwargs, kwarg_seen): + if hasattr(value, "keys"): + kwargs.extend(value.items()) + return True + else: + if kwarg_seen: + raise SyntaxError("positional argument follows keyword argument") + try: + args.extend(value) + except TypeError: + raise TypeError( + f"Value of '...{raw_token_str}' must be a mapping or an iterable, " + f"not {type(value).__name__}." + ) + return False + +"#; + final_code.push_str(&indent_body(&helper_func, 4)); + final_code.push_str("\n"); + } + + final_code.push_str(" args = []\n"); + final_code.push_str(" kwargs = []\n"); + if !body.trim().is_empty() { + final_code.push_str(&indent_body(&body, 4)); + final_code.push_str("\n"); + } + final_code.push_str(" return args, kwargs"); + + Ok(final_code) +} + +fn indent_body(body: &str, indent_level: usize) -> String { + let indent = " ".repeat(indent_level); + body.lines() + .map(|line| { + if line.trim().is_empty() { + String::new() + } else { + format!("{}{}", indent, line) + } + }) + .collect::>() + .join("\n") +} + +fn compile_value(value: &TagValue) -> Result { + let compiled_value = match value.kind { + ValueKind::Int | ValueKind::Float => Ok(value.token.token.clone()), + ValueKind::String => { + // The token includes quotes, which is what we want for a Python string literal + Ok(value.token.token.clone()) + } + ValueKind::Variable => Ok(format!("variable(context, '{}')", value.token.token)), + ValueKind::TemplateString => Ok(format!("template_string(context, {})", value.token.token)), + ValueKind::Translation => { + let inner_string_start = value.token.token.find('(').map(|i| i + 1).unwrap_or(0); + let inner_string_end = value + .token + .token + .rfind(')') + .unwrap_or(value.token.token.len()); + if inner_string_start > 0 && inner_string_end > inner_string_start { + let inner_string = &value.token.token[inner_string_start..inner_string_end]; + Ok(format!("translation(context, {})", inner_string)) + } else { + Err(CompileError::from(format!( + "Invalid translation string format: {}", + value.token.token + ))) + } + } + ValueKind::List => { + let mut items = Vec::new(); + for item in &value.children { + let compiled_item = compile_value(item)?; + if item.spread.is_some() { + items.push(format!("*{}", compiled_item)); + } else { + items.push(compiled_item); + } + } + Ok(format!("[{}]", items.join(", "))) + } + ValueKind::Dict => { + let mut items = Vec::new(); + let mut children_iter = value.children.iter(); + while let Some(child) = children_iter.next() { + if child.spread.is_some() { + items.push(format!("**{}", compile_value(child)?)); + } else { + // This is a key, next must be value + let key = child; + let value = children_iter.next().ok_or_else(|| { + CompileError::from("Dict AST has uneven number of key-value children") + })?; + let compiled_key = compile_value(key)?; + let compiled_value = compile_value(value)?; + items.push(format!("{}: {}", compiled_key, compiled_value)); + } + } + Ok(format!("{{{}}}", items.join(", "))) + } + }; + + let mut result = compiled_value?; + + // Apply filters + for filter in &value.filters { + let filter_name = &filter.token.token; + if let Some(arg) = &filter.arg { + let compiled_arg = compile_value(arg)?; + result = format!( + "filter(context, '{}', {}, {})", + filter_name, result, compiled_arg + ); + } else { + result = format!("filter(context, '{}', {}, None)", filter_name, result); + } + } + + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ast::{TagAttr, TagToken, TagValue, TagValueFilter, ValueKind}; + use crate::tag_parser::TagParser; + use std::collections::HashSet; + + fn create_tag_token(token: &str) -> TagToken { + TagToken { + token: token.to_string(), + start_index: 0, + end_index: token.len(), + line_col: (1, 1), + } + } + + fn create_var_tag_value(token: &str) -> TagValue { + TagValue { + token: create_tag_token(token), + children: vec![], + kind: ValueKind::Variable, + spread: None, + filters: vec![], + start_index: 0, + end_index: token.len(), + line_col: (1, 1), + } + } + + fn create_string_tag_value(token: &str) -> TagValue { + let quoted_token = format!("\"{}\"", token); + TagValue { + token: create_tag_token("ed_token), + children: vec![], + kind: ValueKind::String, + spread: None, + filters: vec![], + start_index: 0, + end_index: quoted_token.len(), + line_col: (1, 1), + } + } + + fn create_expr_tag_value(token: &str) -> TagValue { + let quoted_token = format!("{}", token); + TagValue { + token: create_tag_token("ed_token), + children: vec![], + kind: ValueKind::TemplateString, + spread: None, + filters: vec![], + start_index: 0, + end_index: quoted_token.len(), + line_col: (1, 1), + } + } + + fn create_trans_tag_value(token: &str) -> TagValue { + let trans_token = format!("_({})", token); + TagValue { + token: create_tag_token(&trans_token), + children: vec![], + kind: ValueKind::Translation, + spread: None, + filters: vec![], + start_index: 0, + end_index: trans_token.len(), + line_col: (1, 1), + } + } + + fn create_int_tag_value(val: i32) -> TagValue { + let token = val.to_string(); + TagValue { + token: create_tag_token(&token), + children: vec![], + kind: ValueKind::Int, + spread: None, + filters: vec![], + start_index: 0, + end_index: token.len(), + line_col: (1, 1), + } + } + + fn create_arg_attr(value: TagValue) -> TagAttr { + TagAttr { + key: None, + value, + is_flag: false, + start_index: 0, + end_index: 0, // not important for these tests + line_col: (1, 1), + } + } + + fn create_kwarg_attr(key: &str, value: TagValue) -> TagAttr { + TagAttr { + key: Some(create_tag_token(key)), + value, + is_flag: false, + start_index: 0, + end_index: 0, // not important for these tests + line_col: (1, 1), + } + } + + #[test] + fn test_no_attributes() { + let ast = vec![]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_single_arg() { + let ast = vec![create_arg_attr(create_var_tag_value("my_var"))]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + args.append(variable(context, 'my_var')) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_multiple_args() { + let ast = vec![ + create_arg_attr(create_var_tag_value("my_var")), + create_arg_attr(create_string_tag_value("hello")), + create_arg_attr(create_int_tag_value(123)), + ]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + args.append(variable(context, 'my_var')) + args.append("hello") + args.append(123) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_single_kwarg() { + let ast = vec![create_kwarg_attr("key", create_var_tag_value("my_var"))]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + kwargs.append(('key', variable(context, 'my_var'))) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_multiple_kwargs() { + let ast = vec![ + create_kwarg_attr("key1", create_var_tag_value("my_var")), + create_kwarg_attr("key2", create_string_tag_value("hello")), + ]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + kwargs.append(('key1', variable(context, 'my_var'))) + kwargs.append(('key2', "hello")) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_mixed_args_and_kwargs() { + let ast = vec![ + create_arg_attr(create_int_tag_value(42)), + create_kwarg_attr("key", create_string_tag_value("value")), + ]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + args.append(42) + kwargs.append(('key', "value")) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_spread_kwargs() { + let mut spread_value = create_var_tag_value("options"); + spread_value.spread = Some("...".to_string()); + let ast = vec![ + create_arg_attr(spread_value), + create_kwarg_attr("key", create_string_tag_value("value")), + ]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + def _handle_spread(value, raw_token_str, args, kwargs, kwarg_seen): + if hasattr(value, "keys"): + kwargs.extend(value.items()) + return True + else: + if kwarg_seen: + raise SyntaxError("positional argument follows keyword argument") + try: + args.extend(value) + except TypeError: + raise TypeError( + f"Value of '...{raw_token_str}' must be a mapping or an iterable, " + f"not {type(value).__name__}." + ) + return False + + args = [] + kwargs = [] + kwarg_seen = False + kwarg_seen = _handle_spread(variable(context, 'options'), """options""", args, kwargs, kwarg_seen) + kwargs.append(('key', "value")) + kwarg_seen = True + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_spread_kwargs_order_preserved() { + let mut spread_value = create_var_tag_value("options"); + spread_value.spread = Some("...".to_string()); + let ast = vec![ + create_kwarg_attr("key1", create_string_tag_value("value1")), + create_arg_attr(spread_value), + create_kwarg_attr("key2", create_string_tag_value("value2")), + ]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + def _handle_spread(value, raw_token_str, args, kwargs, kwarg_seen): + if hasattr(value, "keys"): + kwargs.extend(value.items()) + return True + else: + if kwarg_seen: + raise SyntaxError("positional argument follows keyword argument") + try: + args.extend(value) + except TypeError: + raise TypeError( + f"Value of '...{raw_token_str}' must be a mapping or an iterable, " + f"not {type(value).__name__}." + ) + return False + + args = [] + kwargs = [] + kwargs.append(('key1', "value1")) + kwarg_seen = True + kwarg_seen = _handle_spread(variable(context, 'options'), """options""", args, kwargs, kwarg_seen) + kwargs.append(('key2', "value2")) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_template_string_arg() { + let ast = vec![create_arg_attr(create_expr_tag_value("\"{{ my_var }}\""))]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + args.append(template_string(context, "{{ my_var }}")) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_translation_arg() { + let ast = vec![create_arg_attr(create_trans_tag_value("\"hello world\""))]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + args.append(translation(context, "hello world")) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_filter() { + let mut value = create_var_tag_value("my_var"); + value.filters.push(TagValueFilter { + token: create_tag_token("upper"), + arg: None, + start_index: 0, + end_index: 0, + line_col: (1, 1), + }); + let ast = vec![create_arg_attr(value)]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + args.append(filter(context, 'upper', variable(context, 'my_var'), None)) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_filter_with_arg() { + let mut value = create_var_tag_value("my_var"); + value.filters.push(TagValueFilter { + token: create_tag_token("default"), + arg: Some(create_string_tag_value("none")), + start_index: 0, + end_index: 0, + line_col: (1, 1), + }); + let ast = vec![create_arg_attr(value)]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + args.append(filter(context, 'default', variable(context, 'my_var'), "none")) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_multiple_filters() { + let mut value = create_var_tag_value("my_var"); + value.filters.push(TagValueFilter { + token: create_tag_token("upper"), + arg: None, + start_index: 0, + end_index: 0, + line_col: (1, 1), + }); + value.filters.push(TagValueFilter { + token: create_tag_token("default"), + arg: Some(create_string_tag_value("none")), + start_index: 0, + end_index: 0, + line_col: (1, 1), + }); + let ast = vec![create_arg_attr(value)]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + args.append(filter(context, 'default', filter(context, 'upper', variable(context, 'my_var'), None), "none")) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_list_value() { + let list_value = TagValue { + token: create_tag_token("[1, my_var]"), + children: vec![create_int_tag_value(1), create_var_tag_value("my_var")], + kind: ValueKind::List, + spread: None, + filters: vec![], + start_index: 0, + end_index: 0, + line_col: (1, 1), + }; + let ast = vec![create_arg_attr(list_value)]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + args.append([1, variable(context, 'my_var')]) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_dict_value() { + let dict_value = TagValue { + token: create_tag_token("{'key': my_var}"), + children: vec![ + create_string_tag_value("key"), + create_var_tag_value("my_var"), + ], + kind: ValueKind::Dict, + spread: None, + filters: vec![], + start_index: 0, + end_index: 0, + line_col: (1, 1), + }; + let ast = vec![create_kwarg_attr("data", dict_value)]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + kwargs.append(('data', {"key": variable(context, 'my_var')})) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_multiple_spreads() { + let mut spread_value_1 = create_var_tag_value("options1"); + spread_value_1.spread = Some("...".to_string()); + let mut spread_value_2 = create_var_tag_value("options2"); + spread_value_2.spread = Some("...".to_string()); + + let ast = vec![ + create_kwarg_attr("key1", create_string_tag_value("value1")), + create_arg_attr(spread_value_1), + create_kwarg_attr("key2", create_string_tag_value("value2")), + create_arg_attr(spread_value_2), + ]; + let result = compile_ast_to_string(&ast).unwrap(); + + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + def _handle_spread(value, raw_token_str, args, kwargs, kwarg_seen): + if hasattr(value, "keys"): + kwargs.extend(value.items()) + return True + else: + if kwarg_seen: + raise SyntaxError("positional argument follows keyword argument") + try: + args.extend(value) + except TypeError: + raise TypeError( + f"Value of '...{raw_token_str}' must be a mapping or an iterable, " + f"not {type(value).__name__}." + ) + return False + + args = [] + kwargs = [] + kwargs.append(('key1', "value1")) + kwarg_seen = True + kwarg_seen = _handle_spread(variable(context, 'options1'), """options1""", args, kwargs, kwarg_seen) + kwargs.append(('key2', "value2")) + kwarg_seen = _handle_spread(variable(context, 'options2'), """options2""", args, kwargs, kwarg_seen) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_compiler_skips_flags() { + let mut flag_attr = create_arg_attr(create_var_tag_value("disabled")); + flag_attr.is_flag = true; + + let ast = vec![ + create_kwarg_attr("key", create_string_tag_value("value")), + flag_attr, + ]; + let result = compile_ast_to_string(&ast).unwrap(); + let expected = r#"def compiled_func(context, *, template_string, translation, variable, filter): + args = [] + kwargs = [] + kwargs.append(('key', "value")) + return args, kwargs"#; + assert_eq!(result, expected.to_string()); + } + + #[test] + fn test_positional_after_keyword_error() { + let ast = vec![ + create_kwarg_attr("key", create_string_tag_value("value")), + create_arg_attr(create_int_tag_value(42)), + ]; + let result = compile_ast_to_string(&ast); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err(), + CompileError::from("positional argument follows keyword argument") + ); + } + + // ########################################### + // PARAMETER ORDERING TESTS + // ########################################### + + #[test] + fn test_arg_after_kwarg_parses_but_compiler_should_error() { + // The parser should allow this syntax, but the compiler should raise an error + let input = r#"{% component key="value" positional_arg %}"#; + let tag = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + let result = compile_ast_to_string(&tag.attrs); + + assert!(result.is_err()); + let error = result.unwrap_err(); + assert_eq!(error, CompileError::from("positional argument follows keyword argument")); + } + + #[test] + fn test_arg_after_spread_parses_and_compiles() { + // Altho we can see that we're spreading a literal dict, + // spreads are evaluated only at runtime, so this should parse and compile. + let input = r#"{% component ...{"key": "value"} positional_arg %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()); + + assert!(!result.is_err()); + } + + #[test] + fn test_kwarg_after_spread_parse_and_compiles() { + // This is totally fine + let input = r#"{% component ...[1, 2, 3] key="value" %}"#; + let tag = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + let result = compile_ast_to_string(&tag.attrs); + + assert!(!result.is_err()); + } +} diff --git a/crates/djc-template-parser/src/tag_parser.rs b/crates/djc-template-parser/src/tag_parser.rs new file mode 100644 index 0000000..1dc4114 --- /dev/null +++ b/crates/djc-template-parser/src/tag_parser.rs @@ -0,0 +1,8292 @@ +//! # Django Template Tag Parser +//! +//! This module converts Django template tag strings (e.g. `{% component %}`) into an Abstract Syntax Tree (AST). +//! using [Pest](https://pest.rs/) parsing library. +//! +//! The parsing grammar is defined in `grammar.pest` and supports: +//! +//! ## Features +//! +//! - **Complex value types**: strings, numbers, variables, template_strings, translations, lists, dicts +//! - **Filter chains**: `value|filter1|filter2:arg` +//! - **Spread operators**: `...list` and `**dict` +//! - **Comments**: `{# comment #}` within tag content +//! - **Position tracking**: line/column information for error reporting +//! - **Template string detection**: identifies strings with Django template tags inside them +//! - Can be easily extended to support HTML syntax `` +//! +//! ## Error Handling +//! +//! The parser returns `ParseError` for invalid input, which includes: +//! - Pest parsing errors (syntax violations) +//! - Invalid key errors (for malformed attributes) +//! - Automatic conversion to Python `ValueError` for PyO3 integration + +use crate::ast::{Tag, TagAttr, TagSyntax, TagToken, TagValue, TagValueFilter, ValueKind}; +use lazy_static; +use pest::Parser; +use pest_derive::Parser; +use regex; +use std::collections::HashSet; +use thiserror::Error; + +#[derive(Parser)] +#[grammar = "grammar.pest"] +pub struct TagParser; + +#[derive(Error, Debug)] +pub enum ParseError { + #[error("Pest parser error: {0}")] + PestError(#[from] pest::error::Error), + #[error("Invalid key: {0}")] + InvalidKey(String), +} + +impl TagParser { + pub fn parse_tag(input: &str, flags: &HashSet) -> Result { + let wrapper_pair = Self::parse(Rule::tag_wrapper, input)? + .next() + .ok_or_else(|| { + ParseError::PestError(pest::error::Error::new_from_span( + pest::error::ErrorVariant::CustomError { + message: "Empty tag content".to_string(), + }, + pest::Span::new(input, 0, 0).unwrap(), + )) + })?; + + // Get the span from wrapper_pair before moving it + let span = wrapper_pair.as_span(); + let start_index = span.start(); + let end_index = span.end(); + + // Descend into tag_wrapper -> (django_tag | html_tag) + let tag_pair = wrapper_pair.into_inner().next().unwrap(); + + let syntax = match tag_pair.as_rule() { + Rule::django_tag => TagSyntax::Django, + // Rule::html_tag => TagSyntax::Html, // Uncomment to enable HTML syntax `` + _ => unreachable!("Expected django_tag"), + }; + + // Descend into (django_tag | html_tag) -> tag_content + let tag_content_pair = tag_pair.into_inner().next().unwrap(); + + let line_col = tag_content_pair.line_col(); + + let mut inner_pairs = tag_content_pair.into_inner(); + + // First item in a tag is always the tag name + let name_pair = inner_pairs + .next() + .ok_or_else(|| ParseError::InvalidKey("Tag is empty".to_string()))?; + if name_pair.as_rule() != Rule::tag_name { + return Err(ParseError::InvalidKey(format!( + "Expected tag_name, found rule {:?}", + name_pair.as_rule() + ))); + } + + let name_span = name_pair.as_span(); + let name = TagToken { + token: name_pair.as_str().to_string(), + start_index: name_span.start(), + end_index: name_span.end(), + line_col: name_pair.line_col(), + }; + + let mut attributes = Vec::new(); + let mut seen_flags = HashSet::new(); + let mut is_self_closing = false; + + // Parse the attributes + for pair in inner_pairs { + match pair.as_rule() { + Rule::attribute => { + let mut attr = Self::process_attribute(pair)?; + + // Check if this is a flag + if attr.key.is_none() && attr.value.spread.is_none() { + let token = &attr.value.token.token; + if flags.contains(token) { + attr.is_flag = true; + if !seen_flags.insert(token.clone()) { + return Err(ParseError::InvalidKey(format!( + "Flag '{}' may be specified only once.", + token + ))); + } + } + } + + attributes.push(attr); + } + Rule::self_closing_slash => { + is_self_closing = true; + } + _ => { /* Spacing and comments are silent and won't appear here */ } + } + } + + Ok(Tag { + name, + attrs: attributes, + is_self_closing, + syntax, + start_index, + end_index, + line_col, + }) + } + + fn process_attribute(attr_pair: pest::iterators::Pair) -> Result { + let start_index = attr_pair.as_span().start(); + let line_col = attr_pair.line_col(); + + let _attr_str = attr_pair.as_str().to_string(); // Clone the string before moving the pair + let mut inner_pairs = attr_pair.into_inner().peekable(); + + // println!("Processing attribute: {:?}", attr_str); + // if let Some(next_rule) = inner_pairs.peek() { + // println!("Next rule: {:?}", next_rule.as_rule()); + // } + + // Check if this is a key-value pair or just a value + match inner_pairs.peek().map(|p| p.as_rule()) { + Some(Rule::key) => { + // println!("Found key-value pair"); + + // Key + let key_pair = inner_pairs.next().unwrap(); + let key_value = key_pair.as_str().to_string(); + let key_end_index = key_pair.as_span().end(); + + // Value + let value_pair = inner_pairs + .filter(|p| p.as_rule() == Rule::filtered_value) + .next() + .ok_or_else(|| { + ParseError::InvalidKey(format!("Missing value for key: {}", key_value)) + })?; + + let value = Self::process_filtered_value(value_pair)?; + let value_end_index = value.end_index; + + Ok(TagAttr { + key: Some(TagToken { + token: key_value, + start_index, + end_index: key_end_index, + line_col, + }), + value, + is_flag: false, + start_index, + end_index: value_end_index, + line_col, + }) + } + Some(Rule::spread_value) => { + // println!("Found spread value"); + + // Spread value form + let spread_value = inner_pairs.next().unwrap(); + + // println!("Spread value: {:?}", spread_value.as_str()); + // println!("Spread value rule: {:?}", spread_value.as_rule()); + + // Get the value part after the ... operator + let mut value_pairs = spread_value.into_inner(); + let value_pair = value_pairs.next().unwrap(); + + // println!("Value pair: {:?}", value_pair.as_str()); + // println!("Value pair rule: {:?}", value_pair.as_rule()); + + // Process the value part + let mut value = match value_pair.as_rule() { + Rule::filtered_value => Self::process_filtered_value(value_pair)?, + other => { + return Err(ParseError::InvalidKey(format!( + "Expected filtered_value after spread operator, got {:?}", + other + ))) + } + }; + + // Update indices + value.spread = Some("...".to_string()); + value.start_index -= 3; + value.line_col = (value.line_col.0, value.line_col.1 - 3); + + let end_index = value.end_index; + + Ok(TagAttr { + key: None, + value, + is_flag: false, + start_index, + end_index, + line_col, + }) + } + Some(Rule::filtered_value) => { + // println!("Found filtered value"); + + let value_pair = inner_pairs.next().unwrap(); + let value = Self::process_filtered_value(value_pair)?; + let end_index = value.end_index; + + Ok(TagAttr { + key: None, + value, + is_flag: false, + start_index, + end_index, + line_col, + }) + } + _ => unreachable!("Invalid attribute structure"), + } + } + + // Filtered value means that: + // 1. It is "value" - meaning that it is the same as "basic value" + list and dict + // 2. It may have a filter chain after it + // + // E.g. `my_var`, `my_var|filter`, `[1, 2, 3]|filter1|filter2` are all filtered values + fn process_filtered_value( + value_pair: pest::iterators::Pair, + ) -> Result { + // println!("Processing value: {:?}", value_pair.as_str()); + // println!("Rule: {:?}", value_pair.as_rule()); + + let total_span = value_pair.as_span(); + let total_start_index = total_span.start(); + let total_end_index = total_span.end(); + let total_line_col = value_pair.line_col(); + + let mut inner_pairs = value_pair.into_inner(); + + // Get the main value part + let value_part = inner_pairs.next().unwrap(); + + // println!("Value part rule: {:?}", value_part.as_rule()); + // println!("Value part text: {:?}", value_part.as_str()); + // println!("Inner pairs of value_part:"); + // for pair in value_part.clone().into_inner() { + // println!(" Rule: {:?}, Text: {:?}", pair.as_rule(), pair.as_str()); + // } + + let mut result = match value_part.as_rule() { + Rule::value => { + // Get the actual value (stripping the * if present) + let mut inner_pairs = value_part.clone().into_inner(); + let inner_value = inner_pairs.next().unwrap(); + + // println!( + // " Inner value rule: {:?}, Text: {:?}", + // inner_value.as_rule(), + // inner_value.as_str() + // ); + + // Process the value + match inner_value.as_rule() { + Rule::list => { + let list_str = inner_value.as_str().to_string(); + + // println!(" Processing list: {:?}", list_str); + + let span = inner_value.as_span(); + let token_start_index = span.start(); + let token_end_index = span.end(); + let token_line_col = inner_value.line_col(); + + let children = Self::process_list(inner_value)?; + + Ok(TagValue { + token: TagToken { + token: list_str, + start_index: token_start_index, + end_index: token_end_index, + line_col: token_line_col, + }, + spread: None, + filters: vec![], + kind: ValueKind::List, + children, + start_index: total_start_index, + end_index: total_end_index, + line_col: total_line_col, + }) + } + Rule::dict => { + let dict_str = inner_value.as_str().to_string(); + + // println!(" Processing dict: {:?}", dict_str); + + let span = inner_value.as_span(); + let token_start_index = span.start(); + let token_end_index = span.end(); + let token_line_col = inner_value.line_col(); + + let children = Self::process_dict(inner_value)?; + + Ok(TagValue { + token: TagToken { + token: dict_str, + start_index: token_start_index, + end_index: token_end_index, + line_col: token_line_col, + }, + spread: None, + filters: vec![], + kind: ValueKind::Dict, + children, + start_index: total_start_index, + end_index: total_end_index, + line_col: total_line_col, + }) + } + _ => { + let mut result = Self::process_dict_key_inner(inner_value); + + // Update indices + result = result.map(|mut tag_value| { + tag_value.start_index = total_start_index; + tag_value.end_index = total_end_index; + tag_value.line_col = total_line_col; + tag_value + }); + + result + } + } + } + other => Err(ParseError::InvalidKey(format!( + "Expected value, got {:?}", + other + ))), + }; + + // Process any filters + if let Some(filter_chain) = inner_pairs.next() { + result = result.and_then(|mut tag_value| { + tag_value.filters = Self::process_filters(filter_chain)?; + Ok(tag_value) + }); + } + + result + } + + // The value of a dict key is a string, number, or i18n string. + // It cannot be dicts nor lists because keys must be hashable. + // + // NOTE: Basic value is NOT a filtered value + // + // E.g. `my_var`, `42`, `"hello world"`, `_("hello world")` are all basic values + fn process_dict_key_inner( + value_pair: pest::iterators::Pair, + ) -> Result { + // println!( + // "Processing basic value: Rule={:?}, Text={:?}", + // value_pair.as_rule(), + // value_pair.as_str() + // ); + + let start_index = value_pair.as_span().start(); + let end_index = value_pair.as_span().end(); + let line_col = value_pair.line_col(); + + // Determine the value kind, so that downstream processing doesn't need to + let text = value_pair.as_str(); + let kind = match value_pair.as_rule() { + Rule::i18n_string => ValueKind::Translation, + Rule::string_literal => { + if Self::has_template_string(text) { + ValueKind::TemplateString + } else { + ValueKind::String + } + } + Rule::int => ValueKind::Int, + Rule::float => ValueKind::Float, + Rule::variable => ValueKind::Variable, + _ => unreachable!("Invalid basic value {:?}", value_pair.as_rule()), + }; + + // If this is an i18n string, remove the whitespace between `_()` and the text + let mut text = text.to_string(); + if kind == ValueKind::Translation { + // Find the first occurrence of either quote type + let single_quote_pos = text.find('\''); + let double_quote_pos = text.find('"'); + + // Select the quote char that appears first + let quote_char = match (single_quote_pos, double_quote_pos) { + // If both quotes are present, use the one that appears first + (Some(s), Some(d)) if s < d => '\'', + (Some(_), Some(_)) => '"', + // If only one quote is present, use it + (Some(_), None) => '\'', + (None, Some(_)) => '"', + // If no quotes are present, return an error + (None, None) => { + return Err(ParseError::InvalidKey( + "No quotes found in i18n string".to_string(), + )) + } + }; + + let start = text.find(quote_char).unwrap(); + let end = text.rfind(quote_char).unwrap(); + let quoted_part = &text[start..=end]; + text = format!("_({})", quoted_part); + } + + Ok(TagValue { + token: TagToken { + token: text.to_string(), + start_index, + end_index, + line_col, + }, + spread: None, + filters: vec![], + kind, + children: vec![], + line_col, + start_index, + end_index, + }) + } + + // Process a key in a dict that may have filters + fn process_filtered_dict_key( + value_pair: pest::iterators::Pair, + ) -> Result { + // println!( + // "Processing filtered basic value: Rule={:?}, Text={:?}", + // value_pair.as_rule(), + // value_pair.as_str() + // ); + + let total_span = value_pair.as_span(); + let total_start_index = total_span.start(); + let total_end_index = total_span.end(); + let total_line_col = value_pair.line_col(); + + let mut inner_pairs = value_pair.into_inner(); + let dict_key_inner = inner_pairs.next().unwrap(); + let mut result = Self::process_dict_key_inner(dict_key_inner); + + // Update indices + result = result.map(|mut tag_value| { + tag_value.start_index = total_start_index; + tag_value.end_index = total_end_index; + tag_value.line_col = total_line_col; + tag_value + }); + + // Process any filters + if let Some(filter_chain) = inner_pairs.next() { + result = result.and_then(|mut tag_value| { + tag_value.filters = Self::process_filters(filter_chain)?; + Ok(tag_value) + }); + } + + result + } + + fn process_list(inner_value: pest::iterators::Pair) -> Result, ParseError> { + let mut items = Vec::new(); + for item in inner_value.into_inner() { + // println!( + // " ALL list tokens: Rule={:?}, Text={:?}", + // item.as_rule(), + // item.as_str() + // ); + + if item.as_rule() == Rule::list_item { + let has_spread = item.as_str().starts_with('*'); + + // println!(" List item inner tokens:"); + + for inner in item.clone().into_inner() { + // println!( + // " Rule={:?}, Text={:?}", + // inner.as_rule(), + // inner.as_str() + // ); + + if inner.as_rule() == Rule::filtered_value { + let mut tag_value = Self::process_filtered_value(inner)?; + + // Update indices + if has_spread { + tag_value.spread = Some("*".to_string()); + tag_value.start_index -= 1; + tag_value.line_col = (tag_value.line_col.0, tag_value.line_col.1 - 1); + } + items.push(tag_value); + } + } + } + } + Ok(items) + } + + fn process_dict(dict_pair: pest::iterators::Pair) -> Result, ParseError> { + let mut items = Vec::new(); + for item in dict_pair.into_inner() { + // println!( + // " ALL dict tokens: Rule={:?}, Text={:?}", + // item.as_rule(), + // item.as_str() + // ); + + match item.as_rule() { + Rule::dict_item_pair => { + let mut inner = item.into_inner(); + let key_pair = inner.next().unwrap(); + let mut value_pair = inner.next().unwrap(); + + // Skip comments in dict items + while value_pair.as_rule() == Rule::COMMENT { + value_pair = inner.next().unwrap(); + } + + // println!( + // " dict_item_pair: Key={:?}, Value={:?}", + // key_pair.as_str(), + // value_pair.as_str() + // ); + + let key = Self::process_filtered_dict_key(key_pair)?; + let value = Self::process_filtered_value(value_pair)?; + + // println!( + // " dict_item_pair(parsed): Key={:?}, Value={:?}", + // key.token, value.token + // ); + + // Check that key is not a list or dict + match key.kind { + ValueKind::List | ValueKind::Dict => { + return Err(ParseError::InvalidKey( + "Dictionary keys cannot be lists or dictionaries".to_string(), + )); + } + _ => {} + } + items.push(key); + items.push(value); + } + Rule::dict_item_spread => { + let mut inner = item.into_inner(); + let mut value_pair = inner.next().unwrap(); + + // println!(" dict_item_spread: Value={:?}", inner.as_str()); + + // Skip comments in dict items + while value_pair.as_rule() == Rule::COMMENT { + value_pair = inner.next().unwrap(); + } + + let mut value = Self::process_filtered_value(value_pair)?; + + // Update indices + value.spread = Some("**".to_string()); + value.start_index -= 2; + value.line_col = (value.line_col.0, value.line_col.1 - 2); + + // println!(" dict_item_spread(parsed): Value={:?}", value.token); + + items.push(value); + } + Rule::COMMENT => {} + _ => unreachable!("Invalid dictionary item {:?}", item.as_rule()), + } + } + Ok(items) + } + fn process_filters( + filter_chain: pest::iterators::Pair, + ) -> Result, ParseError> { + // Return error if not a filter chain rule + if filter_chain.as_rule() != Rule::filter_chain + && filter_chain.as_rule() != Rule::filter_chain_noarg + { + return Err(ParseError::InvalidKey(format!( + "Expected filter chain, got {:?}", + filter_chain.as_rule() + ))); + } + + let mut filters = Vec::new(); + + // println!( + // "Found rule {:?}, processing filters...", + // filter_chain.as_rule() + // ); + + for filter in filter_chain.into_inner() { + // Skip comments + if filter.as_rule() == Rule::COMMENT { + continue; + } + + // println!("Processing filter: {:?}", filter.as_str()); + + if filter.as_rule() != Rule::filter && filter.as_rule() != Rule::filter_noarg { + return Err(ParseError::InvalidKey(format!( + "Expected filter, got {:?}", + filter.as_rule() + ))); + } + + let filter_span = filter.as_span(); + let filter_start_index = filter_span.start(); + let filter_end_index = filter_span.end(); + let filter_line_col = filter.line_col(); + + // Find the filter name (skipping the pipe token) + let mut filter_parts = filter.into_inner(); + let filter_pair = filter_parts + .find(|p| p.as_rule() == Rule::filter_name) + .unwrap(); + let filter_name = filter_pair.as_str().to_string(); + let token_start_index = filter_pair.as_span().start(); + let token_end_index = filter_pair.as_span().end(); + let token_line_col = filter_pair.line_col(); + + // println!("Found filter name: {:?}", filter_name); + + let filter_arg = if let Some(arg_part) = + filter_parts.find(|p| p.as_rule() == Rule::filter_arg_part) + { + // Position, includeing the `:` + let arg_span = arg_part.as_span(); + let arg_start_index = arg_span.start(); + let arg_end_index = arg_span.end(); + let arg_line_col = arg_part.line_col(); + + let arg_value_pair: pest::iterators::Pair<'_, Rule> = arg_part + .into_inner() + .find(|p| p.as_rule() == Rule::filter_arg) + .unwrap(); + + // Process the filter argument as a TagValue + let mut result = Self::process_filtered_value(arg_value_pair)?; + + // Update indices + result.start_index = arg_start_index; + result.end_index = arg_end_index; + result.line_col = arg_line_col; + Some(result) + } else { + None + }; + + filters.push(TagValueFilter { + arg: filter_arg, + token: TagToken { + token: filter_name, + start_index: token_start_index, + end_index: token_end_index, + line_col: token_line_col, + }, + start_index: filter_start_index, + end_index: filter_end_index, + line_col: filter_line_col, + }); + + // println!("Added filter to chain: {:?}", filters.last().unwrap()); + } + + // println!( + // "Completed processing filter chain, returning {:?} filters", + // filters.len() + // ); + + Ok(filters) + } + + fn has_template_string(s: &str) -> bool { + // Don't check for template strings in i18n strings + if s.starts_with("_(") { + return false; + } + + // Check for any of the Django template tags with their closing tags + // The pattern ensures that: + // 1. Opening and closing tags are properly paired + // 2. Tags are in the correct order (no closing before opening) + lazy_static::lazy_static! { + static ref VAR_TAG: regex::Regex = regex::Regex::new(r"\{\{.*?\}\}").unwrap(); + static ref BLOCK_TAG: regex::Regex = regex::Regex::new(r"\{%.*?%\}").unwrap(); + static ref COMMENT_TAG: regex::Regex = regex::Regex::new(r"\{#.*?#\}").unwrap(); + } + + VAR_TAG.is_match(s) || BLOCK_TAG.is_match(s) || COMMENT_TAG.is_match(s) + } +} + +#[cfg(test)] +mod tests { + use std::vec; + + use super::*; + + #[test] + fn test_arg_single_variable() { + // Test simple variable name + let input = "{% my_tag val %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "val".to_string(), + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 13, + line_col: (1, 11), + },], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 16, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_variable_with_dots() { + // Test variable with dots + let input = "{% my_tag my.nested.value %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "my.nested.value".to_string(), + start_index: 10, + end_index: 25, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 10, + end_index: 25, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 25, + line_col: (1, 11), + },], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 28, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_number_1() { + let input = "{% my_tag 42 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "42".to_string(), + start_index: 10, + end_index: 12, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 10, + end_index: 12, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 12, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 15, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_number_2() { + let input = "{% my_tag 001 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "001".to_string(), + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 13, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 16, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_number_with_decimal_1() { + let input = "{% my_tag -1.5 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "-1.5".to_string(), + start_index: 10, + end_index: 14, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Float, + start_index: 10, + end_index: 14, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 14, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 17, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_number_with_decimal_2() { + let input = "{% my_tag +2. %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "+2.".to_string(), + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Float, + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 13, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 16, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_number_with_decimal_3() { + let input = "{% my_tag .3 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: ".3".to_string(), + start_index: 10, + end_index: 12, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Float, + start_index: 10, + end_index: 12, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 12, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 15, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_number_scientific_1() { + let input = "{% my_tag -1.2e2 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "-1.2e2".to_string(), + start_index: 10, + end_index: 16, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Float, + start_index: 10, + end_index: 16, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 16, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 19, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_number_scientific_2() { + let input = "{% my_tag .2e-02 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: ".2e-02".to_string(), + start_index: 10, + end_index: 16, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Float, + start_index: 10, + end_index: 16, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 16, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 19, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_number_scientific_3() { + let input = "{% my_tag 20.e+02 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "20.e+02".to_string(), + start_index: 10, + end_index: 17, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Float, + start_index: 10, + end_index: 17, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 17, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 20, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_quoted_string() { + // Test single quoted string + let input = r#"{% my_tag 'hello world' %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "'hello world'".to_string(), + start_index: 10, + end_index: 23, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 10, + end_index: 23, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 23, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 26, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_double_quoted_string() { + // Test double quoted string + let input = r#"{% my_tag "hello world" %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#""hello world""#.to_string(), + start_index: 10, + end_index: 23, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 10, + end_index: 23, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 23, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 26, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_i18n_string() { + let input = r#"{% my_tag _('hello world') %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "_('hello world')".to_string(), + start_index: 10, + end_index: 26, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Translation, + start_index: 10, + end_index: 26, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 26, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 29, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_i18n_string_with_double_quotes() { + let input = r#"{% my_tag _("hello world") %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"_("hello world")"#.to_string(), + start_index: 10, + end_index: 26, + line_col: (1, 11), + }, + children: vec![], + kind: ValueKind::Translation, + spread: None, + filters: vec![], + start_index: 10, + end_index: 26, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 26, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 29, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_arg_single_whitespace() { + let input = "{% my_tag val %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "val".to_string(), + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 13, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 16, + line_col: (1, 4), + } + ); + } + #[test] + fn test_arg_multiple() { + let input = r#"{% my_tag component value1 value2 %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![ + TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "component".to_string(), + start_index: 10, + end_index: 19, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 10, + end_index: 19, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 19, + line_col: (1, 11), + }, + TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value1".to_string(), + start_index: 20, + end_index: 26, + line_col: (1, 21), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 20, + end_index: 26, + line_col: (1, 21), + }, + is_flag: false, + start_index: 20, + end_index: 26, + line_col: (1, 21), + }, + TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value2".to_string(), + start_index: 27, + end_index: 33, + line_col: (1, 28), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 27, + end_index: 33, + line_col: (1, 28), + }, + is_flag: false, + start_index: 27, + end_index: 33, + line_col: (1, 28), + } + ], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 36, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_kwarg_single() { + let input = r#"{% my_tag key=val %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: Some(TagToken { + token: "key".to_string(), + start_index: 10, + end_index: 13, + line_col: (1, 11), + }), + value: TagValue { + token: TagToken { + token: "val".to_string(), + start_index: 14, + end_index: 17, + line_col: (1, 15), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 14, + end_index: 17, + line_col: (1, 15), + }, + is_flag: false, + start_index: 10, + end_index: 17, + line_col: (1, 11), + },], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 20, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_kwarg_single_whitespace() { + let input = r#"{% my_tag key=val %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: Some(TagToken { + token: "key".to_string(), + start_index: 10, + end_index: 13, + line_col: (1, 11), + }), + value: TagValue { + token: TagToken { + token: "val".to_string(), + start_index: 14, + end_index: 17, + line_col: (1, 15), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 14, + end_index: 17, + line_col: (1, 15), + }, + is_flag: false, + start_index: 10, + end_index: 17, + line_col: (1, 11), + },], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 20, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_kwarg_multiple() { + let input = r#"{% my_tag key=val key2=val2 %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![ + TagAttr { + key: Some(TagToken { + token: "key".to_string(), + start_index: 10, + end_index: 13, + line_col: (1, 11), + }), + value: TagValue { + token: TagToken { + token: "val".to_string(), + start_index: 14, + end_index: 17, + line_col: (1, 15), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 14, + end_index: 17, + line_col: (1, 15), + }, + is_flag: false, + start_index: 10, + end_index: 17, + line_col: (1, 11), + }, + TagAttr { + key: Some(TagToken { + token: "key2".to_string(), + start_index: 18, + end_index: 22, + line_col: (1, 19), + }), + value: TagValue { + token: TagToken { + token: "val2".to_string(), + start_index: 23, + end_index: 27, + line_col: (1, 24), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 23, + end_index: 27, + line_col: (1, 24), + }, + is_flag: false, + start_index: 18, + end_index: 27, + line_col: (1, 19), + } + ], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 30, + line_col: (1, 4), + } + ); + } + + // Test that we do NOT allow whitespace around the `=`, e.g. `key= val`, `key =val`, `key = val` + #[test] + fn test_kwarg_whitespace_around_equals_1() { + // Test whitespace after key + let input = "{% my_tag key= val %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow whitespace after key before equals" + ); + } + #[test] + fn test_kwarg_whitespace_around_equals_2() { + // Test whitespace before value + let input = "{% my_tag key =val %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow whitespace before value after equals" + ); + } + #[test] + fn test_kwarg_whitespace_around_equals_3() { + // Test whitespace on both sides + let input = "{% my_tag key = val %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow whitespace around equals" + ); + } + #[test] + fn test_kwarg_whitespace_around_equals_4() { + // Test multiple attributes with mixed whitespace + let input = "{% my_tag key1= val1 key2 =val2 key3 = val3 %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow whitespace around equals in any attribute" + ); + } + + #[test] + fn test_kwarg_special_chars() { + let input = r#"{% my_tag @click.stop=handler attr:key=val %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![ + TagAttr { + key: Some(TagToken { + token: "@click.stop".to_string(), + start_index: 10, + end_index: 21, + line_col: (1, 11), + }), + value: TagValue { + token: TagToken { + token: "handler".to_string(), + start_index: 22, + end_index: 29, + line_col: (1, 23), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 22, + end_index: 29, + line_col: (1, 23), + }, + is_flag: false, + start_index: 10, + end_index: 29, + line_col: (1, 11), + }, + TagAttr { + key: Some(TagToken { + token: "attr:key".to_string(), + start_index: 30, + end_index: 38, + line_col: (1, 31), + }), + value: TagValue { + token: TagToken { + token: "val".to_string(), + start_index: 39, + end_index: 42, + line_col: (1, 40), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 39, + end_index: 42, + line_col: (1, 40), + }, + is_flag: false, + start_index: 30, + end_index: 42, + line_col: (1, 31), + } + ], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 45, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_kwarg_invalid() { + let inputs = vec![ + "{% my_tag :key=val %}", + "{% my_tag ...key=val %}", + "{% my_tag _('hello')=val %}", + r#"{% my_tag "key"=val %}"#, + "{% my_tag key[0]=val %}", + ]; + + for input in inputs { + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Input should fail: {}", + input + ); + } + } + + #[test] + fn test_comment_before() { + let input = r#"{% my_tag {# comment #} value %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 24, + end_index: 29, + line_col: (1, 25), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 24, + end_index: 29, + line_col: (1, 25), + }, + is_flag: false, + start_index: 24, + end_index: 29, + line_col: (1, 25), + },], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 32, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_comment_after() { + // Test comment after attribute + let input = "{% my_tag key=val{# comment #} %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: Some(TagToken { + token: "key".to_string(), + start_index: 10, + end_index: 13, + line_col: (1, 11), + }), + value: TagValue { + token: TagToken { + token: "val".to_string(), + start_index: 14, + end_index: 17, + line_col: (1, 15), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 14, + end_index: 17, + line_col: (1, 15), + }, + is_flag: false, + start_index: 10, + end_index: 17, + line_col: (1, 11), + },], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 33, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_comment_between() { + let input = "{% my_tag key1=val1 {# comment #} key2=val2 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![ + TagAttr { + key: Some(TagToken { + token: "key1".to_string(), + start_index: 10, + end_index: 14, + line_col: (1, 11), + }), + value: TagValue { + token: TagToken { + token: "val1".to_string(), + start_index: 15, + end_index: 19, + line_col: (1, 16), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 15, + end_index: 19, + line_col: (1, 16), + }, + is_flag: false, + start_index: 10, + end_index: 19, + line_col: (1, 11), + }, + TagAttr { + key: Some(TagToken { + token: "key2".to_string(), + start_index: 34, + end_index: 38, + line_col: (1, 35), + }), + value: TagValue { + token: TagToken { + token: "val2".to_string(), + start_index: 39, + end_index: 43, + line_col: (1, 40), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 39, + end_index: 43, + line_col: (1, 40), + }, + is_flag: false, + start_index: 34, + end_index: 43, + line_col: (1, 35), + } + ], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 46, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_comment_multiple() { + // Test multiple comments + // {% my_tag {# c1 #}key1=val1{# c2 #} {# c3 #}key2=val2{# c4 #} %} + // Position breakdown: + // 0-2: {% + // 3-9: my_tag + // 10-18: {# c1 #} + // 18-22: key1 + // 23-27: val1 + // 27-36: {# c2 #} + // 37-46: {# c3 #} + // 46-50: key2 (but actual test shows it's at 44-48) + let input = "{% my_tag {# c1 #}key1=val1{# c2 #} {# c3 #}key2=val2{# c4 #} %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![ + TagAttr { + key: Some(TagToken { + token: "key1".to_string(), + start_index: 18, + end_index: 22, + line_col: (1, 19), + }), + value: TagValue { + token: TagToken { + token: "val1".to_string(), + start_index: 23, + end_index: 27, + line_col: (1, 24), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 23, + end_index: 27, + line_col: (1, 24), + }, + is_flag: false, + start_index: 18, + end_index: 27, + line_col: (1, 19), + }, + TagAttr { + key: Some(TagToken { + token: "key2".to_string(), + start_index: 44, + end_index: 48, + line_col: (1, 45), + }), + value: TagValue { + token: TagToken { + token: "val2".to_string(), + start_index: 49, + end_index: 53, + line_col: (1, 50), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 49, + end_index: 53, + line_col: (1, 50), + }, + is_flag: false, + start_index: 44, + end_index: 53, + line_col: (1, 45), + } + ], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 64, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_comment_no_whitespace() { + // Test that comments without whitespace between tag_name and attributes should fail + // because we require at least one WHITESPACE (not just comments) + let input = "{% my_tag {# c1 #}key1=val1{# c2 #}key2=val2{# c3 #} %}"; + let result = TagParser::parse_tag(input, &HashSet::new()); + assert!(result.is_err(), "Should error when there's no whitespace between tag_name and attributes (only comments)"); + } + + #[test] + fn test_comment_with_newlines() { + // Test comment with newlines + let input = "{% my_tag key1=val1 {# multi\nline\ncomment #} key2=val2 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![ + TagAttr { + key: Some(TagToken { + token: "key1".to_string(), + start_index: 10, + end_index: 14, + line_col: (1, 11), + }), + value: TagValue { + token: TagToken { + token: "val1".to_string(), + start_index: 15, + end_index: 19, + line_col: (1, 16), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 15, + end_index: 19, + line_col: (1, 16), + }, + is_flag: false, + start_index: 10, + end_index: 19, + line_col: (1, 11), + }, + TagAttr { + key: Some(TagToken { + token: "key2".to_string(), + start_index: 45, + end_index: 49, + line_col: (3, 12), + }), + value: TagValue { + token: TagToken { + token: "val2".to_string(), + start_index: 50, + end_index: 54, + line_col: (3, 17), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 50, + end_index: 54, + line_col: (3, 17), + }, + is_flag: false, + start_index: 45, + end_index: 54, + line_col: (3, 12), + } + ], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 57, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_comment_not_allowed_between_key_and_value() { + // Test comment between key and equals + let input = "{% my_tag key{# comment #}=val %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow comment between key and equals" + ); + + // Test comment between equals and value + let input = "{% my_tag key={# comment #}val %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow comment between equals and value" + ); + } + + #[test] + fn test_spread_basic() { + let input = "{% my_tag ...myvalue %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "myvalue".to_string(), + start_index: 13, + end_index: 20, + line_col: (1, 14), + }, + children: vec![], + spread: Some("...".to_string()), + filters: vec![], + kind: ValueKind::Variable, + start_index: 10, + end_index: 20, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 20, + line_col: (1, 11), + },], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 23, + line_col: (1, 4), + } + ); + } + #[test] + fn test_spread_between() { + // Test spread with other attributes + let input = "{% my_tag key1=val1 ...myvalue key2=val2 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![ + TagAttr { + key: Some(TagToken { + token: "key1".to_string(), + start_index: 10, + end_index: 14, + line_col: (1, 11), + }), + value: TagValue { + token: TagToken { + token: "val1".to_string(), + start_index: 15, + end_index: 19, + line_col: (1, 16), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 15, + end_index: 19, + line_col: (1, 16), + }, + is_flag: false, + start_index: 10, + end_index: 19, + line_col: (1, 11), + }, + TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "myvalue".to_string(), + start_index: 23, + end_index: 30, + line_col: (1, 24), + }, + children: vec![], + spread: Some("...".to_string()), + filters: vec![], + kind: ValueKind::Variable, + start_index: 20, + end_index: 30, + line_col: (1, 21), + }, + is_flag: false, + start_index: 20, + end_index: 30, + line_col: (1, 21), + }, + TagAttr { + key: Some(TagToken { + token: "key2".to_string(), + start_index: 31, + end_index: 35, + line_col: (1, 32), + }), + value: TagValue { + token: TagToken { + token: "val2".to_string(), + start_index: 36, + end_index: 40, + line_col: (1, 37), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 36, + end_index: 40, + line_col: (1, 37), + }, + is_flag: false, + start_index: 31, + end_index: 40, + line_col: (1, 32), + } + ], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 43, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_spread_multiple() { + let input = "{% my_tag ...dict1 key=val ...dict2 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![ + TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "dict1".to_string(), + start_index: 13, + end_index: 18, + line_col: (1, 14), + }, + children: vec![], + spread: Some("...".to_string()), + filters: vec![], + kind: ValueKind::Variable, + start_index: 10, + end_index: 18, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 18, + line_col: (1, 11), + }, + TagAttr { + key: Some(TagToken { + token: "key".to_string(), + start_index: 19, + end_index: 22, + line_col: (1, 20), + }), + value: TagValue { + token: TagToken { + token: "val".to_string(), + start_index: 23, + end_index: 26, + line_col: (1, 24), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 23, + end_index: 26, + line_col: (1, 24), + }, + is_flag: false, + start_index: 19, + end_index: 26, + line_col: (1, 20), + }, + TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "dict2".to_string(), + start_index: 30, + end_index: 35, + line_col: (1, 31), + }, + children: vec![], + spread: Some("...".to_string()), + filters: vec![], + kind: ValueKind::Variable, + start_index: 27, + end_index: 35, + line_col: (1, 28), + }, + is_flag: false, + start_index: 27, + end_index: 35, + line_col: (1, 28), + } + ], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 38, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_spread_dict() { + // Test spread with dictionary + let input = r#"{% my_tag ...{"key": "value"} %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "{\"key\": \"value\"}".to_string(), + start_index: 13, + end_index: 29, + line_col: (1, 14), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key\"".to_string(), + start_index: 14, + end_index: 19, + line_col: (1, 15), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 14, + end_index: 19, + line_col: (1, 15), + }, + TagValue { + token: TagToken { + token: "\"value\"".to_string(), + start_index: 21, + end_index: 28, + line_col: (1, 22), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 21, + end_index: 28, + line_col: (1, 22), + }, + ], + spread: Some("...".to_string()), + filters: vec![], + kind: ValueKind::Dict, + start_index: 10, + end_index: 29, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 29, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 32, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_spread_list() { + let input = "{% my_tag ...[1, 2, 3] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "[1, 2, 3]".to_string(), + start_index: 13, + end_index: 22, + line_col: (1, 14), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 14, + end_index: 15, + line_col: (1, 15), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 14, + end_index: 15, + line_col: (1, 15), + }, + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 17, + end_index: 18, + line_col: (1, 18), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 17, + end_index: 18, + line_col: (1, 18), + }, + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 20, + end_index: 21, + line_col: (1, 21), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 20, + end_index: 21, + line_col: (1, 21), + } + ], + spread: Some("...".to_string()), + filters: vec![], + kind: ValueKind::List, + start_index: 10, + end_index: 22, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 22, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 25, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_spread_i18n() { + // Test spread with i18n string + let input = "{% my_tag ..._('hello') %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "_('hello')".to_string(), + start_index: 13, + end_index: 23, + line_col: (1, 14), + }, + children: vec![], + spread: Some("...".to_string()), + filters: vec![], + kind: ValueKind::Translation, + start_index: 10, + end_index: 23, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 23, + line_col: (1, 11), + },], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 26, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_spread_variable() { + // Test spread with variable + let input = "{% my_tag ...my_var %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "my_var".to_string(), + start_index: 13, + end_index: 19, + line_col: (1, 14), + }, + children: vec![], + spread: Some("...".to_string()), + filters: vec![], + kind: ValueKind::Variable, + start_index: 10, + end_index: 19, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 19, + line_col: (1, 11), + },], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 22, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_spread_number() { + // Test spread with number + let input = "{% my_tag ...42 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "42".to_string(), + start_index: 13, + end_index: 15, + line_col: (1, 14), + }, + children: vec![], + spread: Some("...".to_string()), + filters: vec![], + kind: ValueKind::Int, + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 15, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 18, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_spread_string() { + // Test spread with string literal + let input = r#"{% my_tag ..."hello" %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "\"hello\"".to_string(), + start_index: 13, + end_index: 20, + line_col: (1, 14), + }, + children: vec![], + spread: Some("...".to_string()), + filters: vec![], + kind: ValueKind::String, + start_index: 10, + end_index: 20, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 20, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 23, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_spread_invalid() { + // Test spread missing value + let input = "{% my_tag ... %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow spread operator without a value" + ); + // Test spread whitespace between operator and value + let input = "{% my_tag ... myvalue %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow spread operator with whitespace between operator and value" + ); + + // Test spread in key position + let input = "{% my_tag ...key=val %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow spread operator in key position" + ); + + // Test spread in value position of key-value pair + let input = "{% my_tag key=...val %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow spread operator in value position of key-value pair" + ); + + // Test spread operator inside list + let input = "{% my_tag [1, ...my_list, 2] %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow ... spread operator inside list" + ); + + // Test spread operator inside list with filters + let input = "{% my_tag [1, ...my_list|filter, 2] %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow ... spread operator inside list with filters" + ); + + // Test spread operator inside nested list + let input = "{% my_tag [1, [...my_list], 2] %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow ... spread operator inside nested list" + ); + } + + #[test] + fn test_filter_basic() { + let input = "{% my_tag value|lower %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + arg: None, + token: TagToken { + token: "lower".to_string(), + start_index: 16, + end_index: 21, + line_col: (1, 17), + }, + start_index: 15, + end_index: 21, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 21, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 21, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 24, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_filter_multiple() { + let input = "{% my_tag value|lower|title|default:'hello' %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + kind: ValueKind::Variable, + spread: None, + filters: vec![ + TagValueFilter { + token: TagToken { + token: "lower".to_string(), + start_index: 16, + end_index: 21, + line_col: (1, 17), + }, + arg: None, + start_index: 15, + end_index: 21, + line_col: (1, 16), + }, + TagValueFilter { + token: TagToken { + token: "title".to_string(), + start_index: 22, + end_index: 27, + line_col: (1, 23), + }, + arg: None, + start_index: 21, + end_index: 27, + line_col: (1, 22), + }, + TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 28, + end_index: 35, + line_col: (1, 29), + }, + arg: Some(TagValue { + token: TagToken { + token: "'hello'".to_string(), + start_index: 36, + end_index: 43, + line_col: (1, 37), + }, + children: vec![], + kind: ValueKind::String, + spread: None, + filters: vec![], + start_index: 35, + end_index: 43, + line_col: (1, 36), + }), + start_index: 27, + end_index: 43, + line_col: (1, 28), + } + ], + start_index: 10, + end_index: 43, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 43, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 46, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_filter_arg_string() { + let input = "{% my_tag value|default:'hello' %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 16, + end_index: 23, + line_col: (1, 17), + }, + arg: Some(TagValue { + token: TagToken { + token: "'hello'".to_string(), + start_index: 24, + end_index: 31, + line_col: (1, 25), + }, + children: vec![], + kind: ValueKind::String, + spread: None, + filters: vec![], + start_index: 23, + end_index: 31, + line_col: (1, 24), + }), + start_index: 15, + end_index: 31, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 31, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 31, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 34, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_filter_arg_number() { + let input = "{% my_tag value|add:42 %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "add".to_string(), + start_index: 16, + end_index: 19, + line_col: (1, 17), + }, + arg: Some(TagValue { + token: TagToken { + token: "42".to_string(), + start_index: 20, + end_index: 22, + line_col: (1, 21), + }, + children: vec![], + kind: ValueKind::Int, + spread: None, + filters: vec![], + start_index: 19, + end_index: 22, + line_col: (1, 20), + }), + start_index: 15, + end_index: 22, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 22, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 22, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 25, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_filter_arg_variable() { + let input = "{% my_tag value|default:my_var.field %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 16, + end_index: 23, + line_col: (1, 17), + }, + arg: Some(TagValue { + token: TagToken { + token: "my_var.field".to_string(), + start_index: 24, + end_index: 36, + line_col: (1, 25), + }, + children: vec![], + kind: ValueKind::Variable, + spread: None, + filters: vec![], + start_index: 23, + end_index: 36, + line_col: (1, 24), + }), + start_index: 15, + end_index: 36, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 36, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 36, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 39, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_filter_arg_i18n() { + let input = "{% my_tag value|default:_('hello') %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 16, + end_index: 23, + line_col: (1, 17), + }, + arg: Some(TagValue { + token: TagToken { + token: "_('hello')".to_string(), + start_index: 24, + end_index: 34, + line_col: (1, 25), + }, + children: vec![], + kind: ValueKind::Translation, + spread: None, + filters: vec![], + start_index: 23, + end_index: 34, + line_col: (1, 24), + }), + start_index: 15, + end_index: 34, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 34, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 34, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 37, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_filter_arg_list() { + let input = "{% my_tag value|default:[1, 2, 3] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 16, + end_index: 23, + line_col: (1, 17), + }, + arg: Some(TagValue { + token: TagToken { + token: "[1, 2, 3]".to_string(), + start_index: 24, + end_index: 33, + line_col: (1, 25), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 25, + end_index: 26, + line_col: (1, 26), + }, + children: vec![], + kind: ValueKind::Int, + spread: None, + filters: vec![], + start_index: 25, + end_index: 26, + line_col: (1, 26), + }, + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 28, + end_index: 29, + line_col: (1, 29), + }, + children: vec![], + kind: ValueKind::Int, + spread: None, + filters: vec![], + start_index: 28, + end_index: 29, + line_col: (1, 29), + }, + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 31, + end_index: 32, + line_col: (1, 32), + }, + children: vec![], + kind: ValueKind::Int, + spread: None, + filters: vec![], + start_index: 31, + end_index: 32, + line_col: (1, 32), + }, + ], + kind: ValueKind::List, + spread: None, + filters: vec![], + start_index: 23, + end_index: 33, + line_col: (1, 24), + }), + start_index: 15, + end_index: 33, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 33, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 33, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 36, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_filter_arg_dict() { + let input = r#"{% my_tag value|default:{"key": "val"} %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 16, + end_index: 23, + line_col: (1, 17), + }, + arg: Some(TagValue { + token: TagToken { + token: "{\"key\": \"val\"}".to_string(), + start_index: 24, + end_index: 38, + line_col: (1, 25), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key\"".to_string(), + start_index: 25, + end_index: 30, + line_col: (1, 26), + }, + children: vec![], + kind: ValueKind::String, + spread: None, + filters: vec![], + start_index: 25, + end_index: 30, + line_col: (1, 26), + }, + TagValue { + token: TagToken { + token: "\"val\"".to_string(), + start_index: 32, + end_index: 37, + line_col: (1, 33), + }, + children: vec![], + kind: ValueKind::String, + spread: None, + filters: vec![], + start_index: 32, + end_index: 37, + line_col: (1, 33), + }, + ], + kind: ValueKind::Dict, + spread: None, + filters: vec![], + start_index: 23, + end_index: 38, + line_col: (1, 24), + }), + start_index: 15, + end_index: 38, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 38, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 38, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 41, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_filter_arg_template_string() { + let input = r#"{% my_tag value|default:"{{ var }}" %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 16, + end_index: 23, + line_col: (1, 17), + }, + arg: Some(TagValue { + token: TagToken { + token: "\"{{ var }}\"".to_string(), + start_index: 24, + end_index: 35, + line_col: (1, 25), + }, + children: vec![], + kind: ValueKind::TemplateString, + spread: None, + filters: vec![], + start_index: 23, + end_index: 35, + line_col: (1, 24), + }), + start_index: 15, + end_index: 35, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 35, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 35, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 38, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_filter_arg_nested() { + let input = r#"{% my_tag value|default:[1, {"key": "val"}, _("hello")] %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 16, + end_index: 23, + line_col: (1, 17), + }, + arg: Some(TagValue { + token: TagToken { + token: "[1, {\"key\": \"val\"}, _(\"hello\")]".to_string(), + start_index: 24, + end_index: 55, + line_col: (1, 25), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 25, + end_index: 26, + line_col: (1, 26), + }, + children: vec![], + kind: ValueKind::Int, + spread: None, + filters: vec![], + start_index: 25, + end_index: 26, + line_col: (1, 26), + }, + TagValue { + token: TagToken { + token: "{\"key\": \"val\"}".to_string(), + start_index: 28, + end_index: 42, + line_col: (1, 29), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key\"".to_string(), + start_index: 29, + end_index: 34, + line_col: (1, 30), + }, + children: vec![], + kind: ValueKind::String, + spread: None, + filters: vec![], + start_index: 29, + end_index: 34, + line_col: (1, 30), + }, + TagValue { + token: TagToken { + token: "\"val\"".to_string(), + start_index: 36, + end_index: 41, + line_col: (1, 37), + }, + children: vec![], + kind: ValueKind::String, + spread: None, + filters: vec![], + start_index: 36, + end_index: 41, + line_col: (1, 37), + }, + ], + kind: ValueKind::Dict, + spread: None, + filters: vec![], + start_index: 28, + end_index: 42, + line_col: (1, 29), + }, + TagValue { + token: TagToken { + token: "_(\"hello\")".to_string(), + start_index: 44, + end_index: 54, + line_col: (1, 45), + }, + children: vec![], + kind: ValueKind::Translation, + spread: None, + filters: vec![], + start_index: 44, + end_index: 54, + line_col: (1, 45), + }, + ], + kind: ValueKind::List, + spread: None, + filters: vec![], + start_index: 23, + end_index: 55, + line_col: (1, 24), + }), + start_index: 15, + end_index: 55, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 55, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 55, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 58, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_filter_invalid() { + // Test using colon instead of pipe + let input = "{% my_tag value:filter %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow colon instead of pipe for filter" + ); + + // Test using colon with filter argument + let input = "{% my_tag value:filter:arg %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow colon instead of pipe for filter with argument" + ); + + // Test using colon after a valid filter + let input = "{% my_tag value|filter:arg:filter2 %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow colon to start a new filter after an argument" + ); + } + + #[test] + fn test_i18n_whitespace() { + let input = "{% my_tag value|default:_( 'hello' ) %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 16, + end_index: 23, + line_col: (1, 17), + }, + arg: Some(TagValue { + token: TagToken { + token: "_('hello')".to_string(), + start_index: 24, + end_index: 36, + line_col: (1, 25), + }, + children: vec![], + kind: ValueKind::Translation, + spread: None, + filters: vec![], + start_index: 23, + end_index: 36, + line_col: (1, 24), + }), + start_index: 15, + end_index: 36, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 36, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 36, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 39, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_i18n_comments() { + let input = "{% my_tag value|default:_({# open paren #}'hello'{# close paren #}) %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 16, + end_index: 23, + line_col: (1, 17), + }, + arg: Some(TagValue { + token: TagToken { + token: "_('hello')".to_string(), + start_index: 24, + end_index: 67, + line_col: (1, 25), + }, + children: vec![], + kind: ValueKind::Translation, + spread: None, + filters: vec![], + start_index: 23, + end_index: 67, + line_col: (1, 24), + }), + start_index: 15, + end_index: 67, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 67, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 67, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 70, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_empty() { + // Empty list + let input = "{% my_tag [] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "[]".to_string(), + start_index: 10, + end_index: 12, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 10, + end_index: 12, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 12, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 15, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_basic() { + // Simple list with numbers + let input = "{% my_tag [1, 2, 3] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "[1, 2, 3]".to_string(), + start_index: 10, + end_index: 19, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 14, + end_index: 15, + line_col: (1, 15), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 14, + end_index: 15, + line_col: (1, 15), + }, + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 17, + end_index: 18, + line_col: (1, 18), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 17, + end_index: 18, + line_col: (1, 18), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 10, + end_index: 19, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 19, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 22, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_mixed() { + // List with mixed types + let input = "{% my_tag [42, 'hello', my_var] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "[42, 'hello', my_var]".to_string(), + start_index: 10, + end_index: 31, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "42".to_string(), + start_index: 11, + end_index: 13, + line_col: (1, 12), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 11, + end_index: 13, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "'hello'".to_string(), + start_index: 15, + end_index: 22, + line_col: (1, 16), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::String, + start_index: 15, + end_index: 22, + line_col: (1, 16), + }, + TagValue { + token: TagToken { + token: "my_var".to_string(), + start_index: 24, + end_index: 30, + line_col: (1, 25), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Variable, + start_index: 24, + end_index: 30, + line_col: (1, 25), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 10, + end_index: 31, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 31, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 34, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_filter() { + // List with filter on the entire list + let input = "{% my_tag [1, 2, 3]|first %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "[1, 2, 3]".to_string(), + start_index: 10, + end_index: 19, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 14, + end_index: 15, + line_col: (1, 15), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 14, + end_index: 15, + line_col: (1, 15), + }, + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 17, + end_index: 18, + line_col: (1, 18), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 17, + end_index: 18, + line_col: (1, 18), + }, + ], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "first".to_string(), + start_index: 20, + end_index: 25, + line_col: (1, 21), + }, + arg: None, + start_index: 19, + end_index: 25, + line_col: (1, 20), + }], + kind: ValueKind::List, + start_index: 10, + end_index: 25, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 25, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 28, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_filter_item() { + // List with filters on individual items + let input = "{% my_tag ['hello'|upper, 'world'|title] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "['hello'|upper, 'world'|title]".to_string(), + start_index: 10, + end_index: 40, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "'hello'".to_string(), + start_index: 11, + end_index: 18, + line_col: (1, 12), + }, + spread: None, + filters: vec![TagValueFilter { + arg: None, + token: TagToken { + token: "upper".to_string(), + start_index: 19, + end_index: 24, + line_col: (1, 20), + }, + start_index: 18, + end_index: 24, + line_col: (1, 19), + }], + children: vec![], + kind: ValueKind::String, + start_index: 11, + end_index: 24, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "'world'".to_string(), + start_index: 26, + end_index: 33, + line_col: (1, 27), + }, + spread: None, + filters: vec![TagValueFilter { + arg: None, + token: TagToken { + token: "title".to_string(), + start_index: 34, + end_index: 39, + line_col: (1, 35), + }, + start_index: 33, + end_index: 39, + line_col: (1, 34), + }], + children: vec![], + kind: ValueKind::String, + start_index: 26, + end_index: 39, + line_col: (1, 27), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 10, + end_index: 40, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 40, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 43, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_filter_everywhere() { + // List with both item filters and list filter + let input = "{% my_tag ['a'|upper, 'b'|upper]|join:',' %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + is_flag: false, + value: TagValue { + token: TagToken { + token: "['a'|upper, 'b'|upper]".to_string(), + start_index: 10, + end_index: 32, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "'a'".to_string(), + start_index: 11, + end_index: 14, + line_col: (1, 12), + }, + spread: None, + filters: vec![TagValueFilter { + arg: None, + token: TagToken { + token: "upper".to_string(), + start_index: 15, + end_index: 20, + line_col: (1, 16), + }, + start_index: 14, + end_index: 20, + line_col: (1, 15), + }], + children: vec![], + kind: ValueKind::String, + start_index: 11, + end_index: 20, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "'b'".to_string(), + start_index: 22, + end_index: 25, + line_col: (1, 23), + }, + spread: None, + filters: vec![TagValueFilter { + arg: None, + token: TagToken { + token: "upper".to_string(), + start_index: 26, + end_index: 31, + line_col: (1, 27), + }, + start_index: 25, + end_index: 31, + line_col: (1, 26), + }], + children: vec![], + kind: ValueKind::String, + start_index: 22, + end_index: 31, + line_col: (1, 23), + }, + ], + spread: None, + filters: vec![TagValueFilter { + arg: Some(TagValue { + token: TagToken { + token: "','".to_string(), + start_index: 38, + end_index: 41, + line_col: (1, 39), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::String, + start_index: 37, + end_index: 41, + line_col: (1, 38), + }), + token: TagToken { + token: "join".to_string(), + start_index: 33, + end_index: 37, + line_col: (1, 34), + }, + start_index: 32, + end_index: 41, + line_col: (1, 33), + }], + kind: ValueKind::List, + start_index: 10, + end_index: 41, + line_col: (1, 11), + }, + start_index: 10, + end_index: 41, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 44, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_nested() { + // Simple nested list + let input = "{% my_tag [1, [2, 3], 4] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + is_flag: false, + value: TagValue { + token: TagToken { + token: "[1, [2, 3], 4]".to_string(), + start_index: 10, + end_index: 24, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "[2, 3]".to_string(), + start_index: 14, + end_index: 20, + line_col: (1, 15), + }, + children: vec![ + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 15, + end_index: 16, + line_col: (1, 16), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 15, + end_index: 16, + line_col: (1, 16), + }, + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 18, + end_index: 19, + line_col: (1, 19), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 18, + end_index: 19, + line_col: (1, 19), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 14, + end_index: 20, + line_col: (1, 15), + }, + TagValue { + token: TagToken { + token: "4".to_string(), + start_index: 22, + end_index: 23, + line_col: (1, 23), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 22, + end_index: 23, + line_col: (1, 23), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 10, + end_index: 24, + line_col: (1, 11), + }, + start_index: 10, + end_index: 24, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 27, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_nested_filter() { + // Nested list with filters + let input = "{% my_tag [[1, 2]|first, [3, 4]|last]|join:',' %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + is_flag: false, + value: TagValue { + token: TagToken { + token: "[[1, 2]|first, [3, 4]|last]".to_string(), + start_index: 10, + end_index: 37, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "[1, 2]".to_string(), + start_index: 11, + end_index: 17, + line_col: (1, 12), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 12, + end_index: 13, + line_col: (1, 13), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 12, + end_index: 13, + line_col: (1, 13), + }, + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 15, + end_index: 16, + line_col: (1, 16), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 15, + end_index: 16, + line_col: (1, 16), + }, + ], + spread: None, + filters: vec![TagValueFilter { + arg: None, + token: TagToken { + token: "first".to_string(), + start_index: 18, + end_index: 23, + line_col: (1, 19), + }, + start_index: 17, + end_index: 23, + line_col: (1, 18), + }], + kind: ValueKind::List, + start_index: 11, + end_index: 23, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "[3, 4]".to_string(), + start_index: 25, + end_index: 31, + line_col: (1, 26), + }, + children: vec![ + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 26, + end_index: 27, + line_col: (1, 27), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 26, + end_index: 27, + line_col: (1, 27), + }, + TagValue { + token: TagToken { + token: "4".to_string(), + start_index: 29, + end_index: 30, + line_col: (1, 30), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 29, + end_index: 30, + line_col: (1, 30), + }, + ], + spread: None, + filters: vec![TagValueFilter { + arg: None, + token: TagToken { + token: "last".to_string(), + start_index: 32, + end_index: 36, + line_col: (1, 33), + }, + start_index: 31, + end_index: 36, + line_col: (1, 32), + }], + kind: ValueKind::List, + start_index: 25, + end_index: 36, + line_col: (1, 26), + }, + ], + spread: None, + filters: vec![TagValueFilter { + arg: Some(TagValue { + token: TagToken { + token: "','".to_string(), + start_index: 43, + end_index: 46, + line_col: (1, 44), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::String, + start_index: 42, + end_index: 46, + line_col: (1, 43), + }), + token: TagToken { + token: "join".to_string(), + start_index: 38, + end_index: 42, + line_col: (1, 39), + }, + start_index: 37, + end_index: 46, + line_col: (1, 38), + }], + kind: ValueKind::List, + start_index: 10, + end_index: 46, + line_col: (1, 11), + }, + start_index: 10, + end_index: 46, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 49, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_whitespace() { + // Test whitespace in list + let input = "{% my_tag [ 1 , 2 , 3 ] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + is_flag: false, + value: TagValue { + token: TagToken { + token: "[ 1 , 2 , 3 ]".to_string(), + start_index: 10, + end_index: 23, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 12, + end_index: 13, + line_col: (1, 13), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 12, + end_index: 13, + line_col: (1, 13), + }, + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 16, + end_index: 17, + line_col: (1, 17), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 16, + end_index: 17, + line_col: (1, 17), + }, + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 20, + end_index: 21, + line_col: (1, 21), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 20, + end_index: 21, + line_col: (1, 21), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 10, + end_index: 23, + line_col: (1, 11), + }, + start_index: 10, + end_index: 23, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 26, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_comments() { + // Test comments in list + let input = + "{% my_tag {# before start #}[{# first #}1,{# second #}2,{# third #}3{# end #}]{# after end #} %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + is_flag: false, + value: TagValue { + token: TagToken { + token: "[{# first #}1,{# second #}2,{# third #}3{# end #}]".to_string(), + start_index: 28, + end_index: 78, + line_col: (1, 29), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 40, + end_index: 41, + line_col: (1, 41), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 40, + end_index: 41, + line_col: (1, 41), + }, + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 54, + end_index: 55, + line_col: (1, 55), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 54, + end_index: 55, + line_col: (1, 55), + }, + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 67, + end_index: 68, + line_col: (1, 68), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 67, + end_index: 68, + line_col: (1, 68), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 28, + end_index: 78, + line_col: (1, 29), + }, + start_index: 28, + end_index: 78, + line_col: (1, 29), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 96, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_trailing_comma() { + let input = "{% my_tag [1, 2, 3,] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + is_flag: false, + value: TagValue { + token: TagToken { + token: "[1, 2, 3,]".to_string(), + start_index: 10, + end_index: 20, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 14, + end_index: 15, + line_col: (1, 15), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 14, + end_index: 15, + line_col: (1, 15), + }, + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 17, + end_index: 18, + line_col: (1, 18), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 17, + end_index: 18, + line_col: (1, 18), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 10, + end_index: 20, + line_col: (1, 11), + }, + start_index: 10, + end_index: 20, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 23, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_spread() { + let input = + "{% my_tag [1, *[2, 3], *{'a': 1}, *my_list, *'xyz', *_('hello'), *'{{ var }}', *3.14, 4] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + is_flag: false, + value: TagValue { + token: TagToken { + token: "[1, *[2, 3], *{'a': 1}, *my_list, *'xyz', *_('hello'), *'{{ var }}', *3.14, 4]".to_string(), + start_index: 10, + end_index: 88, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "[2, 3]".to_string(), + start_index: 15, + end_index: 21, + line_col: (1, 16), + }, + children: vec![ + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 16, + end_index: 17, + line_col: (1, 17), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 16, + end_index: 17, + line_col: (1, 17), + }, + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 19, + end_index: 20, + line_col: (1, 20), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 19, + end_index: 20, + line_col: (1, 20), + }, + ], + spread: Some("*".to_string()), + filters: vec![], + kind: ValueKind::List, + start_index: 14, + end_index: 21, + line_col: (1, 15), + }, + TagValue { + token: TagToken { + token: "{'a': 1}".to_string(), + start_index: 24, + end_index: 32, + line_col: (1, 25), + }, + children: vec![ + TagValue { + token: TagToken { + token: "'a'".to_string(), + start_index: 25, + end_index: 28, + line_col: (1, 26), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::String, + start_index: 25, + end_index: 28, + line_col: (1, 26), + }, + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 30, + end_index: 31, + line_col: (1, 31), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 30, + end_index: 31, + line_col: (1, 31), + }, + ], + spread: Some("*".to_string()), + filters: vec![], + kind: ValueKind::Dict, + start_index: 23, + end_index: 32, + line_col: (1, 24), + }, + TagValue { + token: TagToken { + token: "my_list".to_string(), + start_index: 35, + end_index: 42, + line_col: (1, 36), + }, + spread: Some("*".to_string()), + filters: vec![], + children: vec![], + kind: ValueKind::Variable, + start_index: 34, + end_index: 42, + line_col: (1, 35), + }, + TagValue { + token: TagToken { + token: "'xyz'".to_string(), + start_index: 45, + end_index: 50, + line_col: (1, 46), + }, + spread: Some("*".to_string()), + filters: vec![], + children: vec![], + kind: ValueKind::String, + start_index: 44, + end_index: 50, + line_col: (1, 45), + }, + TagValue { + token: TagToken { + token: "_('hello')".to_string(), + start_index: 53, + end_index: 63, + line_col: (1, 54), + }, + spread: Some("*".to_string()), + filters: vec![], + children: vec![], + kind: ValueKind::Translation, + start_index: 52, + end_index: 63, + line_col: (1, 53), + }, + TagValue { + token: TagToken { + token: "'{{ var }}'".to_string(), + start_index: 66, + end_index: 77, + line_col: (1, 67), + }, + spread: Some("*".to_string()), + filters: vec![], + children: vec![], + kind: ValueKind::TemplateString, + start_index: 65, + end_index: 77, + line_col: (1, 66), + }, + TagValue { + token: TagToken { + token: "3.14".to_string(), + start_index: 80, + end_index: 84, + line_col: (1, 81), + }, + spread: Some("*".to_string()), + filters: vec![], + children: vec![], + kind: ValueKind::Float, + start_index: 79, + end_index: 84, + line_col: (1, 80), + }, + TagValue { + token: TagToken { + token: "4".to_string(), + start_index: 86, + end_index: 87, + line_col: (1, 87), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 86, + end_index: 87, + line_col: (1, 87), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 10, + end_index: 88, + line_col: (1, 11), + }, + start_index: 10, + end_index: 88, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 91, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_spread_filter() { + let input = "{% my_tag [1, *[2|upper, 3|lower], *{'a': 1}|default:empty, *my_list|join:\",\", *'xyz'|upper, *_('hello')|escape, *'{{ var }}'|safe, *3.14|round, 4|default:0] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + is_flag: false, + value: TagValue { + token: TagToken { + token: "[1, *[2|upper, 3|lower], *{'a': 1}|default:empty, *my_list|join:\",\", *'xyz'|upper, *_('hello')|escape, *'{{ var }}'|safe, *3.14|round, 4|default:0]".to_string(), + start_index: 10, + end_index: 157, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "[2|upper, 3|lower]".to_string(), + start_index: 15, + end_index: 33, + line_col: (1, 16), + }, + children: vec![ + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 16, + end_index: 17, + line_col: (1, 17), + }, + spread: None, + children: vec![], + filters: vec![TagValueFilter { + arg: None, + token: TagToken { + token: "upper".to_string(), + start_index: 18, + end_index: 23, + line_col: (1, 19), + }, + start_index: 17, + end_index: 23, + line_col: (1, 18), + }], + kind: ValueKind::Int, + start_index: 16, + end_index: 23, + line_col: (1, 17), + }, + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 25, + end_index: 26, + line_col: (1, 26), + }, + spread: None, + children: vec![], + filters: vec![TagValueFilter { + arg: None, + token: TagToken { + token: "lower".to_string(), + start_index: 27, + end_index: 32, + line_col: (1, 28), + }, + start_index: 26, + end_index: 32, + line_col: (1, 27), + }], + kind: ValueKind::Int, + start_index: 25, + end_index: 32, + line_col: (1, 26), + }, + ], + spread: Some("*".to_string()), + filters: vec![], + kind: ValueKind::List, + start_index: 14, + end_index: 33, + line_col: (1, 15), + }, + TagValue { + token: TagToken { + token: "{'a': 1}".to_string(), + start_index: 36, + end_index: 44, + line_col: (1, 37), + }, + children: vec![ + TagValue { + token: TagToken { + token: "'a'".to_string(), + start_index: 37, + end_index: 40, + line_col: (1, 38), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::String, + start_index: 37, + end_index: 40, + line_col: (1, 38), + }, + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 42, + end_index: 43, + line_col: (1, 43), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 42, + end_index: 43, + line_col: (1, 43), + }, + ], + spread: Some("*".to_string()), + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 45, + end_index: 52, + line_col: (1, 46), + }, + arg: Some(TagValue { + token: TagToken { + token: "empty".to_string(), + start_index: 53, + end_index: 58, + line_col: (1, 54), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Variable, + start_index: 52, + end_index: 58, + line_col: (1, 53), + }), + start_index: 44, + end_index: 58, + line_col: (1, 45), + }], + kind: ValueKind::Dict, + start_index: 35, + end_index: 58, + line_col: (1, 36), + }, + TagValue { + token: TagToken { + token: "my_list".to_string(), + start_index: 61, + end_index: 68, + line_col: (1, 62), + }, + spread: Some("*".to_string()), + children: vec![], + filters: vec![TagValueFilter { + token: TagToken { + token: "join".to_string(), + start_index: 69, + end_index: 73, + line_col: (1, 70), + }, + arg: Some(TagValue { + token: TagToken { + token: "\",\"".to_string(), + start_index: 74, + end_index: 77, + line_col: (1, 75), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::String, + start_index: 73, + end_index: 77, + line_col: (1, 74), + }), + start_index: 68, + end_index: 77, + line_col: (1, 69), + }], + kind: ValueKind::Variable, + start_index: 60, + end_index: 77, + line_col: (1, 61), + }, + TagValue { + token: TagToken { + token: "'xyz'".to_string(), + start_index: 80, + end_index: 85, + line_col: (1, 81), + }, + spread: Some("*".to_string()), + children: vec![], + filters: vec![TagValueFilter { + token: TagToken { + token: "upper".to_string(), + start_index: 86, + end_index: 91, + line_col: (1, 87), + }, + arg: None, + start_index: 85, + end_index: 91, + line_col: (1, 86), + }], + kind: ValueKind::String, + start_index: 79, + end_index: 91, + line_col: (1, 80), + }, + TagValue { + token: TagToken { + token: "_('hello')".to_string(), + start_index: 94, + end_index: 104, + line_col: (1, 95), + }, + spread: Some("*".to_string()), + children: vec![], + filters: vec![TagValueFilter { + token: TagToken { + token: "escape".to_string(), + start_index: 105, + end_index: 111, + line_col: (1, 106), + }, + arg: None, + start_index: 104, + end_index: 111, + line_col: (1, 105), + }], + kind: ValueKind::Translation, + start_index: 93, + end_index: 111, + line_col: (1, 94), + }, + TagValue { + token: TagToken { + token: "'{{ var }}'".to_string(), + start_index: 114, + end_index: 125, + line_col: (1, 115), + }, + spread: Some("*".to_string()), + children: vec![], + filters: vec![TagValueFilter { + token: TagToken { + token: "safe".to_string(), + start_index: 126, + end_index: 130, + line_col: (1, 127), + }, + arg: None, + start_index: 125, + end_index: 130, + line_col: (1, 126), + }], + kind: ValueKind::TemplateString, + start_index: 113, + end_index: 130, + line_col: (1, 114), + }, + TagValue { + token: TagToken { + token: "3.14".to_string(), + start_index: 133, + end_index: 137, + line_col: (1, 134), + }, + spread: Some("*".to_string()), + children: vec![], + filters: vec![TagValueFilter { + token: TagToken { + token: "round".to_string(), + start_index: 138, + end_index: 143, + line_col: (1, 139), + }, + arg: None, + start_index: 137, + end_index: 143, + line_col: (1, 138), + }], + kind: ValueKind::Float, + start_index: 132, + end_index: 143, + line_col: (1, 133), + }, + TagValue { + token: TagToken { + token: "4".to_string(), + start_index: 145, + end_index: 146, + line_col: (1, 146), + }, + spread: None, + children: vec![], + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 147, + end_index: 154, + line_col: (1, 148), + }, + arg: Some(TagValue { + token: TagToken { + token: "0".to_string(), + start_index: 155, + end_index: 156, + line_col: (1, 156), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 154, + end_index: 156, + line_col: (1, 155), + }), + start_index: 146, + end_index: 156, + line_col: (1, 147), + }], + kind: ValueKind::Int, + start_index: 145, + end_index: 156, + line_col: (1, 146), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 10, + end_index: 157, + line_col: (1, 11), + }, + start_index: 10, + end_index: 157, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 160, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_spread_invalid() { + // Test asterisk at top level as value-only + let input = "{% my_tag *value %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow asterisk operator at top level" + ); + + // Test asterisk in value position of key-value pair + let input = "{% my_tag key=*value %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow asterisk operator in value position of key-value pair" + ); + + // Test asterisk in key position + let input = "{% my_tag *key=value %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow asterisk operator in key position" + ); + + // Test asterisk with nested list at top level + let input = "{% my_tag *[1, 2, 3] %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow asterisk operator with list at top level" + ); + + // Test asterisk with nested list in key-value pair + let input = "{% my_tag key=*[1, 2, 3] %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow asterisk operator with list in key-value pair" + ); + + // Test combining spread operators + let input = "{% my_tag ...*[1, 2, 3] %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow combining spread operators" + ); + + // Test combining spread operators with variable + let input = "{% my_tag ...*my_list %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow combining spread operators with variable" + ); + + // Test combining spread operators + let input = "{% my_tag *...[1, 2, 3] %}"; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow combining spread operators" + ); + } + + #[test] + fn test_list_spread_comments() { + // Test comments before / after spread + let input = "{% my_tag [{# ... #}*{# ... #}1,*{# ... #}2,{# ... #}3] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "[{# ... #}*{# ... #}1,*{# ... #}2,{# ... #}3]".to_string(), + start_index: 10, + end_index: 55, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 30, + end_index: 31, + line_col: (1, 31), + }, + spread: Some("*".to_string()), + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 29, + end_index: 31, + line_col: (1, 30), + }, + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 42, + end_index: 43, + line_col: (1, 43), + }, + spread: Some("*".to_string()), + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 41, + end_index: 43, + line_col: (1, 42), + }, + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 53, + end_index: 54, + line_col: (1, 54), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 53, + end_index: 54, + line_col: (1, 54), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 10, + end_index: 55, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 55, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 58, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_list_spread_nested_comments() { + // Test comments with nested spread + let input = + "{% my_tag {# c0 #}[1, {# c1 #}*{# c2 #}[2, {# c3 #}*{# c4 #}[3, 4]], 5]{# c5 #} %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + is_flag: false, + value: TagValue { + token: TagToken { + token: "[1, {# c1 #}*{# c2 #}[2, {# c3 #}*{# c4 #}[3, 4]], 5]" + .to_string(), + start_index: 18, + end_index: 71, + line_col: (1, 19), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 19, + end_index: 20, + line_col: (1, 20), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 19, + end_index: 20, + line_col: (1, 20), + }, + TagValue { + token: TagToken { + token: "[2, {# c3 #}*{# c4 #}[3, 4]]".to_string(), + start_index: 39, + end_index: 67, + line_col: (1, 40), + }, + children: vec![ + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 40, + end_index: 41, + line_col: (1, 41), + }, + spread: None, + children: vec![], + filters: vec![], + kind: ValueKind::Int, + start_index: 40, + end_index: 41, + line_col: (1, 41), + }, + TagValue { + token: TagToken { + token: "[3, 4]".to_string(), + start_index: 60, + end_index: 66, + line_col: (1, 61), + }, + children: vec![ + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 61, + end_index: 62, + line_col: (1, 62), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 61, + end_index: 62, + line_col: (1, 62), + }, + TagValue { + token: TagToken { + token: "4".to_string(), + start_index: 64, + end_index: 65, + line_col: (1, 65), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 64, + end_index: 65, + line_col: (1, 65), + }, + ], + spread: Some("*".to_string()), + filters: vec![], + kind: ValueKind::List, + start_index: 59, + end_index: 66, + line_col: (1, 60), + }, + ], + spread: Some("*".to_string()), + filters: vec![], + kind: ValueKind::List, + start_index: 38, + end_index: 67, + line_col: (1, 39), + }, + TagValue { + token: TagToken { + token: "5".to_string(), + start_index: 69, + end_index: 70, + line_col: (1, 70), + }, + spread: None, + filters: vec![], + children: vec![], + kind: ValueKind::Int, + start_index: 69, + end_index: 70, + line_col: (1, 70), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 18, + end_index: 71, + line_col: (1, 19), + }, + start_index: 18, + end_index: 71, + line_col: (1, 19), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 82, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_template_string_negative() { + // Test simple string without template string + let input = "{% my_tag \"Hello\" %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "\"Hello\"".to_string(), + start_index: 10, + end_index: 17, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 10, + end_index: 17, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 17, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 20, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_template_string_block() { + // Test string with {% tag %} + let input = "{% my_tag \"Hello {% lorem w 1 %}\" %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "\"Hello {% lorem w 1 %}\"".to_string(), + start_index: 10, + end_index: 33, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::TemplateString, + start_index: 10, + end_index: 33, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 33, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 36, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_template_string_variable() { + // Test string with {{ variable }} + let input = "{% my_tag \"Hello {{ last_name }}\" %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "\"Hello {{ last_name }}\"".to_string(), + start_index: 10, + end_index: 33, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::TemplateString, + start_index: 10, + end_index: 33, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 33, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 36, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_template_string_comment() { + // Test string with {# comment #} + let input = "{% my_tag \"Hello {# TODO #}\" %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "\"Hello {# TODO #}\"".to_string(), + start_index: 10, + end_index: 28, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::TemplateString, + start_index: 10, + end_index: 28, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 28, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 31, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_template_string_mixed() { + // Test string with multiple template tags + let input = "{% my_tag \"Hello {{ first_name }} {% lorem 1 w %} {# TODO #}\" %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "\"Hello {{ first_name }} {% lorem 1 w %} {# TODO #}\"" + .to_string(), + start_index: 10, + end_index: 61, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::TemplateString, + start_index: 10, + end_index: 61, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 61, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 64, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_template_string_invalid() { + // Test incomplete template tags (should not be marked as template_string) + let inputs = vec![ + r#"{% my_tag "Hello {{ first_name" %}"#, + r#"{% my_tag "Hello {% first_name" %}"#, + r#"{% my_tag "Hello {# first_name" %}"#, + r#"{% my_tag "Hello {{ first_name %}" %}"#, + r#"{% my_tag "Hello first_name }}" %}"#, + r#"{% my_tag "Hello }} first_name {{" %}"#, + ]; + for input in inputs { + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + // Extract the string value from the input (between quotes) + let string_start = input.find('"').unwrap() + 1; + let string_end = input.rfind('"').unwrap(); + let string_value = &input[string_start..string_end]; + + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: format!("\"{}\"", string_value), + start_index: 10, + end_index: 10 + string_value.len() + 2, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 10, + end_index: 10 + string_value.len() + 2, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 10 + string_value.len() + 2, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: input.len(), + line_col: (1, 4), + } + ); + } + } + + #[test] + fn test_template_string_filter_arg() { + // Test that template strings are detected in filter args + let input = "{% my_tag value|default:\"{{ var }}\" %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "value".to_string(), + start_index: 10, + end_index: 15, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 16, + end_index: 23, + line_col: (1, 17), + }, + arg: Some(TagValue { + token: TagToken { + token: "\"{{ var }}\"".to_string(), + start_index: 24, + end_index: 35, + line_col: (1, 25), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::TemplateString, + start_index: 23, + end_index: 35, + line_col: (1, 24), + }), + start_index: 15, + end_index: 35, + line_col: (1, 16), + }], + kind: ValueKind::Variable, + start_index: 10, + end_index: 35, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 35, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 38, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_template_string_i18n() { + // Test that template strings are not detected in i18n strings + let input = "{% my_tag _(\"{{ var }}\") %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "_(\"{{ var }}\")".to_string(), + start_index: 10, + end_index: 24, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Translation, + start_index: 10, + end_index: 24, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 24, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 27, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_dict_filters_key() { + // Test filters on keys + let input = r#"{% my_tag {"key"|upper|lower: "value"} %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"{"key"|upper|lower: "value"}"#.to_string(), + start_index: 10, + end_index: 38, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key\"".to_string(), + start_index: 11, + end_index: 16, + line_col: (1, 12), + }, + children: vec![], + spread: None, + filters: vec![ + TagValueFilter { + arg: None, + token: TagToken { + token: "upper".to_string(), + start_index: 17, + end_index: 22, + line_col: (1, 18), + }, + start_index: 16, + end_index: 22, + line_col: (1, 17), + }, + TagValueFilter { + arg: None, + token: TagToken { + token: "lower".to_string(), + start_index: 23, + end_index: 28, + line_col: (1, 24), + }, + start_index: 22, + end_index: 28, + line_col: (1, 23), + }, + ], + kind: ValueKind::String, + start_index: 11, + end_index: 28, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "\"value\"".to_string(), + start_index: 30, + end_index: 37, + line_col: (1, 31), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 30, + end_index: 37, + line_col: (1, 31), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::Dict, + start_index: 10, + end_index: 38, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 38, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 41, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_dict_filters_value() { + // Test filters on values + let input = r#"{% my_tag {"key": "value"|upper|lower} %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"{"key": "value"|upper|lower}"#.to_string(), + start_index: 10, + end_index: 38, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key\"".to_string(), + start_index: 11, + end_index: 16, + line_col: (1, 12), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 11, + end_index: 16, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "\"value\"".to_string(), + start_index: 18, + end_index: 25, + line_col: (1, 19), + }, + children: vec![], + spread: None, + filters: vec![ + TagValueFilter { + arg: None, + token: TagToken { + token: "upper".to_string(), + start_index: 26, + end_index: 31, + line_col: (1, 27), + }, + start_index: 25, + end_index: 31, + line_col: (1, 26), + }, + TagValueFilter { + arg: None, + token: TagToken { + token: "lower".to_string(), + start_index: 32, + end_index: 37, + line_col: (1, 33), + }, + start_index: 31, + end_index: 37, + line_col: (1, 32), + }, + ], + kind: ValueKind::String, + start_index: 18, + end_index: 37, + line_col: (1, 19), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::Dict, + start_index: 10, + end_index: 38, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 38, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 41, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_dict_filters() { + // Test filter on entire dict + let input = r#"{% my_tag {"key": "value"}|default:empty_dict %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"{"key": "value"}"#.to_string(), + start_index: 10, + end_index: 26, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key\"".to_string(), + start_index: 11, + end_index: 16, + line_col: (1, 12), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 11, + end_index: 16, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "\"value\"".to_string(), + start_index: 18, + end_index: 25, + line_col: (1, 19), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 18, + end_index: 25, + line_col: (1, 19), + }, + ], + spread: None, + filters: vec![TagValueFilter { + token: TagToken { + token: "default".to_string(), + start_index: 27, + end_index: 34, + line_col: (1, 28), + }, + arg: Some(TagValue { + token: TagToken { + token: "empty_dict".to_string(), + start_index: 35, + end_index: 45, + line_col: (1, 36), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 34, + end_index: 45, + line_col: (1, 35), + }), + start_index: 26, + end_index: 45, + line_col: (1, 27), + }], + kind: ValueKind::Dict, + start_index: 10, + end_index: 45, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 45, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 48, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_dict_filters_all() { + // Test filter on all dict + let input = r#"{% my_tag {"key" | default: "value" | default : empty_dict} | default : empty_dict %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"{"key" | default: "value" | default : empty_dict}"# + .to_string(), + start_index: 10, + end_index: 59, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key\"".to_string(), + start_index: 11, + end_index: 16, + line_col: (1, 12), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + arg: None, + token: TagToken { + token: "default".to_string(), + start_index: 19, + end_index: 26, + line_col: (1, 20), + }, + start_index: 17, + end_index: 26, + line_col: (1, 18), + }], + kind: ValueKind::String, + start_index: 11, + end_index: 26, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "\"value\"".to_string(), + start_index: 28, + end_index: 35, + line_col: (1, 29), + }, + children: vec![], + spread: None, + filters: vec![TagValueFilter { + arg: Some(TagValue { + token: TagToken { + token: "empty_dict".to_string(), + start_index: 48, + end_index: 58, + line_col: (1, 49), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 45, + end_index: 58, + line_col: (1, 46), + }), + token: TagToken { + token: "default".to_string(), + start_index: 38, + end_index: 45, + line_col: (1, 39), + }, + start_index: 36, + end_index: 58, + line_col: (1, 37), + }], + kind: ValueKind::String, + start_index: 28, + end_index: 58, + line_col: (1, 29), + }, + ], + spread: None, + filters: vec![TagValueFilter { + arg: Some(TagValue { + token: TagToken { + token: "empty_dict".to_string(), + start_index: 72, + end_index: 82, + line_col: (1, 73), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 69, + end_index: 82, + line_col: (1, 70), + }), + token: TagToken { + token: "default".to_string(), + start_index: 62, + end_index: 69, + line_col: (1, 63), + }, + start_index: 60, + end_index: 82, + line_col: (1, 61), + }], + kind: ValueKind::Dict, + start_index: 10, + end_index: 82, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 82, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 85, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_dict_nested() { + // Test dict in list + let input = "{% my_tag [1, {\"key\": \"val\"}, 2] %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"[1, {"key": "val"}, 2]"#.to_string(), + start_index: 10, + end_index: 32, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 11, + end_index: 12, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: r#"{"key": "val"}"#.to_string(), + start_index: 14, + end_index: 28, + line_col: (1, 15), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key\"".to_string(), + start_index: 15, + end_index: 20, + line_col: (1, 16), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 15, + end_index: 20, + line_col: (1, 16), + }, + TagValue { + token: TagToken { + token: "\"val\"".to_string(), + start_index: 22, + end_index: 27, + line_col: (1, 23), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 22, + end_index: 27, + line_col: (1, 23), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::Dict, + start_index: 14, + end_index: 28, + line_col: (1, 15), + }, + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 30, + end_index: 31, + line_col: (1, 31), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 30, + end_index: 31, + line_col: (1, 31), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 10, + end_index: 32, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 32, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 35, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_dict_nested_list() { + // Test list in dict + let input = r#"{% my_tag {"key": [1, 2, 3]} %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"{"key": [1, 2, 3]}"#.to_string(), + start_index: 10, + end_index: 28, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key\"".to_string(), + start_index: 11, + end_index: 16, + line_col: (1, 12), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 11, + end_index: 16, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: r#"[1, 2, 3]"#.to_string(), + start_index: 18, + end_index: 27, + line_col: (1, 19), + }, + children: vec![ + TagValue { + token: TagToken { + token: "1".to_string(), + start_index: 19, + end_index: 20, + line_col: (1, 20), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 19, + end_index: 20, + line_col: (1, 20), + }, + TagValue { + token: TagToken { + token: "2".to_string(), + start_index: 22, + end_index: 23, + line_col: (1, 23), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 22, + end_index: 23, + line_col: (1, 23), + }, + TagValue { + token: TagToken { + token: "3".to_string(), + start_index: 25, + end_index: 26, + line_col: (1, 26), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 25, + end_index: 26, + line_col: (1, 26), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::List, + start_index: 18, + end_index: 27, + line_col: (1, 19), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::Dict, + start_index: 10, + end_index: 28, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 28, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 31, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_dict_invalid() { + let invalid_inputs = vec![ + ( + r#"{% my_tag {key|lower:my_arg: 123} %}"#, + "filter arguments in dictionary keys", + ), + ( + r#"{% my_tag {"key"|default:empty_dict: "value"|default:empty_dict} %}"#, + "filter arguments in dictionary keys", + ), + ("{% my_tag {key} %}", "missing value"), + ("{% my_tag {key,} %}", "missing value with comma"), + ("{% my_tag {key:} %}", "missing value after colon"), + ("{% my_tag {:value} %}", "missing key"), + ("{% my_tag {key: key:} %}", "double colon"), + ("{% my_tag {:key :key} %}", "double key"), + ]; + + for (input, msg) in invalid_inputs { + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow {}: {}", + msg, + input + ); + } + } + + #[test] + fn test_dict_key_types() { + // Test string literal key + let input = r#"{% my_tag {"key": "value"} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + + // Test variable key + let input = r#"{% my_tag {my_var: "value"} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + + // Test i18n string key + let input = r#"{% my_tag {_("hello"): "value"} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + + // Test number key + let input = r#"{% my_tag {42: "value"} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + + // Test filtered key + let input = r#"{% my_tag {"key"|upper: "value"} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + + // Test list as key (should fail) + let input = r#"{% my_tag {[1, 2]: "value"} %}"#; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow list as dictionary key" + ); + + // Test dict as key (should fail) + let input = r#"{% my_tag {"nested": "dict"}: "value" %}"#; + assert!( + TagParser::parse_tag(input, &HashSet::new()).is_err(), + "Should not allow dictionary as dictionary key" + ); + } + #[test] + fn test_dict_value_types() { + // Test string literal value + let input = r#"{% my_tag {"key": "value"} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + + // Test variable value + let input = r#"{% my_tag {"key": my_var} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + // Test number value + let input = r#"{% my_tag {"key": 42} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + + // Test list value + let input = r#"{% my_tag {"key": [1, 2, 3]} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + + // Test dict value + let input = r#"{% my_tag {"key": {"nested": "dict"}} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + + // Test filtered value + let input = r#"{% my_tag {"key": "value"|upper} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + + // Test spread value + let input = r#"{% my_tag {"key1": "val1", **other_dict} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + + // Test spread with filter that might return dict + let input = r#"{% my_tag {"key1": "val1", **42|make_dict} %}"#; + assert!(TagParser::parse_tag(input, &HashSet::new()).is_ok()); + } + + #[test] + fn test_dict_spread() { + // Test spreading into dict + let input = r#"{% my_tag {"key1": "val1", **other_dict, "key2": "val2", **"{{ key3 }}", **_( " key4 ")} %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"{"key1": "val1", **other_dict, "key2": "val2", **"{{ key3 }}", **_( " key4 ")}"#.to_string(), + start_index: 10, + end_index: 88, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key1\"".to_string(), + start_index: 11, + end_index: 17, + line_col: (1, 12), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 11, + end_index: 17, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "\"val1\"".to_string(), + start_index: 19, + end_index: 25, + line_col: (1, 20), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 19, + end_index: 25, + line_col: (1, 20), + }, + TagValue { + token: TagToken { + token: "other_dict".to_string(), + start_index: 29, + end_index: 39, + line_col: (1, 30), + }, + children: vec![], + spread: Some("**".to_string()), + filters: vec![], + kind: ValueKind::Variable, + start_index: 27, + end_index: 39, + line_col: (1, 28), + }, + TagValue { + token: TagToken { + token: "\"key2\"".to_string(), + start_index: 41, + end_index: 47, + line_col: (1, 42), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 41, + end_index: 47, + line_col: (1, 42), + }, + TagValue { + token: TagToken { + token: "\"val2\"".to_string(), + start_index: 49, + end_index: 55, + line_col: (1, 50), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 49, + end_index: 55, + line_col: (1, 50), + }, + TagValue { + token: TagToken { + token: "\"{{ key3 }}\"".to_string(), + start_index: 59, + end_index: 71, + line_col: (1, 60), + }, + children: vec![], + spread: Some("**".to_string()), + filters: vec![], + kind: ValueKind::TemplateString, + start_index: 57, + end_index: 71, + line_col: (1, 58), + }, + TagValue { + token: TagToken { + token: "_(\" key4 \")".to_string(), + start_index: 75, + end_index: 87, + line_col: (1, 76), + }, + children: vec![], + spread: Some("**".to_string()), + filters: vec![], + kind: ValueKind::Translation, + start_index: 73, + end_index: 87, + line_col: (1, 74), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::Dict, + start_index: 10, + end_index: 88, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 88, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 91, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_dict_spread_filters() { + // Test spreading into dict + filters + let input = r#"{% my_tag {"key1": "val1", **other_dict, "key2": "val2", **"{{ key3 }}", **_( " key4 ")} %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"{"key1": "val1", **other_dict, "key2": "val2", **"{{ key3 }}", **_( " key4 ")}"#.to_string(), + start_index: 10, + end_index: 88, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key1\"".to_string(), + start_index: 11, + end_index: 17, + line_col: (1, 12), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 11, + end_index: 17, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "\"val1\"".to_string(), + start_index: 19, + end_index: 25, + line_col: (1, 20), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 19, + end_index: 25, + line_col: (1, 20), + }, + TagValue { + token: TagToken { + token: "other_dict".to_string(), + start_index: 29, + end_index: 39, + line_col: (1, 30), + }, + children: vec![], + spread: Some("**".to_string()), + filters: vec![], + kind: ValueKind::Variable, + start_index: 27, + end_index: 39, + line_col: (1, 28), + }, + TagValue { + token: TagToken { + token: "\"key2\"".to_string(), + start_index: 41, + end_index: 47, + line_col: (1, 42), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 41, + end_index: 47, + line_col: (1, 42), + }, + TagValue { + token: TagToken { + token: "\"val2\"".to_string(), + start_index: 49, + end_index: 55, + line_col: (1, 50), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 49, + end_index: 55, + line_col: (1, 50), + }, + TagValue { + token: TagToken { + token: "\"{{ key3 }}\"".to_string(), + start_index: 59, + end_index: 71, + line_col: (1, 60), + }, + children: vec![], + spread: Some("**".to_string()), + filters: vec![], + kind: ValueKind::TemplateString, + start_index: 57, + end_index: 71, + line_col: (1, 58), + }, + TagValue { + token: TagToken { + token: "_(\" key4 \")".to_string(), + start_index: 75, + end_index: 87, + line_col: (1, 76), + }, + children: vec![], + spread: Some("**".to_string()), + filters: vec![], + kind: ValueKind::Translation, + start_index: 73, + end_index: 87, + line_col: (1, 74), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::Dict, + start_index: 10, + end_index: 88, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 88, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 91, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_dict_spread_dict() { + // Test spreading literal dict + let input = r#"{% my_tag {"key1": "val1", **{"inner": "value"}, "key2": "val2"} %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"{"key1": "val1", **{"inner": "value"}, "key2": "val2"}"# + .to_string(), + start_index: 10, + end_index: 64, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key1\"".to_string(), + start_index: 11, + end_index: 17, + line_col: (1, 12), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 11, + end_index: 17, + line_col: (1, 12), + }, + TagValue { + token: TagToken { + token: "\"val1\"".to_string(), + start_index: 19, + end_index: 25, + line_col: (1, 20), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 19, + end_index: 25, + line_col: (1, 20), + }, + TagValue { + token: TagToken { + token: r#"{"inner": "value"}"#.to_string(), + start_index: 29, + end_index: 47, + line_col: (1, 30), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"inner\"".to_string(), + start_index: 30, + end_index: 37, + line_col: (1, 31), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 30, + end_index: 37, + line_col: (1, 31), + }, + TagValue { + token: TagToken { + token: "\"value\"".to_string(), + start_index: 39, + end_index: 46, + line_col: (1, 40), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 39, + end_index: 46, + line_col: (1, 40), + }, + ], + spread: Some("**".to_string()), + filters: vec![], + kind: ValueKind::Dict, + start_index: 27, + end_index: 47, + line_col: (1, 28), + }, + TagValue { + token: TagToken { + token: "\"key2\"".to_string(), + start_index: 49, + end_index: 55, + line_col: (1, 50), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 49, + end_index: 55, + line_col: (1, 50), + }, + TagValue { + token: TagToken { + token: "\"val2\"".to_string(), + start_index: 57, + end_index: 63, + line_col: (1, 58), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 57, + end_index: 63, + line_col: (1, 58), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::Dict, + start_index: 10, + end_index: 64, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 64, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 67, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_dict_with_comments() { + // Test comments after values + let input = r#"{% my_tag {# comment before dict #}{{# comment after dict start #} + "key1": "value1", {# comment after first value #} + "key2": "value2" + {# comment before dict end #}}{# comment after dict #} %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"{{# comment after dict start #} + "key1": "value1", {# comment after first value #} + "key2": "value2" + {# comment before dict end #}}"# + .to_string(), + start_index: 35, + end_index: 196, + line_col: (1, 36), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key1\"".to_string(), + start_index: 79, + end_index: 85, + line_col: (2, 13), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 79, + end_index: 85, + line_col: (2, 13), + }, + TagValue { + token: TagToken { + token: "\"value1\"".to_string(), + start_index: 87, + end_index: 95, + line_col: (2, 21), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 87, + end_index: 95, + line_col: (2, 21), + }, + TagValue { + token: TagToken { + token: "\"key2\"".to_string(), + start_index: 141, + end_index: 147, + line_col: (3, 13), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 141, + end_index: 147, + line_col: (3, 13), + }, + TagValue { + token: TagToken { + token: "\"value2\"".to_string(), + start_index: 149, + end_index: 157, + line_col: (3, 21), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 149, + end_index: 157, + line_col: (3, 21), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::Dict, + start_index: 35, + end_index: 196, + line_col: (1, 36), + }, + is_flag: false, + start_index: 35, + end_index: 196, + line_col: (1, 36), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 223, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_dict_comments_colons_commas() { + // Test comments around colons and commas + let input = r#"{% my_tag { + "key1" {# comment before colon #}: {# comment after colon #} "value1" {# comment before comma #}, {# comment after comma #} + "key2": "value2" + } %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"{ + "key1" {# comment before colon #}: {# comment after colon #} "value1" {# comment before comma #}, {# comment after comma #} + "key2": "value2" + }"#.to_string(), + start_index: 10, + end_index: 186, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key1\"".to_string(), + start_index: 24, + end_index: 30, + line_col: (2, 13), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 24, + end_index: 30, + line_col: (2, 13), + }, + TagValue { + token: TagToken { + token: "\"value1\"".to_string(), + start_index: 85, + end_index: 93, + line_col: (2, 74), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 85, + end_index: 93, + line_col: (2, 74), + }, + TagValue { + token: TagToken { + token: "\"key2\"".to_string(), + start_index: 160, + end_index: 166, + line_col: (3, 13), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 160, + end_index: 166, + line_col: (3, 13), + }, + TagValue { + token: TagToken { + token: "\"value2\"".to_string(), + start_index: 168, + end_index: 176, + line_col: (3, 21), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 168, + end_index: 176, + line_col: (3, 21), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::Dict, + start_index: 10, + end_index: 186, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 186, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 189, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_dict_comments_spread() { + // Test comments around spread operator + let input = r#"{% my_tag { + "key1": "value1", + {# comment before spread #}**{# comment after spread #}{"key2": "value2"} + } %}"#; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: r#"{ + "key1": "value1", + {# comment before spread #}**{# comment after spread #}{"key2": "value2"} + }"# + .to_string(), + start_index: 10, + end_index: 137, + line_col: (1, 11), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key1\"".to_string(), + start_index: 24, + end_index: 30, + line_col: (2, 13), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 24, + end_index: 30, + line_col: (2, 13), + }, + TagValue { + token: TagToken { + token: "\"value1\"".to_string(), + start_index: 32, + end_index: 40, + line_col: (2, 21), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 32, + end_index: 40, + line_col: (2, 21), + }, + TagValue { + token: TagToken { + token: r#"{"key2": "value2"}"#.to_string(), + start_index: 109, + end_index: 127, + line_col: (3, 68), + }, + children: vec![ + TagValue { + token: TagToken { + token: "\"key2\"".to_string(), + start_index: 110, + end_index: 116, + line_col: (3, 69), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 110, + end_index: 116, + line_col: (3, 69), + }, + TagValue { + token: TagToken { + token: "\"value2\"".to_string(), + start_index: 118, + end_index: 126, + line_col: (3, 77), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 118, + end_index: 126, + line_col: (3, 77), + }, + ], + spread: Some("**".to_string()), + filters: vec![], + kind: ValueKind::Dict, + start_index: 107, + end_index: 127, + line_col: (3, 66), + }, + ], + spread: None, + filters: vec![], + kind: ValueKind::Dict, + start_index: 10, + end_index: 137, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 137, + line_col: (1, 11), + }], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 140, + line_col: (1, 4), + } + ); + } + + // ####################################### + // FLAGS + // ####################################### + + #[test] + fn test_flag() { + let input = "{% my_tag 123 my_flag key='val' %}"; + let mut flags = HashSet::new(); + flags.insert("my_flag".to_string()); + let result = TagParser::parse_tag(input, &flags).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![ + TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "123".to_string(), + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "my_flag".to_string(), + start_index: 14, + end_index: 21, + line_col: (1, 15), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 14, + end_index: 21, + line_col: (1, 15), + }, + is_flag: true, // This is what we're testing + start_index: 14, + end_index: 21, + line_col: (1, 15), + }, + TagAttr { + key: Some(TagToken { + token: "key".to_string(), + start_index: 22, + end_index: 25, + line_col: (1, 23), + }), + value: TagValue { + token: TagToken { + token: "'val'".to_string(), + start_index: 26, + end_index: 31, + line_col: (1, 27), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 26, + end_index: 31, + line_col: (1, 27), + }, + is_flag: false, + start_index: 22, + end_index: 31, + line_col: (1, 23), + }, + ], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 34, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_flag_not_as_flag() { + // Same as test_flag, but `my_flag` is not in the flags set + let input = "{% my_tag 123 my_flag key='val' %}"; + let flags = HashSet::new(); // empty set + let result = TagParser::parse_tag(input, &flags).unwrap(); + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![ + TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "123".to_string(), + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + is_flag: false, + start_index: 10, + end_index: 13, + line_col: (1, 11), + }, + TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "my_flag".to_string(), + start_index: 14, + end_index: 21, + line_col: (1, 15), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 14, + end_index: 21, + line_col: (1, 15), + }, + is_flag: false, // This is what we're testing + start_index: 14, + end_index: 21, + line_col: (1, 15), + }, + TagAttr { + key: Some(TagToken { + token: "key".to_string(), + start_index: 22, + end_index: 25, + line_col: (1, 23), + }), + value: TagValue { + token: TagToken { + token: "'val'".to_string(), + start_index: 26, + end_index: 31, + line_col: (1, 27), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::String, + start_index: 26, + end_index: 31, + line_col: (1, 27), + }, + is_flag: false, + start_index: 22, + end_index: 31, + line_col: (1, 23), + }, + ], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 34, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_flag_as_spread() { + let input = "{% my_tag ...my_flag %}"; + let mut flags = HashSet::new(); + flags.insert("my_flag".to_string()); + let result = TagParser::parse_tag(input, &flags).unwrap(); + + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: None, + value: TagValue { + token: TagToken { + token: "my_flag".to_string(), + start_index: 13, + end_index: 20, + line_col: (1, 14), + }, + children: vec![], + spread: Some("...".to_string()), + filters: vec![], + kind: ValueKind::Variable, + start_index: 10, + end_index: 20, + line_col: (1, 11), + }, + is_flag: false, // This is what we're testing + start_index: 10, + end_index: 20, + line_col: (1, 11), + },], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 23, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_flag_as_kwarg() { + let input = "{% my_tag my_flag=123 %}"; + let mut flags = HashSet::new(); + flags.insert("my_flag".to_string()); + let result = TagParser::parse_tag(input, &flags).unwrap(); + + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: Some(TagToken { + token: "my_flag".to_string(), + start_index: 10, + end_index: 17, + line_col: (1, 11), + }), + value: TagValue { + token: TagToken { + token: "123".to_string(), + start_index: 18, + end_index: 21, + line_col: (1, 19), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Int, + start_index: 18, + end_index: 21, + line_col: (1, 19), + }, + is_flag: false, // This is what we're testing + start_index: 10, + end_index: 21, + line_col: (1, 11), + },], + is_self_closing: false, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 24, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_flag_duplicate() { + let input = "{% my_tag my_flag my_flag %}"; + let mut flags = HashSet::new(); + flags.insert("my_flag".to_string()); + let result = TagParser::parse_tag(input, &flags); + assert!(result.is_err()); + if let Err(ParseError::InvalidKey(msg)) = result { + assert_eq!(msg, "Flag 'my_flag' may be specified only once."); + } else { + panic!("Expected InvalidKey error"); + } + } + + #[test] + fn test_flag_case_sensitive() { + let input = "{% my_tag my_flag %}"; + let mut flags = HashSet::new(); + flags.insert("MY_FLAG".to_string()); // Different case + let result = TagParser::parse_tag(input, &flags).unwrap(); + + // my_flag should not be a flag + assert_eq!(result.attrs[0].is_flag, false); + } + + // ####################################### + // SELF-CLOSING TAGS + // ####################################### + + #[test] + fn test_self_closing_tag() { + let input = "{% my_tag / %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![], + is_self_closing: true, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 14, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_self_closing_tag_with_args() { + let input = "{% my_tag key=val / %}"; + let result = TagParser::parse_tag(input, &HashSet::new()).unwrap(); + + assert_eq!( + result, + Tag { + name: TagToken { + token: "my_tag".to_string(), + start_index: 3, + end_index: 9, + line_col: (1, 4), + }, + attrs: vec![TagAttr { + key: Some(TagToken { + token: "key".to_string(), + start_index: 10, + end_index: 13, + line_col: (1, 11), + }), + value: TagValue { + token: TagToken { + token: "val".to_string(), + start_index: 14, + end_index: 17, + line_col: (1, 15), + }, + children: vec![], + spread: None, + filters: vec![], + kind: ValueKind::Variable, + start_index: 14, + end_index: 17, + line_col: (1, 15), + }, + is_flag: false, + start_index: 10, + end_index: 17, + line_col: (1, 11), + },], + is_self_closing: true, + syntax: TagSyntax::Django, + start_index: 0, + end_index: 22, + line_col: (1, 4), + } + ); + } + + #[test] + fn test_self_closing_tag_in_middle_errors() { + let input = "{% my_tag / key=val %}"; + let result = TagParser::parse_tag(input, &HashSet::new()); + assert!( + result.is_err(), + "Self-closing slash in the middle should be an error" + ); + // The error message will vary depending on the parser state, so just check it's an error + } +} diff --git a/djc_core/__init__.py b/djc_core/__init__.py index 2dfb9ee..6afa4c8 100644 --- a/djc_core/__init__.py +++ b/djc_core/__init__.py @@ -7,3 +7,10 @@ __doc__ = djc_core.__doc__ if hasattr(djc_core, "__all__"): __all__ = djc_core.__all__ + +# OVERRIDES START HERE +# Add here any additional public API that we defined purely in Python +from djc_core.djc_template_parser import CompiledFunc, compile_tag + +if hasattr(djc_core, "__all__"): + __all__ += ["CompiledFunc", "compile_tag"] diff --git a/djc_core/__init__.pyi b/djc_core/__init__.pyi index 1774962..887abf9 100644 --- a/djc_core/__init__.pyi +++ b/djc_core/__init__.pyi @@ -1 +1,2 @@ from djc_core.djc_html_transformer import * +from djc_core.djc_template_parser import * diff --git a/djc_core/djc_template_parser.py b/djc_core/djc_template_parser.py new file mode 100644 index 0000000..88ad3df --- /dev/null +++ b/djc_core/djc_template_parser.py @@ -0,0 +1,76 @@ +# OVERRIDES START HERE +# We want our `compile_tag` to use `exec()`, so it can return a pre-compiled function. +from typing import Any, Callable, List, Protocol, Tuple, TypeVar, Union +from .djc_core import Tag, TagAttr, compile_ast_to_string + +TContext = TypeVar("TContext") + + +class CompiledFunc(Protocol[TContext]): + def __call__( + self, + context: TContext, + *, + variable: Callable[[TContext, str], Any], + template_string: Callable[[TContext, str], Any], + translation: Callable[[TContext, str], Any], + filter: Callable[[TContext, str, Any, Any], Any], + ) -> Tuple[List[Any], List[Tuple[str, Any]]]: ... + + """ + The result of compiling the template tag AST into a Python function. + + This function accepts the context object, and function implementations, + and returns a tuple of arguments and keyword arguments. + + Example: + + ```python + tag_ast = parse_tag('...[val1] [1, 2, 3] a=b data={"key": "value"}') + compiled_func = compile_tag(tag_ast) + + context = {"val1": "foo", "b": "bar"} + variable = lambda ctx, var: ctx.get(var) + template_string = lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}" + translation = lambda ctx, text: f"TRANSLATION_RESOLVED:{text}" + filter = lambda ctx, name, value, arg=None: f"{value}|{name}:{arg}" + + args, kwargs = compiled_func( + context, + variable=variable, + template_string=template_string, + translation=translation, + filter=filter, + ) + + print(args) # ['foo', [1, 2, 3]] + print(kwargs) # [('a', 'bar'), ('data', {'key': 'value'})] + ``` + """ + + +def compile_tag(tag_or_attrs: Union[Tag, List[TagAttr]]): + """ + Compile the template tag AST (`Tag` object or list of `TagAttr` objects generated by `parse_tag`) + into a Python function. + + The generated function takes a `context` object and returns a tuple of arguments and keyword arguments. + + Args: + tag_or_attrs: A `Tag` object from `parse_tag` or a list of `TagAttr` objects. + + Returns: + A callable function that matches the `CompiledFunc` protocol. + + """ + if isinstance(tag_or_attrs, Tag): + attributes = tag_or_attrs.attrs + else: + attributes = tag_or_attrs + func_string = compile_ast_to_string(attributes) + local_scope = {} + exec(func_string, {}, local_scope) + + compiled_func = local_scope["compiled_func"] + compiled_func._source_code = func_string + return compiled_func diff --git a/djc_core/djc_template_parser.pyi b/djc_core/djc_template_parser.pyi new file mode 100644 index 0000000..98f0a70 --- /dev/null +++ b/djc_core/djc_template_parser.pyi @@ -0,0 +1,200 @@ +# ruff: noqa +from typing import Any, Callable, List, Literal, Optional, Protocol, Set, Tuple, TypeVar, Union + +TContext = TypeVar("TContext") + +class ValueKind: + def __init__( + self, kind: Literal["list", "dict", "int", "float", "variable", "template_string", "translation", "string"] + ) -> None: ... + +class TagSyntax: + def __init__(self, syntax: Literal["django", "html"]) -> None: ... + +class TagToken: + def __init__(self, token: str, start_index: int, end_index: int, line_col: Tuple[int, int]) -> None: ... + token: str + start_index: int + end_index: int + line_col: Tuple[int, int] + +class TagValueFilter: + def __init__( + self, token: TagToken, arg: Optional[TagValue], start_index: int, end_index: int, line_col: Tuple[int, int] + ) -> None: ... + token: TagToken + arg: Optional[TagValue] + start_index: int + end_index: int + line_col: Tuple[int, int] + +class TagValue: + def __init__( + self, + token: TagToken, + children: List[TagValue], + kind: ValueKind, + spread: Optional[str], + filters: List[TagValueFilter], + start_index: int, + end_index: int, + line_col: Tuple[int, int], + ) -> None: ... + token: TagToken + children: List[TagValue] + kind: ValueKind + spread: Optional[str] + filters: List[TagValueFilter] + start_index: int + end_index: int + line_col: Tuple[int, int] + +class TagAttr: + def __init__( + self, + key: Optional[TagToken], + value: TagValue, + is_flag: bool, + start_index: int, + end_index: int, + line_col: Tuple[int, int], + ) -> None: ... + key: Optional[TagToken] + value: TagValue + is_flag: bool + start_index: int + end_index: int + line_col: Tuple[int, int] + +class Tag: + def __init__( + self, + name: TagToken, + attrs: List[TagAttr], + is_self_closing: bool, + syntax: TagSyntax, + start_index: int, + end_index: int, + line_col: Tuple[int, int], + ) -> None: ... + name: TagToken + attrs: List[TagAttr] + is_self_closing: bool + syntax: TagSyntax + start_index: int + end_index: int + line_col: Tuple[int, int] + +class CompiledFunc(Protocol[TContext]): + def __call__( + self, + context: TContext, + *, + variable: Callable[[TContext, str], Any], + template_string: Callable[[TContext, str], Any], + translation: Callable[[TContext, str], Any], + filter: Callable[[TContext, str, Any, Any], Any], + ) -> Tuple[List[Any], List[Tuple[str, Any]]]: ... + """ + The result of compiling the template tag AST into a Python function. + + This function accepts the context object, and function implementations, + and returns a tuple of arguments and keyword arguments. + + Example: + + ```python + tag_ast = parse_tag('...[val1] [1, 2, 3] a=b data={"key": "value"}') + compiled_func = compile_tag(tag_ast) + + context = {"val1": "foo", "b": "bar"} + variable = lambda ctx, var: ctx.get(var) + template_string = lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}" + translation = lambda ctx, text: f"TRANSLATION_RESOLVED:{text}" + filter = lambda ctx, name, value, arg=None: f"{value}|{name}:{arg}" + + args, kwargs = compiled_func( + context, + variable=variable, + template_string=template_string, + translation=translation, + filter=filter, + ) + + print(args) # ['foo', [1, 2, 3]] + print(kwargs) # [('a', 'bar'), ('data', {'key': 'value'})] + ``` + """ + +def parse_tag(input: str, flags: Optional[Set[str]] = None) -> Tag: + """ + Parse a Django template tag string into a Tag object. + + If you have a template tag string like this: + + ```django + {% my_tag ...[val1] a=b [1, 2, 3] data={"key": "value"} / %} + ``` + + Then this parser accepts the contents of the tag as a string, and returns their AST - a Tag object. + + ```python + tag_ast = parse_tag('my_tag ...[val1] a=b [1, 2, 3] data={"key": "value"} /') + print(tag_ast.name) # TagToken(token='my_tag', ...) + print(tag_ast.attrs) # [TagAttr(...), TagAttr(...), ...] + print(tag_ast.is_self_closing) # True + ``` + + The parser supports: + - Tag name (must be the first token) + - Key-value pairs (e.g. key=value) + - Standalone values (e.g. 1, "my string", val) + - Spread operators (e.g. ...value, **value, *value) + - Filters (e.g. value|filter:arg) + - Lists and dictionaries (e.g. [1, 2, 3], {"key": "value"}) + - String literals (single/double quoted) (e.g. "my string", 'my string') + - Numbers (e.g. 1, 1.23, 1e-10) + - Variables (e.g. val, key) + - Translation strings (e.g. _("text")) + - Comments (e.g. {# comment #}) + - Self-closing tags (e.g. {% my_tag / %}) + + Args: + input: The template tag string to parse, without the {% %} delimiters + flags: An optional set of strings that should be treated as flags. + + Returns: + A Tag object representing the parsed tag. + + Raises: + ValueError: If the input cannot be parsed according to the grammar + + """ + +def compile_tag(tag_or_attrs: Union[Tag, List[TagAttr]]) -> CompiledFunc[Any]: + """ + Compile the template tag AST (`Tag` object or list of `TagAttr` objects generated by `parse_tag`) + into a Python function. + + The generated function takes a `context` object and returns a tuple of arguments and keyword arguments. + + Args: + tag_or_attrs: A `Tag` object from `parse_tag` or a list of `TagAttr` objects. + + Returns: + A callable function that matches the `CompiledFunc` protocol. + + """ + +__all__ = [ + "parse_tag", + "compile_tag", + "CompiledFunc", + "Tag", + "TagAttr", + "TagSyntax", + "TagToken", + "TagValue", + "TagValueFilter", + "ValueKind", +] diff --git a/tests/test_template_parser_tag_compiler.py b/tests/test_template_parser_tag_compiler.py new file mode 100644 index 0000000..21d2ba1 --- /dev/null +++ b/tests/test_template_parser_tag_compiler.py @@ -0,0 +1,262 @@ +# ruff: noqa: ANN001,ANN201,ANN202,ARG001,ARG005,S101 +from unittest.mock import Mock, call + +import pytest +from djc_core import compile_tag, parse_tag + + +def test_full_compilation_flow(): + tag_content = ( + '{% my_tag "a string" var_one 123 ' + 'key_one="a value" ' + "key_two=var_two " + 'key_three=_("a translation") ' + 'key_four="{{ an_expression }}" ' + "...spread_var|dict_filter " + 'key_five=my_val|other_filter:"my_arg" ' + "key_five=123 %}" + ) + + ast = parse_tag(tag_content) + compiled_func = compile_tag(ast) + + context = { + "var_one": "resolved_var_one", + "var_two": "resolved_var_two", + "spread_var": {"a": 1, "b": 2}, + "my_val": "original_value", + } + + mock_variable = Mock(side_effect=lambda ctx, var: ctx.get(var)) + mock_template_string = Mock(side_effect=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}") + mock_translation = Mock(side_effect=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}") + + def dummy_filter_side_effect(context, name, value, arg=None): + # This filter is used on the spread argument, so it must return a dict + if name == "dict_filter": + return {"a": 1, "b": 2} + return f"{value}|{name}:{arg}" + + mock_filter = Mock(side_effect=dummy_filter_side_effect) + + # The generated function has dependencies as keyword-only args + args, kwargs = compiled_func( + context, + variable=mock_variable, + template_string=mock_template_string, + translation=mock_translation, + filter=mock_filter, + ) + + # Assert that our Python callbacks were called with the correct arguments + mock_variable.assert_has_calls( + [ + call(context, "var_one"), + call(context, "var_two"), + call(context, "spread_var"), + call(context, "my_val"), + ] + ) + mock_template_string.assert_called_once_with(context, "{{ an_expression }}") + mock_translation.assert_called_once_with(context, "a translation") + mock_filter.assert_has_calls( + [ + call(context, "dict_filter", {"a": 1, "b": 2}, None), + call(context, "other_filter", "original_value", "my_arg"), + ] + ) + + # Assert the final resolved values + assert args == [ + "a string", + "resolved_var_one", + 123, + ] + + assert kwargs == [ + ("key_one", "a value"), + ("key_two", "resolved_var_two"), + ("key_three", "TRANSLATION_RESOLVED:a translation"), + ("key_four", "TEMPLATE_RESOLVED:{{ an_expression }}"), + # Spread variables from `...spread_var|my_filter` are expanded into tuples + ("a", 1), + ("b", 2), + # Kwargs after the spread variable + ("key_five", "original_value|other_filter:my_arg"), + # Compiler doesn't omit repeated kwargs, this is to be handled in Python + ("key_five", 123), + ] + + +# Since flags are NOT treated as args, this should be OK +def test_flag_after_kwarg(): + tag_content = "{% my_tag key='value' my_flag %}" + + ast1 = parse_tag(tag_content, flags={"my_flag"}) + assert ast1.attrs[1].value.token.token == "my_flag" + assert ast1.attrs[1].is_flag + + compiled_func1 = compile_tag(ast1) + args1, kwargs1 = compiled_func1( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=Mock(), + translation=Mock(), + filter=Mock(), + ) + assert args1 == [] + assert kwargs1 == [("key", "value")] + + # Same as before, but with flags=None + ast2 = parse_tag(tag_content, flags=None) + assert ast2.attrs[1].value.token.token == "my_flag" + assert not ast2.attrs[1].is_flag + + with pytest.raises(SyntaxError, match="positional argument follows keyword argument"): + compile_tag(ast2) + + +class TestParamsOrder: + def test_arg_after_kwarg_is_error(self): + tag_content = "{% my_tag key='value' positional_arg %}" + ast = parse_tag(input=tag_content) + with pytest.raises(SyntaxError, match="positional argument follows keyword argument"): + compile_tag(tag_or_attrs=ast) + + def test_arg_after_dict_spread_is_error(self): + tag_content = "{% my_tag ...{'key': 'value'} positional_arg %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + + with pytest.raises(SyntaxError, match="positional argument follows keyword argument"): + tag_func( + context={}, + variable=Mock(), + template_string=Mock(), + translation=Mock(), + filter=Mock(), + ) + + def test_arg_after_list_spread_is_ok(self): + tag_content = "{% my_tag ...[1, 2, 3] positional_arg %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={"positional_arg": 4}, + variable=lambda ctx, var: ctx[var], + template_string=Mock(), + translation=Mock(), + filter=Mock(), + ) + assert args == [1, 2, 3, 4] + assert kwargs == [] + + def test_dict_spread_after_arg_is_ok(self): + tag_content = "{% my_tag positional_arg ...{'key': 'value'} %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={"positional_arg": 1}, + variable=lambda ctx, var: ctx[var], + template_string=Mock(), + translation=Mock(), + filter=Mock(), + ) + assert args == [1] + assert kwargs == [("key", "value")] + + def test_dict_spread_after_kwarg_is_ok(self): + tag_content = "{% my_tag key='value' ...{'key2': 'value2'} %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={}, + variable=Mock(), + template_string=Mock(), + translation=Mock(), + filter=Mock(), + ) + assert args == [] + assert kwargs == [("key", "value"), ("key2", "value2")] + + def test_list_spread_after_arg_is_ok(self): + tag_content = "{% my_tag positional_arg ...[1, 2, 3] %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={"positional_arg": 4}, + variable=lambda ctx, var: ctx[var], + template_string=Mock(), + translation=Mock(), + filter=Mock(), + ) + assert args == [4, 1, 2, 3] + assert kwargs == [] + + def test_list_spread_after_kwarg_is_error(self): + tag_content = "{% my_tag key='value' ...[1, 2, 3] %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + with pytest.raises(SyntaxError, match="positional argument follows keyword argument"): + tag_func( + context={}, + variable=Mock(), + template_string=Mock(), + translation=Mock(), + filter=Mock(), + ) + + def test_list_spread_after_list_spread_is_ok(self): + tag_content = "{% my_tag ...[1, 2, 3] ...[4, 5, 6] %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={}, + variable=Mock(), + template_string=Mock(), + translation=Mock(), + filter=Mock(), + ) + assert args == [1, 2, 3, 4, 5, 6] + assert kwargs == [] + + def test_dict_spread_after_dict_spread_is_ok(self): + tag_content = "{% my_tag ...{'key': 'value'} ...{'key2': 'value2'} %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={}, + variable=Mock(), + template_string=Mock(), + translation=Mock(), + filter=Mock(), + ) + assert args == [] + assert kwargs == [("key", "value"), ("key2", "value2")] + + def test_list_spread_after_dict_spread_is_error(self): + tag_content = "{% my_tag ...{'key': 'value'} ...[1, 2, 3] %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + with pytest.raises(SyntaxError, match="positional argument follows keyword argument"): + tag_func( + context={}, + variable=Mock(), + template_string=Mock(), + translation=Mock(), + filter=Mock(), + ) + + def test_dict_spread_after_list_spread_is_ok(self): + tag_content = "{% my_tag ...[1, 2, 3] ...{'key': 'value'} %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={}, + variable=Mock(), + template_string=Mock(), + translation=Mock(), + filter=Mock(), + ) + assert args == [1, 2, 3] + assert kwargs == [("key", "value")] diff --git a/tests/test_template_parser_tag_parser.py b/tests/test_template_parser_tag_parser.py new file mode 100644 index 0000000..893c7cb --- /dev/null +++ b/tests/test_template_parser_tag_parser.py @@ -0,0 +1,3675 @@ +""" +This file is defined both in django-components and djc_core_template_parser to ensure compatibility. + +Source of truth is djc_core_template_parser. +""" + +# ruff: noqa: ANN201,ARG005,S101,S105,S106,E501 +import re + +import pytest +from djc_core import ( + Tag, + TagAttr, + TagSyntax, + TagToken, + TagValue, + TagValueFilter, + ValueKind, + compile_tag, + parse_tag, +) + + +class TestTagParser: + def test_args_kwargs(self): + tag = parse_tag("{% component 'my_comp' key=val key2='val2 two' %}") + + expected_tag = Tag( + name=TagToken( + token="component", + start_index=3, + end_index=12, + line_col=(1, 4), + ), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken( + token="'my_comp'", + start_index=13, + end_index=22, + line_col=(1, 14), + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=13, + end_index=22, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=22, + line_col=(1, 14), + ), + TagAttr( + key=TagToken( + token="key", + start_index=23, + end_index=26, + line_col=(1, 24), + ), + value=TagValue( + token=TagToken( + token="val", + start_index=27, + end_index=30, + line_col=(1, 28), + ), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=27, + end_index=30, + line_col=(1, 28), + ), + is_flag=False, + start_index=23, + end_index=30, + line_col=(1, 24), + ), + TagAttr( + key=TagToken( + token="key2", + start_index=31, + end_index=35, + line_col=(1, 32), + ), + value=TagValue( + token=TagToken( + token="'val2 two'", + start_index=36, + end_index=46, + line_col=(1, 37), + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=36, + end_index=46, + line_col=(1, 37), + ), + is_flag=False, + start_index=31, + end_index=46, + line_col=(1, 32), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=49, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"val": [1, 2, 3]}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + assert args == ["my_comp"] + assert kwargs == [("key", [1, 2, 3]), ("key2", "val2 two")] + + def test_nested_quotes(self): + tag = parse_tag("{% component 'my_comp' key=val key2='val2 \"two\"' text=\"organisation's\" %}") + + expected_tag = Tag( + name=TagToken( + token="component", + start_index=3, + end_index=12, + line_col=(1, 4), + ), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken( + token="'my_comp'", + start_index=13, + end_index=22, + line_col=(1, 14), + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=13, + end_index=22, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=22, + line_col=(1, 14), + ), + TagAttr( + key=TagToken( + token="key", + start_index=23, + end_index=26, + line_col=(1, 24), + ), + value=TagValue( + token=TagToken( + token="val", + start_index=27, + end_index=30, + line_col=(1, 28), + ), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=27, + end_index=30, + line_col=(1, 28), + ), + is_flag=False, + start_index=23, + end_index=30, + line_col=(1, 24), + ), + TagAttr( + key=TagToken( + token="key2", + start_index=31, + end_index=35, + line_col=(1, 32), + ), + value=TagValue( + token=TagToken( + token="'val2 \"two\"'", + start_index=36, + end_index=48, + line_col=(1, 37), + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=36, + end_index=48, + line_col=(1, 37), + ), + is_flag=False, + start_index=31, + end_index=48, + line_col=(1, 32), + ), + TagAttr( + key=TagToken( + token="text", + start_index=49, + end_index=53, + line_col=(1, 50), + ), + value=TagValue( + token=TagToken( + token='"organisation\'s"', + start_index=54, + end_index=70, + line_col=(1, 55), + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=54, + end_index=70, + line_col=(1, 55), + ), + is_flag=False, + start_index=49, + end_index=70, + line_col=(1, 50), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=73, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"val": "some_value"}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + assert args == ["my_comp"] + assert kwargs == [ + ("key", "some_value"), + ("key2", 'val2 "two"'), + ("text", "organisation's"), + ] + + def test_trailing_quote_single(self): + # Test that the Rust parser correctly identifies malformed input with unclosed quote + with pytest.raises(SyntaxError, match="expected self_closing_slash, attribute, filter, or COMMENT"): + parse_tag("{% component 'my_comp' key=val key2='val2 \"two\"' text=\"organisation's\" 'abc %}") + + def test_trailing_quote_double(self): + # Test that the Rust parser correctly identifies malformed input with unclosed double quote + with pytest.raises(SyntaxError, match="expected self_closing_slash, attribute, filter, or COMMENT"): + parse_tag('{% component "my_comp" key=val key2="val2 \'two\'" text=\'organisation"s\' "abc %}') + + def test_trailing_quote_as_value_single(self): + # Test that the Rust parser correctly identifies malformed input with unclosed quote in key=value pair + with pytest.raises(SyntaxError, match="expected value"): + parse_tag("{% component 'my_comp' key=val key2='val2 \"two\"' text=\"organisation's\" value='abc %}") + + def test_trailing_quote_as_value_double(self): + # Test that the Rust parser correctly identifies malformed input with unclosed double quote in key=value pair + with pytest.raises(SyntaxError, match="expected value"): + parse_tag('{% component "my_comp" key=val key2="val2 \'two\'" text=\'organisation"s\' value="abc %}') + + def test_translation(self): + tag = parse_tag('{% component "my_comp" _("one") key=_("two") %}') + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken(token='"my_comp"', start_index=13, end_index=22, line_col=(1, 14)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=13, + end_index=22, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=22, + line_col=(1, 14), + ), + TagAttr( + key=None, + value=TagValue( + token=TagToken(token='_("one")', start_index=23, end_index=31, line_col=(1, 24)), + children=[], + kind=ValueKind("translation"), + spread=None, + filters=[], + start_index=23, + end_index=31, + line_col=(1, 24), + ), + is_flag=False, + start_index=23, + end_index=31, + line_col=(1, 24), + ), + TagAttr( + key=TagToken(token="key", start_index=32, end_index=35, line_col=(1, 33)), + value=TagValue( + token=TagToken(token='_("two")', start_index=36, end_index=44, line_col=(1, 37)), + children=[], + kind=ValueKind("translation"), + spread=None, + filters=[], + start_index=36, + end_index=44, + line_col=(1, 37), + ), + is_flag=False, + start_index=32, + end_index=44, + line_col=(1, 33), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=47, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + assert args == ["my_comp", "TRANSLATION_RESOLVED:one"] + assert kwargs == [("key", "TRANSLATION_RESOLVED:two")] + + def test_translation_whitespace(self): + tag = parse_tag('{% component value=_( "test" ) %}') + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=TagToken(token="value", start_index=13, end_index=18, line_col=(1, 14)), + value=TagValue( + token=TagToken(token='_("test")', start_index=19, end_index=32, line_col=(1, 20)), + children=[], + kind=ValueKind("translation"), + spread=None, + filters=[], + start_index=19, + end_index=32, + line_col=(1, 20), + ), + is_flag=False, + start_index=13, + end_index=32, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=35, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + assert args == [] + assert kwargs == [("value", "TRANSLATION_RESOLVED:test")] + + +class TestFilter: + def test_tag_parser_filters(self): + tag = parse_tag('{% component "my_comp" value|lower key=val|yesno:"yes,no" key2=val2|default:"N/A"|upper %}') + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken(token='"my_comp"', start_index=13, end_index=22, line_col=(1, 14)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=13, + end_index=22, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=22, + line_col=(1, 14), + ), + TagAttr( + key=None, + value=TagValue( + token=TagToken(token="value", start_index=23, end_index=28, line_col=(1, 24)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken(token="lower", start_index=29, end_index=34, line_col=(1, 30)), + arg=None, + start_index=28, + end_index=34, + line_col=(1, 29), + ) + ], + start_index=23, + end_index=34, + line_col=(1, 24), + ), + is_flag=False, + start_index=23, + end_index=34, + line_col=(1, 24), + ), + TagAttr( + key=TagToken(token="key", start_index=35, end_index=38, line_col=(1, 36)), + value=TagValue( + token=TagToken(token="val", start_index=39, end_index=42, line_col=(1, 40)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken(token="yesno", start_index=43, end_index=48, line_col=(1, 44)), + arg=TagValue( + token=TagToken(token='"yes,no"', start_index=49, end_index=57, line_col=(1, 50)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=48, + end_index=57, + line_col=(1, 49), + ), + start_index=42, + end_index=57, + line_col=(1, 43), + ) + ], + start_index=39, + end_index=57, + line_col=(1, 40), + ), + is_flag=False, + start_index=35, + end_index=57, + line_col=(1, 36), + ), + TagAttr( + key=TagToken(token="key2", start_index=58, end_index=62, line_col=(1, 59)), + value=TagValue( + token=TagToken(token="val2", start_index=63, end_index=67, line_col=(1, 64)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken(token="default", start_index=68, end_index=75, line_col=(1, 69)), + arg=TagValue( + token=TagToken(token='"N/A"', start_index=76, end_index=81, line_col=(1, 77)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=75, + end_index=81, + line_col=(1, 76), + ), + start_index=67, + end_index=81, + line_col=(1, 68), + ), + TagValueFilter( + token=TagToken(token="upper", start_index=82, end_index=87, line_col=(1, 83)), + arg=None, + start_index=81, + end_index=87, + line_col=(1, 82), + ), + ], + start_index=63, + end_index=87, + line_col=(1, 64), + ), + is_flag=False, + start_index=58, + end_index=87, + line_col=(1, 59), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=90, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"value": "HELLO", "val": True, "val2": None}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + assert args == ["my_comp", "lower(HELLO, None)"] + assert kwargs == [ + ("key", "yesno(True, yes,no)"), + ("key2", "upper(default(None, N/A), None)"), + ] + + def test_filter_whitespace(self): + tag = parse_tag("{% component value | lower key=val | upper key2=val2 %}") + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken(token="value", start_index=13, end_index=18, line_col=(1, 14)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken(token="lower", start_index=23, end_index=28, line_col=(1, 24)), + arg=None, + start_index=20, + end_index=28, + line_col=(1, 21), + ) + ], + start_index=13, + end_index=28, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=28, + line_col=(1, 14), + ), + TagAttr( + key=TagToken(token="key", start_index=32, end_index=35, line_col=(1, 33)), + value=TagValue( + token=TagToken(token="val", start_index=36, end_index=39, line_col=(1, 37)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken(token="upper", start_index=44, end_index=49, line_col=(1, 45)), + arg=None, + start_index=41, + end_index=49, + line_col=(1, 42), + ) + ], + start_index=36, + end_index=49, + line_col=(1, 37), + ), + is_flag=False, + start_index=32, + end_index=49, + line_col=(1, 33), + ), + TagAttr( + key=TagToken(token="key2", start_index=53, end_index=57, line_col=(1, 54)), + value=TagValue( + token=TagToken(token="val2", start_index=58, end_index=62, line_col=(1, 59)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=58, + end_index=62, + line_col=(1, 59), + ), + is_flag=False, + start_index=53, + end_index=62, + line_col=(1, 54), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=65, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"value": "HELLO", "val": "world", "val2": "test"}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + assert args == ["lower(HELLO, None)"] + assert kwargs == [ + ("key", "upper(world, None)"), + ("key2", "test"), + ] + + def test_filter_argument_must_follow_filter(self): + with pytest.raises( + SyntaxError, + match=re.escape("expected filter or COMMENT"), + ): + parse_tag('{% component value=val|yesno:"yes,no":arg %}') + + +class TestDict: + def test_dict_simple(self): + tag = parse_tag('{% component data={ "key": "val" } %}') + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=TagToken(token="data", start_index=13, end_index=17, line_col=(1, 14)), + value=TagValue( + token=TagToken(token='{ "key": "val" }', start_index=18, end_index=34, line_col=(1, 19)), + children=[ + TagValue( + token=TagToken(token='"key"', start_index=20, end_index=25, line_col=(1, 21)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=20, + end_index=25, + line_col=(1, 21), + ), + TagValue( + token=TagToken(token='"val"', start_index=27, end_index=32, line_col=(1, 28)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=27, + end_index=32, + line_col=(1, 28), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=18, + end_index=34, + line_col=(1, 19), + ), + is_flag=False, + start_index=13, + end_index=34, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=37, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + assert args == [] + assert kwargs == [("data", {"key": "val"})] + + def test_dict_trailing_comma(self): + tag = parse_tag('{% component data={ "key": "val", } %}') + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=TagToken(token="data", start_index=13, end_index=17, line_col=(1, 14)), + value=TagValue( + token=TagToken(token='{ "key": "val", }', start_index=18, end_index=35, line_col=(1, 19)), + children=[ + TagValue( + token=TagToken(token='"key"', start_index=20, end_index=25, line_col=(1, 21)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=20, + end_index=25, + line_col=(1, 21), + ), + TagValue( + token=TagToken(token='"val"', start_index=27, end_index=32, line_col=(1, 28)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=27, + end_index=32, + line_col=(1, 28), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=18, + end_index=35, + line_col=(1, 19), + ), + is_flag=False, + start_index=13, + end_index=35, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=38, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + assert args == [] + assert kwargs == [("data", {"key": "val"})] + + def test_dict_missing_colon(self): + with pytest.raises( + SyntaxError, + match=re.escape("expected filter_noarg or COMMENT"), + ): + parse_tag('{% component data={ "key" } %}') + + def test_dict_missing_colon_2(self): + with pytest.raises( + SyntaxError, + match=re.escape("expected filter_chain_noarg or COMMENT"), + ): + parse_tag('{% component data={ "key", "val" } %}') + + def test_dict_extra_colon(self): + with pytest.raises( + SyntaxError, + match=re.escape("expected value or COMMENT"), + ): + parse_tag("{% component data={ key:: key } %}") + + def test_dict_spread(self): + tag = parse_tag("{% component data={ **spread } %}") + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=TagToken(token="data", start_index=13, end_index=17, line_col=(1, 14)), + value=TagValue( + token=TagToken(token="{ **spread }", start_index=18, end_index=30, line_col=(1, 19)), + children=[ + TagValue( + token=TagToken(token="spread", start_index=22, end_index=28, line_col=(1, 23)), + children=[], + kind=ValueKind("variable"), + spread="**", + filters=[], + start_index=20, + end_index=28, + line_col=(1, 21), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=18, + end_index=30, + line_col=(1, 19), + ), + is_flag=False, + start_index=13, + end_index=30, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=33, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args1, kwargs1 = tag_func( + context={"spread": {"key": "val"}}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args1 == [] + assert kwargs1 == [("data", {"key": "val"})] + + args2, kwargs2 = tag_func( + context={"spread": {}}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args2 == [] + assert kwargs2 == [("data", {})] + + with pytest.raises( + TypeError, + match=re.escape("'list' object is not a mapping"), + ): + tag_func( + context={"spread": [1, 2, 3]}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + with pytest.raises( + TypeError, + match=re.escape("'int' object is not a mapping"), + ): + tag_func( + context={"spread": 3}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + with pytest.raises( + TypeError, + match=re.escape("'NoneType' object is not a mapping"), + ): + tag_func( + context={"spread": None}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + def test_dict_spread_between_key_value_pairs(self): + tag = parse_tag('{% component data={ "key": val, **spread, "key2": val2 } %}') + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=TagToken(token="data", start_index=13, end_index=17, line_col=(1, 14)), + value=TagValue( + token=TagToken( + token='{ "key": val, **spread, "key2": val2 }', + start_index=18, + end_index=56, + line_col=(1, 19), + ), + children=[ + TagValue( + token=TagToken(token='"key"', start_index=20, end_index=25, line_col=(1, 21)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=20, + end_index=25, + line_col=(1, 21), + ), + TagValue( + token=TagToken(token="val", start_index=27, end_index=30, line_col=(1, 28)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=27, + end_index=30, + line_col=(1, 28), + ), + TagValue( + token=TagToken(token="spread", start_index=34, end_index=40, line_col=(1, 35)), + children=[], + kind=ValueKind("variable"), + spread="**", + filters=[], + start_index=32, + end_index=40, + line_col=(1, 33), + ), + TagValue( + token=TagToken(token='"key2"', start_index=42, end_index=48, line_col=(1, 43)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=42, + end_index=48, + line_col=(1, 43), + ), + TagValue( + token=TagToken(token="val2", start_index=50, end_index=54, line_col=(1, 51)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=50, + end_index=54, + line_col=(1, 51), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=18, + end_index=56, + line_col=(1, 19), + ), + is_flag=False, + start_index=13, + end_index=56, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=59, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args1, kwargs1 = tag_func( + context={"spread": {"a": 1}, "val": "HELLO", "val2": "WORLD"}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args1 == [] + assert kwargs1 == [("data", {"key": "HELLO", "a": 1, "key2": "WORLD"})] + + # Test that dictionary keys cannot have filter arguments - The `:` is parsed as dictionary key separator + # So instead, the content below will be parsed as key `"key"|filter`, and value `"arg":"value"' + # And the latter is invalid because it's missing the `|` separator. + def test_colon_in_dictionary_keys(self): + with pytest.raises( + SyntaxError, + match=re.escape("expected filter_chain or COMMENT"), + ): + parse_tag('{% component data={"key"|filter:"arg": "value"} %}') + + def test_dicts_complex(self): + # NOTE: In this example, it looks like e.g. `"e"` should be a filter argument + # to `c|default`. BUT! variables like `c|default` are inside a dictionary, + # so the `:` is preferentially interpreted as dictionary key separator (`{key: val}`). + # So e.g. line `{c|default: "e"|yesno:"yes,no"}` + # actually means `{: }`, + # where `` is `c|default` and `val` is `"e"|yesno:"yes,no"`. + tag = parse_tag( + """ + {% component + simple={ + "a": 1|add:2 + } + nested={ + "key"|upper: val|lower, + **spread, + "obj": {"x": 1|add:2} + } + filters={ + "a"|lower: "b"|upper, + c|default: "e"|yesno:"yes,no" + } + %}""" + ) + + expected_tag = Tag( + name=TagToken(token="component", start_index=16, end_index=25, line_col=(2, 16)), + attrs=[ + TagAttr( + key=TagToken(token="simple", start_index=38, end_index=44, line_col=(3, 13)), + value=TagValue( + token=TagToken( + token='{\n "a": 1|add:2\n }', + start_index=45, + end_index=89, + line_col=(3, 20), + ), + children=[ + TagValue( + token=TagToken(token='"a"', start_index=63, end_index=66, line_col=(4, 17)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=63, + end_index=66, + line_col=(4, 17), + ), + TagValue( + token=TagToken(token="1", start_index=68, end_index=69, line_col=(4, 22)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken(token="add", start_index=70, end_index=73, line_col=(4, 24)), + arg=TagValue( + token=TagToken(token="2", start_index=74, end_index=75, line_col=(4, 28)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=73, + end_index=75, + line_col=(4, 27), + ), + start_index=69, + end_index=75, + line_col=(4, 23), + ) + ], + start_index=68, + end_index=75, + line_col=(4, 22), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=45, + end_index=89, + line_col=(3, 20), + ), + is_flag=False, + start_index=38, + end_index=89, + line_col=(3, 13), + ), + TagAttr( + key=TagToken(token="nested", start_index=102, end_index=108, line_col=(6, 13)), + value=TagValue( + token=TagToken( + token='{\n "key"|upper: val|lower,\n **spread,\n "obj": {"x": 1|add:2}\n }', + start_index=109, + end_index=228, + line_col=(6, 20), + ), + children=[ + TagValue( + token=TagToken(token='"key"', start_index=127, end_index=132, line_col=(7, 17)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="upper", start_index=133, end_index=138, line_col=(7, 23) + ), + arg=None, + start_index=132, + end_index=138, + line_col=(7, 22), + ) + ], + start_index=127, + end_index=138, + line_col=(7, 17), + ), + TagValue( + token=TagToken(token="val", start_index=140, end_index=143, line_col=(7, 30)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="lower", start_index=144, end_index=149, line_col=(7, 34) + ), + arg=None, + start_index=143, + end_index=149, + line_col=(7, 33), + ) + ], + start_index=140, + end_index=149, + line_col=(7, 30), + ), + TagValue( + token=TagToken(token="spread", start_index=169, end_index=175, line_col=(8, 19)), + children=[], + kind=ValueKind("variable"), + spread="**", + filters=[], + start_index=167, + end_index=175, + line_col=(8, 17), + ), + TagValue( + token=TagToken(token='"obj"', start_index=193, end_index=198, line_col=(9, 17)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=193, + end_index=198, + line_col=(9, 17), + ), + TagValue( + token=TagToken( + token='{"x": 1|add:2}', start_index=200, end_index=214, line_col=(9, 24) + ), + children=[ + TagValue( + token=TagToken(token='"x"', start_index=201, end_index=204, line_col=(9, 25)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=201, + end_index=204, + line_col=(9, 25), + ), + TagValue( + token=TagToken(token="1", start_index=206, end_index=207, line_col=(9, 30)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="add", start_index=208, end_index=211, line_col=(9, 32) + ), + arg=TagValue( + token=TagToken( + token="2", start_index=212, end_index=213, line_col=(9, 36) + ), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=211, + end_index=213, + line_col=(9, 35), + ), + start_index=207, + end_index=213, + line_col=(9, 31), + ) + ], + start_index=206, + end_index=213, + line_col=(9, 30), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=200, + end_index=214, + line_col=(9, 24), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=109, + end_index=228, + line_col=(6, 20), + ), + is_flag=False, + start_index=102, + end_index=228, + line_col=(6, 13), + ), + TagAttr( + key=TagToken(token="filters", start_index=241, end_index=248, line_col=(11, 13)), + value=TagValue( + token=TagToken( + token='{\n "a"|lower: "b"|upper,\n c|default: "e"|yesno:"yes,no"\n }', + start_index=249, + end_index=348, + line_col=(11, 21), + ), + children=[ + TagValue( + token=TagToken(token='"a"', start_index=267, end_index=270, line_col=(12, 17)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="lower", start_index=271, end_index=276, line_col=(12, 21) + ), + arg=None, + start_index=270, + end_index=276, + line_col=(12, 20), + ) + ], + start_index=267, + end_index=276, + line_col=(12, 17), + ), + TagValue( + token=TagToken(token='"b"', start_index=278, end_index=281, line_col=(12, 28)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="upper", start_index=282, end_index=287, line_col=(12, 32) + ), + arg=None, + start_index=281, + end_index=287, + line_col=(12, 31), + ) + ], + start_index=278, + end_index=287, + line_col=(12, 28), + ), + TagValue( + token=TagToken(token="c", start_index=305, end_index=306, line_col=(13, 17)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="default", start_index=307, end_index=314, line_col=(13, 19) + ), + arg=None, + start_index=306, + end_index=314, + line_col=(13, 18), + ) + ], + start_index=305, + end_index=314, + line_col=(13, 17), + ), + TagValue( + token=TagToken(token='"e"', start_index=316, end_index=319, line_col=(13, 28)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="yesno", start_index=320, end_index=325, line_col=(13, 32) + ), + arg=TagValue( + token=TagToken( + token='"yes,no"', start_index=326, end_index=334, line_col=(13, 38) + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=325, + end_index=334, + line_col=(13, 37), + ), + start_index=319, + end_index=334, + line_col=(13, 31), + ) + ], + start_index=316, + end_index=334, + line_col=(13, 28), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=249, + end_index=348, + line_col=(11, 21), + ), + is_flag=False, + start_index=241, + end_index=348, + line_col=(11, 13), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=363, + line_col=(2, 16), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args1, kwargs1 = tag_func( + context={"spread": {6: 7}, "c": None, "val": "bar"}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args1 == [] + assert kwargs1 == [ + ("simple", {"a": "add(1, 2)"}), + ("nested", {"upper(key, None)": "lower(bar, None)", 6: 7, "obj": {"x": "add(1, 2)"}}), + ("filters", {"lower(a, None)": "upper(b, None)", "default(None, None)": "yesno(e, yes,no)"}), + ] + + +class TestList: + def test_list_simple(self): + tag = parse_tag("{% component data=[1, 2, 3] %}") + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=TagToken(token="data", start_index=13, end_index=17, line_col=(1, 14)), + value=TagValue( + token=TagToken(token="[1, 2, 3]", start_index=18, end_index=27, line_col=(1, 19)), + children=[ + TagValue( + token=TagToken(token="1", start_index=19, end_index=20, line_col=(1, 20)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=19, + end_index=20, + line_col=(1, 20), + ), + TagValue( + token=TagToken(token="2", start_index=22, end_index=23, line_col=(1, 23)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=22, + end_index=23, + line_col=(1, 23), + ), + TagValue( + token=TagToken(token="3", start_index=25, end_index=26, line_col=(1, 26)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=25, + end_index=26, + line_col=(1, 26), + ), + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=18, + end_index=27, + line_col=(1, 19), + ), + is_flag=False, + start_index=13, + end_index=27, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=30, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args1, kwargs1 = tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args1 == [] + assert kwargs1 == [("data", [1, 2, 3])] + + def test_list_trailing_comma(self): + tag = parse_tag("{% component data=[1, 2, 3, ] %}") + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=TagToken(token="data", start_index=13, end_index=17, line_col=(1, 14)), + value=TagValue( + token=TagToken(token="[1, 2, 3, ]", start_index=18, end_index=29, line_col=(1, 19)), + children=[ + TagValue( + token=TagToken(token="1", start_index=19, end_index=20, line_col=(1, 20)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=19, + end_index=20, + line_col=(1, 20), + ), + TagValue( + token=TagToken(token="2", start_index=22, end_index=23, line_col=(1, 23)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=22, + end_index=23, + line_col=(1, 23), + ), + TagValue( + token=TagToken(token="3", start_index=25, end_index=26, line_col=(1, 26)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=25, + end_index=26, + line_col=(1, 26), + ), + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=18, + end_index=29, + line_col=(1, 19), + ), + is_flag=False, + start_index=13, + end_index=29, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=32, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args1, kwargs1 = tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args1 == [] + assert kwargs1 == [("data", [1, 2, 3])] + + def test_lists_complex(self): + tag = parse_tag( + """ + {% component + nums=[ + 1, + 2|add:3, + *spread + ] + items=[ + "a"|upper, + 'b'|lower, + c|default:"d" + ] + mixed=[ + 1, + [*nested], + {"key": "val"} + ] + %}""" + ) + + expected_tag = Tag( + name=TagToken(token="component", start_index=20, end_index=29, line_col=(2, 20)), + attrs=[ + TagAttr( + key=TagToken(token="nums", start_index=46, end_index=50, line_col=(3, 17)), + value=TagValue( + token=TagToken( + token="[\n 1,\n 2|add:3,\n *spread\n ]", + start_index=51, + end_index=150, + line_col=(3, 22), + ), + children=[ + TagValue( + token=TagToken(token="1", start_index=73, end_index=74, line_col=(4, 21)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=73, + end_index=74, + line_col=(4, 21), + ), + TagValue( + token=TagToken(token="2", start_index=96, end_index=97, line_col=(5, 21)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken(token="add", start_index=98, end_index=101, line_col=(5, 23)), + arg=TagValue( + token=TagToken( + token="3", start_index=102, end_index=103, line_col=(5, 27) + ), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=101, + end_index=103, + line_col=(5, 26), + ), + start_index=97, + end_index=103, + line_col=(5, 22), + ) + ], + start_index=96, + end_index=103, + line_col=(5, 21), + ), + TagValue( + token=TagToken(token="spread", start_index=126, end_index=132, line_col=(6, 22)), + children=[], + kind=ValueKind("variable"), + spread="*", + filters=[], + start_index=125, + end_index=132, + line_col=(6, 21), + ), + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=51, + end_index=150, + line_col=(3, 22), + ), + is_flag=False, + start_index=46, + end_index=150, + line_col=(3, 17), + ), + TagAttr( + key=TagToken(token="items", start_index=167, end_index=172, line_col=(8, 17)), + value=TagValue( + token=TagToken( + token='[\n "a"|upper,\n \'b\'|lower,\n c|default:"d"\n ]', + start_index=173, + end_index=288, + line_col=(8, 23), + ), + children=[ + TagValue( + token=TagToken(token='"a"', start_index=195, end_index=198, line_col=(9, 21)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="upper", start_index=199, end_index=204, line_col=(9, 25) + ), + arg=None, + start_index=198, + end_index=204, + line_col=(9, 24), + ) + ], + start_index=195, + end_index=204, + line_col=(9, 21), + ), + TagValue( + token=TagToken(token="'b'", start_index=226, end_index=229, line_col=(10, 21)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="lower", start_index=230, end_index=235, line_col=(10, 25) + ), + arg=None, + start_index=229, + end_index=235, + line_col=(10, 24), + ) + ], + start_index=226, + end_index=235, + line_col=(10, 21), + ), + TagValue( + token=TagToken(token="c", start_index=257, end_index=258, line_col=(11, 21)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="default", start_index=259, end_index=266, line_col=(11, 23) + ), + arg=TagValue( + token=TagToken( + token='"d"', start_index=267, end_index=270, line_col=(11, 31) + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=266, + end_index=270, + line_col=(11, 30), + ), + start_index=258, + end_index=270, + line_col=(11, 22), + ) + ], + start_index=257, + end_index=270, + line_col=(11, 21), + ), + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=173, + end_index=288, + line_col=(8, 23), + ), + is_flag=False, + start_index=167, + end_index=288, + line_col=(8, 17), + ), + TagAttr( + key=TagToken(token="mixed", start_index=305, end_index=310, line_col=(13, 17)), + value=TagValue( + token=TagToken( + token='[\n 1,\n [*nested],\n {"key": "val"}\n ]', + start_index=311, + end_index=419, + line_col=(13, 23), + ), + children=[ + TagValue( + token=TagToken(token="1", start_index=333, end_index=334, line_col=(14, 21)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=333, + end_index=334, + line_col=(14, 21), + ), + TagValue( + token=TagToken(token="[*nested]", start_index=356, end_index=365, line_col=(15, 21)), + children=[ + TagValue( + token=TagToken( + token="nested", start_index=358, end_index=364, line_col=(15, 23) + ), + children=[], + kind=ValueKind("variable"), + spread="*", + filters=[], + start_index=357, + end_index=364, + line_col=(15, 22), + ) + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=356, + end_index=365, + line_col=(15, 21), + ), + TagValue( + token=TagToken( + token='{"key": "val"}', start_index=387, end_index=401, line_col=(16, 21) + ), + children=[ + TagValue( + token=TagToken( + token='"key"', start_index=388, end_index=393, line_col=(16, 22) + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=388, + end_index=393, + line_col=(16, 22), + ), + TagValue( + token=TagToken( + token='"val"', start_index=395, end_index=400, line_col=(16, 29) + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=395, + end_index=400, + line_col=(16, 29), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=387, + end_index=401, + line_col=(16, 21), + ), + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=311, + end_index=419, + line_col=(13, 23), + ), + is_flag=False, + start_index=305, + end_index=419, + line_col=(13, 17), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=434, + line_col=(2, 20), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args1, kwargs1 = tag_func( + context={"nested": [1, 2, 3], "spread": [5, 6], "c": None}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args1 == [] + assert kwargs1 == [ + ("nums", [1, "add(2, 3)", 5, 6]), + ("items", ["upper(a, None)", "lower(b, None)", "default(None, d)"]), + ("mixed", [1, [1, 2, 3], {"key": "val"}]), + ] + + def test_mixed_complex(self): + tag = parse_tag( + """ + {% component + data={ + "items": [ + 1|add:2, + {"x"|upper: 2|add:3}, + *spread_items|default:"" + ], + "nested": { + "a": [ + 1|add:2, + *nums|default:"" + ], + "b": { + "x": [ + *more|default:"" + ] + } + }, + **rest|injectd, + "key": _('value')|upper + } + %}""" + ) + + expected_tag = Tag( + name=TagToken(token="component", start_index=16, end_index=25, line_col=(2, 16)), + attrs=[ + TagAttr( + key=TagToken(token="data", start_index=38, end_index=42, line_col=(3, 13)), + value=TagValue( + token=TagToken( + token='{\n "items": [\n 1|add:2,\n {"x"|upper: 2|add:3},\n *spread_items|default:""\n ],\n "nested": {\n "a": [\n 1|add:2,\n *nums|default:""\n ],\n "b": {\n "x": [\n *more|default:""\n ]\n }\n },\n **rest|injectd,\n "key": _(\'value\')|upper\n }', + start_index=43, + end_index=614, + line_col=(3, 18), + ), + children=[ + TagValue( + token=TagToken(token='"items"', start_index=61, end_index=68, line_col=(4, 17)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=61, + end_index=68, + line_col=(4, 17), + ), + TagValue( + token=TagToken( + token='[\n 1|add:2,\n {"x"|upper: 2|add:3},\n *spread_items|default:""\n ]', + start_index=70, + end_index=205, + line_col=(4, 26), + ), + children=[ + TagValue( + token=TagToken(token="1", start_index=92, end_index=93, line_col=(5, 21)), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="add", start_index=94, end_index=97, line_col=(5, 23) + ), + arg=TagValue( + token=TagToken( + token="2", start_index=98, end_index=99, line_col=(5, 27) + ), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=97, + end_index=99, + line_col=(5, 26), + ), + start_index=93, + end_index=99, + line_col=(5, 22), + ) + ], + start_index=92, + end_index=99, + line_col=(5, 21), + ), + TagValue( + token=TagToken( + token='{"x"|upper: 2|add:3}', + start_index=121, + end_index=141, + line_col=(6, 21), + ), + children=[ + TagValue( + token=TagToken( + token='"x"', start_index=122, end_index=125, line_col=(6, 22) + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="upper", + start_index=126, + end_index=131, + line_col=(6, 26), + ), + arg=None, + start_index=125, + end_index=131, + line_col=(6, 25), + ) + ], + start_index=122, + end_index=131, + line_col=(6, 22), + ), + TagValue( + token=TagToken( + token="2", start_index=133, end_index=134, line_col=(6, 33) + ), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="add", + start_index=135, + end_index=138, + line_col=(6, 35), + ), + arg=TagValue( + token=TagToken( + token="3", + start_index=139, + end_index=140, + line_col=(6, 39), + ), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=138, + end_index=140, + line_col=(6, 38), + ), + start_index=134, + end_index=140, + line_col=(6, 34), + ) + ], + start_index=133, + end_index=140, + line_col=(6, 33), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=121, + end_index=141, + line_col=(6, 21), + ), + TagValue( + token=TagToken( + token="spread_items", start_index=164, end_index=176, line_col=(7, 22) + ), + children=[], + kind=ValueKind("variable"), + spread="*", + filters=[ + TagValueFilter( + token=TagToken( + token="default", start_index=177, end_index=184, line_col=(7, 35) + ), + arg=TagValue( + token=TagToken( + token='""', start_index=185, end_index=187, line_col=(7, 43) + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=184, + end_index=187, + line_col=(7, 42), + ), + start_index=176, + end_index=187, + line_col=(7, 34), + ) + ], + start_index=163, + end_index=187, + line_col=(7, 21), + ), + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=70, + end_index=205, + line_col=(4, 26), + ), + TagValue( + token=TagToken(token='"nested"', start_index=223, end_index=231, line_col=(9, 17)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=223, + end_index=231, + line_col=(9, 17), + ), + TagValue( + token=TagToken( + token='{\n "a": [\n 1|add:2,\n *nums|default:""\n ],\n "b": {\n "x": [\n *more|default:""\n ]\n }\n }', + start_index=233, + end_index=527, + line_col=(9, 27), + ), + children=[ + TagValue( + token=TagToken(token='"a"', start_index=255, end_index=258, line_col=(10, 21)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=255, + end_index=258, + line_col=(10, 21), + ), + TagValue( + token=TagToken( + token='[\n 1|add:2,\n *nums|default:""\n ]', + start_index=260, + end_index=357, + line_col=(10, 26), + ), + children=[ + TagValue( + token=TagToken( + token="1", start_index=286, end_index=287, line_col=(11, 25) + ), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="add", + start_index=288, + end_index=291, + line_col=(11, 27), + ), + arg=TagValue( + token=TagToken( + token="2", + start_index=292, + end_index=293, + line_col=(11, 31), + ), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=291, + end_index=293, + line_col=(11, 30), + ), + start_index=287, + end_index=293, + line_col=(11, 26), + ) + ], + start_index=286, + end_index=293, + line_col=(11, 25), + ), + TagValue( + token=TagToken( + token="nums", start_index=320, end_index=324, line_col=(12, 26) + ), + children=[], + kind=ValueKind("variable"), + spread="*", + filters=[ + TagValueFilter( + token=TagToken( + token="default", + start_index=325, + end_index=332, + line_col=(12, 31), + ), + arg=TagValue( + token=TagToken( + token='""', + start_index=333, + end_index=335, + line_col=(12, 39), + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=332, + end_index=335, + line_col=(12, 38), + ), + start_index=324, + end_index=335, + line_col=(12, 30), + ) + ], + start_index=319, + end_index=335, + line_col=(12, 25), + ), + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=260, + end_index=357, + line_col=(10, 26), + ), + TagValue( + token=TagToken(token='"b"', start_index=379, end_index=382, line_col=(14, 21)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=379, + end_index=382, + line_col=(14, 21), + ), + TagValue( + token=TagToken( + token='{\n "x": [\n *more|default:""\n ]\n }', + start_index=384, + end_index=509, + line_col=(14, 26), + ), + children=[ + TagValue( + token=TagToken( + token='"x"', start_index=410, end_index=413, line_col=(15, 25) + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=410, + end_index=413, + line_col=(15, 25), + ), + TagValue( + token=TagToken( + token='[\n *more|default:""\n ]', + start_index=415, + end_index=487, + line_col=(15, 30), + ), + children=[ + TagValue( + token=TagToken( + token="more", + start_index=446, + end_index=450, + line_col=(16, 30), + ), + children=[], + kind=ValueKind("variable"), + spread="*", + filters=[ + TagValueFilter( + token=TagToken( + token="default", + start_index=451, + end_index=458, + line_col=(16, 35), + ), + arg=TagValue( + token=TagToken( + token='""', + start_index=459, + end_index=461, + line_col=(16, 43), + ), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=458, + end_index=461, + line_col=(16, 42), + ), + start_index=450, + end_index=461, + line_col=(16, 34), + ) + ], + start_index=445, + end_index=461, + line_col=(16, 29), + ), + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=415, + end_index=487, + line_col=(15, 30), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=384, + end_index=509, + line_col=(14, 26), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=233, + end_index=527, + line_col=(9, 27), + ), + TagValue( + token=TagToken(token="rest", start_index=547, end_index=551, line_col=(20, 19)), + children=[], + kind=ValueKind("variable"), + spread="**", + filters=[ + TagValueFilter( + token=TagToken( + token="injectd", start_index=552, end_index=559, line_col=(20, 24) + ), + arg=None, + start_index=551, + end_index=559, + line_col=(20, 23), + ) + ], + start_index=545, + end_index=559, + line_col=(20, 17), + ), + TagValue( + token=TagToken(token='"key"', start_index=577, end_index=582, line_col=(21, 17)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=577, + end_index=582, + line_col=(21, 17), + ), + TagValue( + token=TagToken(token="_('value')", start_index=584, end_index=594, line_col=(21, 24)), + children=[], + kind=ValueKind("translation"), + spread=None, + filters=[ + TagValueFilter( + token=TagToken( + token="upper", start_index=595, end_index=600, line_col=(21, 35) + ), + arg=None, + start_index=594, + end_index=600, + line_col=(21, 34), + ) + ], + start_index=584, + end_index=600, + line_col=(21, 24), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=43, + end_index=614, + line_col=(3, 18), + ), + is_flag=False, + start_index=38, + end_index=614, + line_col=(3, 13), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=629, + line_col=(2, 16), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args1, kwargs1 = tag_func( + context={"spread_items": None, "nums": [1, 2, 3], "more": "x", "rest": {"a": "b"}}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: {**value, "injected": True} + if name == "injectd" + else f"{name}({value}, {arg})", + ) + assert args1 == [] + assert kwargs1 == [ + ( + "data", + { + "items": [ + "add(1, 2)", + {"upper(x, None)": "add(2, 3)"}, + *list("default(None, )"), + ], + "nested": { + "a": ["add(1, 2)", *list("default([1, 2, 3], )")], + "b": {"x": [*list("default(x, )")]}, + }, + "a": "b", + "injected": True, + "key": "upper(TRANSLATION_RESOLVED:value, None)", + }, + ), + ] + + +class TestSpread: + # Test that spread operator cannot be used as dictionary value + def test_spread_as_dictionary_value(self): + with pytest.raises( + SyntaxError, + match=re.escape("expected value or COMMENT"), + ): + parse_tag('{% component data={"key": **spread} %}') + + # NOTE: The Rust parser actually parses this successfully, + # treating `**spread|abc: 123` as a `spread` variable with a filter `abc` + # that has an argument `123`. + def test_spread_with_colon_interpreted_as_key(self): + tag = parse_tag("{% component data={**spread|abc: 123 } %}") + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=TagToken(token="data", start_index=13, end_index=17, line_col=(1, 14)), + value=TagValue( + token=TagToken(token="{**spread|abc: 123 }", start_index=18, end_index=38, line_col=(1, 19)), + children=[ + TagValue( + token=TagToken(token="spread", start_index=21, end_index=27, line_col=(1, 22)), + children=[], + kind=ValueKind("variable"), + spread="**", + filters=[ + TagValueFilter( + token=TagToken(token="abc", start_index=28, end_index=31, line_col=(1, 29)), + arg=TagValue( + token=TagToken( + token="123", start_index=33, end_index=36, line_col=(1, 34) + ), + children=[], + kind=ValueKind("int"), + spread=None, + filters=[], + start_index=31, + end_index=36, + line_col=(1, 32), + ), + start_index=27, + end_index=36, + line_col=(1, 28), + ) + ], + start_index=19, + end_index=36, + line_col=(1, 20), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=18, + end_index=38, + line_col=(1, 19), + ), + is_flag=False, + start_index=13, + end_index=38, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=41, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args1, kwargs1 = tag_func( + context={"spread": {6: 7}}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: {**value, "ABC": arg} + if name == "abc" + else f"{name}({value}, {arg})", + ) + assert args1 == [] + assert kwargs1 == [ + ("data", {6: 7, "ABC": 123}), + ] + + def test_spread_in_filter_position(self): + with pytest.raises( + SyntaxError, + match=re.escape("expected filter_name or COMMENT"), + ): + parse_tag("{% component data=val|...spread|abc } %}") + + def test_spread_whitespace_1(self): + # NOTE: Separating `...` from its variable is NOT valid, and will result in error. + with pytest.raises( + SyntaxError, + match=re.escape("expected value"), + ): + parse_tag("{% component ... attrs %}") + + # NOTE: But there CAN be whitespace between `*` / `**` and the value, + # because we're scoped inside `{ ... }` dict or `[ ... ]` list. + def test_spread_whitespace_2(self): + tag = parse_tag('{% component dict={"a": "b", ** my_attr} list=["a", * my_list] %}') + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=TagToken(token="dict", start_index=13, end_index=17, line_col=(1, 14)), + value=TagValue( + token=TagToken(token='{"a": "b", ** my_attr}', start_index=18, end_index=40, line_col=(1, 19)), + children=[ + TagValue( + token=TagToken(token='"a"', start_index=19, end_index=22, line_col=(1, 20)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=19, + end_index=22, + line_col=(1, 20), + ), + TagValue( + token=TagToken(token='"b"', start_index=24, end_index=27, line_col=(1, 25)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=24, + end_index=27, + line_col=(1, 25), + ), + TagValue( + token=TagToken(token="my_attr", start_index=32, end_index=39, line_col=(1, 33)), + children=[], + kind=ValueKind("variable"), + spread="**", + filters=[], + start_index=30, + end_index=39, + line_col=(1, 31), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=18, + end_index=40, + line_col=(1, 19), + ), + is_flag=False, + start_index=13, + end_index=40, + line_col=(1, 14), + ), + TagAttr( + key=TagToken(token="list", start_index=41, end_index=45, line_col=(1, 42)), + value=TagValue( + token=TagToken(token='["a", * my_list]', start_index=46, end_index=62, line_col=(1, 47)), + children=[ + TagValue( + token=TagToken(token='"a"', start_index=47, end_index=50, line_col=(1, 48)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=47, + end_index=50, + line_col=(1, 48), + ), + TagValue( + token=TagToken(token="my_list", start_index=54, end_index=61, line_col=(1, 55)), + children=[], + kind=ValueKind("variable"), + spread="*", + filters=[], + start_index=53, + end_index=61, + line_col=(1, 54), + ), + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=46, + end_index=62, + line_col=(1, 47), + ), + is_flag=False, + start_index=41, + end_index=62, + line_col=(1, 42), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=65, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args1, kwargs1 = tag_func( + context={"my_attr": {6: 7}, "my_list": [8, 9]}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args1 == [] + assert kwargs1 == [ + ("dict", {"a": "b", 6: 7}), + ("list", ["a", 8, 9]), + ] + + with pytest.raises( + TypeError, + match=re.escape("list' object is not a mapping"), + ): + tag_func( + context={"my_attr": [6, 7], "my_list": [8, 9]}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + # NOTE: This still works because even tho my_list is not a list, + # dictionaries are still iterable (same as dict.keys()). + args2, kwargs2 = tag_func( + context={"my_attr": {6: 7}, "my_list": {8: 9}}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args2 == [] + assert kwargs2 == [ + ("dict", {"a": "b", 6: 7}), + ("list", ["a", 8]), + ] + + # Test that one cannot use e.g. `...`, `**`, `*` in wrong places + def test_spread_incorrect_syntax(self): + with pytest.raises( + SyntaxError, + match=re.escape("expected dict_item_spread, dict_key, or COMMENT"), + ): + parse_tag('{% component dict={"a": "b", *my_attr} %}') + + with pytest.raises( + SyntaxError, + match=re.escape("expected dict_item_spread, dict_key, or COMMENT"), + ): + _ = parse_tag('{% component dict={"a": "b", ...my_attr} %}') + + with pytest.raises( + SyntaxError, + match=re.escape("expected value or COMMENT"), + ): + _ = parse_tag('{% component list=["a", "b", **my_list] %}') + + with pytest.raises( + SyntaxError, + match=re.escape("expected list_item or COMMENT"), + ): + _ = parse_tag('{% component list=["a", "b", ...my_list] %}') + + with pytest.raises( + SyntaxError, + match=re.escape("expected self_closing_slash, attribute, or COMMENT"), + ): + _ = parse_tag("{% component *attrs %}") + + with pytest.raises( + SyntaxError, + match=re.escape("expected self_closing_slash, attribute, or COMMENT"), + ): + _ = parse_tag("{% component **attrs %}") + + with pytest.raises( + SyntaxError, + match=re.escape("expected value"), + ): + _ = parse_tag("{% component key=*attrs %}") + + with pytest.raises( + SyntaxError, + match=re.escape("expected value"), + ): + _ = parse_tag("{% component key=**attrs %}") + + # Test that one cannot do `key=...{"a": "b"}` + def test_spread_onto_key(self): + with pytest.raises( + SyntaxError, + match=re.escape("expected value"), + ): + parse_tag('{% component key=...{"a": "b"} %}') + + with pytest.raises( + SyntaxError, + match=re.escape("expected value"), + ): + parse_tag('{% component key=...["a", "b"] %}') + + with pytest.raises( + SyntaxError, + match=re.escape("expected value"), + ): + parse_tag("{% component key=...attrs %}") + + def test_spread_dict_literal_nested(self): + tag = parse_tag('{% component { **{"key": val2}, "key": val1 } %}') + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken( + token='{ **{"key": val2}, "key": val1 }', start_index=13, end_index=45, line_col=(1, 14) + ), + children=[ + TagValue( + token=TagToken(token='{"key": val2}', start_index=17, end_index=30, line_col=(1, 18)), + children=[ + TagValue( + token=TagToken(token='"key"', start_index=18, end_index=23, line_col=(1, 19)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=18, + end_index=23, + line_col=(1, 19), + ), + TagValue( + token=TagToken(token="val2", start_index=25, end_index=29, line_col=(1, 26)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=25, + end_index=29, + line_col=(1, 26), + ), + ], + kind=ValueKind("dict"), + spread="**", + filters=[], + start_index=15, + end_index=30, + line_col=(1, 16), + ), + TagValue( + token=TagToken(token='"key"', start_index=32, end_index=37, line_col=(1, 33)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=32, + end_index=37, + line_col=(1, 33), + ), + TagValue( + token=TagToken(token="val1", start_index=39, end_index=43, line_col=(1, 40)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=39, + end_index=43, + line_col=(1, 40), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=13, + end_index=45, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=45, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=48, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"val1": 1, "val2": 2}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [{"key": 1}] + assert kwargs == [] + + def test_spread_dict_literal_as_attribute(self): + tag = parse_tag('{% component ...{"key": val2} %}') + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken(token='{"key": val2}', start_index=16, end_index=29, line_col=(1, 17)), + children=[ + TagValue( + token=TagToken(token='"key"', start_index=17, end_index=22, line_col=(1, 18)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=17, + end_index=22, + line_col=(1, 18), + ), + TagValue( + token=TagToken(token="val2", start_index=24, end_index=28, line_col=(1, 25)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=24, + end_index=28, + line_col=(1, 25), + ), + ], + kind=ValueKind("dict"), + spread="...", + filters=[], + start_index=13, + end_index=29, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=29, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=32, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"val1": 1, "val2": 2}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [] + assert kwargs == [("key", 2)] + + def test_spread_list_literal_nested(self): + tag = parse_tag("{% component [ *[val1], val2 ] %}") + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken(token="[ *[val1], val2 ]", start_index=13, end_index=30, line_col=(1, 14)), + children=[ + TagValue( + token=TagToken(token="[val1]", start_index=16, end_index=22, line_col=(1, 17)), + children=[ + TagValue( + token=TagToken(token="val1", start_index=17, end_index=21, line_col=(1, 18)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=17, + end_index=21, + line_col=(1, 18), + ), + ], + kind=ValueKind("list"), + spread="*", + filters=[], + start_index=15, + end_index=22, + line_col=(1, 16), + ), + TagValue( + token=TagToken(token="val2", start_index=24, end_index=28, line_col=(1, 25)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=24, + end_index=28, + line_col=(1, 25), + ), + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=13, + end_index=30, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=30, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=33, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"val1": 1, "val2": 2}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [[1, 2]] + assert kwargs == [] + + def test_spread_list_literal_as_attribute(self): + tag = parse_tag("{% component ...[val1] %}") + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken(token="[val1]", start_index=16, end_index=22, line_col=(1, 17)), + children=[ + TagValue( + token=TagToken(token="val1", start_index=17, end_index=21, line_col=(1, 18)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=17, + end_index=21, + line_col=(1, 18), + ), + ], + kind=ValueKind("list"), + spread="...", + filters=[], + start_index=13, + end_index=22, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=22, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=25, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"val1": 1, "val2": 2}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [1] + assert kwargs == [] + + +class TestTemplateString: + def test_template_string(self): + tag = parse_tag("{% component '{% lorem w 4 %}' %}") + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken(token="'{% lorem w 4 %}'", start_index=13, end_index=30, line_col=(1, 14)), + children=[], + kind=ValueKind("template_string"), + spread=None, + filters=[], + start_index=13, + end_index=30, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=30, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=33, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"val1": 1, "val2": 2}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == ["TEMPLATE_RESOLVED:{% lorem w 4 %}"] + assert kwargs == [] + + def test_template_string_in_dict(self): + tag = parse_tag('{% component { "key": "{% lorem w 4 %}" } %}') + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken( + token='{ "key": "{% lorem w 4 %}" }', start_index=13, end_index=41, line_col=(1, 14) + ), + children=[ + TagValue( + token=TagToken(token='"key"', start_index=15, end_index=20, line_col=(1, 16)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=15, + end_index=20, + line_col=(1, 16), + ), + TagValue( + token=TagToken( + token='"{% lorem w 4 %}"', start_index=22, end_index=39, line_col=(1, 23) + ), + children=[], + kind=ValueKind("template_string"), + spread=None, + filters=[], + start_index=22, + end_index=39, + line_col=(1, 23), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=13, + end_index=41, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=41, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=44, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"val1": 1, "val2": 2}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [{"key": "TEMPLATE_RESOLVED:{% lorem w 4 %}"}] + assert kwargs == [] + + def test_template_string_in_list(self): + tag = parse_tag("{% component [ '{% lorem w 4 %}' ] %}") + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken(token="[ '{% lorem w 4 %}' ]", start_index=13, end_index=34, line_col=(1, 14)), + children=[ + TagValue( + token=TagToken( + token="'{% lorem w 4 %}'", start_index=15, end_index=32, line_col=(1, 16) + ), + children=[], + kind=ValueKind("template_string"), + spread=None, + filters=[], + start_index=15, + end_index=32, + line_col=(1, 16), + ), + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=13, + end_index=34, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=34, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=37, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"val1": 1, "val2": 2}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [["TEMPLATE_RESOLVED:{% lorem w 4 %}"]] + assert kwargs == [] + + +class TestComments: + def test_comments(self): + tag = parse_tag("{% component {# comment #} val %}") + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken(token="val", start_index=27, end_index=30, line_col=(1, 28)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=27, + end_index=30, + line_col=(1, 28), + ), + is_flag=False, + start_index=27, + end_index=30, + line_col=(1, 28), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=33, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"val": 1, "val2": 2}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [1] + assert kwargs == [] + + def test_comments_within_list(self): + tag = parse_tag("{% component [ *[val1], {# comment #} val2 ] %}") + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken( + token="[ *[val1], {# comment #} val2 ]", start_index=13, end_index=44, line_col=(1, 14) + ), + children=[ + TagValue( + token=TagToken(token="[val1]", start_index=16, end_index=22, line_col=(1, 17)), + children=[ + TagValue( + token=TagToken(token="val1", start_index=17, end_index=21, line_col=(1, 18)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=17, + end_index=21, + line_col=(1, 18), + ), + ], + kind=ValueKind("list"), + spread="*", + filters=[], + start_index=15, + end_index=22, + line_col=(1, 16), + ), + TagValue( + token=TagToken(token="val2", start_index=38, end_index=42, line_col=(1, 39)), + children=[], + kind=ValueKind("variable"), + spread=None, + filters=[], + start_index=38, + end_index=42, + line_col=(1, 39), + ), + ], + kind=ValueKind("list"), + spread=None, + filters=[], + start_index=13, + end_index=44, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=44, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=47, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"val1": 1, "val2": 2}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [[1, 2]] + assert kwargs == [] + + def test_comments_within_dict(self): + tag = parse_tag('{% component { "key": "123" {# comment #} } %}') + + expected_tag = Tag( + name=TagToken(token="component", start_index=3, end_index=12, line_col=(1, 4)), + attrs=[ + TagAttr( + key=None, + value=TagValue( + token=TagToken( + token='{ "key": "123" {# comment #} }', start_index=13, end_index=43, line_col=(1, 14) + ), + children=[ + TagValue( + token=TagToken(token='"key"', start_index=15, end_index=20, line_col=(1, 16)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=15, + end_index=20, + line_col=(1, 16), + ), + TagValue( + token=TagToken(token='"123"', start_index=22, end_index=27, line_col=(1, 23)), + children=[], + kind=ValueKind("string"), + spread=None, + filters=[], + start_index=22, + end_index=27, + line_col=(1, 23), + ), + ], + kind=ValueKind("dict"), + spread=None, + filters=[], + start_index=13, + end_index=43, + line_col=(1, 14), + ), + is_flag=False, + start_index=13, + end_index=43, + line_col=(1, 14), + ), + ], + is_self_closing=False, + syntax=TagSyntax("django"), + start_index=0, + end_index=46, + line_col=(1, 4), + ) + + assert tag == expected_tag + + tag_func = compile_tag(tag) + args, kwargs = tag_func( + context={"val1": 1, "val2": 2}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [{"key": "123"}] + assert kwargs == [] + + +class TestParamsOrder: + def test_arg_after_kwarg_is_error(self): + tag_content = "{% my_tag key='value' positional_arg %}" + ast = parse_tag(input=tag_content) + with pytest.raises(SyntaxError, match="positional argument follows keyword argument"): + compile_tag(tag_or_attrs=ast) + + def test_arg_after_dict_spread_is_error(self): + tag_content = "{% my_tag ...{'key': 'value'} positional_arg %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + + with pytest.raises(SyntaxError, match="positional argument follows keyword argument"): + tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + def test_arg_after_list_spread_is_ok(self): + tag_content = "{% my_tag ...[1, 2, 3] positional_arg %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={"positional_arg": 4}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [1, 2, 3, 4] + assert kwargs == [] + + def test_dict_spread_after_arg_is_ok(self): + tag_content = "{% my_tag positional_arg ...{'key': 'value'} %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={"positional_arg": 1}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [1] + assert kwargs == [("key", "value")] + + def test_dict_spread_after_kwarg_is_ok(self): + tag_content = "{% my_tag key='value' ...{'key2': 'value2'} %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [] + assert kwargs == [("key", "value"), ("key2", "value2")] + + def test_list_spread_after_arg_is_ok(self): + tag_content = "{% my_tag positional_arg ...[1, 2, 3] %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={"positional_arg": 4}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [4, 1, 2, 3] + assert kwargs == [] + + def test_list_spread_after_kwarg_is_error(self): + tag_content = "{% my_tag key='value' ...[1, 2, 3] %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + with pytest.raises(SyntaxError, match="positional argument follows keyword argument"): + tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + def test_list_spread_after_list_spread_is_ok(self): + tag_content = "{% my_tag ...[1, 2, 3] ...[4, 5, 6] %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [1, 2, 3, 4, 5, 6] + assert kwargs == [] + + def test_dict_spread_after_dict_spread_is_ok(self): + tag_content = "{% my_tag ...{'key': 'value'} ...{'key2': 'value2'} %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [] + assert kwargs == [("key", "value"), ("key2", "value2")] + + def test_list_spread_after_dict_spread_is_error(self): + tag_content = "{% my_tag ...{'key': 'value'} ...[1, 2, 3] %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + with pytest.raises(SyntaxError, match="positional argument follows keyword argument"): + tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + def test_dict_spread_after_list_spread_is_ok(self): + tag_content = "{% my_tag ...[1, 2, 3] ...{'key': 'value'} %}" + ast = parse_tag(input=tag_content) + tag_func = compile_tag(ast) + args, kwargs = tag_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args == [1, 2, 3] + assert kwargs == [("key", "value")] + + +class TestFlags: + def test_flag(self): + tag_content = "{% my_tag 123 my_flag key='val' %}" + + ast = parse_tag(tag_content, flags={"my_flag"}) + assert ast.attrs[1].value.token.token == "my_flag" + assert ast.attrs[1].is_flag + + # The compiled function should omit the flag + compiled_func = compile_tag(ast) + args, kwargs = compiled_func( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + assert args == [123] + assert kwargs == [("key", "val")] + + # Same as before, but with flags=None + ast2 = parse_tag(tag_content, flags=None) + assert ast2.attrs[1].value.token.token == "my_flag" + assert not ast2.attrs[1].is_flag + + # The compiled function should omit the flag + compiled_func2 = compile_tag(ast2) + args2, kwargs2 = compiled_func2( + context={"my_flag": "x"}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + assert args2 == [123, "x"] + assert kwargs2 == [("key", "val")] + + # my_flag is NOT treated as flag because it's used as spread + def test_flag_as_spread(self): + tag_content = "{% my_tag ...my_flag %}" + + ast1 = parse_tag(tag_content, flags={"my_flag"}) + assert ast1.attrs[0].value.token.token == "my_flag" + assert not ast1.attrs[0].is_flag + + compiled_func1 = compile_tag(ast1) + args1, kwargs1 = compiled_func1( + context={"my_flag": ["arg1", "arg2"]}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + assert args1 == ["arg1", "arg2"] + assert kwargs1 == [] + + # Same as before, but with flags=None + ast2 = parse_tag(tag_content, flags=None) + assert ast2.attrs[0].value.token.token == "my_flag" + assert not ast2.attrs[0].is_flag + + compiled_func2 = compile_tag(ast2) + args2, kwargs2 = compiled_func2( + context={"my_flag": ["arg1", "arg2"]}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + + assert args2 == ["arg1", "arg2"] + assert kwargs2 == [] + + # my_flag is NOT treated as flag because it's used as kwarg + def test_flag_as_kwarg(self): + tag_content = "{% my_tag my_flag=123 %}" + + ast1 = parse_tag(tag_content, flags={"my_flag"}) + assert ast1.attrs[0].key + assert ast1.attrs[0].key.token == "my_flag" + assert not ast1.attrs[0].is_flag + + compiled_func1 = compile_tag(ast1) + args1, kwargs1 = compiled_func1( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args1 == [] + assert kwargs1 == [("my_flag", 123)] + + # Same as before, but with flags=None + ast2 = parse_tag(tag_content, flags=None) + assert ast2.attrs[0].key + assert ast2.attrs[0].key.token == "my_flag" + assert not ast2.attrs[0].is_flag + + compiled_func2 = compile_tag(ast2) + args2, kwargs2 = compiled_func2( + context={}, + variable=lambda ctx, var: ctx[var], + template_string=lambda ctx, expr: f"TEMPLATE_RESOLVED:{expr}", + translation=lambda ctx, text: f"TRANSLATION_RESOLVED:{text}", + filter=lambda ctx, name, value, arg=None: f"{name}({value}, {arg})", + ) + assert args2 == [] + assert kwargs2 == [("my_flag", 123)] + + def test_flag_duplicate(self): + tag_content = "{% my_tag my_flag my_flag %}" + with pytest.raises(SyntaxError, match=r"Flag 'my_flag' may be specified only once."): + parse_tag(tag_content, flags={"my_flag"}) + + def test_flag_case_sensitive(self): + tag_content = "{% my_tag my_flag %}" + ast = parse_tag(tag_content, flags={"MY_FLAG"}) + assert ast.attrs[0].value.token.token == "my_flag" + assert not ast.attrs[0].is_flag + + +class TestSelfClosing: + def test_self_closing_simple(self): + ast = parse_tag("{% my_tag / %}") + assert ast.name.token == "my_tag" + assert ast.is_self_closing is True + assert ast.attrs == [] + + def test_self_closing_with_args(self): + ast = parse_tag("{% my_tag key=val / %}") + assert ast.name.token == "my_tag" + assert ast.is_self_closing is True + assert len(ast.attrs) == 1 + assert ast.attrs[0].key + assert ast.attrs[0].key.token == "key" + assert ast.attrs[0].value.token.token == "val" + + def test_self_closing_in_middle_errors(self): + with pytest.raises( + SyntaxError, + match=r"expected attribute or COMMENT", + ): + parse_tag("{% my_tag / key=val %}")