diff --git a/DEV.md b/DEV.md index dbe2004..d99f842 100644 --- a/DEV.md +++ b/DEV.md @@ -18,6 +18,8 @@ Install dependencies: pip install -r requirements.txt ``` +To update requirements.txt, use https://azurda.github.io/ + ## Test ```bash diff --git a/LICENSE.md b/LICENSE.md index 211de03..34b851f 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ The ISC License -Copyright (c) 2024 by Jos de Jong +Copyright (c) 2024-2025 by Jos de Jong Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. diff --git a/README.md b/README.md index 4e5fd17..9d0c12a 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,8 @@ Where: - `data` is a JSON object or array - `query` is a JSON query or string containing a text query - `options` is an optional object which can have the following options: - - `functions` an object with custom functions + - `functions` an object with custom functions, see section [Custom functions](#custom-functions). + - `operators` a list with custom operators, see section [Custom operators](#custom-operators). Example: @@ -91,13 +92,13 @@ Example: from pprint import pprint from jsonquerylang import jsonquery -input = [ +data = [ {"name": "Chris", "age": 23, "scores": [7.2, 5, 8.0]}, {"name": "Joe", "age": 32, "scores": [6.1, 8.1]}, {"name": "Emily", "age": 19}, ] query = ["sort", ["get", "age"], "desc"] -output = jsonquery(input, query) +output = jsonquery(data, query) pprint(output) # [{'age': 32, 'name': 'Joe', 'scores': [6.1, 8.1]}, # {'age': 23, 'name': 'Chris', 'scores': [7.2, 5, 8.0]}, @@ -118,7 +119,7 @@ Where: - `query` is a JSON query or string containing a text query - `options` is an optional object which can have the following options: - - `functions` an object with custom functions + - `functions` an object with custom functions, see section [Custom functions](#custom-functions). The function returns a lambda function which can be executed by passing JSON data as first argument. @@ -128,14 +129,14 @@ Example: from pprint import pprint from jsonquerylang import compile -input = [ +data = [ {"name": "Chris", "age": 23, "scores": [7.2, 5, 8.0]}, {"name": "Joe", "age": 32, "scores": [6.1, 8.1]}, {"name": "Emily", "age": 19}, ] query = ["sort", ["get", "age"], "desc"] queryMe = compile(query) -output = queryMe(input) +output = queryMe(data) pprint(output) # [{'age': 32, 'name': 'Joe', 'scores': [6.1, 8.1]}, # {'age': 23, 'name': 'Chris', 'scores': [7.2, 5, 8.0]}, @@ -156,8 +157,8 @@ Where: - `textQuery`: A query in text format - `options`: An optional object which can have the following properties: - - `functions` an object with custom functions - - `operators` an object with the names of custom operators both as key and value + - `functions` an object with custom functions, see section [Custom functions](#custom-functions). + - `operators` a list with custom operators, see section [Custom operators](#custom-operators) Example: @@ -189,8 +190,8 @@ Where: - `query` is a JSON Query - `options` is an optional object that can have the following properties: - - `operators` an object with the names of custom operators both as key and value - - `indentation` a string containing the desired indentation, defaults to two spaces: `" "` + - `operators` a list with custom operators, see section [Custom operators](#custom-operators). + - `indentation` a string containing the desired indentation, defaults to two spaces: `" "`. - `max_line_length` a number with the maximum line length, used for wrapping contents. Default value: `40`. Example: @@ -210,6 +211,85 @@ print(textQuery) # '.friends | filter(.city == "new York") | sort(.age) | pick(.name, .age)' ``` +## Custom functions + +The functions `jsonquery`, `compile`, and `parse` accept custom functions. Custom functions are passed as an object with the key being the function name, and the value being a factory function. + +Here is a minimal example which adds a function `times` to JSON Query: + +```python +from jsonquerylang import jsonquery, JsonQueryOptions + + +def fn_times(value): + return lambda array: list(map(lambda item: item * value, array)) + + +data = [2, 3, 8] +query = 'times(2)' +options: JsonQueryOptions = {"functions": {"times": fn_times}} + +print(jsonquery(data, query, options)) +# [4, 6, 16] +``` + +In the example above, the argument `value` is static. When the parameters are not static, the function `compile` can be used to compile them. For example, the function filter is implemented as follows: + +```python +from jsonquerylang import compile, JsonQueryOptions + +def truthy(value): + return value not in [False, 0, None] + +def fn_filter(predicate): + _predicate = compile(predicate) + + return lambda data: list(filter(lambda item: truthy(_predicate(item)), data)) + +options: JsonQueryOptions = {"functions": {"filter": fn_filter}} +``` + +You can have a look at the source code of the functions in [`/jsonquerylang/functions.py`](/jsonquerylang/functions.py) for more examples. + +## Custom operators + +The functions `jsonquery`, `parse`, and `stringify` accept custom operators. Custom operators are passed as a list with operators definitions. In practice, often both a custom operator and a corresponding custom function are configured. Each custom operator is an object with: + +- Two required properties `name` and `op` to specify the function name and operator name, for example `{ "name": "add", "op": "+", ... }` +- One of the three properties `at`, `before`, or `after`, specifying the precedence compared to an existing operator. +- optionally, the property `left_associative` can be set to `True` to allow using a chain of multiple operators without parenthesis, like `a and b and c`. Without this, an exception will be thrown, which can be solved by using parenthesis like `(a and b) and c`. +- optionally, the property `vararg` can be set to `True` when the function supports a variable number of arguments, like `and(a, b, c, ...)`. In that case, a chain of operators like `a and b and c` will be parsed into the JSON Format `["and", a, b, c, ...]`. Operators that do not support variable arguments, like `1 + 2 + 3`, will be parsed into a nested JSON Format like `["add", ["add", 1, 2], 3]`. + +Here is a minimal example configuring a custom operator `~=` and a corresponding function `aboutEq`: + +```python +from jsonquerylang import jsonquery, compile, JsonQueryOptions + + +def about_eq(a, b): + epsilon = 0.001 + a_compiled = compile(a, options) + b_compiled = compile(b, options) + + return lambda data: abs(a_compiled(data) - b_compiled(data)) < epsilon + + +options: JsonQueryOptions = { + "functions": {"aboutEq": about_eq}, + "operators": [{"name": "aboutEq", "op": "~=", "at": "=="}], +} + +scores = [ + {"name": "Joe", "score": 2.0001, "previousScore": 1.9999}, + {"name": "Sarah", "score": 3, "previousScore": 1.5}, +] +query = "filter(.score ~= .previousScore)" +unchanged_scores = jsonquery(scores, query, options) + +print(unchanged_scores) +# [{'name': 'Joe', 'score': 2.0001, 'previousScore': 1.9999}] +``` + ## License Released under the [ISC license](LICENSE.md). diff --git a/example4_custom_functions.py b/example4_custom_functions.py index 148ec9d..8f10f5d 100644 --- a/example4_custom_functions.py +++ b/example4_custom_functions.py @@ -10,3 +10,4 @@ def times(value): options: JsonQueryOptions = {"functions": {"times": times}} print(jsonquery(data, query, options)) +# [4, 6, 16] diff --git a/example5_custom_operators.py b/example5_custom_operators.py new file mode 100644 index 0000000..6c677d6 --- /dev/null +++ b/example5_custom_operators.py @@ -0,0 +1,25 @@ +from jsonquerylang import jsonquery, compile, JsonQueryOptions + + +def about_eq(a, b): + epsilon = 0.001 + a_compiled = compile(a, options) + b_compiled = compile(b, options) + + return lambda data: abs(a_compiled(data) - b_compiled(data)) < epsilon + + +options: JsonQueryOptions = { + "functions": {"aboutEq": about_eq}, + "operators": [{"name": "aboutEq", "op": "~=", "at": "=="}], +} + +scores = [ + {"name": "Joe", "score": 2.0001, "previousScore": 1.9999}, + {"name": "Sarah", "score": 3, "previousScore": 1.5}, +] +query = "filter(.score ~= .previousScore)" +unchanged_scores = jsonquery(scores, query, options) + +print(unchanged_scores) +# [{'name': 'Joe', 'score': 2.0001, 'previousScore': 1.9999}] diff --git a/jsonquerylang/__init__.py b/jsonquerylang/__init__.py index d0784db..8a99d2d 100644 --- a/jsonquerylang/__init__.py +++ b/jsonquerylang/__init__.py @@ -1,5 +1,5 @@ from jsonquerylang.jsonquery import jsonquery -from jsonquerylang.compile import compile +from jsonquerylang.compile import compile, build_function from jsonquerylang.stringify import stringify from jsonquerylang.parse import parse from jsonquerylang.types import ( diff --git a/jsonquerylang/compile.py b/jsonquerylang/compile.py index 85d07e2..3e2d796 100644 --- a/jsonquerylang/compile.py +++ b/jsonquerylang/compile.py @@ -33,6 +33,8 @@ def compile( :return: Returns a function which can execute the query """ + functions = get_functions(lambda q: compile(q, options), build_function) + custom_functions: Final = (options.get("functions") if options else None) or {} all_functions: Final = {**functions, **custom_functions} @@ -57,4 +59,12 @@ def compile( return lambda _: query -functions = get_functions(compile) +def build_function(fn): + def evaluate_fn(*args): + compiled_args = list(map(compile, args)) + + return lambda data: fn( + *list(map(lambda compiled_arg: compiled_arg(data), compiled_args)) + ) + + return evaluate_fn diff --git a/jsonquerylang/functions.py b/jsonquerylang/functions.py index ee75abc..a274ee9 100644 --- a/jsonquerylang/functions.py +++ b/jsonquerylang/functions.py @@ -3,17 +3,7 @@ import re -def get_functions(compile): - def build_function(fn): - def evaluate_fn(*args): - compiled_args = list(map(compile, args)) - - return lambda data: fn( - *list(map(lambda compiled_arg: compiled_arg(data), compiled_args)) - ) - - return evaluate_fn - +def get_functions(compile, build_function): def fn_get(*path: []): def getter(item): value = item @@ -175,8 +165,8 @@ def key_by(data): fn_min = lambda: lambda data: min(data) fn_max = lambda: lambda data: max(data) - fn_and = build_function(lambda a, b: a and b) - fn_or = build_function(lambda a, b: a or b) + fn_and = build_function(lambda *args: reduce(lambda a, b: a and b, args)) + fn_or = build_function(lambda *args: reduce(lambda a, b: a or b, args)) fn_not = build_function(lambda a: not a) def fn_exists(query_get): diff --git a/jsonquerylang/operators.py b/jsonquerylang/operators.py new file mode 100644 index 0000000..0713257 --- /dev/null +++ b/jsonquerylang/operators.py @@ -0,0 +1,55 @@ +from functools import reduce +from typing import Mapping +from jsonquerylang.types import OperatorGroup, CustomOperator +from jsonquerylang.utils import find_index + +operators: list[OperatorGroup] = [ + {"pow": "^"}, + {"multiply": "*", "divide": "/", "mod": "%"}, + {"add": "+", "subtract": "-"}, + {"gt": ">", "gte": ">=", "lt": "<", "lte": "<=", "in": "in", "not in": "not in"}, + {"eq": "==", "ne": "!="}, + {"and": "and"}, + {"or": "or"}, + {"pipe": "|"}, +] + +vararg_operators = ["|", "and", "or"] + +left_associative_operators = ["|", "and", "or", "*", "/", "%", "+", "-"] + + +def extend_operators( + all_operators: list[OperatorGroup], custom_operators: list[CustomOperator] +) -> list[OperatorGroup]: + # backward compatibility error with v4 where `operators` was an object + if type(custom_operators) is not list: + raise RuntimeError("Invalid custom operators") + + return reduce(extend_operator, custom_operators, all_operators) + + +def extend_operator( + all_operators: list[OperatorGroup], custom_operator: CustomOperator +) -> list[OperatorGroup]: + name = custom_operator.get("name") + op = custom_operator.get("op") + at = custom_operator.get("at") + after = custom_operator.get("after") + before = custom_operator.get("before") + + if at: + callback = lambda group: {**group, name: op} if at in group.values() else group + + return list(map(callback, all_operators)) + + search_op = after or before + index = find_index(lambda group: search_op in group.values(), all_operators) + if index != -1: + updated_operators = all_operators.copy() + new_group: Mapping[str, str] = {name: op} + updated_operators.insert(index + (1 if after else 0), new_group) + + return updated_operators + + raise RuntimeError("Invalid custom operator") diff --git a/jsonquerylang/parse.py b/jsonquerylang/parse.py index cc4074a..a87f912 100644 --- a/jsonquerylang/parse.py +++ b/jsonquerylang/parse.py @@ -1,17 +1,25 @@ import json +from functools import reduce from typing import Optional, Callable, Pattern, Final -from jsonquerylang.compile import functions -from jsonquerylang.constants import ( +from jsonquerylang.compile import compile, build_function +from jsonquerylang.functions import get_functions +from jsonquerylang.regexps import ( starts_with_whitespace_regex, starts_with_keyword_regex, starts_with_int_regex, starts_with_number_regex, starts_with_unquoted_property_regex, starts_with_string_regex, +) +from jsonquerylang.operators import ( operators, + extend_operators, + left_associative_operators, + vararg_operators, ) -from jsonquerylang.types import JsonQueryParseOptions, JsonQueryType +from jsonquerylang.types import JsonQueryParseOptions, JsonQueryType, OperatorGroup +from jsonquerylang.utils import merge def parse(query: str, options: Optional[JsonQueryParseOptions] = None) -> JsonQueryType: @@ -36,65 +44,94 @@ def parse(query: str, options: Optional[JsonQueryParseOptions] = None) -> JsonQu :param options: Can an object with custom operators and functions :return: Returns the query in JSON format """ - jsonQuery = [ - "pipe", - ["get", "friends"], - ["filter", ["eq", ["get", "city"], "New York"]], - ["sort", ["get", "age"]], - ["pick", ["get", "name"], ["get", "age"]], - ] - custom_operators: Final = (options.get("operators") if options else None) or {} - custom_functions: Final = (options.get("functions") if options else None) or {} - all_functions: Final = {**functions, **custom_functions} - all_operators: Final = {**operators, **custom_operators} - sorted_operator_names: Final = sorted( - all_operators.keys(), key=lambda name: len(name), reverse=True + + functions = get_functions(lambda q: compile(q, options), build_function) + custom_functions: Final = options.get("functions", {}) if options else {} + custom_operators: Final = options.get("operators", []) if options else [] + + all_functions: Final = merge(functions, custom_functions) + all_operators: Final = extend_operators(operators, custom_operators) + all_operators_map: Final = reduce(merge, all_operators) + all_vararg_operators: Final = vararg_operators + list( + map( + lambda op: op.get("op"), + filter(lambda op: op.get("vararg"), custom_operators), + ) + ) + all_left_associative_operators: Final = left_associative_operators + list( + map( + lambda op: op.get("op"), + filter(lambda op: op.get("left_associative"), custom_operators), + ) ) i = 0 - def parse_pipe(): + def parse_operator(precedence_level: int = len(all_operators) - 1): nonlocal i - skip_whitespace() - first = parse_operator() - skip_whitespace() + if precedence_level < 0: + return parse_parenthesis() - if get_char() == "|": - pipe = [first] + current_operators = all_operators[precedence_level] - while i < len(query) and get_char() == "|": - i += 1 - skip_whitespace() - pipe.append(parse_operator()) + left_parenthesis = get_char() == "(" + left = parse_operator(precedence_level - 1) - return ["pipe", *pipe] + while True: + skip_whitespace() - return first + start = i + name = parse_operator_name(current_operators) + if not name: + break - def parse_operator(): - nonlocal i + right = parse_operator(precedence_level - 1) - left = parse_parenthesis() + child_name = left[0] if type(left) is list else None + chained = name == child_name and not left_parenthesis + if ( + chained + and all_operators_map[name] not in all_left_associative_operators + ): + i = start + break - skip_whitespace() + left = ( + [*left, right] + if chained and all_operators_map[name] in all_vararg_operators + else [name, left, right] + ) + + return left + + def parse_operator_name(current_operators: OperatorGroup) -> str | None: + nonlocal i + + # we sort the operators from longest to shortest, so we first handle "<=" and next "<" + sorted_operator_names: Final = sorted( + current_operators.keys(), key=lambda _name: len(_name), reverse=True + ) for name in sorted_operator_names: - op = all_operators[name] + op = current_operators[name] if query[i : i + len(op)] == op: i += len(op) + skip_whitespace() - right = parse_parenthesis() - return [name, left, right] - return left + return name + + return None def parse_parenthesis(): nonlocal i + skip_whitespace() + if get_char() == "(": i += 1 - inner = parse_pipe() + inner = parse_operator() eat_char(")") return inner @@ -135,11 +172,11 @@ def parse_function(): skip_whitespace() - args = [parse_pipe()] if get_char() != ")" else [] + args = [parse_operator()] if get_char() != ")" else [] while i < len(query) and get_char() != ")": skip_whitespace() eat_char(",") - args.append(parse_pipe()) + args.append(parse_operator()) eat_char(")") @@ -166,7 +203,7 @@ def parse_object(): skip_whitespace() eat_char(":") - object[key] = parse_pipe() + object[key] = parse_operator() eat_char("}") @@ -205,7 +242,7 @@ def parse_array(): eat_char(",") skip_whitespace() - array.append(parse_pipe()) + array.append(parse_operator()) eat_char("]") @@ -273,7 +310,7 @@ def get_char(): def raise_error(message: str, pos: Optional[int] = i): raise SyntaxError(f"{message} (pos: {pos if pos else i})") - output = parse_pipe() + output = parse_operator() parse_end() return output diff --git a/jsonquerylang/constants.py b/jsonquerylang/regexps.py similarity index 66% rename from jsonquerylang/constants.py rename to jsonquerylang/regexps.py index e4ddd75..084f795 100644 --- a/jsonquerylang/constants.py +++ b/jsonquerylang/regexps.py @@ -1,24 +1,5 @@ import re -operators = { - "and": "and", - "or": "or", - "eq": "==", - "gt": ">", - "gte": ">=", - "lt": "<", - "lte": "<=", - "ne": "!=", - "add": "+", - "subtract": "-", - "multiply": "*", - "divide": "/", - "pow": "^", - "mod": "%", - "in": "in", - "not in": "not in", -} - unquoted_property_regex = re.compile(r"^[a-zA-Z_$][a-zA-Z\d_$]*$") starts_with_unquoted_property_regex = re.compile(r"^[a-zA-Z_$][a-zA-Z\d_$]*") starts_with_string_regex = re.compile( diff --git a/jsonquerylang/stringify.py b/jsonquerylang/stringify.py index e3a7ce0..6dfcd8f 100644 --- a/jsonquerylang/stringify.py +++ b/jsonquerylang/stringify.py @@ -1,7 +1,13 @@ import json +from functools import reduce from typing import List, Optional, Union, Final -from jsonquerylang.constants import operators, unquoted_property_regex +from jsonquerylang.regexps import unquoted_property_regex +from jsonquerylang.operators import ( + operators, + extend_operators, + left_associative_operators, +) from jsonquerylang.types import ( JsonQueryType, JsonQueryStringifyOptions, @@ -9,6 +15,7 @@ JsonPath, JsonQueryFunctionType, ) +from jsonquerylang.utils import find_index, merge DEFAULT_MAX_LINE_LENGTH = 40 DEFAULT_INDENTATION = " " @@ -46,25 +53,30 @@ def stringify( max_line_length: Final = ( options.get("max_line_length") if options else None ) or DEFAULT_MAX_LINE_LENGTH - custom_operators: Final = (options.get("operators") if options else None) or {} - all_operators: Final = {**operators, **custom_operators} + custom_operators: Final = (options.get("operators") if options else None) or [] + all_operators: Final = extend_operators(operators, custom_operators) + all_operators_map: Final = reduce(merge, all_operators) + all_left_associative_operators: Final = left_associative_operators + list( + map( + lambda op: op["op"], + filter(lambda op: op.get("left_associative"), custom_operators), + ) + ) - def _stringify(_query: JsonQueryType, indent: str) -> str: + def _stringify(_query: JsonQueryType, indent: str, parenthesis=False) -> str: if type(_query) is list: - return stringify_function(_query, indent) + return stringify_function(_query, indent, parenthesis) else: return json.dumps(_query) # value (string, number, boolean, null) - def stringify_function(query_fn: JsonQueryFunctionType, indent: str) -> str: + def stringify_function( + query_fn: JsonQueryFunctionType, indent: str, parenthesis: bool + ) -> str: name, *args = query_fn if name == "get" and len(args) > 0: return stringify_path(args) - if name == "pipe": - args_str = stringify_args(args, indent + space) - return join(args_str, ["", " | ", ""], ["", f"\n{indent + space}| ", ""]) - if name == "object": return stringify_object(args[0], indent) @@ -76,31 +88,51 @@ def stringify_function(query_fn: JsonQueryFunctionType, indent: str) -> str: [f"[\n{indent + space}", f",\n{indent + space}", f"\n{indent}]"], ) - op = all_operators.get(name) - if op is not None and len(args) == 2: - left, right = args - left_str = _stringify(left, indent) - right_str = _stringify(right, indent) - return f"({left_str} {op} {right_str})" + # operator like ".age >= 18" + op = all_operators_map.get(name) + if op: + start = "(" if parenthesis else "" + end = ")" if parenthesis else "" + + def stringify_operator_arg(arg: JsonQueryType, index: int): + child_name = arg[0] if type(arg) is list else None + precedence = find_index(lambda group: name in group, all_operators) + child_precedence = find_index( + lambda group: child_name in group, all_operators + ) + child_parenthesis = ( + precedence < child_precedence + or (precedence == child_precedence and index > 0) + or (name == child_name and op not in all_left_associative_operators) + ) + + return _stringify(arg, indent + space, child_parenthesis) + + args_str = [ + stringify_operator_arg(arg, index) for index, arg in enumerate(args) + ] - child_indent = indent if len(args) == 1 else indent + space - args_str = stringify_args(args, child_indent) - return ( - f"{name}{args_str[0]}" - if len(args) == 1 and args_str[0][0] == "(" - else join( + return join( args_str, - [f"{name}(", ", ", ")"], - ( - [f"{name}(", f",\n{indent}", ")"] - if len(args) == 1 - else [ - f"{name}(\n{child_indent}", - f",\n{child_indent}", - f"\n{indent})", - ] - ), + [start, f" {op} ", end], + [start, f"\n{indent + space}{op} ", end], ) + + # regular function like "sort(.age)" + child_indent = indent if len(args) == 1 else indent + space + args_str = stringify_args(args, child_indent) + return join( + args_str, + [f"{name}(", ", ", ")"], + ( + [f"{name}(", f",\n{indent}", ")"] + if len(args) == 1 + else [ + f"{name}(\n{child_indent}", + f",\n{child_indent}", + f"\n{indent})", + ] + ), ) def stringify_object(query_obj: JsonQueryObjectType, indent: str) -> str: diff --git a/jsonquerylang/types.py b/jsonquerylang/types.py index 65951ea..9a52c2c 100644 --- a/jsonquerylang/types.py +++ b/jsonquerylang/types.py @@ -13,16 +13,49 @@ JsonQueryObjectType: TypeAlias = Mapping[str, JsonQueryType] +OperatorGroup: TypeAlias = Mapping[str, str] + + +class CustomOperatorAt(TypedDict): + name: str + op: str + at: str + vararg: NotRequired[bool] + left_associative: NotRequired[bool] + + +class CustomOperatorBefore(TypedDict): + name: str + op: str + before: str + vararg: NotRequired[bool] + left_associative: NotRequired[bool] + + +class CustomOperatorAfter(TypedDict): + name: str + op: str + after: str + vararg: NotRequired[bool] + left_associative: NotRequired[bool] + + +CustomOperator: TypeAlias = ( + CustomOperatorAt | CustomOperatorBefore | CustomOperatorAfter +) + + class JsonQueryOptions(TypedDict): functions: NotRequired[Mapping[str, Callable]] + operators: NotRequired[list[CustomOperator]] class JsonQueryStringifyOptions(TypedDict): - operators: NotRequired[Mapping[str, str]] + operators: NotRequired[list[CustomOperator]] max_line_length: NotRequired[int] indentation: NotRequired[str] class JsonQueryParseOptions(TypedDict): functions: NotRequired[Mapping[str, bool] | Mapping[str, Callable]] - operators: NotRequired[Mapping[str, str]] + operators: NotRequired[list[CustomOperator]] diff --git a/jsonquerylang/utils.py b/jsonquerylang/utils.py new file mode 100644 index 0000000..42fb05c --- /dev/null +++ b/jsonquerylang/utils.py @@ -0,0 +1,13 @@ +from typing import Mapping + + +def find_index(predicate, iterable): + for i, item in enumerate(iterable): + if predicate(item): + return i + + return -1 + + +def merge(a: Mapping[str, str], b: Mapping[str, str]): + return {**a, **b} diff --git a/requirements.txt b/requirements.txt index 140aab1..daad4e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,31 +1,31 @@ build==1.2.2.post1 -certifi==2024.8.30 -charset-normalizer==3.4.0 -colorama==0.4.0 +certifi==2025.1.31 +charset-normalizer==3.4.1 +colorama==0.4.6 docutils==0.21.2 idna==3.10 -importlib_metadata==8.5.0 +importlib_metadata==8.6.1 jaraco.classes==3.4.0 jaraco.context==6.0.1 jaraco.functools==4.1.0 -keyring==25.5.0 +keyring==25.6.0 markdown-it-py==3.0.0 mdurl==0.1.2 -more-itertools==10.5.0 -nh3==0.2.18 +more-itertools==10.6.0 +nh3==0.2.21 packaging==24.2 -pkginfo==1.10.0 -Pygments==2.18.0 +pkginfo==1.12.1.2 +Pygments==2.19.1 pyproject_hooks==1.2.0 pywin32-ctypes==0.2.3 readme_renderer==44.0 requests==2.32.3 requests-toolbelt==1.0.0 rfc3986==2.0.0 -rich==13.9.4 -ruff==0.7.2 -setuptools==75.3.0 -twine==5.1.1 -urllib3==2.2.3 -wheel==0.44.0 +rich==14.0.0 +ruff==0.11.5 +setuptools==78.1.0 +twine==6.1.0 +urllib3==2.4.0 +wheel==0.45.1 zipp==3.21.0 diff --git a/setup.py b/setup.py index 29429e8..4676e35 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Operating System :: OS Independent", ], include_package_data=True, diff --git a/tests/test-suite/compile.test.json b/tests/test-suite/compile.test.json index 9b26593..c6e4874 100644 --- a/tests/test-suite/compile.test.json +++ b/tests/test-suite/compile.test.json @@ -1,6 +1,6 @@ { - "source": "https://github.com/jsonquerylang/jsonquery/tree/main/test-suite/compile.test.json", - "updated": "2024-12-03T09:00:00Z", + "source": "https://github.com/jsonquerylang/jsonquery/blob/v5.0.0/test-suite/compile.test.json", + "version": "5.0.0", "tests": [ { "category": "value", @@ -948,6 +948,13 @@ "query": ["and", true, true], "output": true }, + { + "category": "and", + "description": "should calculate and with more than two arguments", + "input": null, + "query": ["and", true, true, false], + "output": false + }, { "category": "or", @@ -998,6 +1005,13 @@ "query": ["or", false, true], "output": true }, + { + "category": "or", + "description": "should calculate or with more than two arguments", + "input": null, + "query": ["or", false, false, true], + "output": true + }, { "category": "not", diff --git a/tests/test-suite/parse.test.json b/tests/test-suite/parse.test.json index f0cfd6a..fec40b4 100644 --- a/tests/test-suite/parse.test.json +++ b/tests/test-suite/parse.test.json @@ -1,6 +1,6 @@ { - "source": "https://github.com/jsonquerylang/jsonquery/tree/main/test-suite/parse.test.json", - "updated": "2024-11-15T09:00:00Z", + "source": "https://github.com/jsonquerylang/jsonquery/blob/v5.0.0/test-suite/parse.test.json", + "version": "5.0.0", "groups": [ { "category": "property", @@ -163,14 +163,93 @@ }, { "category": "operator", - "description": "should throw an error when using multiple operators without parenthesis", + "description": "should parse operators and and or with more than two arguments", "tests": [ - { "input": ".a == \"A\" and .b == \"B\"", "throws": "Unexpected part 'and .b == \"B\"'" }, + { "input": "1 and 2 and 3", "output": ["and", 1, 2, 3] }, + { "input": "1 or 2 or 3", "output": ["or", 1, 2, 3] }, + { "input": "1 * 2 * 3", "output": ["multiply", ["multiply", 1, 2], 3] }, + { "input": "1 / 2 / 3", "output": ["divide", ["divide", 1, 2], 3] }, + { "input": "1 % 2 % 3", "output": ["mod", ["mod", 1, 2], 3] }, + { "input": "1 + 2 + 3", "output": ["add", ["add", 1, 2], 3] }, + { "input": "1 - 2 - 3", "output": ["subtract", ["subtract", 1, 2], 3] } + ] + }, + { + "category": "operator", + "description": "should throw when chaining operators without vararg support", + "tests": [ + { "input": "1 ^ 2 ^ 3", "throws": "Unexpected part '^ 3'" }, + { "input": "1 == 2 == 3", "throws": "Unexpected part '== 3'" }, + { "input": "1 != 2 != 3", "throws": "Unexpected part '!= 3'" }, + { "input": "1 < 2 < 3", "throws": "Unexpected part '< 3'" }, + { "input": "1 <= 2 <= 3", "throws": "Unexpected part '<= 3'" }, + { "input": "1 > 2 > 3", "throws": "Unexpected part '> 3'" }, + { "input": "1 >= 2 >= 3", "throws": "Unexpected part '>= 3'" }, + { "input": "1 == 2 == 3", "throws": "Unexpected part '== 3'" } + ] + }, + { + "category": "operator", + "description": "should parse operators with the same precedence", + "tests": [ + { "input": "2 * 3 / 4", "output": ["divide", ["multiply", 2, 3], 4] }, + { "input": "2 / 3 * 4", "output": ["multiply", ["divide", 2, 3], 4] }, + { "input": "2 * 3 % 4", "output": ["mod", ["multiply", 2, 3], 4] }, + { "input": "2 % 3 * 4", "output": ["multiply", ["mod", 2, 3], 4] }, + { "input": "2 + 3 - 4", "output": ["subtract", ["add", 2, 3], 4] }, + { "input": "2 - 3 + 4", "output": ["add", ["subtract", 2, 3], 4] } + ] + }, + { + "category": "operator", + "description": "should parse operators with differing precedence", + "tests": [ + { "input": "2 ^ 3 * 4", "output": ["multiply", ["pow", 2, 3], 4] }, + { "input": "2 * 3 ^ 4", "output": ["multiply", 2, ["pow", 3, 4]] }, + { "input": "2 * 3 + 4", "output": ["add", ["multiply", 2, 3], 4] }, + { "input": "2 + 3 * 4", "output": ["add", 2, ["multiply", 3, 4]] }, + { "input": "2 + 3 > 4", "output": ["gt", ["add", 2, 3], 4] }, + { "input": "2 > 3 + 4", "output": ["gt", 2, ["add", 3, 4]] }, + { "input": "2 > 3 == 4", "output": ["eq", ["gt", 2, 3], 4] }, + { "input": "2 == 3 > 4", "output": ["eq", 2, ["gt", 3, 4]] }, + { "input": "2 == 3 and 4", "output": ["and", ["eq", 2, 3], 4] }, + { "input": "2 and 3 == 4", "output": ["and", 2, ["eq", 3, 4]] }, + { "input": "2 and 3 or 4", "output": ["or", ["and", 2, 3], 4] }, + { "input": "2 or 3 and 4", "output": ["or", 2, ["and", 3, 4]] }, + { "input": "2 > 3 and 4", "output": ["and", ["gt", 2, 3], 4] }, + { "input": "2 and 3 > 4", "output": ["and", 2, ["gt", 3, 4]] }, + { "input": "2 or 3 | 4", "output": ["pipe", ["or", 2, 3], 4] }, + { "input": "2 | 3 or 4", "output": ["pipe", 2, ["or", 3, 4]] } + ] + }, + { + "category": "operator", + "description": "should override operator precedence using parenthesis", + "tests": [ + { "input": "2 + 3 * 4", "output": ["add", 2, ["multiply", 3, 4]] }, + { "input": "2 + (3 * 4)", "output": ["add", 2, ["multiply", 3, 4]] }, + { "input": "(2 + 3) * 4", "output": ["multiply", ["add", 2, 3], 4] }, + { "input": "2 * (3 + 4)", "output": ["multiply", 2, ["add", 3, 4]] } + ] + }, + { + "category": "operator", + "description": "should keep the structure based on parenthesis", + "tests": [ + { "input": "(2 * 3) * 4", "output": ["multiply", ["multiply", 2, 3], 4] }, { - "input": "2 and 3 and 4", - "throws": "Unexpected part 'and 4' (pos: 8)" + "input": "((2 * 3) * 4) * 5", + "output": ["multiply", ["multiply", ["multiply", 2, 3], 4], 5] }, - { "input": ".a + 2 * 3", "throws": "Unexpected part '* 3' (pos: 7)" } + { + "input": "(2 * 3) * (4 * 5)", + "output": ["multiply", ["multiply", 2, 3], ["multiply", 4, 5]] + }, + { "input": "2 * (3 * 4)", "output": ["multiply", 2, ["multiply", 3, 4]] }, + { "input": "(2 + 3) + 4", "output": ["add", ["add", 2, 3], 4] }, + { "input": "2 + (3 + 4)", "output": ["add", 2, ["add", 3, 4]] }, + { "input": "(2 - 3) - 4", "output": ["subtract", ["subtract", 2, 3], 4] }, + { "input": "2 - (3 - 4)", "output": ["subtract", 2, ["subtract", 3, 4]] } ] }, { @@ -253,6 +332,10 @@ { "input": " { a : 1 } ", "output": ["object", { "a": 1 }] }, { "input": "{a:1,b:2}", "output": ["object", { "a": 1, "b": 2 }] }, { "input": "{ a : 1 , b : 2 }", "output": ["object", { "a": 1, "b": 2 }] }, + { + "input": "{ ok: .error == null }", + "output": ["object", { "ok": ["eq", ["get", "error"], null] }] + }, { "input": "{ \"a\" : 1 , \"b\" : 2 }", "output": ["object", { "a": 1, "b": 2 }] }, { "input": "{ 2: \"two\" }", "output": ["object", { "2": "two" }] }, { "input": "{ 0: \"zero\" }", "output": ["object", { "0": "zero" }] }, diff --git a/tests/test-suite/stringify.test.json b/tests/test-suite/stringify.test.json index 763e275..d829d21 100644 --- a/tests/test-suite/stringify.test.json +++ b/tests/test-suite/stringify.test.json @@ -1,6 +1,6 @@ { - "source": "https://github.com/jsonquerylang/jsonquery/tree/main/test-suite/stringify.test.json", - "updated": "2024-11-12T09:00:00Z", + "source": "https://github.com/jsonquerylang/jsonquery/blob/v5.0.0/test-suite/stringify.test.json", + "version": "5.0.0", "groups": [ { "category": "property", @@ -18,43 +18,112 @@ }, { "category": "operator", - "description": "should stringify all operators", + "description": "should stringify all operators", "tests": [ - { "input": ["eq", ["get", "score"], 8], "output": "(.score == 8)" }, - { "input": ["lt", ["get", "score"], 8], "output": "(.score < 8)" }, - { "input": ["lte", ["get", "score"], 8], "output": "(.score <= 8)" }, - { "input": ["gt", ["get", "score"], 8], "output": "(.score > 8)" }, - { "input": ["gte", ["get", "score"], 8], "output": "(.score >= 8)" }, - { "input": ["ne", ["get", "score"], 8], "output": "(.score != 8)" }, - { "input": ["add", ["get", "score"], 8], "output": "(.score + 8)" }, - { "input": ["subtract", ["get", "score"], 8], "output": "(.score - 8)" }, - { "input": ["multiply", ["get", "score"], 8], "output": "(.score * 8)" }, - { "input": ["divide", ["get", "score"], 8], "output": "(.score / 8)" }, - { "input": ["pow", ["get", "score"], 8], "output": "(.score ^ 8)" }, - { "input": ["mod", ["get", "score"], 8], "output": "(.score % 8)" }, - { "input": ["and", ["get", "score"], 8], "output": "(.score and 8)" }, - { "input": ["or", ["get", "score"], 8], "output": "(.score or 8)" }, + { "input": ["eq", ["get", "score"], 8], "output": ".score == 8" }, + { "input": ["lt", ["get", "score"], 8], "output": ".score < 8" }, + { "input": ["lte", ["get", "score"], 8], "output": ".score <= 8" }, + { "input": ["gt", ["get", "score"], 8], "output": ".score > 8" }, + { "input": ["gte", ["get", "score"], 8], "output": ".score >= 8" }, + { "input": ["ne", ["get", "score"], 8], "output": ".score != 8" }, + { "input": ["add", ["get", "score"], 8], "output": ".score + 8" }, + { "input": ["subtract", ["get", "score"], 8], "output": ".score - 8" }, + { "input": ["multiply", ["get", "score"], 8], "output": ".score * 8" }, + { "input": ["divide", ["get", "score"], 8], "output": ".score / 8" }, + { "input": ["pow", ["get", "score"], 8], "output": ".score ^ 8" }, + { "input": ["mod", ["get", "score"], 8], "output": ".score % 8" }, + { "input": ["and", ["get", "score"], 8], "output": ".score and 8" }, + { "input": ["or", ["get", "score"], 8], "output": ".score or 8" }, { "input": ["in", ["get", "score"], ["array", 8, 9, 10]], - "output": "(.score in [8, 9, 10])" + "output": ".score in [8, 9, 10]" }, { "input": ["not in", ["get", "score"], ["array", 8, 9, 10]], - "output": "(.score not in [8, 9, 10])" + "output": ".score not in [8, 9, 10]" } ] }, { "category": "operator", - "description": "should stringify a custom operator", - "options": { - "operators": { "aboutEq": "~=" } - }, + "description": "should wrap operators with the same precedence in parenthesis when needed", + "tests": [ + { "input": ["pow", ["pow", 2, 3], 4], "output": "(2 ^ 3) ^ 4" }, + { "input": ["pow", 2, ["pow", 3, 4]], "output": "2 ^ (3 ^ 4)" }, + { "input": ["multiply", ["multiply", 2, 3], 4], "output": "2 * 3 * 4" }, + { "input": ["multiply", 2, ["multiply", 3, 4]], "output": "2 * (3 * 4)" }, + { "input": ["divide", ["divide", 2, 3], 4], "output": "2 / 3 / 4" }, + { "input": ["divide", 2, ["divide", 3, 4]], "output": "2 / (3 / 4)" }, + { "input": ["divide", ["multiply", 2, 3], 4], "output": "2 * 3 / 4" }, + { "input": ["divide", 2, ["multiply", 3, 4]], "output": "2 / (3 * 4)" }, + { "input": ["divide", 2, 3, ["multiply", 4, 5]], "output": "2 / 3 / (4 * 5)" }, + { + "input": ["divide", 2, ["multiply", 3, 4], ["multiply", 5, 6]], + "output": "2 / (3 * 4) / (5 * 6)" + }, + { "input": ["multiply", ["divide", 2, 3], 4], "output": "2 / 3 * 4" }, + { "input": ["mod", ["mod", 2, 3], 4], "output": "2 % 3 % 4" }, + { "input": ["mod", 2, ["mod", 3, 4]], "output": "2 % (3 % 4)" }, + { "input": ["mod", ["multiply", 2, 3], 4], "output": "2 * 3 % 4" }, + { "input": ["multiply", ["mod", 2, 3], 4], "output": "2 % 3 * 4" }, + { "input": ["add", ["add", 2, 3], 4], "output": "2 + 3 + 4" }, + { "input": ["add", 2, ["add", 3, 4]], "output": "2 + (3 + 4)" }, + { "input": ["subtract", ["subtract", 2, 3], 4], "output": "2 - 3 - 4" }, + { "input": ["subtract", 2, ["subtract", 3, 4]], "output": "2 - (3 - 4)" }, + { "input": ["subtract", ["add", 2, 3], 4], "output": "2 + 3 - 4" }, + { "input": ["subtract", 2, ["add", 3, 4]], "output": "2 - (3 + 4)" }, + { "input": ["add", ["subtract", 2, 3], 4], "output": "2 - 3 + 4" }, + { "input": ["eq", ["eq", 2, 3], 4], "output": "(2 == 3) == 4" }, + { "input": ["eq", 2, ["eq", 3, 4]], "output": "2 == (3 == 4)" } + ] + }, + { + "category": "operator", + "description": "should wrap operators with differing precedence in parenthesis when needed", + "tests": [ + { "input": ["abs", ["add", 2, 3]], "output": "abs(2 + 3)" }, + { "input": ["multiply", ["pow", 2, 3], 4], "output": "2 ^ 3 * 4" }, + { "input": ["multiply", 2, ["pow", 3, 4]], "output": "2 * 3 ^ 4" }, + { "input": ["pow", 2, ["multiply", 3, 4]], "output": "2 ^ (3 * 4)" }, + { "input": ["pow", ["multiply", 2, 3], 4], "output": "(2 * 3) ^ 4" }, + { "input": ["add", ["multiply", 2, 3], 4], "output": "2 * 3 + 4" }, + { "input": ["add", 2, ["multiply", 3, 4]], "output": "2 + 3 * 4" }, + { "input": ["multiply", 2, ["add", 3, 4]], "output": "2 * (3 + 4)" }, + { "input": ["multiply", ["add", 2, 3], 4], "output": "(2 + 3) * 4" }, + { "input": ["gt", ["add", 2, 3], 4], "output": "2 + 3 > 4" }, + { "input": ["gt", 2, ["add", 3, 4]], "output": "2 > 3 + 4" }, + { "input": ["add", 2, ["gt", 3, 4]], "output": "2 + (3 > 4)" }, + { "input": ["add", ["gt", 2, 3], 4], "output": "(2 > 3) + 4" }, + { "input": ["eq", ["gt", 2, 3], 4], "output": "2 > 3 == 4" }, + { "input": ["gt", 2, ["eq", 3, 4]], "output": "2 > (3 == 4)" }, + { "input": ["and", ["eq", 2, 3], 4], "output": "2 == 3 and 4" }, + { "input": ["eq", 2, ["and", 3, 4]], "output": "2 == (3 and 4)" }, + { "input": ["eq", ["and", 2, 3], 4], "output": "(2 and 3) == 4" }, + { "input": ["or", ["and", 2, 3], 4], "output": "2 and 3 or 4" }, + { "input": ["and", 2, ["or", 3, 4]], "output": "2 and (3 or 4)" }, + { "input": ["and", ["gt", 2, 3], 4], "output": "2 > 3 and 4" }, + { "input": ["gt", 2, ["and", 3, 4]], "output": "2 > (3 and 4)" }, + { "input": ["gt", ["and", 2, 3], 4], "output": "(2 and 3) > 4" }, + { "input": ["pipe", ["and", 2, 3], 4], "output": "2 and 3 | 4" }, + { "input": ["pipe", 2, ["and", 3, 4]], "output": "2 | 3 and 4" }, + { "input": ["and", ["pipe", 2, 3], 4], "output": "(2 | 3) and 4" }, + { "input": ["and", 2, ["pipe", 3, 4]], "output": "2 and (3 | 4)" } + ] + }, + { + "category": "operator", + "description": "should stringify a variable number of arguments in operators", "tests": [ - { "input": ["aboutEq", 2, 3], "output": "(2 ~= 3)" }, - { "input": ["filter", ["aboutEq", 2, 3]], "output": "filter(2 ~= 3)" }, - { "input": ["object", { "result": ["aboutEq", 2, 3] }], "output": "{ result: (2 ~= 3) }" }, - { "input": ["eq", 2, 3], "output": "(2 == 3)" } + { "input": ["pipe", 2, 3, 4], "output": "2 | 3 | 4" }, + { "input": ["get", 2, 3, 4], "output": ".2.3.4" }, + { "input": ["and", 2, 3, 4], "output": "2 and 3 and 4" }, + { "input": ["and", 2, 3, 4, 5], "output": "2 and 3 and 4 and 5" }, + { "input": ["or", 2, 3, 4], "output": "2 or 3 or 4" }, + { "input": ["add", 2, 3, 4], "output": "2 + 3 + 4" }, + { "input": ["subtract", 2, 3, 4], "output": "2 - 3 - 4" }, + { "input": ["multiply", 2, 3, 4], "output": "2 * 3 * 4" }, + { "input": ["divide", 2, 3, 4], "output": "2 / 3 / 4" }, + { "input": ["mod", 2, 3, 4], "output": "2 % 3 % 4" } ] }, { @@ -235,7 +304,7 @@ }, { "input": ["filter", ["and", ["gte", ["get", "age"], 23], ["lte", ["get", "age"], 27]]], - "output": "filter((.age >= 23) and (.age <= 27))" + "output": "filter(.age >= 23 and .age <= 27)" }, { "input": [ diff --git a/tests/test_compile.py b/tests/test_compile.py index d6a2805..960e80a 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -3,6 +3,7 @@ import re from os import path from jsonquerylang import compile +from jsonquerylang.compile import build_function friends = [ {"name": "Chris", "age": 23, "scores": [7.2, 5, 8.0]}, @@ -12,56 +13,96 @@ class CompileTestCase(unittest.TestCase): - def test_compile(self): + def test_unknown_function(self): """Raise an exception in case of an unknown function""" self.assertRaisesRegex( SyntaxError, 'Unknown function "foo"', lambda: go([], ["foo"]) ) - def test_options1(self): - """Test defining a custom function""" + def test_pass_empty_options(self): + """Test define empty options object""" - def times(value): - return lambda data: list(map(lambda item: item * value, data)) + query = ["get", "name"] + evaluate = compile(query, {}) - query = ["times", 2] + self.assertEqual(evaluate({"name": "Joe"}), "Joe") - evaluate = compile(query, {"functions": {"times": times}}) + def test_options1(self): + """should define a custom function""" + options = { + "functions": { + "times": lambda value: lambda data: [item * value for item in data] + } + } - self.assertEqual(evaluate([2, 3, 4]), [4, 6, 8]) - self.assertEqual(evaluate([-4, 5]), [-8, 10]) + self.assertEqual(go([1, 2, 3], ["times", 2], options), [2, 4, 6]) + with self.assertRaises(Exception) as context: + go([1, 2, 3], ["times", 2]) + self.assertIn('Unknown function "times"', str(context.exception)) def test_options2(self): - """Test define options but no custom function""" + """should extend with a custom function with more than 2 arguments""" - query = ["get", "name"] - evaluate = compile(query, {}) + def one_of(value, a, b, c): + return value == a or value == b or value == c - self.assertEqual(evaluate({"name": "Joe"}), "Joe") + options = {"functions": {"oneOf": build_function(one_of)}} + + self.assertTrue(go("C", ["oneOf", ["get"], "A", "B", "C"], options)) + self.assertFalse(go("D", ["oneOf", ["get"], "A", "B", "C"], options)) def test_options3(self): - """Test defining a custom function that uses compile""" + """should override an existing function""" + options = {"functions": {"sort": lambda: lambda _data: "custom sort"}} + + self.assertEqual(go([2, 3, 1], ["sort"], options), "custom sort") - def by_times(data_path, value): - getter = compile(data_path) + def test_options4(self): + """should be able to insert a function in a nested compile""" - return lambda data: list(map(lambda item: getter(item) * value, data)) + def times(value): + _options = {"functions": {"foo": lambda: lambda _data: 42}} + _value = compile(value, _options) + return lambda data: [item * _value(data) for item in data] - query = ["by_times", ["get", "score"], 2] - evaluate = compile(query, {"functions": {"by_times": by_times}}) + options = {"functions": {"times": times}} - self.assertEqual( - evaluate( - [ - {"score": 2}, - {"score": 4}, - ] - ), - [4, 8], - ) + self.assertEqual(go([1, 2, 3], ["times", 2], options), [2, 4, 6]) + self.assertEqual(go([1, 2, 3], ["times", ["foo"]], options), [42, 84, 126]) + + with self.assertRaises(Exception) as context: + go([1, 2, 3], ["foo"], options) + self.assertIn('Unknown function "foo"', str(context.exception)) + + def test_options6(self): + """should clean up the custom function stack when creating a query throws an error""" + options = { + "functions": { + "sort": lambda: (_ for _ in ()).throw(Exception("Test Error")) + } + } + + with self.assertRaises(Exception) as context: + go({}, ["sort"], options) + self.assertEqual(str(context.exception), "Test Error") + + # Should fall back to default behavior + self.assertEqual(go([2, 3, 1], ["sort"]), [1, 2, 3]) + + def test_options7(self): + """should extend with a custom function aboutEq""" + + def about_eq(a, b): + epsilon = 0.001 + return abs(a - b) < epsilon + + options = {"functions": {"aboutEq": build_function(about_eq)}} + + self.assertTrue(go({"a": 2}, ["aboutEq", ["get", "a"], 2], options)) + self.assertTrue(go({"a": 1.999}, ["aboutEq", ["get", "a"], 2], options)) def test_error_handling1(self): - """Should throw a helpful error when a pipe contains a compile time error""" + """should throw a helpful error when a pipe contains a compile time error""" query = ["foo", 42] @@ -146,14 +187,14 @@ def test_suite(self): suite = json.load(read_file) for test in suite["tests"]: - message = f"[{test["category"]}] {test["description"]}" + message = f"[{test['category']}] {test['description']}" with self.subTest(message=message): evaluate = compile(test["query"]) self.assertEqual(evaluate(test["input"]), test["output"]) -def go(data, query): - evaluate = compile(query) +def go(data, query, options=None): + evaluate = compile(query, options) return evaluate(data) diff --git a/tests/test_operators.py b/tests/test_operators.py new file mode 100644 index 0000000..f8ceb98 --- /dev/null +++ b/tests/test_operators.py @@ -0,0 +1,55 @@ +import unittest +from jsonquerylang.operators import extend_operators + + +class CompileTestCase(unittest.TestCase): + def test_custom_operator_at(self): + """Test defining a custom operator at a given precedence""" + + ops = [{"add": "+", "subtract": "-"}, {"eq": "=="}] + + self.assertEqual( + extend_operators(ops, [{"name": "aboutEq", "op": "~=", "at": "=="}]), + [{"add": "+", "subtract": "-"}, {"eq": "==", "aboutEq": "~="}], + ) + + def test_custom_operator_after(self): + """Test defining a custom operator after a given precedence""" + + ops = [{"add": "+", "subtract": "-"}, {"eq": "=="}] + + self.assertEqual( + extend_operators(ops, [{"name": "aboutEq", "op": "~=", "after": "+"}]), + [{"add": "+", "subtract": "-"}, {"aboutEq": "~="}, {"eq": "=="}], + ) + + def test_custom_operator_before(self): + """Test defining a custom operator before a given precedence""" + + ops = [{"add": "+", "subtract": "-"}, {"eq": "=="}] + + self.assertEqual( + extend_operators(ops, [{"name": "aboutEq", "op": "~=", "before": "=="}]), + [{"add": "+", "subtract": "-"}, {"aboutEq": "~="}, {"eq": "=="}], + ) + + def test_multiple_custom_operators(self): + """Test defining multiple custom operators""" + + ops = [{"add": "+", "subtract": "-"}, {"eq": "=="}] + + self.assertEqual( + extend_operators( + ops, + [ + {"name": "first", "op": "op1", "before": "=="}, + {"name": "second", "op": "op2", "before": "op1"}, + ], + ), + [ + {"add": "+", "subtract": "-"}, + {"second": "op2"}, + {"first": "op1"}, + {"eq": "=="}, + ], + ) diff --git a/tests/test_parse.py b/tests/test_parse.py index 6679dd5..e7f1df0 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -3,7 +3,7 @@ import re from os import path -from jsonquerylang import parse +from jsonquerylang import parse, JsonQueryParseOptions class ParseTestCase(unittest.TestCase): @@ -18,7 +18,7 @@ def test_suite(self): for group in suite["groups"]: for test in group["tests"]: - message = f"[{group["category"]}] {group["description"]} (input: {test["input"]})" + message = f"[{group['category']}] {group['description']} (input: {test['input']})" if "output" in test: with self.subTest(message=message): @@ -31,6 +31,109 @@ def test_suite(self): lambda: parse(test["input"]), ) + def test_options1(self): + """should parse a custom function""" + + options: JsonQueryParseOptions = {"functions": {"customFn": lambda: lambda: 42}} + + self.assertEqual( + parse('customFn(.age, "desc")', options), + ["customFn", ["get", "age"], "desc"], + ) + + # built-in functions should still be available + self.assertEqual(parse("add(2, 3)", options), ["add", 2, 3]) + + def test_options2(self): + """should parse a custom operator without vararg""" + + options: JsonQueryParseOptions = { + "operators": [{"name": "aboutEq", "op": "~=", "at": "=="}] + } + + self.assertEqual( + parse(".score ~= 8", options), ["aboutEq", ["get", "score"], 8] + ) + + # built-in operators should still be available + self.assertEqual(parse(".score == 8", options), ["eq", ["get", "score"], 8]) + + self.assertRaisesRegex( + SyntaxError, + re.escape("Unexpected part '~= 4'"), + lambda: parse("2 ~= 3 ~= 4", options), + ) + + def test_options3(self): + """should parse a custom operator with vararg without left_associative""" + + options: JsonQueryParseOptions = { + "operators": [{"name": "aboutEq", "op": "~=", "at": "==", "vararg": True}] + } + + self.assertEqual(parse("2 and 3 and 4", options), ["and", 2, 3, 4]) + self.assertEqual(parse("2 ~= 3", options), ["aboutEq", 2, 3]) + self.assertEqual(parse("2 ~= 3 and 4", options), ["and", ["aboutEq", 2, 3], 4]) + self.assertEqual(parse("2 and 3 ~= 4", options), ["and", 2, ["aboutEq", 3, 4]]) + self.assertEqual(parse("2 == 3 ~= 4", options), ["aboutEq", ["eq", 2, 3], 4]) + self.assertEqual(parse("2 ~= 3 == 4", options), ["eq", ["aboutEq", 2, 3], 4]) + self.assertRaisesRegex( + SyntaxError, + re.escape("Unexpected part '~= 4'"), + lambda: parse("2 ~= 3 ~= 4", options), + ) + self.assertRaisesRegex( + SyntaxError, + re.escape("Unexpected part '== 4'"), + lambda: parse("2 == 3 == 4", options), + ) + + def test_options4(self): + """should parse a custom operator with vararg without left_associative""" + + options: JsonQueryParseOptions = { + "operators": [ + { + "name": "aboutEq", + "op": "~=", + "at": "==", + "vararg": True, + "left_associative": True, + } + ] + } + + self.assertEqual(parse("2 and 3 and 4", options), ["and", 2, 3, 4]) + self.assertEqual(parse("2 ~= 3", options), ["aboutEq", 2, 3]) + self.assertEqual(parse("2 ~= 3 ~= 4", options), ["aboutEq", 2, 3, 4]) + self.assertRaisesRegex( + SyntaxError, + re.escape("Unexpected part '== 4'"), + lambda: parse("2 == 3 == 4", options), + ) + + def test_options5(self): + """should throw an error in case of an invalid custom operator""" + + options: JsonQueryParseOptions = {"operators": [dict()]} + + self.assertRaisesRegex( + RuntimeError, + re.escape("Invalid custom operator"), + lambda: parse(".score > 8", options), + ) + + def test_options6(self): + """should throw an error in case of an invalid custom operator (2)""" + + options: JsonQueryParseOptions = {"operators": dict()} + + self.assertRaisesRegex( + RuntimeError, + re.escape("Invalid custom operators"), + lambda: parse(".score > 8", options), + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_stringify.py b/tests/test_stringify.py index a74bbaa..46bb11c 100644 --- a/tests/test_stringify.py +++ b/tests/test_stringify.py @@ -24,12 +24,80 @@ def test_suite(self): ) for test in group["tests"]: - message = f"[{group["category"]}] {group["description"]} (input: {test["input"]})" + message = f"[{group['category']}] {group['description']} (input: {test['input']})" with self.subTest(message=message): self.assertEqual( stringify(test["input"], options), test["output"] ) + def test_options1(self): + """should stringify a custom operator""" + options: JsonQueryStringifyOptions = { + "operators": [{"name": "aboutEq", "op": "~=", "at": "=="}] + } + + self.assertEqual(stringify(["aboutEq", 2, 3], options), "2 ~= 3") + self.assertEqual( + stringify(["filter", ["aboutEq", 2, 3]], options), "filter(2 ~= 3)" + ) + self.assertEqual( + stringify(["object", {"result": ["aboutEq", 2, 3]}], options), + "{ result: 2 ~= 3 }", + ) + + # existing operators should still be there + self.assertEqual(stringify(["eq", 2, 3], options), "2 == 3") + + # precedence and parenthesis + self.assertEqual( + stringify(["aboutEq", ["aboutEq", 2, 3], 4], options), "(2 ~= 3) ~= 4" + ) + self.assertEqual( + stringify(["aboutEq", 2, ["aboutEq", 3, 4]], options), "2 ~= (3 ~= 4)" + ) + self.assertEqual( + stringify(["aboutEq", ["and", 2, 3], 4], options), "(2 and 3) ~= 4" + ) + self.assertEqual( + stringify(["aboutEq", 2, ["and", 3, 4]], options), "2 ~= (3 and 4)" + ) + self.assertEqual( + stringify(["and", ["aboutEq", 2, 3], 4], options), "2 ~= 3 and 4" + ) + self.assertEqual( + stringify(["and", 2, ["aboutEq", 3, 4]], options), "2 and 3 ~= 4" + ) + self.assertEqual( + stringify(["aboutEq", ["add", 2, 3], 4], options), "2 + 3 ~= 4" + ) + self.assertEqual( + stringify(["aboutEq", 2, ["add", 3, 4]], options), "2 ~= 3 + 4" + ) + self.assertEqual( + stringify(["add", ["aboutEq", 2, 3], 4], options), "(2 ~= 3) + 4" + ) + self.assertEqual( + stringify(["add", 2, ["aboutEq", 3, 4]], options), "2 + (3 ~= 4)" + ) + + def test_options2(self): + """should stringify left associative custom operator""" + options: JsonQueryStringifyOptions = { + "operators": [ + {"name": "aboutEq", "op": "~=", "at": "==", "left_associative": True} + ] + } + + self.assertEqual( + stringify(["aboutEq", ["aboutEq", 2, 3], 4], options), "2 ~= 3 ~= 4" + ) + self.assertEqual( + stringify(["aboutEq", 2, ["aboutEq", 3, 4]], options), "2 ~= (3 ~= 4)" + ) + + # Note: we do not test the option `CustomOperator.vararg` + # since they have no effect on stringification, only on parsing. + if __name__ == "__main__": unittest.main()