Add POSIX Regex support for PostgreSQL and MySQL (#1714)

* add posix regex filter operator for postgresql and mysql

* update docs

* add tests

* adjust tests

* apply formatting with make styles

* fix mypy issue

docs/query.rst

@@ -278,6 +278,17 @@ In PostgreSQL and MYSQL, you can use the ``contains``, ``contained_by`` and ``fi
  obj5 = await JSONModel.filter(data__filter={"owner__name__isnull": True}).first()
  obj6 = await JSONModel.filter(data__filter={"owner__last__not_isnull": False}).first()
 
+In PostgreSQL and MySQL, you can use ``postgres_posix_regex`` to make comparisons using POSIX regular expressions:
+On PostgreSQL, this uses the ``~`` operator, on MySQL it uses the ``REGEXP`` operator.
+
+.. code-block:: python3
+ class DemoModel:
+ demo_text = fields.TextField()
+
+ await DemoModel.create(demo_text="Hello World")
+ obj = await DemoModel.filter(demo_text__posix_regex="^Hello World$").first()
+
+
 Complex prefetch
 ================
 

tests/test_posix_regex_filter.py

@@ -0,0 +1,18 @@
+from tests import testmodels
+from tortoise.contrib import test
+
+
+class TestPosixRegexFilter(test.TestCase):
+
+ @test.requireCapability(dialect="mysql")
+ @test.requireCapability(dialect="postgres")
+ async def test_regex_filter(self):
+ author = await testmodels.Author.create(name="Johann Wolfgang von Goethe")
+ self.assertEqual(
+ set(
+ await testmodels.Author.filter(
+ name__posix_regex="^Johann [a-zA-Z]+ von Goethe$"
+ ).values_list("name", flat=True)
+ ),
+ {author.name},
+ )

tortoise/backends/base_postgres/executor.py

@@ -12,8 +12,15 @@
  postgres_json_contains,
  postgres_json_filter,
 )
+from tortoise.contrib.postgres.regex import postgres_posix_regex
 from tortoise.contrib.postgres.search import SearchCriterion
-from tortoise.filters import json_contained_by, json_contains, json_filter, search
+from tortoise.filters import (
+ json_contained_by,
+ json_contains,
+ json_filter,
+ posix_regex,
+ search,
+)
 
 
 def postgres_search(field: Term, value: Term):
@@ -28,6 +35,7 @@ class BasePostgresExecutor(BaseExecutor):
  json_contains: postgres_json_contains,
  json_contained_by: postgres_json_contained_by,
  json_filter: postgres_json_filter,
+ posix_regex: postgres_posix_regex,
  }
 
  def parameter(self, pos: int) -> Parameter:

tortoise/backends/mysql/executor.py

@@ -1,6 +1,6 @@
 from pypika import Parameter, functions
 from pypika.enums import SqlTypes
-from pypika.terms import Criterion
+from pypika.terms import BasicCriterion, Criterion
 from pypika.utils import format_quotes
 
 from tortoise import Model
@@ -25,6 +25,7 @@
  json_contained_by,
  json_contains,
  json_filter,
+ posix_regex,
  search,
  starts_with,
 )
@@ -95,6 +96,10 @@ def mysql_search(field: Term, value: str):
  return SearchCriterion(field, expr=StrWrapper(value))
 
 
+def mysql_posix_regex(field: Term, value: str):
+ return BasicCriterion(" REGEXP ", field, StrWrapper(value))
+
+
 class MySQLExecutor(BaseExecutor):
  FILTER_FUNC_OVERRIDE = {
  contains: mysql_contains,
@@ -108,6 +113,7 @@ class MySQLExecutor(BaseExecutor):
  json_contains: mysql_json_contains,
  json_contained_by: mysql_json_contained_by,
  json_filter: mysql_json_filter,
+ posix_regex: mysql_posix_regex,
  }
  EXPLAIN_PREFIX = "EXPLAIN FORMAT=JSON"
 

tortoise/contrib/postgres/regex.py

@@ -0,0 +1,11 @@
+import enum
+
+from pypika.terms import BasicCriterion, Term
+
+
+class PostgresRegexMatching(enum.Enum):
+ posix_regex = "~"
+
+
+def postgres_posix_regex(field: Term, value: str):
+ return BasicCriterion(PostgresRegexMatching.posix_regex, field, field.wrap_constant(value))

tortoise/filters.py

@@ -130,6 +130,13 @@ def search(field: Term, value: str):
  pass
 
 
+def posix_regex(field: Term, value: str):
+ # Will be overridden in each executor
+ raise NotImplementedError(
+ "The postgres_posix_regex filter operator is not supported by your database backend"
+ )
+
+
 def starts_with(field: Term, value: str) -> Criterion:
  return Like(Cast(field, SqlTypes.VARCHAR), field.wrap_constant(f"{escape_like(value)}%"))
 
@@ -473,6 +480,12 @@ def get_filters_for_field(
  "operator": insensitive_ends_with,
  "value_encoder": string_encoder,
  },
+ f"{field_name}__posix_regex": {
+ "field": actual_field_name,
+ "source_field": source_field,
+ "operator": posix_regex,
+ "value_encoder": string_encoder,
+ },
  f"{field_name}__year": {
  "field": actual_field_name,
  "source_field": source_field,