From acd56c893731f25bb6614074c537941e79f66c02 Mon Sep 17 00:00:00 2001 From: Grigi Date: Thu, 27 Sep 2018 12:40:51 +0200 Subject: [PATCH 1/5] __init__ restructure give 25-30% speedup in select benchmark. I also cleared up a few lint warnings. --- requirements-dev.txt | 6 +-- requirements.txt | 2 +- tortoise/backends/asyncpg/client.py | 7 ++-- tortoise/backends/base/client.py | 8 ++-- tortoise/backends/base/config_generator.py | 28 +++++++------- tortoise/backends/base/schema_generator.py | 3 -- tortoise/fields.py | 4 +- tortoise/models.py | 44 +++++++++++----------- tortoise/queryset.py | 4 +- 9 files changed, 52 insertions(+), 54 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3e033cddf..dbe0583c6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -19,7 +19,7 @@ certifi==2018.8.24 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests ciso8601==2.0.1 -click==6.7 # via pip-tools +click==7.0 # via pip-tools cloud-sptheme==1.9.4 colorama==0.3.9 # via green coverage==4.5.1 # via coveralls, green @@ -41,7 +41,7 @@ markupsafe==1.0 # via jinja2 mccabe==0.6.1 # via flake8, pylint mypy-extensions==0.4.1 # via mypy mypy==0.630 -packaging==17.1 # via sphinx +packaging==18.0 # via sphinx pbr==4.2.0 # via stevedore pip-tools==3.0.0 pluggy==0.7.1 # via tox @@ -53,7 +53,7 @@ pygments==2.2.0 pylint==2.1.1 pymysql==0.9.2 # via aiomysql pyparsing==2.2.1 # via packaging -pypika==0.15.5 +pypika==0.15.6 pytz==2018.5 # via babel pyyaml==3.13 # via bandit requests==2.19.1 # via coveralls, sphinx diff --git a/requirements.txt b/requirements.txt index f8d392eed..fe4ad85eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pypika>=0.15.4,<1.0 +pypika>=0.15.6,<1.0 ciso8601>=2.0 aiocontextvars==0.1.2;python_version<"3.7" aiosqlite>=0.6.0 diff --git a/tortoise/backends/asyncpg/client.py b/tortoise/backends/asyncpg/client.py index c78bd5d5c..cf37988ac 100644 --- a/tortoise/backends/asyncpg/client.py +++ b/tortoise/backends/asyncpg/client.py @@ -107,7 +107,7 @@ def _in_transaction(self): else: return self._transaction_class(self.connection_name, pool=self._db_pool) - async def execute_query(self, query): + async def execute_query(self, query, get_inserted_id=False): try: async with self.acquire_connection() as connection: self.log.debug(query) @@ -153,6 +153,7 @@ def __init__(self, connection_name, pool=None, connection=None): self._transaction_class = self.__class__ self._old_context_value = None self.connection_name = connection_name + self.transaction = None def acquire_connection(self): return ConnectionWrapper(self._connection) @@ -172,7 +173,7 @@ async def start(self): async def commit(self): try: await self.transaction.commit() - except asyncpg.exceptions._base.InterfaceError as exc: + except (AttributeError, asyncpg.exceptions._base.InterfaceError) as exc: raise TransactionManagementError(exc) if self._pool: await self._pool.release(self._connection) @@ -182,7 +183,7 @@ async def commit(self): async def rollback(self): try: await self.transaction.rollback() - except asyncpg.exceptions._base.InterfaceError as exc: + except (AttributeError, asyncpg.exceptions._base.InterfaceError) as exc: raise TransactionManagementError(exc) if self._pool: await self._pool.release(self._connection) diff --git a/tortoise/backends/base/client.py b/tortoise/backends/base/client.py index 07ae006ac..28f59de02 100644 --- a/tortoise/backends/base/client.py +++ b/tortoise/backends/base/client.py @@ -37,7 +37,7 @@ def acquire_connection(self): def _in_transaction(self): raise NotImplementedError() # pragma: nocoverage - async def execute_query(self, query): + async def execute_query(self, query, get_inserted_id=False): raise NotImplementedError() # pragma: nocoverage async def execute_script(self, script): @@ -62,11 +62,11 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): class SingleConnectionWrapper(BaseDBAsyncClient): + # pylint: disable=W0223 + def __init__(self, connection_name, connection, closing_callback=None): - self.connection_name = connection_name + super().__init__(connection_name, single_connection=True) self.connection = connection - self.log = logging.getLogger('db_client') - self.single_connection = True self.closing_callback = closing_callback def acquire_connection(self): diff --git a/tortoise/backends/base/config_generator.py b/tortoise/backends/base/config_generator.py index 43404fa1d..7fb92f03a 100644 --- a/tortoise/backends/base/config_generator.py +++ b/tortoise/backends/base/config_generator.py @@ -10,7 +10,7 @@ DB_LOOKUP = { 'postgres': { 'engine': 'tortoise.backends.asyncpg', - 'vars': { + 'vmap': { 'path': 'database', 'hostname': 'host', 'port': 'port', @@ -21,13 +21,13 @@ 'sqlite': { 'engine': 'tortoise.backends.sqlite', 'skip_first_char': False, - 'vars': { + 'vmap': { 'path': 'file_path', }, }, 'mysql': { 'engine': 'tortoise.backends.mysql', - 'vars': { + 'vmap': { 'path': 'database', 'hostname': 'host', 'port': 'port', @@ -61,20 +61,20 @@ def expand_db_url(db_url: str, testing: bool = False) -> dict: path = path.replace('\\{', '{').replace('\\}', '}') path = path.format(uuid.uuid4().hex) - vars = {} # type: dict - vars.update(db['vars']) - params[vars['path']] = path - if vars.get('hostname'): - params[vars['hostname']] = str(url.hostname or '') + vmap = {} # type: dict + vmap.update(db['vmap']) + params[vmap['path']] = path + if vmap.get('hostname'): + params[vmap['hostname']] = str(url.hostname or '') try: - if vars.get('port'): - params[vars['port']] = str(url.port or '') + if vmap.get('port'): + params[vmap['port']] = str(url.port or '') except ValueError: raise ConfigurationError('Port is not an integer') - if vars.get('username'): - params[vars['username']] = str(url.username or '') - if vars.get('password'): - params[vars['password']] = str(url.password or '') + if vmap.get('username'): + params[vmap['username']] = str(url.username or '') + if vmap.get('password'): + params[vmap['password']] = str(url.password or '') return { 'engine': db['engine'], diff --git a/tortoise/backends/base/schema_generator.py b/tortoise/backends/base/schema_generator.py index c38c0f438..4c915b10d 100644 --- a/tortoise/backends/base/schema_generator.py +++ b/tortoise/backends/base/schema_generator.py @@ -47,9 +47,6 @@ def _get_primary_key_create_string(self, field_name): # has to implement in children raise NotImplementedError() # pragma: nocoverage - def _get_auto_now_add_default(self): - raise NotImplementedError() # pragma: nocoverage - def _get_table_sql(self, model) -> dict: fields_to_create = [] m2m_tables_for_create = [] diff --git a/tortoise/fields.py b/tortoise/fields.py index a90b5afcb..adc24dfa9 100644 --- a/tortoise/fields.py +++ b/tortoise/fields.py @@ -29,7 +29,7 @@ class Field: """ def __init__( self, - type=None, + type=None, # pylint: disable=W0622 source_field: Optional[str] = None, generated: bool = False, pk: bool = False, @@ -301,7 +301,7 @@ def __init__( class BackwardFKRelation: - def __init__(self, type, relation_field, **kwargs): + def __init__(self, type, relation_field, **kwargs): # pylint: disable=W0622 self.type = type self.relation_field = relation_field diff --git a/tortoise/models.py b/tortoise/models.py index f3f61252a..3954de1e7 100644 --- a/tortoise/models.py +++ b/tortoise/models.py @@ -328,21 +328,19 @@ class Model(metaclass=ModelMeta): def __init__(self, *args, **kwargs) -> None: is_new = not bool(kwargs.get('id')) + passed_fields = set(kwargs.keys()) - for key, field in self._meta.fields_map.items(): - if isinstance(field, fields.BackwardFKRelation): - setattr( - self, key, - RelationQueryContainer(field.type, field.relation_field, self, is_new) - ) - elif isinstance(field, fields.ManyToManyField): - setattr(self, key, ManyToManyRelationManager(field.type, self, field, is_new)) - elif isinstance(field, fields.Field): - setattr(self, key, field.default) - else: - setattr(self, key, None) + for key in self._meta.backward_fk_fields: + field = self._meta.fields_map[key] + setattr( + self, key, + RelationQueryContainer(field.type, field.relation_field, self, is_new) + ) + + for key in self._meta.m2m_fields: + field = self._meta.fields_map[key] + setattr(self, key, ManyToManyRelationManager(field.type, self, field, is_new)) - passed_fields = set(kwargs.keys()) for key, value in kwargs.items(): if key in self._meta.fk_fields: if hasattr(value, 'id') and not value.id: @@ -351,6 +349,13 @@ def __init__(self, *args, **kwargs) -> None: relation_field = '{}_id'.format(key) setattr(self, relation_field, value.id) passed_fields.add(relation_field) + elif key in self._meta.fields: + field_object = self._meta.fields_map[key] + if value is None and not field_object.null: + raise ValueError('{} is non nullable field, but null was passed'.format(key)) + setattr(self, key, field_object.to_python_value(value)) + elif key in self._meta.db_fields: + setattr(self, self._meta.fields_db_projection_reverse[key], value) elif key in self._meta.backward_fk_fields: raise ConfigurationError( 'You can\'t set backward relations through init, change related model instead' @@ -359,18 +364,11 @@ def __init__(self, *args, **kwargs) -> None: raise ConfigurationError( 'You can\'t set m2m relations through init, use m2m_manager instead' ) - elif key in self._meta.fields: - field_object = self._meta.fields_map[key] - if value is None and not field_object.null: - raise ValueError('{} is non nullable field, but null was passed'.format(key)) - setattr(self, key, field_object.to_python_value(value)) - elif key in self._meta.db_fields: - setattr(self, self._meta.fields_db_projection_reverse[key], value) + + passed_fields.update(self._meta.fetch_fields) for key, field_object in self._meta.fields_map.items(): - if key in passed_fields or key in self._meta.fetch_fields: - continue - else: + if key not in passed_fields: if callable(field_object.default): setattr(self, key, field_object.default()) else: diff --git a/tortoise/queryset.py b/tortoise/queryset.py index bcc278537..e09742246 100644 --- a/tortoise/queryset.py +++ b/tortoise/queryset.py @@ -284,7 +284,7 @@ def annotate(self, **kwargs) -> 'QuerySet': queryset._available_custom_filters.update(get_filters_for_field(key, None, key)) return queryset - def values_list(self, *fields: str, flat: bool = False): + def values_list(self, *fields: str, flat: bool = False): # pylint: disable=W0621 """ Make QuerySet returns list of tuples for given args instead of objects. If ```flat=True`` and only one arg is passed can return flat list. @@ -581,6 +581,8 @@ async def _execute(self): class FieldSelectQuery(AwaitableQuery): + # pylint: disable=W0223 + def _join_table_with_forwarded_fields(self, model, field, forwarded_fields): table = Table(model._meta.table) if field in model._meta.fields_db_projection and not forwarded_fields: From 16da4d30bbd178d95e4a44e3bf9f6282d8dde95e Mon Sep 17 00:00:00 2001 From: Grigi Date: Thu, 27 Sep 2018 13:13:11 +0200 Subject: [PATCH 2/5] Fix check --- tortoise/models.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tortoise/models.py b/tortoise/models.py index 3954de1e7..80ce45f2f 100644 --- a/tortoise/models.py +++ b/tortoise/models.py @@ -332,10 +332,8 @@ def __init__(self, *args, **kwargs) -> None: for key in self._meta.backward_fk_fields: field = self._meta.fields_map[key] - setattr( - self, key, - RelationQueryContainer(field.type, field.relation_field, self, is_new) - ) + setattr(self, key, RelationQueryContainer( + field.type, field.relation_field, self, is_new)) # type: ignore for key in self._meta.m2m_fields: field = self._meta.fields_map[key] From bed66a430eaab1ec0e9690fbace3b2563b40f945 Mon Sep 17 00:00:00 2001 From: Grigi Date: Thu, 27 Sep 2018 13:15:41 +0200 Subject: [PATCH 3/5] v0.10.8 --- CHANGELOG.rst | 5 +++++ tortoise/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b9258efb1..cbfc00ac8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,10 @@ Changelog ========= + +0.10.8 +------ +- Performance fixes from ``pypika`` and object retrieval + 0.10.7 ------ - Fixed SQLite relative db path and :memory: now also works diff --git a/tortoise/__init__.py b/tortoise/__init__.py index b0984fc18..32a334724 100644 --- a/tortoise/__init__.py +++ b/tortoise/__init__.py @@ -374,4 +374,4 @@ async def do_stuff(): loop.run_until_complete(Tortoise.close_connections()) -__version__ = "0.10.7" +__version__ = "0.10.8" From eb06787d78ebdb0cf63f87856550f3eba263ee5d Mon Sep 17 00:00:00 2001 From: Grigi Date: Thu, 27 Sep 2018 13:29:30 +0200 Subject: [PATCH 4/5] Fix bad refactor --- tortoise/backends/asyncpg/client.py | 4 ++-- tortoise/backends/base/client.py | 6 ++++-- tortoise/backends/mysql/client.py | 4 ++-- tortoise/backends/sqlite/client.py | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tortoise/backends/asyncpg/client.py b/tortoise/backends/asyncpg/client.py index cf37988ac..6f7fe70b9 100644 --- a/tortoise/backends/asyncpg/client.py +++ b/tortoise/backends/asyncpg/client.py @@ -18,8 +18,8 @@ class AsyncpgDBClient(BaseDBAsyncClient): executor_class = AsyncpgExecutor schema_generator = AsyncpgSchemaGenerator - def __init__(self, user, password, database, host, port, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self, user, password, database, host, port, **kwargs): + super().__init__(**kwargs) self.user = user self.password = password diff --git a/tortoise/backends/base/client.py b/tortoise/backends/base/client.py index 28f59de02..7545238ff 100644 --- a/tortoise/backends/base/client.py +++ b/tortoise/backends/base/client.py @@ -62,11 +62,13 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): class SingleConnectionWrapper(BaseDBAsyncClient): - # pylint: disable=W0223 + # pylint: disable=W0223,W0231 def __init__(self, connection_name, connection, closing_callback=None): - super().__init__(connection_name, single_connection=True) + self.log = logging.getLogger('db_client') + self.connection_name = connection_name self.connection = connection + self.single_connection = True self.closing_callback = closing_callback def acquire_connection(self): diff --git a/tortoise/backends/mysql/client.py b/tortoise/backends/mysql/client.py index 952bbfa18..b59cff463 100644 --- a/tortoise/backends/mysql/client.py +++ b/tortoise/backends/mysql/client.py @@ -18,8 +18,8 @@ class MySQLClient(BaseDBAsyncClient): executor_class = MySQLExecutor schema_generator = MySQLSchemaGenerator - def __init__(self, user, password, database, host, port, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self, user, password, database, host, port, **kwargs): + super().__init__(**kwargs) self.user = user self.password = password diff --git a/tortoise/backends/sqlite/client.py b/tortoise/backends/sqlite/client.py index 26fe11035..c4c0c830c 100644 --- a/tortoise/backends/sqlite/client.py +++ b/tortoise/backends/sqlite/client.py @@ -16,8 +16,8 @@ class SqliteClient(BaseDBAsyncClient): executor_class = SqliteExecutor schema_generator = SqliteSchemaGenerator - def __init__(self, file_path, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self, file_path, **kwargs): + super().__init__(**kwargs) self.filename = file_path self._transaction_class = type( 'TransactionWrapper', (TransactionWrapper, self.__class__), {} From f081426f959fb4d4d99a569586c764ce6195c1af Mon Sep 17 00:00:00 2001 From: Grigi Date: Thu, 27 Sep 2018 20:16:45 +0200 Subject: [PATCH 5/5] Allow NPE to pass through --- tortoise/backends/asyncpg/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tortoise/backends/asyncpg/client.py b/tortoise/backends/asyncpg/client.py index 6f7fe70b9..4a3d3c36f 100644 --- a/tortoise/backends/asyncpg/client.py +++ b/tortoise/backends/asyncpg/client.py @@ -173,7 +173,7 @@ async def start(self): async def commit(self): try: await self.transaction.commit() - except (AttributeError, asyncpg.exceptions._base.InterfaceError) as exc: + except asyncpg.exceptions._base.InterfaceError as exc: raise TransactionManagementError(exc) if self._pool: await self._pool.release(self._connection) @@ -183,7 +183,7 @@ async def commit(self): async def rollback(self): try: await self.transaction.rollback() - except (AttributeError, asyncpg.exceptions._base.InterfaceError) as exc: + except asyncpg.exceptions._base.InterfaceError as exc: raise TransactionManagementError(exc) if self._pool: await self._pool.release(self._connection)