From 1045cb0cb9f81ad2704d0300226ec4e4f474a7c4 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Thu, 13 Oct 2022 17:25:58 +0300 Subject: [PATCH 1/6] setup: clarify Python versions We run tests with Python 3.6 and newer. Python 3.5 has reached its end of life two years ago [1]. Our code already doesn't work on Python 3.5 [2]. 1. https://endoflife.date/python 2. https://github.com/tarantool/tarantool-python/actions/runs/3243224481/jobs/5317632662 Part of #232 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0ce35187..0d1d7e20 100755 --- a/setup.py +++ b/setup.py @@ -104,5 +104,5 @@ def find_version(*file_paths): setup_requires=[ 'setuptools_scm==6.4.2', ], - python_requires='>=3', + python_requires='>=3.6', ) From 249e1e8f9318045bcda8da99f1e875d51e7cf3aa Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Thu, 13 Oct 2022 17:24:05 +0300 Subject: [PATCH 2/6] deps: backport dataclasses for Python 3.6 Dataclasses would be a really convenient approach to provide box.error support. They are supported in Python since 3.7, but this package make it able to introduce their support in Python 3.6 which we still support. Part of #232 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2a9c9c38..5885f0e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ msgpack>=1.0.4 pandas pytz +dataclasses; python_version <= '3.6' From cd6a62a60c337c3ede10ce1204cf65abb6ec30de Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Thu, 13 Oct 2022 17:35:31 +0300 Subject: [PATCH 3/6] api: ConnectionPool support for Python 3.6 After backporting dataclasses, it became possible to use ConnectionPool on Python 3.6. Follows #196 --- CHANGELOG.md | 1 + tarantool/__init__.py | 10 ++++------ test/suites/lib/skip.py | 8 -------- test/suites/test_pool.py | 3 +-- test/suites/test_ssl.py | 2 -- 5 files changed, 6 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eed24c57..9709d445 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,6 +135,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ``` - Support iproto feature discovery (#206). +- Backport ConnectionPool support for Python 3.6. - Support pandas way to build datetime from timestamp (PR #252). diff --git a/tarantool/__init__.py b/tarantool/__init__.py index 62f5aea5..0c63b675 100644 --- a/tarantool/__init__.py +++ b/tarantool/__init__.py @@ -40,6 +40,8 @@ Interval, ) +from tarantool.connection_pool import ConnectionPool, Mode + try: from tarantool.version import __version__ except ImportError: @@ -136,9 +138,5 @@ def connectmesh(addrs=({'host': 'localhost', 'port': 3301},), user=None, __all__ = ['connect', 'Connection', 'connectmesh', 'MeshConnection', 'Schema', 'Error', 'DatabaseError', 'NetworkError', 'NetworkWarning', - 'SchemaError', 'dbapi', 'Datetime', 'Interval', 'IntervalAdjust'] - -# ConnectionPool is supported only for Python 3.7 or newer. -if sys.version_info.major >= 3 and sys.version_info.minor >= 7: - from tarantool.connection_pool import ConnectionPool, Mode - __all__.extend(['ConnectionPool', 'Mode']) + 'SchemaError', 'dbapi', 'Datetime', 'Interval', 'IntervalAdjust', + 'ConnectionPool', 'Mode'] diff --git a/test/suites/lib/skip.py b/test/suites/lib/skip.py index b34a445b..536b059c 100644 --- a/test/suites/lib/skip.py +++ b/test/suites/lib/skip.py @@ -122,14 +122,6 @@ def skip_or_run_varbinary_test(func): 'does not support VARBINARY type') -def skip_or_run_conn_pool_test(func): - """Decorator to skip or run ConnectionPool tests depending on - the Python version. - """ - - return skip_or_run_test_python(func, '3.7', - 'does not support ConnectionPool') - def skip_or_run_decimal_test(func): """Decorator to skip or run decimal-related tests depending on the tarantool version. diff --git a/test/suites/test_pool.py b/test/suites/test_pool.py index 5c5aaeb6..f5e27f16 100644 --- a/test/suites/test_pool.py +++ b/test/suites/test_pool.py @@ -13,7 +13,7 @@ PoolTolopogyWarning, ) -from .lib.skip import skip_or_run_sql_test, skip_or_run_conn_pool_test +from .lib.skip import skip_or_run_sql_test from .lib.tarantool_server import TarantoolServer @@ -75,7 +75,6 @@ def setUpClass(self): print(' POOL '.center(70, '='), file=sys.stderr) print('-' * 70, file=sys.stderr) - @skip_or_run_conn_pool_test def setUp(self): # Create five servers and extract helpful fields for tests. self.servers = [] diff --git a/test/suites/test_ssl.py b/test/suites/test_ssl.py index 977474f3..fd0aa450 100644 --- a/test/suites/test_ssl.py +++ b/test/suites/test_ssl.py @@ -10,7 +10,6 @@ SSL_TRANSPORT ) import tarantool -from .lib.skip import skip_or_run_conn_pool_test from .lib.tarantool_server import TarantoolServer @@ -291,7 +290,6 @@ def __init__(self, @unittest.skipIf(sys.platform.startswith("win"), 'Pool tests on windows platform are not supported') - @skip_or_run_conn_pool_test def test_pool(self): servers = [] cnt = 5 From 40d6ce895abacbe3126f11a2d255ed4243f3da92 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Thu, 13 Oct 2022 12:46:56 +0300 Subject: [PATCH 4/6] msgpack: improve unknown ext type error message Part of #232 --- tarantool/msgpack_ext/unpacker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tarantool/msgpack_ext/unpacker.py b/tarantool/msgpack_ext/unpacker.py index bc1fb0a0..fdc204c6 100644 --- a/tarantool/msgpack_ext/unpacker.py +++ b/tarantool/msgpack_ext/unpacker.py @@ -35,4 +35,4 @@ def ext_hook(code, data): if code in decoders: return decoders[code](data) - raise NotImplementedError("Unknown msgpack type: %d" % (code,)) + raise NotImplementedError("Unknown msgpack extension type code %d" % (code,)) From 1c202d18456d3f5f3cef42686f498caeb5b77a67 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Fri, 14 Oct 2022 10:27:30 +0300 Subject: [PATCH 5/6] iproto: support errors extra information Since Tarantool 2.4.1, iproto error responses contain extra info with backtrace. After this patch, DatabaseError would contain `extra_info` property, if it was provided. Error extra information is parsed based on common encoder/decoder rules. String fields are converted to either `str` or `bytes` based on `encoding` mode. 1. https://www.tarantool.io/en/doc/latest/dev_guide/internals/box_protocol/#responses-for-errors Part of #232 --- CHANGELOG.md | 1 + docs/source/api/submodule-types.rst | 4 ++ docs/source/index.rst | 1 + tarantool/const.py | 4 +- tarantool/error.py | 15 +++- tarantool/response.py | 14 +++- tarantool/types.py | 108 ++++++++++++++++++++++++++++ test/suites/__init__.py | 3 +- test/suites/lib/skip.py | 11 +++ test/suites/test_dml.py | 73 +++++++++++++++++++ test/suites/test_encoding.py | 24 ++++++- 11 files changed, 250 insertions(+), 8 deletions(-) create mode 100644 docs/source/api/submodule-types.rst create mode 100644 tarantool/types.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9709d445..be66011a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -136,6 +136,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support iproto feature discovery (#206). - Backport ConnectionPool support for Python 3.6. +- Support extra information for iproto errors (#232). - Support pandas way to build datetime from timestamp (PR #252). diff --git a/docs/source/api/submodule-types.rst b/docs/source/api/submodule-types.rst new file mode 100644 index 00000000..08f081db --- /dev/null +++ b/docs/source/api/submodule-types.rst @@ -0,0 +1,4 @@ +module :py:mod:`tarantool.types` +================================ + +.. automodule:: tarantool.types diff --git a/docs/source/index.rst b/docs/source/index.rst index a3be248e..5e5a04ca 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -43,6 +43,7 @@ API Reference api/submodule-response.rst api/submodule-schema.rst api/submodule-space.rst + api/submodule-types.rst api/submodule-utils.rst .. Indices and tables diff --git a/tarantool/const.py b/tarantool/const.py index 52d5ea81..4cbf511d 100644 --- a/tarantool/const.py +++ b/tarantool/const.py @@ -27,7 +27,7 @@ IPROTO_OPS = 0x28 # IPROTO_DATA = 0x30 -IPROTO_ERROR = 0x31 +IPROTO_ERROR_24 = 0x31 # IPROTO_METADATA = 0x32 IPROTO_SQL_TEXT = 0x40 @@ -36,6 +36,8 @@ IPROTO_SQL_INFO_ROW_COUNT = 0x00 IPROTO_SQL_INFO_AUTOINCREMENT_IDS = 0x01 # +IPROTO_ERROR = 0x52 +# IPROTO_VERSION = 0x54 IPROTO_FEATURES = 0x55 diff --git a/tarantool/error.py b/tarantool/error.py index c8690a0b..c8737b49 100644 --- a/tarantool/error.py +++ b/tarantool/error.py @@ -41,10 +41,17 @@ class DatabaseError(Error): Exception raised for errors that are related to the database. """ - def __init__(self, *args): + def __init__(self, *args, extra_info=None): """ :param args: ``(code, message)`` or ``(message,)``. :type args: :obj:`tuple` + + :param extra_info: Additional `box.error`_ information + with backtrace. + :type extra_info: :class:`~tarantool.types.BoxError` or + :obj:`None`, optional + + .. _box.error: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_error/error/ """ super().__init__(*args) @@ -59,6 +66,8 @@ def __init__(self, *args): self.code = 0 self.message = '' + self.extra_info = extra_info + class DataError(DatabaseError): """ @@ -235,7 +244,7 @@ class NetworkError(DatabaseError): Error related to network. """ - def __init__(self, orig_exception=None, *args): + def __init__(self, orig_exception=None, *args, **kwargs): """ :param orig_exception: Exception to wrap. :type orig_exception: optional @@ -256,7 +265,7 @@ def __init__(self, orig_exception=None, *args): super(NetworkError, self).__init__( orig_exception.errno, self.message) else: - super(NetworkError, self).__init__(orig_exception, *args) + super(NetworkError, self).__init__(orig_exception, *args, **kwargs) class NetworkWarning(UserWarning): diff --git a/tarantool/response.py b/tarantool/response.py index ce7320f7..f318839a 100644 --- a/tarantool/response.py +++ b/tarantool/response.py @@ -11,6 +11,7 @@ from tarantool.const import ( IPROTO_REQUEST_TYPE, IPROTO_DATA, + IPROTO_ERROR_24, IPROTO_ERROR, IPROTO_SYNC, IPROTO_SCHEMA_ID, @@ -21,6 +22,7 @@ IPROTO_VERSION, IPROTO_FEATURES, ) +from tarantool.types import decode_box_error from tarantool.error import ( DatabaseError, InterfaceError, @@ -117,14 +119,22 @@ def __init__(self, conn, response): # self.append(self._data) else: # Separate return_code and completion_code - self._return_message = self._body.get(IPROTO_ERROR, "") + self._return_message = self._body.get(IPROTO_ERROR_24, "") self._return_code = self._code & (REQUEST_TYPE_ERROR - 1) + + self._return_error = None + return_error_map = self._body.get(IPROTO_ERROR) + if return_error_map is not None: + self._return_error = decode_box_error(return_error_map) + self._data = [] if self._return_code == 109: raise SchemaReloadException(self._return_message, self._schema_version) if self.conn.error: - raise DatabaseError(self._return_code, self._return_message) + raise DatabaseError(self._return_code, + self._return_message, + extra_info=self._return_error) def __getitem__(self, idx): if self._data is None: diff --git a/tarantool/types.py b/tarantool/types.py new file mode 100644 index 00000000..140baca9 --- /dev/null +++ b/tarantool/types.py @@ -0,0 +1,108 @@ +""" +Additional Tarantool type definitions. +""" + +import typing +from dataclasses import dataclass + +@dataclass +class BoxError(): + """ + Type representing Tarantool `box.error`_ object: a single + MP_ERROR_STACK object with a link to the previous stack error. + + .. _box.error: https://www.tarantool.io/en/doc/latest/reference/reference_lua/box_error/error/ + """ + + type: typing.Union[str, bytes] + """ + Type that implies source, for example ``"ClientError"``. + + Value type depends on :class:`~tarantool.Connection` + :paramref:`~tarantool.Connection.params.encoding`. + """ + + file: typing.Union[str, bytes] + """ + Source code file where error was caught. + + Value type depends on :class:`~tarantool.Connection` + :paramref:`~tarantool.Connection.params.encoding`. + """ + + line: int + """ + Line number in source code file. + """ + + message: typing.Union[str, bytes] + """ + Text of reason. + + Value type depends on :class:`~tarantool.Connection` + :paramref:`~tarantool.Connection.params.encoding`. + """ + + errno: int + """ + Ordinal number of the error. + """ + + errcode: int + """ + Number of the error as defined in ``errcode.h``. + """ + + fields: typing.Optional[dict] = None + """ + Additional fields depending on error type. For example, if + :attr:`~tarantool.types.BoxError.type` is ``"AccessDeniedError"``, + then it will include ``"object_type"``, ``"object_name"``, + ``"access_type"``. + """ + + prev: typing.Optional[typing.List['BoxError']] = None + """ + Previous error in stack. + """ + + +MP_ERROR_STACK = 0x00 +MP_ERROR_TYPE = 0x00 +MP_ERROR_FILE = 0x01 +MP_ERROR_LINE = 0x02 +MP_ERROR_MESSAGE = 0x03 +MP_ERROR_ERRNO = 0x04 +MP_ERROR_ERRCODE = 0x05 +MP_ERROR_FIELDS = 0x06 + +def decode_box_error(err_map): + """ + Decode MessagePack map received from Tarantool to `box.error`_ + object representation. + + :param err_map: Error MessagePack map received from Tarantool. + :type err_map: :obj:`dict` + + :rtype: :class:`~tarantool.BoxError` + + :raises: :exc:`KeyError` + """ + + encoded_stack = err_map[MP_ERROR_STACK] + + prev = None + for item in encoded_stack[::-1]: + err = BoxError( + type=item[MP_ERROR_TYPE], + file=item[MP_ERROR_FILE], + line=item[MP_ERROR_LINE], + message=item[MP_ERROR_MESSAGE], + errno=item[MP_ERROR_ERRNO], + errcode=item[MP_ERROR_ERRCODE], + fields=item.get(MP_ERROR_FIELDS), # omitted if empty + prev=prev, + ) + prev = err + + return prev diff --git a/test/suites/__init__.py b/test/suites/__init__.py index c8d85561..f825f2ac 100644 --- a/test/suites/__init__.py +++ b/test/suites/__init__.py @@ -38,4 +38,5 @@ def load_tests(loader, tests, pattern): os.chdir(__tmp) - +# Workaround to disable unittest output truncating +__import__('sys').modules['unittest.util']._MAX_LENGTH = 99999 diff --git a/test/suites/lib/skip.py b/test/suites/lib/skip.py index 536b059c..1b63a55e 100644 --- a/test/suites/lib/skip.py +++ b/test/suites/lib/skip.py @@ -154,3 +154,14 @@ def skip_or_run_datetime_test(func): return skip_or_run_test_pcall_require(func, 'datetime', 'does not support datetime type') + +def skip_or_run_error_extra_info_test(func): + """Decorator to skip or run tests related to extra error info + provided over iproto depending on the tarantool version. + + Tarantool provides extra error info only since 2.4.1 version. + See https://github.com/tarantool/tarantool/issues/4398 + """ + + return skip_or_run_test_tarantool(func, '2.4.1', + 'does not provide extra error info') diff --git a/test/suites/test_dml.py b/test/suites/test_dml.py index 2a4c9481..1f419084 100644 --- a/test/suites/test_dml.py +++ b/test/suites/test_dml.py @@ -1,8 +1,10 @@ import sys import unittest import tarantool +from tarantool.error import DatabaseError from .lib.tarantool_server import TarantoolServer +from .lib.skip import skip_or_run_error_extra_info_test class TestSuite_Request(unittest.TestCase): @classmethod @@ -325,6 +327,77 @@ def test_14_idempotent_close(self): con.close() self.assertEqual(con.is_closed(), True) + @skip_or_run_error_extra_info_test + def test_14_extra_error_info(self): + try: + self.con.eval("not a Lua code") + except DatabaseError as exc: + self.assertEqual(exc.extra_info.type, 'LuajitError') + self.assertRegex(exc.extra_info.file, r'/tarantool') + self.assertTrue(exc.extra_info.line > 0) + self.assertEqual(exc.extra_info.message, "eval:1: unexpected symbol near 'not'") + self.assertEqual(exc.extra_info.errno, 0) + self.assertEqual(exc.extra_info.errcode, 32) + self.assertEqual(exc.extra_info.fields, None) + self.assertEqual(exc.extra_info.prev, None) + else: + self.fail('Expected error') + + @skip_or_run_error_extra_info_test + def test_15_extra_error_info_stacked(self): + try: + self.con.eval(r""" + local e1 = box.error.new(box.error.UNKNOWN) + local e2 = box.error.new(box.error.TIMEOUT) + e2:set_prev(e1) + error(e2) + """) + except DatabaseError as exc: + self.assertEqual(exc.extra_info.type, 'ClientError') + self.assertRegex(exc.extra_info.file, 'eval') + self.assertEqual(exc.extra_info.line, 3) + self.assertEqual(exc.extra_info.message, "Timeout exceeded") + self.assertEqual(exc.extra_info.errno, 0) + self.assertEqual(exc.extra_info.errcode, 78) + self.assertEqual(exc.extra_info.fields, None) + self.assertNotEqual(exc.extra_info.prev, None) + prev = exc.extra_info.prev + self.assertEqual(prev.type, 'ClientError') + self.assertEqual(prev.file, 'eval') + self.assertEqual(prev.line, 2) + self.assertEqual(prev.message, "Unknown error") + self.assertEqual(prev.errno, 0) + self.assertEqual(prev.errcode, 0) + self.assertEqual(prev.fields, None) + else: + self.fail('Expected error') + + @skip_or_run_error_extra_info_test + def test_16_extra_error_info_fields(self): + try: + self.con.eval(""" + box.schema.func.create('forbidden_function') + """) + except DatabaseError as exc: + self.assertEqual(exc.extra_info.type, 'AccessDeniedError') + self.assertRegex(exc.extra_info.file, r'/tarantool') + self.assertTrue(exc.extra_info.line > 0) + self.assertEqual( + exc.extra_info.message, + "Create access to function 'forbidden_function' is denied for user 'test'") + self.assertEqual(exc.extra_info.errno, 0) + self.assertEqual(exc.extra_info.errcode, 42) + self.assertEqual( + exc.extra_info.fields, + { + 'object_type': 'function', + 'object_name': 'forbidden_function', + 'access_type': 'Create' + }) + self.assertEqual(exc.extra_info.prev, None) + else: + self.fail('Expected error') + @classmethod def tearDownClass(self): self.con.close() diff --git a/test/suites/test_encoding.py b/test/suites/test_encoding.py index e434b2d5..45bc6053 100644 --- a/test/suites/test_encoding.py +++ b/test/suites/test_encoding.py @@ -1,8 +1,10 @@ import sys import unittest + import tarantool +from tarantool.error import DatabaseError -from .lib.skip import skip_or_run_varbinary_test +from .lib.skip import skip_or_run_varbinary_test, skip_or_run_error_extra_info_test from .lib.tarantool_server import TarantoolServer class TestSuite_Encoding(unittest.TestCase): @@ -172,6 +174,26 @@ def test_02_04_varbinary_decode_for_encoding_none_behavior(self): """ % (space, data_hex)) self.assertSequenceEqual(resp, [[data_id, data]]) + @skip_or_run_error_extra_info_test + def test_01_05_error_extra_info_decode_for_encoding_utf8_behavior(self): + try: + self.con_encoding_utf8.eval("not a Lua code") + except DatabaseError as exc: + self.assertEqual(exc.extra_info.type, 'LuajitError') + self.assertEqual(exc.extra_info.message, "eval:1: unexpected symbol near 'not'") + else: + self.fail('Expected error') + + @skip_or_run_error_extra_info_test + def test_02_05_error_extra_info_decode_for_encoding_none_behavior(self): + try: + self.con_encoding_none.eval("not a Lua code") + except DatabaseError as exc: + self.assertEqual(exc.extra_info.type, b'LuajitError') + self.assertEqual(exc.extra_info.message, b"eval:1: unexpected symbol near 'not'") + else: + self.fail('Expected error') + @classmethod def tearDownClass(self): for con in self.conns: From 951aa1a359d00a7db15c1c7b371b1f9a2783ec53 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Mon, 17 Oct 2022 17:17:25 +0300 Subject: [PATCH 6/6] msgpack: support error extension type Tarantool supports error extension type since version 2.4.1 [1], encoding was introduced in Tarantool 2.10.0 [2]. This patch introduces the support of Tarantool error extension type in msgpack decoders and encoders. Tarantool error extension type objects are decoded to `tarantool.BoxError` type. `tarantool.BoxError` may be encoded to Tarantool error extension type objects. Error extension type internals are the same as errors extra information: the only difference is that extra information is encoded as a separate error dictionary field and error extension type objects is encoded as MessagePack extension type objects. Error extension type objects are parsed based on common encoder/decoder rules. String fields are converted to either `str` or `bytes` based on `encoding` mode. The only way to receive an error extension type object from Tarantool is to receive an explicitly built `box.error` object: either from `return box.error.new(...)` or a tuple with it. All errors raised within Tarantool (including those raised with `box.error(...)`) are encoded based on the same rules as simple errors due to backward compatibility. It is possible to create error extension type objects with Python code, but it not likely to be really useful since most of their fields is computed on error initialization on the server side (even for custom error types): ``` tarantool.BoxError( type='ClientError', file='[string " local err = box.error.ne..."]', line=1, message='Unknown error', errno=0, errcode=0, ) ``` 1. https://github.com/tarantool/tarantool/issues/4398 2. https://github.com/tarantool/tarantool/issues/6433 Closes #232 --- CHANGELOG.md | 1 + docs/source/dev-guide.rst | 3 + tarantool/__init__.py | 4 +- tarantool/const.py | 2 +- tarantool/msgpack_ext/datetime.py | 4 +- tarantool/msgpack_ext/decimal.py | 4 +- tarantool/msgpack_ext/error.py | 52 ++++ tarantool/msgpack_ext/interval.py | 10 +- tarantool/msgpack_ext/packer.py | 14 +- tarantool/msgpack_ext/unpacker.py | 13 +- tarantool/msgpack_ext/uuid.py | 4 +- tarantool/request.py | 106 ++++---- tarantool/response.py | 88 ++++--- tarantool/types.py | 35 ++- test/suites/__init__.py | 3 +- test/suites/lib/skip.py | 13 + test/suites/test_error_ext.py | 399 ++++++++++++++++++++++++++++++ test/suites/test_interval.py | 11 +- test/suites/test_protocol.py | 3 +- 19 files changed, 666 insertions(+), 103 deletions(-) create mode 100644 tarantool/msgpack_ext/error.py create mode 100644 test/suites/test_error_ext.py diff --git a/CHANGELOG.md b/CHANGELOG.md index be66011a..6d44e5de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,6 +137,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support iproto feature discovery (#206). - Backport ConnectionPool support for Python 3.6. - Support extra information for iproto errors (#232). +- Error extension type support (#232). - Support pandas way to build datetime from timestamp (PR #252). diff --git a/docs/source/dev-guide.rst b/docs/source/dev-guide.rst index a3f7d704..f19fe34f 100644 --- a/docs/source/dev-guide.rst +++ b/docs/source/dev-guide.rst @@ -83,6 +83,8 @@ they are represented with in-built and custom types: +-----------------------------+----+-------------+----+-----------------------------+ | :obj:`uuid.UUID` | -> | `UUID`_ | -> | :obj:`uuid.UUID` | +-----------------------------+----+-------------+----+-----------------------------+ + | :class:`tarantool.BoxError` | -> | `ERROR`_ | -> | :class:`tarantool.BoxError` | + +-----------------------------+----+-------------+----+-----------------------------+ | :class:`tarantool.Datetime` | -> | `DATETIME`_ | -> | :class:`tarantool.Datetime` | +-----------------------------+----+-------------+----+-----------------------------+ | :class:`tarantool.Interval` | -> | `INTERVAL`_ | -> | :class:`tarantool.Interval` | @@ -109,5 +111,6 @@ and iterate through it as with any other serializable object. .. _extension types: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/ .. _DECIMAL: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type .. _UUID: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-uuid-type +.. _ERROR: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-error-type .. _DATETIME: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-datetime-type .. _INTERVAL: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-interval-type diff --git a/tarantool/__init__.py b/tarantool/__init__.py index 0c63b675..915961fd 100644 --- a/tarantool/__init__.py +++ b/tarantool/__init__.py @@ -42,6 +42,8 @@ from tarantool.connection_pool import ConnectionPool, Mode +from tarantool.types import BoxError + try: from tarantool.version import __version__ except ImportError: @@ -139,4 +141,4 @@ def connectmesh(addrs=({'host': 'localhost', 'port': 3301},), user=None, __all__ = ['connect', 'Connection', 'connectmesh', 'MeshConnection', 'Schema', 'Error', 'DatabaseError', 'NetworkError', 'NetworkWarning', 'SchemaError', 'dbapi', 'Datetime', 'Interval', 'IntervalAdjust', - 'ConnectionPool', 'Mode'] + 'ConnectionPool', 'Mode', 'BoxError',] diff --git a/tarantool/const.py b/tarantool/const.py index 4cbf511d..8f9f6ce0 100644 --- a/tarantool/const.py +++ b/tarantool/const.py @@ -129,4 +129,4 @@ # Tarantool 2.10 protocol version is 3 CONNECTOR_IPROTO_VERSION = 3 # List of connector-supported features -CONNECTOR_FEATURES = [] +CONNECTOR_FEATURES = [IPROTO_FEATURE_ERROR_EXTENSION,] diff --git a/tarantool/msgpack_ext/datetime.py b/tarantool/msgpack_ext/datetime.py index fc1045d4..64422f5d 100644 --- a/tarantool/msgpack_ext/datetime.py +++ b/tarantool/msgpack_ext/datetime.py @@ -78,7 +78,7 @@ def get_int_as_bytes(data, size): return data.to_bytes(size, byteorder=BYTEORDER, signed=True) -def encode(obj): +def encode(obj, _): """ Encode a datetime object. @@ -134,7 +134,7 @@ def get_bytes_as_int(data, cursor, size): part = data[cursor:cursor + size] return int.from_bytes(part, BYTEORDER, signed=True), cursor + size -def decode(data): +def decode(data, _): """ Decode a datetime object. diff --git a/tarantool/msgpack_ext/decimal.py b/tarantool/msgpack_ext/decimal.py index 80e40051..bad947fb 100644 --- a/tarantool/msgpack_ext/decimal.py +++ b/tarantool/msgpack_ext/decimal.py @@ -225,7 +225,7 @@ def strip_decimal_str(str_repr, scale, first_digit_ind): # Do not strips zeroes before the decimal point return str_repr -def encode(obj): +def encode(obj, _): """ Encode a decimal object. @@ -335,7 +335,7 @@ def add_str_digit(digit, digits_reverted, scale): digits_reverted.append(str(digit)) -def decode(data): +def decode(data, _): """ Decode a decimal object. diff --git a/tarantool/msgpack_ext/error.py b/tarantool/msgpack_ext/error.py new file mode 100644 index 00000000..a3f13a04 --- /dev/null +++ b/tarantool/msgpack_ext/error.py @@ -0,0 +1,52 @@ +""" +Tarantool `error`_ extension type support module. + +Refer to :mod:`~tarantool.msgpack_ext.types.error`. + +.. _error: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-error-type +""" + +from tarantool.types import ( + encode_box_error, + decode_box_error, +) + +EXT_ID = 3 +""" +`error`_ type id. +""" + +def encode(obj, packer): + """ + Encode an error object. + + :param obj: Error to encode. + :type obj: :class:`tarantool.BoxError` + + :param packer: msgpack packer to encode error dictionary. + :type packer: :class:`msgpack.Packer` + + :return: Encoded error. + :rtype: :obj:`bytes` + """ + + err_map = encode_box_error(obj) + return packer.pack(err_map) + +def decode(data, unpacker): + """ + Decode an error object. + + :param obj: Error to decode. + :type obj: :obj:`bytes` + + :param unpacker: msgpack unpacker to decode error dictionary. + :type unpacker: :class:`msgpack.Unpacker` + + :return: Decoded error. + :rtype: :class:`tarantool.BoxError` + """ + + unpacker.feed(data) + err_map = unpacker.unpack() + return decode_box_error(err_map) diff --git a/tarantool/msgpack_ext/interval.py b/tarantool/msgpack_ext/interval.py index 725edc9d..cd7ab16d 100644 --- a/tarantool/msgpack_ext/interval.py +++ b/tarantool/msgpack_ext/interval.py @@ -51,7 +51,7 @@ `datetime.interval`_ type id. """ -def encode(obj): +def encode(obj, _): """ Encode an interval object. @@ -80,13 +80,16 @@ def encode(obj): return buf -def decode(data): +def decode(data, unpacker): """ Decode an interval object. :param obj: Interval to decode. :type obj: :obj:`bytes` + :param unpacker: msgpack unpacker to decode fields. + :type unpacker: :class:`msgpack.Unpacker` + :return: Decoded interval. :rtype: :class:`tarantool.Interval` @@ -108,9 +111,8 @@ def decode(data): } if len(data) != 0: - # To create an unpacker is the only way to parse + # Unpacker object is the only way to parse # a sequence of values in Python msgpack module. - unpacker = msgpack.Unpacker() unpacker.feed(data) field_count = unpacker.unpack() for _ in range(field_count): diff --git a/tarantool/msgpack_ext/packer.py b/tarantool/msgpack_ext/packer.py index 4706496f..12faa29e 100644 --- a/tarantool/msgpack_ext/packer.py +++ b/tarantool/msgpack_ext/packer.py @@ -8,28 +8,36 @@ from uuid import UUID from msgpack import ExtType +from tarantool.types import BoxError from tarantool.msgpack_ext.types.datetime import Datetime from tarantool.msgpack_ext.types.interval import Interval import tarantool.msgpack_ext.decimal as ext_decimal import tarantool.msgpack_ext.uuid as ext_uuid +import tarantool.msgpack_ext.error as ext_error import tarantool.msgpack_ext.datetime as ext_datetime import tarantool.msgpack_ext.interval as ext_interval encoders = [ {'type': Decimal, 'ext': ext_decimal }, {'type': UUID, 'ext': ext_uuid }, + {'type': BoxError, 'ext': ext_error }, {'type': Datetime, 'ext': ext_datetime}, {'type': Interval, 'ext': ext_interval}, ] -def default(obj): +def default(obj, packer=None): """ :class:`msgpack.Packer` encoder. :param obj: Object to encode. :type obj: :class:`decimal.Decimal` or :class:`uuid.UUID` or - :class:`tarantool.Datetime` or :class:`tarantool.Interval` + or :class:`tarantool.BoxError` or :class:`tarantool.Datetime` + or :class:`tarantool.Interval` + + :param packer: msgpack packer to work with common types + (like dictionary in extended error payload) + :type packer: :class:`msgpack.Packer`, optional :return: Encoded value. :rtype: :class:`msgpack.ExtType` @@ -39,5 +47,5 @@ def default(obj): for encoder in encoders: if isinstance(obj, encoder['type']): - return ExtType(encoder['ext'].EXT_ID, encoder['ext'].encode(obj)) + return ExtType(encoder['ext'].EXT_ID, encoder['ext'].encode(obj, packer)) raise TypeError("Unknown type: %r" % (obj,)) diff --git a/tarantool/msgpack_ext/unpacker.py b/tarantool/msgpack_ext/unpacker.py index fdc204c6..6950f485 100644 --- a/tarantool/msgpack_ext/unpacker.py +++ b/tarantool/msgpack_ext/unpacker.py @@ -6,17 +6,19 @@ import tarantool.msgpack_ext.decimal as ext_decimal import tarantool.msgpack_ext.uuid as ext_uuid +import tarantool.msgpack_ext.error as ext_error import tarantool.msgpack_ext.datetime as ext_datetime import tarantool.msgpack_ext.interval as ext_interval decoders = { ext_decimal.EXT_ID : ext_decimal.decode , ext_uuid.EXT_ID : ext_uuid.decode , + ext_error.EXT_ID : ext_error.decode , ext_datetime.EXT_ID: ext_datetime.decode, ext_interval.EXT_ID: ext_interval.decode, } -def ext_hook(code, data): +def ext_hook(code, data, unpacker=None): """ :class:`msgpack.Unpacker` decoder. @@ -26,13 +28,18 @@ def ext_hook(code, data): :param data: MessagePack extension type data. :type data: :obj:`bytes` + :param unpacker: msgpack unpacker to work with common types + (like dictionary in extended error payload) + :type unpacker: :class:`msgpack.Unpacker`, optional + :return: Decoded value. :rtype: :class:`decimal.Decimal` or :class:`uuid.UUID` or - :class:`tarantool.Datetime` or :class:`tarantool.Interval` + or :class:`tarantool.BoxError` or :class:`tarantool.Datetime` + or :class:`tarantool.Interval` :raise: :exc:`NotImplementedError` """ if code in decoders: - return decoders[code](data) + return decoders[code](data, unpacker) raise NotImplementedError("Unknown msgpack extension type code %d" % (code,)) diff --git a/tarantool/msgpack_ext/uuid.py b/tarantool/msgpack_ext/uuid.py index 8a1951d0..91b4ac94 100644 --- a/tarantool/msgpack_ext/uuid.py +++ b/tarantool/msgpack_ext/uuid.py @@ -20,7 +20,7 @@ `uuid`_ type id. """ -def encode(obj): +def encode(obj, _): """ Encode an UUID object. @@ -33,7 +33,7 @@ def encode(obj): return obj.bytes -def decode(data): +def decode(data, _): """ Decode an UUID object. diff --git a/tarantool/request.py b/tarantool/request.py index 164047cd..7274b8d2 100644 --- a/tarantool/request.py +++ b/tarantool/request.py @@ -63,6 +63,67 @@ from tarantool.msgpack_ext.packer import default as packer_default +def build_packer(conn): + """ + Build packer to pack request. + + :param conn: Request sender. + :type conn: :class:`~tarantool.Connection` + + :rtype: :class:`msgpack.Packer` + """ + + packer_kwargs = dict() + + # use_bin_type=True is default since msgpack-1.0.0. + # + # The option controls whether to pack binary (non-unicode) + # string values as mp_bin or as mp_str. + # + # The default behaviour of the Python 3 connector (since + # default encoding is "utf-8") is to pack bytes as mp_bin + # and Unicode strings as mp_str. encoding=None mode must + # be used to work with non-utf strings. + # + # encoding = 'utf-8' + # + # Python 3 -> Tarantool -> Python 3 + # str -> mp_str (string) -> str + # bytes -> mp_bin (varbinary) -> bytes + # + # encoding = None + # + # Python 3 -> Tarantool -> Python 3 + # bytes -> mp_str (string) -> bytes + # str -> mp_str (string) -> bytes + # mp_bin (varbinary) -> bytes + # + # msgpack-0.5.0 (and only this version) warns when the + # option is unset: + # + # | FutureWarning: use_bin_type option is not specified. + # | Default value of the option will be changed in future + # | version. + # + # The option is supported since msgpack-0.4.0, so we can + # just always set it for all msgpack versions to get rid + # of the warning on msgpack-0.5.0 and to keep our + # behaviour on msgpack-1.0.0. + if conn.encoding is None: + packer_kwargs['use_bin_type'] = False + else: + packer_kwargs['use_bin_type'] = True + + # We need configured packer to work with error extention + # type payload, but module do not provide access to self + # inside extension type packers. + packer_no_ext = msgpack.Packer(**packer_kwargs) + default = lambda obj: packer_default(obj, packer_no_ext) + packer_kwargs['default'] = default + + return msgpack.Packer(**packer_kwargs) + + class Request(object): """ Represents a single request to the server in compliance with the @@ -87,50 +148,7 @@ def __init__(self, conn): self._body = '' self.response_class = Response - packer_kwargs = dict() - - # use_bin_type=True is default since msgpack-1.0.0. - # - # The option controls whether to pack binary (non-unicode) - # string values as mp_bin or as mp_str. - # - # The default behaviour of the Python 3 connector (since - # default encoding is "utf-8") is to pack bytes as mp_bin - # and Unicode strings as mp_str. encoding=None mode must - # be used to work with non-utf strings. - # - # encoding = 'utf-8' - # - # Python 3 -> Tarantool -> Python 3 - # str -> mp_str (string) -> str - # bytes -> mp_bin (varbinary) -> bytes - # - # encoding = None - # - # Python 3 -> Tarantool -> Python 3 - # bytes -> mp_str (string) -> bytes - # str -> mp_str (string) -> bytes - # mp_bin (varbinary) -> bytes - # - # msgpack-0.5.0 (and only this version) warns when the - # option is unset: - # - # | FutureWarning: use_bin_type option is not specified. - # | Default value of the option will be changed in future - # | version. - # - # The option is supported since msgpack-0.4.0, so we can - # just always set it for all msgpack versions to get rid - # of the warning on msgpack-0.5.0 and to keep our - # behaviour on msgpack-1.0.0. - if conn.encoding is None: - packer_kwargs['use_bin_type'] = False - else: - packer_kwargs['use_bin_type'] = True - - packer_kwargs['default'] = packer_default - - self.packer = msgpack.Packer(**packer_kwargs) + self.packer = build_packer(conn) def _dumps(self, src): """ diff --git a/tarantool/response.py b/tarantool/response.py index f318839a..7fef6e90 100644 --- a/tarantool/response.py +++ b/tarantool/response.py @@ -32,6 +32,58 @@ from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook +def build_unpacker(conn): + """ + Build unpacker to unpack request response. + + :param conn: Request sender. + :type conn: :class:`~tarantool.Connection` + + :rtype: :class:`msgpack.Unpacker` + """ + + unpacker_kwargs = dict() + + # Decode MsgPack arrays into Python lists by default (not tuples). + # Can be configured in the Connection init + unpacker_kwargs['use_list'] = conn.use_list + + # Use raw=False instead of encoding='utf-8'. + if msgpack.version >= (0, 5, 2) and conn.encoding == 'utf-8': + # Get rid of the following warning. + # > PendingDeprecationWarning: encoding is deprecated, + # > Use raw=False instead. + unpacker_kwargs['raw'] = False + elif conn.encoding is not None: + unpacker_kwargs['encoding'] = conn.encoding + + # raw=False is default since msgpack-1.0.0. + # + # The option decodes mp_str to bytes, not a Unicode + # string (when True). + if msgpack.version >= (1, 0, 0) and conn.encoding is None: + unpacker_kwargs['raw'] = True + + # encoding option is not supported since msgpack-1.0.0, + # but it is handled in the Connection constructor. + assert(msgpack.version < (1, 0, 0) or conn.encoding in (None, 'utf-8')) + + # strict_map_key=True is default since msgpack-1.0.0. + # + # The option forbids non-string keys in a map (when True). + if msgpack.version >= (1, 0, 0): + unpacker_kwargs['strict_map_key'] = False + + # We need configured unpacker to work with error extention + # type payload, but module do not provide access to self + # inside extension type unpackers. + unpacker_no_ext = msgpack.Unpacker(**unpacker_kwargs) + ext_hook = lambda code, data: unpacker_ext_hook(code, data, unpacker_no_ext) + unpacker_kwargs['ext_hook'] = ext_hook + + return msgpack.Unpacker(**unpacker_kwargs) + + class Response(Sequence): """ Represents a single response from the server in compliance with the @@ -56,41 +108,7 @@ def __init__(self, conn, response): # created in the __new__(). # super(Response, self).__init__() - unpacker_kwargs = dict() - - # Decode MsgPack arrays into Python lists by default (not tuples). - # Can be configured in the Connection init - unpacker_kwargs['use_list'] = conn.use_list - - # Use raw=False instead of encoding='utf-8'. - if msgpack.version >= (0, 5, 2) and conn.encoding == 'utf-8': - # Get rid of the following warning. - # > PendingDeprecationWarning: encoding is deprecated, - # > Use raw=False instead. - unpacker_kwargs['raw'] = False - elif conn.encoding is not None: - unpacker_kwargs['encoding'] = conn.encoding - - # raw=False is default since msgpack-1.0.0. - # - # The option decodes mp_str to bytes, not a Unicode - # string (when True). - if msgpack.version >= (1, 0, 0) and conn.encoding is None: - unpacker_kwargs['raw'] = True - - # encoding option is not supported since msgpack-1.0.0, - # but it is handled in the Connection constructor. - assert(msgpack.version < (1, 0, 0) or conn.encoding in (None, 'utf-8')) - - # strict_map_key=True is default since msgpack-1.0.0. - # - # The option forbids non-string keys in a map (when True). - if msgpack.version >= (1, 0, 0): - unpacker_kwargs['strict_map_key'] = False - - unpacker_kwargs['ext_hook'] = unpacker_ext_hook - - unpacker = msgpack.Unpacker(**unpacker_kwargs) + unpacker = build_unpacker(conn) unpacker.feed(response) header = unpacker.unpack() diff --git a/tarantool/types.py b/tarantool/types.py index 140baca9..5ca86150 100644 --- a/tarantool/types.py +++ b/tarantool/types.py @@ -56,7 +56,7 @@ class BoxError(): fields: typing.Optional[dict] = None """ Additional fields depending on error type. For example, if - :attr:`~tarantool.types.BoxError.type` is ``"AccessDeniedError"``, + :attr:`~tarantool.BoxError.type` is ``"AccessDeniedError"``, then it will include ``"object_type"``, ``"object_name"``, ``"access_type"``. """ @@ -106,3 +106,36 @@ def decode_box_error(err_map): prev = err return prev + +def encode_box_error(err): + """ + Encode Python `box.error`_ representation to MessagePack map. + + :param err: Error to encode + :type err: :obj:`tarantool.BoxError` + + :rtype: :obj:`dict` + + :raises: :exc:`KeyError` + """ + + stack = [] + + while err is not None: + dict_item = { + MP_ERROR_TYPE: err.type, + MP_ERROR_FILE: err.file, + MP_ERROR_LINE: err.line, + MP_ERROR_MESSAGE: err.message, + MP_ERROR_ERRNO: err.errno, + MP_ERROR_ERRCODE: err.errcode, + } + + if err.fields is not None: # omitted if empty + dict_item[MP_ERROR_FIELDS] = err.fields + + stack.append(dict_item) + + err = err.prev + + return {MP_ERROR_STACK: stack} diff --git a/test/suites/__init__.py b/test/suites/__init__.py index f825f2ac..aae5fe23 100644 --- a/test/suites/__init__.py +++ b/test/suites/__init__.py @@ -20,6 +20,7 @@ from .test_datetime import TestSuite_Datetime from .test_interval import TestSuite_Interval from .test_package import TestSuite_Package +from .test_error_ext import TestSuite_ErrorExt test_cases = (TestSuite_Schema_UnicodeConnection, TestSuite_Schema_BinaryConnection, @@ -27,7 +28,7 @@ TestSuite_Mesh, TestSuite_Execute, TestSuite_DBAPI, TestSuite_Encoding, TestSuite_Pool, TestSuite_Ssl, TestSuite_Decimal, TestSuite_UUID, TestSuite_Datetime, - TestSuite_Interval, TestSuite_Package,) + TestSuite_Interval, TestSuite_ErrorExt,) def load_tests(loader, tests, pattern): suite = unittest.TestSuite() diff --git a/test/suites/lib/skip.py b/test/suites/lib/skip.py index 1b63a55e..e111746e 100644 --- a/test/suites/lib/skip.py +++ b/test/suites/lib/skip.py @@ -165,3 +165,16 @@ def skip_or_run_error_extra_info_test(func): return skip_or_run_test_tarantool(func, '2.4.1', 'does not provide extra error info') + +def skip_or_run_error_ext_type_test(func): + """Decorator to skip or run tests related to error extension + type depending on the tarantool version. + + Tarantool supports error extension type only since 2.4.1 version, + yet encoding was introduced only in 2.10.0. + See https://github.com/tarantool/tarantool/issues/4398, + https://github.com/tarantool/tarantool/issues/6433 + """ + + return skip_or_run_test_tarantool(func, '2.10.0', + 'does not support error extension type') diff --git a/test/suites/test_error_ext.py b/test/suites/test_error_ext.py new file mode 100644 index 00000000..199e7b50 --- /dev/null +++ b/test/suites/test_error_ext.py @@ -0,0 +1,399 @@ +import sys +import unittest +import uuid +import msgpack +import warnings +import tarantool +import pkg_resources + +from tarantool.msgpack_ext.packer import default as packer_default +from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook +from tarantool.request import build_packer +from tarantool.response import build_unpacker + +from .lib.tarantool_server import TarantoolServer +from .lib.skip import skip_or_run_error_ext_type_test + +class TestSuite_ErrorExt(unittest.TestCase): + @classmethod + def setUpClass(self): + print(' ERROR EXT TYPE '.center(70, '='), file=sys.stderr) + print('-' * 70, file=sys.stderr) + self.srv = TarantoolServer() + self.srv.script = 'test/suites/box.lua' + self.srv.start() + + self.adm = self.srv.admin + self.adm(r""" + box.schema.space.create('test') + box.space['test']:create_index('primary', { + type = 'tree', + parts = {1, 'string'}, + unique = true}) + + box.schema.user.create('test', {password = 'test', if_not_exists = true}) + box.schema.user.grant('test', 'read,write,execute,create', 'universe') + + box.schema.user.create('no_grants', {if_not_exists = true}) + """) + + self.conn_encoding_utf8 = tarantool.Connection( + self.srv.host, self.srv.args['primary'], + user='test', password='test', + encoding='utf-8') + self.conn_encoding_none = tarantool.Connection( + self.srv.host, self.srv.args['primary'], + user='test', password='test', + encoding=None) + + if self.adm.tnt_version >= pkg_resources.parse_version('2.10.0'): + self.conn_encoding_utf8.eval(r""" + local err = box.error.new(box.error.UNKNOWN) + rawset(_G, 'simple_error', err) + """) + + # https://github.com/tarantool/tarantool/blob/125c13c81abb302708771ba04d59382d44a4a512/test/box-tap/extended_error.test.lua + self.conn_encoding_utf8.eval(r""" + local user = box.session.user() + box.schema.func.create('forbidden_function', {body = 'function() end'}) + box.session.su('no_grants') + _, access_denied_error = pcall(function() box.func.forbidden_function:call() end) + box.session.su(user) + rawset(_G, 'access_denied_error', access_denied_error) + """) + + # https://github.com/tarantool/tarantool/blob/125c13c81abb302708771ba04d59382d44a4a512/test/box-tap/extended_error.test.lua + self.conn_encoding_utf8.eval(r""" + local e1 = box.error.new(box.error.UNKNOWN) + local e2 = box.error.new(box.error.UNKNOWN) + e2:set_prev(e1) + rawset(_G, 'chained_error', e2) + """) + + def setUp(self): + # prevent a remote tarantool from clean our session + if self.srv.is_started(): + self.srv.touch_lock() + + self.adm("box.space['test']:truncate()") + + + # msgpack data for different encodings are actually the same, + # but sometimes python msgpack module use different string + # types (str8 and str16) for the same strings depending on use_bin_type: + # + # >>> msgpack.Packer(use_bin_type=True).pack('[string " local err = box.error.ne..."]') + # b'\xd9;[string " local err = box.error.ne..."]' + # >>> msgpack.Packer(use_bin_type=False).pack('[string " local err = box.error.ne..."]') + # b'\xda\x00;[string " local err = box.error.ne..."]' + + cases = { + 'simple_error_for_encoding_utf8': { + 'conn': 'conn_encoding_utf8', + 'str_type': str, + 'python': tarantool.BoxError( + type='ClientError', + file='eval', + line=1, + message='Unknown error', + errno=0, + errcode=0, + ), + 'msgpack': (b'\x81\x00\x91\x86\x00\xab\x43\x6c\x69\x65\x6e\x74' + + b'\x45\x72\x72\x6f\x72\x01\xa4\x65\x76\x61\x6c\x02' + + b'\x01\x03\xad\x55\x6e\x6b\x6e\x6f\x77\x6e\x20\x65' + + b'\x72\x72\x6f\x72\x04\x00\x05\x00'), + 'tarantool': "simple_error", + }, + 'simple_error_for_encoding_none': { + 'conn': 'conn_encoding_none', + 'str_type': lambda obj: bytes(obj, encoding='utf8'), + 'python': tarantool.BoxError( + type=b'ClientError', + file=b'eval', + line=1, + message=b'Unknown error', + errno=0, + errcode=0, + ), + 'msgpack': (b'\x81\x00\x91\x86\x00\xab\x43\x6c\x69\x65\x6e\x74' + + b'\x45\x72\x72\x6f\x72\x01\xa4\x65\x76\x61\x6c\x02' + + b'\x01\x03\xad\x55\x6e\x6b\x6e\x6f\x77\x6e\x20\x65' + + b'\x72\x72\x6f\x72\x04\x00\x05\x00'), + 'tarantool': "simple_error", + }, + 'error_with_fields_for_encoding_utf8': { + 'conn': 'conn_encoding_utf8', + 'str_type': str, + 'python': tarantool.BoxError( + type='AccessDeniedError', + file='/__w/sdk/sdk/tarantool-2.10/tarantool/src/box/func.c', + line=535, + message="Execute access to function 'forbidden_function' is denied for user 'no_grants'", + errno=0, + errcode=42, + fields={ + 'object_type': 'function', + 'object_name': 'forbidden_function', + 'access_type': 'Execute', + }, + ), + 'msgpack': (b'\x81\x00\x91\x87\x00\xb1\x41\x63\x63\x65\x73\x73' + + b'\x44\x65\x6e\x69\x65\x64\x45\x72\x72\x6f\x72\x01' + + b'\xd9\x34\x2f\x5f\x5f\x77\x2f\x73\x64\x6b\x2f\x73' + + b'\x64\x6b\x2f\x74\x61\x72\x61\x6e\x74\x6f\x6f\x6c' + + b'\x2d\x32\x2e\x31\x30\x2f\x74\x61\x72\x61\x6e\x74' + + b'\x6f\x6f\x6c\x2f\x73\x72\x63\x2f\x62\x6f\x78\x2f' + + b'\x66\x75\x6e\x63\x2e\x63\x02\xcd\x02\x17\x03\xd9' + + b'\x4e\x45\x78\x65\x63\x75\x74\x65\x20\x61\x63\x63' + + b'\x65\x73\x73\x20\x74\x6f\x20\x66\x75\x6e\x63\x74' + + b'\x69\x6f\x6e\x20\x27\x66\x6f\x72\x62\x69\x64\x64' + + b'\x65\x6e\x5f\x66\x75\x6e\x63\x74\x69\x6f\x6e\x27' + + b'\x20\x69\x73\x20\x64\x65\x6e\x69\x65\x64\x20\x66' + + b'\x6f\x72\x20\x75\x73\x65\x72\x20\x27\x6e\x6f\x5f' + + b'\x67\x72\x61\x6e\x74\x73\x27\x04\x00\x05\x2a\x06' + + b'\x83\xab\x6f\x62\x6a\x65\x63\x74\x5f\x74\x79\x70' + + b'\x65\xa8\x66\x75\x6e\x63\x74\x69\x6f\x6e\xab\x6f' + + b'\x62\x6a\x65\x63\x74\x5f\x6e\x61\x6d\x65\xb2\x66' + + b'\x6f\x72\x62\x69\x64\x64\x65\x6e\x5f\x66\x75\x6e' + + b'\x63\x74\x69\x6f\x6e\xab\x61\x63\x63\x65\x73\x73' + + b'\x5f\x74\x79\x70\x65\xa7\x45\x78\x65\x63\x75\x74' + + b'\x65'), + 'tarantool': "access_denied_error", + 'ignore_file_info': True, + }, + 'error_with_fields_for_encoding_none': { + 'conn': 'conn_encoding_none', + 'str_type': lambda obj: bytes(obj, encoding='utf8'), + 'python': tarantool.BoxError( + type=b'AccessDeniedError', + file=b'/__w/sdk/sdk/tarantool-2.10/tarantool/src/box/func.c', + line=535, + message=b"Execute access to function 'forbidden_function' is denied for user 'no_grants'", + errno=0, + errcode=42, + fields={ + b'object_type': b'function', + b'object_name': b'forbidden_function', + b'access_type': b'Execute', + }, + ), + 'msgpack': (b'\x81\x00\x91\x87\x00\xb1\x41\x63\x63\x65\x73\x73' + + b'\x44\x65\x6e\x69\x65\x64\x45\x72\x72\x6f\x72\x01' + + b'\xda\x00\x34\x2f\x5f\x5f\x77\x2f\x73\x64\x6b\x2f' + + b'\x73\x64\x6b\x2f\x74\x61\x72\x61\x6e\x74\x6f\x6f' + + b'\x6c\x2d\x32\x2e\x31\x30\x2f\x74\x61\x72\x61\x6e' + + b'\x74\x6f\x6f\x6c\x2f\x73\x72\x63\x2f\x62\x6f\x78' + + b'\x2f\x66\x75\x6e\x63\x2e\x63\x02\xcd\x02\x17\x03' + + b'\xda\x00\x4e\x45\x78\x65\x63\x75\x74\x65\x20\x61' + + b'\x63\x63\x65\x73\x73\x20\x74\x6f\x20\x66\x75\x6e' + + b'\x63\x74\x69\x6f\x6e\x20\x27\x66\x6f\x72\x62\x69' + + b'\x64\x64\x65\x6e\x5f\x66\x75\x6e\x63\x74\x69\x6f' + + b'\x6e\x27\x20\x69\x73\x20\x64\x65\x6e\x69\x65\x64' + + b'\x20\x66\x6f\x72\x20\x75\x73\x65\x72\x20\x27\x6e' + + b'\x6f\x5f\x67\x72\x61\x6e\x74\x73\x27\x04\x00\x05' + + b'\x2a\x06\x83\xab\x6f\x62\x6a\x65\x63\x74\x5f\x74' + + b'\x79\x70\x65\xa8\x66\x75\x6e\x63\x74\x69\x6f\x6e' + + b'\xab\x6f\x62\x6a\x65\x63\x74\x5f\x6e\x61\x6d\x65' + + b'\xb2\x66\x6f\x72\x62\x69\x64\x64\x65\x6e\x5f\x66' + + b'\x75\x6e\x63\x74\x69\x6f\x6e\xab\x61\x63\x63\x65' + + b'\x73\x73\x5f\x74\x79\x70\x65\xa7\x45\x78\x65\x63' + + b'\x75\x74\x65'), + 'tarantool': "access_denied_error", + 'ignore_file_info': True, + }, + 'error_chain_for_encoding_utf8': { + 'conn': 'conn_encoding_utf8', + 'str_type': str, + 'python': tarantool.BoxError( + type='ClientError', + file='eval', + line=3, + message='Unknown error', + errno=0, + errcode=0, + prev=tarantool.BoxError( + type='ClientError', + file='eval', + line=2, + message='Unknown error', + errno=0, + errcode=0, + ), + ), + 'msgpack': (b'\x81\x00\x92\x86\x00\xab\x43\x6c\x69\x65\x6e\x74' + + b'\x45\x72\x72\x6f\x72\x01\xa4\x65\x76\x61\x6c\x02' + + b'\x03\x03\xad\x55\x6e\x6b\x6e\x6f\x77\x6e\x20\x65' + + b'\x72\x72\x6f\x72\x04\x00\x05\x00\x86\x00\xab\x43' + + b'\x6c\x69\x65\x6e\x74\x45\x72\x72\x6f\x72\x01\xa4' + + b'\x65\x76\x61\x6c\x02\x02\x03\xad\x55\x6e\x6b\x6e' + + b'\x6f\x77\x6e\x20\x65\x72\x72\x6f\x72\x04\x00\x05\x00'), + 'tarantool': "chained_error", + 'ignore_file_info': False, + }, + 'error_chain_for_encoding_none': { + 'conn': 'conn_encoding_none', + 'str_type': lambda obj: bytes(obj, encoding='utf8'), + 'python': tarantool.BoxError( + type=b'ClientError', + file=b'eval', + line=3, + message=b'Unknown error', + errno=0, + errcode=0, + prev=tarantool.BoxError( + type=b'ClientError', + file=b'eval', + line=2, + message=b'Unknown error', + errno=0, + errcode=0, + ), + ), + 'msgpack': (b'\x81\x00\x92\x86\x00\xab\x43\x6c\x69\x65\x6e\x74' + + b'\x45\x72\x72\x6f\x72\x01\xa4\x65\x76\x61\x6c\x02' + + b'\x03\x03\xad\x55\x6e\x6b\x6e\x6f\x77\x6e\x20\x65' + + b'\x72\x72\x6f\x72\x04\x00\x05\x00\x86\x00\xab\x43' + + b'\x6c\x69\x65\x6e\x74\x45\x72\x72\x6f\x72\x01\xa4' + + b'\x65\x76\x61\x6c\x02\x02\x03\xad\x55\x6e\x6b\x6e' + + b'\x6f\x77\x6e\x20\x65\x72\x72\x6f\x72\x04\x00\x05\x00'), + 'tarantool': "chained_error", + 'ignore_file_info': False, + } + } + + + def test_msgpack_decode(self): + for name in self.cases.keys(): + with self.subTest(msg=name): + case = self.cases[name] + conn = getattr(self, case['conn']) + + self.assertEqual( + unpacker_ext_hook( + 3, + case['msgpack'], + build_unpacker(conn) + ), + case['python']) + + @skip_or_run_error_ext_type_test + def test_tarantool_decode(self): + for name in self.cases.keys(): + with self.subTest(msg=name): + case = self.cases[name] + conn = getattr(self, case['conn']) + + self.adm(f""" + local err = rawget(_G, '{case['tarantool']}') + box.space['test']:replace{{'{name}', err, 'payload'}} + """) + + res = conn.select('test', case['str_type'](name)) + self.assertEqual(len(res), 1) + + # Tarantool error file and line could differ even between + # different patches. + # + # Also, in Tarantool errors are not comparable at all. + # + # tarantool> msgpack.decode(error_str) == msgpack.decode(error_str) + # --- + # - false + # ... + + self.assertEqual(res[0][0], case['str_type'](name)) + self.assertEqual(res[0][2], case['str_type']('payload')) + + err = res[0][1] + self.assertTrue( + isinstance(err, tarantool.BoxError), + f'{err} is expected to be a BoxError object') + + expected_err = case['python'] + while err is not None: + self.assertEqual(err.type, expected_err.type) + self.assertEqual(err.message, expected_err.message) + self.assertEqual(err.errno, expected_err.errno) + self.assertEqual(err.errcode, expected_err.errcode) + self.assertEqual(err.fields, expected_err.fields) + + err = err.prev + expected_err = expected_err.prev + + self.assertEqual(err, expected_err) + + + def test_msgpack_encode(self): + for name in self.cases.keys(): + with self.subTest(msg=name): + case = self.cases[name] + conn = getattr(self, case['conn']) + + self.assertEqual(packer_default(case['python'], build_packer(conn)), + msgpack.ExtType(code=3, data=case['msgpack'])) + + @skip_or_run_error_ext_type_test + def test_tarantool_encode(self): + for name in self.cases.keys(): + with self.subTest(msg=name): + case = self.cases[name] + conn = getattr(self, case['conn']) + + conn.insert( + 'test', + [case['str_type'](name), case['python'], case['str_type']('payload')]) + + lua_eval = f""" + local err = rawget(_G, '{case['tarantool']}') + + local tuple = box.space['test']:get('{name}') + assert(tuple ~= nil) + + local tuple_err = tuple[2] + + local fields = {{'type', 'message', 'errno', 'errcode', 'fields'}} + + local json = require('json') + + local function compare_errors(err1, err2) + if (err1 == nil) and (err2 ~= nil) then + return nil, ('Test error stack is empty, but expected error ' .. + 'has previous %s (%s) error'):format( + err2.type, err2.message) + end + + if (err1 ~= nil) and (err2 == nil) then + return nil, ('Expected error stack is empty, but test error ' .. + 'has previous %s (%s) error'):format( + err1.type, err1.message) + end + + for _, field in ipairs(fields) do + if json.encode(err1[field]) ~= json.encode(err2[field]) then + return nil, ('%s %s is not equal to expected %s'):format( + field, + json.encode(err1[field]), + json.encode(err2[field])) + end + end + + if (err1.prev ~= nil) or (err2.prev ~= nil) then + return compare_errors(err1.prev, err2.prev) + end + + return true + end + + return compare_errors(tuple_err, err) + """ + + self.assertSequenceEqual(conn.eval(lua_eval), [True]) + + + @classmethod + def tearDownClass(self): + self.conn_encoding_utf8.close() + self.conn_encoding_none.close() + self.srv.stop() + self.srv.clean() diff --git a/test/suites/test_interval.py b/test/suites/test_interval.py index 2de70a11..a3458ad7 100644 --- a/test/suites/test_interval.py +++ b/test/suites/test_interval.py @@ -9,6 +9,7 @@ from tarantool.msgpack_ext.packer import default as packer_default from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook +from tarantool.response import build_unpacker from .lib.tarantool_server import TarantoolServer from .lib.skip import skip_or_run_datetime_test @@ -150,7 +151,11 @@ def test_msgpack_decode(self): with self.subTest(msg=name): case = self.cases[name] - self.assertEqual(unpacker_ext_hook(6, case['msgpack']), + self.assertEqual(unpacker_ext_hook( + 6, + case['msgpack'], + build_unpacker(self.con), + ), case['python']) @skip_or_run_datetime_test @@ -201,13 +206,13 @@ def test_unknown_field_decode(self): case = b'\x01\x09\xce\x00\x98\x96\x80' self.assertRaisesRegex( MsgpackError, 'Unknown interval field id 9', - lambda: unpacker_ext_hook(6, case)) + lambda: unpacker_ext_hook(6, case, build_unpacker(self.con))) def test_unknown_adjust_decode(self): case = b'\x02\x07\xce\x00\x98\x96\x80\x08\x03' self.assertRaisesRegex( MsgpackError, '3 is not a valid Adjust', - lambda: unpacker_ext_hook(6, case)) + lambda: unpacker_ext_hook(6, case, build_unpacker(self.con))) arithmetic_cases = { diff --git a/test/suites/test_protocol.py b/test/suites/test_protocol.py index 61ac3cd8..f1902afc 100644 --- a/test/suites/test_protocol.py +++ b/test/suites/test_protocol.py @@ -78,12 +78,13 @@ def test_04_protocol(self): # Tarantool 2.10.3 still has version 3. if self.adm.tnt_version >= pkg_resources.parse_version('2.10.0'): self.assertTrue(self.con._protocol_version >= 3) + self.assertEqual(self.con._features[IPROTO_FEATURE_ERROR_EXTENSION], True) else: self.assertIsNone(self.con._protocol_version) + self.assertEqual(self.con._features[IPROTO_FEATURE_ERROR_EXTENSION], False) self.assertEqual(self.con._features[IPROTO_FEATURE_STREAMS], False) self.assertEqual(self.con._features[IPROTO_FEATURE_TRANSACTIONS], False) - self.assertEqual(self.con._features[IPROTO_FEATURE_ERROR_EXTENSION], False) self.assertEqual(self.con._features[IPROTO_FEATURE_WATCHERS], False) self.assertEqual(self.con._features[IPROTO_FEATURE_GRACEFUL_SHUTDOWN], False)