diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fcacc57..61d42e9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed * Optimize `crud.select()` without conditions and with `after`. +* Behaviour of potentially long `select` and `count` calls: a critical log entry + containing the current stack traceback is created upon such function calls — + an user can explicitly request a full scan through by passing `fullscan=true` + to `select` or `count` options table argument in which a case a log entry will + not be created (#276). ### Fixed * `crud.select()` if a condition is '<=' and it's value < `after`. diff --git a/README.md b/README.md index 0cbcce9e..fe4c6db4 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ It can be used to convert received tuples to objects via `crud.unflatten_rows` f For example: ```lua -res, err = crud.select('customers') +res, err = crud.select('customers', nil, {first = 2}) res --- - metadata: @@ -450,6 +450,8 @@ where: if full primary key equal condition is specified * `timeout` (`?number`) - `vshard.call` timeout (in seconds) * `fields` (`?table`) - field names for getting only a subset of fields + * `fullscan` (`?boolean`) - if `true` then a critical log entry will be skipped + on potentially long `select`, see [avoiding full scan](doc/select.md#avoiding-full-scan). * `mode` (`?string`, `read` or `write`) - if `write` is specified then `select` is performed on master * `prefer_replica` (`?boolean`) - if `true` then the preferred target is one of @@ -472,7 +474,7 @@ Each condition is a table `{operator, field-identifier, value}`: **Example:** ```lua -crud.select('customers', {{'<=', 'age', 35}}) +crud.select('customers', {{'<=', 'age', 35}}, {first = 10}) --- - metadata: - {'name': 'id', 'type': 'unsigned'} @@ -496,8 +498,10 @@ See more examples of select queries [here.](https://github.com/tarantool/crud/bl ### Pairs You can iterate across a distributed space using the `crud.pairs` function. -Its arguments are the same as [`crud.select`](#select) arguments, -but negative `first` values aren't allowed. +Its arguments are the same as [`crud.select`](#select) arguments except +`fullscan` (it does not exist because `crud.pairs` does not generate a critical +log entry on potentially long requests) and negative `first` values aren't +allowed. User could pass use_tomap flag (false by default) to iterate over flat tuples or objects. **Example:** @@ -597,7 +601,7 @@ Returns true or nil with error. **Example:** ```lua -#crud.select('customers', {{'<=', 'age', 35}}) +#crud.select('customers', {{'<=', 'age', 35}}, {first = 10}) --- - 1 ... @@ -605,7 +609,7 @@ crud.truncate('customers', {timeout = 2}) --- - true ... -#crud.select('customers', {{'<=', 'age', 35}}) +#crud.select('customers', {{'<=', 'age', 35}}, {first = 10}) --- - 0 ... @@ -633,7 +637,7 @@ Returns number or nil with error. Using `memtx`: ```lua -#crud.select('customers') +#crud.select('customers', nil, {fullscan = true}) --- - 5 ... @@ -686,6 +690,8 @@ where: * `force_map_call` (`?boolean`) - if `true` then the map call is performed without any optimizations even, default value is `false` + * `fullscan` (`?boolean`) - if `true` then a critical log entry will be skipped + on potentially long `count`, see [avoiding full scan](doc/select.md#avoiding-full-scan). * `mode` (`?string`, `read` or `write`) - if `write` is specified then `count` is performed on master, default value is `read` * `prefer_replica` (`?boolean`) - if `true` then the preferred target is one of @@ -695,9 +701,9 @@ where: default value is `false` ```lua -crud.count('customers', {{'<=', 'age', 35}}) +crud.count('customers', {{'==', 'age', 35}}) --- -- 5 +- 1 ... ``` diff --git a/crud/count.lua b/crud/count.lua index b13514b4..6e35ac6e 100644 --- a/crud/count.lua +++ b/crud/count.lua @@ -10,6 +10,7 @@ local sharding = require('crud.common.sharding') local filters = require('crud.compare.filters') local count_plan = require('crud.compare.plan') local dev_checks = require('crud.common.dev_checks') +local ratelimit = require('crud.ratelimit') local schema = require('crud.common.schema') local sharding_metadata_module = require('crud.common.sharding.sharding_metadata') @@ -90,6 +91,22 @@ function count.init() _G._crud.count_on_storage = count_on_storage end +local check_count_safety_rl = ratelimit.new() +local function check_count_safety(space_name, plan, opts) + if opts.fullscan == true then + return + end + + local iter = plan.tarantool_iter + if iter == box.index.EQ or iter == box.index.REQ then + return + end + + local rl = check_count_safety_rl + local traceback = debug.traceback() + rl:log_crit("Potentially long count from space '%s'\n %s", space_name, traceback) +end + -- returns result, err, need_reload -- need_reload indicates if reloading schema could help -- see crud.common.schema.wrap_func_reload() @@ -98,6 +115,7 @@ local function call_count_on_router(space_name, user_conditions, opts) timeout = '?number', bucket_id = '?number|cdata', force_map_call = '?boolean', + fullscan = '?boolean', yield_every = '?number', prefer_replica = '?boolean', balance = '?boolean', @@ -147,6 +165,7 @@ local function call_count_on_router(space_name, user_conditions, opts) if err ~= nil then return nil, CountError:new("Failed to plan count: %s", err), const.NEED_SCHEMA_RELOAD end + check_count_safety(space_name, plan, opts) -- set replicasets to count from local replicasets_to_count = replicasets @@ -297,6 +316,7 @@ function count.call(space_name, user_conditions, opts) timeout = '?number', bucket_id = '?number|cdata', force_map_call = '?boolean', + fullscan = '?boolean', yield_every = '?number', prefer_replica = '?boolean', balance = '?boolean', diff --git a/crud/ratelimit.lua b/crud/ratelimit.lua new file mode 100644 index 00000000..e39f0a63 --- /dev/null +++ b/crud/ratelimit.lua @@ -0,0 +1,125 @@ +-- Mostly it's a copy-paste from tarantool/tarantool log.lua: +-- https://github.com/tarantool/tarantool/blob/29654ffe3638e5a218dd32f1788830ff05c1c05c/src/lua/log.lua +-- +-- We have three reasons for the copy-paste: +-- 1. Tarantool has not log.crit() (a function for logging with CRIT level). +-- 2. Only new versions of Tarantool have Ratelimit type. +-- 3. We want own copy of Ratelimit in case the implementation in Tarantool +-- changes. Less pain between Tarantool versions. +local ffi = require('ffi') + +local S_CRIT = ffi.C.S_CRIT +local S_WARN = ffi.C.S_WARN + +local function say(level, fmt, ...) + if ffi.C.log_level < level then + -- don't waste cycles on debug.getinfo() + return + end + local type_fmt = type(fmt) + local format = "%s" + if select('#', ...) ~= 0 then + local stat + stat, fmt = pcall(string.format, fmt, ...) + if not stat then + error(fmt, 3) + end + elseif type_fmt == 'table' then + -- An implementation in tarantool/tarantool supports encoding a table in + -- JSON, but it requires more dependencies from FFI. So we just deleted + -- it because we don't need such encoding in the module. + error("table not supported", 3) + elseif type_fmt ~= 'string' then + fmt = tostring(fmt) + end + + local debug = require('debug') + local frame = debug.getinfo(3, "Sl") + local line, file = 0, 'eval' + if type(frame) == 'table' then + line = frame.currentline or 0 + file = frame.short_src or frame.src or 'eval' + end + + ffi.C._say(level, file, line, nil, format, fmt) +end + +local ratelimit_enabled = true + +local function ratelimit_enable() + ratelimit_enabled = true +end + +local function ratelimit_disable() + ratelimit_enabled = false +end + +local Ratelimit = { + interval = 60, + burst = 10, + emitted = 0, + suppressed = 0, + start = 0, +} + +local function ratelimit_new(object) + return Ratelimit:new(object) +end + +function Ratelimit:new(object) + object = object or {} + setmetatable(object, self) + self.__index = self + return object +end + +function Ratelimit:check() + if not ratelimit_enabled then + return 0, true + end + + local clock = require('clock') + local now = clock.monotonic() + local saved_suppressed = 0 + if now > self.start + self.interval then + saved_suppressed = self.suppressed + self.suppressed = 0 + self.emitted = 0 + self.start = now + end + + if self.emitted < self.burst then + self.emitted = self.emitted + 1 + return saved_suppressed, true + end + self.suppressed = self.suppressed + 1 + return saved_suppressed, false +end + +function Ratelimit:log_check(lvl) + local suppressed, ok = self:check() + if lvl >= S_WARN and suppressed > 0 then + say(S_WARN, '%d messages suppressed due to rate limiting', suppressed) + end + return ok +end + +function Ratelimit:log(lvl, fmt, ...) + if self:log_check(lvl) then + say(lvl, fmt, ...) + end +end + +local function log_ratelimited_closure(lvl) + return function(self, fmt, ...) + self:log(lvl, fmt, ...) + end +end + +Ratelimit.log_crit = log_ratelimited_closure(S_CRIT) + +return { + new = ratelimit_new, + enable = ratelimit_enable, + disable = ratelimit_disable, +} diff --git a/crud/select/compat/common.lua b/crud/select/compat/common.lua index d74e9cc8..50a66d0d 100644 --- a/crud/select/compat/common.lua +++ b/crud/select/compat/common.lua @@ -1,6 +1,28 @@ +local ratelimit = require('crud.ratelimit') +local check_select_safety_rl = ratelimit.new() + local common = {} common.SELECT_FUNC_NAME = '_crud.select_on_storage' common.DEFAULT_BATCH_SIZE = 100 +common.check_select_safety = function(space_name, plan, opts) + if opts.fullscan == true then + return + end + + if opts.first ~= nil and math.abs(opts.first) <= 1000 then + return + end + + local iter = plan.tarantool_iter + if iter == box.index.EQ or iter == box.index.REQ then + return + end + + local rl = check_select_safety_rl + local traceback = debug.traceback() + rl:log_crit("Potentially long select from space '%s'\n %s", space_name, traceback) +end + return common diff --git a/crud/select/compat/select.lua b/crud/select/compat/select.lua index be500f5d..4f9964ed 100644 --- a/crud/select/compat/select.lua +++ b/crud/select/compat/select.lua @@ -180,6 +180,7 @@ local function build_select_iterator(space_name, user_conditions, opts) return { tuples_limit = tuples_limit, merger = merger, + plan = plan, space_format = filtered_space_format, } end @@ -252,14 +253,16 @@ local function select_module_call_xc(space_name, user_conditions, opts) checks('string', '?table', { after = '?table|cdata', first = '?number', - timeout = '?number', batch_size = '?number', bucket_id = '?number|cdata', force_map_call = '?boolean', fields = '?table', + fullscan = '?boolean', + + mode = '?vshard_call_mode', prefer_replica = '?boolean', balance = '?boolean', - mode = '?vshard_call_mode', + timeout = '?number', }) opts = opts or {} @@ -292,6 +295,7 @@ local function select_module_call_xc(space_name, user_conditions, opts) if err ~= nil then return nil, err end + common.check_select_safety(space_name, iter.plan, opts) local tuples = {} diff --git a/crud/select/compat/select_old.lua b/crud/select/compat/select_old.lua index b5f82b94..4d054b3e 100644 --- a/crud/select/compat/select_old.lua +++ b/crud/select/compat/select_old.lua @@ -332,6 +332,7 @@ local function select_module_call_xc(space_name, user_conditions, opts) if err ~= nil then return nil, err end + common.check_select_safety(space_name, iter.plan, opts) local tuples = {} @@ -366,14 +367,16 @@ function select_module.call(space_name, user_conditions, opts) checks('string', '?table', { after = '?table', first = '?number', - timeout = '?number', batch_size = '?number', bucket_id = '?number|cdata', force_map_call = '?boolean', fields = '?table', + fullscan = '?boolean', + + mode = '?vshard_call_mode', prefer_replica = '?boolean', balance = '?boolean', - mode = '?vshard_call_mode', + timeout = '?number', }) return sharding.wrap_method(select_module_call_xc, space_name, user_conditions, opts) diff --git a/doc/pairs.md b/doc/pairs.md index 40f8699e..861edb87 100644 --- a/doc/pairs.md +++ b/doc/pairs.md @@ -1,7 +1,8 @@ # Pairs examples With ``crud.pairs``, you can iterate across a distributed space. -The arguments are the same as [``crud.select``](https://github.com/tarantool/crud/blob/master/doc/select.md), except of the ``use_tomap`` parameter. +The arguments are the same as [``crud.select``](https://github.com/tarantool/crud/blob/master/doc/select.md) arguments except ``fullscan`` (it does not exist because ``crud.pairs`` does not generate a critical log entry on potentially long requests) and negative ``first`` values aren't allowed. +User could pass ``use_tomap`` flag (false by default) to iterate over flat tuples or objects. Below are examples that may help you. Examples schema is similar to the [select documentation](select.md/#examples-space-format) diff --git a/doc/select.md b/doc/select.md index 3fc84faa..ed8dec37 100644 --- a/doc/select.md +++ b/doc/select.md @@ -24,7 +24,7 @@ ## Filtering -``CRUD`` allows to filter tuples by conditions. Each condition can use field name (or number) or index name. The first condition that uses index name is used to iterate over space. If there is no conditions that match index names, full scan is performed. Other conditions are used as additional filters. Search condition for the indexed field must be placed first to avoid a full scan. +``CRUD`` allows to filter tuples by conditions. Each condition can use field name (or number) or index name. The first condition that uses index name is used to iterate over space. If there is no conditions that match index names, full scan is performed. Other conditions are used as additional filters. Search condition for the indexed field must be placed first to avoid a full scan. In additional, don't forget to limit amount of results with ``first`` parameter. This will help to avoid too long selects in production. **Note:** If you specify sharding key or ``bucket_id`` select will be performed on single node. Otherwise Map-Reduce over all nodes will be occurred. @@ -107,7 +107,7 @@ We have an ``age_index`` index. Example below gets a list of ``customers`` over **Example:** ```lua -crud.select('developers', {{'>=', 'age_index', 30}}) +crud.select('developers', {{'>=', 'age_index', 30}}, {first = 10}) --- - metadata: - {'name': 'id', 'type': 'unsigned'} @@ -130,8 +130,8 @@ field match, the search will also be performed using index. These two queries are equivalent, the search will be done using index in both cases: ```lua -crud.select('developers', {{'>=', 'age_index', 30}}) -crud.select('developers', {{'>=', 'age', 30}}) +crud.select('developers', {{'>=', 'age_index', 30}}, {first = 10}) +crud.select('developers', {{'>=', 'age', 30}}, {first = 10}) ``` ### Select using composite index @@ -141,7 +141,7 @@ Suppose we have a composite index consisting of the ``name`` and ``surname`` fie **Example**: ```lua -crud.select('developers', {{'==', 'full_name', {"Alexey", "Adams"}}}) +crud.select('developers', {{'==', 'full_name', {"Alexey", "Adams"}}}, {first = 10}) --- - metadata: - {'name': 'id', 'type': 'unsigned'} @@ -161,7 +161,7 @@ Alternatively, you can use a partial key for a composite index. **Example**: ```lua -crud.select('developers', {{'==', 'full_name', "Alexey"}}) +crud.select('developers', {{'==', 'full_name', "Alexey"}}, {first = 10}) --- - metadata: - {'name': 'id', 'type': 'unsigned'} @@ -184,7 +184,7 @@ You can also make a selection using a non-indexed field. **Example:** ```lua -crud.select('developers', {{'==', 'surname', "Adams"}}) +crud.select('developers', {{'==', 'surname', "Adams"}}, {first = 10}) --- - metadata: - {'name': 'id', 'type': 'unsigned'} @@ -202,11 +202,92 @@ crud.select('developers', {{'==', 'surname', "Adams"}}) ### Avoiding full scan +Most requests lead to a full scan. A critical log entry containing the current stack traceback is created upon such calls with a message: `Potentially long select from space '%space_name%'`. + **Example:** ```lua crud.select('developers', {{'==', 'surname', "Adams"}, {'>=', 'age', 25}}) --- +2022-05-24 14:06:31.748 [25108] main/103/playground.lua C> Potentially long select from space 'developers' + stack traceback: + .rocks/share/tarantool/crud/select/compat/common.lua:24: in function 'check_select_safety' + .rocks/share/tarantool/crud/select/compat/select.lua:298: in function <.rocks/share/tarantool/crud/select/compat/select.lua:252> + [C]: in function 'pcall' + .rocks/share/tarantool/crud/common/sharding/init.lua:163: in function <.rocks/share/tarantool/crud/common/sharding/init.lua:158> + [C]: in function 'xpcall' + .rocks/share/tarantool/errors.lua:145: in function <.rocks/share/tarantool/errors.lua:139> + [C]: in function 'pcall' + builtin/box/console.lua:403: in function 'eval' + builtin/box/console.lua:709: in function 'repl' + builtin/box/console.lua:758: in function 'start' + doc/playground.lua:164: in main chunk +- metadata: + - {'name': 'id', 'type': 'unsigned'} + - {'name': 'bucket_id', 'type': 'unsigned'} + - {'name': 'name', 'type': 'string'} + - {'name': 'surname', 'type': 'string'} + - {'name': 'age', 'type': 'number'} + rows: + - [3, 9661, 'Pavel', 'Adams', 27] +... +``` + +You can avoid the full scan with '=' or '==' condition on index fields. + +**Example:** + +```lua +crud.select('developers', {{'==', 'surname', "Adams"}, {'==', 'age', 27}}) +--- +- metadata: + - {'name': 'id', 'type': 'unsigned'} + - {'name': 'bucket_id', 'type': 'unsigned'} + - {'name': 'name', 'type': 'string'} + - {'name': 'surname', 'type': 'string'} + - {'name': 'age', 'type': 'number'} + rows: + - [3, 9661, 'Pavel', 'Adams', 27] +... +``` + +The order of conditions is important. A first condition on index fields determines whether a full scan will be performed or not. + +**Example:** + +```lua +crud.select('developers', {{'==', 'surname', "Adams"}, {'>', 'id', 0}, {'==', 'age', 27}}) +2022-05-24 14:07:26.289 [29561] main/103/playground.lua C> Potentially long select from space 'developers' + stack traceback: + .rocks/share/tarantool/crud/select/compat/common.lua:24: in function 'check_select_safety' + .rocks/share/tarantool/crud/select/compat/select.lua:298: in function <.rocks/share/tarantool/crud/select/compat/select.lua:252> + [C]: in function 'pcall' + .rocks/share/tarantool/crud/common/sharding/init.lua:163: in function <.rocks/share/tarantool/crud/common/sharding/init.lua:158> + [C]: in function 'xpcall' + .rocks/share/tarantool/errors.lua:145: in function <.rocks/share/tarantool/errors.lua:139> + [C]: in function 'pcall' + builtin/box/console.lua:403: in function 'eval' + builtin/box/console.lua:709: in function 'repl' + builtin/box/console.lua:758: in function 'start' + doc/playground.lua:164: in main chunk +- metadata: + - {'name': 'id', 'type': 'unsigned'} + - {'name': 'bucket_id', 'type': 'unsigned'} + - {'name': 'name', 'type': 'string'} + - {'name': 'surname', 'type': 'string'} + - {'name': 'age', 'type': 'number'} + rows: + - [3, 9661, 'Pavel', 'Adams', 27] +... +``` + +Also, you can avoid the critical message with parameter ``first`` <= 1000. + +**Example:** + +```lua +crud.select('developers', {{'==', 'surname', "Adams"}, {'>=', 'age', 25}}, {first = 10}) +--- - metadata: - {'name': 'id', 'type': 'unsigned'} - {'name': 'bucket_id', 'type': 'unsigned'} @@ -218,12 +299,12 @@ crud.select('developers', {{'==', 'surname', "Adams"}, {'>=', 'age', 25}}) ... ``` -In this case, a full scan will be performed, since non-indexed field is placed first in search conditions. Example below shows how you can avoid a full scan. +Or you can do it with parameter ``fullscan=true`` if you know what you're doing (a small space). **Example:** ```lua -crud.select('developers', {{'>=', 'age', 30}, {'==', 'surname', "Adams"}}) +crud.select('developers', {{'==', 'surname', "Adams"}, {'>=', 'age', 25}}, {fullscan = true}) --- - metadata: - {'name': 'id', 'type': 'unsigned'} @@ -271,7 +352,7 @@ Using ``after``, we can get the objects after specified tuple. **Example:** ```lua -res, err = crud.select('developers', nil, { after = res.rows[3] }) +res, err = crud.select('developers', nil, { after = res.rows[3], first = 5 }) res --- - metadata: @@ -398,7 +479,7 @@ format - {'name': 'age', 'type': 'number'} ... -- get names of users that are 27 years old or older -res, err = crud.select('developers', {{'>=', 'age', 27}}, { fields = {'id', 'name'} }) +res, err = crud.select('developers', {{'>=', 'age', 27}}, { fields = {'id', 'name'}, first = 10 }) res - metadata: - {'name': 'id', 'type': 'unsigned'} @@ -426,7 +507,7 @@ format - {'name': 'age', 'type': 'number'} ... -- get names of users that are 27 years old or older -res, err = crud.select('developers', {{'>=', 'age', 27}}, { fields = {'id', 'name'} }) +res, err = crud.select('developers', {{'>=', 'age', 27}}, { fields = {'id', 'name'}, first = 10 }) res - metadata: - {'name': 'id', 'type': 'unsigned'} @@ -438,7 +519,7 @@ res - [4, 'Mikhail', 51] ... -- get names of users that are 27 years old or older -res, err = crud.select('developers', {{'>=', 'age', 27}}, { fields = {'id', 'name'}, after = res.rows[1] }) +res, err = crud.select('developers', {{'>=', 'age', 27}}, { fields = {'id', 'name'}, after = res.rows[1], first = 10 }) res - metadata: - {'name': 'id', 'type': 'unsigned'} @@ -452,12 +533,12 @@ res **THIS WOULD FAIL** ```lua -- 'fields' isn't specified -res, err = crud.select('developers', {{'>=', 'age', 27}}) +res, err = crud.select('developers', {{'>=', 'age', 27}}, {first = 10}) -- THIS WOULD FAIL -- call 'select' with 'fields' option specified -- and pass to 'after' tuple that were got without 'fields' option -res, err = crud.select('developers', {{'>=', 'age', 27}}, { fields = {'id', 'name'}, after = res.rows[1] }) +res, err = crud.select('developers', {{'>=', 'age', 27}}, { fields = {'id', 'name'}, after = res.rows[1], first = 10 }) ``` You could use `crud.cut_rows` function to cut off scan key and primary key values that were merged to the result fields. @@ -465,7 +546,7 @@ You could use `crud.cut_rows` function to cut off scan key and primary key value ```lua -- get names of users that are 27 years old or older -res, err = crud.select('developers', {{'>=', 'age', 27}}, { fields = {'id', 'name'} }) +res, err = crud.select('developers', {{'>=', 'age', 27}}, { fields = {'id', 'name'}, first = 10 }) res - metadata: - {'name': 'id', 'type': 'unsigned'} diff --git a/test/helper.lua b/test/helper.lua index b6a39d41..46743999 100644 --- a/test/helper.lua +++ b/test/helper.lua @@ -509,4 +509,17 @@ function helpers.assert_timeout_error(value, message) error(err, 2) end +function helpers.fflush_main_server_stdout(cluster, capture) + -- Sometimes we have a delay here. This hack helps to wait for the end of + -- the output. It shouldn't take much time. + cluster.main_server.net_box:eval([[ + require('log').error("crud fflush stdout message") + ]]) + local captured = "" + while not string.find(captured, "crud fflush stdout message", 1, true) do + captured = captured .. (capture:flush().stdout or "") + end + return captured +end + return helpers diff --git a/test/integration/count_test.lua b/test/integration/count_test.lua index 008ea788..568ce7b9 100644 --- a/test/integration/count_test.lua +++ b/test/integration/count_test.lua @@ -2,6 +2,7 @@ local fio = require('fio') local clock = require('clock') local t = require('luatest') +local luatest_capture = require('luatest.capture') local helpers = require('test.helper') @@ -26,6 +27,9 @@ pgroup.before_all(function(g) g.cluster:server('router').net_box:eval([[ require('crud').cfg{ stats = true } ]]) + g.cluster:server('router').net_box:eval([[ + require('crud.ratelimit').disable() + ]]) end) pgroup.after_all(function(g) helpers.stop_cluster(g.cluster) end) @@ -37,14 +41,14 @@ pgroup.before_each(function(g) end) pgroup.test_count_non_existent_space = function(g) - local result, err = g.cluster.main_server.net_box:call('crud.count', {'non_existent_space'}) + local result, err = g.cluster.main_server.net_box:call('crud.count', {'non_existent_space', nil, {fullscan = true}}) t.assert_equals(result, nil) t.assert_str_contains(err.err, "Space \"non_existent_space\" doesn't exist") end pgroup.test_count_empty_space = function(g) - local result, err = g.cluster.main_server.net_box:call('crud.count', {'customers'}) + local result, err = g.cluster.main_server.net_box:call('crud.count', {'customers', nil, {fullscan = true}}) t.assert_equals(err, nil) t.assert_equals(result, 0) @@ -69,7 +73,7 @@ pgroup.test_not_valid_operation = function(g) } local result, err = g.cluster.main_server.net_box:call('crud.count', - {'customers', conditions} + {'customers', conditions, {fullscan = true}} ) t.assert_equals(result, nil) @@ -89,6 +93,79 @@ pgroup.test_conditions_with_non_existed_field = function(g) t.assert_equals(result, 0) end + +local count_safety_cases = { + nil_and_nil_opts = { + has_crit = true, + user_conditions = nil, + opts = nil, + }, + fullscan_false = { + has_crit = true, + user_conditions = nil, + opts = {fullscan = false}, + }, + fullscan_true = { + has_crit = false, + user_conditions = nil, + opts = {fullscan = true}, + }, + non_equal_conditions = { + has_crit = true, + user_conditions = { + {'>=', 'last_name', 'A'}, + {'<=', 'last_name', 'Z'}, + {'>', 'age', 20}, + {'<', 'age', 30}, + }, + opts = nil, + }, + equal_condition = { + has_crit = false, + user_conditions = { + {'>=', 'last_name', 'A'}, + {'<=', 'last_name', 'Z'}, + {'=', 'age', 25}, + }, + opts = nil, + }, + equal_condition2 = { + has_crit = false, + user_conditions = { + {'>=', 'last_name', 'A'}, + {'<=', 'last_name', 'Z'}, + {'==', 'age', 25}, + }, + opts = nil, + }, +} + +for name, case in pairs(count_safety_cases) do + local space = 'customers' + local crit_log = "C> Potentially long count from space '" .. space .. "'" + local test_name = ('test_count_safety_%s'):format(name) + + pgroup[test_name] = function(g) + local uc = case.user_conditions + local opts = case.opts + local capture = luatest_capture:new() + capture:enable() + + local _, err = g.cluster.main_server.net_box:call('crud.count', {space, uc, opts}) + t.assert_equals(err, nil) + + local captured = helpers.fflush_main_server_stdout(g.cluster, capture) + + if case.has_crit then + t.assert_str_contains(captured, crit_log) + else + t.assert_equals(string.find(captured, crit_log, 1, true), nil) + end + + capture:disable() + end +end + pgroup.test_count_all = function(g) -- let's insert five tuples on different replicasets -- (two tuples on one replica and three on the other) @@ -118,7 +195,7 @@ pgroup.test_count_all = function(g) }) local result, err = g.cluster.main_server.net_box:call('crud.count', - {'customers'} + {'customers', nil, {fullscan = true}} ) t.assert_equals(err, nil) @@ -154,7 +231,7 @@ pgroup.test_count_all_with_yield_every = function(g) }) local result, err = g.cluster.main_server.net_box:call('crud.count', - {'customers', nil, {yield_every = 1}} + {'customers', nil, {yield_every = 1, fullscan = true}} ) t.assert_equals(err, nil) @@ -190,7 +267,7 @@ pgroup.test_count_all_with_yield_every_0 = function(g) }) local result, err = g.cluster.main_server.net_box:call('crud.count', - {'customers', nil, {yield_every = 0}} + {'customers', nil, {yield_every = 0, fullscan = true}} ) t.assert_equals(result, nil) @@ -312,7 +389,7 @@ pgroup.test_ge_condition_with_index = function(g) local expected_len = 3 local result, err = g.cluster.main_server.net_box:call('crud.count', - {'customers', conditions} + {'customers', conditions, {fullscan = true}} ) t.assert_equals(err, nil) @@ -354,7 +431,7 @@ pgroup.test_gt_condition_with_index = function(g) local expected_len = 1 local result, err = g.cluster.main_server.net_box:call('crud.count', - {'customers', conditions} + {'customers', conditions, {fullscan = true}} ) t.assert_equals(err, nil) @@ -396,7 +473,7 @@ pgroup.test_le_condition_with_index = function(g) local expected_len = 4 local result, err = g.cluster.main_server.net_box:call('crud.count', - {'customers', conditions} + {'customers', conditions, {fullscan = true}} ) t.assert_equals(err, nil) @@ -438,7 +515,7 @@ pgroup.test_lt_condition_with_index = function(g) local expected_len = 2 local result, err = g.cluster.main_server.net_box:call('crud.count', - {'customers', conditions} + {'customers', conditions, {fullscan = true}} ) t.assert_equals(err, nil) @@ -543,6 +620,7 @@ pgroup.test_opts_not_damaged = function(g) mode = 'read', prefer_replica = false, balance = false, + fullscan = true } local new_count_opts, err = g.cluster.main_server:eval([[ local crud = require('crud') @@ -586,7 +664,7 @@ pgroup.test_count_no_map_reduce = function(g) local result, err = g.cluster.main_server.net_box:call('crud.count', { 'customers', nil, - {bucket_id = 2804, timeout = 1}, + {bucket_id = 2804, timeout = 1, fullscan = true}, }) t.assert_equals(err, nil) t.assert_equals(result, 1) @@ -647,7 +725,7 @@ pgroup.test_count_timeout = function(g) local begin = clock.proc() local result, err = g.cluster.main_server.net_box:call('crud.count', - {'customers', conditions, {timeout = timeout}} + {'customers', conditions, {timeout = timeout, fullscan = true}} ) t.assert_equals(err, nil) @@ -688,7 +766,7 @@ pgroup.test_composite_index = function(g) } -- no after - local result, err = g.cluster.main_server.net_box:call('crud.count', {'customers', conditions}) + local result, err = g.cluster.main_server.net_box:call('crud.count', {'customers', conditions}, {fullscan = true}) t.assert_equals(err, nil) t.assert_equals(result, 4) diff --git a/test/integration/read_calls_strategies_test.lua b/test/integration/read_calls_strategies_test.lua index 0c8e8255..d2b826d4 100644 --- a/test/integration/read_calls_strategies_test.lua +++ b/test/integration/read_calls_strategies_test.lua @@ -78,7 +78,8 @@ pgroup.test_select = function(g) local _, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil, { mode = g.params.mode, balance = g.params.balance, - prefer_replica = g.params.prefer_replica + prefer_replica = g.params.prefer_replica, + fullscan = true }}) t.assert_equals(err, nil) local vshard_calls = g.get_vshard_calls('call_impl') @@ -110,7 +111,8 @@ pgroup.test_count = function(g) local _, err = g.cluster.main_server.net_box:call('crud.count', {'customers', nil, { mode = g.params.mode, balance = g.params.balance, - prefer_replica = g.params.prefer_replica + prefer_replica = g.params.prefer_replica, + fullscan = true }}) t.assert_equals(err, nil) local vshard_calls = g.get_vshard_calls('call_impl') diff --git a/test/integration/reload_test.lua b/test/integration/reload_test.lua index 5d8b25fb..0e2b4a6c 100644 --- a/test/integration/reload_test.lua +++ b/test/integration/reload_test.lua @@ -93,7 +93,7 @@ function g.test_router() g.highload_fiber:cancel() - local result, err = g.router.net_box:call('crud.select', {'customers'}) + local result, err = g.router.net_box:call('crud.select', {'customers', nil, {fullscan = true}}) t.assert_equals(err, nil) t.assert_items_include(result.rows, g.insertions_passed) end @@ -122,7 +122,7 @@ function g.test_storage() g.highload_fiber:cancel() - local result, err = g.router.net_box:call('crud.select', {'customers'}) + local result, err = g.router.net_box:call('crud.select', {'customers', nil, {fullscan = true}}) t.assert_equals(err, nil) t.assert_items_include(result.rows, g.insertions_passed) end diff --git a/test/integration/select_test.lua b/test/integration/select_test.lua index cb095385..e42648d9 100644 --- a/test/integration/select_test.lua +++ b/test/integration/select_test.lua @@ -1,6 +1,7 @@ local fio = require('fio') local t = require('luatest') +local luatest_capture = require('luatest.capture') local crud = require('crud') local crud_utils = require('crud.common.utils') @@ -30,6 +31,9 @@ pgroup.before_all(function(g) g.cluster:server('router').net_box:eval([[ require('crud').cfg{ stats = true } ]]) + g.cluster:server('router').net_box:eval([[ + require('crud.ratelimit').disable() + ]]) end) pgroup.after_all(function(g) helpers.stop_cluster(g.cluster) end) @@ -44,7 +48,7 @@ end) pgroup.test_non_existent_space = function(g) -- insert local obj, err = g.cluster.main_server.net_box:call( - 'crud.select', {'non_existent_space'} + 'crud.select', {'non_existent_space', nil, {fullscan=true}} ) t.assert_equals(obj, nil) @@ -53,7 +57,7 @@ end pgroup.test_select_no_index = function(g) local obj, err = g.cluster.main_server.net_box:call( - 'crud.select', {'no_index_space'} + 'crud.select', {'no_index_space', nil, {fullscan=true}} ) t.assert_equals(obj, nil) @@ -77,6 +81,111 @@ pgroup.test_not_valid_value_type = function(g) t.assert_str_contains(err.err, "Supplied key type of part 0 does not match index part type: expected unsigned") end +local select_safety_cases = { + nil_and_nil_opts = { + has_crit = true, + user_conditions = nil, + opts = nil, + }, + fullscan_false = { + has_crit = true, + user_conditions = nil, + opts = {fullscan = false}, + }, + fullscan_true = { + has_crit = false, + user_conditions = nil, + opts = {fullscan = true}, + }, + limit = { + has_crit = false, + user_conditions = nil, + opts = {first = 123}, + }, + max_negative_limit = { + has_crit = false, + user_conditions = nil, + opts = {first = -1000}, + }, + max_limit = { + has_crit = false, + user_conditions = nil, + opts = {first = 1000}, + }, + too_big_limit = { + has_crit = true, + user_conditions = nil, + opts = {first = 1001}, + }, + too_low_limit = { + has_crit = true, + user_conditions = nil, + opts = {first = -1001}, + }, + non_equal_conditions = { + has_crit = true, + user_conditions = { + {'>=', 'last_name', 'A'}, + {'<=', 'last_name', 'Z'}, + {'>', 'age', 20}, + {'<', 'age', 30}, + }, + opts = nil, + }, + equal_condition = { + has_crit = false, + user_conditions = { + {'>=', 'last_name', 'A'}, + {'<=', 'last_name', 'Z'}, + {'=', 'age', 25}, + }, + opts = nil, + }, + equal_condition2 = { + has_crit = false, + user_conditions = { + {'>=', 'last_name', 'A'}, + {'<=', 'last_name', 'Z'}, + {'==', 'age', 25}, + }, + opts = nil, + }, +} + +for name, case in pairs(select_safety_cases) do + local space = 'customers' + local crit_log = "C> Potentially long select from space '" .. space .. "'" + local test_name = ('test_select_safety_%s'):format(name) + + pgroup[test_name] = function(g) + local uc = case.user_conditions + local opts = case.opts + local capture = luatest_capture:new() + capture:enable() + + if opts ~= nil and opts.first ~= nil and opts.first < 0 then + local after_tuple = { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + } + opts.after = crud_utils.flatten(after_tuple, g.space_format) + end + + local _, err = g.cluster.main_server.net_box:call('crud.select', {space, uc, opts}) + t.assert_equals(err, nil) + + local captured = helpers.fflush_main_server_stdout(g.cluster, capture) + + if case.has_crit then + t.assert_str_contains(captured, crit_log) + else + t.assert_equals(string.find(captured, crit_log, 1, true), nil) + end + + capture:disable() + end +end + pgroup.test_select_all = function(g) local customers = helpers.insert_objects(g, 'customers', { { @@ -96,7 +205,7 @@ pgroup.test_select_all = function(g) table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil}) + local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil, {fullscan=true}}) t.assert_equals(err, nil) t.assert_equals(result.rows, { @@ -115,7 +224,7 @@ pgroup.test_select_all = function(g) }) -- no after - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil}) + local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil, {fullscan=true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -123,7 +232,8 @@ pgroup.test_select_all = function(g) -- after obj 2 local after = crud_utils.flatten(customers[2], g.space_format) - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil, {after=after}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', nil, {after=after, fullscan=true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -131,7 +241,8 @@ pgroup.test_select_all = function(g) -- after obj 4 (last) local after = crud_utils.flatten(customers[4], g.space_format) - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil, {after=after}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', nil, {after=after, fullscan=true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -504,14 +615,16 @@ pgroup.test_select_all_with_batch_size = function(g) table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) -- batch size 1 - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil, {batch_size=1}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', nil, {batch_size=1, fullscan=true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) t.assert_equals(objects, customers) -- batch size 3 - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil, {batch_size=3}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', nil, {batch_size=3, fullscan=true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -676,7 +789,7 @@ pgroup.test_ge_condition_with_index = function(g) } -- no after - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions}) + local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions}, {fullscan = true}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -684,7 +797,8 @@ pgroup.test_ge_condition_with_index = function(g) -- after obj 3 local after = crud_utils.flatten(customers[3], g.space_format) - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions, {after=after}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', conditions, {after=after, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -715,7 +829,7 @@ pgroup.test_le_condition_with_index = function(g) } -- no after - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions}) + local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions}, {fullscan = true}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -723,7 +837,8 @@ pgroup.test_le_condition_with_index = function(g) -- after obj 3 local after = crud_utils.flatten(customers[3], g.space_format) - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions, {after=after}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', conditions, {after=after, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -762,7 +877,7 @@ pgroup.test_lt_condition_with_index = function(g) } -- no after - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions}) + local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions, {fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -770,7 +885,8 @@ pgroup.test_lt_condition_with_index = function(g) -- after obj 1 local after = crud_utils.flatten(customers[1], g.space_format) - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions, {after=after}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', conditions, {after=after, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -814,7 +930,8 @@ pgroup.test_multiple_conditions = function(g) -- after obj 5 local after = crud_utils.flatten(customers[5], g.space_format) - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions, {after=after}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', conditions, {after=after, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -845,7 +962,7 @@ pgroup.test_composite_index = function(g) } -- no after - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions}) + local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions}, {fullscan = true}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -853,7 +970,8 @@ pgroup.test_composite_index = function(g) -- after obj 2 local after = crud_utils.flatten(customers[2], g.space_format) - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions, {after=after}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', conditions, {after=after, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -949,7 +1067,8 @@ pgroup.test_select_with_batch_size_1 = function(g) -- LE local conditions = {{'<=', 'age', 35}} - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions, {batch_size=1}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', conditions, {batch_size=1, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -957,7 +1076,8 @@ pgroup.test_select_with_batch_size_1 = function(g) -- LT local conditions = {{'<', 'age', 35}} - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions, {batch_size=1}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', conditions, {batch_size=1, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -965,7 +1085,8 @@ pgroup.test_select_with_batch_size_1 = function(g) -- GE local conditions = {{'>=', 'age', 35}} - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions, {batch_size=1}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', conditions, {batch_size=1, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -973,7 +1094,8 @@ pgroup.test_select_with_batch_size_1 = function(g) -- GT local conditions = {{'>', 'age', 35}} - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', conditions, {batch_size=1}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', conditions, {batch_size=1, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -1087,13 +1209,15 @@ pgroup.test_multipart_primary_index = function(g) t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {3})) local conditions_ge = {{'>=', 'primary', 0}} - local result_ge_0, err = g.cluster.main_server.net_box:call('crud.select', {'coord', conditions_ge}) + local result_ge_0, err = g.cluster.main_server.net_box:call('crud.select', {'coord', conditions_ge, + {fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result_ge_0.rows, result_ge_0.metadata) t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {1, 2, 3, 4, 5})) local result, err = g.cluster.main_server.net_box:call('crud.select', {'coord', conditions_ge, - {after = result_ge_0.rows[1]}}) + {after = result_ge_0.rows[1], + fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {2, 3, 4, 5})) @@ -1124,7 +1248,7 @@ pgroup.test_select_partial_result_bad_input = function(g) local conditions = {{'>=', 'age', 33}} local result, err = g.cluster.main_server.net_box:call('crud.select', - {'customers', conditions, {fields = {'id', 'mame'}}} + {'customers', conditions, {fields = {'id', 'mame'}, fullscan = true}} ) t.assert_equals(result, nil) @@ -1163,7 +1287,7 @@ pgroup.test_select_partial_result = function(g) } local result, err = g.cluster.main_server.net_box:call('crud.select', - {'customers', conditions, {fields = fields}} + {'customers', conditions, {fields = fields, fullscan = true}} ) t.assert_equals(err, nil) @@ -1177,7 +1301,7 @@ pgroup.test_select_partial_result = function(g) } result, err = g.cluster.main_server.net_box:call('crud.select', - {'customers', conditions, {after = result.rows[1], fields = fields}} + {'customers', conditions, {after = result.rows[1], fields = fields, fullscan = true}} ) t.assert_equals(err, nil) @@ -1196,7 +1320,7 @@ pgroup.test_select_partial_result = function(g) } result, err = g.cluster.main_server.net_box:call('crud.select', - {'customers', conditions, {fields = fields}} + {'customers', conditions, {fields = fields, fullscan = true}} ) t.assert_equals(err, nil) @@ -1210,7 +1334,7 @@ pgroup.test_select_partial_result = function(g) } result, err = g.cluster.main_server.net_box:call('crud.select', - {'customers', conditions, {after = result.rows[1], fields = fields}} + {'customers', conditions, {after = result.rows[1], fields = fields, fullscan = true}} ) t.assert_equals(err, nil) @@ -1232,7 +1356,7 @@ pgroup.test_select_partial_result = function(g) } result, err = g.cluster.main_server.net_box:call('crud.select', - {'customers', conditions, {fields = fields}} + {'customers', conditions, {fields = fields, fullscan = true}} ) t.assert_equals(err, nil) @@ -1246,7 +1370,7 @@ pgroup.test_select_partial_result = function(g) } result, err = g.cluster.main_server.net_box:call('crud.select', - {'customers', conditions, {after = result.rows[1], fields = fields}} + {'customers', conditions, {after = result.rows[1], fields = fields, fullscan = true}} ) t.assert_equals(err, nil) @@ -1265,7 +1389,7 @@ pgroup.test_select_partial_result = function(g) } result, err = g.cluster.main_server.net_box:call('crud.select', - {'customers', conditions, {fields = fields}} + {'customers', conditions, {fields = fields, fullscan = true}} ) t.assert_equals(err, nil) @@ -1279,7 +1403,7 @@ pgroup.test_select_partial_result = function(g) } result, err = g.cluster.main_server.net_box:call('crud.select', - {'customers', conditions, {after = result.rows[1], fields = fields}} + {'customers', conditions, {after = result.rows[1], fields = fields, fullscan = true}} ) t.assert_equals(err, nil) @@ -1318,7 +1442,7 @@ pgroup.test_cut_selected_rows = function(g) -- with fields option local result, err = g.cluster.main_server.net_box:call('crud.select', - {'customers', conditions, {fields = fields}} + {'customers', conditions, {fields = fields, fullscan = true}} ) t.assert_equals(err, nil) @@ -1341,7 +1465,7 @@ pgroup.test_cut_selected_rows = function(g) } local result, err = g.cluster.main_server.net_box:call('crud.select', - {'customers', conditions} + {'customers', conditions, {fullscan = true}} ) t.assert_equals(err, nil) @@ -1418,7 +1542,7 @@ pgroup.test_jsonpath = function(g) }) local result, err = g.cluster.main_server.net_box:call('crud.select', - {'developers', {{'>=', '[5]', 40}}, {fields = {'name', 'last_name'}}}) + {'developers', {{'>=', '[5]', 40}}, {fields = {'name', 'last_name'}, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -1429,7 +1553,7 @@ pgroup.test_jsonpath = function(g) t.assert_equals(objects, expected_objects) local result, err = g.cluster.main_server.net_box:call('crud.select', - {'developers', {{'<', '["age"]', 21}}, {fields = {'name', 'last_name'}}}) + {'developers', {{'<', '["age"]', 21}}, {fields = {'name', 'last_name'}, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -1440,7 +1564,7 @@ pgroup.test_jsonpath = function(g) t.assert_equals(objects, expected_objects) local result, err = g.cluster.main_server.net_box:call('crud.select', - {'developers', {{'>=', '[6].a.b', 55}}, {fields = {'name', 'last_name'}}}) + {'developers', {{'>=', '[6].a.b', 55}}, {fields = {'name', 'last_name'}, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -1486,7 +1610,7 @@ pgroup.test_jsonpath_index_field = function(g) -- PK jsonpath index local result, err = g.cluster.main_server.net_box:call('crud.select', - {'cars', {{'<=', 'id_ind', 3}, {'<=', 'age', 5}}, {fields = {'id', 'age'}}}) + {'cars', {{'<=', 'id_ind', 3}, {'<=', 'age', 5}}, {fields = {'id', 'age'}, fullscan = true}}) t.assert_equals(err, nil) local objects = crud.unflatten_rows(result.rows, result.metadata) @@ -1685,7 +1809,8 @@ pgroup.test_select_timeout = function(g) table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil, {timeout = 1}}) + local result, err = g.cluster.main_server.net_box:call( + 'crud.select', {'customers', nil, {timeout = 1, fullscan = true}}) t.assert_equals(err, nil) t.assert_equals(result.rows, { @@ -1771,7 +1896,7 @@ pgroup.test_select_no_map_reduce = function(g) local result, err = g.cluster.main_server.net_box:call('crud.select', { 'customers', nil, - {bucket_id = 2804, timeout = 1}, + {bucket_id = 2804, timeout = 1, fullscan = true}, }) t.assert_equals(err, nil) t.assert_equals(result.rows, { diff --git a/test/integration/stats_test.lua b/test/integration/stats_test.lua index 2ddd7c4b..ec71e610 100644 --- a/test/integration/stats_test.lua +++ b/test/integration/stats_test.lua @@ -394,6 +394,7 @@ local select_cases = { map_reduces = 1, tuples_fetched = 0, tuples_lookup = 4, + opts = {fullscan = true}, }, pairs_by_primary_index = { eval = eval.pairs, @@ -452,9 +453,9 @@ local function generate_stats(g) for _, case in pairs(select_cases) do local _, err if case.eval ~= nil then - _, err = g.router:eval(case.eval, { space_name, case.conditions }) + _, err = g.router:eval(case.eval, { space_name, case.conditions, case.opts }) else - _, err = g.router:call(case.func, { space_name, case.conditions }) + _, err = g.router:call(case.func, { space_name, case.conditions, case.opts }) end t.assert_equals(err, nil) diff --git a/test/integration/truncate_test.lua b/test/integration/truncate_test.lua index 934048b3..9cfb306c 100644 --- a/test/integration/truncate_test.lua +++ b/test/integration/truncate_test.lua @@ -61,7 +61,7 @@ pgroup.test_truncate = function(g) table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil}) + local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil, {fullscan = true}}) t.assert_equals(err, nil) t.assert_gt(#result.rows, 0) @@ -69,7 +69,7 @@ pgroup.test_truncate = function(g) t.assert_equals(err, nil) t.assert_equals(result, true) - local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil}) + local result, err = g.cluster.main_server.net_box:call('crud.select', {'customers', nil, {fullscan = true}}) t.assert_equals(err, nil) t.assert_equals(#result.rows, 0) end diff --git a/test/integration/updated_shema_test.lua b/test/integration/updated_shema_test.lua index 7d1ff5b9..6803efd1 100644 --- a/test/integration/updated_shema_test.lua +++ b/test/integration/updated_shema_test.lua @@ -283,7 +283,7 @@ end pgroup.test_select_non_existent_space = function(g) -- non-existent space err local obj, err = g.cluster.main_server.net_box:call( - 'crud.select', {'customers'} + 'crud.select', {'customers', nil, {fullscan = true}} ) t.assert_equals(obj, nil) @@ -298,7 +298,7 @@ pgroup.test_select_non_existent_space = function(g) -- check that schema changes were applied local obj, err = g.cluster.main_server.net_box:call( - 'crud.select', {'customers'} + 'crud.select', {'customers', nil, {fullscan = true}} ) t.assert_is_not(obj, nil) @@ -599,7 +599,7 @@ pgroup.test_select_field_added = function(g) -- unknown field (no results) local obj, err = g.cluster.main_server.net_box:call( - 'crud.select', {'customers', {{'==', 'extra', 'EXTRRRRA'}}} + 'crud.select', {'customers', {{'==', 'extra', 'EXTRRRRA'}}, {fullscan = true}} ) t.assert_equals(obj.rows, {}) @@ -612,7 +612,7 @@ pgroup.test_select_field_added = function(g) -- check that schema changes were applied local obj, err = g.cluster.main_server.net_box:call( - 'crud.select', {'customers', {{'==', 'extra', 'EXTRRRRA'}}} + 'crud.select', {'customers', {{'==', 'extra', 'EXTRRRRA'}}, {fullscan = true}} ) t.assert_is_not(obj, nil) @@ -763,7 +763,7 @@ pgroup.test_alter_index_parts = function(g) -- Check sort order before alter local result, err = g.cluster.main_server.net_box:call( - 'crud.select', {'customers', {{'>=', 'number_value_index', {0, "0"}}}} + 'crud.select', {'customers', {{'>=', 'number_value_index', {0, "0"}}}, {fullscan = true}} ) t.assert_equals(err, nil) t.assert_equals(#result.rows, 10) @@ -784,7 +784,7 @@ pgroup.test_alter_index_parts = function(g) -- Sort order should be new local result, err = g.cluster.main_server.net_box:call( - 'crud.select', {'customers', {{'>=', 'number_value_index', {"0", 0}}}} + 'crud.select', {'customers', {{'>=', 'number_value_index', {"0", 0}}}, {fullscan = true}} ) t.assert_equals(err, nil) t.assert_equals(#result.rows, 10)