diff --git a/CHANGELOG.md b/CHANGELOG.md index 3093ddef..5ebfd203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +* Statistics for CRUD operations on router (#224). ### Changed diff --git a/README.md b/README.md index 7ce5f820..6a419017 100644 --- a/README.md +++ b/README.md @@ -674,6 +674,97 @@ Combinations of `mode`, `prefer_replica` and `balance` options lead to: * prefer_replica, balance - [vshard call `callbre`](https://www.tarantool.io/en/doc/latest/reference/reference_rock/vshard/vshard_api/#router-api-callbre) +### Statistics + +`crud` routers can provide statistics on called operations. +```lua +-- Enable statistics collect. +crud.cfg{ stats = true } + +-- Returns table with statistics information. +crud.stats() + +-- Returns table with statistics information for specific space. +crud.stats('my_space') + +-- Disables statistics collect and destroys all collectors. +crud.cfg{ stats = false } + +-- Destroys all statistics collectors and creates them again. +crud.reset_stats() +``` + +You can use `crud.cfg` to check current stats state. +```lua +crud.cfg +--- +- stats: true +... +``` +Beware that iterating through `crud.cfg` with pairs is not supported yet, +refer to [tarantool/crud#265](https://github.com/tarantool/crud/issues/265). + +Format is as follows. +```lua +crud.stats() +--- +- spaces: + my_space: + insert: + ok: + latency: 0.002 + count: 19800 + time: 39.6 + error: + latency: 0.000001 + count: 4 + time: 0.000004 +... +crud.stats('my_space') +--- +- insert: + ok: + latency: 0.002 + count: 19800 + time: 39.6 + error: + latency: 0.000001 + count: 4 + time: 0.000004 +... +``` +`spaces` section contains statistics for each observed space. +If operation has never been called for a space, the corresponding +field will be empty. If no requests has been called for a +space, it will not be represented. Space data is based on +client requests rather than storages schema, so requests +for non-existing spaces are also collected. + +Possible statistics operation labels are +`insert` (for `insert` and `insert_object` calls), +`get`, `replace` (for `replace` and `replace_object` calls), `update`, +`upsert` (for `upsert` and `upsert_object` calls), `delete`, +`select` (for `select` and `pairs` calls), `truncate`, `len`, `count` +and `borders` (for `min` and `max` calls). + +Each operation section contains of different collectors +for success calls and error (both error throw and `nil, err`) +returns. `count` is total requests count since instance start +or stats restart. `latency` is average time of requests execution, +`time` is the total time of requests execution. + +Since `pairs` request behavior differs from any other crud request, its +statistics collection also has specific behavior. Statistics (`select` +section) are updated after `pairs` cycle is finished: you +either have iterated through all records or an error was thrown. +If your pairs cycle was interrupted with `break`, statistics will +be collected when pairs objects are cleaned up with Lua garbage +collector. + +Statistics are preserved between package reloads. Statistics are preserved +between [Tarantool Cartridge role reloads](https://www.tarantool.io/en/doc/latest/book/cartridge/cartridge_api/modules/cartridge.roles/#reload) +if you use CRUD Cartridge roles. + ## Cartridge roles `cartridge.roles.crud-storage` is a Tarantool Cartridge role that depends on the diff --git a/cartridge/roles/crud-router.lua b/cartridge/roles/crud-router.lua index ef510e51..1c4d43fe 100644 --- a/cartridge/roles/crud-router.lua +++ b/cartridge/roles/crud-router.lua @@ -1,8 +1,10 @@ local crud = require('crud') +local stash = require('crud.common.stash') -- removes routes that changed in config and adds new routes local function init() crud.init_router() + stash.setup_cartridge_reload() end local function stop() diff --git a/cartridge/roles/crud-storage.lua b/cartridge/roles/crud-storage.lua index 8371c428..3728c88c 100644 --- a/cartridge/roles/crud-storage.lua +++ b/cartridge/roles/crud-storage.lua @@ -1,7 +1,9 @@ local crud = require('crud') +local stash = require('crud.common.stash') local function init() crud.init_storage() + stash.setup_cartridge_reload() end local function stop() diff --git a/crud.lua b/crud.lua index b045e7fe..3f2b5c59 100644 --- a/crud.lua +++ b/crud.lua @@ -2,6 +2,7 @@ -- -- @module crud +local cfg = require('crud.cfg') local insert = require('crud.insert') local replace = require('crud.replace') local get = require('crud.get') @@ -15,6 +16,7 @@ local count = require('crud.count') local borders = require('crud.borders') local sharding_metadata = require('crud.common.sharding.sharding_metadata') local utils = require('crud.common.utils') +local stats = require('crud.stats') local crud = {} @@ -23,47 +25,47 @@ local crud = {} -- @refer insert.tuple -- @function insert -crud.insert = insert.tuple +crud.insert = stats.wrap(insert.tuple, stats.op.INSERT) -- @refer insert.object -- @function insert_object -crud.insert_object = insert.object +crud.insert_object = stats.wrap(insert.object, stats.op.INSERT) -- @refer get.call -- @function get -crud.get = get.call +crud.get = stats.wrap(get.call, stats.op.GET) -- @refer replace.tuple -- @function replace -crud.replace = replace.tuple +crud.replace = stats.wrap(replace.tuple, stats.op.REPLACE) -- @refer replace.object -- @function replace_object -crud.replace_object = replace.object +crud.replace_object = stats.wrap(replace.object, stats.op.REPLACE) -- @refer update.call -- @function update -crud.update = update.call +crud.update = stats.wrap(update.call, stats.op.UPDATE) -- @refer upsert.tuple -- @function upsert -crud.upsert = upsert.tuple +crud.upsert = stats.wrap(upsert.tuple, stats.op.UPSERT) -- @refer upsert.object -- @function upsert -crud.upsert_object = upsert.object +crud.upsert_object = stats.wrap(upsert.object, stats.op.UPSERT) -- @refer delete.call -- @function delete -crud.delete = delete.call +crud.delete = stats.wrap(delete.call, stats.op.DELETE) -- @refer select.call -- @function select -crud.select = select.call +crud.select = stats.wrap(select.call, stats.op.SELECT) -- @refer select.pairs -- @function pairs -crud.pairs = select.pairs +crud.pairs = stats.wrap(select.pairs, stats.op.SELECT, { pairs = true }) -- @refer utils.unflatten_rows -- @function unflatten_rows @@ -71,23 +73,23 @@ crud.unflatten_rows = utils.unflatten_rows -- @refer truncate.call -- @function truncate -crud.truncate = truncate.call +crud.truncate = stats.wrap(truncate.call, stats.op.TRUNCATE) -- @refer len.call -- @function len -crud.len = len.call +crud.len = stats.wrap(len.call, stats.op.LEN) -- @refer count.call -- @function count -crud.count = count.call +crud.count = stats.wrap(count.call, stats.op.COUNT) -- @refer borders.min -- @function min -crud.min = borders.min +crud.min = stats.wrap(borders.min, stats.op.BORDERS) -- @refer borders.max -- @function max -crud.max = borders.max +crud.max = stats.wrap(borders.max, stats.op.BORDERS) -- @refer utils.cut_rows -- @function cut_rows @@ -97,6 +99,18 @@ crud.cut_rows = utils.cut_rows -- @function cut_objects crud.cut_objects = utils.cut_objects +-- @refer cfg.cfg +-- @function cfg +crud.cfg = cfg.cfg + +-- @refer stats.get +-- @function stats +crud.stats = stats.get + +-- @refer stats.reset +-- @function reset_stats +crud.reset_stats = stats.reset + --- Initializes crud on node -- -- Exports all functions that are used for calls diff --git a/crud/cfg.lua b/crud/cfg.lua new file mode 100644 index 00000000..2525a947 --- /dev/null +++ b/crud/cfg.lua @@ -0,0 +1,70 @@ +---- Module for CRUD configuration. +-- @module crud.cfg +-- + +local checks = require('checks') +local errors = require('errors') + +local stash = require('crud.common.stash') +local stats = require('crud.stats') + +local CfgError = errors.new_class('CfgError', {capture_stack = false}) + +local cfg_module = {} + +local function set_defaults_if_empty(cfg) + if cfg.stats == nil then + cfg.stats = false + end + + return cfg +end + +local cfg = set_defaults_if_empty(stash.get(stash.name.cfg)) + +--- Configure CRUD module. +-- +-- @function __call +-- +-- @tab self +-- +-- @tab[opt] opts +-- +-- @bool[opt] opts.stats +-- Enable or disable statistics collect. +-- Statistics are observed only on router instances. +-- +-- @return Configuration table. +-- +local function __call(self, opts) + checks('table', { stats = '?boolean' }) + + opts = opts or {} + + if opts.stats ~= nil then + if opts.stats == true then + stats.enable() + else + stats.disable() + end + + rawset(cfg, 'stats', opts.stats) + end + + return self +end + +local function __newindex() + CfgError:assert(false, 'Use crud.cfg{} instead') +end + +-- Iterating through `crud.cfg` with pairs is not supported +-- yet, refer to tarantool/crud#265. +cfg_module.cfg = setmetatable({}, { + __index = cfg, + __newindex = __newindex, + __call = __call, + __serialize = function() return cfg end +}) + +return cfg_module diff --git a/crud/common/stash.lua b/crud/common/stash.lua new file mode 100644 index 00000000..3cb65ed1 --- /dev/null +++ b/crud/common/stash.lua @@ -0,0 +1,63 @@ +---- Module for preserving data between reloads. +-- @module crud.common.stash +-- +local dev_checks = require('crud.common.dev_checks') + +local stash = {} + +--- Available stashes list. +-- +-- @tfield string cfg +-- Stash for CRUD module configuration. +-- +-- @tfield string stats_internal +-- Stash for main stats module. +-- +-- @tfield string stats_local_registry +-- Stash for local metrics registry. +-- +stash.name = { + cfg = '__crud_cfg', + stats_internal = '__crud_stats_internal', + stats_local_registry = '__crud_stats_local_registry' +} + +--- Setup Tarantool Cartridge reload. +-- +-- Call on Tarantool Cartridge roles that are expected +-- to use stashes. +-- +-- @function setup_cartridge_reload +-- +-- @return Returns +-- +function stash.setup_cartridge_reload() + local hotreload = require('cartridge.hotreload') + for _, name in pairs(stash.name) do + hotreload.whitelist_globals({ name }) + end +end + +--- Get a stash instance, initialize if needed. +-- +-- Stashes are persistent to package reload. +-- To use them with Cartridge roles reload, +-- call `stash.setup_cartridge_reload` in role. +-- +-- @function get +-- +-- @string name +-- Stash identifier. Use one from `stash.name` table. +-- +-- @treturn table A stash instance. +-- +function stash.get(name) + dev_checks('string') + + local instance = rawget(_G, name) or {} + rawset(_G, name, instance) + + return instance +end + +return stash diff --git a/crud/stats/init.lua b/crud/stats/init.lua new file mode 100644 index 00000000..c2555791 --- /dev/null +++ b/crud/stats/init.lua @@ -0,0 +1,288 @@ +---- CRUD statistics module. +-- @module crud.stats +-- + +local clock = require('clock') +local checks = require('checks') +local fun = require('fun') + +local dev_checks = require('crud.common.dev_checks') +local stash = require('crud.common.stash') +local op_module = require('crud.stats.operation') +local registry = require('crud.stats.local_registry') + +local stats = {} +local internal = stash.get(stash.name.stats_internal) + +--- Check if statistics module was enabled. +-- +-- @function is_enabled +-- +-- @treturn boolean Returns `true` or `false`. +-- +function stats.is_enabled() + return internal.is_enabled == true +end + +--- Initializes statistics registry, enables callbacks and wrappers. +-- +-- If already enabled, do nothing. +-- +-- @function enable +-- +-- @treturn boolean Returns `true`. +-- +function stats.enable() + if stats.is_enabled() then + return true + end + + internal.is_enabled = true + registry.init() + + return true +end + +--- Resets statistics registry. +-- +-- After reset collectors are the same as right +-- after initial `stats.enable()`. +-- +-- @function reset +-- +-- @treturn boolean Returns true. +-- +function stats.reset() + if not stats.is_enabled() then + return true + end + + registry.destroy() + registry.init() + + return true +end + +--- Destroys statistics registry and disable callbacks. +-- +-- If already disabled, do nothing. +-- +-- @function disable +-- +-- @treturn boolean Returns true. +-- +function stats.disable() + if not stats.is_enabled() then + return true + end + + registry.destroy() + internal.is_enabled = false + + return true +end + +--- Get statistics on CRUD operations. +-- +-- @function get +-- +-- @string[opt] space_name +-- If specified, returns table with statistics +-- of operations on space, separated by operation type and +-- execution status. If there wasn't any requests of "op" type +-- for space, there won't be corresponding collectors. +-- If not specified, returns table with statistics +-- about all observed spaces. +-- +-- @treturn table Statistics on CRUD operations. +-- If statistics disabled, returns `{}`. +-- +function stats.get(space_name) + checks('?string') + + if not stats.is_enabled() then + return {} + end + + return registry.get(space_name) +end + +-- Hack to set __gc for a table in Lua 5.1 +-- See https://stackoverflow.com/questions/27426704/lua-5-1-workaround-for-gc-metamethod-for-tables +-- or https://habr.com/ru/post/346892/ +local function setmt__gc(t, mt) + local prox = newproxy(true) + getmetatable(prox).__gc = function() mt.__gc(t) end + t[prox] = true + return setmetatable(t, mt) +end + +local function wrap_pairs_gen(build_latency, space_name, op, gen, param, state) + local total_latency = build_latency + + -- If pairs() cycle will be interrupted with break, + -- we'll never get a proper obervation. + -- We create an object with the same lifespan as gen() + -- function so if someone break pairs cycle, + -- it still will be observed. + local observed = false + + local gc_observer = setmt__gc({}, { + __gc = function() + if observed == false then + registry.observe(total_latency, space_name, op, 'ok') + observed = true + end + end + }) + + local wrapped_gen = function(param, state) + -- Mess with gc_observer so its lifespan will + -- be the same as wrapped_gen() function. + gc_observer[1] = state + + local start_time = clock.monotonic() + + local status, next_state, var = pcall(gen, param, state) + + local finish_time = clock.monotonic() + + total_latency = total_latency + (finish_time - start_time) + + if status == false then + registry.observe(total_latency, space_name, op, 'error') + observed = true + error(next_state, 2) + end + + -- Observe stats in the end of pairs cycle + if var == nil then + registry.observe(total_latency, space_name, op, 'ok') + observed = true + return nil + end + + return next_state, var + end + + return fun.wrap(wrapped_gen, param, state) +end + +local function wrap_tail(space_name, op, pairs, start_time, call_status, ...) + dev_checks('string', 'string', 'boolean', 'number', 'boolean') + + local finish_time = clock.monotonic() + local latency = finish_time - start_time + + if call_status == false then + registry.observe(latency, space_name, op, 'error') + error((...), 2) + end + + if pairs == false then + if select(2, ...) ~= nil then + -- If not `pairs` call, return values `nil, err` + -- treated as error case. + registry.observe(latency, space_name, op, 'error') + return ... + else + registry.observe(latency, space_name, op, 'ok') + return ... + end + else + return wrap_pairs_gen(latency, space_name, op, ...) + end +end + +--- Wrap CRUD operation call to collect statistics. +-- +-- Approach based on `box.atomic()`: +-- https://github.com/tarantool/tarantool/blob/b9f7204b5e0d10b443c6f198e9f7f04e0d16a867/src/box/lua/schema.lua#L369 +-- +-- @function wrap +-- +-- @func func +-- Function to wrap. First argument is expected to +-- be a space name string. If statistics enabled, +-- errors are caught and thrown again. +-- +-- @string op +-- Label of registry collectors. +-- Use `require('crud.stats').op` to pick one. +-- +-- @tab[opt] opts +-- +-- @bool[opt=false] opts.pairs +-- If false, wraps only function passed as argument. +-- Second return value of wrapped function is treated +-- as error (`nil, err` case). +-- If true, also wraps gen() function returned by +-- call. Statistics observed on cycle end (last +-- element was fetched or error was thrown). If pairs +-- cycle was interrupted with `break`, statistics will +-- be collected when pairs objects are cleaned up with +-- Lua garbage collector. +-- +-- @return Wrapped function output. +-- +function stats.wrap(func, op, opts) + dev_checks('function', 'string', { pairs = '?boolean' }) + + local pairs + if type(opts) == 'table' and opts.pairs ~= nil then + pairs = opts.pairs + else + pairs = false + end + + return function(space_name, ...) + if not stats.is_enabled() then + return func(space_name, ...) + end + + local start_time = clock.monotonic() + + return wrap_tail( + space_name, op, pairs, start_time, + pcall(func, space_name, ...) + ) + end +end + +--- Table with CRUD operation lables. +-- +-- @tfield string INSERT +-- Identifies both `insert` and `insert_object`. +-- +-- @tfield string GET +-- +-- @tfield string REPLACE +-- Identifies both `replace` and `replace_object`. +-- +-- @tfield string UPDATE +-- +-- @tfield string UPSERT +-- Identifies both `upsert` and `upsert_object`. +-- +-- @tfield string DELETE +-- +-- @tfield string SELECT +-- Identifies both `pairs` and `select`. +-- +-- @tfield string TRUNCATE +-- +-- @tfield string LEN +-- +-- @tfield string COUNT +-- +-- @tfield string BORDERS +-- Identifies both `min` and `max`. +-- +stats.op = op_module + +--- Stats module internal state (for debug/test). +-- +-- @tfield[opt] boolean is_enabled Is currently enabled. +stats.internal = internal + +return stats diff --git a/crud/stats/local_registry.lua b/crud/stats/local_registry.lua new file mode 100644 index 00000000..c5e125f1 --- /dev/null +++ b/crud/stats/local_registry.lua @@ -0,0 +1,101 @@ +---- Internal module used to store statistics. +-- @module crud.stats.local_registry +-- + +local dev_checks = require('crud.common.dev_checks') +local stash = require('crud.common.stash') +local registry_utils = require('crud.stats.registry_utils') + +local registry = {} +local internal = stash.get(stash.name.stats_local_registry) + +--- Initialize local metrics registry. +-- +-- Registries are not meant to used explicitly +-- by users, init is not guaranteed to be idempotent. +-- +-- @function init +-- +-- @treturn boolean Returns true. +-- +function registry.init() + internal.registry = {} + internal.registry.spaces = {} + + return true +end + +--- Destroy local metrics registry. +-- +-- Registries are not meant to used explicitly +-- by users, destroy is not guaranteed to be idempotent. +-- +-- @function destroy +-- +-- @treturn boolean Returns `true`. +-- +function registry.destroy() + internal.registry = nil + + return true +end + +--- Get copy of local metrics registry. +-- +-- Registries are not meant to used explicitly +-- by users, get is not guaranteed to work without init. +-- +-- @function get +-- +-- @string[opt] space_name +-- If specified, returns table with statistics +-- of operations on table, separated by operation type and +-- execution status. If there wasn't any requests for table, +-- returns `{}`. If not specified, returns table with statistics +-- about all observed spaces. +-- +-- @treturn table Returns copy of metrics registry (or registry section). +-- +function registry.get(space_name) + dev_checks('?string') + + if space_name ~= nil then + return table.deepcopy(internal.registry.spaces[space_name]) or {} + end + + return table.deepcopy(internal.registry) +end + +--- Increase requests count and update latency info. +-- +-- @function observe +-- +-- @string space_name +-- Name of space. +-- +-- @number latency +-- Time of call execution. +-- +-- @string op +-- Label of registry collectors. +-- Use `require('crud.stats').op` to pick one. +-- +-- @string success +-- `'ok'` if no errors on execution, `'error'` otherwise. +-- +-- @treturn boolean Returns `true`. +-- +function registry.observe(latency, space_name, op, status) + dev_checks('number', 'string', 'string', 'string') + + registry_utils.init_collectors_if_required(internal.registry.spaces, space_name, op) + local collectors = internal.registry.spaces[space_name][op][status] + + collectors.count = collectors.count + 1 + collectors.time = collectors.time + latency + collectors.latency = collectors.time / collectors.count + + return true +end + +return registry diff --git a/crud/stats/operation.lua b/crud/stats/operation.lua new file mode 100644 index 00000000..a6a9627a --- /dev/null +++ b/crud/stats/operation.lua @@ -0,0 +1,23 @@ +-- It is not clear how to describe modules +-- with constants for ldoc. ldoc-styled description +-- for this module is available at `crud.stats.init`. +-- See https://github.com/lunarmodules/LDoc/issues/369 +-- for possible updates. +return { + -- INSERT identifies both `insert` and `insert_object`. + INSERT = 'insert', + GET = 'get', + -- REPLACE identifies both `replace` and `replace_object`. + REPLACE = 'replace', + UPDATE = 'update', + -- UPSERT identifies both `upsert` and `upsert_object`. + UPSERT = 'upsert', + DELETE = 'delete', + -- SELECT identifies both `pairs` and `select`. + SELECT = 'select', + TRUNCATE = 'truncate', + LEN = 'len', + COUNT = 'count', + -- BORDERS identifies both `min` and `max`. + BORDERS = 'borders', +} diff --git a/crud/stats/registry_utils.lua b/crud/stats/registry_utils.lua new file mode 100644 index 00000000..2c99f8a3 --- /dev/null +++ b/crud/stats/registry_utils.lua @@ -0,0 +1,60 @@ +---- Internal module used by statistics registries. +-- @module crud.stats.registry_utils +-- + +local dev_checks = require('crud.common.dev_checks') + +local registry_utils = {} + +--- Build collectors for local registry. +-- +-- @function build_collectors +-- +-- @treturn table Returns collectors for success and error requests. +-- Collectors store 'count', 'latency' and 'time' values. +-- +function registry_utils.build_collectors() + local collectors = { + ok = { + count = 0, + latency = 0, + time = 0, + }, + error = { + count = 0, + latency = 0, + time = 0, + }, + } + + return collectors +end + +--- Initialize all statistic collectors for a space operation. +-- +-- @function init_collectors_if_required +-- +-- @tab spaces +-- `spaces` section of registry. +-- +-- @string space_name +-- Name of space. +-- +-- @string op +-- Label of registry collectors. +-- Use `require('crud.stats').op` to pick one. +-- +function registry_utils.init_collectors_if_required(spaces, space_name, op) + dev_checks('table', 'string', 'string') + + if spaces[space_name] == nil then + spaces[space_name] = {} + end + + local space_collectors = spaces[space_name] + if space_collectors[op] == nil then + space_collectors[op] = registry_utils.build_collectors() + end +end + +return registry_utils diff --git a/deps.sh b/deps.sh index 87ce6b92..abff8f4b 100755 --- a/deps.sh +++ b/deps.sh @@ -4,7 +4,7 @@ set -e # Test dependencies: -tarantoolctl rocks install luatest 0.5.5 +tarantoolctl rocks install luatest 0.5.7 tarantoolctl rocks install luacheck 0.25.0 tarantoolctl rocks install luacov 0.13.0 diff --git a/test/entrypoint/srv_stats.lua b/test/entrypoint/srv_stats.lua new file mode 100755 index 00000000..d9649b70 --- /dev/null +++ b/test/entrypoint/srv_stats.lua @@ -0,0 +1,63 @@ +#!/usr/bin/env tarantool + +require('strict').on() +_G.is_initialized = function() return false end + +local log = require('log') +local errors = require('errors') +local cartridge = require('cartridge') + +package.preload['customers-storage'] = function() + local engine = os.getenv('ENGINE') or 'memtx' + return { + role_name = 'customers-storage', + init = function() + local customers_space = box.schema.space.create('customers', { + format = { + {name = 'id', type = 'unsigned'}, + {name = 'bucket_id', type = 'unsigned'}, + {name = 'name', type = 'string'}, + {name = 'last_name', type = 'string'}, + {name = 'age', type = 'number'}, + {name = 'city', type = 'string'}, + }, + if_not_exists = true, + engine = engine, + }) + -- primary index + customers_space:create_index('id_index', { + parts = { {field = 'id'} }, + if_not_exists = true, + }) + customers_space:create_index('bucket_id', { + parts = { {field = 'bucket_id'} }, + unique = false, + if_not_exists = true, + }) + customers_space:create_index('age_index', { + parts = { {field = 'age'} }, + unique = false, + if_not_exists = true, + }) + end, + } +end + +local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { + advertise_uri = 'localhost:3301', + http_port = 8081, + bucket_count = 3000, + roles = { + 'cartridge.roles.crud-router', + 'cartridge.roles.crud-storage', + 'customers-storage', + }, + roles_reload_allowed = true, +}) + +if not ok then + log.error('%s', err) + os.exit(1) +end + +_G.is_initialized = cartridge.is_healthy diff --git a/test/helper.lua b/test/helper.lua index f2cdb6ab..669dec07 100644 --- a/test/helper.lua +++ b/test/helper.lua @@ -378,4 +378,92 @@ function helpers.get_sharding_func_cache_size(cluster) ]]) end +function helpers.simple_functions_params() + return { + sleep_time = 0.01, + error = { err = 'err' }, + error_msg = 'throw me', + } +end + +function helpers.prepare_simple_functions(router) + local params = helpers.simple_functions_params() + + local _, err = router:eval([[ + local clock = require('clock') + local fiber = require('fiber') + + local params = ... + local sleep_time = params.sleep_time + local error_table = params.error + local error_msg = params.error_msg + + -- Using `fiber.sleep(time)` between two `clock.monotonic()` + -- may return diff less than `time`. + sleep_for = function(time) + local start = clock.monotonic() + while (clock.monotonic() - start) < time do + fiber.sleep(time / 10) + end + end + + return_true = function(space_name) + sleep_for(sleep_time) + return true + end + + return_err = function(space_name) + sleep_for(sleep_time) + return nil, error_table + end + + throws_error = function() + sleep_for(sleep_time) + error(error_msg) + end + ]], { params }) + + t.assert_equals(err, nil) +end + +function helpers.is_space_exist(router, space_name) + local res, err = router:eval([[ + local vshard = require('vshard') + local utils = require('crud.common.utils') + + local space, err = utils.get_space(..., vshard.router.routeall()) + if err ~= nil then + return nil, err + end + return space ~= nil + ]], { space_name }) + + t.assert_equals(err, nil) + return res +end + +function helpers.reload_package(srv) + srv.net_box:eval([[ + local function startswith(text, prefix) + return text:find(prefix, 1, true) == 1 + end + + for k, _ in pairs(package.loaded) do + if startswith(k, 'crud') then + package.loaded[k] = nil + end + end + + crud = require('crud') + ]]) +end + +function helpers.reload_roles(srv) + local ok, err = srv.net_box:eval([[ + return require('cartridge.roles').reload() + ]]) + + t.assert_equals({ok, err}, {true, nil}) +end + return helpers diff --git a/test/integration/cfg_test.lua b/test/integration/cfg_test.lua new file mode 100644 index 00000000..718a21c1 --- /dev/null +++ b/test/integration/cfg_test.lua @@ -0,0 +1,74 @@ +local fio = require('fio') + +local t = require('luatest') + +local helpers = require('test.helper') + +local group = t.group('cfg') + +group.before_all(function(g) + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_stats'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + }) + + g.cluster:start() +end) + +group.after_all(function(g) helpers.stop_cluster(g.cluster) end) + +group.test_defaults = function(g) + local cfg = g.cluster:server('router'):eval("return require('crud').cfg") + t.assert_equals(cfg, { stats = false }) +end + +group.test_change_value = function(g) + local new_cfg = g.cluster:server('router'):eval("return require('crud').cfg({ stats = true })") + t.assert_equals(new_cfg.stats, true) +end + +group.test_table_is_immutable = function(g) + local router = g.cluster:server('router') + + t.assert_error_msg_contains( + 'Use crud.cfg{} instead', + router.eval, router, + [[ + local cfg = require('crud').cfg() + cfg.stats = 'newvalue' + ]]) + + t.assert_error_msg_contains( + 'Use crud.cfg{} instead', + router.eval, router, + [[ + local cfg = require('crud').cfg() + cfg.newfield = 'newvalue' + ]]) +end + +group.test_package_reload_preserves_values = function(g) + local router = g.cluster:server('router') + + -- Generate some non-default values. + router:eval("return require('crud').cfg({ stats = true })") + + helpers.reload_package(router) + + local cfg = router:eval("return require('crud').cfg") + t.assert_equals(cfg.stats, true) +end + +group.test_role_reload_preserves_values = function(g) + local router = g.cluster:server('router') + + -- Generate some non-default values. + router:eval("return require('crud').cfg({ stats = true })") + + helpers.reload_roles(router) + + local cfg = router:eval("return require('crud').cfg") + t.assert_equals(cfg.stats, true) +end diff --git a/test/integration/reload_test.lua b/test/integration/reload_test.lua index c1f20c67..5d8b25fb 100644 --- a/test/integration/reload_test.lua +++ b/test/integration/reload_test.lua @@ -8,14 +8,6 @@ local g = t.group() local helpers = require('test.helper') -local function reload(srv) - local ok, err = srv.net_box:eval([[ - return require("cartridge.roles").reload() - ]]) - - t.assert_equals({ok, err}, {true, nil}) -end - g.before_all(function() g.cluster = helpers.Cluster:new({ datadir = fio.tempdir(), @@ -92,7 +84,7 @@ function g.test_router() t.assert_equals(last_insert[3], 'A', 'No workload for label A') end) - reload(g.router) + helpers.reload_roles(g.router) local cnt = #g.insertions_passed g.cluster:retrying({}, function() @@ -117,7 +109,7 @@ function g.test_storage() -- snapshot with a signal g.s1_master.process:kill('USR1') - reload(g.s1_master) + helpers.reload_roles(g.s1_master) g.cluster:retrying({}, function() g.s1_master.net_box:call('box.snapshot') diff --git a/test/integration/stats_test.lua b/test/integration/stats_test.lua new file mode 100644 index 00000000..5af04db9 --- /dev/null +++ b/test/integration/stats_test.lua @@ -0,0 +1,509 @@ +local fio = require('fio') +local clock = require('clock') +local t = require('luatest') + +local stats_registry_utils = require('crud.stats.registry_utils') + +local g = t.group('stats_integration') +local helpers = require('test.helper') + +local space_name = 'customers' +local non_existing_space_name = 'non_existing_space' +local new_space_name = 'newspace' + +g.before_all(function(g) + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_stats'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + }) + g.cluster:start() + g.router = g.cluster:server('router').net_box + + helpers.prepare_simple_functions(g.router) + g.router:eval("require('crud').cfg{ stats = true }") +end) + +g.after_all(function(g) + helpers.stop_cluster(g.cluster) +end) + +g.before_each(function(g) + g.router:eval("crud = require('crud')") + helpers.truncate_space_on_cluster(g.cluster, space_name) + helpers.drop_space_on_cluster(g.cluster, new_space_name) +end) + +function g:get_stats(space_name) + return self.router:eval("return require('crud').stats(...)", { space_name }) +end + + +local function create_new_space(g) + helpers.call_on_storages(g.cluster, function(server) + server.net_box:eval([[ + local space_name = ... + if not box.cfg.read_only then + local sp = box.schema.space.create(space_name, { format = { + {name = 'id', type = 'unsigned'}, + {name = 'bucket_id', type = 'unsigned'}, + }}) + + sp:create_index('pk', { + parts = { {field = 'id'} }, + }) + + sp:create_index('bucket_id', { + parts = { {field = 'bucket_id'} }, + unique = false, + }) + end + ]], { new_space_name }) + end) +end + +-- If there weren't any operations, space stats is {}. +-- To compute stats diff, this helper return real stats +-- if they're already present or default stats if +-- this operation of space hasn't been observed yet. +local function set_defaults_if_empty(space_stats, op) + if space_stats[op] ~= nil then + return space_stats[op] + else + return stats_registry_utils.build_collectors(op) + end +end + +local eval = { + pairs = [[ + local space_name = select(1, ...) + local conditions = select(2, ...) + + local result = {} + for _, v in crud.pairs(space_name, conditions, { batch_size = 1 }) do + table.insert(result, v) + end + + return result + ]], + + pairs_pcall = [[ + local space_name = select(1, ...) + local conditions = select(2, ...) + + local _, err = pcall(crud.pairs, space_name, conditions, { batch_size = 1 }) + + return nil, tostring(err) + ]], +} + +local simple_operation_cases = { + insert = { + func = 'crud.insert', + args = { + space_name, + { 12, box.NULL, 'Ivan', 'Ivanov', 20, 'Moscow' }, + }, + op = 'insert', + }, + insert_object = { + func = 'crud.insert_object', + args = { + space_name, + { id = 13, name = 'Ivan', last_name = 'Ivanov', age = 20, city = 'Moscow' }, + }, + op = 'insert', + }, + get = { + func = 'crud.get', + args = { space_name, { 12 } }, + op = 'get', + }, + select = { + func = 'crud.select', + args = { space_name, {{ '==', 'id_index', 3 }} }, + op = 'select', + }, + pairs = { + eval = eval.pairs, + args = { space_name, {{ '==', 'id_index', 3 }} }, + op = 'select', + }, + replace = { + func = 'crud.replace', + args = { + space_name, + { 12, box.NULL, 'Ivan', 'Ivanov', 20, 'Moscow' }, + }, + op = 'replace', + }, + replace_object = { + func = 'crud.replace_object', + args = { + space_name, + { id = 12, name = 'Ivan', last_name = 'Ivanov', age = 20, city = 'Moscow' }, + }, + op = 'replace', + }, + update = { + prepare = function(g) + helpers.insert_objects(g, space_name, {{ + id = 15, name = 'Ivan', last_name = 'Ivanov', + age = 20, city = 'Moscow' + }}) + end, + func = 'crud.update', + args = { space_name, 12, {{'+', 'age', 10}} }, + op = 'update', + }, + upsert = { + func = 'crud.upsert', + args = { + space_name, + { 16, box.NULL, 'Ivan', 'Ivanov', 20, 'Moscow' }, + {{'+', 'age', 1}}, + }, + op = 'upsert', + }, + upsert_object = { + func = 'crud.upsert_object', + args = { + space_name, + { id = 17, name = 'Ivan', last_name = 'Ivanov', age = 20, city = 'Moscow' }, + {{'+', 'age', 1}} + }, + op = 'upsert', + }, + delete = { + func = 'crud.delete', + args = { space_name, { 12 } }, + op = 'delete', + }, + truncate = { + func = 'crud.truncate', + args = { space_name }, + op = 'truncate', + }, + len = { + func = 'crud.len', + args = { space_name }, + op = 'len', + }, + count = { + func = 'crud.count', + args = { space_name, {{ '==', 'id_index', 3 }} }, + op = 'count', + }, + min = { + func = 'crud.min', + args = { space_name }, + op = 'borders', + }, + max = { + func = 'crud.max', + args = { space_name }, + op = 'borders', + }, + insert_error = { + func = 'crud.insert', + args = { space_name, { 'id' } }, + op = 'insert', + expect_error = true, + }, + insert_object_error = { + func = 'crud.insert_object', + args = { space_name, { 'id' } }, + op = 'insert', + expect_error = true, + }, + get_error = { + func = 'crud.get', + args = { space_name, { 'id' } }, + op = 'get', + expect_error = true, + }, + select_error = { + func = 'crud.select', + args = { space_name, {{ '==', 'id_index', 'sdf' }} }, + op = 'select', + expect_error = true, + }, + pairs_error = { + eval = eval.pairs, + args = { space_name, {{ '%=', 'id_index', 'sdf' }} }, + op = 'select', + expect_error = true, + pcall = true, + }, + replace_error = { + func = 'crud.replace', + args = { space_name, { 'id' } }, + op = 'replace', + expect_error = true, + }, + replace_object_error = { + func = 'crud.replace_object', + args = { space_name, { 'id' } }, + op = 'replace', + expect_error = true, + }, + update_error = { + func = 'crud.update', + args = { space_name, { 'id' }, {{'+', 'age', 1}} }, + op = 'update', + expect_error = true, + }, + upsert_error = { + func = 'crud.upsert', + args = { space_name, { 'id' }, {{'+', 'age', 1}} }, + op = 'upsert', + expect_error = true, + }, + upsert_object_error = { + func = 'crud.upsert_object', + args = { space_name, { 'id' }, {{'+', 'age', 1}} }, + op = 'upsert', + expect_error = true, + }, + delete_error = { + func = 'crud.delete', + args = { space_name, { 'id' } }, + op = 'delete', + expect_error = true, + }, + count_error = { + func = 'crud.count', + args = { space_name, {{ '==', 'id_index', 'sdf' }} }, + op = 'count', + expect_error = true, + }, + min_error = { + func = 'crud.min', + args = { space_name, 'badindex' }, + op = 'borders', + expect_error = true, + }, + max_error = { + func = 'crud.max', + args = { space_name, 'badindex' }, + op = 'borders', + expect_error = true, + }, +} + +-- Generate non-null stats for all cases. +local function generate_stats(g) + for _, case in pairs(simple_operation_cases) do + if case.prepare ~= nil then + case.prepare(g) + end + + local _, err + if case.eval ~= nil then + if case.pcall then + _, err = pcall(g.router.eval, g.router, case.eval, case.args) + else + _, err = g.router:eval(case.eval, case.args) + end + else + _, err = g.router:call(case.func, case.args) + end + + if case.expect_error ~= true then + t.assert_equals(err, nil) + else + t.assert_not_equals(err, nil) + end + end +end + + +-- Call some operations for existing +-- spaces and ensure statistics is updated. +for name, case in pairs(simple_operation_cases) do + local test_name = ('test_%s'):format(name) + + if case.prepare ~= nil then + g.before_test(test_name, case.prepare) + end + + g[test_name] = function(g) + -- Collect stats before call. + local stats_before = g:get_stats(space_name) + t.assert_type(stats_before, 'table') + + -- Call operation. + local before_start = clock.monotonic() + + local _, err + if case.eval ~= nil then + if case.pcall then + _, err = pcall(g.router.eval, g.router, case.eval, case.args) + else + _, err = g.router:eval(case.eval, case.args) + end + else + _, err = g.router:call(case.func, case.args) + end + + local after_finish = clock.monotonic() + + if case.expect_error ~= true then + t.assert_equals(err, nil) + else + t.assert_not_equals(err, nil) + end + + -- Collect stats after call. + local stats_after = g:get_stats(space_name) + t.assert_type(stats_after, 'table') + t.assert_not_equals(stats_after[case.op], nil) + + -- Expecting 'ok' metrics to change on `expect_error == false` + -- or 'error' to change otherwise. + local changed, unchanged + if case.expect_error == true then + changed = 'error' + unchanged = 'ok' + else + unchanged = 'error' + changed = 'ok' + end + + local op_before = set_defaults_if_empty(stats_before, case.op) + local changed_before = op_before[changed] + local op_after = set_defaults_if_empty(stats_after, case.op) + local changed_after = op_after[changed] + + t.assert_equals(changed_after.count - changed_before.count, 1, + 'Expected count incremented') + + local ok_latency_max = math.max(changed_before.latency, after_finish - before_start) + + t.assert_gt(changed_after.latency, 0, + 'Changed latency has appropriate value') + t.assert_le(changed_after.latency, ok_latency_max, + 'Changed latency has appropriate value') + + local time_diff = changed_after.time - changed_before.time + + t.assert_gt(time_diff, 0, 'Total time increase has appropriate value') + t.assert_le(time_diff, after_finish - before_start, + 'Total time increase has appropriate value') + + local unchanged_before = op_before[unchanged] + local unchanged_after = stats_after[case.op][unchanged] + + t.assert_equals(unchanged_before, unchanged_after, 'Other stats remained the same') + end +end + + +-- Call some operation on non-existing +-- space and ensure statistics are updated. +g.before_test('test_non_existing_space', function(g) + t.assert_equals( + helpers.is_space_exist(g.router, non_existing_space_name), + false, + ('Space %s does not exist'):format(non_existing_space_name) + ) +end) + +g.test_non_existing_space = function(g) + local op = 'get' + + -- Collect stats before call. + local stats_before = g:get_stats(non_existing_space_name) + t.assert_type(stats_before, 'table') + local op_before = set_defaults_if_empty(stats_before, op) + + -- Call operation. + local _, err = g.router:call('crud.get', { non_existing_space_name, { 1 } }) + t.assert_not_equals(err, nil) + + -- Collect stats after call. + local stats_after = g:get_stats(non_existing_space_name) + t.assert_type(stats_after, 'table') + local op_after = stats_after[op] + t.assert_type(op_after, 'table', 'Section has been created if not existed') + + t.assert_equals(op_after.error.count - op_before.error.count, 1, + 'Error count for non-existing space incremented') +end + + +g.before_test( + 'test_role_reload_do_not_reset_observations', + generate_stats) + +g.test_role_reload_do_not_reset_observations = function(g) + local stats_before = g:get_stats() + + helpers.reload_roles(g.cluster:server('router')) + + local stats_after = g:get_stats() + t.assert_equals(stats_after, stats_before) +end + + +g.before_test( + 'test_module_reload_do_not_reset_observations', + generate_stats) + +g.test_module_reload_do_not_reset_observations = function(g) + local stats_before = g:get_stats() + + helpers.reload_package(g.cluster:server('router')) + + local stats_after = g:get_stats() + t.assert_equals(stats_after, stats_before) +end + + +g.test_spaces_created_in_runtime_supported_with_stats = function(g) + local op = 'insert' + local stats_before = g:get_stats(new_space_name) + local op_before = set_defaults_if_empty(stats_before, op) + + create_new_space(g) + + local _, err = g.router:call('crud.insert', { new_space_name, { 1, box.NULL }}) + t.assert_equals(err, nil) + + local stats_after = g:get_stats(new_space_name) + local op_after = stats_after[op] + t.assert_type(op_after, 'table', "'insert' stats found for new space") + t.assert_type(op_after.ok, 'table', "success 'insert' stats found for new space") + t.assert_equals(op_after.ok.count - op_before.ok.count, 1, + "Success requests count incremented for new space") +end + + +g.before_test( + 'test_spaces_dropped_in_runtime_supported_with_stats', + function(g) + create_new_space(g) + + local _, err = g.router:call('crud.insert', { new_space_name, { 1, box.NULL }}) + t.assert_equals(err, nil) + end) + +g.test_spaces_dropped_in_runtime_supported_with_stats = function(g) + local op = 'insert' + local stats_before = g:get_stats(new_space_name) + local op_before = set_defaults_if_empty(stats_before, op) + t.assert_type(op_before, 'table', "'insert' stats found for new space") + + helpers.drop_space_on_cluster(g.cluster, new_space_name) + + local _, err = g.router:call('crud.insert', { new_space_name, { 2, box.NULL }}) + t.assert_not_equals(err, nil, "Should trigger 'space not found' error") + + local stats_after = g:get_stats(new_space_name) + local op_after = stats_after[op] + t.assert_type(op_after, 'table', "'insert' stats found for dropped new space") + t.assert_type(op_after.error, 'table', "error 'insert' stats found for dropped new space") + t.assert_equals(op_after.error.count - op_before.error.count, 1, + "Error requests count incremented since space was known to registry before drop") +end diff --git a/test/unit/stats_test.lua b/test/unit/stats_test.lua new file mode 100644 index 00000000..628a9266 --- /dev/null +++ b/test/unit/stats_test.lua @@ -0,0 +1,555 @@ +local clock = require('clock') +local fio = require('fio') +local fun = require('fun') +local t = require('luatest') + +local stats_module = require('crud.stats') + +local g = t.group('stats_unit') +local helpers = require('test.helper') + +local space_name = 'customers' + +g.before_all(function(g) + -- Enable test cluster for "is space exist?" checks. + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_stats'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + }) + g.cluster:start() + g.router = g.cluster:server('router').net_box + + helpers.prepare_simple_functions(g.router) + g.router:eval("stats_module = require('crud.stats')") +end) + +g.after_all(function(g) + helpers.stop_cluster(g.cluster) +end) + +-- Reset statistics between tests, reenable if needed. +g.before_each(function(g) + g:enable_stats() +end) + +g.after_each(function(g) + g:disable_stats() +end) + +function g:get_stats(space_name) + return self.router:eval("return stats_module.get(...)", { space_name }) +end + +function g:enable_stats() + self.router:eval("stats_module.enable()") +end + +function g:disable_stats() + self.router:eval("stats_module.disable()") +end + +function g:reset_stats() + self.router:eval("return stats_module.reset()") +end + + +g.test_get_format_after_enable = function(g) + local stats = g:get_stats() + + t.assert_type(stats, 'table') + t.assert_equals(stats.spaces, {}) +end + +g.test_get_by_space_name_format_after_enable = function(g) + local stats = g:get_stats(space_name) + + t.assert_type(stats, 'table') + t.assert_equals(stats, {}) +end + +-- Test statistics values after wrapped functions call. +local observe_cases = { + wrapper_observes_expected_values_on_ok = { + operations = stats_module.op, + func = 'return_true', + changed_coll = 'ok', + unchanged_coll = 'error', + }, + wrapper_observes_expected_values_on_error_return = { + operations = stats_module.op, + func = 'return_err', + changed_coll = 'error', + unchanged_coll = 'ok', + }, + wrapper_observes_expected_values_on_error_throw = { + operations = stats_module.op, + func = 'throws_error', + changed_coll = 'error', + unchanged_coll = 'ok', + pcall = true, + }, +} + +local call_wrapped = [[ + local func = rawget(_G, select(1, ...)) + local op = select(2, ...) + local opts = select(3, ...) + local space_name = select(4, ...) + + stats_module.wrap(func, op, opts)(space_name) +]] + +for name, case in pairs(observe_cases) do + for _, op in pairs(case.operations) do + local test_name = ('test_%s_%s'):format(op, name) + + g[test_name] = function(g) + -- Call wrapped functions on server side. + -- Collect execution times from outside. + local run_count = 10 + local time_diffs = {} + + local args = { case.func, op, case.opts, space_name } + + for _ = 1, run_count do + local before_start = clock.monotonic() + + if case.pcall then + pcall(g.router.eval, g.router, call_wrapped, args) + else + g.router:eval(call_wrapped, args) + end + + local after_finish = clock.monotonic() + + table.insert(time_diffs, after_finish - before_start) + end + + table.sort(time_diffs) + local total_time = fun.sum(time_diffs) + + -- Validate stats format after execution. + local total_stats = g:get_stats() + t.assert_type(total_stats, 'table', 'Total stats present after observations') + + local space_stats = g:get_stats(space_name) + t.assert_type(space_stats, 'table', 'Space stats present after observations') + + t.assert_equals(total_stats.spaces[space_name], space_stats, + 'Space stats is a section of total stats') + + local op_stats = space_stats[op] + t.assert_type(op_stats, 'table', 'Op stats present after observations for the space') + + -- Expected collectors (changed_coll: 'ok' or 'error') have changed. + local changed = op_stats[case.changed_coll] + t.assert_type(changed, 'table', 'Status stats present after observations') + + t.assert_equals(changed.count, run_count, 'Count incremented by count of runs') + + local sleep_time = helpers.simple_functions_params().sleep_time + t.assert_ge(changed.latency, sleep_time, 'Latency has appropriate value') + t.assert_le(changed.latency, time_diffs[#time_diffs], 'Latency has appropriate value') + + t.assert_ge(changed.time, sleep_time * run_count, + 'Total time increase has appropriate value') + t.assert_le(changed.time, total_time, 'Total time increase has appropriate value') + + -- Other collectors (unchanged_coll: 'error' or 'ok') + -- have been initialized and have default values. + local unchanged = op_stats[case.unchanged_coll] + t.assert_type(unchanged, 'table', 'Other status stats present after observations') + + t.assert_equals( + unchanged, + { + count = 0, + latency = 0, + time = 0 + }, + 'Other status collectors initialized after observations' + ) + end + end +end + +local pairs_cases = { + success_run = { + prepare = [[ + local params = ... + local sleep_time = params.sleep_time + + local function sleep_ten_times(param, state) + if state == 10 then + return nil + end + + sleep_for(sleep_time) + + return state + 1, param + end + rawset(_G, 'sleep_ten_times', sleep_ten_times) + ]], + eval = [[ + local params, space_name, op = ... + local sleep_time = params.sleep_time + + local build_sleep_multiplier = 2 + + local wrapped = stats_module.wrap( + function(space_name) + sleep_for(build_sleep_multiplier * sleep_time) + + return sleep_ten_times, {}, 0 + end, + op, + { pairs = true } + ) + + for _, _ in wrapped(space_name) do end + ]], + build_sleep_multiplier = 2, + iterations_expected = 10, + changed_coll = 'ok', + unchanged_coll = 'error', + }, + error_throw = { + prepare = [[ + local params = ... + local sleep_time = params.sleep_time + local error_table = params.error + + + local function sleep_five_times_and_throw_error(param, state) + if state == 5 then + error(error_table) + end + + sleep_for(sleep_time) + + return state + 1, param + end + rawset(_G, 'sleep_five_times_and_throw_error', sleep_five_times_and_throw_error) + ]], + eval = [[ + local params, space_name, op = ... + local sleep_time = params.sleep_time + + local build_sleep_multiplier = 2 + + local wrapped = stats_module.wrap( + function(space_name) + sleep_for(build_sleep_multiplier * sleep_time) + + return sleep_five_times_and_throw_error, {}, 0 + end, + op, + { pairs = true } + ) + + for _, _ in wrapped(space_name) do end + ]], + build_sleep_multiplier = 2, + iterations_expected = 5, + changed_coll = 'error', + unchanged_coll = 'ok', + pcall = true, + }, + break_after_gc = { + prepare = [[ + local params = ... + local sleep_time = params.sleep_time + + local function sleep_ten_times(param, state) + if state == 10 then + return nil + end + + sleep_for(sleep_time) + + return state + 1, param + end + rawset(_G, 'sleep_ten_times', sleep_ten_times) + ]], + eval = [[ + local params, space_name, op = ... + local sleep_time = params.sleep_time + + local build_sleep_multiplier = 2 + + local wrapped = stats_module.wrap( + function(space_name) + sleep_for(build_sleep_multiplier * sleep_time) + + return sleep_ten_times, {}, 0 + end, + op, + { pairs = true } + ) + + for i, _ in wrapped(space_name) do + if i == 5 then + break + end + end + ]], + post_eval = [[ + collectgarbage('collect') + collectgarbage('collect') + ]], + build_sleep_multiplier = 2, + iterations_expected = 5, + changed_coll = 'ok', + unchanged_coll = 'error', + } +} + +for name, case in pairs(pairs_cases) do + local test_name = ('test_pairs_wrapper_observes_all_iterations_on_%s'):format(name) + + g.before_test(test_name, function(g) + g.router:eval(case.prepare, { helpers.simple_functions_params() }) + end) + + g[test_name] = function(g) + local op = stats_module.op.SELECT + + local params = helpers.simple_functions_params() + local args = { params, space_name, op } + + local before_start = clock.monotonic() + + if case.pcall then + pcall(g.router.eval, g.router, case.eval, args) + else + g.router:eval(case.eval, args) + end + + if case.post_eval then + g.router:eval(case.post_eval) + end + + local after_finish = clock.monotonic() + local time_diff = after_finish - before_start + + -- Validate stats format after execution. + local total_stats = g:get_stats() + t.assert_type(total_stats, 'table', 'Total stats present after observations') + + local space_stats = g:get_stats(space_name) + t.assert_type(space_stats, 'table', 'Space stats present after observations') + + t.assert_equals(total_stats.spaces[space_name], space_stats, + 'Space stats is a section of total stats') + + local op_stats = space_stats[op] + t.assert_type(op_stats, 'table', 'Op stats present after observations for the space') + + -- Expected collectors (changed_coll: 'ok' or 'error') have changed. + local changed = op_stats[case.changed_coll] + t.assert_type(changed, 'table', 'Status stats present after observations') + + t.assert_equals(changed.count, 1, 'Count incremented by 1') + + t.assert_ge(changed.latency, + params.sleep_time * (case.build_sleep_multiplier + case.iterations_expected), + 'Latency has appropriate value') + t.assert_le(changed.latency, time_diff, 'Latency has appropriate value') + + t.assert_ge(changed.time, + params.sleep_time * (case.build_sleep_multiplier + case.iterations_expected), + 'Total time has appropriate value') + t.assert_le(changed.time, time_diff, 'Total time has appropriate value') + + -- Other collectors (unchanged_coll: 'error' or 'ok') + -- have been initialized and have default values. + local unchanged = op_stats[case.unchanged_coll] + t.assert_type(unchanged, 'table', 'Other status stats present after observations') + + t.assert_equals( + unchanged, + { + count = 0, + latency = 0, + time = 0 + }, + 'Other status collectors initialized after observations' + ) + end +end + +-- Test wrapper preserves return values. +local disable_stats_cases = { + stats_disable_before_wrap_ = { + before_wrap = 'stats_module.disable()', + after_wrap = '', + }, + stats_disable_after_wrap_ = { + before_wrap = '', + after_wrap = 'stats_module.disable()', + }, + [''] = { + before_wrap = '', + after_wrap = '', + }, +} + +local preserve_return_cases = { + wrapper_preserves_return_values_on_ok = { + func = 'return_true', + res = true, + err = nil, + }, + wrapper_preserves_return_values_on_error = { + func = 'return_err', + res = nil, + err = helpers.simple_functions_params().error, + }, +} + +local preserve_throw_cases = { + wrapper_preserves_error_throw = { + opts = { pairs = false }, + }, + pairs_wrapper_preserves_error_throw = { + opts = { pairs = true }, + }, +} + +for name_head, disable_case in pairs(disable_stats_cases) do + for name_tail, return_case in pairs(preserve_return_cases) do + local test_name = ('test_%s%s'):format(name_head, name_tail) + + g[test_name] = function(g) + local op = stats_module.op.INSERT + + local eval = ([[ + local func = rawget(_G, select(1, ...)) + local op = select(2, ...) + local space_name = select(3, ...) + + %s -- before_wrap + local w_func = stats_module.wrap(func, op) + %s -- after_wrap + + return w_func(space_name) + ]]):format(disable_case.before_wrap, disable_case.after_wrap) + + local res, err = g.router:eval(eval, { return_case.func, op, space_name }) + + t.assert_equals(res, return_case.res, 'Wrapper preserves first return value') + t.assert_equals(err, return_case.err, 'Wrapper preserves second return value') + end + end + + local test_name = ('test_%spairs_wrapper_preserves_return_values'):format(name_head) + + g[test_name] = function(g) + local op = stats_module.op.INSERT + + local input = { a = 'a', b = 'b' } + local eval = ([[ + local input = select(1, ...) + local func = function() return pairs(input) end + local op = select(2, ...) + local space_name = select(3, ...) + + %s -- before_wrap + local w_func = stats_module.wrap(func, op, { pairs = true }) + %s -- after_wrap + + local res = {} + for k, v in w_func(space_name) do + res[k] = v + end + + return res + ]]):format(disable_case.before_wrap, disable_case.after_wrap) + + local res = g.router:eval(eval, { input, op, space_name }) + + t.assert_equals(input, res, 'Wrapper preserves pairs return values') + end + + for name_tail, throw_case in pairs(preserve_throw_cases) do + local test_name = ('test_%s%s'):format(name_head, name_tail) + + g[test_name] = function(g) + local op = stats_module.op.INSERT + + local eval = ([[ + local func = rawget(_G, 'throws_error') + local opts = select(1, ...) + local op = select(2, ...) + local space_name = select(3, ...) + + %s -- before_wrap + local w_func = stats_module.wrap(func, op, opts) + %s -- after_wrap + + w_func(space_name) + ]]):format(disable_case.before_wrap, disable_case.after_wrap) + + t.assert_error_msg_contains( + helpers.simple_functions_params().error_msg, + g.router.eval, g.router, eval, { throw_case.opts, op, space_name } + ) + end + end +end + + +g.test_stats_is_empty_after_disable = function(g) + g:disable_stats() + + local op = stats_module.op.INSERT + g.router:eval(call_wrapped, { 'return_true', op, {}, space_name }) + + local stats = g:get_stats() + t.assert_equals(stats, {}) +end + + +local function prepare_non_default_stats(g) + local op = stats_module.op.INSERT + g.router:eval(call_wrapped, { 'return_true', op, {}, space_name }) + + local stats = g:get_stats(space_name) + t.assert_equals(stats[op].ok.count, 1, 'Non-zero stats prepared') + + return stats +end + +g.test_enable_is_idempotent = function(g) + local stats_before = prepare_non_default_stats(g) + + g:enable_stats() + + local stats_after = g:get_stats(space_name) + + t.assert_equals(stats_after, stats_before, 'Stats have not been reset') +end + +g.test_reset = function(g) + prepare_non_default_stats(g) + + g:reset_stats() + + local stats = g:get_stats(space_name) + + t.assert_equals(stats, {}, 'Stats have been reset') +end + +g.test_reset_for_disabled_stats_does_not_init_module = function(g) + g:disable_stats() + + local stats_before = g:get_stats() + t.assert_equals(stats_before, {}, "Stats is empty") + + g:reset_stats() + + local stats_after = g:get_stats() + t.assert_equals(stats_after, {}, "Stats is still empty") +end