From 6da4f5684a00fe39106ac139538b06821c8c829c Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Mon, 13 Dec 2021 15:20:23 +0300 Subject: [PATCH] stats: add statistics for CRUD router operations Add statistics module for collecting metrics of CRUD operations on router. Wrap all CRUD operation calls in the statistics collector. Statistics must be enabled manually with `crud.cfg`. They can be disabled, restarted or re-enabled later. This patch introduces `crud.cfg`. `crud.cfg` is a tool to set module configuration. It is similar to Tarantool `box.cfg`, although we don't need to call it to bootstrap the module -- it is used only to change configuration. `crud.cfg` is a callable table. To change configuration, call it: `crud.cfg{ stats = true }`. You can check table contents as with ordinary table, but do not change them directly -- use call instead. Table contents is immutable and use proxy approach (see [1, 2]). Iterating through `crud.cfg` with pairs is not supported yet, refer to tarantool/crud#265. `crud.stats()` returns --- - spaces: my_space: insert: ok: latency: 0.002 count: 19800 time: 39.6 error: latency: 0.000001 count: 4 time: 0.000004 ... `spaces` section contains statistics for each observed space. If operation has never been called for a space, the corresponding field will be empty. If no requests has been called for a space, it will not be represented. Space data is based on client requests rather than storages schema, so requests for non-existing spaces are also collected. Possible statistics operation labels are `insert` (for `insert` and `insert_object` calls), `get`, `replace` (for `replace` and `replace_object` calls), `update`, `upsert` (for `upsert` and `upsert_object` calls), `delete`, `select` (for `select` and `pairs` calls), `truncate`, `len`, `count` and `borders` (for `min` and `max` calls). Each operation section consists of different collectors for success calls and error (both error throw and `nil, err`) returns. `count` is the total requests count since instance start or stats restart. `latency` is the average time of requests execution, `time` is the total time of requests execution. Since `pairs` request behavior differs from any other crud request, its statistics collection also has specific behavior. Statistics (`select` section) are updated after `pairs` cycle is finished: you either have iterated through all records or an error was thrown. If your pairs cycle was interrupted with `break`, statistics will be collected when pairs objects are cleaned up with Lua garbage collector. Statistics are preserved between package reloads. Statistics are preserved between Tarantool Cartridge role reloads [3] if CRUD Cartridge roles are used. 1. http://lua-users.org/wiki/ReadOnlyTables 2. https://github.com/tarantool/tarantool/issues/2867 3. https://www.tarantool.io/en/doc/latest/book/cartridge/cartridge_api/modules/cartridge.roles/#reload Part of #224 --- CHANGELOG.md | 1 + README.md | 91 +++++ cartridge/roles/crud-router.lua | 2 + cartridge/roles/crud-storage.lua | 2 + crud.lua | 46 ++- crud/cfg.lua | 70 ++++ crud/common/stash.lua | 63 ++++ crud/stats/init.lua | 288 ++++++++++++++++ crud/stats/local_registry.lua | 101 ++++++ crud/stats/operation.lua | 23 ++ crud/stats/registry_utils.lua | 60 ++++ deps.sh | 2 +- test/entrypoint/srv_stats.lua | 63 ++++ test/helper.lua | 88 +++++ test/integration/cfg_test.lua | 74 +++++ test/integration/reload_test.lua | 12 +- test/integration/stats_test.lua | 509 ++++++++++++++++++++++++++++ test/unit/stats_test.lua | 555 +++++++++++++++++++++++++++++++ 18 files changed, 2023 insertions(+), 27 deletions(-) create mode 100644 crud/cfg.lua create mode 100644 crud/common/stash.lua create mode 100644 crud/stats/init.lua create mode 100644 crud/stats/local_registry.lua create mode 100644 crud/stats/operation.lua create mode 100644 crud/stats/registry_utils.lua create mode 100755 test/entrypoint/srv_stats.lua create mode 100644 test/integration/cfg_test.lua create mode 100644 test/integration/stats_test.lua create mode 100644 test/unit/stats_test.lua diff --git a/CHANGELOG.md b/CHANGELOG.md index 3093ddef..5ebfd203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +* Statistics for CRUD operations on router (#224). ### Changed diff --git a/README.md b/README.md index 7ce5f820..6a419017 100644 --- a/README.md +++ b/README.md @@ -674,6 +674,97 @@ Combinations of `mode`, `prefer_replica` and `balance` options lead to: * prefer_replica, balance - [vshard call `callbre`](https://www.tarantool.io/en/doc/latest/reference/reference_rock/vshard/vshard_api/#router-api-callbre) +### Statistics + +`crud` routers can provide statistics on called operations. +```lua +-- Enable statistics collect. +crud.cfg{ stats = true } + +-- Returns table with statistics information. +crud.stats() + +-- Returns table with statistics information for specific space. +crud.stats('my_space') + +-- Disables statistics collect and destroys all collectors. +crud.cfg{ stats = false } + +-- Destroys all statistics collectors and creates them again. +crud.reset_stats() +``` + +You can use `crud.cfg` to check current stats state. +```lua +crud.cfg +--- +- stats: true +... +``` +Beware that iterating through `crud.cfg` with pairs is not supported yet, +refer to [tarantool/crud#265](https://github.com/tarantool/crud/issues/265). + +Format is as follows. +```lua +crud.stats() +--- +- spaces: + my_space: + insert: + ok: + latency: 0.002 + count: 19800 + time: 39.6 + error: + latency: 0.000001 + count: 4 + time: 0.000004 +... +crud.stats('my_space') +--- +- insert: + ok: + latency: 0.002 + count: 19800 + time: 39.6 + error: + latency: 0.000001 + count: 4 + time: 0.000004 +... +``` +`spaces` section contains statistics for each observed space. +If operation has never been called for a space, the corresponding +field will be empty. If no requests has been called for a +space, it will not be represented. Space data is based on +client requests rather than storages schema, so requests +for non-existing spaces are also collected. + +Possible statistics operation labels are +`insert` (for `insert` and `insert_object` calls), +`get`, `replace` (for `replace` and `replace_object` calls), `update`, +`upsert` (for `upsert` and `upsert_object` calls), `delete`, +`select` (for `select` and `pairs` calls), `truncate`, `len`, `count` +and `borders` (for `min` and `max` calls). + +Each operation section contains of different collectors +for success calls and error (both error throw and `nil, err`) +returns. `count` is total requests count since instance start +or stats restart. `latency` is average time of requests execution, +`time` is the total time of requests execution. + +Since `pairs` request behavior differs from any other crud request, its +statistics collection also has specific behavior. Statistics (`select` +section) are updated after `pairs` cycle is finished: you +either have iterated through all records or an error was thrown. +If your pairs cycle was interrupted with `break`, statistics will +be collected when pairs objects are cleaned up with Lua garbage +collector. + +Statistics are preserved between package reloads. Statistics are preserved +between [Tarantool Cartridge role reloads](https://www.tarantool.io/en/doc/latest/book/cartridge/cartridge_api/modules/cartridge.roles/#reload) +if you use CRUD Cartridge roles. + ## Cartridge roles `cartridge.roles.crud-storage` is a Tarantool Cartridge role that depends on the diff --git a/cartridge/roles/crud-router.lua b/cartridge/roles/crud-router.lua index ef510e51..1c4d43fe 100644 --- a/cartridge/roles/crud-router.lua +++ b/cartridge/roles/crud-router.lua @@ -1,8 +1,10 @@ local crud = require('crud') +local stash = require('crud.common.stash') -- removes routes that changed in config and adds new routes local function init() crud.init_router() + stash.setup_cartridge_reload() end local function stop() diff --git a/cartridge/roles/crud-storage.lua b/cartridge/roles/crud-storage.lua index 8371c428..3728c88c 100644 --- a/cartridge/roles/crud-storage.lua +++ b/cartridge/roles/crud-storage.lua @@ -1,7 +1,9 @@ local crud = require('crud') +local stash = require('crud.common.stash') local function init() crud.init_storage() + stash.setup_cartridge_reload() end local function stop() diff --git a/crud.lua b/crud.lua index b045e7fe..3f2b5c59 100644 --- a/crud.lua +++ b/crud.lua @@ -2,6 +2,7 @@ -- -- @module crud +local cfg = require('crud.cfg') local insert = require('crud.insert') local replace = require('crud.replace') local get = require('crud.get') @@ -15,6 +16,7 @@ local count = require('crud.count') local borders = require('crud.borders') local sharding_metadata = require('crud.common.sharding.sharding_metadata') local utils = require('crud.common.utils') +local stats = require('crud.stats') local crud = {} @@ -23,47 +25,47 @@ local crud = {} -- @refer insert.tuple -- @function insert -crud.insert = insert.tuple +crud.insert = stats.wrap(insert.tuple, stats.op.INSERT) -- @refer insert.object -- @function insert_object -crud.insert_object = insert.object +crud.insert_object = stats.wrap(insert.object, stats.op.INSERT) -- @refer get.call -- @function get -crud.get = get.call +crud.get = stats.wrap(get.call, stats.op.GET) -- @refer replace.tuple -- @function replace -crud.replace = replace.tuple +crud.replace = stats.wrap(replace.tuple, stats.op.REPLACE) -- @refer replace.object -- @function replace_object -crud.replace_object = replace.object +crud.replace_object = stats.wrap(replace.object, stats.op.REPLACE) -- @refer update.call -- @function update -crud.update = update.call +crud.update = stats.wrap(update.call, stats.op.UPDATE) -- @refer upsert.tuple -- @function upsert -crud.upsert = upsert.tuple +crud.upsert = stats.wrap(upsert.tuple, stats.op.UPSERT) -- @refer upsert.object -- @function upsert -crud.upsert_object = upsert.object +crud.upsert_object = stats.wrap(upsert.object, stats.op.UPSERT) -- @refer delete.call -- @function delete -crud.delete = delete.call +crud.delete = stats.wrap(delete.call, stats.op.DELETE) -- @refer select.call -- @function select -crud.select = select.call +crud.select = stats.wrap(select.call, stats.op.SELECT) -- @refer select.pairs -- @function pairs -crud.pairs = select.pairs +crud.pairs = stats.wrap(select.pairs, stats.op.SELECT, { pairs = true }) -- @refer utils.unflatten_rows -- @function unflatten_rows @@ -71,23 +73,23 @@ crud.unflatten_rows = utils.unflatten_rows -- @refer truncate.call -- @function truncate -crud.truncate = truncate.call +crud.truncate = stats.wrap(truncate.call, stats.op.TRUNCATE) -- @refer len.call -- @function len -crud.len = len.call +crud.len = stats.wrap(len.call, stats.op.LEN) -- @refer count.call -- @function count -crud.count = count.call +crud.count = stats.wrap(count.call, stats.op.COUNT) -- @refer borders.min -- @function min -crud.min = borders.min +crud.min = stats.wrap(borders.min, stats.op.BORDERS) -- @refer borders.max -- @function max -crud.max = borders.max +crud.max = stats.wrap(borders.max, stats.op.BORDERS) -- @refer utils.cut_rows -- @function cut_rows @@ -97,6 +99,18 @@ crud.cut_rows = utils.cut_rows -- @function cut_objects crud.cut_objects = utils.cut_objects +-- @refer cfg.cfg +-- @function cfg +crud.cfg = cfg.cfg + +-- @refer stats.get +-- @function stats +crud.stats = stats.get + +-- @refer stats.reset +-- @function reset_stats +crud.reset_stats = stats.reset + --- Initializes crud on node -- -- Exports all functions that are used for calls diff --git a/crud/cfg.lua b/crud/cfg.lua new file mode 100644 index 00000000..2525a947 --- /dev/null +++ b/crud/cfg.lua @@ -0,0 +1,70 @@ +---- Module for CRUD configuration. +-- @module crud.cfg +-- + +local checks = require('checks') +local errors = require('errors') + +local stash = require('crud.common.stash') +local stats = require('crud.stats') + +local CfgError = errors.new_class('CfgError', {capture_stack = false}) + +local cfg_module = {} + +local function set_defaults_if_empty(cfg) + if cfg.stats == nil then + cfg.stats = false + end + + return cfg +end + +local cfg = set_defaults_if_empty(stash.get(stash.name.cfg)) + +--- Configure CRUD module. +-- +-- @function __call +-- +-- @tab self +-- +-- @tab[opt] opts +-- +-- @bool[opt] opts.stats +-- Enable or disable statistics collect. +-- Statistics are observed only on router instances. +-- +-- @return Configuration table. +-- +local function __call(self, opts) + checks('table', { stats = '?boolean' }) + + opts = opts or {} + + if opts.stats ~= nil then + if opts.stats == true then + stats.enable() + else + stats.disable() + end + + rawset(cfg, 'stats', opts.stats) + end + + return self +end + +local function __newindex() + CfgError:assert(false, 'Use crud.cfg{} instead') +end + +-- Iterating through `crud.cfg` with pairs is not supported +-- yet, refer to tarantool/crud#265. +cfg_module.cfg = setmetatable({}, { + __index = cfg, + __newindex = __newindex, + __call = __call, + __serialize = function() return cfg end +}) + +return cfg_module diff --git a/crud/common/stash.lua b/crud/common/stash.lua new file mode 100644 index 00000000..3cb65ed1 --- /dev/null +++ b/crud/common/stash.lua @@ -0,0 +1,63 @@ +---- Module for preserving data between reloads. +-- @module crud.common.stash +-- +local dev_checks = require('crud.common.dev_checks') + +local stash = {} + +--- Available stashes list. +-- +-- @tfield string cfg +-- Stash for CRUD module configuration. +-- +-- @tfield string stats_internal +-- Stash for main stats module. +-- +-- @tfield string stats_local_registry +-- Stash for local metrics registry. +-- +stash.name = { + cfg = '__crud_cfg', + stats_internal = '__crud_stats_internal', + stats_local_registry = '__crud_stats_local_registry' +} + +--- Setup Tarantool Cartridge reload. +-- +-- Call on Tarantool Cartridge roles that are expected +-- to use stashes. +-- +-- @function setup_cartridge_reload +-- +-- @return Returns +-- +function stash.setup_cartridge_reload() + local hotreload = require('cartridge.hotreload') + for _, name in pairs(stash.name) do + hotreload.whitelist_globals({ name }) + end +end + +--- Get a stash instance, initialize if needed. +-- +-- Stashes are persistent to package reload. +-- To use them with Cartridge roles reload, +-- call `stash.setup_cartridge_reload` in role. +-- +-- @function get +-- +-- @string name +-- Stash identifier. Use one from `stash.name` table. +-- +-- @treturn table A stash instance. +-- +function stash.get(name) + dev_checks('string') + + local instance = rawget(_G, name) or {} + rawset(_G, name, instance) + + return instance +end + +return stash diff --git a/crud/stats/init.lua b/crud/stats/init.lua new file mode 100644 index 00000000..c2555791 --- /dev/null +++ b/crud/stats/init.lua @@ -0,0 +1,288 @@ +---- CRUD statistics module. +-- @module crud.stats +-- + +local clock = require('clock') +local checks = require('checks') +local fun = require('fun') + +local dev_checks = require('crud.common.dev_checks') +local stash = require('crud.common.stash') +local op_module = require('crud.stats.operation') +local registry = require('crud.stats.local_registry') + +local stats = {} +local internal = stash.get(stash.name.stats_internal) + +--- Check if statistics module was enabled. +-- +-- @function is_enabled +-- +-- @treturn boolean Returns `true` or `false`. +-- +function stats.is_enabled() + return internal.is_enabled == true +end + +--- Initializes statistics registry, enables callbacks and wrappers. +-- +-- If already enabled, do nothing. +-- +-- @function enable +-- +-- @treturn boolean Returns `true`. +-- +function stats.enable() + if stats.is_enabled() then + return true + end + + internal.is_enabled = true + registry.init() + + return true +end + +--- Resets statistics registry. +-- +-- After reset collectors are the same as right +-- after initial `stats.enable()`. +-- +-- @function reset +-- +-- @treturn boolean Returns true. +-- +function stats.reset() + if not stats.is_enabled() then + return true + end + + registry.destroy() + registry.init() + + return true +end + +--- Destroys statistics registry and disable callbacks. +-- +-- If already disabled, do nothing. +-- +-- @function disable +-- +-- @treturn boolean Returns true. +-- +function stats.disable() + if not stats.is_enabled() then + return true + end + + registry.destroy() + internal.is_enabled = false + + return true +end + +--- Get statistics on CRUD operations. +-- +-- @function get +-- +-- @string[opt] space_name +-- If specified, returns table with statistics +-- of operations on space, separated by operation type and +-- execution status. If there wasn't any requests of "op" type +-- for space, there won't be corresponding collectors. +-- If not specified, returns table with statistics +-- about all observed spaces. +-- +-- @treturn table Statistics on CRUD operations. +-- If statistics disabled, returns `{}`. +-- +function stats.get(space_name) + checks('?string') + + if not stats.is_enabled() then + return {} + end + + return registry.get(space_name) +end + +-- Hack to set __gc for a table in Lua 5.1 +-- See https://stackoverflow.com/questions/27426704/lua-5-1-workaround-for-gc-metamethod-for-tables +-- or https://habr.com/ru/post/346892/ +local function setmt__gc(t, mt) + local prox = newproxy(true) + getmetatable(prox).__gc = function() mt.__gc(t) end + t[prox] = true + return setmetatable(t, mt) +end + +local function wrap_pairs_gen(build_latency, space_name, op, gen, param, state) + local total_latency = build_latency + + -- If pairs() cycle will be interrupted with break, + -- we'll never get a proper obervation. + -- We create an object with the same lifespan as gen() + -- function so if someone break pairs cycle, + -- it still will be observed. + local observed = false + + local gc_observer = setmt__gc({}, { + __gc = function() + if observed == false then + registry.observe(total_latency, space_name, op, 'ok') + observed = true + end + end + }) + + local wrapped_gen = function(param, state) + -- Mess with gc_observer so its lifespan will + -- be the same as wrapped_gen() function. + gc_observer[1] = state + + local start_time = clock.monotonic() + + local status, next_state, var = pcall(gen, param, state) + + local finish_time = clock.monotonic() + + total_latency = total_latency + (finish_time - start_time) + + if status == false then + registry.observe(total_latency, space_name, op, 'error') + observed = true + error(next_state, 2) + end + + -- Observe stats in the end of pairs cycle + if var == nil then + registry.observe(total_latency, space_name, op, 'ok') + observed = true + return nil + end + + return next_state, var + end + + return fun.wrap(wrapped_gen, param, state) +end + +local function wrap_tail(space_name, op, pairs, start_time, call_status, ...) + dev_checks('string', 'string', 'boolean', 'number', 'boolean') + + local finish_time = clock.monotonic() + local latency = finish_time - start_time + + if call_status == false then + registry.observe(latency, space_name, op, 'error') + error((...), 2) + end + + if pairs == false then + if select(2, ...) ~= nil then + -- If not `pairs` call, return values `nil, err` + -- treated as error case. + registry.observe(latency, space_name, op, 'error') + return ... + else + registry.observe(latency, space_name, op, 'ok') + return ... + end + else + return wrap_pairs_gen(latency, space_name, op, ...) + end +end + +--- Wrap CRUD operation call to collect statistics. +-- +-- Approach based on `box.atomic()`: +-- https://github.com/tarantool/tarantool/blob/b9f7204b5e0d10b443c6f198e9f7f04e0d16a867/src/box/lua/schema.lua#L369 +-- +-- @function wrap +-- +-- @func func +-- Function to wrap. First argument is expected to +-- be a space name string. If statistics enabled, +-- errors are caught and thrown again. +-- +-- @string op +-- Label of registry collectors. +-- Use `require('crud.stats').op` to pick one. +-- +-- @tab[opt] opts +-- +-- @bool[opt=false] opts.pairs +-- If false, wraps only function passed as argument. +-- Second return value of wrapped function is treated +-- as error (`nil, err` case). +-- If true, also wraps gen() function returned by +-- call. Statistics observed on cycle end (last +-- element was fetched or error was thrown). If pairs +-- cycle was interrupted with `break`, statistics will +-- be collected when pairs objects are cleaned up with +-- Lua garbage collector. +-- +-- @return Wrapped function output. +-- +function stats.wrap(func, op, opts) + dev_checks('function', 'string', { pairs = '?boolean' }) + + local pairs + if type(opts) == 'table' and opts.pairs ~= nil then + pairs = opts.pairs + else + pairs = false + end + + return function(space_name, ...) + if not stats.is_enabled() then + return func(space_name, ...) + end + + local start_time = clock.monotonic() + + return wrap_tail( + space_name, op, pairs, start_time, + pcall(func, space_name, ...) + ) + end +end + +--- Table with CRUD operation lables. +-- +-- @tfield string INSERT +-- Identifies both `insert` and `insert_object`. +-- +-- @tfield string GET +-- +-- @tfield string REPLACE +-- Identifies both `replace` and `replace_object`. +-- +-- @tfield string UPDATE +-- +-- @tfield string UPSERT +-- Identifies both `upsert` and `upsert_object`. +-- +-- @tfield string DELETE +-- +-- @tfield string SELECT +-- Identifies both `pairs` and `select`. +-- +-- @tfield string TRUNCATE +-- +-- @tfield string LEN +-- +-- @tfield string COUNT +-- +-- @tfield string BORDERS +-- Identifies both `min` and `max`. +-- +stats.op = op_module + +--- Stats module internal state (for debug/test). +-- +-- @tfield[opt] boolean is_enabled Is currently enabled. +stats.internal = internal + +return stats diff --git a/crud/stats/local_registry.lua b/crud/stats/local_registry.lua new file mode 100644 index 00000000..c5e125f1 --- /dev/null +++ b/crud/stats/local_registry.lua @@ -0,0 +1,101 @@ +---- Internal module used to store statistics. +-- @module crud.stats.local_registry +-- + +local dev_checks = require('crud.common.dev_checks') +local stash = require('crud.common.stash') +local registry_utils = require('crud.stats.registry_utils') + +local registry = {} +local internal = stash.get(stash.name.stats_local_registry) + +--- Initialize local metrics registry. +-- +-- Registries are not meant to used explicitly +-- by users, init is not guaranteed to be idempotent. +-- +-- @function init +-- +-- @treturn boolean Returns true. +-- +function registry.init() + internal.registry = {} + internal.registry.spaces = {} + + return true +end + +--- Destroy local metrics registry. +-- +-- Registries are not meant to used explicitly +-- by users, destroy is not guaranteed to be idempotent. +-- +-- @function destroy +-- +-- @treturn boolean Returns `true`. +-- +function registry.destroy() + internal.registry = nil + + return true +end + +--- Get copy of local metrics registry. +-- +-- Registries are not meant to used explicitly +-- by users, get is not guaranteed to work without init. +-- +-- @function get +-- +-- @string[opt] space_name +-- If specified, returns table with statistics +-- of operations on table, separated by operation type and +-- execution status. If there wasn't any requests for table, +-- returns `{}`. If not specified, returns table with statistics +-- about all observed spaces. +-- +-- @treturn table Returns copy of metrics registry (or registry section). +-- +function registry.get(space_name) + dev_checks('?string') + + if space_name ~= nil then + return table.deepcopy(internal.registry.spaces[space_name]) or {} + end + + return table.deepcopy(internal.registry) +end + +--- Increase requests count and update latency info. +-- +-- @function observe +-- +-- @string space_name +-- Name of space. +-- +-- @number latency +-- Time of call execution. +-- +-- @string op +-- Label of registry collectors. +-- Use `require('crud.stats').op` to pick one. +-- +-- @string success +-- `'ok'` if no errors on execution, `'error'` otherwise. +-- +-- @treturn boolean Returns `true`. +-- +function registry.observe(latency, space_name, op, status) + dev_checks('number', 'string', 'string', 'string') + + registry_utils.init_collectors_if_required(internal.registry.spaces, space_name, op) + local collectors = internal.registry.spaces[space_name][op][status] + + collectors.count = collectors.count + 1 + collectors.time = collectors.time + latency + collectors.latency = collectors.time / collectors.count + + return true +end + +return registry diff --git a/crud/stats/operation.lua b/crud/stats/operation.lua new file mode 100644 index 00000000..a6a9627a --- /dev/null +++ b/crud/stats/operation.lua @@ -0,0 +1,23 @@ +-- It is not clear how to describe modules +-- with constants for ldoc. ldoc-styled description +-- for this module is available at `crud.stats.init`. +-- See https://github.com/lunarmodules/LDoc/issues/369 +-- for possible updates. +return { + -- INSERT identifies both `insert` and `insert_object`. + INSERT = 'insert', + GET = 'get', + -- REPLACE identifies both `replace` and `replace_object`. + REPLACE = 'replace', + UPDATE = 'update', + -- UPSERT identifies both `upsert` and `upsert_object`. + UPSERT = 'upsert', + DELETE = 'delete', + -- SELECT identifies both `pairs` and `select`. + SELECT = 'select', + TRUNCATE = 'truncate', + LEN = 'len', + COUNT = 'count', + -- BORDERS identifies both `min` and `max`. + BORDERS = 'borders', +} diff --git a/crud/stats/registry_utils.lua b/crud/stats/registry_utils.lua new file mode 100644 index 00000000..2c99f8a3 --- /dev/null +++ b/crud/stats/registry_utils.lua @@ -0,0 +1,60 @@ +---- Internal module used by statistics registries. +-- @module crud.stats.registry_utils +-- + +local dev_checks = require('crud.common.dev_checks') + +local registry_utils = {} + +--- Build collectors for local registry. +-- +-- @function build_collectors +-- +-- @treturn table Returns collectors for success and error requests. +-- Collectors store 'count', 'latency' and 'time' values. +-- +function registry_utils.build_collectors() + local collectors = { + ok = { + count = 0, + latency = 0, + time = 0, + }, + error = { + count = 0, + latency = 0, + time = 0, + }, + } + + return collectors +end + +--- Initialize all statistic collectors for a space operation. +-- +-- @function init_collectors_if_required +-- +-- @tab spaces +-- `spaces` section of registry. +-- +-- @string space_name +-- Name of space. +-- +-- @string op +-- Label of registry collectors. +-- Use `require('crud.stats').op` to pick one. +-- +function registry_utils.init_collectors_if_required(spaces, space_name, op) + dev_checks('table', 'string', 'string') + + if spaces[space_name] == nil then + spaces[space_name] = {} + end + + local space_collectors = spaces[space_name] + if space_collectors[op] == nil then + space_collectors[op] = registry_utils.build_collectors() + end +end + +return registry_utils diff --git a/deps.sh b/deps.sh index 87ce6b92..abff8f4b 100755 --- a/deps.sh +++ b/deps.sh @@ -4,7 +4,7 @@ set -e # Test dependencies: -tarantoolctl rocks install luatest 0.5.5 +tarantoolctl rocks install luatest 0.5.7 tarantoolctl rocks install luacheck 0.25.0 tarantoolctl rocks install luacov 0.13.0 diff --git a/test/entrypoint/srv_stats.lua b/test/entrypoint/srv_stats.lua new file mode 100755 index 00000000..d9649b70 --- /dev/null +++ b/test/entrypoint/srv_stats.lua @@ -0,0 +1,63 @@ +#!/usr/bin/env tarantool + +require('strict').on() +_G.is_initialized = function() return false end + +local log = require('log') +local errors = require('errors') +local cartridge = require('cartridge') + +package.preload['customers-storage'] = function() + local engine = os.getenv('ENGINE') or 'memtx' + return { + role_name = 'customers-storage', + init = function() + local customers_space = box.schema.space.create('customers', { + format = { + {name = 'id', type = 'unsigned'}, + {name = 'bucket_id', type = 'unsigned'}, + {name = 'name', type = 'string'}, + {name = 'last_name', type = 'string'}, + {name = 'age', type = 'number'}, + {name = 'city', type = 'string'}, + }, + if_not_exists = true, + engine = engine, + }) + -- primary index + customers_space:create_index('id_index', { + parts = { {field = 'id'} }, + if_not_exists = true, + }) + customers_space:create_index('bucket_id', { + parts = { {field = 'bucket_id'} }, + unique = false, + if_not_exists = true, + }) + customers_space:create_index('age_index', { + parts = { {field = 'age'} }, + unique = false, + if_not_exists = true, + }) + end, + } +end + +local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { + advertise_uri = 'localhost:3301', + http_port = 8081, + bucket_count = 3000, + roles = { + 'cartridge.roles.crud-router', + 'cartridge.roles.crud-storage', + 'customers-storage', + }, + roles_reload_allowed = true, +}) + +if not ok then + log.error('%s', err) + os.exit(1) +end + +_G.is_initialized = cartridge.is_healthy diff --git a/test/helper.lua b/test/helper.lua index f2cdb6ab..669dec07 100644 --- a/test/helper.lua +++ b/test/helper.lua @@ -378,4 +378,92 @@ function helpers.get_sharding_func_cache_size(cluster) ]]) end +function helpers.simple_functions_params() + return { + sleep_time = 0.01, + error = { err = 'err' }, + error_msg = 'throw me', + } +end + +function helpers.prepare_simple_functions(router) + local params = helpers.simple_functions_params() + + local _, err = router:eval([[ + local clock = require('clock') + local fiber = require('fiber') + + local params = ... + local sleep_time = params.sleep_time + local error_table = params.error + local error_msg = params.error_msg + + -- Using `fiber.sleep(time)` between two `clock.monotonic()` + -- may return diff less than `time`. + sleep_for = function(time) + local start = clock.monotonic() + while (clock.monotonic() - start) < time do + fiber.sleep(time / 10) + end + end + + return_true = function(space_name) + sleep_for(sleep_time) + return true + end + + return_err = function(space_name) + sleep_for(sleep_time) + return nil, error_table + end + + throws_error = function() + sleep_for(sleep_time) + error(error_msg) + end + ]], { params }) + + t.assert_equals(err, nil) +end + +function helpers.is_space_exist(router, space_name) + local res, err = router:eval([[ + local vshard = require('vshard') + local utils = require('crud.common.utils') + + local space, err = utils.get_space(..., vshard.router.routeall()) + if err ~= nil then + return nil, err + end + return space ~= nil + ]], { space_name }) + + t.assert_equals(err, nil) + return res +end + +function helpers.reload_package(srv) + srv.net_box:eval([[ + local function startswith(text, prefix) + return text:find(prefix, 1, true) == 1 + end + + for k, _ in pairs(package.loaded) do + if startswith(k, 'crud') then + package.loaded[k] = nil + end + end + + crud = require('crud') + ]]) +end + +function helpers.reload_roles(srv) + local ok, err = srv.net_box:eval([[ + return require('cartridge.roles').reload() + ]]) + + t.assert_equals({ok, err}, {true, nil}) +end + return helpers diff --git a/test/integration/cfg_test.lua b/test/integration/cfg_test.lua new file mode 100644 index 00000000..718a21c1 --- /dev/null +++ b/test/integration/cfg_test.lua @@ -0,0 +1,74 @@ +local fio = require('fio') + +local t = require('luatest') + +local helpers = require('test.helper') + +local group = t.group('cfg') + +group.before_all(function(g) + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_stats'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + }) + + g.cluster:start() +end) + +group.after_all(function(g) helpers.stop_cluster(g.cluster) end) + +group.test_defaults = function(g) + local cfg = g.cluster:server('router'):eval("return require('crud').cfg") + t.assert_equals(cfg, { stats = false }) +end + +group.test_change_value = function(g) + local new_cfg = g.cluster:server('router'):eval("return require('crud').cfg({ stats = true })") + t.assert_equals(new_cfg.stats, true) +end + +group.test_table_is_immutable = function(g) + local router = g.cluster:server('router') + + t.assert_error_msg_contains( + 'Use crud.cfg{} instead', + router.eval, router, + [[ + local cfg = require('crud').cfg() + cfg.stats = 'newvalue' + ]]) + + t.assert_error_msg_contains( + 'Use crud.cfg{} instead', + router.eval, router, + [[ + local cfg = require('crud').cfg() + cfg.newfield = 'newvalue' + ]]) +end + +group.test_package_reload_preserves_values = function(g) + local router = g.cluster:server('router') + + -- Generate some non-default values. + router:eval("return require('crud').cfg({ stats = true })") + + helpers.reload_package(router) + + local cfg = router:eval("return require('crud').cfg") + t.assert_equals(cfg.stats, true) +end + +group.test_role_reload_preserves_values = function(g) + local router = g.cluster:server('router') + + -- Generate some non-default values. + router:eval("return require('crud').cfg({ stats = true })") + + helpers.reload_roles(router) + + local cfg = router:eval("return require('crud').cfg") + t.assert_equals(cfg.stats, true) +end diff --git a/test/integration/reload_test.lua b/test/integration/reload_test.lua index c1f20c67..5d8b25fb 100644 --- a/test/integration/reload_test.lua +++ b/test/integration/reload_test.lua @@ -8,14 +8,6 @@ local g = t.group() local helpers = require('test.helper') -local function reload(srv) - local ok, err = srv.net_box:eval([[ - return require("cartridge.roles").reload() - ]]) - - t.assert_equals({ok, err}, {true, nil}) -end - g.before_all(function() g.cluster = helpers.Cluster:new({ datadir = fio.tempdir(), @@ -92,7 +84,7 @@ function g.test_router() t.assert_equals(last_insert[3], 'A', 'No workload for label A') end) - reload(g.router) + helpers.reload_roles(g.router) local cnt = #g.insertions_passed g.cluster:retrying({}, function() @@ -117,7 +109,7 @@ function g.test_storage() -- snapshot with a signal g.s1_master.process:kill('USR1') - reload(g.s1_master) + helpers.reload_roles(g.s1_master) g.cluster:retrying({}, function() g.s1_master.net_box:call('box.snapshot') diff --git a/test/integration/stats_test.lua b/test/integration/stats_test.lua new file mode 100644 index 00000000..5af04db9 --- /dev/null +++ b/test/integration/stats_test.lua @@ -0,0 +1,509 @@ +local fio = require('fio') +local clock = require('clock') +local t = require('luatest') + +local stats_registry_utils = require('crud.stats.registry_utils') + +local g = t.group('stats_integration') +local helpers = require('test.helper') + +local space_name = 'customers' +local non_existing_space_name = 'non_existing_space' +local new_space_name = 'newspace' + +g.before_all(function(g) + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_stats'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + }) + g.cluster:start() + g.router = g.cluster:server('router').net_box + + helpers.prepare_simple_functions(g.router) + g.router:eval("require('crud').cfg{ stats = true }") +end) + +g.after_all(function(g) + helpers.stop_cluster(g.cluster) +end) + +g.before_each(function(g) + g.router:eval("crud = require('crud')") + helpers.truncate_space_on_cluster(g.cluster, space_name) + helpers.drop_space_on_cluster(g.cluster, new_space_name) +end) + +function g:get_stats(space_name) + return self.router:eval("return require('crud').stats(...)", { space_name }) +end + + +local function create_new_space(g) + helpers.call_on_storages(g.cluster, function(server) + server.net_box:eval([[ + local space_name = ... + if not box.cfg.read_only then + local sp = box.schema.space.create(space_name, { format = { + {name = 'id', type = 'unsigned'}, + {name = 'bucket_id', type = 'unsigned'}, + }}) + + sp:create_index('pk', { + parts = { {field = 'id'} }, + }) + + sp:create_index('bucket_id', { + parts = { {field = 'bucket_id'} }, + unique = false, + }) + end + ]], { new_space_name }) + end) +end + +-- If there weren't any operations, space stats is {}. +-- To compute stats diff, this helper return real stats +-- if they're already present or default stats if +-- this operation of space hasn't been observed yet. +local function set_defaults_if_empty(space_stats, op) + if space_stats[op] ~= nil then + return space_stats[op] + else + return stats_registry_utils.build_collectors(op) + end +end + +local eval = { + pairs = [[ + local space_name = select(1, ...) + local conditions = select(2, ...) + + local result = {} + for _, v in crud.pairs(space_name, conditions, { batch_size = 1 }) do + table.insert(result, v) + end + + return result + ]], + + pairs_pcall = [[ + local space_name = select(1, ...) + local conditions = select(2, ...) + + local _, err = pcall(crud.pairs, space_name, conditions, { batch_size = 1 }) + + return nil, tostring(err) + ]], +} + +local simple_operation_cases = { + insert = { + func = 'crud.insert', + args = { + space_name, + { 12, box.NULL, 'Ivan', 'Ivanov', 20, 'Moscow' }, + }, + op = 'insert', + }, + insert_object = { + func = 'crud.insert_object', + args = { + space_name, + { id = 13, name = 'Ivan', last_name = 'Ivanov', age = 20, city = 'Moscow' }, + }, + op = 'insert', + }, + get = { + func = 'crud.get', + args = { space_name, { 12 } }, + op = 'get', + }, + select = { + func = 'crud.select', + args = { space_name, {{ '==', 'id_index', 3 }} }, + op = 'select', + }, + pairs = { + eval = eval.pairs, + args = { space_name, {{ '==', 'id_index', 3 }} }, + op = 'select', + }, + replace = { + func = 'crud.replace', + args = { + space_name, + { 12, box.NULL, 'Ivan', 'Ivanov', 20, 'Moscow' }, + }, + op = 'replace', + }, + replace_object = { + func = 'crud.replace_object', + args = { + space_name, + { id = 12, name = 'Ivan', last_name = 'Ivanov', age = 20, city = 'Moscow' }, + }, + op = 'replace', + }, + update = { + prepare = function(g) + helpers.insert_objects(g, space_name, {{ + id = 15, name = 'Ivan', last_name = 'Ivanov', + age = 20, city = 'Moscow' + }}) + end, + func = 'crud.update', + args = { space_name, 12, {{'+', 'age', 10}} }, + op = 'update', + }, + upsert = { + func = 'crud.upsert', + args = { + space_name, + { 16, box.NULL, 'Ivan', 'Ivanov', 20, 'Moscow' }, + {{'+', 'age', 1}}, + }, + op = 'upsert', + }, + upsert_object = { + func = 'crud.upsert_object', + args = { + space_name, + { id = 17, name = 'Ivan', last_name = 'Ivanov', age = 20, city = 'Moscow' }, + {{'+', 'age', 1}} + }, + op = 'upsert', + }, + delete = { + func = 'crud.delete', + args = { space_name, { 12 } }, + op = 'delete', + }, + truncate = { + func = 'crud.truncate', + args = { space_name }, + op = 'truncate', + }, + len = { + func = 'crud.len', + args = { space_name }, + op = 'len', + }, + count = { + func = 'crud.count', + args = { space_name, {{ '==', 'id_index', 3 }} }, + op = 'count', + }, + min = { + func = 'crud.min', + args = { space_name }, + op = 'borders', + }, + max = { + func = 'crud.max', + args = { space_name }, + op = 'borders', + }, + insert_error = { + func = 'crud.insert', + args = { space_name, { 'id' } }, + op = 'insert', + expect_error = true, + }, + insert_object_error = { + func = 'crud.insert_object', + args = { space_name, { 'id' } }, + op = 'insert', + expect_error = true, + }, + get_error = { + func = 'crud.get', + args = { space_name, { 'id' } }, + op = 'get', + expect_error = true, + }, + select_error = { + func = 'crud.select', + args = { space_name, {{ '==', 'id_index', 'sdf' }} }, + op = 'select', + expect_error = true, + }, + pairs_error = { + eval = eval.pairs, + args = { space_name, {{ '%=', 'id_index', 'sdf' }} }, + op = 'select', + expect_error = true, + pcall = true, + }, + replace_error = { + func = 'crud.replace', + args = { space_name, { 'id' } }, + op = 'replace', + expect_error = true, + }, + replace_object_error = { + func = 'crud.replace_object', + args = { space_name, { 'id' } }, + op = 'replace', + expect_error = true, + }, + update_error = { + func = 'crud.update', + args = { space_name, { 'id' }, {{'+', 'age', 1}} }, + op = 'update', + expect_error = true, + }, + upsert_error = { + func = 'crud.upsert', + args = { space_name, { 'id' }, {{'+', 'age', 1}} }, + op = 'upsert', + expect_error = true, + }, + upsert_object_error = { + func = 'crud.upsert_object', + args = { space_name, { 'id' }, {{'+', 'age', 1}} }, + op = 'upsert', + expect_error = true, + }, + delete_error = { + func = 'crud.delete', + args = { space_name, { 'id' } }, + op = 'delete', + expect_error = true, + }, + count_error = { + func = 'crud.count', + args = { space_name, {{ '==', 'id_index', 'sdf' }} }, + op = 'count', + expect_error = true, + }, + min_error = { + func = 'crud.min', + args = { space_name, 'badindex' }, + op = 'borders', + expect_error = true, + }, + max_error = { + func = 'crud.max', + args = { space_name, 'badindex' }, + op = 'borders', + expect_error = true, + }, +} + +-- Generate non-null stats for all cases. +local function generate_stats(g) + for _, case in pairs(simple_operation_cases) do + if case.prepare ~= nil then + case.prepare(g) + end + + local _, err + if case.eval ~= nil then + if case.pcall then + _, err = pcall(g.router.eval, g.router, case.eval, case.args) + else + _, err = g.router:eval(case.eval, case.args) + end + else + _, err = g.router:call(case.func, case.args) + end + + if case.expect_error ~= true then + t.assert_equals(err, nil) + else + t.assert_not_equals(err, nil) + end + end +end + + +-- Call some operations for existing +-- spaces and ensure statistics is updated. +for name, case in pairs(simple_operation_cases) do + local test_name = ('test_%s'):format(name) + + if case.prepare ~= nil then + g.before_test(test_name, case.prepare) + end + + g[test_name] = function(g) + -- Collect stats before call. + local stats_before = g:get_stats(space_name) + t.assert_type(stats_before, 'table') + + -- Call operation. + local before_start = clock.monotonic() + + local _, err + if case.eval ~= nil then + if case.pcall then + _, err = pcall(g.router.eval, g.router, case.eval, case.args) + else + _, err = g.router:eval(case.eval, case.args) + end + else + _, err = g.router:call(case.func, case.args) + end + + local after_finish = clock.monotonic() + + if case.expect_error ~= true then + t.assert_equals(err, nil) + else + t.assert_not_equals(err, nil) + end + + -- Collect stats after call. + local stats_after = g:get_stats(space_name) + t.assert_type(stats_after, 'table') + t.assert_not_equals(stats_after[case.op], nil) + + -- Expecting 'ok' metrics to change on `expect_error == false` + -- or 'error' to change otherwise. + local changed, unchanged + if case.expect_error == true then + changed = 'error' + unchanged = 'ok' + else + unchanged = 'error' + changed = 'ok' + end + + local op_before = set_defaults_if_empty(stats_before, case.op) + local changed_before = op_before[changed] + local op_after = set_defaults_if_empty(stats_after, case.op) + local changed_after = op_after[changed] + + t.assert_equals(changed_after.count - changed_before.count, 1, + 'Expected count incremented') + + local ok_latency_max = math.max(changed_before.latency, after_finish - before_start) + + t.assert_gt(changed_after.latency, 0, + 'Changed latency has appropriate value') + t.assert_le(changed_after.latency, ok_latency_max, + 'Changed latency has appropriate value') + + local time_diff = changed_after.time - changed_before.time + + t.assert_gt(time_diff, 0, 'Total time increase has appropriate value') + t.assert_le(time_diff, after_finish - before_start, + 'Total time increase has appropriate value') + + local unchanged_before = op_before[unchanged] + local unchanged_after = stats_after[case.op][unchanged] + + t.assert_equals(unchanged_before, unchanged_after, 'Other stats remained the same') + end +end + + +-- Call some operation on non-existing +-- space and ensure statistics are updated. +g.before_test('test_non_existing_space', function(g) + t.assert_equals( + helpers.is_space_exist(g.router, non_existing_space_name), + false, + ('Space %s does not exist'):format(non_existing_space_name) + ) +end) + +g.test_non_existing_space = function(g) + local op = 'get' + + -- Collect stats before call. + local stats_before = g:get_stats(non_existing_space_name) + t.assert_type(stats_before, 'table') + local op_before = set_defaults_if_empty(stats_before, op) + + -- Call operation. + local _, err = g.router:call('crud.get', { non_existing_space_name, { 1 } }) + t.assert_not_equals(err, nil) + + -- Collect stats after call. + local stats_after = g:get_stats(non_existing_space_name) + t.assert_type(stats_after, 'table') + local op_after = stats_after[op] + t.assert_type(op_after, 'table', 'Section has been created if not existed') + + t.assert_equals(op_after.error.count - op_before.error.count, 1, + 'Error count for non-existing space incremented') +end + + +g.before_test( + 'test_role_reload_do_not_reset_observations', + generate_stats) + +g.test_role_reload_do_not_reset_observations = function(g) + local stats_before = g:get_stats() + + helpers.reload_roles(g.cluster:server('router')) + + local stats_after = g:get_stats() + t.assert_equals(stats_after, stats_before) +end + + +g.before_test( + 'test_module_reload_do_not_reset_observations', + generate_stats) + +g.test_module_reload_do_not_reset_observations = function(g) + local stats_before = g:get_stats() + + helpers.reload_package(g.cluster:server('router')) + + local stats_after = g:get_stats() + t.assert_equals(stats_after, stats_before) +end + + +g.test_spaces_created_in_runtime_supported_with_stats = function(g) + local op = 'insert' + local stats_before = g:get_stats(new_space_name) + local op_before = set_defaults_if_empty(stats_before, op) + + create_new_space(g) + + local _, err = g.router:call('crud.insert', { new_space_name, { 1, box.NULL }}) + t.assert_equals(err, nil) + + local stats_after = g:get_stats(new_space_name) + local op_after = stats_after[op] + t.assert_type(op_after, 'table', "'insert' stats found for new space") + t.assert_type(op_after.ok, 'table', "success 'insert' stats found for new space") + t.assert_equals(op_after.ok.count - op_before.ok.count, 1, + "Success requests count incremented for new space") +end + + +g.before_test( + 'test_spaces_dropped_in_runtime_supported_with_stats', + function(g) + create_new_space(g) + + local _, err = g.router:call('crud.insert', { new_space_name, { 1, box.NULL }}) + t.assert_equals(err, nil) + end) + +g.test_spaces_dropped_in_runtime_supported_with_stats = function(g) + local op = 'insert' + local stats_before = g:get_stats(new_space_name) + local op_before = set_defaults_if_empty(stats_before, op) + t.assert_type(op_before, 'table', "'insert' stats found for new space") + + helpers.drop_space_on_cluster(g.cluster, new_space_name) + + local _, err = g.router:call('crud.insert', { new_space_name, { 2, box.NULL }}) + t.assert_not_equals(err, nil, "Should trigger 'space not found' error") + + local stats_after = g:get_stats(new_space_name) + local op_after = stats_after[op] + t.assert_type(op_after, 'table', "'insert' stats found for dropped new space") + t.assert_type(op_after.error, 'table', "error 'insert' stats found for dropped new space") + t.assert_equals(op_after.error.count - op_before.error.count, 1, + "Error requests count incremented since space was known to registry before drop") +end diff --git a/test/unit/stats_test.lua b/test/unit/stats_test.lua new file mode 100644 index 00000000..628a9266 --- /dev/null +++ b/test/unit/stats_test.lua @@ -0,0 +1,555 @@ +local clock = require('clock') +local fio = require('fio') +local fun = require('fun') +local t = require('luatest') + +local stats_module = require('crud.stats') + +local g = t.group('stats_unit') +local helpers = require('test.helper') + +local space_name = 'customers' + +g.before_all(function(g) + -- Enable test cluster for "is space exist?" checks. + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_stats'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + }) + g.cluster:start() + g.router = g.cluster:server('router').net_box + + helpers.prepare_simple_functions(g.router) + g.router:eval("stats_module = require('crud.stats')") +end) + +g.after_all(function(g) + helpers.stop_cluster(g.cluster) +end) + +-- Reset statistics between tests, reenable if needed. +g.before_each(function(g) + g:enable_stats() +end) + +g.after_each(function(g) + g:disable_stats() +end) + +function g:get_stats(space_name) + return self.router:eval("return stats_module.get(...)", { space_name }) +end + +function g:enable_stats() + self.router:eval("stats_module.enable()") +end + +function g:disable_stats() + self.router:eval("stats_module.disable()") +end + +function g:reset_stats() + self.router:eval("return stats_module.reset()") +end + + +g.test_get_format_after_enable = function(g) + local stats = g:get_stats() + + t.assert_type(stats, 'table') + t.assert_equals(stats.spaces, {}) +end + +g.test_get_by_space_name_format_after_enable = function(g) + local stats = g:get_stats(space_name) + + t.assert_type(stats, 'table') + t.assert_equals(stats, {}) +end + +-- Test statistics values after wrapped functions call. +local observe_cases = { + wrapper_observes_expected_values_on_ok = { + operations = stats_module.op, + func = 'return_true', + changed_coll = 'ok', + unchanged_coll = 'error', + }, + wrapper_observes_expected_values_on_error_return = { + operations = stats_module.op, + func = 'return_err', + changed_coll = 'error', + unchanged_coll = 'ok', + }, + wrapper_observes_expected_values_on_error_throw = { + operations = stats_module.op, + func = 'throws_error', + changed_coll = 'error', + unchanged_coll = 'ok', + pcall = true, + }, +} + +local call_wrapped = [[ + local func = rawget(_G, select(1, ...)) + local op = select(2, ...) + local opts = select(3, ...) + local space_name = select(4, ...) + + stats_module.wrap(func, op, opts)(space_name) +]] + +for name, case in pairs(observe_cases) do + for _, op in pairs(case.operations) do + local test_name = ('test_%s_%s'):format(op, name) + + g[test_name] = function(g) + -- Call wrapped functions on server side. + -- Collect execution times from outside. + local run_count = 10 + local time_diffs = {} + + local args = { case.func, op, case.opts, space_name } + + for _ = 1, run_count do + local before_start = clock.monotonic() + + if case.pcall then + pcall(g.router.eval, g.router, call_wrapped, args) + else + g.router:eval(call_wrapped, args) + end + + local after_finish = clock.monotonic() + + table.insert(time_diffs, after_finish - before_start) + end + + table.sort(time_diffs) + local total_time = fun.sum(time_diffs) + + -- Validate stats format after execution. + local total_stats = g:get_stats() + t.assert_type(total_stats, 'table', 'Total stats present after observations') + + local space_stats = g:get_stats(space_name) + t.assert_type(space_stats, 'table', 'Space stats present after observations') + + t.assert_equals(total_stats.spaces[space_name], space_stats, + 'Space stats is a section of total stats') + + local op_stats = space_stats[op] + t.assert_type(op_stats, 'table', 'Op stats present after observations for the space') + + -- Expected collectors (changed_coll: 'ok' or 'error') have changed. + local changed = op_stats[case.changed_coll] + t.assert_type(changed, 'table', 'Status stats present after observations') + + t.assert_equals(changed.count, run_count, 'Count incremented by count of runs') + + local sleep_time = helpers.simple_functions_params().sleep_time + t.assert_ge(changed.latency, sleep_time, 'Latency has appropriate value') + t.assert_le(changed.latency, time_diffs[#time_diffs], 'Latency has appropriate value') + + t.assert_ge(changed.time, sleep_time * run_count, + 'Total time increase has appropriate value') + t.assert_le(changed.time, total_time, 'Total time increase has appropriate value') + + -- Other collectors (unchanged_coll: 'error' or 'ok') + -- have been initialized and have default values. + local unchanged = op_stats[case.unchanged_coll] + t.assert_type(unchanged, 'table', 'Other status stats present after observations') + + t.assert_equals( + unchanged, + { + count = 0, + latency = 0, + time = 0 + }, + 'Other status collectors initialized after observations' + ) + end + end +end + +local pairs_cases = { + success_run = { + prepare = [[ + local params = ... + local sleep_time = params.sleep_time + + local function sleep_ten_times(param, state) + if state == 10 then + return nil + end + + sleep_for(sleep_time) + + return state + 1, param + end + rawset(_G, 'sleep_ten_times', sleep_ten_times) + ]], + eval = [[ + local params, space_name, op = ... + local sleep_time = params.sleep_time + + local build_sleep_multiplier = 2 + + local wrapped = stats_module.wrap( + function(space_name) + sleep_for(build_sleep_multiplier * sleep_time) + + return sleep_ten_times, {}, 0 + end, + op, + { pairs = true } + ) + + for _, _ in wrapped(space_name) do end + ]], + build_sleep_multiplier = 2, + iterations_expected = 10, + changed_coll = 'ok', + unchanged_coll = 'error', + }, + error_throw = { + prepare = [[ + local params = ... + local sleep_time = params.sleep_time + local error_table = params.error + + + local function sleep_five_times_and_throw_error(param, state) + if state == 5 then + error(error_table) + end + + sleep_for(sleep_time) + + return state + 1, param + end + rawset(_G, 'sleep_five_times_and_throw_error', sleep_five_times_and_throw_error) + ]], + eval = [[ + local params, space_name, op = ... + local sleep_time = params.sleep_time + + local build_sleep_multiplier = 2 + + local wrapped = stats_module.wrap( + function(space_name) + sleep_for(build_sleep_multiplier * sleep_time) + + return sleep_five_times_and_throw_error, {}, 0 + end, + op, + { pairs = true } + ) + + for _, _ in wrapped(space_name) do end + ]], + build_sleep_multiplier = 2, + iterations_expected = 5, + changed_coll = 'error', + unchanged_coll = 'ok', + pcall = true, + }, + break_after_gc = { + prepare = [[ + local params = ... + local sleep_time = params.sleep_time + + local function sleep_ten_times(param, state) + if state == 10 then + return nil + end + + sleep_for(sleep_time) + + return state + 1, param + end + rawset(_G, 'sleep_ten_times', sleep_ten_times) + ]], + eval = [[ + local params, space_name, op = ... + local sleep_time = params.sleep_time + + local build_sleep_multiplier = 2 + + local wrapped = stats_module.wrap( + function(space_name) + sleep_for(build_sleep_multiplier * sleep_time) + + return sleep_ten_times, {}, 0 + end, + op, + { pairs = true } + ) + + for i, _ in wrapped(space_name) do + if i == 5 then + break + end + end + ]], + post_eval = [[ + collectgarbage('collect') + collectgarbage('collect') + ]], + build_sleep_multiplier = 2, + iterations_expected = 5, + changed_coll = 'ok', + unchanged_coll = 'error', + } +} + +for name, case in pairs(pairs_cases) do + local test_name = ('test_pairs_wrapper_observes_all_iterations_on_%s'):format(name) + + g.before_test(test_name, function(g) + g.router:eval(case.prepare, { helpers.simple_functions_params() }) + end) + + g[test_name] = function(g) + local op = stats_module.op.SELECT + + local params = helpers.simple_functions_params() + local args = { params, space_name, op } + + local before_start = clock.monotonic() + + if case.pcall then + pcall(g.router.eval, g.router, case.eval, args) + else + g.router:eval(case.eval, args) + end + + if case.post_eval then + g.router:eval(case.post_eval) + end + + local after_finish = clock.monotonic() + local time_diff = after_finish - before_start + + -- Validate stats format after execution. + local total_stats = g:get_stats() + t.assert_type(total_stats, 'table', 'Total stats present after observations') + + local space_stats = g:get_stats(space_name) + t.assert_type(space_stats, 'table', 'Space stats present after observations') + + t.assert_equals(total_stats.spaces[space_name], space_stats, + 'Space stats is a section of total stats') + + local op_stats = space_stats[op] + t.assert_type(op_stats, 'table', 'Op stats present after observations for the space') + + -- Expected collectors (changed_coll: 'ok' or 'error') have changed. + local changed = op_stats[case.changed_coll] + t.assert_type(changed, 'table', 'Status stats present after observations') + + t.assert_equals(changed.count, 1, 'Count incremented by 1') + + t.assert_ge(changed.latency, + params.sleep_time * (case.build_sleep_multiplier + case.iterations_expected), + 'Latency has appropriate value') + t.assert_le(changed.latency, time_diff, 'Latency has appropriate value') + + t.assert_ge(changed.time, + params.sleep_time * (case.build_sleep_multiplier + case.iterations_expected), + 'Total time has appropriate value') + t.assert_le(changed.time, time_diff, 'Total time has appropriate value') + + -- Other collectors (unchanged_coll: 'error' or 'ok') + -- have been initialized and have default values. + local unchanged = op_stats[case.unchanged_coll] + t.assert_type(unchanged, 'table', 'Other status stats present after observations') + + t.assert_equals( + unchanged, + { + count = 0, + latency = 0, + time = 0 + }, + 'Other status collectors initialized after observations' + ) + end +end + +-- Test wrapper preserves return values. +local disable_stats_cases = { + stats_disable_before_wrap_ = { + before_wrap = 'stats_module.disable()', + after_wrap = '', + }, + stats_disable_after_wrap_ = { + before_wrap = '', + after_wrap = 'stats_module.disable()', + }, + [''] = { + before_wrap = '', + after_wrap = '', + }, +} + +local preserve_return_cases = { + wrapper_preserves_return_values_on_ok = { + func = 'return_true', + res = true, + err = nil, + }, + wrapper_preserves_return_values_on_error = { + func = 'return_err', + res = nil, + err = helpers.simple_functions_params().error, + }, +} + +local preserve_throw_cases = { + wrapper_preserves_error_throw = { + opts = { pairs = false }, + }, + pairs_wrapper_preserves_error_throw = { + opts = { pairs = true }, + }, +} + +for name_head, disable_case in pairs(disable_stats_cases) do + for name_tail, return_case in pairs(preserve_return_cases) do + local test_name = ('test_%s%s'):format(name_head, name_tail) + + g[test_name] = function(g) + local op = stats_module.op.INSERT + + local eval = ([[ + local func = rawget(_G, select(1, ...)) + local op = select(2, ...) + local space_name = select(3, ...) + + %s -- before_wrap + local w_func = stats_module.wrap(func, op) + %s -- after_wrap + + return w_func(space_name) + ]]):format(disable_case.before_wrap, disable_case.after_wrap) + + local res, err = g.router:eval(eval, { return_case.func, op, space_name }) + + t.assert_equals(res, return_case.res, 'Wrapper preserves first return value') + t.assert_equals(err, return_case.err, 'Wrapper preserves second return value') + end + end + + local test_name = ('test_%spairs_wrapper_preserves_return_values'):format(name_head) + + g[test_name] = function(g) + local op = stats_module.op.INSERT + + local input = { a = 'a', b = 'b' } + local eval = ([[ + local input = select(1, ...) + local func = function() return pairs(input) end + local op = select(2, ...) + local space_name = select(3, ...) + + %s -- before_wrap + local w_func = stats_module.wrap(func, op, { pairs = true }) + %s -- after_wrap + + local res = {} + for k, v in w_func(space_name) do + res[k] = v + end + + return res + ]]):format(disable_case.before_wrap, disable_case.after_wrap) + + local res = g.router:eval(eval, { input, op, space_name }) + + t.assert_equals(input, res, 'Wrapper preserves pairs return values') + end + + for name_tail, throw_case in pairs(preserve_throw_cases) do + local test_name = ('test_%s%s'):format(name_head, name_tail) + + g[test_name] = function(g) + local op = stats_module.op.INSERT + + local eval = ([[ + local func = rawget(_G, 'throws_error') + local opts = select(1, ...) + local op = select(2, ...) + local space_name = select(3, ...) + + %s -- before_wrap + local w_func = stats_module.wrap(func, op, opts) + %s -- after_wrap + + w_func(space_name) + ]]):format(disable_case.before_wrap, disable_case.after_wrap) + + t.assert_error_msg_contains( + helpers.simple_functions_params().error_msg, + g.router.eval, g.router, eval, { throw_case.opts, op, space_name } + ) + end + end +end + + +g.test_stats_is_empty_after_disable = function(g) + g:disable_stats() + + local op = stats_module.op.INSERT + g.router:eval(call_wrapped, { 'return_true', op, {}, space_name }) + + local stats = g:get_stats() + t.assert_equals(stats, {}) +end + + +local function prepare_non_default_stats(g) + local op = stats_module.op.INSERT + g.router:eval(call_wrapped, { 'return_true', op, {}, space_name }) + + local stats = g:get_stats(space_name) + t.assert_equals(stats[op].ok.count, 1, 'Non-zero stats prepared') + + return stats +end + +g.test_enable_is_idempotent = function(g) + local stats_before = prepare_non_default_stats(g) + + g:enable_stats() + + local stats_after = g:get_stats(space_name) + + t.assert_equals(stats_after, stats_before, 'Stats have not been reset') +end + +g.test_reset = function(g) + prepare_non_default_stats(g) + + g:reset_stats() + + local stats = g:get_stats(space_name) + + t.assert_equals(stats, {}, 'Stats have been reset') +end + +g.test_reset_for_disabled_stats_does_not_init_module = function(g) + g:disable_stats() + + local stats_before = g:get_stats() + t.assert_equals(stats_before, {}, "Stats is empty") + + g:reset_stats() + + local stats_after = g:get_stats() + t.assert_equals(stats_after, {}, "Stats is still empty") +end