diff --git a/.github/workflows/test_on_push.yaml b/.github/workflows/test_on_push.yaml index bec97dc57..2743aa572 100644 --- a/.github/workflows/test_on_push.yaml +++ b/.github/workflows/test_on_push.yaml @@ -78,6 +78,41 @@ jobs: run: make -C build coveralls if: ${{ matrix.coveralls }} + run-perf-tests-ce: + if: | + github.event_name == 'push' || + github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository + strategy: + matrix: + tarantool-version: ["1.10", "2.8"] + metrics-version: ["0.12.0"] + fail-fast: false + runs-on: [ubuntu-latest] + steps: + - uses: actions/checkout@master + + - name: Setup Tarantool CE + uses: tarantool/setup-tarantool@v1 + with: + tarantool-version: ${{ matrix.tarantool-version }} + + - name: Install requirements for community + run: | + tarantool --version + ./deps.sh + + - name: Install metrics + run: tarantoolctl rocks install metrics ${{ matrix.metrics-version }} + + # This server starts and listen on 8084 port that is used for tests + - name: Stop Mono server + run: sudo kill -9 $(sudo lsof -t -i tcp:8084) || true + + - run: cmake -S . -B build + + - name: Run performance tests + run: make -C build performance + run-tests-ee: if: github.event_name == 'push' strategy: diff --git a/CMakeLists.txt b/CMakeLists.txt index 714474a18..2e11fbf3f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,14 @@ add_custom_target(luatest COMMENT "Run regression tests" ) +set(PERFORMANCE_TESTS_SUBDIR "test/performance") + +add_custom_target(performance + COMMAND PERF_MODE_ON=true ${LUATEST} -v -c ${PERFORMANCE_TESTS_SUBDIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMENT "Run performance tests" +) + add_custom_target(coverage COMMAND ${LUACOV} ${PROJECT_SOURCE_DIR} && grep -A999 '^Summary' ${CODE_COVERAGE_REPORT} DEPENDS ${CODE_COVERAGE_STATS} diff --git a/README.md b/README.md index 5f9891dcc..df9eca4ae 100644 --- a/README.md +++ b/README.md @@ -718,6 +718,9 @@ crud.cfg stats_driver: local ... ``` +Performance overhead is 3-10% in case of `local` driver and +5-15% in case of `metrics` driver, up to 20% for `metrics` with quantiles. + Beware that iterating through `crud.cfg` with pairs is not supported yet, refer to [tarantool/crud#265](https://github.com/tarantool/crud/issues/265). diff --git a/test/entrypoint/srv_ddl.lua b/test/entrypoint/srv_ddl.lua index 30f432b8a..8f5d2cb41 100755 --- a/test/entrypoint/srv_ddl.lua +++ b/test/entrypoint/srv_ddl.lua @@ -102,6 +102,12 @@ package.preload['customers-storage'] = function() }, } + local customers_id_schema = table.deepcopy(customers_schema) + customers_id_schema.sharding_key = {'id'} + table.insert(customers_id_schema.indexes, primary_index_id) + table.insert(customers_id_schema.indexes, bucket_id_index) + table.insert(customers_id_schema.indexes, age_index) + local customers_name_key_schema = table.deepcopy(customers_schema) customers_name_key_schema.sharding_key = {'name'} table.insert(customers_name_key_schema.indexes, primary_index) @@ -157,6 +163,7 @@ package.preload['customers-storage'] = function() local schema = { spaces = { + customers = customers_id_schema, customers_name_key = customers_name_key_schema, customers_name_key_uniq_index = customers_name_key_uniq_index_schema, customers_name_key_non_uniq_index = customers_name_key_non_uniq_index_schema, @@ -195,8 +202,8 @@ local ok, err = errors.pcall('CartridgeCfgError', cartridge.cfg, { 'customers-storage', 'cartridge.roles.crud-router', 'cartridge.roles.crud-storage', - }, -}) + }}, { readahead = 20 * 1024 * 1024 } +) if not ok then log.error('%s', err) diff --git a/test/integration/ddl_sharding_key_test.lua b/test/integration/ddl_sharding_key_test.lua index da02b54da..28b2676f2 100644 --- a/test/integration/ddl_sharding_key_test.lua +++ b/test/integration/ddl_sharding_key_test.lua @@ -696,6 +696,7 @@ pgroup.test_update_cache_with_incorrect_key = function(g) -- records for all spaces exist sharding_key_as_index_obj = helpers.get_sharding_key_cache(g.cluster) t.assert_equals(sharding_key_as_index_obj, { + customers = {parts = {{fieldno = 1}}}, customers_G_func = {parts = {{fieldno = 1}}}, customers_body_func = {parts = {{fieldno = 1}}}, customers_age_key = {parts = {{fieldno = 4}}}, @@ -722,6 +723,7 @@ pgroup.test_update_cache_with_incorrect_key = function(g) -- other records for correct spaces exist in cache sharding_key_as_index_obj = helpers.get_sharding_key_cache(g.cluster) t.assert_equals(sharding_key_as_index_obj, { + customers = {parts = {{fieldno = 1}}}, customers_G_func = {parts = {{fieldno = 1}}}, customers_body_func = {parts = {{fieldno = 1}}}, customers_age_key = {parts = {{fieldno = 4}}}, @@ -747,6 +749,7 @@ pgroup.test_update_cache_with_incorrect_key = function(g) -- other records for correct spaces exist in cache sharding_key_as_index_obj = helpers.get_sharding_key_cache(g.cluster) t.assert_equals(sharding_key_as_index_obj, { + customers = {parts = {{fieldno = 1}}}, customers_G_func = {parts = {{fieldno = 1}}}, customers_body_func = {parts = {{fieldno = 1}}}, customers_age_key = {parts = {{fieldno = 4}}}, diff --git a/test/performance/perf_test.lua b/test/performance/perf_test.lua new file mode 100644 index 000000000..4d161951d --- /dev/null +++ b/test/performance/perf_test.lua @@ -0,0 +1,531 @@ +local fio = require('fio') +local clock = require('clock') +local fiber = require('fiber') +local errors = require('errors') +local net_box = require('net.box') +local log = require('log') + +local t = require('luatest') +local g = t.group('perf') + +local helpers = require('test.helper') + + +local id = 0 +local function gen() + id = id + 1 + return id +end + +local function reset_gen() + id = 0 +end + +g.before_all(function(g) + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_ddl'), + use_vshard = true, + replicasets = { + { + uuid = helpers.uuid('a'), + alias = 'router', + roles = { 'crud-router' }, + servers = { + { instance_uuid = helpers.uuid('a', 1), alias = 'router' }, + }, + }, + { + uuid = helpers.uuid('b'), + alias = 's-1', + roles = { 'customers-storage', 'crud-storage' }, + servers = { + { instance_uuid = helpers.uuid('b', 1), alias = 's1-master' }, + { instance_uuid = helpers.uuid('b', 2), alias = 's1-replica' }, + }, + }, + { + uuid = helpers.uuid('c'), + alias = 's-2', + roles = { 'customers-storage', 'crud-storage' }, + servers = { + { instance_uuid = helpers.uuid('c', 1), alias = 's2-master' }, + { instance_uuid = helpers.uuid('c', 2), alias = 's2-replica' }, + }, + }, + { + uuid = helpers.uuid('d'), + alias = 's-2', + roles = { 'customers-storage', 'crud-storage' }, + servers = { + { instance_uuid = helpers.uuid('d', 1), alias = 's3-master' }, + { instance_uuid = helpers.uuid('d', 2), alias = 's3-replica' }, + }, + } + }, + }) + g.cluster:start() + + g.router = g.cluster:server('router').net_box + + g.router:eval([[ + rawset(_G, 'crud', require('crud')) + ]]) + + -- Run real perf tests only with flag, otherwise run short version + -- to test compatibility as part of unit/integration test run. + g.perf_mode_on = os.getenv('PERF_MODE_ON') + + g.total_report = {} +end) + +g.before_each(function(g) + helpers.truncate_space_on_cluster(g.cluster, 'customers') + reset_gen() +end) + +local function normalize(s, n) + if type(s) == 'number' then + s = ('%.2f'):format(s) + end + + local len = s:len() + if len > n then + return s:sub(1, n) + end + + return (' '):rep(n - len) .. s +end + +local row_name = { + insert = 'insert', + select_pk = 'select by pk', + select_gt_pk = 'select gt by pk (limit 10)', + pairs_gt = 'pairs gt by pk (limit 100)', +} + +local column_name = { + without_stats_wrapper = 'without stats wrapper', + stats_disabled = 'stats disabled', + local_stats = 'local stats', + metrics_stats = 'metrics stats (no quantiles)', + metrics_quantile_stats = 'metrics stats (with quantiles)', +} + +local function visualize_section(total_report, name, comment, section, params) + local report_str = ('== %s ==\n(%s)\n\n'):format(name, comment or '') + + local normalized_row_header = normalize('', params.row_header_width) + local headers = '| ' .. normalized_row_header .. ' |' + local after_headers = '| ' .. ('-'):rep(normalized_row_header:len()) .. ' |' + + for _, column in ipairs(params.columns) do + local normalized_column_header = normalize(column, params.col_width[column]) + headers = headers .. ' ' .. normalized_column_header .. ' |' + after_headers = after_headers .. ' ' .. ('-'):rep(normalized_column_header:len()) .. ' |' + end + + report_str = report_str .. headers .. '\n' + report_str = report_str .. after_headers .. '\n' + + for _, row in ipairs(params.rows) do + local row_str = '| ' .. normalize(row, params.row_header_width) .. ' |' + + for _, column in ipairs(params.columns) do + local report = total_report[row][column] + + local report_str + if report ~= nil then + report_str = report.str[section] + else + report_str = 'unknown' + end + + row_str = row_str .. ' ' .. normalize(report_str, params.col_width[column]) .. ' |' + end + + report_str = report_str .. row_str .. '\n' + end + + report_str = report_str .. '\n\n\n' + + return report_str +end + +local function visualize_report(report) + local params = {} + + params.col_width = 2 + for _, name in pairs(column_name) do + params.col_width = math.max(name:len() + 2, params.col_width) + end + + params.row_header_width = 30 + + -- Set columns and rows explicitly to preserve custom order. + params.columns = { + column_name.without_stats_wrapper, + column_name.stats_disabled, + column_name.local_stats, + column_name.metrics_stats, + column_name.metrics_quantile_stats, + } + + params.rows = { + row_name.select_pk, + row_name.select_gt_pk, + row_name.pairs_gt, + row_name.insert, + } + + params.row_header_width = 1 + for _, name in pairs(row_name) do + params.row_header_width = math.max(name:len(), params.row_header_width) + end + + local min_col_width = 12 + params.col_width = {} + for _, name in ipairs(params.columns) do + params.col_width[name] = math.max(name:len(), min_col_width) + end + + local report_str = '\n==== PERFORMANCE REPORT ====\n\n\n' + + report_str = report_str .. visualize_section(report, 'SUCCESS REQUESTS', + 'The higher the better', 'success_count', params) + report_str = report_str .. visualize_section(report, 'SUCCESS REQUESTS PER SECOND', + 'The higher the better', 'success_rps', params) + report_str = report_str .. visualize_section(report, 'ERRORS', + 'Bad if higher than zero', 'error_count', params) + report_str = report_str .. visualize_section(report, 'AVERAGE CALL TIME', + 'The lower the better', 'average_time', params) + report_str = report_str .. visualize_section(report, 'MAX CALL TIME', + 'The lower the better', 'max_time', params) + + log.info(report_str) +end + +g.after_each(function(g) + g.router:call('crud.cfg', {{ stats = false }}) +end) + +g.after_all(function(g) + g.cluster:stop() + fio.rmtree(g.cluster.datadir) + + visualize_report(g.total_report) +end) + +local function generate_customer() + return { gen(), box.NULL, 'David Smith', 33 } +end + +local select_prepare = function(g) + local count + if g.perf_mode_on then + count = 10100 + else + count = 100 + end + + for _ = 1, count do + g.router:call('crud.insert', { 'customers', generate_customer() }) + end + reset_gen() +end + +local insert_params = function() + return { 'customers', generate_customer() } +end + +local select_params_pk_eq = function() + return { 'customers', {{'==', 'id', gen() % 10000}} } +end + +local select_params_pk_gt = function() + return { 'customers', {{'>', 'id', gen() % 10000}}, { first = 10 } } +end + +local pairs_params_pk_gt = function() + return { 'customers', {{'>', 'id', gen() % 10000}}, { first = 100, batch_size = 50 } } +end + +local stats_cases = { + stats_disabled = { + column_name = column_name.stats_disabled, + }, + local_stats = { + prepare = function(g) + g.router:call('crud.cfg', {{ stats = true, stats_driver = 'local', stats_quantiles = false }}) + end, + column_name = column_name.local_stats, + }, + metrics_stats = { + prepare = function(g) + local is_metrics_supported = g.router:eval([[ + return require('crud.stats.metrics_registry').is_supported() + ]]) + t.skip_if(is_metrics_supported == false, 'Metrics registry is unsupported') + g.router:call('crud.cfg', {{ stats = true, stats_driver = 'metrics', stats_quantiles = false }}) + end, + column_name = column_name.metrics_stats, + }, + metrics_quantile_stats = { + prepare = function(g) + local is_metrics_supported = g.router:eval([[ + return require('crud.stats.metrics_registry').is_supported() + ]]) + t.skip_if(is_metrics_supported == false, 'Metrics registry is unsupported') + g.router:call('crud.cfg', {{ stats = true, stats_driver = 'metrics', stats_quantiles = true }}) + end, + column_name = column_name.metrics_quantile_stats, + }, +} + +local integration_params = { + timeout = 2, + fiber_count = 5, + connection_count = 2, +} + +local pairs_integration = { + timeout = 5, + fiber_count = 1, + connection_count = 1, +} + +local insert_perf = { + timeout = 30, + fiber_count = 600, + connection_count = 10, +} + +-- Higher load may lead to net_msg_max limit break. +local select_perf = { + timeout = 30, + fiber_count = 200, + connection_count = 10, +} + +local pairs_perf = { + timeout = 30, + fiber_count = 100, + connection_count = 10, +} + +local cases = { + crud_insert = { + call = 'crud.insert', + params = insert_params, + matrix = stats_cases, + integration_params = integration_params, + perf_params = insert_perf, + row_name = row_name.insert, + }, + + crud_insert_without_stats_wrapper = { + prepare = function(g) + g.router:eval([[ + rawset(_G, '_plain_insert', require('crud.insert').tuple) + ]]) + end, + call = '_plain_insert', + params = insert_params, + matrix = { [''] = { column_name = column_name.without_stats_wrapper } }, + integration_params = integration_params, + perf_params = insert_perf, + row_name = row_name.insert, + }, + + crud_select_pk_eq = { + prepare = select_prepare, + call = 'crud.select', + params = select_params_pk_eq, + matrix = stats_cases, + integration_params = integration_params, + perf_params = select_perf, + row_name = row_name.select_pk, + }, + + crud_select_without_stats_wrapper_pk_eq = { + prepare = function(g) + g.router:eval("_plain_select = require('crud.select').call") + select_prepare(g) + end, + call = '_plain_select', + params = select_params_pk_eq, + matrix = { [''] = { column_name = column_name.without_stats_wrapper } }, + integration_params = integration_params, + perf_params = select_perf, + row_name = row_name.select_pk, + }, + + crud_select_pk_gt = { + prepare = select_prepare, + call = 'crud.select', + params = select_params_pk_gt, + matrix = stats_cases, + integration_params = integration_params, + perf_params = select_perf, + row_name = row_name.select_gt_pk, + }, + + crud_select_without_stats_wrapper_pk_gt = { + prepare = function(g) + g.router:eval([[ + rawset(_G, '_plain_select', require('crud.select').call) + ]]) + select_prepare(g) + end, + call = '_plain_select', + params = select_params_pk_gt, + matrix = { [''] = { column_name = column_name.without_stats_wrapper } }, + integration_params = integration_params, + perf_params = select_perf, + row_name = row_name.select_gt_pk, + }, + + crud_pairs_gt = { + prepare = function(g) + g.router:eval([[ + _run_pairs = function(...) + local t = {} + for _, tuple in require('crud').pairs(...) do + table.insert(t, tuple) + end + end + ]]) + select_prepare(g) + end, + call = '_run_pairs', + params = pairs_params_pk_gt, + matrix = stats_cases, + integration_params = pairs_integration, + perf_params = pairs_perf, + row_name = row_name.pairs_gt, + }, + + crud_pairs_without_stats_wrapper_pk_gt = { + prepare = function(g) + g.router:eval([[ + _run_pairs = function(...) + local t = {} + for _, tuple in require('crud.select').pairs(...) do + table.insert(t, tuple) + end + end + ]]) + select_prepare(g) + end, + call = '_run_pairs', + params = pairs_params_pk_gt, + matrix = { [''] = { column_name = column_name.without_stats_wrapper } }, + integration_params = pairs_integration, + perf_params = pairs_perf, + row_name = row_name.pairs_gt, + }, +} + +local function generator_f(conn, call, params, report, timeout) + local start = clock.monotonic() + + while (clock.monotonic() - start) < timeout do + local call_start = clock.monotonic() + local ok, res, err = pcall(conn.call, conn, call, params()) + local call_time = clock.monotonic() - call_start + + if not ok then + log.error(res) + table.insert(report.errors, res) + elseif err ~= nil then + errors.wrap(err) + log.error(err) + table.insert(report.errors, err) + else + report.count = report.count + 1 + end + + report.total_time = report.total_time + call_time + report.max_time = math.max(report.max_time, call_time) + end +end + +for name, case in pairs(cases) do + local matrix = case.matrix or { [''] = { { column_name = '' } } } + + for subname, subcase in pairs(matrix) do + local name_tail = '' + if subname ~= '' then + name_tail = ('_with_%s'):format(subname) + end + + local test_name = ('test_%s%s'):format(name, name_tail) + + g.before_test(test_name, function(g) + if case.prepare ~= nil then + case.prepare(g) + end + + if subcase.prepare ~= nil then + subcase.prepare(g) + end + end) + + g[test_name] = function(g) + local params + if g.perf_mode_on then + params = case.perf_params + else + params = case.integration_params + end + + local connections = {} + + local router = g.cluster:server('router') + for _ = 1, params.connection_count do + local c = net_box:connect(router.net_box_uri, router.net_box_credentials) + if c == nil then + t.fail('Failed to prepare connections') + end + table.insert(connections, c) + end + + local fibers = {} + local report = { errors = {}, count = 0, total_time = 0, max_time = 0 } + for id = 1, params.fiber_count do + local conn_id = id % params.connection_count + 1 + local conn = connections[conn_id] + local f = fiber.new(generator_f, conn, case.call, case.params, report, params.timeout) + f:set_joinable(true) + table.insert(fibers, f) + end + + local start_time = clock.monotonic() + for i = 1, params.fiber_count do + fibers[i]:join() + end + local run_time = clock.monotonic() - start_time + + report.str = { + success_count = ('%d'):format(report.count), + error_count = ('%d'):format(#report.errors), + success_rps = ('%.2f'):format(report.count / run_time), + max_time = ('%.3f ms'):format(report.max_time * 1e3), + } + + local total_count = report.count + #report.errors + if total_count > 0 then + report.str.average_time = ('%.3f ms'):format(report.total_time / total_count * 1e3) + else + report.str.average_time = 'unknown' + end + + g.total_report[case.row_name] = g.total_report[case.row_name] or {} + g.total_report[case.row_name][subcase.column_name] = report + + log.info('\n%s: %s success requests (rps %s), %s errors, call average time %s, call max time %s \n', + test_name, report.str.success_count, report.str.success_rps, report.str.error_count, + report.str.average_time, report.str.max_time) + end + end +end diff --git a/test/performance/select_perf_test.lua b/test/performance/select_perf_test.lua deleted file mode 100644 index 683e77088..000000000 --- a/test/performance/select_perf_test.lua +++ /dev/null @@ -1,167 +0,0 @@ -local fio = require('fio') -local fiber = require('fiber') -local errors = require('errors') -local net_box = require('net.box') -local log = require('log') - -local t = require('luatest') -local g = t.group('perf') - -local helpers = require('test.helper') - -g.before_all = function() - g.cluster = helpers.Cluster:new({ - datadir = fio.tempdir(), - server_command = helpers.entrypoint('srv_select'), - use_vshard = true, - replicasets = { - { - uuid = helpers.uuid('a'), - alias = 'router', - roles = { 'crud-router' }, - servers = { - { instance_uuid = helpers.uuid('a', 1), alias = 'router' }, - }, - }, - { - uuid = helpers.uuid('b'), - alias = 's-1', - roles = { 'customers-storage', 'crud-storage' }, - servers = { - { instance_uuid = helpers.uuid('b', 1), alias = 's1-master' }, - { instance_uuid = helpers.uuid('b', 2), alias = 's1-replica' }, - }, - }, - { - uuid = helpers.uuid('c'), - alias = 's-2', - roles = { 'customers-storage', 'crud-storage' }, - servers = { - { instance_uuid = helpers.uuid('c', 1), alias = 's2-master' }, - { instance_uuid = helpers.uuid('c', 2), alias = 's2-replica' }, - }, - }, - { - uuid = helpers.uuid('d'), - alias = 's-2', - roles = { 'customers-storage', 'crud-storage' }, - servers = { - { instance_uuid = helpers.uuid('d', 1), alias = 's3-master' }, - { instance_uuid = helpers.uuid('d', 2), alias = 's3-replica' }, - }, - } - }, - }) - g.cluster:start() -end - -g.after_all = function() - g.cluster:stop() - fio.rmtree(g.cluster.datadir) -end - -g.before_each(function() end) - -local function insert_customers(conn, id, count, timeout, report) - local customer = {id, box.NULL, 'David', 'Smith', 33, 'Los Angeles'} - local start = fiber.clock() - - while (fiber.clock() - start) < timeout do - local ok, res, err = pcall(conn.call, conn, [[package.loaded.crud.insert]], {'customers', customer}) - if not ok then - log.error('Insert error: %s', res) - table.insert(report.errors, res) - elseif err ~= nil then - errors.wrap(err) - log.error('Insert error: %s', err) - table.insert(report.errors, err) - else - report.count = report.count + 1 - end - customer[1] = customer[1] + count - end -end - -local function select_customers(conn, id, timeout, report) - local start = fiber.clock() - local ok, err = pcall(function() - while (fiber.clock() - start) < timeout do - local _, err = conn:call([[package.loaded.crud.select]], {'customers', {{'>', 'id', id}}, {first = 10}}) - if err ~= nil then - errors.wrap(err) - log.error(err) - table.insert(report.errors, err) - else - report.count = report.count + 1 - end - end - end) - if not ok then - table.insert(report.errors, err) - log.error(err) - end -end - -g.test_insert = function() - local timeout = 30 - local fiber_count = 600 - local connection_count = 10 - local connections = {} - - local server = g.cluster.main_server - server.net_box:eval([[require('crud')]]) - for _ = 1, connection_count do - local c = net_box:connect(server.net_box_uri, server.net_box_credentials) - assert(c) - table.insert(connections, c) - end - - local fibers = {} - local report = {errors = {}, count = 0} - for id = 1, fiber_count do - local conn_id = id % connection_count + 1 - local conn = connections[conn_id] - local f = fiber.new(insert_customers, conn, id, fiber_count, timeout, report) - f:set_joinable(true) - table.insert(fibers, f) - end - - for i = 1, fiber_count do - fibers[i]:join() - end - - log.error('\nINSERT: requests %d, rps %d, errors %d', - report.count, report.count / timeout, #report.errors) -end - -g.test_select = function() - local timeout = 30 - local fiber_count = 200 - local connection_count = 10 - local connections = {} - - local server = g.cluster.main_server - server.net_box:eval([[require('crud')]]) - for _ = 1, connection_count do - local c = net_box:connect(server.net_box_uri, server.net_box_credentials) - assert(c) - table.insert(connections, c) - end - - local fibers = {} - local report = {errors = {}, count = 0} - for id = 1, fiber_count do - local conn_id = id % connection_count + 1 - local conn = connections[conn_id] - local f = fiber.new(select_customers, conn, id, timeout, report) - f:set_joinable(true) - table.insert(fibers, f) - end - - for i = 1, fiber_count do - fibers[i]:join() - end - - log.error('\nSELECT: requests %d, rps %d, errors %d', - report.count, report.count / timeout, #report.errors) -end