From a9de166ce7d58b4ccabbd1d586a2ee9dc3b878e8 Mon Sep 17 00:00:00 2001 From: danielhanrahan Date: Sat, 21 Sep 2024 16:47:43 +0100 Subject: [PATCH] [k6] CM handles searches using different filters Each 5 VUs for searches uses a different filter: no filter, module, property, cps path, and trust level. This is needed since each search may have different performance. This also helps reduce load versus doing a combined module and property search all the time. Issue-ID: CPS-2349 Signed-off-by: danielhanrahan Change-Id: I33a75260f97697d2061d10b80ba7eeb2f6c346f5 --- k6-tests/ncmp/common/cmhandle-crud.js | 2 +- k6-tests/ncmp/common/search-base.js | 21 ++- k6-tests/ncmp/common/utils.js | 12 +- k6-tests/ncmp/ncmp-kpi.js | 176 ++++++++++++++++++++++---- 4 files changed, 181 insertions(+), 30 deletions(-) diff --git a/k6-tests/ncmp/common/cmhandle-crud.js b/k6-tests/ncmp/common/cmhandle-crud.js index 7fab62abd8..0fea1a44f2 100644 --- a/k6-tests/ncmp/common/cmhandle-crud.js +++ b/k6-tests/ncmp/common/cmhandle-crud.js @@ -66,7 +66,7 @@ function createCmHandlePayload(cmHandleIds) { } function getNumberOfReadyCmHandles() { - const response = executeCmHandleIdSearch('readyCmHandles'); + const response = executeCmHandleIdSearch('cps-path-for-ready-cm-handles'); const arrayOfCmHandleIds = JSON.parse(response.body); return arrayOfCmHandleIds.length; } diff --git a/k6-tests/ncmp/common/search-base.js b/k6-tests/ncmp/common/search-base.js index a6424fe5d0..1e98e9b07b 100644 --- a/k6-tests/ncmp/common/search-base.js +++ b/k6-tests/ncmp/common/search-base.js @@ -36,19 +36,34 @@ function executeSearchRequest(searchType, scenario) { } const SEARCH_PARAMETERS_PER_SCENARIO = { - "module-and-properties": { + "no-filter": { + "cmHandleQueryParameters": [] + }, + "module": { "cmHandleQueryParameters": [ { "conditionName": "hasAllModules", "conditionParameters": [{"moduleName": "ietf-yang-types"}] - }, + } + ] + }, + "property": { + "cmHandleQueryParameters": [ { "conditionName": "hasAllProperties", "conditionParameters": [{"Color": "yellow"}] } ] }, - "readyCmHandles": { + "trust-level": { + "cmHandleQueryParameters": [ + { + "conditionName": "cmHandleWithTrustLevel", + "conditionParameters": [ {"trustLevel": "COMPLETE"} ] + } + ] + }, + "cps-path-for-ready-cm-handles": { "cmHandleQueryParameters": [ { "conditionName": "cmHandleWithCpsPath", diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js index 8ee6d10123..b52866c3fb 100644 --- a/k6-tests/ncmp/common/utils.js +++ b/k6-tests/ncmp/common/utils.js @@ -87,8 +87,16 @@ export function makeCustomSummaryReport(testResults, scenarioConfig) { makeSummaryCsvLine('0', 'HTTP request failures for all tests', 'rate of failed requests', 'http_req_failed', 0, testResults, scenarioConfig), makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', 110, testResults, scenarioConfig), makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', 80, testResults, scenarioConfig), - makeSummaryCsvLine('3', 'CM-handle ID search with Module and Property filter', 'milliseconds', 'id_search_duration', 4000, testResults, scenarioConfig), - makeSummaryCsvLine('4', 'CM-handle search with Module and Property filter', 'milliseconds', 'cm_search_duration', 30000, testResults, scenarioConfig), + makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 4000, testResults, scenarioConfig), + makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 4000, testResults, scenarioConfig), + makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 4000, testResults, scenarioConfig), + makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 4000, testResults, scenarioConfig), + makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 4000, testResults, scenarioConfig), + makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 30000, testResults, scenarioConfig), + makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 30000, testResults, scenarioConfig), + makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 30000, testResults, scenarioConfig), + makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 30000, testResults, scenarioConfig), + makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 30000, testResults, scenarioConfig), makeSummaryCsvLine('5a', 'NCMP overhead for Synchronous single CM-handle pass-through read', 'milliseconds', 'ncmp_overhead_passthrough_read', 40, testResults, scenarioConfig), makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 60, testResults, scenarioConfig), makeSummaryCsvLine('6a', 'NCMP overhead for Synchronous single CM-handle pass-through write', 'milliseconds', 'ncmp_overhead_passthrough_write', 30, testResults, scenarioConfig), diff --git a/k6-tests/ncmp/ncmp-kpi.js b/k6-tests/ncmp/ncmp-kpi.js index f9a19c394f..8815511bdd 100644 --- a/k6-tests/ncmp/ncmp-kpi.js +++ b/k6-tests/ncmp/ncmp-kpi.js @@ -37,8 +37,16 @@ let passthroughReadNcmpOverheadTrend = new Trend('ncmp_overhead_passthrough_read let passthroughReadNcmpOverheadTrendWithAlternateId = new Trend('ncmp_overhead_passthrough_read_alt_id', true); let passthroughWriteNcmpOverheadTrend = new Trend('ncmp_overhead_passthrough_write', true); let passthroughWriteNcmpOverheadTrendWithAlternateId = new Trend('ncmp_overhead_passthrough_write_alt_id', true); -let idSearchDurationTrend = new Trend('id_search_duration', true); -let cmSearchDurationTrend = new Trend('cm_search_duration', true); +let idSearchNoFilterDurationTrend = new Trend('id_search_nofilter_duration', true); +let idSearchModuleDurationTrend = new Trend('id_search_module_duration', true); +let idSearchPropertyDurationTrend = new Trend('id_search_property_duration', true); +let idSearchCpsPathDurationTrend = new Trend('id_search_cpspath_duration', true); +let idSearchTrustLevelDurationTrend = new Trend('id_search_trustlevel_duration', true); +let cmSearchNoFilterDurationTrend = new Trend('cm_search_nofilter_duration', true); +let cmSearchModuleDurationTrend = new Trend('cm_search_module_duration', true); +let cmSearchPropertyDurationTrend = new Trend('cm_search_property_duration', true); +let cmSearchCpsPathDurationTrend = new Trend('cm_search_cpspath_duration', true); +let cmSearchTrustLevelDurationTrend = new Trend('cm_search_trustlevel_duration', true); let legacyBatchReadCmHandlesPerSecondTrend = new Trend('legacy_batch_read_cmhandles_per_second', false); const legacyBatchEventReader = new Reader({ @@ -56,37 +64,85 @@ export const options = { passthrough_read_scenario: { executor: 'constant-vus', exec: 'passthroughReadScenario', - vus: 4, + vus: 2, duration: DURATION, }, passthrough_read_alt_id_scenario: { executor: 'constant-vus', exec: 'passthroughReadAltIdScenario', - vus: 4, + vus: 2, duration: DURATION, }, passthrough_write_scenario: { executor: 'constant-vus', exec: 'passthroughWriteScenario', - vus: 4, + vus: 2, duration: DURATION, }, passthrough_write_alt_id_scenario: { executor: 'constant-vus', exec: 'passthroughWriteAltIdScenario', - vus: 4, + vus: 2, + duration: DURATION, + }, + cm_handle_id_search_nofilter_scenario: { + executor: 'constant-vus', + exec: 'cmHandleIdSearchNoFilterScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_search_nofilter_scenario: { + executor: 'constant-vus', + exec: 'cmHandleSearchNoFilterScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_id_search_module_scenario: { + executor: 'constant-vus', + exec: 'cmHandleIdSearchModuleScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_search_module_scenario: { + executor: 'constant-vus', + exec: 'cmHandleSearchModuleScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_id_search_property_scenario: { + executor: 'constant-vus', + exec: 'cmHandleIdSearchPropertyScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_search_property_scenario: { + executor: 'constant-vus', + exec: 'cmHandleSearchPropertyScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_id_search_cpspath_scenario: { + executor: 'constant-vus', + exec: 'cmHandleIdSearchCpsPathScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_search_cpspath_scenario: { + executor: 'constant-vus', + exec: 'cmHandleSearchCpsPathScenario', + vus: 1, duration: DURATION, }, - cm_handle_id_search_scenario: { + cm_handle_id_search_trustlevel_scenario: { executor: 'constant-vus', - exec: 'cmHandleIdSearchScenario', - vus: 5, + exec: 'cmHandleIdSearchTrustLevelScenario', + vus: 1, duration: DURATION, }, - cm_handle_search_scenario: { + cm_handle_search_trustlevel_scenario: { executor: 'constant-vus', - exec: 'cmHandleSearchScenario', - vus: 5, + exec: 'cmHandleSearchTrustLevelScenario', + vus: 1, duration: DURATION, }, legacy_batch_produce_scenario: { @@ -112,8 +168,16 @@ export const options = { 'ncmp_overhead_passthrough_write': ['avg <= 40'], 'ncmp_overhead_passthrough_read_alt_id': ['avg <= 40'], 'ncmp_overhead_passthrough_write_alt_id': ['avg <= 40'], - 'id_search_duration': ['avg <= 2000'], - 'cm_search_duration': ['avg <= 15000'], + 'id_search_nofilter_duration': ['avg <= 2000'], + 'id_search_module_duration': ['avg <= 2000'], + 'id_search_property_duration': ['avg <= 2000'], + 'id_search_cpspath_duration': ['avg <= 2000'], + 'id_search_trustlevel_duration': ['avg <= 2000'], + 'cm_search_nofilter_duration': ['avg <= 15000'], + 'cm_search_module_duration': ['avg <= 15000'], + 'cm_search_property_duration': ['avg <= 15000'], + 'cm_search_cpspath_duration': ['avg <= 15000'], + 'cm_search_trustlevel_duration': ['avg <= 15000'], 'legacy_batch_read_cmhandles_per_second': ['avg >= 150'], }, }; @@ -188,19 +252,83 @@ export function passthroughWriteAltIdScenario() { } } -export function cmHandleIdSearchScenario() { - const response = executeCmHandleIdSearch('module-and-properties'); - if (check(response, { 'CM handle ID search status equals 200': (r) => r.status === 200 }) - && check(response, { 'CM handle ID search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { - idSearchDurationTrend.add(response.timings.duration); +export function cmHandleIdSearchNoFilterScenario() { + const response = executeCmHandleIdSearch('no-filter'); + if (check(response, { 'CM handle ID no-filter search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle ID no-filter search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + idSearchNoFilterDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleSearchNoFilterScenario() { + const response = executeCmHandleSearch('no-filter'); + if (check(response, { 'CM handle no-filter search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle no-filter search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + cmSearchNoFilterDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleIdSearchModuleScenario() { + const response = executeCmHandleIdSearch('module'); + if (check(response, { 'CM handle ID module search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle ID module search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + idSearchModuleDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleSearchModuleScenario() { + const response = executeCmHandleSearch('module'); + if (check(response, { 'CM handle module search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle module search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + cmSearchModuleDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleIdSearchPropertyScenario() { + const response = executeCmHandleIdSearch('property'); + if (check(response, { 'CM handle ID property search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle ID property search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + idSearchPropertyDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleSearchPropertyScenario() { + const response = executeCmHandleSearch('property'); + if (check(response, { 'CM handle property search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle property search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + cmSearchPropertyDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleIdSearchCpsPathScenario() { + const response = executeCmHandleIdSearch('cps-path-for-ready-cm-handles'); + if (check(response, { 'CM handle ID cps path search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle ID cps path search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + idSearchCpsPathDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleSearchCpsPathScenario() { + const response = executeCmHandleSearch('cps-path-for-ready-cm-handles'); + if (check(response, { 'CM handle cps path search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle cps path search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + cmSearchCpsPathDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleIdSearchTrustLevelScenario() { + const response = executeCmHandleIdSearch('trust-level'); + if (check(response, { 'CM handle ID trust level search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle ID trust level search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + idSearchTrustLevelDurationTrend.add(response.timings.duration); } } -export function cmHandleSearchScenario() { - const response = executeCmHandleSearch('module-and-properties'); - if (check(response, { 'CM handle search status equals 200': (r) => r.status === 200 }) - && check(response, { 'CM handle search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { - cmSearchDurationTrend.add(response.timings.duration); +export function cmHandleSearchTrustLevelScenario() { + const response = executeCmHandleSearch('trust-level'); + if (check(response, { 'CM handle trust level search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle trust level search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + cmSearchTrustLevelDurationTrend.add(response.timings.duration); } }