diff --git a/migrations/1674942200154_histogram-plot.js b/migrations/1674942200154_histogram-plot.js new file mode 100644 index 00000000..5e379f43 --- /dev/null +++ b/migrations/1674942200154_histogram-plot.js @@ -0,0 +1,9 @@ +exports.up = (pgm) => { + pgm.addColumn({ schema: "jtl", name: "charts" }, { + histogram_plot_data: { + type: "jsonb", + "default": null, + notNull: false, + }, + }) +} diff --git a/src/server/controllers/item/get-item-controller.spec.ts b/src/server/controllers/item/get-item-controller.spec.ts index e8e47114..b89eed54 100644 --- a/src/server/controllers/item/get-item-controller.spec.ts +++ b/src/server/controllers/item/get-item-controller.spec.ts @@ -27,6 +27,7 @@ describe("getItemController", () => { (db.one as any).mockResolvedValueOnce({ plot_data: {}, extra_plot_data: {}, + histogram_plot_data: {}, note: "my note", environment: "environment", // eslint-disable-next-line @typescript-eslint/naming-convention @@ -49,6 +50,7 @@ describe("getItemController", () => { baseId: null, environment: "environment", extraPlotData: {}, + histogramPlotData: {}, hostname: null, isBase: false, monitoring: { diff --git a/src/server/controllers/item/get-item-controller.ts b/src/server/controllers/item/get-item-controller.ts index 33eacab9..91caffe8 100644 --- a/src/server/controllers/item/get-item-controller.ts +++ b/src/server/controllers/item/get-item-controller.ts @@ -13,6 +13,7 @@ export const getItemController = async (req: IGetUserAuthInfoRequest, res: Respo const { plot_data: plot, extra_plot_data: extraPlotData, + histogram_plot_data: histogramPlotData, note, environment, // eslint-disable-next-line @typescript-eslint/naming-convention @@ -40,6 +41,7 @@ export const getItemController = async (req: IGetUserAuthInfoRequest, res: Respo overview, sutOverview, statistics, status, apdexSettings, plot, extraPlotData, note, environment, hostname, reportStatus, thresholds, analysisEnabled, baseId: base_id, isBase: base_id === itemId, zeroErrorToleranceEnabled, topMetricsSettings, + histogramPlotData, name, monitoring: { cpu: { diff --git a/src/server/controllers/item/shared/item-data-processing.ts b/src/server/controllers/item/shared/item-data-processing.ts index 7a1e625f..f8093788 100644 --- a/src/server/controllers/item/shared/item-data-processing.ts +++ b/src/server/controllers/item/shared/item-data-processing.ts @@ -3,14 +3,27 @@ import { db } from "../../../../db/db" import { logger } from "../../../../logger" import { prepareDataForSavingToDb, - prepareChartDataForSaving, + prepareChartDataForSaving, prepareHistogramDataForSaving, } from "../../../data-stats/prepare-data" import { chartQueryOptionInterval } from "../../../data-stats/helper/duration" import { - saveThresholdsResult, saveItemStats, savePlotData, updateItem, - aggOverviewQuery, aggLabelQuery, chartOverviewQuery, - charLabelQuery, sutOverviewQuery, distributedThreadsQuery, responseCodeDistribution, responseMessageFailures, - deleteSamples, calculateApdexValues, updateItemApdexSettings, chartOverviewStatusCodesQuery, + saveThresholdsResult, + saveItemStats, + savePlotData, + updateItem, + aggOverviewQuery, + aggLabelQuery, + chartOverviewQuery, + charLabelQuery, + sutOverviewQuery, + distributedThreadsQuery, + responseCodeDistribution, + responseMessageFailures, + deleteSamples, + calculateApdexValues, + updateItemApdexSettings, + chartOverviewStatusCodesQuery, + responseTimePerLabelHistogram, } from "../../../queries/items" import { ReportStatus } from "../../../queries/items.model" import { getScenarioSettings, currentScenarioMetrics } from "../../../queries/scenario" @@ -29,9 +42,11 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId }) const aggOverview = await db.one(aggOverviewQuery(itemId)) const aggLabel = await db.many(aggLabelQuery(itemId)) const statusCodeDistribution = await db.manyOrNone(responseCodeDistribution(itemId)) + const responseTimePerLabelDistribution = await db.manyOrNone(responseTimePerLabelHistogram(itemId)) const responseFailures = await db.manyOrNone(responseMessageFailures(itemId)) const scenarioSettings = await db.one(getScenarioSettings(projectName, scenarioName)) + console.log({ responseTimePerLabelDistribution }) if (aggOverview.number_of_sut_hostnames > 1) { sutMetrics = await db.many(sutOverviewQuery(itemId)) @@ -55,6 +70,7 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId }) labelStats, sutOverview, } = prepareDataForSavingToDb(aggOverview, aggLabel, sutMetrics, statusCodeDistribution, responseFailures, apdex) + const responseTimeHistogram = prepareHistogramDataForSaving(responseTimePerLabelDistribution) const defaultInterval = chartQueryOptionInterval(duration) let chartData const extraChartData = [] @@ -112,7 +128,8 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId }) await db.tx(async t => { await t.none(saveItemStats(itemId, JSON.stringify(labelStats), overview, JSON.stringify(sutOverview))) - await t.none(savePlotData(itemId, JSON.stringify(chartData), JSON.stringify(extraChartData))) + await t.none(savePlotData(itemId, JSON.stringify(chartData), JSON.stringify(extraChartData), + JSON.stringify(responseTimeHistogram))) await t.none(updateItem(itemId, ReportStatus.Ready, overview.startDate)) }) diff --git a/src/server/data-stats/prepare-data.spec.ts b/src/server/data-stats/prepare-data.spec.ts index 1e2927c3..c4ea350e 100644 --- a/src/server/data-stats/prepare-data.spec.ts +++ b/src/server/data-stats/prepare-data.spec.ts @@ -2,498 +2,523 @@ import { - calculateDistributedThreads, - prepareChartDataForSaving, - prepareDataForSavingToDb, - stringToNumber, transformDataForDb, transformMonitoringDataForDb, + calculateDistributedThreads, + prepareChartDataForSaving, + prepareDataForSavingToDb, prepareHistogramDataForSaving, + stringToNumber, transformDataForDb, transformMonitoringDataForDb, } from "./prepare-data" describe("prepare data", () => { - describe("transformDataForDb", () => { - it("should return undefined when unable to process data", () => { - const result = transformDataForDb({}, "itemId", []) - expect(result).toBeUndefined() + describe("transformDataForDb", () => { + it("should return undefined when unable to process data", () => { + const result = transformDataForDb({}, "itemId", []) + expect(result).toBeUndefined() + }) + it("should call shouldSkipLabel function", () => { + const shouldSkipLabelSpy = jest.spyOn(require("../controllers/item/utils/labelFilter"), "shouldSkipLabel") + const inputData = { + bytes: "792", + sentBytes: "124", + label: "endpoint3", + // eslint-disable-next-line @typescript-eslint/naming-convention + Connect: "155", + // eslint-disable-next-line @typescript-eslint/naming-convention + Latency: "190", + elapsed: "191", + success: "true", + // eslint-disable-next-line @typescript-eslint/naming-convention + Hostname: "localhost", + timeStamp: "1555399218911", + allThreads: "1", + grpThreads: "1", + threadName: "Thread 1-1", + responseCode: "200", + responseMessage: "", + } + transformDataForDb(inputData, "itemId", []) + expect(shouldSkipLabelSpy).toHaveBeenCalledWith(inputData.label, []) + }) + it("should correctly proccess data", () => { + const inputData = { + bytes: "792", + sentBytes: "124", + label: "endpoint3", + // eslint-disable-next-line @typescript-eslint/naming-convention + Connect: "155", + // eslint-disable-next-line @typescript-eslint/naming-convention + Latency: "190", + elapsed: "191", + success: "true", + // eslint-disable-next-line @typescript-eslint/naming-convention + Hostname: "localhost", + timeStamp: "1555399218911", + allThreads: "1", + grpThreads: "1", + threadName: "Thread 1-1", + responseCode: "200", + responseMessage: "", + } + const result = transformDataForDb(inputData, "itemId", []) + expect(result).toEqual({ + bytes: 792, + sentBytes: 124, + label: "endpoint3", + // eslint-disable-next-line @typescript-eslint/naming-convention + Connect: 155, + // eslint-disable-next-line @typescript-eslint/naming-convention + Latency: 190, + elapsed: 191, + success: true, + // eslint-disable-next-line @typescript-eslint/naming-convention + Hostname: "localhost", + timeStamp: new Date(1555399218911), + allThreads: 1, + grpThreads: 1, + itemId: "itemId", + threadName: "Thread 1-1", + responseCode: "200", + responseMessage: "", + sutHostname: undefined, + }) + }) + it("should return sutHostname when URL provided with valid url", () => { + const inputData = { + bytes: "792", + sentBytes: "123", + label: "endpoint3", + // eslint-disable-next-line @typescript-eslint/naming-convention + Connect: "155", + // eslint-disable-next-line @typescript-eslint/naming-convention + Latency: "190", + elapsed: "191", + success: "true", + // eslint-disable-next-line @typescript-eslint/naming-convention + Hostname: "localhost", + timeStamp: "1555399218911", + allThreads: "1", + grpThreads: "1", + threadName: "Thread 1-1", + responseCode: "200", + responseMessage: "", + URL: "https://example.com/styles.css", + } + const result = transformDataForDb(inputData, "itemId", []) + expect(result.sutHostname).toEqual("example.com") + }) + it("should return sutHostname undefined when URL contains empty url", () => { + const inputData = { + bytes: "792", + sentBytes: "123", + label: "endpoint3", + // eslint-disable-next-line @typescript-eslint/naming-convention + Connect: "155", + // eslint-disable-next-line @typescript-eslint/naming-convention + Latency: "190", + elapsed: "191", + success: "true", + // eslint-disable-next-line @typescript-eslint/naming-convention + Hostname: "localhost", + timeStamp: "1555399218911", + allThreads: "1", + grpThreads: "1", + threadName: "Thread 1-1", + responseCode: "200", + responseMessage: "", + URL: "", + } + const result = transformDataForDb(inputData, "itemId", []) + expect(result.sutHostname).toBeUndefined() + }) + it("should return sutHostname undefined when URL contains invalid url", () => { + const inputData = { + bytes: "792", + sentBytes: "123", + label: "endpoint3", + // eslint-disable-next-line @typescript-eslint/naming-convention + Connect: "155", + // eslint-disable-next-line @typescript-eslint/naming-convention + Latency: "190", + elapsed: "191", + success: "true", + // eslint-disable-next-line @typescript-eslint/naming-convention + Hostname: "localhost", + timeStamp: "1555399218911", + allThreads: "1", + grpThreads: "1", + threadName: "Thread 1-1", + responseCode: "200", + responseMessage: "", + URL: "file", + } + const result = transformDataForDb(inputData, "itemId", []) + expect(result.sutHostname).toBeUndefined() + }) + it("should process the data even when sentBytes not provided", () => { + const inputData = { + bytes: "792", + label: "endpoint3", + // eslint-disable-next-line @typescript-eslint/naming-convention + Connect: "155", + // eslint-disable-next-line @typescript-eslint/naming-convention + Latency: "190", + elapsed: "191", + success: "true", + // eslint-disable-next-line @typescript-eslint/naming-convention + Hostname: "localhost", + timeStamp: "1555399218911", + allThreads: "1", + grpThreads: "1", + threadName: "Thread 1-1", + responseCode: "200", + responseMessage: "", + URL: "file", + } + const result = transformDataForDb(inputData, "itemId", []) + expect(result.sentBytes).toEqual(0) + }) }) - it("should call shouldSkipLabel function", () => { - const shouldSkipLabelSpy = jest.spyOn(require("../controllers/item/utils/labelFilter"), "shouldSkipLabel") - const inputData = { - bytes: "792", - sentBytes: "124", - label: "endpoint3", - // eslint-disable-next-line @typescript-eslint/naming-convention - Connect: "155", - // eslint-disable-next-line @typescript-eslint/naming-convention - Latency: "190", - elapsed: "191", - success: "true", - // eslint-disable-next-line @typescript-eslint/naming-convention - Hostname: "localhost", - timeStamp: "1555399218911", - allThreads: "1", - grpThreads: "1", - threadName: "Thread 1-1", - responseCode: "200", - responseMessage: "", - } - transformDataForDb(inputData, "itemId", []) - expect(shouldSkipLabelSpy).toHaveBeenCalledWith(inputData.label, []) + describe("stringToNumber", () => { + it("should convert string to number", () => { + const result = stringToNumber("1", 10) + expect(result).toBe(1) + }) + it("should throw an error when unable to convert ", () => { + expect(() => { + stringToNumber(undefined, 10) + }).toThrow() + }) }) - it("should correctly proccess data", () => { - const inputData = { - bytes: "792", - sentBytes: "124", - label: "endpoint3", - // eslint-disable-next-line @typescript-eslint/naming-convention - Connect: "155", - // eslint-disable-next-line @typescript-eslint/naming-convention - Latency: "190", - elapsed: "191", - success: "true", - // eslint-disable-next-line @typescript-eslint/naming-convention - Hostname: "localhost", - timeStamp: "1555399218911", - allThreads: "1", - grpThreads: "1", - threadName: "Thread 1-1", - responseCode: "200", - responseMessage: "", - } - const result = transformDataForDb(inputData, "itemId", []) - expect(result).toEqual({ - bytes: 792, - sentBytes: 124, - label: "endpoint3", - // eslint-disable-next-line @typescript-eslint/naming-convention - Connect: 155, - // eslint-disable-next-line @typescript-eslint/naming-convention - Latency: 190, - elapsed: 191, - success: true, - // eslint-disable-next-line @typescript-eslint/naming-convention - Hostname: "localhost", - timeStamp: new Date(1555399218911), - allThreads: 1, - grpThreads: 1, - itemId: "itemId", - threadName: "Thread 1-1", - responseCode: "200", - responseMessage: "", - sutHostname: undefined, - }) + describe("calculateDistributedThreads", () => { + it("should correctly calculate distributed threads", () => { + const inputData = [ + { + time: new Date(1555399218911), + hostname: "generator-1", + threads: 10, + }, + { + time: new Date(1555399218911), + hostname: "generator-2", + threads: 10, + }, + { + time: new Date(1555399218911), + hostname: "generator-3", + threads: 5, + }, + ] + const distributedThreads = calculateDistributedThreads(inputData) + expect(distributedThreads).toEqual([[1555399218911, 25]]) + }) }) - it("should return sutHostname when URL provided with valid url", () => { - const inputData = { - bytes: "792", - sentBytes: "123", - label: "endpoint3", - // eslint-disable-next-line @typescript-eslint/naming-convention - Connect: "155", - // eslint-disable-next-line @typescript-eslint/naming-convention - Latency: "190", - elapsed: "191", - success: "true", - // eslint-disable-next-line @typescript-eslint/naming-convention - Hostname: "localhost", - timeStamp: "1555399218911", - allThreads: "1", - grpThreads: "1", - threadName: "Thread 1-1", - responseCode: "200", - responseMessage: "", - URL: "https://example.com/styles.css", - } - const result = transformDataForDb(inputData, "itemId", []) - expect(result.sutHostname).toEqual("example.com") + describe("prepareDataForSavingToDb", () => { + it("should correctly parse data", () => { + const overviewData = { + _id: null, + start: new Date("2021-03-29T10:57:10.882Z"), + end: new Date("2021-03-29T11:27:10.171Z"), + avg_connect: 5.802922997682204, + avg_latency: 105.62091166623745, + avg_response: 105.72559876384238, + bytes_received_total: 123123, + bytes_sent_total: 69848465, + total: 46596, + n90: 271, + number_of_failed: 3, + } + const labelsData = [ + { + label: "label1", + min_response: 227, + max_response: 1325, + avg_response: 286.97317596566523, + bytes_received_total: 123, + bytes_sent_total: 1231, + total_samples: 932, + start: new Date("2021-03-29T10:59:01.561Z"), + end: new Date("2021-03-29T11:27:07.702Z"), + n90: 343, + n95: 367, + n99: 418, + latency: 1, + connect: 2, + number_of_failed: 0, + }, + { + label: "label2", + min_response: 35, + max_response: 118, + avg_response: 44.93503480278422, + total_samples: 431, + bytes_received_total: 123, + bytes_sent_total: 1231, + start: new Date("2021-03-29T11:00:53.221Z"), + end: new Date("2021-03-29T11:27:01.650Z"), + n90: 50, + n95: 56, + n99: 93, + latency: 3, + connect: 4, + number_of_failed: 0, + }, + ] + const statusCodes = [ + { label: "label2", status_code: "200", count: 433 }, + { label: "label1", status_code: "200", count: 932 }] + const responseFailures = [ + { + label: "label1", + response_message: "failure", + count: 31, + status_code: "100", + failure_message: "failure", + }, + { + label: "label2", + response_message: "failure2", + count: 1, + status_code: "101", + failure_message: "failure1", + }, + ] + const apdex = [ + { label: "label1", toleration: "40", satisfaction: "200" }, + { label: "label2", toleration: "30", satisfaction: "100" }] + const { overview, labelStats } = prepareDataForSavingToDb(overviewData, labelsData, [], + statusCodes, responseFailures, apdex) + expect(overview).toEqual({ + percentil: 271, + maxVu: undefined, + avgResponseTime: 106, + errorRate: 0.01, + errorCount: 3, + throughput: 25.9, + bytesPerSecond: 38820.04, + bytesSentPerSecond: 68.43, + avgLatency: 105.62, + avgConnect: 5.8, + startDate: new Date("2021-03-29T10:57:10.882Z"), + endDate: new Date("2021-03-29T11:27:10.171Z"), + duration: 29.99, + }) + expect(labelStats).toEqual([ + { + label: "label1", + samples: 932, + avgResponseTime: 287, + minResponseTime: 227, + maxResponseTime: 1325, + errorRate: 0, + bytesPerSecond: 0.07, + bytesSentPerSecond: 0.73, + throughput: 0.55, + n9: 418, + n5: 367, + n0: 343, + latency: 1, + connect: 2, + statusCodes: [{ count: 932, statusCode: "200" }], + responseMessageFailures: [{ + count: 31, responseMessage: "failure", statusCode: "100", failureMessage: "failure", + }], + apdex: { + satisfaction: 200, + toleration: 40, + }, + }, + { + label: "label2", + samples: 431, + avgResponseTime: 45, + minResponseTime: 35, + maxResponseTime: 118, + errorRate: 0, + bytesPerSecond: 0.08, + bytesSentPerSecond: 0.78, + throughput: 0.27, + n9: 93, + n5: 56, + n0: 50, + latency: 3, + connect: 4, + statusCodes: [{ count: 433, statusCode: "200" }], + responseMessageFailures: [{ + count: 1, responseMessage: "failure2", statusCode: "101", failureMessage: "failure1", + }], + apdex: { + toleration: 30, + satisfaction: 100, + }, + }]) + }) }) - it("should return sutHostname undefined when URL contains empty url", () => { - const inputData = { - bytes: "792", - sentBytes: "123", - label: "endpoint3", - // eslint-disable-next-line @typescript-eslint/naming-convention - Connect: "155", - // eslint-disable-next-line @typescript-eslint/naming-convention - Latency: "190", - elapsed: "191", - success: "true", - // eslint-disable-next-line @typescript-eslint/naming-convention - Hostname: "localhost", - timeStamp: "1555399218911", - allThreads: "1", - grpThreads: "1", - threadName: "Thread 1-1", - responseCode: "200", - responseMessage: "", - URL: "", - } - const result = transformDataForDb(inputData, "itemId", []) - expect(result.sutHostname).toBeUndefined() + describe("transformMonitoringDataForDb", () => { + it("should parse input data correctly", () => { + const row = { + name: "test", + cpu: "10", + mem: "5", + ts: "1555399218", + } + const itemId = "myTestId" + const transformedMonitoringData = transformMonitoringDataForDb(row, itemId) + expect(transformedMonitoringData).toEqual({ + name: "test", + cpu: 10, + mem: 5, + itemId, + timestamp: new Date("2019-04-16T07:20:18.000Z"), + }) + }) + it("should return indefined when an error occurs during parsing", () => { + const itemId = "myTestId" + const row = {} + const transformedMonitoringData = transformMonitoringDataForDb(row, itemId) + expect(transformedMonitoringData).toBeUndefined() + }) + it("should return localhost if no name provided", () => { + const row = { + cpu: "10", + mem: "5", + ts: "1555399218", + } + const itemId = "myTestId" + const transformedMonitoringData = transformMonitoringDataForDb(row, itemId) + expect(transformedMonitoringData).toEqual({ + name: "localhost", + cpu: 10, + mem: 5, + itemId, + timestamp: new Date("2019-04-16T07:20:18.000Z"), + }) + }) }) - it("should return sutHostname undefined when URL contains invalid url", () => { - const inputData = { - bytes: "792", - sentBytes: "123", - label: "endpoint3", - // eslint-disable-next-line @typescript-eslint/naming-convention - Connect: "155", - // eslint-disable-next-line @typescript-eslint/naming-convention - Latency: "190", - elapsed: "191", - success: "true", - // eslint-disable-next-line @typescript-eslint/naming-convention - Hostname: "localhost", - timeStamp: "1555399218911", - allThreads: "1", - grpThreads: "1", - threadName: "Thread 1-1", - responseCode: "200", - responseMessage: "", - URL: "file", - } - const result = transformDataForDb(inputData, "itemId", []) - expect(result.sutHostname).toBeUndefined() - }) - it("should process the data even when sentBytes not provided", () => { - const inputData = { - bytes: "792", - label: "endpoint3", - // eslint-disable-next-line @typescript-eslint/naming-convention - Connect: "155", - // eslint-disable-next-line @typescript-eslint/naming-convention - Latency: "190", - elapsed: "191", - success: "true", - // eslint-disable-next-line @typescript-eslint/naming-convention - Hostname: "localhost", - timeStamp: "1555399218911", - allThreads: "1", - grpThreads: "1", - threadName: "Thread 1-1", - responseCode: "200", - responseMessage: "", - URL: "file", - } - const result = transformDataForDb(inputData, "itemId", []) - expect(result.sentBytes).toEqual(0) - }) - }) - describe("stringToNumber", () => { - it("should convert string to number", () => { - const result = stringToNumber("1", 10) - expect(result).toBe(1) - }) - it("should throw an error when unable to convert ", () => { - expect(() => { - stringToNumber(undefined, 10) - }).toThrow() - }) - }) - describe("calculateDistributedThreads", () => { - it("should correctly calculate distributed threads", () => { - const inputData = [ - { - time: new Date(1555399218911), - hostname: "generator-1", - threads: 10, - }, - { - time: new Date(1555399218911), - hostname: "generator-2", - threads: 10, - }, - { - time: new Date(1555399218911), - hostname: "generator-3", - threads: 5, - }, - ] - const distributedThreads = calculateDistributedThreads(inputData) - expect(distributedThreads).toEqual([[1555399218911, 25]]) - }) - }) - describe("prepareDataForSavingToDb", () => { - it("should correctly parse data", () => { - const overviewData = { - _id: null, - start: new Date("2021-03-29T10:57:10.882Z"), - end: new Date("2021-03-29T11:27:10.171Z"), - avg_connect: 5.802922997682204, - avg_latency: 105.62091166623745, - avg_response: 105.72559876384238, - bytes_received_total: 123123, - bytes_sent_total: 69848465, - total: 46596, - n90: 271, - number_of_failed: 3, - } - const labelsData = [ - { - label: "label1", - min_response: 227, - max_response: 1325, - avg_response: 286.97317596566523, - bytes_received_total: 123, - bytes_sent_total: 1231, - total_samples: 932, - start: new Date("2021-03-29T10:59:01.561Z"), - end: new Date("2021-03-29T11:27:07.702Z"), - n90: 343, - n95: 367, - n99: 418, - latency: 1, - connect: 2, - number_of_failed: 0, - }, - { - label: "label2", - min_response: 35, - max_response: 118, - avg_response: 44.93503480278422, - total_samples: 431, - bytes_received_total: 123, - bytes_sent_total: 1231, - start: new Date("2021-03-29T11:00:53.221Z"), - end: new Date("2021-03-29T11:27:01.650Z"), - n90: 50, - n95: 56, - n99: 93, - latency: 3, - connect: 4, - number_of_failed: 0, - }, - ] - const statusCodes = [ - { label: "label2", status_code: "200", count: 433 }, - { label: "label1", status_code: "200", count: 932 }] - const responseFailures = [ - { label: "label1", response_message: "failure", count: 31, status_code: "100", failure_message: "failure" }, - { label: "label2", response_message: "failure2", count: 1, status_code: "101", failure_message: "failure1" }, - ] - const apdex = [ - { label: "label1", toleration: "40", satisfaction: "200" }, - { label: "label2", toleration: "30", satisfaction: "100" }] - const { overview, labelStats } = prepareDataForSavingToDb(overviewData, labelsData, [], - statusCodes, responseFailures, apdex) - expect(overview).toEqual({ - percentil: 271, - maxVu: undefined, - avgResponseTime: 106, - errorRate: 0.01, - errorCount: 3, - throughput: 25.9, - bytesPerSecond: 38820.04, - bytesSentPerSecond: 68.43, - avgLatency: 105.62, - avgConnect: 5.8, - startDate: new Date("2021-03-29T10:57:10.882Z"), - endDate: new Date("2021-03-29T11:27:10.171Z"), - duration: 29.99, - }) - expect(labelStats).toEqual([ - { - label: "label1", - samples: 932, - avgResponseTime: 287, - minResponseTime: 227, - maxResponseTime: 1325, - errorRate: 0, - bytesPerSecond: 0.07, - bytesSentPerSecond: 0.73, - throughput: 0.55, - n9: 418, - n5: 367, - n0: 343, - latency: 1, - connect: 2, - statusCodes: [{ count: 932, statusCode: "200" }], - responseMessageFailures: [{ - count: 31, responseMessage: "failure", statusCode: "100", failureMessage: "failure", - }], - apdex: { - satisfaction: 200, - toleration: 40, - }, - }, - { - label: "label2", - samples: 431, - avgResponseTime: 45, - minResponseTime: 35, - maxResponseTime: 118, - errorRate: 0, - bytesPerSecond: 0.08, - bytesSentPerSecond: 0.78, - throughput: 0.27, - n9: 93, - n5: 56, - n0: 50, - latency: 3, - connect: 4, - statusCodes: [{ count: 433, statusCode: "200" }], - responseMessageFailures: [{ - count: 1, responseMessage: "failure2", statusCode: "101", failureMessage: "failure1", - }], - apdex: { - toleration: 30, - satisfaction: 100, - }, - }]) - }) - }) - describe("transformMonitoringDataForDb", () => { - it("should parse input data correctly", () => { - const row = { - name: "test", - cpu: "10", - mem: "5", - ts: "1555399218", - } - const itemId = "myTestId" - const transformedMonitoringData = transformMonitoringDataForDb(row, itemId) - expect(transformedMonitoringData).toEqual({ - name: "test", - cpu: 10, - mem: 5, - itemId, - timestamp: new Date("2019-04-16T07:20:18.000Z"), - }) - }) - it("should return indefined when an error occurs during parsing", () => { - const itemId = "myTestId" - const row = {} - const transformedMonitoringData = transformMonitoringDataForDb(row, itemId) - expect(transformedMonitoringData).toBeUndefined() - }) - it("should return localhost if no name provided", () => { - const row = { - cpu: "10", - mem: "5", - ts: "1555399218", - } - const itemId = "myTestId" - const transformedMonitoringData = transformMonitoringDataForDb(row, itemId) - expect(transformedMonitoringData).toEqual({ - name: "localhost", - cpu: 10, - mem: 5, - itemId, - timestamp: new Date("2019-04-16T07:20:18.000Z"), - }) - }) - }) - describe("prepareChartDataForSaving", () => { - it("should correctly parse data", () => { + describe("prepareChartDataForSaving", () => { + it("should correctly parse data", () => { - const overviewData = [{ - min: new Date("2019-04-16T07:20:18.000Z"), - max: new Date("2019-04-16T07:20:18.000Z"), - total: 1000, - threads: 30, - avg_response: 30.2, - time: new Date("2019-04-16T07:20:18.000Z"), - number_of_failed: 4, - bytes_received_total: 32423412, - bytes_sent_total: 342342341, - error_rate: 1.2, - }] - const labelData = [{ - time: new Date("2019-04-16T07:20:18.000Z"), - label: "test-label", - min: new Date("2019-04-16T07:20:18.000Z"), - max: new Date("2019-04-16T07:20:18.000Z"), - total: 200, - threads: 30, - avg_response: 123.2, - min_response: 12.3, - max_response: 1233.1, - bytes_received_total: 32423123, - bytes_sent_total: 56456546546, - n90: 120.1, - n95: 251, - n99: 300.3, - error_rate: 0.02, - }] - const statusCodeData = [{ - time: new Date("2019-04-16T07:20:18.000Z").toString(), - statusCode: "200", - count: 10, - }] - const chartData = prepareChartDataForSaving({ overviewData, labelData, interval: 450, statusCodeData }) - expect(chartData).toEqual({ - maxResponseTime: [{ - data: [[1555399218000, 1233.1]], - name: "test-label", - }], - minResponseTime: [{ - data: [[1555399218000, 12.3]], - name: "test-label", - }], - networkDown: [{ - data: [[1555399218000, 72051384.44]], - name: "test-label", - }], - networkUp: [{ - data: [[1555399218000, 125458992324.44]], - name: "test-label", - }], - networkV2: [{ - data: [[1555399218000, 125531043708.89]], - name: "test-label", - }], - overAllFailRate: { - data: [[1555399218000, 120]], - name: "errors", - }, - overAllNetworkDown: { - data: [[1555399218000, 72052026.67]], - name: "network down", - }, - overAllNetworkUp: { - data: [[1555399218000, 760760757.78]], - name: "network up", - }, - overAllNetworkV2: { - data: [[1555399218000, 832812784.44]], - name: "network", - }, - overallThroughput: { - data: [[1555399218000, 2222.22]], - name: "throughput", - }, - overallTimeResponse: { - data: [[1555399218000, 30.2]], - name: "response time", - }, - percentile90: [{ - data: [[1555399218000, 120.1]], - name: "test-label", - }], - percentile95: [{ - data: [[1555399218000, 251]], - name: "test-label", - }], - percentile99: [{ - data: [[1555399218000, 300.3]], - name: "test-label", - }], - responseTime: [{ - data: [[1555399218000, 123.2]], - name: "test-label", - }], - threads: [[1555399218000, 30]], - throughput: [{ - data: [[1555399218000, 444.44]], - name: "test-label", - }], - statusCodes: [{ - data: [[1555399218000, 10]], - name: "200", - }], - errorRate: [{ - data: [[1555399218000, 2]], - name: "test-label", - }], - }) + const overviewData = [{ + min: new Date("2019-04-16T07:20:18.000Z"), + max: new Date("2019-04-16T07:20:18.000Z"), + total: 1000, + threads: 30, + avg_response: 30.2, + time: new Date("2019-04-16T07:20:18.000Z"), + number_of_failed: 4, + bytes_received_total: 32423412, + bytes_sent_total: 342342341, + error_rate: 1.2, + }] + const labelData = [{ + time: new Date("2019-04-16T07:20:18.000Z"), + label: "test-label", + min: new Date("2019-04-16T07:20:18.000Z"), + max: new Date("2019-04-16T07:20:18.000Z"), + total: 200, + threads: 30, + avg_response: 123.2, + min_response: 12.3, + max_response: 1233.1, + bytes_received_total: 32423123, + bytes_sent_total: 56456546546, + n90: 120.1, + n95: 251, + n99: 300.3, + error_rate: 0.02, + }] + const statusCodeData = [{ + time: new Date("2019-04-16T07:20:18.000Z").toString(), + statusCode: "200", + count: 10, + }] + const chartData = prepareChartDataForSaving({ overviewData, labelData, interval: 450, statusCodeData }) + expect(chartData).toEqual({ + maxResponseTime: [{ + data: [[1555399218000, 1233.1]], + name: "test-label", + }], + minResponseTime: [{ + data: [[1555399218000, 12.3]], + name: "test-label", + }], + networkDown: [{ + data: [[1555399218000, 72051384.44]], + name: "test-label", + }], + networkUp: [{ + data: [[1555399218000, 125458992324.44]], + name: "test-label", + }], + networkV2: [{ + data: [[1555399218000, 125531043708.89]], + name: "test-label", + }], + overAllFailRate: { + data: [[1555399218000, 120]], + name: "errors", + }, + overAllNetworkDown: { + data: [[1555399218000, 72052026.67]], + name: "network down", + }, + overAllNetworkUp: { + data: [[1555399218000, 760760757.78]], + name: "network up", + }, + overAllNetworkV2: { + data: [[1555399218000, 832812784.44]], + name: "network", + }, + overallThroughput: { + data: [[1555399218000, 2222.22]], + name: "throughput", + }, + overallTimeResponse: { + data: [[1555399218000, 30.2]], + name: "response time", + }, + percentile90: [{ + data: [[1555399218000, 120.1]], + name: "test-label", + }], + percentile95: [{ + data: [[1555399218000, 251]], + name: "test-label", + }], + percentile99: [{ + data: [[1555399218000, 300.3]], + name: "test-label", + }], + responseTime: [{ + data: [[1555399218000, 123.2]], + name: "test-label", + }], + threads: [[1555399218000, 30]], + throughput: [{ + data: [[1555399218000, 444.44]], + name: "test-label", + }], + statusCodes: [{ + data: [[1555399218000, 10]], + name: "200", + }], + errorRate: [{ + data: [[1555399218000, 2]], + name: "test-label", + }], + }) + }) + }) + describe("prepareHistogramDataForSaving", () => { + it("should should correctly parse data", () => { + const inputData = [{ label: "label1", histogram: [0, 1, 30, 3, 0] }, + { label: "label2", histogram: [0, 30, 0, 5, 10, 2, 0] }] + const parsedData = prepareHistogramDataForSaving(inputData) + expect(parsedData).toEqual({ + responseTimePerLabelDistribution: [ + { label: "label1", values: [1, 30, 3] }, + { label: "label2", values: [30, 0, 5, 10, 2] }, + ], + }) + }) }) - }) }) diff --git a/src/server/data-stats/prepare-data.ts b/src/server/data-stats/prepare-data.ts index 44fac5fd..030e19b1 100644 --- a/src/server/data-stats/prepare-data.ts +++ b/src/server/data-stats/prepare-data.ts @@ -81,6 +81,16 @@ export const prepareDataForSavingToDb = (overviewData, labelData, sutStats, stat } } +export const prepareHistogramDataForSaving = (responseTimePerLabelDistribution: ResponseTimeHistogram[]) => { + // removing first and last numbers, see https://docs.timescale.com/api/latest/hyperfunctions/histogram/ + return { + responseTimePerLabelDistribution: responseTimePerLabelDistribution.map(data => ({ + label: data.label, + values: data.histogram.slice(1, data.histogram.length - 1), + })), + } +} + export const prepareChartDataForSaving = ( { overviewData, @@ -409,3 +419,8 @@ interface PrepareChartsData { overviewData: ChartOverviewData[] statusCodeData: StatuCodesData[] } + +interface ResponseTimeHistogram { + histogram: number[] + label: string +} diff --git a/src/server/queries/items.ts b/src/server/queries/items.ts index 30f9a82b..1b56645e 100644 --- a/src/server/queries/items.ts +++ b/src/server/queries/items.ts @@ -12,17 +12,17 @@ export const createNewItem = (scenarioName, startTime, environment, note, status } } -export const savePlotData = (itemId, data, extraPlotData) => { +export const savePlotData = (itemId, data, extraPlotData, histogramData) => { return { - text: "INSERT INTO jtl.charts(item_id, plot_data, extra_plot_data) VALUES($1, $2, $3)", - values: [itemId, data, extraPlotData], + text: "INSERT INTO jtl.charts(item_id, plot_data, extra_plot_data, histogram_plot_data) VALUES($1, $2, $3, $4)", + values: [itemId, data, extraPlotData, histogramData], } } export const findItem = (itemId, projectName, scenarioName) => { return { // eslint-disable-next-line max-len - text: `SELECT charts.plot_data, charts.extra_plot_data, note, environment, status, hostname, s.analysis_enabled as "analysisEnabled", + text: `SELECT charts.plot_data, charts.extra_plot_data, charts.histogram_plot_data, note, environment, status, hostname, s.analysis_enabled as "analysisEnabled", s.zero_error_tolerance_enabled as "zeroErrorToleranceEnabled", threshold_result as "thresholds", report_status as "reportStatus", p.item_top_statistics_settings as "topMetricsSettings", items.name, items.apdex_settings as "apdexSettings", @@ -523,3 +523,20 @@ export const calculateApdexValues = (itemId, satisfyingThreshold, toleratingThre values: [itemId, satisfyingThreshold, toleratingThreshold], } } + +export const responseTimePerLabelHistogram = (itemId) => { + return { + text: ` + SELECT label, histogram(t_elapsed, 0, x.max + 1, x.buckets | 1) + FROM (SELECT label, + ceil(max(elapsed) / 100)::integer as buckets, + array_agg(elapsed) as elapsed, + max(elapsed) as max + FROM jtl.samples + WHERE item_id = $1 + GROUP BY label) x, + LATERAL unnest(x.elapsed) as t_elapsed + GROUP BY x.label;`, + values: [itemId], + } +}