diff --git a/.vscode/settings.json b/.vscode/settings.json index 8b2a4f5..93fa4ec 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "cSpell.words": [ + "analysed", "mstest", "nunit", "pactum", diff --git a/src/beats/beats.js b/src/beats/beats.js index 5ef86ee..1b97545 100644 --- a/src/beats/beats.js +++ b/src/beats/beats.js @@ -1,7 +1,7 @@ const { getCIInformation } = require('../helpers/ci'); const logger = require('../utils/logger'); const { BeatsApi } = require('./beats.api'); -const { HOOK } = require('../helpers/constants'); +const { HOOK, PROCESS_STATUS } = require('../helpers/constants'); const TestResult = require('test-results-parser/src/models/TestResult'); const { BeatsAttachments } = require('./beats.attachments'); @@ -31,9 +31,7 @@ class Beats { await this.#publishTestResults(); await this.#uploadAttachments(); this.#updateTitleLink(); - await this.#attachFailureSummary(); - await this.#attachSmartAnalysis(); - await this.#attachErrorClusters(); + await this.#attachExtensions(); } #setCIInfo() { @@ -104,13 +102,20 @@ class Beats { } } - async #attachFailureSummary() { + async #attachExtensions() { if (!this.test_run_id) { return; } if (!this.config.targets) { return; } + await this.#attachFailureSummary(); + await this.#attachFailureAnalysis(); + await this.#attachSmartAnalysis(); + await this.#attachErrorClusters(); + } + + async #attachFailureSummary() { if (this.result.status !== 'FAIL') { return; } @@ -132,13 +137,29 @@ class Beats { } } - async #attachSmartAnalysis() { - if (!this.test_run_id) { + async #attachFailureAnalysis() { + if (this.result.status !== 'FAIL') { return; } - if (!this.config.targets) { + if (this.config.show_failure_analysis === false) { return; } + try { + logger.info('šŸŖ„ Fetching Failure Analysis...'); + await this.#setTestRun('Failure Analysis Status', 'failure_analysis_status'); + this.config.extensions.push({ + name: 'failure-analysis', + hook: HOOK.AFTER_SUMMARY, + inputs: { + data: this.test_run + } + }); + } catch (error) { + logger.error(`āŒ Unable to attach failure analysis: ${error.message}`, error); + } + } + + async #attachSmartAnalysis() { if (this.config.show_smart_analysis === false) { return; } @@ -165,7 +186,7 @@ class Beats { } async #setTestRun(text, wait_for = 'smart_analysis_status') { - if (this.test_run && this.test_run[wait_for] === 'COMPLETED') { + if (this.test_run && this.test_run[wait_for] === PROCESS_STATUS.COMPLETED) { return; } let retry = 3; @@ -175,13 +196,13 @@ class Beats { this.test_run = await this.api.getTestRun(this.test_run_id); const status = this.test_run && this.test_run[wait_for]; switch (status) { - case 'COMPLETED': + case PROCESS_STATUS.COMPLETED: logger.debug(`ā˜‘ļø ${text} generated successfully`); return; - case 'FAILED': + case PROCESS_STATUS.FAILED: logger.error(`āŒ Failed to generate ${text}`); return; - case 'SKIPPED': + case PROCESS_STATUS.SKIPPED: logger.warn(`ā— Skipped generating ${text}`); return; } @@ -191,12 +212,6 @@ class Beats { } async #attachErrorClusters() { - if (!this.test_run_id) { - return; - } - if (!this.config.targets) { - return; - } if (this.result.status !== 'FAIL') { return; } diff --git a/src/beats/beats.types.d.ts b/src/beats/beats.types.d.ts index ef2a245..ea6ab37 100644 --- a/src/beats/beats.types.d.ts +++ b/src/beats/beats.types.d.ts @@ -9,6 +9,12 @@ export type IBeatExecutionMetric = { added: number removed: number flaky: number + product_bugs: number + environment_issues: number + automation_bugs: number + not_a_defects: number + to_investigate: number + auto_analysed: number failure_summary: any failure_summary_provider: any failure_summary_model: any diff --git a/src/extensions/failure-analysis.extension.js b/src/extensions/failure-analysis.extension.js new file mode 100644 index 0000000..0ae6c86 --- /dev/null +++ b/src/extensions/failure-analysis.extension.js @@ -0,0 +1,58 @@ +const { BaseExtension } = require('./base.extension'); +const { STATUS, HOOK } = require("../helpers/constants"); + +class FailureAnalysisExtension extends BaseExtension { + + constructor(target, extension, result, payload, root_payload) { + super(target, extension, result, payload, root_payload); + this.#setDefaultOptions(); + this.#setDefaultInputs(); + this.updateExtensionInputs(); + } + + #setDefaultOptions() { + this.default_options.hook = HOOK.AFTER_SUMMARY, + this.default_options.condition = STATUS.PASS_OR_FAIL; + } + + #setDefaultInputs() { + this.default_inputs.title = ''; + this.default_inputs.title_link = ''; + } + + run() { + this.#setText(); + this.attach(); + } + + #setText() { + const data = this.extension.inputs.data; + if (!data) { + return; + } + + /** + * @type {import('../beats/beats.types').IBeatExecutionMetric} + */ + const execution_metrics = data.execution_metrics[0]; + + if (!execution_metrics) { + logger.warn('āš ļø No execution metrics found. Skipping.'); + return; + } + + const failure_analysis = []; + + if (execution_metrics.to_investigate) { + failure_analysis.push(`šŸ”Ž To Investigate: ${execution_metrics.to_investigate}`); + } + if (execution_metrics.auto_analysed) { + failure_analysis.push(`šŸŖ„ Auto Analysed: ${execution_metrics.auto_analysed}`); + } + + this.text = failure_analysis.join(' ā€„ā€¢ā€„ '); + } + +} + +module.exports = { FailureAnalysisExtension }; \ No newline at end of file diff --git a/src/extensions/index.js b/src/extensions/index.js index d67e619..2e813d1 100644 --- a/src/extensions/index.js +++ b/src/extensions/index.js @@ -13,6 +13,7 @@ const { EXTENSION } = require('../helpers/constants'); const { checkCondition } = require('../helpers/helper'); const logger = require('../utils/logger'); const { ErrorClustersExtension } = require('./error-clusters.extension'); +const { FailureAnalysisExtension } = require('./failure-analysis.extension'); async function run(options) { const { target, result, hook } = options; @@ -59,6 +60,8 @@ function getExtensionRunner(extension, options) { return new CIInfoExtension(options.target, extension, options.result, options.payload, options.root_payload); case EXTENSION.AI_FAILURE_SUMMARY: return new AIFailureSummaryExtension(options.target, extension, options.result, options.payload, options.root_payload); + case EXTENSION.FAILURE_ANALYSIS: + return new FailureAnalysisExtension(options.target, extension, options.result, options.payload, options.root_payload); case EXTENSION.SMART_ANALYSIS: return new SmartAnalysisExtension(options.target, extension, options.result, options.payload, options.root_payload); case EXTENSION.ERROR_CLUSTERS: diff --git a/src/extensions/smart-analysis.extension.js b/src/extensions/smart-analysis.extension.js index f8f99e8..b355fab 100644 --- a/src/extensions/smart-analysis.extension.js +++ b/src/extensions/smart-analysis.extension.js @@ -65,13 +65,13 @@ class SmartAnalysisExtension extends BaseExtension { for (const item of smart_analysis) { rows.push(item); if (rows.length === 3) { - texts.push(rows.join(' ļ½œ ')); + texts.push(rows.join(' ā€„ā€¢ā€„ ')); rows.length = 0; } } if (rows.length > 0) { - texts.push(rows.join(' ļ½œ ')); + texts.push(rows.join(' ā€„ā€¢ā€„ ')); } this.text = this.mergeTexts(texts); diff --git a/src/helpers/constants.js b/src/helpers/constants.js index 11073ff..af060c9 100644 --- a/src/helpers/constants.js +++ b/src/helpers/constants.js @@ -21,6 +21,7 @@ const TARGET = Object.freeze({ const EXTENSION = Object.freeze({ AI_FAILURE_SUMMARY: 'ai-failure-summary', + FAILURE_ANALYSIS: 'failure-analysis', SMART_ANALYSIS: 'smart-analysis', ERROR_CLUSTERS: 'error-clusters', HYPERLINKS: 'hyperlinks', @@ -39,6 +40,13 @@ const URLS = Object.freeze({ QUICK_CHART: 'https://quickchart.io' }); +const PROCESS_STATUS = Object.freeze({ + RUNNING: 'RUNNING', + COMPLETED: 'COMPLETED', + FAILED: 'FAILED', + SKIPPED: 'SKIPPED', +}); + const MIN_NODE_VERSION = 14; module.exports = Object.freeze({ @@ -47,5 +55,6 @@ module.exports = Object.freeze({ TARGET, EXTENSION, URLS, + PROCESS_STATUS, MIN_NODE_VERSION }); \ No newline at end of file diff --git a/src/index.d.ts b/src/index.d.ts index 4df1c58..ab6be91 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -229,6 +229,7 @@ export interface PublishReport { project?: string; run?: string; show_failure_summary?: boolean; + show_failure_analysis?: boolean; show_smart_analysis?: boolean; show_error_clusters?: boolean; targets?: Target[]; diff --git a/test/beats.spec.js b/test/beats.spec.js index f283bb6..4341ff7 100644 --- a/test/beats.spec.js +++ b/test/beats.spec.js @@ -230,4 +230,38 @@ describe('TestBeats', () => { assert.equal(mock.getInteraction(id4).exercised, true); }); + it('should send results with failure analysis to beats', async () => { + const id1 = mock.addInteraction('post test results to beats'); + const id2 = mock.addInteraction('get test results with failure analysis from beats'); + const id3 = mock.addInteraction('get empty error clusters from beats'); + const id4 = mock.addInteraction('post test-summary with beats to teams with ai failure summary and smart analysis and failure analysis'); + await publish({ + config: { + api_key: 'api-key', + project: 'project-name', + run: 'build-name', + targets: [ + { + name: 'teams', + inputs: { + url: 'http://localhost:9393/message' + } + } + ], + results: [ + { + type: 'testng', + files: [ + 'test/data/testng/single-suite-failures.xml' + ] + } + ] + } + }); + assert.equal(mock.getInteraction(id1).exercised, true); + assert.equal(mock.getInteraction(id2).exercised, true); + assert.equal(mock.getInteraction(id3).exercised, true); + assert.equal(mock.getInteraction(id4).exercised, true); + }); + }); \ No newline at end of file diff --git a/test/mocks/beats.mock.js b/test/mocks/beats.mock.js index e26a294..64313c7 100644 --- a/test/mocks/beats.mock.js +++ b/test/mocks/beats.mock.js @@ -29,6 +29,7 @@ addInteractionHandler('get test results from beats', () => { body: { id: 'test-run-id', "failure_summary_status": "COMPLETED", + "failure_analysis_status": "COMPLETED", "smart_analysis_status": "COMPLETED", "execution_metrics": [ { @@ -69,6 +70,42 @@ addInteractionHandler('get test results with smart analysis from beats', () => { } }); +addInteractionHandler('get test results with failure analysis from beats', () => { + return { + strict: false, + request: { + method: 'GET', + path: '/api/core/v1/test-runs/test-run-id' + }, + response: { + status: 200, + body: { + id: 'test-run-id', + "failure_summary_status": "COMPLETED", + "failure_analysis_status": "COMPLETED", + "smart_analysis_status": "SKIPPED", + "execution_metrics": [ + { + "failure_summary": "", + "newly_failed": 1, + "always_failing": 1, + "recovered": 1, + "added": 0, + "removed": 0, + "flaky": 1, + "product_bugs": 1, + "environment_issues": 1, + "automation_bugs": 1, + "not_a_defects": 1, + "to_investigate": 1, + "auto_analysed": 1 + } + ] + } + } + } +}); + addInteractionHandler('get error clusters from beats', () => { return { strict: false, diff --git a/test/mocks/teams.mock.js b/test/mocks/teams.mock.js index 486772e..029eee4 100644 --- a/test/mocks/teams.mock.js +++ b/test/mocks/teams.mock.js @@ -1642,17 +1642,59 @@ addInteractionHandler('post test-summary with beats to teams with ai failure sum { "@DATA:TEMPLATE@": "TEAMS_ROOT_RESULTS_SINGLE_SUITE_FAILURES", }, - // { - // "type": "TextBlock", - // "text": "Smart Analysis", - // "isSubtle": true, - // "weight": "bolder", - // "separator": true, - // "wrap": true - // }, { "type": "TextBlock", - "text": "ā­• Newly Failed: 1 ļ½œ šŸ”“ Always Failing: 1 ļ½œ šŸŸ” Flaky: 1\n\nšŸŸ¢ Recovered: 1", + "text": "ā­• Newly Failed: 1 ā€„ā€¢ā€„ šŸ”“ Always Failing: 1 ā€„ā€¢ā€„ šŸŸ” Flaky: 1\n\nšŸŸ¢ Recovered: 1", + "wrap": true, + "separator": true, + } + ], + "actions": [] + } + } + ] + } + }, + response: { + status: 200 + } + } +}); + +addInteractionHandler('post test-summary with beats to teams with ai failure summary and smart analysis and failure analysis', () => { + return { + request: { + method: 'POST', + path: '/message', + body: { + "type": "message", + "attachments": [ + { + "contentType": "application/vnd.microsoft.card.adaptive", + "content": { + "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "type": "AdaptiveCard", + "version": "1.0", + "body": [ + { + "type": "TextBlock", + "text": "[āŒ build-name](http://localhost:9393/reports/test-run-id)", + "size": "medium", + "weight": "bolder", + "wrap": true + }, + { + "@DATA:TEMPLATE@": "TEAMS_ROOT_RESULTS_SINGLE_SUITE_FAILURES", + }, + { + "type": "TextBlock", + "text": "šŸ”Ž To Investigate: 1 ā€„ā€¢ā€„ šŸŖ„ Auto Analysed: 1", + "wrap": true, + "separator": true, + }, + { + "type": "TextBlock", + "text": "ā­• Newly Failed: 1 ā€„ā€¢ā€„ šŸ”“ Always Failing: 1 ā€„ā€¢ā€„ šŸŸ” Flaky: 1\n\nšŸŸ¢ Recovered: 1", "wrap": true, "separator": true, }