From 2124eb7bf7407cc4a72cb33c55298aee93586296 Mon Sep 17 00:00:00 2001 From: Henry Dobosn Date: Thu, 12 Mar 2020 16:12:13 +0000 Subject: [PATCH 1/4] feat(qa): cross browser tests' --- Jenkinsfile_nightly | 21 ++-- RunFunctionalTests.sh | 14 +++ build.gradle | 23 +++- functional-output/zapreports | 2 - package.json | 12 +- saucelabs.conf.js | 63 ++++++++++ src/test/js/config/runUITestsSauceLabs.sh | 30 +++++ src/test/js/config/supportedBrowsers.js | 32 +++++ src/test/js/config/test_data.js | 2 +- src/test/js/cross_browser_test.js | 119 ++++++++++++++++++ src/test/js/policy_check_functional_test.js | 2 - src/test/js/shared/random_data.js | 2 +- .../js/shared/sauceLabsReportingHelper.js | 30 +++++ 13 files changed, 334 insertions(+), 18 deletions(-) create mode 100755 RunFunctionalTests.sh delete mode 100644 functional-output/zapreports create mode 100644 saucelabs.conf.js create mode 100644 src/test/js/config/runUITestsSauceLabs.sh create mode 100644 src/test/js/config/supportedBrowsers.js create mode 100644 src/test/js/cross_browser_test.js create mode 100644 src/test/js/shared/sauceLabsReportingHelper.js diff --git a/Jenkinsfile_nightly b/Jenkinsfile_nightly index ff71a346d..f564d5305 100644 --- a/Jenkinsfile_nightly +++ b/Jenkinsfile_nightly @@ -10,7 +10,7 @@ properties([ ]) ]) -@Library("Infrastructure") +@Library("Infrastructure") _ def type = "java" @@ -36,18 +36,25 @@ static LinkedHashMap secret(String secretName, String envVar) { } withNightlyPipeline(type, product, component) { - env.TEST_URL = params.URL_TO_TEST - env.IDAMAPI = params.API_URL_TO_TEST - loadVaultSecrets(secrets) - enableSecurityScan() - enableMutationTest() - enableFullFunctionalTest(200) + enableCrossBrowserTest() + + after('crossBrowserTest') { + try { + withSauceConnect("reform_tunnel") { + sh "./gradlew functionalSauce" + steps.archiveArtifacts allowEmptyArchive: true, artifacts: 'functional-output/**/*' + } + } + finally { + steps.saucePublisher() + } + } after('fullFunctionalTest') { diff --git a/RunFunctionalTests.sh b/RunFunctionalTests.sh new file mode 100755 index 000000000..8ea91d9f6 --- /dev/null +++ b/RunFunctionalTests.sh @@ -0,0 +1,14 @@ +#!/bin/bash + + +export SMOKE_TEST_USER_USERNAME=idamOwner@HMCTS.NET +export SMOKE_TEST_USER_PASSWORD=Ref0rmIsFun +export PROXY_SERVER=http://proxyout.reform.hmcts.net:8080 +export NOTIFY_API_KEY=sidam_sandbox-b7ab8862-25b4-41c9-8311-cb78815f7d2d-1f3ed33e-7fb8-4c42-912f-a8300b78340f + +export IDAMAPI=http://idam-api-sbox.service.core-compute-sandbox.internal +export TEST_URL=https://idam-web-public-aks.sandbox.platform.hmcts.net + +node_modules/codeceptjs/bin/codecept.js run --grep @functional + +# ./gradlew --no-daemon --info --rerun-tasks functional diff --git a/build.gradle b/build.gradle index de4e58fe3..ab8df07ae 100644 --- a/build.gradle +++ b/build.gradle @@ -150,7 +150,27 @@ allprojects { task codeceptFunctional(type: Exec, dependsOn: [':yarnInstall', ':notifyClientInstall']) { workingDir '.' - commandLine 'node_modules/codeceptjs/bin/codecept.js', 'run', '--grep', '@functional', '--verbose', '--reporter', 'mocha-multi' + commandLine 'node_modules/codeceptjs/bin/codecept.js', 'run', '--grep', '@functional', '--scan', '--verbose', '--reporter', 'mocha-multi' + } + + task smokeSauce(dependsOn: ':codeceptSmokeSauce') { + group = 'Delivery pipeline' + description = 'Executes non-destructive smoke tests against a running instance' + } + + task codeceptSmokeSauce(type: Exec, dependsOn: ':yarnInstall') { + workingDir '.' + commandLine 'node_modules/codeceptjs/bin/codecept.js', 'run', '--config', 'saucelabs.conf.js','--steps', '--grep', '@smoke', '--verbose', '--debug', '--reporter', 'mochawesome' + } + + task functionalSauce(dependsOn: ':codeceptFunctionalSauce') { + group = 'Delivery pipeline' + description = 'Executes non-destructive smoke tests against a running instance' + } + + task codeceptFunctionalSauce(type: Exec, dependsOn: [':yarnInstall', ':notifyClientInstall']) { + workingDir '.' + commandLine 'node_modules/codeceptjs/bin/codecept.js', 'run-multiple', '--all', '--config', 'saucelabs.conf.js', '--grep', '@crossbrowser', '--verbose', '--debug', '--reporter', 'mochawesome' } task pa11y(type: Exec, dependsOn: 'pa11yInstall') { @@ -215,4 +235,3 @@ test.finalizedBy jacocoTestReport bootRun { systemProperties = System.properties } - diff --git a/functional-output/zapreports b/functional-output/zapreports deleted file mode 100644 index e71affc08..000000000 --- a/functional-output/zapreports +++ /dev/null @@ -1,2 +0,0 @@ -zap reports - diff --git a/package.json b/package.json index 64190545f..131bb2012 100644 --- a/package.json +++ b/package.json @@ -8,10 +8,10 @@ }, "devDependencies": { "chai": "^4.2.0", - "codeceptjs": "^1.2.0", + "codeceptjs": "^2.4.3", "codeceptjs-resemblehelper": "^1.9.0", "deep-equal-in-any-order": "^1.0.13", - "electron": "^2.0.8", + "mocha": "^5.2.0", "mocha-junit-reporter": "^1.17.0", "mocha-multi": "^1.0.1", "mochawesome": "^3.0.2", @@ -20,6 +20,12 @@ "pa11y": "^4.13.2", "proxy-agent": "^3.0.1", "puppeteer": "^2.0.0", - "node-fetch": "^2.6.0" + "node-fetch": "^2.6.0", + "wdio-sauce-service": "^0.4.4", + "webdriverio": "^4.14.2" + }, + "scripts": { + "test-crossbrowser-e2e": "NODE_PATH=. codeceptjs run-multiple --all -c saucelabs.conf.js --steps --grep '@crossbrowser' --reporter mochawesome", + "test:crossbrowser": "runSauceLabsTests.sh" } } diff --git a/saucelabs.conf.js b/saucelabs.conf.js new file mode 100644 index 000000000..ec86276dc --- /dev/null +++ b/saucelabs.conf.js @@ -0,0 +1,63 @@ +const supportedBrowsers = require('./src/test/js/config/supportedBrowsers.js'); +const browser = process.env.SAUCE_BROWSER || 'chrome'; +const tunnelName = process.env.TUNNEL_IDENTIFIER || 'reformtunnel'; + +const waitForTimeout = 60000; +const smartWait = 5000; + +const getBrowserConfig = browserGroup => { + const browserConfig = []; + for (const candidateBrowser in supportedBrowsers[browserGroup]) { + if (candidateBrowser) { + const desiredCapability = supportedBrowsers[browserGroup][candidateBrowser]; + desiredCapability.acceptSslCerts = true; + desiredCapability.tunnelIdentifier = tunnelName; + desiredCapability.tags = ['idam-web-public']; + browserConfig.push({ + browser: desiredCapability.browserName, + desiredCapabilities: desiredCapability + }); + } else { + console.error('ERROR: supportedBrowsers.js is empty or incorrectly defined'); + } + } + return browserConfig; +}; + +const setupConfig = { + tests: './src/test/js/cross_browser_test.js', + output: `${process.cwd()}/functional-output`, + helpers: { + WebDriverIO: { + url: process.env.TEST_URL, + browser : 'chrome', + waitForTimeout, + smartWait, + cssSelectorsEnabled: 'true', + host: 'ondemand.eu-central-1.saucelabs.com', + port: 80, + region: 'eu', + sauceConnect: true, + services: ['sauce'], + acceptSslCerts : true, + user: process.env.SAUCE_USERNAME, + key: process.env.SAUCE_ACCESS_KEY, + desiredCapabilities: { } + }, + SauceLabsReportingHelper: { require: './src/test/js/shared/sauceLabsReportingHelper.js' }, + idam_helper: { require: './src/test/js/shared/idam_helper.js' } + }, + include: { I: './src/test/js/shared/custom_steps.js' }, + + multiple: { + chrome: { + browsers: getBrowserConfig('chrome') + }, + firefox: { + browsers: getBrowserConfig('firefox') + } + }, + name: 'Idam web public' +}; + +exports.config = setupConfig; \ No newline at end of file diff --git a/src/test/js/config/runUITestsSauceLabs.sh b/src/test/js/config/runUITestsSauceLabs.sh new file mode 100644 index 000000000..65e9e0c32 --- /dev/null +++ b/src/test/js/config/runUITestsSauceLabs.sh @@ -0,0 +1,30 @@ +#!/bin/bash +supportedBrowsers=`sed '/\/\//d' ./supportedBrowsers.js | sed '/\/\//d' | sed "s/[\'\:\{ ]//g"` +browsersArray=(${supportedBrowsers//$'\n'/ }) + +outputDirectory="${E2E_CROSSBROWSER_OUTPUT_DIR:-functional-output/crossbrowser/reports}" + +echo +echo "*****************************************" +echo "* The following browsers will be tested *" +echo "*****************************************" +echo "$supportedBrowsers" +echo "****************************************" +echo +echo + +for i in "${browsersArray[@]}" +do + echo "*** Testing $i ***" + + FOLDERNAME="$i-$(date +%s)" + + mkdir ../../../../output/$FOLDERNAME + + SAUCELABS_BROWSER=$i TUNNEL_IDENTIFIER=reformtunnel npm run test-crossbrowser-e2e + + for f in ../../../../output/*.*; do + echo $f + mv $f ../../../../output/$FOLDERNAME + done +done \ No newline at end of file diff --git a/src/test/js/config/supportedBrowsers.js b/src/test/js/config/supportedBrowsers.js new file mode 100644 index 000000000..7fac711a4 --- /dev/null +++ b/src/test/js/config/supportedBrowsers.js @@ -0,0 +1,32 @@ +const supportedBrowsers = { + chrome: { + chrome_win_latest: { + browserName: 'chrome', + name: 'DIV_WIN_CHROME_LATEST', + platform: 'Windows 10', + version: 'latest' + }, + chrome_mac_latest: { + browserName: 'chrome', + name: 'MAC_CHROME_LATEST', + platform: 'macOS 10.13', + version: 'latest' + } + }, + firefox: { + firefox_win_latest: { + browserName: 'firefox', + name: 'WIN_FIREFOX_LATEST', + platform: 'Windows 10', + version: 'latest' + }, + firefox_mac_latest: { + browserName: 'firefox', + name: 'MAC_FIREFOX_LATEST', + platform: 'macOS 10.13', + version: 'latest' + } + } +}; + +module.exports = supportedBrowsers; \ No newline at end of file diff --git a/src/test/js/config/test_data.js b/src/test/js/config/test_data.js index e0f28baaa..d37eab69b 100644 --- a/src/test/js/config/test_data.js +++ b/src/test/js/config/test_data.js @@ -6,6 +6,6 @@ module.exports = { NOTIFY_API_KEY: process.env.NOTIFY_API_KEY, SCENARIO_RETRY_LIMIT: 3, PASSWORD: "Passw0rdIDAM", - SERVICE_REDIRECT_URI: 'https://idam.testservice.gov.uk', + SERVICE_REDIRECT_URI: 'http://idam.testservice.gov.uk', SERVICE_CLIENT_SECRET: 'autotestingservice', }; \ No newline at end of file diff --git a/src/test/js/cross_browser_test.js b/src/test/js/cross_browser_test.js new file mode 100644 index 000000000..9268c3b91 --- /dev/null +++ b/src/test/js/cross_browser_test.js @@ -0,0 +1,119 @@ +const chai = require('chai'); +const {expect} = chai; +const TestData = require('./config/test_data'); +const randomData = require('./shared/random_data'); + +Feature('Users can sign in'); + +let randomUserFirstName; +let citizenEmail; +let userFirstNames = []; +let roleNames = []; +let serviceNames = []; +let randomUserLastName; +let existingCitizenEmail; +let pinUserFirstName; +let pinUserLastName; +let pinaccessToken; + +const serviceName = randomData.getRandomServiceName(); + +const selfRegUrl = `${TestData.WEB_PUBLIC_URL}/users/selfRegister?redirect_uri=${TestData.SERVICE_REDIRECT_URI}&client_id=${serviceName}`; + +BeforeSuite(async (I) => { + randomUserFirstName = randomData.getRandomUserName(); + const adminEmail = 'admin.' + randomData.getRandomEmailAddress(); + citizenEmail = 'citizen.' + randomData.getRandomEmailAddress(); + existingCitizenEmail = 'existingcitizen.' + randomData.getRandomEmailAddress(); + pinUserLastName = randomData.getRandomUserName() + 'pinępinç'; + pinUserFirstName = randomData.getRandomUserName() + 'ępinçłpin'; + + let token = await I.getAuthToken(); + let response; + response = await I.createRole(randomData.getRandomRoleName() + "_beta", 'beta description', '', token); + const serviceBetaRole = response.name; + response = await I.createRole(randomData.getRandomRoleName() + "_admin", 'admin description', serviceBetaRole, token); + const serviceAdminRole = response.name; + response = await I.createRole(randomData.getRandomRoleName() + "_super", 'super description', serviceAdminRole, token); + const serviceSuperRole = response.name; + const serviceRoles = [serviceBetaRole, serviceAdminRole, serviceSuperRole]; + roleNames.push(serviceRoles); + await I.createServiceWithRoles(serviceName, serviceRoles, serviceBetaRole, token); + serviceNames.push(serviceName); + await I.createUserWithRoles(adminEmail, randomUserFirstName + 'Admin', [serviceAdminRole, "IDAM_ADMIN_USER"]); + userFirstNames.push(randomUserFirstName + 'Admin'); + await I.createUserWithRoles(citizenEmail, randomUserFirstName + 'Citizen', ["citizen"]); + userFirstNames.push(randomUserFirstName + 'Citizen'); + randomUserLastName = randomData.getRandomUserName(); + await I.createUserWithRoles(citizenEmail, randomUserFirstName, ["citizen"]); + userFirstNames.push(randomUserFirstName); + await I.createUserWithRoles(existingCitizenEmail, randomUserFirstName + 'Citizen', ["citizen"]); + userFirstNames.push(randomUserFirstName + 'Citizen'); + + const pinUser = await I.getPinUser(pinUserFirstName, pinUserLastName); + const code = await I.loginAsPin(pinUser.pin, serviceName, TestData.SERVICE_REDIRECT_URI); + pinaccessToken = await I.getAccessToken(code, serviceName, TestData.SERVICE_REDIRECT_URI, TestData.SERVICE_CLIENT_SECRET); +}); + +AfterSuite(async (I) => { + return I.deleteAllTestData(randomData.TEST_BASE_PREFIX) +}); + +Scenario('@functional @crossbrowser Idam Web public cross browser tests', async (I) => { + + const email = 'test_citizen.' + randomData.getRandomEmailAddress(); + citizenEmail = 'citizen.' + randomData.getRandomEmailAddress(); + + const loginPage = `${TestData.WEB_PUBLIC_URL}/login?redirect_uri=${TestData.SERVICE_REDIRECT_URI}&client_id=${serviceName}&state=selfreg`; + I.amOnPage(selfRegUrl); + I.waitInUrl('users/selfRegister', 180); + I.waitForText('Create an account or sign in', 20, 'h1'); + I.see('Create an account'); + I.fillField('firstName', randomUserFirstName); + I.fillField('lastName', randomUserLastName); + I.fillField('email', citizenEmail); + I.click("Continue"); + I.waitForText('Check your email', 20, 'h1'); + I.wait(5); + const userActivationUrl = await I.extractUrl(citizenEmail); + I.amOnPage(userActivationUrl); + I.waitForText('Create a password', 20, 'h1'); + I.seeTitleEquals('User Activation - HMCTS Access'); + I.fillField('#password1', TestData.PASSWORD); + I.fillField('#password2', TestData.PASSWORD); + I.click('Continue'); + I.waitForText('Account created', 20, 'h1'); + I.see('You can now sign in to your account.'); + I.wait(5); + I.lockAccount(citizenEmail, serviceName); + I.click('reset your password'); + I.waitForText('Reset your password', 20, 'h1'); + I.fillField('#email', citizenEmail); + I.click('Submit'); + I.waitForText('Check your email', 20, 'h1'); + I.wait(5); + const resetPasswordUrl = await I.extractUrl(citizenEmail); + I.amOnPage(resetPasswordUrl); + I.waitForText('Create a new password', 20, 'h1'); + I.seeTitleEquals('Reset Password - HMCTS Access'); + I.fillField('#password1', 'Passw0rd1234'); + I.fillField('#password2', 'Passw0rd1234'); + I.click('Continue'); + I.waitForText('Your password has been changed', 20, 'h1'); + I.see('You can now sign in with your new password.'); + + const userInfo = await I.retry({retries: 3, minTimeout: 10000}).getUserByEmail(citizenEmail); + expect(userInfo.active).to.equal(true); + expect(userInfo.email).to.equal(citizenEmail); + expect(userInfo.forename).to.equal(randomUserFirstName); + expect(userInfo.id).to.not.equal(null); + expect(userInfo.roles).to.eql(['citizen']); + + I.amOnPage(`${TestData.WEB_PUBLIC_URL}/login/uplift?client_id=${serviceName}&redirect_uri=${TestData.SERVICE_REDIRECT_URI}&jwt=${pinaccessToken}`); + I.waitForText('Sign in to your account.', 30); + I.click('Sign in to your account.'); + I.fillField('#username', existingCitizenEmail); + I.wait(2) + I.fillField('#password', TestData.PASSWORD); + I.click('Sign in'); +}); \ No newline at end of file diff --git a/src/test/js/policy_check_functional_test.js b/src/test/js/policy_check_functional_test.js index 96ceca804..14179155a 100644 --- a/src/test/js/policy_check_functional_test.js +++ b/src/test/js/policy_check_functional_test.js @@ -36,8 +36,6 @@ BeforeSuite(async (I) => { roleNames.push(serviceRoles); await I.createServiceWithRoles(serviceName, serviceRoles, serviceBetaRole, token); serviceNames.push(serviceName); - await I.createUserWithRoles(adminEmail, randomUserFirstName + 'Admin', [serviceAdminRole, "IDAM_ADMIN_USER"]); - userFirstNames.push(randomUserFirstName + 'Admin'); await I.createUserWithRoles(citizenEmail, randomUserFirstName + 'Citizen', ["citizen"]); userFirstNames.push(randomUserFirstName + 'Citizen'); diff --git a/src/test/js/shared/random_data.js b/src/test/js/shared/random_data.js index ca4b77918..2a0694d74 100644 --- a/src/test/js/shared/random_data.js +++ b/src/test/js/shared/random_data.js @@ -12,7 +12,7 @@ function randomAlphabeticString(length = 10) { return randomString } -const testBasePrefix = "SIDMTESTWP" + randomAlphabeticString(); +const testBasePrefix = "SIDMTESTWP_" + randomAlphabeticString(); const testUserPrefix = testBasePrefix + "USER"; const testRolePrefix = testBasePrefix + "ROLE_"; const testServicePrefix = testBasePrefix + "SERVICE_"; diff --git a/src/test/js/shared/sauceLabsReportingHelper.js b/src/test/js/shared/sauceLabsReportingHelper.js new file mode 100644 index 000000000..1e7477ae9 --- /dev/null +++ b/src/test/js/shared/sauceLabsReportingHelper.js @@ -0,0 +1,30 @@ +/* eslint-disable no-process-env, no-console, prefer-template */ + +const event = require('codeceptjs').event; +const container = require('codeceptjs').container; +const exec = require('child_process').exec; + +const sauceUsername = process.env.SAUCE_USERNAME; +const sauceKey = process.env.SAUCE_ACCESS_KEY; + + +function updateSauceLabsResult(result, sessionId) { + console.log('SauceOnDemandSessionID=' + sessionId + ' job-name=idam-web-public-Uitests'); + // eslint-disable-next-line max-len + return 'curl -X PUT -s -d \'{"passed": ' + result + '}\' -u ' + sauceUsername + ':' + sauceKey + ' https://eu-central-1.saucelabs.com/rest/v1/' + sauceUsername + '/jobs/' + sessionId; +} + +// eslint-disable-next-line +module.exports = function() { + // Setting test success on SauceLabs + event.dispatcher.on(event.test.passed, () => { + const sessionId = container.helpers('WebDriverIO').browser.requestHandler.sessionID; + exec(updateSauceLabsResult('true', sessionId)); + }); + + // Setting test failure on SauceLabs + event.dispatcher.on(event.test.failed, () => { + const sessionId = container.helpers('WebDriverIO').browser.requestHandler.sessionID; + exec(updateSauceLabsResult('false', sessionId)); + }); +}; \ No newline at end of file From ccb120163f827bc243ffa75b0dfde074a160e4f9 Mon Sep 17 00:00:00 2001 From: Henry Dobosn Date: Thu, 12 Mar 2020 16:17:18 +0000 Subject: [PATCH 2/4] revert(delete): remove file --- RunFunctionalTests.sh | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100755 RunFunctionalTests.sh diff --git a/RunFunctionalTests.sh b/RunFunctionalTests.sh deleted file mode 100755 index 8ea91d9f6..000000000 --- a/RunFunctionalTests.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - - -export SMOKE_TEST_USER_USERNAME=idamOwner@HMCTS.NET -export SMOKE_TEST_USER_PASSWORD=Ref0rmIsFun -export PROXY_SERVER=http://proxyout.reform.hmcts.net:8080 -export NOTIFY_API_KEY=sidam_sandbox-b7ab8862-25b4-41c9-8311-cb78815f7d2d-1f3ed33e-7fb8-4c42-912f-a8300b78340f - -export IDAMAPI=http://idam-api-sbox.service.core-compute-sandbox.internal -export TEST_URL=https://idam-web-public-aks.sandbox.platform.hmcts.net - -node_modules/codeceptjs/bin/codecept.js run --grep @functional - -# ./gradlew --no-daemon --info --rerun-tasks functional From 6487e64fa30f1cf61f1c2f53a40653dfea2fd65c Mon Sep 17 00:00:00 2001 From: Shravan Mechineni Date: Thu, 12 Mar 2020 16:37:29 +0000 Subject: [PATCH 3/4] fix dependencies --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index 131bb2012..e132dfc4b 100644 --- a/package.json +++ b/package.json @@ -8,10 +8,9 @@ }, "devDependencies": { "chai": "^4.2.0", - "codeceptjs": "^2.4.3", + "codeceptjs": "^1.2.0", "codeceptjs-resemblehelper": "^1.9.0", "deep-equal-in-any-order": "^1.0.13", - "mocha": "^5.2.0", "mocha-junit-reporter": "^1.17.0", "mocha-multi": "^1.0.1", "mochawesome": "^3.0.2", From a8c69c4c7ac2de828e6150fc4dc2022c0d1ea719 Mon Sep 17 00:00:00 2001 From: Shravan Mechineni Date: Thu, 12 Mar 2020 17:13:06 +0000 Subject: [PATCH 4/4] fix codecept unknown option scan error --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ab8df07ae..e611cc5e7 100644 --- a/build.gradle +++ b/build.gradle @@ -150,7 +150,7 @@ allprojects { task codeceptFunctional(type: Exec, dependsOn: [':yarnInstall', ':notifyClientInstall']) { workingDir '.' - commandLine 'node_modules/codeceptjs/bin/codecept.js', 'run', '--grep', '@functional', '--scan', '--verbose', '--reporter', 'mocha-multi' + commandLine 'node_modules/codeceptjs/bin/codecept.js', 'run', '--grep', '@functional', '--verbose', '--reporter', 'mocha-multi' } task smokeSauce(dependsOn: ':codeceptSmokeSauce') {