be able to run newman tests on prod

ReadMe.md

@@ -73,7 +73,7 @@ The following parameters can be set in config files or in env variables:
 - RESOURCE_ROLE_UPDATE_TOPIC: the resource role update topic, default value is 'challenge.action.resource.role.update'
 
 Configuration for testing is at `config/test.js`, only add such new configurations different from `config/default.js`
-- WAIT_TIME: wait time used in test, default is 1500 or 1.5 second
+- WAIT_TIME: wait time used in test, default is 6000 or 6 seconds
 - MOCK_CHALLENGE_API_PORT: the mock server port, default is 4000.
 - AUTH_V2_URL: The auth v2 url
 - AUTH_V2_CLIENT_ID: The auth v2 client id
@@ -84,6 +84,8 @@ Configuration for testing is at `config/test.js`, only add such new configuratio
 - COPILOT_CREDENTIALS_PASSWORD: The user's password with copilot role
 - USER_CREDENTIALS_USERNAME: The user's username with user role
 - USER_CREDENTIALS_PASSWORD: The user's password with user role
+- POSTMAN_ROLE_NAME_PREFIX: the role name prefix for every `ResourceRole` record
+- MOCK_BUS_API_BY_NOCK: indicates whether Nock is used for mocking Bus API.
 
 ## Available commands
 - Install dependencies `npm install`
@@ -99,6 +101,8 @@ Configuration for testing is at `config/test.js`, only add such new configuratio
 - App is running at `http://localhost:3000`
 - Start mock server `npm run mock-challenge-api`
 - The mock server is running at `http://localhost:4000`
+- Run the Postman tests `npm run test:newman`
+- Clear the testing data by Postman tests: `npm run test:newman:clear`
 
 ## Local Deployment
 ### Foreman Setup
@@ -214,6 +218,12 @@ To run postman e2e tests.
 npm run test:newman
 ```
 
+To clear the testing data from postman e2e tests.
+
+```bash
+npm run test:newman:clear
+```
+
 ## Running tests in CI
 - TBD
 

Verification.md

@@ -90,4 +90,8 @@ Iteration 4/4
 ├───────────────────────────────────────────────────────────────┤
 │ average response time: 18ms [min: 11ms, max: 24ms, s.d.: 4ms] │
 └───────────────────────────────────────────────────────────────┘
-```
+```
+
+Then you can run `npm run test:newman:clear` to delete all testing data by above postman tests.  
+If 'socket hang up' appears while running the `npm run test:newman`. You can increase the `WAIT_TIME` from the `default/test.js`.
+ Then run `npm run test:newman:clear` before calling `npm run test:newman` again.

config/test.js

@@ -6,7 +6,7 @@ module.exports = {
   TERMS_API_URL: 'http://localhost:4000/v5/terms',
   BUSAPI_URL: 'http://localhost:4000/v5',
   CHALLENGE_PHASES_API_URL: 'http://localhost:4000/v5/challenge-phases',
-  WAIT_TIME: 1500,
+  WAIT_TIME: 6000,
   MOCK_CHALLENGE_API_PORT: 4000,
   AUTH_V2_URL: process.env.AUTH_V2_URL || 'https://topcoder-dev.auth0.com/oauth/ro',
   AUTH_V2_CLIENT_ID: process.env.AUTH_V2_CLIENT_ID || '',
@@ -16,5 +16,7 @@ module.exports = {
   COPILOT_CREDENTIALS_USERNAME: process.env.COPILOT_CREDENTIALS_USERNAME || '',
   COPILOT_CREDENTIALS_PASSWORD: process.env.COPILOT_CREDENTIALS_PASSWORD || '',
   USER_CREDENTIALS_USERNAME: process.env.USER_CREDENTIALS_USERNAME || '',
-  USER_CREDENTIALS_PASSWORD: process.env.USER_CREDENTIALS_PASSWORD || ''
+  USER_CREDENTIALS_PASSWORD: process.env.USER_CREDENTIALS_PASSWORD || '',
+  POSTMAN_ROLE_NAME_PREFIX: process.env.POSTMAN_ROLE_NAME_PREFIX || 'POSTMANE2E-',
+  MOCK_BUS_API_BY_NOCK: process.env.MOCK_BUS_API_BY_NOCK ? process.env.MOCK_BUS_API_BY_NOCK === 'true' : true
 }

package.json

@@ -17,7 +17,8 @@
     "view-data": "node src/scripts/view-data.js",
     "mock-challenge-api": "NODE_ENV=test node mock/mock-challenge-api",
     "test": "nyc --reporter=html --reporter=text mocha test/unit/test.js --require test/common/prepare.js --timeout 60000 --exit",
-    "test:newman": "NODE_ENV=test node test/postman/newman.js"
+    "test:newman": "NODE_ENV=test node test/postman/newman.js",
+    "test:newman:clear": "NODE_ENV=test node test/postman/clearTestData.js"
   },
   "author": "TCSCODER",
   "license": "none",

test/postman/ClearPostmanData.md

@@ -0,0 +1,69 @@
+# Clear Testing data which are from Postman Tests
+
+## How to clear the Postman related testing data
+- To summarize, simply run below command after running the Postman tests.
+```
+  npm run test:newman:clear
+```
+- You should follow the ReadMe.md and Verification.md to run the tests. Then you will get output like below:
+```
+> NODE_ENV=test node test/postman/clearTestData.js
+
+info: Clear the Postman test data.
+clear the test data from postman test!
+ResourceRole to be deleted addd9ae8-9610-4c20-9849-95587fbfa318
+ResourceRolePhaseDependency to be deleted d775f701-e440-451d-b5cb-e675fd5db89e
+ResourceRolePhaseDependency to be deleted 4aac6a0b-5375-4cc5-8af0-c3feb64ac51e
+Resource to be deleted 82823bde-4acb-437a-8ab1-03aef7f30ea0
+Resource to be deleted c02514c9-93ef-4da9-8771-df0fbb931d86
+Resource to be deleted 262528be-94c3-4ae7-96be-9c643a54c457
+Resource to be deleted 03eac62d-93ed-4be1-a061-4d58595d0833
+Resource to be deleted e61a2997-f995-47ff-98ca-aeb42831aec1
+Resource to be deleted 2572f829-1076-470e-b142-59d07cc59a1f
+Resource to be deleted 2a9f228d-0981-48fd-80e7-988b23f1dc8c
+Resource to be deleted 15093b6b-13aa-4f7c-964a-6b5708e302e4
+Resource to be deleted eaf6b9c9-b0ca-4804-ba6f-654127d4feaf
+ResourceRole to be deleted 9a72f8cb-93b3-4f3e-b9ed-7bbe09255521
+ResourceRole to be deleted f722e872-897f-442f-980c-e92d00fe70fb
+ResourceRole to be deleted 1d28eb17-4085-4269-b5a2-c73cc28aa5b9
+ResourceRole to be deleted c4b53497-d6d2-43dc-b5bc-f09a681ae33a
+ResourceRole to be deleted 47f3191d-1596-4155-b8da-35dc9288d820
+ResourceRole to be deleted c5032dba-5da1-4846-a46c-845d74e880be
+ResourceRole to be deleted 1da92cb3-8658-46d7-b147-4c913634fac1
+clear the test data from postman test completed!
+info: Done!
+```
+## Strategy
+1. Setup the `POSTMAN_ROLE_NAME_PREFIX` from the test environment. This prefix should be a name that will never be used 
+set as part of the role name. e.g. 'POSTMANE2E-'. In this case, the created `ResourceRole` will have a name like 'POSTMANE2E-submitter'.
+
+2. Choose either one solution for mocking the Bus API. We can not ignore this, becuase in production environment, it is 
+not allowed to send the Kafka messge to the Bus API.
+a. Set `MOCK_BUS_API_BY_NOCK` to `true` from the test environment. In this way, Nock will return the response if any events
+    is posted to the Bus API.
+b. You can use use Postman's mock server. You can refer to https://drive.google.com/file/d/1GXMzyqpzwix-LDBwieiRFfpJlJxrTIgI/view?usp=sharing
+   for details. You need to update the environment variable `BUSAPI_URL` to your Postman mock server.
+
+3. Steps of clearing the test data from Postman tests.
+   * Find all `ResourceRole` record whose names are starting with `POSTMAN_ROLE_NAME_PREFIX`.
+   * For each `ResourceRole` record, find all `ResourceRolePhaseDependency` records whose `resourceRoleId` are the same 
+     as the `id` of `RecourceRole`. Delete those `ResourceRolePhaseDependency` records.
+   * For each `ResourceRole` record, find all `Resource` records whose `roleId` are the same
+     as the `id` of `RecourceRole`.
+     * Delete those `Resource` records.
+     * Delete the ES index by the resource id too. (Only **Resource** are indexed by ES.)
+   * Delete the `ResourceRole` record.
+
+4. Note, in production enviroment, there is no need to run `npm run init-es force` or `npm run init-db`.
+
+## Questions from the spec
+* The DB is getting filled up with dummy/test data. You need to suggest a way to delete/clean up the data created from executing the tests without affecting existing data.  
+  _Check above strategy section._
+* Lookup data may not be required to get created as it may already exist or in some cases must not get created as there will be conflicts with existing data. You need to suggest how to overcome this issue.  
+  _All data from the Postman tests can be easily deleted._
+* Existing lookup data should not be deleted. You need to suggest how to avoid accidentally deleting lookup data.  
+  _The Postman tests only use the testing data created by itself._
+* Existing production data should not be affected by the tests. You need to suggest how to avoid this possible issue.  
+  _Check the strategy section. All testing data are removed at last._
+* If possible, we should be able to differentiate the test data from the actual data so we can filter it out from the search results of the production API. Please suggest how to achieve this.  
+  _Check the strategy section. We can easy find all `ResourceRole` records with the given role name prefix. Then we can find all the related `ResourceRolePhaseDependency` and `Resource`._

test/postman/clearTestData.js

@@ -0,0 +1,109 @@
+/**
+ * Clear the postman test data. All data created by postman e2e tests will be cleared.
+ */
+
+const models = require('../../src/models')
+const logger = require('../../src/common/logger')
+const helper = require('../../src/common/helper')
+const config = require('config')
+const _ = require('lodash')
+
+logger.info('Clear the Postman test data.')
+
+/**
+ * Delete the Resource from the ES by the given id
+ * @param id the resource id
+ * @returns {Promise<void>}
+ */
+const deleteFromESById = async (id) => {
+  // delete from ES
+  const esClient = await helper.getESClient()
+  await esClient.delete({
+    index: config.ES.ES_INDEX,
+    type: config.ES.ES_TYPE,
+    id: id,
+    refresh: 'true' // refresh ES so that it is effective for read operations instantly
+  })
+}
+
+/**
+ * Get Data by model id.
+ * @param {Object} modelName The dynamoose model name
+ * @param {String} id The id value
+ * @returns {Promise<void>}
+ */
+const getById = async (modelName, id) => {
+  return new Promise((resolve, reject) => {
+    models[modelName].query('id').eq(id).exec((err, result) => {
+      if (err) {
+        console.log('ERROR')
+        return reject(err)
+      }
+      if (result.length > 0) {
+        return resolve(result[0])
+      } else {
+        return resolve(null)
+      }
+    })
+  })
+}
+
+/**
+ * Delete the record from database by the given id.
+ * @param modelName the model name
+ * @param id the id
+ * @returns {Promise<void>}
+ */
+const deleteFromDBById = async (modelName, id) => {
+  if (id && id.length > 0) {
+    try {
+      const entity = await getById(modelName, id)
+      if (entity) {
+        await entity.delete()
+      }
+    } catch (err) {
+      throw err
+    }
+  }
+}
+
+/**
+ * Clear the postman test data. The main function of this class.
+ * @returns {Promise<void>}
+ */
+const clearTestData = async () => {
+  console.log('clear the test data from postman test!')
+  let roles = await helper.scanAll('ResourceRole')
+  roles = _.filter(roles, r => (r.name.startsWith(config.POSTMAN_ROLE_NAME_PREFIX)))
+  for (const role of roles) {
+    let roleId = role.id
+    let rolePhaseDeps = await helper.scanAll('ResourceRolePhaseDependency')
+    rolePhaseDeps = _.filter(rolePhaseDeps, d => (d.resourceRoleId === roleId))
+    for (const dep of rolePhaseDeps) {
+      console.log('ResourceRolePhaseDependency to be deleted', dep.id)
+      await deleteFromDBById('ResourceRolePhaseDependency', dep.id)
+    }
+    let resources = await helper.scanAll('Resource')
+    resources = _.filter(resources, r => (r.roleId === roleId))
+    for (const res of resources) {
+      console.log('Resource to be deleted', res.id)
+      await deleteFromDBById('Resource', res.id)
+      await deleteFromESById(res.id)
+    }
+    console.log('ResourceRole to be deleted', roleId)
+    await deleteFromDBById('ResourceRole', roleId)
+  }
+  console.log('clear the test data from postman test completed!')
+}
+
+clearTestData().then(() => {
+  logger.info('Done!')
+  process.exit()
+}).catch((e) => {
+  logger.logFullError(e)
+  process.exit(1)
+})
+
+module.exports = {
+  clearTestData
+}

test/postman/newman.js

@@ -1,6 +1,8 @@
 const newman = require('newman')
 const _ = require('lodash')
 const envHelper = require('./envHelper')
+const nock = require('nock')
+const config = require('config')
 
 const requests = [
   {
@@ -184,17 +186,41 @@ const runner = (options) => new Promise((resolve, reject) => {
   })
 })
 
+/**
+ * Sleep for the given time
+ * @param ms the miliseconds
+ * @returns {Promise<unknown>}
+ */
+function sleep (ms) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+/**
+ * Clean the Nock.
+ */
+function cleanNock () {
+  if (config.MOCK_BUS_API_BY_NOCK) {
+    nock.cleanAll()
+  }
+}
+
 ;(async () => {
   const m2mToken = await envHelper.getM2MToken()
   const adminToken = await envHelper.getAdminToken()
   const copilotToken = await envHelper.getCopilotToken()
   const userToken = await envHelper.getUserToken()
   const originalEnvVars = [
-    { key: 'M2M_TOKEN', value: `Bearer ${m2mToken}` },
-    { key: 'admin_token', value: `Bearer ${adminToken}` },
-    { key: 'copilot_token', value: `Bearer ${copilotToken}` },
-    { key: 'user_token', value: `Bearer ${userToken}` }
+    { key: 'M2M_TOKEN', value: `${m2mToken}` },
+    { key: 'admin_token', value: `${adminToken}` },
+    { key: 'copilot_token', value: `${copilotToken}` },
+    { key: 'user_token', value: `${userToken}` }
   ]
+  if (config.MOCK_BUS_API_BY_NOCK) {
+    nock(config.BUSAPI_URL)
+      .persist()
+      .post('/bus/events')
+      .reply(204)
+  }
   for (const request of requests) {
     options.envVar = [
       ...originalEnvVars,
@@ -207,16 +233,20 @@ const runner = (options) => new Promise((resolve, reject) => {
     try {
       const results = await runner(options)
       if (_.get(results, 'run.failures.length', 0) > 0) {
+        cleanNock()
         process.exit(-1)
       }
     } catch (err) {
       console.log(err)
     }
+    await sleep(config.WAIT_TIME)
   }
-})().then(() => {
+})().then(async () => {
+  cleanNock()
   console.log('newman test completed!')
   process.exit(0)
-}).catch((err) => {
+}).catch(async (err) => {
+  cleanNock()
   console.log(err)
   process.exit(1)
 })

test/postman/resource-api.postman_collection.json

@@ -1,6 +1,6 @@
 {
 	"info": {
-		"_postman_id": "c24b7751-3c89-4248-9d7c-6d2499985af2",
+		"_postman_id": "751f9f67-b079-4916-be2e-58d19e423484",
 		"name": "resource-api",
 		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
 	},
@@ -821,6 +821,7 @@
 										"exec": [
 											"const iterationData = pm.iterationData",
 											"const httpCode = iterationData.get('httpCode')",
+											"const idLabel = iterationData.get('idLabel')",
 											"pm.test(`Status code is ${httpCode}`, function () {",
 											"    pm.response.to.have.status(httpCode);",
 											"    if(httpCode === 200){",
@@ -829,6 +830,7 @@
 											"        pm.expect(response.resourceRoleId).to.eq(pm.environment.get('COPILOT_RESOURCE_ROLE_ID'))",
 											"        pm.expect(response.phaseState).to.eq(iterationData.get('phaseState'))",
 											"        pm.environment.set(\"DEPENDENCY_ID_2\", pm.response.json().id);",
+											"        pm.environment.set(idLabel, pm.response.json().id);",
 											"    }",
 											"});"
 										],
@@ -914,7 +916,7 @@
 								],
 								"body": {
 									"mode": "raw",
-									"raw": "{\n\t\"phaseId\": \"{{PHASE_ID}}\",\n\t\"resourceRoleId\": \"{{SUBMITTER_RESOURCE_ROLE_ID}}\",\n\t\"phaseState\": false\n}"
+									"raw": "{\n\t\"phaseId\": \"{{PHASE_ID}}\",\n\t\"resourceRoleId\": \"{{COPILOT_RESOURCE_ROLE_ID}}\",\n\t\"phaseState\": false\n}"
 								},
 								"url": {
 									"raw": "{{URL}}/resource-roles/Phase-dependencies",
@@ -1707,14 +1709,14 @@
 									"raw": ""
 								},
 								"url": {
-									"raw": "{{URL}}/resource-roles/Phase-dependencies/{{DEPENDENCY_ID_2}}",
+									"raw": "{{URL}}/resource-roles/Phase-dependencies/{{DEPENDENCY_ID_3}}",
 									"host": [
 										"{{URL}}"
 									],
 									"path": [
 										"resource-roles",
 										"Phase-dependencies",
-										"{{DEPENDENCY_ID_2}}"
+										"{{DEPENDENCY_ID_3}}"
 									]
 								}
 							},