New Relic deployment records & Production TTA Plan import

.circleci/config.yml

@@ -31,6 +31,33 @@ commands:
       - run:
           name: Combine package-lock.json files to single file
           command: cat yarn.lock frontend/yarn.lock > << parameters.filename >>
+  notify_new_relic:
+    description: "Notify new relic of a deploy"
+    parameters:
+      env_name:
+        description: "Name of the environment. Ex. sandbox, dev, staging, prod"
+        type: string
+      new_relic_app_id:
+        description: "App ID used in New Relic"
+        type: string
+      new_relic_api_key:
+        description: "API key from New Relic"
+        type: string
+    steps:
+      - run:
+          name: Notify New Relic
+          command: |
+            curl -X POST "https://api.newrelic.com/v2/applications/<< parameters.new_relic_app_id >>/deployments.json" \
+            -H "X-Api-Key: << parameters.new_relic_api_key >>" -i \
+            -H "Content-Type: application/json" \
+            -d \
+            "{
+              \"deployment\": {
+                \"revision\": \"<< pipeline.git.revision >>\",
+                \"description\": \"<< parameters.env_name >> Successfully Deployed\"
+              }
+            }"
+
   cf_deploy:
     description: "Login to cloud foundry space with service account credentials
       and push application using deployment configuration file."
@@ -135,6 +162,18 @@ parameters:
   sandbox_git_branch:  # change to feature branch to test deployment
     default: "cm-363-record-deployments"
     type: string
+  prod_new_relic_app_id:
+    default: "877570491"
+    type: string
+  staging_new_relic_app_id:
+    default: "868729138"
+    type: string
+  dev_new_relic_app_id:
+    default: "867221900"
+    type: string
+  sandbox_new_relic_app_id:
+    default: "867346799"
+    type: string
 jobs:
   build_and_lint:
     executor: docker-executor
@@ -319,6 +358,10 @@ jobs:
                   cf run-task tta-smarthub-sandbox \
                     --command "yarn db:migrate:prod" \
                     --name "Reset DB"
+            - notify_new_relic:
+                env_name: sandbox
+                new_relic_app_id: << pipeline.parameters.sandbox_new_relic_app_id >>
+                new_relic_api_key: $NEW_RELIC_REST_API_KEY
       - when:  # dev
           condition:
             and:
@@ -344,6 +387,10 @@ jobs:
                   cf run-task tta-smarthub-dev \
                     --command "yarn db:migrate:prod" \
                     --name "Reset DB"
+            - notify_new_relic:
+                env_name: dev
+                new_relic_app_id: << pipeline.parameters.dev_new_relic_app_id >>
+                new_relic_api_key: $NEW_RELIC_REST_API_KEY
       - when:  # staging
           condition:
             and:
@@ -367,6 +414,10 @@ jobs:
                 name: Run database migrations
                 command: |
                   cf run-task tta-smarthub-staging --command "yarn db:migrate:prod" --name migrate
+            - notify_new_relic:
+                env_name: staging
+                new_relic_app_id: << pipeline.parameters.staging_new_relic_app_id >>
+                new_relic_api_key: $NEW_RELIC_REST_API_KEY
       - when:  # prod
           condition:
             and:
@@ -390,6 +441,10 @@ jobs:
                 name: Run database migrations
                 command: |
                   cf run-task tta-smarthub-prod --command "yarn db:migrate:prod" --name migrate
+            - notify_new_relic:
+                env_name: prod
+                new_relic_app_id: << pipeline.parameters.prod_new_relic_app_id >>
+                new_relic_api_key: $NEW_RELIC_REST_API_KEY
 workflows:
   build_test_deploy:
     jobs:

package.json

@@ -20,7 +20,7 @@
     "server:debug": "nodemon src/index.js --exec babel-node --inspect",
     "client": "yarn --cwd frontend start",
     "test": "jest src --runInBand",
-    "test:ci": "cross-env JEST_JUNIT_OUTPUT_DIR=reports JEST_JUNIT_OUTPUT_NAME=unit.xml POSTGRES_USERNAME=postgres POSTGRES_DB=ttasmarthub CURRENT_USER_ID=5 CI=true jest src --coverage --reporters=default --reporters=jest-junit --forceExit",
+    "test:ci": "cross-env JEST_JUNIT_OUTPUT_DIR=reports JEST_JUNIT_OUTPUT_NAME=unit.xml POSTGRES_USERNAME=postgres POSTGRES_DB=ttasmarthub CURRENT_USER_ID=5 CI=true jest src --coverage --reporters=default --reporters=jest-junit",
     "test:all": "yarn test:ci && yarn --cwd frontend test:ci",
     "lint": "eslint src",
     "lint:ci": "eslint -f eslint-formatter-multiple src",

src/tools/importPlanGoals.js

@@ -1,7 +1,7 @@
 /* eslint-disable no-restricted-syntax */
 /* eslint-disable no-loop-func */
-import { readFileSync } from 'fs';
 import parse from 'csv-parse/lib/sync';
+import { downloadFile } from '../lib/s3';
 import {
   Role, Topic, RoleTopic, Goal, TopicGoal, Grant, GrantGoal,
 } from '../models';
@@ -25,9 +25,9 @@ const hubRoles = [
   { name: 'SS', fullName: 'System Specialist' },
 ];
 
-function parseCsv(file) {
+async function parseCsv(fileKey) {
   let grantees = {};
-  const csv = readFileSync(file);
+  const { Body: csv } = await downloadFile(fileKey);
   [...grantees] = parse(csv, {
     skipEmptyLines: true,
     columns: true,
@@ -59,9 +59,9 @@ async function prePopulateRoles() {
  * ...
  * (Total 5 goals. Some of them could be empty)
  */
-export default async function importGoals(file, region) {
-  const grantees = parseCsv(file);
-  const regionId = region || 14; // default to region 14
+export default async function importGoals(fileKey, region) {
+  const grantees = await parseCsv(fileKey);
+  const regionId = region;
   try {
     const cleanRoleTopics = [];
     const cleanGrantGoals = [];

src/tools/importPlanGoals.test.js

@@ -1,18 +1,28 @@
+import { readFileSync } from 'fs';
 import importGoals from './importPlanGoals';
+import { downloadFile } from '../lib/s3';
 import db, {
   Role, Topic, RoleTopic, Goal, Grantee, Grant,
 } from '../models';
 
+jest.mock('../lib/s3');
+
 describe('Import TTA plan goals', () => {
-  afterAll(() => {
-    db.sequelize.close();
+  beforeEach(() => {
+    downloadFile.mockReset();
+  });
+  afterAll(async () => {
+    await db.sequelize.close();
   });
   it('should import Roles table', async () => {
+    const fileName = 'GranteeTTAPlanTest.csv';
+    downloadFile.mockResolvedValue({ Body: readFileSync(fileName) });
+
     await Role.destroy({ where: {} });
     const rolesBefore = await Role.findAll();
 
     expect(rolesBefore.length).toBe(0);
-    await importGoals('GranteeTTAPlanTest.csv');
+    await importGoals(fileName, 14);
 
     const roles = await Role.findAll({
       attributes: ['id', 'name', 'fullName'],
@@ -59,11 +69,14 @@ describe('Import TTA plan goals', () => {
   });
 
   it('should import Topics table', async () => {
+    const fileName = 'GranteeTTAPlanTest.csv';
+    downloadFile.mockResolvedValue({ Body: readFileSync(fileName) });
+
     await Topic.destroy({ where: {} });
     const topicsBefore = await Topic.findAll();
 
     expect(topicsBefore.length).toBe(0);
-    await importGoals('GranteeTTAPlanTest.csv');
+    await importGoals(fileName, 14);
 
     const topics = await Topic.findAll();
     expect(topics).toBeDefined();
@@ -104,11 +117,14 @@ describe('Import TTA plan goals', () => {
   });
 
   it('should import Goals table', async () => {
+    const fileName = 'GranteeTTAPlanTest.csv';
+    downloadFile.mockResolvedValue({ Body: readFileSync(fileName) });
+
     await Goal.destroy({ where: {} });
     const goalsBefore = await Goal.findAll();
 
     expect(goalsBefore.length).toBe(0);
-    await importGoals('GranteeTTAPlanTest.csv');
+    await importGoals(fileName, 14);
 
     const allGoals = await Goal.findAll();
     expect(allGoals).toBeDefined();
@@ -166,7 +182,10 @@ describe('Import TTA plan goals', () => {
   });
 
   it('should have Grantees Goals connection', async () => {
-    await importGoals('GranteeTTAPlanTest.csv');
+    const fileName = 'GranteeTTAPlanTest.csv';
+    downloadFile.mockResolvedValue({ Body: readFileSync(fileName) });
+
+    await importGoals(fileName, 14);
 
     // test eager loading
     const grantee = await Grantee.findOne({
@@ -188,23 +207,29 @@ describe('Import TTA plan goals', () => {
   });
 
   it('should import RoleTopics table', async () => {
+    const fileName = 'GranteeTTAPlanTest.csv';
+    downloadFile.mockResolvedValue({ Body: readFileSync(fileName) });
+
     await Topic.destroy({ where: {} });
 
     const roleTopicsBefore = await RoleTopic.findAll();
 
     expect(roleTopicsBefore.length).toBe(0);
-    await importGoals('GranteeTTAPlanTest.csv', 14);
+    await importGoals(fileName, 14);
 
     const roleTopics = await RoleTopic.findAll();
     expect(roleTopics).toBeDefined();
     expect(roleTopics.length).toBe(20);
   });
 
   it('should import goals from another region', async () => {
+    const fileName = 'R9GranteeTTAPlanTest.csv';
+    downloadFile.mockResolvedValue({ Body: readFileSync(fileName) });
+
     const goalsBefore = await Goal.findAll();
 
     expect(goalsBefore.length).toBe(12);
-    await importGoals('R9GranteeTTAPlanTest.csv', 9);
+    await importGoals(fileName, 9);
 
     const allGoals = await Goal.findAll();
     expect(allGoals).toBeDefined();
@@ -273,4 +298,25 @@ describe('Import TTA plan goals', () => {
     });
     expect(goalWithTopic.topics[0].roles[0].fullName).toBe('Grantee Specialist');
   });
+
+  it('is idempotent', async () => {
+    const fileName = 'GranteeTTAPlanTest.csv';
+    downloadFile.mockResolvedValue({ Body: readFileSync(fileName) });
+
+    await Goal.destroy({ where: {} });
+    const goalsBefore = await Goal.findAll();
+
+    expect(goalsBefore.length).toBe(0);
+    await importGoals(fileName, 14);
+
+    const allGoals = await Goal.findAll();
+    expect(allGoals).toBeDefined();
+    expect(allGoals.length).toBe(12);
+
+    await importGoals(fileName, 14);
+
+    const allGoals2 = await Goal.findAll();
+    expect(allGoals2).toBeDefined();
+    expect(allGoals2.length).toBe(12);
+  });
 });

src/tools/importTTAPlanGoals.js

@@ -1,5 +1,7 @@
+import {} from 'dotenv/config';
 import { option } from 'yargs';
 import importGoals from './importPlanGoals';
+import { logger } from '../logger';
 
 const { argv } = option('file', {
   alias: 'f',
@@ -13,21 +15,16 @@ const { argv } = option('file', {
   .help()
   .alias('help', 'h');
 
-const defaultInputFile = './GranteeTTAPlan.csv';
-const defaultRegion = 14;
+const { file, region } = argv;
 
-let file;
-let region;
-if (argv.file) {
-  file = argv.file;
-} else {
-  file = defaultInputFile;
+if (!file) {
+  logger.error('File not provided to importTTAPlanGoals');
+  process.exit(1);
 }
 
-if (argv.region) {
-  region = argv.region;
-} else {
-  region = defaultRegion;
+if (!region) {
+  logger.error('Region not provided to importTTAPlanGoals');
+  process.exit(1);
 }
 
 importGoals(file, region);