diff --git a/.cfignore b/.cfignore index e1bd2210c8..4074811e76 100644 --- a/.cfignore +++ b/.cfignore @@ -4,6 +4,7 @@ deployment_config/ frontend/ src/ maintenance_page/ +smartsheet_scripts/ terraform/ hses.zip temp/ diff --git a/.circleci/config.yml b/.circleci/config.yml index 647fa79175..acea256d48 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -164,7 +164,7 @@ parameters: default: "main" type: string sandbox_git_branch: # change to feature branch to test deployment - default: "kw-ie-keyboard-nav" + default: "cfignore-legacy-scripts" type: string prod_new_relic_app_id: default: "877570491" diff --git a/R14ActivityReportsTest.csv b/R14ActivityReportsTest.csv index 111380c120..d4c5243f1f 100644 --- a/R14ActivityReportsTest.csv +++ b/R14ActivityReportsTest.csv @@ -1,5 +1,5 @@ -ReportID,Grantee Name,CDI Grantee Name (if applicable),Multi-Grantee Activities,Program Type(s),Non-Grantee Activity,Source of Request,Reason/s,T -TA,Topics,Other topics,Grantee Participants,Non-Grantee Participants,Number of Participants,Start Date,End Date,Duration,Other Specialists,Target Populations,Resources Used,Non-OHS Resources,GOAL 1,Grantee's learning level GOAL 1 (Optional),Objective 1.1,Objective 1.1 status,Objective 1.2,Objective 1.2 status,GOAL 2,Grantee's learning level GOAL 2 (Optional),Objective 2.1,Objective 2.1 status,Objective 2.2,Objective 2.2 status,TTA Provided and Grantee Progress Made,Grantee Follow Up Tasks & Objectives,Specialist Follow Up Tasks & Objectives,Format,Additional notes for this activity,,Manager,Manager approval,Created,Created By,Override Created By,Modified,Modified By -R14-AR-000435,"Action, Inc. | 14CH010848",,,,,Grantee,Need: Professional Development,Both,"Coaching / Teaching / Instructional Support | ECS, FES",,Manager / Coordinator / Specialist / Case Manager,,,2020-11-06,2020-11-06,2.0,,Preschool,,,"Grantee will have a minimum of 5% of staff receive mentoring and coaching (PG#1, Obj #4).",,Completed: Train the Education Manager on an overview of the TLC’s due to the cancelled even in May.,,,,,,,,,,"Pre TA: Working on PD plans and has a Creative Curriculum Training coming up. +ReportID,Grantee Name,CDI Grantee Name (if applicable),Multi-Grantee Activities,Program Type(s),Non-Grantee Activity,Source of Request,Reason/s,T -TA,Topics,Other topics,Grantee Participants,Non-Grantee Participants,Number of Participants,Start Date,End Date,Duration,Other Specialists,Target Populations,Resources Used,Non-OHS Resources,GOAL 1,Grantee's learning level GOAL 1 (Optional),Objective 1.1,Objective 1.1 status,Objective 1.2,Objective 1.2 status,GOAL 2,Grantee's learning level GOAL 2 (Optional),Objective 2.1,Objective 2.1 status,Objective 2.2,Objective 2.2 status,TTA Provided and Grantee Progress Made,Grantee Follow Up Tasks & Objectives,Specialist Follow Up Tasks & Objectives,Format,Additional notes for this activity,,Manager,Manager approval,Created,Created By,Override Created By,Modified,Modified By +R14-AR-000435,"Action, Inc. | 14CH010848",,,,,Grantee,Need: Professional Development,Both,"Coaching / Teaching / Instructional Support | ECS, FES",,Manager / Coordinator / Specialist / Case Manager,,,2020-11-06,2020-11-06,2,Specialist2@test.gov,Preschool,,,"Grantee will have a minimum of 5% of staff receive mentoring and coaching (PG#1, Obj #4).",,Completed: Train the Education Manager on an overview of the TLC’s due to the cancelled even in May.,,,,,,,,,,"Pre TA: Working on PD plans and has a Creative Curriculum Training coming up. Trained and provided TA for the education manager on Together Learning and Collaborating (TCL’s) with the following discussions: • Reviewed the content of the TLC group sessions 3 and 4. @@ -17,12 +17,12 @@ Post TA: • After program implements them, meet to give feedback and strategies ECS: -• Send Plan form• Mail a thing",Virtual,,,manager1@test.region14,Approved,2020-11-09 08:20:57,creator@first.row,,2020-12-21 07:47:05,creator@first.row +• Send Plan form• Mail a thing",Virtual,,,manager@test.gov,Approved,2020-11-09 08:20:57,specialist1@test.gov,,2020-12-21 07:47:05,specialist1@test.gov R14-AR-000442,S College | 14CH011664,,,,,Regional Office,"Monitoring | Area of Concern Monitoring | Deficiency",Technical Assistance,"Behavioral / Mental Health | HS, FES Corrective Actions | GS Quality Improvement / QIP | GS",,"Coach / Trainer -Manager / Coordinator / Specialist / Case Manager",,,2020-11-05,2020-11-05,2.0,,"Infant/Toddlers +Manager / Coordinator / Specialist / Case Manager",,,2020-11-05,2020-11-05,2,,"Infant/Toddlers Preschool",,,"Grantee will finalize additions to Technical Service Plan, defining T/TA outcomes, plan steps, strategies and identify dates of service.",,"Continued support for Social/Emotional Plan implementation- In Progress Revise/refine coaching needs assessments- In Progress Support staff’s reflective practices- in Progress",,,,,,,,,,"Anticipated Outcome: Grantee will finalize additions to Technical Service Plan, defining T/TA outcomes, plan steps, strategies and identify dates of service. @@ -44,10 +44,10 @@ Post TA: November 13, 2020- revise education staff needs assessments per discussion. distribute needs assessments to education staff. November 30, 2020- review Head Start Heals materials for integration to services.","Next Steps for Specialist: -November 11, 2020- send participants the arrow-PD chart.",Virtual,,,manager1@test.region14,Approved,2020-11-09 10:39:08,creator@second.row,,2020-12-21 07:47:05,creator@second.row +November 11, 2020- send participants the arrow-PD chart.",Virtual,,,manager@test.gov,Approved,2020-11-09 10:39:08,specialist2@test.gov,,2020-12-21 07:47:05,specialist2@test.gov R14-AR-000770,"Program, Inc. | 14CH014598",,,,,Grantee,Need: Program Planning,Technical Assistance,"CLASS / Learning Environments / Classroom Management | ECS Environmental Health and Safety | HS",,"Manager / Coordinator / Specialist / Case Manager -Teacher / Infant-Toddler Caregiver",,,2020-12-18,2020-12-18,2.0,,Preschool,"https://eclkc.ohs.acf.hhs.gov/publication/guiding-questions-active-supervision-safety +Teacher / Infant-Toddler Caregiver",,,2020-12-18,2020-12-18,2,,Preschool,"https://eclkc.ohs.acf.hhs.gov/publication/guiding-questions-active-supervision-safety https://eclkc.ohs.acf.hhs.gov/safety-practices/article/keep-children-safe-using-active-supervision https://eclkc.ohs.acf.hhs.gov/safety-practices/article/active-supervision https://eclkc.ohs.acf.hhs.gov/safety-practices/article/tips-keeping-children-safe-developmental-guide-preschoolers @@ -71,9 +71,9 @@ ECS facilitated an overview of the 6 strategies for Active Supervision and led p Post TA: *15 Minute In-service Suites *Scanning and counting in the classroom -*Transition ideas utilizing cue cards",Provide ongoing training and professional development opportunities to staff by utilizing the resources shared by ECS from the 6 strategies and provide follow up with staff. Grantee will ensure staff understand and have reviewed all policies and procedures of safety practices.,Follow-up with the Director during ongoing TTA visits and send data collected from training to Director,Virtual,,,manager2@test.region14,Approved,2020-12-18 11:32:54,creator@third.row,,2020-12-21 07:47:05,admin@test.gov +*Transition ideas utilizing cue cards",Provide ongoing training and professional development opportunities to staff by utilizing the resources shared by ECS from the 6 strategies and provide follow up with staff. Grantee will ensure staff understand and have reviewed all policies and procedures of safety practices.,Follow-up with the Director during ongoing TTA visits and send data collected from training to Director,Virtual,,,manager@test.gov,Approved,2020-12-18 11:32:54,specialist3@test.gov,,2020-12-21 07:47:05,manager@test.gov R14-AR-001005,The Public Schools | 14CH014747,,,Head Start (ages 3-5),,Grantee,New Director or Management,Technical Assistance,Other,Orientation/Onboarding,"Manager / Coordinator / Specialist -Program Director (HS/EHS)",,8.0,2021-01-28,2021-01-28,2.5,,Preschool,"TTA ECLKC Resources: +Program Director (HS/EHS)",,8,2021-01-28,2021-01-28,2.5,,Preschool,"TTA ECLKC Resources: https://eclkc.ohs.acf.hhs.gov/human-resources/article/ensuring-new-employees-success-best-practices-employee-onboarding",,"Anticipated Outcomes: GS Goal: The Grantee will strengthen its Human Resource system.",1 - Introductory,"TTA Objective: Assess current practices, identifying future methods, and formalizing plans to support the program goals and Head Start (HS) regulations.",In progress,,,,,,,,,"Pre-TA: The leadership staff from Quincy Public School District 172 provided the following updates related to the program since the last TTA virtual meeting on November 24, 2020: @@ -121,8 +121,8 @@ GS recommended/suggested the following: - continue to discuss HR as it relates to their orientation and onboarding process - complete HR Audit - Continue to build an Orientation and Onboarding Plan","The TTA Specialist will: (What more do you need from TTA Specialists?) -",Virtual,,,manager3@test.region14,Approved,2021-01-29 11:44:28,creator@third.row,,2021-02-03 02:13:36,creator@third.row -R14-AR-000892,Board of Trustees of | 14CH041588,,,Head Start (ages 3-5),,Grantee,Ongoing Quality Improvement,Technical Assistance,Other,Staff Wellness,Manager / Coordinator / Specialist,,4.0,2021-01-14,2021-01-14,2.0,,Preschool,Trauma Briefs 1-5,Sesame St- Elmo & COVID www.sesamestreet.org,"Program will foster the growth and development of its staff through individualized, job-embedded professional development, mental health supports, and daily classroom support.",2 - Intermediate,ECS will provide TA to support program's efforts with resources related to school readiness and building staff resilience and wellness.,In progress,,,,,,,,,"T/TA Provided: ECS facilitated TA to provide further resources sharing, Sesame Street and Elmo COVID-19 information, Self-Care checklist, and introduced the concept of Compassion Fatigue and its effect on staff and remaining well. ECS provided Professional Quality of Life Scale allowing participants to complete and a de-brief where they expressed rating just as expected to thinking scores would be higher on burnout section to “shocked”. ECS guided them to the criteria on each section and elements of burnout vs. secondary trauma stress sharing the ProQOL helper card. ECS asked if they thought the scale could be used with staff. Participants did think it could be used and shared examples of staff challenges, asked for strategies to support. ECS recommended including the mental health consultant as possible, providing affirmation and validation of feelings as well as support to identify any resources. ECS spotlighted resource from NC Pyramid Model Innovations Self-Care for Teacher and HS NCPFCE Trauma Briefs to generate supports for staff and families. ECS offered data on staff wellness, retention, and engaged staff in discussion of characteristics that effect stress levels and job satisfaction. Grantee is in process of self-assessment and plan to review current practices in development of a formalized Wellness plan for PY 21-22 and requested one-page, parent-friendly resources on COVID supports to include on newsletters and/or distribution to parents. +",Virtual,,,manager@test.gov,Approved,2021-01-29 11:44:28,specialist3@test.gov,,2021-02-03 02:13:36,specialist3@test.gov +R14-AR-000892,Board of Trustees of | 14CH041588,,,Head Start (ages 3-5),,Grantee,Ongoing Quality Improvement,Technical Assistance,Other,Staff Wellness,Manager / Coordinator / Specialist,,4,2021-01-14,2021-01-14,2,,Preschool,Trauma Briefs 1-5,Sesame St- Elmo & COVID www.sesamestreet.org,"Program will foster the growth and development of its staff through individualized, job-embedded professional development, mental health supports, and daily classroom support.",2 - Intermediate,ECS will provide TA to support program's efforts with resources related to school readiness and building staff resilience and wellness.,In progress,,,,,,,,,"T/TA Provided: ECS facilitated TA to provide further resources sharing, Sesame Street and Elmo COVID-19 information, Self-Care checklist, and introduced the concept of Compassion Fatigue and its effect on staff and remaining well. ECS provided Professional Quality of Life Scale allowing participants to complete and a de-brief where they expressed rating just as expected to thinking scores would be higher on burnout section to “shocked”. ECS guided them to the criteria on each section and elements of burnout vs. secondary trauma stress sharing the ProQOL helper card. ECS asked if they thought the scale could be used with staff. Participants did think it could be used and shared examples of staff challenges, asked for strategies to support. ECS recommended including the mental health consultant as possible, providing affirmation and validation of feelings as well as support to identify any resources. ECS spotlighted resource from NC Pyramid Model Innovations Self-Care for Teacher and HS NCPFCE Trauma Briefs to generate supports for staff and families. ECS offered data on staff wellness, retention, and engaged staff in discussion of characteristics that effect stress levels and job satisfaction. Grantee is in process of self-assessment and plan to review current practices in development of a formalized Wellness plan for PY 21-22 and requested one-page, parent-friendly resources on COVID supports to include on newsletters and/or distribution to parents. Progress toward Outcome: Grantee participated in exercise, gained awareness of resources, and strategized on possible uses of resources. Health/Nutrition Coordinator reported feeling adequately prepared to incorporate ideas and complete draft staff wellness plan for PY 21-22 based on ECS TA and resource sharing. Final Check scheduled for May 18, 2021 to ensure no further needs are necessary. @@ -132,12 +132,12 @@ Post TA: 3. N/A","Next Steps for Grantee: draft staff wellness action plan.","Next Steps for Specialist: January- Send resources to grantee. -As identified- send short, parent-friendly resources on COVID support, well-being",Virtual,"Grantee Progress: Program has initiated and planned for several staff wellness activities such as weekly staff highlight, etc.",,manager1@test.region14,Approved,2021-01-15 14:46:25,creator@fourth.row,,2021-01-17 06:40:49,creator@fourth.row +As identified- send short, parent-friendly resources on COVID support, well-being",Virtual,"Grantee Progress: Program has initiated and planned for several staff wellness activities such as weekly staff highlight, etc.",,manager@test.gov,Approved,2021-01-15 14:46:25,specialist1@test.gov,,2021-01-17 06:40:49,specialist1@test.gov R14-AR-001002,"High Five, Inc. | 14CH010143",,,Head Start (ages 3-5),,Grantee,Ongoing Quality Improvement,Training,Other,Reflective Supervision,"Coach Family Service Worker / Case Manager Manager / Coordinator / Specialist Program Director (HS/EHS) -Teacher / Infant-Toddler Caregiver",,30.0,2021-01-25,2021-01-25,1.5,,Preschool,,,Enhance reflective practice.,2 - Intermediate,Grantee will increase understanding of the components of Reflective Supervision.,Completed,,,,,,,,,"Began with an opening activity around setting the intention for the year. +Teacher / Infant-Toddler Caregiver",,30,2021-01-25,2021-01-25,1.5,,Preschool,,,Enhance reflective practice.,2 - Intermediate,Grantee will increase understanding of the components of Reflective Supervision.,Completed,,,,,,,,,"Began with an opening activity around setting the intention for the year. Reviewed Reflective Supervision practices through video examples and discussion. Key points included: -The definition of reflective supervision @@ -158,11 +158,11 @@ Post TTA: 1. BETH strategy, Active listening reminder, video and practice. 2. Listen with intention",The Grantee will reflect on strategies and determine next steps,The ECS will conduct a virtual visit for Technical Service Plan development.,"Virtual In Person -Telephone",,,manager1@test.region14,Approved,2021-02-01 16:55:13,creator@row.seven,,2021-02-02 12:22:38,creator@row.seven +Telephone",,,manager@test.gov,Approved,2021-02-01 16:55:13,specialist2@test.gov,,2021-02-02 12:22:38,specialist2@test.gov R14-AR-001162,Big District | 14CH014444,,,"Early Head Start (ages 0-3) Head Start (ages 3-5)",,Grantee,Planning/Coordination (also TTA Plan Agreement),Technical Assistance,Other,Grantee T/TA Planning,"Manager / Coordinator / Specialist Program Director (HS/EHS) -Program Support / Administrative Assistant",,4.0,2021-02-23,2021-02-23,3.0,"specialist1@test.gov,","Infant/Toddlers +Program Support / Administrative Assistant",,4,2021-02-23,2021-02-23,3,"specialist1@test.gov,","Infant/Toddlers Preschool",,,Reflect on the program’s strengths and areas to strengthen and set priorities for next program year.,,Develop a TTA plan for the grantee with action steps and timelines for the next program year.,Completed,,,,,,,,,"TTA Provided: ECS and leadership team, including Head Start Director (HSD), participated in a brainstorming session to review strengths and areas to strengthen in the next program year. The discussion included the following: *ECS shared the intention and purpose of developing a TTA Plan in coordination and the connection to the grantee’s annual planning time. @@ -191,8 +191,8 @@ ECS and leadership team, including Head Start Director (HSD), participated in a Progress Toward Outcomes: Participants contributed to the conversations regarding annual planning and developed the next steps.","Grantee will: *See TTA Plan","ECS will: -*See TTA Plan",Virtual,,,manager4@test.region14,Approved,2021-02-24 10:46:04,creator@row.seven,,2021-02-24 10:56:01,creator@row.seven -R14-AR-000279,Healthy | 14CH010545,,,,,Regional Office,Need: Program Planning,Technical Assistance,Other,TSP Development,Manager / Coordinator / Specialist / Case Manager,,,2020-10-16,2020-10-16,2.0,,"Infant/Toddlers +*See TTA Plan",Virtual,,,manager@test.gov,Approved,2021-02-24 10:46:04,specialist2@test.gov,,2021-02-24 10:56:01,specialist2@test.gov +R14-AR-000279,Healthy | 14CH010545,,,,,Regional Office,Need: Program Planning,Technical Assistance,Other,TSP Development,Manager / Coordinator / Specialist / Case Manager,,,2020-10-16,2020-10-16,2,,"Infant/Toddlers Preschool",,,"Grantee will finalize Technical Service Plan, defining T/TA outcomes, plan steps, strategies and identify dates of service.",,TSP development - In progress,,,,,,,,,,"Anticipated Outcome: Grantee will finalize Technical Service Plan, defining T/TA outcomes, plan steps, strategies and identify dates of service. Grantee Progress: Grantee will close largest site due to COVID positive cases at site. Several counties in area are on state’s warning list due to increase in cases. Program is enrolled at 67%, PBC coach/specialist who recently resigned is returning to program position. @@ -205,9 +205,9 @@ Post TA: 1. Gained more information on Coordinated Approaches (CA). 2. Use in format development of CA","Next Steps for Grantee: - Email ECS child Outcomes PY 19-20 report and Coaching plan.","Next Steps for Specialist: -October- Email team PBC Program Leader’s Guide.",Virtual,,,manager9@test.region14,Approved,2020-10-19 07:23:40,creator@fourth.row,Person Overrider,2020-12-21 07:47:05,creator@fourth.row +October- Email team PBC Program Leader’s Guide.",Virtual,,,manager@test.gov,Approved,2020-10-19 07:23:40,specialist1@test.gov,Person Overrider,2020-12-21 07:47:05,specialist1@test.gov R14-AR-000976,"ACRONYM, Co. | 14CH010140",,,"Early Head Start (ages 0-3) -Head Start (ages 3-5)",,Grantee,Ongoing Quality Improvement,Technical Assistance,Transition Practices,,Manager / Coordinator / Specialist,,1.0,2021-01-25,2021-01-25,2.0,,"Infant/Toddlers +Head Start (ages 3-5)",,Grantee,Ongoing Quality Improvement,Technical Assistance,Transition Practices,,Manager / Coordinator / Specialist,,1,2021-01-25,2021-01-25,2,"specialist1@test.gov, Specialist3@Test.Gov","Infant/Toddlers Preschool",https://eclkc.ohs.acf.hhs.gov/sites/default/files/pdf/supporting-transitions-brief-two.pdf,,We will provide responsive practices that support all children’s positive growth and development (PG#1).,2 - Intermediate,Incorporate new strategies to create smooth transitions.,In progress,,,,,,,,,"Pre TA: No big changes Brainstormed ideas for a smooth transition for children and families from EHS to HS, and HS to the school with the following conversations: @@ -235,7 +235,7 @@ Post TA: ECS: • Send Transition resources supporting multiple places • Send Transitions information found in the HS Act -• Plan for upcoming meeting on Transitions",Both,,,manager10@test.region,Approved,2021-01-26 12:28:28,creator@row.ten,,2021-01-26 12:33:04,creator@row.ten +• Plan for upcoming meeting on Transitions",Both,,,manager@test.gov,Approved,2021-01-26 12:28:28,specialist3@test.gov,,2021-01-26 12:33:04,specialist3@test.gov R14-AR-001132,"Healthy | 14CH010545 Another | 14CH010114 This Is One | 14CH1414140",,,Head Start (ages 3-5),,Grantee,Ongoing Quality Improvement,Training,"CLASS: Classroom Management @@ -243,7 +243,7 @@ CLASS: Emotional Support CLASS: Instructional Support",,"Center Director / Site Director Coach Manager / Coordinator / Specialist -Teacher / Infant-Toddler Caregiver",,10.0,2021-02-16,2021-02-18,16.0,"specialist1@test.gov, specialist2@test.gov, s3@test.gov",Preschool,,Teachstone Provided Materials,"Participants will demonstrate knowledge of the CLASS dimensions, indicators, and behavior markers; practice notetaking and coding process; and prepare for CLASS certification",,"reviewed all domains, dimensions, indicators, and behavior makers",Completed,provided knowledge about and practiced note taking and presented and practiced the CLASS scoring process,Completed,,,,,,,"TA Provided: +Teacher / Infant-Toddler Caregiver",,10,2021-02-16,2021-02-18,16,"specialist1@test.gov, specialist2@test.gov, specialist3@test.gov",Preschool,,Teachstone Provided Materials,"Participants will demonstrate knowledge of the CLASS dimensions, indicators, and behavior markers; practice notetaking and coding process; and prepare for CLASS certification",,"reviewed all domains, dimensions, indicators, and behavior makers",Completed,provided knowledge about and practiced note taking and presented and practiced the CLASS scoring process,Completed,,,,,,,"TA Provided: Trainers shared the three-day virtual training agenda, including the following: @@ -281,4 +281,4 @@ Complete the training process in Teachstone. Support participants until the testing is complete (through April 16th). -Share information with assigned ECS (as test results are available).",Virtual,,,manager11@test.region,Approved,2021-02-22 06:37:35,creatror@creator.biz,,2021-02-22 06:52:19,admin@test.gov +Share information with assigned ECS (as test results are available).",Virtual,,,manager@test.gov,Approved,2021-02-22 06:37:35,specialist3@test.gov,,2021-02-22 06:52:19,manager@test.gov diff --git a/frontend/src/pages/LegacyReport/index.js b/frontend/src/pages/LegacyReport/index.js index f521e542e0..5a5881ebc4 100644 --- a/frontend/src/pages/LegacyReport/index.js +++ b/frontend/src/pages/LegacyReport/index.js @@ -5,6 +5,7 @@ import { Alert, Table } from '@trussworks/react-uswds'; import { map } from 'lodash'; import Container from '../../components/Container'; +import FileReviewItem from '../ActivityReport/Pages/Review/FileReviewItem'; import { legacyReportById } from '../../fetchers/activityReports'; import reportColumns from './reportColumns'; @@ -45,7 +46,7 @@ function LegacyReport({ match }) { ); } - const { imported } = legacyReport; + const { imported, attachments } = legacyReport; const entries = map(reportColumns, (display, field) => { const value = imported[field]; return { @@ -57,9 +58,9 @@ function LegacyReport({ match }) { const tableEntries = entries.filter((item) => item.value).map(({ field, display, value }) => ( - + {display} - + {value.split('\n').map((string) =>
{string}
)} @@ -90,6 +91,27 @@ function LegacyReport({ match }) { {tableEntries} + {attachments && attachments.length > 0 + && ( + + Attachments + + {attachments.map(({ + id, + originalFileName, + url: { url }, + status, + }) => ( + + ))} + + + )} diff --git a/frontend/src/pages/LegacyReport/reportColumns.js b/frontend/src/pages/LegacyReport/reportColumns.js index eaeaadd2e3..3077b39637 100644 --- a/frontend/src/pages/LegacyReport/reportColumns.js +++ b/frontend/src/pages/LegacyReport/reportColumns.js @@ -40,6 +40,7 @@ const reportFields = { additionalNotesForThisActivity: 'Additional notes for this activity', manager: 'Manager', managerApproval: 'Manager approval', + comments: 'Comments', created: 'Time Created', createdBy: 'Created By', modified: 'Time Modified', diff --git a/package.json b/package.json index 51b676328f..d0e917ac41 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,9 @@ "import:goals:local": "./node_modules/.bin/babel-node ./src/tools/importTTAPlanGoals.js", "import:goals": "node ./build/server/tools/importTTAPlanGoals.js", "import:hses:local": "./node_modules/.bin/babel-node ./src/tools/importGrantGranteesCLI.js --skipdownload", - "import:hses": "node ./build/server/tools/importGrantGranteesCLI.js" + "import:hses": "node ./build/server/tools/importGrantGranteesCLI.js", + "reconcile:legacy": "node ./build/server/tools/reconcileLegacyReports.js", + "reconcile:legacy:local": "./node_modules/.bin/babel-node ./src/tools/reconcileLegacyReports.js" }, "repository": { "type": "git", @@ -105,7 +107,7 @@ "statements": 90, "functions": 90, "branches": 81, - "lines": 90 + "lines": 90 } } }, diff --git a/smartsheet_scripts/legacy_report_attachments/.ruby-version b/smartsheet_scripts/legacy_report_attachments/.ruby-version new file mode 100644 index 0000000000..37c2961c24 --- /dev/null +++ b/smartsheet_scripts/legacy_report_attachments/.ruby-version @@ -0,0 +1 @@ +2.7.2 diff --git a/smartsheet_scripts/legacy_report_attachments/Gemfile b/smartsheet_scripts/legacy_report_attachments/Gemfile new file mode 100644 index 0000000000..d2612639e2 --- /dev/null +++ b/smartsheet_scripts/legacy_report_attachments/Gemfile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +git_source(:github) {|repo_name| "https://github.com/#{repo_name}" } + +gem "activesupport", "~> 6.1" +gem "smartsheet", "~> 2.101" +gem "rake", "~> 13.0" +gem "faraday", "~> 1.3" +gem "faraday_middleware", "~> 1.0" +gem "faraday-cookie_jar" diff --git a/smartsheet_scripts/legacy_report_attachments/Gemfile.lock b/smartsheet_scripts/legacy_report_attachments/Gemfile.lock new file mode 100644 index 0000000000..e743f81071 --- /dev/null +++ b/smartsheet_scripts/legacy_report_attachments/Gemfile.lock @@ -0,0 +1,57 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (6.1.3) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + zeitwerk (~> 2.3) + awrence (1.2.1) + concurrent-ruby (1.1.8) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) + faraday (1.3.0) + faraday-net_http (~> 1.0) + multipart-post (>= 1.2, < 3) + ruby2_keywords + faraday-cookie_jar (0.0.7) + faraday (>= 0.8.0) + http-cookie (~> 1.0.0) + faraday-net_http (1.0.1) + faraday_middleware (1.0.0) + faraday (~> 1.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + i18n (1.8.9) + concurrent-ruby (~> 1.0) + minitest (5.14.4) + multipart-post (2.1.1) + plissken (1.4.1) + rake (13.0.3) + ruby2_keywords (0.0.4) + smartsheet (2.101.1) + awrence (~> 1.0) + faraday (>= 0.13.1, < 2) + faraday_middleware (>= 0.10.0, < 2) + plissken (~> 1.2) + tzinfo (2.0.4) + concurrent-ruby (~> 1.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.7) + zeitwerk (2.4.2) + +PLATFORMS + ruby + +DEPENDENCIES + activesupport (~> 6.1) + faraday (~> 1.3) + faraday-cookie_jar + faraday_middleware (~> 1.0) + rake (~> 13.0) + smartsheet (~> 2.101) + +BUNDLED WITH + 2.1.4 diff --git a/smartsheet_scripts/legacy_report_attachments/README.md b/smartsheet_scripts/legacy_report_attachments/README.md new file mode 100644 index 0000000000..96af308b5b --- /dev/null +++ b/smartsheet_scripts/legacy_report_attachments/README.md @@ -0,0 +1,41 @@ +Legacy Report Attachments +========================= + +Script to import comments and file attachments from Smartsheet ARs. + +Configuration +------------- + +### Environment Variables + +To run, the following three ENV vars must be set: + +* `SMARTSHEET_API_TOKEN` api token for smartsheet, must have access to all 12 region AR sheets +* `SMARTHUB_SESSION_COOKIE` the session cookie for a smarthub admin with read access to all 12 regions +* `SMARTHUB_SESSION_SIG` the session.sig cooke for the same smarthub admin + +### smartsheet.yml + +This file contains the sheet ids for smartsheet activity report sheets + +### Attachment::SMARTHUB_URI_BASE + +This constant (set in `lib/attachment.rb`) is the base url of the smarthub + +Running +------- + +* Ensure ruby 2.7.2 is installed +* `bundle install` +* Set ENV vars +* `bundle exec rake run[REGION]` where `REGION` is the region number + + +example: `bundle exec rake run[1]` + + +Disclaimers +----------- + +There are several attachments stored in onedrive that are behind additional login layers. +These will need to be imported by hand diff --git a/smartsheet_scripts/legacy_report_attachments/Rakefile b/smartsheet_scripts/legacy_report_attachments/Rakefile new file mode 100644 index 0000000000..df35c70d32 --- /dev/null +++ b/smartsheet_scripts/legacy_report_attachments/Rakefile @@ -0,0 +1,6 @@ +require_relative "lib/attachment" + +desc "Transfer attachments by region" +task :run, [:region] do |task, args| + Attachment.new(args[:region]).call +end diff --git a/smartsheet_scripts/legacy_report_attachments/lib/attachment.rb b/smartsheet_scripts/legacy_report_attachments/lib/attachment.rb new file mode 100644 index 0000000000..a572b1320a --- /dev/null +++ b/smartsheet_scripts/legacy_report_attachments/lib/attachment.rb @@ -0,0 +1,152 @@ +require "active_support/all" +require "logger" +require "smartsheet" +require "faraday" +require "faraday_middleware" +require "faraday-cookie_jar" +require "tempfile" +require_relative "config" + +class Attachment + + SMARTHUB_URI_BASE = "https://ttahub.ohs.acf.hhs.gov" + + attr_reader :region, :client, :cookie + def initialize(region) + @region = region.to_s + # session cookies taken out of chrome dev tools for an active admin session + fail "Session cookie not provided" if ENV["SMARTHUB_SESSION_COOKIE"].blank? || ENV["SMARTHUB_SESSION_SIG"].blank? + @cookie = "session=#{ENV["SMARTHUB_SESSION_COOKIE"]}; session.sig=#{ENV["SMARTHUB_SESSION_SIG"]}" + token = ENV["SMARTSHEET_API_TOKEN"] + fail "No API Token provided" if token.blank? + logger = Logger.new($stdout) + logger.level = :warn + @client = Smartsheet::Client.new( + token: token, + logger: logger, + base_url: Smartsheet::Constants::GOV_API_URL + ) + end + + def call + sheet = client.sheets.get(sheet_id: sheet_id, params: { + include: "attachments,discussions", + level: "2", + columnIds: report_id_column_id + }) + sheet[:rows].each do |row| + next if row[:discussions].blank? && row[:attachments].blank? + process_row(row) + end + sheet + end + + def process_row(row) + legacy_id = row[:cells].first[:value].gsub(/^R14/, "R#{"%02d" % region}") + puts "Processing #{legacy_id}" + activity_report = if row[:discussions].present? + discussions = row[:discussions].flat_map { |disc| retrieve_discussion(disc[:id]) }.join("\n") + update_comments legacy_id, discussions + else + activity_report legacy_id + end + (row[:attachments] || []).each do |attachment| + process_attachment activity_report, attachment + end + end + + def process_attachment(activity_report, attachment_meta) + if activity_report["attachments"].none? { |attach| attach["originalFileName"] == attachment_meta[:name] } + attach = client.sheets.attachments.get(sheet_id: sheet_id, attachment_id: attachment_meta[:id]) + if attach[:url].blank? + fail "No URL to download file: #{attach.inspect}" + end + Tempfile.create do |file| + success = download_file(file, attach[:url]) + if success + post_file(activity_report["id"], file, attachment_meta) + else + puts "Failed to download file for: #{activity_report["displayId"]}, #{attach.inspect}" + end + end + end + end + + def update_comments(legacy_id, discussions) + response = put("/api/activity-reports/legacy/#{legacy_id}", {comments: discussions}) + if response.success? + JSON.parse(response.body) + else + fail "#{response.status} #{response.body}" + end + end + + def activity_report(legacy_id) + response = get("/api/activity-reports/legacy/#{legacy_id}") + if response.success? + JSON.parse(response.body) + else + fail "#{response.status} #{response.body}" + end + end + + def download_file(file, url) + conn = Faraday.new(url) do |c| + c.use FaradayMiddleware::FollowRedirects + c.use :cookie_jar + end + response = conn.get + if response.success? + file.write(response.body) + file.flush + true + else + false + end + end + + def post_file(report_id, file, attachment_meta) + filepart = Faraday::FilePart.new(file.path, attachment_meta[:mime_type], attachment_meta[:name]) + params = { + "reportId": report_id, + file: filepart + } + multipart_faraday.post("/api/files", params) + end + + def put(url, params) + faraday.put(url, params.to_json, "Content-Type" => "application/json") + end + + def get(url) + faraday.get(url) + end + + def multipart_faraday + @multipart_faraday ||= Faraday.new(url: SMARTHUB_URI_BASE, headers: {'Cookie' => cookie}) do |f| + f.request :multipart + end + end + + def faraday + @faraday ||= Faraday.new(url: SMARTHUB_URI_BASE, headers: {'Cookie' => cookie}) + end + + def retrieve_discussion(disc_id) + client.sheets.discussions.get(sheet_id: sheet_id, discussion_id: disc_id)[:comments].map do |comment| + "#{comment[:created_by][:email]}: #{comment[:text]}" + end + end + + def report_id_column_id + @report_id_column_id ||= sheet_columns.find { |c| c[:title] == "ReportID" }[:id] + end + + def sheet_columns + @sheet_columns ||= client.sheets.columns.list(sheet_id: sheet_id)[:data] + end + + def sheet_id + @sheet_id ||= Config.load[region][:ar_sheet_id] + end +end diff --git a/smartsheet_scripts/legacy_report_attachments/lib/config.rb b/smartsheet_scripts/legacy_report_attachments/lib/config.rb new file mode 100644 index 0000000000..c19d8c12a9 --- /dev/null +++ b/smartsheet_scripts/legacy_report_attachments/lib/config.rb @@ -0,0 +1,17 @@ +require "yaml" +require "erb" +require "active_support/core_ext/hash" + +class Config + CONFIG_FILE = File.expand_path(File.join(__dir__, "..", "smartsheet.yml")).freeze + + def self.load + @config ||= yaml.with_indifferent_access[:regions] + end + + private + + def self.yaml + @yaml ||= YAML.load(ERB.new(File.read(CONFIG_FILE)).result) + end +end diff --git a/smartsheet_scripts/legacy_report_attachments/smartsheet.yml b/smartsheet_scripts/legacy_report_attachments/smartsheet.yml new file mode 100644 index 0000000000..32897fbe18 --- /dev/null +++ b/smartsheet_scripts/legacy_report_attachments/smartsheet.yml @@ -0,0 +1,25 @@ +regions: + "1": + ar_sheet_id: "1515864581675124" + "2": + ar_sheet_id: "5139786187348084" + "3": + ar_sheet_id: "2184092773455988" + "4": + ar_sheet_id: "2458901960923252" + "5": + ar_sheet_id: "1333002054080628" + "6": + ar_sheet_id: "6793245517092980" + "7": + ar_sheet_id: "5175863912634484" + "8": + ar_sheet_id: "4964482802194548" + "9": + ar_sheet_id: "2924064098949236" + "10": + ar_sheet_id: "7427663726319732" + "11": + ar_sheet_id: "354711583266932" + "12": + ar_sheet_id: "2712682988509300" diff --git a/src/constants.js b/src/constants.js index d33a364555..30ce10de03 100644 --- a/src/constants.js +++ b/src/constants.js @@ -12,6 +12,7 @@ export const FILE_STATUSES = { QUEUED: 'SCANNING_QUEUED', QUEUEING_FAILED: 'QUEUEING_FAILED', SCANNING: 'SCANNING', + SCANNING_FAILED: 'SCANNING_FAILED', APPROVED: 'APPROVED', REJECTED: 'REJECTED', }; diff --git a/src/middleware/userAdminAccessMiddleware.js b/src/middleware/userAdminAccessMiddleware.js index 8790dd8011..5a94db2912 100644 --- a/src/middleware/userAdminAccessMiddleware.js +++ b/src/middleware/userAdminAccessMiddleware.js @@ -16,9 +16,7 @@ import handleErrors from '../lib/apiErrorHandler'; export default async function userAdminAccessMiddleware(req, res, next) { try { const { userId } = req.session; - if ((await validateUserAuthForAdmin(userId))) { - auditLogger.info(`User ${userId} successfully checked ADMIN access`); - } else { + if (!(await validateUserAuthForAdmin(userId))) { auditLogger.error(`User ${userId} attempted to access an ADMIN route without permission`); // consider sending a 404 rather than a 403 (Forbidden) to avoid confirming route return res.sendStatus(httpCodes.FORBIDDEN); diff --git a/src/migrations/20210323155040-add-scanning-failed-enum.js b/src/migrations/20210323155040-add-scanning-failed-enum.js new file mode 100644 index 0000000000..c8fbf7f7a0 --- /dev/null +++ b/src/migrations/20210323155040-add-scanning-failed-enum.js @@ -0,0 +1,9 @@ +module.exports = { + up: async (queryInterface) => { + await queryInterface.sequelize.query('ALTER TYPE "enum_Files_status" ADD VALUE \'SCANNING_FAILED\';'); + }, + down: async (queryInterface) => { + const query = 'DELETE FROM pg_enum WHERE enumlabel = \'SCANNING_FAILED\' AND enumtypid = ( SELECT oid FROM pg_type WHERE typname = \'enum_Files_status\')'; + await queryInterface.sequelize.query(query); + }, +}; diff --git a/src/migrations/20210323195745-fix-legacy-report-region-id.js b/src/migrations/20210323195745-fix-legacy-report-region-id.js new file mode 100644 index 0000000000..79a1a0aadd --- /dev/null +++ b/src/migrations/20210323195745-fix-legacy-report-region-id.js @@ -0,0 +1,10 @@ +module.exports = { + up: (queryInterface) => ( + queryInterface.sequelize.query('UPDATE "ActivityReports" SET "legacyId" = regexp_replace("legacyId", \'R(\\d)-\', \'R0\\1-\') WHERE "legacyId" ~ \'R\\d-\'') + ), + down: async () => { + /** + * Non-reversible + */ + }, +}; diff --git a/src/models/file.js b/src/models/file.js index 44099a5c9a..6df1bf045a 100644 --- a/src/models/file.js +++ b/src/models/file.js @@ -33,6 +33,7 @@ module.exports = (sequelize, DataTypes) => { 'QUEUEING_FAILED', 'SCANNING_QUEUED', 'SCANNING', + 'SCANNING_FAILED', 'APPROVED', 'REJECTED', ), diff --git a/src/routes/activityReports/handlers.js b/src/routes/activityReports/handlers.js index 4284804d10..22d2951692 100644 --- a/src/routes/activityReports/handlers.js +++ b/src/routes/activityReports/handlers.js @@ -16,6 +16,7 @@ import { goalsForGrants } from '../../services/goals'; import { userById, usersWithPermissions } from '../../services/users'; import { REPORT_STATUSES, DECIMAL_BASE } from '../../constants'; import { getUserReadRegions } from '../../services/accessValidation'; +import { logger } from '../../logger'; const { APPROVE_REPORTS } = SCOPES; @@ -25,6 +26,25 @@ const logContext = { namespace, }; +export async function updateLegacyFields(req, res) { + try { + const { legacyReportId } = req.params; + const report = await activityReportByLegacyId(legacyReportId); + if (!report) { + res.sendStatus(404); + return; + } + // no authorization here because the entire route is only available to admins + const imported = { ...report.imported, ...req.body }; + logger.debug(`Saving new data: ${JSON.stringify(imported, null, 2)}`); + + const savedReport = await createOrUpdate({ imported }, report); + res.json(savedReport); + } catch (error) { + handleErrors(req, res, error, logContext); + } +} + export async function getLegacyReport(req, res) { try { const { legacyReportId } = req.params; diff --git a/src/routes/activityReports/handlers.test.js b/src/routes/activityReports/handlers.test.js index b39125de54..4819085a73 100644 --- a/src/routes/activityReports/handlers.test.js +++ b/src/routes/activityReports/handlers.test.js @@ -10,6 +10,7 @@ import { getReports, getReportAlerts, getLegacyReport, + updateLegacyFields, } from './handlers'; import { activityReportById, @@ -21,6 +22,7 @@ import { activityReportAlerts, activityReportByLegacyId, } from '../../services/activityReports'; +import { getUserReadRegions } from '../../services/accessValidation'; import { userById, usersWithPermissions } from '../../services/users'; import ActivityReport from '../../policies/activityReport'; import User from '../../policies/user'; @@ -36,6 +38,8 @@ jest.mock('../../services/activityReports', () => ({ activityReportByLegacyId: jest.fn(), })); +jest.mock('../../services/accessValidation'); + jest.mock('../../services/users', () => ({ userById: jest.fn(), usersWithPermissions: jest.fn(), @@ -69,7 +73,7 @@ describe('Activity Report handlers', () => { jest.clearAllMocks(); }); - describe('activityReportByLegacyId', () => { + describe('getLegacyReport', () => { const request = { ...mockRequest, params: { legacyReportId: 1 }, @@ -100,6 +104,29 @@ describe('Activity Report handlers', () => { }); }); + describe('updateLegacyFields', () => { + const comments = 'smartsheet.user@tta.com: My comment'; + const request = { + ...mockRequest, + params: { legacyReportId: 1 }, + body: { comments }, + }; + + it('updates the import data with the comments', async () => { + activityReportByLegacyId.mockResolvedValue(report); + createOrUpdate.mockResolvedValue(report); + await updateLegacyFields(request, mockResponse); + expect(createOrUpdate).toHaveBeenCalledWith({ imported: { comments } }, report); + expect(mockResponse.json).toHaveBeenCalledWith(report); + }); + + it('handles a report not being found', async () => { + activityReportByLegacyId.mockResolvedValue(null); + await updateLegacyFields(request, mockResponse); + expect(mockResponse.sendStatus).toHaveBeenCalledWith(404); + }); + }); + describe('reviewReport', () => { const request = { ...mockRequest, @@ -354,9 +381,7 @@ describe('Activity Report handlers', () => { it('returns the reports', async () => { activityReports.mockResolvedValue({ count: 1, rows: [report] }); - userById.mockResolvedValue({ - id: 1, - }); + getUserReadRegions.mockResolvedValue([1]); await getReports(request, mockResponse); expect(mockResponse.json).toHaveBeenCalledWith({ count: 1, rows: [report] }); @@ -364,6 +389,8 @@ describe('Activity Report handlers', () => { it('handles a list of reports that are not found', async () => { activityReports.mockResolvedValue(null); + getUserReadRegions.mockResolvedValue([1]); + await getReports(request, mockResponse); expect(mockResponse.sendStatus).toHaveBeenCalledWith(404); }); @@ -377,10 +404,6 @@ describe('Activity Report handlers', () => { it('returns my alerts', async () => { activityReportAlerts.mockResolvedValue({ count: 1, rows: [report] }); - userById.mockResolvedValue({ - id: 1, - }); - await getReportAlerts(request, mockResponse); expect(mockResponse.json).toHaveBeenCalledWith({ alertsCount: 1, alerts: [report] }); }); diff --git a/src/routes/activityReports/index.js b/src/routes/activityReports/index.js index f47aee2039..33bb264fb6 100644 --- a/src/routes/activityReports/index.js +++ b/src/routes/activityReports/index.js @@ -12,7 +12,9 @@ import { reviewReport, resetToDraft, getLegacyReport, + updateLegacyFields, } from './handlers'; +import userAdminAccessMiddleware from '../../middleware/userAdminAccessMiddleware'; const router = express.Router(); @@ -26,6 +28,7 @@ router.get('/activity-recipients', getActivityRecipients); router.get('/goals', getGoals); router.get('/alerts', getReportAlerts); router.get('/legacy/:legacyReportId', getLegacyReport); +router.put('/legacy/:legacyReportId', userAdminAccessMiddleware, updateLegacyFields); router.get('/:activityReportId', getReport); router.get('/', getReports); router.put('/:activityReportId', saveReport); diff --git a/src/routes/apiDirectory.test.js b/src/routes/apiDirectory.test.js index d7fa933109..650f99afd5 100644 --- a/src/routes/apiDirectory.test.js +++ b/src/routes/apiDirectory.test.js @@ -10,7 +10,7 @@ const request = require('supertest'); const mockUser = { id: 110, hsesUserId: '110', - hsesUsername: 'user', + hsesUsername: 'user110', homeRegionId: 1, permissions: [ { @@ -39,7 +39,7 @@ describe('apiDirectory tests', () => { process.env.CURRENT_USER_ID = 110; }); afterAll(async () => { - await User.delete({ where: { id: mockUser.id } }); + await User.destroy({ where: { id: mockUser.id } }); await db.sequelize.close(); }); diff --git a/src/routes/files/handlers.js b/src/routes/files/handlers.js index 3c94bd3291..c746087cfb 100644 --- a/src/routes/files/handlers.js +++ b/src/routes/files/handlers.js @@ -9,6 +9,7 @@ import createFileMetaData, { import ActivityReportPolicy from '../../policies/activityReport'; import { activityReportById } from '../../services/activityReports'; import { userById } from '../../services/users'; +import { validateUserAuthForAdmin } from '../../services/accessValidation'; import { auditLogger } from '../../logger'; import { FILE_STATUSES } from '../../constants'; @@ -53,10 +54,10 @@ export const deleteHandler = async (req, res) => { export default async function uploadHandler(req, res) { const form = new multiparty.Form(); - form.parse(req, async (error, fields, files) => { + await form.parse(req, async (error, fields, files) => { const { reportId } = fields; if (error) { - res.status(500).send(error); + return res.status(500).send(error); } let buffer; let metadata; @@ -67,29 +68,25 @@ export default async function uploadHandler(req, res) { const report = await activityReportById(reportId); const authorization = new ActivityReportPolicy(user, report); - if (!authorization.canUpdate()) { - res.sendStatus(403); - return; + if (!(authorization.canUpdate() || (await validateUserAuthForAdmin(req.session.userId)))) { + return res.sendStatus(403); } try { if (!files.file) { - res.status(400).send({ error: 'file required' }); - return; + return res.status(400).send({ error: 'file required' }); } const { path, originalFilename, size } = files.file[0]; if (!size) { - res.status(400).send({ error: 'fileSize required' }); + return res.status(400).send({ error: 'fileSize required' }); } if (!reportId) { - res.status(400).send({ error: 'reportId required' }); - return; + return res.status(400).send({ error: 'reportId required' }); } buffer = fs.readFileSync(path); type = await fileType.fromFile(path); if (!type) { - res.status(400).send('Could not determine file type'); - return; + return res.status(400).send('Could not determine file type'); } fileName = `${uuidv4()}.${type.ext}`; metadata = await createFileMetaData( @@ -99,8 +96,7 @@ export default async function uploadHandler(req, res) { size, ); } catch (err) { - await handleErrors(req, res, err, logContext); - return; + return handleErrors(req, res, err, logContext); } try { const uploadedFile = await uploadFile(buffer, fileName, type); @@ -111,17 +107,14 @@ export default async function uploadHandler(req, res) { if (metadata) { await updateStatus(metadata.id, UPLOAD_FAILED); } - await handleErrors(req, res, err, logContext); - return; + return handleErrors(req, res, err, logContext); } try { await addToScanQueue({ key: metadata.key }); - await updateStatus(metadata.id, QUEUED); + return updateStatus(metadata.id, QUEUED); } catch (err) { - if (metadata) { - await updateStatus(metadata.id, QUEUEING_FAILED); - auditLogger.error(`${logContext} Failed to queue ${metadata.originalFileName}. Error: ${err}`); - } + auditLogger.error(`${logContext} Failed to queue ${metadata.originalFileName}. Error: ${err}`); + return updateStatus(metadata.id, QUEUEING_FAILED); } }); } diff --git a/src/routes/files/handlers.test.js b/src/routes/files/handlers.test.js index 0b9fcb3d60..bffa861033 100644 --- a/src/routes/files/handlers.test.js +++ b/src/routes/files/handlers.test.js @@ -3,17 +3,20 @@ import db, { File, ActivityReport, User, - Permission, } from '../../models'; import app from '../../app'; import { uploadFile, deleteFileFromS3, getPresignedURL } from '../../lib/s3'; import * as queue from '../../services/scanQueue'; -import SCOPES from '../../middleware/scopeConstants'; import { REPORT_STATUSES, FILE_STATUSES } from '../../constants'; import ActivityReportPolicy from '../../policies/activityReport'; import * as Files from '../../services/files'; +import { validateUserAuthForAdmin } from '../../services/accessValidation'; jest.mock('../../policies/activityReport'); +jest.mock('../../services/accessValidation', () => ({ + validateUserAuthForAdmin: jest.fn().mockResolvedValue(false), + validateUserAuthForAccess: jest.fn().mockResolvedValue(true), +})); const request = require('supertest'); @@ -26,23 +29,6 @@ const mockUser = { hsesUserId: '2046', hsesUsername: '2046', homeRegionId: 1, - permissions: [ - { - userId: 2046, - regionId: 5, - scopeId: SCOPES.READ_WRITE_REPORTS, - }, - { - userId: 2046, - regionId: 6, - scopeId: SCOPES.READ_WRITE_REPORTS, - }, - { - userId: 2046, - regionId: 14, - scopeId: SCOPES.SITE_ACCESS, - }, - ], }; const mockSession = jest.fn(); @@ -63,7 +49,7 @@ describe('File Upload', () => { let report; let fileId; beforeAll(async () => { - user = await User.create(mockUser, { include: [{ model: Permission, as: 'permissions' }] }); + user = await User.create(mockUser); report = await ActivityReport.create(reportObject); process.env.NODE_ENV = 'test'; process.env.BYPASS_AUTH = 'true'; @@ -76,7 +62,7 @@ describe('File Upload', () => { process.env = ORIGINAL_ENV; // restore original env await db.sequelize.close(); }); - afterEach(() => { + beforeEach(() => { jest.clearAllMocks(); }); @@ -90,15 +76,35 @@ describe('File Upload', () => { canUpdate: () => true, })); uploadFile.mockResolvedValue({ key: 'key' }); - await request(app) + const response = await request(app) .post('/api/files') .field('reportId', report.dataValues.id) .attach('file', `${__dirname}/testfiles/testfile.pdf`) - .expect(200) - .then((res) => { - fileId = res.body.id; - expect(uploadFile).toHaveBeenCalled(); - }); + .expect(200); + fileId = response.body.id; + expect(uploadFile).toHaveBeenCalled(); + expect(mockAddToScanQueue).toHaveBeenCalled(); + const file = await File.findOne({ where: { id: fileId } }); + const uuid = file.dataValues.key.slice(0, -4); + expect(file.dataValues.id).toBe(fileId); + expect(file.dataValues.status).not.toBe(null); + expect(file.dataValues.originalFileName).toBe('testfile.pdf'); + expect(file.dataValues.activityReportId).toBe(report.dataValues.id); + expect(validate(uuid)).toBe(true); + }); + it('allows an admin to upload a file', async () => { + ActivityReportPolicy.mockImplementation(() => ({ + canUpdate: () => false, + })); + validateUserAuthForAdmin.mockResolvedValue(true); + uploadFile.mockResolvedValue({ key: 'key' }); + const response = await request(app) + .post('/api/files') + .field('reportId', report.dataValues.id) + .attach('file', `${__dirname}/testfiles/testfile.pdf`) + .expect(200); + fileId = response.body.id; + expect(uploadFile).toHaveBeenCalled(); expect(mockAddToScanQueue).toHaveBeenCalled(); const file = await File.findOne({ where: { id: fileId } }); const uuid = file.dataValues.key.slice(0, -4); @@ -154,10 +160,8 @@ describe('File Upload', () => { await request(app) .post('/api/files') .attach('file', `${__dirname}/testfiles/testfile.pdf`) - .expect(400, { error: 'reportId required' }) - .then(() => { - expect(uploadFile).not.toHaveBeenCalled(); - }); + .expect(400, { error: 'reportId required' }); + await expect(uploadFile).not.toHaveBeenCalled(); }); it('tests a file upload without a file', async () => { ActivityReportPolicy.mockImplementation(() => ({ @@ -172,7 +176,7 @@ describe('File Upload', () => { }); }); it('tests an unauthorized upload', async () => { - jest.clearAllMocks(); + validateUserAuthForAdmin.mockResolvedValue(false); ActivityReportPolicy.mockImplementation(() => ({ canUpdate: () => false, })); diff --git a/src/services/accessValidation.js b/src/services/accessValidation.js index 51611de6fd..689cdf2d1e 100644 --- a/src/services/accessValidation.js +++ b/src/services/accessValidation.js @@ -65,7 +65,12 @@ export async function validateUserAuthForAdmin(userId) { scopeId: ADMIN, }, }); - return userPermission !== null; + if (userPermission !== null) { + logger.info(`User ${userId} successfully checked ADMIN access`); + return true; + } + logger.warn(`User ${userId} unsuccessfully checked ADMIN access`); + return false; } catch (error) { logger.error(`${JSON.stringify({ ...logContext })} - ADMIN Access error - ${error}`); throw error; diff --git a/src/services/accessValidation.test.js b/src/services/accessValidation.test.js index 1a898f2e8f..bd590e65b8 100644 --- a/src/services/accessValidation.test.js +++ b/src/services/accessValidation.test.js @@ -163,7 +163,7 @@ describe('accessValidation', () => { const user = { hsesUserId: '33', email: 'invalid', - hsesUsername: 'username', + hsesUsername: 'user33', homeRegionId: 3, }; await User.destroy({ where: { hsesUserId: '33' } }); diff --git a/src/services/activityReports.js b/src/services/activityReports.js index 4455b1af52..398f6732e5 100644 --- a/src/services/activityReports.js +++ b/src/services/activityReports.js @@ -175,6 +175,18 @@ export function activityReportByLegacyId(legacyId) { where: { legacyId, }, + include: [ + { + model: File, + where: { + status: { + [Op.ne]: 'UPLOAD_FAILED', + }, + }, + as: 'attachments', + required: false, + }, + ], }); } diff --git a/src/services/activityReports.test.js b/src/services/activityReports.test.js index c73d075ed3..ed81da77b2 100644 --- a/src/services/activityReports.test.js +++ b/src/services/activityReports.test.js @@ -23,15 +23,15 @@ const GRANTEE_ID = 16; const mockUser = { id: 1000, homeRegionId: 1, - name: 'user', - hsesUsername: 'user', + name: 'user1000', + hsesUsername: 'user1000', hsesUserId: '1000', }; const mockUserTwo = { id: 1002, homeRegionId: 1, - name: 'a user', + name: 'user1002', hsesUserId: 1002, hsesUsername: 'Rex', }; @@ -129,10 +129,10 @@ describe('Activity Reports DB service', () => { }); it('creates a new report', async () => { - const beginningARCount = await ActivityReport.count(); + const beginningARCount = await ActivityReport.findAll({ where: { userId: mockUser.id } }); const report = await createOrUpdate(reportObject); - const endARCount = await ActivityReport.count(); - expect(endARCount - beginningARCount).toBe(1); + const endARCount = await ActivityReport.findAll({ where: { userId: mockUser.id } }); + expect(endARCount.length - beginningARCount.length).toBe(1); expect(report.activityRecipients[0].grant.id).toBe(RECIPIENT_ID); }); @@ -147,7 +147,7 @@ describe('Activity Reports DB service', () => { collaborators: [{ id: mockUser.id }], }); expect(report.collaborators.length).toBe(1); - expect(report.collaborators[0].name).toBe('user'); + expect(report.collaborators[0].name).toBe('user1000'); }); it('handles notes being created', async () => { @@ -375,7 +375,7 @@ describe('Activity Reports DB service', () => { sortBy: 'author', sortDir: 'asc', offset: 0, limit: 2, }); expect(rows.length).toBe(2); - expect(rows[0].author.name).toBe('a user'); + expect(rows[0].author.name).toBe('user1000'); }); it('retrieves reports sorted by collaborators', async () => { @@ -385,7 +385,7 @@ describe('Activity Reports DB service', () => { sortBy: 'collaborators', sortDir: 'asc', offset: 0, limit: 12, }); expect(rows.length).toBe(6); - expect(rows[0].collaborators[0].name).toBe('user'); + expect(rows[0].collaborators[0].name).toBe('user1000'); }); it('retrieves reports sorted by id', async () => { diff --git a/src/services/legacyreport.test.js b/src/services/legacyreport.test.js new file mode 100644 index 0000000000..6f71c71d99 --- /dev/null +++ b/src/services/legacyreport.test.js @@ -0,0 +1,127 @@ +import reconcileLegacyReports, +{ + reconcileAuthors, + reconcileCollaborators, + reconcileApprovingManagers, +} from './legacyreports'; +import db, { User, ActivityReport, ActivityReportCollaborator } from '../models'; +import { REPORT_STATUSES } from '../constants'; +import { activityReportByLegacyId } from './activityReports'; + +const report1 = { + activityRecipientType: 'grantee', + status: REPORT_STATUSES.DRAFT, + regionId: 1, + ECLKCResourcesUsed: ['test'], + legacyId: 'legacy-1', + imported: { + manager: 'Manager4099@Test.Gov', + createdBy: 'user4096@Test.gov', + otherSpecialists: 'user4097@TEST.gov, user4098@test.gov', + }, +}; + +const report2 = { + activityRecipientType: 'grantee', + status: REPORT_STATUSES.DRAFT, + regionId: 1, + ECLKCResourcesUsed: ['test'], + legacyId: 'legacy-2', + imported: { + manager: 'Manager4099@test.gov', + createdBy: 'user4097@Test.gov', + otherSpecialists: 'user4096@test.gov', + }, +}; + +const user1 = { + id: 4096, + homeRegionId: 1, + name: 'user4096', + hsesUsername: 'user4096', + hsesUserId: '4096', + email: 'user4096@test.gov', +}; + +const user2 = { + id: 4097, + homeRegionId: 1, + name: 'user4097', + hsesUsername: 'user4097', + hsesUserId: '4097', + email: 'user4097@test.gov', +}; + +const user3 = { + id: 4098, + homeRegionId: 1, + name: 'user4098', + hsesUsername: 'user4098', + hsesUserId: '4098', + email: 'user4098@test.gov', +}; + +const manager = { + id: 4099, + homeRegionId: 1, + name: 'manager4099', + hsesUsername: 'manager4099', + hsesUserId: '4099', + email: 'manager4099@test.gov', +}; + +describe('reconcile legacy reports', () => { + let mockReport1; + let mockReport2; + let mockUser1; + let mockUser2; + let mockUser3; + let mockManager; + + beforeAll(async () => { + mockReport1 = await ActivityReport.create(report1); + mockReport2 = await ActivityReport.create(report2); + mockUser1 = await User.create(user1); + mockUser2 = await User.create(user2); + mockUser3 = await User.create(user3); + mockManager = await User.create(manager); + }); + + afterAll(async () => { + await ActivityReportCollaborator.destroy({ + where: { activityReportId: [mockReport1.id, mockReport2.id] }, + }); + await ActivityReport.destroy({ where: { id: [mockReport1.id, mockReport2.id] } }); + await User.destroy({ + where: { id: [mockUser1.id, mockUser2.id, mockUser3.id, mockManager.id] }, + }); + await db.sequelize.close(); + }); + it('adds an author if there is one', async () => { + await reconcileAuthors(mockReport1); + const ret = await activityReportByLegacyId(report1.legacyId); + expect(ret.userId).toBe(mockUser1.id); + }); + it('adds an approvingManager if there is one', async () => { + await reconcileApprovingManagers(mockReport1); + const ret = await activityReportByLegacyId(report1.legacyId); + expect(ret.approvingManagerId).toBe(manager.id); + }); + it('adds collaborators', async () => { + await reconcileCollaborators(mockReport1); + const collaborators = await ActivityReportCollaborator.findAll({ + where: { activityReportId: mockReport1.id }, + }); + expect(collaborators.length).toBe(2); + }); + it('tests the reconciliation process', async () => { + await reconcileLegacyReports(); + const ret = await activityReportByLegacyId(report2.legacyId); + expect(ret.userId).toBe(user2.id); + expect(ret.approvingManagerId).toBe(manager.id); + const collaborators = await ActivityReportCollaborator.findAll({ + where: { activityReportId: ret.id }, + }); + expect(collaborators.length).toBe(1); + }); +}); diff --git a/src/services/legacyreports.js b/src/services/legacyreports.js new file mode 100644 index 0000000000..56a267c97e --- /dev/null +++ b/src/services/legacyreports.js @@ -0,0 +1,148 @@ +import { Op } from 'sequelize'; +import { userByEmail } from './users'; +import { ActivityReport, ActivityReportCollaborator } from '../models'; +import { logger } from '../logger'; + +/* +* Returns all legacy reports that either: +* 1. are missing an author +* 2. are missing an approving manager +* 3. have colloborators in the imported field +* These are the only reports that might need reconciliation +*/ +const getLegacyReports = async () => { + const reports = await ActivityReport.findAll({ + where: { + legacyId: { + [Op.ne]: null, + }, + imported: { + [Op.ne]: null, + }, + [Op.or]: [ + { + userId: { + [Op.eq]: null, + }, + }, + { + approvingManagerId: { + [Op.eq]: null, + }, + }, + { + imported: { + otherSpecialists: { + [Op.ne]: '', + }, + }, + }, + ], + + }, + }); + return reports; +}; + +/* +* Checks a report to see if the email address listed in the imported.manager field +* belongs to any user. If it does, it updates the report with that user.id in the +* approvingManager column +*/ +export const reconcileApprovingManagers = async (report) => { + try { + const user = await userByEmail(report.imported.manager); + if (user) { + await ActivityReport.update({ approvingManagerId: user.id }, { where: { id: report.id } }); + logger.info(`Updated approvingManager for report ${report.displayId} to user Id ${user.id}`); + } + } catch (err) { + logger.error(err); + } +}; +/* +* Checks a report to see if the email address listed in the imported.createdBy field +* belongs to any user. If it does, it updates the report with that user.id in the +* userId column +*/ +export const reconcileAuthors = async (report) => { + try { + const user = await userByEmail(report.imported.createdBy); + if (user) { + await ActivityReport.update({ userId: user.id }, { where: { id: report.id } }); + logger.info(`Updated author for report ${report.displayId} to user Id ${user.id}`); + } + } catch (err) { + logger.error(err); + } +}; + +/* +* First checks if the number of collaborators is different than the number of +* entries in the imported.otherSpecialists field. If not, then no reconciliation is needed. +* If there is a difference, it tries to find users matching the email addresses in the +* otherSpecialists field. It then uses findorCreate to add collaborators that haven't yet +* been added. +*/ +export const reconcileCollaborators = async (report) => { + try { + const collaborators = await ActivityReportCollaborator + .findAll({ where: { activityReportId: report.id } }); + // In legacy reports, specialists are in a single column seperated by commas. + // First, get a list of other specialists and split on commas eliminating any blanks. + const splitOtherSpecialists = report.imported.otherSpecialists.split(',').filter((j) => j !== ''); + // Next we map the other specialists to lower case and trim whitespace to standardize them. + const otherSpecialists = splitOtherSpecialists.map((i) => i.toLowerCase().trim()); + if (otherSpecialists.length !== collaborators.length) { + const users = []; + otherSpecialists.forEach((specialist) => { + users.push(userByEmail(specialist)); + }); + const userArray = await Promise.all(users); + const pendingCollaborators = []; + userArray.forEach((user) => { + if (user) { + pendingCollaborators.push(ActivityReportCollaborator + .findOrCreate({ where: { activityReportId: report.id, userId: user.id } })); + } + }); + const newCollaborators = await Promise.all(pendingCollaborators); + // findOrCreate returns an array with the second value being a boolean + // which is true if a new object is created. This counts the number of objects where + // c[1] is true + const numberOfNewCollaborators = newCollaborators.filter((c) => c[1]).length; + if (numberOfNewCollaborators > 0) { + logger.info(`Added ${numberOfNewCollaborators} collaborator for report ${report.displayId}`); + } + } + } catch (err) { + logger.error(err); + } +}; + +export default async function reconcileLegacyReports() { + // Get all reports that might need reconciliation + const reports = await getLegacyReports(); + // Array to help promises from reports that are getting reconciled + const updates = []; + try { + reports.forEach((report) => { + // if there is no author, try to reconcile the author + if (!report.userId) { + updates.push(reconcileAuthors(report)); + } + // if there is no approving manager, try to reconcile the approving manager + if (!report.approvingManagerId) { + updates.push(reconcileApprovingManagers(report)); + } + // if the report has collaborators, check if collaborators need reconcilliation. + if (report.imported.otherSpecialists !== '') { + updates.push(reconcileCollaborators(report)); + } + }); + // let all promises resolve + await Promise.all(updates); + } catch (err) { + logger.error(err); + } +} diff --git a/src/services/users.js b/src/services/users.js index 3a9d1db2e1..fa97a34968 100644 --- a/src/services/users.js +++ b/src/services/users.js @@ -32,6 +32,15 @@ export async function userById(userId) { }); } +export async function userByEmail(email) { + return User.findOne({ + attributes: ['id'], + where: { + email: { [Op.iLike]: email }, + }, + }); +} + export async function usersWithPermissions(regions, scopes) { return User.findAll({ attributes: ['id', 'name'], diff --git a/src/services/users.test.js b/src/services/users.test.js index 945164ea38..ed5a1f02b5 100644 --- a/src/services/users.test.js +++ b/src/services/users.test.js @@ -3,7 +3,7 @@ import db, { } from '../models'; import { - usersWithPermissions, userById, + usersWithPermissions, userById, userByEmail, } from './users'; import SCOPES from '../middleware/scopeConstants'; @@ -13,22 +13,50 @@ describe('Users DB service', () => { jest.clearAllMocks(); }); - afterAll(() => { - db.sequelize.close(); + afterAll(async () => { + await db.sequelize.close(); }); describe('userById', () => { + beforeEach(async () => { + await User.create({ + id: 54, + name: 'user 54', + hsesUsername: 'user.54', + hsesUserId: '54', + }); + await User.create({ + id: 55, + name: 'user 55', + hsesUsername: 'user.55', + hsesUserId: '55', + }); + }); + + afterEach(async () => { + await User.destroy({ where: { id: [54, 55] } }); + }); + + it('retrieves the correct user', async () => { + const user = await userById(54); + expect(user.id).toBe(54); + expect(user.name).toBe('user 54'); + }); + }); + describe('userByEmail', () => { beforeEach(async () => { await User.create({ id: 50, - name: 'user 1', - hsesUsername: 'user.1', + name: 'user 50', + email: 'user50@test.gov', + hsesUsername: 'user50', hsesUserId: '50', }); await User.create({ id: 51, - name: 'user 2', - hsesUsername: 'user.2', + name: 'user 51', + email: 'user51@test.gov', + hsesUsername: 'user51', hsesUserId: '51', }); }); @@ -38,9 +66,12 @@ describe('Users DB service', () => { }); it('retrieves the correct user', async () => { - const user = await userById(50); + const user = await userByEmail('user50@test.gov'); expect(user.id).toBe(50); - expect(user.name).toBe('user 1'); + }); + it('retrieves the correct user if case differs', async () => { + const user = await userByEmail('User51@Test.Gov'); + expect(user.id).toBe(51); }); }); diff --git a/src/tools/importActivityReports.js b/src/tools/importActivityReports.js index 7efadff326..13d278211d 100644 --- a/src/tools/importActivityReports.js +++ b/src/tools/importActivityReports.js @@ -187,7 +187,8 @@ function coerceInt(value) { function coerceReportId(value, region) { if (!value) return null; // R14 is a test region, and shouldn't be used in actual reportIds - return value.replace(invalidRegionRE, `R${region}`); + const paddedRegion = `${region}`.padStart(2, '0'); + return value.replace(invalidRegionRE, `R${paddedRegion}`); } function parseGrantNumbers(value) { diff --git a/src/tools/reconcileLegacyReports.js b/src/tools/reconcileLegacyReports.js new file mode 100644 index 0000000000..0f651a1d87 --- /dev/null +++ b/src/tools/reconcileLegacyReports.js @@ -0,0 +1,7 @@ +import reconcileLegacyReports from '../services/legacyreports'; +import { auditLogger } from '../logger'; + +reconcileLegacyReports().then(process.exit(0)).catch((e) => { + auditLogger.error(e); + process.exit(1); +}); diff --git a/src/workers/files.js b/src/workers/files.js index 24f56e6e85..30bc405626 100644 --- a/src/workers/files.js +++ b/src/workers/files.js @@ -6,6 +6,8 @@ import { File } from '../models'; import { FILE_STATUSES } from '../constants'; const { + SCANNING, + SCANNING_FAILED, APPROVED, REJECTED, } = FILE_STATUSES; @@ -39,6 +41,7 @@ const updateFileStatus = async (key, fileStatus) => { const processFile = async (key) => { let res; try { + await updateFileStatus(key, SCANNING); const data = await downloadFile(key); const form = new FormData(); form.append('name', key); @@ -47,10 +50,11 @@ const processFile = async (key) => { res = await axios.post(`${process.env.CLAMAV_ENDPOINT}/scan`, form, { httpsAgent: agent, headers: { ...form.getHeaders() } }); await updateFileStatus(key, APPROVED); } catch (error) { - if (error.response.status === 406) { + if (error.response && error.response.status === 406) { await updateFileStatus(key, REJECTED); return { status: error.response.status, data: error.response.data }; } + await updateFileStatus(key, SCANNING_FAILED); throw error; } return ({ status: res.status, data: res.data });