Adding proponent version of csv export (#2209)

* Changes to show all survey results to superusers

* removing hard coded values

* fixing linting

* splitting to seperate end points

* fixing auth check

* fixing linting

* merging method in service

* Handle no data error for graphs

* adding new nodata component

* adding new email for submission response

* fixing linting and testing

* Upgrades to Issue Tracking Table

* removing try catch

* Adding proponent version of csv export

* fixing unit test

met-api/migrations/versions/d9777850eb98_add_proponent_template.py

@@ -0,0 +1,56 @@
+"""add_proponent_template
+
+Revision ID: d9777850eb98
+Revises: 7ebd9ecfccdd
+Create Date: 2023-09-17 12:47:48.475329
+
+"""
+from datetime import datetime
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'd9777850eb98'
+down_revision = '7ebd9ecfccdd'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    document_type = sa.Table(
+        'generated_document_type',
+        sa.MetaData(),
+        sa.Column('id', sa.Integer),
+        sa.Column('name', sa.String),
+        sa.Column('description', sa.String),
+        sa.Column('created_date', sa.DateTime, default=datetime.utcnow)
+    )
+
+    document_template = sa.Table(
+        'generated_document_template',
+        sa.MetaData(),
+        sa.Column('id', sa.Integer),
+        sa.Column('type_id', sa.Integer),
+        sa.Column('hash_code', sa.String),
+        sa.Column('extension', sa.String),
+        sa.Column('created_date', sa.DateTime, default=datetime.utcnow)
+    )
+
+    op.bulk_insert(document_type, [
+        {'id': 3, 'name': 'proponent_comments_sheet', 'description': 'Comments export for proponent'}
+    ])
+
+    op.bulk_insert(document_template, [
+        {'id': 3, 'type_id': 3, 'hash_code': None, "extension": "xlsx"}
+    ])
+    op.execute('UPDATE generated_document_template SET hash_code = null where id = 1')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute('DELETE FROM document_type WHERE id = 3')
+    op.execute('DELETE FROM document_template WHERE id = 3')
+    # ### end Alembic commands ###

met-api/src/met_api/models/comment.py

@@ -219,7 +219,7 @@ def update(cls, submission_id, comment: CommentSchema, session=None) -> Comment:
 
     @classmethod
     def get_comments_by_survey_id(cls, survey_id):
-        """Get comments paginated."""
+        """Get comments for staff."""
         null_value = None
         query = db.session.query(Submission)\
             .join(Comment, Submission.id == Comment.submission_id)\
@@ -229,3 +229,23 @@ def get_comments_by_survey_id(cls, survey_id):
         query = query.order_by(Submission.id.asc())
         items = query.all()
         return SubmissionSchema(many=True, exclude=['submission_json']).dump(items)
+
+    @classmethod
+    def get_public_viewable_comments_by_survey_id(cls, survey_id):
+        """Get comments that are viewable on the public report."""
+        query = db.session.query(Comment)\
+            .join(Submission, Submission.id == Comment.submission_id)\
+            .join(CommentStatusModel, Submission.comment_status_id == CommentStatusModel.id)\
+            .join(Survey, Survey.id == Submission.survey_id)\
+            .join(ReportSetting, and_(Comment.survey_id == ReportSetting.survey_id,
+                                      Comment.component_id == ReportSetting.question_key))\
+            .filter(
+                and_(
+                    Comment.survey_id == survey_id,
+                    CommentStatusModel.id == CommentStatus.Approved.value,
+                    ReportSetting.display == true(),
+                    Submission.reviewed_by != 'System'
+                ))
+        query = query.order_by(Comment.text.asc())
+        items = query.all()
+        return CommentSchema(many=True, only=['submission_id', 'label', 'text']).dump(items)

met-api/src/met_api/resources/comment.py

@@ -62,8 +62,8 @@ def get(survey_id):
 
 
 @cors_preflight('GET, OPTIONS')
-@API.route('/survey/<survey_id>/sheet')
-class GeneratedCommentsSheet(Resource):
+@API.route('/survey/<survey_id>/sheet/staff')
+class GeneratedStaffCommentsSheet(Resource):
     """Resource for managing multiple comments."""
 
     @staticmethod
@@ -73,7 +73,34 @@ def get(survey_id):
         """Export comments."""
         try:
 
-            response = CommentService().export_comments_to_spread_sheet(survey_id)
+            response = CommentService().export_comments_to_spread_sheet_staff(survey_id)
+            response_headers = dict(response.headers)
+            headers = {
+                'content-type': response_headers.get('content-type'),
+                'content-disposition': response_headers.get('content-disposition'),
+            }
+            return Response(
+                response=response.content,
+                status=response.status_code,
+                headers=headers
+            )
+        except ValueError as err:
+            return str(err), HTTPStatus.INTERNAL_SERVER_ERROR
+
+
+@cors_preflight('GET, OPTIONS')
+@API.route('/survey/<survey_id>/sheet/proponent')
+class GeneratedProponentCommentsSheet(Resource):
+    """Resource for managing multiple comments."""
+
+    @staticmethod
+    @cross_origin(origins=allowedorigins())
+    @_jwt.requires_auth
+    def get(survey_id):
+        """Export comments."""
+        try:
+
+            response = CommentService().export_comments_to_spread_sheet_proponent(survey_id)
             response_headers = dict(response.headers)
             headers = {
                 'content-type': response_headers.get('content-type'),

met-api/src/met_api/services/comment_service.py

@@ -157,7 +157,7 @@ def extract_comments_from_survey(cls, survey_submission: SubmissionSchema, surve
         return comments
 
     @classmethod
-    def export_comments_to_spread_sheet(cls, survey_id):
+    def export_comments_to_spread_sheet_staff(cls, survey_id):
         """Export comments to spread sheet."""
         engagement = SurveyModel.find_by_id(survey_id)
         one_of_roles = (
@@ -177,7 +177,7 @@ def export_comments_to_spread_sheet(cls, survey_id):
             'comments': data_rows
         }
         document_options = {
-            'document_type': GeneratedDocumentTypes.COMMENT_SHEET.value,
+            'document_type': GeneratedDocumentTypes.COMMENT_SHEET_STAFF.value,
             'template_name': 'staff_comments_sheet.xlsx',
             'convert_to': 'csv',
             'report_name': 'comments_sheet'
@@ -239,3 +239,61 @@ def get_rejection_note(cls, comment):
             rejection_note.append(comment.get('rejected_reason_other'))
 
         return ', '.join(rejection_note)
+
+    @classmethod
+    def export_comments_to_spread_sheet_proponent(cls, survey_id):
+        """Export comments to spread sheet."""
+        engagement = SurveyModel.find_by_id(survey_id)
+        one_of_roles = (
+            MembershipType.TEAM_MEMBER.name,
+            Role.EXPORT_TO_CSV.value
+        )
+        authorization.check_auth(one_of_roles=one_of_roles, engagement_id=engagement.engagement_id)
+        comments = Comment.get_public_viewable_comments_by_survey_id(survey_id)
+        formatted_comments = cls.format_comments(comments)
+        document_options = {
+            'document_type': GeneratedDocumentTypes.COMMENT_SHEET_PROPONENT.value,
+            'template_name': 'proponent_comments_sheet.xlsx',
+            'convert_to': 'csv',
+            'report_name': 'proponent_comments_sheet'
+        }
+        return DocumentGenerationService().generate_document(data=formatted_comments, options=document_options)
+
+    @classmethod
+    def format_comments(cls, comments):
+        """Group the comments together, arranging them in the same order as the titles."""
+        grouped_comments = []
+        titles = []
+        for comment in comments:
+            submission_id = comment['submission_id']
+            text = comment.get('text', '')  # Get the text, or an empty string if it's missing
+            label = comment['label']
+
+            # Check if a group with the same submission ID already exists
+            existing_group = next((group for group in grouped_comments
+                                   if group['submission_id'] == submission_id), None)
+
+            if existing_group:
+                # Add the new comment
+                existing_group['commentText'].append({'text': text, 'label': label})
+            else:
+                new_group = {'submission_id': submission_id, 'commentText': [{'text': text, 'label': label}]}
+                grouped_comments.append(new_group)
+
+                # Add unique labels to titles list in order of appearance
+                if label not in [title['label'] for title in titles]:
+                    titles.append({'label': label})
+
+        # Sort commentText within each group based on the order of titles
+        for group in grouped_comments:
+            sorted_comment_text = []
+            for title in titles:
+                label = title['label']
+                matching_comments = [comment for comment in group['commentText'] if comment['label'] == label]
+                if not matching_comments:
+                    sorted_comment_text.append({'text': '', 'label': label})  # Add empty text for missing labels
+                else:
+                    sorted_comment_text.extend(matching_comments)
+            group['commentText'] = sorted_comment_text
+
+        return {'titles': titles, 'comments': grouped_comments}

met-api/src/met_api/utils/enums.py

@@ -48,8 +48,9 @@ class MembershipStatus(Enum):
 class GeneratedDocumentTypes(IntEnum):
     """Document Types."""
 
-    COMMENT_SHEET = 1
+    COMMENT_SHEET_STAFF = 1
     CAC_FORM_SHEET = 2
+    COMMENT_SHEET_PROPONENT = 3
 
 
 class LoginSource(Enum):

met-api/src/met_api/utils/notification.py

@@ -11,10 +11,15 @@
 
 def get_tenant_site_url(tenant_id, path=''):
     """Get the tenant specific site url (domain / tenant / path)."""
-    if tenant_id is None:
-        raise ValueError('Missing tenant id.')
-    tenant: Tenant = Tenant.find_by_id(tenant_id)
-    return current_app.config.get('SITE_URL', '') + f'/{tenant.short_name}' + path
+    is_single_tenant_environment = current_app.config.get('IS_SINGLE_TENANT_ENVIRONMENT', False)
+    site_url = current_app.config.get('SITE_URL', '')
+    if not is_single_tenant_environment:
+        if tenant_id is None:
+            raise ValueError('Missing tenant id.')
+        tenant: Tenant = Tenant.find_by_id(tenant_id)
+        return site_url + f'/{tenant.short_name}' + path
+    else:
+        return site_url + path
 
 
 def send_email(subject, email, html_body, args, template_id):

met-api/tests/unit/api/test_comment.py

@@ -207,7 +207,8 @@ def test_get_comments_spreadsheet(mocker, client, jwt, session):  # pylint:disab
     submission = factory_submission_model(survey.id, eng.id, participant.id)
     factory_comment_model(survey.id, submission.id)
     headers = factory_auth_header(jwt=jwt, claims=claims)
-    rv = client.get(f'/api/comments/survey/{survey.id}/sheet', headers=headers, content_type=ContentType.JSON.value)
+    rv = client.get(f'/api/comments/survey/{survey.id}/sheet/staff',
+                    headers=headers, content_type=ContentType.JSON.value)
     assert rv.status_code == 200
     mock_post_generate_document.assert_called()
     mock_get_access_token.assert_called()

met-web/src/apiManager/endpoints/index.ts

@@ -72,7 +72,8 @@ const Endpoints = {
     },
     Comment: {
         GET_LIST: `${AppConfig.apiUrl}/comments/survey/survey_id`,
-        GET_SPREAD_SHEET: `${AppConfig.apiUrl}/comments/survey/survey_id/sheet`,
+        GET_STAFF_SPREAD_SHEET: `${AppConfig.apiUrl}/comments/survey/survey_id/sheet/staff`,
+        GET_PROPONENT_SPREAD_SHEET: `${AppConfig.apiUrl}/comments/survey/survey_id/sheet/proponent`,
     },
     Feedback: {
         GET_LIST: `${AppConfig.apiUrl}/feedbacks/`,

met-web/src/components/comments/admin/reviewListing/Submissions.tsx

@@ -4,22 +4,19 @@ import Grid from '@mui/material/Grid';
 import { Link, useLocation } from 'react-router-dom';
 import { MetPageGridContainer, PrimaryButton, MetHeader1, SecondaryButton } from 'components/common';
 import { HeadCell, PaginationOptions } from 'components/common/Table/types';
-import { formatDate, formatToUTC } from 'components/common/dateHelper';
+import { formatDate } from 'components/common/dateHelper';
 import { Collapse, Link as MuiLink } from '@mui/material';
 import TextField from '@mui/material/TextField';
 import SearchIcon from '@mui/icons-material/Search';
 import Stack from '@mui/material/Stack';
 import { SurveySubmission } from 'models/surveySubmission';
 import { COMMENTS_STATUS, CommentStatus } from 'constants/commentStatus';
-import { getCommentsSheet } from 'services/commentService';
-import { downloadFile } from 'utils';
 import { AdvancedSearch } from './AdvancedSearch';
 import { CommentListingContext } from './CommentListingContext';
 import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
 import { useAppSelector } from 'hooks';
 import { USER_ROLES } from 'services/userService/constants';
 import { USER_GROUP } from 'models/user';
-import { PermissionsGate } from 'components/permissionsGate';
 
 const Submissions = () => {
     const {
@@ -36,7 +33,6 @@ const Submissions = () => {
     } = useContext(CommentListingContext);
     const { roles, userDetail, assignedEngagements } = useAppSelector((state) => state.user);
     const { state } = useLocation();
-    const [isExporting, setIsExporting] = useState(false);
     const [isAdvancedSearchOpen, setIsAdvancedSearchOpen] = useState(Boolean(state));
 
     const handleSearchBarClick = (filter: string) => {
@@ -46,13 +42,6 @@ const Submissions = () => {
         });
     };
 
-    const handleExportComments = async () => {
-        setIsExporting(true);
-        const response = await getCommentsSheet({ survey_id: survey.id });
-        downloadFile(response, `INTERNAL ONLY - ${survey.engagement?.name || ''} - ${formatToUTC(Date())}.csv`);
-        setIsExporting(false);
-    };
-
     const headCells: HeadCell<SurveySubmission>[] = [
         {
             key: 'id',
@@ -157,20 +146,6 @@ const Submissions = () => {
                         <PrimaryButton component={Link} to={`/surveys/${survey.id}/comments/all`}>
                             Read All Comments
                         </PrimaryButton>
-                        <PermissionsGate scopes={[USER_ROLES.EXPORT_TO_CSV]} errorProps={{ disabled: true }}>
-                            <SecondaryButton
-                                onClick={handleExportComments}
-                                loading={isExporting}
-                                sx={{
-                                    '&.Mui-disabled': {
-                                        background: '#e0e0e0',
-                                        color: '#a6a6a6',
-                                    },
-                                }}
-                            >
-                                Export to CSV
-                            </SecondaryButton>
-                        </PermissionsGate>
                     </Stack>
                 </Stack>
             </Grid>