tweak(datatrakWeb): RN-1400: save task due date in unix time

packages/admin-panel/src/VizBuilderApp/components/PreviewOptions/PreviewOptions.jsx

@@ -6,13 +6,14 @@ import React, { useState } from 'react';
 import PropTypes from 'prop-types';
 import styled from 'styled-components';
 import InsertDriveFileIcon from '@material-ui/icons/InsertDriveFile';
-import { FlexEnd, FlexSpaceBetween, FlexStart, ImportModal } from '@tupaia/ui-components';
+import { FlexEnd, FlexSpaceBetween, FlexStart } from '@tupaia/ui-components';
 import { usePreviewDataContext, useVizConfigContext } from '../../context';
 import { LinkButton } from '../LinkButton';
 import { useUploadTestData } from '../../api';
 import { ProjectField } from './ProjectField';
 import { LocationField } from './LocationField';
 import { DateRangeField } from './DateRangeField';
+import { ImportModal } from '../../../importExport';
 
 const Container = styled(FlexSpaceBetween)`
   padding: 24px 0;

packages/central-server/src/tests/apiV2/taskComments/CreateTaskComment.test.js

@@ -84,7 +84,7 @@ describe('Permissions checker for CreateTaskComment', async () => {
     };
     await findOrCreateDummyRecord(models.user, assignee);
 
-    const dueDate = new Date('2021-12-31');
+    const dueDate = new Date('2021-12-31').getTime();
 
     tasks = [
       {

packages/central-server/src/tests/apiV2/taskComments/GETTaskComments.test.js

@@ -62,7 +62,7 @@ describe('Permissions checker for GETTaskComments', async () => {
     };
     await findOrCreateDummyRecord(models.user, user);
 
-    const dueDate = new Date('2021-12-31');
+    const dueDate = new Date('2021-12-31').getTime();
 
     const task = {
       id: generateId(),

packages/central-server/src/tests/apiV2/tasks/CreateTask.test.js

@@ -45,10 +45,12 @@ describe('Permissions checker for CreateTask', async () => {
     last_name: 'Pan',
   };
 
+  const dueDate = new Date('2021-12-31').getTime();
+
   const BASE_TASK = {
     assignee_id: assignee.id,
     repeat_schedule: null,
-    due_date: new Date('2021-12-31'),
+    due_date: dueDate,
     status: 'to_do',
   };
 

packages/central-server/src/tests/apiV2/tasks/EditTask.test.js

@@ -55,7 +55,7 @@ describe('Permissions checker for EditTask', async () => {
     last_name: 'Pan',
   };
 
-  const dueDate = new Date('2021-12-31');
+  const dueDate = new Date('2021-12-31').getTime();
 
   let tasks;
 
@@ -218,7 +218,7 @@ describe('Permissions checker for EditTask', async () => {
         });
         await app.put(`tasks/${tasks[1].id}`, {
           body: {
-            due_date: new Date('2021-11-30').toISOString(),
+            due_date: new Date('2021-11-30').getTime(),
           },
         });
 

packages/central-server/src/tests/apiV2/tasks/GETTasks.test.js

@@ -87,7 +87,7 @@ describe('Permissions checker for GETTasks', async () => {
     };
     await findOrCreateDummyRecord(models.user, assignee);
 
-    const dueDate = new Date('2021-12-31');
+    const dueDate = new Date('2021-12-31').getTime();
 
     tasks = [
       {

packages/database/package.json

@@ -38,6 +38,7 @@
     "@tupaia/tsutils": "workspace:*",
     "@tupaia/utils": "workspace:*",
     "date-fns": "^2.29.2",
+    "date-fns-tz": "^2.0.1",
     "db-migrate": "^0.11.5",
     "db-migrate-pg": "^1.2.2",
     "dotenv": "^16.4.5",

packages/database/src/TupaiaDatabase.js

@@ -70,6 +70,8 @@ const COMPARATORS = {
  */
 const supportedFunctions = ['ST_AsGeoJSON', 'COALESCE'];
 
+const RAW_INPUT_PATTERN = /(^CASE)|(^to_timestamp)/;
+
 // no math here, just hand-tuned to be as low as possible while
 // keeping all the tests passing
 const HANDLER_DEBOUNCE_DURATION = 250;
@@ -561,9 +563,8 @@ function buildQuery(connection, queryConfig, where = {}, options = {}) {
         }
       }
 
-      // Special case to handle CASE statements, otherwise they get interpreted as column names
-      const CASE_PATTERN = /^CASE/;
-      if (CASE_PATTERN.test(selector)) {
+      // Special case to handle raw input statements, otherwise they get interpreted as column names
+      if (RAW_INPUT_PATTERN.test(selector)) {
         return { [alias]: connection.raw(selector) };
       }
 
@@ -741,9 +742,9 @@ function getColSelector(connection, inputColStr) {
 
     return connection.raw(`COALESCE(${identifiers})`, bindings);
   }
-  const casePattern = /^CASE/;
 
-  if (casePattern.test(inputColStr)) {
+  // Special handling of raw input statements
+  if (RAW_INPUT_PATTERN.test(inputColStr)) {
     return connection.raw(inputColStr);
   }
 

packages/database/src/__tests__/changeHandlers/TaskAssigneeEmailer.test.js

@@ -36,7 +36,7 @@ const TASK = {
   status: 'to_do',
   repeat_schedule: null,
   assignee_id: null,
-  due_date: new Date('2021-01-01 00:00:00'),
+  due_date: new Date('2021-01-01 00:00:00').getTime(),
   id: generateId(),
 };
 

packages/database/src/__tests__/changeHandlers/TaskCompletionHandler.test.js

@@ -72,7 +72,7 @@ describe('TaskCompletionHandler', () => {
       survey_id: SURVEY.id,
       created_at: '2024-07-08',
       status: 'to_do',
-      due_date: '2024-07-25',
+      due_date: new Date('2024-07-20').getTime(),
       survey_response_id: null,
     });
     await upsertDummyRecord(models.user, { id: userId });

packages/database/src/__tests__/changeHandlers/TaskCreationHandler.test.js

@@ -81,6 +81,7 @@ const buildSurveyResponse = async (models, surveyCode, answers) => {
     surveyCode,
     answers,
     id: generateId(),
+    timezone: 'Pacific/Auckland',
   };
 
   const surveyResponses = await buildAndInsertSurveyResponses(models, [surveyResponse]);
@@ -103,13 +104,15 @@ const TEST_DATA = [
       answers: {
         TEST_CODE_00: entityId,
         TEST_CODE_01: 'Yes',
-        TEST_CODE_02: '2024/06/06 00:00:00+00',
+        // answers come in iso format
+        TEST_CODE_02: new Date('2024-06-06 00:00:00+12:00').toISOString(),
         TEST_CODE_03: userId,
       },
     },
     {
       survey_id: taskSurveyId,
-      due_date: '2024-06-06 00:00:00',
+      // due date will be set to the last second of the day in the survey response timezone and converted to a timestamp
+      due_date: new Date('2024-06-06 23:59:59+12:00').getTime(),
       assignee_id: userId,
       entity_id: entityId,
     },

packages/database/src/__tests__/modelClasses/Task.test.js

@@ -100,27 +100,29 @@ describe('TaskModel', () => {
 
     await Promise.all(SURVEYS.map(survey => upsertDummyRecord(models.survey, survey)));
 
+    const dueDate = new Date('2020-01-01 00:00:00+00').getTime();
+
     const TASKS = [
       {
         id: generateId(),
         survey_id: SURVEYS[0].id,
         status: 'to_do',
         entity_id: facilities[0].id,
-        due_date: new Date('2020-01-01'),
+        due_date: dueDate,
       },
       {
         id: generateId(),
         survey_id: SURVEYS[1].id,
         status: 'to_do',
         entity_id: facilities[1].id,
-        due_date: new Date('2020-01-01'),
+        due_date: dueDate,
       },
       {
         id: generateId(),
         survey_id: SURVEYS[0].id,
         status: 'to_do',
         entity_id: facilities[1].id,
-        due_date: new Date('2020-01-01'),
+        due_date: dueDate,
       },
     ];
 

packages/database/src/changeHandlers/TaskCreationHandler.js

@@ -3,6 +3,7 @@
  *  Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd
  */
 import keyBy from 'lodash.keyby';
+import { formatInTimeZone } from 'date-fns-tz';
 import { ChangeHandler } from './ChangeHandler';
 
 const getAnswerWrapper = (config, answers) => {
@@ -71,6 +72,7 @@ export class TaskCreationHandler extends ChangeHandler {
 
     for (const response of changedResponses) {
       const sr = await models.surveyResponse.findById(response.id);
+      const { timezone } = sr;
       const questions = await getQuestions(models, response.survey_id);
 
       const taskQuestions = questions.filter(question => question.type === 'Task');
@@ -84,7 +86,7 @@ export class TaskCreationHandler extends ChangeHandler {
       for (const taskQuestion of taskQuestions) {
         const config = taskQuestion.config.task;
         const getAnswer = getAnswerWrapper(config, answers);
-        
+
         if (
           !config ||
           getAnswer('shouldCreateTask') === null ||
@@ -100,12 +102,29 @@ export class TaskCreationHandler extends ChangeHandler {
           : getAnswer('entityId');
         const surveyId = await getSurveyId(models, config);
 
+        const dueDateAnswer = getAnswer('dueDate');
+
+        let dueDate = null;
+        if (dueDateAnswer) {
+          // Convert the due date to the timezone of the survey response and set the time to the last second of the day
+          const dateInTimezone = formatInTimeZone(
+            dueDateAnswer,
+            timezone,
+            "yyyy-MM-dd'T23:59:59'XXX",
+          );
+
+          // Convert the date to a timestamp
+          const timestamp = new Date(dateInTimezone).getTime();
+
+          dueDate = timestamp;
+        }
+
         await models.task.create({
           initial_request_id: response.id,
           survey_id: surveyId,
           entity_id: entityId,
           assignee_id: getAnswer('assignee'),
-          due_date: getAnswer('dueDate'),
+          due_date: dueDate,
           status: 'to_do',
         });
       }

packages/database/src/migrations/20240809001011-ChangeTaskDueDateToUnix-modifies-schema.js

@@ -0,0 +1,51 @@
+'use strict';
+
+var dbm;
+var type;
+var seed;
+
+/**
+ * We receive the dbmigrate dependency from dbmigrate initially.
+ * This enables us to not have to rely on NODE_PATH.
+ */
+exports.setup = function (options, seedLink) {
+  dbm = options.dbmigrate;
+  type = dbm.dataType;
+  seed = seedLink;
+};
+
+exports.up = async function (db) {
+  return db.runSql(`
+    ALTER TABLE task
+    RENAME COLUMN due_date TO old_due_date;
+
+    ALTER TABLE task
+    ADD COLUMN due_date DOUBLE PRECISION;
+
+    UPDATE task
+    SET due_date = (EXTRACT(EPOCH FROM old_due_date) * 1000)::DOUBLE PRECISION;
+
+    ALTER TABLE task
+    DROP COLUMN old_due_date;
+  `);
+};
+
+exports.down = function (db) {
+  return db.runSql(`
+    ALTER TABLE task
+    RENAME COLUMN due_date TO old_due_date;
+
+    ALTER TABLE task
+    ADD COLUMN due_date TIMESTAMP WITH TIME ZONE;
+
+    UPDATE task
+    SET due_date = to_timestamp(CAST(old_due_date AS DOUBLE PRECISION) / 1000);
+
+    ALTER TABLE task
+    DROP COLUMN old_due_date;
+  `);
+};
+
+exports._meta = {
+  version: 1,
+};

packages/database/src/modelClasses/Task.js

@@ -341,6 +341,7 @@ export class TaskModel extends DatabaseModel {
   }
 
   customColumnSelectors = {
+    task_due_date: () => `to_timestamp(due_date/1000)`,
     task_status: () =>
       `CASE  
         WHEN status = 'cancelled' then 'cancelled'
@@ -349,7 +350,7 @@ export class TaskModel extends DatabaseModel {
             CASE 
                 WHEN repeat_schedule IS NOT NULL THEN 'repeating'
                 WHEN due_date IS NULL THEN 'to_do'
-                WHEN due_date < '${format(new Date(), 'yyyy-MM-dd')}' THEN 'overdue'
+                WHEN due_date < ${new Date().getTime()} THEN 'overdue'
                 ELSE 'to_do'
             END
         ELSE 'to_do' 

packages/datatrak-web-server/src/__tests__/TasksRoute.test.ts

@@ -164,6 +164,33 @@ describe('TaskRoute', () => {
           },
         },
       ],
+      [
+        'Due date filter is between start and end of day',
+        {
+          headers: {
+            cookie: 'show_completed_tasks=true;show_cancelled_tasks=true;all_assignees_tasks=true',
+          },
+          query: {
+            filters: [
+              {
+                id: 'due_date',
+                value: '2021-01-01 23:59:59.000+12:00',
+              },
+            ],
+          },
+        },
+        {
+          filter: {
+            due_date: {
+              comparator: 'BETWEEN',
+              comparisonValue: [
+                new Date('2021-01-01T00:00:00.000+12:00').getTime(),
+                new Date('2021-01-01T23:59:59.000+12:00').getTime(),
+              ],
+            },
+          },
+        },
+      ],
     ];
 
     // @ts-ignore

packages/datatrak-web-server/src/routes/TaskRoute.ts

@@ -26,7 +26,7 @@ const FIELDS = [
   'assignee_name',
   'assignee_id',
   'task_status',
-  'due_date',
+  'task_due_date',
   'repeat_schedule',
   'survey_id',
   'entity_id',