Update Submissions OData to include deletedAt

lib/data/odata-filter.js

@@ -87,6 +87,50 @@ const odataFilter = (expr, odataToColumnMap) => {
  return op(ast);
 };
 
+// Returns sql expression to exclude deleted records if provided OData filter
+// expression doesn't use `__system/deletedAt` field. Logic of this function
+// can easily be merged with `OdataFilter()` but it is kept separate for the
+// for the sake of simplicity and for separation of concerns.
+const odataExcludeDeleted = (expr, odataToColumnMap) => {
+ const deleteAtColumn = odataToColumnMap.get('__system/deletedAt')
+ ?? odataToColumnMap.get('$root/Submissions/__system/deletedAt');
+
+ const filterOutDeletedRecordsExp = sql`(${sql.identifier(deleteAtColumn.split('.'))} is null)`;
+
+ if (expr == null) return filterOutDeletedRecordsExp;
+
+ let ast; // still hate this.
+ try { ast = odataParser.filter(expr); } // eslint-disable-line brace-style
+ catch (ex) { throw Problem.user.unparseableODataExpression({ reason: ex.message }); }
+
+ const hasDeletedAtClause = (node) => {
+ if (node.type === 'FirstMemberExpression' || node.type === 'RootExpression') {
+ if (node.raw === '__system/deletedAt'
+ || node.raw === '$root/Submissions/__system/deletedAt') return true;
+ } else if (node.type === 'MethodCallExpression') {
+ // n.b. we only support Unary (single-arity functions)
+ return hasDeletedAtClause(node.value.parameters[0]);
+ } else if (node.type === 'EqualsExpression'
+ || node.type === 'NotEqualsExpression'
+ || node.type === 'LesserThanExpression'
+ || node.type === 'LesserOrEqualsExpression'
+ || node.type === 'GreaterThanExpression'
+ || node.type === 'GreaterOrEqualsExpression'
+ || node.type === 'AndExpression'
+ || node.type === 'OrExpression') {
+ return hasDeletedAtClause(node.value.left) || hasDeletedAtClause(node.value.right);
+ } else if (node.type === 'BoolParenExpression'
+ || node.type === 'NotExpression') {
+ return hasDeletedAtClause(node.value);
+ }
+ return false;
+ };
+
+ if (!hasDeletedAtClause(ast)) return filterOutDeletedRecordsExp;
+
+ return sql`true`;
+};
+
 const odataOrderBy = (expr, odataToColumnMap, stableOrderColumn = null) => {
  let initialOrder = null;
  const clauses = expr.split(',').map((exp) => {
@@ -115,5 +159,4 @@ const odataOrderBy = (expr, odataToColumnMap, stableOrderColumn = null) => {
  return sql`ORDER BY ${sql.join(clauses, sql`,`)}`;
 };
 
-module.exports = { odataFilter, odataOrderBy };
-
+module.exports = { odataFilter, odataOrderBy, odataExcludeDeleted };

lib/data/odata.js

@@ -150,7 +150,8 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi
  deviceId: submission.deviceId || null,
  // because of how we compute this value we don't need to default it to 0:
  edits: submission.aux.edit.count,
- formVersion: submission.aux.exports.formVersion
+ formVersion: submission.aux.exports.formVersion,
+ deletedAt: submission.deletedAt
  };
 
  if (!options.metadata || options.metadata.__system) {
@@ -363,6 +364,7 @@ const systemFields = new Set([
  '__system/deviceId',
  '__system/edits',
  '__system/formVersion',
+ '__system/deletedAt'
 ]);
 
 module.exports = { submissionToOData, systemFields };

lib/data/submission.js

@@ -342,7 +342,8 @@ const filterableFields = [
  ['__system/submissionDate', 'submissions.createdAt'],
  ['__system/updatedAt', 'submissions.updatedAt'],
  ['__system/submitterId', 'submissions.submitterId'],
- ['__system/reviewState', 'submissions.reviewState']
+ ['__system/reviewState', 'submissions.reviewState'],
+ ['__system/deletedAt', 'submissions.deletedAt']
 ];
 
 const odataToColumnMap = new Map(filterableFields.concat(filterableFields.map(f => ([`$root/Submissions/${f[0]}`, f[1]]))));

lib/formats/odata.js

@@ -126,6 +126,7 @@ const edmxTemplater = template(`<?xml version="1.0" encoding="UTF-8"?>
  <ComplexType Name="metadata">
  <Property Name="submissionDate" Type="Edm.DateTimeOffset"/>
  <Property Name="updatedAt" Type="Edm.DateTimeOffset"/>
+ <Property Name="deletedAt" Type="Edm.DateTimeOffset"/>
  <Property Name="submitterId" Type="Edm.String"/>
  <Property Name="submitterName" Type="Edm.String"/>
  <Property Name="attachmentsPresent" Type="Edm.Int64"/>

lib/model/frames/submission.js

@@ -25,6 +25,7 @@ class Submission extends Frame.define(
  'deviceId', readable, 'createdAt', readable,
  'updatedAt', readable, 'reviewState', readable, writable,
  'userAgent', readable, 'draft',
+ 'deletedAt', readable,
  embedded('submitter'),
  embedded('currentVersion')
 ) {

lib/model/query/submissions.js

@@ -12,7 +12,7 @@ const { map } = require('ramda');
 const { sql } = require('slonik');
 const { Frame, table } = require('../frame');
 const { Actor, Form, Submission } = require('../frames');
-const { odataFilter, odataOrderBy } = require('../../data/odata-filter');
+const { odataFilter, odataOrderBy, odataExcludeDeleted } = require('../../data/odata-filter');
 const { odataToColumnMap, odataSubTableToColumnMap } = require('../../data/submission');
 const { unjoiner, extender, equals, page, updater, QueryOptions, insertMany, markDeleted, markUndeleted } = require('../../util/db');
 const { blankStringToNull, construct } = require('../../util/util');
@@ -231,7 +231,7 @@ const joinOnSkiptoken = (skiptoken, formId, draft) => {
 
 const countByFormId = (formId, draft, options = QueryOptions.none) => ({ one }) => {
  const filter = sql`${equals({ formId, draft })} AND
-"deletedAt" IS NULL AND
+${odataExcludeDeleted(options.filter, odataToColumnMap)} AND
 ${odataFilter(options.filter, odataToColumnMap)}`;
  return one(sql`
 SELECT * FROM
@@ -382,7 +382,7 @@ where
  ${encrypted ? sql`(form_defs."encKeyId" is null or form_defs."encKeyId" in (${sql.join(keyIds, sql`,`)})) and` : sql``}
  ${odataFilter(options.filter, options.isSubmissionsTable ? odataToColumnMap : odataSubTableToColumnMap)} and
  ${equals(options.condition)}
- and submission_defs.current=true and submissions."formId"=${formId} and submissions."deletedAt" is null
+ and submission_defs.current=true and submissions."formId"=${formId} and ${odataExcludeDeleted(options.filter, options.isSubmissionsTable ? odataToColumnMap : odataSubTableToColumnMap)}
 ${options.orderby ? sql`
  ${odataOrderBy(options.orderby, odataToColumnMap, 'submissions.id')}`
  : sql`order by submissions."createdAt" desc, submissions.id desc`}