Feature/threshold calc

modules/server/.env.schema

@@ -1,8 +1,10 @@
 ALLOW_CUSTOM_MAX_DOWNLOAD_ROWS=false
 CONFIG_PATH=./configs
+DATA_MASK_THRESHOLD=
 DEBUG=false
 DOCUMENT_TYPE=''
 DOWNLOAD_STREAM_BUFFER_SIZE=2000
+ENABLE_DOCUMENT_HITS=true
 ENABLE_LOGS=false
 ES_HOST=http://localhost:9200
 ES_INDEX=''
@@ -15,4 +17,3 @@ MAX_RESULTS_WINDOW=10000
 PING_MS=2200
 PING_PATH=/ping
 PORT=5050
-ENABLE_DOCUMENT_HITS=true

modules/server/src/config/constants.ts

@@ -4,6 +4,7 @@ export const ALLOW_CUSTOM_MAX_DOWNLOAD_ROWS = stringToBool(
 	process.env.ALLOW_CUSTOM_MAX_DOWNLOAD_ROWS,
 );
 export const CONFIG_FILES_PATH = process.env.CONFIG_PATH || './configs';
+export const DATA_MASK_THRESHOLD = process.env.DATA_MASK_THRESHOLD || Number.MAX_SAFE_INTEGER;
 export const DEBUG_MODE = stringToBool(process.env.DEBUG);
 export const DOCUMENT_TYPE = process.env.DOCUMENT_TYPE || '';
 export const DOWNLOAD_STREAM_BUFFER_SIZE =

modules/server/src/mapping/createConnectionResolvers.ts

@@ -1,16 +1,15 @@
 import { IResolvers } from '@graphql-tools/utils';
 
-import { ConfigProperties, ExtendedConfigsInterface } from '@/config/types';
 import { GetServerSideFilterFn } from '@/utils/getDefaultServerSideFilter';
 
-import resolveAggregations from './resolveAggregations';
-import resolveHits from './resolveHits';
+import { createResolvers } from './resolvers';
 
 // TODO: tighten these types
-type CreateConnectionResolversArgs = {
+export type CreateConnectionResolversArgs = {
 	createStateResolvers?: boolean;
 	enableAdmin: boolean;
 	enableDocumentHits: boolean;
+	dataMaskThreshold: number;
 	getServerSideFilter?: GetServerSideFilterFn;
 	Parallel: any;
 	type: Record<string, any>;
@@ -21,38 +20,36 @@ const createConnectionResolvers: CreateConnectionResolversFn = ({
 	createStateResolvers = true,
 	enableAdmin,
 	enableDocumentHits,
+	dataMaskThreshold,
 	getServerSideFilter,
 	Parallel,
 	type,
-}) => ({
-	[type.name]: {
-		aggregations: resolveAggregations({ type, getServerSideFilter }),
-		configs: async (parentObj, { fieldNames }: { fieldNames: string[] }) => {
-			return {
-				downloads: type.config?.[ConfigProperties.DOWNLOADS],
-				extended: fieldNames
-					? type.extendedFields.filter((extendedField: ExtendedConfigsInterface) =>
-							fieldNames.includes(extendedField.fieldName),
-					  )
-					: type.extendedFields,
-				...(createStateResolvers && {
-					facets: type.config?.[ConfigProperties.FACETS],
-					matchbox: type.config?.[ConfigProperties.MATCHBOX],
-					table: type.config?.[ConfigProperties.TABLE],
-				}),
-			};
+}) => {
+	const { aggregations, hits, configs } = createResolvers({
+		createStateResolvers,
+		type,
+		Parallel,
+		getServerSideFilter,
+		dataMaskThreshold,
+		enableDocumentHits,
+	});
+
+	return {
+		[type.name]: {
+			aggregations,
+			configs,
+			hits,
+			// keeping this available for backwards compatibility, but hoping to remove it
+			// TODO: investigate its current usage and need. remove otherwise
+			// Update 2023-02: ENABLE_ADMIN prevents error comes up on facets.
+			// `aggregation` vs numericAggregation` cannot be assessed, requires "mapping".
+			...(enableAdmin && {
+				mapping: async () => {
+					return type.mapping;
+				},
+			}),
 		},
-		...(enableDocumentHits && { hits: resolveHits({ type, Parallel, getServerSideFilter }) }),
-		// keeping this available for backwards compatibility, but hoping to remove it
-		// TODO: investigate its current usage and need. remove otherwise
-		// Update 2023-02: ENABLE_ADMIN prevents error comes up on facets.
-		// `aggregation` vs numericAggregation` cannot be assessed, requires "mapping".
-		...(enableAdmin && {
-			mapping: async () => {
-				return type.mapping;
-			},
-		}),
-	},
-});
+	};
+};
 
 export default createConnectionResolvers;

modules/server/src/mapping/createConnectionTypeDefs.js

@@ -1,51 +1,46 @@
 import mappingToAggsType from './mappingToAggsType';
 
-const generateHitsTypeString = (name, fieldsToExclude) => {
-	if (fieldsToExclude.includes('hits')) {
-		return '';
-	}
+export default ({ type, fields = '', createStateTypeDefs = true, showRecords }) => {
+	const dataMaskingType = !showRecords ? 'type DataMasking { thresholdValue: Int }' : '';
 
-	return `
-  hits(
-    score: String
-    offset: Int
-    sort: [Sort]
-    filters: JSON
-    before: String
-    after: String
-    first: Int
-    last: Int
-    searchAfter: JSON
-    trackTotalHits: Boolean = true
-  ): ${name}Connection`;
-};
-
-export default ({ type, fields = '', createStateTypeDefs = true, fieldsToExclude }) => {
 	return `
     type ${type.name} {
       aggregations(
         filters: JSON
-
         include_missing: Boolean
         # Should term aggregations be affected by queries that contain filters on their field. For example if a query is filtering primary_site by Blood should the term aggregation on primary_site return all values or just Blood. Set to False for UIs that allow users to select multiple values of an aggregation.
         aggregations_filter_themselves: Boolean
       ): ${type.name}Aggregations
 
+      ${!showRecords ? 'dataMasking: DataMasking' : ''}
+
       configs: ${createStateTypeDefs ? 'ConfigsWithState' : 'ConfigsWithoutState'}
 
-      ${generateHitsTypeString(type.name, fieldsToExclude)}
+      hits(
+        score: String
+        offset: Int
+        sort: [Sort]
+        filters: JSON
+        before: String
+        after: String
+        first: Int
+        last: Int
+        searchAfter: JSON
+        trackTotalHits: Boolean = true
+      ): ${type.name}Connection
 
-
       mapping: JSON
     }
 
     type ${type.name}Aggregations {
       ${mappingToAggsType(type.mapping)}
     }
 
+    ${dataMaskingType}
+
     type ${type.name}Connection {
       total: Int!
-      edges: [${type.name}Edge]
+     ${showRecords ? `edges: [${type.name}Edge]` : ''}
     }
 
     type ${type.name}Edge {

modules/server/src/mapping/mappingToFields.js

@@ -6,7 +6,7 @@ import mappingToObjectTypes from './mappingToObjectTypes';
 import mappingToScalarFields from './mappingToScalarFields';
 
 const mappingToFields = ({ enableDocumentHits, type, parent }) => {
-	const fieldsToExclude = enableDocumentHits ? [] : ['hits'];
+	const showRecords = enableDocumentHits;
 	return [
 		mappingToObjectTypes(type.name, type.mapping, parent, type.extendedFields),
 		Object.entries(type.mapping)
@@ -29,7 +29,7 @@ const mappingToFields = ({ enableDocumentHits, type, parent }) => {
 				type.customFields,
 			],
 			createStateTypeDefs: 'createState' in type ? type.createState : true,
-			fieldsToExclude,
+			showRecords,
 		}),
 	].join();
 };

modules/server/src/mapping/masking.ts

@@ -0,0 +1,95 @@
+import { Aggregation } from './types';
+
+/**
+ * This returns a total count that is less than or equal to the actual total hits in the query.
+ * It is calculated by adding +1 for values under threshold and bucket.doc_count
+ * for values greater than or equal to
+ *
+ * @param aggregation an aggregation with the most buckets which has data masking applied
+ * @returns hits total value
+ */
+const calculateHitsFromAggregation = ({
+	aggregation,
+}: {
+	aggregation: Aggregation | undefined;
+}) => {
+	if (!aggregation) {
+		console.error('No aggregation found for calculating hits.');
+		return 0;
+	}
+	return aggregation.buckets.reduce(
+		(totalAcc, bucket) => (bucket.belowThreshold ? totalAcc + 1 : totalAcc + bucket.doc_count),
+		0,
+	);
+};
+
+/**
+ *
+ * 1) Iterate through aggs applying data masking to buckets if applicable
+ * 2) Find the agg with the most bucket count and data masking applied to be used in calculating hits.total
+ *
+ * @param aggregations - aggregations from query
+ * @param thresholdMin - threshold value
+ * @returns aggregations with data masking applied and hits total
+ */
+export const applyAggregationMasking = ({
+	aggregations,
+	thresholdMin,
+}: {
+	aggregations: Record<
+		string,
+		{
+			bucket_count: number;
+			buckets: Array<{
+				doc_count: number;
+				key: string;
+			}>;
+		}
+	>;
+	thresholdMin: number;
+}) => {
+	// set data masked properties to one less than the configured threshold value (under threshold)
+	const THRESHOLD_REPLACEMENT_VALUE = thresholdMin - 1;
+
+	const { aggsTotal: dataMaskedAggregations, totalHitsAgg } = Object.entries(aggregations).reduce<{
+		aggsTotal: Record<string, Aggregation>;
+		totalHitsAgg: { key: string; bucketCount: number };
+	}>(
+		({ aggsTotal, totalHitsAgg }, [type, aggregation]) => {
+			// mask buckets if under threshold
+			const dataMaskedBuckets = aggregation.buckets.map((bucket) =>
+				bucket.doc_count < thresholdMin
+					? { ...bucket, doc_count: THRESHOLD_REPLACEMENT_VALUE, belowThreshold: true }
+					: { ...bucket, belowThreshold: false },
+			);
+
+			// update total hits selected agg if needed
+			const bucketIsMasked = dataMaskedBuckets.some((bucket) => bucket.belowThreshold);
+			const hitsAgg =
+				totalHitsAgg.bucketCount < aggregation.bucket_count && bucketIsMasked
+					? { key: type, bucketCount: aggregation.bucket_count }
+					: totalHitsAgg;
+
+			return {
+				aggsTotal: {
+					...aggsTotal,
+					[type]: {
+						...aggregation,
+						buckets: dataMaskedBuckets,
+					},
+				},
+				totalHitsAgg: hitsAgg,
+			};
+		},
+		{
+			aggsTotal: {},
+			totalHitsAgg: { key: '', bucketCount: 0 },
+		},
+	);
+
+	const hitsTotal = calculateHitsFromAggregation({
+		aggregation: dataMaskedAggregations[totalHitsAgg.key],
+	});
+
+	return { hitsTotal, dataMaskedAggregations };
+};

modules/server/src/mapping/resolveAggregations.js

@@ -1,17 +1,15 @@
 import getFields from 'graphql-fields';
 
-import { buildQuery, buildAggregations, flattenAggregations } from '../middleware';
+import { buildAggregations, buildQuery, flattenAggregations } from '../middleware';
 
 import { resolveSetsInSqon } from './hackyTemporaryEsSetResolution';
-import esSearch from './utils/esSearch';
 import compileFilter from './utils/compileFilter';
+import esSearch from './utils/esSearch';
 
-let toGraphqlField = (acc, [a, b]) => ({ ...acc, [a.replace(/\./g, '__')]: b });
-
-export default ({ type, getServerSideFilter }) =>
-	async (
+export default ({ type, getServerSideFilter }) => {
+	return async (
 		obj,
-		{ offset = 0, filters, aggregations_filter_themselves, include_missing = true },
+		{ filters, aggregations_filter_themselves, include_missing = true },
 		context,
 		info,
 	) => {
@@ -58,5 +56,11 @@ export default ({ type, getServerSideFilter }) =>
 			includeMissing: include_missing,
 		});
 
-		return Object.entries(aggregations).reduce(toGraphqlField, {});
+		return aggregations;
 	};
+};
+
+const toGraphqlField = (acc, [a, b]) => ({ ...acc, [a.replace(/\./g, '__')]: b });
+export const aggregationsToGraphql = (aggregations) => {
+	return Object.entries(aggregations).reduce(toGraphqlField, {});
+};