Feat/fed search cleanup (#920)

ciaranschutte · Ciaran Schutte · web-flow · commit 5ffe5e99b4a0 · 2025-03-15T14:19:51.000-04:00
* add placeholder

* Adjust config template example and file

* remove unused arranger constructor param

* reorder constants file, parse env var

* update arranger network config type

* type, add gql server types

* graphqlRoutes, remove unused import, export func

* resolver creation, improve TS usage, split functionality into distinct files, improve usage'

* add surrounding brackets for func

* cleanup schema resolvers

---------

Co-authored-by: Ciaran Schutte &lt;ciaranschutte@oicr.on.ca&gt;
diff --git a/modules/server/README.md b/modules/server/README.md
@@ -21,6 +21,8 @@ npm run prepare
 
 ## Federated Search
 
+[Placeholder]
+
 ### Config
 
-[Placeholder] 
+[Placeholder]
diff --git a/modules/server/configTemplates/configs.json.schema b/modules/server/configTemplates/configs.json.schema
@@ -73,4 +73,13 @@
 		"maxResultsWindow": 10000,
 		"rowIdFieldName": "analysis.analysis_id"
 	}
+	"network": {
+		"servers": [
+			{
+				"displayName": "Toronto",
+				"graphqlUrl": "http://<URL>/graphql",
+				"documentType": "file"
+			}
+		]
+	}
 }
diff --git a/modules/server/configTemplates/network.json b/modules/server/configTemplates/network.json
@@ -1,11 +1,5 @@
 {
 	"network": {
-		"servers": [
-			{
-				"displayName": "Toronto",
-				"graphqlUrl": "http://.../graphql",
-				"documentType": "file"
-			}
-		]
+		"servers": []
 	}
 }
diff --git a/modules/server/src/app.js b/modules/server/src/app.js
@@ -12,11 +12,9 @@ export default async function (rootPath = '') {
 
 	/**
 	 * @param {boolean} enableAdmin
-	 * @param {boolean} enableDocumentHits - enables including "hits" property in the GQL response
 	 */
 	return arranger({
 		enableAdmin: ENV_CONFIG.ENABLE_ADMIN,
-		enableDocumentHits: ENV_CONFIG.ENABLE_DOCUMENT_HITS,
 	}).then((router) => {
 		app.use(router);
 
diff --git a/modules/server/src/config/constants.ts b/modules/server/src/config/constants.ts
@@ -5,14 +5,15 @@ export const ALLOW_CUSTOM_MAX_DOWNLOAD_ROWS = stringToBool(
 );
 export const CONFIG_FILES_PATH = process.env.CONFIG_PATH || './configs';
 export const DATA_MASK_MIN_THRESHOLD =
-	process.env.DATA_MASK_MIN_THRESHOLD || Number.MAX_SAFE_INTEGER;
+	stringToNumber(process.env.DATA_MASK_MIN_THRESHOLD) || Number.MAX_SAFE_INTEGER;
 export const DEBUG_MODE = stringToBool(process.env.DEBUG);
 export const DOCUMENT_TYPE = process.env.DOCUMENT_TYPE || '';
 export const DOWNLOAD_STREAM_BUFFER_SIZE =
 	stringToNumber(process.env.DOWNLOAD_STREAM_BUFFER_SIZE) || 2000;
 export const ENABLE_ADMIN = stringToBool(process.env.ENABLE_ADMIN);
 export const ENABLE_DOCUMENT_HITS = stringToBool(process.env.ENABLE_DOCUMENT_HITS);
 export const ENABLE_LOGS = stringToBool(process.env.ENABLE_LOGS);
+export const ENABLE_NETWORK_AGGREGATION = stringToBool(process.env.ENABLE_NETWORK_AGGREGATION);
 export const ES_ARRANGER_SET_INDEX = process.env.ES_ARRANGER_SET_INDEX || 'arranger-sets';
 export const ES_ARRANGER_SET_TYPE = process.env.ES_ARRANGER_SET_TYPE || 'arranger-sets';
 export const ES_HOST = process.env.ES_HOST || 'http://127.0.0.1:9200';
@@ -27,4 +28,3 @@ export const PING_MS = stringToNumber(process.env.PING_MS) || 2200;
 export const PING_PATH = process.env.PING_PATH || '/ping';
 export const PORT = stringToNumber(process.env.PORT) || 5050;
 export const ROW_ID_FIELD_NAME = process.env.ROW_ID_FIELD_NAME || 'id';
-export const ENABLE_NETWORK_AGGREGATION = stringToBool(process.env.ENABLE_NETWORK_AGGREGATION);
diff --git a/modules/server/src/config/types.ts b/modules/server/src/config/types.ts
@@ -1,7 +1,6 @@
 // TODO: will gradually tighten these as we migrate to TS
 
 import { ES_TYPES } from '@/mapping/esToAggTypeMap';
-import { DOCUMENT_TYPE } from './constants';
 
 export const ConfigOptionalProperties = {
 	DOWNLOADS: 'downloads',
@@ -55,7 +54,7 @@ export const TableProperties = {
 	ROW_ID_FIELD_NAME: 'rowIdFieldName',
 } as const;
 
-export const NetworkAggregationProperties = {
+const NetworkAggregationProperties = {
 	GRAPHQL_URL: 'graphqlUrl',
 	DOCUMENT_TYPE: 'documentType',
 	DISPLAY_NAME: 'displayName',
@@ -151,10 +150,12 @@ export interface TableConfigsInterface {
 	[ConfigProperties.ROW_ID_FIELD_NAME]?: string;
 }
 
-export interface NetworkAggregationInterface {
-	[NetworkAggregationProperties.GRAPHQL_URL]: string;
-	[NetworkAggregationProperties.DOCUMENT_TYPE]: string;
-	[NetworkAggregationProperties.DISPLAY_NAME]: string;
+interface NetworkAggregationInterface {
+	servers: {
+		[NetworkAggregationProperties.GRAPHQL_URL]: string;
+		[NetworkAggregationProperties.DOCUMENT_TYPE]: string;
+		[NetworkAggregationProperties.DISPLAY_NAME]: string;
+	}[];
 }
 
 export interface ConfigObject {
@@ -165,7 +166,7 @@ export interface ConfigObject {
 	[ConfigProperties.INDEX]: string;
 	[ConfigProperties.MATCHBOX]: any[];
 	[ConfigProperties.TABLE]: TableConfigsInterface;
-	[ConfigProperties.NETWORK_AGGREGATION]: NetworkAggregationInterface[];
+	[ConfigProperties.NETWORK_AGGREGATION]: NetworkAggregationInterface;
 }
 
 export interface FieldFromMapping {
diff --git a/modules/server/src/gqlServer.ts b/modules/server/src/gqlServer.ts
@@ -0,0 +1,24 @@
+import { Client } from '@elastic/elasticsearch';
+import { GraphQLResolveInfo } from 'graphql';
+
+export type Context = {
+	esClient: Client;
+};
+
+export type ResolverOutput<T> = T | Promise<T>;
+
+/**
+ * GQL resolver
+ *
+ * @param root - Parent object of a query.
+ * @param args - Query arguments.
+ * @param context - Context passed to apollo-server for queries.
+ * @param info - GraphQL info object.
+ * @return Returns resolved value;
+ */
+export type Resolver<Root = {}, QueryArgs = Object, ReturnValue = undefined> = (
+	root: Root,
+	args: QueryArgs,
+	context: Context,
+	info: GraphQLResolveInfo,
+) => ResolverOutput<ReturnValue> | ResolverOutput<ReturnValue>;
diff --git a/modules/server/src/graphqlRoutes.js b/modules/server/src/graphqlRoutes.js
@@ -5,7 +5,7 @@ import expressPlayground from 'graphql-playground-middleware-express';
 
 import { mergeSchemas } from '@graphql-tools/schema';
 import getConfigObject, { initializeSets } from './config';
-import { DEBUG_MODE, ENABLE_NETWORK_AGGREGATION, ES_PASS, ES_USER } from './config/constants';
+import { DEBUG_MODE, ES_PASS, ES_USER } from './config/constants';
 import { ConfigProperties } from './config/types';
 import { addMappingsToTypes, extendFields, fetchMapping } from './mapping';
 import { extendColumns, extendFacets, flattenMappingToFields } from './mapping/extendMapping';
@@ -243,7 +243,7 @@ const createEndpoint = async ({ esClient, graphqlOptions = {}, mockSchema, schem
 	return router;
 };
 
-const createSchemasFromConfigs = async ({
+export const createSchemasFromConfigs = async ({
 	configsSource = '',
 	enableAdmin,
 	enableDocumentHits,
@@ -272,7 +272,7 @@ const createSchemasFromConfigs = async ({
 
 		const schemasToMerge = [schema];
 
-		/*
+		/**
 		 * Federated Network Search
 		 */
 		if (enableNetworkAggregation) {
diff --git a/modules/server/src/mapping/createConnectionTypeDefs.js b/modules/server/src/mapping/createConnectionTypeDefs.js
@@ -1,8 +1,17 @@
 import mappingToAggsType from './mappingToAggsType';
 
-export default ({ type, fields = '', createStateTypeDefs = true, showRecords }) => {
-	const dataMaskingType = !showRecords ? 'type DataMasking { thresholdValue: Int }' : '';
+const createConnectionType = (enableDocumentHits) => {
+	return `type ${type.name}Connection {
+    total: Int!
+   ${enableDocumentHits ? `edges: [${type.name}Edge]` : ''}
+  }`;
+};
+
+const createDataMaskingType = (enableDocumentHits) => {
+	return !enableDocumentHits ? `type DataMasking { thresholdValue: Int }` : '';
+};
 
+export default ({ type, fields = '', createStateTypeDefs = true, enableDocumentHits }) => {
 	return `
     type ${type.name} {
       aggregations(
@@ -34,12 +43,10 @@ export default ({ type, fields = '', createStateTypeDefs = true, showRecords })
       ${mappingToAggsType(type.mapping)}
     }
 
-    ${dataMaskingType}
+    ${createDataMaskingType(enableDocumentHits)}
 
-    type ${type.name}Connection {
-      total: Int!
-     ${showRecords ? `edges: [${type.name}Edge]` : ''}
-    }
+    ${createConnectionType(enableDocumentHits)}
+  
 
     type ${type.name}Edge {
       searchAfter: JSON
diff --git a/modules/server/src/mapping/mappingToFields.js b/modules/server/src/mapping/mappingToFields.js
@@ -6,7 +6,6 @@ import mappingToObjectTypes from './mappingToObjectTypes';
 import mappingToScalarFields from './mappingToScalarFields';
 
 const mappingToFields = ({ enableDocumentHits, type, parent }) => {
-	const showRecords = enableDocumentHits;
 	return [
 		mappingToObjectTypes(type.name, type.mapping, parent, type.extendedFields),
 		Object.entries(type.mapping)
@@ -29,7 +28,7 @@ const mappingToFields = ({ enableDocumentHits, type, parent }) => {
 				type.customFields,
 			],
 			createStateTypeDefs: 'createState' in type ? type.createState : true,
-			showRecords,
+			enableDocumentHits,
 		}),
 	].join();
 };
diff --git a/modules/server/src/mapping/masking.ts b/modules/server/src/mapping/masking.ts
@@ -8,7 +8,7 @@ export const Relation = {
 export type Relation = keyof typeof Relation;
 
 /**
- * This returns a total count that is less than or equal to the actual total hits in the query.
+ * Returns a total count that is less than or equal to the actual total hits in the query
  * It is calculated by adding +1 for values under threshold or adding bucket.doc_count amount
  * for values greater than or equal to
  *
diff --git a/modules/server/src/mapping/resolveAggregations.ts b/modules/server/src/mapping/resolveAggregations.ts
@@ -2,12 +2,48 @@ import getFields from 'graphql-fields';
 
 import { buildAggregations, buildQuery, flattenAggregations } from '../middleware';
 
+import { Resolver } from '@/gqlServer';
+import { GetServerSideFilterFn } from '@/utils/getDefaultServerSideFilter';
 import { resolveSetsInSqon } from './hackyTemporaryEsSetResolution';
+import { Relation } from './masking';
+import { AggregationQuery, Root } from './types';
 import compileFilter from './utils/compileFilter';
 import esSearch from './utils/esSearch';
 
-export default ({ type, getServerSideFilter }) => {
-	return async (
+const toGraphqlField = (acc: Aggregations, [a, b]: [string, Aggregation]) => ({
+	...acc,
+	[a.replace(/\./g, '__')]: b,
+});
+export const aggregationsToGraphql = (aggregations: Aggregations) => {
+	return Object.entries(aggregations).reduce<Aggregations>(toGraphqlField, {});
+};
+
+/*
+ * Types
+ */
+export type Bucket = {
+	doc_count: number;
+	key: string;
+	relation: Relation;
+};
+
+export type Aggregation = {
+	bucket_count: number;
+	buckets: Bucket[];
+};
+
+export type Aggregations = Record<string, Aggregation>;
+
+export type AggregationsResolver = Resolver<Root, AggregationQuery, Promise<Aggregations>>;
+
+const getAggregationsResolver = ({
+	type,
+	getServerSideFilter,
+}: {
+	type: Record<string, any>;
+	getServerSideFilter: GetServerSideFilterFn | undefined;
+}) => {
+	const resolver: Resolver<unknown, AggregationQuery, Promise<Aggregations>> = async (
 		obj,
 		{ filters, aggregations_filter_themselves, include_missing = true },
 		context,
@@ -26,7 +62,7 @@ export default ({ type, getServerSideFilter }) => {
 			nestedFieldNames,
 			filters: compileFilter({
 				clientSideFilter: resolvedFilter,
-				serverSideFilter: getServerSideFilter(context),
+				serverSideFilter: getServerSideFilter && getServerSideFilter(),
 			}),
 		});
 
@@ -48,6 +84,7 @@ export default ({ type, getServerSideFilter }) => {
 		const response = await esSearch(esClient)({
 			index: type.index,
 			size: 0,
+			// @ts-expect-error - valid search query parameter in ES 7.17, not in types
 			_source: false,
 			body,
 		});
@@ -58,9 +95,8 @@ export default ({ type, getServerSideFilter }) => {
 
 		return aggregations;
 	};
-};
 
-const toGraphqlField = (acc, [a, b]) => ({ ...acc, [a.replace(/\./g, '__')]: b });
-export const aggregationsToGraphql = (aggregations) => {
-	return Object.entries(aggregations).reduce(toGraphqlField, {});
+	return resolver;
 };
+
+export default getAggregationsResolver;
diff --git a/modules/server/src/mapping/resolveHitsFromAggs.ts b/modules/server/src/mapping/resolveHitsFromAggs.ts
@@ -0,0 +1,51 @@
+import { Resolver } from '@/gqlServer';
+import { get } from 'lodash';
+import { applyAggregationMasking } from './masking';
+import { AggregationsResolver } from './resolveAggregations';
+import { HitsQuery, Root } from './types';
+
+type HitsResolver = Resolver<Root, HitsQuery, Promise<{ total: number }>>;
+
+/**
+ * Resolver for "aggregation only mode" of Arranger where "hits" is based on "aggregations"
+ * Calculate hits from aggregation data, instead of using "hits" ES response field
+ *
+ * If "aggregations" field is not in query, return 0
+ *
+ * @param aggregationsResolver - resolver ES query code for aggregations
+ * @returns Returns a total count that is less than or equal to the actual total hits in the query.
+ */
+export const getHitsFromAggsResolver = (aggregationsResolver: AggregationsResolver) => {
+	const resolver: HitsResolver = async (obj, args, context, info) => {
+		/*
+		 * Get "aggregations" field from full query if found
+		 * Popular gql parsing libs parse the "info" property which may not include full query based on schema
+		 */
+		const aggregationsPath = 'operation.selectionSet.selections[0].selectionSet.selections';
+		const aggregationsSelectionSet = get(info, aggregationsPath, []).find(
+			(selection: { kind: string; name: { value: string } }) =>
+				selection.kind === 'Field' && selection.name.value === 'aggregations',
+		);
+
+		if (aggregationsSelectionSet) {
+			const modifiedInfo = { ...info, fieldNodes: [aggregationsSelectionSet] };
+
+			const aggregations = await aggregationsResolver(
+				obj,
+				// @ts-ignore
+				// modifying the query info field inline so it can query aggregations correctly
+				// not idiomatic so doesn't line up with typings from graphql
+				info.variableValues,
+				context,
+				modifiedInfo,
+			);
+			const { hitsTotal: total } = applyAggregationMasking({
+				aggregations,
+			});
+			return { total };
+		} else {
+			return { total: 0 };
+		}
+	};
+	return resolver;
+};
diff --git a/modules/server/src/mapping/resolvers.ts b/modules/server/src/mapping/resolvers.ts
diff --git a/modules/server/src/mapping/types.ts b/modules/server/src/mapping/types.ts
diff --git a/modules/server/src/mapping/utils/esSearch.ts b/modules/server/src/mapping/utils/esSearch.ts
diff --git a/modules/server/src/schema/Aggregations.ts b/modules/server/src/schema/Aggregations.ts
diff --git a/modules/server/src/schema/Root.js b/modules/server/src/schema/Root.js

Original file line number	Diff line number	Diff line change
`@@ -73,4 +73,13 @@`
`73`	`73`	`"maxResultsWindow": 10000,`
`74`	`74`	`"rowIdFieldName": "analysis.analysis_id"`
`75`	`75`	`}`
	`76`	`+ "network": {`
	`77`	`+ "servers": [`
	`78`	`+ {`
	`79`	`+ "displayName": "Toronto",`
	`80`	`+ "graphqlUrl": "http://<URL>/graphql",`
	`81`	`+ "documentType": "file"`
	`82`	`+ }`
	`83`	`+ ]`
	`84`	`+ }`
`76`	`85`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,5 @@`
`1`	`1`	`{`
`2`	`2`	`"network": {`
`3`		`- "servers": [`
`4`		`- {`
`5`		`- "displayName": "Toronto",`
`6`		`- "graphqlUrl": "http://.../graphql",`
`7`		`- "documentType": "file"`
`8`		`- }`
`9`		`- ]`
	`3`	`+ "servers": []`
`10`	`4`	`}`
`11`	`5`	`}`