From c67014f81262f3f081a6d8d88cd208980b465a7a Mon Sep 17 00:00:00 2001 From: Jay Quan Date: Mon, 16 Oct 2023 09:36:37 +0200 Subject: [PATCH 1/3] refactor: make createFullfacetPipeline more readable --- src/common/utils.ts | 425 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 327 insertions(+), 98 deletions(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index 314012af6..3a533f981 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -469,108 +469,167 @@ export const createFullqueryFilter = ( return filterQuery; }; +const pipelineHandler = { + handleTextSearch: ( + pipeline: PipelineStage[], + model: Model, + fields: Y, + key: string, + ) => { + if (typeof fields[key as keyof Y] !== "string") return; + const match = { + $match: { + $or: [ + { + $text: searchExpression(model, key, fields[key as keyof Y]) as { + $search: string; + }, + }, + ], + }, + }; + return pipeline.unshift(match); + }, + handleIdFieldSearch: ( + pipeline: PipelineStage[], + model: Model, + fields: Y, + key: string, + idField: keyof T, + ) => { + const match = { + $match: { + [idField]: searchExpression(model, key, fields[key as keyof Y]), + }, + }; + pipeline.push(match); + }, + handleModeSearch: ( + pipeline: PipelineStage[], + fields: Y, + key: string, + idField: keyof T, + ) => { + const currentExpression = JSON.parse( + JSON.stringify(fields[key as keyof Y]), + ); + if (idField in currentExpression) { + currentExpression["_id"] = currentExpression[idField]; + delete currentExpression[idField]; + } + const match = { + $match: currentExpression, + }; + pipeline.push(match); + }, + handleUserGroupSearch: ( + pipeline: PipelineStage[], + model: Model, + fields: Y, + key: string, + ) => { + const hasFieldAndNotGlobal = + fields[key as keyof Y] && + (fields[key as keyof Y] as unknown as string[]).indexOf("globalaccess") < + 0; + + const hasOwnerGroupInSchema = "ownerGroup" in model.schema.paths; + + if (!hasFieldAndNotGlobal || !hasOwnerGroupInSchema) { + return; + } + + const ownerGroupMatch = searchExpression( + model, + "ownerGroup", + fields[key as keyof Y], + ); + const accessGroupsMatch = searchExpression( + model, + "accessGroups", + fields[key as keyof Y], + ); + + const match = { + $match: { + $or: [ + { ownerGroup: ownerGroupMatch }, + { accessGroups: accessGroupsMatch }, + ], + }, + }; + + pipeline.push(match); + }, + handleScientificQuery: ( + pipeline: PipelineStage[], + fields: Y, + key: string, + ) => { + const match = { + $match: mapScientificQuery( + fields[key as keyof Y] as unknown as IScientificFilter[], + ), + }; + pipeline.push(match); + }, + + handleGenericSearch: ( + pipeline: PipelineStage[], + model: Model, + fields: Y, + key: string, + ) => { + const match: Record = {}; + match[key] = searchExpression(model, key, fields[key as keyof Y]); + const m = { + $match: match, + }; + pipeline.push(m); + }, +}; + export const createFullfacetPipeline = ( - model: Model< - T, - Record, - Record, - Record - >, + model: Model, idField: keyof T, fields: Y, facets: string[], subField = "", ): PipelineStage[] => { - const pipeline = []; + const pipeline: PipelineStage[] = []; const facetMatch: Record = {}; Object.keys(fields).forEach((key) => { - if (facets.indexOf(key) < 0) { - if (key === "text") { - if (typeof fields[key as keyof Y] === "string") { - const match = { - $match: { - $or: [ - { - $text: searchExpression( - model, - key, - fields[key as keyof Y], - ), - }, - ], - }, - }; - pipeline.unshift(match); - } - } else if (key === idField) { - const match = { - $match: { - [idField]: searchExpression(model, key, fields[key as keyof Y]), - }, - }; - pipeline.push(match); - } else if (key === "mode") { - // substitute potential id field in fields - const currentExpression = JSON.parse( - JSON.stringify(fields[key as keyof Y]), - ); - if (idField in currentExpression) { - currentExpression["_id"] = currentExpression[idField]; - delete currentExpression[idField]; - } - const match = { - $match: currentExpression, - }; - pipeline.push(match); - } else if (key === "userGroups") { - if ( - fields[key as keyof Y] && - (fields[key as keyof Y] as unknown as string[]).indexOf( - "globalaccess", - ) < 0 && - "ownerGroup" in model.schema.paths - ) { - const match = { - $match: { - $or: [ - { - ownerGroup: searchExpression( - model, - "ownerGroup", - fields[key as keyof Y], - ), - }, - { - accessGroups: searchExpression( - model, - "accessGroups", - fields[key as keyof Y], - ), - }, - ], - }, - }; - pipeline.push(match); - } - } else if (key === "scientific" || key === "sampleCharacteristics") { - const match = { - $match: mapScientificQuery( - fields[key as keyof Y] as unknown as IScientificFilter[], - ), - }; - pipeline.push(match); - } else { - const match: Record = {}; - match[key] = searchExpression(model, key, fields[key as keyof Y]); - const m = { - $match: match, - }; - pipeline.push(m); - } - } else { + if (facets.includes(key)) { facetMatch[key] = searchExpression(model, key, fields[key as keyof Y]); } + + switch (key) { + case "text": + pipelineHandler.handleTextSearch(pipeline, model, fields, key); + break; + case idField: + pipelineHandler.handleIdFieldSearch( + pipeline, + model, + fields, + key, + idField, + ); + break; + case "mode": + pipelineHandler.handleModeSearch(pipeline, fields, key, idField); + break; + case "userGroups": + pipelineHandler.handleUserGroupSearch(pipeline, model, fields, key); + break; + case "scientific": + case "sampleCharacteristics": + pipelineHandler.handleScientificQuery(pipeline, fields, key); + break; + default: + pipelineHandler.handleGenericSearch(pipeline, model, fields, key); + } }); const facetObject: Record = {}; @@ -583,9 +642,8 @@ export const createFullfacetPipeline = ( facetMatch, ); return; - } else if ( - facet in model.schema.discriminators[DatasetType.Derived].paths - ) { + } + if (facet in model.schema.discriminators[DatasetType.Derived].paths) { facetObject[facet] = createNewFacetPipelineStage( facet, schemaTypeOf(model, facet), @@ -593,7 +651,8 @@ export const createFullfacetPipeline = ( ); return; } - } else if (facet in model.schema.paths) { + } + if (facet in model.schema.paths) { facetObject[facet] = createNewFacetPipelineStage( facet, schemaTypeOf(model, facet), @@ -601,7 +660,6 @@ export const createFullfacetPipeline = ( ); return; } - if (facet.startsWith("datasetlifecycle.")) { const lifecycleFacet = facet.split(".")[1]; facetObject[lifecycleFacet] = createNewFacetPipelineStage( @@ -633,11 +691,182 @@ export const createFullfacetPipeline = ( facetObject["all"].push({ $count: "totalSets", }); - pipeline.push({ $facet: facetObject }); + pipeline.push({ + $facet: facetObject as Record, + }); - return pipeline as PipelineStage[]; + return pipeline; }; +// export const createFullfacetPipeline = ( +// model: Model< +// T, +// Record, +// Record, +// Record +// >, +// idField: keyof T, +// fields: Y, +// facets: string[], +// subField = "", +// ): PipelineStage[] => { +// const pipeline = []; +// const facetMatch: Record = {}; + +// Object.keys(fields).forEach((key) => { +// if (facets.indexOf(key) < 0) { +// if (key === "text") { +// if (typeof fields[key as keyof Y] === "string") { +// const match = { +// $match: { +// $or: [ +// { +// $text: searchExpression( +// model, +// key, +// fields[key as keyof Y], +// ), +// }, +// ], +// }, +// }; +// pipeline.unshift(match); +// } +// } else if (key === idField) { +// const match = { +// $match: { +// [idField]: searchExpression(model, key, fields[key as keyof Y]), +// }, +// }; +// pipeline.push(match); +// } else if (key === "mode") { +// // substitute potential id field in fields +// const currentExpression = JSON.parse( +// JSON.stringify(fields[key as keyof Y]), +// ); +// if (idField in currentExpression) { +// currentExpression["_id"] = currentExpression[idField]; +// delete currentExpression[idField]; +// } +// const match = { +// $match: currentExpression, +// }; +// pipeline.push(match); +// } else if (key === "userGroups") { +// if ( +// fields[key as keyof Y] && +// (fields[key as keyof Y] as unknown as string[]).indexOf( +// "globalaccess", +// ) < 0 && +// "ownerGroup" in model.schema.paths +// ) { +// const match = { +// $match: { +// $or: [ +// { +// ownerGroup: searchExpression( +// model, +// "ownerGroup", +// fields[key as keyof Y], +// ), +// }, +// { +// accessGroups: searchExpression( +// model, +// "accessGroups", +// fields[key as keyof Y], +// ), +// }, +// ], +// }, +// }; +// pipeline.push(match); +// } +// } else if (key === "scientific" || key === "sampleCharacteristics") { +// const match = { +// $match: mapScientificQuery( +// fields[key as keyof Y] as unknown as IScientificFilter[], +// ), +// }; +// pipeline.push(match); +// } else { +// const match: Record = {}; +// match[key] = searchExpression(model, key, fields[key as keyof Y]); +// const m = { +// $match: match, +// }; +// pipeline.push(m); +// } +// } else { +// facetMatch[key] = searchExpression(model, key, fields[key as keyof Y]); +// } +// }); + +// const facetObject: Record = {}; +// facets.forEach((facet) => { +// if (typeof model.schema.discriminators !== "undefined") { +// if (facet in model.schema.discriminators[DatasetType.Raw].paths) { +// facetObject[facet] = createNewFacetPipelineStage( +// facet, +// schemaTypeOf(model, facet), +// facetMatch, +// ); +// return; +// } else if ( +// facet in model.schema.discriminators[DatasetType.Derived].paths +// ) { +// facetObject[facet] = createNewFacetPipelineStage( +// facet, +// schemaTypeOf(model, facet), +// facetMatch, +// ); +// return; +// } +// } else if (facet in model.schema.paths) { +// facetObject[facet] = createNewFacetPipelineStage( +// facet, +// schemaTypeOf(model, facet), +// facetMatch, +// ); +// return; +// } + +// if (facet.startsWith("datasetlifecycle.")) { +// const lifecycleFacet = facet.split(".")[1]; +// facetObject[lifecycleFacet] = createNewFacetPipelineStage( +// lifecycleFacet, +// schemaTypeOf(model, lifecycleFacet), +// facetMatch, +// ); +// return; +// } else { +// Logger.warn( +// `Warning: Facet not part of any model: ${facet}`, +// "utils.createFullfacetPipeline", +// ); +// } +// }); + +// facetObject["all"] = [ +// { +// $match: facetMatch as Record, +// }, +// ]; + +// if (subField) { +// facetObject["all"].push({ +// $unwind: "$" + subField, +// }); +// } + +// facetObject["all"].push({ +// $count: "totalSets", +// }); +// pipeline.push({ $facet: facetObject }); + +// return pipeline as PipelineStage[]; +// }; + export const addCreatedByFields = ( obj: T, username: string, From 44920098f28485f50df2bbeb47a7741544366a5d Mon Sep 17 00:00:00 2001 From: Jay Quan Date: Mon, 16 Oct 2023 09:42:09 +0200 Subject: [PATCH 2/3] fix: minor note addition --- src/common/utils.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common/utils.ts b/src/common/utils.ts index 3a533f981..3529dbf63 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -698,6 +698,10 @@ export const createFullfacetPipeline = ( return pipeline; }; +// NOTE: remove below comments in the future once its sure that everything is +// working fine with above refactored code. +// refactored date 16 October 2023 + // export const createFullfacetPipeline = ( // model: Model< // T, From 8bdcaf299f17daad8fb3c1e80360fb416066adfc Mon Sep 17 00:00:00 2001 From: Jay Quan Date: Mon, 16 Oct 2023 12:37:30 +0200 Subject: [PATCH 3/3] fix: eslint fix and removed comments --- src/common/utils.ts | 175 +------------------------------------------- 1 file changed, 1 insertion(+), 174 deletions(-) diff --git a/src/common/utils.ts b/src/common/utils.ts index 3529dbf63..aba6cf777 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -561,7 +561,7 @@ const pipelineHandler = { pipeline.push(match); }, - handleScientificQuery: ( + handleScientificQuery: ( pipeline: PipelineStage[], fields: Y, key: string, @@ -698,179 +698,6 @@ export const createFullfacetPipeline = ( return pipeline; }; -// NOTE: remove below comments in the future once its sure that everything is -// working fine with above refactored code. -// refactored date 16 October 2023 - -// export const createFullfacetPipeline = ( -// model: Model< -// T, -// Record, -// Record, -// Record -// >, -// idField: keyof T, -// fields: Y, -// facets: string[], -// subField = "", -// ): PipelineStage[] => { -// const pipeline = []; -// const facetMatch: Record = {}; - -// Object.keys(fields).forEach((key) => { -// if (facets.indexOf(key) < 0) { -// if (key === "text") { -// if (typeof fields[key as keyof Y] === "string") { -// const match = { -// $match: { -// $or: [ -// { -// $text: searchExpression( -// model, -// key, -// fields[key as keyof Y], -// ), -// }, -// ], -// }, -// }; -// pipeline.unshift(match); -// } -// } else if (key === idField) { -// const match = { -// $match: { -// [idField]: searchExpression(model, key, fields[key as keyof Y]), -// }, -// }; -// pipeline.push(match); -// } else if (key === "mode") { -// // substitute potential id field in fields -// const currentExpression = JSON.parse( -// JSON.stringify(fields[key as keyof Y]), -// ); -// if (idField in currentExpression) { -// currentExpression["_id"] = currentExpression[idField]; -// delete currentExpression[idField]; -// } -// const match = { -// $match: currentExpression, -// }; -// pipeline.push(match); -// } else if (key === "userGroups") { -// if ( -// fields[key as keyof Y] && -// (fields[key as keyof Y] as unknown as string[]).indexOf( -// "globalaccess", -// ) < 0 && -// "ownerGroup" in model.schema.paths -// ) { -// const match = { -// $match: { -// $or: [ -// { -// ownerGroup: searchExpression( -// model, -// "ownerGroup", -// fields[key as keyof Y], -// ), -// }, -// { -// accessGroups: searchExpression( -// model, -// "accessGroups", -// fields[key as keyof Y], -// ), -// }, -// ], -// }, -// }; -// pipeline.push(match); -// } -// } else if (key === "scientific" || key === "sampleCharacteristics") { -// const match = { -// $match: mapScientificQuery( -// fields[key as keyof Y] as unknown as IScientificFilter[], -// ), -// }; -// pipeline.push(match); -// } else { -// const match: Record = {}; -// match[key] = searchExpression(model, key, fields[key as keyof Y]); -// const m = { -// $match: match, -// }; -// pipeline.push(m); -// } -// } else { -// facetMatch[key] = searchExpression(model, key, fields[key as keyof Y]); -// } -// }); - -// const facetObject: Record = {}; -// facets.forEach((facet) => { -// if (typeof model.schema.discriminators !== "undefined") { -// if (facet in model.schema.discriminators[DatasetType.Raw].paths) { -// facetObject[facet] = createNewFacetPipelineStage( -// facet, -// schemaTypeOf(model, facet), -// facetMatch, -// ); -// return; -// } else if ( -// facet in model.schema.discriminators[DatasetType.Derived].paths -// ) { -// facetObject[facet] = createNewFacetPipelineStage( -// facet, -// schemaTypeOf(model, facet), -// facetMatch, -// ); -// return; -// } -// } else if (facet in model.schema.paths) { -// facetObject[facet] = createNewFacetPipelineStage( -// facet, -// schemaTypeOf(model, facet), -// facetMatch, -// ); -// return; -// } - -// if (facet.startsWith("datasetlifecycle.")) { -// const lifecycleFacet = facet.split(".")[1]; -// facetObject[lifecycleFacet] = createNewFacetPipelineStage( -// lifecycleFacet, -// schemaTypeOf(model, lifecycleFacet), -// facetMatch, -// ); -// return; -// } else { -// Logger.warn( -// `Warning: Facet not part of any model: ${facet}`, -// "utils.createFullfacetPipeline", -// ); -// } -// }); - -// facetObject["all"] = [ -// { -// $match: facetMatch as Record, -// }, -// ]; - -// if (subField) { -// facetObject["all"].push({ -// $unwind: "$" + subField, -// }); -// } - -// facetObject["all"].push({ -// $count: "totalSets", -// }); -// pipeline.push({ $facet: facetObject }); - -// return pipeline as PipelineStage[]; -// }; - export const addCreatedByFields = ( obj: T, username: string,