Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix data-service CI failures #2691

Merged
merged 1 commit into from
May 4, 2022
Merged

Fix data-service CI failures #2691

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 43 additions & 22 deletions data-serving/data-service/src/controllers/case.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,49 +39,66 @@ class InvalidParamError extends Error {}
type BatchValidationErrors = { index: number; message: string }[];

const caseFromDTO = async (receivedCase: CaseDTO) => {
const aCase = receivedCase as unknown as LeanDocument<CaseDocument>;
const aCase = (receivedCase as unknown) as LeanDocument<CaseDocument>;
if (receivedCase.demographics?.ageRange) {
// won't be many age buckets, so fetch all of them.
const allBuckets = await AgeBucket.find({});
const caseStart = receivedCase.demographics?.ageRange.start;
const caseEnd = receivedCase.demographics?.ageRange.end;
const matchingBucketIDs = allBuckets.filter(b => {
const bucketContainsStart = (b.start <= caseStart && b.end >= caseStart);
const bucketContainsEnd = (b.start <= caseEnd && b.end >= caseEnd);
const bucketWithinCaseRange = (b.start > caseStart && b.end < caseEnd);
return bucketContainsStart || bucketContainsEnd || bucketWithinCaseRange;
}).map((b) => (b._id));
const matchingBucketIDs = allBuckets
.filter((b) => {
const bucketContainsStart =
b.start <= caseStart && b.end >= caseStart;
const bucketContainsEnd =
b.start <= caseEnd && b.end >= caseEnd;
const bucketWithinCaseRange =
b.start > caseStart && b.end < caseEnd;
return (
bucketContainsStart ||
bucketContainsEnd ||
bucketWithinCaseRange
);
})
.map((b) => b._id);
aCase.demographics.ageBuckets = matchingBucketIDs;
}
return aCase;
}
};

const dtoFromCase = async (storedCase: LeanDocument<CaseDocument>) => {
let dto = storedCase as unknown as CaseDTO;
if (storedCase.demographics && storedCase.demographics.ageBuckets && storedCase.demographics.ageBuckets.length > 0) {
const ageBuckets = await Promise.all(storedCase.demographics.ageBuckets.map((bucketId) => {
return AgeBucket.findById(bucketId).lean()
}));
const minimumAge = Math.min(...(ageBuckets.map(b => b!.start)));
const maximumAge = Math.max(...(ageBuckets.map(b => b!.end)));
let dto = (storedCase as unknown) as CaseDTO;
if (
storedCase.demographics &&
storedCase.demographics.ageBuckets &&
storedCase.demographics.ageBuckets.length > 0
) {
const ageBuckets = await Promise.all(
storedCase.demographics.ageBuckets.map((bucketId) => {
return AgeBucket.findById(bucketId).lean();
}),
);
const minimumAge = Math.min(...ageBuckets.map((b) => b!.start));
const maximumAge = Math.max(...ageBuckets.map((b) => b!.end));
dto = {
...dto,
demographics: {
...dto.demographics!,
ageRange: {
start: minimumAge,
end: maximumAge,
}
}
}
},
},
};
// although the type system can't see it, there's an ageBuckets property on the demographics DTO now
delete (dto as unknown as { demographics: { ageBuckets?: [ObjectId] }}).demographics.ageBuckets;
delete ((dto as unknown) as {
demographics: { ageBuckets?: [ObjectId] };
}).demographics.ageBuckets;
}
delete dto.restrictedNotes;
delete dto.notes;

return dto;
}
};

export class CasesController {
private csvHeaders: string[];
Expand Down Expand Up @@ -124,7 +141,7 @@ export class CasesController {
delete aCase.notes;
});

res.json(await Promise.all(c.map(aCase => dtoFromCase(aCase))));
res.json(await Promise.all(c.map((aCase) => dtoFromCase(aCase))));
};

/**
Expand Down Expand Up @@ -538,7 +555,9 @@ export class CasesController {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const upsertLambda = async (c: any) => {
delete c.caseCount;
c = caseWithDenormalisedConfirmationDate(await caseFromDTO(c as CaseDTO));
c = caseWithDenormalisedConfirmationDate(
await caseFromDTO(c as CaseDTO),
);
if (
c.caseReference?.sourceId &&
c.caseReference?.sourceEntryId
Expand Down Expand Up @@ -1099,7 +1118,9 @@ export const casesMatchingSearchQuery = (opts: {
}): any => {
// set data limit to 10K by default
const countLimit = opts.limit ? opts.limit : 10000;
console.log(`Search query: ${opts.searchQuery}`);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

switch to logger.debug?

const parsedSearch = parseSearchQuery(opts.searchQuery);
console.log(`Parsed search (full text?): ${parsedSearch.fullTextSearch}`);
const queryOpts = parsedSearch.fullTextSearch
? {
$text: { $search: parsedSearch.fullTextSearch },
Expand Down
19 changes: 14 additions & 5 deletions data-serving/data-service/test/controllers/case.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { CaseRevision } from '../../src/model/case-revision';
import { Demographics } from '../../src/model/demographics';
import { MongoMemoryServer } from 'mongodb-memory-server';
import { Source } from '../../src/model/source';
import { PathogenDocument } from '../../src/model/pathogen';
import app from './../../src/index';
import fullCase from './../model/data/case.full.json';
import minimalCase from './../model/data/case.minimal.json';
Expand Down Expand Up @@ -310,7 +311,7 @@ describe('GET', () => {
it('Search for matching country and something else that does not match', async () => {
const res = await request(app)
.get(
'/api/cases?page=1&limit=1&q=country%3ADE%occupation%3Anope',
'/api/cases?page=1&limit=1&q=country%3ADE%20occupation%3Anope',
)
.expect(200)
.expect('Content-Type', /json/);
Expand Down Expand Up @@ -589,7 +590,9 @@ describe('POST', () => {

const changedCaseWithEntryId = new Case(fullCase);
await changedCaseWithEntryId.save();
changedCaseWithEntryId.notes = 'new notes';
changedCaseWithEntryId.pathogens = [
{ id: '304', name: 'Pneumonia' } as PathogenDocument,
];

const unchangedCaseWithEntryId = new Case(fullCase);
unchangedCaseWithEntryId.caseReference.sourceEntryId =
Expand Down Expand Up @@ -633,7 +636,9 @@ describe('POST', () => {
const changedCaseWithEntryId = new Case(fullCase);
await changedCaseWithEntryId.save();
changedCaseWithEntryId.caseReference.uploadIds = newUploadIds;
changedCaseWithEntryId.notes = 'new notes';
changedCaseWithEntryId.pathogens = [
{ id: '304', name: 'Pneumonia' } as PathogenDocument,
];

const unchangedCaseWithEntryId = new Case(fullCase);
unchangedCaseWithEntryId.caseReference.sourceEntryId =
Expand Down Expand Up @@ -767,7 +772,9 @@ describe('POST', () => {
it('batch upsert should result in create and update metadata', async () => {
const existingCase = new Case(fullCase);
await existingCase.save();
existingCase.notes = 'new notes';
existingCase.pathogens = [
{ id: '104', name: 'Pneumonia' } as PathogenDocument,
];

const res = await request(app)
.post('/api/cases/batchUpsert')
Expand Down Expand Up @@ -798,7 +805,9 @@ describe('POST', () => {
it('batch upsert should result in case revisions of existing cases', async () => {
const existingCase = new Case(fullCase);
await existingCase.save();
existingCase.notes = 'new notes';
existingCase.pathogens = [
{ id: '104', name: 'Pneumonia' } as PathogenDocument,
];

const res = await request(app)
.post('/api/cases/batchUpsert')
Expand Down
1 change: 0 additions & 1 deletion data-serving/data-service/test/model/data/case.full.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,6 @@
"id": 104
}
],
"notes": "Contact of a confirmed case at work.",
"revisionMetadata": {
"revisionNumber": 0,
"creationMetadata": {
Expand Down