Skip to content
This repository has been archived by the owner on Apr 13, 2023. It is now read-only.

Commit

Permalink
feat: Chained parameter, ES logging, SQS encryption (#510)
Browse files Browse the repository at this point in the history 
* chore: ES logging, SQS encryption (#504)

* test: integration test for chained parameters (#500)
  • Loading branch information
@Bingjiling
Bingjiling authored Nov 4, 2021
1 parent 8e45219 commit 5a30027
Show file tree
Hide file tree
Showing 8 changed files with 258 additions and 55 deletions.
21 changes: 21 additions & 0 deletions cloudformation/elasticsearch.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,8 +156,19 @@ Resources:
IdentityPoolId: !Ref KibanaIdentityPool
Roles:
authenticated: !GetAtt AdminKibanaAccessRole.Arn
SearchLogs:
Type: AWS::Logs::LogGroup
Properties:
LogGroupName: !Sub '${AWS::StackName}-search-logs'
SearchLogResourcePolicy:
Type: AWS::Logs::ResourcePolicy
DependsOn: SearchLogs
Properties:
PolicyDocument: !Sub '{ "Version": "2012-10-17", "Statement": [{ "Sid": "", "Effect": "Allow", "Principal": { "Service": "es.amazonaws.com"}, "Action":[ "logs:PutLogEvents","logs:CreateLogStream"],"Resource": "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:${AWS::StackName}-search-logs:*"}]}'
PolicyName: !Sub '${AWS::StackName}-search-logs-resource-policy'
ElasticSearchDomain:
Type: AWS::Elasticsearch::Domain
DependsOn: SearchLogResourcePolicy
Metadata:
cfn_nag:
rules_to_suppress:
Expand Down Expand Up @@ -224,6 +235,16 @@ Resources:
Resource:
Fn::Sub: arn:${AWS::Partition}:es:${AWS::Region}:${AWS::AccountId}:domain/*
- !Ref AWS::NoValue
LogPublishingOptions:
ES_APPLICATION_LOGS:
CloudWatchLogsLogGroupArn: !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:${AWS::StackName}-search-logs:*"
Enabled: true
SEARCH_SLOW_LOGS:
CloudWatchLogsLogGroupArn: !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:${AWS::StackName}-search-logs:*"
Enabled: true
INDEX_SLOW_LOGS:
CloudWatchLogsLogGroupArn: !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:${AWS::StackName}-search-logs:*"
Enabled: true
UpdateSearchMappingsCustomResource:
DependsOn: ElasticSearchDomain
Type: AWS::CloudFormation::CustomResource
Expand Down
16 changes: 8 additions & 8 deletions integration-tests/bulkExport.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/
import BulkExportTestHelper, { ExportStatusOutput } from './bulkExportTestHelper';
import { getFhirClient } from './utils';
import { getFhirClient, getResourcesFromBundleResponse } from './utils';
import createGroupMembersBundle from './createGroupMembersBundle.json';

const EIGHT_MINUTES_IN_MS = 8 * 60 * 1000;
Expand All @@ -28,12 +28,12 @@ describe('Bulk Export', () => {
test('Successfully export all data added to DB after currentTime', async () => {
// BUILD
const oldCreatedResourceBundleResponse = await bulkExportTestHelper.sendCreateResourcesRequest();
const resTypToResNotExpectedInExport = bulkExportTestHelper.getResources(oldCreatedResourceBundleResponse);
const resTypToResNotExpectedInExport = getResourcesFromBundleResponse(oldCreatedResourceBundleResponse);
// sleep 30 seconds to make tests more resilient to clock skew when running locally.
await sleep(30_000);
const currentTime = new Date();
const newCreatedResourceBundleResponse = await bulkExportTestHelper.sendCreateResourcesRequest();
const resTypToResExpectedInExport = bulkExportTestHelper.getResources(newCreatedResourceBundleResponse);
const resTypToResExpectedInExport = getResourcesFromBundleResponse(newCreatedResourceBundleResponse);

// OPERATE
// Only export resources that were added after 'currentTime'
Expand All @@ -51,7 +51,7 @@ describe('Bulk Export', () => {
test('Successfully export just Patient data', async () => {
// BUILD
const createdResourceBundleResponse = await bulkExportTestHelper.sendCreateResourcesRequest();
const resTypToResExpectedInExport = bulkExportTestHelper.getResources(createdResourceBundleResponse);
const resTypToResExpectedInExport = getResourcesFromBundleResponse(createdResourceBundleResponse);
const type = 'Patient';

// OPERATE
Expand Down Expand Up @@ -103,7 +103,7 @@ describe('Bulk Export', () => {
test('Successfully export a group and patient compartment', async () => {
// BUILD
const createdResourceBundleResponse = await bulkExportTestHelper.sendCreateGroupRequest();
const resTypToResExpectedInExport = bulkExportTestHelper.getResources(
const resTypToResExpectedInExport = getResourcesFromBundleResponse(
createdResourceBundleResponse,
createGroupMembersBundle,
true,
Expand All @@ -121,7 +121,7 @@ describe('Bulk Export', () => {
test('Successfully export group members last updated after _since timestamp in a group last updated before the _since timestamp', async () => {
// BUILD
const createdResourceBundleResponse = await bulkExportTestHelper.sendCreateGroupRequest();
const resTypToResExpectedInExport = bulkExportTestHelper.getResources(
const resTypToResExpectedInExport = getResourcesFromBundleResponse(
createdResourceBundleResponse,
createGroupMembersBundle,
true,
Expand Down Expand Up @@ -152,7 +152,7 @@ describe('Bulk Export', () => {
test('Does not include inactive members in group export', async () => {
// BUILD
const createdResourceBundleResponse = await bulkExportTestHelper.sendCreateGroupRequest({ inactive: true });
const resTypToResExpectedInExport = bulkExportTestHelper.getResources(
const resTypToResExpectedInExport = getResourcesFromBundleResponse(
createdResourceBundleResponse,
createGroupMembersBundle,
true,
Expand All @@ -172,7 +172,7 @@ describe('Bulk Export', () => {
const createdResourceBundleResponse = await bulkExportTestHelper.sendCreateGroupRequest({
period: { start: '1992-02-01T00:00:00.000Z', end: '2020-03-04T00:00:00.000Z' },
});
const resTypToResExpectedInExport = bulkExportTestHelper.getResources(
const resTypToResExpectedInExport = getResourcesFromBundleResponse(
createdResourceBundleResponse,
createGroupMembersBundle,
true,
Expand Down
37 changes: 0 additions & 37 deletions integration-tests/bulkExportTestHelper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,43 +178,6 @@ export default class BulkExportTestHelper {
}
}

getResources(
bundleResponse: any,
originalBundle: any = createBundle,
swapBundleInternalReference: boolean = false,
): Record<string, any> {
let resources = [];
const clonedCreatedBundle = cloneDeep(originalBundle);
const urlToReferenceList = [];
for (let i = 0; i < bundleResponse.entry.length; i += 1) {
const res: any = clonedCreatedBundle.entry[i].resource;
const bundleResponseEntry = bundleResponse.entry[i];
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const [location, resourceType, id] = bundleResponseEntry.response.location.match(/(\w+)\/(.+)/);
res.id = id;
res.meta = {
lastUpdated: bundleResponseEntry.response.lastModified,
versionId: bundleResponseEntry.response.etag,
};
resources.push(res);
urlToReferenceList.push({ url: clonedCreatedBundle.entry[i].fullUrl, reference: `${resourceType}/${id}` });
}
// If internal reference was used in bundle creation, swap it to resource reference
if (swapBundleInternalReference) {
let resourcesString = JSON.stringify(resources);
urlToReferenceList.forEach((item) => {
const regEx = new RegExp(`"reference":"${item.url}"`, 'g');
resourcesString = resourcesString.replace(regEx, `"reference":"${item.reference}"`);
});
resources = JSON.parse(resourcesString);
}
const resourceTypeToExpectedResource: Record<string, any> = {};
resources.forEach((res: { resourceType: string }) => {
resourceTypeToExpectedResource[res.resourceType] = res;
});
return resourceTypeToExpectedResource;
}

async getResourcesInExportedFiles(outputs: ExportStatusOutput[]): Promise<Record<string, any[]>> {
// For each resourceType get all fileUrls
const resourceTypeToFileUrls: Record<string, string[]> = mapValues(
Expand Down
58 changes: 58 additions & 0 deletions integration-tests/search.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import {
getFhirClient,
randomPatient,
waitForResourceToBeSearchable,
getResourcesFromBundleResponse,
randomChainedParamBundle,
} from './utils';

jest.setTimeout(600 * 1000);
Expand Down Expand Up @@ -50,6 +52,62 @@ describe('search', () => {
}
});

test('search for valid chained parameters test', async () => {
const createParamChainBundle = randomChainedParamBundle();

const response = (await client.post('/', createParamChainBundle)).data;
const resources = getResourcesFromBundleResponse(response, createParamChainBundle, true);
const testPatient = resources.Patient;

// wait for the patient to be asynchronously written to ES
await waitForResourceToBeSearchable(client, testPatient);

const aFewMinutesAgo = aFewMinutesAgoAsDate();

const p = (params: any) => ({ url: 'Patient', params: { _lastUpdated: `ge${aFewMinutesAgo}`, ...params } });
const testsParams = [
p({ 'organization.name': resources.Organization.name }),
p({ 'general-practitioner:PractitionerRole.organization.name': resources.Organization.name }),
p({ 'general-practitioner:PractitionerRole.practitioner.family': resources.Practitioner.name[0].family }),
p({ 'general-practitioner:PractitionerRole.location.organization.name': resources.Location.name }),
// Verify that chained parameters are combined with 'OR'
p({
'organization.name': resources.Organization.name,
'general-practitioner:PractitionerRole.practitioner.family': 'random-family-name-that-no-one-has',
}),
];

// run tests serially for easier debugging and to avoid throttling
// eslint-disable-next-line no-restricted-syntax
for (const testParams of testsParams) {
// eslint-disable-next-line no-await-in-loop
await expectResourceToBePartOfSearchResults(client, testParams, testPatient);
}
});

test('search for invalid chained parameters', async () => {
const p = (params: any) => ({ url: 'Patient', params: { ...params } });
const testsParams = [
// Invalid search parameter 'location' for resource type Organization
p({ 'organization.location.name': 'Hawaii' }),
// Chained search parameter 'address' for resource type Organization is not a reference.
p({ 'organization.address.name': 'Hawaii' }),
// Chained search parameter 'link' for resource type Patient points to multiple resource types
p({ 'link.name': 'five-O' }),
// Chained parameter returns more than 100 ids
p({ 'link:Patient.birthdate': 'gt1900-05-01' }),
];

// run tests serially for easier debugging and to avoid throttling
// eslint-disable-next-line no-restricted-syntax
for (const testParams of testsParams) {
// eslint-disable-next-line no-await-in-loop
await expect(client.get('Patient', testParams)).rejects.toMatchObject({
response: { status: 400 },
});
}
});

test('search for various valid parameters in query and request body', async () => {
const testPatient: ReturnType<typeof randomPatient> = (await client.post('Patient', randomPatient())).data;

Expand Down
149 changes: 149 additions & 0 deletions integration-tests/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import qs from 'qs';
import { stringify } from 'query-string';
import { decode } from 'jsonwebtoken';
import waitForExpect from 'wait-for-expect';
import { cloneDeep } from 'lodash';
import createBundle from './createPatientPractitionerEncounterBundle.json';

const DEFAULT_TENANT_ID = 'tenant1';

Expand Down Expand Up @@ -240,6 +242,116 @@ export const randomPatient = () => {
};
};

export const randomChainedParamBundle = () => {
const chance = new Chance();
return {
resourceType: 'Bundle',
type: 'transaction',
entry: [
{
fullUrl: 'urn:uuid:fcfe413c-c62d-4097-9e31-02ff6ff523ad',
resource: {
resourceType: 'Patient',
name: [
{
family: 'Escobedo608',
given: ['Cristina921'],
},
],
managingOrganization: {
reference: 'urn:uuid:e92f7839-c81b-4341-93c3-4c6460bd78dc',
},
generalPractitioner: [
{
reference: 'urn:uuid:fcfe413c-c62d-4097-9e31-02ff6gg786yz',
},
],
},
request: {
method: 'POST',
url: 'Patient',
},
},
{
fullUrl: 'urn:uuid:fcfe413c-c62d-4097-9e31-02ff6gg786yz',
resource: {
practitioner: {
reference: 'urn:uuid:e0352b49-8798-398c-8f10-2fc0648a268a',
display: 'Dr Adam Careful',
},
organization: {
reference: 'urn:uuid:e92f7839-c81b-4341-93c3-4c6460bd78dc',
},
location: [
{
reference: 'urn:uuid:fcfe413c-c62d-4097-9e31-02ff6gg369ls',
display: 'South Wing, second floor',
},
],
resourceType: 'PractitionerRole',
},
request: {
method: 'POST',
url: 'PractitionerRole',
},
},
{
fullUrl: 'urn:uuid:fcfe413c-c62d-4097-9e31-02ff6gg369ls',
resource: {
description: 'Old South Wing, Neuro Radiology Operation Room 1 on second floor',
name: chance.word({ length: 15 }),
managingOrganization: {
reference: 'urn:uuid:e92f7839-c81b-4341-93c3-4c6460bd78dc',
},
resourceType: 'Location',
status: 'suspended',
},
request: {
method: 'POST',
url: 'Location',
},
},
{
fullUrl: 'urn:uuid:e0352b49-8798-398c-8f10-2fc0648a268a',
resource: {
resourceType: 'Practitioner',
name: [
{
family: chance.word({ length: 15 }),
given: ['Julia241'],
},
],
},
request: {
method: 'POST',
url: 'Practitioner',
},
},
{
fullUrl: 'urn:uuid:e92f7839-c81b-4341-93c3-4c6460bd78dc',
resource: {
resourceType: 'Organization',
name: chance.word({ length: 15 }),
alias: ['HL7 International'],
address: [
{
line: ['3300 Washtenaw Avenue, Suite 227'],
city: 'Ann Arbor',
state: 'MI',
postalCode: '48104',
country: 'USA',
},
],
},
request: {
method: 'POST',
url: 'Organization',
},
},
],
};
};

const expectSearchResultsToFulfillExpectation = async (
client: AxiosInstance,
search: { url: string; params?: any; postQueryParams?: any },
Expand Down Expand Up @@ -338,3 +450,40 @@ export const waitForResourceToBeSearchable = async (client: AxiosInstance, resou
3000,
);
};

export const getResourcesFromBundleResponse = (
bundleResponse: any,
originalBundle: any = createBundle,
swapBundleInternalReference = false,
): Record<string, any> => {
let resources = [];
const clonedCreatedBundle = cloneDeep(originalBundle);
const urlToReferenceList = [];
for (let i = 0; i < bundleResponse.entry.length; i += 1) {
const res: any = clonedCreatedBundle.entry[i].resource;
const bundleResponseEntry = bundleResponse.entry[i];
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const [location, resourceType, id] = bundleResponseEntry.response.location.match(/(\w+)\/(.+)/);
res.id = id;
res.meta = {
lastUpdated: bundleResponseEntry.response.lastModified,
versionId: bundleResponseEntry.response.etag,
};
resources.push(res);
urlToReferenceList.push({ url: clonedCreatedBundle.entry[i].fullUrl, reference: `${resourceType}/${id}` });
}
// If internal reference was used in bundle creation, swap it to resource reference
if (swapBundleInternalReference) {
let resourcesString = JSON.stringify(resources);
urlToReferenceList.forEach((item) => {
const regEx = new RegExp(`"reference":"${item.url}"`, 'g');
resourcesString = resourcesString.replace(regEx, `"reference":"${item.reference}"`);
});
resources = JSON.parse(resourcesString);
}
const resourceTypeToExpectedResource: Record<string, any> = {};
resources.forEach((res: { resourceType: string }) => {
resourceTypeToExpectedResource[res.resourceType] = res;
});
return resourceTypeToExpectedResource;
};
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
"fhir-works-on-aws-interface": "11.2.0",
"fhir-works-on-aws-persistence-ddb": "3.9.0",
"fhir-works-on-aws-routing": "6.3.0",
"fhir-works-on-aws-search-es": "3.7.0",
"fhir-works-on-aws-search-es": "3.8.0",
"serverless-http": "^2.3.1",
"yargs": "^16.2.0"
},
Expand Down
Loading

0 comments on commit 5a30027

Please sign in to comment.