Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add fields required for acknowledgements page #2499 #2547

Merged
merged 5 commits into from
Feb 24, 2022
Merged

Add fields required for acknowledgements page #2499 #2547

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8fba3c5
Add extra fields to sources.origin #2185
iamleeg Feb 23, 2022
98594ea
Add fields to the source model in the curator API #2185
iamleeg Feb 23, 2022
486e7ae
Add the new fields to the new source form #2185
iamleeg Feb 23, 2022
b13512b
Add new fields to the source table #2185
iamleeg Feb 23, 2022
410b8c9
Support additional source fields in the inline-new-source workflow #2185
iamleeg Feb 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions data-serving/scripts/setup-db/migrations/20220223130636-add_origin_fields_to_sources.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
const originalOriginSchema = {
"bsonType": "object",
"required": ["url"],
"properties": {
"url": {
"bsonType": "string"
},
"license": {
"bsonType": "string"
}
}
}

const newOriginSchema = {
"bsonType": "object",
"required": ["license", "url"],
"properties": {
"url": {
"bsonType": "string"
},
"license": {
"bsonType": "string"
},
"providerName": {
"bsonType": "string"
},
"providerWebsiteUrl": {
"bsonType": "string"
}
}
}

module.exports = {
async up(db, client) {
let res = await db.command({listCollections: undefined, filter: {name: 'sources'}});
let schema = res.cursor.firstBatch[0].options.validator.$jsonSchema;
schema.properties.origin = newOriginSchema;
await db.command({ collMod: 'sources', validator:{ $jsonSchema: schema } } );
},

async down(db, client) {
let res = await db.command({listCollections: undefined, filter: {name: 'sources'}});
let schema = res.cursor.firstBatch[0].options.validator.$jsonSchema;
schema.properties.origin = originalOriginSchema;
await db.command({ collMod: 'sources', validator:{ $jsonSchema: schema } } );
}
};
13 changes: 11 additions & 2 deletions verification/curator-service/api/openapi/openapi.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1137,17 +1137,26 @@ components:
name:
type: string
description: Human readable name of a source
example: Paraguay Ministry of Health and Labor
example: Paraguay
origin:
type: object
properties:
url:
type: string
example: https://opendata.digilugu.ee/opendata_covid19_test_results.csv
description: Source location, can be http, https, or s3 schemes.
description: Source location, can be http, https, or s3 schemes. This must be the link to the source data download.
license:
type: string
example: MIT
description: The license under which we have the right to ingest the data. Ideally a SPDX identifier (https://spdx.org/licenses/).
providerName:
type: string
example: Paraguay Ministry of Public Health and Social Welfare
description: The organisation or individual who collects the data in this source.
providerWebsiteUrl:
type: string
example: https://www.mspbs.gov.py/index.php
description: The provider's website, or where possible their specific site for information about this outbreak (including the source data used here).
format:
type: string
description: Format of the source to be ingested, useful when automated ingestion
Expand Down
11 changes: 9 additions & 2 deletions verification/curator-service/api/src/model/origin.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,21 @@ export const originSchema = new mongoose.Schema(
{
url: {
type: String,
required: 'Enter an origin URL',
required: 'Enter a data source URL',
},
license: String,
license: {
type: String,
required: 'Enter a source license',
},
providerName: String,
providerWebsiteUrl: String,
},
{ _id: false },
);

export type OriginDocument = mongoose.Document & {
url: string;
license: string;
providerName: string;
providerWebsiteUrl: string;
};
4 changes: 3 additions & 1 deletion verification/curator-service/api/test/model/data/origin.full.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
{
"url": "http://foo.bar",
"license": "MIT"
"license": "MIT",
"providerName": "The Ministry of Sound",
"providerWebsiteUrl": "https://www.ministryofsound.example"
}
3 changes: 2 additions & 1 deletion verification/curator-service/api/test/model/data/origin.minimal.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
{
"url": "http://foo.bar"
"url": "http://foo.bar",
"license": "GPL3"
}
9 changes: 8 additions & 1 deletion verification/curator-service/api/test/model/origin.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,14 @@ describe('validate', () => {
});

it('a minimal origin is valid', async () => {
return new Origin(minimalModel).validate();
const missingLicense = _.cloneDeep(minimalModel);
delete missingLicense.license;

return new Origin(missingLicense).validate((e) => {
expect(e).not.toBeNull();
if (e)
expect(e.name).toBe(Error.ValidationError.name);
});
});

it('a fully specified origin is valid', async () => {
Expand Down
1 change: 1 addition & 0 deletions verification/curator-service/ui/cypress/integration/components/BulkCaseForm.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ describe('Bulk upload form', function () {
cy.get('div[data-testid="caseReference"]').type('www.new-source.com');
cy.contains('li', 'www.new-source.com').click();
cy.get('input[name="caseReference.sourceName"]').type('New source');
cy.get('input[name="caseReference.sourceLicense"]').type('GPL3');
const csvFixture = '../fixtures/bulk_data.csv';
cy.get('input[type="file"]').attachFile(csvFixture);
cy.server();
Expand Down
1 change: 1 addition & 0 deletions verification/curator-service/ui/cypress/integration/components/Curator.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ describe('Curator', function () {
cy.get('div[data-testid="caseReference"]').type('www.example.com');
cy.contains('li', 'www.example.com').click();
cy.get('input[name="caseReference.sourceName"]').type('Example source');
cy.get('input[name="caseReference.sourceLicense"]').type('MPL');
cy.get('div[data-testid="sourceEntryId"]')
.click()
.type('testSourceEntryID123');
Expand Down
1 change: 1 addition & 0 deletions verification/curator-service/ui/cypress/integration/components/NewCaseForm.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ describe('New case form', function () {
cy.get('div[data-testid="caseReference"]').type('www.new-source.com');
cy.get('input[name="caseReference.sourceName"]').click();
cy.get('input[name="caseReference.sourceName"]').type('New source');
cy.get('input[name="caseReference.sourceLicense"]').type('WTFPL');
cy.get('div[data-testid="location"]').type('France');
cy.contains('li', 'France').click();
cy.get('input[name="confirmedDate"]').type('2020-01-01');
Expand Down
3 changes: 2 additions & 1 deletion verification/curator-service/ui/src/components/AutomatedSourceForm.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ describe('<AutomatedSourceForm />', () => {
expect(screen.getByTestId('recipients')).toBeInTheDocument();
expect(screen.getByTestId('excludeFromLineList')).toBeInTheDocument();
expect(screen.getByTestId('hasStableIdentifiers')).toBeInTheDocument();

expect(screen.getByTestId('providerName')).toBeInTheDocument();
expect(screen.getByTestId('providerWebsiteUrl')).toBeInTheDocument();
// Buttons
expect(screen.getByText(/create source/i)).toBeEnabled();
expect(screen.getByText(/cancel/i)).toBeEnabled();
Expand Down
33 changes: 32 additions & 1 deletion verification/curator-service/ui/src/components/AutomatedSourceForm.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ export interface AutomatedSourceFormValues {
url: string;
license: string;
name: string;
providerName: string;
providerWebsiteUrl: string;
countryCodes: string[];
format: string;
notificationRecipients: string[];
Expand All @@ -96,6 +98,8 @@ const AutomatedSourceFormSchema = Yup.object().shape({
countryCodes: Yup.array().of(Yup.string()).required('Required'),
format: Yup.string().required('Required'),
license: Yup.string().required('Required'),
providerName: Yup.string(),
providerWebsiteUrl: Yup.string(),
notificationRecipients: Yup.array().of(Yup.string().email()),
excludeFromLineList: Yup.boolean().required('Required'),
hasStableIdentifiers: Yup.boolean().required('Required'),
Expand All @@ -114,7 +118,12 @@ export default function AutomatedSourceForm(props: Props): JSX.Element {
const newSource = {
name: values.name,
countryCodes: values.countryCodes,
origin: { url: values.url, license: values.license },
origin: {
url: values.url,
license: values.license,
providerName: values.providerName,
providerWebsiteUrl: values.providerWebsiteUrl,
},
format: values.format,
notificationRecipients: values.notificationRecipients,
excludeFromLineList: values.excludeFromLineList,
Expand Down Expand Up @@ -149,6 +158,8 @@ export default function AutomatedSourceForm(props: Props): JSX.Element {
countryCodes: [],
format: '',
license: '',
providerName: '',
providerWebsiteUrl: '',
notificationRecipients: [user.email],
excludeFromLineList: false,
hasStableIdentifiers: false,
Expand Down Expand Up @@ -247,6 +258,26 @@ export default function AutomatedSourceForm(props: Props): JSX.Element {
fullWidth
/>
</div>
<div className={classes.formSection}>
<FastField
name="providerName"
label="Data Provider Name"
type="text"
data-testid="providerName"
component={TextField}
fullWidth
/>
</div>
<div className={classes.formSection}>
<FastField
name="providerWebsiteUrl"
label="Data Provider Website"
type="text"
data-testid="providerWebsiteUrl"
component={TextField}
fullWidth
/>
</div>
<div className={classes.formSection}>
<SelectField
name="format"
Expand Down
3 changes: 3 additions & 0 deletions verification/curator-service/ui/src/components/BulkCaseForm.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,9 @@ class BulkCaseForm extends React.Component<
const newCaseReference = await submitSource({
name: values.caseReference.sourceName as string,
url: values.caseReference.sourceUrl,
license: values.caseReference.sourceLicense as string,
providerName: values.caseReference.sourceProviderName,
providerWebsiteUrl: values.caseReference.sourceProviderUrl,
format: 'CSV',
});
values.caseReference.sourceId = newCaseReference.sourceId;
Expand Down
3 changes: 3 additions & 0 deletions verification/curator-service/ui/src/components/CaseForm.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,9 @@ export default function CaseForm(props: Props): JSX.Element {
const newCaseReference = await submitSource({
name: values.caseReference.sourceName as string,
url: values.caseReference.sourceUrl,
license: values.caseReference.sourceLicense as string,
providerName: values.caseReference.sourceProviderName,
providerWebsiteUrl: values.caseReference.sourceProviderUrl,
});
values.caseReference.sourceId = newCaseReference.sourceId;
} catch (e) {
Expand Down
12 changes: 11 additions & 1 deletion verification/curator-service/ui/src/components/SourceTable.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ describe('<SourceTable />', () => {
const sourceName = 'source_name';
const originUrl = 'origin url';
const format = 'JSON';
const providerName = 'provider_name';
const providerWebsiteUrl = 'website url';
const countryCodes = ['US', 'MX', 'CA'];
const license = 'MIT';
const recipients = ['foo@bar.com', 'bar@baz.com'];
Expand All @@ -36,7 +38,9 @@ describe('<SourceTable />', () => {
countryCodes: countryCodes,
origin: {
url: originUrl,
license: license,
license,
providerName,
providerWebsiteUrl,
},
automation: {
parser: {
Expand Down Expand Up @@ -90,6 +94,12 @@ describe('<SourceTable />', () => {
expect(
await screen.findByText(new RegExp(license)),
).toBeInTheDocument();
expect(
await screen.findByText(new RegExp(providerName)),
).toBeInTheDocument();
expect(
await screen.findByText(new RegExp(providerWebsiteUrl)),
).toBeInTheDocument();
expect(
await screen.findByText(new RegExp(recipients.join('.*'))),
).toBeInTheDocument();
Expand Down
66 changes: 63 additions & 3 deletions verification/curator-service/ui/src/components/SourceTable.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ import ChipInput from 'material-ui-chip-input';

interface Origin {
url: string;
license?: string;
license: string;
providerName?: string;
providerWebsiteUrl?: string;
}

interface Field {
Expand Down Expand Up @@ -88,10 +90,11 @@ interface TableRow {
countryCodes: string; // flattened
// origin
url: string;
license: string;
providerName?: string;
providerWebsiteUrl?: string;

license?: string;
// automation.parser

format?: string;
awsLambdaArn?: string;
// automation.schedule
Expand Down Expand Up @@ -232,6 +235,8 @@ class SourceTable extends React.Component<Props, SourceTableState> {
origin: {
url: rowData.url,
license: rowData.license,
providerName: rowData.providerName,
providerWebsiteUrl: rowData.providerWebsiteUrl,
},
format: rowData.format,
automation:
Expand Down Expand Up @@ -438,6 +443,57 @@ class SourceTable extends React.Component<Props, SourceTableState> {
/>
),
},
{
title: 'Provider Name',
field: 'providerName',
tooltip:
'Miskatonic University Department of Medecine',
editComponent: (props): JSX.Element => (
<TextField
type="text"
size="small"
fullWidth
placeholder="Provider Name"
error={
!this.validateRequired(props.value)
}
helperText={
this.validateRequired(props.value)
? ''
: 'Required field'
}
onChange={(event): void =>
props.onChange(event.target.value)
}
defaultValue={props.value}
/>
),
},
{
title: 'Provider Website',
field: 'providerWebsiteUrl',
tooltip: 'https://medsci.miskatonic.edu/',
editComponent: (props): JSX.Element => (
<TextField
type="text"
size="small"
fullWidth
placeholder="Provider Website"
error={
!this.validateRequired(props.value)
}
helperText={
this.validateRequired(props.value)
? ''
: 'Required field'
}
onChange={(event): void =>
props.onChange(event.target.value)
}
defaultValue={props.value}
/>
),
},
{
title: 'Notification recipients',
field: 'notificationRecipients',
Expand Down Expand Up @@ -658,6 +714,10 @@ class SourceTable extends React.Component<Props, SourceTableState> {
format: s.format,
url: s.origin.url,
license: s.origin.license,
providerName:
s.origin.providerName,
providerWebsiteUrl:
s.origin.providerWebsiteUrl,
awsLambdaArn:
s.automation?.parser
?.awsLambdaArn,
Expand Down
Loading