Skip to content

Commit

Permalink
Merge pull request #67 from koopjs/f/11798-dcat-3-feeds
Browse files Browse the repository at this point in the history 
update to dcat US 3.0
  • Loading branch information
@sansth1010
sansth1010 authored Nov 22, 2024
2 parents 69ea737 + cc1e10a commit b8042a6
Show file tree
Hide file tree
Showing 7 changed files with 314 additions and 48 deletions.
71 changes: 65 additions & 6 deletions src/dcat-us/compile-dcat-feed.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ import { compileDcatFeedEntry } from './compile-dcat-feed';
import * as datasetFromApi from '../test-helpers/mock-dataset.json';
import { DcatUsError } from './dcat-us-error';

describe('generating DCAT-US 1.1 feed', () => {
describe('generating DCAT-US 1.0 feed', () => {
const version = '1.1';
it('should throw 400 DcatUs error if template contains transformer that is not defined', async function () {
const dcatTemplate = {
title: '{{name}}',
Expand All @@ -12,7 +13,7 @@ describe('generating DCAT-US 1.1 feed', () => {
}

try {
compileDcatFeedEntry(datasetFromApi, dcatTemplate, {});
compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version);
} catch (error) {
expect(error).toBeInstanceOf(DcatUsError);
expect(error).toHaveProperty('statusCode', 400);
Expand All @@ -31,7 +32,7 @@ describe('generating DCAT-US 1.1 feed', () => {
]
}

const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}));
const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version));
expect(dcatDataset.distribution).toBeDefined();
expect(dcatDataset.distribution).toStrictEqual(['distro1', 'distro2', 'distro3', 'distro4']);
});
Expand All @@ -43,7 +44,7 @@ describe('generating DCAT-US 1.1 feed', () => {
keyword: '{{tags}}',
distribution: ['distro1', '{{distroname}}']
}
const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}));
const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version));
expect(dcatDataset.distribution).toStrictEqual(['distro1']);
});

Expand All @@ -56,7 +57,7 @@ describe('generating DCAT-US 1.1 feed', () => {
distribution,
spatial: '{{extent}}'
}
const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}));
const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version));
expect(dcatDataset.theme).toBeDefined();
expect(dcatDataset.theme).toStrictEqual(['geospatial']);
});
Expand All @@ -70,9 +71,67 @@ describe('generating DCAT-US 1.1 feed', () => {
};

expect(() => {
compileDcatFeedEntry(undefined, dcatTemplate, {});
compileDcatFeedEntry(undefined, dcatTemplate, {}, version);
}).toThrow(DcatUsError);
});
});

describe('generating DCAT-US 3.0 feed', () => {
const version = '3.0';
it('should throw 400 DcatUs error if template contains transformer that is not defined', async function () {
const dcatTemplate = {
title: '{{name}}',
description: '{{description}}',
keyword: '{{tags}}',
issued: '{{created:toISO}}'
}

try {
compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version);
} catch (error) {
expect(error).toBeInstanceOf(DcatUsError);
expect(error).toHaveProperty('statusCode', 400);
}
});

it('show return distribution in a single array', async function () {
const dcatTemplate = {
title: '{{name}}',
description: '{{description}}',
keyword: '{{tags}}',
'dcat:distribution': [
'distro1',
'distro2',
['distro3', 'distro4']
]
}

const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version));
expect(dcatDataset['dcat:distribution']).toBeDefined();
expect(dcatDataset['dcat:distribution']).toStrictEqual(['distro1', 'distro2', 'distro3', 'distro4']);
});

it('show not return uninterpolated distribution in dataset', async function () {
const dcatTemplate = {
title: '{{name}}',
description: '{{description}}',
keyword: '{{tags}}',
'dcat:distribution': ['distro1', '{{distroname}}']
}
const dcatDataset = JSON.parse(compileDcatFeedEntry(datasetFromApi, dcatTemplate, {}, version));
expect(dcatDataset['dcat:distribution']).toStrictEqual(['distro1']);
});

it('should throw error if geojson from provider is missing', async function () {
const dcatTemplate = {
title: '{{name}}',
description: '{{description}}',
keyword: '{{tags}}',
issued: '{{created:toISO}}'
};

expect(() => {
compileDcatFeedEntry(undefined, dcatTemplate, {}, version);
}).toThrow(DcatUsError);
});
});
31 changes: 25 additions & 6 deletions src/dcat-us/compile-dcat-feed.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,33 @@ type Feature = {
properties: Record<string, any>
};

export function compileDcatFeedEntry(geojsonFeature: Feature | undefined, feedTemplate: DcatDatasetTemplate, feedTemplateTransforms: TransformsList): string {
export function compileDcatFeedEntry(
geojsonFeature: Feature | undefined,
feedTemplate: DcatDatasetTemplate,
feedTemplateTransforms: TransformsList,
version: string
): string {
try {
const dcatFeedItem = generateDcatItem(feedTemplate, feedTemplateTransforms, geojsonFeature);
return indent(JSON.stringify({
...dcatFeedItem,
distribution: Array.isArray(dcatFeedItem.distribution) && removeUninterpolatedDistributions(_.flatten(dcatFeedItem.distribution)),
theme: dcatFeedItem.spatial && ['geospatial']
}, null, '\t'), 2);
let feedEntry: Record<string, any>;
if (version === '1.1') {
feedEntry = {
...dcatFeedItem,
distribution: Array.isArray(dcatFeedItem.distribution) && removeUninterpolatedDistributions(_.flatten(dcatFeedItem.distribution)),
theme: dcatFeedItem.spatial && ['geospatial']
};
}

if (version === '3.0') {
feedEntry = {
...dcatFeedItem,
'dcat:distribution':
Array.isArray(dcatFeedItem['dcat:distribution']) &&
removeUninterpolatedDistributions(_.flatten(dcatFeedItem['dcat:distribution'])),
};
}

return indent(JSON.stringify(feedEntry, null, '\t'), 2);
} catch (err) {
throw new DcatUsError(err.message, 400);
}
Expand Down
59 changes: 59 additions & 0 deletions src/dcat-us/constants/contexts.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Context header for DCAT US 1.1
export const HEADER_V_1X = {
'@context':
'https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld',
'@type': 'dcat:Catalog',
conformsTo: 'https://project-open-data.cio.gov/v1.1/schema',
describedBy: 'https://project-open-data.cio.gov/v1.1/schema/catalog.json',
};

// Context header for DCAT US 3.0
// source: https://raw.githubusercontent.com/DOI-DO/dcat-us/refs/heads/main/context/dcat-us-3.0.jsonld
export const HEADER_V_3_0 = {
'@context': {
'@version': 1.1,
'@protected': true,
'adms': 'http://www.w3.org/ns/adms#',
'cnt': 'http://www.w3.org/2011/content#',
'dash': 'http://datashapes.org/dash#',
'dcat': 'http://www.w3.org/ns/dcat#',
'dcatap': 'http://data.europa.eu/r5r/',
'dcat-us': 'http://data.resources.gov/ontology/dcat-us#',
'dcat-us-shp': 'http://data.resources.gov/shapes/dcat-us#',
'dcterms': 'http://purl.org/dc/terms/',
'dqv': 'http://www.w3.org/ns/dqv#',
'foaf': 'http://xmlns.com/foaf/0.1/',
'gsp': 'http://www.opengis.net/ont/geosparql#',
'locn': 'http://www.w3.org/ns/locn#',
'odrs': 'http://schema.theodi.org/odrs#',
'org': 'http://www.w3c.org/ns/org#',
'owl': 'http://www.w3.org/2002/07/owl#',
'prov': 'http://www.w3.org/ns/prov#',
'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
'rdfs': 'http://www.w3.org/2000/01/rdf-schema#',
'schema': 'http://schema.org/',
'sh': 'http://www.w3.org/ns/shacl#',
'skos': 'http://www.w3.org/2004/02/skos/core#',
'sdmx-attribute': 'http://purl.org/linked-data/sdmx/2009/attribute#',
'spdx': 'http://spdx.org/rdf/terms#',
'vcard': 'http://www.w3.org/2006/vcard/ns#',
"xsd": "http://www.w3.org/2001/XMLSchema#",
'adms:Identifier': {
'@id': 'adms:Identifier',
'@context': {
'schemaAgency': 'http://www.w3.org/ns/adms#schemaAgency',
'creator': {
'@id': 'dcterms:creator',
'@type': '@id'
},
'issued': {
'@id': 'dcterms:issued'
},
'version': 'http://purl.org/dc/terms/version',
'notation': 'http://www.w3.org/2004/02/skos/core#notation'
}
},
},
'@type': 'dcat:Catalog',
conformsTo: 'https://resource.data.gov/profile/dcat-us#',
};
75 changes: 66 additions & 9 deletions src/dcat-us/index.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
import { readableFromArray, streamToString } from '../test-helpers/stream-utils';
import { getDataStreamDcatUs11 } from './';
import { getDataStreamDcatUs } from './';
import * as datasetFromApi from '../test-helpers/mock-dataset.json';
import { HEADER_V_3_0 } from './constants/contexts';

async function generateDcatFeed(dataset, template, templateTransforms) {
const { stream: dcatStream } = getDataStreamDcatUs11(template, templateTransforms);
async function generateDcatFeed(dataset, template, templateTransforms, version) {
const { stream: dcatStream } = getDataStreamDcatUs(template, templateTransforms, version);

const docStream = readableFromArray([dataset]); // no datasets since we're just checking the catalog
const feedString = await streamToString(docStream.pipe(dcatStream));
return { feed: JSON.parse(feedString) };
}

describe('generating DCAT-US 1.1 feed', () => {
const version = '1.1';
it('formats catalog correctly', async function () {
const { feed } = await generateDcatFeed([], {}, {});
const { feed } = await generateDcatFeed([], {}, {}, version);

expect(feed['@context']).toBe('https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld');
expect(feed['@type']).toBe('dcat:Catalog');
Expand All @@ -36,11 +38,13 @@ describe('generating DCAT-US 1.1 feed', () => {
fn: '{{owner}}',
hasEmail: '{{orgContactEmail:optional}}'
}
}, {
toISO: (_key, val) => {
return new Date(val).toISOString();
}
});
},
{
toISO: (_key, val) => {
return new Date(val).toISOString();
}
},
version);

expect(feed['@context']).toBe('https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld');
expect(feed['@type']).toBe('dcat:Catalog');
Expand All @@ -57,5 +61,58 @@ describe('generating DCAT-US 1.1 feed', () => {
expect(feedResponse.publisher).toStrictEqual({ name: 'QA Premium Alpha Hub' });
expect(feedResponse.keyword).toStrictEqual(['Data collection', 'just modified']);
});
});

describe('generating DCAT-US 3.0 feed', () => {
const version = '3.0';
it('formats catalog correctly', async function () {
const { feed } = await generateDcatFeed([], {}, {}, version);

expect(feed['@context']).toStrictEqual(HEADER_V_3_0['@context']);
expect(feed['conformsTo']).toBe('https://resource.data.gov/profile/dcat-us#');
expect(feed['@type']).toBe('dcat:Catalog');
expect(Array.isArray(feed['dcat:dataset'])).toBeTruthy();
});

it('should interprolate dataset stream to feed based upon template', async function () {
const { feed } = await generateDcatFeed(datasetFromApi, {
title: '{{name}}',
description: '{{description}}',
keyword: '{{tags}}',
issued: '{{created:toISO}}',
modified: '{{modified:toISO}}',
publisher: {
name: '{{source}}'
},
contactPoint: {
'@type': 'vcard:Contact',
fn: '{{owner}}',
hasEmail: '{{orgContactEmail:optional}}'
},
header: {
'@id': 'hub.arcgis.com'
}
},
{
toISO: (_key, val) => {
return new Date(val).toISOString();
}
},
version);

expect(feed['@context']).toStrictEqual(HEADER_V_3_0['@context']);
expect(feed['@type']).toBe('dcat:Catalog');
expect(feed['@id']).toBe('hub.arcgis.com');
expect(feed['conformsTo']).toBe('https://resource.data.gov/profile/dcat-us#');
expect(Array.isArray(feed['dcat:dataset'])).toBeTruthy();
expect(feed['dcat:dataset'].length).toBe(1);
const feedResponse = feed['dcat:dataset'][0];
expect(feedResponse.title).toBe('Tahoe places of interest');
expect(feedResponse.description).toBe('Description. Here be Tahoe things. You can do a lot here. Here are some more words. And a few more.<div><br /></div><div>with more words</div><div><br /></div><div>adding a few more to test how long it takes for our jobs to execute.</div><div><br /></div><div>Tom was here!</div>');
expect(feedResponse.issued).toBe('2021-01-29T15:34:38.000Z');
expect(feedResponse.modified).toBe('2021-07-27T20:25:19.723Z');
expect(feedResponse.contactPoint).toStrictEqual({ '@type': 'vcard:Contact', fn: 'thervey_qa_pre_a_hub' });
expect(feedResponse.publisher).toStrictEqual({ name: 'QA Premium Alpha Hub' });
expect(feedResponse.keyword).toStrictEqual(['Data collection', 'just modified']);
});
});
44 changes: 29 additions & 15 deletions src/dcat-us/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,42 @@
import { compileDcatFeedEntry } from './compile-dcat-feed';
import { FeedFormatterStream } from './feed-formatter-stream';
import { TransformsList } from 'adlib';
import { HEADER_V_3_0, HEADER_V_1X } from './constants/contexts';

export function getDataStreamDcatUs11(feedTemplate: any, feedTemplateTransforms: TransformsList) {
const catalogStr = JSON.stringify({
'@context':
'https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld',
'@type': 'dcat:Catalog',
conformsTo: 'https://project-open-data.cio.gov/v1.1/schema',
describedBy: 'https://project-open-data.cio.gov/v1.1/schema/catalog.json',
}, null, '\t');
export function getDataStreamDcatUs(feedTemplate: any, feedTemplateTransforms: TransformsList, version: string) {
const footer = '\n\t]\n}';
let header: string;
let template: Record<string, any>;

const header = `${catalogStr.substr(
0,
catalogStr.length - 2,
)},\n\t"dataset": [\n`;
if (version === '3.0') {
const { header: templateHeader, ...restFeedTemplate } = feedTemplate;
template = restFeedTemplate;
header = generateDcatUs3XHeader(templateHeader);
}

const footer = '\n\t]\n}';
if (version === '1.1') {
template = feedTemplate;
header = generateDcatUs1XHeader();
}

const formatFn = (chunk) => {
return compileDcatFeedEntry(chunk, feedTemplate, feedTemplateTransforms);
return compileDcatFeedEntry(chunk, template, feedTemplateTransforms, version);
};

return {
stream: new FeedFormatterStream(header, footer, ',\n', formatFn)
};
}
}

function generateDcatUs1XHeader() {
const catalogStr = JSON.stringify(HEADER_V_1X, null, '\t');
return `${catalogStr.substring(
0,
catalogStr.length - 2,
)},\n\t"dataset": [\n`;
}

function generateDcatUs3XHeader(header: Record<string, any>) {
const catalogStr = JSON.stringify({ ...HEADER_V_3_0, ...header }, null, '\t');
return `${catalogStr.substring(0, catalogStr.length - 2)},\n\t"dcat:dataset": [\n`;
}
Loading

0 comments on commit b8042a6

Please sign in to comment.