F/1915 reflect custom config updates

CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG.md
 
+## Unreleased
+Added
+- Supports custom configuration values found at `site.data.feeds.dcatUS11` [#4](https://github.com/koopjs/koop-output-dcat-us-11/pull/4)
+
 ## 1.0.1
 
 Fixed

src/dcat-us/dataset-formatter.ts

@@ -8,7 +8,7 @@ import { IItem } from '@esri/arcgis-rest-portal';
 
 // TODO - use real type for hubDataset when it gets defined in Hub.js
 type HubDatasetAttributes = Record<string, any>;
-type DcatDatasetTemplate = Record<string, any>;
+export type DcatDatasetTemplate = Record<string, any>;
 
 export function formatDcatDataset (hubDataset: HubDatasetAttributes, siteUrl: string, dcatCustomizations: DcatDatasetTemplate = {}) {
   const landingPage = `${siteUrl}/datasets/${hubDataset.id}`;

src/dcat-us/index.test.ts

@@ -7,8 +7,9 @@ import * as mockSiteModel from '../test-helpers/mock-site-model.json';
 function generateDcatFeed(
   siteItem,
   datasets,
+  dcatCustomizations?
 ) {
-  const dcatStream = getDataStreamDcatUs11(siteItem);
+  const dcatStream = getDataStreamDcatUs11(siteItem, dcatCustomizations);
 
   const docStream = readableFromArray(datasets); // no datasets since we're just checking the catalog
 
@@ -53,4 +54,75 @@ describe('generating DCAT-US 1.1 feed', () => {
     expect(chk1.distribution).toBeInstanceOf(Array);
     expect(chk1.distribution.length).toBe(6);
   });
+
+  it('respects dcat customizations of overwritable attributes', async function () {
+    const feed = await generateDcatFeed(
+      mockSiteModel.item, 
+      [datasetFromApi],
+      {
+        description: '{{name}}', // overwrite existing attribute
+        customField: '{{name}}' // add new attribute
+      }
+    );
+
+    const chk1 = feed['dataset'][0];
+
+    expect(chk1['@type']).toBe('dcat:Dataset');
+    expect(chk1.identifier).toBe('https://download-test-qa-pre-a-hub.hubqa.arcgis.com/datasets/f4bcc1035b7d46cba95e977f4affb6be_0');
+    expect(chk1.license).toBe(null);
+    expect(chk1.landingPage).toBe('https://download-test-qa-pre-a-hub.hubqa.arcgis.com/datasets/f4bcc1035b7d46cba95e977f4affb6be_0');
+    expect(chk1.title).toBe('Tahoe places of interest');
+    expect(chk1.description).toBe('Tahoe places of interest');
+    expect(chk1.customField).toBe('Tahoe places of interest');
+    expect(chk1.keyword).toEqual([ 'Data collection', 'just modified' ]);
+    expect(chk1.issued).toBe('2021-01-29T15:34:38.000Z');
+    expect(chk1.modified).toBe('2021-07-27T20:25:19.723Z');
+    expect(chk1.publisher).toEqual({ name: 'QA Premium Alpha Hub' });
+    expect(chk1.contactPoint).toEqual({ '@type': 'vcard:Contact', fn: 'thervey_qa_pre_a_hub' });
+    expect(chk1.accessLevel).toBe('public');
+    expect(chk1.spatial).toBe('-121.118,38.7754,-119.009,39.359');
+    expect(chk1.theme).toEqual(['geospatial']);
+
+    expect(chk1.distribution).toBeInstanceOf(Array);
+    expect(chk1.distribution.length).toBe(6);
+  });
+
+  it('scrubs dcat customization of protected fields', async function () {
+    const feed = await generateDcatFeed(
+      mockSiteModel.item, 
+      [datasetFromApi],
+      {
+        '@type': '{{name}}',
+        identifier: '{{name}}',
+        landingPage: '{{name}}',
+        webService: '{{name}}',
+        distributions: '{{name}}',
+        contactPoint: {
+          '@type': '{{name}}',
+          fn: '{{owner}}',
+        }
+      }
+    );
+
+    const chk1 = feed['dataset'][0];
+
+    expect(chk1['@type']).toBe('dcat:Dataset');
+    expect(chk1.identifier).toBe('https://download-test-qa-pre-a-hub.hubqa.arcgis.com/datasets/f4bcc1035b7d46cba95e977f4affb6be_0');
+    expect(chk1.license).toBe(null);
+    expect(chk1.webService).toBe(undefined);
+    expect(chk1.landingPage).toBe('https://download-test-qa-pre-a-hub.hubqa.arcgis.com/datasets/f4bcc1035b7d46cba95e977f4affb6be_0');
+    expect(chk1.title).toBe('Tahoe places of interest');
+    expect(chk1.description).toBe('Description. Here be Tahoe things. You can do a lot here. Here are some more words. And a few more.<div><br /></div><div>with more words</div><div><br /></div><div>adding a few more to test how long it takes for our jobs to execute.</div><div><br /></div><div>Tom was here!</div>');
+    expect(chk1.keyword).toEqual([ 'Data collection', 'just modified' ]);
+    expect(chk1.issued).toBe('2021-01-29T15:34:38.000Z');
+    expect(chk1.modified).toBe('2021-07-27T20:25:19.723Z');
+    expect(chk1.publisher).toEqual({ name: 'QA Premium Alpha Hub' });
+    expect(chk1.contactPoint).toEqual({ fn: 'thervey_qa_pre_a_hub' });
+    expect(chk1.accessLevel).toBe('public');
+    expect(chk1.spatial).toBe('-121.118,38.7754,-119.009,39.359');
+    expect(chk1.theme).toEqual(['geospatial']);
+
+    expect(chk1.distribution).toBeInstanceOf(Array);
+    expect(chk1.distribution.length).toBe(6);
+  });
 });

src/dcat-us/index.ts

@@ -1,8 +1,8 @@
 import { IItem } from '@esri/arcgis-rest-portal';
-import { formatDcatDataset } from './dataset-formatter';
+import { DcatDatasetTemplate, formatDcatDataset } from './dataset-formatter';
 import { FeedFormatterStream } from './feed-formatter-stream';
 
-export function getDataStreamDcatUs11(siteItem: IItem) {
+export function getDataStreamDcatUs11(siteItem: IItem, dcatCustomizations?: DcatDatasetTemplate) {
   const catalogStr = JSON.stringify({
       '@context':
         'https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld',
@@ -19,7 +19,7 @@ export function getDataStreamDcatUs11(siteItem: IItem) {
   const footer = '\n\t]\n}';
 
   const formatFn = (chunk) => {
-    return formatDcatDataset(chunk, siteItem.url);
+    return formatDcatDataset(chunk, siteItem.url, dcatCustomizations);
   };
 
   return new FeedFormatterStream(header, footer, ',\n', formatFn);

src/index.test.ts

@@ -7,6 +7,9 @@ import * as request from 'supertest';
 import * as mockDataset from './test-helpers/mock-dataset.json';
 import * as mockSiteModel from './test-helpers/mock-site-model.json';
 
+import { FeedFormatterStream } from './dcat-us/feed-formatter-stream';
+import * as _ from 'lodash';
+
 describe('Output Plugin', () => {
   let mockFetchSite;
   let mockConfigModule;
@@ -145,4 +148,44 @@ describe('Output Plugin', () => {
 
     // TODO test stream error
   });
+
+  it('Properly passes custom dcat configurations to getDataStreamDcatUs11', async () => {
+    // Mock getDataStreamDcatUs11
+    const { getDataStreamDcatUs11 } = require('./dcat-us');
+    jest.mock('./dcat-us', () => ({
+      getDataStreamDcatUs11: jest.fn(),
+    }));
+    const mockGetDataStreamDcatUs11 = mocked(getDataStreamDcatUs11);
+    mockGetDataStreamDcatUs11.mockReturnValue(new FeedFormatterStream('{', '}', '', () => ''));
+
+    // Change fetchSite's return value to include a custom dcat config
+    const customConfigSiteModel: any = _.cloneDeep(mockSiteModel);
+    customConfigSiteModel.data.feeds = {
+      dcatUS11: {
+        "title": "{{default.name}}",
+        "description": "{{default.description}}",
+        "keyword": "{{item.tags}}",
+        "issued": "{{item.created:toISO}}",
+        "modified": "{{item.modified:toISO}}",
+        "publisher": { "name": "{{default.source.source}}" },
+        "contactPoint": {
+          "fn": "{{item.owner}}",
+          "hasEmail": "{{org.portalProperties.links.contactUs.url}}"
+        },
+        "landingPage": "some silly standard",
+      }
+    }
+    mockFetchSite.mockResolvedValue(customConfigSiteModel);
+
+    [plugin, app] = buildPluginAndApp();
+
+    await request(app)
+      .get('/dcat')
+      .set('host', siteHostName)
+      .expect('Content-Type', /application\/json/)
+      .expect(200)
+      .expect(() => {
+        expect(mockGetDataStreamDcatUs11).toHaveBeenCalledWith(customConfigSiteModel.item, customConfigSiteModel.data.feeds.dcatUS11);
+      });
+  });
 });

src/index.ts

@@ -34,7 +34,7 @@ export = class OutputDcatUs11 {
 
       const datasetStream = await this.model.pullStream(req);
 
-      const dcatStream = getDataStreamDcatUs11(siteModel.item);
+      const dcatStream = getDataStreamDcatUs11(siteModel.item, _.get(siteModel, 'data.feeds.dcatUS11'));
 
       datasetStream
         .pipe(dcatStream)

src/test-helpers/mock-site-model.json

@@ -908,22 +908,6 @@
       },
       "internalUrl": "download-test-qa-pre-a-hub.hubqa.arcgis.com",
       "customHostname": null,
-      "dcatConfig": {
-        "title": "{{default.name}}",
-        "description": "{{default.description}}",
-        "keyword": "{{item.tags}}",
-        "issued": "{{item.created:toISO}}",
-        "modified": "{{item.modified:toISO}}",
-        "publisher": { "name": "{{default.source.source}}" },
-        "contactPoint": {
-          "fn": "{{item.owner}}",
-          "hasEmail": "{{org.portalProperties.links.contactUs.url}}"
-        },
-        "anotherThing": "again",
-        "landingPage": "some silly standard",
-        "bureauCode": ["010:86", "010:04"],
-        "programCode": ["015:001", "015:002"]
-      },
       "telemetry": {
         "consentNotice": {
           "isTheme": true,