filter search down to required fields

src/dcat-us/_generate-distributions.ts

@@ -3,6 +3,12 @@ import * as _ from 'lodash';
 const WFS_SERVER = 'WFSServer';
 const WMS_SERVER = 'WMSServer';
 
+export const DISTRIBUTION_DEPENDENCIES = [
+  'layer.geometryType',
+  'server.spatialReference',
+  'metadata.metadata.distInfo.distTranOps.onLineSrc'
+];
+
 /*
 * Generate DCAT Distributions
 */

src/dcat-us/base-dataset-template.ts

@@ -10,7 +10,7 @@ export const baseDatasetTemplate = {
   contactPoint: {
     '@type': 'vcard:Contact',
     fn: '{{owner}}',
-    // hasEmail: '{{org.portalProperties.links.contactUs.url:optional}}' TODO - re-enable when supported
+    hasEmail: '{{orgContactEmail:optional}}'
   },
   accessLevel: 'public'
 };

src/dcat-us/dataset-formatter.test.ts

@@ -1,4 +1,4 @@
-import { dcatTemplate, formatDcatDataset } from './dataset-formatter';
+import { buildDatasetTemplate, formatDcatDataset } from './dataset-formatter';
 
 it('dcatHelper: it does not allow customizations to overwrite critical fields', () => {
   const customizations = {
@@ -13,27 +13,27 @@ it('dcatHelper: it does not allow customizations to overwrite critical fields',
       hasEmail: 'mailto:dcat.support@dc.gov',
     },
   };
-  const template = dcatTemplate(customizations);
+  const template = buildDatasetTemplate(customizations);
   expect(template['@type']).not.toBe('SABOTAGE');
   expect(template.contactPoint['@type']).not.toBe('SABOTAGE');
 });
 
 it('dcatHelper: it does not throw an error if there are no customizations', () => {
   const customizations = undefined;
-  const template = dcatTemplate(customizations);
+  const template = buildDatasetTemplate(customizations);
   expect(template).toBeTruthy();
 });
 
 it('dcatHelper: it does not throw an error customizations are null', () => {
   const customizations = null;
-  const template = dcatTemplate(customizations);
+  const template = buildDatasetTemplate(customizations);
   expect(template).toBeTruthy();
 });
 
 describe('formatDcatDataset', () => {
   const siteUrl = 'https://foobar.hub.arcgis.com';
 
-  it('object: should render links with the correct SRID', () => {
+  it('should render links with the correct SRID', () => {
     const dataset = {
       owner: 'fpgis.CALFIRE',
       created: 1570747289000,
@@ -131,11 +131,11 @@ describe('formatDcatDataset', () => {
       spatial: '-123.8832,35.0024,-118.3281,42.0122',
       theme: ['geospatial'],
     };
-    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl));
+    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate()));
     expect(actual).toEqual(expected);
   });
 
-  it('object: should render links without the srid', () => {
+  it('should render links without the srid', () => {
     const dataset = {
       owner: 'fpgis.CALFIRE',
       created: 1570747289000,
@@ -220,11 +220,11 @@ describe('formatDcatDataset', () => {
       spatial: '-123.8832,35.0024,-118.3281,42.0122',
       theme: ['geospatial'],
     };
-    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl));
+    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate()));
     expect(actual).toEqual(expected);
   });
 
-  it('object: should override theme when theme customizations is an empty array', () => {
+  it('should override theme when theme customizations is an empty array', () => {
     const dataset = {
       owner: 'fpgis.CALFIRE',
       created: 1570747289000,
@@ -310,12 +310,12 @@ describe('formatDcatDataset', () => {
       theme: ['geospatial'],
     };
     const actual = JSON.parse(
-      formatDcatDataset(dataset, siteUrl, { theme: [] }),
+      formatDcatDataset(dataset, siteUrl, buildDatasetTemplate({ theme: [] })),
     );
     expect(actual).toEqual(expected);
   });
 
-  it('object: should _NOT_ override theme when theme customizations is supplied', () => {
+  it('should _NOT_ override theme when theme customizations is supplied', () => {
     const dataset = {
       owner: 'fpgis.CALFIRE',
       created: 1570747289000,
@@ -401,12 +401,12 @@ describe('formatDcatDataset', () => {
       theme: ['my theme'],
     };
     const actual = JSON.parse(
-      formatDcatDataset(dataset, siteUrl, { theme: ['my theme'] }),
+      formatDcatDataset(dataset, siteUrl, buildDatasetTemplate({ theme: ['my theme'] })),
     );
     expect(actual).toEqual(expected);
   });
 
-  it('object: should render links with the correct SRID', () => {
+  it('should render links with the correct SRID', () => {
     const dataset = {
       owner: 'fpgis.CALFIRE',
       created: 1570747289000,
@@ -495,11 +495,11 @@ describe('formatDcatDataset', () => {
       spatial: '-123.8832,35.0024,-118.3281,42.0122',
       theme: ['geospatial'],
     };
-    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl));
+    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate()));
     expect(actual).toEqual(expected);
   });
 
-  it('object: license should display structuredLicense url', () => {
+  it('license should display structuredLicense url', () => {
     const dataset = {
       owner: 'fpgis.CALFIRE',
       created: 1570747289000,
@@ -531,11 +531,11 @@ describe('formatDcatDataset', () => {
     };
     const expectedLicense = 'https://google.com';
 
-    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl));
+    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate()));
     expect(actual.license).toEqual(expectedLicense);
   });
 
-  it('object: license should be empty when structuredLicense.url is unavailable', () => {
+  it('license should be empty when structuredLicense.url is unavailable', () => {
     const dataset = {
       owner: 'fpgis.CALFIRE',
       created: 1570747289000,
@@ -567,11 +567,11 @@ describe('formatDcatDataset', () => {
     };
     const expectedLicense = null;
 
-    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl));
+    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate()));
     expect(actual.license).toEqual(expectedLicense);
   });
 
-  it('object: license should display null when structuredLicense is unavailable', () => {
+  it('license should display null when structuredLicense is unavailable', () => {
     const dataset = {
       owner: 'fpgis.CALFIRE',
       created: 1570747289000,
@@ -602,11 +602,11 @@ describe('formatDcatDataset', () => {
     };
     const expectedLicense = null;
 
-    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl));
+    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate()));
     expect(actual.license).toEqual(expectedLicense);
   });
 
-  test('object: Hub Page gets default keyword when no tags', () => {
+  test('Hub Page gets default keyword when no tags', () => {
     const datasetWithNoTags = {
       owner: 'fpgis.CALFIRE',
       type: 'Hub Page',
@@ -647,20 +647,20 @@ describe('formatDcatDataset', () => {
     };
     const expectedKeyword = 'ArcGIS Hub page';
     expect(
-      JSON.parse(formatDcatDataset(datasetWithNoTags, siteUrl)).keyword[0],
+      JSON.parse(formatDcatDataset(datasetWithNoTags, siteUrl, buildDatasetTemplate())).keyword[0],
     ).toBe(expectedKeyword);
     expect(
-      JSON.parse(formatDcatDataset({ ...datasetWithNoTags, tags: [] }, siteUrl))
+      JSON.parse(formatDcatDataset({ ...datasetWithNoTags, tags: [] }, siteUrl, buildDatasetTemplate()))
         .keyword[0],
     ).toBe(expectedKeyword);
     expect(
       JSON.parse(
-        formatDcatDataset({ ...datasetWithNoTags, tags: [''] }, siteUrl),
+        formatDcatDataset({ ...datasetWithNoTags, tags: [''] }, siteUrl, buildDatasetTemplate()),
       ).keyword[0],
     ).toBe(expectedKeyword);
   });
 
-  it('object: should create custom distribution when data is supplied', () => {
+  it('should create custom distribution when data is supplied', () => {
     const dataset = {
       owner: 'fpgis.CALFIRE',
       created: 1570747289000,
@@ -712,11 +712,11 @@ describe('formatDcatDataset', () => {
       description: 'endpoint',
     };
 
-    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl));
+    const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate()));
     expect(actual.distribution.pop()).toEqual(expectedDistribution);
   });
 
-  it('object: doesnt blow up when onLineSrc is not an array', () => {
+  it('doesnt blow up when onLineSrc is not an array', () => {
     const dataset = {
       owner: 'fpgis.CALFIRE',
       created: 1570747289000,
@@ -757,7 +757,7 @@ describe('formatDcatDataset', () => {
     };
 
     try {
-      formatDcatDataset(dataset, siteUrl);
+      formatDcatDataset(dataset, siteUrl, buildDatasetTemplate());
     } catch {
       fail('Should not throw!');
     }

src/dcat-us/dataset-formatter.ts

@@ -10,11 +10,9 @@ import { IItem } from '@esri/arcgis-rest-portal';
 type HubDatasetAttributes = Record<string, any>;
 export type DcatDatasetTemplate = Record<string, any>;
 
-export function formatDcatDataset (hubDataset: HubDatasetAttributes, siteUrl: string, dcatCustomizations: DcatDatasetTemplate = {}) {
+export function formatDcatDataset (hubDataset: HubDatasetAttributes, siteUrl: string, datasetTemplate: DcatDatasetTemplate) {
   const landingPage = `${siteUrl}/datasets/${hubDataset.id}`;
 
-  const template = dcatTemplate(dcatCustomizations);
-
   const { structuredLicense: { url = null } = {} } = hubDataset;
 
   const defaultDataset = {
@@ -34,7 +32,7 @@ export function formatDcatDataset (hubDataset: HubDatasetAttributes, siteUrl: st
     }
   };
 
-  const dcatDataset = Object.assign({}, defaultDataset, adlib(template, hubDataset, transforms));
+  const dcatDataset = Object.assign({}, defaultDataset, adlib(datasetTemplate, hubDataset, transforms));
 
   if (isPage(hubDataset as IItem) && !hasTags(hubDataset)) {
     dcatDataset.keyword = ['ArcGIS Hub page'];
@@ -47,7 +45,7 @@ export function formatDcatDataset (hubDataset: HubDatasetAttributes, siteUrl: st
 
     // https://project-open-data.cio.gov/v1.1/schema/#theme
     // allow theme to be overrriden
-    if (_.isEmpty(dcatCustomizations.theme)) {
+    if (_.isEmpty(datasetTemplate.theme)) {
       dcatDataset.theme = ['geospatial'];
     }
   }
@@ -83,7 +81,7 @@ function scrubProtectedKeys (customizations: DcatDatasetTemplate): DcatDatasetTe
   return scrubbedCustomizations;
 }
 
-export function dcatTemplate (customizations: DcatDatasetTemplate): DcatDatasetTemplate {
-  const customConfig = scrubProtectedKeys(customizations || {});
+export function buildDatasetTemplate (customizations: DcatDatasetTemplate = {}): DcatDatasetTemplate {
+  const customConfig = scrubProtectedKeys(customizations);
   return Object.assign({}, baseDatasetTemplate, customConfig);
 }

src/dcat-us/index.test.ts

@@ -4,22 +4,27 @@ import { getDataStreamDcatUs11 } from './';
 import * as datasetFromApi from '../test-helpers/mock-dataset.json';
 import * as mockSiteModel from '../test-helpers/mock-site-model.json';
 
-function generateDcatFeed(
-  siteItem,
-  datasets,
-  dcatCustomizations?
+import { DcatDatasetTemplate } from './dataset-formatter';
+import { IItem } from '@esri/arcgis-rest-types';
+
+async function generateDcatFeed(
+  siteItem: IItem,
+  datasets: any[],
+  dcatCustomizations?: DcatDatasetTemplate
 ) {
-  const dcatStream = getDataStreamDcatUs11(siteItem, dcatCustomizations);
+  const { stream: dcatStream, dependencies } = getDataStreamDcatUs11(siteItem, dcatCustomizations);
 
   const docStream = readableFromArray(datasets); // no datasets since we're just checking the catalog
 
-  return streamToString(docStream.pipe(dcatStream)).then(JSON.parse);
+  const feedString = await streamToString(docStream.pipe(dcatStream));
+
+  return { feed: JSON.parse(feedString), dependencies };
 }
 
 
 describe('generating DCAT-US 1.1 feed', () => {
   it('formats catalog correctly', async function () {
-    const feed = await generateDcatFeed(mockSiteModel.item, []);
+    const { feed } = await generateDcatFeed(mockSiteModel.item, []);
 
     expect(feed['@context']).toBe('https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld');
     expect(feed['@type']).toBe('dcat:Catalog');
@@ -30,7 +35,7 @@ describe('generating DCAT-US 1.1 feed', () => {
   });
 
   it('populates dataset array', async function () {
-    const feed = await generateDcatFeed(mockSiteModel.item, [
+    const { feed } = await generateDcatFeed(mockSiteModel.item, [
       datasetFromApi,
     ]);
 
@@ -56,8 +61,8 @@ describe('generating DCAT-US 1.1 feed', () => {
   });
 
   it('respects dcat customizations of overwritable attributes', async function () {
-    const feed = await generateDcatFeed(
-      mockSiteModel.item, 
+    const { feed } = await generateDcatFeed(
+      mockSiteModel.item,
       [datasetFromApi],
       {
         description: '{{name}}', // overwrite existing attribute
@@ -88,8 +93,8 @@ describe('generating DCAT-US 1.1 feed', () => {
   });
 
   it('scrubs dcat customization of protected fields', async function () {
-    const feed = await generateDcatFeed(
-      mockSiteModel.item, 
+    const { feed } = await generateDcatFeed(
+      mockSiteModel.item,
       [datasetFromApi],
       {
         '@type': '{{name}}',
@@ -125,4 +130,66 @@ describe('generating DCAT-US 1.1 feed', () => {
     expect(chk1.distribution).toBeInstanceOf(Array);
     expect(chk1.distribution.length).toBe(6);
   });
+
+  it('reports default dependencies when no customizations', async () => {
+    const { dependencies } = await generateDcatFeed(
+      mockSiteModel.item,
+      [datasetFromApi]
+    );
+
+    const expectedDependencies = [
+      'id',
+      'layer.geometryType',
+      'server.spatialReference',
+      'metadata.metadata.distInfo.distTranOps.onLineSrc',
+      'name',
+      'description',
+      'tags',
+      'created',
+      'modified',
+      'orgContactEmail',
+      'source',
+      'owner'
+    ];
+
+    expect(dependencies.length).toBe(expectedDependencies.length);
+    expect(dependencies).toEqual(expect.arrayContaining(expectedDependencies));
+  });
+
+  it('reports custom dependencies when customizations provided', async () => {
+    const { dependencies } = await generateDcatFeed(
+      mockSiteModel.item,
+      [datasetFromApi],
+      {
+        // overwrite some defaults
+        keyword: '{{keyword}}',
+        publisher: {
+          name: '{{other.source}}'
+        },
+        // net-new
+        foo: '{{foo.bar.baz}}',
+        lolz: '{{lol.lol.lol}}'
+      }
+    );
+
+    const expectedDependencies = [
+      'id',
+      'layer.geometryType',
+      'server.spatialReference',
+      'metadata.metadata.distInfo.distTranOps.onLineSrc',
+      'name',
+      'description',
+      'created',
+      'modified',
+      'owner',
+      'orgContactEmail',
+      'keyword',
+      'other.source',
+      'foo.bar.baz',
+      'lol.lol.lol'
+    ];
+
+    expect(dependencies.length).toBe(expectedDependencies.length);
+    expect(dependencies).toEqual(expect.arrayContaining(expectedDependencies));
+  });
 });