diff --git a/CHANGELOG.md b/CHANGELOG.md index 430c4ca..1c75e7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG.md +## Unreleased +Fixed +- Landing page and identifier urls are now the canonical hub url for the item [#13](https://github.com/koopjs/koop-output-dcat-us-11/pull/13) +- Hub Pages included in the feed no longer have 404ing urls [#13](https://github.com/koopjs/koop-output-dcat-us-11/pull/13) + ## 1.3.1 Fixed - Fixed landingPage and identifier urls to be custom domain rather than opendata subdomain diff --git a/example-app/package-lock.json b/example-app/package-lock.json index f142d01..fc9681f 100644 --- a/example-app/package-lock.json +++ b/example-app/package-lock.json @@ -233,11 +233,11 @@ "@esri/arcgis-rest-auth": "^3.2.1", "@esri/arcgis-rest-portal": "^3.2.1", "@esri/arcgis-rest-request": "^3.2.1", - "@esri/hub-common": "^8.13.2", - "@esri/hub-initiatives": "^8.13.2", - "@esri/hub-search": "^8.14.0", - "@esri/hub-sites": "^8.13.2", - "@esri/hub-teams": "^8.13.2", + "@esri/hub-common": "^9.8.0", + "@esri/hub-initiatives": "^9.8.0", + "@esri/hub-search": "^9.8.0", + "@esri/hub-sites": "^9.8.0", + "@esri/hub-teams": "^9.8.0", "config": "^3.3.6", "lodash": "^4.17.21", "tslib": "~2.3.0" @@ -296,11 +296,13 @@ "integrity": "sha512-lVOAhu0qvhQEm5BFSOp8VxqxkAd5dBa7ZNXhtuDNNyELy38R0C/0zt3ri4eQLpMIZG4oip/djinOLgzAdQZzBw==" }, "@esri/hub-common": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/@esri/hub-common/-/hub-common-8.15.0.tgz", - "integrity": "sha512-uHnZdI0xsaYaOySEJtPyyFaZzKL3rghsHLdiKix33k2Y0uAP4mrOkVn2gTdIgxVZDDXJrv8QuPnVqytSLpnD+A==", + "version": "9.8.0", + "resolved": "https://registry.npmjs.org/@esri/hub-common/-/hub-common-9.8.0.tgz", + "integrity": "sha512-pfk3YqW1Vf0MJ06jy8netvFnHMnfQzq6ScIfep6CUsEhueJrD5VrGMo3ENtVfKX4xZb3u4ESxaHQcGrgbngigw==", "requires": { + "abab": "^2.0.5", "adlib": "^3.0.7", + "jsonapi-typescript": "^0.1.3", "tslib": "^1.13.0" }, "dependencies": { @@ -312,9 +314,9 @@ } }, "@esri/hub-initiatives": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/@esri/hub-initiatives/-/hub-initiatives-8.15.0.tgz", - "integrity": "sha512-7/pMoq89RCQT46LpzNLPlbXFPyEh0yE9JXi0DD3akaORP0kkee9e91mPImUSiTLgr0s2dEt4cB/gBc4O5kDejA==", + "version": "9.8.0", + "resolved": "https://registry.npmjs.org/@esri/hub-initiatives/-/hub-initiatives-9.8.0.tgz", + "integrity": "sha512-uBEjDXDEyDfW4nr0RH3mU86p+BT8pum7aKAWFYke4uoNqESp46mJVsV9/MMFtGDcHBL93jVW0y/6Fa69wKgL5A==", "requires": { "tslib": "^1.13.0" }, @@ -327,11 +329,10 @@ } }, "@esri/hub-search": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/@esri/hub-search/-/hub-search-8.15.0.tgz", - "integrity": "sha512-gMsU+VjfJwf7nqkn4ZOJhWA/HLAelzisc5ucXXyMnfbd3Hz5BXIfptZRu41GqQMbtwPg/bzaOFwx6ONbrboD6w==", + "version": "9.8.0", + "resolved": "https://registry.npmjs.org/@esri/hub-search/-/hub-search-9.8.0.tgz", + "integrity": "sha512-xtf9gTlz3sfOdsLmxQPTBEf1raONIacpPEs93OFDvGr79wSctJokPSiKan0IBvUX+XAOphE7bRjWmoqKf2tIBg==", "requires": { - "base-64": "^1.0.0", "tslib": "^1.13.0" }, "dependencies": { @@ -343,9 +344,9 @@ } }, "@esri/hub-sites": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/@esri/hub-sites/-/hub-sites-8.15.0.tgz", - "integrity": "sha512-wXhFEnV343cW+u//KzUVVguEVoScUXp/4xXALPgLq5H6lZJfUEpIsCpMSQbkR10Bb1oBWMtfZTOQgFpqLa6xmA==", + "version": "9.8.0", + "resolved": "https://registry.npmjs.org/@esri/hub-sites/-/hub-sites-9.8.0.tgz", + "integrity": "sha512-9iButpUXYuct/U5f2gpqjBma5ten/MzohTq7EFbX7T2KBKvJOi9UR8QqfnyBxhjyY9tTMUFEXYehR83GSZ1vAg==", "requires": { "tslib": "^1.13.0" }, @@ -358,9 +359,9 @@ } }, "@esri/hub-teams": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/@esri/hub-teams/-/hub-teams-8.15.0.tgz", - "integrity": "sha512-B4XtQaIYFnkENswpETxoci7LNU/9RpHjYItB/Jfrwt/kGlpA/KGS11YwTSVfv7nMFxrm1p1TC/T1sXgEVgJrEw==", + "version": "9.8.0", + "resolved": "https://registry.npmjs.org/@esri/hub-teams/-/hub-teams-9.8.0.tgz", + "integrity": "sha512-YqZc8bQARxKsCGbAsEoCv789+sbwSh0W8zRzM78qW6wADbCgd1qKpZ2JhenM4cRSIQzr9FJI0dVnD3Ofn8nCBQ==", "requires": { "tslib": "^1.13.0" }, @@ -372,6 +373,11 @@ } } }, + "abab": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz", + "integrity": "sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q==" + }, "adlib": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/adlib/-/adlib-3.0.7.tgz", @@ -380,11 +386,6 @@ "esm": "^3.2.25" } }, - "base-64": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/base-64/-/base-64-1.0.0.tgz", - "integrity": "sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==" - }, "config": { "version": "3.3.6", "resolved": "https://registry.npmjs.org/config/-/config-3.3.6.tgz", @@ -398,6 +399,11 @@ "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==" }, + "json-typescript": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/json-typescript/-/json-typescript-1.1.2.tgz", + "integrity": "sha512-Np07MUsYMKbB0nNlw/MMIRjUK7ehO48LA4FsrzrhCfTUxMKbvOBAo0sc0b4nQ80ge9d32sModCunCgoyUojgUA==" + }, "json5": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz", @@ -406,6 +412,14 @@ "minimist": "^1.2.5" } }, + "jsonapi-typescript": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/jsonapi-typescript/-/jsonapi-typescript-0.1.3.tgz", + "integrity": "sha512-uPcPS01GeM+4HIyn18s7l1yD2S3uLZy2TX1UkQffCM0bLb3TMwudXUyVXPHTMZ4vdZT8MqKqN2vjB5PogTAdFQ==", + "requires": { + "json-typescript": "^1.0.0" + } + }, "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", diff --git a/package-lock.json b/package-lock.json index ef8db90..10f59c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -597,11 +597,13 @@ "integrity": "sha512-lVOAhu0qvhQEm5BFSOp8VxqxkAd5dBa7ZNXhtuDNNyELy38R0C/0zt3ri4eQLpMIZG4oip/djinOLgzAdQZzBw==" }, "@esri/hub-common": { - "version": "8.13.2", - "resolved": "https://registry.npmjs.org/@esri/hub-common/-/hub-common-8.13.2.tgz", - "integrity": "sha512-bxYoqLD2dl8t2enly6e0WzIqPYoX+Rm0ndddHJItCmhP25I+CPR+5km7iIrBsSFPCzINk6Lr/2fe4ew+km7zOA==", + "version": "9.8.0", + "resolved": "https://registry.npmjs.org/@esri/hub-common/-/hub-common-9.8.0.tgz", + "integrity": "sha512-pfk3YqW1Vf0MJ06jy8netvFnHMnfQzq6ScIfep6CUsEhueJrD5VrGMo3ENtVfKX4xZb3u4ESxaHQcGrgbngigw==", "requires": { + "abab": "^2.0.5", "adlib": "^3.0.7", + "jsonapi-typescript": "^0.1.3", "tslib": "^1.13.0" }, "dependencies": { @@ -613,9 +615,9 @@ } }, "@esri/hub-initiatives": { - "version": "8.13.2", - "resolved": "https://registry.npmjs.org/@esri/hub-initiatives/-/hub-initiatives-8.13.2.tgz", - "integrity": "sha512-o/n8ezOFRo7uEkMcy3WwEAIBL/I+JYt1jSKDUYqXaZuu6Fqz1KGTuqAKwGvIK3Jd7AWJzgkDz1mMbn9OBQfmuA==", + "version": "9.8.0", + "resolved": "https://registry.npmjs.org/@esri/hub-initiatives/-/hub-initiatives-9.8.0.tgz", + "integrity": "sha512-uBEjDXDEyDfW4nr0RH3mU86p+BT8pum7aKAWFYke4uoNqESp46mJVsV9/MMFtGDcHBL93jVW0y/6Fa69wKgL5A==", "requires": { "tslib": "^1.13.0" }, @@ -628,11 +630,10 @@ } }, "@esri/hub-search": { - "version": "8.14.0", - "resolved": "https://registry.npmjs.org/@esri/hub-search/-/hub-search-8.14.0.tgz", - "integrity": "sha512-K43+xuDxhVljCRMbIyiuiI/Fdd5Il34kFCkbJKo/o3RdpAE5akcCosmT6g0mgR+2QSt//Abg9hlDz+pbpgBoGg==", + "version": "9.8.0", + "resolved": "https://registry.npmjs.org/@esri/hub-search/-/hub-search-9.8.0.tgz", + "integrity": "sha512-xtf9gTlz3sfOdsLmxQPTBEf1raONIacpPEs93OFDvGr79wSctJokPSiKan0IBvUX+XAOphE7bRjWmoqKf2tIBg==", "requires": { - "base-64": "^1.0.0", "tslib": "^1.13.0" }, "dependencies": { @@ -644,9 +645,9 @@ } }, "@esri/hub-sites": { - "version": "8.13.2", - "resolved": "https://registry.npmjs.org/@esri/hub-sites/-/hub-sites-8.13.2.tgz", - "integrity": "sha512-zCa6G4DAA38l7DSqZVZuV7c46/IG1YUymVqln7Bw2eltE0B1rLZ895uDCvN0FfdVeeNMhCJ15wVb7V5lZ7Ex0Q==", + "version": "9.8.0", + "resolved": "https://registry.npmjs.org/@esri/hub-sites/-/hub-sites-9.8.0.tgz", + "integrity": "sha512-9iButpUXYuct/U5f2gpqjBma5ten/MzohTq7EFbX7T2KBKvJOi9UR8QqfnyBxhjyY9tTMUFEXYehR83GSZ1vAg==", "requires": { "tslib": "^1.13.0" }, @@ -659,9 +660,9 @@ } }, "@esri/hub-teams": { - "version": "8.13.2", - "resolved": "https://registry.npmjs.org/@esri/hub-teams/-/hub-teams-8.13.2.tgz", - "integrity": "sha512-35Kpl6piqJrrK/LmFo988znKl09Dd/tNBjn/DA3Cj+Y0XGH7KtN3yntGpjyysb4QMgx8/KECb84zzj9TAibVwg==", + "version": "9.8.0", + "resolved": "https://registry.npmjs.org/@esri/hub-teams/-/hub-teams-9.8.0.tgz", + "integrity": "sha512-YqZc8bQARxKsCGbAsEoCv789+sbwSh0W8zRzM78qW6wADbCgd1qKpZ2JhenM4cRSIQzr9FJI0dVnD3Ofn8nCBQ==", "requires": { "tslib": "^1.13.0" }, @@ -1838,8 +1839,7 @@ "abab": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz", - "integrity": "sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q==", - "dev": true + "integrity": "sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q==" }, "accepts": { "version": "1.3.7", @@ -2210,11 +2210,6 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, - "base-64": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/base-64/-/base-64-1.0.0.tgz", - "integrity": "sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==" - }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -5889,6 +5884,11 @@ "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", "dev": true }, + "json-typescript": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/json-typescript/-/json-typescript-1.1.2.tgz", + "integrity": "sha512-Np07MUsYMKbB0nNlw/MMIRjUK7ehO48LA4FsrzrhCfTUxMKbvOBAo0sc0b4nQ80ge9d32sModCunCgoyUojgUA==" + }, "json5": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz", @@ -5897,6 +5897,14 @@ "minimist": "^1.2.5" } }, + "jsonapi-typescript": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/jsonapi-typescript/-/jsonapi-typescript-0.1.3.tgz", + "integrity": "sha512-uPcPS01GeM+4HIyn18s7l1yD2S3uLZy2TX1UkQffCM0bLb3TMwudXUyVXPHTMZ4vdZT8MqKqN2vjB5PogTAdFQ==", + "requires": { + "json-typescript": "^1.0.0" + } + }, "jsonlint-lines": { "version": "1.7.1", "resolved": "https://registry.npmjs.org/jsonlint-lines/-/jsonlint-lines-1.7.1.tgz", diff --git a/package.json b/package.json index c575e60..66466c7 100644 --- a/package.json +++ b/package.json @@ -57,11 +57,11 @@ "@esri/arcgis-rest-auth": "^3.2.1", "@esri/arcgis-rest-portal": "^3.2.1", "@esri/arcgis-rest-request": "^3.2.1", - "@esri/hub-common": "^8.13.2", - "@esri/hub-initiatives": "^8.13.2", - "@esri/hub-search": "^8.14.0", - "@esri/hub-sites": "^8.13.2", - "@esri/hub-teams": "^8.13.2", + "@esri/hub-common": "^9.8.0", + "@esri/hub-initiatives": "^9.8.0", + "@esri/hub-search": "^9.8.0", + "@esri/hub-sites": "^9.8.0", + "@esri/hub-teams": "^9.8.0", "config": "^3.3.6", "lodash": "^4.17.21", "tslib": "~2.3.0" diff --git a/src/dcat-us/dataset-formatter.test.ts b/src/dcat-us/dataset-formatter.test.ts index ab313d9..cd0584f 100644 --- a/src/dcat-us/dataset-formatter.test.ts +++ b/src/dcat-us/dataset-formatter.test.ts @@ -1,3 +1,4 @@ +import { IModel } from '@esri/hub-common'; import { buildDatasetTemplate, formatDcatDataset } from './dataset-formatter'; it('dcatHelper: it does not allow customizations to overwrite critical fields', () => { @@ -32,6 +33,7 @@ it('dcatHelper: it does not throw an error customizations are null', () => { describe('formatDcatDataset', () => { const siteUrl = 'https://foobar.hub.arcgis.com'; + const siteModel = { item: { url: siteUrl } } as unknown as IModel; it('should render links with the correct SRID', () => { const dataset = { @@ -131,7 +133,7 @@ describe('formatDcatDataset', () => { spatial: '-123.8832,35.0024,-118.3281,42.0122', theme: ['geospatial'], }; - const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate())); + const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate())); expect(actual).toEqual(expected); }); @@ -220,7 +222,7 @@ describe('formatDcatDataset', () => { spatial: '-123.8832,35.0024,-118.3281,42.0122', theme: ['geospatial'], }; - const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate())); + const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate())); expect(actual).toEqual(expected); }); @@ -310,7 +312,7 @@ describe('formatDcatDataset', () => { theme: ['geospatial'], }; const actual = JSON.parse( - formatDcatDataset(dataset, siteUrl, buildDatasetTemplate({ theme: [] })), + formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ theme: [] })), ); expect(actual).toEqual(expected); }); @@ -401,7 +403,7 @@ describe('formatDcatDataset', () => { theme: ['my theme'], }; const actual = JSON.parse( - formatDcatDataset(dataset, siteUrl, buildDatasetTemplate({ theme: ['my theme'] })), + formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate({ theme: ['my theme'] })), ); expect(actual).toEqual(expected); }); @@ -495,7 +497,7 @@ describe('formatDcatDataset', () => { spatial: '-123.8832,35.0024,-118.3281,42.0122', theme: ['geospatial'], }; - const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate())); + const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate())); expect(actual).toEqual(expected); }); @@ -531,7 +533,7 @@ describe('formatDcatDataset', () => { }; const expectedLicense = 'https://google.com'; - const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate())); + const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate())); expect(actual.license).toEqual(expectedLicense); }); @@ -567,7 +569,7 @@ describe('formatDcatDataset', () => { }; const expectedLicense = 'licenseInfo text'; - const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate())); + const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate())); expect(actual.license).toEqual(expectedLicense); }); @@ -601,7 +603,7 @@ describe('formatDcatDataset', () => { }; const expectedLicense = ''; - const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate())); + const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate())); expect(actual.license).toEqual(expectedLicense); }); @@ -646,15 +648,15 @@ describe('formatDcatDataset', () => { }; const expectedKeyword = 'ArcGIS Hub page'; expect( - JSON.parse(formatDcatDataset(datasetWithNoTags, siteUrl, buildDatasetTemplate())).keyword[0], + JSON.parse(formatDcatDataset(datasetWithNoTags, siteUrl, siteModel, buildDatasetTemplate())).keyword[0], ).toBe(expectedKeyword); expect( - JSON.parse(formatDcatDataset({ ...datasetWithNoTags, tags: [] }, siteUrl, buildDatasetTemplate())) + JSON.parse(formatDcatDataset({ ...datasetWithNoTags, tags: [] }, siteUrl, siteModel, buildDatasetTemplate())) .keyword[0], ).toBe(expectedKeyword); expect( JSON.parse( - formatDcatDataset({ ...datasetWithNoTags, tags: [''] }, siteUrl, buildDatasetTemplate()), + formatDcatDataset({ ...datasetWithNoTags, tags: [''] }, siteUrl, siteModel, buildDatasetTemplate()), ).keyword[0], ).toBe(expectedKeyword); }); @@ -711,7 +713,7 @@ describe('formatDcatDataset', () => { description: 'endpoint', }; - const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, buildDatasetTemplate())); + const actual = JSON.parse(formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate())); expect(actual.distribution.pop()).toEqual(expectedDistribution); }); @@ -756,7 +758,7 @@ describe('formatDcatDataset', () => { }; try { - formatDcatDataset(dataset, siteUrl, buildDatasetTemplate()); + formatDcatDataset(dataset, siteUrl, siteModel, buildDatasetTemplate()); } catch { fail('Should not throw!'); } diff --git a/src/dcat-us/dataset-formatter.ts b/src/dcat-us/dataset-formatter.ts index 7df1765..77462e2 100644 --- a/src/dcat-us/dataset-formatter.ts +++ b/src/dcat-us/dataset-formatter.ts @@ -3,17 +3,20 @@ import { adlib, TransformsList } from 'adlib'; import { isPage } from '@esri/hub-sites'; import { baseDatasetTemplate } from './base-dataset-template'; import { _generateDistributions } from './_generate-distributions'; -import { cloneObject } from '@esri/hub-common'; +import { cloneObject, DatasetResource, datasetToContent, getContentSiteUrls, IModel } from '@esri/hub-common'; import { IItem } from '@esri/arcgis-rest-portal'; // TODO - use real type for hubDataset when it gets defined in Hub.js type HubDatasetAttributes = Record; export type DcatDatasetTemplate = Record; -export function formatDcatDataset (hubDataset: HubDatasetAttributes, siteUrl: string, datasetTemplate: DcatDatasetTemplate) { - const landingPage = siteUrl.startsWith('https://') - ? `${siteUrl}/datasets/${hubDataset.id}` - : `https://${siteUrl}/datasets/${hubDataset.id}`; +export function formatDcatDataset (hubDataset: HubDatasetAttributes, siteUrl: string, siteModel: IModel, datasetTemplate: DcatDatasetTemplate) { + const content = datasetToContent({ + id: hubDataset.id, + attributes: hubDataset + } as DatasetResource); + const { relative: relativePath } = getContentSiteUrls(content, siteModel); + const landingPage = siteUrl.startsWith('https://') ? siteUrl + relativePath : `https://${siteUrl}${relativePath}`; const { structuredLicense: { url = null } = {}, diff --git a/src/dcat-us/index.test.ts b/src/dcat-us/index.test.ts index d82ea2b..ee7454e 100644 --- a/src/dcat-us/index.test.ts +++ b/src/dcat-us/index.test.ts @@ -3,6 +3,7 @@ import { getDataStreamDcatUs11 } from './'; import * as datasetFromApi from '../test-helpers/mock-dataset.json'; import { DcatDatasetTemplate } from './dataset-formatter'; +import { IModel } from '@esri/hub-common'; const hostname = 'css-monster-qa-pre-hub.hubqa.arcgis.com'; @@ -11,7 +12,9 @@ async function generateDcatFeed( datasets: any[], dcatCustomizations?: DcatDatasetTemplate ) { - const { stream: dcatStream, dependencies } = getDataStreamDcatUs11(hostname, dcatCustomizations); + const siteModel = { item: { url: hostname } } as unknown as IModel; + + const { stream: dcatStream, dependencies } = getDataStreamDcatUs11(hostname, siteModel, dcatCustomizations); const docStream = readableFromArray(datasets); // no datasets since we're just checking the catalog @@ -40,9 +43,9 @@ describe('generating DCAT-US 1.1 feed', () => { const chk1 = feed['dataset'][0]; expect(chk1['@type']).toBe('dcat:Dataset'); - expect(chk1.identifier).toBe('https://css-monster-qa-pre-hub.hubqa.arcgis.com/datasets/f4bcc1035b7d46cba95e977f4affb6be_0'); + expect(chk1.identifier).toBe('https://css-monster-qa-pre-hub.hubqa.arcgis.com/datasets/qa-pre-a-hub::tahoe-places-of-interest'); expect(chk1.license).toBe(''); - expect(chk1.landingPage).toBe('https://css-monster-qa-pre-hub.hubqa.arcgis.com/datasets/f4bcc1035b7d46cba95e977f4affb6be_0'); + expect(chk1.landingPage).toBe('https://css-monster-qa-pre-hub.hubqa.arcgis.com/datasets/qa-pre-a-hub::tahoe-places-of-interest'); expect(chk1.title).toBe('Tahoe places of interest'); expect(chk1.description).toBe('Description. Here be Tahoe things. You can do a lot here. Here are some more words. And a few more.

with more words

adding a few more to test how long it takes for our jobs to execute.

Tom was here!
'); expect(chk1.keyword).toEqual([ 'Data collection', 'just modified' ]); @@ -71,9 +74,9 @@ describe('generating DCAT-US 1.1 feed', () => { const chk1 = feed['dataset'][0]; expect(chk1['@type']).toBe('dcat:Dataset'); - expect(chk1.identifier).toBe('https://css-monster-qa-pre-hub.hubqa.arcgis.com/datasets/f4bcc1035b7d46cba95e977f4affb6be_0'); + expect(chk1.identifier).toBe('https://css-monster-qa-pre-hub.hubqa.arcgis.com/datasets/qa-pre-a-hub::tahoe-places-of-interest'); expect(chk1.license).toBe(''); - expect(chk1.landingPage).toBe('https://css-monster-qa-pre-hub.hubqa.arcgis.com/datasets/f4bcc1035b7d46cba95e977f4affb6be_0'); + expect(chk1.landingPage).toBe('https://css-monster-qa-pre-hub.hubqa.arcgis.com/datasets/qa-pre-a-hub::tahoe-places-of-interest'); expect(chk1.title).toBe('Tahoe places of interest'); expect(chk1.description).toBe('Tahoe places of interest'); expect(chk1.customField).toBe('Tahoe places of interest'); @@ -110,10 +113,10 @@ describe('generating DCAT-US 1.1 feed', () => { const chk1 = feed['dataset'][0]; expect(chk1['@type']).toBe('dcat:Dataset'); - expect(chk1.identifier).toBe('https://css-monster-qa-pre-hub.hubqa.arcgis.com/datasets/f4bcc1035b7d46cba95e977f4affb6be_0'); + expect(chk1.identifier).toBe('https://css-monster-qa-pre-hub.hubqa.arcgis.com/datasets/qa-pre-a-hub::tahoe-places-of-interest'); expect(chk1.license).toBe(''); expect(chk1.webService).toBe(undefined); - expect(chk1.landingPage).toBe('https://css-monster-qa-pre-hub.hubqa.arcgis.com/datasets/f4bcc1035b7d46cba95e977f4affb6be_0'); + expect(chk1.landingPage).toBe('https://css-monster-qa-pre-hub.hubqa.arcgis.com/datasets/qa-pre-a-hub::tahoe-places-of-interest'); expect(chk1.title).toBe('Tahoe places of interest'); expect(chk1.description).toBe('Description. Here be Tahoe things. You can do a lot here. Here are some more words. And a few more.

with more words

adding a few more to test how long it takes for our jobs to execute.

Tom was here!
'); expect(chk1.keyword).toEqual([ 'Data collection', 'just modified' ]); @@ -137,6 +140,8 @@ describe('generating DCAT-US 1.1 feed', () => { const expectedDependencies = [ 'id', + 'type', + 'slug', 'licenseInfo', 'structuredLicense', 'layer.geometryType', @@ -176,6 +181,8 @@ describe('generating DCAT-US 1.1 feed', () => { const expectedDependencies = [ 'id', + 'type', + 'slug', 'licenseInfo', 'structuredLicense', 'layer.geometryType', diff --git a/src/dcat-us/index.ts b/src/dcat-us/index.ts index 87a2dcd..e655719 100644 --- a/src/dcat-us/index.ts +++ b/src/dcat-us/index.ts @@ -1,9 +1,10 @@ +import { IModel } from '@esri/hub-common'; import { listDependencies } from 'adlib'; import { buildDatasetTemplate, DcatDatasetTemplate, formatDcatDataset } from './dataset-formatter'; import { FeedFormatterStream } from './feed-formatter-stream'; import { DISTRIBUTION_DEPENDENCIES } from './_generate-distributions'; -export function getDataStreamDcatUs11(hostname: string, dcatCustomizations?: DcatDatasetTemplate) { +export function getDataStreamDcatUs11(siteUrl: string, siteModel: IModel, dcatCustomizations?: DcatDatasetTemplate) { const catalogStr = JSON.stringify({ '@context': 'https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld', @@ -22,13 +23,15 @@ export function getDataStreamDcatUs11(hostname: string, dcatCustomizations?: Dca const datasetTemplate = buildDatasetTemplate(dcatCustomizations); const formatFn = (chunk) => { - return formatDcatDataset(chunk, hostname, datasetTemplate); + return formatDcatDataset(chunk, siteUrl, siteModel, datasetTemplate); }; return { stream: new FeedFormatterStream(header, footer, ',\n', formatFn), dependencies: [ 'id', // used for the dataset landing page URL + 'type', // used for the dataset landing page URL + 'slug', // used for the dataset landing page URL 'licenseInfo', // required for license resolution 'structuredLicense', // required for license resolution ...DISTRIBUTION_DEPENDENCIES, diff --git a/src/index.test.ts b/src/index.test.ts index 0a40f2b..c2e2396 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -110,7 +110,7 @@ describe('Output Plugin', () => { }, options: { portal: 'https://www.arcgis.com', - fields: "id,licenseInfo,structuredLicense,layer,server,metadata,url,name,description,tags,created,modified,source,owner,orgContactEmail,extent" }, + fields: "id,type,slug,licenseInfo,structuredLicense,layer,server,metadata,url,name,description,tags,created,modified,source,owner,orgContactEmail,extent" }, }); }); @@ -193,7 +193,7 @@ describe('Output Plugin', () => { .expect('Content-Type', /application\/json/) .expect(200) .expect(() => { - expect(mockGetDataStreamDcatUs11).toHaveBeenCalledWith(siteHostName, customConfigSiteModel.data.feeds.dcatUS11); + expect(mockGetDataStreamDcatUs11).toHaveBeenCalledWith(siteHostName, customConfigSiteModel, customConfigSiteModel.data.feeds.dcatUS11); }); }); @@ -208,7 +208,7 @@ describe('Output Plugin', () => { .expect('Content-Type', /application\/json/) .expect(200) .expect(() => { - expect(mockGetDataStreamDcatUs11).toHaveBeenCalledWith(siteHostName, dcatConfig); + expect(mockGetDataStreamDcatUs11).toHaveBeenCalledWith(siteHostName, mockSiteModel, dcatConfig); }); }); @@ -224,7 +224,7 @@ describe('Output Plugin', () => { .expect('Content-Type', /application\/json/) .expect(200) .expect(() => { - expect(mockGetDataStreamDcatUs11).toHaveBeenCalledWith(siteHostName, dcatConfig); + expect(mockGetDataStreamDcatUs11).toHaveBeenCalledWith(siteHostName, mockSiteModel, dcatConfig); }); }); @@ -254,7 +254,7 @@ describe('Output Plugin', () => { .expect('Content-Type', /application\/json/) .expect(200) .expect(() => { - expect(mockGetDataStreamDcatUs11).toHaveBeenCalledWith(siteHostName, customConfigSiteModel.data.feeds.dcatUS11); + expect(mockGetDataStreamDcatUs11).toHaveBeenCalledWith(siteHostName, customConfigSiteModel, customConfigSiteModel.data.feeds.dcatUS11); }); }); @@ -305,7 +305,7 @@ describe('Output Plugin', () => { .expect('Content-Type', /application\/json/) .expect(200) .expect(() => { - expect(mockGetDataStreamDcatUs11).toHaveBeenCalledWith('css-monster-qa-pre-hub.hubqa.arcgis.com', customConfigSiteModel.data.feeds.dcatUS11); + expect(mockGetDataStreamDcatUs11).toHaveBeenCalledWith('css-monster-qa-pre-hub.hubqa.arcgis.com', customConfigSiteModel, customConfigSiteModel.data.feeds.dcatUS11); }); }); }); diff --git a/src/index.ts b/src/index.ts index 1b48ace..7b684cf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -45,7 +45,10 @@ export = class OutputDcatUs11 { dcatConfig = _.get(siteModel, 'data.feeds.dcatUS11'); } - const { stream: dcatStream, dependencies } = getDataStreamDcatUs11(hostname, dcatConfig); + // TODO: We only pass in hostname because some site item urls are out of sync, causing invalid urls for + // landingPage and identifier. If we can resolve the syncing issues, we can omit hostname and just use + // the absolute url we get from getContentSiteUrls() + const { stream: dcatStream, dependencies } = getDataStreamDcatUs11(hostname, siteModel, dcatConfig); const apiTerms = getApiTermsFromDependencies(dependencies);