From 7243e5a8e26145cd7af4ab87e41871fd164e9731 Mon Sep 17 00:00:00 2001 From: stdavis Date: Mon, 25 Mar 2024 09:18:32 -0600 Subject: [PATCH] feat: add downloadMetadata check --- src/scripts/.gitignore | 1 + src/scripts/package-lock.json | 48 +++++++++++++++++++++++++++++ src/scripts/package.json | 1 + src/scripts/validate-sgid-index.mjs | 43 +++++++++++++++++++++++++- 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 src/scripts/.gitignore diff --git a/src/scripts/.gitignore b/src/scripts/.gitignore new file mode 100644 index 0000000000..16d3c4dbbf --- /dev/null +++ b/src/scripts/.gitignore @@ -0,0 +1 @@ +.cache diff --git a/src/scripts/package-lock.json b/src/scripts/package-lock.json index 8a50a420a7..a6b28d052d 100644 --- a/src/scripts/package-lock.json +++ b/src/scripts/package-lock.json @@ -15,6 +15,7 @@ "ky": "^1.2.2", "pg": "^8.11.3", "progress": "^2.0.3", + "ts-import": "^5.0.0-beta.0", "uuid": "^9.0.1" } }, @@ -227,6 +228,14 @@ "node": ">=14" } }, + "node_modules/comment-parser": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/comment-parser/-/comment-parser-1.3.1.tgz", + "integrity": "sha512-B52sN2VNghyq5ofvUsqZjmk6YkihBX5vMSChmSK9v4ShjKf3Vk5Xcmgpw4o+iIgtrnM/u5FiMpz9VKb8lpBveA==", + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -733,6 +742,11 @@ } } }, + "node_modules/options-defaults": { + "version": "2.0.40", + "resolved": "https://registry.npmjs.org/options-defaults/-/options-defaults-2.0.40.tgz", + "integrity": "sha512-a0oW0AMaP/Uqk1gU7s3unE83wzs/MACy3wsCnNREn4wqp4KCcxRdulRjf0d2FeIxENbGJ4EBGtHTQ6J30XB6Cw==" + }, "node_modules/packet-reader": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", @@ -1076,6 +1090,40 @@ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, + "node_modules/ts-import": { + "version": "5.0.0-beta.0", + "resolved": "https://registry.npmjs.org/ts-import/-/ts-import-5.0.0-beta.0.tgz", + "integrity": "sha512-YOe/xCmwDWughfeaAaGJ4UWzlCKNnt9e+oda3St6mUMkRJCTBhBso+7XApIijw7Mr9SS6NLOdav8i5EJrx7UVQ==", + "dependencies": { + "comment-parser": "1.3.1", + "options-defaults": "2.0.40", + "tslib": "2.5.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "typescript": "5" + } + }, + "node_modules/tslib": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", + "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" + }, + "node_modules/typescript": { + "version": "5.4.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.3.tgz", + "integrity": "sha512-KrPd3PKaCLr78MalgiwJnA25Nm8HAmdwN3mYUYZgG/wizIo9EainNVQI9/yDavtVFRN2h3k8uf3GLHuhDMgEHg==", + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", diff --git a/src/scripts/package.json b/src/scripts/package.json index cbbaa41c0c..0a387012c6 100644 --- a/src/scripts/package.json +++ b/src/scripts/package.json @@ -9,6 +9,7 @@ "ky": "^1.2.2", "pg": "^8.11.3", "progress": "^2.0.3", + "ts-import": "^5.0.0-beta.0", "uuid": "^9.0.1" } } diff --git a/src/scripts/validate-sgid-index.mjs b/src/scripts/validate-sgid-index.mjs index e6443d2499..67da9c71f4 100644 --- a/src/scripts/validate-sgid-index.mjs +++ b/src/scripts/validate-sgid-index.mjs @@ -2,9 +2,12 @@ import { GoogleAuth, auth } from 'google-auth-library'; import { GoogleSpreadsheet } from 'google-spreadsheet'; import ky from 'ky'; import ProgressBar from 'progress'; +import * as tsImport from 'ts-import'; import { v4 as uuid } from 'uuid'; import { validateOpenDataUrl, validateOpenSgidTableName, validateUrl } from './utilities.mjs'; +const downloadMetadata = await tsImport.load('../data/downloadMetadata.ts'); + const spreadsheetId = '11ASS7LnxgpnD0jN4utzklREgMf1pcvYjcXcIcESHweQ'; const scopes = ['https://www.googleapis.com/auth/spreadsheets', 'https://www.googleapis.com/auth/drive']; @@ -137,6 +140,36 @@ async function duplicates(row) { } } +async function downloadMetadataCheck(row) { + const name = row.get('hubName'); + const metadata = downloadMetadata.dataPages[name]; + + if (!metadata || metadata.featureServiceId === null) { + return; + } + + const checks = [ + // sgid index field, metadata field + ['itemId', 'itemId'], + ['hubName', 'name'], + ['serverServiceName', 'featureServiceId'], + ['openSgidTableName', 'openSgid'], + ['serverLayerId', 'layerId'], + ]; + + for (const [sgidIndexField, metadataField] of checks) { + const sgidIndexValue = row.get(sgidIndexField); + const metadataValue = metadata[metadataField]; + + if (sgidIndexValue !== metadataValue) { + recordError( + `downloadMetadata(${name}): "${metadataField}" does not match SGID Index column "${sgidIndexField}"`, + row, + ); + } + } +} + const duplicateLookups = { openSgidTableName: {}, itemId: {}, @@ -162,7 +195,15 @@ function buildDuplicateLookups(rows) { } } -const checks = [openSGIDTableName, productPage, idGuid, itemId, duplicates]; +const checks = [ + // these functions must return a promise + openSGIDTableName, + productPage, + idGuid, + itemId, + duplicates, + downloadMetadataCheck, +]; const rows = await worksheet.getRows(); buildDuplicateLookups(rows);