From d10b6abaa76cc0571bff24b941d634dbde17f5d5 Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Fri, 17 Nov 2023 21:13:57 -0500 Subject: [PATCH 1/9] Drop support for Node.js < 18. --- .github/workflows/main.yml | 10 +++++----- CHANGELOG.md | 5 +++++ package.json | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 37bbc675..beb47bdc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [18.x] + node-version: [20.x] steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} @@ -23,7 +23,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [14.x, 16.x, 18.x, 20.x, 22.x] + node-version: [18.x, 20.x, 22.x] steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} @@ -41,7 +41,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [16.x] + node-version: [20.x] bundler: [webpack, browserify] steps: - uses: actions/checkout@v4 @@ -62,7 +62,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [16.x] + node-version: [20.x] steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} @@ -78,7 +78,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [18.x] + node-version: [20.x] steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 02bc075f..28d5ef21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # jsonld ChangeLog +## 9.0.0 - 2023-xx-xx + +### Changed +- **BREAKING**: Drop support for Node.js < 18. + ## 8.3.3 - 2024-12-21 ### Added diff --git a/package.json b/package.json index facc56b5..4ae66466 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "webpack-merge": "^5.8.0" }, "engines": { - "node": ">=14" + "node": ">=18" }, "keywords": [ "JSON", From b1da29fd8bb17be59ba53cc4a2f2fbd8ab3d659f Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Fri, 17 Nov 2023 22:52:10 -0500 Subject: [PATCH 2/9] Update to `rdf-canonize@4`. - **BREAKING**: See the `rdf-canonize` 4.0.0 changelog for **important** changes and upgrade notes. - Update to handle different RDF/JS dataset `BlankNode` format. - Enable pass through of numerous possible `rdf-canonize` options in a `canonize()` `canonizeOptions` parameter. - Update to use `rdf-canonize` options. - The `URDNA2015` default algorithm has been changed to `RDFC-1.0` from `rdf-canon`. - Complexity control defaults `maxWorkFactor` or `maxDeepIterations` may need to be adjusted to process graphs with certain blank node constructs. - A `signal` option is available to use an `AbortSignal` to limit resource usage. - The internal digest algorithm can be changed. --- CHANGELOG.md | 10 ++ lib/fromRdf.js | 43 +++++-- lib/jsonld.js | 46 ++++--- lib/toRdf.js | 66 ++++++---- package.json | 2 +- tests/misc.js | 2 +- tests/test.js | 339 +++++++++++++++++++++++++++++++------------------ 7 files changed, 330 insertions(+), 178 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28d5ef21..fc1dcb96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ ### Changed - **BREAKING**: Drop support for Node.js < 18. +- **BREAKING**: Upgrade dependencies. + - `rdf-canonize@4`: See the [rdf-canonize][] 4.0.0 changelog for + **important** changes and upgrade notes. Of note: + - The `URDNA2015` default algorithm has been changed to `RDFC-1.0` from + [rdf-canon][]. + - Complexity control defaults `maxWorkFactor` or `maxDeepIterations` may + need to be adjusted to process graphs with certain blank node constructs. + - A `signal` option is available to use an `AbortSignal` to limit resource + usage. + - The internal digest algorithm can be changed. ## 8.3.3 - 2024-12-21 diff --git a/lib/fromRdf.js b/lib/fromRdf.js index 01098353..795ec9af 100644 --- a/lib/fromRdf.js +++ b/lib/fromRdf.js @@ -89,7 +89,7 @@ api.fromRDF = async ( const nodeMap = graphMap[name]; // get subject, predicate, object - const s = quad.subject.value; + const s = _nodeId(quad.subject); const p = quad.predicate.value; const o = quad.object; @@ -98,13 +98,14 @@ api.fromRDF = async ( } const node = nodeMap[s]; - const objectIsNode = o.termType.endsWith('Node'); - if(objectIsNode && !(o.value in nodeMap)) { - nodeMap[o.value] = {'@id': o.value}; + const objectNodeId = _nodeId(o); + const objectIsNode = !!objectNodeId; + if(objectIsNode && !(objectNodeId in nodeMap)) { + nodeMap[objectNodeId] = {'@id': objectNodeId}; } if(p === RDF_TYPE && !useRdfType && objectIsNode) { - _addValue(node, '@type', o.value, {propertyIsArray: true}); + _addValue(node, '@type', objectNodeId, {propertyIsArray: true}); continue; } @@ -114,9 +115,9 @@ api.fromRDF = async ( // object may be an RDF list/partial list node but we can't know easily // until all triples are read if(objectIsNode) { - if(o.value === RDF_NIL) { + if(objectNodeId === RDF_NIL) { // track rdf:nil uniquely per graph - const object = nodeMap[o.value]; + const object = nodeMap[objectNodeId]; if(!('usages' in object)) { object.usages = []; } @@ -125,12 +126,12 @@ api.fromRDF = async ( property: p, value }); - } else if(o.value in referencedOnce) { + } else if(objectNodeId in referencedOnce) { // object referenced more than once - referencedOnce[o.value] = false; + referencedOnce[objectNodeId] = false; } else { // keep track of single reference - referencedOnce[o.value] = { + referencedOnce[objectNodeId] = { node, property: p, value @@ -303,8 +304,9 @@ api.fromRDF = async ( */ function _RDFToObject(o, useNativeTypes, rdfDirection, options) { // convert NamedNode/BlankNode object to JSON-LD - if(o.termType.endsWith('Node')) { - return {'@id': o.value}; + const nodeId = _nodeId(o); + if(nodeId) { + return {'@id': nodeId}; } // convert literal to JSON-LD @@ -397,3 +399,20 @@ function _RDFToObject(o, useNativeTypes, rdfDirection, options) { return rval; } + +/** + * Return id for a term. Handles BlankNodes and NamedNodes. Adds a '_:' prefix + * for BlanksNodes. + * + * @param term a term object. + * + * @return the Node term id or null. + */ +function _nodeId(term) { + if(term.termType === 'NamedNode') { + return term.value; + } else if(term.termType === 'BlankNode') { + return '_:' + term.value; + } + return null; +} diff --git a/lib/jsonld.js b/lib/jsonld.js index c6931aeb..a2a563bb 100644 --- a/lib/jsonld.js +++ b/lib/jsonld.js @@ -523,7 +523,7 @@ jsonld.link = async function(input, ctx, options) { /** * Performs RDF dataset normalization on the given input. The input is JSON-LD * unless the 'inputFormat' option is used. The output is an RDF dataset - * unless the 'format' option is used. + * unless a non-null 'format' option is used. * * Note: Canonicalization sets `safe` to `true` and `base` to `null` by * default in order to produce safe outputs and "fail closed" by default. This @@ -531,25 +531,31 @@ jsonld.link = async function(input, ctx, options) { * allow unsafe defaults (for cryptographic usage) in order to comply with the * JSON-LD 1.1 specification. * - * @param input the input to normalize as JSON-LD or as a format specified by - * the 'inputFormat' option. + * @param input the input to normalize as JSON-LD given as an RDF dataset or as + * a format specified by the 'inputFormat' option. * @param [options] the options to use: - * [algorithm] the normalization algorithm to use, `URDNA2015` or - * `URGNA2012` (default: `URDNA2015`). * [base] the base IRI to use (default: `null`). * [expandContext] a context to expand with. * [skipExpansion] true to assume the input is expanded and skip * expansion, false not to, defaults to false. Some well-formed * and safe-mode checks may be omitted. - * [inputFormat] the format if input is not JSON-LD: - * 'application/n-quads' for N-Quads. - * [format] the format if output is a string: - * 'application/n-quads' for N-Quads. + * [inputFormat] the input format. null for a JSON-LD object, + * 'application/n-quads' for N-Quads. (default: null) + * [format] the output format. null for an RDF dataset, + * 'application/n-quads' for an N-Quads string. (default: N-Quads) * [documentLoader(url, options)] the document loader. - * [useNative] true to use a native canonize algorithm * [rdfDirection] null or 'i18n-datatype' to support RDF * transformation of @direction (default: null). * [safe] true to use safe mode. (default: true). + * [canonizeOptions] options to pass to rdf-canonize canonize(). See + * rdf-canonize for more details. Commonly used options, and their + * defaults, are: + * algorithm="RDFC-1.0", + * messageDigestAlgorithm="sha256", + * canonicalIdMap, + * maxWorkFactor=1, + * maxDeepIterations=-1, + * and signal=null. * [contextResolver] internal use only. * * @return a Promise that resolves to the normalized output. @@ -559,15 +565,19 @@ jsonld.normalize = jsonld.canonize = async function(input, options) { throw new TypeError('Could not canonize, too few arguments.'); } - // set default options + // set toRDF options options = _setDefaults(options, { - base: _isString(input) ? input : null, - algorithm: 'URDNA2015', skipExpansion: false, safe: true, contextResolver: new ContextResolver( {sharedCache: _resolvedContextCache}) }); + + // set canonize options + const canonizeOptions = Object.assign({}, { + algorithm: 'RDFC-1.0' + }, options.canonizeOptions || null); + if('inputFormat' in options) { if(options.inputFormat !== 'application/n-quads' && options.inputFormat !== 'application/nquads') { @@ -579,17 +589,18 @@ jsonld.normalize = jsonld.canonize = async function(input, options) { const parsedInput = NQuads.parse(input); // do canonicalization - return canonize.canonize(parsedInput, options); + return canonize.canonize(parsedInput, canonizeOptions); } // convert to RDF dataset then do normalization const opts = {...options}; delete opts.format; + delete opts.canonizeOptions; opts.produceGeneralizedRdf = false; const dataset = await jsonld.toRDF(input, opts); // do canonicalization - return canonize.canonize(dataset, options); + return canonize.canonize(dataset, canonizeOptions); }; /** @@ -653,8 +664,8 @@ jsonld.fromRDF = async function(dataset, options) { * [skipExpansion] true to assume the input is expanded and skip * expansion, false not to, defaults to false. Some well-formed * and safe-mode checks may be omitted. - * [format] the format to use to output a string: - * 'application/n-quads' for N-Quads. + * [format] the output format. null for an RDF dataset, + * 'application/n-quads' for an N-Quads string. (default: null) * [produceGeneralizedRdf] true to output generalized RDF, false * to produce only standard RDF (default: false). * [documentLoader(url, options)] the document loader. @@ -672,7 +683,6 @@ jsonld.toRDF = async function(input, options) { // set default options options = _setDefaults(options, { - base: _isString(input) ? input : '', skipExpansion: false, contextResolver: new ContextResolver( {sharedCache: _resolvedContextCache}) diff --git a/lib/toRdf.js b/lib/toRdf.js index 53f20af4..e8a54844 100644 --- a/lib/toRdf.js +++ b/lib/toRdf.js @@ -63,12 +63,7 @@ api.toRDF = (input, options) => { if(graphName === '@default') { graphTerm = {termType: 'DefaultGraph', value: ''}; } else if(_isAbsoluteIri(graphName)) { - if(graphName.startsWith('_:')) { - graphTerm = {termType: 'BlankNode'}; - } else { - graphTerm = {termType: 'NamedNode'}; - } - graphTerm.value = graphName; + graphTerm = _makeTerm(graphName); } else { // skip relative IRIs (not valid RDF) if(options.eventHandler) { @@ -119,10 +114,7 @@ function _graphToRDF(dataset, graph, graphTerm, issuer, options) { for(const item of items) { // RDF subject - const subject = { - termType: id.startsWith('_:') ? 'BlankNode' : 'NamedNode', - value: id - }; + const subject = _makeTerm(id); // skip relative IRI subjects (not valid RDF) if(!_isAbsoluteIri(id)) { @@ -144,10 +136,7 @@ function _graphToRDF(dataset, graph, graphTerm, issuer, options) { } // RDF predicate - const predicate = { - termType: property.startsWith('_:') ? 'BlankNode' : 'NamedNode', - value: property - }; + const predicate = _makeTerm(property); // skip relative IRI predicates (not valid RDF) if(!_isAbsoluteIri(property)) { @@ -226,13 +215,16 @@ function _listToRDF(list, issuer, dataset, graphTerm, rdfDirection, options) { const last = list.pop(); // Result is the head of the list - const result = last ? {termType: 'BlankNode', value: issuer.getId()} : nil; + const result = last ? { + termType: 'BlankNode', + value: issuer.getId().slice(2) + } : nil; let subject = result; for(const item of list) { const object = _objectToRDF( item, issuer, dataset, graphTerm, rdfDirection, options); - const next = {termType: 'BlankNode', value: issuer.getId()}; + const next = {termType: 'BlankNode', value: issuer.getId().slice(2)}; dataset.push({ subject, predicate: first, @@ -284,14 +276,16 @@ function _listToRDF(list, issuer, dataset, graphTerm, rdfDirection, options) { function _objectToRDF( item, issuer, dataset, graphTerm, rdfDirection, options ) { - const object = {}; + let object; // convert value object to RDF if(graphTypes.isValue(item)) { - object.termType = 'Literal'; - object.value = undefined; - object.datatype = { - termType: 'NamedNode' + object = { + termType: 'Literal', + value: undefined, + datatype: { + termType: 'NamedNode' + } }; let value = item['@value']; const datatype = item['@type'] || null; @@ -374,13 +368,14 @@ function _objectToRDF( } else if(graphTypes.isList(item)) { const _list = _listToRDF( item['@list'], issuer, dataset, graphTerm, rdfDirection, options); - object.termType = _list.termType; - object.value = _list.value; + object = { + termType: _list.termType, + value: _list.value + }; } else { // convert string/node object to RDF const id = types.isObject(item) ? item['@id'] : item; - object.termType = id.startsWith('_:') ? 'BlankNode' : 'NamedNode'; - object.value = id; + object = _makeTerm(id); } // skip relative IRIs, not valid RDF @@ -404,3 +399,24 @@ function _objectToRDF( return object; } + +/** + * Make a term from an id. Handles BlankNodes and NamedNodes based on a + * possible '_:' id prefix. The prefix is removed for BlankNodes. + * + * @param id a term id. + * + * @return a term object. + */ +function _makeTerm(id) { + if(id.startsWith('_:')) { + return { + termType: 'BlankNode', + value: id.slice(2) + }; + } + return { + termType: 'NamedNode', + value: id + }; +} diff --git a/package.json b/package.json index 4ae66466..c6d8168c 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "@digitalbazaar/http-client": "^3.4.1", "canonicalize": "^1.0.1", "lru-cache": "^6.0.0", - "rdf-canonize": "^3.4.0" + "rdf-canonize": "^4.0.1" }, "devDependencies": { "@babel/core": "^7.21.8", diff --git a/tests/misc.js b/tests/misc.js index 908052cd..21b3d25e 100644 --- a/tests/misc.js +++ b/tests/misc.js @@ -4030,7 +4030,7 @@ _:b0 "v" . ] ; const nq = `\ -_:b0 <_:b1> "v" . +_:b0 _:b1 "v" . `; await _test({ diff --git a/tests/test.js b/tests/test.js index 998647ed..ab98a9d4 100644 --- a/tests/test.js +++ b/tests/test.js @@ -352,38 +352,73 @@ const TEST_TYPES = { ], compare: compareCanonizedExpectedNQuads }, - 'rdfc:Urgna2012EvalTest': { - fn: 'normalize', + 'rdfc:RDFC10EvalTest': { + skip: { + // NOTE: idRegex format: + // /manifest-urdna2015#testNNN$/, + // FIXME + idRegex: [ + // Unsupported U escape + // /manifest-urdna2015#test060/ + ] + }, + fn: 'canonize', params: [ readTestNQuads('action'), createTestOptions({ - algorithm: 'URGNA2012', + algorithm: 'RDFC-1.0', inputFormat: 'application/n-quads', format: 'application/n-quads' }) ], compare: compareExpectedNQuads }, - 'rdfc:Urdna2015EvalTest': { + 'rdfc:RDFC10NegativeEvalTest': { skip: { // NOTE: idRegex format: - // /manifest-urdna2015#testNNN$/, - // FIXME - idRegex: [ - // Unsupported U escape - /manifest-urdna2015#test060/ - ] + // /manifest-rdfc10#testNNN$/, + idRegex: [] }, fn: 'canonize', params: [ readTestNQuads('action'), createTestOptions({ - algorithm: 'URDNA2015', + algorithm: 'RDFC-1.0', + inputFormat: 'application/n-quads', + format: 'application/n-quads' + }) + ] + }, + 'rdfc:RDFC10MapTest': { + skip: { + // NOTE: idRegex format: + // /manifest-rdfc10#testNNN$/, + idRegex: [] + }, + fn: 'canonize', + params: [ + readTestNQuads('action'), + createTestOptions({ + algorithm: 'RDFC-1.0', inputFormat: 'application/n-quads', format: 'application/n-quads' }) ], - compare: compareExpectedNQuads + preRunAdjustParams: ({params, extra}) => { + // add canonicalIdMap + const m = new Map(); + extra.canonicalIdMap = m; + params[1].canonizeOptions = params[1].canonizeOptions || {}; + params[1].canonizeOptions.canonicalIdMap = m; + return params; + }, + postRunAdjustParams: ({params}) => { + // restore output param to empty map + const m = new Map(); + params[1].canonizeOptions = params[1].canonizeOptions || {}; + params[1].canonizeOptions.canonicalIdMap = m; + }, + compare: compareExpectedCanonicalIdMap } }; @@ -432,8 +467,6 @@ if(options.earl && options.earl.filename) { } } -return new Promise(resolve => { - // async generated tests // _tests => [{suite}, ...] // suite => { @@ -443,21 +476,20 @@ return new Promise(resolve => { // } const _tests = []; -return addManifest(manifest, _tests) - .then(() => { - return _testsToMocha(_tests); - }).then(result => { - if(options.earl.report) { - describe('Writing EARL report to: ' + options.earl.filename, function() { - // print out EARL even if .only was used - const _it = result.hadOnly ? it.only : it; - _it('should print the earl report', function() { - return options.writeFile( - options.earl.filename, options.earl.report.reportJson()); - }); - }); - } - }).then(() => resolve()); +await addManifest(manifest, _tests); +const result = _testsToMocha(_tests); +if(options.earl.report) { + describe('Writing EARL report to: ' + options.earl.filename, function() { + // print out EARL even if .only was used + const _it = result.hadOnly ? it.only : it; + _it('should print the earl report', function() { + return options.writeFile( + options.earl.filename, options.earl.report.reportJson()); + }); + }); +} + +return; // build mocha tests from local test structure function _testsToMocha(tests) { @@ -488,89 +520,102 @@ function _testsToMocha(tests) { }; } -}); - /** * Adds the tests for all entries in the given manifest. * - * @param manifest {Object} the manifest. - * @param parent {Object} the parent test structure - * @return {Promise} + * @param {object} manifest - The manifest. + * @param {object} parent - The parent test structure. + * @returns {Promise} - A promise with no value. */ -function addManifest(manifest, parent) { - return new Promise((resolve, reject) => { - // create test structure - const suite = { - title: manifest.name || manifest.label, - tests: [], - suites: [], - imports: [] - }; - parent.push(suite); - - // get entries and sequence (alias for entries) - const entries = [].concat( - getJsonLdValues(manifest, 'entries'), - getJsonLdValues(manifest, 'sequence') - ); - - const includes = getJsonLdValues(manifest, 'include'); - // add includes to sequence as jsonld files - for(let i = 0; i < includes.length; ++i) { - entries.push(includes[i] + '.jsonld'); +async function addManifest(manifest, parent) { + // create test structure + const suite = { + title: manifest.name || manifest.label, + tests: [], + suites: [], + imports: [] + }; + parent.push(suite); + + // get entries and sequence (alias for entries) + const entries = [].concat( + getJsonLdValues(manifest, 'entries'), + getJsonLdValues(manifest, 'sequence') + ); + + const includes = getJsonLdValues(manifest, 'include'); + // add includes to sequence as jsonld files + for(let i = 0; i < includes.length; ++i) { + entries.push(includes[i] + '.jsonld'); + } + + // resolve all entry promises and process + for await (const entry of await Promise.all(entries)) { + if(typeof entry === 'string' && entry.endsWith('js')) { + // process later as a plain JavaScript file + suite.imports.push(entry); + continue; + } else if(typeof entry === 'function') { + // process as a function that returns a promise + const childSuite = await entry(options); + if(suite) { + suite.suites.push(childSuite); + } + continue; + } + const manifestEntry = await readManifestEntry(manifest, entry); + if(isJsonLdType(manifestEntry, '__SKIP__')) { + // special local skip logic + suite.tests.push(manifestEntry); + } else if(isJsonLdType(manifestEntry, 'mf:Manifest')) { + // entry is another manifest + await addManifest(manifestEntry, suite.suites); + } else { + // assume entry is a test + await addTest(manifest, manifestEntry, suite.tests); } + } +} - // resolve all entry promises and process - Promise.all(entries).then(entries => { - let p = Promise.resolve(); - entries.forEach(entry => { - if(typeof entry === 'string' && entry.endsWith('js')) { - // process later as a plain JavaScript file - suite.imports.push(entry); - return; - } else if(typeof entry === 'function') { - // process as a function that returns a promise - p = p.then(() => { - return entry(options); - }).then(childSuite => { - if(suite) { - suite.suites.push(childSuite); - } - }); - return; - } - p = p.then(() => { - return readManifestEntry(manifest, entry); - }).then(entry => { - if(isJsonLdType(entry, '__SKIP__')) { - // special local skip logic - suite.tests.push(entry); - } else if(isJsonLdType(entry, 'mf:Manifest')) { - // entry is another manifest - return addManifest(entry, suite.suites); - } else { - // assume entry is a test - return addTest(manifest, entry, suite.tests); - } - }); - }); - return p; - }).then(() => { - resolve(); - }).catch(err => { - console.error(err); - reject(err); - }); - }); +/** + * Common adjust params helper. + * + * @param {object} params - The param to adjust. + * @param {object} test - The test. + */ +function _commonAdjustParams(params, test) { + if(isJsonLdType(test, 'rdfc:RDFC10EvalTest') || + isJsonLdType(test, 'rdfc:RDFC10MapTest') || + isJsonLdType(test, 'rdfc:RDFC10NegativeEvalTest')) { + if(test.hashAlgorithm) { + params.canonizeOptions = params.canonizeOptions || {}; + params.canonizeOptions.messageDigestAlgorithm = test.hashAlgorithm; + } + if(test.computationalComplexity === 'low') { + // simple test cases + params.canonizeOptions = params.canonizeOptions || {}; + params.canonizeOptions.maxWorkFactor = 0; + } + if(test.computationalComplexity === 'medium') { + // tests between O(n) and O(n^2) + params.canonizeOptions = params.canonizeOptions || {}; + params.canonizeOptions.maxWorkFactor = 2; + } + if(test.computationalComplexity === 'high') { + // poison tests between O(n^2) and O(n^3) + params.canonizeOptions = params.canonizeOptions || {}; + params.canonizeOptions.maxWorkFactor = 3; + } + } } /** * Adds a test. * - * @param manifest {Object} the manifest. - * @param test {Object} the test. - * @param tests {Array} the list of tests to add to. - * @return {Promise} + * @param {object} manifest - The manifest. + * @param {object} test - The test. + * @param {Array} tests - The list of tests to add to. + * @returns {Promise} - A promise with no value. */ async function addTest(manifest, test, tests) { // expand @id and input base @@ -600,6 +645,10 @@ async function addTest(manifest, test, tests) { title: description + ` (jobs=${jobs})`, f: makeFn({ test, + adjustParams: params => { + _commonAdjustParams(params[1], test); + return params; + }, run: ({/*test, */testInfo, params}) => { // skip Promise.all if(jobs === 1 && fast1) { @@ -773,7 +822,14 @@ function makeFn({ }); }); - const params = adjustParams(testInfo.params.map(param => param(test))); + let params = testInfo.params.map(param => param(test)); + const extra = {}; + // type specific pre run adjustments + if(testInfo.preRunAdjustParams) { + params = testInfo.preRunAdjustParams({params, extra}); + } + // general adjustments + params = adjustParams(params); // resolve test data const values = await Promise.all(params); // copy used to check inputs do not change @@ -783,6 +839,10 @@ function makeFn({ // run and capture errors and results try { result = await run({test, testInfo, params: values}); + // type specific post run adjustments + if(testInfo.postRunAdjustParams) { + testInfo.postRunAdjustParams({params: values, extra}); + } // check input not changed assert.deepStrictEqual(valuesOrig, values); } catch(e) { @@ -792,21 +852,25 @@ function makeFn({ try { if(isJsonLdType(test, 'jld:NegativeEvaluationTest')) { if(!isBenchmark) { - await compareExpectedError(test, err); + await compareExpectedError({test, err}); + } + } else if(isJsonLdType(test, 'rdfc:RDFC10NegativeEvalTest')) { + if(!isBenchmark) { + await checkError({test, err}); } } else if(isJsonLdType(test, 'jld:PositiveEvaluationTest') || - isJsonLdType(test, 'rdfc:Urgna2012EvalTest') || - isJsonLdType(test, 'rdfc:Urdna2015EvalTest')) { + isJsonLdType(test, 'rdfc:RDFC10EvalTest') || + isJsonLdType(test, 'rdfc:RDFC10MapTest')) { if(err) { throw err; } if(!isBenchmark) { - await testInfo.compare(test, result); + await testInfo.compare({test, result, extra}); } } else if(isJsonLdType(test, 'jld:PositiveSyntaxTest')) { // no checks } else { - throw Error('Unknown test type: ' + test.type); + throw new Error(`Unknown test type: "${test.type}"`); } let benchmarkResult = null; @@ -1016,11 +1080,11 @@ function _getExpectProperty(test) { } else if('result' in test) { return 'result'; } else { - throw Error('No expected output property found'); + throw new Error('No expected output property found'); } } -async function compareExpectedJson(test, result) { +async function compareExpectedJson({test, result}) { let expect; try { expect = await readTestJson(_getExpectProperty(test))(test); @@ -1035,7 +1099,7 @@ async function compareExpectedJson(test, result) { } } -async function compareExpectedNQuads(test, result) { +async function compareExpectedNQuads({test, result}) { let expect; try { expect = await readTestNQuads(_getExpectProperty(test))(test); @@ -1050,11 +1114,15 @@ async function compareExpectedNQuads(test, result) { } } -async function compareCanonizedExpectedNQuads(test, result) { +async function compareCanonizedExpectedNQuads({test, result}) { let expect; try { expect = await readTestNQuads(_getExpectProperty(test))(test); - const opts = {algorithm: 'URDNA2015'}; + const opts = { + algorithm: 'RDFC-1.0', + // some tests need this: expand 0027 and 0062 + maxWorkFactor: 2 + }; const expectDataset = rdfCanonize.NQuads.parse(expect); const expectCmp = await rdfCanonize.canonize(expectDataset, opts); const resultDataset = rdfCanonize.NQuads.parse(result); @@ -1070,7 +1138,35 @@ async function compareCanonizedExpectedNQuads(test, result) { } } -async function compareExpectedError(test, err) { +async function compareExpectedCanonicalIdMap({test, result, extra}) { + let expect; + try { + expect = await readTestJson(_getExpectProperty(test))(test); + const expectMap = new Map(Object.entries(expect)); + assert.deepStrictEqual(extra.canonicalIdMap, expectMap); + } catch(err) { + if(options.bailOnError) { + console.log('\nTEST FAILED\n'); + console.log('EXPECTED:\n ' + JSON.stringify(expect, null, 2)); + console.log('ACTUAL:\n' + JSON.stringify(result, null, 2)); + } + throw err; + } +} + +async function checkError({/*test,*/ err}) { + try { + assert.ok(err, 'no error present'); + } catch(_err) { + if(options.bailOnError) { + console.log('\nTEST FAILED\n'); + console.log('EXPECTED ERROR'); + } + throw _err; + } +} + +async function compareExpectedError({test, err}) { let expect; let result; try { @@ -1091,10 +1187,7 @@ async function compareExpectedError(test, err) { } function isJsonLdType(node, type) { - const nodeType = [].concat( - getJsonLdValues(node, '@type'), - getJsonLdValues(node, 'type') - ); + const nodeType = getJsonLdType(node); type = Array.isArray(type) ? type : [type]; for(let i = 0; i < type.length; ++i) { if(nodeType.indexOf(type[i]) !== -1) { @@ -1104,13 +1197,17 @@ function isJsonLdType(node, type) { return false; } +function getJsonLdType(node) { + return [].concat( + getJsonLdValues(node, '@type'), + getJsonLdValues(node, 'type') + ); +} + function getJsonLdValues(node, property) { let rval = []; if(property in node) { - rval = node[property]; - if(!Array.isArray(rval)) { - rval = [rval]; - } + rval = [].concat(node[property]); } return rval; } From 598e2cc7202db31f7a40ca478beabab66c67f189 Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Fri, 17 Nov 2023 22:58:39 -0500 Subject: [PATCH 3/9] Remove support for `application/nquads` alias. --- CHANGELOG.md | 3 +++ lib/jsonld.js | 7 ++----- tests/misc.js | 31 ++++++++++++------------------- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc1dcb96..f2ffcea6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ usage. - The internal digest algorithm can be changed. +### Removed +- **BREAKING**: Remove `application/nquads` alias for `application/n-quads`. + ## 8.3.3 - 2024-12-21 ### Added diff --git a/lib/jsonld.js b/lib/jsonld.js index a2a563bb..0a001b72 100644 --- a/lib/jsonld.js +++ b/lib/jsonld.js @@ -579,8 +579,7 @@ jsonld.normalize = jsonld.canonize = async function(input, options) { }, options.canonizeOptions || null); if('inputFormat' in options) { - if(options.inputFormat !== 'application/n-quads' && - options.inputFormat !== 'application/nquads') { + if(options.inputFormat !== 'application/n-quads') { throw new JsonLdError( 'Unknown canonicalization input format.', 'jsonld.CanonizeError'); @@ -700,8 +699,7 @@ jsonld.toRDF = async function(input, options) { // output RDF dataset const dataset = _toRDF(expanded, options); if(options.format) { - if(options.format === 'application/n-quads' || - options.format === 'application/nquads') { + if(options.format === 'application/n-quads') { return NQuads.serialize(dataset); } throw new JsonLdError( @@ -1007,7 +1005,6 @@ jsonld.unregisterRDFParser = function(contentType) { // register the N-Quads RDF parser jsonld.registerRDFParser('application/n-quads', NQuads.parse); -jsonld.registerRDFParser('application/nquads', NQuads.parse); /* URL API */ jsonld.url = require('./url'); diff --git a/tests/misc.js b/tests/misc.js index 21b3d25e..b9ebbbc3 100644 --- a/tests/misc.js +++ b/tests/misc.js @@ -143,19 +143,18 @@ describe('other toRDF tests', () => { }); }); - it('should handle deprecated N-Quads format', done => { + it('should fail for deprecated N-Quads format', done => { const doc = { "@id": "https://example.com/", "https://example.com/test": "test" }; const p = jsonld.toRDF(doc, {format: 'application/nquads'}); assert(p instanceof Promise); - p.catch(e => { - assert.ifError(e); - }).then(output => { - assert.equal( - output, - ' "test" .\n'); + p.then(() => { + assert.fail(); + }).catch(e => { + assert(e); + assert.equal(e.name, 'jsonld.UnknownFormat'); done(); }); }); @@ -232,21 +231,15 @@ describe('other fromRDF tests', () => { }); }); - it('should handle deprecated N-Quads format', done => { + it('should fail for deprecated N-Quads format', done => { const nq = ' "test" .\n'; const p = jsonld.fromRDF(nq, {format: 'application/nquads'}); assert(p instanceof Promise); - p.catch(e => { - assert.ifError(e); - }).then(output => { - assert.deepEqual( - output, - [{ - "@id": "https://example.com/", - "https://example.com/test": [{ - "@value": "test" - }] - }]); + p.then(() => { + assert.fail(); + }).catch(e => { + assert(e); + assert.equal(e.name, 'jsonld.UnknownFormat'); done(); }); }); From 76cf794d0371a121ccd71d85fb1157e855623ad0 Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Fri, 17 Nov 2023 22:59:17 -0500 Subject: [PATCH 4/9] Update dependencies. --- CHANGELOG.md | 2 ++ package.json | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2ffcea6..11504307 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ### Changed - **BREAKING**: Drop support for Node.js < 18. - **BREAKING**: Upgrade dependencies. + - `@digitalbazaar/http-client@4`. + - `canonicalize@2`. - `rdf-canonize@4`: See the [rdf-canonize][] 4.0.0 changelog for **important** changes and upgrade notes. Of note: - The `URDNA2015` default algorithm has been changed to `RDFC-1.0` from diff --git a/package.json b/package.json index c6d8168c..0efa490f 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,8 @@ "lib/**/*.js" ], "dependencies": { - "@digitalbazaar/http-client": "^3.4.1", - "canonicalize": "^1.0.1", + "@digitalbazaar/http-client": "^4.0.0", + "canonicalize": "^2.0.0", "lru-cache": "^6.0.0", "rdf-canonize": "^4.0.1" }, From 92c08083baec806486c0eec33f42cf907c8b3d2c Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Sat, 21 Dec 2024 16:28:58 -0500 Subject: [PATCH 5/9] Update actions. - Use Node.js 22.x for most jobs. - Rename workflow extension to .yaml. --- .github/workflows/{main.yml => main.yaml} | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) rename .github/workflows/{main.yml => main.yaml} (95%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yaml similarity index 95% rename from .github/workflows/main.yml rename to .github/workflows/main.yaml index beb47bdc..5c658058 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yaml @@ -8,7 +8,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [20.x] + node-version: [22.x] steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} @@ -41,7 +41,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [20.x] + node-version: [22.x] bundler: [webpack, browserify] steps: - uses: actions/checkout@v4 @@ -62,7 +62,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [20.x] + node-version: [22.x] steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} @@ -78,7 +78,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [20.x] + node-version: [22.x] steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} diff --git a/README.md b/README.md index 98299837..78cdc5ff 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ jsonld.js ========= -[![Build status](https://img.shields.io/github/actions/workflow/status/digitalbazaar/jsonld.js/main.yml)](https://github.com/digitalbazaar/jsonld.js/actions/workflows/main.yml) +[![Build status](https://img.shields.io/github/actions/workflow/status/digitalbazaar/jsonld.js/main.yaml)](https://github.com/digitalbazaar/jsonld.js/actions/workflows/main.yaml) [![Coverage status](https://img.shields.io/codecov/c/github/digitalbazaar/jsonld.js)](https://codecov.io/gh/digitalbazaar/jsonld.js) [![npm](https://img.shields.io/npm/v/jsonld)](https://npm.im/jsonld) From ceb3ab3da65b2bba7cc8e6ab8d641508e74b8759 Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Mon, 6 Jan 2025 22:06:49 -0500 Subject: [PATCH 6/9] Update development dependencies. - Update development dependencies. - Update karma testing. - Remove older fixes in favor of more default behavior. - Update bundle build. - Use newer corejs version. - Build with modern browserslist defaults and no IE support. - Support for older browsers requires a custom build. --- CHANGELOG.md | 7 +++++++ karma.conf.js | 51 ++++++++++++----------------------------------- package.json | 50 ++++++++++++++++++++++------------------------ webpack.config.js | 48 ++++++++++++++------------------------------ 4 files changed, 59 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11504307..22046736 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,13 @@ - A `signal` option is available to use an `AbortSignal` to limit resource usage. - The internal digest algorithm can be changed. +- Update development dependencies. +- Update karma testing. + - Remove older fixes in favor of more default behavior. +- Update bundle build. + - Use newer corejs version. + - Build with modern browserslist defaults and no IE support. + - Support for older browsers requires a custom build. ### Removed - **BREAKING**: Remove `application/nquads` alias for `application/n-quads`. diff --git a/karma.conf.js b/karma.conf.js index 81a6ce96..37d12842 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -81,18 +81,11 @@ module.exports = function(config) { rules: [ { test: /\.js$/, - include: [{ - // exclude node_modules by default - exclude: /(node_modules)/ - }, { - // include specific packages - include: [ - /(node_modules\/canonicalize)/, - /(node_modules\/lru-cache)/, - /(node_modules\/rdf-canonize)/, - /(node_modules\/yallist)/ - ] - }], + // avoid processing core-js + include: { + and: [/node_modules/], + not: [/core-js/] + }, use: { loader: 'babel-loader', options: { @@ -101,21 +94,17 @@ module.exports = function(config) { '@babel/preset-env', { useBuiltIns: 'usage', - corejs: '3.9', + corejs: '3.39', bugfixes: true, //debug: true, targets: { // test with slightly looser browserslist defaults - browsers: 'defaults, > 0.25%' + browsers: 'defaults, > 0.25%, not IE 11' } } ] ], plugins: [ - [ - '@babel/plugin-proposal-object-rest-spread', - {useBuiltIns: true} - ], '@babel/plugin-transform-modules-commonjs', '@babel/plugin-transform-runtime' ] @@ -123,16 +112,13 @@ module.exports = function(config) { } } ], - noParse: [ - // avoid munging internal benchmark script magic - /benchmark/ - ] + //noParse: [ + // // avoid munging internal benchmark script magic + // /benchmark/ + //] }, - node: { - Buffer: false, - process: false, - crypto: false, - setImmediate: false + output: { + globalObject: 'this' } }, @@ -192,17 +178,6 @@ module.exports = function(config) { //browsers: ['ChromeHeadless', 'Chrome', 'Firefox', 'Safari'], browsers: ['ChromeHeadless'], - customLaunchers: { - IE9: { - base: 'IE', - 'x-ua-compatible': 'IE=EmulateIE9' - }, - IE8: { - base: 'IE', - 'x-ua-compatible': 'IE=EmulateIE8' - } - }, - // Continuous Integration mode // if true, Karma captures browsers, runs the tests and exits singleRun: true, diff --git a/package.json b/package.json index 0efa490f..ffb5fa91 100644 --- a/package.json +++ b/package.json @@ -29,54 +29,52 @@ "lib/**/*.js" ], "dependencies": { - "@digitalbazaar/http-client": "^4.0.0", + "@digitalbazaar/http-client": "^4.1.1", "canonicalize": "^2.0.0", "lru-cache": "^6.0.0", "rdf-canonize": "^4.0.1" }, "devDependencies": { - "@babel/core": "^7.21.8", - "@babel/plugin-proposal-object-rest-spread": "^7.20.7", - "@babel/plugin-transform-modules-commonjs": "^7.21.5", - "@babel/plugin-transform-runtime": "^7.21.4", - "@babel/preset-env": "^7.21.5", - "@babel/runtime": "^7.21.5", - "babel-loader": "^8.2.2", + "@babel/core": "^7.26.0", + "@babel/plugin-transform-modules-commonjs": "^7.26.3", + "@babel/plugin-transform-runtime": "^7.25.9", + "@babel/preset-env": "^7.26.0", + "@babel/runtime": "^7.26.0", + "babel-loader": "^9.2.1", "benchmark": "^2.1.4", - "browserify": "^17.0.0", - "chai": "^4.3.7", - "core-js": "^3.30.2", + "browserify": "^17.0.1", + "chai": "^4.5.0", + "core-js": "^3.39.0", "cors": "^2.8.5", "cross-env": "^7.0.3", "envify": "^4.1.0", - "eslint": "^8.41.0", - "eslint-config-digitalbazaar": "^3.0.0", + "eslint": "^8.57.1", + "eslint-config-digitalbazaar": "^5.2.0", "esmify": "^2.1.1", - "express": "^4.18.2", - "fs-extra": "^9.1.0", + "express": "^4.21.2", + "fs-extra": "^11.2.0", "join-path-js": "0.0.0", - "karma": "^5.2.3", + "karma": "^6.4.4", "karma-babel-preprocessor": "^8.0.2", "karma-browserify": "^8.1.0", "karma-chrome-launcher": "^3.2.0", "karma-edge-launcher": "^0.4.2", - "karma-firefox-launcher": "^2.1.2", - "karma-ie-launcher": "^1.0.0", + "karma-firefox-launcher": "^2.1.3", "karma-mocha": "^2.0.1", "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-server-side": "^1.8.0", - "karma-sourcemap-loader": "^0.3.7", + "karma-sourcemap-loader": "^0.4.0", "karma-tap-reporter": "0.0.6", - "karma-webpack": "^4.0.2", + "karma-webpack": "^5.0.1", "klona": "^2.0.6", - "mocha": "^8.3.2", + "mocha": "^11.0.1", "mocha-lcov-reporter": "^1.3.0", - "nyc": "^15.1.0", - "watchify": "^3.11.1", - "webpack": "^4.46.0", - "webpack-cli": "^4.5.0", - "webpack-merge": "^5.8.0" + "nyc": "^17.1.0", + "watchify": "^4.0.0", + "webpack": "^5.97.1", + "webpack-cli": "^6.0.1", + "webpack-merge": "^6.0.1" }, "engines": { "node": ">=18" diff --git a/webpack.config.js b/webpack.config.js index 83102b83..fb3ac5d3 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -26,7 +26,7 @@ const outputs = [ filenameBase: 'jsonld', targets: { // use slightly looser browserslist defaults - browsers: 'defaults, > 0.25%' + browsers: 'defaults, > 0.25%, not IE 11' } }, // core jsonld library (esm) @@ -38,7 +38,7 @@ const outputs = [ ], filenameBase: 'jsonld.esm', targets: { - esmodules: true + browsers: 'defaults and fully supports es6-module' } }, // - custom builds can be created by specifying the high level files you need @@ -72,18 +72,11 @@ outputs.forEach(info => { rules: [ { test: /\.js$/, - include: [{ - // exclude node_modules by default - exclude: /(node_modules)/ - }, { - // include specific packages - include: [ - /(node_modules\/canonicalize)/, - /(node_modules\/lru-cache)/, - /(node_modules\/rdf-canonize)/, - /(node_modules\/yallist)/ - ] - }], + // avoid processing core-js + include: { + and: [/node_modules/], + not: [/core-js/] + }, use: { loader: 'babel-loader', options: { @@ -92,7 +85,7 @@ outputs.forEach(info => { '@babel/preset-env', { useBuiltIns: 'usage', - corejs: '3.9', + corejs: '3.39', // TODO: remove for babel 8 bugfixes: true, //debug: true, @@ -101,10 +94,6 @@ outputs.forEach(info => { ] ], plugins: [ - [ - '@babel/plugin-proposal-object-rest-spread', - {useBuiltIns: true} - ], '@babel/plugin-transform-modules-commonjs', '@babel/plugin-transform-runtime' ] @@ -112,17 +101,6 @@ outputs.forEach(info => { } } ] - }, - plugins: [ - //new webpack.DefinePlugin({ - //}) - ], - // disable various node shims as jsonld handles this manually - node: { - Buffer: false, - crypto: false, - process: false, - setImmediate: false } }; @@ -133,8 +111,11 @@ outputs.forEach(info => { path: path.join(__dirname, 'dist'), filename: info.filenameBase + '.js', library: info.library || '[name]', - libraryTarget: info.libraryTarget || 'umd' - } + libraryTarget: info.libraryTarget || 'umd', + globalObject: 'this' + }, + // shut off to debug bundles + devtool: false }); if(info.library === null) { delete bundle.output.library; @@ -150,7 +131,8 @@ outputs.forEach(info => { path: path.join(__dirname, 'dist'), filename: info.filenameBase + '.min.js', library: info.library || '[name]', - libraryTarget: info.libraryTarget || 'umd' + libraryTarget: info.libraryTarget || 'umd', + globalObject: 'this' }, devtool: 'cheap-module-source-map' }); From d6479b04bb9b9f586fb4ea0ace35c45477623b98 Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Wed, 22 Oct 2025 22:42:08 -0400 Subject: [PATCH 7/9] Update dependencies. --- karma.conf.js | 2 +- package.json | 26 +++++++++++++------------- webpack.config.js | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/karma.conf.js b/karma.conf.js index 37d12842..58d18665 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -94,7 +94,7 @@ module.exports = function(config) { '@babel/preset-env', { useBuiltIns: 'usage', - corejs: '3.39', + corejs: '3.46', bugfixes: true, //debug: true, targets: { diff --git a/package.json b/package.json index ffb5fa91..452c1b92 100644 --- a/package.json +++ b/package.json @@ -29,30 +29,30 @@ "lib/**/*.js" ], "dependencies": { - "@digitalbazaar/http-client": "^4.1.1", - "canonicalize": "^2.0.0", + "@digitalbazaar/http-client": "^4.2.0", + "canonicalize": "^2.1.0", "lru-cache": "^6.0.0", "rdf-canonize": "^4.0.1" }, "devDependencies": { - "@babel/core": "^7.26.0", - "@babel/plugin-transform-modules-commonjs": "^7.26.3", - "@babel/plugin-transform-runtime": "^7.25.9", - "@babel/preset-env": "^7.26.0", - "@babel/runtime": "^7.26.0", - "babel-loader": "^9.2.1", + "@babel/core": "^7.28.4", + "@babel/plugin-transform-modules-commonjs": "^7.27.1", + "@babel/plugin-transform-runtime": "^7.28.3", + "@babel/preset-env": "^7.28.3", + "@babel/runtime": "^7.28.4", + "babel-loader": "^10.0.0", "benchmark": "^2.1.4", "browserify": "^17.0.1", "chai": "^4.5.0", - "core-js": "^3.39.0", + "core-js": "^3.46.0", "cors": "^2.8.5", "cross-env": "^7.0.3", "envify": "^4.1.0", "eslint": "^8.57.1", "eslint-config-digitalbazaar": "^5.2.0", "esmify": "^2.1.1", - "express": "^4.21.2", - "fs-extra": "^11.2.0", + "express": "^5.1.0", + "fs-extra": "^11.3.2", "join-path-js": "0.0.0", "karma": "^6.4.4", "karma-babel-preprocessor": "^8.0.2", @@ -68,11 +68,11 @@ "karma-tap-reporter": "0.0.6", "karma-webpack": "^5.0.1", "klona": "^2.0.6", - "mocha": "^11.0.1", + "mocha": "^11.7.4", "mocha-lcov-reporter": "^1.3.0", "nyc": "^17.1.0", "watchify": "^4.0.0", - "webpack": "^5.97.1", + "webpack": "^5.102.1", "webpack-cli": "^6.0.1", "webpack-merge": "^6.0.1" }, diff --git a/webpack.config.js b/webpack.config.js index fb3ac5d3..6cf69f62 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -85,7 +85,7 @@ outputs.forEach(info => { '@babel/preset-env', { useBuiltIns: 'usage', - corejs: '3.39', + corejs: '3.46', // TODO: remove for babel 8 bugfixes: true, //debug: true, From a131d419a9b86adc1c36cec3a735f364e74efd3c Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Wed, 22 Oct 2025 23:04:55 -0400 Subject: [PATCH 8/9] Support and test Node.js 24.x. --- .github/workflows/main.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 5c658058..3600d5dc 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -8,7 +8,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [22.x] + node-version: [24.x] steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} @@ -23,7 +23,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [18.x, 20.x, 22.x] + node-version: [18.x, 20.x, 22.x, 24.x] steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} @@ -41,7 +41,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [22.x] + node-version: [24.x] bundler: [webpack, browserify] steps: - uses: actions/checkout@v4 @@ -62,7 +62,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [22.x] + node-version: [24.x] steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} @@ -78,7 +78,7 @@ jobs: timeout-minutes: 10 strategy: matrix: - node-version: [22.x] + node-version: [24.x] steps: - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} From 1b5e91263b7f0321c802fd58901966d3cd06086a Mon Sep 17 00:00:00 2001 From: "David I. Lehn" Date: Sat, 25 Oct 2025 01:47:21 -0400 Subject: [PATCH 9/9] Refactor test framework. - Test runtime loads test files from a web server. - Allows testing of manifests on remote web servers. - Trading off some performance to align node and browser testing. - Moves some test setup code into config data and manifest. --- CHANGELOG.md | 7 +- karma.conf.js | 50 +++++++- package.json | 1 - tests/test-karma.js | 114 +++++------------ tests/test-node.js | 119 +++++++++--------- tests/test-server.js | 201 ++++++++++++++++++++++++++++++ tests/test-webidl.js | 14 ++- tests/test.js | 287 ++++++++++++++++++++++++++++--------------- 8 files changed, 545 insertions(+), 248 deletions(-) create mode 100644 tests/test-server.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 22046736..dc1045c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # jsonld ChangeLog -## 9.0.0 - 2023-xx-xx +## 9.0.0 - 2025-xx-xx ### Changed - **BREAKING**: Drop support for Node.js < 18. @@ -23,6 +23,11 @@ - Use newer corejs version. - Build with modern browserslist defaults and no IE support. - Support for older browsers requires a custom build. +- Refactor test framework. + - Test runtime loads test files from a web server. + - Allows testing of manifests on remote web servers. + - Trading off some performance to align node and browser testing. + - Moves some test setup code into config data and manifest. ### Removed - **BREAKING**: Remove `application/nquads` alias for `application/n-quads`. diff --git a/karma.conf.js b/karma.conf.js index 58d18665..af793396 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -10,12 +10,40 @@ */ const os = require('os'); const webpack = require('webpack'); +const {TestServer} = require('./tests/test-server.js'); + +// karma test server proxy details +const _proxyTestsPrefix = '/tests'; + +let testServer; + +// shutdown test server "reporter" hook +function ShutdownTestServer(baseReporterDecorator) { + baseReporterDecorator(this); + + this.onRunComplete = async function() { + await testServer.close(); + }; +} + +// Inject the base reporter +ShutdownTestServer.$inject = ['baseReporterDecorator', 'config']; + +// local "reporter" plugin +const shutdownTestServer = { + 'reporter:shutdown-test-server': ['type', ShutdownTestServer] +}; + +module.exports = async function(config) { + testServer = new TestServer({ + earlFilename: process.env.EARL + }); + await testServer.start(); -module.exports = function(config) { // bundler to test: webpack, browserify const bundler = process.env.BUNDLER || 'webpack'; - const frameworks = ['mocha', 'server-side']; + const frameworks = ['mocha']; // main bundle preprocessors const preprocessors = ['babel']; @@ -66,7 +94,8 @@ module.exports = function(config) { 'process.env.EARL': JSON.stringify(process.env.EARL), 'process.env.TESTS': JSON.stringify(process.env.TESTS), 'process.env.TEST_ENV': JSON.stringify(process.env.TEST_ENV), - 'process.env.TEST_ROOT_DIR': JSON.stringify(__dirname), + 'process.env.TEST_SERVER_URL': JSON.stringify(_proxyTestsPrefix), + 'process.env.AUTH_TOKEN': JSON.stringify(testServer.authToken), 'process.env.VERBOSE_SKIP': JSON.stringify(process.env.VERBOSE_SKIP), // for 'auto' test env 'process.env._TEST_ENV_ARCH': JSON.stringify(process.arch), @@ -151,11 +180,20 @@ module.exports = function(config) { ] }, + // local server shutdown plugin + plugins: [ + 'karma-*', + shutdownTestServer + ], + // test results reporter to use // possible values: 'dots', 'progress' // available reporters: https://npmjs.org/browse/keyword/karma-reporter //reporters: ['progress'], - reporters: ['mocha'], + reporters: [ + 'mocha', + 'shutdown-test-server' + ], // web server port port: 9876, @@ -197,6 +235,8 @@ module.exports = function(config) { }, // Proxied paths - proxies: {} + proxies: { + '/tests': testServer.url + } }); }; diff --git a/package.json b/package.json index 452c1b92..657c715a 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,6 @@ "karma-mocha": "^2.0.1", "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", - "karma-server-side": "^1.8.0", "karma-sourcemap-loader": "^0.4.0", "karma-tap-reporter": "0.0.6", "karma-webpack": "^5.0.1", diff --git a/tests/test-karma.js b/tests/test-karma.js index 19ba045b..630569b8 100644 --- a/tests/test-karma.js +++ b/tests/test-karma.js @@ -6,18 +6,15 @@ * @author Dave Longley * @author David I. Lehn * - * Copyright (c) 2011-2023 Digital Bazaar, Inc. All rights reserved. + * Copyright (c) 2011-2025 Digital Bazaar, Inc. All rights reserved. */ -/* global serverRequire */ // FIXME: hack to ensure delay is set first mocha.setup({delay: true, ui: 'bdd'}); const assert = require('chai').assert; const benchmark = require('benchmark'); const common = require('./test.js'); -const server = require('karma-server-side'); const webidl = require('./test-webidl'); -const join = require('join-path-js'); // special benchmark setup const _ = require('lodash'); @@ -26,70 +23,25 @@ window.Benchmark = Benchmark; const entries = []; +// setup test server url, add localhost if needed +let testServerUrl = process.env.TEST_SERVER_URL; +if(!testServerUrl.endsWith('/')) { + testServerUrl += '/'; +} +if(!(testServerUrl.startsWith('http:') || testServerUrl.startsWith('https:'))) { + const pathname = testServerUrl; + testServerUrl = new URL(window.location); + testServerUrl.pathname = pathname; +} + if(process.env.TESTS) { entries.push(...process.env.TESTS.split(' ')); } else { - const _top = process.env.TEST_ROOT_DIR; - // TODO: support just adding certain entries in EARL mode? - - // json-ld-api main test suite - entries.push((async () => { - const testPath = join(_top, 'test-suites/json-ld-api/tests'); - const siblingPath = join(_top, '../json-ld-api/tests'); - return server.run(testPath, siblingPath, function(testPath, siblingPath) { - const fs = serverRequire('fs-extra'); - // use local tests if setup - if(fs.existsSync(testPath)) { - return testPath; - } - // default to sibling dir - return siblingPath; - }); - })()); - - // json-ld-framing main test suite - entries.push((async () => { - const testPath = join(_top, 'test-suites/json-ld-framing/tests'); - const siblingPath = join(_top, '../json-ld-framing/tests'); - return server.run(testPath, siblingPath, function(testPath, siblingPath) { - const fs = serverRequire('fs-extra'); - // use local tests if setup - if(fs.existsSync(testPath)) { - return testPath; - } - // default to sibling dir - return siblingPath; - }); - })()); - - /* - // TODO: use json-ld-framing once tests are moved - // json-ld.org framing test suite - // FIXME: add path detection - entries.push(join( - _top, 'test-suites/json-ld.org/test-suite/tests/frame-manifest.jsonld')); - entries.push(join( - _top, '../json-ld.org/test-suite/tests/frame-manifests.jsonld')); - */ + entries.push(new URL('tests/default/', testServerUrl)); - // W3C RDF Dataset Canonicalization "rdf-canon" test suite - entries.push((async () => { - const testPath = join(_top, 'test-suites/rdf-canon/tests'); - const siblingPath = join(_top, '../rdf-canon/tests'); - return server.run(testPath, siblingPath, function(testPath, siblingPath) { - const fs = serverRequire('fs-extra'); - // use local tests if setup - if(fs.existsSync(testPath)) { - return testPath; - } - // default to sibling dir - return siblingPath; - }); - })()); + // TODO: support just adding certain entries in EARL mode? - // other tests - entries.push(join(_top, 'tests/misc.js')); - entries.push(join(_top, 'tests/graph-container.js')); + // other tests (including js ones) added with options.addExtraTests // WebIDL tests entries.push(webidl); @@ -126,33 +78,35 @@ const options = { throw new Error('exit not implemented'); }, earl: { + enabled: !!process.env.EARL, filename: process.env.EARL }, entries, + addExtraTests: async () => { + // direct load for bundling + // called after handling other entry loading + require('./misc.js'); + require('./graph-container.js'); + }, testEnvDefaults, - readFile: filename => { - return server.run(filename, function(filename) { - const fs = serverRequire('fs-extra'); - return fs.readFile(filename, 'utf8').then(data => { - return data; - }); - }); + get testServerUrl() { + return testServerUrl; }, - writeFile: (filename, data) => { - return server.run(filename, data, function(filename, data) { - const fs = serverRequire('fs-extra'); - return fs.outputFile(filename, data); - }); + get authToken() { + return process.env.AUTH_TOKEN; }, - /* eslint-disable-next-line no-unused-vars */ import: f => { console.error('import not implemented for "' + f + '"'); - } + }, + cleanup: async () => {} }; -// wait for setup of all tests then run mocha -common(options).then(() => { +async function main() { + // wait for setup of all tests then run mocha + await common.setup(options); run(); -}).catch(err => { +} + +main().catch(err => { console.error(err); }); diff --git a/tests/test-node.js b/tests/test-node.js index 09ca54d3..e8e8605d 100644 --- a/tests/test-node.js +++ b/tests/test-node.js @@ -6,67 +6,36 @@ * @author Dave Longley * @author David I. Lehn * - * Copyright (c) 2011-2023 Digital Bazaar, Inc. All rights reserved. + * Copyright (c) 2011-2025 Digital Bazaar, Inc. All rights reserved. */ const assert = require('chai').assert; const benchmark = require('benchmark'); const common = require('./test.js'); -const fs = require('fs-extra'); const os = require('os'); const path = require('path'); +const {TestServer} = require('./test-server.js'); -const entries = []; - -if(process.env.TESTS) { - entries.push(...process.env.TESTS.split(' ')); -} else { - const _top = path.resolve(__dirname, '..'); - - // json-ld-api main test suite - const apiPath = path.resolve(_top, 'test-suites/json-ld-api/tests'); - if(fs.existsSync(apiPath)) { - entries.push(apiPath); - } else { - // default to sibling dir - entries.push(path.resolve(_top, '../json-ld-api/tests')); - } - - // json-ld-framing main test suite - const framingPath = path.resolve(_top, 'test-suites/json-ld-framing/tests'); - if(fs.existsSync(framingPath)) { - entries.push(framingPath); - } else { - // default to sibling dir - entries.push(path.resolve(_top, '../json-ld-framing/tests')); - } +// local HTTP test server +let testServer; - /* - // TODO: use json-ld-framing once tests are moved - // json-ld.org framing test suite - const framingPath = path.resolve( - _top, 'test-suites/json-ld.org/test-suite/tests/frame-manifest.jsonld'); - if(fs.existsSync(framingPath)) { - entries.push(framingPath); - } else { - // default to sibling dir - entries.push(path.resolve( - _top, '../json-ld.org/test-suite/tests/frame-manifest.jsonld')); - } - */ +const entries = []; +const allowedImports = []; - // W3C RDF Dataset Canonicalization "rdf-canon" test suite - const rdfCanonPath = path.resolve(_top, 'test-suites/rdf-canon/tests'); - if(fs.existsSync(rdfCanonPath)) { - entries.push(rdfCanonPath); - } else { - // default to sibling dir - entries.push(path.resolve(_top, '../rdf-canon/tests')); +async function init({testServer}) { + if(process.env.TESTS) { + entries.push(...process.env.TESTS.split(' ')); + return; } + entries.push(new URL('/tests/default/', testServer.url)); // other tests - entries.push(path.resolve(_top, 'tests/misc.js')); - entries.push(path.resolve(_top, 'tests/graph-container.js')); - entries.push(path.resolve(_top, 'tests/node-document-loader-tests.js')); + // setup allow list + const _tests = path.resolve(__dirname); + allowedImports.push(path.resolve(_tests, 'misc.js')); + allowedImports.push(path.resolve(_tests, 'graph-container.js')); + allowedImports.push(path.resolve(_tests, 'node-document-loader-tests.js')); + // add all allow list entries + entries.push(...allowedImports); } // test environment defaults @@ -98,26 +67,54 @@ const options = { benchmark, exit: code => process.exit(code), earl: { + enabled: !!process.env.EARL, filename: process.env.EARL }, entries, + addExtraTests: async () => {}, testEnvDefaults, - readFile: filename => { - return fs.readFile(filename, 'utf8'); + get testServerUrl() { + return testServer.url; + }, + get authToken() { + return testServer.authToken; }, - writeFile: (filename, data) => { - return fs.outputFile(filename, data); + import: f => { + if(!allowedImports.includes(f)) { + throw new Error(`Import not allowed: "${f}"`); + } + return require(f); }, - import: f => require(f) + cleanup: async () => { + await testServer.close(); + } }; -// wait for setup of all tests then run mocha -common(options).then(() => { - run(); -}).catch(err => { - console.error(err); -}); - process.on('unhandledRejection', (reason, p) => { console.error('Unhandled Rejection at:', p, 'reason:', reason); }); + +async function main() { + // start test server + testServer = new TestServer({ + earlFilename: process.env.EARL + }); + await testServer.start(); + + await init({ + testServer + }); + + // wait for setup of all tests then run mocha + await common.setup(options); + run(); + + // FIXME: run returns before tests are complete + //await testServer.close(); +} + +main().catch(async err => { + console.error(err); + // close server so mocha can cleanly shutdown + await options.cleanup(); +}); diff --git a/tests/test-server.js b/tests/test-server.js new file mode 100644 index 00000000..b02dda00 --- /dev/null +++ b/tests/test-server.js @@ -0,0 +1,201 @@ +/** + * Test server. + * + * @author Dave Longley + * @author David I. Lehn + * + * Copyright (c) 2025 Digital Bazaar, Inc. All rights reserved. + */ +const crypto = require('node:crypto'); +const express = require('express'); +const fs = require('node:fs/promises'); +const {join} = require('node:path'); + +// all known served static paths without proxy prefix +const pathMappings = [ + // JSON-LD Test Suites + // W3C JSON-LD API + ['/tests/test-suites/json-ld-api', '../test-suites/json-ld-api/tests'], + ['/tests/siblings/json-ld-api', '../../json-ld-api/tests'], + // W3C JSON-LD Framing + ['/tests/test-suites/json-ld-framing', + '../test-suites/json-ld-framing/tests'], + ['/tests/siblings/json-ld-framing', '../../json-ld-framing/tests'], + // json-ld.org test suite (old) + // includes various *-manifest.jsonld files + ['/tests/test-suites/json-ld.org', '../test-suites/json-ld.org/test-suite'], + ['/tests/siblings/json-ld.org', '../../json-ld.org/test-suite/'], + // W3C RDF Dataset Canonicalization "rdf-canon" test suite + ['/tests/test-suites/rdf-canon', '../test-suites/rdf-canon/tests'], + ['/tests/siblings/rdf-canon', '../../rdf-canon/tests'], + // WebIDL + ['/tests/webidl', './webidl'] +]; + +/* eslint-disable */ +const defaultManifestPath = '/tests/default/manifest.jsonld'; +const defaultManifest = { + "@context": [ + "https://w3c.github.io/json-ld-api/tests/context.jsonld" + ], + "@id": "", + "@type": "mf:Manifest", + "name": "jsonld.js common", + "description": "", + "baseIri": "", + "sequence": [ + { + "@id": "", + "@type": "mf:Manifest", + "name": "JSON-LD API", + "urn:test:sequence:allowMissing": true, + "urn:test:sequence:min": 1, + "urn:test:sequence:max": 1, + "sequence": [ + "../test-suites/json-ld-api", + "../siblings/json-ld-api" + ] + }, + { + "@id": "", + "@type": "mf:Manifest", + "name": "JSON-LD Framing", + "urn:test:sequence:allowMissing": true, + "urn:test:sequence:min": 1, + "urn:test:sequence:max": 1, + "sequence": [ + "../test-suites/json-ld-framing", + "../siblings/json-ld-framing" + ] + }, + { + "@id": "", + "@type": "mf:Manifest", + "name": "Old JSON-LD Test Suite", + "skip": true, + "urn:test:sequence:allowMissing": true, + "urn:test:sequence:min": 1, + "urn:test:sequence:max": 1, + "sequence": [ + "../test-suites/json-ld.org", + "../siblings/json-ld.org" + ] + }, + { + "@id": "", + "@type": "mf:Manifest", + "name": "rdf-cannon", + "urn:test:sequence:allowMissing": true, + "urn:test:sequence:min": 1, + "urn:test:sequence:max": 1, + "sequence": [ + "../test-suites/rdf-canon", + "../siblings/rdf-canon" + ] + } + ] +}; +/* eslint-enable */ + +class TestServer { + constructor({ + earlFilename = null + } = {}) { + // allow list for EARL and benchmark file names + this.earlFilename = earlFilename; + // random auth token for this session + this.authToken = crypto.randomUUID(); + this.url = null; + this.httpServer = null; + // static paths to serve. [[serverPath, localPath], ...] + this.staticPaths = []; + // served test config + this.config = {}; + } + + checkAuthToken(req, res, next) { + const auth = req.headers.authorization; + if(auth !== `Bearer ${this.authToken}`) { + throw new Error('bad auth'); + } + next(); + } + + async start({ + port = 0, + //server = '0.0.0.0' + server = 'localhost' + } = {}) { + this.app = express(); + // limit adjusted to handle large EARL POSTs. + this.app.use(express.json({limit: '10mb'})); + // debug + this.app.get('/ping', (req, res) => { + res.send('pong'); + }); + + // setup static routes + for(const [route, relpath] of pathMappings) { + this.app.use(route, + this.checkAuthToken.bind(this), + express.static(join(__dirname, relpath), { + setHeaders: function(res, path/*, stat*/) { + // handle extra extensions + if(path.endsWith('.nq')) { + res.setHeader('Content-Type', 'application/n-quads'); + } + } + })); + } + // setup routes to save data + // uses static configured path to address security issues + this.app.post('/earl', + this.checkAuthToken.bind(this), + async (req, res) => { + if(!req.body) { + res.status(400).send('no content'); + return; + } + await fs.writeFile( + this.earlFilename, + JSON.stringify(req.body, null, 2)); + res.status(200).end(); + }); + + // test config + this.app.get('/config', + this.checkAuthToken.bind(this), + (req, res) => { + res.json(this.config); + }); + + // default manifest + this.app.get(defaultManifestPath, + this.checkAuthToken.bind(this), + (req, res) => { + res.json(defaultManifest); + }); + + const httpServerPromise = new Promise(resolve => { + this.httpServer = this.app.listen({port, server}, () => { + const address = this.httpServer.address(); + //const url = `http://${address.address}:${address.port}`; + this.url = `http://${server}:${address.port}`; + resolve(); + }); + }); + + return httpServerPromise; + } + + async close() { + if(this.httpServer) { + this.httpServer.close(); + this.httpServer = null; + } + } +} + +module.exports = { + TestServer +}; diff --git a/tests/test-webidl.js b/tests/test-webidl.js index 6b7a2420..9262e845 100644 --- a/tests/test-webidl.js +++ b/tests/test-webidl.js @@ -98,7 +98,19 @@ return new Promise((resolve, reject) => { return toString.apply(this, arguments); }; - options.readFile('./tests/webidl/JsonLdProcessor.idl').then(idl => { + const idlUrl = + new URL('tests/webidl/JsonLdProcessor.idl', options.testServerUrl); + fetch(idlUrl, { + headers: { + Authorization: `Bearer ${options.authToken}` + } + }).then(response => { + if(!response.ok) { + throw new Error(`IDL fetch failed: URL="${idlUrl}"`); + } + const idl = response.text(); + return idl; + }).then(idl => { setup({explicit_done: true}); var idl_array = new IdlArray(); idl_array.add_idls(idl); diff --git a/tests/test.js b/tests/test.js index ab98a9d4..005c8ed3 100644 --- a/tests/test.js +++ b/tests/test.js @@ -50,7 +50,7 @@ * @author Dave Longley * @author David I. Lehn * - * Copyright (c) 2011-2023 Digital Bazaar, Inc. All rights reserved. + * Copyright (c) 2011-2025 Digital Bazaar, Inc. All rights reserved. */ /* eslint-disable indent */ const EarlReport = require('./earl-report'); @@ -60,7 +60,14 @@ const {klona} = require('klona'); const {prependBase} = require('../lib/url'); const rdfCanonize = require('rdf-canonize'); -// helper functions, inspired by 'boolean' package +const stats = { + getJson: 0, + getData: 0, + postJson: 0, + postData: 0 +}; + +// boolean helper functions, inspired by 'boolean' package function isTrue(value) { return value && [ 'true', 't', 'yes', 'y', 'on', '1' @@ -73,7 +80,8 @@ function isFalse(value) { ].includes(value.trim().toLowerCase()); } -module.exports = async function(options) { +// main setup +async function setup(options) { 'use strict'; @@ -112,7 +120,7 @@ if(options.env.BENCHMARK) { // Only support one job size for EARL output to simplify reporting and avoid // multi-variable issues. Can compare multiple runs with different job sizes. -if(options.earl.filename && benchmarkOptions.jobs.length > 1) { +if(options.earl.enabled && benchmarkOptions.jobs.length > 1) { throw new Error('Only one job size allowed when outputting EARL.'); } @@ -458,7 +466,7 @@ if(options.env.TEST_ENV) { } // create earl report -if(options.earl && options.earl.filename) { +if(options.earl.enabled) { options.earl.report = new EarlReport({ env: testEnv }); @@ -478,17 +486,34 @@ const _tests = []; await addManifest(manifest, _tests); const result = _testsToMocha(_tests); -if(options.earl.report) { +// add extra tests +// useful for karma to load js tests in proper order +await options.addExtraTests(); + +// add pseudo test to output EARL results +if(options.earl.enabled) { describe('Writing EARL report to: ' + options.earl.filename, function() { // print out EARL even if .only was used const _it = result.hadOnly ? it.only : it; - _it('should print the earl report', function() { - return options.writeFile( - options.earl.filename, options.earl.report.reportJson()); + _it('should print the earl report', async function() { + await postJson({ + url: new URL('earl', options.testServerUrl), + data: options.earl.report.report() + }); }); }); } +// add pseudo test to do cleanup +// NOTE: This is a hack to get around issues with dynamic test setup, running +// a HTTP test server, and problems using mocha root hooks or global fixtures. +describe('cleanup', function() { + const _it = result.hadOnly ? it.only : it; + _it('cleanup', async function() { + await options.cleanup(); + }); +}); + return; // build mocha tests from local test structure @@ -537,6 +562,14 @@ async function addManifest(manifest, parent) { }; parent.push(suite); + if(manifest.skip === true) { + if(verboseSkip) { + console.log('Skipping manifest due to manifest:', + {id: manifest['@id'], name: manifest.name}); + } + return; + } + // get entries and sequence (alias for entries) const entries = [].concat( getJsonLdValues(manifest, 'entries'), @@ -549,6 +582,12 @@ async function addManifest(manifest, parent) { entries.push(includes[i] + '.jsonld'); } + // custom manifest sequence options + const seqAllowMissing = manifest['urn:test:sequence:allowMissing'] || false; + const seqMin = manifest['urn:test:sequence:min'] || 0; + const seqMax = manifest['urn:test:sequence:max'] || Infinity; + let seqCount = 0; + // resolve all entry promises and process for await (const entry of await Promise.all(entries)) { if(typeof entry === 'string' && entry.endsWith('js')) { @@ -558,22 +597,41 @@ async function addManifest(manifest, parent) { } else if(typeof entry === 'function') { // process as a function that returns a promise const childSuite = await entry(options); - if(suite) { - suite.suites.push(childSuite); - } + suite.suites.push(childSuite); continue; } - const manifestEntry = await readManifestEntry(manifest, entry); - if(isJsonLdType(manifestEntry, '__SKIP__')) { - // special local skip logic - suite.tests.push(manifestEntry); - } else if(isJsonLdType(manifestEntry, 'mf:Manifest')) { + + let manifestEntry; + try { + manifestEntry = await readManifestEntry(manifest, entry); + } catch(e) { + // TODO: check error + if(seqAllowMissing) { + continue; + } + // TODO: add details + throw new Error('Invalid sequence entry.', {cause: e}); + } + + if(isJsonLdType(manifestEntry, 'mf:Manifest')) { // entry is another manifest await addManifest(manifestEntry, suite.suites); } else { // assume entry is a test await addTest(manifest, manifestEntry, suite.tests); } + + seqCount++; + // short circuit if max entries found + if(seqCount === seqMax) { + // TODO: debug logging + break; + } + } + // check if minimum required entries found + if(seqCount < seqMin) { + // TODO: add details + throw new Error('Too few sequence entries.'); } } @@ -623,7 +681,7 @@ async function addTest(manifest, test, tests) { //var number = test_id.substr(2); test['@id'] = (manifest.baseIri || '') + - basename(manifest.filename).replace('.jsonld', '') + + basename(manifest._url.pathname).replace('.jsonld', '') + test_id; test.base = manifest.baseIri + test.input; test.manifest = manifest; @@ -899,7 +957,7 @@ function makeFn({ }; } - if(options.earl.report) { + if(options.earl.enabled) { options.earl.report.addAssertion(test, true, { benchmarkResult }); @@ -921,7 +979,7 @@ function makeFn({ } options.exit(); } - if(options.earl.report) { + if(options.earl.enabled) { options.earl.report.addAssertion(test, false); } console.error('Error: ', JSON.stringify(err, null, 2)); @@ -973,47 +1031,28 @@ function getJsonLdTestType(test) { return null; } -function readManifestEntry(manifest, entry) { - let p = Promise.resolve(); - let _entry = entry; - if(typeof entry === 'string') { - let _filename; - p = p.then(() => { - if(entry.endsWith('json') || entry.endsWith('jsonld')) { - // load as file - return entry; - } - // load as dir with manifest.jsonld - return joinPath(entry, 'manifest.jsonld'); - }).then(entry => { - const dir = dirname(manifest.filename); - return joinPath(dir, entry); - }).then(filename => { - _filename = filename; - return readJson(filename); - }).then(entry => { - _entry = entry; - _entry.filename = _filename; - return _entry; - }).catch(err => { - if(err.code === 'ENOENT') { - //console.log('File does not exist, skipping: ' + _filename); - // return a "skip" entry - _entry = { - type: '__SKIP__', - title: 'Not found, skipping: ' + _filename, - filename: _filename, - skip: true - }; - return; - } - throw err; - }); +async function readManifestEntry(manifest, entry) { + let _entry; + if(typeof entry === 'string' || entry instanceof URL) { + let url; + if(typeof entry === 'string') { + url = new URL(entry, manifest._url); + } else { + url = new URL(entry); + } + const pathname = url.pathname; + // load as dir with manifest.jsonld if not file-like + if(!pathname.endsWith('.json') && !pathname.endsWith('.jsonld')) { + const pathname = await joinPath(url.pathname, 'manifest.jsonld'); + url.pathname = pathname; + } + _entry = await getJson({url}); + _entry._url = url; + } else { + _entry = structuredClone(entry); + _entry._url = manifest._url; } - return p.then(() => { - _entry.dirname = dirname(_entry.filename || manifest.filename); - return _entry; - }); + return _entry; } function readTestUrl(property) { @@ -1024,7 +1063,7 @@ function readTestUrl(property) { if(options && options.load) { // always load const filename = await joinPath(test.dirname, test[property]); - return readJson(filename); + return getJson({url: filename}); } return test.manifest.baseIri + test[property]; }; @@ -1035,8 +1074,8 @@ function readTestJson(property) { if(!test[property]) { return null; } - const filename = await joinPath(test.dirname, test[property]); - return readJson(filename); + const url = new URL(test[property], test._url); + return getJson({url}); }; } @@ -1045,8 +1084,9 @@ function readTestNQuads(property) { if(!test[property]) { return null; } - const filename = await joinPath(test.dirname, test[property]); - return readFile(filename); + const url = new URL(test[property], test._url); + const response = await getData({url}); + return response.text(); }; } @@ -1227,30 +1267,79 @@ function getJsonLdErrorCode(err) { return err.name; } -async function readJson(filename) { - const data = await readFile(filename); - return JSON.parse(data); +async function getJson({url, headers = {}}) { + stats.getJson++; + const response = await getData({url, headers: { + accept: 'application/ld+json,application/json', + ...headers + }}); + return response.json(); } -async function readFile(filename) { - return options.readFile(filename); +async function getData({url, headers = {}}) { + stats.getData++; + if(url.protocol === 'http:' || url.protocol === 'https:') { + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${options.authToken}`, + ...headers + } + }); + if(!response.ok) { + throw new Error(`getData: bad response: URL="${url}"`); + } + return response; + } + throw new Error(`getData: unsupported protocol: URL="${url}"`); } -async function joinPath() { - return join.apply(null, Array.prototype.slice.call(arguments)); +async function postJson({url, data, headers = {}}) { + stats.postJson++; + const response = await postData({ + url, + data: JSON.stringify(data), + headers: { + 'Content-Type': 'application/json', + ...headers + } + }); + return response; } -function dirname(filename) { - if(options.nodejs) { - return options.nodejs.path.dirname(filename); - } - const idx = filename.lastIndexOf('/'); - if(idx === -1) { - return filename; +async function postData({url, data, headers = {}}) { + stats.postData++; + if(url.protocol === 'http:' || url.protocol === 'https:') { + const response = await fetch(url, { + method: 'POST', + headers: { + Authorization: `Bearer ${options.authToken}`, + ...headers + }, + body: data + }); + if(!response.ok) { + throw new Error(`postData: bad response: URL="${url}"`); + } + return response; } - return filename.substr(0, idx); + throw new Error(`postData: unsupported protocol: URL="${url}"`); } +async function joinPath() { + return join.apply(null, Array.prototype.slice.call(arguments)); +} + +//function dirname(filename) { +// if(options.nodejs) { +// return options.nodejs.path.dirname(filename); +// } +// const idx = filename.lastIndexOf('/'); +// if(idx === -1) { +// return filename; +// } +// return filename.substr(0, idx); +//} + function basename(filename) { if(options.nodejs) { return options.nodejs.path.basename(filename); @@ -1327,7 +1416,7 @@ function createDocumentLoader(test) { return localLoader; - function loadLocally(url) { + async function loadLocally(url) { const doc = {contextUrl: null, documentUrl: url, document: null}; const options = test.option; if(options && url === test.base) { @@ -1364,31 +1453,31 @@ function createDocumentLoader(test) { } } - let p = Promise.resolve(); + let filename; if(doc.documentUrl.indexOf(':') === -1) { - p = p.then(() => { - return joinPath(test.manifest.dirname, doc.documentUrl); - }).then(filename => { - doc.documentUrl = 'file://' + filename; - return filename; - }); + // FIXME: needed? improve or remove? + throw new Error(`Non-URL="${doc.documentUrl}"`); + //filename = await joinPath(test.manifest.dirname, doc.documentUrl); + //doc.documentUrl = 'file://' + filename; } else { - p = p.then(() => { - return joinPath( - test.manifest.dirname, - doc.documentUrl.substr(test.manifest.baseIri.length)); - }).then(fn => { - return fn; - }); + filename = new URL( + doc.documentUrl.substr(test.manifest.baseIri.length), + test.manifest._url); } - return p.then(readJson).then(json => { + try { + const json = await getJson({url: filename}); doc.document = json; return doc; - }).catch(() => { - throw {name: 'loading document failed', url}; - }); + } catch(err) { + throw {name: 'loading document failed', url, cause: err}; + } } } +} + +module.exports = { + setup, + stats };