Merge pull request #208 from VisLab/update-tokenizer

Rewrite of tokenizer and introduction of object-based test cases

.github/workflows/ci.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Download dependencies
         run: npm ci
       - name: Test & publish code coverage
-        uses: paambaati/codeclimate-action@v8.0.0
+        uses: paambaati/codeclimate-action@v9.0.0
         env:
           CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
         with:

.gitignore

@@ -7,6 +7,7 @@ dist/
 spec_tests/*.txt
 spec_tests/temp*.json
 spec_tests/temp.spec.js
+tests/temp.spec.js
 
 # Unit test / coverage reports
 htmlcov/

README.md

@@ -5,7 +5,7 @@
 
 # hed-validator
 
-This package contains a JavaScript validator for HED (hierarchical event descriptor) strings.
+This package contains a JavaScript validator for HED (Hierarchical Event Descriptor) strings.
 
 [HED](https://www.hedtags.org/) is a system for annotating events using comma-separated path strings.
 Any type of event can be annotated using HED-type syntax.
@@ -57,3 +57,8 @@ To use the `hed-validator`, you must install the npm `hed-validator` package and
 
 A sample of current `hed-validator` usage can be found in the BIDS validator in
 [`hed.js`](https://github.com/bids-standard/bids-validator/blob/5dfc3938ea8ce128c7db295e7bebc8eed2de1ce6/bids-validator/validators/hed.js).
+
+## Repository notes:
+
+The `develop` branch is now the default branch. All changes to the repository should
+be done as PRs (pull requests) to the `develop` branch.

bids/validator/bidsHedTsvValidator.js

@@ -82,6 +82,10 @@ export class BidsHedTsvValidator {
    * @private
    */
   _validateHedColumn() {
+    if (this.tsvFile.hedColumnHedStrings.length === 0) {
+      // no HED column strings to validate
+      return []
+    }
     return this.tsvFile.hedColumnHedStrings.flatMap((hedString, rowIndexMinusTwo) =>
       this._validateHedColumnString(hedString, rowIndexMinusTwo + 2),
     )
@@ -230,7 +234,6 @@ export class BidsHedTsvParser {
    */
   _parseHedRows(tsvHedRows) {
     const hedStrings = []
-
     tsvHedRows.forEach((row, index) => {
       const hedString = this._parseHedRow(row, index + 2)
       if (hedString !== null) {
@@ -248,9 +251,9 @@ export class BidsHedTsvParser {
    * @private
    */
   _mergeEventRows(rowStrings) {
+    const eventStrings = []
     const groupedTsvRows = groupBy(rowStrings, (rowString) => rowString.onset)
     const sortedOnsetTimes = Array.from(groupedTsvRows.keys()).sort((a, b) => a - b)
-    const eventStrings = []
     for (const onset of sortedOnsetTimes) {
       const onsetRows = groupedTsvRows.get(onset)
       const onsetEventString = new BidsTsvEvent(this.tsvFile, onsetRows)
@@ -275,6 +278,7 @@ export class BidsHedTsvParser {
         hedStringParts.push(hedStringPart)
       }
     }
+    if (hedStringParts.length === 0) return null
 
     const hedString = hedStringParts.join(',')
 

common/issues/data.js

@@ -43,11 +43,26 @@ export default {
     level: 'error',
     message: stringTemplate`Invalid tag - "${'tag'}".`,
   },
+  extraSlash: {
+    hedCode: 'TAG_INVALID',
+    level: 'error',
+    message: stringTemplate`Tag has extra slash at index ${'index'} of string "${'string'}".`,
+  },
+  extraBlank: {
+    hedCode: 'TAG_INVALID',
+    level: 'error',
+    message: stringTemplate`Tag has extra blank at index ${'index'} of string "${'string'}".`,
+  },
   extraCommaOrInvalid: {
     hedCode: 'TAG_INVALID',
     level: 'error',
     message: stringTemplate`Either "${'previousTag'}" contains a comma when it should not or "${'tag'}" is not a valid tag.`,
   },
+  invalidTagPrefix: {
+    hedCode: 'TAG_NAMESPACE_PREFIX_INVALID',
+    level: 'error',
+    message: stringTemplate`Either tag prefix at index ${'index'} contains non-alphabetic characters or does not have an associated schema.`,
+  },
   multipleUniqueTags: {
     hedCode: 'TAG_NOT_UNIQUE',
     level: 'error',

parser/columnSplicer.js

@@ -111,25 +111,36 @@ export class ColumnSplicer {
    */
   _spliceTemplate(columnTemplate) {
     const columnName = columnTemplate.originalTag
+
+    // HED column handled specially
+    if (columnName === 'HED') {
+      return this._spliceHedColumnTemplate()
+    }
+
+    // Not the HED column so treat as usual
     const replacementString = this.columnReplacements.get(columnName)
+
+    // Handle null or undefined replacement strings
     if (replacementString === null) {
       return null
     }
-    if (columnName === 'HED') {
-      return this._spliceHedColumnTemplate()
-    }
     if (replacementString === undefined) {
       this.issues.push(generateIssue('undefinedCurlyBraces', { column: columnName }))
       return []
     }
+
+    // Handle recursive curly braces
     if (replacementString.columnSplices.length > 0) {
       this.issues.push(generateIssue('recursiveCurlyBraces', { column: columnName }))
       return []
     }
-    const tagsHavePlaceholder = replacementString.tags.some((tag) => tag.originalTagName === '#')
-    if (tagsHavePlaceholder) {
+
+    // Handle value templates with placeholder
+    if (replacementString.tags.some((tag) => tag.originalTagName === '#')) {
       return this._spliceValueTemplate(columnTemplate)
     }
+
+    // Default case
     return replacementString.parseTree
   }
 
@@ -142,6 +153,15 @@ export class ColumnSplicer {
   _spliceHedColumnTemplate() {
     const columnName = 'HED'
     const replacementString = this.columnValues.get(columnName)
+    if (
+      replacementString === undefined ||
+      replacementString === null ||
+      replacementString === 'n/a' ||
+      replacementString === ''
+    ) {
+      return null
+    }
+
     return this._reparseAndSpliceString(replacementString)
   }
 

parser/parsedHedTag.js

@@ -3,7 +3,7 @@ import { Schema } from '../common/schema/types'
 import { getTagLevels, replaceTagNameWithPound } from '../utils/hedStrings'
 import ParsedHedSubstring from './parsedHedSubstring'
 import { SchemaValueTag } from '../validator/schema/types'
-import TagConverter from './converter'
+import { TagConverter } from './tagConverter'
 
 /**
  * A parsed HED tag.
@@ -65,7 +65,7 @@ export class ParsedHedTag extends ParsedHedSubstring {
   }
 
   /**
-   * Format this HED tag by removing newlines, double quotes, and slashes.
+   * Format this HED tag by removing newlines and double quotes.
    *
    * @returns {string} The formatted version of this tag.
    */
@@ -78,12 +78,12 @@ export class ParsedHedTag extends ParsedHedSubstring {
     if (hedTagString.endsWith('"')) {
       hedTagString = hedTagString.slice(0, -1)
     }
-    if (hedTagString.startsWith('/')) {
-      hedTagString = hedTagString.slice(1)
-    }
-    if (hedTagString.endsWith('/')) {
-      hedTagString = hedTagString.slice(0, -1)
-    }
+    // if (hedTagString.startsWith('/')) {
+    //   hedTagString = hedTagString.slice(1)
+    // }
+    // if (hedTagString.endsWith('/')) {
+    //   hedTagString = hedTagString.slice(0, -1)
+    // }
     return hedTagString.toLowerCase()
   }
 
@@ -315,12 +315,12 @@ export class ParsedHed3Tag extends ParsedHedTag {
    * @throws {IssueError} If tag conversion or parsing fails.
    */
   _convertTag(hedSchemas, hedString, tagSpec) {
-    const hed3ValidCharacters = /^[^{}[\]()~,\0\t]+$/
-    if (!hed3ValidCharacters.test(this.originalTag)) {
-      IssueError.generateAndThrow('internalConsistencyError', {
-        message: 'The parser failed to properly remove an illegal or special character.',
-      })
-    }
+    // const hed3ValidCharacters = /^[^{}[\]()~,\0\t]+$/
+    // if (!hed3ValidCharacters.test(this.originalTag)) {
+    //   IssueError.generateAndThrow('internalConsistencyError', {
+    //     message: 'The parser failed to properly remove an illegal or special character.',
+    //   })
+    // }
 
     const schemaName = tagSpec.library
     this.schema = hedSchemas.getSchema(schemaName)

parser/converter.js → parser/tagConverter.js

@@ -5,7 +5,7 @@ import { SchemaValueTag } from '../validator/schema/types'
 /**
  * Converter from a tag specification to a schema-based tag object.
  */
-export default class TagConverter {
+export class TagConverter {
   /**
    * A parsed tag token.
    * @type {TagSpec}