Add highlight tests

These tests use files from the top-repos corpus and annotate lines that
have intentional highlight formatting.

Highlight tests can only run on tree-sitter 0.20.4+ (before that there were
bugs that made it very slow), so must be disabled during the check on older
tree-sitter versions.

.gitattributes

@@ -0,0 +1 @@
+test/highlight/* linguist-vendored

.github/workflows/test-older-tree-sitter.yml

@@ -20,4 +20,6 @@ jobs:
         cache: 'npm'
     - run: cp package-json-old-tree-sitter.json package.json
     - run: npm install
-    - run: npx tree-sitter test -f "Does not match any named tests; only runs queries."
+    # corpus tests fail and highlight tests are extremely slow; just remove them!
+    - run: rm -r test
+    - run: npx tree-sitter test

.gitignore

@@ -3,7 +3,6 @@ node_modules
 !/src/scanner.c
 *.swp
 /build
-/test
 Cargo.lock
 /target/*
 *.a

scripts/test-with-memcheck.sh

@@ -8,17 +8,4 @@ if [[ "$1" == "--install-valgrind" ]]; then
     shift
 fi
 
-# Query tests hang forever when run with valgrind, so move them out of
-# the way.
-mv ./queries ./queries.bak
-trap "mv ./queries.bak ./queries || true" EXIT
-
 valgrind tree-sitter test
-
-# Now move the query tests back, and move the corpus tests out of the
-# way.
-mv ./queries.bak ./queries
-mv ./corpus ./corpus.bak
-trap "mv ./corpus.bak ./corpus || true" EXIT
-
-tree-sitter test

scripts/write-generated-grammar.sh

@@ -35,7 +35,7 @@ git add ./src/tree_sitter/* --force
 git add ./src/*.json --force
 git add grammar.js
 git add package.json
-git add corpus
+git add test
 git add queries
 git add Makefile
 git add bindings/c/*.in

test/highlight/AppDelegate.swift

@@ -0,0 +1,29 @@
+import Cocoa
+// ^ include
+import GRDB
+
+@NSApplicationMain
+// ^ type
+class AppDelegate: NSObject, NSApplicationDelegate {
+// ^ keyword
+//    ^ type
+//               ^ punctuation.delimiter
+//                 ^ type
+//                         ^ punctuation.delimiter
+//                                                 ^ punctuation.bracket
+    func applicationDidFinishLaunching(_ aNotification: Notification) {
+//  ^ keyword.function
+//       ^ method
+//                                     ^ parameter
+//                                       ^ parameter
+        _ = try! DatabaseQueue()
+//        ^ operator
+//          ^ operator
+//               ^ function.call
+        _ = FTS5()
+        _ = sqlite3_preupdate_new(nil, 0, nil)
+//                                ^ variable.builtin
+//                                     ^ number
+    }
+//  ^ punctuation.bracket
+}

test/highlight/Bag.swift

@@ -0,0 +1,201 @@
+//
+//  Bag.swift
+//  Platform
+//
+//  Created by Krunoslav Zaher on 2/28/15.
+//  Copyright © 2015 Krunoslav Zaher. All rights reserved.
+//
+
+import Swift
+
+let arrayDictionaryMaxSize = 30
+
+struct BagKey {
+// ^ keyword
+    /**
+    Unique identifier for object added to `Bag`.
+
+    It's underlying type is UInt64. If we assume there in an idealized CPU that works at 4GHz,
+     it would take ~150 years of continuous running time for it to overflow.
+    */
+//  ^ comment
+    fileprivate let rawValue: UInt64
+//  ^ keyword
+//              ^ keyword
+//                  ^ property
+}
+
+/**
+Data structure that represents a bag of elements typed `T`.
+
+Single element can be stored multiple times.
+
+Time and space complexity of insertion and deletion is O(n). 
+
+It is suitable for storing small number of elements.
+*/
+struct Bag<T> : CustomDebugStringConvertible {
+//         ^ type
+    /// Type of identifier for inserted elements.
+//  ^ comment
+    typealias KeyType = BagKey
+
+    typealias Entry = (key: BagKey, value: T)
+
+    private var _nextKey: BagKey = BagKey(rawValue: 0)
+//  ^ keyword
+//          ^ keyword
+//              ^ property
+
+    // data
+
+    // first fill inline variables
+    var _key0: BagKey?
+    var _value0: T?
+
+    // then fill "array dictionary"
+    var _pairs = ContiguousArray<Entry>()
+
+    // last is sparse dictionary
+    var _dictionary: [BagKey: T]?
+
+    var _onlyFastPath = true
+
+    /// Creates new empty `Bag`.
+    init() {
+//  ^ constructor
+    }
+
+    /**
+    Inserts `value` into bag.
+
+    - parameter element: Element to insert.
+    - returns: Key that can be used to remove element from bag.
+    */
+    mutating func insert(_ element: T) -> BagKey {
+//                                     ^ operator
+        let key = _nextKey
+
+        _nextKey = BagKey(rawValue: _nextKey.rawValue &+ 1)
+//                                                    ^ operator
+
+        if _key0 == nil {
+//      ^ conditional
+            _key0 = key
+            _value0 = element
+            return key
+//          ^ keyword.return
+        }
+
+        _onlyFastPath = false
+
+        if _dictionary != nil {
+            _dictionary![key] = element
+            return key
+        }
+
+        if _pairs.count < arrayDictionaryMaxSize {
+            _pairs.append((key: key, value: element))
+            return key
+        }
+
+        _dictionary = [key: element]
+
+        return key
+    }
+
+    /// - returns: Number of elements in bag.
+    var count: Int {
+        let dictionaryCount: Int = _dictionary?.count ?? 0
+        return (_value0 != nil ? 1 : 0) + _pairs.count + dictionaryCount
+    }
+
+    /// Removes all elements from bag and clears capacity.
+    mutating func removeAll() {
+        _key0 = nil
+        _value0 = nil
+
+        _pairs.removeAll(keepingCapacity: false)
+        _dictionary?.removeAll(keepingCapacity: false)
+    }
+
+    /**
+    Removes element with a specific `key` from bag.
+
+    - parameter key: Key that identifies element to remove from bag.
+    - returns: Element that bag contained, or nil in case element was already removed.
+    */
+    mutating func removeKey(_ key: BagKey) -> T? {
+        if _key0 == key {
+            _key0 = nil
+            let value = _value0!
+            _value0 = nil
+            return value
+        }
+
+        if let existingObject = _dictionary?.removeValue(forKey: key) {
+            return existingObject
+        }
+
+        for i in 0 ..< _pairs.count where _pairs[i].key == key {
+            let value = _pairs[i].value
+            _pairs.remove(at: i)
+            return value
+        }
+
+        return nil
+    }
+}
+
+extension Bag {
+// ^ keyword
+//        ^ type
+    /// A textual representation of `self`, suitable for debugging.
+    var debugDescription : String {
+        "\(self.count) elements in Bag"
+//      ^ string
+//       ^ punctuation.bracket
+//         ^ variable.builtin
+    }
+}
+
+extension Bag {
+    /// Enumerates elements inside the bag.
+    ///
+    /// - parameter action: Enumeration closure.
+    func forEach(_ action: (T) -> Void) {
+        if _onlyFastPath {
+            if let value0 = _value0 {
+                action(value0)
+            }
+            return
+        }
+
+        let value0 = _value0
+        let dictionary = _dictionary
+
+        if let value0 = value0 {
+            action(value0)
+        }
+
+        for i in 0 ..< _pairs.count {
+            action(_pairs[i].value)
+        }
+
+        if dictionary?.count ?? 0 > 0 {
+            for element in dictionary!.values {
+                action(element)
+            }
+        }
+    }
+}
+
+extension BagKey: Hashable {
+    func hash(into hasher: inout Hasher) {
+        hasher.combine(rawValue)
+    }
+}
+
+func ==(lhs: BagKey, rhs: BagKey) -> Bool {
+    lhs.rawValue == rhs.rawValue
+}

test/highlight/HeroStringConvertible.swift

@@ -0,0 +1,65 @@
+// The MIT License (MIT)
+//
+// Copyright (c) 2016 Luke Zhao <me@lkzhao.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+import Foundation
+
+public protocol HeroStringConvertible {
+  static func from(node: ExprNode) -> Self?
+// ^ keyword
+}
+
+extension String {
+  func parse<T: HeroStringConvertible>() -> [T]? {
+//            ^ punctuation.delimiter
+//              ^ type
+    let lexer = Lexer(input: self)
+    let parser = Parser(tokens: lexer.tokenize())
+//                                   ^ punctuation.delimiter
+//                                    ^ function.call
+    do {
+//  ^ keyword
+      let nodes = try parser.parse()
+//                ^ operator
+      var results = [T]()
+//                  ^ punctuation.bracket
+      for node in nodes {
+        if let modifier = T.from(node: node) {
+          results.append(modifier)
+        } else {
+          print("\(node.name) doesn't exist in \(T.self)")
+//              ^ string
+        }
+      }
+      return results
+    } catch let error {
+//    ^ keyword
+//          ^ keyword
+//              ^ variable
+      print("failed to parse \"\(self)\", error: \(error)")
+    }
+    return nil
+  }
+
+  func parseOne<T: HeroStringConvertible>() -> T? {
+    return parse()?.last
+  }
+}