Bring to head of internal repo

README.md

@@ -74,23 +74,23 @@ Queries try to follow the [conventions established by tree-sitter.](https://tree
 
 Most captures also include documentation as `@doc`. `@definition.function` and `@definition.method` also capture `@codeium.parameters`.
 
-| Top-level capture         | Python | TypeScript | JavaScript | Go  | Java | C++   | PHP | Ruby | C#  |
-| ------------------------- | ------ | ---------- | ---------- | --- | ---- | ----- | --- | ---- | --- |
-| `@definition.class`       | ✓      | ✓          | ✓          | ✓   | ✓    | ✓     | ✓   | ✓    | ✓   |
-| `@definition.function`    | ✓      | ✓[^3]      | ✓          | ✓   | N/A  | ✓     | ✓   | N/A  | N/A |
-| `@definition.method`      | ✓[^1]  | ✓[^3]      | ✓          | ✓   | ✓    | ✓[^1] | ✓   | ✓    | ✓   |
-| `@definition.constructor` | ✓      | ✓          | ✓          | N/A | ✗    | ✗     | ✗   | ✗    | ✓   |
-| `@definition.interface`   | N/A    | ✓          | N/A        | ✓   | ✓    | N/A   | ✓   | ✗    | ✗   |
-| `@definition.namespace`   | N/A    | ✓          | N/A        | N/A | N/A  | ✓     | ✓   | N/A  | ✓   |
-| `@definition.module`      | N/A    | ✓          | N/A        | N/A | N/A  | ✗     | N/A | ✓    | N/A |
-| `@definition.type`        | N/A    | ✓          | N/A        | ✓   | N/A  | ✗     | ✗   | N/A  | N/A |
-| `@definition.constant`    | ✗      | ✗          | ✗          | ✗   | ✗    | ✗     | ✗   | ✗    | ✗   |
-| `@definition.enum`        | ✗      | ✗          | ✗          | ✗   | ✗    | ✗     | ✗   | N/A  | ✓   |
-| `@definition.import`      | ✓      | ✓          | ✓          | ✗   | ✗    | ✗     | N/A | ✓    | ✗   |
-| `@definition.include`     | N/A    | N/A        | N/A        | N/A | N/A  | ✗     | ✗   | N/A  | N/A |
-| `@definition.package`     | N/A    | N/A        | N/A        | ✓   | ✓    | N/A   | N/A | N/A  | N/A |
-| `@reference.call`         | ✓      | ✓          | ✓          | ✓   | ✗    | ✗     | ✗   | ✓    | ✗   |
-| `@reference.class`        | ✓[^2]  | ✓          | ✓          | ✓   | ✗    | ✗     | ✗   | ✗    | ✗   |
+| Top-level capture         | Python | TypeScript | JavaScript | Go  | Java | C++   | PHP | Ruby | C#  | Perl  | Kotlin | Dart  | Bash | C   |
+| ------------------------- | ------ | ---------- | ---------- | --- | ---- | ----- | --- | ---- | --- | ----- | ------ | ----- | ---- | --- |
+| `@definition.class`       | ✓      | ✓          | ✓          | ✓   | ✓    | ✓     | ✓   | ✓    | ✓   | ✓     | ✓      | ✓     | ✗    | ✗   |
+| `@definition.function`    | ✓      | ✓[^3]      | ✓          | ✓   | N/A  | ✓     | ✓   | N/A  | N/A | ✓     | ✓      | ✓     | ✓    | ✓   |
+| `@definition.method`      | ✓[^1]  | ✓[^3]      | ✓          | ✓   | ✓    | ✓[^1] | ✓   | ✓    | ✓   | ✓[^1] | ✓      | ✓[^1] | ✓    | ✓   |
+| `@definition.constructor` | ✓      | ✓          | ✓          | N/A | ✗    | ✗     | ✗   | ✗    | ✓   | ✗     | ✓      | ✗     | N/A  | N/A |
+| `@definition.interface`   | N/A    | ✓          | N/A        | ✓   | ✓    | N/A   | ✓   | ✗    | ✓   | N/A   | ✗      | ✗     | N/A  | N/A |
+| `@definition.namespace`   | N/A    | ✓          | N/A        | N/A | N/A  | ✓     | ✓   | N/A  | ✓   | ✗     | ✗      | N/A   | N/A  | N/A |
+| `@definition.module`      | N/A    | ✓          | N/A        | N/A | N/A  | ✗     | N/A | ✓    | N/A | N/A   | N/A    | ✗     | N/A  | N/A |
+| `@definition.type`        | N/A    | ✓          | N/A        | ✓   | N/A  | ✗     | ✗   | N/A  | N/A | N/A   | N/A    | ✗     | N/A  | N/A |
+| `@definition.constant`    | ✗      | ✗          | ✗          | ✗   | ✗    | ✗     | ✗   | ✗    | ✗   | ✗     | ✗      | ✗     | N/A  | ✗   |
+| `@definition.enum`        | ✗      | ✗          | ✗          | ✗   | ✗    | ✗     | ✗   | N/A  | ✓   | N/A   | ✗      | ✗     | N/A  | ✗   |
+| `@definition.import`      | ✓      | ✓          | ✓          | ✗   | ✗    | ✗     | N/A | ✓    | ✗   | ✓     | ✓      | ✗     | N/A  | ✓   |
+| `@definition.include`     | N/A    | N/A        | N/A        | N/A | N/A  | ✗     | ✗   | N/A  | N/A | N/A   | N/A    | N/A   | N/A  | N/A |
+| `@definition.package`     | N/A    | N/A        | N/A        | ✓   | ✓    | N/A   | N/A | N/A  | N/A | N/A   | N/A    | N/A   | N/A  | N/A |
+| `@reference.call`         | ✓      | ✓          | ✓          | ✓   | ✗    | ✗     | ✗   | ✓    | ✗   | ✗     | ✗      | ✗     | ✗    | ✗   |
+| `@reference.class`        | ✓[^2]  | ✓          | ✓          | ✓   | ✗    | ✗     | ✗   | ✗    | ✗   | ✗     | ✗      | ✗     | N/A  | N/A |
 
 | Language | Supported injections   |
 | -------- | ---------------------- |

download_parse.sh

@@ -2,7 +2,7 @@
 set -euo pipefail
 
 cd "$(dirname "${BASH_SOURCE[0]}")"
-VERSION="v0.0.14"
+VERSION="v0.0.15"
 rm -f parse.gz parse
 curl -Lo parse.gz "https://github.com/Exafunction/codeium-parse/releases/download/$VERSION/parse.gz"
 gzip -d parse.gz

goldens/test.c.golden

@@ -0,0 +1,26 @@
+Name: "runtime/parser.h"
+Definition (definition.import):
+#include "runtime/parser.h"
+
+
+
+Name: parser__push
+Parameters: (Parser* self, StackVersion version, Tree* tree,
+                         TSStateId state)
+Return type: void
+Definition (definition.function):
+static void parser__push(Parser* self, StackVersion version, Tree* tree,
+                         TSStateId state) {
+  ts_stack_push(self->stack, version, tree, false, state);
+  ts_tree_release(tree);
+}
+
+Name: parser__lex
+Parameters: (Parser* self, StackVersion version)
+Return type: Tree
+Definition (definition.function):
+static Tree* parser__lex(Parser* self, StackVersion version) {
+  TSStateId parse_state = ts_stack_top_state(self->stack, version);
+  Length start_position = ts_stack_top_position(self->stack, version);
+  TSLexMode lex_mode = self->language->lex_modes[parse_state];
+}

goldens/test.cpp.golden

@@ -41,6 +41,24 @@ class Foo {
   }
 }
 
+Name: Foo
+Parameters: ()
+Doc:
+// Constructor comment.
+Declaration (definition.function):
+Foo() = default;
+Lineage: [Foo]
+Lineage types: [class]
+
+Name: Foo
+Parameters: (int /*unused*/)
+Doc:
+// Constructor comment.
+Definition (definition.function):
+explicit Foo(int /*unused*/) {}
+Lineage: [Foo]
+Lineage types: [class]
+
 Name: foo
 Parameters: ()
 Return type: void

goldens/test.cs.golden

@@ -34,6 +34,8 @@ namespace Namespace {
     // Comment Bar
     public void Bar() => bar = 0;
   }
+  // Comment IFoo.
+  public interface IFoo {}
 }
 
 Name: F
@@ -136,3 +138,11 @@ Definition (definition.method):
 public void Bar() => bar = 0;
 Lineage: [Namespace Class]
 Lineage types: [namespace class]
+
+Name: IFoo
+Doc:
+// Comment IFoo.
+Definition (definition.interface):
+public interface IFoo {}
+Lineage: [Namespace]
+Lineage types: [namespace]

goldens/test.dart.golden

@@ -0,0 +1,71 @@
+Name: foo
+Parameters: ()
+Definition (definition.function):
+void foo()
+
+Name: bar
+Parameters: ()
+Doc:
+/// bar is a docstring
+Definition (definition.function):
+int bar()
+
+Name: fooClassBar
+Parameters: ()
+Doc:
+/// FCB docstring
+Definition (definition.function):
+int fooClassBar()
+Lineage: [FooClass]
+Lineage types: [class]
+
+Name: FooClass
+Definition (definition.class):
+class FooClass {
+    int someField = 0;
+
+    // Constructor
+    FooClass() {
+        this.someField = 0;
+    }
+
+    /// FCB docstring
+    int fooClassBar() {
+        return 0;
+    }
+}
+
+Name: FooMixin
+Definition (definition.class):
+mixin FooMixin {
+    int barMixin() {
+        return 0;
+    }
+}
+
+Name: barMixin
+Parameters: ()
+Definition (definition.function):
+int barMixin()
+Lineage: [FooMixin]
+Lineage types: [class]
+
+Name: bar
+Parameters: ()
+Definition (definition.function):
+int bar()
+Lineage: [FooBar]
+Lineage types: [class]
+
+Name: FooBar
+Definition (definition.class):
+class FooBar extends FooClass with Foo {
+    int bar() {
+        return 0;
+    }
+}
+
+Name: main
+Parameters: ()
+Definition (definition.function):
+void main()

goldens/test.kt.golden

@@ -0,0 +1,55 @@
+Name: kotlinx.coroutines
+Named imports:
+    kotlinx.coroutines
+
+Name: kotlinx.coroutines.sync
+Named imports:
+    kotlinx.coroutines.sync
+
+Name: Foo
+Definition (definition.class):
+class Foo(param1: Int, param2: Int) {
+  /**
+   * Constructor docstring
+   *
+   * @param param1
+   * @param param2
+   */
+  constructor(param1: Int, param2: Int) {}
+
+  /** Method docstring */
+  fun methodWithDocstring(): void {}
+}
+
+Name: constructor
+Parameters: (param1: Int, param2: Int)
+Doc:
+/**
+   * Constructor docstring
+   *
+   * @param param1
+   * @param param2
+   */
+Declaration (definition.constructor):
+constructor(param1: Int, param2: Int) {}
+Lineage: [Foo]
+Lineage types: [class]
+
+Name: methodWithDocstring
+Parameters: ()
+Doc:
+/** Method docstring */
+Definition (definition.function):
+fun methodWithDocstring(): void {}
+Lineage: [Foo]
+Lineage types: [class]
+
+Name: fib
+Parameters: (n: Int)
+Definition (definition.function):
+fun fib(n: Int): Int {
+  if (n == 0 || n == 1) {
+    return n
+  }
+  return fib(n - 1) + fib(n - 2)
+}

goldens/test.pl.golden

@@ -0,0 +1,62 @@
+Name: strict
+Definition (definition.import):
+use strict 1.0;
+
+Name: feature
+Definition (definition.import):
+use feature 'class';
+
+Name: FooClass
+Definition (definition.class):
+class FooClass {
+    field $someField = 42;
+
+    method bar {
+        say "The answer is $someField";
+    }
+}
+
+Name: bar
+Definition (definition.function):
+method bar {
+        say "The answer is $someField";
+    }
+Lineage: [FooClass]
+Lineage types: [class]
+
+Name: foo2
+Parameters: ()
+Doc:
+# foo2 comment
+Definition (definition.function):
+sub foo2 () {
+    my $foo = Foo.new;
+    $foo.bar
+}
+
+Name: new
+Parameters: ($class, $firstName, $lastName)
+Definition (definition.function):
+sub new ($class, $firstName, $lastName) {
+    bless { firstName => $firstName, lastName => $lastName }, $class
+}
+
+Name: getFirstName
+Parameters: ($self)
+Definition (definition.function):
+sub getFirstName($self) {
+    $self->{firstName}
+}
+
+Name: Average
+Definition (definition.function):
+sub Average {
+   # get total number of arguments passed.
+   $n = scalar(@_);
+   $sum = 0;
+   foreach $item (@_) {
+      $sum += $item;
+   }
+   $average = $sum / $n;
+   return $average;
+}

goldens/test.proto.golden

@@ -0,0 +1,39 @@
+Name: b
+Definition (definition.package):
+package a.b;
+Lineage: [a]
+Lineage types: [package]
+
+Name: TestService
+Declaration (definition.class):
+service TestService {
+  rpc Echo(EchoRequest) returns (EchoResponse) {}
+}
+Lineage: [a b]
+Lineage types: [package package]
+
+Name: Echo
+Declaration (definition.function):
+rpc Echo(EchoRequest) returns (EchoResponse) {}
+Lineage: [a b TestService]
+Lineage types: [package package class]
+
+Name: EchoRequest
+Doc:
+// EchoRequest comment.
+Definition (definition.class):
+message EchoRequest {
+  string message = 1;
+}
+Lineage: [a b]
+Lineage types: [package package]
+
+Name: EchoResponse
+Doc:
+// EchoResponse comment.
+Definition (definition.class):
+message EchoResponse {
+  string message = 1;
+}
+Lineage: [a b]
+Lineage types: [package package]

goldens/test.sh.golden

@@ -0,0 +1,16 @@
+Name: do_something
+Definition (definition.function):
+do_something() {
+  echo ok
+}
+
+Name: do_nothing
+Definition (definition.function):
+do_nothing() { return 0; }
+
+Name: add
+Definition (definition.function):
+add() {
+  sum=$(($1 + $2))
+  echo $sum
+}

queries/bash_tags.scm

@@ -0,0 +1,10 @@
+;; Functions
+
+(
+  (function_definition
+    name: (word) @name
+    body: (_) @body
+  ) @definition.function
+)
+
+;; What else do we need here?

queries/c_class_fields.scm

@@ -0,0 +1,11 @@
+;; C struct fields. Unlike for C++, there is no `class_specifier` here.
+
+(
+    (struct_specifier
+        body: (
+            (field_declaration_list
+                (_) @field
+            )
+        )
+    )
+)

queries/c_tags.scm

@@ -0,0 +1,39 @@
+;; Basic function definitions
+(
+  (comment)* @doc
+  .
+  (function_definition
+    type: (_)? @codeium.return_type
+    declarator: (function_declarator
+      declarator: (identifier) @name
+      parameters: (parameter_list) @codeium.parameters
+    )
+    body: (_)? @body
+  ) @definition.function
+  (select-adjacent! @doc @definition.function)
+)
+
+;; Function to pointer
+(
+  (comment)* @doc
+  .
+  (function_definition
+    type: (_)? @codeium.return_type
+    declarator: (pointer_declarator
+      declarator: (function_declarator
+        declarator: (identifier) @name
+        parameters: (parameter_list) @codeium.parameters
+      )
+    )
+    body: (_)? @body
+  ) @definition.function
+  (select-adjacent! @doc @definition.function)
+)
+
+;; Imports
+
+(
+  (preproc_include
+    path: (_) @name
+  ) @definition.import
+)

queries/cpp_tags.scm

@@ -15,7 +15,7 @@
   (comment)* @doc
   .
   (_
-    (_type_specifier) @codeium.return_type
+    type: (_)? @codeium.return_type
     declarator: [
       (function_declarator
         declarator: (_) @name
@@ -64,7 +64,7 @@
   .
   (template_declaration
     (_
-      (_type_specifier) @codeium.return_type
+      type: (_)? @codeium.return_type
       declarator: [
         (function_declarator
           declarator: (_) @name
@@ -110,7 +110,7 @@
   (template_declaration
     (template_declaration
       (_
-        (_type_specifier) @codeium.return_type
+        type: (_)? @codeium.return_type
         declarator: [
           (function_declarator
             declarator: (_) @name

queries/csharp_tags.scm

@@ -4,73 +4,82 @@
   name: (identifier) @name) @definition.namespace
 
 (
-    (comment)* @doc
-    .
-    (struct_declaration
-      name: (identifier) @name
-      body: (declaration_list) @body) @definition.class
-    (#select-adjacent! @doc @definition.class)
+  (comment)* @doc
+  .
+  (struct_declaration
+    name: (identifier) @name
+    body: (declaration_list) @body) @definition.class
+  (#select-adjacent! @doc @definition.class)
 )
 
 (
-    (comment)* @doc
-    .
-    (record_declaration
-      name: (identifier) @name
-      body: (declaration_list)? @body) @definition.class
-    (#select-adjacent! @doc @definition.class)
+  (comment)* @doc
+  .
+  (record_declaration
+    name: (identifier) @name
+    body: (declaration_list)? @body) @definition.class
+  (#select-adjacent! @doc @definition.class)
 )
 
 (
-    (comment)* @doc
-    .
-    (enum_declaration
-      name: (identifier) @name
-      body: (enum_member_declaration_list) @body) @definition.enum
-    (#select-adjacent! @doc @definition.enum)
+  (comment)* @doc
+  .
+  (enum_declaration
+    name: (identifier) @name
+    body: (enum_member_declaration_list) @body) @definition.enum
+  (#select-adjacent! @doc @definition.enum)
 )
 
 (
-    (comment)* @doc
-    .
-    (class_declaration
-      name: (identifier) @name
-      body: (declaration_list) @body) @definition.class
-    (#select-adjacent! @doc @definition.class)
+  (comment)* @doc
+  .
+  (class_declaration
+    name: (identifier) @name
+    body: (declaration_list) @body) @definition.class
+  (#select-adjacent! @doc @definition.class)
 )
 
 (
-    (comment)* @doc
-    .
-    (operator_declaration
-      operator: _ @name
-      body: (_) @body) @definition.method
-    (#select-adjacent! @doc @definition.method)
+  (comment)* @doc
+  .
+  (interface_declaration
+    name: (identifier) @name
+    body: (declaration_list) @body) @definition.interface
+  (#select-adjacent! @doc @definition.interface)
 )
 
 (
-    (comment)* @doc
-    .
-    (constructor_declaration
-      name: (identifier) @name
-      body: (_) @body) @definition.constructor
-    (#select-adjacent! @doc @definition.constructor)
+  (comment)* @doc
+  .
+  (operator_declaration
+    operator: _ @name
+    body: (_) @body) @definition.method
+  (#select-adjacent! @doc @definition.method)
 )
 
 (
-    (comment)* @doc
-    .
-    (destructor_declaration
-      name: (identifier) @name
-      body: (_) @body) @definition.destructor
-    (#select-adjacent! @doc @definition.destructor)
+  (comment)* @doc
+  .
+  (constructor_declaration
+    name: (identifier) @name
+    body: (_) @body) @definition.constructor
+  (#select-adjacent! @doc @definition.constructor)
 )
 
 (
-    (comment)* @doc
-    .
-    (method_declaration
-      name: (identifier) @name
-      body: (_) @body) @definition.method
-    (#select-adjacent! @doc @definition.method)
+  (comment)* @doc
+  .
+  (destructor_declaration
+    name: (identifier) @name
+    body: (_) @body) @definition.destructor
+  (#select-adjacent! @doc @definition.destructor)
+)
+
+(
+  (comment)* @doc
+  .
+  (method_declaration
+    name: (identifier) @name
+    body: (_) @body) @definition.method
+  (#select-adjacent! @doc @definition.method)
 )

queries/dart_tags.scm

@@ -0,0 +1,63 @@
+;; We treat mixins and classes as the same
+(
+  [
+    (documentation_comment)
+  ]* @doc
+  .
+  (class_definition
+    name: (identifier) @name
+    body: (class_body) @body
+    type_parameters: (type_parameters)?
+  ) @definition.class
+  (#select-adjacent! @doc @definition.class)
+)
+
+(
+  [
+    (documentation_comment)
+  ]* @doc
+  .
+  (mixin_declaration
+    (mixin)
+    .
+    (identifier) @name
+    (class_body)? @body
+  ) @definition.class
+  (#select-adjacent! @doc @definition.class)
+)
+
+;; Functions
+(
+  [
+    (documentation_comment)
+  ]* @doc
+  .
+  (
+    (function_signature
+      name: (identifier) @name
+      (formal_parameter_list) @codeium.parameters
+    )
+    .
+    (function_body)? @body
+  ) @definition.function
+  (#select-adjacent! @doc @definition.function)
+  (#not-has-parent? @definition.function method_signature)
+)
+
+(
+  [
+    (documentation_comment)
+  ]* @doc
+  .
+  (
+    (method_signature
+      (function_signature
+        name: (identifier) @name
+        (formal_parameter_list) @codeium.parameters
+      )
+    )
+    .
+    (function_body)? @body
+  ) @definition.function
+  (#select-adjacent! @doc @definition.function)
+)

queries/java_tags.scm

@@ -36,7 +36,6 @@
   (#select-adjacent! @doc @definition.class)
 )
 
-
 (
   [
     (line_comment)

queries/kotlin_tags.scm

@@ -0,0 +1,54 @@
+;; Functions
+
+(
+  [
+    (line_comment)
+    (multiline_comment)
+  ]* @doc
+  .
+  (function_declaration
+    name: (simple_identifier) @name
+    parameters: (function_value_parameters) @codeium.parameters
+    body: (function_body)? @body
+  ) @definition.function
+  (#select-adjacent! @doc @definition.function)
+)
+
+;; Constructors
+
+(
+  [
+    (line_comment)
+    (multiline_comment)
+  ]* @doc
+  .
+  (secondary_constructor
+    "constructor" @name
+    parameters: (function_value_parameters) @codeium.parameters
+  ) @definition.constructor
+  (#select-adjacent! @doc @definition.constructor)
+)
+
+;; Classes (Also includes interfaces)
+
+(
+  [
+    (line_comment)
+    (multiline_comment)
+  ]* @doc
+  .
+  (class_declaration
+    name: (type_identifier) @name
+    body: (class_body)? @body
+  ) @definition.class
+  (#select-adjacent! @doc @definition.class)
+)
+
+;; Imports
+
+(
+  (import_header
+    (identifier) @import.named.source @name
+    (import_alias)? @import.named.alias
+  )
+)

queries/perl_tags.scm

@@ -0,0 +1,35 @@
+;; Tags for parsing Perl class statements
+(
+  (class_statement
+    name: (package) @name
+    (block) @body
+  ) @definition.class
+)
+
+;; Tags for parsing Perl functions (subroutines)
+
+(
+  (comment)* @doc
+  .
+  [
+    (subroutine_declaration_statement
+      name: (bareword) @name
+      (prototype_or_signature)? @codeium.parameters
+      body: (block) @body) @definition.function
+    (method_declaration_statement
+      name: (bareword) @name
+      (prototype_or_signature)? @codeium.parameters
+      body: (block) @body) @definition.function
+  ]
+  (#select-adjacent! @doc @definition.function)
+)
+
+;; Tags for parsing Perl imports
+
+(
+  source_file
+  (
+    use_statement
+      module: (package) @name
+  ) @definition.import
+)

queries/protobuf_class_fields.scm

@@ -0,0 +1,14 @@
+;; 1. Any possible message body value can be parsed as a class field.
+(message_body
+    (field)? @field
+    (message)? @field
+    (oneof)? @field
+    (enum)? @field
+    (map_field)? @field
+    (enum_field)? @field
+)
+
+;; 2. Parse enum values as class fields
+(enum_body
+    (enum_field) @field
+)

queries/protobuf_tags.scm

@@ -0,0 +1,39 @@
+(source_file
+  (package
+    (full_ident) @name) @definition.package
+  (#lineage-from-name! ".")
+) @codeium.lineage_node
+
+(
+  (comment)* @doc
+  .
+  (message
+    (message_name) @name
+    (message_body) @body) @definition.class
+  (#select-adjacent! @doc @definition.class)
+)
+
+(
+  (comment)* @doc
+  .
+  (enum
+    (enum_name) @name
+    (enum_body) @body) @definition.class
+  (#select-adjacent! @doc @definition.class)
+)
+
+(
+  (comment)* @doc
+  .
+  (service
+    (service_name) @name) @definition.class
+  (#select-adjacent! @doc @definition.class)
+)
+
+(
+  (comment)* @doc
+  .
+  (rpc
+    (rpc_name) @name) @definition.function
+  (#select-adjacent! @doc @definition.function)
+)

test_files/test.c

@@ -0,0 +1,18 @@
+#include "runtime/parser.h"
+
+typedef struct {
+  Parser* parser;
+  TSSymbol lookahead_symbol;
+} SkipPrecedingTreesSession;
+
+static void parser__push(Parser* self, StackVersion version, Tree* tree,
+                         TSStateId state) {
+  ts_stack_push(self->stack, version, tree, false, state);
+  ts_tree_release(tree);
+}
+
+static Tree* parser__lex(Parser* self, StackVersion version) {
+  TSStateId parse_state = ts_stack_top_state(self->stack, version);
+  Length start_position = ts_stack_top_position(self->stack, version);
+  TSLexMode lex_mode = self->language->lex_modes[parse_state];
+}

test_files/test.cs

@@ -1,3 +1,4 @@
+// clang-format off
 namespace Namespace {
   // Comment A.
   internal delegate void A(params int[] test);
@@ -32,4 +33,7 @@ public partial class Class<in TParam>
     // Comment Bar
     public void Bar() => bar = 0;
   }
+  // Comment IFoo.
+  public interface IFoo {}
 }
+// clang-format on

test_files/test.dart

@@ -0,0 +1,47 @@
+import 'package:test/test.dart';
+
+// Functions
+
+// foo not a docstring
+void foo() {
+    print("Hello, World!");
+}
+
+/// bar is a docstring
+int bar() {
+    return 0;
+}
+
+// Classes with methods
+
+class FooClass {
+    int someField = 0;
+
+    // Constructor
+    FooClass() {
+        this.someField = 0;
+    }
+
+    /// FCB docstring
+    int fooClassBar() {
+        return 0;
+    }
+}
+
+// Mixins
+mixin FooMixin {
+    int barMixin() {
+        return 0;
+    }
+}
+
+// Combinations
+class FooBar extends FooClass with Foo {
+    int bar() {
+        return 0;
+    }
+}
+
+void main() {
+    print("Hello, World!");
+}

test_files/test.kt

@@ -0,0 +1,30 @@
+package test.kotlin
+
+import kotlinx.coroutines.*
+import kotlinx.coroutines.sync.*
+
+/**
+ * Class docstring
+ *
+ * @param param1
+ * @param param2
+ */
+class Foo(param1: Int, param2: Int) {
+  /**
+   * Constructor docstring
+   *
+   * @param param1
+   * @param param2
+   */
+  constructor(param1: Int, param2: Int) {}
+
+  /** Method docstring */
+  fun methodWithDocstring(): void {}
+}
+
+fun fib(n: Int): Int {
+  if (n == 0 || n == 1) {
+    return n
+  }
+  return fib(n - 1) + fib(n - 2)
+}

test_files/test.pl

@@ -0,0 +1,37 @@
+use strict 1.0;
+use feature 'class';
+
+class FooClass {
+    field $someField = 42;
+
+    method bar {
+        say "The answer is $someField";
+    }
+}
+
+# foo2 comment
+sub foo2 () {
+    my $foo = Foo.new;
+    $foo.bar
+}
+
+package person;
+
+sub new ($class, $firstName, $lastName) {
+    bless { firstName => $firstName, lastName => $lastName }, $class
+}
+
+sub getFirstName($self) {
+    $self->{firstName}
+}
+
+sub Average {
+   # get total number of arguments passed.
+   $n = scalar(@_);
+   $sum = 0;
+   foreach $item (@_) {
+      $sum += $item;
+   }
+   $average = $sum / $n;
+   return $average;
+}

test_files/test.proto

@@ -0,0 +1,19 @@
+// Copyright Exafunction, Inc.
+
+syntax = "proto3";
+
+package a.b;
+
+service TestService {
+  rpc Echo(EchoRequest) returns (EchoResponse) {}
+}
+
+// EchoRequest comment.
+message EchoRequest {
+  string message = 1;
+}
+
+// EchoResponse comment.
+message EchoResponse {
+  string message = 1;
+}

test_files/test.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "Hello World!"
+
+do_something() {
+  echo ok
+}
+
+do_nothing() { return 0; }
+
+add() {
+  sum=$(($1 + $2))
+  echo $sum
+}