Merge pull request #222 from SteveL-MSFT/tree-sitter

Tree-sitter grammar for expressions

.gitignore

@@ -2,3 +2,7 @@ Cargo.lock
 target
 bin/
 .DS_Store
+
+# Node.js generated files for tree-sitter
+build/
+node_modules/

build.ps1

@@ -93,7 +93,7 @@ New-Item -ItemType Directory $target > $null
 # make sure dependencies are built first so clippy runs correctly
 $windows_projects = @("pal", "ntreg", "ntstatuserror", "ntuserinfo", "registry")
 $projects = @("dsc_lib", "dsc", "osinfo", "process", "tools/test_group_resource", "y2j", "powershellgroup", "tools/dsctest")
-$pedantic_unclean_projects = @("ntreg")
+$pedantic_unclean_projects = @("tree-sitter-dscexpression", "ntreg")
 
 if ($IsWindows) {
     $projects += $windows_projects

tree-sitter-dscexpression/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "tree-sitter-dscexpression"
+description = "DSCExpression grammar for the tree-sitter parsing library"
+version = "0.0.1"
+keywords = ["incremental", "parsing", "DSCExpression"]
+categories = ["parsing", "text-editors"]
+repository = "https://github.com/powershell/dsc"
+edition = "2018"
+license = "MIT"
+
+build = "bindings/rust/build.rs"
+include = [
+  "bindings/rust/*",
+  "grammar.js",
+  "queries/*",
+  "src/*",
+]
+
+[lib]
+path = "bindings/rust/lib.rs"
+
+[dependencies]
+tree-sitter = "~0.20.10"
+
+[build-dependencies]
+cc = "1.0"

tree-sitter-dscexpression/binding.gyp

@@ -0,0 +1,19 @@
+{
+  "targets": [
+    {
+      "target_name": "tree_sitter_DSCExpression_binding",
+      "include_dirs": [
+        "<!(node -e \"require('nan')\")",
+        "src"
+      ],
+      "sources": [
+        "bindings/node/binding.cc",
+        "src/parser.c",
+        # If your language uses an external scanner, add it here.
+      ],
+      "cflags_c": [
+        "-std=c99",
+      ]
+    }
+  ]
+}

tree-sitter-dscexpression/bindings/node/binding.cc

@@ -0,0 +1,28 @@
+#include "tree_sitter/parser.h"
+#include <node.h>
+#include "nan.h"
+
+using namespace v8;
+
+extern "C" TSLanguage * tree_sitter_DSCExpression();
+
+namespace {
+
+NAN_METHOD(New) {}
+
+void Init(Local<Object> exports, Local<Object> module) {
+  Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New);
+  tpl->SetClassName(Nan::New("Language").ToLocalChecked());
+  tpl->InstanceTemplate()->SetInternalFieldCount(1);
+
+  Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
+  Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
+  Nan::SetInternalFieldPointer(instance, 0, tree_sitter_DSCExpression());
+
+  Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("DSCExpression").ToLocalChecked());
+  Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
+}
+
+NODE_MODULE(tree_sitter_DSCExpression_binding, Init)
+
+}  // namespace

tree-sitter-dscexpression/bindings/node/index.js

@@ -0,0 +1,19 @@
+try {
+  module.exports = require("../../build/Release/tree_sitter_DSCExpression_binding");
+} catch (error1) {
+  if (error1.code !== 'MODULE_NOT_FOUND') {
+    throw error1;
+  }
+  try {
+    module.exports = require("../../build/Debug/tree_sitter_DSCExpression_binding");
+  } catch (error2) {
+    if (error2.code !== 'MODULE_NOT_FOUND') {
+      throw error2;
+    }
+    throw error1
+  }
+}
+
+try {
+  module.exports.nodeTypeInfo = require("../../src/node-types.json");
+} catch (_) {}

tree-sitter-dscexpression/bindings/rust/build.rs

@@ -0,0 +1,40 @@
+fn main() {
+    let src_dir = std::path::Path::new("src");
+
+    let mut c_config = cc::Build::new();
+    c_config.include(&src_dir);
+    c_config
+        .flag_if_supported("-Wno-unused-parameter")
+        .flag_if_supported("-Wno-unused-but-set-variable")
+        .flag_if_supported("-Wno-trigraphs");
+    let parser_path = src_dir.join("parser.c");
+    c_config.file(&parser_path);
+
+    // If your language uses an external scanner written in C,
+    // then include this block of code:
+
+    /*
+    let scanner_path = src_dir.join("scanner.c");
+    c_config.file(&scanner_path);
+    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
+    */
+
+    c_config.compile("parser");
+    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
+
+    // If your language uses an external scanner written in C++,
+    // then include this block of code:
+
+    /*
+    let mut cpp_config = cc::Build::new();
+    cpp_config.cpp(true);
+    cpp_config.include(&src_dir);
+    cpp_config
+        .flag_if_supported("-Wno-unused-parameter")
+        .flag_if_supported("-Wno-unused-but-set-variable");
+    let scanner_path = src_dir.join("scanner.cc");
+    cpp_config.file(&scanner_path);
+    cpp_config.compile("scanner");
+    println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
+    */
+}

tree-sitter-dscexpression/bindings/rust/lib.rs

@@ -0,0 +1,52 @@
+//! This crate provides DSCExpression language support for the [tree-sitter][] parsing library.
+//!
+//! Typically, you will use the [language][language func] function to add this language to a
+//! tree-sitter [Parser][], and then use the parser to parse some code:
+//!
+//! ```
+//! let code = "";
+//! let mut parser = tree_sitter::Parser::new();
+//! parser.set_language(tree_sitter_DSCExpression::language()).expect("Error loading DSCExpression grammar");
+//! let tree = parser.parse(code, None).unwrap();
+//! ```
+//!
+//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
+//! [language func]: fn.language.html
+//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
+//! [tree-sitter]: https://tree-sitter.github.io/
+
+use tree_sitter::Language;
+
+extern "C" {
+    fn tree_sitter_DSCExpression() -> Language;
+}
+
+/// Get the tree-sitter [Language][] for this grammar.
+///
+/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
+pub fn language() -> Language {
+    unsafe { tree_sitter_DSCExpression() }
+}
+
+/// The content of the [`node-types.json`][] file for this grammar.
+///
+/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
+pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
+
+// Uncomment these to include any queries that this grammar contains
+
+// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
+// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm");
+// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm");
+// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn test_can_load_grammar() {
+        let mut parser = tree_sitter::Parser::new();
+        parser
+            .set_language(super::language())
+            .expect("Error loading DSCExpression language");
+    }
+}

tree-sitter-dscexpression/build.ps1

@@ -0,0 +1,14 @@
+#requires -version 7.4
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# check if tools are installed
+
+$PSNativeCommandUseErrorActionPreference = $true
+$ErrorActionPreference = 'Stop'
+
+npx tree-sitter generate
+node-gyp configure
+node-gyp build
+npx tree-sitter test

tree-sitter-dscexpression/corpus/invalid_expressions.txt

@@ -0,0 +1,75 @@
+=====
+String literal looking like function
+=====
+
+functionOne(1,2)
+
+---
+
+    (statement
+      (stringLiteral))
+
+=====
+String parameter not in quotes
+=====
+
+[functionOne(alpha, beta)]
+
+---
+
+    (statement
+      (expression
+        (function
+          (functionName)
+          (ERROR
+            (functionName)
+            (functionName)))))
+
+=====
+Missing parenthesis
+=====
+
+[functionOne]
+
+---
+
+    (ERROR
+      (functionName))
+
+=====
+Missing parameter
+=====
+
+[functionOne(1, ,2)]
+
+---
+
+    (statement
+      (expression
+        (function
+          (functionName)
+          (number)
+          (ERROR)
+          (number))))
+
+=====
+Escaped bracket
+=====
+
+[[notFunction()]
+
+---
+
+    (statement
+      (escapedStringLiteral))
+
+=====
+String literal
+=====
+
+This is a string
+
+---
+
+    (statement
+      (stringLiteral))

tree-sitter-dscexpression/corpus/valid_expressions.txt

@@ -0,0 +1,96 @@
+=====
+Function no args
+=====
+
+[functionOne()]
+
+---
+
+    (statement
+      (expression
+        (function
+          (functionName))))
+
+=====
+Simple expression
+=====
+
+[myFunction('argString')]
+
+---
+
+    (statement
+      (expression
+        (function
+          (functionName)
+          (string))))
+
+=====
+Whitespace
+=====
+
+  [  functionOne   ( 'arg'  , 2 )  ]
+
+---
+
+    (statement
+      (expression
+        (function
+          (functionName)
+          (string)
+          (number))))
+
+=====
+Multiple arguments
+=====
+
+[myFunction('argString', 1, true)]
+
+---
+
+    (statement
+      (expression
+        (function
+          (functionName)
+          (string)
+          (number)
+          (boolean))))
+
+=====
+Nested functions
+=====
+
+[functionOne('argString', functionTwo(1, functionThree('threeString', 3), 2), 'oneString')]
+
+---
+
+    (statement
+      (expression
+        (function
+          (functionName)
+          (string)
+          (function
+            (functionName)
+            (number)
+            (function
+              (functionName)
+              (string)
+              (number))
+            (number))
+          (string))))
+
+=====
+Function with dot-notation
+=====
+
+[functionOne('argString').prop1.prop2]
+
+---
+
+    (statement
+      (expression
+        (function
+          (functionName)
+          (string))
+        (memberName)
+        (memberName)))