Merge pull request Wilfred#1 from Razzeee/add-lexer

Add lexer

.gitignore

@@ -3,4 +3,20 @@ node_modules
 build
 *.log
 prebuilds
-.idea
+.idea
+examples/elm-spa-example
+examples/elm-browser
+examples/elm-bytes
+examples/elm-core
+examples/elm-file
+examples/elm-html
+examples/elm-http
+examples/elm-json
+examples/elm-parser
+examples/elm-projectelm-project-metadata-utils
+examples/elm-random
+examples/elm-regex
+examples/elm-svg
+examples/elm-time
+examples/elm-url
+examples/elm-virtual-dom

.travis.yml

@@ -3,25 +3,22 @@ language: node_js
 sudo: false
 
 node_js:
-- node
+  - node
 
 os:
-- linux
+  - linux
 
 matrix:
   include:
-  - os: osx
-    env: COMPILER=clang++
-    osx_image: xcode9.2
-    compiler: clang
-  - os: linux
-    env: CXX=clang
-    compiler: clang
+    - os: osx
+      osx_image: xcode9.2
+    - os: linux
+      env: CXX=clang++
 
 branches:
   only:
-  - master
-  - /^v.*$/
+    - master
+    - /^v.*$/
 
 deploy:
   provider: script
@@ -33,4 +30,4 @@ deploy:
 
 env:
   global:
-    secure: "NTNcwplBIpGejo1ciZlkVFLGJ14nbvRf42/evwUU61gvuMe0xrCSCgoEUTL9vNJUBotG2WqwZnlfScop1Ag5E/9H/jFvJ/1UoJ99Zs+b0A+vOPgMTXjYqW+GYfAC/3HvQPDq+32Xe0a0Tq+PxuKMu0jskO1vspQMRvcYaglIkCGIO6uzwiolViTehVhdhZw1gjv+L37cXsBmccXI9j9TsYe+XceUOqi3/efik6gPcH1KJZGa5jdobVmUBhV524drzO51+q5eigAcUUP0Z8IhXrjAvB50C1Y/Opao8AYhc2IHCx6CEu6T2IeAjXaFghFt4Z+g2fzJgkdyO4ap4lweuQ/e4MHm5jBBVkBAORqtsukz/vJhQOoe4Nu9kLPUl+biW2NS3Tj+Usqb9tuSmyiw5EFeCDhMcg3M9cTLIs4ZibgYnUN/7IjsLsnTYrQ1Wht2rabn+KOfjatkVEJMnYFrizsQo0AQ7edi6X9ncPUNAViQTvpyPxknMWGEdyva4YWUcuH7SxsDqQtEeTqNMXJ+FglRuZzzDIXRn6FaxUUTpBFtuj8jblL1C9f2UvlUrmyqDJ5bBoFmeRmffbr/u357TiQybQQyQ3GroSm/KGPQjSoQFMGCgKAQv64Y/pfgLgw5PbUf5agjNNbxcJoeCFT2ceGOP8ucwHzACCTng6ZtbDU="
+    secure: "NTNcwplBIpGejo1ciZlkVFLGJ14nbvRf42/evwUU61gvuMe0xrCSCgoEUTL9vNJUBotG2WqwZnlfScop1Ag5E/9H/jFvJ/1UoJ99Zs+b0A+vOPgMTXjYqW+GYfAC/3HvQPDq+32Xe0a0Tq+PxuKMu0jskO1vspQMRvcYaglIkCGIO6uzwiolViTehVhdhZw1gjv+L37cXsBmccXI9j9TsYe+XceUOqi3/efik6gPcH1KJZGa5jdobVmUBhV524drzO51+q5eigAcUUP0Z8IhXrjAvB50C1Y/Opao8AYhc2IHCx6CEu6T2IeAjXaFghFt4Z+g2fzJgkdyO4ap4lweuQ/e4MHm5jBBVkBAORqtsukz/vJhQOoe4Nu9kLPUl+biW2NS3Tj+Usqb9tuSmyiw5EFeCDhMcg3M9cTLIs4ZibgYnUN/7IjsLsnTYrQ1Wht2rabn+KOfjatkVEJMnYFrizsQo0AQ7edi6X9ncPUNAViQTvpyPxknMWGEdyva4YWUcuH7SxsDqQtEeTqNMXJ+FglRuZzzDIXRn6FaxUUTpBFtuj8jblL1C9f2UvlUrmyqDJ5bBoFmeRmffbr/u357TiQybQQyQ3GroSm/KGPQjSoQFMGCgKAQv64Y/pfgLgw5PbUf5agjNNbxcJoeCFT2ceGOP8ucwHzACCTng6ZtbDU="

.vscode/tasks.json

@@ -17,6 +17,26 @@
             "type": "npm",
             "script": "parse-1",
             "problemMatcher": []
+        },
+        {
+            "type": "npm",
+            "script": "parse-asset",
+            "problemMatcher": []
+        },
+        {
+            "type": "npm",
+            "script": "parse-article",
+            "problemMatcher": []
+        },
+        {
+            "type": "npm",
+            "script": "parse-basic",
+            "problemMatcher": []
+        },
+        {
+            "type": "npm",
+            "script": "parse-test",
+            "problemMatcher": []
         }
     ]
 }

HOW_TO_RELEASE.MD

@@ -0,0 +1,5 @@
+1. Increase the version number in the package.json
+2. Push the code to master
+3. Run `npm dist`
+4. Create a release on github with the name being the version number from before prefixed with `v` for e.g. `v1.1.0`
+5. Enjoy, builds should appear on that realease as soon as the CI is done running them.

README.MD

@@ -3,4 +3,30 @@
 
 # Elm tree sitter
 
-This is very alpha, incomplete and just an exploration. Please keep this in mind.
+## Why am I doing this?
+
+I believe that Elm would greatly benefit from better tooling, the ultimate goal is to write a language server integration. This is a possible building block for that.
+
+What it brings to the table:
+
+- Very fast parsing, should enable parsing on each keystroke
+- Resilient, even if you use wrong syntax, most of the file should still be recognized alright
+- Should also be useful to the elm atom maintainers, as atom is using tree sitter as the new default for code highlighting (our ast might be too expressive). Highlight implementation still needs to be done if wanted.
+
+## What is this tested with?
+
+This is tested against the tests included in the repo and:
+
+- [elm-spa-example](https://github.com/rtfeldman/elm-spa-example)
+- All core elm packets from [here](https://github.com/elm)
+
+So it should work fine for a fair amount of code. What's not tested right now is behavior in error cases.
+
+## Thanks
+
+Very very big thanks goes out to @klazuka and the people of [intellij-elm](https://github.com/klazuka/intellij-elm/) as I basically stole [how they're creating their parser](https://github.com/klazuka/intellij-elm/blob/master/src/main/grammars/ElmParser.bnf) minus the GLSL implementation.
+
+## Want to help?
+
+Help writing some tests or simply find valid elm files, that fail parsing.
+Test are located in the `corpus` folder.

binding.gyp

@@ -8,7 +8,8 @@
       ],
       "sources": [
         "src/parser.c",
-        "src/binding.cc"
+        "src/binding.cc",
+        "src/scanner.cc"
       ],
       "cflags_c": [
         "-std=c99",

corpus/annotations.txt

@@ -1,7 +1,6 @@
 =====================================
 Annotation for func returning multiple
 =====================================
-
 main : Program Value Model Msg
 main =
     Api.application Viewer.decoder
@@ -15,71 +14,77 @@ main =
 ---
 
 (file
-    (type_annotation 
-        (lower_case_identifier) (colon) 
-            (type_expression 
-                (type_ref (upper_case_qid (upper_case_identifier)) (type_ref (upper_case_qid (upper_case_identifier)))
-                )
+    (type_annotation
+        (lower_case_identifier) 
+        (colon)
+        (type_expression
+            (type_ref 
+                (upper_case_qid (upper_case_identifier))
+                (type_ref (upper_case_qid (upper_case_identifier)))
+                (type_ref (upper_case_qid (upper_case_identifier)))
+                (type_ref (upper_case_qid (upper_case_identifier)))
             )
-    ) 
-    (value_declaration 
-        (pattern (union_pattern (upper_case_qid (upper_case_identifier)) 
-        (union_argument_pattern (upper_case_qid (upper_case_identifier))) 
-        (union_argument_pattern (lower_pattern (lower_case_identifier))))
-        ) (eq) 
-        (function_call_expr (value_expr (value_qid (upper_case_identifier) (dot) (lower_case_identifier)))
+        )
+    )
+    (virtual_end_decl)
+    (value_declaration
+        (function_declaration_left (lower_case_identifier))
+        (eq)
+        (function_call_expr 
+            (value_expr (value_qid (upper_case_identifier) (dot) (lower_case_identifier)))
             (value_expr (value_qid (upper_case_identifier) (dot) (lower_case_identifier)))
-            (record_expr 
+            (record_expr
                 (left_brace) (field (lower_case_identifier) (eq) (value_expr (value_qid (lower_case_identifier))))
-                (comma) (field (lower_case_identifier) (eq) (value_expr (upper_case_qid (upper_case_identifier)))) 
-                (comma) (field (lower_case_identifier) (eq) (value_expr (upper_case_qid (upper_case_identifier)))) 
-                (comma) (field (lower_case_identifier) (eq) (value_expr (value_qid (lower_case_identifier)))) 
-                (comma) (field (lower_case_identifier) (eq) (value_expr (value_qid (lower_case_identifier)))) 
-                (comma) (field (lower_case_identifier) (eq) (value_expr (value_qid (lower_case_identifier)))) 
+                (comma) (field (lower_case_identifier) (eq) (value_expr (upper_case_qid (upper_case_identifier))))
+                (comma) (field (lower_case_identifier) (eq) (value_expr (upper_case_qid (upper_case_identifier))))
+                (comma) (field (lower_case_identifier) (eq) (value_expr (value_qid (lower_case_identifier))))
+                (comma) (field (lower_case_identifier) (eq) (value_expr (value_qid (lower_case_identifier))))
+                (comma) (field (lower_case_identifier) (eq) (value_expr (value_qid (lower_case_identifier))))
                 (right_brace)
             )
         )
     )
+    (virtual_end_decl)
 )
 
 =====================================
 Annotation for func with parameters returning tuple
 =====================================
-
-
 init : Maybe Viewer -> Url -> Nav.Key -> ( Model, Cmd Msg )
 init maybeViewer url navKey =
     changeRouteTo (Route.fromUrl url)
         (Redirect (Session.fromViewer navKey maybeViewer))
 ---
 
-(file 
-    (type_annotation 
-        (lower_case_identifier) (colon) 
-            (type_expression 
+(file
+    (type_annotation
+        (lower_case_identifier) (colon)
+            (type_expression
                 (type_ref (upper_case_qid (upper_case_identifier)) (type_ref (upper_case_qid (upper_case_identifier))))
-                (arrow) (type_ref (upper_case_qid (upper_case_identifier))) 
-                (arrow) (type_ref (upper_case_qid (upper_case_identifier) (dot) (upper_case_identifier))) 
-                (arrow) (tuple_type (left_parenthesis) (type_expression (type_ref (upper_case_qid (upper_case_identifier)))) 
+                (arrow) (type_ref (upper_case_qid (upper_case_identifier)))
+                (arrow) (type_ref (upper_case_qid (upper_case_identifier) (dot) (upper_case_identifier)))
+                (arrow) (tuple_type (left_parenthesis) (type_expression (type_ref (upper_case_qid (upper_case_identifier))))
                     (comma) (type_expression (type_ref (upper_case_qid (upper_case_identifier)) (type_ref (upper_case_qid (upper_case_identifier))))) (right_parenthesis))
             )
-    ) 
-    (value_declaration 
-        (function_declaration_left (lower_case_identifier) 
-            (lower_pattern (lower_case_identifier)) 
-            (lower_pattern (lower_case_identifier)) 
-            (lower_pattern (lower_case_identifier))) (eq) 
-        (function_call_expr (value_expr (value_qid (lower_case_identifier))) 
-            (parenthesized_expr (left_parenthesis) 
+    )
+    (virtual_end_decl)
+    (value_declaration
+        (function_declaration_left (lower_case_identifier)
+            (lower_pattern (lower_case_identifier))
+            (lower_pattern (lower_case_identifier))
+            (lower_pattern (lower_case_identifier))) (eq)
+        (function_call_expr (value_expr (value_qid (lower_case_identifier)))
+            (parenthesized_expr (left_parenthesis)
                 (function_call_expr (value_expr (value_qid (upper_case_identifier) (dot) (lower_case_identifier)))
-                (value_expr (value_qid (lower_case_identifier)))) 
-            (right_parenthesis)) 
-                (parenthesized_expr (left_parenthesis) (function_call_expr (value_expr (upper_case_qid (upper_case_identifier))) 
+                (value_expr (value_qid (lower_case_identifier))))
+            (right_parenthesis))
+                (parenthesized_expr (left_parenthesis) (function_call_expr (value_expr (upper_case_qid (upper_case_identifier)))
                     (parenthesized_expr (left_parenthesis) (function_call_expr (value_expr (value_qid (upper_case_identifier) (dot) (lower_case_identifier)))
-                        (value_expr (value_qid (lower_case_identifier))) 
-                        (value_expr (value_qid (lower_case_identifier)))) 
-                    (right_parenthesis))) 
+                        (value_expr (value_qid (lower_case_identifier)))
+                        (value_expr (value_qid (lower_case_identifier))))
+                    (right_parenthesis)))
                 (right_parenthesis))
         )
     )
-) 
+    (virtual_end_decl)
+)