From 52104f7d250470bc35c640904ee2bc32633dbd7e Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sun, 28 Jan 2018 11:43:59 +0300
Subject: [PATCH 1/3] Explain TOMBSTONE and EOF

---
 src/bin/gen.rs      | 8 ++++++--
 src/syntax_kinds.rs | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/bin/gen.rs b/src/bin/gen.rs
index 4b8a5afecf81..89609bd77e61 100644
--- a/src/bin/gen.rs
+++ b/src/bin/gen.rs
@@ -51,8 +51,12 @@ impl Grammar {
             write!(acc, "    {},\n", scream(kind)).unwrap();
         }
         acc.push_str("\n");
-        acc.push_str("    TOMBSTONE = !0 - 1,\n");
-        acc.push_str("    EOF = !0,\n");
+        acc.push_str("    // Technical SyntaxKinds: they appear temporally during parsing,\n");
+        acc.push_str("    // but never end up in the final tree\n");
+        acc.push_str("    #[doc(hidden)]\n");
+        acc.push_str("    TOMBSTONE,\n");
+        acc.push_str("    #[doc(hidden)]\n");
+        acc.push_str("    EOF,\n");
         acc.push_str("}\n");
         acc.push_str("pub(crate) use self::SyntaxKind::*;\n");
         acc.push_str("\n");
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs
index aa19c2adff17..cc9e74f8e84b 100644
--- a/src/syntax_kinds.rs
+++ b/src/syntax_kinds.rs
@@ -92,8 +92,12 @@ pub enum SyntaxKind {
     ALIAS,
     VISIBILITY,
 
-    TOMBSTONE = !0 - 1,
-    EOF = !0,
+    // Technical SyntaxKinds: they appear temporally during parsing,
+    // but never end up in the final tree
+    #[doc(hidden)]
+    TOMBSTONE,
+    #[doc(hidden)]
+    EOF,
 }
 pub(crate) use self::SyntaxKind::*;
 

From cd4e13f10ac61805de41e73c7483f56624e8f5d2 Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sun, 28 Jan 2018 12:06:26 +0300
Subject: [PATCH 2/3] Document events a bit

---
 src/parser/event_parser/mod.rs | 48 ++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs
index 65aea017b193..1bb943b5de79 100644
--- a/src/parser/event_parser/mod.rs
+++ b/src/parser/event_parser/mod.rs
@@ -4,17 +4,65 @@ use {SyntaxKind, Token};
 mod parser;
 mod grammar;
 
+
+/// `Parser` produces a flat list of `Event`s.
+/// They are converted to a tree-structure in
+/// a separate pass, via `TreeBuilder`.
 #[derive(Debug)]
 pub(crate) enum Event {
+    /// This event signifies the start of the node.
+    /// It should be either abandoned (in which case the
+    /// `kind` is `TOMBSTONE`, and the event is ignored),
+    /// or completed via a `Finish` event.
+    ///
+    /// All tokens between a `Start` and a `Finish` would
+    /// become the children of the respective node.
+    ///
+    /// For left-recursive syntactic constructs, the parser produces
+    /// a child node before it sees a parent. `forward_parent`
+    /// exists to allow to tweak parent-child relationships.
+    ///
+    /// Consider this path
+    ///
+    /// foo::bar
+    ///
+    /// The events for it would look like this:
+    ///
+    ///
+    /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH
+    ///       |                          /\
+    ///       |                          |
+    ///       +------forward-parent------+
+    ///
+    /// And the tree would look like this
+    ///
+    ///    +--PATH---------+
+    ///    |   |           |
+    ///    |   |           |
+    ///    |  '::'       'bar'
+    ///    |
+    ///   PATH
+    ///    |
+    ///   'foo'
+    ///
+    /// See also `CompleteMarker::precede`.
     Start {
         kind: SyntaxKind,
         forward_parent: Option<u32>,
     },
+
+    /// Complete the previous `Start` event
     Finish,
+
+    /// Produce a single leaf-element.
+    /// `n_raw_tokens` is used to glue complex contextual tokens.
+    /// For example, lexer tokenizes `>>` as `>`, `>`, and
+    /// `n_raw_tokens = 2` is used to produced a single `>>`.
     Token {
         kind: SyntaxKind,
         n_raw_tokens: u8,
     },
+
     Error {
         message: String,
     },

From d3dedcace8a5fb1f047ecad05d23dce8745252d5 Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sun, 28 Jan 2018 12:06:43 +0300
Subject: [PATCH 3/3] Document events a bit

---
 src/parser/event_parser/mod.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs
index 1bb943b5de79..7823c476c200 100644
--- a/src/parser/event_parser/mod.rs
+++ b/src/parser/event_parser/mod.rs
@@ -4,7 +4,6 @@ use {SyntaxKind, Token};
 mod parser;
 mod grammar;
 
-
 /// `Parser` produces a flat list of `Event`s.
 /// They are converted to a tree-structure in
 /// a separate pass, via `TreeBuilder`.