Implementation of Void-token (tokay-lang#129)

* ImlValue::(SelfToken|SelfValue|VoidToken)

* Implemented basic ImlValue::VoidToken behavior

* Context::get_capture() using Capture

* Clarify Op::Next and Op::Reject

* Clarification of VoidToken

* Let $0 always return entire range of current parselet

* Context::run() continue with end-execution only when necessary

* Drafting Until and String

* Updated ROADMAP.md

ROADMAP.md

@@ -10,9 +10,16 @@ This document describes upcoming changes to achieve with a specific version.
   - [ ] `Until<P, Escape: '\\'>`
   - [ ] `String<Start, End: Void, Escape: '\\'>`
 - [ ] Implement inlined parselets (#120)
-- [ ] New list syntax `,`, redefining sequence/`dict` syntax (#100)
+- [x] New list syntax `,`, redefining sequence/`dict` syntax (#100)
   - Top-level `list` definition `l = ,`
   - Top-level `dict` definition `d = ()`
+- [ ] Compiler refactoring and improvements
+  - [x] Scope-struct
+  - [ ] Clean-up Scope-Compiler-intermezzo
+  - [x] `void` and `Void`
+  - [x] `Empty`
+  - [ ] ImlRefValue
+  - [ ] ImlVariable
 
 ## 0.8
 

src/builtin/mod.rs

@@ -157,8 +157,11 @@ tokay_function!("ord : @c", {
 
 tokay_function!("print : @*args", {
     if args.len() == 0 && context.is_some() {
-        if let Some(capture) = context.unwrap().get_capture(0) {
-            print!("{}", capture.to_string());
+        let context = context.unwrap();
+
+        if let Some(mut capture) = context.get_capture(0) {
+            let value = capture.extract(context.thread.reader);
+            print!("{}", value.to_string());
         }
     } else {
         for i in 0..args.len() {

src/compiler/ast.rs

@@ -12,7 +12,8 @@ use crate::vm::*;
 use charclass::CharClass;
 
 pub static RESERVED_TOKENS: &[&'static str] = &[
-    "Char", "Chars", "EOF", "Expect", "Not", "Kle", "Opt", "Peek", "Pos", "Repeat", "Self", "Void",
+    "Char", "Chars", "Empty", "EOF", "Expect", "Not", "Kle", "Opt", "Peek", "Pos", "Repeat",
+    "Self", "Void",
 ];
 
 pub static RESERVED_KEYWORDS: &[&'static str] = &[
@@ -79,7 +80,7 @@ fn traverse_node_value(scope: &Scope, node: &Dict, name: Option<String>) -> ImlV
         "value_null" => ImlValue::Value(scope.compiler.statics.borrow()[1].clone()),
         "value_true" => ImlValue::Value(scope.compiler.statics.borrow()[2].clone()),
         "value_false" => ImlValue::Value(scope.compiler.statics.borrow()[3].clone()),
-        "value_self" => ImlValue::This(false),
+        "value_self" => ImlValue::SelfValue,
         "value_integer" => match node["value"].to_i64() {
             Ok(0) => ImlValue::Value(scope.compiler.statics.borrow()[4].clone()),
             Ok(1) => ImlValue::Value(scope.compiler.statics.borrow()[5].clone()),
@@ -89,7 +90,8 @@ fn traverse_node_value(scope: &Scope, node: &Dict, name: Option<String>) -> ImlV
         "value_string" => scope.compiler.register_static(node["value"].clone()),
 
         // Tokens
-        "value_token_self" => ImlValue::This(true),
+        "value_token_self" => ImlValue::SelfToken,
+        "value_token_void" => ImlValue::VoidToken,
         "value_token_match" | "value_token_touch" => {
             let mut value = node["value"].to_string();
 

src/compiler/iml/imlparselet.rs

@@ -230,8 +230,8 @@ impl ImlParselet {
             }
 
             match value {
-                ImlValue::This(_) => {
-                    // Replace any references of self
+                ImlValue::SelfValue | ImlValue::SelfToken => {
+                    // Replace any references of self by from
                     *value = ImlValue::Parselet(from.clone());
                     changes = true;
                 }

src/compiler/iml/imlprogram.rs

@@ -163,7 +163,7 @@ impl ImlProgram {
                 ImlValue::Unresolved(value) => {
                     finalize_value(&*value.borrow(), current, visited, configs)
                 }
-                ImlValue::This(_) => Some(Consumable {
+                ImlValue::SelfToken => Some(Consumable {
                     leftrec: true,
                     nullable: false,
                 }),

src/compiler/iml/imlvalue.rs

@@ -22,9 +22,11 @@ modified and resolved during the compilation process.
 pub(in crate::compiler) enum ImlValue {
     Unset,                             // Unset
     Unresolved(Rc<RefCell<ImlValue>>), // Unresolved ImlValues are shared
+    SelfValue,                         // self-reference (value)
+    SelfToken,                         // Self-reference (consuming)
+    VoidToken,                         // Void (consuming)
     Value(RefValue),                   // Static value
     Parselet(ImlParselet),             // Parselet
-    This(bool),                        // self-reference to function (false) or parselet (true)
     Variable {
         // Resolved variable
         offset: Option<Offset>, // Source offset
@@ -265,7 +267,7 @@ impl ImlValue {
     pub fn is_callable(&self, without_arguments: bool) -> bool {
         match self {
             Self::Unresolved(value) => value.borrow().is_callable(without_arguments),
-            Self::This(_) => true, // fixme?
+            Self::SelfValue | Self::SelfToken => true, // fixme?
             Self::Value(value) => value.is_callable(without_arguments),
             Self::Parselet(parselet) => {
                 let parselet = parselet.borrow();
@@ -290,7 +292,8 @@ impl ImlValue {
     pub fn is_consuming(&self) -> bool {
         match self {
             Self::Unresolved(value) => value.borrow().is_consuming(),
-            Self::This(consuming) => *consuming,
+            Self::SelfValue => false,
+            Self::SelfToken | Self::VoidToken => true,
             Self::Value(value) => value.is_consuming(),
             Self::Parselet(parselet) => parselet.borrow().model.borrow().is_consuming,
             Self::Name { name, .. } | Self::Generic { name, .. } => {
@@ -325,6 +328,7 @@ impl ImlValue {
             ImlValue::Unresolved(value) => {
                 return value.borrow().compile(program, current, offset, call, ops)
             }
+            ImlValue::VoidToken => ops.push(Op::Next),
             ImlValue::Value(value) => match &*value.borrow() {
                 Value::Void => ops.push(Op::PushVoid),
                 Value::Null => ops.push(Op::PushNull),
@@ -350,14 +354,14 @@ impl ImlValue {
                 return current.0.borrow().generics[name]
                     .compile(program, current, offset, call, ops)
             }
-            ImlValue::This(_) | ImlValue::Parselet(_) => {}
+            ImlValue::SelfValue | ImlValue::SelfToken | ImlValue::Parselet(_) => {}
             _ => unreachable!("{}", self),
         }
 
         // Check if something has been pushed before.
         if start == ops.len() {
             let idx = match self {
-                ImlValue::This(_) => current.1, // use current index
+                ImlValue::SelfValue | ImlValue::SelfToken => current.1, // use current index
                 ImlValue::Parselet(parselet) => match parselet.derive(current.0) {
                     Ok(parselet) => program.register(&ImlValue::Parselet(parselet)),
                     Err(msg) => {
@@ -404,8 +408,9 @@ impl std::fmt::Display for ImlValue {
         match self {
             Self::Unset => write!(f, "unset"),
             Self::Unresolved(value) => value.borrow().fmt(f),
-            Self::This(true) => write!(f, "Self"),
-            Self::This(false) => write!(f, "self"),
+            Self::SelfValue => write!(f, "self"),
+            Self::SelfToken => write!(f, "Self"),
+            Self::VoidToken => write!(f, "Void"),
             Self::Value(value) => write!(f, "{}", value.repr()),
             Self::Parselet(parselet) => write!(
                 f,
@@ -460,6 +465,7 @@ impl std::hash::Hash for ImlValue {
         match self {
             Self::Unset => state.write_u8('u' as u8),
             Self::Unresolved(value) => value.borrow().hash(state),
+            Self::VoidToken => state.write_u8('V' as u8),
             Self::Value(value) => {
                 state.write_u8('v' as u8);
                 value.hash(state)

src/compiler/parser.rs

@@ -4205,6 +4205,48 @@ impl Parser {
                                                                             ]))
                                                                     ]))
                                                                 ]))
+                                                        ])),
+                                                        (value!([
+                                                            "emit" => "sequence",
+                                                            "children" =>
+                                                                (value!([
+                                                                    (value!([
+                                                                        "emit" => "value_generic",
+                                                                        "children" =>
+                                                                            (value!([
+                                                                                (value!([
+                                                                                    "emit" => "identifier",
+                                                                                    "value" => "Keyword"
+                                                                                ])),
+                                                                                (value!([
+                                                                                    "emit" => "genarg",
+                                                                                    "children" =>
+                                                                                        (value!([
+                                                                                            "emit" => "value_token_touch",
+                                                                                            "value" => "Void"
+                                                                                        ]))
+                                                                                ]))
+                                                                            ]))
+                                                                    ])),
+                                                                    (value!([
+                                                                        "emit" => "call",
+                                                                        "children" =>
+                                                                            (value!([
+                                                                                (value!([
+                                                                                    "emit" => "identifier",
+                                                                                    "value" => "ast"
+                                                                                ])),
+                                                                                (value!([
+                                                                                    "emit" => "callarg",
+                                                                                    "children" =>
+                                                                                        (value!([
+                                                                                            "emit" => "value_string",
+                                                                                            "value" => "value_token_void"
+                                                                                        ]))
+                                                                                ]))
+                                                                            ]))
+                                                                    ]))
+                                                                ]))
                                                         ]))
                                                     ]))
                                             ]))

src/compiler/prelude.rs

@@ -56,7 +56,7 @@ impl Compiler {
                                                             ])),
                                                             (value!([
                                                                 "emit" => "identifier",
-                                                                "value" => "Void"
+                                                                "value" => "Empty"
                                                             ]))
                                                         ]))
                                                 ]))
@@ -865,7 +865,7 @@ impl Compiler {
                                                                     ])),
                                                                     (value!([
                                                                         "emit" => "identifier",
-                                                                        "value" => "Void"
+                                                                        "value" => "Empty"
                                                                     ]))
                                                                 ]))
                                                         ]))

src/compiler/tokay.tok

@@ -244,6 +244,7 @@ TokenLiteral : @{
     Keyword<'Char'> '<' Ccl '>'  ast("value_token_ccl")
     Keyword<'Char'>  ast("value_token_any")
     Keyword<'Self'>  ast("value_token_self")
+    Keyword<'Void'>  ast("value_token_void")
 }
 
 Token : @{

src/prelude.tok

@@ -3,10 +3,10 @@
 # The `Not`-builtin runs its parser and returns its negated result,
 # so that `P` occurence becomes rejected.
 #
-# Everything else becomes accepted (Void).
+# Everything else becomes accepted (Empty).
 Not : @<P> {
     P reject
-    Void
+    Empty
 }
 
 # The `Peek`-builtin runs `P` and returns its result, but resets the reading-context afterwards.
@@ -74,7 +74,7 @@ Pos : @<P, blur: true>{ Repeat<P, blur: blur> }
 Kle : @<P, blur: true>{ Repeat<P, min: 0, blur: blur> || void }
 
 # Optionally accepts `P` or nothing.
-Opt : @<P>{ P | Void }
+Opt : @<P>{ P | Empty }
 
 # Implements a recursive, separated list.
 #

src/value/token.rs

@@ -11,7 +11,7 @@ extern crate self as tokay;
 
 #[derive(Debug, Clone, Hash, PartialEq, PartialOrd)]
 pub enum Token {
-    Void,                               // Matches the empty word
+    Empty,                              // Matches the empty word
     EOF,                                // Matches End of File
     Char(CharClass),                    // Matches one character from a character class
     BuiltinChar(fn(ch: char) -> bool),  // Matches one character from a callback function
@@ -59,8 +59,8 @@ impl Token {
         }
 
         match ident {
+            "Empty" => Some(Token::Empty),
             "EOF" => Some(Token::EOF),
-            "Void" => Some(Token::Void),
             ident => builtin_ccl(ident),
         }
     }
@@ -73,7 +73,7 @@ impl Object for Token {
 
     fn repr(&self) -> String {
         match self {
-            Token::Void => "Void".to_string(),
+            Token::Empty => "Empty".to_string(),
             Token::EOF => "EOF".to_string(),
             Token::Char(ccl) => format!("{:?}", ccl),
             Token::Chars(ccl) => format!("{:?}+", ccl),
@@ -93,7 +93,7 @@ impl Object for Token {
 
     fn is_nullable(&self) -> bool {
         match self {
-            Token::Void => true,
+            Token::Empty => true,
             Token::EOF => false,
             Token::Char(ccl) | Token::Chars(ccl) => ccl.len() == 0, //True shouldn't be possible here by definition!
             Token::BuiltinChar(_) | Token::BuiltinChars(_) => true,
@@ -113,7 +113,7 @@ impl Object for Token {
         let reader = &mut context.thread.reader;
 
         match self {
-            Token::Void => Ok(Accept::Next),
+            Token::Empty => Ok(Accept::Next),
             Token::EOF => {
                 if let Some(_) = reader.peek() {
                     Err(Reject::Next)