feat(scanner): add '?' and 'include' token

scanner/scanner.go

@@ -121,6 +121,8 @@ func (s *Scanner) Parse() {
 			tokenKind = UNDERSCORE
 		case '*':
 			tokenKind = STAR
+		case '?':
+			tokenKind = QUESTIONMARK
 		case '\n':
 			s.addToken(NEWLINE, "")
 			s.advanceLine()
@@ -145,7 +147,7 @@ func (s *Scanner) Parse() {
 		out:
 			for {
 				switch s.curChar {
-				case '\n', '!', '#', '_', '*', '-', '[', ']', '(', ')', '`', '>':
+				case '\n', '!', '#', '_', '*', '-', '[', ']', '(', ')', '`', '>', '?':
 					break out
 				}
 
@@ -155,6 +157,14 @@ func (s *Scanner) Parse() {
 
 			// skip empty texts
 			if res.Len() != 0 {
+				result := res.String()
+				if result[0] == 'i' && strings.HasPrefix(result, "include") {
+					split := strings.Split(result, " ")
+					if len(split) >= 2 {
+						s.addToken(INCLUDE, split[1])
+						continue
+					}
+				}
 				s.addToken(TEXT, res.String())
 			}
 			// INFO: this skips adding the text again

scanner/scanner_test.go

@@ -77,13 +77,22 @@ func TestHeadings(t *testing.T) {
 		STRAIGHTBRACECLOSE,
 		PARENOPEN,
 		TEXT,
+		QUESTIONMARK,
+		TEXT,
 		PARENCLOSE,
 		NEWLINE,
 		BACKTICK,
 		TEXT,
 		BACKTICK,
 		NEWLINE,
+		QUESTIONMARK,
+		INCLUDE,
+		NEWLINE,
+		QUESTIONMARK,
+		TEXT,
+		NEWLINE,
 	}
+	s.PrintTokens()
 	if len(tokens) != len(expectedTokens) {
 		t.Errorf("expected %d tokens, got: %d", len(expectedTokens), len(tokens))
 	}

scanner/tests/markdown.md

@@ -20,3 +20,6 @@
 ![xnacly](https://avatars.githubusercontent.com/u/47723417?v=4)
 
 `inline code`
+
+?include test.md
+?include

scanner/tokens.go

@@ -21,6 +21,8 @@ const (
 	BACKTICK
 	GREATERTHAN
 	BANG
+	QUESTIONMARK
+	INCLUDE
 )
 
 var TOKEN_LOOKUP_MAP = map[uint]string{
@@ -35,6 +37,8 @@ var TOKEN_LOOKUP_MAP = map[uint]string{
 	PARENCLOSE:         "PARENCLOSE",
 	GREATERTHAN:        "GREATERTHAN",
 	BACKTICK:           "BACKTICK",
+	QUESTIONMARK:       "QUESTIONMARK",
+	INCLUDE:            "INCLUDE",
 	TEXT:               "TEXT",
 	BANG:               "BANG",
 }