fix bug with determining type of inline token

- *} determined as open type as it matched single symbol pattern '*' - also, rules for ~/^ inline tokens adjusted to comply with rules for emphasis/strong markup
sivukhin · Mar 26, 2024 · 8e9588a · 8e9588a
1 parent cf23eab
commit 8e9588a
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 6 deletions.
diff --git a/djot_parser/examples/inline-type-force.djot b/djot_parser/examples/inline-type-force.djot
@@ -0,0 +1,8 @@
+*}hi*
+*hi{*
+_}hi_
+_hi{_
+~}hi~
+~hi{~
+^}hi^
+^hi{^
diff --git a/djot_parser/examples/inline-type-force.html b/djot_parser/examples/inline-type-force.html
@@ -0,0 +1,8 @@
+<p>*}hi*
+*hi{*
+_}hi_
+_hi{_
+~}hi~
+~hi{~
+^}hi^
+^hi{^</p>
diff --git a/djot_tokenizer/djot_inline_token.go b/djot_tokenizer/djot_inline_token.go
@@ -89,7 +89,7 @@ func matchInlineToken(
 		if next, ok := r.Token2(s, [...]byte{'{', '_'}); ok {
 			return next, ok
 		}
-		if next, ok := r.Token1(s, [...]byte{'_'}); ok && !r.HasMask(next, tokenizer.SpaceNewLineByteMask) {
+		if next, ok := r.Token1(s, [...]byte{'_'}); ok && !r.HasMask(next, tokenizer.SpaceNewLineByteMask) && !r.HasToken1(next, [...]byte{'}'}) {
 			return next, ok
 		}
 		return fail()
@@ -105,7 +105,7 @@ func matchInlineToken(
 		if next, ok := r.Token2(s, [...]byte{'{', '*'}); ok {
 			return next, ok
 		}
-		if next, ok := r.Token1(s, [...]byte{'*'}); ok && !r.HasMask(next, tokenizer.SpaceNewLineByteMask) {
+		if next, ok := r.Token1(s, [...]byte{'*'}); ok && !r.HasMask(next, tokenizer.SpaceNewLineByteMask) && !r.HasToken1(next, [...]byte{'}'}) {
 			return next, ok
 		}
 		return fail()
@@ -125,22 +125,34 @@ func matchInlineToken(
 		if next, ok := r.Token2(s, [...]byte{'{', '~'}); ok {
 			return next, ok
 		}
-		return r.Token1(s, [...]byte{'~'})
+		if next, ok := r.Token1(s, [...]byte{'~'}); ok && !r.HasMask(next, tokenizer.SpaceNewLineByteMask) && !r.HasToken1(next, [...]byte{'}'}) {
+			return next, ok
+		}
+		return fail()
 	case SubscriptInline ^ tokenizer.Open:
 		if next, ok := r.Token2(s, [...]byte{'~', '}'}); ok {
 			return next, ok
 		}
-		return r.Token1(s, [...]byte{'~'})
+		if next, ok := r.Token1(s, [...]byte{'~'}); ok && s > 0 && !r.HasMask(s-1, tokenizer.SpaceNewLineByteMask) {
+			return next, ok
+		}
+		return fail()
 	case SuperscriptInline:
 		if next, ok := r.Token2(s, [...]byte{'{', '^'}); ok {
 			return next, ok
 		}
-		return r.Token1(s, [...]byte{'^'})
+		if next, ok := r.Token1(s, [...]byte{'^'}); ok && !r.HasMask(next, tokenizer.SpaceNewLineByteMask) && !r.HasToken1(next, [...]byte{'}'}) {
+			return next, ok
+		}
+		return fail()
 	case SuperscriptInline ^ tokenizer.Open:
 		if next, ok := r.Token2(s, [...]byte{'^', '}'}); ok {
 			return next, ok
 		}
-		return r.Token1(s, [...]byte{'^'})
+		if next, ok := r.Token1(s, [...]byte{'^'}); ok && s > 0 && !r.HasMask(s-1, tokenizer.SpaceNewLineByteMask) {
+			return next, ok
+		}
+		return fail()
 	case InsertInline:
 		return r.Token2(s, [...]byte{'{', '+'})
 	case InsertInline ^ tokenizer.Open: