Add a new token type to catch Kaleido symbols

vhiribarren · May 1, 2021 · f010aff · f010aff
1 parent 6f2df76
commit f010aff
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 12 deletions.
diff --git a/lexer/kaleido_lexer.go b/lexer/kaleido_lexer.go
@@ -13,6 +13,7 @@ const (
 	KTokenExtern
 	KTokenIdentifier
 	KTokenNumber
+	KTokenSymbol
 )
 
 type KaleidoTokenContext struct {
@@ -40,6 +41,10 @@ func emitNumber(val string) *KaleidoTokenContext {
 	return &KaleidoTokenContext{Token: KTokenNumber, Value: val}
 }
 
+func emitSymbol(val rune) *KaleidoTokenContext {
+	return &KaleidoTokenContext{Token: KTokenSymbol, Value: string(val)}
+}
+
 type KaleidoLexer struct {
 	BaseLexer
 }
@@ -48,30 +53,31 @@ func NewKaleidoLexer(data string) KaleidoLexer {
 	return KaleidoLexer{BaseLexer: NewBaseLexer(data)}
 }
 
-func (l *KaleidoLexer) NextToken() (*KaleidoTokenContext, error) {
+func (l *KaleidoLexer) NextToken() *KaleidoTokenContext {
 	for {
 		l.ConsumeWhitespaces()
 		val, err := l.PeekNext()
 		switch {
 		case err != nil:
-			return emitEOF(), nil
+			return emitEOF()
 		case isAlphabetic(val):
 			result := l.consumeGreedAlphanum()
 			switch result {
 			case "def":
-				return emitDef(), nil
+				return emitDef()
 			case "extern":
-				return emitExtern(), nil
+				return emitExtern()
 			default:
-				return emitIdentifier(result), nil
+				return emitIdentifier(result)
 			}
 		case isNumeral(val):
 			result := l.consumeGreedNumber()
-			return emitNumber(result), nil
+			return emitNumber(result)
 		case val == '#':
 			l.consumeGreedCommentLine()
 		default:
-			return nil, newErrorBadRune("Received: %v but was not waiting for that", val)
+			l.ConsumeNext()
+			return emitSymbol(val)
 		}
 
 	}

diff --git a/lexer/kaleido_lexer_test.go b/lexer/kaleido_lexer_test.go
@@ -5,11 +5,13 @@ import (
 )
 
 func TestValidMedley(t *testing.T) {
-	input := "machin123    def   defextern  123  extern 456hello  #comment def"
+	input := "machin123    def   defextern <= 123  extern 456hello  #comment def"
 	targetResults := []KaleidoTokenContext{
 		{KTokenIdentifier, "machin123"},
 		{KTokenDef, ""},
 		{KTokenIdentifier, "defextern"},
+		{KTokenSymbol, "<"},
+		{KTokenSymbol, "="},
 		{KTokenNumber, "123"},
 		{KTokenExtern, ""},
 		{KTokenNumber, "456"},
@@ -18,10 +20,7 @@ func TestValidMedley(t *testing.T) {
 	}
 	lexer := NewKaleidoLexer(input)
 	for i := 0; i < len(targetResults); i++ {
-		result, err := lexer.NextToken()
-		if err != nil {
-			t.Fatalf("Error: %v", err)
-		}
+		result := lexer.NextToken()
 		if result.Token != targetResults[i].Token || result.Value != targetResults[i].Value {
 			t.Fatalf("Was waiting for: %v but creceived: %v", result, &targetResults[i])
 		}