Skip to content

Commit

Permalink
fix(lexers/go): "~" is a valid token
Browse files Browse the repository at this point in the history
With the introduction of generics,
tilde is a valid punctuation token in Go programs.
https://go.dev/ref/spec#Operators_and_punctuation

This updates the punctuation regex for the Go lexer,
and adds a test to ensure that it's treated as such.
  • Loading branch information
abhinav committed Feb 12, 2024
1 parent f4788c0 commit db7646d
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 1 deletion.
2 changes: 1 addition & 1 deletion lexers/go.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func goRules() Rules {
{`"(\\\\|\\"|[^"])*"`, LiteralString, nil},
{`(<<=|>>=|<<|>>|<=|>=|&\^=|&\^|\+=|-=|\*=|/=|%=|&=|\|=|&&|\|\||<-|\+\+|--|==|!=|:=|\.\.\.|[+\-*/%&])`, Operator, nil},
{`([a-zA-Z_]\w*)(\s*)(\()`, ByGroups(NameFunction, UsingSelf("root"), Punctuation), nil},
{`[|^<>=!()\[\]{}.,;:]`, Punctuation, nil},
{`[|^<>=!()\[\]{}.,;:~]`, Punctuation, nil},
{`[^\W\d]\w*`, NameOther, nil},
},
}
Expand Down
26 changes: 26 additions & 0 deletions lexers/go_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,3 +120,29 @@ func TestGoHTMLTemplateNegativeNumber(t *testing.T) {
assert.True(t, found)
}
}

func TestGoGenericTokens(t *testing.T) {
for _, source := range []string{
`type Foo[T any] struct { ~[]T | ~string }`,
} {
tokens, err := chroma.Tokenise(Go, nil, source)
assert.NoError(t, err)
assert.Equal(t, source, chroma.Stringify(tokens...))

// No error tokens.
for _, token := range tokens {
assert.NotEqual(t, chroma.Error, token.Type,
"unexpected error token: %v (%q)", token, token.Value)
}

// "~" must be treated as punctuation.
var found bool
for _, token := range tokens {
if token.Value == "~" {
found = true
assert.Equal(t, chroma.Punctuation, token.Type)
}
}
assert.True(t, found)
}
}

0 comments on commit db7646d

Please sign in to comment.