From 9f8fc9ab76b4863227a8a15aae9c39a729ab5143 Mon Sep 17 00:00:00 2001 From: mrsdizzie Date: Tue, 7 Jul 2020 13:48:09 -0400 Subject: [PATCH 1/3] Remove newline when highlighting random chunks of code Somewhere when tokenizing a newline gets added to code formatted by chroma. This breaks the case of 'added-code' inside of an 'added-line' in a diff. Just remove any newline when processing chunks of code since we don't need it. Fixes #12172 --- modules/highlight/highlight.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index 90590d220ba4e..054b23bd5c86f 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -81,7 +81,9 @@ func Code(fileName, code string) string { } htmlw.Flush() - return htmlbuf.String() + // Strip any newline that chroma might have added since this may be one of multiple + // highlights on what should display as a single line of code + return strings.TrimSuffix(htmlbuf.String(), "\n") } // File returns map with line lumbers and HTML version of code with chroma syntax highlighting classes From 928d3ca8d51b9275566c597bc8d44623289e6981 Mon Sep 17 00:00:00 2001 From: mrsdizzie Date: Tue, 7 Jul 2020 16:00:43 -0400 Subject: [PATCH 2/3] don't process empty lines --- modules/highlight/highlight.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index 054b23bd5c86f..98f15f6383dd5 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -43,6 +43,11 @@ func NewContext() { func Code(fileName, code string) string { NewContext() + // don't process empty lines + if code == "\n" { + return code + } + if len(code) > sizeLimit { return code } From d3e0d34cbe5d1ad0c318c0db257b1ca0d4e3113c Mon Sep 17 00:00:00 2001 From: mrsdizzie Date: Tue, 7 Jul 2020 16:37:27 -0400 Subject: [PATCH 3/3] This is the proper way to fix this by telling chroma not to add the newline in the first place --- go.mod | 2 +- modules/highlight/highlight.go | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 9be45c502895d..d4a2d3cfb27c1 100644 --- a/go.mod +++ b/go.mod @@ -100,8 +100,8 @@ require ( github.com/urfave/cli v1.20.0 github.com/xanzy/go-gitlab v0.31.0 github.com/yohcop/openid-go v1.0.0 - github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691 github.com/yuin/goldmark v1.1.32 + github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691 github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 golang.org/x/net v0.0.0-20200602114024-627f9648deb9 diff --git a/modules/highlight/highlight.go b/modules/highlight/highlight.go index 98f15f6383dd5..5d66e4c32d40c 100644 --- a/modules/highlight/highlight.go +++ b/modules/highlight/highlight.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "github.com/alecthomas/chroma" "github.com/alecthomas/chroma/formatters/html" "github.com/alecthomas/chroma/lexers" "github.com/alecthomas/chroma/styles" @@ -43,11 +44,6 @@ func NewContext() { func Code(fileName, code string) string { NewContext() - // don't process empty lines - if code == "\n" { - return code - } - if len(code) > sizeLimit { return code } @@ -73,7 +69,7 @@ func Code(fileName, code string) string { lexer = lexers.Fallback } - iterator, err := lexer.Tokenise(nil, string(code)) + iterator, err := lexer.Tokenise(&chroma.TokeniseOptions{State: "root", Nested: true}, string(code)) if err != nil { log.Error("Can't tokenize code: %v", err) return code @@ -86,9 +82,7 @@ func Code(fileName, code string) string { } htmlw.Flush() - // Strip any newline that chroma might have added since this may be one of multiple - // highlights on what should display as a single line of code - return strings.TrimSuffix(htmlbuf.String(), "\n") + return htmlbuf.String() } // File returns map with line lumbers and HTML version of code with chroma syntax highlighting classes