Render embedded code preview by permlink in markdown (go-gitea#30234)

The permlink in markdown will be rendered as a code preview block, like GitHub

Co-authored-by: silverwind <me@silverwind.io>

modules/charset/escape_test.go

@@ -4,6 +4,7 @@
 package charset
 
 import (
+ "regexp"
  "strings"
  "testing"
 
@@ -156,13 +157,16 @@ func TestEscapeControlReader(t *testing.T) {
  tests = append(tests, test)
  }
 
+ re := regexp.MustCompile(`repo.ambiguous_character:\d+,\d+`) // simplify the output for the tests, remove the translation variants
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
  output := &strings.Builder{}
  status, err := EscapeControlReader(strings.NewReader(tt.text), output, &translation.MockLocale{})
  assert.NoError(t, err)
  assert.Equal(t, tt.status, *status)
- assert.Equal(t, tt.result, output.String())
+ outStr := output.String()
+ outStr = re.ReplaceAllString(outStr, "repo.ambiguous_character")
+ assert.Equal(t, tt.result, outStr)
  })
  }
 }

modules/csv/csv_test.go

@@ -561,14 +561,14 @@ func TestFormatError(t *testing.T) {
  err: &csv.ParseError{
  Err: csv.ErrFieldCount,
  },
- expectedMessage: "repo.error.csv.invalid_field_count",
+ expectedMessage: "repo.error.csv.invalid_field_count:0",
  expectsError: false,
  },
  {
  err: &csv.ParseError{
  Err: csv.ErrBareQuote,
  },
- expectedMessage: "repo.error.csv.unexpected",
+ expectedMessage: "repo.error.csv.unexpected:0,0",
  expectsError: false,
  },
  {

modules/indexer/code/search.go

@@ -22,7 +22,7 @@ type Result struct {
  UpdatedUnix timeutil.TimeStamp
  Language string
  Color string
- Lines []ResultLine
+ Lines []*ResultLine
 }
 
 type ResultLine struct {
@@ -70,16 +70,18 @@ func writeStrings(buf *bytes.Buffer, strs ...string) error {
  return nil
 }
 
-func HighlightSearchResultCode(filename string, lineNums []int, code string) []ResultLine {
+func HighlightSearchResultCode(filename, language string, lineNums []int, code string) []*ResultLine {
  // we should highlight the whole code block first, otherwise it doesn't work well with multiple line highlighting
- hl, _ := highlight.Code(filename, "", code)
+ hl, _ := highlight.Code(filename, language, code)
  highlightedLines := strings.Split(string(hl), "\n")
 
  // The lineNums outputted by highlight.Code might not match the original lineNums, because "highlight" removes the last `\n`
- lines := make([]ResultLine, min(len(highlightedLines), len(lineNums)))
+ lines := make([]*ResultLine, min(len(highlightedLines), len(lineNums)))
  for i := 0; i < len(lines); i++ {
- lines[i].Num = lineNums[i]
- lines[i].FormattedContent = template.HTML(highlightedLines[i])
+ lines[i] = &ResultLine{
+ Num: lineNums[i],
+ FormattedContent: template.HTML(highlightedLines[i]),
+ }
  }
  return lines
 }
@@ -122,7 +124,7 @@ func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Res
  UpdatedUnix: result.UpdatedUnix,
  Language: result.Language,
  Color: result.Color,
- Lines: HighlightSearchResultCode(result.Filename, lineNums, formattedLinesBuffer.String()),
+ Lines: HighlightSearchResultCode(result.Filename, result.Language, lineNums, formattedLinesBuffer.String()),
  }, nil
 }
 

modules/markup/html.go

@@ -171,6 +171,7 @@ type processor func(ctx *RenderContext, node *html.Node)
 var defaultProcessors = []processor{
  fullIssuePatternProcessor,
  comparePatternProcessor,
+ codePreviewPatternProcessor,
  fullHashPatternProcessor,
  shortLinkProcessor,
  linkProcessor,

modules/markup/html_codepreview.go

@@ -0,0 +1,92 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package markup
+
+import (
+ "html/template"
+ "net/url"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "code.gitea.io/gitea/modules/httplib"
+ "code.gitea.io/gitea/modules/log"
+
+ "golang.org/x/net/html"
+)
+
+// codePreviewPattern matches "http://domain/.../{owner}/{repo}/src/commit/{commit}/{filepath}#L10-L20"
+var codePreviewPattern = regexp.MustCompile(`https?://\S+/([^\s/]+)/([^\s/]+)/src/commit/([0-9a-f]{7,64})(/\S+)#(L\d+(-L\d+)?)`)
+
+type RenderCodePreviewOptions struct {
+ FullURL string
+ OwnerName string
+ RepoName string
+ CommitID string
+ FilePath string
+
+ LineStart, LineStop int
+}
+
+func renderCodeBlock(ctx *RenderContext, node *html.Node) (urlPosStart, urlPosStop int, htm template.HTML, err error) {
+ m := codePreviewPattern.FindStringSubmatchIndex(node.Data)
+ if m == nil {
+ return 0, 0, "", nil
+ }
+
+ opts := RenderCodePreviewOptions{
+ FullURL: node.Data[m[0]:m[1]],
+ OwnerName: node.Data[m[2]:m[3]],
+ RepoName: node.Data[m[4]:m[5]],
+ CommitID: node.Data[m[6]:m[7]],
+ FilePath: node.Data[m[8]:m[9]],
+ }
+ if !httplib.IsCurrentGiteaSiteURL(opts.FullURL) {
+ return 0, 0, "", nil
+ }
+ u, err := url.Parse(opts.FilePath)
+ if err != nil {
+ return 0, 0, "", err
+ }
+ opts.FilePath = strings.TrimPrefix(u.Path, "/")
+
+ lineStartStr, lineStopStr, _ := strings.Cut(node.Data[m[10]:m[11]], "-")
+ lineStart, _ := strconv.Atoi(strings.TrimPrefix(lineStartStr, "L"))
+ lineStop, _ := strconv.Atoi(strings.TrimPrefix(lineStopStr, "L"))
+ opts.LineStart, opts.LineStop = lineStart, lineStop
+ h, err := DefaultProcessorHelper.RenderRepoFileCodePreview(ctx.Ctx, opts)
+ return m[0], m[1], h, err
+}
+
+func codePreviewPatternProcessor(ctx *RenderContext, node *html.Node) {
+ for node != nil {
+ if node.Type != html.TextNode {
+ node = node.NextSibling
+ continue
+ }
+ urlPosStart, urlPosEnd, h, err := renderCodeBlock(ctx, node)
+ if err != nil || h == "" {
+ if err != nil {
+ log.Error("Unable to render code preview: %v", err)
+ }
+ node = node.NextSibling
+ continue
+ }
+ next := node.NextSibling
+ textBefore := node.Data[:urlPosStart]
+ textAfter := node.Data[urlPosEnd:]
+ // "textBefore" could be empty if there is only a URL in the text node, then an empty node (p, or li) will be left here.
+ // However, the empty node can't be simply removed, because:
+ // 1. the following processors will still try to access it (need to double-check undefined behaviors)
+ // 2. the new node is inserted as "<p>{TextBefore}<div NewNode/>{TextAfter}</p>" (the parent could also be "li")
+ // then it is resolved as: "<p>{TextBefore}</p><div NewNode/><p>{TextAfter}</p>",
+ // so unless it could correctly replace the parent "p/li" node, it is very difficult to eliminate the "TextBefore" empty node.
+ node.Data = textBefore
+ node.Parent.InsertBefore(&html.Node{Type: html.RawNode, Data: string(h)}, next)
+ if textAfter != "" {
+ node.Parent.InsertBefore(&html.Node{Type: html.TextNode, Data: textAfter}, next)
+ }
+ node = next
+ }
+}

modules/markup/html_codepreview_test.go

@@ -0,0 +1,34 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package markup_test
+
+import (
+ "context"
+ "html/template"
+ "strings"
+ "testing"
+
+ "code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/markup"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestRenderCodePreview(t *testing.T) {
+ markup.Init(&markup.ProcessorHelper{
+ RenderRepoFileCodePreview: func(ctx context.Context, opts markup.RenderCodePreviewOptions) (template.HTML, error) {
+ return "<div>code preview</div>", nil
+ },
+ })
+ test := func(input, expected string) {
+ buffer, err := markup.RenderString(&markup.RenderContext{
+ Ctx: git.DefaultContext,
+ Type: "markdown",
+ }, input)
+ assert.NoError(t, err)
+ assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
+ }
+ test("http://localhost:3000/owner/repo/src/commit/0123456789/foo/bar.md#L10-L20", "<p><div>code preview</div></p>")
+ test("http://other/owner/repo/src/commit/0123456789/foo/bar.md#L10-L20", `<p><a href="http://other/owner/repo/src/commit/0123456789/foo/bar.md#L10-L20" rel="nofollow">http://other/owner/repo/src/commit/0123456789/foo/bar.md#L10-L20</a></p>`)
+}

modules/markup/renderer.go

@@ -8,6 +8,7 @@ import (
  "context"
  "errors"
  "fmt"
+ "html/template"
  "io"
  "net/url"
  "path/filepath"
@@ -33,6 +34,8 @@ type ProcessorHelper struct {
  IsUsernameMentionable func(ctx context.Context, username string) bool
 
  ElementDir string // the direction of the elements, eg: "ltr", "rtl", "auto", default to no direction attribute
+
+ RenderRepoFileCodePreview func(ctx context.Context, options RenderCodePreviewOptions) (template.HTML, error)
 }
 
 var DefaultProcessorHelper ProcessorHelper

modules/markup/sanitizer.go

@@ -60,6 +60,21 @@ func createDefaultPolicy() *bluemonday.Policy {
  // For JS code copy and Mermaid loading state
  policy.AllowAttrs("class").Matching(regexp.MustCompile(`^code-block( is-loading)?$`)).OnElements("pre")
 
+ // For code preview
+ policy.AllowAttrs("class").Matching(regexp.MustCompile(`^code-preview-[-\w]+( file-content)?$`)).Globally()
+ policy.AllowAttrs("class").Matching(regexp.MustCompile(`^lines-num$`)).OnElements("td")
+ policy.AllowAttrs("data-line-number").OnElements("span")
+ policy.AllowAttrs("class").Matching(regexp.MustCompile(`^lines-code chroma$`)).OnElements("td")
+ policy.AllowAttrs("class").Matching(regexp.MustCompile(`^code-inner$`)).OnElements("code")
+
+ // For code preview (unicode escape)
+ policy.AllowAttrs("class").Matching(regexp.MustCompile(`^file-view( unicode-escaped)?$`)).OnElements("table")
+ policy.AllowAttrs("class").Matching(regexp.MustCompile(`^lines-escape$`)).OnElements("td")
+ policy.AllowAttrs("class").Matching(regexp.MustCompile(`^toggle-escape-button btn interact-bg$`)).OnElements("a") // don't use button, button might submit a form
+ policy.AllowAttrs("class").Matching(regexp.MustCompile(`^(ambiguous-code-point|escaped-code-point|broken-code-point)$`)).OnElements("span")
+ policy.AllowAttrs("class").Matching(regexp.MustCompile(`^char$`)).OnElements("span")
+ policy.AllowAttrs("data-tooltip-content", "data-escaped").OnElements("span")
+
  // For color preview
  policy.AllowAttrs("class").Matching(regexp.MustCompile(`^color-preview$`)).OnElements("span")
 

modules/translation/mock.go

@@ -6,6 +6,7 @@ package translation
 import (
  "fmt"
  "html/template"
+ "strings"
 )
 
 // MockLocale provides a mocked locale without any translations
@@ -19,18 +20,25 @@ func (l MockLocale) Language() string {
  return "en"
 }
 
-func (l MockLocale) TrString(s string, _ ...any) string {
- return s
+func (l MockLocale) TrString(s string, args ...any) string {
+ return sprintAny(s, args...)
 }
 
-func (l MockLocale) Tr(s string, a ...any) template.HTML {
- return template.HTML(s)
+func (l MockLocale) Tr(s string, args ...any) template.HTML {
+ return template.HTML(sprintAny(s, args...))
 }
 
 func (l MockLocale) TrN(cnt any, key1, keyN string, args ...any) template.HTML {
- return template.HTML(key1)
+ return template.HTML(sprintAny(key1, args...))
 }
 
 func (l MockLocale) PrettyNumber(v any) string {
  return fmt.Sprint(v)
 }
+
+func sprintAny(s string, args ...any) string {
+ if len(args) == 0 {
+ return s
+ }
+ return s + ":" + fmt.Sprintf(strings.Repeat(",%v", len(args))[1:], args...)
+}

options/locale/locale_en-US.ini

@@ -1233,6 +1233,8 @@ file_view_rendered = View Rendered
 file_view_raw = View Raw
 file_permalink = Permalink
 file_too_large = The file is too large to be shown.
+code_preview_line_from_to = Lines %[1]d to %[2]d in %[3]s
+code_preview_line_in = Line %[1]d in %[2]s
 invisible_runes_header = `This file contains invisible Unicode characters`
 invisible_runes_description = `This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.`
 ambiguous_runes_header = `This file contains ambiguous Unicode characters`

routers/web/repo/search.go

@@ -81,7 +81,7 @@ func Search(ctx *context.Context) {
  // UpdatedUnix: not supported yet
  // Language: not supported yet
  // Color: not supported yet
- Lines: code_indexer.HighlightSearchResultCode(r.Filename, r.LineNumbers, strings.Join(r.LineCodes, "\n")),
+ Lines: code_indexer.HighlightSearchResultCode(r.Filename, "", r.LineNumbers, strings.Join(r.LineCodes, "\n")),
  })
  }
  }

routers/web/repo/wiki_test.go

@@ -145,7 +145,7 @@ func TestNewWikiPost_ReservedName(t *testing.T) {
  })
  NewWikiPost(ctx)
  assert.EqualValues(t, http.StatusOK, ctx.Resp.Status())
- assert.EqualValues(t, ctx.Tr("repo.wiki.reserved_page"), ctx.Flash.ErrorMsg)
+ assert.EqualValues(t, ctx.Tr("repo.wiki.reserved_page", "_edit"), ctx.Flash.ErrorMsg)
  assertWikiNotExists(t, ctx.Repo.Repository, "_edit")
 }
 

services/contexttest/context_tests.go

@@ -63,6 +63,7 @@ func MockContext(t *testing.T, reqPath string, opts ...MockContextOption) (*cont
  base.Locale = &translation.MockLocale{}
 
  ctx := context.NewWebContext(base, opt.Render, nil)
+ ctx.AppendContextValue(context.WebContextKey, ctx)
  ctx.PageData = map[string]any{}
  ctx.Data["PageStartTime"] = time.Now()
  chiCtx := chi.NewRouteContext()

services/markup/main_test.go

@@ -11,6 +11,6 @@ import (
 
 func TestMain(m *testing.M) {
  unittest.MainTest(m, &unittest.TestOptions{
- FixtureFiles: []string{"user.yml"},
+ FixtureFiles: []string{"user.yml", "repository.yml", "access.yml", "repo_unit.yml"},
  })
 }

services/markup/processorhelper.go

@@ -14,6 +14,8 @@ import (
 func ProcessorHelper() *markup.ProcessorHelper {
  return &markup.ProcessorHelper{
  ElementDir: "auto", // set dir="auto" for necessary (eg: <p>, <h?>, etc) tags
+
+ RenderRepoFileCodePreview: renderRepoFileCodePreview,
  IsUsernameMentionable: func(ctx context.Context, username string) bool {
  mentionedUser, err := user.GetUserByName(ctx, username)
  if err != nil {