Avoid emoji mismatch and allow to only enable chosen emojis (go-gitea#35692) (go-gitea#35705)

Backport go-gitea#35692 by wxiaoguang

Fix go-gitea#23635

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

Author: GiteaBot

custom/conf/app.example.ini

@@ -1343,6 +1343,10 @@ LEVEL = Info
 ;; Dont mistake it for Reactions.
 ;CUSTOM_EMOJIS = gitea, codeberg, gitlab, git, github, gogs
 ;;
+;; Comma separated list of enabled emojis, for example: smile, thumbsup, thumbsdown
+;; Leave it empty to enable all emojis.
+;ENABLED_EMOJIS =
+;;
 ;; Whether the full name of the users should be shown where possible. If the full name isn't set, the username will be used.
 ;DEFAULT_SHOW_FULL_NAME = false
 ;;

modules/emoji/emoji.go

@@ -8,7 +8,9 @@ import (
 	"io"
 	"sort"
 	"strings"
-	"sync"
+	"sync/atomic"
+
+	"code.gitea.io/gitea/modules/setting"
 )
 
 // Gemoji is a set of emoji data.
@@ -23,74 +25,78 @@ type Emoji struct {
 	SkinTones      bool
 }
 
-var (
-	// codeMap provides a map of the emoji unicode code to its emoji data.
-	codeMap map[string]int
-
-	// aliasMap provides a map of the alias to its emoji data.
-	aliasMap map[string]int
-
-	// emptyReplacer is the string replacer for emoji codes.
-	emptyReplacer *strings.Replacer
-
-	// codeReplacer is the string replacer for emoji codes.
-	codeReplacer *strings.Replacer
-
-	// aliasReplacer is the string replacer for emoji aliases.
-	aliasReplacer *strings.Replacer
-
-	once sync.Once
-)
+type globalVarsStruct struct {
+	codeMap       map[string]int    // emoji unicode code to its emoji data.
+	aliasMap      map[string]int    // the alias to its emoji data.
+	emptyReplacer *strings.Replacer // string replacer for emoji codes, used for finding emoji positions.
+	codeReplacer  *strings.Replacer // string replacer for emoji codes.
+	aliasReplacer *strings.Replacer // string replacer for emoji aliases.
+}
 
-func loadMap() {
-	once.Do(func() {
-		// initialize
-		codeMap = make(map[string]int, len(GemojiData))
-		aliasMap = make(map[string]int, len(GemojiData))
+var globalVarsStore atomic.Pointer[globalVarsStruct]
 
-		// process emoji codes and aliases
-		codePairs := make([]string, 0)
-		emptyPairs := make([]string, 0)
-		aliasPairs := make([]string, 0)
+func globalVars() *globalVarsStruct {
+	vars := globalVarsStore.Load()
+	if vars != nil {
+		return vars
+	}
+	// although there can be concurrent calls, the result should be the same, and there is no performance problem
+	vars = &globalVarsStruct{}
+	vars.codeMap = make(map[string]int, len(GemojiData))
+	vars.aliasMap = make(map[string]int, len(GemojiData))
+
+	// process emoji codes and aliases
+	codePairs := make([]string, 0)
+	emptyPairs := make([]string, 0)
+	aliasPairs := make([]string, 0)
+
+	// sort from largest to small so we match combined emoji first
+	sort.Slice(GemojiData, func(i, j int) bool {
+		return len(GemojiData[i].Emoji) > len(GemojiData[j].Emoji)
+	})
 
-		// sort from largest to small so we match combined emoji first
-		sort.Slice(GemojiData, func(i, j int) bool {
-			return len(GemojiData[i].Emoji) > len(GemojiData[j].Emoji)
-		})
+	for idx, emoji := range GemojiData {
+		if emoji.Emoji == "" || len(emoji.Aliases) == 0 {
+			continue
+		}
 
-		for i, e := range GemojiData {
-			if e.Emoji == "" || len(e.Aliases) == 0 {
+		// process aliases
+		firstAlias := ""
+		for _, alias := range emoji.Aliases {
+			if alias == "" {
 				continue
 			}
-
-			// setup codes
-			codeMap[e.Emoji] = i
-			codePairs = append(codePairs, e.Emoji, ":"+e.Aliases[0]+":")
-			emptyPairs = append(emptyPairs, e.Emoji, e.Emoji)
-
-			// setup aliases
-			for _, a := range e.Aliases {
-				if a == "" {
-					continue
-				}
-
-				aliasMap[a] = i
-				aliasPairs = append(aliasPairs, ":"+a+":", e.Emoji)
+			enabled := len(setting.UI.EnabledEmojisSet) == 0 || setting.UI.EnabledEmojisSet.Contains(alias)
+			if !enabled {
+				continue
 			}
+			if firstAlias == "" {
+				firstAlias = alias
+			}
+			vars.aliasMap[alias] = idx
+			aliasPairs = append(aliasPairs, ":"+alias+":", emoji.Emoji)
 		}
 
-		// create replacers
-		emptyReplacer = strings.NewReplacer(emptyPairs...)
-		codeReplacer = strings.NewReplacer(codePairs...)
-		aliasReplacer = strings.NewReplacer(aliasPairs...)
-	})
+		// process emoji code
+		if firstAlias != "" {
+			vars.codeMap[emoji.Emoji] = idx
+			codePairs = append(codePairs, emoji.Emoji, ":"+emoji.Aliases[0]+":")
+			emptyPairs = append(emptyPairs, emoji.Emoji, emoji.Emoji)
+		}
+	}
+
+	// create replacers
+	vars.emptyReplacer = strings.NewReplacer(emptyPairs...)
+	vars.codeReplacer = strings.NewReplacer(codePairs...)
+	vars.aliasReplacer = strings.NewReplacer(aliasPairs...)
+	globalVarsStore.Store(vars)
+	return vars
 }
 
 // FromCode retrieves the emoji data based on the provided unicode code (ie,
 // "\u2618" will return the Gemoji data for "shamrock").
 func FromCode(code string) *Emoji {
-	loadMap()
-	i, ok := codeMap[code]
+	i, ok := globalVars().codeMap[code]
 	if !ok {
 		return nil
 	}
@@ -102,12 +108,11 @@ func FromCode(code string) *Emoji {
 // "alias" or ":alias:" (ie, "shamrock" or ":shamrock:" will return the Gemoji
 // data for "shamrock").
 func FromAlias(alias string) *Emoji {
-	loadMap()
 	if strings.HasPrefix(alias, ":") && strings.HasSuffix(alias, ":") {
 		alias = alias[1 : len(alias)-1]
 	}
 
-	i, ok := aliasMap[alias]
+	i, ok := globalVars().aliasMap[alias]
 	if !ok {
 		return nil
 	}
@@ -119,15 +124,13 @@ func FromAlias(alias string) *Emoji {
 // alias (in the form of ":alias:") (ie, "\u2618" will be converted to
 // ":shamrock:").
 func ReplaceCodes(s string) string {
-	loadMap()
-	return codeReplacer.Replace(s)
+	return globalVars().codeReplacer.Replace(s)
 }
 
 // ReplaceAliases replaces all aliases of the form ":alias:" with its
 // corresponding unicode value.
 func ReplaceAliases(s string) string {
-	loadMap()
-	return aliasReplacer.Replace(s)
+	return globalVars().aliasReplacer.Replace(s)
 }
 
 type rememberSecondWriteWriter struct {
@@ -163,7 +166,6 @@ func (n *rememberSecondWriteWriter) WriteString(s string) (int, error) {
 
 // FindEmojiSubmatchIndex returns index pair of longest emoji in a string
 func FindEmojiSubmatchIndex(s string) []int {
-	loadMap()
 	secondWriteWriter := rememberSecondWriteWriter{}
 
 	// A faster and clean implementation would copy the trie tree formation in strings.NewReplacer but
@@ -175,7 +177,7 @@ func FindEmojiSubmatchIndex(s string) []int {
 	// Therefore we can simply take the index of the second write as our first emoji
 	//
 	// FIXME: just copy the trie implementation from strings.NewReplacer
-	_, _ = emptyReplacer.WriteString(&secondWriteWriter, s)
+	_, _ = globalVars().emptyReplacer.WriteString(&secondWriteWriter, s)
 
 	// if we wrote less than twice then we never "replaced"
 	if secondWriteWriter.writecount < 2 {

modules/emoji/emoji_test.go

@@ -7,14 +7,13 @@ package emoji
 import (
 	"testing"
 
+	"code.gitea.io/gitea/modules/container"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/test"
+
 	"github.com/stretchr/testify/assert"
 )
 
-func TestDumpInfo(t *testing.T) {
-	t.Logf("codes: %d", len(codeMap))
-	t.Logf("aliases: %d", len(aliasMap))
-}
-
 func TestLookup(t *testing.T) {
 	a := FromCode("\U0001f37a")
 	b := FromCode("🍺")
@@ -24,15 +23,27 @@ func TestLookup(t *testing.T) {
 	assert.Equal(t, a, b)
 	assert.Equal(t, b, c)
 	assert.Equal(t, c, d)
-	assert.Equal(t, a, d)
 
 	m := FromCode("\U0001f44d")
 	n := FromAlias(":thumbsup:")
 	o := FromAlias("+1")
 
 	assert.Equal(t, m, n)
 	assert.Equal(t, m, o)
-	assert.Equal(t, n, o)
+
+	defer test.MockVariableValue(&setting.UI.EnabledEmojisSet, container.SetOf("thumbsup"))()
+	defer globalVarsStore.Store(nil)
+	globalVarsStore.Store(nil)
+	a = FromCode("\U0001f37a")
+	c = FromAlias(":beer:")
+	m = FromCode("\U0001f44d")
+	n = FromAlias(":thumbsup:")
+	o = FromAlias("+1")
+	assert.Nil(t, a)
+	assert.Nil(t, c)
+	assert.NotNil(t, m)
+	assert.NotNil(t, n)
+	assert.Nil(t, o)
 }
 
 func TestReplacers(t *testing.T) {

modules/markup/html_emoji.go

@@ -5,6 +5,7 @@ package markup
 
 import (
 	"strings"
+	"unicode"
 
 	"code.gitea.io/gitea/modules/emoji"
 	"code.gitea.io/gitea/modules/setting"
@@ -66,26 +67,31 @@ func emojiShortCodeProcessor(ctx *RenderContext, node *html.Node) {
 		}
 		m[0] += start
 		m[1] += start
-
 		start = m[1]
 
 		alias := node.Data[m[0]:m[1]]
-		alias = strings.ReplaceAll(alias, ":", "")
-		converted := emoji.FromAlias(alias)
-		if converted == nil {
-			// check if this is a custom reaction
-			if _, exist := setting.UI.CustomEmojisMap[alias]; exist {
-				replaceContent(node, m[0], m[1], createCustomEmoji(ctx, alias))
-				node = node.NextSibling.NextSibling
-				start = 0
-				continue
-			}
+
+		var nextChar byte
+		if m[1] < len(node.Data) {
+			nextChar = node.Data[m[1]]
+		}
+		if nextChar == ':' || unicode.IsLetter(rune(nextChar)) || unicode.IsDigit(rune(nextChar)) {
 			continue
 		}
 
-		replaceContent(node, m[0], m[1], createEmoji(ctx, converted.Emoji, converted.Description))
-		node = node.NextSibling.NextSibling
-		start = 0
+		alias = strings.Trim(alias, ":")
+		converted := emoji.FromAlias(alias)
+		if converted != nil {
+			// standard emoji
+			replaceContent(node, m[0], m[1], createEmoji(ctx, converted.Emoji, converted.Description))
+			node = node.NextSibling.NextSibling
+			start = 0 // restart searching start since node has changed
+		} else if _, exist := setting.UI.CustomEmojisMap[alias]; exist {
+			// custom reaction
+			replaceContent(node, m[0], m[1], createCustomEmoji(ctx, alias))
+			node = node.NextSibling.NextSibling
+			start = 0 // restart searching start since node has changed
+		}
 	}
 }
 

modules/markup/html_test.go

@@ -357,12 +357,9 @@ func TestRender_emoji(t *testing.T) {
 		`<p><span class="emoji" aria-label="smiling face with sunglasses">😎</span><span class="emoji" aria-label="zany face">🤪</span><span class="emoji" aria-label="locked with key">🔐</span><span class="emoji" aria-label="money-mouth face">🤑</span><span class="emoji" aria-label="red question mark">❓</span></p>`)
 
 	// should match nothing
-	test(
-		"2001:0db8:85a3:0000:0000:8a2e:0370:7334",
-		`<p>2001:0db8:85a3:0000:0000:8a2e:0370:7334</p>`)
-	test(
-		":not exist:",
-		`<p>:not exist:</p>`)
+	test(":100:200", `<p>:100:200</p>`)
+	test("std::thread::something", `<p>std::thread::something</p>`)
+	test(":not exist:", `<p>:not exist:</p>`)
 }
 
 func TestRender_ShortLinks(t *testing.T) {

modules/setting/ui.go

@@ -33,6 +33,8 @@ var UI = struct {
 	ReactionsLookup         container.Set[string] `ini:"-"`
 	CustomEmojis            []string
 	CustomEmojisMap         map[string]string `ini:"-"`
+	EnabledEmojis           []string
+	EnabledEmojisSet        container.Set[string] `ini:"-"`
 	SearchRepoDescription   bool
 	OnlyShowRelevantRepos   bool
 	ExploreDefaultSort      string `ini:"EXPLORE_PAGING_DEFAULT_SORT"`
@@ -169,4 +171,5 @@ func loadUIFrom(rootCfg ConfigProvider) {
 	for _, emoji := range UI.CustomEmojis {
 		UI.CustomEmojisMap[emoji] = ":" + emoji + ":"
 	}
+	UI.EnabledEmojisSet = container.SetOf(UI.EnabledEmojis...)
 }