Skip to content

Commit b272224

Browse files
GiteaBotearl-warren
GiteaBot
authored and
earl-warren
committed
Preserve BOM in web editor (#28935) (#28959)
Backport #28935 by @silverwind The `ToUTF8*` functions were stripping BOM, while BOM is actually valid in UTF8, so the stripping must be optional depending on use case. This does: - Add a options struct to all `ToUTF8*` functions, that by default will strip BOM to preserve existing behaviour - Remove `ToUTF8` function, it was dead code - Rename `ToUTF8WithErr` to `ToUTF8` - Preserve BOM in Monaco Editor - Remove a unnecessary newline in the textarea value. Browsers did ignore it, it seems but it's better not to rely on this behaviour. Fixes: go-gitea/gitea#28743 Related: go-gitea/gitea#6716 which seems to have once introduced a mechanism that strips and re-adds the BOM, but from what I can tell, this mechanism was removed at some point after that PR. Co-authored-by: silverwind <me@silverwind.io> (cherry picked from commit b8e6cff)
1 parent afdce6b commit b272224

File tree

13 files changed

+69
-134
lines changed

13 files changed

+69
-134
lines changed

‎modules/charset/charset.go

+22-21
Original file line numberDiff line numberDiff line change
@@ -22,17 +22,21 @@ import (
2222
// UTF8BOM is the utf-8 byte-order marker
2323
var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
2424

25+
type ConvertOpts struct {
26+
KeepBOM bool
27+
}
28+
2529
// ToUTF8WithFallbackReader detects the encoding of content and converts to UTF-8 reader if possible
26-
func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
30+
func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader {
2731
buf := make([]byte, 2048)
2832
n, err := util.ReadAtMost(rd, buf)
2933
if err != nil {
30-
return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd)
34+
return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
3135
}
3236

3337
charsetLabel, err := DetectEncoding(buf[:n])
3438
if err != nil || charsetLabel == "UTF-8" {
35-
return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd)
39+
return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
3640
}
3741

3842
encoding, _ := charset.Lookup(charsetLabel)
@@ -42,20 +46,20 @@ func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
4246

4347
return transform.NewReader(
4448
io.MultiReader(
45-
bytes.NewReader(RemoveBOMIfPresent(buf[:n])),
49+
bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)),
4650
rd,
4751
),
4852
encoding.NewDecoder(),
4953
)
5054
}
5155

52-
// ToUTF8WithErr converts content to UTF8 encoding
53-
func ToUTF8WithErr(content []byte) (string, error) {
56+
// ToUTF8 converts content to UTF8 encoding
57+
func ToUTF8(content []byte, opts ConvertOpts) (string, error) {
5458
charsetLabel, err := DetectEncoding(content)
5559
if err != nil {
5660
return "", err
5761
} else if charsetLabel == "UTF-8" {
58-
return string(RemoveBOMIfPresent(content)), nil
62+
return string(MaybeRemoveBOM(content, opts)), nil
5963
}
6064

6165
encoding, _ := charset.Lookup(charsetLabel)
@@ -70,28 +74,22 @@ func ToUTF8WithErr(content []byte) (string, error) {
7074
result = append(result, content[n:]...)
7175
}
7276

73-
result = RemoveBOMIfPresent(result)
77+
result = MaybeRemoveBOM(result, opts)
7478

7579
return string(result), err
7680
}
7781

7882
// ToUTF8WithFallback detects the encoding of content and converts to UTF-8 if possible
79-
func ToUTF8WithFallback(content []byte) []byte {
80-
bs, _ := io.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content)))
83+
func ToUTF8WithFallback(content []byte, opts ConvertOpts) []byte {
84+
bs, _ := io.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content), opts))
8185
return bs
8286
}
8387

84-
// ToUTF8 converts content to UTF8 encoding and ignore error
85-
func ToUTF8(content string) string {
86-
res, _ := ToUTF8WithErr([]byte(content))
87-
return res
88-
}
89-
9088
// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible
91-
func ToUTF8DropErrors(content []byte) []byte {
89+
func ToUTF8DropErrors(content []byte, opts ConvertOpts) []byte {
9290
charsetLabel, err := DetectEncoding(content)
9391
if err != nil || charsetLabel == "UTF-8" {
94-
return RemoveBOMIfPresent(content)
92+
return MaybeRemoveBOM(content, opts)
9593
}
9694

9795
encoding, _ := charset.Lookup(charsetLabel)
@@ -117,11 +115,14 @@ func ToUTF8DropErrors(content []byte) []byte {
117115
}
118116
}
119117

120-
return RemoveBOMIfPresent(decoded)
118+
return MaybeRemoveBOM(decoded, opts)
121119
}
122120

123-
// RemoveBOMIfPresent removes a UTF-8 BOM from a []byte
124-
func RemoveBOMIfPresent(content []byte) []byte {
121+
// MaybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false
122+
func MaybeRemoveBOM(content []byte, opts ConvertOpts) []byte {
123+
if opts.KeepBOM {
124+
return content
125+
}
125126
if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
126127
return content[3:]
127128
}

‎modules/charset/charset_test.go

+34-99
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,15 @@ func resetDefaultCharsetsOrder() {
3030
}
3131
}
3232

33-
func TestRemoveBOMIfPresent(t *testing.T) {
34-
res := RemoveBOMIfPresent([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
33+
func TestMaybeRemoveBOM(t *testing.T) {
34+
res := MaybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
3535
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
3636

37-
res = RemoveBOMIfPresent([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
37+
res = MaybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
3838
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
3939
}
4040

41-
func TestToUTF8WithErr(t *testing.T) {
41+
func TestToUTF8(t *testing.T) {
4242
resetDefaultCharsetsOrder()
4343
var res string
4444
var err error
@@ -47,84 +47,84 @@ func TestToUTF8WithErr(t *testing.T) {
4747
// locale, so some conversions might behave differently. For that reason, we don't
4848
// depend on particular conversions but in expected behaviors.
4949

50-
res, err = ToUTF8WithErr([]byte{0x41, 0x42, 0x43})
50+
res, err = ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
5151
assert.NoError(t, err)
5252
assert.Equal(t, "ABC", res)
5353

5454
// "áéíóú"
55-
res, err = ToUTF8WithErr([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
55+
res, err = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
5656
assert.NoError(t, err)
5757
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
5858

5959
// "áéíóú"
60-
res, err = ToUTF8WithErr([]byte{
60+
res, err = ToUTF8([]byte{
6161
0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
6262
0xc3, 0xba,
63-
})
63+
}, ConvertOpts{})
6464
assert.NoError(t, err)
6565
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
6666

67-
res, err = ToUTF8WithErr([]byte{
67+
res, err = ToUTF8([]byte{
6868
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
6969
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
70-
})
70+
}, ConvertOpts{})
7171
assert.NoError(t, err)
7272
stringMustStartWith(t, "Hola,", res)
7373
stringMustEndWith(t, "AAA.", res)
7474

75-
res, err = ToUTF8WithErr([]byte{
75+
res, err = ToUTF8([]byte{
7676
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
7777
0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
78-
})
78+
}, ConvertOpts{})
7979
assert.NoError(t, err)
8080
stringMustStartWith(t, "Hola,", res)
8181
stringMustEndWith(t, "AAA.", res)
8282

83-
res, err = ToUTF8WithErr([]byte{
83+
res, err = ToUTF8([]byte{
8484
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
8585
0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
86-
})
86+
}, ConvertOpts{})
8787
assert.NoError(t, err)
8888
stringMustStartWith(t, "Hola,", res)
8989
stringMustEndWith(t, "AAA.", res)
9090

9191
// Japanese (Shift-JIS)
9292
// 日属秘ぞしちゅ。
93-
res, err = ToUTF8WithErr([]byte{
93+
res, err = ToUTF8([]byte{
9494
0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
9595
0xBF, 0x82, 0xE3, 0x81, 0x42,
96-
})
96+
}, ConvertOpts{})
9797
assert.NoError(t, err)
9898
assert.Equal(t, []byte{
9999
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
100100
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
101101
},
102102
[]byte(res))
103103

104-
res, err = ToUTF8WithErr([]byte{0x00, 0x00, 0x00, 0x00})
104+
res, err = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
105105
assert.NoError(t, err)
106106
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
107107
}
108108

109109
func TestToUTF8WithFallback(t *testing.T) {
110110
resetDefaultCharsetsOrder()
111111
// "ABC"
112-
res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43})
112+
res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
113113
assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
114114

115115
// "áéíóú"
116-
res = ToUTF8WithFallback([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
116+
res = ToUTF8WithFallback([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
117117
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
118118

119119
// UTF8 BOM + "áéíóú"
120-
res = ToUTF8WithFallback([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
120+
res = ToUTF8WithFallback([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
121121
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
122122

123123
// "Hola, así cómo ños"
124124
res = ToUTF8WithFallback([]byte{
125125
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
126126
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73,
127-
})
127+
}, ConvertOpts{})
128128
assert.Equal(t, []byte{
129129
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63,
130130
0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73,
@@ -133,126 +133,65 @@ func TestToUTF8WithFallback(t *testing.T) {
133133
// "Hola, así cómo "
134134
minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
135135

136-
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
136+
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}, ConvertOpts{})
137137
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
138138
assert.Equal(t, minmatch, res[0:len(minmatch)])
139139

140-
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
140+
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}, ConvertOpts{})
141141
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
142142
assert.Equal(t, minmatch, res[0:len(minmatch)])
143143

144144
// Japanese (Shift-JIS)
145145
// "日属秘ぞしちゅ。"
146-
res = ToUTF8WithFallback([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
146+
res = ToUTF8WithFallback([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}, ConvertOpts{})
147147
assert.Equal(t, []byte{
148148
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
149149
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
150150
}, res)
151151

152-
res = ToUTF8WithFallback([]byte{0x00, 0x00, 0x00, 0x00})
152+
res = ToUTF8WithFallback([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
153153
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
154154
}
155155

156-
func TestToUTF8(t *testing.T) {
157-
resetDefaultCharsetsOrder()
158-
// Note: golang compiler seems so behave differently depending on the current
159-
// locale, so some conversions might behave differently. For that reason, we don't
160-
// depend on particular conversions but in expected behaviors.
161-
162-
res := ToUTF8(string([]byte{0x41, 0x42, 0x43}))
163-
assert.Equal(t, "ABC", res)
164-
165-
// "áéíóú"
166-
res = ToUTF8(string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}))
167-
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
168-
169-
// BOM + "áéíóú"
170-
res = ToUTF8(string([]byte{
171-
0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
172-
0xc3, 0xba,
173-
}))
174-
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
175-
176-
// Latin1
177-
// Hola, así cómo ños
178-
res = ToUTF8(string([]byte{
179-
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
180-
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73,
181-
}))
182-
assert.Equal(t, []byte{
183-
0x48, 0x6f, 0x6c, 0x61, 0x2c, 0x20, 0x61, 0x73, 0xc3, 0xad, 0x20, 0x63,
184-
0xc3, 0xb3, 0x6d, 0x6f, 0x20, 0xc3, 0xb1, 0x6f, 0x73,
185-
}, []byte(res))
186-
187-
// Latin1
188-
// Hola, así cómo \x07ños
189-
res = ToUTF8(string([]byte{
190-
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
191-
0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73,
192-
}))
193-
// Hola,
194-
bytesMustStartWith(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C}, []byte(res))
195-
196-
// This test FAILS
197-
// res = ToUTF8("Hola, así cómo \x81ños")
198-
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
199-
// assert.Regexp(t, "^Hola, así cómo", res)
200-
201-
// Japanese (Shift-JIS)
202-
// 日属秘ぞしちゅ。
203-
res = ToUTF8(string([]byte{
204-
0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
205-
0xBF, 0x82, 0xE3, 0x81, 0x42,
206-
}))
207-
assert.Equal(t, []byte{
208-
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
209-
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
210-
},
211-
[]byte(res))
212-
213-
res = ToUTF8("\x00\x00\x00\x00")
214-
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
215-
}
216-
217156
func TestToUTF8DropErrors(t *testing.T) {
218157
resetDefaultCharsetsOrder()
219158
// "ABC"
220-
res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43})
159+
res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
221160
assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
222161

223162
// "áéíóú"
224-
res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
163+
res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
225164
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
226165

227166
// UTF8 BOM + "áéíóú"
228-
res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
167+
res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
229168
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
230169

231170
// "Hola, así cómo ños"
232-
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
171+
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}, ConvertOpts{})
233172
assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73}, res[:8])
234173
assert.Equal(t, []byte{0x73}, res[len(res)-1:])
235174

236175
// "Hola, así cómo "
237176
minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
238177

239-
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
178+
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}, ConvertOpts{})
240179
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
241180
assert.Equal(t, minmatch, res[0:len(minmatch)])
242181

243-
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
182+
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}, ConvertOpts{})
244183
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
245184
assert.Equal(t, minmatch, res[0:len(minmatch)])
246185

247186
// Japanese (Shift-JIS)
248187
// "日属秘ぞしちゅ。"
249-
res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
188+
res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}, ConvertOpts{})
250189
assert.Equal(t, []byte{
251190
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
252191
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
253192
}, res)
254193

255-
res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00})
194+
res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
256195
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
257196
}
258197

@@ -302,10 +241,6 @@ func stringMustEndWith(t *testing.T, expected, value string) {
302241
assert.Equal(t, expected, value[len(value)-len(expected):])
303242
}
304243

305-
func bytesMustStartWith(t *testing.T, expected, value []byte) {
306-
assert.Equal(t, expected, value[:len(expected)])
307-
}
308-
309244
func TestToUTF8WithFallbackReader(t *testing.T) {
310245
resetDefaultCharsetsOrder()
311246

@@ -317,7 +252,7 @@ func TestToUTF8WithFallbackReader(t *testing.T) {
317252
}
318253
input = input[:testLen]
319254
input += "// Выключаем"
320-
rd := ToUTF8WithFallbackReader(bytes.NewReader([]byte(input)))
255+
rd := ToUTF8WithFallbackReader(bytes.NewReader([]byte(input)), ConvertOpts{})
321256
r, _ := io.ReadAll(rd)
322257
assert.EqualValuesf(t, input, string(r), "testing string len=%d", testLen)
323258
}

‎modules/indexer/code/bleve/bleve.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro
174174
return batch.Index(id, &RepoIndexerData{
175175
RepoID: repo.ID,
176176
CommitID: commitSha,
177-
Content: string(charset.ToUTF8DropErrors(fileContents)),
177+
Content: string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})),
178178
Language: analyze.GetCodeLanguage(update.Filename, fileContents),
179179
UpdatedAt: time.Now().UTC(),
180180
})

0 commit comments

Comments
 (0)