Skip to content

Commit

Permalink
server: Return VECTOR as LONGTEXT for clients (#55223)
Browse files Browse the repository at this point in the history 
ref #54245
  • Loading branch information
@EricZequan
EricZequan authored Aug 20, 2024
1 parent 573e1ab commit c4a1f18
Show file tree
Hide file tree
Showing 7 changed files with 290 additions and 3 deletions.
52 changes: 52 additions & 0 deletions DEPS.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4195,6 +4195,32 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/jingyugao/rowserrcheck/com_github_jingyugao_rowserrcheck-v1.1.1.zip",
],
)
go_repository(
name = "com_github_jinzhu_inflection",
build_file_proto_mode = "disable_global",
importpath = "github.com/jinzhu/inflection",
sha256 = "cf1087a6f6653ed5f366f85cf0110bbbf581d4e9bc8a4d1a9b56765d94b546c3",
strip_prefix = "github.com/jinzhu/inflection@v1.0.0",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/jinzhu/inflection/com_github_jinzhu_inflection-v1.0.0.zip",
"http://ats.apps.svc/gomod/github.com/jinzhu/inflection/com_github_jinzhu_inflection-v1.0.0.zip",
"https://cache.hawkingrei.com/gomod/github.com/jinzhu/inflection/com_github_jinzhu_inflection-v1.0.0.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/jinzhu/inflection/com_github_jinzhu_inflection-v1.0.0.zip",
],
)
go_repository(
name = "com_github_jinzhu_now",
build_file_proto_mode = "disable_global",
importpath = "github.com/jinzhu/now",
sha256 = "1223dc55db616f156c1f1467adc2c88f786905df3cc3cb4fd5161badd654c62b",
strip_prefix = "github.com/jinzhu/now@v1.1.5",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/jinzhu/now/com_github_jinzhu_now-v1.1.5.zip",
"http://ats.apps.svc/gomod/github.com/jinzhu/now/com_github_jinzhu_now-v1.1.5.zip",
"https://cache.hawkingrei.com/gomod/github.com/jinzhu/now/com_github_jinzhu_now-v1.1.5.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/jinzhu/now/com_github_jinzhu_now-v1.1.5.zip",
],
)
go_repository(
name = "com_github_jirfag_go_printf_func_name",
build_file_proto_mode = "disable_global",
Expand Down Expand Up @@ -9711,6 +9737,32 @@ def go_deps():
"https://storage.googleapis.com/pingcapmirror/gomod/filippo.io/edwards25519/io_filippo_edwards25519-v1.1.0.zip",
],
)
go_repository(
name = "io_gorm_driver_mysql",
build_file_proto_mode = "disable_global",
importpath = "gorm.io/driver/mysql",
sha256 = "4a529d09c3a0082e313ed76f4d3fe5bfb2667711ef36d7ac6e6804dba43e7978",
strip_prefix = "gorm.io/driver/mysql@v1.5.7",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/gorm.io/driver/mysql/io_gorm_driver_mysql-v1.5.7.zip",
"http://ats.apps.svc/gomod/gorm.io/driver/mysql/io_gorm_driver_mysql-v1.5.7.zip",
"https://cache.hawkingrei.com/gomod/gorm.io/driver/mysql/io_gorm_driver_mysql-v1.5.7.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/gorm.io/driver/mysql/io_gorm_driver_mysql-v1.5.7.zip",
],
)
go_repository(
name = "io_gorm_gorm",
build_file_proto_mode = "disable_global",
importpath = "gorm.io/gorm",
sha256 = "e33460f1d29271c188f4920275adc1d8f83890a17dfa6a9f03face8cdda32cb5",
strip_prefix = "gorm.io/gorm@v1.25.11",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/gorm.io/gorm/io_gorm_gorm-v1.25.11.zip",
"http://ats.apps.svc/gomod/gorm.io/gorm/io_gorm_gorm-v1.25.11.zip",
"https://cache.hawkingrei.com/gomod/gorm.io/gorm/io_gorm_gorm-v1.25.11.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/gorm.io/gorm/io_gorm_gorm-v1.25.11.zip",
],
)
go_repository(
name = "io_k8s_api",
build_file_proto_mode = "disable_global",
Expand Down
4 changes: 4 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,8 @@ require (
google.golang.org/api v0.169.0
google.golang.org/grpc v1.63.2
gopkg.in/yaml.v2 v2.4.0
gorm.io/driver/mysql v1.5.7
gorm.io/gorm v1.25.11
honnef.co/go/tools v0.4.7
k8s.io/api v0.28.6
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0
Expand All @@ -155,6 +157,8 @@ require (
github.com/getsentry/sentry-go v0.27.0 // indirect
github.com/goccy/go-reflect v1.2.0 // indirect
github.com/google/flatbuffers v2.0.8+incompatible // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
Expand Down
10 changes: 10 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,7 @@ github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhO
github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
Expand Down Expand Up @@ -523,6 +524,10 @@ github.com/jfcg/sorty/v2 v2.1.0 h1:EjrVSL3cDRxBt/ehiYCIv10F7YHYbTzEmdv7WbkkN1k=
github.com/jfcg/sorty/v2 v2.1.0/go.mod h1:JpcSKlmtGOOAGyTdWN2ErjvxeMSJVYBsylAKepIxmNg=
github.com/jingyugao/rowserrcheck v1.1.1 h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjzq7gFzUs=
github.com/jingyugao/rowserrcheck v1.1.1/go.mod h1:4yvlZSDb3IyDTUZJUmpZfm2Hwok+Dtp+nu2qOq+er9c=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
Expand Down Expand Up @@ -1453,6 +1458,11 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg=
gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
Expand Down
31 changes: 30 additions & 1 deletion pkg/server/internal/column/column.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func (column *Info) dump(buffer []byte, d *ResultEncoder, withDefault bool) []by

buffer = append(buffer, 0x0c)
buffer = dump.Uint16(buffer, d.ColumnTypeInfoCharsetID(column))
buffer = dump.Uint32(buffer, column.ColumnLength)
buffer = dump.Uint32(buffer, column.dumpLength())
buffer = append(buffer, dumpType(column.Type))
buffer = dump.Uint16(buffer, DumpFlag(column.Type, column.Flag))
buffer = append(buffer, column.Decimal)
Expand All @@ -95,13 +95,39 @@ func (column *Info) dump(buffer []byte, d *ResultEncoder, withDefault bool) []by
return buffer
}

func (column *Info) dumpCharset() uint16 {
switch column.Type {
case mysql.TypeTiDBVectorFloat32:
// When passing Vector column to the SQL Client, pretend to be a non-binary String.
return mysql.DefaultCollationID
default:
return column.Charset
}
}

func (column *Info) dumpLength() uint32 {
switch column.Type {
case mysql.TypeTiDBVectorFloat32:
// When passing Vector column to the SQL Client, pretend to be a non-binary String.
// Thus, we use maximum string length here.
// As a downside, there is no way to get the max dimension of the
// vector through binary protocol.
return mysql.MaxLongBlobWidth
default:
return column.ColumnLength
}
}

// DumpFlag dumps flag of a column.
func DumpFlag(tp byte, flag uint16) uint16 {
switch tp {
case mysql.TypeSet:
return flag | uint16(mysql.SetFlag)
case mysql.TypeEnum:
return flag | uint16(mysql.EnumFlag)
case mysql.TypeTiDBVectorFloat32:
// When passing Vector column to the SQL Client, pretend to be a non-binary String.
return flag & ^uint16(mysql.BinaryFlag)
default:
return flag
}
Expand All @@ -111,6 +137,9 @@ func dumpType(tp byte) byte {
switch tp {
case mysql.TypeSet, mysql.TypeEnum:
return mysql.TypeString
case mysql.TypeTiDBVectorFloat32:
// When passing Vector column to the SQL Client, pretend to be a non-binary String.
return mysql.TypeLongBlob
default:
return tp
}
Expand Down
8 changes: 6 additions & 2 deletions pkg/server/internal/column/result_encoder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,11 @@ func (d *ResultEncoder) UpdateDataEncoding(chsID uint16) {
func (d *ResultEncoder) ColumnTypeInfoCharsetID(info *Info) uint16 {
// Only replace the charset when @@character_set_results is valid and
// the target column is a non-binary string.
charset := info.dumpCharset()
if d.isNull || len(d.chsName) == 0 || !isStringColumnType(info.Type) {
return info.Charset
return charset
}
if info.Charset == mysql.BinaryDefaultCollationID {
if charset == mysql.BinaryDefaultCollationID {
return mysql.BinaryDefaultCollationID
}
return uint16(mysql.CharsetNameToID(d.chsName))
Expand Down Expand Up @@ -137,6 +138,9 @@ func isStringColumnType(tp byte) bool {
mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeBlob,
mysql.TypeEnum, mysql.TypeSet, mysql.TypeJSON:
return true
case mysql.TypeTiDBVectorFloat32:
// When passing Vector column to the SQL Client, pretend to be a non-binary String.
return true
}
return false
}
2 changes: 2 additions & 0 deletions pkg/server/tests/commontest/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ go_test(
"@com_github_stretchr_testify//require",
"@com_github_tikv_client_go_v2//tikv",
"@com_github_tikv_client_go_v2//tikvrpc",
"@io_gorm_driver_mysql//:mysql",
"@io_gorm_gorm//:gorm",
"@io_opencensus_go//stats/view",
"@org_uber_go_goleak//:goleak",
],
Expand Down
Loading

0 comments on commit c4a1f18

Please sign in to comment.