Cleanup usage of go.rice in favor of go:embed (vitessio#10956) (vites…

…sio#937)

* Cleanup usage of go.rice in favor of go:embed

The usage of go.rice predates the availability of go:embed, but we
should switch to using go:embed instead to ship specific assets like
config files that we need.

go.rice is also incompatible with Go 1.19 and while it might see a fix
in the future, it seems better to go with the recommended Go approach
that is available these days.

Signed-off-by: Dirkjan Bussink <d.bussink@gmail.com>

* Move vtctld to also use `go embed` instead of go.rice

Signed-off-by: Dirkjan Bussink <d.bussink@gmail.com>

* Remove last rice-box related comments

Signed-off-by: Dirkjan Bussink <d.bussink@gmail.com>

* Remove config moving

This right now breaks building the actual tests since the tests might
also end up loading the regular code which has a `go embed` and refers
to the package with the config embeds.

This doesn't mean that the config isn't properly included in the
binaries. Also with using `go embed` we have a build time dependency on
the files and we always know the latest is included, so we don't have
the issue of potentially outdated files either.

All in all, it seems simplest to remove this logic and trust that Go
itself works as advertised.

Signed-off-by: Dirkjan Bussink <d.bussink@gmail.com>

.gitattributes

@@ -4,4 +4,3 @@ web/vtadmin/src/proto/*.ts -diff
 go/vt/proto/** linguist-generated=true
 go/vt/proto/*/*.pb.go -diff
 go/vt/sqlparser/sql.go linguist-generated=true -diff
-go/vt/mysqlctl/rice-box.go linguist-generated=true -diff

.github/workflows/local_example.yml

@@ -60,6 +60,4 @@ jobs:
  if [ ${{matrix.os}} = "macos-latest" ]; then
  export PATH="/usr/local/opt/mysql@5.7/bin:$PATH"
  fi
- # Make sure that testing is entirely non-reliant on config
- mv config config-moved
  eatmydata -- go run test.go -print-log -follow -retry=1 local_example

Makefile

@@ -47,18 +47,13 @@ endif
 # Safe, since this code isn't performance critical.
 export CGO_CFLAGS := -O1
 
-# regenerate rice-box.go when any of the .cnf files change
-embed_config:
- cd go/vt/mysqlctl && go run github.com/GeertJohan/go.rice/rice embed-go && go build .
-
 # build the vitess binaries with dynamic dependency on libc
 build-dyn:
 ifndef NOBANNER
  echo $$(date): Building source tree
 endif
  bash ./build.env
  go install -trimpath $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/...
- (cd go/cmd/vttablet && go run github.com/GeertJohan/go.rice/rice append --exec=../../../bin/vttablet)
 
 # build the vitess binaries statically
 build:
@@ -69,8 +64,6 @@ endif
  go env -w GOPRIVATE=github.com/planetscale/*
  # build all the binaries by default with CGO disabled
  CGO_ENABLED=0 go install -trimpath $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/...
- # embed local resources in the vttablet executable
- (cd go/cmd/vttablet && go run github.com/GeertJohan/go.rice/rice append --exec=../../../bin/vttablet)
  # build vtorc with CGO, because it depends on sqlite
  CGO_ENABLED=1 go install -trimpath $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/cmd/vtorc/...
 
@@ -86,13 +79,6 @@ endif
  export GOBIN=""
  # For the specified GOOS + GOARCH, build all the binaries by default with CGO disabled
  CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go install -trimpath $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/...
- # unset GOOS and embed local resources in the vttablet executable
- if [ -d /go/bin ]; then
- # Probably in the bootstrap container
- (cd go/cmd/vttablet && go run github.com/GeertJohan/go.rice/rice --verbose append --exec=/go/bin/${GOOS}_${GOARCH}/vttablet)
- else
- (cd go/cmd/vttablet && unset GOOS && unset GOARCH && go run github.com/GeertJohan/go.rice/rice --verbose append --exec=$${HOME}/go/bin/${GOOS}_${GOARCH}/vttablet)
- fi
  # Cross-compiling w/ cgo isn't trivial and we don't need vtorc, so we can skip building it
 
 debug:
@@ -427,7 +413,7 @@ web_bootstrap:
 
 # Do a production build of the vtctld UI.
 # This target needs to be manually run every time any file within web/vtctld2/app
-# is modified to regenerate rice-box.go
+# is modified to regenerate assets.
 web_build: web_bootstrap
  ./tools/web_build.sh
 

config/embed.go

@@ -0,0 +1,30 @@
+package config
+
+import _ "embed"
+
+//go:embed init_db.sql
+var DefaultInitDB string
+
+//go:embed mycnf/default.cnf
+var MycnfDefault string
+
+//go:embed mycnf/mariadb100.cnf
+var MycnfMariaDB100 string
+
+//go:embed mycnf/mariadb101.cnf
+var MycnfMariaDB101 string
+
+//go:embed mycnf/mariadb102.cnf
+var MycnfMariaDB102 string
+
+//go:embed mycnf/mariadb103.cnf
+var MycnfMariaDB103 string
+
+//go:embed mycnf/mariadb104.cnf
+var MycnfMariaDB104 string
+
+//go:embed mycnf/mysql57.cnf
+var MycnfMySQL57 string
+
+//go:embed mycnf/mysql80.cnf
+var MycnfMySQL80 string

go.mod

@@ -8,7 +8,6 @@ require (
  github.com/Azure/azure-pipeline-go v0.2.2
  github.com/Azure/azure-storage-blob-go v0.10.0
  github.com/DataDog/datadog-go v2.2.0+incompatible
- github.com/GeertJohan/go.rice v1.0.0
  github.com/HdrHistogram/hdrhistogram-go v0.9.0 // indirect
  github.com/PuerkitoBio/goquery v1.5.1
  github.com/aquarapid/vaultlib v0.5.1
@@ -134,15 +133,12 @@ require (
 require (
  cloud.google.com/go v0.81.0 // indirect
  github.com/BurntSushi/toml v0.3.1 // indirect
- github.com/GeertJohan/go.incremental v1.0.0 // indirect
- github.com/akavel/rsrc v0.8.0 // indirect
  github.com/andybalholm/cascadia v1.1.0 // indirect
  github.com/beorn7/perks v1.0.1 // indirect
  github.com/blang/semver v3.5.1+incompatible // indirect
  github.com/coreos/go-semver v0.3.0 // indirect
  github.com/coreos/go-systemd/v22 v22.3.2 // indirect
  github.com/cyphar/filepath-securejoin v0.2.3 // indirect
- github.com/daaku/go.zipexe v1.0.0 // indirect
  github.com/davecgh/go-spew v1.1.1 // indirect
  github.com/fatih/color v1.9.0 // indirect
  github.com/felixge/httpsnoop v1.0.1 // indirect
@@ -157,7 +153,6 @@ require (
  github.com/hashicorp/golang-lru v0.5.1 // indirect
  github.com/hashicorp/hcl v1.0.0 // indirect
  github.com/inconshreveable/mousetrap v1.0.0 // indirect
- github.com/jessevdk/go-flags v1.4.0 // indirect
  github.com/json-iterator/go v1.1.11 // indirect
  github.com/jstemmer/go-junit-report v0.9.1 // indirect
  github.com/mattn/go-colorable v0.1.6 // indirect
@@ -169,7 +164,6 @@ require (
  github.com/mitchellh/mapstructure v1.4.1 // indirect
  github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
  github.com/modern-go/reflect2 v1.0.1 // indirect
- github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 // indirect
  github.com/onsi/ginkgo v1.12.1 // indirect
  github.com/onsi/gomega v1.10.3 // indirect
  github.com/pelletier/go-toml v1.9.3 // indirect
@@ -184,8 +178,6 @@ require (
  github.com/subosito/gotenv v1.2.0 // indirect
  github.com/tidwall/match v1.1.1 // indirect
  github.com/tidwall/pretty v1.2.0 // indirect
- github.com/valyala/bytebufferpool v1.0.0 // indirect
- github.com/valyala/fasttemplate v1.0.1 // indirect
  github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
  github.com/xdg/stringprep v1.0.0 // indirect
  go.opencensus.io v0.23.0 // indirect

go.sum

@@ -79,10 +79,6 @@ github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e h1:4ZrkT/RzpnRO
 github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e/go.mod h1:uw9h2sd4WWHOPdJ13MQpwK5qYWKYDumDqxWWIknEQ+k=
 github.com/DataDog/datadog-go v2.2.0+incompatible h1:V5BKkxACZLjzHjSgBbr2gvLA2Ae49yhc6CSY7MLy5k4=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
-github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg=
-github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
-github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ=
-github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
 github.com/HdrHistogram/hdrhistogram-go v0.9.0 h1:dpujRju0R4M/QZzcnR1LH1qm+TVG3UzkWdp5tH1WMcg=
 github.com/HdrHistogram/hdrhistogram-go v0.9.0/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4=
 github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
@@ -98,8 +94,6 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
-github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw=
-github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -181,8 +175,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY=
-github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E=
 github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw=
 github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -470,8 +462,6 @@ github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -585,8 +575,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
-github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg=
-github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
@@ -773,10 +761,6 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK
 github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
-github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=

go/cmd/vttablet/vttablet.go

@@ -24,8 +24,7 @@ import (
  "os"
 
  "vitess.io/vitess/go/vt/vttablet/tabletmanager/vdiff"
-
- rice "github.com/GeertJohan/go.rice"
+ "vitess.io/vitess/resources"
 
  "vitess.io/vitess/go/vt/binlog"
  "vitess.io/vitess/go/vt/dbconfigs"
@@ -171,24 +170,14 @@ func initConfig(tabletAlias *topodatapb.TabletAlias) (*tabletenv.TabletConfig, *
 }
 
 // extractOnlineDDL extracts the gh-ost binary from this executable. gh-ost is appended
-// to vttablet executable by `make build` and via ricebox
+// to vttablet executable by `make build` with a go:embed
 func extractOnlineDDL() error {
  if binaryFileName, isOverride := onlineddl.GhostBinaryFileName(); !isOverride {
- riceBox, err := rice.FindBox("../../../resources/bin")
- if err != nil {
- return err
- }
-
- // there is no path override for gh-ost. We're expected to auto-extract gh-ost.
- ghostBinary, err := riceBox.Bytes("gh-ost")
- if err != nil {
- return err
- }
- if err := os.WriteFile(binaryFileName, ghostBinary, 0755); err != nil {
+ if err := os.WriteFile(binaryFileName, resources.GhostBinary, 0755); err != nil {
  // One possibility of failure is that gh-ost is up and running. In that case,
  // let's pause and check if the running gh-ost is exact same binary as the one we wish to extract.
  foundBytes, _ := os.ReadFile(binaryFileName)
- if bytes.Equal(ghostBinary, foundBytes) {
+ if bytes.Equal(resources.GhostBinary, foundBytes) {
  // OK, it's the same binary, there is no need to extract the file anyway
  return nil
  }

go/vt/mysqlctl/mysqld.go

@@ -41,7 +41,7 @@ import (
  "sync"
  "time"
 
- rice "github.com/GeertJohan/go.rice"
+ "vitess.io/vitess/config"
 
  "vitess.io/vitess/go/mysql"
  "vitess.io/vitess/go/vt/dbconfigs"
@@ -670,12 +670,7 @@ func (mysqld *Mysqld) Init(ctx context.Context, cnf *Mycnf, initDBSQLFile string
  }
 
  if initDBSQLFile == "" { // default to built-in
- riceBox := rice.MustFindBox("../../../config")
- sqlFile, err := riceBox.Open("init_db.sql")
- if err != nil {
- return fmt.Errorf("could not open built-in init_db.sql file")
- }
- if err := mysqld.executeMysqlScript(params, sqlFile); err != nil {
+ if err := mysqld.executeMysqlScript(params, strings.NewReader(config.DefaultInitDB)); err != nil {
  return fmt.Errorf("failed to initialize mysqld: %v", err)
  }
  return nil
@@ -806,26 +801,47 @@ func (mysqld *Mysqld) getMycnfTemplate() string {
  }
  myTemplateSource := new(bytes.Buffer)
  myTemplateSource.WriteString("[mysqld]\n")
-
- riceBox := rice.MustFindBox("../../../config")
- b, err := riceBox.Bytes("mycnf/default.cnf")
- if err != nil {
- log.Warningf("could not open embedded default.cnf config file")
- }
- myTemplateSource.Write(b)
+ myTemplateSource.WriteString(config.MycnfDefault)
 
  // database flavor + version specific file.
  // {flavor}{major}{minor}.cnf
  f := FlavorMariaDB
  if mysqld.capabilities.isMySQLLike() {
  f = FlavorMySQL
  }
- fn := fmt.Sprintf("mycnf/%s%d%d.cnf", f, mysqld.capabilities.version.Major, mysqld.capabilities.version.Minor)
- b, err = riceBox.Bytes(fn)
- if err != nil {
- log.Infof("this version of Vitess does not include built-in support for %v %v", mysqld.capabilities.flavor, mysqld.capabilities.version)
+ var versionConfig string
+ switch f {
+ case FlavorPercona, FlavorMySQL:
+ switch mysqld.capabilities.version.Major {
+ case 5:
+ if mysqld.capabilities.version.Minor == 7 {
+ versionConfig = config.MycnfMySQL57
+ } else {
+ log.Infof("this version of Vitess does not include built-in support for %v %v", mysqld.capabilities.flavor, mysqld.capabilities.version)
+ }
+ case 8:
+ versionConfig = config.MycnfMySQL80
+ default:
+ log.Infof("this version of Vitess does not include built-in support for %v %v", mysqld.capabilities.flavor, mysqld.capabilities.version)
+ }
+ case FlavorMariaDB:
+ switch mysqld.capabilities.version.Minor {
+ case 0:
+ versionConfig = config.MycnfMariaDB100
+ case 1:
+ versionConfig = config.MycnfMariaDB101
+ case 2:
+ versionConfig = config.MycnfMariaDB102
+ case 3:
+ versionConfig = config.MycnfMariaDB103
+ case 4:
+ versionConfig = config.MycnfMariaDB104
+ default:
+ log.Infof("this version of Vitess does not include built-in support for %v %v", mysqld.capabilities.flavor, mysqld.capabilities.version)
+ }
  }
- myTemplateSource.Write(b)
+
+ myTemplateSource.WriteString(versionConfig)
 
  if extraCnf := os.Getenv("EXTRA_MY_CNF"); extraCnf != "" {
  parts := strings.Split(extraCnf, ":")