Skip to content

Commit

Permalink
Support deploy TiFlash on multi-disks with "storage" configurations s…
Browse files Browse the repository at this point in the history
…ince v4.0.9 (#931)
  • Loading branch information
JaySon-Huang authored Nov 25, 2020
1 parent c989f9d commit 1f0cbac
Show file tree
Hide file tree
Showing 9 changed files with 731 additions and 64 deletions.
57 changes: 31 additions & 26 deletions examples/topology.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -418,8 +418,10 @@ server_configs:
# - key: "dc"
# value: "sha"
tiflash:
# path_realtime_mode: false
logger.level: "info"
## Deprecated multi-disks storage path setting style since v4.0.9,
## check storage.* configurations in host level for new style.
# path_realtime_mode: false
tiflash-learner:
log-level: "info"
# raftstore.apply-pool-size: 4
Expand Down Expand Up @@ -488,37 +490,40 @@ tiflash_servers:
# flash_proxy_status_port: 20292
# metrics_port: 8234
# deploy_dir: /tidb-deploy/tiflash-9000
## With cluster version >= v4.0.9 and you want to deploy a multi-disk TiFlash node, it is recommended to
## check config.storage.* for details. The data_dir will be ignored if you defined those configurations.
## Setting data_dir to a ','-joined string is still supported but deprecated.
## Check https://docs.pingcap.com/tidb/stable/tiflash-configuration#multi-disk-deployment for more details.
# data_dir: /tidb-data/tiflash-9000
# log_dir: /tidb-deploy/tiflash-9000/log
# numa_node: "0,1"
# #
# # `path_realtime_mode` only works if `data_dir` is specified with multiple paths.
# #
# # path_realtime_mode:
# # "true" means only other paths instead of first path can be used to store older data.
# # "false" means all paths can be used to store older data.
# #
# # TiFlash only uses the first path to store the latest data (i.e. "delta"). And others for the older data (i.e. "stable". which is the majority of data),
# #
# # E.g, if you intend to use an fast and smaller NVMe SSD (256GB) to speed up data ingestion speed in TiFlash,
# # and 4 extra normal SSDs (512GB each) for main storage. Then your configurations should be look like:
# #
# # data_dir: /nvme_ssd_256/data,/ssd1_512/data,/ssd2_512/data,/ssd3_512/data,/ssd4_512/data
# # config:
# # path_realtime_mode: true
# #
# #
# # And if your first disk is big enough, to fully use the capacity of it, use configurations look like:
# #
# # data_dir: /nvme_ssd_512/data,/ssd1_512/data,/ssd2_512/data,/ssd3_512/data,/ssd4_512/data
# # config:
# # path_realtime_mode: false
# #
# #
# # The following configs are used to overwrite the `server_configs.tiflash` values.
# config:
# path_realtime_mode: false
# logger.level: "info"
# ## Deprecated multi-disks storage path setting. Deprecated since v4.0.9,
# ## check following storage.* configurations for new style.
# # path_realtime_mode: false
# #
# ## Multi-disks storage paths settings. These will overwrite the `data_dir`.
# ## If there are multiple SSD disks on the machine,
# ## specify the path list on `storage.main.dir` can improve TiFlash performance.
# # storage.main.dir: [ "/nvme_ssd0_512/tiflash", "/nvme_ssd1_512/tiflash" ]
# ## Store capacity of each path, i.e. max data size allowed.
# ## If it is not set, or is set to 0s, the actual disk capacity is used.
# # storage.main.capacity = [ 536870912000, 536870912000 ]
# #
# ## If there are multiple disks with different IO metrics (e.g. one SSD and some HDDs)
# ## on the machine,
# ## set `storage.latest.dir` to store the latest data on SSD (disks with higher IOPS metrics)
# ## set `storage.main.dir` to store the main data on HDD (disks with lower IOPS metrics)
# ## can improve TiFlash performance.
# # storage.main.dir: [ "/hdd0_512/tiflash", "/hdd1_512/tiflash", "/hdd2_512/tiflash" ]
# # storage.latest.dir: [ "/nvme_ssd0_100/tiflash" ]
# ## Store capacity of each path, i.e. max data size allowed.
# ## If it is not set, or is set to 0s, the actual disk capacity is used.
# # storage.main.capacity = [ 536870912000, 536870912000 ]
# # storage.latest.capacity = [ 107374182400 ]
#
# # The following configs are used to overwrite the `server_configs.tiflash-learner` values.
# learner_config:
# log-level: "info"
Expand Down
144 changes: 144 additions & 0 deletions pkg/cluster/spec/parse_topology_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,150 @@ tikv_servers:
})
}

func (s *topoSuite) TestTiFlashStorage(c *check.C) {
// test tiflash storage section, 'storage.main.dir' should not be defined in server_configs
withTempFile(`
server_configs:
tiflash:
storage.main.dir: [/data1/tiflash]
tiflash_servers:
- host: 172.16.5.140
data_dir: /ssd0/tiflash,/ssd1/tiflash,/ssd2/tiflash
config:
storage.main.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash]
storage.latest.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash]
`, func(file string) {
topo := Specification{}
err := ParseTopologyYaml(file, &topo)
c.Assert(err, check.NotNil)
})

// test tiflash storage section, 'storage.latest.dir' should not be defined in server_configs
withTempFile(`
server_configs:
tiflash:
storage.latest.dir: [/data1/tiflash]
tiflash_servers:
- host: 172.16.5.140
data_dir: /ssd0/tiflash,/ssd1/tiflash,/ssd2/tiflash
config:
storage.main.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash]
storage.latest.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash]
`, func(file string) {
topo := Specification{}
err := ParseTopologyYaml(file, &topo)
c.Assert(err, check.NotNil)
})

// test tiflash storage section defined data dir
// test for depreacated setting, for backward compatibility
withTempFile(`
tiflash_servers:
- host: 172.16.5.140
data_dir: /ssd0/tiflash
config:
`, func(file string) {
topo := Specification{}
err := ParseTopologyYaml(file, &topo)
c.Assert(err, check.IsNil)
ExpandRelativeDir(&topo)

c.Assert(topo.TiFlashServers[0].DeployDir, check.Equals, "/home/tidb/deploy/tiflash-9000")
c.Assert(topo.TiFlashServers[0].DataDir, check.Equals, "/ssd0/tiflash")
c.Assert(topo.TiFlashServers[0].LogDir, check.Equals, "")
})

// test tiflash storage section defined data dir
withTempFile(`
tiflash_servers:
- host: 172.16.5.140
data_dir: /ssd0/tiflash,/ssd1/tiflash,/ssd2/tiflash
config:
storage.main.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash]
storage.latest.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash]
`, func(file string) {
topo := Specification{}
err := ParseTopologyYaml(file, &topo)
c.Assert(err, check.IsNil)
ExpandRelativeDir(&topo)

c.Assert(topo.TiFlashServers[0].DeployDir, check.Equals, "/home/tidb/deploy/tiflash-9000")
c.Assert(topo.TiFlashServers[0].DataDir, check.Equals, "/ssd0/tiflash,/ssd1/tiflash,/ssd2/tiflash")
c.Assert(topo.TiFlashServers[0].LogDir, check.Equals, "")
})

// test tiflash storage section defined data dir, "data_dir" will be ignored
withTempFile(`
tiflash_servers:
- host: 172.16.5.140
# if storage.main.dir is defined, data_dir will be ignored
data_dir: /hdd0/tiflash
config:
storage.main.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash]
`, func(file string) {
topo := Specification{}
err := ParseTopologyYaml(file, &topo)
c.Assert(err, check.IsNil)
ExpandRelativeDir(&topo)

c.Assert(topo.TiFlashServers[0].DeployDir, check.Equals, "/home/tidb/deploy/tiflash-9000")
c.Assert(topo.TiFlashServers[0].DataDir, check.Equals, "/ssd0/tiflash,/ssd1/tiflash,/ssd2/tiflash")
c.Assert(topo.TiFlashServers[0].LogDir, check.Equals, "")
})

// test tiflash storage section defined data dir
// if storage.latest.dir is not empty, the first path in
// storage.latest.dir will be the first path in 'DataDir'
// DataDir is the union set of storage.latest.dir and storage.main.dir
withTempFile(`
tiflash_servers:
- host: 172.16.5.140
data_dir: /ssd0/tiflash
config:
storage.main.dir: [/hdd0/tiflash, /hdd1/tiflash, /hdd2/tiflash]
storage.latest.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash, /hdd0/tiflash]
`, func(file string) {
topo := Specification{}
err := ParseTopologyYaml(file, &topo)
c.Assert(err, check.IsNil)
ExpandRelativeDir(&topo)

c.Assert(topo.TiFlashServers[0].DeployDir, check.Equals, "/home/tidb/deploy/tiflash-9000")
c.Assert(topo.TiFlashServers[0].DataDir, check.Equals, "/ssd0/tiflash,/hdd0/tiflash,/hdd1/tiflash,/hdd2/tiflash,/ssd1/tiflash,/ssd2/tiflash")
c.Assert(topo.TiFlashServers[0].LogDir, check.Equals, "")
})

// test tiflash storage section defined data dir
// should always define storage.main.dir if any 'storage.*' is defined
withTempFile(`
tiflash_servers:
- host: 172.16.5.140
data_dir: /ssd0/tiflash
config:
#storage.main.dir: [/hdd0/tiflash, /hdd1/tiflash, /hdd2/tiflash]
storage.latest.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash, /hdd0/tiflash]
`, func(file string) {
topo := Specification{}
err := ParseTopologyYaml(file, &topo)
c.Assert(err, check.NotNil)
})

// test tiflash storage section defined data dir
// storage.main.dir should always use absolute path
withTempFile(`
tiflash_servers:
- host: 172.16.5.140
data_dir: /ssd0/tiflash
config:
storage.main.dir: [tiflash/data, ]
storage.latest.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash, /hdd0/tiflash]
`, func(file string) {
topo := Specification{}
err := ParseTopologyYaml(file, &topo)
c.Assert(err, check.NotNil)
})
}

func merge4test(base, scale string) (*Specification, error) {
baseTopo := Specification{}
if err := ParseTopologyYaml(base, &baseTopo); err != nil {
Expand Down
14 changes: 14 additions & 0 deletions pkg/cluster/spec/spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/tiup/pkg/cluster/executor"
"github.com/pingcap/tiup/pkg/cluster/template/scripts"
"github.com/pingcap/tiup/pkg/logger/log"
"github.com/pingcap/tiup/pkg/meta"
"go.etcd.io/etcd/clientv3"
)
Expand Down Expand Up @@ -309,6 +310,19 @@ func (s *Specification) UnmarshalYAML(unmarshal func(interface{}) error) error {
return err
}

// Rewrite TiFlashSpec.DataDir since we may override it with configurations.
// Should do it before validatation because we need to detect dir conflicts.
for i := 0; i < len(s.TiFlashServers); i++ {
dataDir, err := s.TiFlashServers[i].GetOverrideDataDir()
if err != nil {
return err
}
if s.TiFlashServers[i].DataDir != dataDir {
log.Infof("'tiflash_server:%s.data_dir' is overwritten by its storage configuration. Now the data_dir is %s", s.TiFlashServers[i].Host, dataDir)
s.TiFlashServers[i].DataDir = dataDir
}
}

return s.Validate()
}

Expand Down
126 changes: 126 additions & 0 deletions pkg/cluster/spec/spec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (

"github.com/BurntSushi/toml"
. "github.com/pingcap/check"
"github.com/pingcap/tiup/pkg/cluster/template/scripts"
"gopkg.in/yaml.v2"
)

Expand Down Expand Up @@ -557,3 +558,128 @@ pd_servers:
_, err = spec.LocationLabels()
c.Assert(err, NotNil)
}

func (s *metaSuiteTopo) TestTiFlashStorageSection(c *C) {
spec := &Specification{}
err := yaml.Unmarshal([]byte(`
tiflash_servers:
- host: 172.16.5.138
data_dir: /hdd0/tiflash,/hdd1/tiflash
config:
storage.main.dir: [/ssd0/tiflash, /ssd1/tiflash]
storage.latest.dir: [/ssd0/tiflash]
`), spec)
c.Assert(err, IsNil)

flashComp := FindComponent(spec, ComponentTiFlash)
instances := flashComp.Instances()
c.Assert(len(instances), Equals, 1)
// parse using clusterVersion<"v4.0.9"
{
ins := instances[0]
// This should be the same with tiflash_server instance's "data_dir"
dataDir := "/hdd0/tiflash,/hdd1/tiflash"
cfg := scripts.NewTiFlashScript(ins.GetHost(), "", dataDir, "", "", "")
conf, err := ins.(*TiFlashInstance).initTiFlashConfig(cfg, "v4.0.8", spec.ServerConfigs.TiFlash)
c.Assert(err, IsNil)

path, ok := conf["path"]
c.Assert(ok, IsTrue)
c.Assert(path, Equals, dataDir)
}
// parse using clusterVersion>="v4.0.9"
checkWithVersion := func(ver string) {
ins := instances[0].(*TiFlashInstance)
dataDir := "/ssd0/tiflash"
cfg := scripts.NewTiFlashScript(ins.GetHost(), "", dataDir, "", "", "")
conf, err := ins.initTiFlashConfig(cfg, ver, spec.ServerConfigs.TiFlash)
c.Assert(err, IsNil)

_, ok := conf["path"]
c.Assert(ok, IsTrue)

// After merging instance configurations with "storgae", the "path" property should be removed.
conf, err = ins.mergeTiFlashInstanceConfig(ver, conf, ins.InstanceSpec.(TiFlashSpec).Config)
c.Assert(err, IsNil)
_, ok = conf["path"]
c.Assert(ok, IsFalse)

if storageSection, ok := conf["storage"]; ok {
if mainSection, ok := storageSection.(map[string]interface{})["main"]; ok {
if mainDirsSection, ok := mainSection.(map[string]interface{})["dir"]; ok {
var mainDirs []interface{} = mainDirsSection.([]interface{})
c.Assert(len(mainDirs), Equals, 2)
c.Assert(mainDirs[0].(string), Equals, "/ssd0/tiflash")
c.Assert(mainDirs[1].(string), Equals, "/ssd1/tiflash")
} else {
c.Error("Can not get storage.main.dir section")
}
} else {
c.Error("Can not get storage.main section")
}
if latestSection, ok := storageSection.(map[string]interface{})["latest"]; ok {
if latestDirsSection, ok := latestSection.(map[string]interface{})["dir"]; ok {
var latestDirs []interface{} = latestDirsSection.([]interface{})
c.Assert(len(latestDirs), Equals, 1)
c.Assert(latestDirs[0].(string), Equals, "/ssd0/tiflash")
} else {
c.Error("Can not get storage.main.dir section")
}

} else {
c.Error("Can not get storage.main section")
}
} else {
c.Error("Can not get storage section")
}
}
checkWithVersion("v4.0.9")
checkWithVersion("nightly")
}

func (s *metaSuiteTopo) TestTiFlashInvalidStorageSection(c *C) {
spec := &Specification{}

testCases := [][]byte{
[]byte(`
tiflash_servers:
- host: 172.16.5.138
data_dir: /hdd0/tiflash,/hdd1/tiflash
config:
# storage.main.dir is not defined
storage.latest.dir: ["/ssd0/tiflash"]
`),
[]byte(`
tiflash_servers:
- host: 172.16.5.138
data_dir: /hdd0/tiflash,/hdd1/tiflash
config:
# storage.main.dir is empty string array
storage.main.dir: []
storage.latest.dir: ["/ssd0/tiflash"]
`),
[]byte(`
tiflash_servers:
- host: 172.16.5.138
data_dir: /hdd0/tiflash,/hdd1/tiflash
config:
# storage.main.dir is not a string array
storage.main.dir: /hdd0/tiflash,/hdd1/tiflash
storage.latest.dir: ["/ssd0/tiflash"]
`),
[]byte(`
tiflash_servers:
- host: 172.16.5.138
data_dir: /hdd0/tiflash,/hdd1/tiflash
config:
# storage.main.dir is not a string array
storage.main.dir: [0, 1]
storage.latest.dir: ["/ssd0/tiflash"]
`),
}

for _, testCase := range testCases {
err := yaml.Unmarshal([]byte(testCase), spec)
c.Check(err, NotNil)
}
}
Loading

0 comments on commit 1f0cbac

Please sign in to comment.