Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add binlog deploy and check process #329

Merged
merged 16 commits into from
Mar 20, 2019
Merged

add binlog deploy and check process #329

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
a7e16bf
add schduler test case
shuijing198799 Mar 15, 2019
2746c82
delete unnessary step and add scheduled-backup-job check step
shuijing198799 Mar 18, 2019
79dfe9a
change name to backupPVC
shuijing198799 Mar 19, 2019
3dbddde
return a string instead of int in getbackupdir
shuijing198799 Mar 19, 2019
2eb505d
conflict resovle
shuijing198799 Mar 19, 2019
84d4698
resolve some merge error and naming conflict
shuijing198799 Mar 19, 2019
aa70d18
resolve some merge error and naming conflict
shuijing198799 Mar 19, 2019
43ba7c7
add binlog process
shuijing198799 Mar 19, 2019
085d0bf
conflict resolve
shuijing198799 Mar 19, 2019
5d6a0ed
add setting to helmsetstring and add create secret
shuijing198799 Mar 19, 2019
994c9ae
fullfil the comments
shuijing198799 Mar 20, 2019
8c924f8
Merge branch 'stability' into yinliang/backup-and-restore
shuijing198799 Mar 20, 2019
9ec00d4
Merge branch 'stability' into yinliang/backup-and-restore
shuijing198799 Mar 20, 2019
c2a14ef
prewrite secret name
shuijing198799 Mar 20, 2019
ac39382
Merge branch 'yinliang/backup-and-restore' of https://github.com/shui…
shuijing198799 Mar 20, 2019
51ed069
change password method
shuijing198799 Mar 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 51 additions & 63 deletions tests/actions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
package tests

import (
"bytes"
"database/sql"
"encoding/json"
"fmt"
Expand Down Expand Up @@ -92,7 +91,7 @@ type OperatorActions interface {
Restore(from *TidbClusterInfo, to *TidbClusterInfo) error
CheckRestore(from *TidbClusterInfo, to *TidbClusterInfo) error
ForceDeploy(info *TidbClusterInfo) error
CreateSecret(info *TidbClusterInfo) error
CreateSecret(info *TidbClusterInfo) (string, string, error)
shuijing198799 marked this conversation as resolved.
Show resolved Hide resolved
getBackupDir(info *TidbClusterInfo) ([]string, error)
}

Expand Down Expand Up @@ -152,9 +151,12 @@ type TidbClusterInfo struct {
Args map[string]string
blockWriter *blockwriter.BlockWriterCase
Monitor bool
UserName string
InitSecretName string
BackupSecretName string
}

func (tc *TidbClusterInfo) HelmSetString() string {
func (tc *TidbClusterInfo) HelmSetString(m map[string]string) string {

set := map[string]string{
"clusterName": tc.ClusterName,
Expand All @@ -166,9 +168,10 @@ func (tc *TidbClusterInfo) HelmSetString() string {
"pd.image": tc.PDImage,
"tikv.image": tc.TiKVImage,
"tidb.image": tc.TiDBImage,
"tidb.passwordSecretName": "set-secret",
"tidb.passwordSecretName": tc.InitSecretName,
"tidb.initSql": tc.InitSql,
"monitor.create": strconv.FormatBool(tc.Monitor),
"secretName": tc.BackupSecretName,
}

for k, v := range tc.Resources {
Expand All @@ -177,6 +180,9 @@ func (tc *TidbClusterInfo) HelmSetString() string {
for k, v := range tc.Args {
set[k] = v
}
for k, v := range m {
set[k] = v
}

arr := make([]string, 0, len(set))
for k, v := range set {
Expand Down Expand Up @@ -245,8 +251,16 @@ func (oa *operatorActions) DeployTidbCluster(info *TidbClusterInfo) error {
defer func() {
glog.Infof("deploy tidb cluster end cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
}()

initSecretName, backupSecretName, err := oa.CreateSecret(info)
if err != nil {
return fmt.Errorf("failed to create secret of cluster [%s]: %v", info.ClusterName, err)
}

info.InitSecretName, info.BackupSecretName = initSecretName, backupSecretName

cmd := fmt.Sprintf("helm install /charts/%s/tidb-cluster --name %s --namespace %s --set-string %s",
info.OperatorTag, info.ClusterName, info.Namespace, info.HelmSetString())
info.OperatorTag, info.ClusterName, info.Namespace, info.HelmSetString(map[string]string{}))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

map[string]string{} -> nil

if res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil {
return fmt.Errorf("failed to deploy tidbcluster: %s/%s, %v, %s",
info.Namespace, info.ClusterName, err, string(res))
Expand Down Expand Up @@ -414,7 +428,7 @@ func chartPath(name string, tag string) string {

func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error {
cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s",
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString())
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString(map[string]string{}))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto

glog.Info("[SCALE] " + cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand All @@ -425,7 +439,7 @@ func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error {

func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterInfo) error {
cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s",
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString())
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString(map[string]string{}))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto

glog.Info("[UPGRADE] " + cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand Down Expand Up @@ -1058,26 +1072,18 @@ func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterInfo) error {
glog.Infof("deploy adhoc backup end cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
}()
sets := map[string]string{
"clusterName": info.ClusterName,
"name": info.BackupPVC,
"mode": "backup",
"user": "root",
"password": info.Password,
"storage.size": "10Gi",
}
var buffer bytes.Buffer
for k, v := range sets {
set := fmt.Sprintf(" --set %s=%s", k, v)
_, err := buffer.WriteString(set)
if err != nil {
return err
}
}

setStr := buffer.String()
setString := info.HelmSetString(sets)

fullbackupName := fmt.Sprintf("%s-backup", info.ClusterName)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup %s",
fullbackupName, info.Namespace, info.OperatorTag, setStr)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup --set-string %s",
fullbackupName, info.Namespace, info.OperatorTag, setString)
glog.Infof("install adhoc deployment [%s]", cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand Down Expand Up @@ -1122,26 +1128,18 @@ func (oa *operatorActions) Restore(from *TidbClusterInfo, to *TidbClusterInfo) e
glog.Infof("deploy restore end cluster[%s] namespace[%s]", to.ClusterName, to.Namespace)
}()
sets := map[string]string{
"clusterName": to.ClusterName,
"name": to.BackupPVC,
"mode": "restore",
"user": "root",
"password": to.Password,
"storage.size": "10Gi",
}
var buffer bytes.Buffer
for k, v := range sets {
set := fmt.Sprintf(" --set %s=%s", k, v)
_, err := buffer.WriteString(set)
if err != nil {
return err
}
}

setStr := buffer.String()
setString := to.HelmSetString(sets)

restoreName := fmt.Sprintf("%s-restore", from.ClusterName)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup %s",
restoreName, to.Namespace, to.OperatorTag, setStr)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup --set-string %s",
restoreName, to.Namespace, to.OperatorTag, setString)
glog.Infof("install restore [%s]", cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand Down Expand Up @@ -1233,23 +1231,23 @@ func (info *TidbClusterInfo) QueryCount() (int, error) {
return 0, fmt.Errorf("can not find count of ")
}

func (oa *operatorActions) CreateSecret(info *TidbClusterInfo) error {
initSecretName := "set-secret"
backupSecretName := "backup-secret"
func (oa *operatorActions) CreateSecret(info *TidbClusterInfo) (string, string, error) {
initSecretName := fmt.Sprintf("%s-set-secret", info.ClusterName)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the field should be set when info created

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

backupSecretName := fmt.Sprintf("%s-backup-secret", info.ClusterName)
initSecret := corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: initSecretName,
Namespace: info.Namespace,
},
Data: map[string][]byte{
"root": []byte(info.Password),
info.UserName: []byte(info.Password),
},
Type: corev1.SecretTypeOpaque,
}

_, err := oa.kubeCli.CoreV1().Secrets(info.Namespace).Create(&initSecret)
if err != nil && !releaseIsExist(err) {
return err
return "", "", err
}

backupSecret := corev1.Secret{
Expand All @@ -1258,18 +1256,18 @@ func (oa *operatorActions) CreateSecret(info *TidbClusterInfo) error {
Namespace: info.Namespace,
},
Data: map[string][]byte{
"user": []byte("root"),
"user": []byte(info.UserName),
"password": []byte(info.Password),
},
Type: corev1.SecretTypeOpaque,
}

_, err = oa.kubeCli.CoreV1().Secrets(info.Namespace).Create(&backupSecret)
if err != nil && !releaseIsExist(err) {
return err
return "", "", err
}

return nil
return initSecretName, backupSecretName, nil
}

func releaseIsExist(err error) bool {
Expand All @@ -1291,19 +1289,11 @@ func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterInfo) error {
"scheduledBackup.schedule": cron,
"scheduledBackup.storage": "10Gi",
}
var buffer bytes.Buffer
for k, v := range sets {
set := fmt.Sprintf(" --set %s=%s", k, v)
_, err := buffer.WriteString(set)
if err != nil {
return err
}
}

setStr := buffer.String()
setString := info.HelmSetString(sets)

cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster %s",
info.ClusterName, info.OperatorTag, setStr)
cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster --set-string %s",
info.ClusterName, info.OperatorTag, setString)

glog.Infof("scheduled-backup delploy [%s]", cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
Expand Down Expand Up @@ -1458,13 +1448,19 @@ func (oa *operatorActions) getBackupDir(info *TidbClusterInfo) ([]string, error)
}

cmd := fmt.Sprintf("kubectl exec %s -n %s ls /data", getBackupDirPodName, info.Namespace)
glog.Infof(cmd)

time.Sleep(20 * time.Second)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we poll the following condition so we can proceed on earlier if condition meet?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I try to poll at first but the time is hard to controller in one minute. for every minute generate a cronjob to backup. At last we have to check the backup number.


res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
glog.Errorf("cluster:[%s/%s] exec :%s failed,error:%v,result:%s", info.Namespace, info.ClusterName, cmd, err, res)
glog.Errorf("cluster:[%s/%s] exec :%s failed,error:%v,result:%s", info.Namespace, info.ClusterName, cmd, err, string(res))
return nil, err
}

if err != nil {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unreachable

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems you are right

return nil, fmt.Errorf("failed to get dir via cmd [%s]", cmd)
}

dirs := strings.Split(string(res), "\n")
glog.Infof("dirs in pod info name [%s] dir name [%s]", info.BackupPVC, strings.Join(dirs, ","))
return dirs, nil
Expand All @@ -1489,18 +1485,10 @@ func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterInfo, to *Ti
"binlog.drainer.mysql.port": "4000",
}

var buffer bytes.Buffer
for k, v := range sets {
set := fmt.Sprintf(" --set %s=%s", k, v)
_, err := buffer.WriteString(set)
if err != nil {
return err
}
}
setString := from.HelmSetString(sets)

setStr := buffer.String()
cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster %s",
from.ClusterName, from.OperatorTag, setStr)
cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster --set-string %s",
from.ClusterName, from.OperatorTag, setString)
glog.Infof(cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand Down
2 changes: 2 additions & 0 deletions tests/cmd/e2e/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ func main() {
StorageClassName: "local-storage",
Password: "admin",
InitSql: initSql,
UserName: "root",
Resources: map[string]string{
"pd.resources.limits.cpu": "1000m",
"pd.resources.limits.memory": "2Gi",
Expand Down Expand Up @@ -129,6 +130,7 @@ func main() {
StorageClassName: "local-storage",
Password: "admin",
InitSql: initSql,
UserName: "root",
Resources: map[string]string{
"pd.resources.limits.cpu": "1000m",
"pd.resources.limits.memory": "2Gi",
Expand Down