Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tidb stability test main function #306

Merged
merged 14 commits into from
Mar 12, 2019
Merged

tidb stability test main function #306

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
4e9e00b
add ignore file and add bakcup test to the stability branch
shuijing198799 Mar 8, 2019
ae77f56
change parameter via the new yaml and value
shuijing198799 Mar 8, 2019
f73a3f5
scheduler yaml still has some argument so I need test backup and rest…
shuijing198799 Mar 8, 2019
3ad827d
fix some bug and make the flow successful
shuijing198799 Mar 10, 2019
654ee35
fix some bug and make the flow successful
shuijing198799 Mar 10, 2019
7462197
fix some bug and avoid some path
shuijing198799 Mar 11, 2019
e33d64b
delete commend line
shuijing198799 Mar 11, 2019
fb7d9cb
delete comment and get yaml files
shuijing198799 Mar 11, 2019
08dd687
delete some unready file and delete double dash and fix some bug intr…
shuijing198799 Mar 12, 2019
940d3dc
ignore a l in command
shuijing198799 Mar 12, 2019
2f66d82
add label to string function and used in action
shuijing198799 Mar 12, 2019
eed42f6
delete unuse functions
shuijing198799 Mar 12, 2019
3c20255
delete more functions and modify the label function
shuijing198799 Mar 12, 2019
8541758
modify the format and delete commend line
shuijing198799 Mar 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ images/tidb-operator-e2e/tidb-operator/
tmp/
data/
.idea
cscope.files
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What are these files?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's ctags generate files only use for me

tags
filenametags

# ginkgo test coverage
*.coverprofile
Expand Down
24 changes: 24 additions & 0 deletions pkg/label/label.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ package label
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"bytes"
"fmt"
)

const (
Expand Down Expand Up @@ -138,3 +140,25 @@ func (l Label) LabelSelector() *metav1.LabelSelector {
func (l Label) Labels() map[string]string {
return l
}

//Labels convers label to string
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

// String converts label to a string

func (l Label) String() (string,error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

gofmt

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

func (l Label) String() string {
	var arr []string

	for k, v := range l {
		arr = append(arr, fmt.Sprintf("%s=%s", k, v))
	}

	return strings.Join(arr, ",")
}

var buffer bytes.Buffer
count := 1
for k,v := range l {
var set string
if count == len(l) {
set = fmt.Sprintf("%s=%s",k , v)
} else {
set = fmt.Sprintf("%s=%s,",k , v)
}
count += 1
_,err := buffer.WriteString(set)
if err != nil {
return "",err
}
}

setStr := buffer.String()
return setStr,nil
}
204 changes: 181 additions & 23 deletions tests/actions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,12 @@ import (
"strconv"
"strings"
"time"
"net/http"
"io/ioutil"
"encoding/json"

"github.com/golang/glog"
_ "github.com/go-sql-driver/mysql"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
Expand All @@ -43,6 +47,11 @@ func NewOperatorActions(cli versioned.Interface, kubeCli kubernetes.Interface) O
}
}

const (
DefaultPollTimeout time.Duration = 10 * time.Minute
DefaultPollInterval time.Duration = 1 * time.Minute
)

type OperatorActions interface {
DeployOperator(info *OperatorInfo) error
CleanOperator(info *OperatorInfo) error
Expand All @@ -56,14 +65,16 @@ type OperatorActions interface {
ScaleTidbCluster(info *TidbClusterInfo) error
UpgradeTidbCluster(info *TidbClusterInfo) error
DeployAdHocBackup(info *TidbClusterInfo) error
CleanAdHocBackup(info *TidbClusterInfo) error
CheckAdHocBackup(info *TidbClusterInfo) error
DeployScheduledBackup(info *TidbClusterInfo) error
CleanScheduledBackup(info *TidbClusterInfo) error
DeployIncrementalBackup(info *TidbClusterInfo) error
CleanIncrementalBackup(info *TidbClusterInfo) error
Restore(from *TidbClusterInfo, jobName string, to *TidbClusterInfo) error
CheckScheduledBackup(info *TidbClusterInfo) error
DeployIncrementalBackup(from *TidbClusterInfo,to *TidbClusterInfo) error
CheckIncrementalBackup(info *TidbClusterInfo) error
Restore(from *TidbClusterInfo, to *TidbClusterInfo) error
CheckRestore(from *TidbClusterInfo, to *TidbClusterInfo) error
DeployMonitor(info *TidbClusterInfo) error
CleanMonitor(info *TidbClusterInfo) error
ForceDeploy(info *TidbClusterInfo) error
}

type FaultTriggerActions interface {
Expand Down Expand Up @@ -204,6 +215,10 @@ func (oa *operatorActions) DumpAllLogs(info *OperatorInfo, clusterInfo *TidbClus
}

func (oa *operatorActions) DeployTidbCluster(info *TidbClusterInfo) error {
glog.Infof("begin to deploy tidb cluster")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

print more info about the cluster, like name and namespace

defer func() {
glog.Infof("deploy tidb cluster end")
}()
cmd := fmt.Sprintf("helm install /charts/%s/tidb-cluster --name %s --namespace %s --set-string %s",
info.OperatorTag, info.ClusterName, info.Namespace, info.HelmSetString())
if res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil {
Expand All @@ -215,10 +230,13 @@ func (oa *operatorActions) DeployTidbCluster(info *TidbClusterInfo) error {
}

func (oa *operatorActions) CleanTidbCluster(info *TidbClusterInfo) error {
glog.Infof("begin to clean tidb cluster")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto

defer func() {
glog.Infof("clean tidb cluster end")
}()
charts := []string{
info.ClusterName,
fmt.Sprintf("%s-backup", info.ClusterName),
fmt.Sprintf("%s-restore", info.ClusterName),
}
for _, chartName := range charts {
res, err := exec.Command("helm", "del", "--purge", chartName).CombinedOutput()
Expand All @@ -228,10 +246,17 @@ func (oa *operatorActions) CleanTidbCluster(info *TidbClusterInfo) error {
}
}

resources := []string{"cronjobs", "jobs", "pods", "pvc"}
//for the test should add two clusters in a namespace, so we
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

delete these lines.

//has to use clustername as an label.
setStr,err := label.New().Instance(info.ClusterName).String()
if err != nil {
return fmt.Errorf("failed to arrange label to string : %v", err)
}

resources := []string{ "pvc"}
for _, resource := range resources {
if res, err := exec.Command("kubectl", "delete", resource, "-n", info.Namespace,
"--all").CombinedOutput(); err != nil {
if res, err := exec.Command("kubectl", "delete", resource, "-n", info.Namespace, "-l",
setStr).CombinedOutput(); err != nil {
return fmt.Errorf("failed to delete %s: %v, %s", resource, err, string(res))
}
}
Expand All @@ -245,7 +270,7 @@ func (oa *operatorActions) CleanTidbCluster(info *TidbClusterInfo) error {
}

pollFn := func() (bool, error) {
if res, err := exec.Command("kubectl", "get", "po", "--output=name", "-n", info.Namespace).
if res, err := exec.Command("kubectl", "get", "po", "--output=name", "-n", info.Namespace, "-l", setStr).
CombinedOutput(); err != nil || len(res) != 0 {
glog.Infof("waiting for tidbcluster: %s/%s pods deleting, %v, [%s]",
info.Namespace, info.ClusterName, err, string(res))
Expand All @@ -262,16 +287,19 @@ func (oa *operatorActions) CleanTidbCluster(info *TidbClusterInfo) error {
info.Namespace, info.ClusterName, err, string(res))
return false, nil
}

return true, nil
}
return wait.PollImmediate(1*time.Minute, 5*time.Minute, pollFn)
return wait.PollImmediate(DefaultPollInterval , DefaultPollTimeout , pollFn)
}

func (oa *operatorActions) CheckTidbClusterStatus(info *TidbClusterInfo) error {
glog.Infof("begin to check tidb cluster")
defer func() {
glog.Infof("check tidb cluster end")
}()
ns := info.Namespace
tcName := info.ClusterName
if err := wait.PollImmediate(1*time.Minute, 10*time.Minute, func() (bool, error) {
if err := wait.PollImmediate(DefaultPollInterval , DefaultPollTimeout , func() (bool, error) {
var tc *v1alpha1.TidbCluster
var err error
if tc, err = oa.cli.PingcapV1alpha1().TidbClusters(ns).Get(tcName, metav1.GetOptions{}); err != nil {
Expand All @@ -285,20 +313,30 @@ func (oa *operatorActions) CheckTidbClusterStatus(info *TidbClusterInfo) error {
if b, err := oa.tikvMembersReadyFn(tc); !b && err == nil {
return false, nil
}

glog.Infof("check tidb cluster begin tidbMembersReadyFn")
if b, err := oa.tidbMembersReadyFn(tc); !b && err == nil {
return false, nil
}

glog.Infof("check tidb cluster begin reclaimPolicySyncFn")
if b, err := oa.reclaimPolicySyncFn(tc); !b && err == nil {
return false, nil
}

glog.Infof("check tidb cluster begin metaSyncFn")
if b, err := oa.metaSyncFn(tc); err != nil {
return false, err
} else if !b && err == nil {
return false, nil
}

glog.Infof("check tidb cluster begin schedulerHAFn")
if b, err := oa.schedulerHAFn(tc); !b && err == nil {
return false, nil
}

glog.Infof("check tidb cluster begin passwordIsSet")
if b, err := oa.passwordIsSet(info); !b && err == nil {
return false, nil
}
Expand All @@ -321,15 +359,6 @@ func (oa *operatorActions) StopInsertDataTo(info *TidbClusterInfo) error {

func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error { return nil }
func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterInfo) error { return nil }
func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterInfo) error { return nil }
func (oa *operatorActions) CleanAdHocBackup(info *TidbClusterInfo) error { return nil }
func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterInfo) error { return nil }
func (oa *operatorActions) CleanScheduledBackup(info *TidbClusterInfo) error { return nil }
func (oa *operatorActions) DeployIncrementalBackup(info *TidbClusterInfo) error { return nil }
func (oa *operatorActions) CleanIncrementalBackup(info *TidbClusterInfo) error { return nil }
func (oa *operatorActions) Restore(from *TidbClusterInfo, jobName string, to *TidbClusterInfo) error {
return nil
}
func (oa *operatorActions) DeployMonitor(info *TidbClusterInfo) error { return nil }
func (oa *operatorActions) CleanMonitor(info *TidbClusterInfo) error { return nil }

Expand Down Expand Up @@ -729,7 +758,7 @@ func (oa *operatorActions) schedulerHAFn(tc *v1alpha1.TidbCluster) (bool, error)
nodeName, len(nodeMap[nodeName]), totalCount)
}
}
return false, nil
return true, nil
}

components := []string{label.PDLabelVal, label.TiKVLabelVal}
Expand Down Expand Up @@ -811,3 +840,132 @@ func checkoutTag(tagName string) error {

return nil
}

//scheduler backup, make sure there are serveral backup record is generated, and
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please remove all the backup related codes from this PR. It should be another PR.

//store the datas of each record
func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterInfo) error {
return nil
}

//restore genereted record by scheduledbackup, and make sure the data is correct
func (oa *operatorActions) CheckScheduledBackup(info *TidbClusterInfo) error {
return nil
}

//full data backup
func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterInfo) error {
return nil
}

//restore full data backup is correct and check full data backup data is correct
func (oa *operatorActions) CheckAdHocBackup(info *TidbClusterInfo) error {
return nil
}

//return fmt.Errorf("failed to launch scheduler backup job: %v, %s", err, string(res))incretment data backup
func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterInfo, to *TidbClusterInfo) error {
return nil
}

//restore increment data backup and make sure data is correct.
func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterInfo) error {
return nil

}

//full restore data from record to database.
func (oa *operatorActions) Restore(from *TidbClusterInfo, to *TidbClusterInfo) error {
return nil
}

//check wether restore data is correct.
func (oa *operatorActions) CheckRestore(from *TidbClusterInfo, to *TidbClusterInfo) error {
return nil
}

type pumpStatus struct {
StatusMap map[string]*nodeStatus
}

type nodeStatus struct {
State string
}

func (oa *operatorActions) pumpHealth(info *TidbClusterInfo , hostName string) bool {
pumpHealthUrl := fmt.Sprintf("%s.%s-pump.%s", hostName, info.ClusterName, info.Namespace)
res, err := http.Get(pumpHealthUrl)
if err != nil {
glog.Errorf("cluster:[%s] call %s failed,error:%v", info.ClusterName, pumpHealthUrl, err)
return false
}
if res.StatusCode >= 400 {
glog.Errorf("Error response %v", res.StatusCode)
return false
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
glog.Errorf("cluster:[%s] read response body failed,error:%v", info.ClusterName, err)
return false
}
healths := pumpStatus{}
err = json.Unmarshal(body, &healths)
if err != nil {
glog.Errorf("cluster:[%s] unmarshal failed,error:%v", info.ClusterName, err)
return false
}
for _, status := range healths.StatusMap {
if status.State != "online" {
glog.Errorf("cluster:[%s] pump's state is not online", info.ClusterName)
return false
}
}
return true
}

type drainerStatus struct {
PumpPos map[string]int64 `json:"PumpPos"`
Synced bool `json:"Synced"`
LastTS int64 `json:"LastTS"`
TsMap string `json:"TsMap"`
}

func (oa *operatorActions) drainerHealth(info *TidbClusterInfo, hostName string) bool {
drainerHealthUrl := fmt.Sprintf("%s.%s-drainer.%s", hostName, info.ClusterName, info.Namespace)
res, err := http.Get(drainerHealthUrl)
if err != nil {
glog.Errorf("cluster:[%s] call %s failed,error:%v", info.ClusterName, drainerHealthUrl, err)
return false
}
if res.StatusCode >= 400 {
glog.Errorf("Error response %v", res.StatusCode)
return false
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
glog.Errorf("cluster:[%s] read response body failed,error:%v", info.ClusterName, err)
return false
}
healths := drainerStatus{}
err = json.Unmarshal(body, &healths)
if err != nil {
glog.Errorf("cluster:[%s] unmarshal failed,error:%v", info.ClusterName, err)
return false
}
return len(healths.PumpPos) > 0 && healths.Synced
}

func (oa *operatorActions) ForceDeploy(info *TidbClusterInfo) error {
if err := oa.CleanTidbCluster(info); err != nil {
return err
}

if err := oa.DeployTidbCluster(info); err != nil {
return err
}

return nil
}

func (info *TidbClusterInfo) FullName() string {
return fmt.Sprintf("%s/%s",info.Namespace,info.ClusterName)
}
22 changes: 22 additions & 0 deletions tests/cmd/e2e/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,26 @@ func main() {
if err := oa.CheckTidbClusterStatus(clusterInfo); err != nil {
glog.Fatal(err)
}

restoreClusterInfo := &tests.TidbClusterInfo{
Namespace: "tidb",
ClusterName: "demo2",
OperatorTag: "v1.0.0-beta.1-p2",
PDImage: "pingcap/pd:v2.1.3",
TiKVImage: "pingcap/tikv:v2.1.3",
TiDBImage: "pingcap/tidb:v2.1.3",
StorageClassName: "local-storage",
Password: "admin",
Args: map[string]string{},
}

if err := oa.CleanTidbCluster(restoreClusterInfo); err != nil {
glog.Fatal(err)
}
if err := oa.DeployTidbCluster(restoreClusterInfo); err != nil {
glog.Fatal(err)
}
if err := oa.CheckTidbClusterStatus(restoreClusterInfo); err != nil {
glog.Fatal(err)
}
}