Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add binlog deploy and check process #329

Merged
merged 16 commits into from
Mar 20, 2019
+249 −52
Merged

add binlog deploy and check process #329

Changes from 1 commit
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
a7e16bf
add schduler test case
shuijing198799 Mar 15, 2019
2746c82
delete unnessary step and add scheduled-backup-job check step
shuijing198799 Mar 18, 2019
79dfe9a
change name to backupPVC
shuijing198799 Mar 19, 2019
3dbddde
return a string instead of int in getbackupdir
shuijing198799 Mar 19, 2019
2eb505d
conflict resovle
shuijing198799 Mar 19, 2019
84d4698
resolve some merge error and naming conflict
shuijing198799 Mar 19, 2019
aa70d18
resolve some merge error and naming conflict
shuijing198799 Mar 19, 2019
43ba7c7
add binlog process
shuijing198799 Mar 19, 2019
085d0bf
conflict resolve
shuijing198799 Mar 19, 2019
5d6a0ed
add setting to helmsetstring and add create secret
shuijing198799 Mar 19, 2019
994c9ae
fullfil the comments
shuijing198799 Mar 20, 2019
8c924f8
Merge branch 'stability' into yinliang/backup-and-restore
shuijing198799 Mar 20, 2019
9ec00d4
Merge branch 'stability' into yinliang/backup-and-restore
shuijing198799 Mar 20, 2019
c2a14ef
prewrite secret name
shuijing198799 Mar 20, 2019
ac39382
Merge branch 'yinliang/backup-and-restore' of https://github.com/shui…
shuijing198799 Mar 20, 2019
51ed069
change password method
shuijing198799 Mar 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add setting to helmsetstring and add create secret
@shuijing198799
shuijing198799 committed Mar 19, 2019
commit 5d6a0edc898167edf49c1c5a927dfd49a869f3c6
114 changes: 51 additions & 63 deletions tests/actions.go
View file
Original file line number Diff line number Diff line change
@@ -14,7 +14,6 @@
package tests

import (
"bytes"
"database/sql"
"encoding/json"
"fmt"
@@ -92,7 +91,7 @@ type OperatorActions interface {
Restore(from *TidbClusterInfo, to *TidbClusterInfo) error
CheckRestore(from *TidbClusterInfo, to *TidbClusterInfo) error
ForceDeploy(info *TidbClusterInfo) error
CreateSecret(info *TidbClusterInfo) error
CreateSecret(info *TidbClusterInfo) (string, string, error)
getBackupDir(info *TidbClusterInfo) ([]string, error)
}

@@ -152,9 +151,12 @@ type TidbClusterInfo struct {
Args map[string]string
blockWriter *blockwriter.BlockWriterCase
Monitor bool
UserName string
InitSecretName string
BackupSecretName string
}

func (tc *TidbClusterInfo) HelmSetString() string {
func (tc *TidbClusterInfo) HelmSetString(m map[string]string) string {

set := map[string]string{
"clusterName": tc.ClusterName,
@@ -166,9 +168,10 @@ func (tc *TidbClusterInfo) HelmSetString() string {
"pd.image": tc.PDImage,
"tikv.image": tc.TiKVImage,
"tidb.image": tc.TiDBImage,
"tidb.passwordSecretName": "set-secret",
"tidb.passwordSecretName": tc.InitSecretName,
"tidb.initSql": tc.InitSql,
"monitor.create": strconv.FormatBool(tc.Monitor),
"secretName": tc.BackupSecretName,
}

for k, v := range tc.Resources {
@@ -177,6 +180,9 @@ func (tc *TidbClusterInfo) HelmSetString() string {
for k, v := range tc.Args {
set[k] = v
}
for k, v := range m {
set[k] = v
}

arr := make([]string, 0, len(set))
for k, v := range set {
@@ -245,8 +251,16 @@ func (oa *operatorActions) DeployTidbCluster(info *TidbClusterInfo) error {
defer func() {
glog.Infof("deploy tidb cluster end cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
}()

initSecretName, backupSecretName, err := oa.CreateSecret(info)
if err != nil {
return fmt.Errorf("failed to create secret of cluster [%s]: %v", info.ClusterName, err)
}

info.InitSecretName, info.BackupSecretName = initSecretName, backupSecretName

cmd := fmt.Sprintf("helm install /charts/%s/tidb-cluster --name %s --namespace %s --set-string %s",
info.OperatorTag, info.ClusterName, info.Namespace, info.HelmSetString())
info.OperatorTag, info.ClusterName, info.Namespace, info.HelmSetString(map[string]string{}))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

map[string]string{} -> nil

if res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput(); err != nil {
return fmt.Errorf("failed to deploy tidbcluster: %s/%s, %v, %s",
info.Namespace, info.ClusterName, err, string(res))
@@ -414,7 +428,7 @@ func chartPath(name string, tag string) string {

func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error {
cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s",
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString())
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString(map[string]string{}))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto

glog.Info("[SCALE] " + cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
@@ -425,7 +439,7 @@ func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterInfo) error {

func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterInfo) error {
cmd := fmt.Sprintf("helm upgrade %s %s --set-string %s",
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString())
info.ClusterName, chartPath("tidb-cluster", info.OperatorTag), info.HelmSetString(map[string]string{}))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto

glog.Info("[UPGRADE] " + cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
@@ -1058,26 +1072,18 @@ func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterInfo) error {
glog.Infof("deploy adhoc backup end cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
}()
sets := map[string]string{
"clusterName": info.ClusterName,
"name": info.BackupPVC,
"mode": "backup",
"user": "root",
"password": info.Password,
"storage.size": "10Gi",
}
var buffer bytes.Buffer
for k, v := range sets {
set := fmt.Sprintf(" --set %s=%s", k, v)
_, err := buffer.WriteString(set)
if err != nil {
return err
}
}

setStr := buffer.String()
setString := info.HelmSetString(sets)

fullbackupName := fmt.Sprintf("%s-backup", info.ClusterName)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup %s",
fullbackupName, info.Namespace, info.OperatorTag, setStr)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup --set-string %s",
fullbackupName, info.Namespace, info.OperatorTag, setString)
glog.Infof("install adhoc deployment [%s]", cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
@@ -1122,26 +1128,18 @@ func (oa *operatorActions) Restore(from *TidbClusterInfo, to *TidbClusterInfo) e
glog.Infof("deploy restore end cluster[%s] namespace[%s]", to.ClusterName, to.Namespace)
}()
sets := map[string]string{
"clusterName": to.ClusterName,
"name": to.BackupPVC,
"mode": "restore",
"user": "root",
"password": to.Password,
"storage.size": "10Gi",
}
var buffer bytes.Buffer
for k, v := range sets {
set := fmt.Sprintf(" --set %s=%s", k, v)
_, err := buffer.WriteString(set)
if err != nil {
return err
}
}

setStr := buffer.String()
setString := to.HelmSetString(sets)

restoreName := fmt.Sprintf("%s-restore", from.ClusterName)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup %s",
restoreName, to.Namespace, to.OperatorTag, setStr)
cmd := fmt.Sprintf("helm install -n %s --namespace %s /charts/%s/tidb-backup --set-string %s",
restoreName, to.Namespace, to.OperatorTag, setString)
glog.Infof("install restore [%s]", cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
@@ -1233,23 +1231,23 @@ func (info *TidbClusterInfo) QueryCount() (int, error) {
return 0, fmt.Errorf("can not find count of ")
}

func (oa *operatorActions) CreateSecret(info *TidbClusterInfo) error {
initSecretName := "set-secret"
backupSecretName := "backup-secret"
func (oa *operatorActions) CreateSecret(info *TidbClusterInfo) (string, string, error) {
initSecretName := fmt.Sprintf("%s-set-secret", info.ClusterName)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the field should be set when info created

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

backupSecretName := fmt.Sprintf("%s-backup-secret", info.ClusterName)
initSecret := corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: initSecretName,
Namespace: info.Namespace,
},
Data: map[string][]byte{
"root": []byte(info.Password),
info.UserName: []byte(info.Password),
},
Type: corev1.SecretTypeOpaque,
}

_, err := oa.kubeCli.CoreV1().Secrets(info.Namespace).Create(&initSecret)
if err != nil && !releaseIsExist(err) {
return err
return "", "", err
}

backupSecret := corev1.Secret{
@@ -1258,18 +1256,18 @@ func (oa *operatorActions) CreateSecret(info *TidbClusterInfo) error {
Namespace: info.Namespace,
},
Data: map[string][]byte{
"user": []byte("root"),
"user": []byte(info.UserName),
"password": []byte(info.Password),
},
Type: corev1.SecretTypeOpaque,
}

_, err = oa.kubeCli.CoreV1().Secrets(info.Namespace).Create(&backupSecret)
if err != nil && !releaseIsExist(err) {
return err
return "", "", err
}

return nil
return initSecretName, backupSecretName, nil
}

func releaseIsExist(err error) bool {
@@ -1291,19 +1289,11 @@ func (oa *operatorActions) DeployScheduledBackup(info *TidbClusterInfo) error {
"scheduledBackup.schedule": cron,
"scheduledBackup.storage": "10Gi",
}
var buffer bytes.Buffer
for k, v := range sets {
set := fmt.Sprintf(" --set %s=%s", k, v)
_, err := buffer.WriteString(set)
if err != nil {
return err
}
}

setStr := buffer.String()
setString := info.HelmSetString(sets)

cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster %s",
info.ClusterName, info.OperatorTag, setStr)
cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster --set-string %s",
info.ClusterName, info.OperatorTag, setString)

glog.Infof("scheduled-backup delploy [%s]", cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
@@ -1458,13 +1448,19 @@ func (oa *operatorActions) getBackupDir(info *TidbClusterInfo) ([]string, error)
}

cmd := fmt.Sprintf("kubectl exec %s -n %s ls /data", getBackupDirPodName, info.Namespace)
glog.Infof(cmd)

time.Sleep(20 * time.Second)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we poll the following condition so we can proceed on earlier if condition meet?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I try to poll at first but the time is hard to controller in one minute. for every minute generate a cronjob to backup. At last we have to check the backup number.


res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
glog.Errorf("cluster:[%s/%s] exec :%s failed,error:%v,result:%s", info.Namespace, info.ClusterName, cmd, err, res)
glog.Errorf("cluster:[%s/%s] exec :%s failed,error:%v,result:%s", info.Namespace, info.ClusterName, cmd, err, string(res))
return nil, err
}

if err != nil {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unreachable

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems you are right

return nil, fmt.Errorf("failed to get dir via cmd [%s]", cmd)
}

dirs := strings.Split(string(res), "\n")
glog.Infof("dirs in pod info name [%s] dir name [%s]", info.BackupPVC, strings.Join(dirs, ","))
return dirs, nil
@@ -1489,18 +1485,10 @@ func (oa *operatorActions) DeployIncrementalBackup(from *TidbClusterInfo, to *Ti
"binlog.drainer.mysql.port": "4000",
}

var buffer bytes.Buffer
for k, v := range sets {
set := fmt.Sprintf(" --set %s=%s", k, v)
_, err := buffer.WriteString(set)
if err != nil {
return err
}
}
setString := from.HelmSetString(sets)

setStr := buffer.String()
cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster %s",
from.ClusterName, from.OperatorTag, setStr)
cmd := fmt.Sprintf("helm upgrade %s /charts/%s/tidb-cluster --set-string %s",
from.ClusterName, from.OperatorTag, setString)
glog.Infof(cmd)
res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
if err != nil {
2 changes: 2 additions & 0 deletions tests/cmd/e2e/main.go
View file
Original file line number Diff line number Diff line change
@@ -88,6 +88,7 @@ func main() {
StorageClassName: "local-storage",
Password: "admin",
InitSql: initSql,
UserName: "root",
Resources: map[string]string{
"pd.resources.limits.cpu": "1000m",
"pd.resources.limits.memory": "2Gi",
@@ -129,6 +130,7 @@ func main() {
StorageClassName: "local-storage",
Password: "admin",
InitSql: initSql,
UserName: "root",
Resources: map[string]string{
"pd.resources.limits.cpu": "1000m",
"pd.resources.limits.memory": "2Gi",