/* * Copyright (c) 2019 Huawei Technologies Co., Ltd. * A-Tune is licensed under the Mulan PSL v2. * You can use this software according to the terms and conditions of the Mulan PSL v2. * You may obtain a copy of Mulan PSL v2 at: * http://license.coscl.org.cn/MulanPSL2 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR * PURPOSE. * See the Mulan PSL v2 for more details. * Create: 2019-10-29 */ package utils import ( "archive/tar" "bufio" "compress/gzip" "encoding/csv" "encoding/xml" "fmt" PB "gitee.com/openeuler/A-Tune/api/profile" "gitee.com/openeuler/A-Tune/common/log" "gopkg.in/yaml.v2" "io" "io/ioutil" "math" "net" "os" "os/exec" "path" "path/filepath" "plugin" "regexp" "strconv" "strings" "syscall" "time" "github.com/urfave/cli" ) // disk filename const ( diskFilename = "diskstats" ) // the number of args const ( ConstExactArgs = iota ConstMinArgs ) // const for function ParseXxx const ( DecimalBase = 10 HexBase = 16 Uint64Bits = 64 ) // ProcDir is path of proc sysfs var ProcDir = "/proc/" // Status of AckCheck const ( SUCCESS = "SUCCESS" FAILD = "FAILED" WARNING = "WARNING" ERROR = "ERROR" INFO = "INFO" SUGGEST = "SUGGEST" UNKNOWN = "UNKNOWN" ) var accurency = 0.0001 // config for waiting rest service const ( period = 2 timeout = 120 ) //file attribute const ( PipeMode uint32 = 0600 FilePerm os.FileMode = 0600 MaxFileSize int64 = 100 * 1024 * 1024 ) // Rest service host and port var RestHost = "localhost" var RestPort = "8383" // CheckArgs method check command args num func CheckArgs(context *cli.Context, expected, checkType int) error { var err error cmdName := context.Command.Name switch checkType { case ConstExactArgs: if context.NArg() != expected { err = fmt.Errorf("%s: %q requires exactly %d argument(s)", os.Args[0], cmdName, expected) } case ConstMinArgs: if context.NArg() < expected { err = fmt.Errorf("%s: %q requires a minimum of %d argument(s)", os.Args[0], cmdName, expected) } } if err != nil { fmt.Printf("Incorrect Usage.\n\n") _ = cli.ShowCommandHelp(context, cmdName) return err } return nil } // Fatal method exit the application func Fatal(err error) { fmt.Fprintln(os.Stderr, err) os.Exit(1) } // LoadPlugins method load the plugin service func LoadPlugins(path string) error { abs, err := filepath.Abs(path) if err != nil { return err } pattern := filepath.Join(abs, "*") libs, err := filepath.Glob(pattern) if err != nil { return err } for _, lib := range libs { if _, err := plugin.Open(lib); err != nil { fmt.Printf("load %s failed : err:%s\n", lib, err) continue } } return nil } // PathExist method check path if exist or not func PathExist(path string) (bool, error) { fileInfo, err := os.Stat(path) if err == nil { if !fileInfo.IsDir() && fileInfo.Size() > MaxFileSize { return true, fmt.Errorf("the size of %s exceeds"+ " the maximum value which is %d", fileInfo.Name(), MaxFileSize) } return true, nil } if os.IsNotExist(err) { return false, nil } return false, err } // PathIsDir method check path is dir or not func PathIsDir(path string) (bool, error) { info, err := os.Stat(path) if err == nil { return info.IsDir(), nil } if os.IsNotExist(err) { return false, fmt.Errorf("path %s doens't exist", path) } return false, err } // Print method print AckCheck to stdout func Print(ackCheck *PB.AckCheck) { fc := 32 if ackCheck.Status == FAILD { fc = 31 } else if ackCheck.Status == WARNING { fc = 33 } else if ackCheck.Status == SUGGEST { fc = 36 } else if ackCheck.Status == INFO { fc = 37 } if ackCheck.Description == "" { fmt.Printf("%c[1;40;%dm %s %c[0m\n", 0x1B, fc, ackCheck.GetName(), 0x1B) } else { description := strings.Replace(ackCheck.Description, "\n", ",", -1) fmt.Printf(" [%c[1;40;%dm %-7s] %-40s %-50s %c[0m\n", 0x1B, fc, ackCheck.Status, ackCheck.Name, description, 0x1B) } } // PrintStr method print string to stdout func PrintStr(description string) { fmt.Printf("%c[1;40;%dm %s %c[0m\n", 0x1B, 32, description, 0x1B) } // IsHost method check whether it is a physical machine or a virtual machine func IsHost() bool { cmd := exec.Command("virt-what") _, err := cmd.Output() return err == nil } // CreateNamedPipe method create a named pip to communicate func CreateNamedPipe() (string, error) { npipe := strconv.FormatInt(time.Now().Unix(), 10) npipe = path.Join("/run", npipe) exist, err := PathExist(npipe) if err != nil { return "", err } if exist { _ = os.Remove(npipe) } err = syscall.Mkfifo(npipe, PipeMode) if err != nil { return "", err } return npipe, nil } // CopyFile method copy file from srcName to dstname func CopyFile(dstName string, srcName string) error { src, err := os.Open(srcName) if err != nil { return err } defer src.Close() dst, err := os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0640) if err != nil { return err } defer dst.Close() if _, err := io.Copy(dst, src); err != nil { return err } return nil } // RemoveDuplicateElement method remove duplicate elem from slice func RemoveDuplicateElement(message []string) []string { result := make([]string, 0, len(message)) messageMap := map[string]struct{}{} for _, item := range message { if _, ok := messageMap[item]; !ok { messageMap[item] = struct{}{} result = append(result, item) } } return result } // WaitForPyservice method waiting for pyEngine service start success func WaitForPyservice() error { ticker := time.NewTicker(time.Second * period) timeout := time.After(time.Second * timeout) addr := RestHost + ":" + RestPort for { select { case <-ticker.C: conn, err := net.Dial("tcp", addr) if err != nil { continue } conn.Close() return nil case <-timeout: return fmt.Errorf("waiting for pyservice timeout") } } } // InterfaceByName method check the interface state, up or down // if interface is not exist or down return err, otherwise return nil func InterfaceByName(name string) error { netIntface, err := net.InterfaceByName(name) if err != nil { return fmt.Errorf("interface %s is not exist", name) } if netIntface.Flags&net.FlagUp == 0 { return fmt.Errorf("interface %s may be down", name) } return nil } // DiskByName method check wether the disk is exist // if disk is exist return nil, otherwise return error func DiskByName(disk string) error { diskFile := path.Join("/proc", diskFilename) file, err := os.Open(diskFile) if err != nil { return fmt.Errorf("open file %s error: %v", diskFile, err) } defer file.Close() scanner := bufio.NewScanner(file) for scanner.Scan() { parts := strings.Fields(scanner.Text()) if len(parts) < 4 { continue } if parts[2] == disk { return nil } } return fmt.Errorf("disk %s is not exist", disk) } // IsInputStringValid: common input string validator func IsInputStringValid(input string) bool { if input != "" { if isOk, _ := regexp.MatchString("^[a-zA-Z0-9/._-]*$", input); isOk { return isOk } } return false } // WriteFile: write or append string to file func WriteFile(filename string, data string, perm os.FileMode, wrapper int) error { f, err := os.OpenFile(filename, wrapper, perm) if err != nil { return err } n, err := f.WriteString(data) if err == nil && n < len(data) { err = io.ErrShortWrite } if err1 := f.Close(); err == nil { err = err1 } return err } // ParseFile: parse yaml or xml file func ParseFile(filePath string, fileFormat string, out interface{}) error { exist, err := PathExist(filePath) if err != nil { return err } if !exist { return fmt.Errorf("%s is not exist", filePath) } data, err := ioutil.ReadFile(filePath) if err != nil { return err } switch fileFormat { case "yaml": if err := yaml.Unmarshal(data, out); err != nil { return err } case "xml": if err := xml.Unmarshal(data, out); err != nil { return err } default: return fmt.Errorf("this conversion mode of %s is not supported", fileFormat) } return nil } //create dir if the dir is not exist func CreateDir(dir string, perm os.FileMode) error { exist, err := PathExist(dir) if err != nil { return err } if !exist { if err = os.MkdirAll(dir, perm); err != nil { return err } } return nil } // ReadAllFile read all file to string. func ReadAllFile(path string) string { content, err := ioutil.ReadFile(path) if err != nil { return "" } return string(content) } // ReadCSV read data from test.csv func ReadCSV(path string) ([][]string, error) { file, err := os.Open(path) if err != nil { return nil, err } defer file.Close() lines, err := csv.NewReader(file).ReadAll() if err != nil { return nil, err } data := make([][]string, 0) for _, line := range lines { if len(line) > 0 { data = append(data, line) } } return data, nil } // CreateCompressFile create a .tar.gz file as target of compress func CreateCompressFile(path string) (string, error) { temp := strings.Split(path, "/") compressName := temp[len(temp)-1] + "-" + time.Now().Format("20060101") + ".tar.gz" zipFile, _ := os.Create(compressName) defer zipFile.Close() gzipWrt := gzip.NewWriter(zipFile) defer gzipWrt.Close() tarWrt := tar.NewWriter(gzipWrt) defer tarWrt.Close() file, err := os.Open(path) if err != nil { log.Errorf("Open file error: %v", err) os.Remove(compressName) return "", err } err = Compress(file, "", tarWrt) if err != nil { log.Errorf("Compress error : %v", err) os.Remove(compressName) return "", err } absPath, _ := filepath.Abs(compressName) return absPath, nil } // Compress recursionly compress all files under given path func Compress(file *os.File, prefix string, tarWrt *tar.Writer) error { info, err := file.Stat() if err != nil { log.Errorf("%v", err) return err } if info.IsDir() { if strings.EqualFold(prefix, "") { prefix = info.Name() } else { prefix = prefix + "/" + info.Name() } subfiles, err := file.Readdir(-1) if err != nil { log.Errorf("%v", err) return err } for _, subfile := range subfiles { f, err := os.Open(file.Name() + "/" + subfile.Name()) if err != nil { log.Errorf("%v", err) return err } err = Compress(f, prefix, tarWrt) if err != nil { log.Errorf("%v", err) return err } } } else { header, err := tar.FileInfoHeader(info, "") if err != nil { return err } if !strings.EqualFold(prefix, "") { header.Name = prefix + "/" + header.Name } err = tarWrt.WriteHeader(header) if err != nil { return err } _, err = io.Copy(tarWrt, file) file.Close() if err != nil { return err } } return nil } // Pair: pair of name and score type Pair struct { Name string Score float64 } // SortedPair: Pairs sorted by score type SortedPair []Pair func (p SortedPair) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p SortedPair) Len() int { return len(p) } func (p SortedPair) Less(i, j int) bool { return p[i].Score >= p[j].Score } func IsEquals(a, b float64) bool { return math.Abs(a-b) < accurency } // CalculateBenchMark: parse benchmark string func CalculateBenchMark(eval string) (float64, error) { kvs := strings.Split(eval, "=") if len(kvs) != 2 { return 0.0, fmt.Errorf("evaluation format error") } floatEval, err := strconv.ParseFloat(kvs[1], 64) if err != nil { return 0.0, err } return floatEval, nil } // Mean calculate the average value func Mean(data []float64) float64 { var mean float64 = 0 for i := 0; i < len(data); i++ { mean += (data[i] - mean) / float64(i+1) } return mean } // Variance calculate the variance func Variance(data []float64) float64 { if len(data) <= 0 { return 0 } mean := Mean(data) var sum float64 = 0 for i := 0; i < len(data); i++ { diff := data[i] - mean sum += diff * diff } return sum / float64(len(data)) } // StandardDeviation calculate the standard deviation func StandardDeviation(data []float64) float64 { return math.Sqrt(Variance(data)) } // ChangeFileName: Change file name func ChangeFileName(dataPath string) (string, string, error) { dir, fileName := filepath.Split(dataPath) extension := filepath.Ext(dataPath) name := strings.TrimSuffix(fileName, extension) timestamp := strconv.FormatInt(time.Now().UnixNano() / 1e6, 10) newFilePath := dir + name + "-" + timestamp + extension log.Infof("new file path: %s", newFilePath) err := os.Rename(dataPath, newFilePath) if err != nil { return "", "", err } return newFilePath, timestamp, nil } // GetLogFilePath: Get log file path func GetLogFilePath(dirPath string) (string, error) { dir, err := ioutil.ReadDir(dirPath) if err != nil { log.Errorf("dir path does not exists") return "", err } for _, file := range dir { if !file.IsDir() && strings.HasSuffix(file.Name(), ".log") && strings.HasPrefix(file.Name(), "test-") { return dirPath + "/" + file.Name(), nil } } return "", fmt.Errorf("cannot find log file") } // Check whether the value is in the list func CheckValueInSlice(value string, strList []string) bool { for _, str := range strList { if str == value { return true } } return false } // Divide requires into groups func DivideToGroups(strs string, groups []string) []string { strsArr := strings.Split(strs, ",") div := make([]string, len(groups)) for _, val := range strsArr { for ind, group := range groups { if strings.Contains(group, val) { if div[ind] == "" { div[ind] += val } else { div[ind] += " " + val } break } } } divRes := make([]string, 0) for _, val := range div { if val != "" { divRes = append(divRes, val) } } return divRes }