Rainbond/util/comman.go

// Copyright (C) 2014-2018 Goodrain Co., Ltd.
// RAINBOND, Application Management Platform

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package util

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"reflect"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/goodrain/rainbond/util/zip"
)

//CheckAndCreateDir check and create dir
func CheckAndCreateDir(path string) error {
	if subPathExists, err := FileExists(path); err != nil {
		return fmt.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", path)
	} else if !subPathExists {
		// Create the sub path now because if it's auto-created later when referenced, it may have an
		// incorrect ownership and mode. For example, the sub path directory must have at least g+rwx
		// when the pod specifies an fsGroup, and if the directory is not created here, Docker will
		// later auto-create it with the incorrect mode 0750
		if err := os.MkdirAll(path, 0755); err != nil {
			return fmt.Errorf("failed to mkdir:%s", path)
		}

		if err := os.Chmod(path, 0755); err != nil {
			return err
		}
	}
	return nil
}

//CheckAndCreateDirByMode check and create dir
func CheckAndCreateDirByMode(path string, mode os.FileMode) error {
	if subPathExists, err := FileExists(path); err != nil {
		return fmt.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", path)
	} else if !subPathExists {
		// Create the sub path now because if it's auto-created later when referenced, it may have an
		// incorrect ownership and mode. For example, the sub path directory must have at least g+rwx
		// when the pod specifies an fsGroup, and if the directory is not created here, Docker will
		// later auto-create it with the incorrect mode 0750
		if err := os.MkdirAll(path, mode); err != nil {
			return fmt.Errorf("failed to mkdir:%s", path)
		}

		if err := os.Chmod(path, mode); err != nil {
			return err
		}
	}
	return nil
}

//DirIsEmpty 验证目录是否为空
func DirIsEmpty(dir string) bool {
	infos, err := ioutil.ReadDir(dir)
	if len(infos) == 0 || err != nil {
		return true
	}
	return false
}

//OpenOrCreateFile open or create file
func OpenOrCreateFile(filename string) (*os.File, error) {
	return os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0777)
}

//FileExists check file exist
func FileExists(filename string) (bool, error) {
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		return false, nil
	} else if err != nil {
		return false, err
	}
	return true, nil
}

//SearchFileBody 搜索文件中是否含有指定字符串
func SearchFileBody(filename, searchStr string) bool {
	body, _ := ioutil.ReadFile(filename)
	return strings.Contains(string(body), searchStr)
}

//IsHaveFile 指定目录是否含有文件
//.开头文件除外
func IsHaveFile(path string) bool {
	files, _ := ioutil.ReadDir(path)
	for _, file := range files {
		if !strings.HasPrefix(file.Name(), ".") {
			return true
		}
	}
	return false
}

//SearchFile 搜索指定目录是否有指定文件，指定搜索目录层数，-1为全目录搜索
func SearchFile(pathDir, name string, level int) bool {
	if level == 0 {
		return false
	}
	files, _ := ioutil.ReadDir(pathDir)
	var dirs []os.FileInfo
	for _, file := range files {
		if file.IsDir() {
			dirs = append(dirs, file)
			continue
		}
		if file.Name() == name {
			return true
		}
	}
	if level == 1 {
		return false
	}
	for _, dir := range dirs {
		ok := SearchFile(path.Join(pathDir, dir.Name()), name, level-1)
		if ok {
			return ok
		}
	}
	return false
}

//FileExistsWithSuffix 指定目录是否含有指定后缀的文件
func FileExistsWithSuffix(pathDir, suffix string) bool {
	files, _ := ioutil.ReadDir(pathDir)
	for _, file := range files {
		if strings.HasSuffix(file.Name(), suffix) {
			return true
		}
	}
	return false
}

//CmdRunWithTimeout exec cmd with timeout
func CmdRunWithTimeout(cmd *exec.Cmd, timeout time.Duration) (bool, error) {
	done := make(chan error)
	if cmd.Process != nil { //还原执行状态
		cmd.Process = nil
		cmd.ProcessState = nil
	}
	if err := cmd.Start(); err != nil {
		return false, err
	}
	go func() {
		done <- cmd.Wait()
	}()
	var err error
	select {
	case <-time.After(timeout):
		// timeout
		if err = cmd.Process.Kill(); err != nil {
			logrus.Errorf("failed to kill: %s, error: %s", cmd.Path, err.Error())
		}
		go func() {
			<-done // allow goroutine to exit
		}()
		logrus.Infof("process:%s killed", cmd.Path)
		return true, err
	case err = <-done:
		return false, err
	}
}

//ReadHostID 读取当前机器ID
//ID是节点的唯一标识，acp_node将把ID与机器信息的绑定关系维护于etcd中
func ReadHostID(filePath string) (string, error) {
	if filePath == "" {
		if runtime.GOOS == "windows" {
			filePath = "c:\\rainbond\\node_host_uuid.conf"
		} else {
			filePath = "/opt/rainbond/etc/node/node_host_uuid.conf"
		}
	}
	_, err := os.Stat(filePath)
	if err != nil {
		if os.IsNotExist(err) {
			uid, err := CreateHostID()
			if err != nil {
				return "", err
			}
			err = ioutil.WriteFile(filePath, []byte("host_uuid="+uid), 0777)
			if err != nil {
				logrus.Error("Write host_uuid file error.", err.Error())
			}
			return uid, nil
		}
		return "", err
	}
	body, err := ioutil.ReadFile(filePath)
	if err != nil {
		return "", err
	}
	info := strings.Split(strings.TrimSpace(string(body)), "=")
	if len(info) == 2 {
		return info[1], nil
	}
	return "", fmt.Errorf("Invalid host uuid from file")
}

//CreateHostID create host id by mac addr
func CreateHostID() (string, error) {
	macAddrs := getMacAddrs()
	if macAddrs == nil || len(macAddrs) == 0 {
		return "", fmt.Errorf("read macaddr error when create node id")
	}
	ip, _ := LocalIP()
	hash := md5.New()
	hash.Write([]byte(macAddrs[0] + ip.String()))
	uid := fmt.Sprintf("%x", hash.Sum(nil))
	if len(uid) >= 32 {
		return uid[:32], nil
	}
	for i := len(uid); i < 32; i++ {
		uid = uid + "0"
	}
	return uid, nil
}

func getMacAddrs() (macAddrs []string) {
	netInterfaces, err := net.Interfaces()
	if err != nil {
		fmt.Printf("fail to get net interfaces: %v", err)
		return macAddrs
	}

	for _, netInterface := range netInterfaces {
		macAddr := netInterface.HardwareAddr.String()
		if len(macAddr) == 0 {
			continue
		}
		macAddrs = append(macAddrs, macAddr)
	}
	return macAddrs
}

//LocalIP 获取本机 ip
// 获取第一个非 loopback ip
func LocalIP() (net.IP, error) {
	tables, err := net.Interfaces()
	if err != nil {
		return nil, err
	}
	for _, t := range tables {
		addrs, err := t.Addrs()
		if err != nil {
			return nil, err
		}
		for _, a := range addrs {
			ipnet, ok := a.(*net.IPNet)
			if !ok || ipnet.IP.IsLoopback() {
				continue
			}
			if v4 := ipnet.IP.To4(); v4 != nil {
				return v4, nil
			}
		}
	}
	return nil, fmt.Errorf("cannot find local IP address")
}

//GetIDFromKey 从 etcd 的 key 中取 id
func GetIDFromKey(key string) string {
	index := strings.LastIndex(key, "/")
	if index < 0 {
		return ""
	}
	if strings.Contains(key, "-") { //build in任务，为了给不同node做一个区分
		return strings.Split(key[index+1:], "-")[0]
	}

	return key[index+1:]
}

//Deweight 去除数组重复
func Deweight(data *[]string) {
	var result []string
	if len(*data) < 1024 {
		// 切片长度小于1024的时候，循环来过滤
		for i := range *data {
			flag := true
			for j := range result {
				if result[j] == (*data)[i] {
					flag = false // 存在重复元素，标识为false
					break
				}
			}
			if flag && (*data)[i] != "" { // 标识为false，不添加进结果
				result = append(result, (*data)[i])
			}
		}
	} else {
		// 大于的时候，通过map来过滤
		var tmp = make(map[string]byte)
		for _, d := range *data {
			l := len(tmp)
			tmp[d] = 0
			if len(tmp) != l && d != "" { // 加入map后，map长度变化，则元素不重复
				result = append(result, d)
			}
		}
	}
	*data = result
}

//GetDirSizeByCmd get dir sizes by du command
//return kb
func GetDirSizeByCmd(path string) float64 {
	out, err := CmdExec(fmt.Sprintf("du -sk %s", path))
	if err != nil {
		fmt.Println(err)
		return 0
	}
	info := strings.Split(out, "	")
	fmt.Println(info)
	if len(info) < 2 {
		return 0
	}
	i, _ := strconv.Atoi(info[0])
	return float64(i)
}

//GetFileSize get file size
func GetFileSize(path string) int64 {
	if fileInfo, err := os.Stat(path); err == nil {
		return fileInfo.Size()
	}
	return 0
}

//GetDirSize kb为单位
func GetDirSize(path string) float64 {
	if ok, err := FileExists(path); err != nil || !ok {
		return 0
	}

	fileSizes := make(chan int64)
	concurrent := make(chan int, 10)
	var wg sync.WaitGroup

	wg.Add(1)
	go walkDir(path, &wg, fileSizes, concurrent)

	go func() {
		wg.Wait() //等待goroutine结束
		close(fileSizes)
	}()
	var nfiles, nbytes int64
loop:
	for {
		select {
		case size, ok := <-fileSizes:
			if !ok {
				break loop
			}
			nfiles++
			nbytes += size
		}
	}
	return float64(nbytes / 1024)
}

//获取目录dir下的文件大小
func walkDir(dir string, wg *sync.WaitGroup, fileSizes chan<- int64, concurrent chan int) {
	defer wg.Done()
	concurrent <- 1
	defer func() {
		<-concurrent
	}()
	for _, entry := range listDirNonSymlink(dir) {
		if entry.IsDir() { //目录
			wg.Add(1)
			subDir := filepath.Join(dir, entry.Name())
			go walkDir(subDir, wg, fileSizes, concurrent)
		} else {
			fileSizes <- entry.Size()
		}
	}
}

//sema is a counting semaphore for limiting concurrency in listDir
var sema = make(chan struct{}, 20)

//读取目录dir下的文件信息
func listDir(dir string) []os.FileInfo {
	sema <- struct{}{}
	defer func() { <-sema }()
	entries, err := ioutil.ReadDir(dir)
	if err != nil {
		logrus.Errorf("get file sizt: %v\n", err)
		return nil
	}
	return entries
}

// 列出指定目录下的非软链类型的所有条目
func listDirNonSymlink(dir string) []os.FileInfo {
	sema <- struct{}{}
	defer func() { <-sema }()
	entries, err := ioutil.ReadDir(dir)
	if err != nil {
		logrus.Errorf("get file sizt: %v\n", err)
		return nil
	}

	var result []os.FileInfo
	for i := range entries {
		if entries[i].Mode()&os.ModeSymlink == 0 {
			result = append(result, entries[i])
		}
	}
	return result
}

//RemoveSpaces 去除空格项
func RemoveSpaces(sources []string) (re []string) {
	for _, s := range sources {
		if s != " " && s != "" {
			re = append(re, s)
		}
	}
	return
}

//CmdExec CmdExec
func CmdExec(args string) (string, error) {
	out, err := exec.Command("bash", "-c", args).Output()
	if err != nil {
		return "", err
	}
	return string(out), nil
}

//Zip zip compressing source dir to target file
func Zip(source, target string) error {
	if err := CheckAndCreateDir(filepath.Dir(target)); err != nil {
		return err
	}
	zipfile, err := os.OpenFile(target, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return err
	}
	defer zipfile.Close()

	archive := zip.NewWriter(zipfile)
	defer archive.Close()

	info, err := os.Stat(source)
	if err != nil {
		return nil
	}

	var baseDir string
	if info.IsDir() {
		baseDir = filepath.Base(source)
	}

	filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		header, err := zip.FileInfoHeader(info)
		if err != nil {
			return err
		}
		if baseDir != "" {
			header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
		}
		if info.IsDir() {
			header.Name += "/"
		} else {
			header.Method = zip.Deflate
		}
		//set file uid and
		elem := reflect.ValueOf(info.Sys()).Elem()
		uid := elem.FieldByName("Uid").Uint()
		gid := elem.FieldByName("Gid").Uint()
		header.Comment = fmt.Sprintf("%d/%d", uid, gid)
		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}
		file, err := os.OpenFile(path, os.O_RDONLY, 0)
		if err != nil {
			return err
		}
		defer file.Close()
		_, err = io.Copy(writer, file)
		return err
	})

	return err
}

//Unzip archive file to target dir
func Unzip(archive, target string) error {
	reader, err := zip.OpenDirectReader(archive)
	if err != nil {
		return fmt.Errorf("error opening archive: %v", err)
	}
	if err := os.MkdirAll(target, 0755); err != nil {
		return err
	}
	for _, file := range reader.File {
		run := func() error {
			path := filepath.Join(target, file.Name)
			if file.FileInfo().IsDir() {
				os.MkdirAll(path, file.Mode())
				if file.Comment != "" && strings.Contains(file.Comment, "/") {
					guid := strings.Split(file.Comment, "/")
					if len(guid) == 2 {
						uid, _ := strconv.Atoi(guid[0])
						gid, _ := strconv.Atoi(guid[1])
						if err := os.Chown(path, uid, gid); err != nil {
							return fmt.Errorf("error changing owner: %v", err)
						}
					}
				}
				return nil
			}

			fileReader, err := file.Open()
			if err != nil {
				return fmt.Errorf("fileReader; error opening file: %v", err)
			}
			defer fileReader.Close()
			targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
			if err != nil {
				return fmt.Errorf("targetFile; error opening file: %v", err)
			}
			defer targetFile.Close()

			if _, err := io.Copy(targetFile, fileReader); err != nil {
				return fmt.Errorf("error copy file: %v", err)
			}
			if file.Comment != "" && strings.Contains(file.Comment, "/") {
				guid := strings.Split(file.Comment, "/")
				if len(guid) == 2 {
					uid, _ := strconv.Atoi(guid[0])
					gid, _ := strconv.Atoi(guid[1])
					if err := os.Chown(path, uid, gid); err != nil {
						return err
					}
				}
			}
			return nil
		}
		if err := run(); err != nil {
			return err
		}
	}

	return nil
}

// CopyFile copy source file to target
// direct io read and write file
// Keep the permissions user and group
func CopyFile(source, target string) error {
	sfi, err := os.Stat(source)
	if err != nil {
		return err
	}
	elem := reflect.ValueOf(sfi.Sys()).Elem()
	uid := elem.FieldByName("Uid").Uint()
	gid := elem.FieldByName("Gid").Uint()
	sf, err := os.OpenFile(source, os.O_RDONLY, 0)
	if err != nil {
		return err
	}
	defer sf.Close()
	tf, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, sfi.Mode())
	if err != nil {
		return err
	}
	defer tf.Close()
	_, err = io.Copy(tf, sf)
	if err != nil {
		return err
	}

	if err := os.Chown(target, int(uid), int(gid)); err != nil {
		return err
	}

	return nil
}

//GetParentDirectory GetParentDirectory
func GetParentDirectory(dirctory string) string {
	return substr(dirctory, 0, strings.LastIndex(dirctory, "/"))
}

func substr(s string, pos, length int) string {
	runes := []rune(s)
	l := pos + length
	if l > len(runes) {
		l = len(runes)
	}
	return string(runes[pos:l])
}

//Rename move file
func Rename(old, new string) error {
	_, err := os.Stat(GetParentDirectory(new))
	if err != nil {
		if err == os.ErrNotExist || strings.Contains(err.Error(), "no such file or directory") {
			if err := os.MkdirAll(GetParentDirectory(new), 0755); err != nil {
				return err
			}
		} else {
			return err
		}
	}
	return os.Rename(old, new)
}

//MergeDir MergeDir
//if Subdirectories already exist, Don't replace
func MergeDir(fromdir, todir string) error {
	files, err := ioutil.ReadDir(fromdir)
	if err != nil {
		return err
	}
	for _, f := range files {
		if err := os.Rename(path.Join(fromdir, f.Name()), path.Join(todir, f.Name())); err != nil {
			if !strings.Contains(err.Error(), "file exists") {
				return err
			}
		}
	}
	return nil
}

//CreateVersionByTime create version number
func CreateVersionByTime() string {
	now := time.Now()
	return now.Format("20060102150405")
}

// GetDirList get all lower level dir
func GetDirList(dirpath string, level int) ([]string, error) {
	var dirlist []string
	list, err := ioutil.ReadDir(dirpath)
	if err != nil {
		return nil, err
	}
	for _, f := range list {
		if f.IsDir() {
			if level <= 1 {
				dirlist = append(dirlist, filepath.Join(dirpath, f.Name()))
			} else {
				list, err := GetDirList(filepath.Join(dirpath, f.Name()), level-1)
				if err != nil {
					return nil, err
				}
				dirlist = append(dirlist, list...)
			}
		}
	}
	return dirlist, nil
}

//GetFileList -
func GetFileList(dirpath string, level int) ([]string, error) {
	var dirlist []string
	list, err := ioutil.ReadDir(dirpath)
	if err != nil {
		return nil, err
	}
	for _, f := range list {
		if !f.IsDir() && level <= 1 {
			dirlist = append(dirlist, filepath.Join(dirpath, f.Name()))
		} else if level > 1 && f.IsDir() {
			list, err := GetFileList(filepath.Join(dirpath, f.Name()), level-1)
			if err != nil {
				return nil, err
			}
			dirlist = append(dirlist, list...)
		}
	}
	return dirlist, nil
}

// GetDirNameList get all lower level dir
func GetDirNameList(dirpath string, level int) ([]string, error) {
	var dirlist []string
	list, err := ioutil.ReadDir(dirpath)
	if err != nil {
		return nil, err
	}
	for _, f := range list {
		if f.IsDir() {
			if level <= 1 {
				dirlist = append(dirlist, f.Name())
			} else {
				list, err := GetDirList(filepath.Join(dirpath, f.Name()), level-1)
				if err != nil {
					return nil, err
				}
				dirlist = append(dirlist, list...)
			}
		}
	}
	return dirlist, nil
}

//GetCurrentDir get current dir
func GetCurrentDir() string {
	dir, err := filepath.Abs("./")
	if err != nil {
		log.Fatal(err)
	}
	return strings.Replace(dir, "\\", "/", -1)
}

//IsDir path is dir
func IsDir(path string) (bool, error) {
	info, err := os.Stat(path)
	if err != nil {
		return false, err
	}
	return info.IsDir(), nil
}

var reg = regexp.MustCompile(`(?U)\$\{.*\}`)

//ParseVariable parse and replace variable in source str
func ParseVariable(source string, configs map[string]string) string {
	resultKey := reg.FindAllString(source, -1)
	for _, sourcekey := range resultKey {
		key, defaultValue := getVariableKey(sourcekey)
		if value, ok := configs[key]; ok {
			source = strings.Replace(source, sourcekey, value, -1)
		} else if defaultValue != "" {
			source = strings.Replace(source, sourcekey, defaultValue, -1)
		}
	}
	return source
}

func getVariableKey(source string) (key, value string) {
	if len(source) < 4 {
		return "", ""
	}
	left := strings.Index(source, "{")
	right := strings.Index(source, "}")
	k := source[left+1 : right]
	if strings.Contains(k, ":") {
		re := strings.Split(k, ":")
		if len(re) > 1 {
			return re[0], re[1]
		}
		return re[0], ""
	}
	return k, ""
}