add golang benchmark

benchmark.py

@@ -14,7 +14,7 @@ from gevent import monkey
 
 monkey.patch_all()
 import os
-out=commands.getoutput('find ./files -type f')
+out=commands.getoutput('find . -type f')
 
 lines=out.split("\n")
 
@@ -27,7 +27,7 @@ def task():
         name=q.get(block=False)
         if name=="":
             break
-        url = 'http://10.1.5.20:8080/upload'
+        url = 'http://10.1.5.20:8080/group1/upload'
         files = {'file': open(name, 'rb')}
         options = {'output': 'json', 'path': '', 'scene': ''}
         try:

doc/benchmark.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"github.com/astaxie/beego/httplib"
+	"github.com/sjqzhang/goutil"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"time"
+)
+
+var url *string
+var dir *string
+var worker *int
+var queue chan string
+var filesize *int
+var filecount *int
+var gen *bool
+var done chan bool=make(chan bool,1)
+var wg sync.WaitGroup=sync.WaitGroup{}
+var util goutil.Common
+
+func init()  {
+	util=goutil.Common{}
+}
+
+func getDir(dir string) []string  {
+    cmd:=exec.Command("find",dir,"-type","f")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	if err:=cmd.Run();err!=nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	return strings.Split( out.String(),"\n")
+}
+
+func sendFile()  {
+        for {
+        	filePath:= <-queue
+			req := httplib.Post(*url)
+			req.PostFile("file", filePath) //注意不是全路径
+			req.Param("output", "text")
+			req.Param("scene", "")
+			req.Param("path", "")
+			if s,err:=req.String();err!=nil {
+				fmt.Println(err,filePath)
+			} else {
+				fmt.Println(s,filePath)
+			}
+			wg.Done()
+		}
+
+}
+
+func genMemFile() []byte {
+	bufstr:=bytes.Buffer{}
+	bufstr.Grow(*filesize)
+	for i:=0;i<*filesize;i++ {
+		bufstr.Write([]byte("a"))
+	}
+	return bufstr.Bytes()
+}
+
+func genFile()  {
+	j:=0
+
+    buf:=genMemFile()
+	for i:=0;i<*filecount;i++ {
+		if i%1000==0 {
+			j=i
+			os.Mkdir(fmt.Sprintf("%d",i),0666)
+		}
+		uuid:=util.GetUUID()
+		for k:=0;k<len(uuid);k++{
+			buf[k]=uuid[k]
+		}
+		ioutil.WriteFile(fmt.Sprintf("%d/%d.txt",j,i),buf,0666)
+	}
+}
+
+func startWorker()  {
+	for i:=0;i<*worker;i++ {
+    	go sendFile()
+	}
+}
+
+func main()  {
+	url=flag.String("url", "http://127.0.0.1:8080/group1/upload", "url")
+	dir=flag.String("dir", "./", "dir to upload")
+	worker=flag.Int("worker", 100, "num of worker")
+	filesize=flag.Int("filesize", 1024*1024, "file of size")
+	filecount=flag.Int("filecount", 1000000, "file of count")
+	gen=flag.Bool("gen", false, "is gen file")
+	flag.Parse()
+	st:=time.Now()
+	if *gen {
+		genFile()
+		fmt.Println(time.Since(st))
+		os.Exit(0)
+	}
+	files:=getDir(*dir)
+	wg.Add(len(files))
+	queue=make(chan string,len(files))
+	for i:=0;i<len(files);i++{
+		queue<-files[i]
+	}
+	startWorker()
+	wg.Wait()
+	fmt.Println(time.Since(st))
+}

fileserver.go

@@ -313,6 +313,7 @@ type GloablConfig struct {
 	SyncWorker           int      `json:"sync_worker"`
 	UploadWorker         int      `json:"upload_worker"`
 	UploadQueueSize      int      `json:"upload_queue_size"`
+	RetryCount           int      `json:"retry_count"`
 }
 type FileInfoResult struct {
 	Name    string `json:"name"`
@@ -809,7 +810,7 @@ func (this *Server) DownloadFromPeer(peer string, fileInfo *FileInfo) {
 		log.Warn("ReadOnly", fileInfo)
 		return
 	}
-	if fileInfo.retry > 5 {
+	if Config().RetryCount > 0 && fileInfo.retry >= Config().RetryCount {
 		log.Error("DownloadFromPeer Error ", fileInfo)
 		return
 	} else {
@@ -1476,7 +1477,7 @@ func (this *Server) postFileToPeer(fileInfo *FileInfo) {
 		b.Param("fileInfo", string(data))
 		result, err = b.String()
 		if err != nil {
-			if fileInfo.retry < 5 {
+			if fileInfo.retry <= Config().RetryCount {
 				fileInfo.retry = fileInfo.retry + 1
 				this.AppendToQueue(fileInfo)
 			}
@@ -3082,7 +3083,7 @@ func (this *Server) CheckClusterStatus() {
 			req = httplib.Get(fmt.Sprintf("%s%s", peer, this.getRequestURI("status")))
 			req.SetTimeout(time.Second*5, time.Second*5)
 			err = req.ToJSON(&status)
-			if status.Status != "ok" {
+			if err != nil || status.Status != "ok" {
 				for _, to := range Config().AlarmReceivers {
 					subject = "fastdfs server error"
 					if err != nil {
@@ -4079,7 +4080,9 @@ func (this *Server) initComponent(isReload bool) {
 	if Config().UploadQueueSize == 0 {
 		Config().UploadQueueSize = 200
 	}
-
+	if Config().RetryCount == 0 {
+		Config().RetryCount = 3
+	}
 }
 
 type HttpHandler struct {

gen_file.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 import os
+import random
 j=0
 for i in range(0,1000000):
     if i%1000==0:
         j=i
         os.system('mkdir -p %s'%(i))
     with open('%s/%s.txt'%(j,i),'w+') as f:
-        f.write(str(i))
+        f.write(str(random.random())*100000)