From d921823f58123d39915bb4cb77b9738433d40a56 Mon Sep 17 00:00:00 2001
From: s_jqzhang <s_jqzhang@163.com>
Date: Wed, 20 Feb 2019 17:32:13 +0800
Subject: [PATCH] add fileserver test

---
 fileserver.go      |  40 ++++++++---
 fileserver_test.go | 174 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 205 insertions(+), 9 deletions(-)
 create mode 100644 fileserver_test.go

diff --git a/fileserver.go b/fileserver.go
index 4ddc1cc..98e7d35 100644
--- a/fileserver.go
+++ b/fileserver.go
@@ -136,7 +136,7 @@ const (
 	"组号": "用于区别不同的集群(上传或下载)与support_group_upload配合使用,带在下载路径中",
 	"group": "group1",
 	"是否合并小文件": "默认不合并,合并可以解决inode不够用的情况（当前对于小于1M文件）进行合并",
-	"enable_merge_small_file": false,
+	"enable_merge_small_file": true,
 	"重试同步失败文件的时间": "单位秒",
 	"refresh_interval": 1800,
 	"是否自动重命名": "默认不自动重命名,使用原文件名",
@@ -1631,8 +1631,13 @@ func (this *Server) SaveFileMd5Log(fileInfo *FileInfo, filename string) {
 	tmpFile.WriteString(msg)
 
 	if filename == CONST_FILE_Md5_FILE_NAME {
-		this.SaveFileInfoToLevelDB(fileInfo.Md5, fileInfo)
-		this.SaveFileInfoToLevelDB(this.util.MD5(fullpath), fileInfo)
+
+		if _,err:=this.SaveFileInfoToLevelDB(fileInfo.Md5, fileInfo);err!=nil {
+			log.Error("saveToLevelDB",err,fileInfo)
+		}
+		if _,err=this.SaveFileInfoToLevelDB(this.util.MD5(fullpath), fileInfo);err!=nil {
+			log.Error("saveToLevelDB",err,fileInfo)
+		}
 		this.statMap.AddCountInt64(logDate+"_"+CONST_STAT_FILE_COUNT_KEY, 1)
 		this.statMap.AddCountInt64(logDate+"_"+CONST_STAT_FILE_TOTAL_SIZE_KEY, fileInfo.Size)
 		this.statMap.AddCountInt64(CONST_STAT_FILE_TOTAL_SIZE_KEY, fileInfo.Size)
@@ -1680,17 +1685,29 @@ func (this *Server) CheckFileExist(w http.ResponseWriter, r *http.Request) {
 	md5sum = r.FormValue("md5")
 
 	if fileInfo, err = this.GetFileInfoFromLevelDB(md5sum); fileInfo != nil {
-		fpath = fileInfo.Path + "/" + fileInfo.Name
+
+		if fileInfo.OffSet!=-1 {
+			if data, err = json.Marshal(fileInfo); err != nil {
+				log.Error(err)
+			}
+			w.Write(data)
+			return
+		}
+		fpath = DOCKER_DIR+ fileInfo.Path + "/" + fileInfo.Name
 		if fileInfo.ReName != "" {
-			fpath = fileInfo.Path + "/" + fileInfo.ReName
+			fpath = DOCKER_DIR+ fileInfo.Path + "/" + fileInfo.ReName
 		}
 		if this.util.IsExist(fpath) {
 			if data, err = json.Marshal(fileInfo); err == nil {
 				w.Write(data)
 				return
+			} else {
+				log.Error(err)
 			}
 		} else {
-			this.RemoveKeyFromLevelDB(md5sum) // when file delete,delete from leveldb
+			if fileInfo.OffSet==-1 {
+				this.RemoveKeyFromLevelDB(md5sum) // when file delete,delete from leveldb
+			}
 		}
 	}
 	data, _ = json.Marshal(FileInfo{})
@@ -1826,7 +1843,6 @@ func (this *Server) SaveFileInfoToLevelDB(key string, fileInfo *FileInfo) (*File
 		return fileInfo, err
 
 	}
-
 	if err = this.ldb.Put([]byte(key), data, nil); err != nil {
 		return fileInfo, err
 	}
@@ -2546,9 +2562,9 @@ func (this *Server) Upload(w http.ResponseWriter, r *http.Request) {
 		if v, _ := this.GetFileInfoFromLevelDB(fileInfo.Md5); v != nil && v.Md5 != "" {
 			fileResult = this.BuildFileResult(v, r)
 			if Config().RenameFile {
-				os.Remove(fileInfo.Path + "/" + fileInfo.ReName)
+				os.Remove(DOCKER_DIR+ fileInfo.Path + "/" + fileInfo.ReName)
 			} else {
-				os.Remove(fileInfo.Path + "/" + fileInfo.Name)
+				os.Remove(DOCKER_DIR+ fileInfo.Path + "/" + fileInfo.Name)
 			}
 
 			if output == "json" {
@@ -3616,12 +3632,15 @@ func (this *Server) initTus() {
 		BasePath:              bigDir,
 		StoreComposer:         composer,
 		NotifyCompleteUploads: true,
+
 	})
 
 	notify := func(handler *tusd.Handler) {
 		for {
 			select {
+
 			case info := <-handler.CompleteUploads:
+
 				log.Info("CompleteUploads", info)
 				name := ""
 				if v, ok := info.MetaData["filename"]; ok {
@@ -3644,6 +3663,9 @@ func (this *Server) initTus() {
 					log.Error(err)
 				} else {
 					if fi.Md5 != "" {
+						if _,err:=this.SaveFileInfoToLevelDB(info.ID,fi);err!=nil {
+							log.Error(err)
+						}
 						log.Info(fmt.Sprintf("file is found md5:%s", fi.Md5))
 						log.Info("remove file:", oldFullPath)
 						log.Info("remove file:", infoFullPath)
diff --git a/fileserver_test.go b/fileserver_test.go
new file mode 100644
index 0000000..3e3d476
--- /dev/null
+++ b/fileserver_test.go
@@ -0,0 +1,174 @@
+package main
+
+import (
+	"fmt"
+	"github.com/astaxie/beego/httplib"
+	"github.com/eventials/go-tus"
+	"github.com/prometheus/common/log"
+	"io/ioutil"
+	_ "net/http/pprof"
+	"os"
+	"testing"
+	"time"
+)
+
+const (
+	CONST_SMALL_FILE_NAME="small.txt"
+	CONST_BIG_FILE_NAME="big.txt"
+	CONST_DOWNLOAD_BIG_FILE_NAME="big_dowload.txt"
+	CONST_DOWNLOAD_SMALL_FILE_NAME="small_dowload.txt"
+)
+
+var testUtil=Common{}
+
+
+var testSmallFileMd5=""
+var testBigFileMd5=""
+
+func init()  {
+
+	var (
+		err error
+	)
+
+	smallBytes:=make([]byte,1025*512)
+	for i:=0;i<len(smallBytes);i++ {
+		smallBytes[i]='a'
+	}
+	bigBytes:=make([]byte,1025*1024*2)
+	for i:=0;i<len(smallBytes);i++ {
+		bigBytes[i]='a'
+	}
+
+	ioutil.WriteFile(CONST_SMALL_FILE_NAME,smallBytes,0664)
+	ioutil.WriteFile(CONST_BIG_FILE_NAME,bigBytes,0664)
+	testSmallFileMd5,err=testUtil.GetFileSumByName(CONST_SMALL_FILE_NAME,"")
+	if err!=nil {
+	//	testing.T.Error(err)
+	fmt.Println(err)
+	}
+	testBigFileMd5,err=testUtil.GetFileSumByName(CONST_BIG_FILE_NAME,"")
+	if err!=nil {
+		//testing.T.Error(err)
+		fmt.Println(err)
+	}
+	fmt.Println(CONST_SMALL_FILE_NAME,testSmallFileMd5)
+	fmt.Println(CONST_BIG_FILE_NAME,testBigFileMd5)
+
+}
+
+func uploadContinue(t *testing.T)  {
+	f, err := os.Open(CONST_BIG_FILE_NAME)
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+	client, err := tus.NewClient("http://127.0.0.1:8080/big/upload/", nil)
+	if err!=nil {
+		t.Error(err)
+	}
+	upload, err := tus.NewUploadFromFile(f)
+	if err!=nil {
+		t.Error(err)
+	}
+	uploader, err := client.CreateUpload(upload)
+	if err!=nil {
+		t.Error(err)
+	}
+	url:=uploader.Url()
+    err=uploader.Upload()
+	time.Sleep(time.Second*1)
+	if err!=nil {
+		t.Error(err)
+	}
+    if err:=httplib.Get(url).ToFile(CONST_DOWNLOAD_BIG_FILE_NAME);err!=nil {
+    	t.Error(err)
+	}
+    fmt.Println(url)
+
+    if md5sum,err:= testUtil.GetFileSumByName(CONST_DOWNLOAD_BIG_FILE_NAME,"");md5sum!=testBigFileMd5 {
+    	t.Error("uploadContinue download fail")
+    	t.Error(err)
+	}
+
+}
+
+func uploadSmall(t *testing.T)  {
+	var obj FileResult
+	req:=httplib.Post("http://127.0.0.1:8080/upload")
+	req.PostFile("file",CONST_SMALL_FILE_NAME)
+	req.Param("output","json")
+	req.Param("scene","")
+	req.Param("path","")
+	req.ToJSON(&obj)
+	if obj.Md5!=testSmallFileMd5 {
+		t.Error("file not equal")
+	}else {
+		req=httplib.Get(obj.Url)
+		req.ToFile(CONST_DOWNLOAD_SMALL_FILE_NAME)
+		if md5sum,err:= testUtil.GetFileSumByName(CONST_DOWNLOAD_SMALL_FILE_NAME,"");md5sum!=testSmallFileMd5 {
+			log.Error(err)
+			t.Error("small file not equal")
+		}
+	}
+}
+
+func uploadLarge(t *testing.T)  {
+	var obj FileResult
+	req:=httplib.Post("http://127.0.0.1:8080/upload")
+	req.PostFile("file",CONST_BIG_FILE_NAME)
+	req.Param("output","json")
+	req.Param("scene","")
+	req.Param("path","")
+	req.ToJSON(&obj)
+	if obj.Md5!=testBigFileMd5 {
+		t.Error("file not equal")
+	} else {
+		req=httplib.Get(obj.Url)
+		req.ToFile(CONST_DOWNLOAD_BIG_FILE_NAME)
+		if md5sum,err:= testUtil.GetFileSumByName(CONST_DOWNLOAD_BIG_FILE_NAME,"");md5sum!=testBigFileMd5 {
+			log.Error(err)
+			t.Error("big file not equal")
+		}
+	}
+}
+
+func checkFileExist(t *testing.T)  {
+	var obj FileInfo
+	req:=httplib.Post("http://127.0.0.1:8080/check_file_exist")
+	req.Param("md5",testBigFileMd5)
+	req.ToJSON(&obj)
+	if obj.Md5!=testBigFileMd5 {
+		t.Error("file not equal testBigFileMd5")
+	}
+
+
+	req=httplib.Get("http://127.0.0.1:8080/check_file_exist?md5="+testSmallFileMd5)
+	req.ToJSON(&obj)
+	if obj.Md5!=testSmallFileMd5 {
+		t.Error("file not equal testSmallFileMd5")
+	}
+}
+
+
+
+func Test_main(t *testing.T) {
+
+	tests := []struct {
+		name string
+	}{
+     {"main"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+		 go	main()
+		 time.Sleep(time.Second*1)
+		 uploadContinue(t)
+		 uploadSmall(t)
+		 uploadLarge(t)
+         checkFileExist(t)
+         time.Sleep(time.Second*2)
+		})
+	}
+}