mirror of
https://gitee.com/sjqzhang/go-fastdfs.git
synced 2024-12-02 03:08:11 +08:00
add fileserver test
This commit is contained in:
s_jqzhang
parent
9d51158f3f
commit
d921823f58
@ -136,7 +136,7 @@ const (
|
||||
"组号": "用于区别不同的集群(上传或下载)与support_group_upload配合使用,带在下载路径中",
|
||||
"group": "group1",
|
||||
"是否合并小文件": "默认不合并,合并可以解决inode不够用的情况（当前对于小于1M文件）进行合并",
|
||||
"enable_merge_small_file": false,
|
||||
"enable_merge_small_file": true,
|
||||
"重试同步失败文件的时间": "单位秒",
|
||||
"refresh_interval": 1800,
|
||||
"是否自动重命名": "默认不自动重命名,使用原文件名",
|
||||
@ -1631,8 +1631,13 @@ func (this *Server) SaveFileMd5Log(fileInfo *FileInfo, filename string) {
|
||||
tmpFile.WriteString(msg)
|
||||
|
||||
if filename == CONST_FILE_Md5_FILE_NAME {
|
||||
this.SaveFileInfoToLevelDB(fileInfo.Md5, fileInfo)
|
||||
this.SaveFileInfoToLevelDB(this.util.MD5(fullpath), fileInfo)
|
||||
|
||||
if _,err:=this.SaveFileInfoToLevelDB(fileInfo.Md5, fileInfo);err!=nil {
|
||||
log.Error("saveToLevelDB",err,fileInfo)
|
||||
}
|
||||
if _,err=this.SaveFileInfoToLevelDB(this.util.MD5(fullpath), fileInfo);err!=nil {
|
||||
log.Error("saveToLevelDB",err,fileInfo)
|
||||
}
|
||||
this.statMap.AddCountInt64(logDate+"_"+CONST_STAT_FILE_COUNT_KEY, 1)
|
||||
this.statMap.AddCountInt64(logDate+"_"+CONST_STAT_FILE_TOTAL_SIZE_KEY, fileInfo.Size)
|
||||
this.statMap.AddCountInt64(CONST_STAT_FILE_TOTAL_SIZE_KEY, fileInfo.Size)
|
||||
@ -1680,17 +1685,29 @@ func (this *Server) CheckFileExist(w http.ResponseWriter, r *http.Request) {
|
||||
md5sum = r.FormValue("md5")
|
||||
|
||||
if fileInfo, err = this.GetFileInfoFromLevelDB(md5sum); fileInfo != nil {
|
||||
fpath = fileInfo.Path + "/" + fileInfo.Name
|
||||
|
||||
if fileInfo.OffSet!=-1 {
|
||||
if data, err = json.Marshal(fileInfo); err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
w.Write(data)
|
||||
return
|
||||
}
|
||||
fpath = DOCKER_DIR+ fileInfo.Path + "/" + fileInfo.Name
|
||||
if fileInfo.ReName != "" {
|
||||
fpath = fileInfo.Path + "/" + fileInfo.ReName
|
||||
fpath = DOCKER_DIR+ fileInfo.Path + "/" + fileInfo.ReName
|
||||
}
|
||||
if this.util.IsExist(fpath) {
|
||||
if data, err = json.Marshal(fileInfo); err == nil {
|
||||
w.Write(data)
|
||||
return
|
||||
} else {
|
||||
log.Error(err)
|
||||
}
|
||||
} else {
|
||||
this.RemoveKeyFromLevelDB(md5sum) // when file delete,delete from leveldb
|
||||
if fileInfo.OffSet==-1 {
|
||||
this.RemoveKeyFromLevelDB(md5sum) // when file delete,delete from leveldb
|
||||
}
|
||||
}
|
||||
}
|
||||
data, _ = json.Marshal(FileInfo{})
|
||||
@ -1826,7 +1843,6 @@ func (this *Server) SaveFileInfoToLevelDB(key string, fileInfo *FileInfo) (*File
|
||||
return fileInfo, err
|
||||
|
||||
}
|
||||
|
||||
if err = this.ldb.Put([]byte(key), data, nil); err != nil {
|
||||
return fileInfo, err
|
||||
}
|
||||
@ -2546,9 +2562,9 @@ func (this *Server) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
if v, _ := this.GetFileInfoFromLevelDB(fileInfo.Md5); v != nil && v.Md5 != "" {
|
||||
fileResult = this.BuildFileResult(v, r)
|
||||
if Config().RenameFile {
|
||||
os.Remove(fileInfo.Path + "/" + fileInfo.ReName)
|
||||
os.Remove(DOCKER_DIR+ fileInfo.Path + "/" + fileInfo.ReName)
|
||||
} else {
|
||||
os.Remove(fileInfo.Path + "/" + fileInfo.Name)
|
||||
os.Remove(DOCKER_DIR+ fileInfo.Path + "/" + fileInfo.Name)
|
||||
}
|
||||
|
||||
if output == "json" {
|
||||
@ -3616,12 +3632,15 @@ func (this *Server) initTus() {
|
||||
BasePath: bigDir,
|
||||
StoreComposer: composer,
|
||||
NotifyCompleteUploads: true,
|
||||
|
||||
})
|
||||
|
||||
notify := func(handler *tusd.Handler) {
|
||||
for {
|
||||
select {
|
||||
|
||||
case info := <-handler.CompleteUploads:
|
||||
|
||||
log.Info("CompleteUploads", info)
|
||||
name := ""
|
||||
if v, ok := info.MetaData["filename"]; ok {
|
||||
@ -3644,6 +3663,9 @@ func (this *Server) initTus() {
|
||||
log.Error(err)
|
||||
} else {
|
||||
if fi.Md5 != "" {
|
||||
if _,err:=this.SaveFileInfoToLevelDB(info.ID,fi);err!=nil {
|
||||
log.Error(err)
|
||||
}
|
||||
log.Info(fmt.Sprintf("file is found md5:%s", fi.Md5))
|
||||
log.Info("remove file:", oldFullPath)
|
||||
log.Info("remove file:", infoFullPath)
|
||||
|
||||
174
fileserver_test.go
Normal file
174
fileserver_test.go
Normal file
@ -0,0 +1,174 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/astaxie/beego/httplib"
|
||||
"github.com/eventials/go-tus"
|
||||
"github.com/prometheus/common/log"
|
||||
"io/ioutil"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
CONST_SMALL_FILE_NAME="small.txt"
|
||||
CONST_BIG_FILE_NAME="big.txt"
|
||||
CONST_DOWNLOAD_BIG_FILE_NAME="big_dowload.txt"
|
||||
CONST_DOWNLOAD_SMALL_FILE_NAME="small_dowload.txt"
|
||||
)
|
||||
|
||||
var testUtil=Common{}
|
||||
|
||||
|
||||
var testSmallFileMd5=""
|
||||
var testBigFileMd5=""
|
||||
|
||||
func init() {
|
||||
|
||||
var (
|
||||
err error
|
||||
)
|
||||
|
||||
smallBytes:=make([]byte,1025*512)
|
||||
for i:=0;i<len(smallBytes);i++ {
|
||||
smallBytes[i]='a'
|
||||
}
|
||||
bigBytes:=make([]byte,1025*1024*2)
|
||||
for i:=0;i<len(smallBytes);i++ {
|
||||
bigBytes[i]='a'
|
||||
}
|
||||
|
||||
ioutil.WriteFile(CONST_SMALL_FILE_NAME,smallBytes,0664)
|
||||
ioutil.WriteFile(CONST_BIG_FILE_NAME,bigBytes,0664)
|
||||
testSmallFileMd5,err=testUtil.GetFileSumByName(CONST_SMALL_FILE_NAME,"")
|
||||
if err!=nil {
|
||||
// testing.T.Error(err)
|
||||
fmt.Println(err)
|
||||
}
|
||||
testBigFileMd5,err=testUtil.GetFileSumByName(CONST_BIG_FILE_NAME,"")
|
||||
if err!=nil {
|
||||
//testing.T.Error(err)
|
||||
fmt.Println(err)
|
||||
}
|
||||
fmt.Println(CONST_SMALL_FILE_NAME,testSmallFileMd5)
|
||||
fmt.Println(CONST_BIG_FILE_NAME,testBigFileMd5)
|
||||
|
||||
}
|
||||
|
||||
func uploadContinue(t *testing.T) {
|
||||
f, err := os.Open(CONST_BIG_FILE_NAME)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer f.Close()
|
||||
client, err := tus.NewClient("http://127.0.0.1:8080/big/upload/", nil)
|
||||
if err!=nil {
|
||||
t.Error(err)
|
||||
}
|
||||
upload, err := tus.NewUploadFromFile(f)
|
||||
if err!=nil {
|
||||
t.Error(err)
|
||||
}
|
||||
uploader, err := client.CreateUpload(upload)
|
||||
if err!=nil {
|
||||
t.Error(err)
|
||||
}
|
||||
url:=uploader.Url()
|
||||
err=uploader.Upload()
|
||||
time.Sleep(time.Second*1)
|
||||
if err!=nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if err:=httplib.Get(url).ToFile(CONST_DOWNLOAD_BIG_FILE_NAME);err!=nil {
|
||||
t.Error(err)
|
||||
}
|
||||
fmt.Println(url)
|
||||
|
||||
if md5sum,err:= testUtil.GetFileSumByName(CONST_DOWNLOAD_BIG_FILE_NAME,"");md5sum!=testBigFileMd5 {
|
||||
t.Error("uploadContinue download fail")
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func uploadSmall(t *testing.T) {
|
||||
var obj FileResult
|
||||
req:=httplib.Post("http://127.0.0.1:8080/upload")
|
||||
req.PostFile("file",CONST_SMALL_FILE_NAME)
|
||||
req.Param("output","json")
|
||||
req.Param("scene","")
|
||||
req.Param("path","")
|
||||
req.ToJSON(&obj)
|
||||
if obj.Md5!=testSmallFileMd5 {
|
||||
t.Error("file not equal")
|
||||
}else {
|
||||
req=httplib.Get(obj.Url)
|
||||
req.ToFile(CONST_DOWNLOAD_SMALL_FILE_NAME)
|
||||
if md5sum,err:= testUtil.GetFileSumByName(CONST_DOWNLOAD_SMALL_FILE_NAME,"");md5sum!=testSmallFileMd5 {
|
||||
log.Error(err)
|
||||
t.Error("small file not equal")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func uploadLarge(t *testing.T) {
|
||||
var obj FileResult
|
||||
req:=httplib.Post("http://127.0.0.1:8080/upload")
|
||||
req.PostFile("file",CONST_BIG_FILE_NAME)
|
||||
req.Param("output","json")
|
||||
req.Param("scene","")
|
||||
req.Param("path","")
|
||||
req.ToJSON(&obj)
|
||||
if obj.Md5!=testBigFileMd5 {
|
||||
t.Error("file not equal")
|
||||
} else {
|
||||
req=httplib.Get(obj.Url)
|
||||
req.ToFile(CONST_DOWNLOAD_BIG_FILE_NAME)
|
||||
if md5sum,err:= testUtil.GetFileSumByName(CONST_DOWNLOAD_BIG_FILE_NAME,"");md5sum!=testBigFileMd5 {
|
||||
log.Error(err)
|
||||
t.Error("big file not equal")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func checkFileExist(t *testing.T) {
|
||||
var obj FileInfo
|
||||
req:=httplib.Post("http://127.0.0.1:8080/check_file_exist")
|
||||
req.Param("md5",testBigFileMd5)
|
||||
req.ToJSON(&obj)
|
||||
if obj.Md5!=testBigFileMd5 {
|
||||
t.Error("file not equal testBigFileMd5")
|
||||
}
|
||||
|
||||
|
||||
req=httplib.Get("http://127.0.0.1:8080/check_file_exist?md5="+testSmallFileMd5)
|
||||
req.ToJSON(&obj)
|
||||
if obj.Md5!=testSmallFileMd5 {
|
||||
t.Error("file not equal testSmallFileMd5")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
func Test_main(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
}{
|
||||
{"main"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
go main()
|
||||
time.Sleep(time.Second*1)
|
||||
uploadContinue(t)
|
||||
uploadSmall(t)
|
||||
uploadLarge(t)
|
||||
checkFileExist(t)
|
||||
time.Sleep(time.Second*2)
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user