test/go-fastdfs
1
0
Fork 0
mirror of https://gitee.com/sjqzhang/go-fastdfs.git synced 2024-12-02 11:17:39 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

optimizer code

Browse Source
This commit is contained in:
s_jqzhang 2019-01-16 10:28:51 +08:00
parent 4628bc265f
commit ddf5838141
1 changed files with 94 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

159
fileserver.go
View File

@ -145,7 +145,7 @@ type Common struct {
} }
type Server struct { type Server struct {
db *leveldb.DB ldb *leveldb.DB
util *Common util *Common
} }
@ -627,7 +627,7 @@ func (this *Server) Download(w http.ResponseWriter, r *http.Request) {
go this.DownloadFromPeer(peer, fileInfo) go this.DownloadFromPeer(peer, fileInfo)
http.Redirect(w, r, peer+r.RequestURI, 302) http.Redirect(w, r, peer+r.RequestURI, 302)
break return
} }
} }
@ -762,7 +762,7 @@ func (this *Server) postFileToPeer(fileInfo *FileInfo, write_log bool) {
log.Error(err) log.Error(err)
return return
} }
this.db.Put([]byte(fileInfo.Md5), data, nil) this.ldb.Put([]byte(fileInfo.Md5), data, nil)
} }
} }
@ -810,7 +810,7 @@ func (this *Server) GetFileInfoByMd5(md5sum string) (*FileInfo, error) {
fileInfo FileInfo fileInfo FileInfo
) )
if data, err = this.db.Get([]byte(md5sum), nil); err != nil { if data, err = this.ldb.Get([]byte(md5sum), nil); err != nil {
return nil, err return nil, err
} else { } else {
if err = json.Unmarshal(data, &fileInfo); err == nil { if err = json.Unmarshal(data, &fileInfo); err == nil {
@ -893,7 +893,7 @@ func (this *Server) Sync(w http.ResponseWriter, r *http.Request) {
if len(r.Form["date"]) > 0 { if len(r.Form["date"]) > 0 {
date = r.Form["date"][0] date = r.Form["date"][0]
} else { } else {
w.Write([]byte("require paramete date , date?=20181230")) w.Write([]byte("require paramete date &force , date?=20181230"))
return return
} }
date = strings.Replace(date, ".", "", -1) date = strings.Replace(date, ".", "", -1)
@ -920,7 +920,7 @@ func (this *Server) GetFileInfoFromLevelDB(key string) (*FileInfo, error) {
fileInfo FileInfo fileInfo FileInfo
) )
if data, err = this.db.Get([]byte(key), nil); err != nil { if data, err = this.ldb.Get([]byte(key), nil); err != nil {
return nil, err return nil, err
} }
@ -933,23 +933,42 @@ func (this *Server) GetFileInfoFromLevelDB(key string) (*FileInfo, error) {
func (this *Server) SaveStat() { func (this *Server) SaveStat() {
stat := statMap.Get() SaveStatFunc := func() {
if v, ok := stat[CONST_STAT_FILE_TOTAL_SIZE_KEY]; ok {
switch v.(type) {
case int64:
if v.(int64) > 0 {
if data, err := json.Marshal(stat); err != nil {
log.Error(err)
} else {
this.util.WriteBinFile(CONST_STAT_FILE_NAME, data)
}
defer func() {
if re := recover(); re != nil {
buffer := debug.Stack()
log.Error("SaveStatFunc")
log.Error(re)
log.Error(string(buffer))
} }
}()
stat := statMap.Get()
if v, ok := stat[CONST_STAT_FILE_TOTAL_SIZE_KEY]; ok {
switch v.(type) {
case int64:
if v.(int64) > 0 {
if data, err := json.Marshal(stat); err != nil {
log.Error(err)
} else {
this.util.WriteBinFile(CONST_STAT_FILE_NAME, data)
}
}
}
} }
} }
for {
time.Sleep(time.Minute * 1)
SaveStatFunc()
}
} }
func (this *Server) SaveFileInfoToLevelDB(key string, fileInfo *FileInfo) (*FileInfo, error) { func (this *Server) SaveFileInfoToLevelDB(key string, fileInfo *FileInfo) (*FileInfo, error) {
@ -964,7 +983,7 @@ func (this *Server) SaveFileInfoToLevelDB(key string, fileInfo *FileInfo) (*File
} }
if err = this.db.Put([]byte(key), data, nil); err != nil { if err = this.ldb.Put([]byte(key), data, nil); err != nil {
return fileInfo, err return fileInfo, err
} }
@ -1416,7 +1435,7 @@ func (this *Server) Upload(w http.ResponseWriter, r *http.Request) {
statMap.AddCountInt64(this.util.GetToDay()+"_"+CONST_STAT_FILE_COUNT_KEY, 1) statMap.AddCountInt64(this.util.GetToDay()+"_"+CONST_STAT_FILE_COUNT_KEY, 1)
} }
this.SaveStat() // this.SaveStat()
this.SaveFileMd5Log(&fileInfo, CONST_FILE_Md5_FILE_NAME) this.SaveFileMd5Log(&fileInfo, CONST_FILE_Md5_FILE_NAME)
@ -1497,7 +1516,7 @@ func (this *Server) BenchMark(w http.ResponseWriter, r *http.Request) {
if i%10000 == 0 { if i%10000 == 0 {
if batch.Len() > 0 { if batch.Len() > 0 {
server.db.Write(batch, nil) server.ldb.Write(batch, nil)
// batch = new(leveldb.Batch) // batch = new(leveldb.Batch)
batch.Reset() batch.Reset()
} }
@ -1589,6 +1608,7 @@ func (this *Server) RegisterExit() {
switch s { switch s {
case syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT: case syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT:
this.SaveStat() this.SaveStat()
this.ldb.Close()
log.Info("Exit", s) log.Info("Exit", s)
os.Exit(1) os.Exit(1)
} }
@ -1754,13 +1774,13 @@ func init() {
staticHandler = http.StripPrefix("/"+Config().Group+"/", http.FileServer(http.Dir(STORE_DIR))) staticHandler = http.StripPrefix("/"+Config().Group+"/", http.FileServer(http.Dir(STORE_DIR)))
initComponent() initComponent(false)
} }
func initComponent() { func initComponent(is_reload bool) {
var ( var (
err error err error
db *leveldb.DB ldb *leveldb.DB
ip string ip string
stat map[string]interface{} stat map[string]interface{}
data []byte data []byte
@ -1780,43 +1800,52 @@ func initComponent() {
} }
} }
Config().Peers = peers Config().Peers = peers
if !is_reload {
db, err = leveldb.OpenFile(CONST_LEVELDB_FILE_NAME, nil) ldb, err = leveldb.OpenFile(CONST_LEVELDB_FILE_NAME, nil)
if err != nil { if err != nil {
log.Error(err)
panic(err)
}
server.db = db
if util.FileExists(CONST_STAT_FILE_NAME) {
if data, err = util.ReadBinFile(CONST_STAT_FILE_NAME); err != nil {
log.Error(err) log.Error(err)
} else { panic(err)
}
server.ldb = ldb
}
if err = json.Unmarshal(data, &stat); err != nil { FormatStatInfo := func() {
if util.FileExists(CONST_STAT_FILE_NAME) {
if data, err = util.ReadBinFile(CONST_STAT_FILE_NAME); err != nil {
log.Error(err) log.Error(err)
} else { } else {
for k, v := range stat {
switch v.(type) {
case float64:
vv := strings.Split(fmt.Sprintf("%f", v), ".")[0]
if count, err = strconv.ParseInt(vv, 10, 64); err != nil { if err = json.Unmarshal(data, &stat); err != nil {
log.Error(err) log.Error(err)
} else { } else {
statMap.Put(k, count) for k, v := range stat {
switch v.(type) {
case float64:
vv := strings.Split(fmt.Sprintf("%f", v), ".")[0]
if count, err = strconv.ParseInt(vv, 10, 64); err != nil {
log.Error(err)
} else {
statMap.Put(k, count)
}
default:
statMap.Put(k, v)
} }
default:
statMap.Put(k, v)
} }
} }
} }
} }
} }
if !is_reload {
FormatStatInfo()
}
//Timer
} }
@ -1854,35 +1883,35 @@ func (HttpHandler) ServeHTTP(res http.ResponseWriter, req *http.Request) {
http.DefaultServeMux.ServeHTTP(res, req) http.DefaultServeMux.ServeHTTP(res, req)
} }
func main() { func (this *Server) Main() {
if !util.FileExists(STORE_DIR) {
os.Mkdir(STORE_DIR, 0777)
}
go func() { go func() {
for { for {
server.CheckFileAndSendToPeer("", false) this.CheckFileAndSendToPeer("", false)
time.Sleep(time.Second * time.Duration(Config().RefreshInterval)) time.Sleep(time.Second * time.Duration(Config().RefreshInterval))
util.RemoveEmptyDir(STORE_DIR) util.RemoveEmptyDir(STORE_DIR)
server.SaveStat()
} }
}() }()
go this.SaveStat()
go this.Check()
go server.Check() http.HandleFunc("/", this.Index)
http.HandleFunc("/check_file_exist", this.CheckFileExist)
http.HandleFunc("/", server.Index) http.HandleFunc("/upload", this.Upload)
http.HandleFunc("/check_file_exist", server.CheckFileExist) http.HandleFunc("/delete", this.RemoveFile)
http.HandleFunc("/upload", server.Upload) http.HandleFunc("/sync", this.Sync)
http.HandleFunc("/delete", server.RemoveFile) http.HandleFunc("/stat", this.Stat)
http.HandleFunc("/sync", server.Sync) http.HandleFunc("/status", this.Status)
http.HandleFunc("/stat", server.Stat) http.HandleFunc("/syncfile", this.SyncFile)
http.HandleFunc("/status", server.Status) http.HandleFunc("/"+Config().Group+"/", this.Download)
http.HandleFunc("/syncfile", server.SyncFile)
http.HandleFunc("/"+Config().Group+"/", server.Download)
fmt.Println("Listen on " + Config().Addr) fmt.Println("Listen on " + Config().Addr)
err := http.ListenAndServe(Config().Addr, new(HttpHandler)) err := http.ListenAndServe(Config().Addr, new(HttpHandler))
log.Error(err) log.Error(err)
fmt.Println(err) fmt.Println(err)
}
func main() {
server.Main()
} }