test/go-fastdfs
1
0
Fork 0
mirror of https://gitee.com/sjqzhang/go-fastdfs.git synced 2024-11-29 17:57:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add search

Browse Source
This commit is contained in:
s_jqzhang 2019-04-22 15:59:51 +08:00
parent d662c3d34e
commit 3e8f26d40a
1 changed files with 61 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
fileserver.go
View File

@ -12,6 +12,8 @@ import (
"github.com/astaxie/beego/httplib"
"github.com/deckarep/golang-set"
_ "github.com/eventials/go-tus"
"github.com/go-ego/riot"
"github.com/go-ego/riot/types"
"github.com/json-iterator/go"
"github.com/nfnt/resize"
"github.com/sjqzhang/googleAuthenticator"
@ -53,25 +55,27 @@ import (
var staticHandler http.Handler
var json = jsoniter.ConfigCompatibleWithStandardLibrary
var server *Server
var searcher = riot.Engine{}
var logacc log.LoggerInterface
var FOLDERS = []string{DATA_DIR, STORE_DIR, CONF_DIR, STATIC_DIR}
var CONST_QUEUE_SIZE = 10000
var (
FileName string
ptr unsafe.Pointer
DOCKER_DIR = ""
STORE_DIR = STORE_DIR_NAME
CONF_DIR = CONF_DIR_NAME
LOG_DIR = LOG_DIR_NAME
DATA_DIR = DATA_DIR_NAME
STATIC_DIR = STATIC_DIR_NAME
LARGE_DIR_NAME = "haystack"
LARGE_DIR = STORE_DIR + "/haystack"
CONST_LEVELDB_FILE_NAME = DATA_DIR + "/fileserver.db"
CONST_LOG_LEVELDB_FILE_NAME = DATA_DIR + "/log.db"
CONST_STAT_FILE_NAME = DATA_DIR + "/stat.json"
CONST_CONF_FILE_NAME = CONF_DIR + "/cfg.json"
logConfigStr = `
FileName string
ptr unsafe.Pointer
DOCKER_DIR = ""
STORE_DIR = STORE_DIR_NAME
CONF_DIR = CONF_DIR_NAME
LOG_DIR = LOG_DIR_NAME
DATA_DIR = DATA_DIR_NAME
STATIC_DIR = STATIC_DIR_NAME
LARGE_DIR_NAME = "haystack"
LARGE_DIR = STORE_DIR + "/haystack"
CONST_LEVELDB_FILE_NAME = DATA_DIR + "/fileserver.db"
CONST_LOG_LEVELDB_FILE_NAME = DATA_DIR + "/log.db"
CONST_SEARCH_LEVELDB_FILE_NAME = DATA_DIR + "/search.db"
CONST_STAT_FILE_NAME = DATA_DIR + "/stat.json"
CONST_CONF_FILE_NAME = CONF_DIR + "/cfg.json"
logConfigStr = `
<seelog type="asynctimer" asyncinterval="1000" minlevel="trace" maxlevel="error">
<outputs formatid="common">
<buffered formatid="common" size="1048576" flushperiod="1000">
@ -333,6 +337,18 @@ func NewServer() *Server {
panic(err)
}
opts := types.EngineOpts{
Using: 1,
IndexerOpts: &types.IndexerOpts{
IndexType: types.LocsIndex,
},
UseStore: true,
StoreFolder: CONST_SEARCH_LEVELDB_FILE_NAME,
StoreEngine: "ldb", // bg: badger, lbd: leveldb, bolt: bolt
GseDict: "zh",
//NotUseGse: true,
}
searcher.Init(opts)
return server
}
@ -1727,6 +1743,8 @@ func (this *Server) saveFileMd5Log(fileInfo *FileInfo, filename string) {
fullpath = fileInfo.Path + "/" + outname
logKey = fmt.Sprintf("%s_%s_%s", logDate, filename, fileInfo.Md5)
if filename == CONST_FILE_Md5_FILE_NAME {
searcher.Index(fileInfo.Md5, types.DocData{Content: fileInfo.Name, Attri: this.util.JsonEncodePretty(fileInfo)}, true)
searcher.Flush()
if ok, err = this.IsExistFromLevelDB(fileInfo.Md5, this.ldb); !ok {
this.statMap.AddCountInt64(logDate+"_"+CONST_STAT_FILE_COUNT_KEY, 1)
this.statMap.AddCountInt64(logDate+"_"+CONST_STAT_FILE_TOTAL_SIZE_KEY, fileInfo.Size)
@ -1744,6 +1762,7 @@ func (this *Server) saveFileMd5Log(fileInfo *FileInfo, filename string) {
return
}
if filename == CONST_REMOME_Md5_FILE_NAME {
searcher.RemoveDoc(fileInfo.Md5)
if ok, err = this.IsExistFromLevelDB(fileInfo.Md5, this.ldb); ok {
this.statMap.AddCountInt64(logDate+"_"+CONST_STAT_FILE_COUNT_KEY, -1)
this.statMap.AddCountInt64(logDate+"_"+CONST_STAT_FILE_TOTAL_SIZE_KEY, -fileInfo.Size)
@ -3028,6 +3047,31 @@ func (this *Server) RepairFileInfo(w http.ResponseWriter, r *http.Request) {
go this.RepairFileInfoFromFile()
w.Write([]byte(this.util.JsonEncodePretty(result)))
}
func (this *Server) Search(w http.ResponseWriter, r *http.Request) {
var (
kw string
)
if !this.IsPeer(r) {
w.Write([]byte(this.GetClusterNotPermitMessage(r)))
return
}
kw = r.FormValue("kw")
//strArr := []string{kw}
logic := types.Logic{
//Should: true,
//Expr: types.Expr{
// Should: strArr,
// //Must: strArr,
//},
}
rankOpts := types.RankOpts{
OutputOffset: 0,
MaxOutputs: 100,
}
req := types.SearchReq{Text: kw, RankOpts: &rankOpts, Logic: logic}
fmt.Println(searcher.SearchDoc(req))
//fmt.Println(searcher.GetDBAllDocs())
}
func (this *Server) Reload(w http.ResponseWriter, r *http.Request) {
var (
err error
@ -3429,6 +3473,7 @@ func init() {
LARGE_DIR = STORE_DIR + "/haystack"
CONST_LEVELDB_FILE_NAME = DATA_DIR + "/fileserver.db"
CONST_LOG_LEVELDB_FILE_NAME = DATA_DIR + "/log.db"
CONST_SEARCH_LEVELDB_FILE_NAME = DATA_DIR + "/search.db"
CONST_STAT_FILE_NAME = DATA_DIR + "/stat.json"
CONST_CONF_FILE_NAME = CONF_DIR + "/cfg.json"
FOLDERS = []string{DATA_DIR, STORE_DIR, CONF_DIR, STATIC_DIR}
@ -3820,6 +3865,7 @@ func (this *Server) Main() {
http.HandleFunc(fmt.Sprintf("%s/remove_empty_dir", groupRoute), this.RemoveEmptyDir)
http.HandleFunc(fmt.Sprintf("%s/repair_fileinfo", groupRoute), this.RepairFileInfo)
http.HandleFunc(fmt.Sprintf("%s/reload", groupRoute), this.Reload)
http.HandleFunc(fmt.Sprintf("%s/search", groupRoute), this.Search)
http.HandleFunc(fmt.Sprintf("%s/syncfile_info", groupRoute), this.SyncFileInfo)
http.HandleFunc(fmt.Sprintf("%s/get_md5s_by_date", groupRoute), this.GetMd5sForWeb)
http.HandleFunc(fmt.Sprintf("%s/receive_md5s", groupRoute), this.ReceiveMd5s)