diff --git a/fileserver.go b/fileserver.go index a5ef4ef..a1bcf86 100644 --- a/fileserver.go +++ b/fileserver.go @@ -12,6 +12,8 @@ import ( "github.com/astaxie/beego/httplib" "github.com/deckarep/golang-set" _ "github.com/eventials/go-tus" + "github.com/go-ego/riot" + "github.com/go-ego/riot/types" "github.com/json-iterator/go" "github.com/nfnt/resize" "github.com/sjqzhang/googleAuthenticator" @@ -53,25 +55,27 @@ import ( var staticHandler http.Handler var json = jsoniter.ConfigCompatibleWithStandardLibrary var server *Server +var searcher = riot.Engine{} var logacc log.LoggerInterface var FOLDERS = []string{DATA_DIR, STORE_DIR, CONF_DIR, STATIC_DIR} var CONST_QUEUE_SIZE = 10000 var ( - FileName string - ptr unsafe.Pointer - DOCKER_DIR = "" - STORE_DIR = STORE_DIR_NAME - CONF_DIR = CONF_DIR_NAME - LOG_DIR = LOG_DIR_NAME - DATA_DIR = DATA_DIR_NAME - STATIC_DIR = STATIC_DIR_NAME - LARGE_DIR_NAME = "haystack" - LARGE_DIR = STORE_DIR + "/haystack" - CONST_LEVELDB_FILE_NAME = DATA_DIR + "/fileserver.db" - CONST_LOG_LEVELDB_FILE_NAME = DATA_DIR + "/log.db" - CONST_STAT_FILE_NAME = DATA_DIR + "/stat.json" - CONST_CONF_FILE_NAME = CONF_DIR + "/cfg.json" - logConfigStr = ` + FileName string + ptr unsafe.Pointer + DOCKER_DIR = "" + STORE_DIR = STORE_DIR_NAME + CONF_DIR = CONF_DIR_NAME + LOG_DIR = LOG_DIR_NAME + DATA_DIR = DATA_DIR_NAME + STATIC_DIR = STATIC_DIR_NAME + LARGE_DIR_NAME = "haystack" + LARGE_DIR = STORE_DIR + "/haystack" + CONST_LEVELDB_FILE_NAME = DATA_DIR + "/fileserver.db" + CONST_LOG_LEVELDB_FILE_NAME = DATA_DIR + "/log.db" + CONST_SEARCH_LEVELDB_FILE_NAME = DATA_DIR + "/search.db" + CONST_STAT_FILE_NAME = DATA_DIR + "/stat.json" + CONST_CONF_FILE_NAME = CONF_DIR + "/cfg.json" + logConfigStr = ` @@ -333,6 +337,18 @@ func NewServer() *Server { panic(err) } + opts := types.EngineOpts{ + Using: 1, + IndexerOpts: &types.IndexerOpts{ + IndexType: types.LocsIndex, + }, + UseStore: true, + StoreFolder: CONST_SEARCH_LEVELDB_FILE_NAME, + StoreEngine: "ldb", // bg: badger, lbd: leveldb, bolt: bolt + GseDict: "zh", + //NotUseGse: true, + } + searcher.Init(opts) return server } @@ -1727,6 +1743,8 @@ func (this *Server) saveFileMd5Log(fileInfo *FileInfo, filename string) { fullpath = fileInfo.Path + "/" + outname logKey = fmt.Sprintf("%s_%s_%s", logDate, filename, fileInfo.Md5) if filename == CONST_FILE_Md5_FILE_NAME { + searcher.Index(fileInfo.Md5, types.DocData{Content: fileInfo.Name, Attri: this.util.JsonEncodePretty(fileInfo)}, true) + searcher.Flush() if ok, err = this.IsExistFromLevelDB(fileInfo.Md5, this.ldb); !ok { this.statMap.AddCountInt64(logDate+"_"+CONST_STAT_FILE_COUNT_KEY, 1) this.statMap.AddCountInt64(logDate+"_"+CONST_STAT_FILE_TOTAL_SIZE_KEY, fileInfo.Size) @@ -1744,6 +1762,7 @@ func (this *Server) saveFileMd5Log(fileInfo *FileInfo, filename string) { return } if filename == CONST_REMOME_Md5_FILE_NAME { + searcher.RemoveDoc(fileInfo.Md5) if ok, err = this.IsExistFromLevelDB(fileInfo.Md5, this.ldb); ok { this.statMap.AddCountInt64(logDate+"_"+CONST_STAT_FILE_COUNT_KEY, -1) this.statMap.AddCountInt64(logDate+"_"+CONST_STAT_FILE_TOTAL_SIZE_KEY, -fileInfo.Size) @@ -3028,6 +3047,31 @@ func (this *Server) RepairFileInfo(w http.ResponseWriter, r *http.Request) { go this.RepairFileInfoFromFile() w.Write([]byte(this.util.JsonEncodePretty(result))) } +func (this *Server) Search(w http.ResponseWriter, r *http.Request) { + var ( + kw string + ) + if !this.IsPeer(r) { + w.Write([]byte(this.GetClusterNotPermitMessage(r))) + return + } + kw = r.FormValue("kw") + //strArr := []string{kw} + logic := types.Logic{ + //Should: true, + //Expr: types.Expr{ + // Should: strArr, + // //Must: strArr, + //}, + } + rankOpts := types.RankOpts{ + OutputOffset: 0, + MaxOutputs: 100, + } + req := types.SearchReq{Text: kw, RankOpts: &rankOpts, Logic: logic} + fmt.Println(searcher.SearchDoc(req)) + //fmt.Println(searcher.GetDBAllDocs()) +} func (this *Server) Reload(w http.ResponseWriter, r *http.Request) { var ( err error @@ -3429,6 +3473,7 @@ func init() { LARGE_DIR = STORE_DIR + "/haystack" CONST_LEVELDB_FILE_NAME = DATA_DIR + "/fileserver.db" CONST_LOG_LEVELDB_FILE_NAME = DATA_DIR + "/log.db" + CONST_SEARCH_LEVELDB_FILE_NAME = DATA_DIR + "/search.db" CONST_STAT_FILE_NAME = DATA_DIR + "/stat.json" CONST_CONF_FILE_NAME = CONF_DIR + "/cfg.json" FOLDERS = []string{DATA_DIR, STORE_DIR, CONF_DIR, STATIC_DIR} @@ -3820,6 +3865,7 @@ func (this *Server) Main() { http.HandleFunc(fmt.Sprintf("%s/remove_empty_dir", groupRoute), this.RemoveEmptyDir) http.HandleFunc(fmt.Sprintf("%s/repair_fileinfo", groupRoute), this.RepairFileInfo) http.HandleFunc(fmt.Sprintf("%s/reload", groupRoute), this.Reload) + http.HandleFunc(fmt.Sprintf("%s/search", groupRoute), this.Search) http.HandleFunc(fmt.Sprintf("%s/syncfile_info", groupRoute), this.SyncFileInfo) http.HandleFunc(fmt.Sprintf("%s/get_md5s_by_date", groupRoute), this.GetMd5sForWeb) http.HandleFunc(fmt.Sprintf("%s/receive_md5s", groupRoute), this.ReceiveMd5s)