mirror of
https://gitee.com/sjqzhang/go-fastdfs.git
synced 2024-12-02 11:17:39 +08:00
add api doc and repair fileinfo
This commit is contained in:
s_jqzhang
parent
405355f67a
commit
97546ca239
11
README.md
11
README.md
@ -4,7 +4,7 @@
|
||||
|
||||
|
||||
|
||||
# go-fastdfs是一个基于http的分布式文件系统，它基于大道至简的设计理念，一切从简设计，使得它的运维及扩展变得更加简单，它具有高性能、高可靠、无中心、免维护等优点。
|
||||
# go-fastdfs是一个基于http协议的分布式文件系统，它基于大道至简的设计理念，一切从简设计，使得它的运维及扩展变得更加简单，它具有高性能、高可靠、无中心、免维护等优点。
|
||||
|
||||
|
||||
- 支持curl命令上传
|
||||
@ -88,7 +88,8 @@ func main() {
|
||||
fmt.Print(obj)
|
||||
}
|
||||
```
|
||||
[更多请参考](doc/upload.md)
|
||||
[更多语言请参考](doc/upload.md)
|
||||
|
||||
|
||||
部署图
|
||||
|
||||
@ -262,6 +263,12 @@ for i in range(0,1000000):
|
||||
也可以多机同时进行压测，所有节点都是可以同时读写的
|
||||
```
|
||||
|
||||
- 代码为什么写在一个文件中？
|
||||
```
|
||||
一、目前的代码还非常简单，没必要弄得太复杂。
|
||||
二、个人理解模块化不是分开多个文件就表示模块化，大家可以用IDE去看一下代码结构，其实是已经模块化的。
|
||||
```
|
||||
|
||||
- 支持断点下载？
|
||||
```
|
||||
答案：支持
|
||||
|
||||
55
doc/api.md
Normal file
55
doc/api.md
Normal file
@ -0,0 +1,55 @@
|
||||
#API通用说明
|
||||
```
|
||||
一、统一使用POST请求
|
||||
二、url中的group只有在support_group_manage设置为true才有。
|
||||
例如：
|
||||
http://10.1.5.9:8080/group/reload
|
||||
默认：
|
||||
http://10.1.5.9:8080/reload
|
||||
|
||||
```
|
||||
|
||||
## 配置管理API
|
||||
```
|
||||
http://10.1.5.9:8080/group/reload
|
||||
|
||||
参数：
|
||||
action: set(修改参数),get获取参数,reload重新加载参数
|
||||
cfg:json参数 与 action=set配合完成参数设置
|
||||
|
||||
```
|
||||
|
||||
## 文件统计信息API
|
||||
```
|
||||
http://10.1.50.90:8080/group/stat
|
||||
|
||||
```
|
||||
|
||||
## 文件上传API
|
||||
```
|
||||
http://10.1.50.90:8080/group/upload
|
||||
参数：
|
||||
file:上传的文件
|
||||
scene:场景
|
||||
output:输出
|
||||
path:自定义路径
|
||||
具体请参阅示例代码
|
||||
```
|
||||
|
||||
## 修复统计信息
|
||||
```
|
||||
http://10.1.50.90:8080/group/repair_stat
|
||||
```
|
||||
|
||||
## 同步失败修复
|
||||
```
|
||||
http://10.1.50.90:8080/group/repair
|
||||
参数：
|
||||
force:是否强行修复(0|1)
|
||||
```
|
||||
|
||||
## 从文件目录中修复元数据（性能较差,所有文件重新计算md5,别重复执行）
|
||||
```
|
||||
http://10.1.50.90:8080/group/repair_fileinfo
|
||||
|
||||
```
|
||||
147
fileserver.go
147
fileserver.go
@ -128,7 +128,7 @@ const (
|
||||
"auto_repair": true,
|
||||
"文件去重算法md5可能存在冲突，默认md5": "sha1|md5",
|
||||
"file_sum_arithmetic": "md5",
|
||||
"是否支持按组管理,主要用途是Nginx支持多集群": "默认不支持,不支持上传路径http://10.1.5.4:8080/action,支持http://10.1.5.4:8080/group(配置中的group参数)/action,action为动作名，如status,delete,sync等",
|
||||
"是否支持按组（集群）管理,主要用途是Nginx支持多集群": "默认不支持,不支持时路径为http://10.1.5.4:8080/action,支持时为http://10.1.5.4:8080/group(配置中的group参数)/action,action为动作名，如status,delete,sync等",
|
||||
"support_group_manage": false,
|
||||
"管理ip列表": "用于管理集的ip白名单,",
|
||||
"admin_ips": ["127.0.0.1"]
|
||||
@ -603,6 +603,24 @@ func (this *Common) GetFileSum(file *os.File, alg string) string {
|
||||
return this.GetFileMd5(file)
|
||||
}
|
||||
|
||||
}
|
||||
func (this *Common) GetFileSumByName(filepath string, alg string) (string,error) {
|
||||
var (
|
||||
err error
|
||||
file *os.File
|
||||
)
|
||||
file,err= os.Open(filepath)
|
||||
if err!=nil {
|
||||
return "",err
|
||||
}
|
||||
defer file.Close()
|
||||
alg = strings.ToLower(alg)
|
||||
if alg == "sha1" {
|
||||
return this.GetFileSha1Sum(file),nil
|
||||
} else {
|
||||
return this.GetFileMd5(file),nil
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (this *Common) GetFileSha1Sum(file *os.File) string {
|
||||
@ -880,6 +898,68 @@ func (this *Server) CheckFileExistByMd5(md5s string, fileInfo *FileInfo) bool {
|
||||
|
||||
}
|
||||
|
||||
|
||||
func (this *Server) RepairFileInfoFromFile() {
|
||||
defer func() {
|
||||
if re := recover(); re != nil {
|
||||
buffer := debug.Stack()
|
||||
log.Error("RepairFileInfoFromDisk")
|
||||
log.Error(re)
|
||||
log.Error(string(buffer))
|
||||
}
|
||||
}()
|
||||
|
||||
handlefunc := func(file_path string, f os.FileInfo, err error) error {
|
||||
|
||||
var (
|
||||
files []os.FileInfo
|
||||
fi os.FileInfo
|
||||
fileInfo FileInfo
|
||||
sum string
|
||||
)
|
||||
|
||||
if f.IsDir() {
|
||||
|
||||
files, err = ioutil.ReadDir(file_path)
|
||||
if err!=nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _,fi=range files {
|
||||
|
||||
if fi.IsDir() || fi.Size()==0 {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
sum,err=this.util.GetFileSumByName(file_path+ "/"+ fi.Name(),Config().FileSumArithmetic)
|
||||
if err!=nil {
|
||||
log.Error(err)
|
||||
continue
|
||||
}
|
||||
fileInfo=FileInfo{
|
||||
Size:fi.Size(),
|
||||
Name:fi.Name(),
|
||||
Path:strings.Replace( file_path,"\\","/",-1),
|
||||
Md5:sum,
|
||||
TimeStamp:fi.ModTime().Unix(),
|
||||
}
|
||||
this.SaveFileMd5Log(&fileInfo,CONST_FILE_Md5_FILE_NAME)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
pathname:=STORE_DIR
|
||||
fi, _ := os.Stat(pathname)
|
||||
if fi.IsDir() {
|
||||
filepath.Walk(pathname, handlefunc)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (this *Server) DownloadFromPeer(peer string, fileInfo *FileInfo) {
|
||||
var (
|
||||
err error
|
||||
@ -2668,6 +2748,14 @@ func (this *Server) Check() {
|
||||
}()
|
||||
|
||||
}
|
||||
func (this *Server) RepairFileInfo(w http.ResponseWriter, r *http.Request) {
|
||||
if !this.IsPeer(r) {
|
||||
w.Write([]byte(this.GetClusterNotPermitMessage(r)))
|
||||
return
|
||||
}
|
||||
go this.RepairFileInfoFromFile()
|
||||
w.Write([]byte("repair job start,don't try again"))
|
||||
}
|
||||
|
||||
func (this *Server) Reload(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@ -2676,28 +2764,66 @@ func (this *Server) Reload(w http.ResponseWriter, r *http.Request) {
|
||||
data []byte
|
||||
|
||||
cfg GloablConfig
|
||||
action string
|
||||
cfgjson string
|
||||
)
|
||||
|
||||
r.ParseForm()
|
||||
if !this.IsPeer(r) {
|
||||
w.Write([]byte(this.GetClusterNotPermitMessage(r)))
|
||||
return
|
||||
}
|
||||
|
||||
if data, err = ioutil.ReadFile(CONST_CONF_FILE_NAME); err != nil {
|
||||
w.Write([]byte(err.Error()))
|
||||
cfgjson=r.FormValue("cfg")
|
||||
action=r.FormValue("action")
|
||||
_=cfgjson
|
||||
|
||||
if action=="get" {
|
||||
w.Write([]byte(this.util.JsonEncodePretty( Config())))
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
if action=="set" {
|
||||
if cfgjson=="" {
|
||||
w.Write([]byte("(error)parameter cfg(json) require"))
|
||||
return
|
||||
}
|
||||
if err=json.Unmarshal([]byte(cfgjson),cfg);err!=nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
cfgjson= this.util.JsonEncodePretty(cfg)
|
||||
this.util.WriteFile(CONST_CONF_FILE_NAME,cfgjson)
|
||||
w.Write([]byte("ok"))
|
||||
return
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(data, &cfg); err != nil {
|
||||
w.Write([]byte(err.Error()))
|
||||
if action=="reload" {
|
||||
|
||||
if data, err = ioutil.ReadFile(CONST_CONF_FILE_NAME); err != nil {
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(data, &cfg); err != nil {
|
||||
w.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
ParseConfig(CONST_CONF_FILE_NAME)
|
||||
|
||||
this.initComponent(true)
|
||||
|
||||
w.Write([]byte("ok"))
|
||||
return
|
||||
|
||||
}
|
||||
if action=="" {
|
||||
w.Write([]byte("(error)action support set(json) get reload"))
|
||||
}
|
||||
|
||||
ParseConfig(CONST_CONF_FILE_NAME)
|
||||
|
||||
this.initComponent(true)
|
||||
|
||||
w.Write([]byte("ok"))
|
||||
|
||||
}
|
||||
|
||||
@ -3030,6 +3156,7 @@ func (this *Server) Main() {
|
||||
http.HandleFunc(fmt.Sprintf("/%s/repair_stat", Config().Group), this.RepairStatWeb)
|
||||
http.HandleFunc(fmt.Sprintf("/%s/status", Config().Group), this.Status)
|
||||
http.HandleFunc(fmt.Sprintf("/%s/repair", Config().Group), this.Repair)
|
||||
http.HandleFunc(fmt.Sprintf("/%s/repair_fileinfo", Config().Group), this.RepairFileInfo)
|
||||
http.HandleFunc(fmt.Sprintf("/%s/reload", Config().Group), this.Reload)
|
||||
http.HandleFunc(fmt.Sprintf("/%s/syncfile", Config().Group), this.SyncFile)
|
||||
http.HandleFunc(fmt.Sprintf("/%s/syncfile_info", Config().Group), this.SyncFileInfo)
|
||||
@ -3046,6 +3173,7 @@ func (this *Server) Main() {
|
||||
http.HandleFunc("/repair_stat", this.RepairStatWeb)
|
||||
http.HandleFunc("/status", this.Status)
|
||||
http.HandleFunc("/repair", this.Repair)
|
||||
http.HandleFunc("/repair_fileinfo", this.RepairFileInfo)
|
||||
http.HandleFunc("/reload", this.Reload)
|
||||
http.HandleFunc("/syncfile", this.SyncFile)
|
||||
http.HandleFunc("/syncfile_info", this.SyncFileInfo)
|
||||
@ -3064,6 +3192,7 @@ func (this *Server) Main() {
|
||||
|
||||
func main() {
|
||||
|
||||
|
||||
server.Main()
|
||||
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user