From c237f148ff42b036cc656d495b1a2b1212755397 Mon Sep 17 00:00:00 2001 From: sjqzhang Date: Wed, 22 Dec 2021 16:51:18 +0800 Subject: [PATCH] support http proxy cache content --- server/config.go | 11 ++++ server/handler.go | 132 ++++++++++++++++++++++++++++++++++++++++++++++ server/server.go | 19 +++++++ 3 files changed, 162 insertions(+) diff --git a/server/config.go b/server/config.go index 4aa5c15..07b59ba 100644 --- a/server/config.go +++ b/server/config.go @@ -128,6 +128,10 @@ const ( "password": "abc", "host": "smtp.163.com:25" }, + "反向代理缓存内容":"目前只支持pypi ex: pip install -i http://127.0.0.1:9000/simple pandas", + "proxies":[ + {"dir":"pypi","origin":"https://pypi.douban.com","addr":":9000"} + ], "告警接收邮件列表": "接收人数组", "alarm_receivers": [], "告警接收URL": "方法post,参数:subject,message", @@ -218,6 +222,13 @@ type GlobalConfig struct { WatchChanSize int `json:"watch_chan_size"` ImageMaxWidth int `json:"image_max_width"` ImageMaxHeight int `json:"image_max_height"` + Proxies []Proxy `json:"proxies"` +} + +type Proxy struct { + Dir string `json:"dir"` + Addr string `json:"addr"` + Origin string `json:"origin"` } func Config() *GlobalConfig { diff --git a/server/handler.go b/server/handler.go index 68ecb1a..739ae3c 100644 --- a/server/handler.go +++ b/server/handler.go @@ -1,9 +1,16 @@ package server import ( + "crypto/tls" "fmt" + "github.com/astaxie/beego/httplib" + "io" + "net" "net/http" + "os" + "path" "runtime/debug" + "strings" "time" log "github.com/sjqzhang/seelog" @@ -40,3 +47,128 @@ func (HttpHandler) ServeHTTP(res http.ResponseWriter, req *http.Request) { } http.DefaultServeMux.ServeHTTP(res, req) } + +type HttpProxyHandler struct { + Proxy Proxy +} + + +func (h *HttpProxyHandler)handleTunneling(w http.ResponseWriter, r *http.Request) { + dest_conn, err := net.DialTimeout("tcp", r.Host, 10*time.Second) + if err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + hijacker, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "Hijacking not supported", http.StatusInternalServerError) + return + } + client_conn, _, err := hijacker.Hijack() + if err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + } + go h.transfer(dest_conn, client_conn) + go h.transfer(client_conn, dest_conn) +} +func (h *HttpProxyHandler)transfer(destination io.WriteCloser, source io.ReadCloser) { + defer destination.Close() + defer source.Close() + io.Copy(destination, source) +} +func (h *HttpProxyHandler)handleHTTP(w http.ResponseWriter, req *http.Request) { + resp, err := http.DefaultTransport.RoundTrip(req) + if err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + defer resp.Body.Close() + h.copyHeader(w.Header(), resp.Header) + w.WriteHeader(resp.StatusCode) + io.Copy(w, resp.Body) +} +func (h *HttpProxyHandler)copyHeader(dst, src http.Header) { + for k, vv := range src { + for _, v := range vv { + dst.Add(k, v) + } + } +} + +func (h *HttpProxyHandler) ServeHTTP(res http.ResponseWriter, req *http.Request) { + status_code := "200" + defer func(t time.Time) { + logStr := fmt.Sprintf("[Access] %s | %s | %s | %s | %s |%s", + time.Now().Format("2006/01/02 - 15:04:05"), + //res.Header(), + time.Since(t).String(), + server.util.GetClientIp(req), + req.Method, + status_code, + req.RequestURI, + ) + logacc.Info(logStr) + }(time.Now()) + defer func() { + if err := recover(); err != nil { + status_code = "500" + res.WriteHeader(500) + print(err) + buff := debug.Stack() + log.Error(err) + log.Error(string(buff)) + } + }() + //if Config().EnableCrossOrigin { + // server.CrossOrigin(res, req) + //} + if req.Method == http.MethodConnect { + h.handleTunneling(res, req) + return + } + href := strings.TrimRight(h.Proxy.Origin, "/") + req.RequestURI + md5 := server.util.MD5(href) + fpath := STORE_DIR + "/" + h.Proxy.Dir + "/" + md5[0:2] + "/" + md5[2:4] + "/" + md5 + _, err := os.Stat(fpath) + if err == nil { + fp, err := os.Open(fpath) + if err != nil { + log.Error(err) + return + } + defer fp.Close() + io.Copy(res, fp) + return + } + go func(href string) { + os.MkdirAll(path.Dir(fpath), 0755) + err := httplib.Get(href).ToFile(fpath) + if err == nil { + fi, err := os.Stat(fpath) + if err == nil { + fileInfo := FileInfo{ + Name: fi.Name(), + Size: fi.Size(), + Path: path.Dir(fpath),// files/default/20211222/15/57/1 + TimeStamp: fi.ModTime().Unix(), + Scene: Config().DefaultScene, + Peers: []string{Config().Host}, // ["http://10.12.188.85:8080"] + } + server.postFileToPeer(&fileInfo) + } + } + }(href) + r := httplib.Get(href) + r.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) + if err != nil { + log.Error(err) + } + response, err := r.DoRequest() + if err != nil { + return + } + defer response.Body.Close() + io.Copy(res, response.Body) + +} diff --git a/server/server.go b/server/server.go index b97fad0..6080031 100644 --- a/server/server.go +++ b/server/server.go @@ -1,6 +1,7 @@ package server import ( + "crypto/tls" "fmt" "net/http" "os" @@ -232,6 +233,24 @@ func (c *Server) Start() { c.initRouter() + if Config().Proxies != nil && len(Config().Proxies) > 0 { + for _, proxy := range Config().Proxies { + go func(proxy Proxy) { + handler := HttpProxyHandler{ + Proxy: proxy, + } + fmt.Println("Proxy on " + proxy.Addr) + server := &http.Server{ + Addr: proxy.Addr, + Handler: &handler, + TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), + } + server.ListenAndServe() + + }(proxy) + } + } + fmt.Println("Listen on " + Config().Addr) if Config().EnableHttps { err := http.ListenAndServeTLS(Config().Addr, CONST_SERVER_CRT_FILE_NAME, CONST_SERVER_KEY_FILE_NAME, new(HttpHandler))