support http proxy cache content

server/config.go

@@ -128,6 +128,10 @@ const (
 		"password": "abc",
 		"host": "smtp.163.com:25"
 	},
+	"反向代理缓存内容":"目前只支持pypi ex:  pip install -i http://127.0.0.1:9000/simple pandas",
+	"proxies":[
+		{"dir":"pypi","origin":"https://pypi.douban.com","addr":":9000"}
+	],
 	"告警接收邮件列表": "接收人数组",
 	"alarm_receivers": [],
 	"告警接收URL": "方法post,参数:subject,message",
@@ -218,6 +222,13 @@ type GlobalConfig struct {
 	WatchChanSize        int      `json:"watch_chan_size"`
 	ImageMaxWidth        int      `json:"image_max_width"`
 	ImageMaxHeight       int      `json:"image_max_height"`
+	Proxies              []Proxy  `json:"proxies"`
+}
+
+type Proxy struct {
+	Dir    string `json:"dir"`
+	Addr   string    `json:"addr"`
+	Origin string `json:"origin"`
 }
 
 func Config() *GlobalConfig {

server/handler.go

@@ -1,9 +1,16 @@
 package server
 
 import (
+	"crypto/tls"
 	"fmt"
+	"github.com/astaxie/beego/httplib"
+	"io"
+	"net"
 	"net/http"
+	"os"
+	"path"
 	"runtime/debug"
+	"strings"
 	"time"
 
 	log "github.com/sjqzhang/seelog"
@@ -40,3 +47,128 @@ func (HttpHandler) ServeHTTP(res http.ResponseWriter, req *http.Request) {
 	}
 	http.DefaultServeMux.ServeHTTP(res, req)
 }
+
+type HttpProxyHandler struct {
+	Proxy Proxy
+}
+
+
+func (h *HttpProxyHandler)handleTunneling(w http.ResponseWriter, r *http.Request) {
+	dest_conn, err := net.DialTimeout("tcp", r.Host, 10*time.Second)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusServiceUnavailable)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+	hijacker, ok := w.(http.Hijacker)
+	if !ok {
+		http.Error(w, "Hijacking not supported", http.StatusInternalServerError)
+		return
+	}
+	client_conn, _, err := hijacker.Hijack()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusServiceUnavailable)
+	}
+	go h.transfer(dest_conn, client_conn)
+	go h.transfer(client_conn, dest_conn)
+}
+func (h *HttpProxyHandler)transfer(destination io.WriteCloser, source io.ReadCloser) {
+	defer destination.Close()
+	defer source.Close()
+	io.Copy(destination, source)
+}
+func (h *HttpProxyHandler)handleHTTP(w http.ResponseWriter, req *http.Request) {
+	resp, err := http.DefaultTransport.RoundTrip(req)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusServiceUnavailable)
+		return
+	}
+	defer resp.Body.Close()
+	h.copyHeader(w.Header(), resp.Header)
+	w.WriteHeader(resp.StatusCode)
+	io.Copy(w, resp.Body)
+}
+func (h *HttpProxyHandler)copyHeader(dst, src http.Header) {
+	for k, vv := range src {
+		for _, v := range vv {
+			dst.Add(k, v)
+		}
+	}
+}
+
+func (h *HttpProxyHandler) ServeHTTP(res http.ResponseWriter, req *http.Request) {
+	status_code := "200"
+	defer func(t time.Time) {
+		logStr := fmt.Sprintf("[Access] %s | %s | %s | %s | %s |%s",
+			time.Now().Format("2006/01/02 - 15:04:05"),
+			//res.Header(),
+			time.Since(t).String(),
+			server.util.GetClientIp(req),
+			req.Method,
+			status_code,
+			req.RequestURI,
+		)
+		logacc.Info(logStr)
+	}(time.Now())
+	defer func() {
+		if err := recover(); err != nil {
+			status_code = "500"
+			res.WriteHeader(500)
+			print(err)
+			buff := debug.Stack()
+			log.Error(err)
+			log.Error(string(buff))
+		}
+	}()
+	//if Config().EnableCrossOrigin {
+	//	server.CrossOrigin(res, req)
+	//}
+	if req.Method == http.MethodConnect {
+		h.handleTunneling(res, req)
+		return
+	}
+	href := strings.TrimRight(h.Proxy.Origin, "/") + req.RequestURI
+	md5 := server.util.MD5(href)
+	fpath := STORE_DIR + "/" + h.Proxy.Dir + "/" + md5[0:2] + "/" + md5[2:4] + "/" + md5
+	_, err := os.Stat(fpath)
+	if err == nil {
+		fp, err := os.Open(fpath)
+		if err != nil {
+			log.Error(err)
+			return
+		}
+		defer fp.Close()
+		io.Copy(res, fp)
+		return
+	}
+	go func(href string) {
+		os.MkdirAll(path.Dir(fpath), 0755)
+		err := httplib.Get(href).ToFile(fpath)
+		if err == nil {
+			fi, err := os.Stat(fpath)
+			if err == nil {
+				fileInfo := FileInfo{
+					Name: fi.Name(),
+					Size: fi.Size(),
+					Path: path.Dir(fpath),//  files/default/20211222/15/57/1
+					TimeStamp: fi.ModTime().Unix(),
+					Scene: Config().DefaultScene,
+					Peers: []string{Config().Host},  // ["http://10.12.188.85:8080"]
+				}
+				server.postFileToPeer(&fileInfo)
+			}
+		}
+	}(href)
+	r := httplib.Get(href)
+	r.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+	if err != nil {
+		log.Error(err)
+	}
+	response, err := r.DoRequest()
+	if err != nil {
+		return
+	}
+	defer response.Body.Close()
+	io.Copy(res, response.Body)
+
+}

server/server.go

@@ -1,6 +1,7 @@
 package server
 
 import (
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"os"
@@ -232,6 +233,24 @@ func (c *Server) Start() {
 
 	c.initRouter()
 
+	if Config().Proxies != nil && len(Config().Proxies) > 0 {
+		for _, proxy := range Config().Proxies {
+			go func(proxy Proxy) {
+				handler := HttpProxyHandler{
+					Proxy: proxy,
+				}
+				fmt.Println("Proxy on " + proxy.Addr)
+				server := &http.Server{
+					Addr:         proxy.Addr,
+					Handler:      &handler,
+					TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
+				}
+				server.ListenAndServe()
+
+			}(proxy)
+		}
+	}
+
 	fmt.Println("Listen on " + Config().Addr)
 	if Config().EnableHttps {
 		err := http.ListenAndServeTLS(Config().Addr, CONST_SERVER_CRT_FILE_NAME, CONST_SERVER_KEY_FILE_NAME, new(HttpHandler))