From c4cd34574dfa894089ee573f89975256abf711ee Mon Sep 17 00:00:00 2001 From: John Date: Thu, 7 Dec 2017 09:53:35 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E6=96=87=E4=BB=B6=E6=8C=87?= =?UTF-8?q?=E9=92=88=E6=B1=A0=E5=B9=B6=E5=8F=91=E5=AE=89=E5=85=A8=E9=97=AE?= =?UTF-8?q?=E9=A2=98=EF=BC=8C=E5=AE=8C=E5=96=84README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.MD | 53 +++++- g/{net => encoding}/gurl/url.go | 0 g/os/gfilepool/gfilepool.go | 44 +++-- geg/other/test.go | 275 +++----------------------------- 4 files changed, 106 insertions(+), 266 deletions(-) rename g/{net => encoding}/gurl/url.go (100%) diff --git a/README.MD b/README.MD index 3171b824a..0c1456570 100644 --- a/README.MD +++ b/README.MD @@ -12,7 +12,52 @@ import "gitee.com/johng/gf/g/xxx" # 说明 . - ├── g 框架目录 - ├── geg 框架示例 - ├── vendor 第三方包 - └── version.go 版本信息 \ No newline at end of file + ├── g 【框架目录】 + │   ├── container 【常用数据结构容器】 + │   │   ├── gbtree B+树 + │   │   ├── glist 并发安全的双向链表 + │   │   ├── gmap 并发安全的哈希表 + │   │   └── gset 并发安全的集合 + │   │   + │   ├── database 【数据库操作】 + │   │   └── gdb MySQL、PostgreSQL的操作封装 + │   │   + │   ├── encoding 【数据编码】 + │   │   ├── gbase64 BASE64 + │   │   ├── gbinary 二进制操作 + │   │   ├── gcompress 数据压缩 + │   │   ├── gcrc32 CRC32 + │   │   ├── ghash 常用哈希函数 + │   │   ├── ghtml html编码 + │   │   ├── gjson JSON + │   │   ├── gmd5 MD5 + │   │   ├── gsha1 SHA1 + │   │   └── gurl URL + │   │   + │   ├── net 【网络通信】 + │   │   ├── ghttp HTTP客户端及服务端 + │   │   ├── gip IP操作 + │   │   ├── gscanner 端口扫描 + │   │   ├── gsmtp SMTP + │   │   ├── gtcp TCP客户端及服务端 + │   │   └── gudp UDP客户端及服务端 + │   │   + │   ├── os 【系统管理】 + │   │   ├── gcache 本地缓存管理 + │   │   ├── gconsole 命令行控制 + │   │   ├── gfile 文件管理 + │   │   ├── gfilepool 文件指针池 + │   │   ├── gfilespace 文件碎片管理 + │   │   ├── glog 日志管理 + │   │   └── gmmap MMAP + │   │   + │   └── util 【常用工具】 + │   ├── gpage 分页 + │   ├── grand 随机数 + │   ├── gregx 正则表达式 + │   ├── gtime 时间管理 + │   └── gutil 其他工具 + │   + ├── geg 【框架示例】 + ├── vendor 【第三方包】 + └── version.go 【版本信息】 \ No newline at end of file diff --git a/g/net/gurl/url.go b/g/encoding/gurl/url.go similarity index 100% rename from g/net/gurl/url.go rename to g/encoding/gurl/url.go diff --git a/g/os/gfilepool/gfilepool.go b/g/os/gfilepool/gfilepool.go index 9e1992bdd..960b26283 100644 --- a/g/os/gfilepool/gfilepool.go +++ b/g/os/gfilepool/gfilepool.go @@ -7,6 +7,8 @@ import ( "gitee.com/johng/gf/g/util/gtime" "gitee.com/johng/gf/g/container/gmap" "gitee.com/johng/gf/g/container/glist" + "sync/atomic" + "sync" ) // 文件指针池 @@ -20,9 +22,10 @@ type Pool struct { // 文件指针池指针 type File struct { + sync.RWMutex pool *Pool // 所属池 file *os.File // 指针对象 - expire int64 // 过期时间 + expire int64 // 过期时间(秒) } // 全局指针池，expire < 0表示不过期，expire = 0表示使用完立即回收，expire > 0表示超时回收 @@ -51,14 +54,14 @@ func New(path string, flag int, expire int) *Pool { // 独立的线程执行过期清理工作 if expire != -1 { go func(p *Pool) { + // 遍历可用指针列表，判断是否过期 for !p.closed { r := p.list.Front() if r != nil && r.Value != nil { f := r.Value.(*File) - if f.expire <= gtime.Second() { - if f.file != nil { - f.file.Close() - } + // 必须小于，中间有1秒的缓存时间，防止同时获取和判断过期时冲突 + if f.getExpire() < gtime.Second() { + f.destroy() p.list.Remove(r) continue } @@ -73,15 +76,16 @@ func New(path string, flag int, expire int) *Pool { // 获得一个文件打开指针 func (p *Pool) File() (*File, error) { if p.list.Len() > 0 { + // 遍历可用指针列表，返回一个未过期的指针 for { r := p.list.PopBack() if r != nil { f := r.(*File) - if f.expire > gtime.Second() { + // 必须大于 + if f.getExpire() > gtime.Second() { return f, nil } else if f.file != nil { - f.file.Close() - f.file = nil + f.destroy() } } else { break; @@ -108,8 +112,28 @@ func (f *File) File() *os.File { return f.file } -// 关闭指针链接(软关闭) +// 关闭指针链接(软关闭)，放回池中重复使用 func (f *File) Close() { - f.expire = gtime.Second() + int64(f.pool.idlemax) + f.setExpire(gtime.Second() + int64(f.pool.idlemax)) f.pool.list.PushFront(f) +} + +// 销毁指针 +func (f *File) destroy() { + f.Lock() + defer f.Unlock() + if f.file != nil { + f.file.Close() + f.file = nil + } +} + +// 获取指针过期时间 +func (f *File) setExpire(expire int64) { + atomic.StoreInt64(&f.expire, expire) +} + +// 获取指针过期时间 +func (f *File) getExpire() int64 { + return atomic.LoadInt64(&f.expire) } \ No newline at end of file diff --git a/geg/other/test.go b/geg/other/test.go index c5a0ceb77..50e595a64 100644 --- a/geg/other/test.go +++ b/geg/other/test.go @@ -1,263 +1,34 @@ package main import ( - "fmt" "sync" - "github.com/boltdb/bolt" - "log" + "fmt" + "strconv" ) - +type LockDemo struct { + var1 string + var2 string + mu1 sync.RWMutex + mu2 sync.RWMutex +} func main() { - var mu sync.RWMutex - - mu.Lock() - mu.RLock() - - fmt.Println(1) - - mu.RUnlock() - mu.Unlock() - - return - //a := 1497965 - //for i := 0; i < 10000; i++ { - // fmt.Println(a*i) - //} - // - //fmt.Println(math.MaxUint64) - //m := make(map[int]int, 0) - //for i := 0; i < 10000000; i ++ { - // m[i] = i - //} - //t1 := gtime.Microsecond() - //for i := 0; i < 10; i ++ { - // if _, ok := m[i]; ok { - // - // } - //} - - //b := make([]byte, 100000) - //removeBlock(b, 80000) - //fmt.Println(gtime.Microsecond() - t1) - - return - //slice := []int{1,2,3,4,5,6,7,8,9} - //index := 1 - ////fmt.Println(append(slice[:index], slice[index+1:]...)) - // - ////rear:=append([]int{}, slice[index:]...) - ////slice=append(slice[0:index], 88) - ////slice=append(slice, rear...) - //// - ////fmt.Println(slice) - // - //fmt.Println(append(append(slice[0 : index], 88), append([]int{}, slice[index : ]...)...)) - //return - //a := gbinary.EncodeBits(nil, 100, 10) - //fmt.Println(a) - //b := gbinary.EncodeBitsToBytes(a) - //fmt.Println(b) - //fmt.Println(gbinary.EncodeInt32(1)) - //return - //return - - //fmt.Println(gbinary.DecodeToInt64([]byte{1})) - //return - //fmt.Println(gbinary.EncodeInt32(1)[0:3]) - //b := []int{1,2,3} - //c := []int{4} - //copy(b[1:], c) - //fmt.Println(b) - //return - //space, err := gfilespace.New("/tmp/test") - //if err != nil { - // fmt.Println(err) - //} - //for i := 0; i < 10; i++ { - // space.AddBlock(int64(i), uint32((i + 1)*10)) - //} - //fmt.Println(space.GetBlock(50)) - //return - - - //db.Set([]byte("1"), []byte(grand.RandStr(10))) - //grand.RandStr(10) - //db.Set([]byte("r88U89b6Vv"), []byte("john211111111111111111111111")) - //db.Get([]byte("name2")) - //fmt.Println(e) - - //fmt.Println(string(v)) - //r := int32(binary.LittleEndian.Uint32(b)) - //fmt.Println(int32(r)) - //binary.BigEndian.Uint16(b) - //gbinary.DecodeToInt32([]byte{1,2,3,4}) - //fmt.Println(gtime.Microsecond() - t1) - //fmt.Println([]byte{byte(i)}) - - //b := make([]byte, 0) - //a := ghash.BKDRHash([]byte("john")) - //for i := 0; i < 1000; i++ { - // r, e := gbinary.Encode([]byte("key_" + strconv.Itoa(i)), a, a) - // if e != nil { - // fmt.Println(e) - // return - // } - // b = append(b, r...) - //} - //fmt.Printf("length: %d\n", len(b)/1024) - //fmt.Printf("compressed: %d\n", len(gcompress.Zlib(b))/1024) - //t1 := gtime.Microsecond() - ////gcompress.Zlib(b) - //gbinary.Encode([]byte("key_" + strconv.Itoa(100)), a, a) - //fmt.Println(gtime.Microsecond() - t1) - //return - //t1 := gtime.Second() - //m := make(map[uint64]bool) - //c := 0 - //for i := 0; i < 100000000; i++ { - // key := ghash.SDBMHash64([]byte("this is test key" + strconv.Itoa(i))) - // if _, ok := m[key]; ok { - // c++ - // } else { - // m[key] = true - // } - //} - //fmt.Println(gtime.Second() - t1) - //fmt.Println("conflicts:", c) - //fmt.Println(ghash.BKDRHash([]byte("johnWRWEREWREWRWEREWRRRRRRRRRRRRRRRRRRRRRRRRRRRRR"))) - //t1 := gtime.Microsecond() - //ghash.BKDRHash64([]byte("john")) - ////fmt.Println(ghash.ELFHash([]byte("john"))) - ////fmt.Println(ghash.JSHash([]byte("john"))) - //fmt.Println(gtime.Microsecond() - t1) - //fmt.Println(ghash.BKDRHash64([]byte("john29384723894723894789sdkjfhsjkdh"))) - //return - //btree := gbtree.New(3) - //t1 := gtime.Microsecond() - //for i := 1; i <= 11; i++ { - // btree.Set([]byte{byte(i)}, []byte{byte(i)}) - //} - //fmt.Println(gtime.Microsecond() - t1) - //btree.Print() - //fmt.Println() - //fmt.Println() - //btree.Remove([]byte{11}) - //btree.Print() - - //t2 := gtime.Microsecond() - //btree.Get([]byte("key2")) - //fmt.Println(btree.Get([]byte{200})) - //fmt.Println(gtime.Microsecond() - t2) - - //return - ////m := gmap.NewStringInterfaceMap() - //t1 := gtime.Microsecond() - //gcrc32.EncodeString("123") - //fmt.Println(gtime.Microsecond() - t1) - //return - //db, err := leveldb.OpenFile("/tmp/lv.db", nil) - //fmt.Println(err) - //defer db.Close() - //t1 := gtime.Microsecond() - //size := 10000000 - - //for i := 0; i < size; i++ { - // //r := []byte(grand.RandStr(10)) - // //if err := db.Set(r, r); err != nil { - // t3 := gtime.Microsecond() - // if err := db.Put([]byte("key1_" + strconv.Itoa(i)), []byte("value1_" + strconv.Itoa(i)), nil); err != nil { - // //if err := db.Set(gbinary.EncodeInt32(int32(i)), gbinary.EncodeInt32(int32(i))); err != nil { - // fmt.Println(err) - // } - // t4 := gtime.Microsecond() - // if t4 - t3 > 1000 { - // fmt.Println(t4-t3) - // } - //} - - //for i := 0; i < size; i++ { - // //r := []byte(grand.RandStr(10)) - // //if err := db.Set(r, r); err != nil { - // t3 := gtime.Microsecond() - // v, err := db.Get([]byte("key1_" + strconv.Itoa(i)), nil) - // if err != nil { - // //if err := db.Set(gbinary.EncodeInt32(int32(i)), gbinary.EncodeInt32(int32(i))); err != nil { - // fmt.Println(err) - // } - // if len(v) == 0 { - // fmt.Println("none") - // } - // t4 := gtime.Microsecond() - // if t4 - t3 > 1000 { - // fmt.Println(t4-t3) - // } - //} - //fmt.Println(gtime.Microsecond() - t1) - // - //return - db, err := bolt.Open("/tmp/my.db", 0600, nil) - if err != nil { - log.Fatal(err) + l := LockDemo{} + wg := sync.WaitGroup{} + for i := 0; i < 1000; i++ { + wg.Add(1) + go func(i int) { + l.mu1.Lock() + l.mu2.Lock() + defer l.mu2.Unlock() + defer l.mu1.Unlock() + l.var1 = strconv.Itoa(i) + l.var2 = strconv.Itoa(i + 1) + wg.Done() + }(i) } - defer db.Close() - - tx, err := db.Begin(true) - if err != nil { - log.Fatal(err) - } - defer tx.Rollback() - - //Use the transaction... - _, err = tx.CreateBucket([]byte("MyBucket")) - if err != nil { - log.Fatal(err) - } - - //Commit the transaction and check for error. - if err := tx.Commit(); err != nil { - log.Fatal(err) - } - t1 := gtime.Microsecond() - db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("MyBucket")) - err := b.Put([]byte("answer"), []byte("11")) - return err - }) - fmt.Println(gtime.Microsecond() - t1) - - t2 := gtime.Microsecond() - db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("MyBucket")) - v := b.Get([]byte("answer")) - fmt.Printf("The answer is: %s\n", v) - return nil - }) - fmt.Println(gtime.Microsecond() - t2) - - - //return - ////db, err := gkvdb.New("/tmp/test2", "t") - //fmt.Println(err) - ////fmt.Println(db.Set("1", []byte("1"))) - //t1 := gtime.Microsecond() - ////fmt.Println(db.Get("1")) - //fmt.Println(db.Set("1", []byte("1"))) - //fmt.Println(gtime.Microsecond() - t1) - ////fmt.Println(db.Set("name", []byte("222"))) - //return - //for i := 0; i < 10000000; i++ { - // gfile.PutContentsAppend("/tmp/test", "1234567890") - //} - // - //file, _ := gfile.OpenWithFlag("/tmp/test", os.O_RDWR|os.O_CREATE) - //fmt.Println(gfile.GetBinContentByTwoOffsets(file, 100, 110)) - ////n, err := file.WriteAt([]byte("123"), 2286 445 522*8) - ////n, err := file.WriteAt([]byte("123"), 1000000*(16)) - ////fmt.Println(n) - ////fmt.Println(err) - //defer file.Close() - //fmt.Println(gcrc32.EncodeString("123")) + wg.Wait() + fmt.Println(l) } \ No newline at end of file