From 35cf228d83afce8d60bb2e9a31b0f906347f2175 Mon Sep 17 00:00:00 2001 From: John Guo Date: Sat, 27 Nov 2021 01:10:00 +0800 Subject: [PATCH] add AdapterRedis for package gcache --- database/gredis/gredis_config.go | 2 +- net/gtrace/gtrace.go | 7 +- os/gcache/gcache_adapter_memory.go | 2 +- os/gcache/gcache_adapter_redis.go | 433 ++++++++++++++++++ os/gcache/gcache_cache.go | 2 +- ...cache_z_unit_feature_adapter_redis_test.go | 174 +++++++ .../gsession_storage_redis_hashtable.go | 2 +- 7 files changed, 615 insertions(+), 7 deletions(-) create mode 100644 os/gcache/gcache_adapter_redis.go create mode 100644 os/gcache/gcache_z_unit_feature_adapter_redis_test.go diff --git a/database/gredis/gredis_config.go b/database/gredis/gredis_config.go index e3431609e..2d94dda84 100644 --- a/database/gredis/gredis_config.go +++ b/database/gredis/gredis_config.go @@ -20,7 +20,7 @@ import ( // Config is redis configuration. type Config struct { - Address string `json:"address"` // It supports single and cluster redis server. Multiple addresses joined with char ','. + Address string `json:"address"` // It supports single and cluster redis server. Multiple addresses joined with char ','. Eg: 192.168.1.1:6379, 192.168.1.2:6379. Db int `json:"db"` // Redis db. Pass string `json:"pass"` // Password for AUTH. MinIdle int `json:"minIdle"` // Minimum number of connections allowed to be idle (default is 0) diff --git a/net/gtrace/gtrace.go b/net/gtrace/gtrace.go index 846cefb47..6b14ca8ac 100644 --- a/net/gtrace/gtrace.go +++ b/net/gtrace/gtrace.go @@ -12,6 +12,8 @@ import ( "os" "strings" + "github.com/gogf/gf/v2/internal/command" + "github.com/gogf/gf/v2/util/gconv" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" @@ -20,7 +22,6 @@ import ( "github.com/gogf/gf/v2/container/gmap" "github.com/gogf/gf/v2/container/gvar" "github.com/gogf/gf/v2/net/gipv4" - "github.com/gogf/gf/v2/os/gcmd" ) const ( @@ -44,8 +45,8 @@ var ( ) func init() { - tracingInternal = gcmd.GetOptWithEnv(commandEnvKeyForTracingInternal, true).Bool() - if maxContentLogSize := gcmd.GetOptWithEnv(commandEnvKeyForMaxContentLogSize).Int(); maxContentLogSize > 0 { + tracingInternal = gconv.Bool(command.GetOptWithEnv(commandEnvKeyForTracingInternal, "true")) + if maxContentLogSize := gconv.Int(command.GetOptWithEnv(commandEnvKeyForMaxContentLogSize)); maxContentLogSize > 0 { tracingMaxContentLogSize = maxContentLogSize } CheckSetDefaultTextMapPropagator() diff --git a/os/gcache/gcache_adapter_memory.go b/os/gcache/gcache_adapter_memory.go index 4d189fc87..539adf85c 100644 --- a/os/gcache/gcache_adapter_memory.go +++ b/os/gcache/gcache_adapter_memory.go @@ -54,7 +54,7 @@ const ( ) // NewAdapterMemory creates and returns a new memory cache object. -func NewAdapterMemory(lruCap ...int) *AdapterMemory { +func NewAdapterMemory(lruCap ...int) Adapter { c := &AdapterMemory{ data: newAdapterMemoryData(), lruGetList: glist.New(true), diff --git a/os/gcache/gcache_adapter_redis.go b/os/gcache/gcache_adapter_redis.go new file mode 100644 index 000000000..03a0e5d27 --- /dev/null +++ b/os/gcache/gcache_adapter_redis.go @@ -0,0 +1,433 @@ +// Copyright 2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gcache + +import ( + "context" + "time" + + "github.com/gogf/gf/v2/container/gvar" + "github.com/gogf/gf/v2/database/gredis" +) + +// AdapterRedis is the gcache adapter implements using Redis server. +type AdapterRedis struct { + redis *gredis.Redis +} + +// NewAdapterRedis creates and returns a new memory cache object. +func NewAdapterRedis(redis *gredis.Redis) Adapter { + return &AdapterRedis{ + redis: redis, + } +} + +// Set sets cache with `key`-`value` pair, which is expired after `duration`. +// +// It does not expire if `duration` == 0. +// It deletes the keys of `data` if `duration` < 0 or given `value` is nil. +func (c *AdapterRedis) Set(ctx context.Context, key interface{}, value interface{}, duration time.Duration) (err error) { + if value == nil || duration < 0 { + _, err = c.redis.Do(ctx, "DEL", key) + } else { + if duration == 0 { + _, err = c.redis.Do(ctx, "SET", key, value) + } else { + _, err = c.redis.Do(ctx, "SETEX", key, uint64(duration.Seconds()), value) + } + } + return err +} + +// SetMap batch sets cache with key-value pairs by `data` map, which is expired after `duration`. +// +// It does not expire if `duration` == 0. +// It deletes the keys of `data` if `duration` < 0 or given `value` is nil. +func (c *AdapterRedis) SetMap(ctx context.Context, data map[interface{}]interface{}, duration time.Duration) error { + if len(data) == 0 { + return nil + } + // DEL. + if duration < 0 { + var ( + index = 0 + keys = make([]interface{}, len(data)) + ) + for k, _ := range data { + keys[index] = k + index += 1 + } + _, err := c.redis.Do(ctx, "DEL", keys...) + if err != nil { + return err + } + } + if duration == 0 { + var ( + index = 0 + keyValues = make([]interface{}, len(data)*2) + ) + for k, v := range data { + keyValues[index] = k + keyValues[index+1] = v + index += 2 + } + _, err := c.redis.Do(ctx, "MSET", keyValues...) + if err != nil { + return err + } + } + if duration > 0 { + var err error + for k, v := range data { + if err = c.Set(ctx, k, v, duration); err != nil { + return err + } + } + } + return nil +} + +// SetIfNotExist sets cache with `key`-`value` pair which is expired after `duration` +// if `key` does not exist in the cache. It returns true the `key` does not exist in the +// cache, and it sets `value` successfully to the cache, or else it returns false. +// +// It does not expire if `duration` == 0. +// It deletes the `key` if `duration` < 0 or given `value` is nil. +func (c *AdapterRedis) SetIfNotExist(ctx context.Context, key interface{}, value interface{}, duration time.Duration) (bool, error) { + var ( + v *gvar.Var + err error + ) + // Execute the function and retrieve the result. + if f, ok := value.(func() (interface{}, error)); ok { + value, err = f() + if value == nil { + return false, err + } + } + // DEL. + if duration < 0 || value == nil { + if v, err = c.redis.Do(ctx, "DEL", key, value); err != nil { + return false, err + } + if v.Int() == 1 { + return true, err + } else { + return false, err + } + } + if v, err = c.redis.Do(ctx, "SETNX", key, value); err != nil { + return false, err + } + if v.Int() > 0 && duration > 0 { + // Set the expiration. + _, err = c.redis.Do(ctx, "EXPIRE", key, uint64(duration.Seconds())) + if err != nil { + return false, err + } + return true, err + } + return false, err +} + +// SetIfNotExistFunc sets `key` with result of function `f` and returns true +// if `key` does not exist in the cache, or else it does nothing and returns false if `key` already exists. +// +// The parameter `value` can be type of `func() interface{}`, but it does nothing if its +// result is nil. +// +// It does not expire if `duration` == 0. +// It deletes the `key` if `duration` < 0 or given `value` is nil. +func (c *AdapterRedis) SetIfNotExistFunc(ctx context.Context, key interface{}, f func() (interface{}, error), duration time.Duration) (ok bool, err error) { + value, err := f() + if err != nil { + return false, err + } + return c.SetIfNotExist(ctx, key, value, duration) +} + +// SetIfNotExistFuncLock sets `key` with result of function `f` and returns true +// if `key` does not exist in the cache, or else it does nothing and returns false if `key` already exists. +// +// It does not expire if `duration` == 0. +// It deletes the `key` if `duration` < 0 or given `value` is nil. +// +// Note that it differs from function `SetIfNotExistFunc` is that the function `f` is executed within +// writing mutex lock for concurrent safety purpose. +func (c *AdapterRedis) SetIfNotExistFuncLock(ctx context.Context, key interface{}, f func() (interface{}, error), duration time.Duration) (ok bool, err error) { + value, err := f() + if err != nil { + return false, err + } + return c.SetIfNotExist(ctx, key, value, duration) +} + +// Get retrieves and returns the associated value of given . +// It returns nil if it does not exist or its value is nil. +func (c *AdapterRedis) Get(ctx context.Context, key interface{}) (*gvar.Var, error) { + return c.redis.Do(ctx, "GET", key) +} + +// GetOrSet retrieves and returns the value of `key`, or sets `key`-`value` pair and +// returns `value` if `key` does not exist in the cache. The key-value pair expires +// after `duration`. +// +// It does not expire if `duration` == 0. +// It deletes the `key` if `duration` < 0 or given `value` is nil, but it does nothing +// if `value` is a function and the function result is nil. +func (c *AdapterRedis) GetOrSet(ctx context.Context, key interface{}, value interface{}, duration time.Duration) (result *gvar.Var, err error) { + result, err = c.Get(ctx, key) + if err != nil { + return nil, err + } + if result == nil { + return gvar.New(value), c.Set(ctx, key, value, duration) + } + return +} + +// GetOrSetFunc retrieves and returns the value of `key`, or sets `key` with result of +// function `f` and returns its result if `key` does not exist in the cache. The key-value +// pair expires after `duration`. +// +// It does not expire if `duration` == 0. +// It deletes the `key` if `duration` < 0 or given `value` is nil, but it does nothing +// if `value` is a function and the function result is nil. +func (c *AdapterRedis) GetOrSetFunc(ctx context.Context, key interface{}, f func() (interface{}, error), duration time.Duration) (result *gvar.Var, err error) { + v, err := c.Get(ctx, key) + if err != nil { + return nil, err + } + if v == nil { + value, err := f() + if err != nil { + return nil, err + } + if value == nil { + return nil, nil + } + return gvar.New(value), c.Set(ctx, key, value, duration) + } else { + return v, nil + } +} + +// GetOrSetFuncLock retrieves and returns the value of `key`, or sets `key` with result of +// function `f` and returns its result if `key` does not exist in the cache. The key-value +// pair expires after `duration`. +// +// It does not expire if `duration` == 0. +// It deletes the `key` if `duration` < 0 or given `value` is nil, but it does nothing +// if `value` is a function and the function result is nil. +// +// Note that it differs from function `GetOrSetFunc` is that the function `f` is executed within +// writing mutex lock for concurrent safety purpose. +func (c *AdapterRedis) GetOrSetFuncLock(ctx context.Context, key interface{}, f func() (interface{}, error), duration time.Duration) (result *gvar.Var, err error) { + return c.GetOrSetFunc(ctx, key, f, duration) +} + +// Contains checks and returns true if `key` exists in the cache, or else returns false. +func (c *AdapterRedis) Contains(ctx context.Context, key interface{}) (bool, error) { + v, err := c.redis.Do(ctx, "EXISTS", key) + if err != nil { + return false, err + } + return v.Bool(), nil +} + +// Size returns the number of items in the cache. +func (c *AdapterRedis) Size(ctx context.Context) (size int, err error) { + v, err := c.redis.Do(ctx, "DBSIZE") + if err != nil { + return 0, err + } + return v.Int(), nil +} + +// Data returns a copy of all key-value pairs in the cache as map type. +// Note that this function may lead lots of memory usage, you can implement this function +// if necessary. +func (c *AdapterRedis) Data(ctx context.Context) (map[interface{}]interface{}, error) { + // Keys. + v, err := c.redis.Do(ctx, "KEYS", "*") + if err != nil { + return nil, err + } + keys := v.Slice() + // Values. + v, err = c.redis.Do(ctx, "MGET", keys...) + if err != nil { + return nil, err + } + values := v.Slice() + // Compose keys and values. + data := make(map[interface{}]interface{}) + for i := 0; i < len(keys); i++ { + data[keys[i]] = values[i] + } + return data, nil +} + +// Keys returns all keys in the cache as slice. +func (c *AdapterRedis) Keys(ctx context.Context) ([]interface{}, error) { + v, err := c.redis.Do(ctx, "KEYS", "*") + if err != nil { + return nil, err + } + return v.Slice(), nil +} + +// Values returns all values in the cache as slice. +func (c *AdapterRedis) Values(ctx context.Context) ([]interface{}, error) { + // Keys. + v, err := c.redis.Do(ctx, "KEYS", "*") + if err != nil { + return nil, err + } + keys := v.Slice() + // Values. + v, err = c.redis.Do(ctx, "MGET", keys...) + if err != nil { + return nil, err + } + return v.Slice(), nil +} + +// Update updates the value of `key` without changing its expiration and returns the old value. +// The returned value `exist` is false if the `key` does not exist in the cache. +// +// It deletes the `key` if given `value` is nil. +// It does nothing if `key` does not exist in the cache. +func (c *AdapterRedis) Update(ctx context.Context, key interface{}, value interface{}) (oldValue *gvar.Var, exist bool, err error) { + var ( + v *gvar.Var + oldDuration time.Duration + ) + // TTL. + v, err = c.redis.Do(ctx, "TTL", key) + if err != nil { + return + } + oldDuration = v.Duration() + if oldDuration == -2 { + // It does not exist. + return + } + // Check existence. + v, err = c.redis.Do(ctx, "GET", key) + if err != nil { + return + } + oldValue = v + // DEL. + if value == nil { + _, err = c.redis.Do(ctx, "DEL", key) + if err != nil { + return + } + return + } + // Update the value. + if oldDuration == -1 { + _, err = c.redis.Do(ctx, "SET", key, value) + } else { + oldDuration *= time.Second + _, err = c.redis.Do(ctx, "SETEX", key, uint64(oldDuration.Seconds()), value) + } + return oldValue, true, err +} + +// UpdateExpire updates the expiration of `key` and returns the old expiration duration value. +// +// It returns -1 and does nothing if the `key` does not exist in the cache. +// It deletes the `key` if `duration` < 0. +func (c *AdapterRedis) UpdateExpire(ctx context.Context, key interface{}, duration time.Duration) (oldDuration time.Duration, err error) { + var ( + v *gvar.Var + ) + // TTL. + v, err = c.redis.Do(ctx, "TTL", key) + if err != nil { + return + } + oldDuration = v.Duration() + if oldDuration == -2 { + // It does not exist. + oldDuration = -1 + return + } + oldDuration *= time.Second + // DEL. + if duration < 0 { + _, err = c.redis.Do(ctx, "DEL", key) + return + } + // Update the expire. + if duration > 0 { + _, err = c.redis.Do(ctx, "EXPIRE", key, uint64(duration.Seconds())) + } + // No expire. + if duration == 0 { + v, err = c.redis.Do(ctx, "GET", key) + if err != nil { + return + } + _, err = c.redis.Do(ctx, "SET", key, v.Val()) + } + return +} + +// GetExpire retrieves and returns the expiration of `key` in the cache. +// +// Note that, +// It returns 0 if the `key` does not expire. +// It returns -1 if the `key` does not exist in the cache. +func (c *AdapterRedis) GetExpire(ctx context.Context, key interface{}) (time.Duration, error) { + v, err := c.redis.Do(ctx, "TTL", key) + if err != nil { + return 0, err + } + switch v.Int() { + case -1: + return 0, nil + case -2: + return -1, nil + default: + return v.Duration() * time.Second, nil + } +} + +// Remove deletes the one or more keys from cache, and returns its value. +// If multiple keys are given, it returns the value of the deleted last item. +func (c *AdapterRedis) Remove(ctx context.Context, keys ...interface{}) (lastValue *gvar.Var, err error) { + if len(keys) == 0 { + return nil, nil + } + // Retrieves the last key value. + if lastValue, err = c.redis.Do(ctx, "GET", keys[len(keys)-1]); err != nil { + return nil, err + } + // Deletes all given keys. + _, err = c.redis.Do(ctx, "DEL", keys...) + return +} + +// Clear clears all data of the cache. +// Note that this function is sensitive and should be carefully used. +// It uses `FLUSHDB` command in redis server, which might be disabled in server. +func (c *AdapterRedis) Clear(ctx context.Context) (err error) { + // The "FLUSHDB" may not be available. + _, err = c.redis.Do(ctx, "FLUSHDB") + return +} + +// Close closes the cache. +func (c *AdapterRedis) Close(ctx context.Context) error { + // It does nothing. + return nil +} diff --git a/os/gcache/gcache_cache.go b/os/gcache/gcache_cache.go index b01e53870..f716d63ac 100644 --- a/os/gcache/gcache_cache.go +++ b/os/gcache/gcache_cache.go @@ -31,7 +31,7 @@ func New(lruCap ...int) *Cache { } // Here may be a "timer leak" if adapter is manually changed from memory adapter. // Do not worry about this, as adapter is less changed, and it does nothing if it's not used. - gtimer.AddSingleton(context.Background(), time.Second, memAdapter.syncEventAndClearExpired) + gtimer.AddSingleton(context.Background(), time.Second, memAdapter.(*AdapterMemory).syncEventAndClearExpired) return c } diff --git a/os/gcache/gcache_z_unit_feature_adapter_redis_test.go b/os/gcache/gcache_z_unit_feature_adapter_redis_test.go new file mode 100644 index 000000000..9a570b7df --- /dev/null +++ b/os/gcache/gcache_z_unit_feature_adapter_redis_test.go @@ -0,0 +1,174 @@ +// Copyright 2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gcache_test + +import ( + "testing" + "time" + + "github.com/gogf/gf/v2/database/gredis" + "github.com/gogf/gf/v2/os/gcache" + "github.com/gogf/gf/v2/test/gtest" +) + +var ( + cacheRedis = gcache.New() + redisConfig = &gredis.Config{ + Address: "127.0.0.1:6379", + Db: 1, + } +) + +func init() { + redis, err := gredis.New(redisConfig) + if err != nil { + panic(err) + } + cacheRedis.SetAdapter(gcache.NewAdapterRedis(redis)) +} + +func Test_AdapterRedis_Basic1(t *testing.T) { + size := 10 + gtest.C(t, func(t *gtest.T) { + for i := 0; i < size; i++ { + cacheRedis.Set(ctx, i, i*10, 0) + } + for i := 0; i < size; i++ { + v, _ := cacheRedis.Get(ctx, i) + t.Assert(v, i*10) + } + n, _ := cacheRedis.Size(ctx) + t.Assert(n, size) + }) + gtest.C(t, func(t *gtest.T) { + data, _ := cacheRedis.Data(ctx) + t.Assert(len(data), size) + t.Assert(data["0"], "0") + t.Assert(data["1"], "10") + t.Assert(data["9"], "90") + }) + gtest.C(t, func(t *gtest.T) { + cacheRedis.Clear(ctx) + n, _ := cacheRedis.Size(ctx) + t.Assert(n, 0) + }) +} + +func Test_AdapterRedis_Basic2(t *testing.T) { + defer cacheRedis.Clear(ctx) + size := 10 + gtest.C(t, func(t *gtest.T) { + for i := 0; i < size; i++ { + cacheRedis.Set(ctx, i, i*10, -1) + } + for i := 0; i < size; i++ { + v, _ := cacheRedis.Get(ctx, i) + t.Assert(v, nil) + } + n, _ := cacheRedis.Size(ctx) + t.Assert(n, 0) + }) +} + +func Test_AdapterRedis_Basic3(t *testing.T) { + defer cacheRedis.Clear(ctx) + size := 10 + gtest.C(t, func(t *gtest.T) { + for i := 0; i < size; i++ { + cacheRedis.Set(ctx, i, i*10, time.Second) + } + for i := 0; i < size; i++ { + v, _ := cacheRedis.Get(ctx, i) + t.Assert(v, i*10) + } + n, _ := cacheRedis.Size(ctx) + t.Assert(n, size) + }) + time.Sleep(time.Second * 2) + gtest.C(t, func(t *gtest.T) { + for i := 0; i < size; i++ { + v, _ := cacheRedis.Get(ctx, i) + t.Assert(v, nil) + } + n, _ := cacheRedis.Size(ctx) + t.Assert(n, 0) + }) +} + +func Test_AdapterRedis_Update(t *testing.T) { + defer cacheRedis.Clear(ctx) + gtest.C(t, func(t *gtest.T) { + var ( + key = "key" + value1 = "value1" + value2 = "value2" + ) + cacheRedis.Set(ctx, key, value1, time.Second) + v, _ := cacheRedis.Get(ctx, key) + t.Assert(v, value1) + + d, _ := cacheRedis.GetExpire(ctx, key) + t.Assert(d > time.Millisecond*500, true) + t.Assert(d <= time.Second, true) + + cacheRedis.Update(ctx, key, value2) + + v, _ = cacheRedis.Get(ctx, key) + t.Assert(v, value2) + d, _ = cacheRedis.GetExpire(ctx, key) + t.Assert(d > time.Millisecond*500, true) + t.Assert(d <= time.Second, true) + }) +} + +func Test_AdapterRedis_UpdateExpire(t *testing.T) { + defer cacheRedis.Clear(ctx) + gtest.C(t, func(t *gtest.T) { + var ( + key = "key" + value = "value" + ) + cacheRedis.Set(ctx, key, value, time.Second) + v, _ := cacheRedis.Get(ctx, key) + t.Assert(v, value) + + d, _ := cacheRedis.GetExpire(ctx, key) + t.Assert(d > time.Millisecond*500, true) + t.Assert(d <= time.Second, true) + + cacheRedis.UpdateExpire(ctx, key, time.Second*2) + + d, _ = cacheRedis.GetExpire(ctx, key) + t.Assert(d > time.Second, true) + t.Assert(d <= 2*time.Second, true) + }) +} + +func Test_AdapterRedis_SetIfNotExist(t *testing.T) { + defer cacheRedis.Clear(ctx) + gtest.C(t, func(t *gtest.T) { + var ( + key = "key" + value1 = "value1" + value2 = "value2" + ) + cacheRedis.Set(ctx, key, value1, time.Second) + v, _ := cacheRedis.Get(ctx, key) + t.Assert(v, value1) + + r, _ := cacheRedis.SetIfNotExist(ctx, key, value2, time.Second*2) + + t.Assert(r, false) + + v, _ = cacheRedis.Get(ctx, key) + t.Assert(v, value1) + + d, _ := cacheRedis.GetExpire(ctx, key) + t.Assert(d > time.Millisecond*500, true) + t.Assert(d <= time.Second, true) + }) +} diff --git a/os/gsession/gsession_storage_redis_hashtable.go b/os/gsession/gsession_storage_redis_hashtable.go index e814b264a..b9d740a56 100644 --- a/os/gsession/gsession_storage_redis_hashtable.go +++ b/os/gsession/gsession_storage_redis_hashtable.go @@ -56,7 +56,7 @@ func (s *StorageRedisHashTable) Get(ctx context.Context, id string, key string) return v.String(), nil } -// GetMap retrieves all key-value pairs as map from storage. +// Data retrieves all key-value pairs as map from storage. func (s *StorageRedisHashTable) Data(ctx context.Context, id string) (data map[string]interface{}, err error) { v, err := s.redis.Do(ctx, "HGETALL", s.key(id)) if err != nil {