From 35cf228d83afce8d60bb2e9a31b0f906347f2175 Mon Sep 17 00:00:00 2001 From: John Guo Date: Sat, 27 Nov 2021 01:10:00 +0800 Subject: [PATCH 1/6] add AdapterRedis for package gcache --- database/gredis/gredis_config.go | 2 +- net/gtrace/gtrace.go | 7 +- os/gcache/gcache_adapter_memory.go | 2 +- os/gcache/gcache_adapter_redis.go | 433 ++++++++++++++++++ os/gcache/gcache_cache.go | 2 +- ...cache_z_unit_feature_adapter_redis_test.go | 174 +++++++ .../gsession_storage_redis_hashtable.go | 2 +- 7 files changed, 615 insertions(+), 7 deletions(-) create mode 100644 os/gcache/gcache_adapter_redis.go create mode 100644 os/gcache/gcache_z_unit_feature_adapter_redis_test.go diff --git a/database/gredis/gredis_config.go b/database/gredis/gredis_config.go index e3431609e..2d94dda84 100644 --- a/database/gredis/gredis_config.go +++ b/database/gredis/gredis_config.go @@ -20,7 +20,7 @@ import ( // Config is redis configuration. type Config struct { - Address string `json:"address"` // It supports single and cluster redis server. Multiple addresses joined with char ','. + Address string `json:"address"` // It supports single and cluster redis server. Multiple addresses joined with char ','. Eg: 192.168.1.1:6379, 192.168.1.2:6379. Db int `json:"db"` // Redis db. Pass string `json:"pass"` // Password for AUTH. MinIdle int `json:"minIdle"` // Minimum number of connections allowed to be idle (default is 0) diff --git a/net/gtrace/gtrace.go b/net/gtrace/gtrace.go index 846cefb47..6b14ca8ac 100644 --- a/net/gtrace/gtrace.go +++ b/net/gtrace/gtrace.go @@ -12,6 +12,8 @@ import ( "os" "strings" + "github.com/gogf/gf/v2/internal/command" + "github.com/gogf/gf/v2/util/gconv" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" @@ -20,7 +22,6 @@ import ( "github.com/gogf/gf/v2/container/gmap" "github.com/gogf/gf/v2/container/gvar" "github.com/gogf/gf/v2/net/gipv4" - "github.com/gogf/gf/v2/os/gcmd" ) const ( @@ -44,8 +45,8 @@ var ( ) func init() { - tracingInternal = gcmd.GetOptWithEnv(commandEnvKeyForTracingInternal, true).Bool() - if maxContentLogSize := gcmd.GetOptWithEnv(commandEnvKeyForMaxContentLogSize).Int(); maxContentLogSize > 0 { + tracingInternal = gconv.Bool(command.GetOptWithEnv(commandEnvKeyForTracingInternal, "true")) + if maxContentLogSize := gconv.Int(command.GetOptWithEnv(commandEnvKeyForMaxContentLogSize)); maxContentLogSize > 0 { tracingMaxContentLogSize = maxContentLogSize } CheckSetDefaultTextMapPropagator() diff --git a/os/gcache/gcache_adapter_memory.go b/os/gcache/gcache_adapter_memory.go index 4d189fc87..539adf85c 100644 --- a/os/gcache/gcache_adapter_memory.go +++ b/os/gcache/gcache_adapter_memory.go @@ -54,7 +54,7 @@ const ( ) // NewAdapterMemory creates and returns a new memory cache object. -func NewAdapterMemory(lruCap ...int) *AdapterMemory { +func NewAdapterMemory(lruCap ...int) Adapter { c := &AdapterMemory{ data: newAdapterMemoryData(), lruGetList: glist.New(true), diff --git a/os/gcache/gcache_adapter_redis.go b/os/gcache/gcache_adapter_redis.go new file mode 100644 index 000000000..03a0e5d27 --- /dev/null +++ b/os/gcache/gcache_adapter_redis.go @@ -0,0 +1,433 @@ +// Copyright 2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gcache + +import ( + "context" + "time" + + "github.com/gogf/gf/v2/container/gvar" + "github.com/gogf/gf/v2/database/gredis" +) + +// AdapterRedis is the gcache adapter implements using Redis server. +type AdapterRedis struct { + redis *gredis.Redis +} + +// NewAdapterRedis creates and returns a new memory cache object. +func NewAdapterRedis(redis *gredis.Redis) Adapter { + return &AdapterRedis{ + redis: redis, + } +} + +// Set sets cache with `key`-`value` pair, which is expired after `duration`. +// +// It does not expire if `duration` == 0. +// It deletes the keys of `data` if `duration` < 0 or given `value` is nil. +func (c *AdapterRedis) Set(ctx context.Context, key interface{}, value interface{}, duration time.Duration) (err error) { + if value == nil || duration < 0 { + _, err = c.redis.Do(ctx, "DEL", key) + } else { + if duration == 0 { + _, err = c.redis.Do(ctx, "SET", key, value) + } else { + _, err = c.redis.Do(ctx, "SETEX", key, uint64(duration.Seconds()), value) + } + } + return err +} + +// SetMap batch sets cache with key-value pairs by `data` map, which is expired after `duration`. +// +// It does not expire if `duration` == 0. +// It deletes the keys of `data` if `duration` < 0 or given `value` is nil. +func (c *AdapterRedis) SetMap(ctx context.Context, data map[interface{}]interface{}, duration time.Duration) error { + if len(data) == 0 { + return nil + } + // DEL. + if duration < 0 { + var ( + index = 0 + keys = make([]interface{}, len(data)) + ) + for k, _ := range data { + keys[index] = k + index += 1 + } + _, err := c.redis.Do(ctx, "DEL", keys...) + if err != nil { + return err + } + } + if duration == 0 { + var ( + index = 0 + keyValues = make([]interface{}, len(data)*2) + ) + for k, v := range data { + keyValues[index] = k + keyValues[index+1] = v + index += 2 + } + _, err := c.redis.Do(ctx, "MSET", keyValues...) + if err != nil { + return err + } + } + if duration > 0 { + var err error + for k, v := range data { + if err = c.Set(ctx, k, v, duration); err != nil { + return err + } + } + } + return nil +} + +// SetIfNotExist sets cache with `key`-`value` pair which is expired after `duration` +// if `key` does not exist in the cache. It returns true the `key` does not exist in the +// cache, and it sets `value` successfully to the cache, or else it returns false. +// +// It does not expire if `duration` == 0. +// It deletes the `key` if `duration` < 0 or given `value` is nil. +func (c *AdapterRedis) SetIfNotExist(ctx context.Context, key interface{}, value interface{}, duration time.Duration) (bool, error) { + var ( + v *gvar.Var + err error + ) + // Execute the function and retrieve the result. + if f, ok := value.(func() (interface{}, error)); ok { + value, err = f() + if value == nil { + return false, err + } + } + // DEL. + if duration < 0 || value == nil { + if v, err = c.redis.Do(ctx, "DEL", key, value); err != nil { + return false, err + } + if v.Int() == 1 { + return true, err + } else { + return false, err + } + } + if v, err = c.redis.Do(ctx, "SETNX", key, value); err != nil { + return false, err + } + if v.Int() > 0 && duration > 0 { + // Set the expiration. + _, err = c.redis.Do(ctx, "EXPIRE", key, uint64(duration.Seconds())) + if err != nil { + return false, err + } + return true, err + } + return false, err +} + +// SetIfNotExistFunc sets `key` with result of function `f` and returns true +// if `key` does not exist in the cache, or else it does nothing and returns false if `key` already exists. +// +// The parameter `value` can be type of `func() interface{}`, but it does nothing if its +// result is nil. +// +// It does not expire if `duration` == 0. +// It deletes the `key` if `duration` < 0 or given `value` is nil. +func (c *AdapterRedis) SetIfNotExistFunc(ctx context.Context, key interface{}, f func() (interface{}, error), duration time.Duration) (ok bool, err error) { + value, err := f() + if err != nil { + return false, err + } + return c.SetIfNotExist(ctx, key, value, duration) +} + +// SetIfNotExistFuncLock sets `key` with result of function `f` and returns true +// if `key` does not exist in the cache, or else it does nothing and returns false if `key` already exists. +// +// It does not expire if `duration` == 0. +// It deletes the `key` if `duration` < 0 or given `value` is nil. +// +// Note that it differs from function `SetIfNotExistFunc` is that the function `f` is executed within +// writing mutex lock for concurrent safety purpose. +func (c *AdapterRedis) SetIfNotExistFuncLock(ctx context.Context, key interface{}, f func() (interface{}, error), duration time.Duration) (ok bool, err error) { + value, err := f() + if err != nil { + return false, err + } + return c.SetIfNotExist(ctx, key, value, duration) +} + +// Get retrieves and returns the associated value of given . +// It returns nil if it does not exist or its value is nil. +func (c *AdapterRedis) Get(ctx context.Context, key interface{}) (*gvar.Var, error) { + return c.redis.Do(ctx, "GET", key) +} + +// GetOrSet retrieves and returns the value of `key`, or sets `key`-`value` pair and +// returns `value` if `key` does not exist in the cache. The key-value pair expires +// after `duration`. +// +// It does not expire if `duration` == 0. +// It deletes the `key` if `duration` < 0 or given `value` is nil, but it does nothing +// if `value` is a function and the function result is nil. +func (c *AdapterRedis) GetOrSet(ctx context.Context, key interface{}, value interface{}, duration time.Duration) (result *gvar.Var, err error) { + result, err = c.Get(ctx, key) + if err != nil { + return nil, err + } + if result == nil { + return gvar.New(value), c.Set(ctx, key, value, duration) + } + return +} + +// GetOrSetFunc retrieves and returns the value of `key`, or sets `key` with result of +// function `f` and returns its result if `key` does not exist in the cache. The key-value +// pair expires after `duration`. +// +// It does not expire if `duration` == 0. +// It deletes the `key` if `duration` < 0 or given `value` is nil, but it does nothing +// if `value` is a function and the function result is nil. +func (c *AdapterRedis) GetOrSetFunc(ctx context.Context, key interface{}, f func() (interface{}, error), duration time.Duration) (result *gvar.Var, err error) { + v, err := c.Get(ctx, key) + if err != nil { + return nil, err + } + if v == nil { + value, err := f() + if err != nil { + return nil, err + } + if value == nil { + return nil, nil + } + return gvar.New(value), c.Set(ctx, key, value, duration) + } else { + return v, nil + } +} + +// GetOrSetFuncLock retrieves and returns the value of `key`, or sets `key` with result of +// function `f` and returns its result if `key` does not exist in the cache. The key-value +// pair expires after `duration`. +// +// It does not expire if `duration` == 0. +// It deletes the `key` if `duration` < 0 or given `value` is nil, but it does nothing +// if `value` is a function and the function result is nil. +// +// Note that it differs from function `GetOrSetFunc` is that the function `f` is executed within +// writing mutex lock for concurrent safety purpose. +func (c *AdapterRedis) GetOrSetFuncLock(ctx context.Context, key interface{}, f func() (interface{}, error), duration time.Duration) (result *gvar.Var, err error) { + return c.GetOrSetFunc(ctx, key, f, duration) +} + +// Contains checks and returns true if `key` exists in the cache, or else returns false. +func (c *AdapterRedis) Contains(ctx context.Context, key interface{}) (bool, error) { + v, err := c.redis.Do(ctx, "EXISTS", key) + if err != nil { + return false, err + } + return v.Bool(), nil +} + +// Size returns the number of items in the cache. +func (c *AdapterRedis) Size(ctx context.Context) (size int, err error) { + v, err := c.redis.Do(ctx, "DBSIZE") + if err != nil { + return 0, err + } + return v.Int(), nil +} + +// Data returns a copy of all key-value pairs in the cache as map type. +// Note that this function may lead lots of memory usage, you can implement this function +// if necessary. +func (c *AdapterRedis) Data(ctx context.Context) (map[interface{}]interface{}, error) { + // Keys. + v, err := c.redis.Do(ctx, "KEYS", "*") + if err != nil { + return nil, err + } + keys := v.Slice() + // Values. + v, err = c.redis.Do(ctx, "MGET", keys...) + if err != nil { + return nil, err + } + values := v.Slice() + // Compose keys and values. + data := make(map[interface{}]interface{}) + for i := 0; i < len(keys); i++ { + data[keys[i]] = values[i] + } + return data, nil +} + +// Keys returns all keys in the cache as slice. +func (c *AdapterRedis) Keys(ctx context.Context) ([]interface{}, error) { + v, err := c.redis.Do(ctx, "KEYS", "*") + if err != nil { + return nil, err + } + return v.Slice(), nil +} + +// Values returns all values in the cache as slice. +func (c *AdapterRedis) Values(ctx context.Context) ([]interface{}, error) { + // Keys. + v, err := c.redis.Do(ctx, "KEYS", "*") + if err != nil { + return nil, err + } + keys := v.Slice() + // Values. + v, err = c.redis.Do(ctx, "MGET", keys...) + if err != nil { + return nil, err + } + return v.Slice(), nil +} + +// Update updates the value of `key` without changing its expiration and returns the old value. +// The returned value `exist` is false if the `key` does not exist in the cache. +// +// It deletes the `key` if given `value` is nil. +// It does nothing if `key` does not exist in the cache. +func (c *AdapterRedis) Update(ctx context.Context, key interface{}, value interface{}) (oldValue *gvar.Var, exist bool, err error) { + var ( + v *gvar.Var + oldDuration time.Duration + ) + // TTL. + v, err = c.redis.Do(ctx, "TTL", key) + if err != nil { + return + } + oldDuration = v.Duration() + if oldDuration == -2 { + // It does not exist. + return + } + // Check existence. + v, err = c.redis.Do(ctx, "GET", key) + if err != nil { + return + } + oldValue = v + // DEL. + if value == nil { + _, err = c.redis.Do(ctx, "DEL", key) + if err != nil { + return + } + return + } + // Update the value. + if oldDuration == -1 { + _, err = c.redis.Do(ctx, "SET", key, value) + } else { + oldDuration *= time.Second + _, err = c.redis.Do(ctx, "SETEX", key, uint64(oldDuration.Seconds()), value) + } + return oldValue, true, err +} + +// UpdateExpire updates the expiration of `key` and returns the old expiration duration value. +// +// It returns -1 and does nothing if the `key` does not exist in the cache. +// It deletes the `key` if `duration` < 0. +func (c *AdapterRedis) UpdateExpire(ctx context.Context, key interface{}, duration time.Duration) (oldDuration time.Duration, err error) { + var ( + v *gvar.Var + ) + // TTL. + v, err = c.redis.Do(ctx, "TTL", key) + if err != nil { + return + } + oldDuration = v.Duration() + if oldDuration == -2 { + // It does not exist. + oldDuration = -1 + return + } + oldDuration *= time.Second + // DEL. + if duration < 0 { + _, err = c.redis.Do(ctx, "DEL", key) + return + } + // Update the expire. + if duration > 0 { + _, err = c.redis.Do(ctx, "EXPIRE", key, uint64(duration.Seconds())) + } + // No expire. + if duration == 0 { + v, err = c.redis.Do(ctx, "GET", key) + if err != nil { + return + } + _, err = c.redis.Do(ctx, "SET", key, v.Val()) + } + return +} + +// GetExpire retrieves and returns the expiration of `key` in the cache. +// +// Note that, +// It returns 0 if the `key` does not expire. +// It returns -1 if the `key` does not exist in the cache. +func (c *AdapterRedis) GetExpire(ctx context.Context, key interface{}) (time.Duration, error) { + v, err := c.redis.Do(ctx, "TTL", key) + if err != nil { + return 0, err + } + switch v.Int() { + case -1: + return 0, nil + case -2: + return -1, nil + default: + return v.Duration() * time.Second, nil + } +} + +// Remove deletes the one or more keys from cache, and returns its value. +// If multiple keys are given, it returns the value of the deleted last item. +func (c *AdapterRedis) Remove(ctx context.Context, keys ...interface{}) (lastValue *gvar.Var, err error) { + if len(keys) == 0 { + return nil, nil + } + // Retrieves the last key value. + if lastValue, err = c.redis.Do(ctx, "GET", keys[len(keys)-1]); err != nil { + return nil, err + } + // Deletes all given keys. + _, err = c.redis.Do(ctx, "DEL", keys...) + return +} + +// Clear clears all data of the cache. +// Note that this function is sensitive and should be carefully used. +// It uses `FLUSHDB` command in redis server, which might be disabled in server. +func (c *AdapterRedis) Clear(ctx context.Context) (err error) { + // The "FLUSHDB" may not be available. + _, err = c.redis.Do(ctx, "FLUSHDB") + return +} + +// Close closes the cache. +func (c *AdapterRedis) Close(ctx context.Context) error { + // It does nothing. + return nil +} diff --git a/os/gcache/gcache_cache.go b/os/gcache/gcache_cache.go index b01e53870..f716d63ac 100644 --- a/os/gcache/gcache_cache.go +++ b/os/gcache/gcache_cache.go @@ -31,7 +31,7 @@ func New(lruCap ...int) *Cache { } // Here may be a "timer leak" if adapter is manually changed from memory adapter. // Do not worry about this, as adapter is less changed, and it does nothing if it's not used. - gtimer.AddSingleton(context.Background(), time.Second, memAdapter.syncEventAndClearExpired) + gtimer.AddSingleton(context.Background(), time.Second, memAdapter.(*AdapterMemory).syncEventAndClearExpired) return c } diff --git a/os/gcache/gcache_z_unit_feature_adapter_redis_test.go b/os/gcache/gcache_z_unit_feature_adapter_redis_test.go new file mode 100644 index 000000000..9a570b7df --- /dev/null +++ b/os/gcache/gcache_z_unit_feature_adapter_redis_test.go @@ -0,0 +1,174 @@ +// Copyright 2020 gf Author(https://github.com/gogf/gf). All Rights Reserved. +// +// This Source Code Form is subject to the terms of the MIT License. +// If a copy of the MIT was not distributed with this file, +// You can obtain one at https://github.com/gogf/gf. + +package gcache_test + +import ( + "testing" + "time" + + "github.com/gogf/gf/v2/database/gredis" + "github.com/gogf/gf/v2/os/gcache" + "github.com/gogf/gf/v2/test/gtest" +) + +var ( + cacheRedis = gcache.New() + redisConfig = &gredis.Config{ + Address: "127.0.0.1:6379", + Db: 1, + } +) + +func init() { + redis, err := gredis.New(redisConfig) + if err != nil { + panic(err) + } + cacheRedis.SetAdapter(gcache.NewAdapterRedis(redis)) +} + +func Test_AdapterRedis_Basic1(t *testing.T) { + size := 10 + gtest.C(t, func(t *gtest.T) { + for i := 0; i < size; i++ { + cacheRedis.Set(ctx, i, i*10, 0) + } + for i := 0; i < size; i++ { + v, _ := cacheRedis.Get(ctx, i) + t.Assert(v, i*10) + } + n, _ := cacheRedis.Size(ctx) + t.Assert(n, size) + }) + gtest.C(t, func(t *gtest.T) { + data, _ := cacheRedis.Data(ctx) + t.Assert(len(data), size) + t.Assert(data["0"], "0") + t.Assert(data["1"], "10") + t.Assert(data["9"], "90") + }) + gtest.C(t, func(t *gtest.T) { + cacheRedis.Clear(ctx) + n, _ := cacheRedis.Size(ctx) + t.Assert(n, 0) + }) +} + +func Test_AdapterRedis_Basic2(t *testing.T) { + defer cacheRedis.Clear(ctx) + size := 10 + gtest.C(t, func(t *gtest.T) { + for i := 0; i < size; i++ { + cacheRedis.Set(ctx, i, i*10, -1) + } + for i := 0; i < size; i++ { + v, _ := cacheRedis.Get(ctx, i) + t.Assert(v, nil) + } + n, _ := cacheRedis.Size(ctx) + t.Assert(n, 0) + }) +} + +func Test_AdapterRedis_Basic3(t *testing.T) { + defer cacheRedis.Clear(ctx) + size := 10 + gtest.C(t, func(t *gtest.T) { + for i := 0; i < size; i++ { + cacheRedis.Set(ctx, i, i*10, time.Second) + } + for i := 0; i < size; i++ { + v, _ := cacheRedis.Get(ctx, i) + t.Assert(v, i*10) + } + n, _ := cacheRedis.Size(ctx) + t.Assert(n, size) + }) + time.Sleep(time.Second * 2) + gtest.C(t, func(t *gtest.T) { + for i := 0; i < size; i++ { + v, _ := cacheRedis.Get(ctx, i) + t.Assert(v, nil) + } + n, _ := cacheRedis.Size(ctx) + t.Assert(n, 0) + }) +} + +func Test_AdapterRedis_Update(t *testing.T) { + defer cacheRedis.Clear(ctx) + gtest.C(t, func(t *gtest.T) { + var ( + key = "key" + value1 = "value1" + value2 = "value2" + ) + cacheRedis.Set(ctx, key, value1, time.Second) + v, _ := cacheRedis.Get(ctx, key) + t.Assert(v, value1) + + d, _ := cacheRedis.GetExpire(ctx, key) + t.Assert(d > time.Millisecond*500, true) + t.Assert(d <= time.Second, true) + + cacheRedis.Update(ctx, key, value2) + + v, _ = cacheRedis.Get(ctx, key) + t.Assert(v, value2) + d, _ = cacheRedis.GetExpire(ctx, key) + t.Assert(d > time.Millisecond*500, true) + t.Assert(d <= time.Second, true) + }) +} + +func Test_AdapterRedis_UpdateExpire(t *testing.T) { + defer cacheRedis.Clear(ctx) + gtest.C(t, func(t *gtest.T) { + var ( + key = "key" + value = "value" + ) + cacheRedis.Set(ctx, key, value, time.Second) + v, _ := cacheRedis.Get(ctx, key) + t.Assert(v, value) + + d, _ := cacheRedis.GetExpire(ctx, key) + t.Assert(d > time.Millisecond*500, true) + t.Assert(d <= time.Second, true) + + cacheRedis.UpdateExpire(ctx, key, time.Second*2) + + d, _ = cacheRedis.GetExpire(ctx, key) + t.Assert(d > time.Second, true) + t.Assert(d <= 2*time.Second, true) + }) +} + +func Test_AdapterRedis_SetIfNotExist(t *testing.T) { + defer cacheRedis.Clear(ctx) + gtest.C(t, func(t *gtest.T) { + var ( + key = "key" + value1 = "value1" + value2 = "value2" + ) + cacheRedis.Set(ctx, key, value1, time.Second) + v, _ := cacheRedis.Get(ctx, key) + t.Assert(v, value1) + + r, _ := cacheRedis.SetIfNotExist(ctx, key, value2, time.Second*2) + + t.Assert(r, false) + + v, _ = cacheRedis.Get(ctx, key) + t.Assert(v, value1) + + d, _ := cacheRedis.GetExpire(ctx, key) + t.Assert(d > time.Millisecond*500, true) + t.Assert(d <= time.Second, true) + }) +} diff --git a/os/gsession/gsession_storage_redis_hashtable.go b/os/gsession/gsession_storage_redis_hashtable.go index e814b264a..b9d740a56 100644 --- a/os/gsession/gsession_storage_redis_hashtable.go +++ b/os/gsession/gsession_storage_redis_hashtable.go @@ -56,7 +56,7 @@ func (s *StorageRedisHashTable) Get(ctx context.Context, id string, key string) return v.String(), nil } -// GetMap retrieves all key-value pairs as map from storage. +// Data retrieves all key-value pairs as map from storage. func (s *StorageRedisHashTable) Data(ctx context.Context, id string) (data map[string]interface{}, err error) { v, err := s.redis.Do(ctx, "HGETALL", s.key(id)) if err != nil { From c33378540aa9156fc1efd27dcc3c537f80104ef6 Mon Sep 17 00:00:00 2001 From: John Guo Date: Sat, 27 Nov 2021 01:33:53 +0800 Subject: [PATCH 2/6] add Available function for Adapter of package gcfg --- os/gcfg/gcfg.go | 9 ++++++--- os/gcfg/gcfg_adaper.go | 7 +++++++ os/gcfg/gcfg_adapter_file.go | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/os/gcfg/gcfg.go b/os/gcfg/gcfg.go index df038492a..f21bb8013 100644 --- a/os/gcfg/gcfg.go +++ b/os/gcfg/gcfg.go @@ -54,7 +54,10 @@ func NewWithAdapter(adapter Adapter) *Config { // exists in the configuration directory, it then sets it as the default configuration file. The // toml file type is the default configuration file type. func Instance(name ...string) *Config { - key := DefaultName + var ( + ctx = context.TODO() + key = DefaultName + ) if len(name) > 0 && name[0] != "" { key = name[0] } @@ -67,9 +70,9 @@ func Instance(name ...string) *Config { // If it's not using default configuration or its configuration file is not available, // it searches the possible configuration file according to the name and all supported // file types. - if key != DefaultName || !adapter.Available() { + if key != DefaultName || !adapter.Available(ctx) { for _, fileType := range supportedFileTypes { - if file := fmt.Sprintf(`%s.%s`, key, fileType); adapter.Available(file) { + if file := fmt.Sprintf(`%s.%s`, key, fileType); adapter.Available(ctx, file) { adapter.SetFileName(file) break } diff --git a/os/gcfg/gcfg_adaper.go b/os/gcfg/gcfg_adaper.go index 3d37d3edd..4c354e62c 100644 --- a/os/gcfg/gcfg_adaper.go +++ b/os/gcfg/gcfg_adaper.go @@ -10,6 +10,13 @@ import "context" // Adapter is the interface for configuration retrieving. type Adapter interface { + // Available checks and returns the configuration service is available. + // The optional parameter `pattern` specifies certain configuration resource. + // + // It returns true if configuration file is present in default AdapterFile, or else false. + // Note that this function does not return error as it just does simply check for backend configuration service. + Available(ctx context.Context, pattern ...string) (ok bool) + // Get retrieves and returns value by specified `pattern`. Get(ctx context.Context, pattern string) (value interface{}, err error) diff --git a/os/gcfg/gcfg_adapter_file.go b/os/gcfg/gcfg_adapter_file.go index f2a15f2ae..54c306b9f 100644 --- a/os/gcfg/gcfg_adapter_file.go +++ b/os/gcfg/gcfg_adapter_file.go @@ -175,7 +175,7 @@ func (c *AdapterFile) Dump() { } // Available checks and returns whether configuration of given `file` is available. -func (c *AdapterFile) Available(fileName ...string) bool { +func (c *AdapterFile) Available(ctx context.Context, fileName ...string) bool { var ( usedFileName string ) From 817c3ce698904bdb8beaffb6e782b94a28eed9ec Mon Sep 17 00:00:00 2001 From: John Guo Date: Sat, 27 Nov 2021 01:36:21 +0800 Subject: [PATCH 3/6] add Available function for Adapter of package gcfg --- os/gcfg/gcfg.go | 9 +++++++++ os/gcfg/gcfg_adaper.go | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/os/gcfg/gcfg.go b/os/gcfg/gcfg.go index f21bb8013..4bf0f5f68 100644 --- a/os/gcfg/gcfg.go +++ b/os/gcfg/gcfg.go @@ -92,6 +92,15 @@ func (c *Config) GetAdapter() Adapter { return c.adapter } +// Available checks and returns the configuration service is available. +// The optional parameter `pattern` specifies certain configuration resource. +// +// It returns true if configuration file is present in default AdapterFile, or else false. +// Note that this function does not return error as it just does simply check for backend configuration service. +func (c *Config) Available(ctx context.Context, resource ...string) (ok bool) { + return c.adapter.Available(ctx, resource...) +} + // Set sets value with specified `pattern`. // It supports hierarchical data access by char separator, which is '.' in default. // It is commonly used for updates certain configuration value in runtime. diff --git a/os/gcfg/gcfg_adaper.go b/os/gcfg/gcfg_adaper.go index 4c354e62c..e45c5cd5c 100644 --- a/os/gcfg/gcfg_adaper.go +++ b/os/gcfg/gcfg_adaper.go @@ -11,11 +11,11 @@ import "context" // Adapter is the interface for configuration retrieving. type Adapter interface { // Available checks and returns the configuration service is available. - // The optional parameter `pattern` specifies certain configuration resource. + // The optional parameter `resource` specifies certain configuration resource. // // It returns true if configuration file is present in default AdapterFile, or else false. // Note that this function does not return error as it just does simply check for backend configuration service. - Available(ctx context.Context, pattern ...string) (ok bool) + Available(ctx context.Context, resource ...string) (ok bool) // Get retrieves and returns value by specified `pattern`. Get(ctx context.Context, pattern string) (value interface{}, err error) From bf8afc96e4fd0857a3778a35508e931156d7da1e Mon Sep 17 00:00:00 2001 From: John Guo Date: Sat, 27 Nov 2021 11:53:05 +0800 Subject: [PATCH 4/6] improve instance creating of database for package gins --- frame/gins/gins_database.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/frame/gins/gins_database.go b/frame/gins/gins_database.go index e807188fc..f1507d2ac 100644 --- a/frame/gins/gins_database.go +++ b/frame/gins/gins_database.go @@ -60,20 +60,21 @@ func Database(name ...string) gdb.DB { configFilePath string ) if fileConfig, ok := Config().GetAdapter().(*gcfg.AdapterFile); ok { - if configFilePath, err = fileConfig.GetFilePath(); configFilePath == "" { - exampleFileName := "config.example.toml" - if exampleConfigFilePath, _ := fileConfig.GetFilePath(exampleFileName); exampleConfigFilePath != "" { - err = gerror.WrapCodef( + if configFilePath, _ = fileConfig.GetFilePath(); configFilePath == "" { + var ( + exampleFileName = "config.example.toml" + exampleConfigFilePath string + ) + if exampleConfigFilePath, _ = fileConfig.GetFilePath(exampleFileName); exampleConfigFilePath != "" { + err = gerror.NewCodef( gcode.CodeMissingConfiguration, - err, `configuration file "%s" not found, but found "%s", did you miss renaming the example configuration file?`, fileConfig.GetFileName(), exampleFileName, ) } else { - err = gerror.WrapCodef( + err = gerror.NewCodef( gcode.CodeMissingConfiguration, - err, `configuration file "%s" not found, did you miss the configuration file or the misspell the configuration file name?`, fileConfig.GetFileName(), ) @@ -85,9 +86,8 @@ func Database(name ...string) gdb.DB { } // Panic if nothing found in Config object or in gdb configuration. if len(configMap) == 0 && !gdb.IsConfigured() { - err = gerror.WrapCodef( + err = gerror.NewCodef( gcode.CodeMissingConfiguration, - err, `database initialization failed: "%s" node not found, is configuration file or configuration node missing?`, configNodeNameDatabase, ) From 7bdb4fe660f2803296aaec49af9efb70f1fcbc56 Mon Sep 17 00:00:00 2001 From: John Guo Date: Sat, 27 Nov 2021 12:10:05 +0800 Subject: [PATCH 5/6] fix unit testing case for package gcache --- ...cache_z_unit_feature_adapter_redis_test.go | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/os/gcache/gcache_z_unit_feature_adapter_redis_test.go b/os/gcache/gcache_z_unit_feature_adapter_redis_test.go index 9a570b7df..b88500751 100644 --- a/os/gcache/gcache_z_unit_feature_adapter_redis_test.go +++ b/os/gcache/gcache_z_unit_feature_adapter_redis_test.go @@ -19,7 +19,7 @@ var ( cacheRedis = gcache.New() redisConfig = &gredis.Config{ Address: "127.0.0.1:6379", - Db: 1, + Db: 2, } ) @@ -32,10 +32,11 @@ func init() { } func Test_AdapterRedis_Basic1(t *testing.T) { + // Set size := 10 gtest.C(t, func(t *gtest.T) { for i := 0; i < size; i++ { - cacheRedis.Set(ctx, i, i*10, 0) + t.AssertNil(cacheRedis.Set(ctx, i, i*10, 0)) } for i := 0; i < size; i++ { v, _ := cacheRedis.Get(ctx, i) @@ -44,6 +45,7 @@ func Test_AdapterRedis_Basic1(t *testing.T) { n, _ := cacheRedis.Size(ctx) t.Assert(n, size) }) + // Data gtest.C(t, func(t *gtest.T) { data, _ := cacheRedis.Data(ctx) t.Assert(len(data), size) @@ -51,8 +53,9 @@ func Test_AdapterRedis_Basic1(t *testing.T) { t.Assert(data["1"], "10") t.Assert(data["9"], "90") }) + // Clear gtest.C(t, func(t *gtest.T) { - cacheRedis.Clear(ctx) + t.AssertNil(cacheRedis.Clear(ctx)) n, _ := cacheRedis.Size(ctx) t.Assert(n, 0) }) @@ -63,7 +66,7 @@ func Test_AdapterRedis_Basic2(t *testing.T) { size := 10 gtest.C(t, func(t *gtest.T) { for i := 0; i < size; i++ { - cacheRedis.Set(ctx, i, i*10, -1) + t.AssertNil(cacheRedis.Set(ctx, i, i*10, -1)) } for i := 0; i < size; i++ { v, _ := cacheRedis.Get(ctx, i) @@ -79,7 +82,7 @@ func Test_AdapterRedis_Basic3(t *testing.T) { size := 10 gtest.C(t, func(t *gtest.T) { for i := 0; i < size; i++ { - cacheRedis.Set(ctx, i, i*10, time.Second) + t.AssertNil(cacheRedis.Set(ctx, i, i*10, time.Second)) } for i := 0; i < size; i++ { v, _ := cacheRedis.Get(ctx, i) @@ -107,7 +110,7 @@ func Test_AdapterRedis_Update(t *testing.T) { value1 = "value1" value2 = "value2" ) - cacheRedis.Set(ctx, key, value1, time.Second) + t.AssertNil(cacheRedis.Set(ctx, key, value1, time.Second)) v, _ := cacheRedis.Get(ctx, key) t.Assert(v, value1) @@ -115,7 +118,8 @@ func Test_AdapterRedis_Update(t *testing.T) { t.Assert(d > time.Millisecond*500, true) t.Assert(d <= time.Second, true) - cacheRedis.Update(ctx, key, value2) + _, _, err := cacheRedis.Update(ctx, key, value2) + t.AssertNil(err) v, _ = cacheRedis.Get(ctx, key) t.Assert(v, value2) @@ -132,7 +136,7 @@ func Test_AdapterRedis_UpdateExpire(t *testing.T) { key = "key" value = "value" ) - cacheRedis.Set(ctx, key, value, time.Second) + t.AssertNil(cacheRedis.Set(ctx, key, value, time.Second)) v, _ := cacheRedis.Get(ctx, key) t.Assert(v, value) @@ -140,7 +144,8 @@ func Test_AdapterRedis_UpdateExpire(t *testing.T) { t.Assert(d > time.Millisecond*500, true) t.Assert(d <= time.Second, true) - cacheRedis.UpdateExpire(ctx, key, time.Second*2) + _, err := cacheRedis.UpdateExpire(ctx, key, time.Second*2) + t.AssertNil(err) d, _ = cacheRedis.GetExpire(ctx, key) t.Assert(d > time.Second, true) @@ -156,7 +161,7 @@ func Test_AdapterRedis_SetIfNotExist(t *testing.T) { value1 = "value1" value2 = "value2" ) - cacheRedis.Set(ctx, key, value1, time.Second) + t.AssertNil(cacheRedis.Set(ctx, key, value1, time.Second)) v, _ := cacheRedis.Get(ctx, key) t.Assert(v, value1) From 5073413ffc0def03806e5813558389c745bc819b Mon Sep 17 00:00:00 2001 From: John Guo Date: Sat, 27 Nov 2021 12:59:41 +0800 Subject: [PATCH 6/6] improve TableFields feature for package gdb --- database/gdb/gdb.go | 4 +- database/gdb/gdb_core_structure.go | 44 ++++++++-------- database/gdb/gdb_core_utility.go | 6 ++- database/gdb/gdb_driver_mssql.go | 71 +++++++++++++------------- database/gdb/gdb_driver_mysql.go | 59 +++++++++++---------- database/gdb/gdb_driver_oracle.go | 55 ++++++++++---------- database/gdb/gdb_driver_pgsql.go | 67 ++++++++++++------------ database/gdb/gdb_driver_sqlite.go | 51 +++++++++--------- database/gdb/gdb_model_fields.go | 6 ++- database/gdb/gdb_model_time.go | 1 + database/gdb/gdb_model_utility.go | 6 +-- database/gdb/gdb_z_mysql_model_test.go | 1 + 12 files changed, 189 insertions(+), 182 deletions(-) diff --git a/database/gdb/gdb.go b/database/gdb/gdb.go index be855644b..7e679c86e 100644 --- a/database/gdb/gdb.go +++ b/database/gdb/gdb.go @@ -227,10 +227,10 @@ type TableField struct { Name string // Field name. Type string // Field type. Null bool // Field can be null or not. - Key string // The index information(empty if it's not a index). + Key string // The index information(empty if it's not an index). Default interface{} // Default value for the field. Extra string // Extra information. - Comment string // Comment. + Comment string // Field comment. } // Counter is the type for update count. diff --git a/database/gdb/gdb_core_structure.go b/database/gdb/gdb_core_structure.go index ebed7daa8..6fb33872e 100644 --- a/database/gdb/gdb_core_structure.go +++ b/database/gdb/gdb_core_structure.go @@ -147,29 +147,31 @@ func (c *Core) convertFieldValueToLocalValue(fieldValue interface{}, fieldType s // mappingAndFilterData automatically mappings the map key to table field and removes // all key-value pairs that are not the field of given table. func (c *Core) mappingAndFilterData(schema, table string, data map[string]interface{}, filter bool) (map[string]interface{}, error) { - if fieldsMap, err := c.db.TableFields(c.GetCtx(), c.guessPrimaryTableName(table), schema); err == nil { - fieldsKeyMap := make(map[string]interface{}, len(fieldsMap)) - for k := range fieldsMap { - fieldsKeyMap[k] = nil - } - // Automatic data key to table field name mapping. - var foundKey string - for dataKey, dataValue := range data { - if _, ok := fieldsKeyMap[dataKey]; !ok { - foundKey, _ = gutil.MapPossibleItemByKey(fieldsKeyMap, dataKey) - if foundKey != "" { - data[foundKey] = dataValue - delete(data, dataKey) - } + fieldsMap, err := c.db.TableFields(c.GetCtx(), c.guessPrimaryTableName(table), schema) + if err != nil { + return nil, err + } + fieldsKeyMap := make(map[string]interface{}, len(fieldsMap)) + for k := range fieldsMap { + fieldsKeyMap[k] = nil + } + // Automatic data key to table field name mapping. + var foundKey string + for dataKey, dataValue := range data { + if _, ok := fieldsKeyMap[dataKey]; !ok { + foundKey, _ = gutil.MapPossibleItemByKey(fieldsKeyMap, dataKey) + if foundKey != "" { + data[foundKey] = dataValue + delete(data, dataKey) } } - // Data filtering. - // It deletes all key-value pairs that has incorrect field name. - if filter { - for dataKey := range data { - if _, ok := fieldsMap[dataKey]; !ok { - delete(data, dataKey) - } + } + // Data filtering. + // It deletes all key-value pairs that has incorrect field name. + if filter { + for dataKey := range data { + if _, ok := fieldsMap[dataKey]; !ok { + delete(data, dataKey) } } } diff --git a/database/gdb/gdb_core_utility.go b/database/gdb/gdb_core_utility.go index 9ddfd1489..a8f555684 100644 --- a/database/gdb/gdb_core_utility.go +++ b/database/gdb/gdb_core_utility.go @@ -42,6 +42,10 @@ func (c *Core) SlaveLink(schema ...string) (Link, error) { // // The meaning of a `word` can be considered as a column name. func (c *Core) QuoteWord(s string) string { + s = gstr.Trim(s) + if s == "" { + return s + } charLeft, charRight := c.db.GetChars() return doQuoteWord(s, charLeft, charRight) } @@ -83,7 +87,7 @@ func (c *Core) Tables(schema ...string) (tables []string, err error) { return } -// TableFields retrieves and returns the fields information of specified table of current schema. +// TableFields retrieves and returns the fields' information of specified table of current schema. // // Note that it returns a map containing the field name and its corresponding fields. // As a map is unsorted, the TableField struct has a "Index" field marks its sequence in the fields. diff --git a/database/gdb/gdb_driver_mssql.go b/database/gdb/gdb_driver_mssql.go index d55ac4af4..74a5f6566 100644 --- a/database/gdb/gdb_driver_mssql.go +++ b/database/gdb/gdb_driver_mssql.go @@ -38,7 +38,7 @@ func (d *DriverMssql) New(core *Core, node *ConfigNode) (DB, error) { }, nil } -// Open creates and returns a underlying sql.DB object for mssql. +// Open creates and returns an underlying sql.DB object for mssql. func (d *DriverMssql) Open(config *ConfigNode) (*sql.DB, error) { source := "" if config.Link != "" { @@ -83,7 +83,7 @@ func (d *DriverMssql) DoCommit(ctx context.Context, link Link, sql string, args newSql, newArgs, err = d.Core.DoCommit(ctx, link, newSql, newArgs) }() var index int - // Convert place holder char '?' to string "@px". + // Convert placeholder char '?' to string "@px". str, _ := gregex.ReplaceStringFunc("\\?", sql, func(s string) string { index++ return fmt.Sprintf("@p%d", index) @@ -219,19 +219,17 @@ func (d *DriverMssql) TableFields(ctx context.Context, table string, schema ...s if len(schema) > 0 && schema[0] != "" { useSchema = schema[0] } - tableFieldsCacheKey := fmt.Sprintf( - `mssql_table_fields_%s_%s@group:%s`, - table, useSchema, d.GetGroup(), - ) - v := tableFieldsMap.GetOrSetFuncLock(tableFieldsCacheKey, func() interface{} { - var ( - result Result - link, err = d.SlaveLink(useSchema) - ) - if err != nil { - return nil - } - structureSql := fmt.Sprintf(` + v := tableFieldsMap.GetOrSetFuncLock( + fmt.Sprintf(`mssql_table_fields_%s_%s@group:%s`, table, useSchema, d.GetGroup()), + func() interface{} { + var ( + result Result + link Link + ) + if link, err = d.SlaveLink(useSchema); err != nil { + return nil + } + structureSql := fmt.Sprintf(` SELECT a.name Field, CASE b.name @@ -259,28 +257,29 @@ LEFT JOIN sys.extended_properties g ON a.id=g.major_id AND a.colid=g.minor_id LEFT JOIN sys.extended_properties f ON d.id=f.major_id AND f.minor_id =0 WHERE d.name='%s' ORDER BY a.id,a.colorder`, - strings.ToUpper(table), - ) - structureSql, _ = gregex.ReplaceString(`[\n\r\s]+`, " ", gstr.Trim(structureSql)) - result, err = d.DoGetAll(ctx, link, structureSql) - if err != nil { - return nil - } - fields = make(map[string]*TableField) - for i, m := range result { - fields[strings.ToLower(m["Field"].String())] = &TableField{ - Index: i, - Name: strings.ToLower(m["Field"].String()), - Type: strings.ToLower(m["Type"].String()), - Null: m["Null"].Bool(), - Key: m["Key"].String(), - Default: m["Default"].Val(), - Extra: m["Extra"].String(), - Comment: m["Comment"].String(), + strings.ToUpper(table), + ) + structureSql, _ = gregex.ReplaceString(`[\n\r\s]+`, " ", gstr.Trim(structureSql)) + result, err = d.DoGetAll(ctx, link, structureSql) + if err != nil { + return nil } - } - return fields - }) + fields = make(map[string]*TableField) + for i, m := range result { + fields[strings.ToLower(m["Field"].String())] = &TableField{ + Index: i, + Name: strings.ToLower(m["Field"].String()), + Type: strings.ToLower(m["Type"].String()), + Null: m["Null"].Bool(), + Key: m["Key"].String(), + Default: m["Default"].Val(), + Extra: m["Extra"].String(), + Comment: m["Comment"].String(), + } + } + return fields + }, + ) if v != nil { fields = v.(map[string]*TableField) } diff --git a/database/gdb/gdb_driver_mysql.go b/database/gdb/gdb_driver_mysql.go index b93c6c2f0..2f399882e 100644 --- a/database/gdb/gdb_driver_mysql.go +++ b/database/gdb/gdb_driver_mysql.go @@ -128,37 +128,36 @@ func (d *DriverMysql) TableFields(ctx context.Context, table string, schema ...s if len(schema) > 0 && schema[0] != "" { useSchema = schema[0] } - tableFieldsCacheKey := fmt.Sprintf( - `mysql_table_fields_%s_%s@group:%s`, - table, useSchema, d.GetGroup(), - ) - v := tableFieldsMap.GetOrSetFuncLock(tableFieldsCacheKey, func() interface{} { - var ( - result Result - link, err = d.SlaveLink(useSchema) - ) - if err != nil { - return nil - } - result, err = d.DoGetAll(ctx, link, fmt.Sprintf(`SHOW FULL COLUMNS FROM %s`, d.QuoteWord(table))) - if err != nil { - return nil - } - fields = make(map[string]*TableField) - for i, m := range result { - fields[m["Field"].String()] = &TableField{ - Index: i, - Name: m["Field"].String(), - Type: m["Type"].String(), - Null: m["Null"].Bool(), - Key: m["Key"].String(), - Default: m["Default"].Val(), - Extra: m["Extra"].String(), - Comment: m["Comment"].String(), + v := tableFieldsMap.GetOrSetFuncLock( + fmt.Sprintf(`mysql_table_fields_%s_%s@group:%s`, table, useSchema, d.GetGroup()), + func() interface{} { + var ( + result Result + link Link + ) + if link, err = d.SlaveLink(useSchema); err != nil { + return nil } - } - return fields - }) + result, err = d.DoGetAll(ctx, link, fmt.Sprintf(`SHOW FULL COLUMNS FROM %s`, d.QuoteWord(table))) + if err != nil { + return nil + } + fields = make(map[string]*TableField) + for i, m := range result { + fields[m["Field"].String()] = &TableField{ + Index: i, + Name: m["Field"].String(), + Type: m["Type"].String(), + Null: m["Null"].Bool(), + Key: m["Key"].String(), + Default: m["Default"].Val(), + Extra: m["Extra"].String(), + Comment: m["Comment"].String(), + } + } + return fields + }, + ) if v != nil { fields = v.(map[string]*TableField) } diff --git a/database/gdb/gdb_driver_oracle.go b/database/gdb/gdb_driver_oracle.go index aced866f1..9d3de0617 100644 --- a/database/gdb/gdb_driver_oracle.go +++ b/database/gdb/gdb_driver_oracle.go @@ -193,15 +193,13 @@ func (d *DriverOracle) TableFields(ctx context.Context, table string, schema ... if len(schema) > 0 && schema[0] != "" { useSchema = schema[0] } - tableFieldsCacheKey := fmt.Sprintf( - `oracle_table_fields_%s_%s@group:%s`, - table, useSchema, d.GetGroup(), - ) - v := tableFieldsMap.GetOrSetFuncLock(tableFieldsCacheKey, func() interface{} { - var ( - result Result - link, err = d.SlaveLink(useSchema) - structureSql = fmt.Sprintf(` + v := tableFieldsMap.GetOrSetFuncLock( + fmt.Sprintf(`oracle_table_fields_%s_%s@group:%s`, table, useSchema, d.GetGroup()), + func() interface{} { + var ( + result Result + link Link + structureSql = fmt.Sprintf(` SELECT COLUMN_NAME AS FIELD, CASE DATA_TYPE @@ -209,27 +207,28 @@ SELECT WHEN 'FLOAT' THEN DATA_TYPE||'('||DATA_PRECISION||','||DATA_SCALE||')' ELSE DATA_TYPE||'('||DATA_LENGTH||')' END AS TYPE FROM USER_TAB_COLUMNS WHERE TABLE_NAME = '%s' ORDER BY COLUMN_ID`, - strings.ToUpper(table), + strings.ToUpper(table), + ) ) - ) - if err != nil { - return nil - } - structureSql, _ = gregex.ReplaceString(`[\n\r\s]+`, " ", gstr.Trim(structureSql)) - result, err = d.DoGetAll(ctx, link, structureSql) - if err != nil { - return nil - } - fields = make(map[string]*TableField) - for i, m := range result { - fields[strings.ToLower(m["FIELD"].String())] = &TableField{ - Index: i, - Name: strings.ToLower(m["FIELD"].String()), - Type: strings.ToLower(m["TYPE"].String()), + if link, err = d.SlaveLink(useSchema); err != nil { + return nil } - } - return fields - }) + structureSql, _ = gregex.ReplaceString(`[\n\r\s]+`, " ", gstr.Trim(structureSql)) + result, err = d.DoGetAll(ctx, link, structureSql) + if err != nil { + return nil + } + fields = make(map[string]*TableField) + for i, m := range result { + fields[strings.ToLower(m["FIELD"].String())] = &TableField{ + Index: i, + Name: strings.ToLower(m["FIELD"].String()), + Type: strings.ToLower(m["TYPE"].String()), + } + } + return fields + }, + ) if v != nil { fields = v.(map[string]*TableField) } diff --git a/database/gdb/gdb_driver_pgsql.go b/database/gdb/gdb_driver_pgsql.go index ab1e9801e..60ce1e266 100644 --- a/database/gdb/gdb_driver_pgsql.go +++ b/database/gdb/gdb_driver_pgsql.go @@ -86,7 +86,7 @@ func (d *DriverPgsql) DoCommit(ctx context.Context, link Link, sql string, args }() var index int - // Convert place holder char '?' to string "$x". + // Convert placeholder char '?' to string "$x". sql, _ = gregex.ReplaceStringFunc("\\?", sql, func(s string) string { index++ return fmt.Sprintf("$%d", index) @@ -119,7 +119,7 @@ func (d *DriverPgsql) Tables(ctx context.Context, schema ...string) (tables []st return } -// TableFields retrieves and returns the fields information of specified table of current schema. +// TableFields retrieves and returns the fields' information of specified table of current schema. // // Also see DriverMysql.TableFields. func (d *DriverPgsql) TableFields(ctx context.Context, table string, schema ...string) (fields map[string]*TableField, err error) { @@ -133,15 +133,13 @@ func (d *DriverPgsql) TableFields(ctx context.Context, table string, schema ...s if len(schema) > 0 && schema[0] != "" { useSchema = schema[0] } - tableFieldsCacheKey := fmt.Sprintf( - `pgsql_table_fields_%s_%s@group:%s`, - table, useSchema, d.GetGroup(), - ) - v := tableFieldsMap.GetOrSetFuncLock(tableFieldsCacheKey, func() interface{} { - var ( - result Result - link, err = d.SlaveLink(useSchema) - structureSql = fmt.Sprintf(` + v := tableFieldsMap.GetOrSetFuncLock( + fmt.Sprintf(`pgsql_table_fields_%s_%s@group:%s`, table, useSchema, d.GetGroup()), + func() interface{} { + var ( + result Result + link Link + structureSql = fmt.Sprintf(` SELECT a.attname AS field, t.typname AS type,a.attnotnull as null, (case when d.contype is not null then 'pri' else '' end) as key ,ic.column_default as default_value,b.description as comment @@ -155,31 +153,32 @@ FROM pg_attribute a left join information_schema.columns ic on ic.column_name = a.attname and ic.table_name = c.relname WHERE c.relname = '%s' and a.attnum > 0 ORDER BY a.attnum`, - strings.ToLower(table), + strings.ToLower(table), + ) ) - ) - if err != nil { - return nil - } - structureSql, _ = gregex.ReplaceString(`[\n\r\s]+`, " ", gstr.Trim(structureSql)) - result, err = d.DoGetAll(ctx, link, structureSql) - if err != nil { - return nil - } - fields = make(map[string]*TableField) - for i, m := range result { - fields[m["field"].String()] = &TableField{ - Index: i, - Name: m["field"].String(), - Type: m["type"].String(), - Null: m["null"].Bool(), - Key: m["key"].String(), - Default: m["default_value"].Val(), - Comment: m["comment"].String(), + if link, err = d.SlaveLink(useSchema); err != nil { + return nil } - } - return fields - }) + structureSql, _ = gregex.ReplaceString(`[\n\r\s]+`, " ", gstr.Trim(structureSql)) + result, err = d.DoGetAll(ctx, link, structureSql) + if err != nil { + return nil + } + fields = make(map[string]*TableField) + for i, m := range result { + fields[m["field"].String()] = &TableField{ + Index: i, + Name: m["field"].String(), + Type: m["type"].String(), + Null: m["null"].Bool(), + Key: m["key"].String(), + Default: m["default_value"].Val(), + Comment: m["comment"].String(), + } + } + return fields + }, + ) if v != nil { fields = v.(map[string]*TableField) } diff --git a/database/gdb/gdb_driver_sqlite.go b/database/gdb/gdb_driver_sqlite.go index 15851f7f7..155270dca 100644 --- a/database/gdb/gdb_driver_sqlite.go +++ b/database/gdb/gdb_driver_sqlite.go @@ -93,7 +93,7 @@ func (d *DriverSqlite) Tables(ctx context.Context, schema ...string) (tables []s return } -// TableFields retrieves and returns the fields information of specified table of current schema. +// TableFields retrieves and returns the fields' information of specified table of current schema. // // Also see DriverMysql.TableFields. func (d *DriverSqlite) TableFields(ctx context.Context, table string, schema ...string) (fields map[string]*TableField, err error) { @@ -106,32 +106,31 @@ func (d *DriverSqlite) TableFields(ctx context.Context, table string, schema ... if len(schema) > 0 && schema[0] != "" { useSchema = schema[0] } - tableFieldsCacheKey := fmt.Sprintf( - `sqlite_table_fields_%s_%s@group:%s`, - table, useSchema, d.GetGroup(), - ) - v := tableFieldsMap.GetOrSetFuncLock(tableFieldsCacheKey, func() interface{} { - var ( - result Result - link, err = d.SlaveLink(useSchema) - ) - if err != nil { - return nil - } - result, err = d.DoGetAll(ctx, link, fmt.Sprintf(`PRAGMA TABLE_INFO(%s)`, table)) - if err != nil { - return nil - } - fields = make(map[string]*TableField) - for i, m := range result { - fields[strings.ToLower(m["name"].String())] = &TableField{ - Index: i, - Name: strings.ToLower(m["name"].String()), - Type: strings.ToLower(m["type"].String()), + v := tableFieldsMap.GetOrSetFuncLock( + fmt.Sprintf(`sqlite_table_fields_%s_%s@group:%s`, table, useSchema, d.GetGroup()), + func() interface{} { + var ( + result Result + link Link + ) + if link, err = d.SlaveLink(useSchema); err != nil { + return nil } - } - return fields - }) + result, err = d.DoGetAll(ctx, link, fmt.Sprintf(`PRAGMA TABLE_INFO(%s)`, table)) + if err != nil { + return nil + } + fields = make(map[string]*TableField) + for i, m := range result { + fields[strings.ToLower(m["name"].String())] = &TableField{ + Index: i, + Name: strings.ToLower(m["name"].String()), + Type: strings.ToLower(m["type"].String()), + } + } + return fields + }, + ) if v != nil { fields = v.(map[string]*TableField) } diff --git a/database/gdb/gdb_model_fields.go b/database/gdb/gdb_model_fields.go index a9daa9aea..9d2cd34a8 100644 --- a/database/gdb/gdb_model_fields.go +++ b/database/gdb/gdb_model_fields.go @@ -34,6 +34,7 @@ func (m *Model) Fields(fieldNamesOrMapStruct ...interface{}) *Model { m.mappingAndFilterToTableFields(gconv.Strings(fieldNamesOrMapStruct), true), ",", )) + // It needs type asserting. case length == 1: structOrMap := fieldNamesOrMapStruct[0] @@ -80,7 +81,10 @@ func (m *Model) FieldsEx(fieldNamesOrMapStruct ...interface{}) *Model { model := m.getModel() switch { case length >= 2: - model.fieldsEx = gstr.Join(m.mappingAndFilterToTableFields(gconv.Strings(fieldNamesOrMapStruct), true), ",") + model.fieldsEx = gstr.Join( + m.mappingAndFilterToTableFields(gconv.Strings(fieldNamesOrMapStruct), true), + ",", + ) return model case length == 1: switch r := fieldNamesOrMapStruct[0].(type) { diff --git a/database/gdb/gdb_model_time.go b/database/gdb/gdb_model_time.go index e1120fe42..34ff11566 100644 --- a/database/gdb/gdb_model_time.go +++ b/database/gdb/gdb_model_time.go @@ -94,6 +94,7 @@ func (m *Model) getSoftFieldNameDeleted(table ...string) (field string) { // getSoftFieldName retrieves and returns the field name of the table for possible key. func (m *Model) getSoftFieldName(table string, keys []string) (field string) { + // Ignore the error from TableFields. fieldsMap, _ := m.TableFields(table) if len(fieldsMap) > 0 { for _, key := range keys { diff --git a/database/gdb/gdb_model_utility.go b/database/gdb/gdb_model_utility.go index 02e307396..b04b7e7a5 100644 --- a/database/gdb/gdb_model_utility.go +++ b/database/gdb/gdb_model_utility.go @@ -26,7 +26,7 @@ func (m *Model) QuoteWord(s string) string { return m.db.GetCore().QuoteWord(s) } -// TableFields retrieves and returns the fields information of specified table of current +// TableFields retrieves and returns the fields' information of specified table of current // schema. // // Also see DriverMysql.TableFields. @@ -57,8 +57,8 @@ func (m *Model) getModel() *Model { // ID -> id // NICK_Name -> nickname. func (m *Model) mappingAndFilterToTableFields(fields []string, filter bool) []string { - fieldsMap, err := m.TableFields(m.tablesInit) - if err != nil || len(fieldsMap) == 0 { + fieldsMap, _ := m.TableFields(m.tablesInit) + if len(fieldsMap) == 0 { return fields } var ( diff --git a/database/gdb/gdb_z_mysql_model_test.go b/database/gdb/gdb_z_mysql_model_test.go index 1adae244b..dc2cf59c2 100644 --- a/database/gdb/gdb_z_mysql_model_test.go +++ b/database/gdb/gdb_z_mysql_model_test.go @@ -955,6 +955,7 @@ func Test_Model_StructsWithOrmTag(t *testing.T) { dbInvalid.GetLogger().SetWriter(buffer) defer dbInvalid.GetLogger().SetWriter(os.Stdout) dbInvalid.Model(table).Order("id asc").Scan(&users) + //fmt.Println(buffer.String()) t.Assert( gstr.Contains(buffer.String(), "SELECT `id`,`Passport`,`password`,`nick_name`,`create_time` FROM `user"), true,