From f1a60b295c572f27843559b71a1921abcec59e64 Mon Sep 17 00:00:00 2001 From: Enwei Jiao Date: Tue, 7 Mar 2023 18:23:51 +0800 Subject: [PATCH] Support multi yaml files (#22583) Signed-off-by: Enwei Jiao --- internal/config/config.go | 4 +- internal/config/config_test.go | 2 +- internal/config/file_source.go | 78 ++++++++++++------------- internal/config/source.go | 6 +- internal/config/source_test.go | 24 ++++++-- internal/util/paramtable/base_table.go | 23 ++++---- internal/util/paramtable/hook_config.go | 2 +- 7 files changed, 78 insertions(+), 61 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 51a9142c17..de93941e6e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -34,8 +34,8 @@ func Init(opts ...Option) (*Manager, error) { opt(o) } sourceManager := NewManager() - if o.File != nil { - s := NewFileSource(o.File) + if o.FileInfo != nil { + s := NewFileSource(o.FileInfo) sourceManager.AddSource(s) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f483e63623..94acb44b62 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -57,7 +57,7 @@ func TestConfigFromRemote(t *testing.T) { t.Setenv("TMP_KEY", "1") t.Setenv("log.level", "info") mgr, _ := Init(WithEnvSource(formatKey), - WithFilesSource(&FileInfo{"../../configs/milvus.yaml", -1}), + WithFilesSource(&FileInfo{[]string{"../../configs/milvus.yaml"}, -1}), WithEtcdSource(&EtcdInfo{ Endpoints: []string{cfg.ACUrls[0].Host}, KeyPrefix: "test", diff --git a/internal/config/file_source.go b/internal/config/file_source.go index b6e6f5ab3b..63764502a8 100644 --- a/internal/config/file_source.go +++ b/internal/config/file_source.go @@ -31,7 +31,7 @@ import ( type FileSource struct { sync.RWMutex - file string + files []string configs map[string]string configRefresher refresher @@ -39,7 +39,7 @@ type FileSource struct { func NewFileSource(fileInfo *FileInfo) *FileSource { fs := &FileSource{ - file: fileInfo.Filepath, + files: fileInfo.Files, configs: make(map[string]string), } fs.configRefresher = newRefresher(fileInfo.RefreshInterval, fs.loadFromFile) @@ -95,46 +95,46 @@ func (fs *FileSource) SetEventHandler(eh EventHandler) { } func (fs *FileSource) loadFromFile() error { yamlReader := viper.New() - configFile := fs.file - if _, err := os.Stat(configFile); err != nil { - return errors.New("cannot access config file: " + configFile) - } - - yamlReader.SetConfigFile(configFile) - if err := yamlReader.ReadInConfig(); err != nil { - log.Warn("Read config failed", zap.Error(err)) - return err - } - newConfig := make(map[string]string) - for _, key := range yamlReader.AllKeys() { - val := yamlReader.Get(key) - str, err := cast.ToStringE(val) - if err != nil { - switch val := val.(type) { - case []any: - str = str[:0] - for _, v := range val { - ss, err := cast.ToStringE(v) - if err != nil { - log.Warn("cast to string failed", zap.Any("value", v)) - } - if str == "" { - str = ss - } else { - str = str + "," + ss - } - } - - default: - log.Warn("val is not a slice", zap.Any("value", val)) - continue - } + for _, configFile := range fs.files { + if _, err := os.Stat(configFile); err != nil { + log.Info("cannot access config file", zap.String("configFile", configFile)) + continue } - newConfig[key] = str - newConfig[formatKey(key)] = str - } + yamlReader.SetConfigFile(configFile) + if err := yamlReader.ReadInConfig(); err != nil { + return errors.Wrap(err, "Read config failed: "+configFile) + } + + for _, key := range yamlReader.AllKeys() { + val := yamlReader.Get(key) + str, err := cast.ToStringE(val) + if err != nil { + switch val := val.(type) { + case []any: + str = str[:0] + for _, v := range val { + ss, err := cast.ToStringE(v) + if err != nil { + log.Warn("cast to string failed", zap.Any("value", v)) + } + if str == "" { + str = ss + } else { + str = str + "," + ss + } + } + + default: + log.Warn("val is not a slice", zap.Any("value", val)) + continue + } + } + newConfig[key] = str + newConfig[formatKey(key)] = str + } + } fs.Lock() defer fs.Unlock() err := fs.configRefresher.fireEvents(fs.GetSourceName(), fs.configs, newConfig) diff --git a/internal/config/source.go b/internal/config/source.go index f4e9f05c93..1395684951 100644 --- a/internal/config/source.go +++ b/internal/config/source.go @@ -49,13 +49,13 @@ type EtcdInfo struct { // FileInfo has attribute for file source type FileInfo struct { - Filepath string + Files []string RefreshInterval time.Duration } // Options hold options type Options struct { - File *FileInfo + FileInfo *FileInfo EtcdInfo *EtcdInfo EnvKeyFormatter func(string) string } @@ -66,7 +66,7 @@ type Option func(options *Options) // WithRequiredFiles tell archaius to manage files, if not exist will return error func WithFilesSource(fi *FileInfo) Option { return func(options *Options) { - options.File = fi + options.FileInfo = fi } } diff --git a/internal/config/source_test.go b/internal/config/source_test.go index cbe85d8123..0836bc9be8 100644 --- a/internal/config/source_test.go +++ b/internal/config/source_test.go @@ -17,6 +17,8 @@ package config import ( + "os" + "path" "testing" "github.com/stretchr/testify/assert" @@ -24,13 +26,27 @@ import ( func TestLoadFromFileSource(t *testing.T) { t.Run("file not exist", func(t *testing.T) { - fs := NewFileSource(&FileInfo{"file_not_exist.yaml", -1}) - err := fs.loadFromFile() - assert.Error(t, err, "cannot access config file: file_not_exist.yaml") + fs := NewFileSource(&FileInfo{[]string{"file_not_exist.yaml"}, -1}) + _ = fs.loadFromFile() + assert.Zero(t, len(fs.configs)) }) + t.Run("file type not support", func(t *testing.T) { - fs := NewFileSource(&FileInfo{"../../go.mod", -1}) + fs := NewFileSource(&FileInfo{[]string{"../../go.mod"}, -1}) err := fs.loadFromFile() assert.Error(t, err) }) + + t.Run("multiple files", func(t *testing.T) { + dir, _ := os.MkdirTemp("", "milvus") + os.WriteFile(path.Join(dir, "milvus.yaml"), []byte("a.b: 1\nc.d: 2"), 0600) + os.WriteFile(path.Join(dir, "user.yaml"), []byte("a.b: 3"), 0600) + + fs := NewFileSource(&FileInfo{[]string{path.Join(dir, "milvus.yaml"), path.Join(dir, "user.yaml")}, -1}) + fs.loadFromFile() + v1, _ := fs.GetConfigurationByKey("a.b") + assert.Equal(t, "3", v1) + v2, _ := fs.GetConfigurationByKey("c.d") + assert.Equal(t, "2", v2) + }) } diff --git a/internal/util/paramtable/base_table.go b/internal/util/paramtable/base_table.go index 4204b04c95..82054d128a 100644 --- a/internal/util/paramtable/base_table.go +++ b/internal/util/paramtable/base_table.go @@ -23,6 +23,7 @@ import ( "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/util/etcd" "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/samber/lo" "go.uber.org/zap" ) @@ -30,7 +31,6 @@ import ( type UniqueID = typeutil.UniqueID const ( - DefaultMilvusYaml = "milvus.yaml" DefaultEasyloggingYaml = "easylogging.yaml" DefaultMinioHost = "localhost" DefaultMinioPort = "9000" @@ -53,7 +53,7 @@ func globalConfigPrefixs() []string { return []string{"metastore.", "localStorage.", "etcd.", "mysql.", "minio.", "pulsar.", "kafka.", "rocksmq.", "log.", "grpc.", "common.", "quotaAndLimits."} } -var defaultYaml = DefaultMilvusYaml +var defaultYaml = []string{"milvus.yaml", "default.yaml", "user.yaml"} // BaseTable the basics of paramtable type BaseTable struct { @@ -61,17 +61,17 @@ type BaseTable struct { mgr *config.Manager configDir string - YamlFile string + YamlFiles []string } // NewBaseTableFromYamlOnly only used in migration tool. // Maybe we shouldn't limit the configDir internally. func NewBaseTableFromYamlOnly(yaml string) *BaseTable { mgr, _ := config.Init(config.WithFilesSource(&config.FileInfo{ - Filepath: yaml, + Files: []string{yaml}, RefreshInterval: 10 * time.Second, })) - gp := &BaseTable{mgr: mgr, YamlFile: yaml} + gp := &BaseTable{mgr: mgr, YamlFiles: []string{yaml}} return gp } @@ -81,7 +81,7 @@ func NewBaseTableFromYamlOnly(yaml string) *BaseTable { // GlobalInitWithYaml should be called only in standalone and embedded Milvus. func (gp *BaseTable) GlobalInitWithYaml(yaml string) { gp.once.Do(func() { - defaultYaml = yaml + defaultYaml = []string{yaml} }) } @@ -96,8 +96,8 @@ func (gp *BaseTable) init(refreshInterval int) { ret = strings.ReplaceAll(ret, ".", "") return ret } - if gp.YamlFile == "" { - gp.YamlFile = DefaultMilvusYaml + if len(gp.YamlFiles) == 0 { + gp.YamlFiles = defaultYaml } var err error gp.mgr, err = config.Init(config.WithEnvSource(formatter)) @@ -110,13 +110,14 @@ func (gp *BaseTable) init(refreshInterval int) { func (gp *BaseTable) initConfigsFromLocal(refreshInterval int) { gp.configDir = gp.initConfPath() - configFilePath := gp.configDir + "/" + gp.YamlFile err := gp.mgr.AddSource(config.NewFileSource(&config.FileInfo{ - Filepath: configFilePath, + Files: lo.Map(gp.YamlFiles, func(file string, _ int) string { + return path.Join(gp.configDir, file) + }), RefreshInterval: time.Duration(refreshInterval) * time.Second, })) if err != nil { - log.Warn("init baseTable with file failed", zap.String("configFile", configFilePath), zap.Error(err)) + log.Warn("init baseTable with file failed", zap.Strings("configFile", gp.YamlFiles), zap.Error(err)) return } } diff --git a/internal/util/paramtable/hook_config.go b/internal/util/paramtable/hook_config.go index bea4ad5e26..1002a0b87e 100644 --- a/internal/util/paramtable/hook_config.go +++ b/internal/util/paramtable/hook_config.go @@ -13,7 +13,7 @@ type hookConfig struct { } func (h *hookConfig) init() { - base := &BaseTable{YamlFile: hookYamlFile} + base := &BaseTable{YamlFiles: []string{hookYamlFile}} base.init(0) log.Info("hook config", zap.Any("hook", base.FileConfigs()))