test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 03:48:37 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Fix config

Browse Source 
Signed-off-by: zhenshan.cao <zhenshan.cao@zilliz.com>
This commit is contained in:
zhenshan.cao 2020-09-09 18:59:56 +08:00 committed by yefu.chen
parent ef25004205
commit 2863029665
20 changed files with 275 additions and 800 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
conf/conf.go Normal file
View File

@ -0,0 +1,74 @@
package conf
import (
"io/ioutil"
"github.com/czs007/suvlim/storage/pkg/types"
yaml "gopkg.in/yaml.v2"
)
// yaml.MapSlice
type MasterConfig struct {
Address string
Port int32
}
type EtcdConfig struct {
Address string
Port int32
Rootpath string
Segthreshold int64
}
type TimeSyncConfig struct {
Interval int32
}
type StorageConfig struct {
Driver types.DriverType
Address string
Port int32
Accesskey string
Secretkey string
}
type PulsarConfig struct {
Address string
Port int32
}
//type ProxyConfig struct {
// Timezone string
// Address string
// Port int32
//}
type ServerConfig struct {
Master MasterConfig
Etcd EtcdConfig
Timesync TimeSyncConfig
Storage StorageConfig
Pulsar PulsarConfig
//Proxy ProxyConfig
}
var Config ServerConfig
func init() {
load_config()
}
func load_config() {
//var config ServerConfig
filename := "github.com/czs007/suvlim/conf/config.yaml"
source, err := ioutil.ReadFile(filename)
if err != nil {
panic(err)
}
err = yaml.Unmarshal(source, &Config)
if err != nil {
panic(err)
}
//fmt.Printf("Result: %v\n", Config)
}

0
conf/conf.toml
View File

11
conf/conf_test.go Normal file
View File

@ -0,0 +1,11 @@
package conf
import (
"fmt"
"testing"
)
func TestMain(m *testing.M) {
fmt.Printf("Result: %v\n", Config)
}

52
conf/config.yaml Normal file
View File

@ -0,0 +1,52 @@
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
master:
address: localhost
port: 6000
etcd:
address: localhost
port: 0
rootpath: a
segthreshold: 10000
timesync:
interval: 10
storage:
driver: MinIO
address: localhost
port: 0
accesskey: ab
secretkey: dd
pulsar:
address: localhost
port: 6650
proxy:
timezone: UTC+8
network:
address: 0.0.0.0
port: 19530
logs:
level: debug
trace.enable: true
path: /tmp/logs
max_log_file_size: 1024MB
log_rotate_num: 0
storage:
path: /var/lib/milvus
auto_flush_interval: 1

188
conf/server_config.template
View File

@ -1,188 +0,0 @@
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
version: 0.5
#----------------------+------------------------------------------------------------+------------+-----------------+
# Cluster Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | If running with Mishards, set true, otherwise false. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# role | Milvus deployment role: rw / ro | Role | rw |
#----------------------+------------------------------------------------------------+------------+-----------------+
cluster:
enable: false
role: rw
#----------------------+------------------------------------------------------------+------------+-----------------+
# General Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# timezone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# meta_uri | URI for metadata storage, using SQLite (for single server | URI | sqlite://:@:/ |
# | Milvus) or MySQL (for distributed cluster Milvus). | | |
# | Format: dialect://username:password@host:port/database | | |
# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | |
# | 'mysql', replace other texts with real values. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
general:
timezone: UTC+8
meta_uri: sqlite://:@:/
#----------------------+------------------------------------------------------------+------------+-----------------+
# Network Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.enable | Enable HTTP server or not. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.port | Port that Milvus HTTP server monitors. | Integer | 19121 |
# | Port range (1024, 65535) | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
network:
bind.address: 0.0.0.0
bind.port: 19530
http.enable: true
http.port: 19121
#----------------------+------------------------------------------------------------+------------+-----------------+
# Storage Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus |
#----------------------+------------------------------------------------------------+------------+-----------------+
# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) |
# | flushes data to disk. | | |
# | 0 means disable the regular flush. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage:
path: @MILVUS_DB_PATH@
auto_flush_interval: 1
#----------------------+------------------------------------------------------------+------------+-----------------+
# WAL Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true |
# | If WAL is enabled, Milvus writes all data changes to log | | |
# | files in advance before implementing data changes. WAL | | |
# | ensures the atomicity and durability for Milvus operations.| | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | false |
# | recovery. If true, when Milvus restarts for recovery and | | |
# | there are errors in WAL log files, log files with errors | | |
# | are ignored. If false, Milvus does not restart when there | | |
# | are errors in WAL log files. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# buffer_size | Sum total of the read buffer and the write buffer in Bytes.| String | 256MB |
# | buffer_size must be in range [64MB, 4096MB]. | | |
# | If the value you specified is out of range, Milvus | | |
# | automatically uses the boundary value closest to the | | |
# | specified value. It is recommended you set buffer_size to | | |
# | a value greater than the inserted data size of a single | | |
# | insert operation for better performance. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Location of WAL log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
wal:
enable: true
recovery_error_ignore: false
buffer_size: 256MB
path: @MILVUS_DB_PATH@/wal
#----------------------+------------------------------------------------------------+------------+-----------------+
# Cache Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of CPU memory used for caching data for faster | String | 4GB |
# | query. The sum of 'cache_size' and 'insert_buffer_size' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# insert_buffer_size | Buffer size used for data insertion. | String | 1GB |
# | The sum of 'insert_buffer_size' and 'cache_size' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# preload_collection | A comma-separated list of collection names that need to | StringList | |
# | be pre-loaded when Milvus server starts up. | | |
# | '*' means preload all existing tables (single-quote or | | |
# | double-quote required). | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
cache:
cache_size: 4GB
insert_buffer_size: 1GB
preload_collection:
#----------------------+------------------------------------------------------------+------------+-----------------+
# GPU Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Use GPU devices or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of GPU memory per card used for cache. | String | 1GB |
#----------------------+------------------------------------------------------------+------------+-----------------+
# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 |
# | compared with 'nq' to decide if the search computation will| | |
# | be executed on GPUs only. | | |
# | If nq >= gpu_search_threshold, the search computation will | | |
# | be executed on GPUs only; | | |
# | if nq < gpu_search_threshold, the search computation will | | |
# | be executed on both CPUs and GPUs. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# search_devices | The list of GPU devices used for search computation. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# build_index_devices | The list of GPU devices used for index building. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
gpu:
enable: @GPU_ENABLE@
cache_size: 1GB
gpu_search_threshold: 1000
search_devices:
- gpu0
build_index_devices:
- gpu0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Logs Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug |
# | error, fatal | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Absolute path to the folder holding the log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# max_log_file_size | The maximum size of each log file, size range | String | 1024MB |
# | [512MB, 4096MB]. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 |
# | logging level, num range [0, 1024], 0 means unlimited. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
logs:
level: debug
trace.enable: true
path: @MILVUS_DB_PATH@/logs
max_log_file_size: 1024MB
log_rotate_num: 0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Enable monitoring function or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# address | Pushgateway address | IP | 127.0.0.1 +
#----------------------+------------------------------------------------------------+------------+-----------------+
# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 |
#----------------------+------------------------------------------------------------+------------+-----------------+
metric:
enable: false
address: 127.0.0.1
port: 9091

1
conf/storage.toml
View File

@ -1 +0,0 @@
driver="S3"

1
go.mod
View File

@ -43,6 +43,7 @@ require (
google.golang.org/grpc/examples v0.0.0-20200828165940-d8ef479ab79a // indirect
google.golang.org/protobuf v1.25.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/yaml.v2 v2.3.0
sigs.k8s.io/yaml v1.2.0 // indirect
)

187
proxy/conf/demo/server_config.yaml
View File

@ -1,187 +0,0 @@
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
version: 0.5
#----------------------+------------------------------------------------------------+------------+-----------------+
# Cluster Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | If runinng with Mishards, set true, otherwise false. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# role | Milvus deployment role: rw / ro | role | rw |
#----------------------+------------------------------------------------------------+------------+-----------------+
cluster:
enable: false
role: rw
#----------------------+------------------------------------------------------------+------------+-----------------+
# General Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# time_zone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# meta_uri | URI for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ |
# | Milvus) or MySQL (for distributed cluster Milvus). | | |
# | Format: dialect://username:password@host:port/database | | |
# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | |
# | 'mysql', replace other texts with real values. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
general:
timezone: UTC+8
meta_uri: sqlite://:@:/
#----------------------+------------------------------------------------------------+------------+-----------------+
# Network Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.enable | Enable web server or not. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.port | Port that Milvus web server monitors. | Integer | 19121 |
# | Port range (1024, 65535) | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
network:
bind.address: 0.0.0.0
bind.port: 19530
http.enable: true
http.port: 19121
#----------------------+------------------------------------------------------------+------------+-----------------+
# Storage Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus |
#----------------------+------------------------------------------------------------+------------+-----------------+
# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) |
# | flushes data to disk. | | |
# | 0 means disable the regular flush. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage:
path: /var/lib/milvus
auto_flush_interval: 1
#----------------------+------------------------------------------------------------+------------+-----------------+
# WAL Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true |
# | If WAL is enabled, Milvus writes all data changes to log | | |
# | files in advance before implementing data changes. WAL | | |
# | ensures the atomicity and durability for Milvus operations.| | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | false |
# | recovery. If true, when Milvus restarts for recovery and | | |
# | there are errors in WAL log files, log files with errors | | |
# | are ignored. If false, Milvus does not restart when there | | |
# | are errors in WAL log files. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# buffer_size | Sum total of the read buffer and the write buffer in MBs. | Integer | 256 (MB) |
# | buffer_size must be in range [64, 4096] (MB). | | |
# | If the value you specified is out of range, Milvus | | |
# | automatically uses the boundary value closest to the | | |
# | specified value. It is recommended you set buffer_size to | | |
# | a value greater than the inserted data size of a single | | |
# | insert operation for better performance. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Location of WAL log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
wal:
enable: true
recovery_error_ignore: false
buffer_size: 256MB
path: /var/lib/milvus/wal
#----------------------+------------------------------------------------------------+------------+-----------------+
# Cache Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of CPU memory used for caching data for faster | Integer | 4 (GB) |
# | query. The sum of 'cpu_cache_capacity' and | | |
# | 'insert_buffer_size' must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) |
# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# preload_collection | A comma-separated list of collection names that need to | StringList | |
# | be pre-loaded when Milvus server starts up. | | |
# | '*' means preload all existing tables (single-quote or | | |
# | double-quote required). | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
cache:
cache_size: 4GB
insert_buffer_size: 1GB
preload_collection:
#----------------------+------------------------------------------------------------+------------+-----------------+
# GPU Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Enable GPU resources or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of GPU memory per card used for cache. | Integer | 1 (GB) |
#----------------------+------------------------------------------------------------+------------+-----------------+
# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 |
# | compared with 'nq' to decide if the search computation will| | |
# | be executed on GPUs only. | | |
# | If nq >= gpu_search_threshold, the search computation will | | |
# | be executed on GPUs only; | | |
# | if nq < gpu_search_threshold, the search computation will | | |
# | be executed on both CPUs and GPUs. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# search_resources | The list of GPU devices used for search computation. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# build_index_resources| The list of GPU devices used for index building. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
gpu:
enable: false
cache_size: 1GB
gpu_search_threshold: 1000
search_devices:
- gpu0
build_index_devices:
- gpu0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Logs Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug |
# | error, fatal | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Absolute path to the folder holding the log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# max_log_file_size | The maximum size of each log file, size range [512, 4096] | Integer | 1024 (MB) |
#----------------------+------------------------------------------------------------+------------+-----------------+
# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 |
# | logging level, num range [0, 1024], 0 means unlimited. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
logs:
level: debug
trace.enable: true
path: /var/lib/milvus/logs
max_log_file_size: 1024MB
log_rotate_num: 0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Enable monitoring function or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# address | Pushgateway address | IP | 127.0.0.1 +
#----------------------+------------------------------------------------------------+------------+-----------------+
# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 |
#----------------------+------------------------------------------------------------+------------+-----------------+
metric:
enable: false
address: 127.0.0.1
port: 9091

27
proxy/conf/log_config.conf
View File

@ -1,27 +0,0 @@
* GLOBAL:
FORMAT = "%datetime | %level | %logger | %msg"
FILENAME = "/usr/local/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log"
ENABLED = true
TO_FILE = true
TO_STANDARD_OUTPUT = false
SUBSECOND_PRECISION = 3
PERFORMANCE_TRACKING = false
MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB
* DEBUG:
FILENAME = "/usr/local/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log"
ENABLED = true
* WARNING:
FILENAME = "/usr/local/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log"
* TRACE:
FILENAME = "/usr/local/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log"
* VERBOSE:
FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
TO_FILE = false
TO_STANDARD_OUTPUT = false
## Error logs
* ERROR:
ENABLED = true
FILENAME = "/usr/local/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log"
* FATAL:
ENABLED = true
FILENAME = "/usr/local/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log"

188
proxy/conf/server_config.template
View File

@ -1,188 +0,0 @@
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
version: 0.5
#----------------------+------------------------------------------------------------+------------+-----------------+
# Cluster Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | If running with Mishards, set true, otherwise false. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# role | Milvus deployment role: rw / ro | Role | rw |
#----------------------+------------------------------------------------------------+------------+-----------------+
cluster:
enable: false
role: rw
#----------------------+------------------------------------------------------------+------------+-----------------+
# General Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# timezone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# meta_uri | URI for metadata storage, using SQLite (for single server | URI | sqlite://:@:/ |
# | Milvus) or MySQL (for distributed cluster Milvus). | | |
# | Format: dialect://username:password@host:port/database | | |
# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | |
# | 'mysql', replace other texts with real values. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
general:
timezone: UTC+8
meta_uri: sqlite://:@:/
#----------------------+------------------------------------------------------------+------------+-----------------+
# Network Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.enable | Enable HTTP server or not. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.port | Port that Milvus HTTP server monitors. | Integer | 19121 |
# | Port range (1024, 65535) | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
network:
bind.address: 0.0.0.0
bind.port: 19530
http.enable: true
http.port: 19121
#----------------------+------------------------------------------------------------+------------+-----------------+
# Storage Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus |
#----------------------+------------------------------------------------------------+------------+-----------------+
# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) |
# | flushes data to disk. | | |
# | 0 means disable the regular flush. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage:
path: @MILVUS_DB_PATH@
auto_flush_interval: 1
#----------------------+------------------------------------------------------------+------------+-----------------+
# WAL Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true |
# | If WAL is enabled, Milvus writes all data changes to log | | |
# | files in advance before implementing data changes. WAL | | |
# | ensures the atomicity and durability for Milvus operations.| | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | false |
# | recovery. If true, when Milvus restarts for recovery and | | |
# | there are errors in WAL log files, log files with errors | | |
# | are ignored. If false, Milvus does not restart when there | | |
# | are errors in WAL log files. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# buffer_size | Sum total of the read buffer and the write buffer in Bytes.| String | 256MB |
# | buffer_size must be in range [64MB, 4096MB]. | | |
# | If the value you specified is out of range, Milvus | | |
# | automatically uses the boundary value closest to the | | |
# | specified value. It is recommended you set buffer_size to | | |
# | a value greater than the inserted data size of a single | | |
# | insert operation for better performance. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Location of WAL log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
wal:
enable: true
recovery_error_ignore: false
buffer_size: 256MB
path: @MILVUS_DB_PATH@/wal
#----------------------+------------------------------------------------------------+------------+-----------------+
# Cache Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of CPU memory used for caching data for faster | String | 4GB |
# | query. The sum of 'cache_size' and 'insert_buffer_size' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# insert_buffer_size | Buffer size used for data insertion. | String | 1GB |
# | The sum of 'insert_buffer_size' and 'cache_size' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# preload_collection | A comma-separated list of collection names that need to | StringList | |
# | be pre-loaded when Milvus server starts up. | | |
# | '*' means preload all existing tables (single-quote or | | |
# | double-quote required). | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
cache:
cache_size: 4GB
insert_buffer_size: 1GB
preload_collection:
#----------------------+------------------------------------------------------------+------------+-----------------+
# GPU Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Use GPU devices or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of GPU memory per card used for cache. | String | 1GB |
#----------------------+------------------------------------------------------------+------------+-----------------+
# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 |
# | compared with 'nq' to decide if the search computation will| | |
# | be executed on GPUs only. | | |
# | If nq >= gpu_search_threshold, the search computation will | | |
# | be executed on GPUs only; | | |
# | if nq < gpu_search_threshold, the search computation will | | |
# | be executed on both CPUs and GPUs. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# search_devices | The list of GPU devices used for search computation. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# build_index_devices | The list of GPU devices used for index building. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
gpu:
enable: @GPU_ENABLE@
cache_size: 1GB
gpu_search_threshold: 1000
search_devices:
- gpu0
build_index_devices:
- gpu0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Logs Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug |
# | error, fatal | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Absolute path to the folder holding the log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# max_log_file_size | The maximum size of each log file, size range | String | 1024MB |
# | [512MB, 4096MB]. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 |
# | logging level, num range [0, 1024], 0 means unlimited. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
logs:
level: debug
trace.enable: true
path: @MILVUS_DB_PATH@/logs
max_log_file_size: 1024MB
log_rotate_num: 0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Enable monitoring function or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# address | Pushgateway address | IP | 127.0.0.1 +
#----------------------+------------------------------------------------------------+------------+-----------------+
# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 |
#----------------------+------------------------------------------------------------+------------+-----------------+
metric:
enable: false
address: 127.0.0.1
port: 9091

109
proxy/conf/server_config.yaml
View File

@ -1,109 +0,0 @@
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
version: 0.5
#----------------------+------------------------------------------------------------+------------+-----------------+
# General Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# timezone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# meta_uri | URI for metadata storage, using SQLite (for single server | URI | sqlite://:@:/ |
# | Milvus) or MySQL (for distributed cluster Milvus). | | |
# | Format: dialect://username:password@host:port/database | | |
# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | |
# | 'mysql', replace other texts with real values. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
general:
timezone: UTC+8
meta_uri: sqlite://:@:/
#----------------------+------------------------------------------------------------+------------+-----------------+
# Network Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.enable | Enable HTTP server or not. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.port | Port that Milvus HTTP server monitors. | Integer | 19121 |
# | Port range (1024, 65535) | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
network:
bind.address: 0.0.0.0
bind.port: 19530
http.enable: true
http.port: 19121
#----------------------+------------------------------------------------------------+------------+-----------------+
# Storage Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus |
#----------------------+------------------------------------------------------------+------------+-----------------+
# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) |
# | flushes data to disk. | | |
# | 0 means disable the regular flush. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage:
path: /tmp
auto_flush_interval: 1
#----------------------+------------------------------------------------------------+------------+-----------------+
# Cache Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of CPU memory used for caching data for faster | String | 4GB |
# | query. The sum of 'cache_size' and 'insert_buffer_size' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# insert_buffer_size | Buffer size used for data insertion. | String | 1GB |
# | The sum of 'insert_buffer_size' and 'cache_size' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# preload_collection | A comma-separated list of collection names that need to | StringList | |
# | be pre-loaded when Milvus server starts up. | | |
# | '*' means preload all existing tables (single-quote or | | |
# | double-quote required). | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
cache:
cache_size: 4GB
insert_buffer_size: 1GB
preload_collection:
#----------------------+------------------------------------------------------------+------------+-----------------+
# Logs Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug |
# | error, fatal | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Absolute path to the folder holding the log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# max_log_file_size | The maximum size of each log file, size range | String | 1024MB |
# | [512MB, 4096MB]. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 |
# | logging level, num range [0, 1024], 0 means unlimited. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
logs:
level: debug
trace.enable: true
path: /tmp/logs
max_log_file_size: 1024MB
log_rotate_num: 0
metric:
level: debug
pulsar:
address: localhost
port: 6650

26
proxy/conf/tracing_config.json
View File

@ -1,26 +0,0 @@
{
"host": "127.0.0.1",
"port": "5666",
"tracer_library": "/path/to/shared_tracing_library",
"tracer_configuration": {
"service_name": "milvus_server",
"sampler": {
"type": "const",
"param": "1"
},
"disabled": false,
"reporter": {
"localAgentHostPort": "127.0.0.1:6831"
},
"headers": {
"jaegerDebugHeader": "jaeger_debug_header",
"jaegerBaggageHeader": "jarger_baggage_header",
"TraceContextHeaderName": "trace_context_header_name",
"traceBaggageHeaderPrefix": "trace_baggage_header_prefix"
},
"baggage_restrictions": {
"denyBaggageOnInitializationFailure": false,
"hostPort": ""
}
}
}

43
proxy/src/config/ConfigMgr.cpp
View File

@ -68,22 +68,16 @@ ConfigMgr ConfigMgr::instance;
ConfigMgr::ConfigMgr() {
config_list_ = {
/* version */
{"version", CreateStringConfig("version", false, &config.version.value, "unknown", nullptr, nullptr)},
/* general */
{"general.timezone",
CreateStringConfig("general.timezone", false, &config.general.timezone.value, "UTC+8", nullptr, nullptr)},
{"timezone",
CreateStringConfig("timezone", false, &config.timezone.value, "UTC+8", nullptr, nullptr)},
/* network */
{"network.bind.address", CreateStringConfig("network.bind.address", false, &config.network.bind.address.value,
{"network.address", CreateStringConfig("network.address", false, &config.network.address.value,
"0.0.0.0", nullptr, nullptr)},
{"network.bind.port", CreateIntegerConfig("network.bind.port", false, 0, 65535, &config.network.bind.port.value,
{"network.port", CreateIntegerConfig("network.port", false, 0, 65535, &config.network.port.value,
19530, nullptr, nullptr)},
{"network.http.enable",
CreateBoolConfig("network.http.enable", false, &config.network.http.enable.value, true, nullptr, nullptr)},
{"network.http.port", CreateIntegerConfig("network.http.port", false, 0, 65535, &config.network.http.port.value,
19121, nullptr, nullptr)},
/* pulsar */
@ -92,27 +86,6 @@ ConfigMgr::ConfigMgr() {
{"pulsar.port", CreateIntegerConfig("pulsar.port", false, 0, 65535, &config.pulsar.port.value,
6650, nullptr, nullptr)},
/* storage */
{"storage.path",
CreateStringConfig("storage.path", false, &config.storage.path.value, "/var/lib/milvus", nullptr, nullptr)},
{"storage.auto_flush_interval",
CreateIntegerConfig("storage.auto_flush_interval", true, 0, std::numeric_limits<int64_t>::max(),
&config.storage.auto_flush_interval.value, 1, nullptr, nullptr)},
/* cache */
{"cache.cache_size", CreateSizeConfig("cache.cache_size", true, 0, std::numeric_limits<int64_t>::max(),
&config.cache.cache_size.value, 4 * GB, nullptr, nullptr)},
{"cache.cpu_cache_threshold",
CreateFloatingConfig("cache.cpu_cache_threshold", false, 0.0, 1.0, &config.cache.cpu_cache_threshold.value,
0.7, nullptr, nullptr)},
{"cache.insert_buffer_size",
CreateSizeConfig("cache.insert_buffer_size", false, 0, std::numeric_limits<int64_t>::max(),
&config.cache.insert_buffer_size.value, 1 * GB, nullptr, nullptr)},
{"cache.cache_insert_data", CreateBoolConfig("cache.cache_insert_data", false,
&config.cache.cache_insert_data.value, false, nullptr, nullptr)},
{"cache.preload_collection", CreateStringConfig("cache.preload_collection", false,
&config.cache.preload_collection.value, "", nullptr, nullptr)},
/* log */
{"logs.level", CreateStringConfig("logs.level", false, &config.logs.level.value, "debug", nullptr, nullptr)},
@ -162,13 +135,19 @@ ConfigMgr::Load(const std::string& path) {
auto yaml = YAML::LoadFile(path);
/* make it flattened */
std::unordered_map<std::string, std::string> flattened;
Flatten(yaml, flattened, "");
// auto proxy_yaml = yaml["porxy"];
auto other_yaml = YAML::Node{};
other_yaml["pulsar"] = yaml["pulsar"];
Flatten(yaml["proxy"], flattened, "");
Flatten(other_yaml, flattened, "");
// Flatten(yaml["proxy"], flattened, "");
/* update config */
for (auto& it : flattened) Set(it.first, it.second, false);
}
void
ConfigMgr::Set(const std::string& name, const std::string& value, bool update) {
std::cout<<"InSet Config "<< name <<std::endl;
if (config_list_.find(name) == config_list_.end()){
std::cout<<"Config "<< name << " not found!"<<std::endl;
return;

28
proxy/src/config/ServerConfig.h
View File

@ -64,21 +64,11 @@ struct ServerConfig {
using Integer = ConfigValue<int64_t>;
using Floating = ConfigValue<double>;
String version{"unknown"};
struct General {
String timezone{"unknown"};
} general;
String timezone{"unknown"};
struct Network {
struct Bind {
String address{"unknown"};
Integer port{0};
} bind;
struct Http {
Bool enable{false};
Integer port{0};
} http;
String address{"unknown"};
Integer port{0};
} network;
struct Pulsar{
@ -86,18 +76,6 @@ struct ServerConfig {
Integer port{6650};
}pulsar;
struct Storage {
String path{"unknown"};
Integer auto_flush_interval{0};
} storage;
struct Cache {
Integer cache_size{0};
Floating cpu_cache_threshold{0.0};
Integer insert_buffer_size{0};
Bool cache_insert_data{false};
String preload_collection{"unknown"};
} cache;
struct Engine {
Integer build_index_threshold{4096};

5
proxy/src/server/Server.cpp
View File

@ -24,6 +24,7 @@
// #include "scheduler/SchedInst.h"
#include "server/grpc_impl/GrpcServer.h"
#include "server/init/CpuChecker.h"
#include "server/init/Directory.h"
// #include "server/init/GpuChecker.h"
#include "server/init/StorageChecker.h"
#include "src/version.h"
@ -146,7 +147,7 @@ Server::Start() {
tracing_config_path.empty() ? tracing::TracerUtil::InitGlobal()
: tracing::TracerUtil::InitGlobal(tracing_config_path);
auto time_zone = config.general.timezone();
auto time_zone = config.timezone();
if (time_zone.length() == 3) {
time_zone = "CUT";
@ -170,6 +171,8 @@ Server::Start() {
STATUS_CHECK(LogMgr::InitLog(config.logs.trace.enable(), config.logs.level(), config.logs.path(),
config.logs.max_log_file_size(), config.logs.log_rotate_num()));
STATUS_CHECK(Directory::Initialize(config.logs.path()));
// print version information
LOG_SERVER_INFO_ << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME;
#ifdef MILVUS_GPU_VERSION

2
proxy/src/server/grpc_impl/GrpcServer.cpp
View File

@ -84,7 +84,7 @@ Status
GrpcServer::StartService() {
SetThreadName("grpcserv_thread");
std::string server_address(config.network.bind.address() + ":" + std::to_string(config.network.bind.port()));
std::string server_address(config.network.address() + ":" + std::to_string(config.network.port()));
::grpc::ServerBuilder builder;
builder.SetOption(std::unique_ptr<::grpc::ServerBuilderOption>(new NoReusePortOption));

74
proxy/src/server/init/Directory.cpp Normal file
View File

@ -0,0 +1,74 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/init/Directory.h"
#include <fcntl.h>
#include <unistd.h>
#include <boost/filesystem.hpp>
#include <string>
#include "config/ServerConfig.h"
namespace milvus::server {
Status
Directory::Initialize(const std::string& path) {
try {
init(path);
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
return Status::OK();
}
Status
Directory::Access(const std::string& path) {
try {
access_check(path);
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
return Status::OK();
}
void
Directory::init(const std::string& path) {
if (path.empty()) {
return;
}
try {
// Returns True if a new directory was created, otherwise false.
boost::filesystem::create_directories(path);
} catch (std::exception& ex) {
std::string msg = "Cannot create directory: " + path + ", reason: " + ex.what();
throw std::runtime_error(msg);
} catch (...) {
std::string msg = "Cannot create directory: " + path;
throw std::runtime_error(msg);
}
}
void
Directory::access_check(const std::string& path) {
if (path.empty()) {
return;
}
int ret = access(path.c_str(), F_OK | R_OK | W_OK);
if (0 != ret) {
std::string msg = "Cannot access path: " + path + ", error(" + std::to_string(errno) +
"): " + std::string(strerror(errno)) + ".";
throw std::runtime_error(msg);
}
}
} // namespace milvus::server

42
proxy/src/server/init/Directory.h Normal file
View File

@ -0,0 +1,42 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <string>
#include "utils/Status.h"
namespace milvus::server {
class Directory {
public:
static Status
Initialize(const std::string& path);
static Status
Lock(const std::string& path);
static Status
Access(const std::string& path);
private:
static void
init(const std::string& path);
static void
lock(const std::string& path);
static void
access_check(const std::string& path);
};
} // namespace milvus::server

12
proxy/src/server/init/StorageChecker.cpp
View File

@ -37,18 +37,6 @@ StorageChecker::CheckStoragePermission() {
return Status(SERVER_UNEXPECTED_ERROR, err_msg);
}
/* Check db directory write permission */
const std::string& primary_path = config.storage.path();
ret = access(primary_path.c_str(), F_OK | R_OK | W_OK);
if (0 != ret) {
std::string err_msg = " Access DB storage path " + primary_path + " fail. " + strerror(errno) +
"(code: " + std::to_string(errno) + ")";
LOG_SERVER_FATAL_ << err_msg;
std::cerr << err_msg << std::endl;
return Status(SERVER_UNEXPECTED_ERROR, err_msg);
}
return Status::OK();
}

5
writer/main.go
View File

@ -4,9 +4,9 @@ import (
"context"
"fmt"
"github.com/czs007/suvlim/storage/pkg"
"github.com/czs007/suvlim/storage/pkg/types"
"github.com/czs007/suvlim/writer/message_client"
"github.com/czs007/suvlim/writer/write_node"
"github.com/czs007/suvlim/conf"
"log"
"sync"
"time"
@ -16,14 +16,13 @@ func main() {
mc := message_client.MessageClient{}
mc.InitClient("pulsar://localhost:6650")
//mc.InitClient("pulsar://192.168.2.18:6650")
//TODO::close client / consumer/ producer
//mc.Close()
go mc.ReceiveMessage()
wg := sync.WaitGroup{}
ctx := context.Background()
kv, err := storage.NewStore(ctx, types.MinIODriver)
kv, err := storage.NewStore(ctx, conf.Config.Storage.Driver)
// if err != nil, should retry link
if err != nil {
log.Fatal(err)