test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 11:59:00 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Upgrade config (#2511)

Browse Source 
* enable cron test on 0.10.0

Signed-off-by: zw <zw@milvus.io>

* fix merge result

Signed-off-by: shengjun.li <shengjun.li@zilliz.com>

* import test_during_creating_index_restart

Signed-off-by: zw <zw@milvus.io>

* config cluster

Signed-off-by: wxyu <xy.wang@zilliz.com>

* config general

Signed-off-by: wxyu <xy.wang@zilliz.com>

* config network

Signed-off-by: wxyu <xy.wang@zilliz.com>

* config storage

Signed-off-by: wxyu <xy.wang@zilliz.com>

* config wal

Signed-off-by: wxyu <xy.wang@zilliz.com>

* config cache

Signed-off-by: wxyu <xy.wang@zilliz.com>

* config gpu and metrics

Signed-off-by: wxyu <xy.wang@zilliz.com>

* config logs

Signed-off-by: wxyu <xy.wang@zilliz.com>

* update server_config.template

Signed-off-by: wxyu <xy.wang@zilliz.com>

* update changelog

Signed-off-by: wxyu <xy.wang@zilliz.com>

* config with unit

Signed-off-by: wxyu <xy.wang@zilliz.com>

* fix clang-format

Signed-off-by: wxyu <xy.wang@zilliz.com>

* update unittests

Signed-off-by: wxyu <xy.wang@zilliz.com>

* disable restart case

Signed-off-by: zw <zw@milvus.io>

* fix gpu compile failed

Signed-off-by: wxyu <xy.wang@zilliz.com>

* Fix lint and comments

Signed-off-by: JinHai-CN <hai.jin@zilliz.com>

* fix wal unittest failed

Signed-off-by: wxyu <xy.wang@zilliz.com>

* fix some unittests

Signed-off-by: wxyu <xy.wang@zilliz.com>

* update config test cases

Signed-off-by: zw <zw@milvus.io>

* Fix config error

Signed-off-by: yhz <413554850@qq.com>

* remove unused code

Signed-off-by: wxyu <xy.wang@zilliz.com>

* Update CHANGELOG

Signed-off-by: JinHai-CN <hai.jin@zilliz.com>

* [skip-ci] fix test cases

Signed-off-by: zw <zw@milvus.io>

* fix config storage

Signed-off-by: wxyu <xy.wang@zilliz.com>

* make -j3 on ci

Signed-off-by: wxyu <xy.wang@zilliz.com>

* update python test

Signed-off-by: wxyu <xy.wang@zilliz.com>

* remove unused code

Signed-off-by: wxyu <xy.wang@zilliz.com>

* fix ut

Signed-off-by: wxyu <xy.wang@zilliz.com>

* fix ut

Signed-off-by: wxyu <xy.wang@zilliz.com>

* update status message

Signed-off-by: wxyu <xy.wang@zilliz.com>

* update

Signed-off-by: wxyu <xy.wang@zilliz.com>

* fix set config bug

Signed-off-by: wxyu <xy.wang@zilliz.com>

Co-authored-by: zw <zw@milvus.io>
Co-authored-by: shengjun.li <shengjun.li@zilliz.com>
Co-authored-by: JinHai-CN <hai.jin@zilliz.com>
Co-authored-by: yhz <413554850@qq.com>
This commit is contained in:
Wang XiangYu 2020-06-12 09:22:26 +08:00 committed by GitHub
parent a9629951e4
commit 83cc812935
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 2130 additions and 2253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -1,6 +1,7 @@
# Changelog
Please mark all change in change log and use the issue from GitHub
# Milvus 0.10.0 (TBD)
## Bug
@ -10,12 +11,13 @@ Please mark all change in change log and use the issue from GitHub
## Feature
- \#2363 Update branch version
- \#2510 Upgrade Milvus config
## Improvement
- \#2381 Upgrade FAISS to 1.6.3
- \#2429 Fix Milvus 0.9.1 performance degrade issue
- \#2441 Improve Knowhere code coverage
- \#2466 optimize k-selection implemention of faiss gpu version
- \#2466 Optimize k-selection implemention of faiss gpu version
- \#2489 Add exception throw on mysql meta error
- \#2495 Add creating lock file failure reason.
- \#2516 Improve unit test coverage

2
ci/jenkins/Jenkinsfile vendored
View File

@ -30,7 +30,7 @@ pipeline {
LOWER_BUILD_TYPE = params.BUILD_TYPE.toLowerCase()
SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
PIPELINE_NAME = "milvus-ci"
HELM_BRANCH = "master"
HELM_BRANCH = "0.10.0"
}
stages {

4
ci/jenkins/step/build.groovy
View File

@ -4,9 +4,9 @@ timeout(time: 120, unit: 'MINUTES') {
def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true)
if ("${BINARY_VERSION}" == "gpu") {
sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -g -u\""
sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j3 -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -g -u\""
} else {
sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u\""
sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j3 -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u\""
}
sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}"
}

267
core/conf/demo/server_config.yaml
View File

@ -9,158 +9,65 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
version: 0.4
version: 0.5
#----------------------+------------------------------------------------------------+------------+-----------------+
# Server Config | Description | Type | Default |
# Cluster Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# address | IP address that Milvus server monitors. | IP | 0.0.0.0 |
# enable | If runinng with Mishards, set true, otherwise false. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 |
# role | Milvus deployment role: rw / ro | role | rw |
#----------------------+------------------------------------------------------------+------------+-----------------+
# deploy_mode | Milvus deployment type: | DeployMode | single |
# | single, cluster_readonly, cluster_writable | | |
cluster:
enable: false
role: rw
#----------------------+------------------------------------------------------------+------------+-----------------+
# General Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# time_zone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# web_enable | Enable web server or not. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# web_port | Port that Milvus web server monitors. | Integer | 19121 |
# | Port range (1024, 65535) | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
server_config:
address: 0.0.0.0
port: 19530
deploy_mode: single
time_zone: UTC+8
web_enable: true
web_port: 19121
#----------------------+------------------------------------------------------------+------------+-----------------+
# DataBase Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# backend_url | URL for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ |
# meta_uri | URI for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ |
# | Milvus) or MySQL (for distributed cluster Milvus). | | |
# | Format: dialect://username:password@host:port/database | | |
# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | |
# | 'mysql', replace other texts with real values. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# preload_collection | A comma-separated list of collection names that need to | StringList | |
# | be pre-loaded when Milvus server starts up. | | |
# | '*' means preload all existing tables (single-quote or | | |
# | double-quote required). | | |
general:
timezone: UTC+8
meta_uri: sqlite://:@:/
#----------------------+------------------------------------------------------------+------------+-----------------+
# Network Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.enable | Enable web server or not. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.port | Port that Milvus web server monitors. | Integer | 19121 |
# | Port range (1024, 65535) | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
network:
bind.address: 0.0.0.0
bind.port: 19530
http.enable: true
http.port: 19121
#----------------------+------------------------------------------------------------+------------+-----------------+
# Storage Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus |
#----------------------+------------------------------------------------------------+------------+-----------------+
# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) |
# | flushes data to disk. | | |
# | 0 means disable the regular flush. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
db_config:
backend_url: sqlite://:@:/
preload_collection:
storage:
path: /var/lib/milvus
auto_flush_interval: 1
#----------------------+------------------------------------------------------------+------------+-----------------+
# Storage Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus |
# | index data. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# secondary_path | A semicolon-separated list of secondary directories used | Path | |
# | to save vector data and index data. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# file_cleanup_timeout | The time gap between marking a file as 'deleted' and | Integer | 10 (s) |
# | physically deleting this file from disk, range [0, 3600] | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage_config:
primary_path: /var/lib/milvus
secondary_path:
file_cleanup_timeout: 10
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable_monitor | Enable monitoring function or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# address | Pushgateway address | IP | 127.0.0.1 +
#----------------------+------------------------------------------------------------+------------+-----------------+
# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 |
#----------------------+------------------------------------------------------------+------------+-----------------+
metric_config:
enable_monitor: false
address: 127.0.0.1
port: 9091
#----------------------+------------------------------------------------------------+------------+-----------------+
# Cache Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integer | 4 (GB) |
# | query. The sum of 'cpu_cache_capacity' and | | |
# | 'insert_buffer_size' must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) |
# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_insert_data | Whether to load data to cache for hot query | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
cache_config:
cpu_cache_capacity: 4
insert_buffer_size: 1
cache_insert_data: false
#----------------------+------------------------------------------------------------+------------+-----------------+
# Engine Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 |
# | compared with 'nq' to decide if OpenBLAS should be used. | | |
# | If nq >= use_blas_threshold, OpenBLAS will be used, search | | |
# | response time will be stable but the search speed will be | | |
# | slower; if nq < use_blas_threshold, AVX or SSE will be | | |
# | used, search speed will be faster but search response time | | |
# | will fluctuate. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 |
# | compared with 'nq' to decide if the search computation will| | |
# | be executed on GPUs only. | | |
# | If nq >= gpu_search_threshold, the search computation will | | |
# | be executed on GPUs only; | | |
# | if nq < gpu_search_threshold, the search computation will | | |
# | be executed on both CPUs and GPUs. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
engine_config:
use_blas_threshold: 1100
gpu_search_threshold: 1000
#----------------------+------------------------------------------------------------+------------+-----------------+
# GPU Resource Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Enable GPU resources or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_capacity | The size of GPU memory per card used for cache. | Integer | 1 (GB) |
#----------------------+------------------------------------------------------------+------------+-----------------+
# search_resources | The list of GPU devices used for search computation. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# build_index_resources| The list of GPU devices used for index building. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
gpu_resource_config:
enable: false
cache_capacity: 1
search_resources:
- gpu0
build_index_resources:
- gpu0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Tracing Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# json_config_path | Absolute path for tracing config file. | Path | |
# | Leave it empty, a no-op tracer will be created. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
tracing_config:
json_config_path:
#----------------------+------------------------------------------------------------+------------+-----------------+
# WAL Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
@ -169,7 +76,7 @@ tracing_config:
# | files in advance before implementing data changes. WAL | | |
# | ensures the atomicity and durability for Milvus operations.| | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | true |
# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | false |
# | recovery. If true, when Milvus restarts for recovery and | | |
# | there are errors in WAL log files, log files with errors | | |
# | are ignored. If false, Milvus does not restart when there | | |
@ -183,29 +90,73 @@ tracing_config:
# | a value greater than the inserted data size of a single | | |
# | insert operation for better performance. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# wal_path | Location of WAL log files. | String | |
# path | Location of WAL log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
wal_config:
wal:
enable: true
recovery_error_ignore: true
recovery_error_ignore: false
buffer_size: 256
wal_path: /var/lib/milvus/wal
path: /var/lib/milvus/wal
#----------------------+------------------------------------------------------------+------------+-----------------+
# Logs | Description | Type | Default |
# Cache Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of CPU memory used for caching data for faster | Integer | 4 (GB) |
# | query. The sum of 'cpu_cache_capacity' and | | |
# | 'insert_buffer_size' must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) |
# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# preload_collection | A comma-separated list of collection names that need to | StringList | |
# | be pre-loaded when Milvus server starts up. | | |
# | '*' means preload all existing tables (single-quote or | | |
# | double-quote required). | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
cache:
cache_size: 4
insert_buffer_size: 1
preload_collection:
#----------------------+------------------------------------------------------------+------------+-----------------+
# GPU Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Enable GPU resources or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of GPU memory per card used for cache. | Integer | 1 (GB) |
#----------------------+------------------------------------------------------------+------------+-----------------+
# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 |
# | compared with 'nq' to decide if the search computation will| | |
# | be executed on GPUs only. | | |
# | If nq >= gpu_search_threshold, the search computation will | | |
# | be executed on GPUs only; | | |
# | if nq < gpu_search_threshold, the search computation will | | |
# | be executed on both CPUs and GPUs. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# search_resources | The list of GPU devices used for search computation. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# build_index_resources| The list of GPU devices used for index building. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
gpu:
enable: false
cache_size: 1
gpu_search_threshold: 1000
search_devices:
- gpu0
build_index_devices:
- gpu0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Logs Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug |
# | error, fatal | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# debug.enable | Whether to enable debug level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# info.enable | Whether to enable info level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# warning.enable | Whether to enable warning level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# error.enable | Whether to enable error level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# fatal.enable | Whether to enable fatal level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Absolute path to the folder holding the log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# max_log_file_size | The maximum size of each log file, size range [512, 4096] | Integer | 1024 (MB) |
@ -214,13 +165,23 @@ wal_config:
# | logging level, num range [0, 1024], 0 means unlimited. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
logs:
level: debug
trace.enable: true
debug.enable: true
info.enable: true
warning.enable: true
error.enable: true
fatal.enable: true
path: /var/lib/milvus/logs
max_log_file_size: 1024
log_rotate_num: 0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Enable monitoring function or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# address | Pushgateway address | IP | 127.0.0.1 +
#----------------------+------------------------------------------------------------+------------+-----------------+
# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 |
#----------------------+------------------------------------------------------------+------------+-----------------+
metric:
enable: false
address: 127.0.0.1
port: 9091

278
core/conf/server_config.template
View File

@ -9,158 +9,65 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
version: 0.4
version: 0.5
#----------------------+------------------------------------------------------------+------------+-----------------+
# Server Config | Description | Type | Default |
# Cluster Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# address | IP address that Milvus server monitors. | IP | 0.0.0.0 |
# enable | If runinng with Mishards, set true, otherwise false. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 |
# role | Milvus deployment role: rw / ro | role | rw |
#----------------------+------------------------------------------------------------+------------+-----------------+
# deploy_mode | Milvus deployment type: | DeployMode | single |
# | single, cluster_readonly, cluster_writable | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# time_zone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# web_enable | Enable web server or not. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# web_port | Port that Milvus web server monitors. | Integer | 19121 |
# | Port range (1024, 65535) | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
server_config:
address: 0.0.0.0
port: 19530
deploy_mode: single
time_zone: UTC+8
web_enable: true
web_port: 19121
cluster:
enable: false
role: rw
#----------------------+------------------------------------------------------------+------------+-----------------+
# DataBase Config | Description | Type | Default |
# General Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# backend_url | URL for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ |
# timezone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# meta_uri | URI for metadata storage, using SQLite (for single server | URL | sqlite://:@:/ |
# | Milvus) or MySQL (for distributed cluster Milvus). | | |
# | Format: dialect://username:password@host:port/database | | |
# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | |
# | 'mysql', replace other texts with real values. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# preload_collection | A comma-separated list of collection names that need to | StringList | |
# | be pre-loaded when Milvus server starts up. | | |
# | '*' means preload all existing tables (single-quote or | | |
# | double-quote required). | | |
general:
timezone: UTC+8
meta_uri: sqlite://:@:/
#----------------------+------------------------------------------------------------+------------+-----------------+
# Network Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.enable | Enable web server or not. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.port | Port that Milvus web server monitors. | Integer | 19121 |
# | Port range (1024, 65535) | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
network:
bind.address: 0.0.0.0
bind.port: 19530
http.enable: true
http.port: 19121
#----------------------+------------------------------------------------------------+------------+-----------------+
# Storage Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus |
#----------------------+------------------------------------------------------------+------------+-----------------+
# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) |
# | flushes data to disk. | | |
# | 0 means disable the regular flush. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
db_config:
backend_url: sqlite://:@:/
preload_collection:
storage:
path: @MILVUS_DB_PATH@
auto_flush_interval: 1
#----------------------+------------------------------------------------------------+------------+-----------------+
# Storage Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# primary_path | Primary directory used to save meta data, vector data and | Path | /var/lib/milvus |
# | index data. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# secondary_path | A semicolon-separated list of secondary directories used | Path | |
# | to save vector data and index data. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# file_cleanup_timeout | The time gap between marking a file as 'deleted' and | Integer | 10 (s) |
# | physically deleting this file from disk, range [0, 3600] | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage_config:
primary_path: @MILVUS_DB_PATH@
secondary_path:
file_cleanup_timeout: 10
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable_monitor | Enable monitoring function or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# address | Pushgateway address | IP | 127.0.0.1 +
#----------------------+------------------------------------------------------------+------------+-----------------+
# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 |
#----------------------+------------------------------------------------------------+------------+-----------------+
metric_config:
enable_monitor: false
address: 127.0.0.1
port: 9091
#----------------------+------------------------------------------------------------+------------+-----------------+
# Cache Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integer | 4 (GB) |
# | query. The sum of 'cpu_cache_capacity' and | | |
# | 'insert_buffer_size' must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# insert_buffer_size | Buffer size used for data insertion. | Integer | 1 (GB) |
# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_insert_data | Whether to load data to cache for hot query | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
cache_config:
cpu_cache_capacity: 4
insert_buffer_size: 1
cache_insert_data: false
#----------------------+------------------------------------------------------------+------------+-----------------+
# Engine Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# use_blas_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1100 |
# | compared with 'nq' to decide if OpenBLAS should be used. | | |
# | If nq >= use_blas_threshold, OpenBLAS will be used, search | | |
# | response time will be stable but the search speed will be | | |
# | slower; if nq < use_blas_threshold, AVX or SSE will be | | |
# | used, search speed will be faster but search response time | | |
# | will fluctuate. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 |
# | compared with 'nq' to decide if the search computation will| | |
# | be executed on GPUs only. | | |
# | If nq >= gpu_search_threshold, the search computation will | | |
# | be executed on GPUs only; | | |
# | if nq < gpu_search_threshold, the search computation will | | |
# | be executed on both CPUs and GPUs. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
engine_config:
use_blas_threshold: 1100
gpu_search_threshold: 1000
#----------------------+------------------------------------------------------------+------------+-----------------+
# GPU Resource Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Enable GPU resources or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_capacity | The size of GPU memory per card used for cache. | Integer | 1 (GB) |
#----------------------+------------------------------------------------------------+------------+-----------------+
# search_resources | The list of GPU devices used for search computation. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# build_index_resources| The list of GPU devices used for index building. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
gpu_resource_config:
enable: @GPU_ENABLE@
cache_capacity: 1
search_resources:
- gpu0
build_index_resources:
- gpu0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Tracing Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# json_config_path | Absolute path for tracing config file. | Path | |
# | Leave it empty, a no-op tracer will be created. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
tracing_config:
json_config_path:
#----------------------+------------------------------------------------------------+------------+-----------------+
# WAL Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
@ -169,58 +76,113 @@ tracing_config:
# | files in advance before implementing data changes. WAL | | |
# | ensures the atomicity and durability for Milvus operations.| | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | true |
# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | false |
# | recovery. If true, when Milvus restarts for recovery and | | |
# | there are errors in WAL log files, log files with errors | | |
# | are ignored. If false, Milvus does not restart when there | | |
# | are errors in WAL log files. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# buffer_size | Sum total of the read buffer and the write buffer in MBs. | Integer | 256 (MB) |
# | buffer_size must be in range [64, 4096] (MB). | | |
# buffer_size | Sum total of the read buffer and the write buffer in Bytes.| String | 256MB |
# | buffer_size must be in range [64MB, 4096MB]. | | |
# | If the value you specified is out of range, Milvus | | |
# | automatically uses the boundary value closest to the | | |
# | specified value. It is recommended you set buffer_size to | | |
# | a value greater than the inserted data size of a single | | |
# | insert operation for better performance. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# wal_path | Location of WAL log files. | String | |
# path | Location of WAL log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
wal_config:
wal:
enable: true
recovery_error_ignore: true
buffer_size: 256
wal_path: @MILVUS_DB_PATH@/wal
recovery_error_ignore: false
buffer_size: 256MB
path: @MILVUS_DB_PATH@/wal
#----------------------+------------------------------------------------------------+------------+-----------------+
# Logs | Description | Type | Default |
# Cache Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of CPU memory used for caching data for faster | String | 4GB |
# | query. The sum of 'cpu_cache_capacity' and | | |
# | 'insert_buffer_size' must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# insert_buffer_size | Buffer size used for data insertion. | String | 1GB |
# | The sum of 'insert_buffer_size' and 'cpu_cache_capacity' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# preload_collection | A comma-separated list of collection names that need to | StringList | |
# | be pre-loaded when Milvus server starts up. | | |
# | '*' means preload all existing tables (single-quote or | | |
# | double-quote required). | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
cache:
cache_size: 4GB
insert_buffer_size: 1GB
preload_collection:
#----------------------+------------------------------------------------------------+------------+-----------------+
# GPU Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Enable GPU resources or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of GPU memory per card used for cache. | String | 1GB |
#----------------------+------------------------------------------------------------+------------+-----------------+
# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 |
# | compared with 'nq' to decide if the search computation will| | |
# | be executed on GPUs only. | | |
# | If nq >= gpu_search_threshold, the search computation will | | |
# | be executed on GPUs only; | | |
# | if nq < gpu_search_threshold, the search computation will | | |
# | be executed on both CPUs and GPUs. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# search_resources | The list of GPU devices used for search computation. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# build_index_resources| The list of GPU devices used for index building. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
gpu:
enable: @GPU_ENABLE@
cache_size: 1GB
gpu_search_threshold: 1000
search_devices:
- gpu0
build_index_devices:
- gpu0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Logs Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug |
# | error, fatal | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# debug.enable | Whether to enable debug level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# info.enable | Whether to enable info level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# warning.enable | Whether to enable warning level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# error.enable | Whether to enable error level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# fatal.enable | Whether to enable fatal level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Absolute path to the folder holding the log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# max_log_file_size | The maximum size of each log file, size range [512, 4096] | Integer | 1024 (MB) |
# max_log_file_size | The maximum size of each log file, size range | String | 1024MB |
# | [512MB, 4096MB]. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 |
# | logging level, num range [0, 1024], 0 means unlimited. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
logs:
level: debug
trace.enable: true
debug.enable: true
info.enable: true
warning.enable: true
error.enable: true
fatal.enable: true
path: @MILVUS_DB_PATH@/logs
max_log_file_size: 1024
max_log_file_size: 1024MB
log_rotate_num: 0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Enable monitoring function or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# address | Pushgateway address | IP | 127.0.0.1 +
#----------------------+------------------------------------------------------------+------------+-----------------+
# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 |
#----------------------+------------------------------------------------------------+------------+-----------------+
metric:
enable: false
address: 127.0.0.1
port: 9091

7
core/src/cache/CpuCacheMgr.cpp vendored
View File

@ -22,8 +22,9 @@ namespace milvus {
namespace cache {
namespace {
constexpr int64_t unit = 1024 * 1024 * 1024;
}
// constexpr int64_t unit = 1024 * 1024 * 1024;
constexpr int64_t unit = 1;
} // namespace
CpuCacheMgr::CpuCacheMgr() {
// All config values have been checked in Config::ValidateConfig()
@ -32,6 +33,8 @@ CpuCacheMgr::CpuCacheMgr() {
int64_t cpu_cache_cap;
config.GetCacheConfigCpuCacheCapacity(cpu_cache_cap);
int64_t cap = cpu_cache_cap * unit;
LOG_SERVER_DEBUG_ << "cpu cache.size: " << cap;
LOG_SERVER_INFO_ << "cpu cache.size: " << cap;
cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL << 32, "[CACHE CPU]");
float cpu_cache_threshold;

1489
core/src/config/Config.cpp
View File

File diff suppressed because it is too large Load Diff

278
core/src/config/Config.h
View File

@ -28,55 +28,49 @@ using ConfigCallBackF = std::function<Status(const std::string&)>;
extern const char* CONFIG_NODE_DELIMITER;
extern const char* CONFIG_VERSION;
/* server config */
extern const char* CONFIG_SERVER;
extern const char* CONFIG_SERVER_ADDRESS;
extern const char* CONFIG_SERVER_ADDRESS_DEFAULT;
extern const char* CONFIG_SERVER_PORT;
extern const char* CONFIG_SERVER_PORT_DEFAULT;
extern const char* CONFIG_SERVER_DEPLOY_MODE;
extern const char* CONFIG_SERVER_DEPLOY_MODE_DEFAULT;
extern const char* CONFIG_SERVER_TIME_ZONE;
extern const char* CONFIG_SERVER_TIME_ZONE_DEFAULT;
extern const char* CONFIG_SERVER_WEB_ENABLE;
extern const char* CONFIG_SERVER_WEB_ENABLE_DEFAULT;
extern const char* CONFIG_SERVER_WEB_PORT;
extern const char* CONFIG_SERVER_WEB_PORT_DEFAULT;
/* cluster config */
extern const char* CONFIG_CLUSTER;
extern const char* CONFIG_CLUSTER_ENABLE;
extern const char* CONFIG_CLUSTER_ENABLE_DEFAULT;
extern const char* CONFIG_CLUSTER_ROLE;
extern const char* CONFIG_CLUSTER_ROLE_DEFAULT;
/* general config */
extern const char* CONFIG_GENERAL;
extern const char* CONFIG_GENERAL_TIMEZONE;
extern const char* CONFIG_GENERAL_TIMEZONE_DEFAULT;
extern const char* CONFIG_GENERAL_METAURI;
extern const char* CONFIG_GENERAL_METAURI_DEFAULT;
/* network config */
extern const char* CONFIG_NETWORK;
extern const char* CONFIG_NETWORK_BIND_ADDRESS;
extern const char* CONFIG_NETWORK_BIND_ADDRESS_DEFAULT;
extern const char* CONFIG_NETWORK_BIND_PORT;
extern const char* CONFIG_NETWORK_BIND_PORT_DEFAULT;
extern const char* CONFIG_NETWORK_HTTP_ENABLE;
extern const char* CONFIG_NETWORK_HTTP_ENABLE_DEFAULT;
extern const char* CONFIG_NETWORK_HTTP_PORT;
extern const char* CONFIG_NETWORK_HTTP_PORT_DEFAULT;
/* db config */
extern const char* CONFIG_DB;
extern const char* CONFIG_DB_BACKEND_URL;
extern const char* CONFIG_DB_BACKEND_URL_DEFAULT;
// extern const char* CONFIG_DB_BACKEND_URL;
// extern const char* CONFIG_DB_BACKEND_URL_DEFAULT;
extern const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD;
extern const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT;
extern const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD;
extern const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT;
extern const char* CONFIG_DB_PRELOAD_COLLECTION;
extern const char* CONFIG_DB_PRELOAD_COLLECTION_DEFAULT;
extern const char* CONFIG_DB_AUTO_FLUSH_INTERVAL;
extern const char* CONFIG_DB_AUTO_FLUSH_INTERVAL_DEFAULT;
/* storage config */
extern const char* CONFIG_STORAGE;
extern const char* CONFIG_STORAGE_PRIMARY_PATH;
extern const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT;
extern const char* CONFIG_STORAGE_SECONDARY_PATH;
extern const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT;
extern const char* CONFIG_STORAGE_PATH;
extern const char* CONFIG_STORAGE_PATH_DEFAULT;
extern const char* CONFIG_STORAGE_AUTO_FLUSH_INTERVAL;
extern const char* CONFIG_STORAGE_AUTO_FLUSH_INTERVAL_DEFAULT;
extern const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT;
extern const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN;
extern const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX;
// extern const char* CONFIG_STORAGE_S3_ENABLE;
// extern const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT;
// extern const char* CONFIG_STORAGE_S3_ADDRESS;
// extern const char* CONFIG_STORAGE_S3_ADDRESS_DEFAULT;
// extern const char* CONFIG_STORAGE_S3_PORT;
// extern const char* CONFIG_STORAGE_S3_PORT_DEFAULT;
// extern const char* CONFIG_STORAGE_S3_ACCESS_KEY;
// extern const char* CONFIG_STORAGE_S3_ACCESS_KEY_DEFAULT;
// extern const char* CONFIG_STORAGE_S3_SECRET_KEY;
// extern const char* CONFIG_STORAGE_S3_SECRET_KEY_DEFAULT;
// extern const char* CONFIG_STORAGE_S3_BUCKET;
// extern const char* CONFIG_STORAGE_S3_BUCKET_DEFAULT;
/* cache config */
extern const char* CONFIG_CACHE;
@ -88,6 +82,8 @@ extern const char* CONFIG_CACHE_INSERT_BUFFER_SIZE;
extern const char* CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT;
extern const char* CONFIG_CACHE_CACHE_INSERT_DATA;
extern const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT;
extern const char* CONFIG_CACHE_PRELOAD_COLLECTION;
extern const char* CONFIG_CACHE_PRELOAD_COLLECTION_DEFAULT;
/* metric config */
extern const char* CONFIG_METRIC;
@ -106,8 +102,6 @@ extern const char* CONFIG_ENGINE_OMP_THREAD_NUM;
extern const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT;
extern const char* CONFIG_ENGINE_SIMD_TYPE;
extern const char* CONFIG_ENGINE_SIMD_TYPE_DEFAULT;
extern const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD;
extern const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT;
/* gpu resource config */
extern const char* CONFIG_GPU_RESOURCE;
@ -117,6 +111,8 @@ extern const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY;
extern const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT;
extern const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD;
extern const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT;
extern const char* CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD;
extern const char* CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD_DEFAULT;
extern const char* CONFIG_GPU_RESOURCE_DELIMITER;
extern const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES;
extern const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT;
@ -142,18 +138,10 @@ extern const char* CONFIG_WAL_WAL_PATH_DEFAULT;
/* logs config */
extern const char* CONFIG_LOGS;
extern const char* CONFIG_LOGS_LEVEL;
extern const char* CONFIG_LOGS_LEVEL_DEFAULT;
extern const char* CONFIG_LOGS_TRACE_ENABLE;
extern const char* CONFIG_LOGS_TRACE_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_DEBUG_ENABLE;
extern const char* CONFIG_LOGS_DEBUG_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_INFO_ENABLE;
extern const char* CONFIG_LOGS_INFO_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_WARNING_ENABLE;
extern const char* CONFIG_LOGS_WARNING_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_ERROR_ENABLE;
extern const char* CONFIG_LOGS_ERROR_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_FATAL_ENABLE;
extern const char* CONFIG_LOGS_FATAL_ENABLE_DEFAULT;
extern const char* CONFIG_LOGS_PATH;
extern const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE;
extern const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT;
@ -215,51 +203,43 @@ class Config {
Status
CheckConfigVersion(const std::string& value);
/* server config */
/* cluster config */
Status
CheckServerConfigAddress(const std::string& value);
CheckClusterConfigEnable(const std::string& value);
Status
CheckServerConfigPort(const std::string& value);
CheckClusterConfigRole(const std::string& value);
/* general config */
Status
CheckServerConfigDeployMode(const std::string& value);
CheckGeneralConfigTimezone(const std::string& value);
Status
CheckServerConfigTimeZone(const std::string& value);
CheckGeneralConfigMetaURI(const std::string& value);
/* network config */
Status
CheckServerConfigWebEnable(const std::string& value);
CheckNetworkConfigBindAddress(const std::string& value);
Status
CheckServerConfigWebPort(const std::string& value);
CheckNetworkConfigBindPort(const std::string& value);
Status
CheckNetworkConfigHTTPEnable(const std::string& value);
Status
CheckNetworkConfigHTTPPort(const std::string& value);
/* db config */
Status
CheckDBConfigBackendUrl(const std::string& value);
Status
CheckDBConfigPreloadCollection(const std::string& value);
// Status
// CheckDBConfigBackendUrl(const std::string& value);
Status
CheckDBConfigArchiveDiskThreshold(const std::string& value);
Status
CheckDBConfigArchiveDaysThreshold(const std::string& value);
Status
CheckDBConfigAutoFlushInterval(const std::string& value);
/* storage config */
Status
CheckStorageConfigPrimaryPath(const std::string& value);
CheckStorageConfigPath(const std::string& value);
Status
CheckStorageConfigSecondaryPath(const std::string& value);
CheckStorageConfigAutoFlushInterval(const std::string& value);
Status
CheckStorageConfigFileCleanupTimeout(const std::string& value);
// Status
// CheckStorageConfigS3Enable(const std::string& value);
// Status
// CheckStorageConfigS3Address(const std::string& value);
// Status
// CheckStorageConfigS3Port(const std::string& value);
// Status
// CheckStorageConfigS3AccessKey(const std::string& value);
// Status
// CheckStorageConfigS3SecretKey(const std::string& value);
// Status
// CheckStorageConfigS3Bucket(const std::string& value);
/* metric config */
Status
@ -278,6 +258,8 @@ class Config {
CheckCacheConfigInsertBufferSize(const std::string& value);
Status
CheckCacheConfigCacheInsertData(const std::string& value);
Status
CheckCacheConfigPreloadCollection(const std::string& value);
/* engine config */
Status
@ -288,9 +270,6 @@ class Config {
CheckEngineConfigSimdType(const std::string& value);
#ifdef MILVUS_GPU_VERSION
Status
CheckEngineConfigGpuSearchThreshold(const std::string& value);
/* gpu resource config */
Status
CheckGpuResourceConfigEnable(const std::string& value);
@ -299,6 +278,8 @@ class Config {
Status
CheckGpuResourceConfigCacheThreshold(const std::string& value);
Status
CheckGpuResourceConfigGpuSearchThreshold(const std::string& value);
Status
CheckGpuResourceConfigSearchResources(const std::vector<std::string>& value);
Status
CheckGpuResourceConfigBuildIndexResources(const std::vector<std::string>& value);
@ -320,18 +301,10 @@ class Config {
/* logs config */
Status
CheckLogsLevel(const std::string& level);
Status
CheckLogsTraceEnable(const std::string& value);
Status
CheckLogsDebugEnable(const std::string& value);
Status
CheckLogsInfoEnable(const std::string& value);
Status
CheckLogsWarningEnable(const std::string& value);
Status
CheckLogsErrorEnable(const std::string& value);
Status
CheckLogsFatalEnable(const std::string& value);
Status
CheckLogsPath(const std::string& value);
Status
CheckLogsMaxLogFileSize(const std::string& value);
@ -350,51 +323,43 @@ class Config {
ExecCallBacks(const std::string& node, const std::string& sub_node, const std::string& value);
public:
/* server config */
/* cluster config */
Status
GetServerConfigAddress(std::string& value);
GetClusterConfigEnable(bool& value);
Status
GetServerConfigPort(std::string& value);
GetClusterConfigRole(std::string& value);
/* general config */
Status
GetServerConfigDeployMode(std::string& value);
GetGeneralConfigTimezone(std::string& value);
Status
GetServerConfigTimeZone(std::string& value);
GetGeneralConfigMetaURI(std::string& value);
/* network config */
Status
GetServerConfigWebEnable(bool& value);
GetNetworkConfigBindAddress(std::string& value);
Status
GetServerConfigWebPort(std::string& value);
GetNetworkConfigBindPort(std::string& value);
Status
GetNetworkConfigHTTPEnable(bool& value);
Status
GetNetworkConfigHTTPPort(std::string& value);
/* db config */
Status
GetDBConfigBackendUrl(std::string& value);
// Status
// GetDBConfigBackendUrl(std::string& value);
Status
GetDBConfigArchiveDiskThreshold(int64_t& value);
Status
GetDBConfigArchiveDaysThreshold(int64_t& value);
Status
GetDBConfigPreloadCollection(std::string& value);
Status
GetDBConfigAutoFlushInterval(int64_t& value);
/* storage config */
Status
GetStorageConfigPrimaryPath(std::string& value);
GetStorageConfigPath(std::string& value);
Status
GetStorageConfigSecondaryPath(std::string& value);
GetStorageConfigAutoFlushInterval(int64_t& value);
Status
GetStorageConfigFileCleanupTimeup(int64_t& value);
// Status
// GetStorageConfigS3Enable(bool& value);
// Status
// GetStorageConfigS3Address(std::string& value);
// Status
// GetStorageConfigS3Port(std::string& value);
// Status
// GetStorageConfigS3AccessKey(std::string& value);
// Status
// GetStorageConfigS3SecretKey(std::string& value);
// Status
// GetStorageConfigS3Bucket(std::string& value);
/* metric config */
Status
@ -413,6 +378,8 @@ class Config {
GetCacheConfigInsertBufferSize(int64_t& value);
Status
GetCacheConfigCacheInsertData(bool& value);
Status
GetCacheConfigPreloadCollection(std::string& value);
/* engine config */
Status
@ -423,9 +390,6 @@ class Config {
GetEngineConfigSimdType(std::string& value);
#ifdef MILVUS_GPU_VERSION
Status
GetEngineConfigGpuSearchThreshold(int64_t& value);
/* gpu resource config */
Status
GetGpuResourceConfigEnable(bool& value);
@ -434,6 +398,8 @@ class Config {
Status
GetGpuResourceConfigCacheThreshold(float& value);
Status
GetGpuResourceConfigGpuSearchThreshold(int64_t& value);
Status
GetGpuResourceConfigSearchResources(std::vector<int64_t>& value);
Status
GetGpuResourceConfigBuildIndexResources(std::vector<int64_t>& value);
@ -455,18 +421,10 @@ class Config {
/* logs config */
Status
GetLogsLevel(std::string& value);
Status
GetLogsTraceEnable(bool& value);
Status
GetLogsDebugEnable(bool& value);
Status
GetLogsInfoEnable(bool& value);
Status
GetLogsWarningEnable(bool& value);
Status
GetLogsErrorEnable(bool& value);
Status
GetLogsFatalEnable(bool& value);
Status
GetLogsPath(std::string& value);
Status
GetLogsMaxLogFileSize(int64_t& value);
@ -477,51 +435,43 @@ class Config {
GetServerRestartRequired(bool& required);
public:
/* server config */
/* cluster config */
Status
SetServerConfigAddress(const std::string& value);
SetClusterConfigEnable(const std::string& value);
Status
SetServerConfigPort(const std::string& value);
SetClusterConfigRole(const std::string& value);
/* general config */
Status
SetServerConfigDeployMode(const std::string& value);
SetGeneralConfigTimezone(const std::string& value);
Status
SetServerConfigTimeZone(const std::string& value);
SetGeneralConfigMetaURI(const std::string& value);
/* network config */
Status
SetServerConfigWebEnable(const std::string& value);
SetNetworkConfigBindAddress(const std::string& value);
Status
SetServerConfigWebPort(const std::string& value);
SetNetworkConfigBindPort(const std::string& value);
Status
SetNetworkConfigHTTPEnable(const std::string& value);
Status
SetNetworkConfigHTTPPort(const std::string& value);
/* db config */
Status
SetDBConfigBackendUrl(const std::string& value);
Status
SetDBConfigPreloadCollection(const std::string& value);
// Status
// SetDBConfigBackendUrl(const std::string& value);
Status
SetDBConfigArchiveDiskThreshold(const std::string& value);
Status
SetDBConfigArchiveDaysThreshold(const std::string& value);
Status
SetDBConfigAutoFlushInterval(const std::string& value);
/* storage config */
Status
SetStorageConfigPrimaryPath(const std::string& value);
SetStorageConfigPath(const std::string& value);
Status
SetStorageConfigSecondaryPath(const std::string& value);
SetStorageConfigAutoFlushInterval(const std::string& value);
Status
SetStorageConfigFileCleanupTimeout(const std::string& value);
// Status
// SetStorageConfigS3Enable(const std::string& value);
// Status
// SetStorageConfigS3Address(const std::string& value);
// Status
// SetStorageConfigS3Port(const std::string& value);
// Status
// SetStorageConfigS3AccessKey(const std::string& value);
// Status
// SetStorageConfigS3SecretKey(const std::string& value);
// Status
// SetStorageConfigS3Bucket(const std::string& value);
/* metric config */
Status
@ -540,6 +490,8 @@ class Config {
SetCacheConfigInsertBufferSize(const std::string& value);
Status
SetCacheConfigCacheInsertData(const std::string& value);
Status
SetCacheConfigPreloadCollection(const std::string& value);
/* engine config */
Status
@ -549,8 +501,6 @@ class Config {
Status
SetEngineConfigSimdType(const std::string& value);
#ifdef MILVUS_GPU_VERSION
Status
SetEngineConfigGpuSearchThreshold(const std::string& value);
/* gpu resource config */
Status
@ -560,6 +510,8 @@ class Config {
Status
SetGpuResourceConfigCacheThreshold(const std::string& value);
Status
SetGpuResourceConfigGpuSearchThreshold(const std::string& value);
Status
SetGpuResourceConfigSearchResources(const std::string& value);
Status
SetGpuResourceConfigBuildIndexResources(const std::string& value);
@ -581,18 +533,10 @@ class Config {
/* logs config */
Status
SetLogsLevel(const std::string& value);
Status
SetLogsTraceEnable(const std::string& value);
Status
SetLogsDebugEnable(const std::string& value);
Status
SetLogsInfoEnable(const std::string& value);
Status
SetLogsWarningEnable(const std::string& value);
Status
SetLogsErrorEnable(const std::string& value);
Status
SetLogsFatalEnable(const std::string& value);
Status
SetLogsPath(const std::string& value);
Status
SetLogsMaxLogFileSize(const std::string& value);

85
core/src/config/Utils.cpp Normal file
View File

@ -0,0 +1,85 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "config/Utils.h"
#include <algorithm>
#include <sstream>
#include <unordered_map>
namespace milvus {
namespace server {
std::unordered_map<std::string, int64_t> BYTE_UNITS = {
{"b", 1},
{"k", 1024},
{"m", 1024 * 1024},
{"g", 1024 * 1024 * 1024},
};
bool
is_number(const std::string& s) {
return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end();
}
bool
is_alpha(const std::string& s) {
return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isalpha(c); }) == s.end();
}
std::string
str_tolower(std::string s) {
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); });
return s;
}
int64_t
parse_bytes(const std::string& str, std::string& err) {
try {
std::string s = str;
if (is_number(s))
return std::stoll(s);
if (s.length() == 0)
return 0;
auto last_two = s.substr(s.length() - 2, 2);
auto last_one = s.substr(s.length() - 1);
if (is_alpha(last_two) && is_alpha(last_one))
if (last_one == "b" or last_one == "B")
s = s.substr(0, s.length() - 1);
auto& units = BYTE_UNITS;
auto suffix = str_tolower(s.substr(s.length() - 1));
std::string digits_part;
if (is_number(suffix)) {
digits_part = s;
suffix = 'b';
} else {
digits_part = s.substr(0, s.length() - 1);
}
if (units.find(suffix) != units.end() or is_number(suffix)) {
auto digits = std::stoll(digits_part);
return digits * units[suffix];
} else {
std::stringstream ss;
ss << "The specified value for memory (" << str << ") should specify the units."
<< "The postfix should be one of the `b` `k` `m` `g` characters";
err = ss.str();
}
} catch (...) {
err = "Unknown error happened on parse bytes.";
}
return 0;
}
} // namespace server
} // namespace milvus

23
core/src/config/Utils.h Normal file
View File

@ -0,0 +1,23 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <string>
namespace milvus {
namespace server {
int64_t
parse_bytes(const std::string& str, std::string& err);
} // namespace server
} // namespace milvus

9
core/src/config/handler/GpuResourceConfigHandler.cpp
View File

@ -94,19 +94,20 @@ GpuResourceConfigHandler::AddGpuSearchThresholdListener() {
ConfigCallBackF lambda_gpu_threshold = [this](const std::string& value) -> Status {
if (!gpu_enable_) {
std::string msg =
std::string("GPU resources is disable. Cannot set config ") + CONFIG_ENGINE_GPU_SEARCH_THRESHOLD;
std::string("GPU resources is disabled. Cannot set config ") + CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
auto& config = Config::GetInstance();
auto status = config.GetEngineConfigGpuSearchThreshold(threshold_);
auto status = config.GetGpuResourceConfigGpuSearchThreshold(threshold_);
if (status.ok()) {
OnGpuSearchThresholdChanged(threshold_);
}
return status;
};
config.RegisterCallBack(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, identity_, lambda_gpu_threshold);
config.RegisterCallBack(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD, identity_,
lambda_gpu_threshold);
}
void
@ -152,7 +153,7 @@ GpuResourceConfigHandler::RemoveGpuBuildResourcesListener() {
void
GpuResourceConfigHandler::RemoveGpuSearchThresholdListener() {
auto& config = Config::GetInstance();
config.CancelCallBack(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, identity_);
config.CancelCallBack(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD, identity_);
}
void

1
core/src/db/meta/MetaFactory.cpp
View File

@ -48,6 +48,7 @@ MetaFactory::Build(const DBMetaOptions& meta_options, const int& mode) {
std::string uri = meta_options.backend_uri_;
utils::MetaUriInfo uri_info;
LOG_ENGINE_DEBUG_ << "MetaUri: " << uri << std::endl;
auto status = utils::ParseMetaUri(uri, uri_info);
if (!status.ok()) {
LOG_ENGINE_ERROR_ << "Wrong URI format: URI = " << uri;

1
core/src/db/wal/WalDefinations.h
View File

@ -27,6 +27,7 @@ using TableSchemaPtr = std::shared_ptr<milvus::engine::meta::CollectionSchema>;
using TableMetaPtr = std::shared_ptr<std::unordered_map<std::string, TableSchemaPtr>>;
#define UNIT_MB (1024 * 1024)
#define UNIT_B 1
#define LSN_OFFSET_MASK 0x00000000ffffffff
enum class MXLogType { None, InsertBinary, InsertVector, Delete, Update, Flush, Entity };

2
core/src/metrics/prometheus/PrometheusMetrics.cpp
View File

@ -33,7 +33,7 @@ PrometheusMetrics::Init() {
// Following should be read from config file.
std::string server_port, push_port, push_address;
STATUS_CHECK(config.GetServerConfigPort(server_port));
STATUS_CHECK(config.GetNetworkConfigBindPort(server_port));
STATUS_CHECK(config.GetMetricConfigPort(push_port));
STATUS_CHECK(config.GetMetricConfigAddress(push_address));

2
core/src/scheduler/SchedInst.h
View File

@ -106,7 +106,7 @@ class OptimizerInst {
int64_t gpu_search_threshold;
config.GetGpuResourceConfigBuildIndexResources(build_gpus);
config.GetGpuResourceConfigSearchResources(search_gpus);
config.GetEngineConfigGpuSearchThreshold(gpu_search_threshold);
config.GetGpuResourceConfigGpuSearchThreshold(gpu_search_threshold);
std::string build_msg = "Build index gpu:";
for (auto build_id : build_gpus) {
build_msg.append(" gpu" + std::to_string(build_id));

2
core/src/scheduler/selector/FaissFlatPass.cpp
View File

@ -25,7 +25,7 @@ namespace scheduler {
void
FaissFlatPass::Init() {
server::Config& config = server::Config::GetInstance();
Status s = config.GetEngineConfigGpuSearchThreshold(threshold_);
Status s = config.GetGpuResourceConfigGpuSearchThreshold(threshold_);
if (!s.ok()) {
threshold_ = std::numeric_limits<int32_t>::max();
}

2
core/src/scheduler/selector/FaissIVFFlatPass.cpp
View File

@ -25,7 +25,7 @@ void
FaissIVFFlatPass::Init() {
#ifdef MILVUS_GPU_VERSION
server::Config& config = server::Config::GetInstance();
Status s = config.GetEngineConfigGpuSearchThreshold(threshold_);
Status s = config.GetGpuResourceConfigGpuSearchThreshold(threshold_);
if (!s.ok()) {
threshold_ = std::numeric_limits<int32_t>::max();
}

2
core/src/scheduler/selector/FaissIVFPQPass.cpp
View File

@ -27,7 +27,7 @@ void
FaissIVFPQPass::Init() {
#ifdef MILVUS_GPU_VERSION
server::Config& config = server::Config::GetInstance();
Status s = config.GetEngineConfigGpuSearchThreshold(threshold_);
Status s = config.GetGpuResourceConfigGpuSearchThreshold(threshold_);
if (!s.ok()) {
threshold_ = std::numeric_limits<int32_t>::max();
}

2
core/src/scheduler/selector/FaissIVFSQ8HPass.cpp
View File

@ -25,7 +25,7 @@ namespace scheduler {
void
FaissIVFSQ8HPass::Init() {
server::Config& config = server::Config::GetInstance();
Status s = config.GetEngineConfigGpuSearchThreshold(threshold_);
Status s = config.GetGpuResourceConfigGpuSearchThreshold(threshold_);
if (!s.ok()) {
threshold_ = std::numeric_limits<int64_t>::max();
}

2
core/src/scheduler/selector/FaissIVFSQ8Pass.cpp
View File

@ -25,7 +25,7 @@ void
FaissIVFSQ8Pass::Init() {
#ifdef MILVUS_GPU_VERSION
server::Config& config = server::Config::GetInstance();
Status s = config.GetEngineConfigGpuSearchThreshold(threshold_);
Status s = config.GetGpuResourceConfigGpuSearchThreshold(threshold_);
if (!s.ok()) {
threshold_ = std::numeric_limits<int32_t>::max();
}

43
core/src/server/DBWrapper.cpp
View File

@ -34,36 +34,26 @@ DBWrapper::StartService() {
// db config
engine::DBOptions opt;
s = config.GetDBConfigBackendUrl(opt.meta_.backend_uri_);
if (!s.ok()) {
std::cerr << s.ToString() << std::endl;
return s;
}
s = config.GetDBConfigAutoFlushInterval(opt.auto_flush_interval_);
s = config.GetGeneralConfigMetaURI(opt.meta_.backend_uri_);
if (!s.ok()) {
std::cerr << s.ToString() << std::endl;
return s;
}
std::string path;
s = config.GetStorageConfigPrimaryPath(path);
s = config.GetStorageConfigPath(path);
if (!s.ok()) {
std::cerr << s.ToString() << std::endl;
return s;
}
opt.meta_.path_ = path + "/db";
std::string db_slave_path;
s = config.GetStorageConfigSecondaryPath(db_slave_path);
s = config.GetStorageConfigAutoFlushInterval(opt.auto_flush_interval_);
if (!s.ok()) {
std::cerr << s.ToString() << std::endl;
return s;
}
StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_);
s = config.GetStorageConfigFileCleanupTimeup(opt.file_cleanup_timeout_);
if (!s.ok()) {
std::cerr << s.ToString() << std::endl;
@ -90,8 +80,25 @@ DBWrapper::StartService() {
std::cerr << s.ToString() << std::endl;
return s;
}
opt.insert_buffer_size_ = insert_buffer_size * engine::GB;
opt.insert_buffer_size_ = insert_buffer_size;
#if 1
bool cluster_enable = false;
std::string cluster_role;
STATUS_CHECK(config.GetClusterConfigEnable(cluster_enable));
STATUS_CHECK(config.GetClusterConfigRole(cluster_role));
if (not cluster_enable) {
opt.mode_ = engine::DBOptions::MODE::SINGLE;
} else if (cluster_role == "ro") {
opt.mode_ = engine::DBOptions::MODE::CLUSTER_READONLY;
} else if (cluster_role == "rw") {
opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE;
} else {
std::cerr << "Error: cluster.role is not one of rw and ro." << std::endl;
kill(0, SIGUSR1);
}
#else
std::string mode;
s = config.GetServerConfigDeployMode(mode);
if (!s.ok()) {
@ -110,6 +117,7 @@ DBWrapper::StartService() {
<< "single, cluster_readonly, and cluster_writable." << std::endl;
kill(0, SIGUSR1);
}
#endif
// get wal configurations
s = config.GetWalConfigEnable(opt.wal_enable_);
@ -127,12 +135,15 @@ DBWrapper::StartService() {
kill(0, SIGUSR1);
}
s = config.GetWalConfigBufferSize(opt.buffer_size_);
int64_t wal_buffer_size = 0;
s = config.GetWalConfigBufferSize(wal_buffer_size);
if (!s.ok()) {
std::cerr << "ERROR! Failed to get buffer_size configuration." << std::endl;
std::cerr << s.ToString() << std::endl;
kill(0, SIGUSR1);
}
wal_buffer_size /= (1024 * 1024);
opt.buffer_size_ = wal_buffer_size;
s = config.GetWalConfigWalPath(opt.mxlog_path_);
if (!s.ok()) {
@ -227,7 +238,7 @@ DBWrapper::StartService() {
// preload collection
std::string preload_collections;
s = config.GetDBConfigPreloadCollection(preload_collections);
s = config.GetCacheConfigPreloadCollection(preload_collections);
if (!s.ok()) {
std::cerr << s.ToString() << std::endl;
return s;

59
core/src/server/Server.cpp
View File

@ -16,6 +16,7 @@
#include <unistd.h>
#include <boost/filesystem.hpp>
#include <cstring>
#include <unordered_map>
#include "config/Config.h"
#include "index/archive/KnowhereResource.h"
@ -154,7 +155,7 @@ Server::Start() {
Config& config = Config::GetInstance();
std::string meta_uri;
STATUS_CHECK(config.GetDBConfigBackendUrl(meta_uri));
STATUS_CHECK(config.GetGeneralConfigMetaURI(meta_uri));
if (meta_uri.length() > 6 && strcasecmp("sqlite", meta_uri.substr(0, 6).c_str()) == 0) {
std::cout << "WARNNING: You are using SQLite as the meta data management, "
"which can't be used in production. Please change it to MySQL!"
@ -169,7 +170,7 @@ Server::Start() {
/* log path is defined in Config file, so InitLog must be called after LoadConfig */
std::string time_zone;
s = config.GetServerConfigTimeZone(time_zone);
s = config.GetGeneralConfigTimezone(time_zone);
if (!s.ok()) {
std::cerr << "Fail to get server config timezone" << std::endl;
return s;
@ -194,6 +195,11 @@ Server::Start() {
tzset();
{
std::unordered_map<std::string, int64_t> level_to_int{
{"debug", 5}, {"info", 4}, {"warning", 3}, {"error", 2}, {"fatal", 1},
};
std::string level;
bool trace_enable = false;
bool debug_enable = false;
bool info_enable = false;
@ -203,12 +209,25 @@ Server::Start() {
std::string logs_path;
int64_t max_log_file_size = 0;
int64_t delete_exceeds = 0;
STATUS_CHECK(config.GetLogsLevel(level));
switch (level_to_int[level]) {
case 5:
debug_enable = true;
case 4:
info_enable = true;
case 3:
warning_enable = true;
case 2:
error_enable = true;
case 1:
fatal_enable = true;
break;
default:
return Status(SERVER_UNEXPECTED_ERROR, "invalid log level");
}
STATUS_CHECK(config.GetLogsTraceEnable(trace_enable));
STATUS_CHECK(config.GetLogsDebugEnable(debug_enable));
STATUS_CHECK(config.GetLogsInfoEnable(info_enable));
STATUS_CHECK(config.GetLogsWarningEnable(warning_enable));
STATUS_CHECK(config.GetLogsErrorEnable(error_enable));
STATUS_CHECK(config.GetLogsFatalEnable(fatal_enable));
STATUS_CHECK(config.GetLogsPath(logs_path));
STATUS_CHECK(config.GetLogsMaxLogFileSize(max_log_file_size));
STATUS_CHECK(config.GetLogsLogRotateNum(delete_exceeds));
@ -216,12 +235,18 @@ Server::Start() {
max_log_file_size, delete_exceeds);
}
std::string deploy_mode;
STATUS_CHECK(config.GetServerConfigDeployMode(deploy_mode));
bool cluster_enable = false;
std::string cluster_role;
STATUS_CHECK(config.GetClusterConfigEnable(cluster_enable));
STATUS_CHECK(config.GetClusterConfigRole(cluster_role));
if (deploy_mode == "single" || deploy_mode == "cluster_writable") {
// std::string deploy_mode;
// STATUS_CHECK(config.GetServerConfigDeployMode(deploy_mode));
// if (deploy_mode == "single" || deploy_mode == "cluster_writable") {
if ((not cluster_enable) || cluster_role == "rw") {
std::string db_path;
STATUS_CHECK(config.GetStorageConfigPrimaryPath(db_path));
STATUS_CHECK(config.GetStorageConfigPath(db_path));
try {
// True if a new directory was created, otherwise false.
@ -232,7 +257,11 @@ Server::Start() {
s = InstanceLockCheck::Check(db_path);
if (!s.ok()) {
std::cerr << "deploy_mode: " << deploy_mode << " instance lock db path failed." << std::endl;
if (not cluster_enable) {
std::cerr << "single instance lock db path failed." << s.message() << std::endl;
} else {
std::cerr << cluster_role << " instance lock db path failed." << s.message() << std::endl;
}
return s;
}
@ -251,7 +280,11 @@ Server::Start() {
}
s = InstanceLockCheck::Check(wal_path);
if (!s.ok()) {
std::cerr << "deploy_mode: " << deploy_mode << " instance lock wal path failed." << std::endl;
if (not cluster_enable) {
std::cerr << "single instance lock wal path failed." << s.message() << std::endl;
} else {
std::cerr << cluster_role << " instance lock wal path failed." << s.message() << std::endl;
}
return s;
}
}

12
core/src/server/delivery/request/ReLoadSegmentsRequest.cpp
View File

@ -37,6 +37,17 @@ Status
ReLoadSegmentsRequest::OnExecute() {
auto& config = Config::GetInstance();
#if 1
bool cluster_enable = false;
std::string cluster_role;
STATUS_CHECK(config.GetClusterConfigEnable(cluster_enable));
STATUS_CHECK(config.GetClusterConfigRole(cluster_role));
if ((not cluster_enable) || cluster_role == "rw") {
// TODO: No need to reload segment files
return Status(SERVER_SUCCESS, "");
}
#else
std::string deploy_mode;
auto status = config.GetServerConfigDeployMode(deploy_mode);
if (!status.ok()) {
@ -48,6 +59,7 @@ ReLoadSegmentsRequest::OnExecute() {
// TODO: No need to reload segment files
return Status(SERVER_SUCCESS, "");
}
#endif
try {
std::string hdr = "ReloadSegmentsRequest(collection=" + collection_name_ + ")";

4
core/src/server/grpc_impl/GrpcServer.cpp
View File

@ -77,8 +77,8 @@ GrpcServer::StartService() {
Config& config = Config::GetInstance();
std::string address, port;
STATUS_CHECK(config.GetServerConfigAddress(address));
STATUS_CHECK(config.GetServerConfigPort(port));
STATUS_CHECK(config.GetNetworkConfigBindAddress(address));
STATUS_CHECK(config.GetNetworkConfigBindPort(port));
std::string server_address(address + ":" + port);

37
core/src/server/init/StorageChecker.cpp
View File

@ -44,6 +44,16 @@ StorageChecker::CheckStoragePermission() {
return Status(SERVER_UNEXPECTED_ERROR, err_msg);
}
#if 1
bool cluster_enable = false;
std::string cluster_role;
STATUS_CHECK(config.GetClusterConfigEnable(cluster_enable));
STATUS_CHECK(config.GetClusterConfigRole(cluster_role));
if (cluster_enable && cluster_role == "ro") {
return Status::OK();
}
#else
std::string deploy_mode;
status = config.GetServerConfigDeployMode(deploy_mode);
if (!status.ok()) {
@ -53,10 +63,11 @@ StorageChecker::CheckStoragePermission() {
if (deploy_mode == "cluster_readonly") {
return Status::OK();
}
#endif
/* Check db directory write permission */
std::string primary_path;
status = config.GetStorageConfigPrimaryPath(primary_path);
status = config.GetStorageConfigPath(primary_path);
if (!status.ok()) {
return status;
}
@ -64,35 +75,13 @@ StorageChecker::CheckStoragePermission() {
ret = access(primary_path.c_str(), F_OK | R_OK | W_OK);
fiu_do_on("StorageChecker.CheckStoragePermission.db_primary_path_access_fail", ret = -1);
if (0 != ret) {
std::string err_msg = " Access DB storage primary path " + primary_path + " fail. " + strerror(errno) +
std::string err_msg = " Access DB storage path " + primary_path + " fail. " + strerror(errno) +
"(code: " + std::to_string(errno) + ")";
LOG_SERVER_FATAL_ << err_msg;
std::cerr << err_msg << std::endl;
return Status(SERVER_UNEXPECTED_ERROR, err_msg);
}
std::string secondary_paths;
status = config.GetStorageConfigSecondaryPath(secondary_paths);
if (!status.ok()) {
return status;
}
if (!secondary_paths.empty()) {
std::vector<std::string> secondary_path_vector;
StringHelpFunctions::SplitStringByDelimeter(secondary_paths, ",", secondary_path_vector);
for (auto& path : secondary_path_vector) {
ret = access(path.c_str(), F_OK | R_OK | W_OK);
fiu_do_on("StorageChecker.CheckStoragePermission.db_secondary_path_access_fail", ret = -1);
if (0 != ret) {
std::string err_msg = " Access DB storage secondary path " + path + " fail. " + strerror(errno) +
"(code: " + std::to_string(errno) + ")";
LOG_SERVER_FATAL_ << err_msg;
std::cerr << err_msg << std::endl;
return Status(SERVER_UNEXPECTED_ERROR, err_msg);
}
}
}
/* Check wal directory write permission */
bool wal_enable = false;
status = config.GetWalConfigEnable(wal_enable);

4
core/src/server/web_impl/WebServer.cpp
View File

@ -24,7 +24,7 @@ void
WebServer::Start() {
auto& config = Config::GetInstance();
bool enable = true;
config.GetServerConfigWebEnable(enable);
config.GetNetworkConfigHTTPEnable(enable);
if (enable && nullptr == thread_ptr_) {
thread_ptr_ = std::make_shared<std::thread>(&WebServer::StartService, this);
}
@ -47,7 +47,7 @@ WebServer::StartService() {
Config& config = Config::GetInstance();
std::string port;
STATUS_CHECK(config.GetServerConfigWebPort(port));
STATUS_CHECK(config.GetNetworkConfigHTTPPort(port));
{
AppComponent components = AppComponent(std::stoi(port));

9
core/src/server/web_impl/handler/WebRequestHandler.cpp
View File

@ -866,7 +866,7 @@ WebRequestHandler::GetAdvancedConfig(AdvancedConfigDto::ObjectWrapper& advanced_
advanced_config->use_blas_threshold = std::stol(reply);
#ifdef MILVUS_GPU_VERSION
engine_cmd_string = engine_cmd_prefix + std::string(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD);
engine_cmd_string = engine_cmd_prefix + std::string(CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD);
CommandLine(engine_cmd_string, reply);
if (!status.ok()) {
ASSIGN_RETURN_STATUS_DTO(status)
@ -924,9 +924,10 @@ WebRequestHandler::SetAdvancedConfig(const AdvancedConfigDto::ObjectWrapper& adv
}
#ifdef MILVUS_GPU_VERSION
engine_cmd_string = engine_cmd_prefix + std::string(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD) + " " +
std::to_string(advanced_config->gpu_search_threshold->getValue());
status = CommandLine(engine_cmd_string, reply);
auto gpu_cmd_prefix = "set_config " + std::string(CONFIG_GPU_RESOURCE) + ".";
auto gpu_cmd_string = gpu_cmd_prefix + std::string(CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD) + " " +
std::to_string(advanced_config->gpu_search_threshold->getValue());
status = CommandLine(gpu_cmd_string, reply);
if (!status.ok()) {
ASSIGN_RETURN_STATUS_DTO(status)
}

1
core/src/utils/LogUtil.cpp
View File

@ -199,7 +199,6 @@ InitLog(bool trace_enable, bool debug_enable, bool info_enable, bool warning_ena
std::to_string(CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) + "], now is " +
std::to_string(max_log_file_size));
}
max_log_file_size *= 1024 * 1024;
defaultConf.setGlobally(el::ConfigurationType::MaxLogFileSize, std::to_string(max_log_file_size));
el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
el::Helpers::installPreRollOutCallback(RolloutHandler);

10
core/src/utils/ValidationUtil.cpp
View File

@ -31,6 +31,7 @@
#include <cmath>
#include <limits>
#include <regex>
#include <set>
#include <string>
namespace milvus {
@ -651,6 +652,15 @@ ValidationUtil::ValidateStoragePath(const std::string& path) {
return std::regex_match(path, regex) ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid file path");
}
Status
ValidationUtil::ValidateLogLevel(const std::string& level) {
std::set<std::string> supported_level{"debug", "info", "warning", "error", "fatal"};
return supported_level.find(level) != supported_level.end()
? Status::OK()
: Status(SERVER_INVALID_ARGUMENT, "Log level must be one of debug, info, warning, error and fatal.");
}
bool
ValidationUtil::IsNumber(const std::string& s) {
return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);

3
core/src/utils/ValidationUtil.h
View File

@ -93,6 +93,9 @@ class ValidationUtil {
static Status
ValidateStoragePath(const std::string& path);
static Status
ValidateLogLevel(const std::string& level);
static bool
IsNumber(const std::string& s);
};

55
core/unittest/db/utils.cpp
View File

@ -39,6 +39,60 @@ INITIALIZE_EASYLOGGINGPP
namespace {
static const char* CONFIG_STR =
"version: 0.5\n"
"\n"
"cluster:\n"
" enable: false\n"
" role: rw\n"
"\n"
"general:\n"
" timezone: UTC+8\n"
" meta_uri: sqlite://:@:/\n"
"\n"
"network:\n"
" bind.address: 0.0.0.0\n"
" bind.port: 19530\n"
" http.enable: true\n"
" http.port: 19121\n"
"\n"
"storage:\n"
" path: /tmp/milvus\n"
" auto_flush_interval: 1\n"
"\n"
"wal:\n"
" enable: true\n"
" recovery_error_ignore: false\n"
" buffer_size: 256MB\n"
" path: /tmp/milvus/wal\n"
"\n"
"cache:\n"
" cache_size: 4GB\n"
" insert_buffer_size: 1GB\n"
" preload_collection:\n"
"\n"
"gpu:\n"
" enable: true\n"
" cache_size: 1GB\n"
" gpu_search_threshold: 1000\n"
" search_devices:\n"
" - gpu0\n"
" build_index_devices:\n"
" - gpu0\n"
"\n"
"logs:\n"
" level: debug\n"
" trace.enable: true\n"
" path: /tmp/milvus/logs\n"
" max_log_file_size: 1024MB\n"
" log_rotate_num: 0\n"
"\n"
"metric:\n"
" enable: false\n"
" address: 127.0.0.1\n"
" port: 9091\n"
"\n";
/*static const char* CONFIG_STR =
"version: 0.4\n"
"server_config:\n"
" address: 0.0.0.0\n"
@ -100,6 +154,7 @@ static const char* CONFIG_STR =
" max_log_file_size: 256\n"
" delete_exceeds: 10\n"
"";
*/
void
WriteToFile(const std::string &file_path, const char *content) {

15
core/unittest/server/test_check.cpp
View File

@ -37,11 +37,7 @@ class ServerCheckerTest : public testing::Test {
db_primary_path = "/tmp/milvus-test/db";
boost::filesystem::create_directories(db_primary_path);
config.SetStorageConfigPrimaryPath(db_primary_path);
db_secondary_path = "/tmp/milvus-test/db-secondary";
boost::filesystem::create_directories(db_secondary_path);
config.SetStorageConfigSecondaryPath(db_secondary_path);
config.SetStorageConfigPath(db_primary_path);
wal_path = "/tmp/milvus-test/wal";
boost::filesystem::create_directories(wal_path);
@ -77,15 +73,6 @@ TEST_F(ServerCheckerTest, STORAGE_FAIL_TEST) {
ASSERT_FALSE(ms::StorageChecker::CheckStoragePermission().ok());
fiu_disable("StorageChecker.CheckStoragePermission.db_primary_path_access_fail");
auto& config = ms::Config::GetInstance();
std::string storage_secondary_path;
ASSERT_TRUE(config.GetStorageConfigSecondaryPath(storage_secondary_path).ok());
ASSERT_TRUE(config.SetStorageConfigSecondaryPath("/tmp/milvus-test01,/tmp/milvus-test02").ok());
fiu_enable("StorageChecker.CheckStoragePermission.db_secondary_path_access_fail", 1, NULL, 0);
ASSERT_FALSE(ms::StorageChecker::CheckStoragePermission().ok());
fiu_disable("StorageChecker.CheckStoragePermission.db_secondary_path_access_fail");
ASSERT_TRUE(config.SetStorageConfigSecondaryPath(storage_secondary_path).ok());
fiu_enable("StorageChecker.CheckStoragePermission.wal_path_access_fail", 1, NULL, 0);
ASSERT_FALSE(ms::StorageChecker::CheckStoragePermission().ok());
fiu_disable("StorageChecker.CheckStoragePermission.wal_path_access_fail");

355
core/unittest/server/test_config.cpp
View File

@ -51,7 +51,7 @@ TEST_F(ConfigTest, CONFIG_TEST) {
config_mgr->DumpString();
milvus::server::ConfigNode& root_config = config_mgr->GetRootNode();
milvus::server::ConfigNode& server_config = root_config.GetChild("server_config");
milvus::server::ConfigNode& server_config = root_config.GetChild("network");
milvus::server::ConfigNode invalid_config = root_config.GetChild("invalid_config");
const auto& im_config_mgr = *static_cast<milvus::server::YamlConfigMgr*>(config_mgr);
@ -74,9 +74,9 @@ TEST_F(ConfigTest, CONFIG_TEST) {
bool not_exit_bool = server_config.GetBoolValue("not_exit", false);
ASSERT_FALSE(not_exit_bool);
std::string address = server_config.GetValue("address");
std::string address = server_config.GetValue("bind.address");
ASSERT_TRUE(!address.empty());
int64_t port = server_config.GetInt64Value("port");
int64_t port = server_config.GetInt64Value("bind.port");
ASSERT_NE(port, 0);
server_config.SetValue("float_test", "2.5");
@ -138,34 +138,34 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
/* server config */
std::string server_addr = "192.168.1.155";
ASSERT_TRUE(config.SetServerConfigAddress(server_addr).ok());
ASSERT_TRUE(config.GetServerConfigAddress(str_val).ok());
ASSERT_TRUE(config.SetNetworkConfigBindAddress(server_addr).ok());
ASSERT_TRUE(config.GetNetworkConfigBindAddress(str_val).ok());
ASSERT_TRUE(str_val == server_addr);
std::string server_port = "12345";
ASSERT_TRUE(config.SetServerConfigPort(server_port).ok());
ASSERT_TRUE(config.GetServerConfigPort(str_val).ok());
ASSERT_TRUE(config.SetNetworkConfigBindPort(server_port).ok());
ASSERT_TRUE(config.GetNetworkConfigBindPort(str_val).ok());
ASSERT_TRUE(str_val == server_port);
std::string web_port = "19999";
ASSERT_TRUE(config.SetServerConfigWebPort(web_port).ok());
ASSERT_TRUE(config.GetServerConfigWebPort(str_val).ok());
ASSERT_TRUE(config.SetNetworkConfigHTTPPort(web_port).ok());
ASSERT_TRUE(config.GetNetworkConfigHTTPPort(str_val).ok());
ASSERT_TRUE(str_val == web_port);
std::string server_mode = "cluster_readonly";
ASSERT_TRUE(config.SetServerConfigDeployMode(server_mode).ok());
ASSERT_TRUE(config.GetServerConfigDeployMode(str_val).ok());
std::string server_mode = "ro";
ASSERT_TRUE(config.SetClusterConfigRole(server_mode).ok());
ASSERT_TRUE(config.GetClusterConfigRole(str_val).ok());
ASSERT_TRUE(str_val == server_mode);
std::string server_time_zone = "UTC+6";
ASSERT_TRUE(config.SetServerConfigTimeZone(server_time_zone).ok());
ASSERT_TRUE(config.GetServerConfigTimeZone(str_val).ok());
ASSERT_TRUE(config.SetGeneralConfigTimezone(server_time_zone).ok());
ASSERT_TRUE(config.GetGeneralConfigTimezone(str_val).ok());
ASSERT_TRUE(str_val == server_time_zone);
/* db config */
std::string db_backend_url = "mysql://root:123456@127.0.0.1:19530/milvus";
ASSERT_TRUE(config.SetDBConfigBackendUrl(db_backend_url).ok());
ASSERT_TRUE(config.GetDBConfigBackendUrl(str_val).ok());
ASSERT_TRUE(config.SetGeneralConfigMetaURI(db_backend_url).ok());
ASSERT_TRUE(config.GetGeneralConfigMetaURI(str_val).ok());
ASSERT_TRUE(str_val == db_backend_url);
int64_t db_archive_disk_threshold = 100;
@ -179,31 +179,16 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
ASSERT_TRUE(int64_val == db_archive_days_threshold);
int64_t db_auto_flush_interval = 1;
ASSERT_TRUE(config.SetDBConfigAutoFlushInterval(std::to_string(db_auto_flush_interval)).ok());
ASSERT_TRUE(config.GetDBConfigAutoFlushInterval(int64_val).ok());
ASSERT_TRUE(config.SetStorageConfigAutoFlushInterval(std::to_string(db_auto_flush_interval)).ok());
ASSERT_TRUE(config.GetStorageConfigAutoFlushInterval(int64_val).ok());
ASSERT_TRUE(int64_val == db_auto_flush_interval);
/* storage config */
std::string storage_primary_path = "/home/zilliz";
ASSERT_TRUE(config.SetStorageConfigPrimaryPath(storage_primary_path).ok());
ASSERT_TRUE(config.GetStorageConfigPrimaryPath(str_val).ok());
ASSERT_TRUE(config.SetStorageConfigPath(storage_primary_path).ok());
ASSERT_TRUE(config.GetStorageConfigPath(str_val).ok());
ASSERT_TRUE(str_val == storage_primary_path);
std::string storage_secondary_path = "/home/zilliz";
ASSERT_TRUE(config.SetStorageConfigSecondaryPath(storage_secondary_path).ok());
ASSERT_TRUE(config.GetStorageConfigSecondaryPath(str_val).ok());
ASSERT_TRUE(str_val == storage_secondary_path);
storage_secondary_path = "/home/zilliz,/tmp/milvus";
ASSERT_TRUE(config.SetStorageConfigSecondaryPath(storage_secondary_path).ok());
ASSERT_TRUE(config.GetStorageConfigSecondaryPath(str_val).ok());
ASSERT_TRUE(str_val == storage_secondary_path);
storage_secondary_path = "";
ASSERT_TRUE(config.SetStorageConfigSecondaryPath(storage_secondary_path).ok());
ASSERT_TRUE(config.GetStorageConfigSecondaryPath(str_val).ok());
ASSERT_TRUE(str_val == storage_secondary_path);
// bool storage_s3_enable = true;
// ASSERT_TRUE(config.SetStorageConfigS3Enable(std::to_string(storage_s3_enable)).ok());
// ASSERT_TRUE(config.GetStorageConfigS3Enable(bool_val).ok());
@ -289,8 +274,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
#ifdef MILVUS_GPU_VERSION
int64_t engine_gpu_search_threshold = 800;
ASSERT_TRUE(config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)).ok());
ASSERT_TRUE(config.GetEngineConfigGpuSearchThreshold(int64_val).ok());
auto status = config.SetGpuResourceConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold));
ASSERT_TRUE(status.ok()) << status.message();
ASSERT_TRUE(config.GetGpuResourceConfigGpuSearchThreshold(int64_val).ok());
ASSERT_TRUE(int64_val == engine_gpu_search_threshold);
#endif
@ -345,7 +331,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
ASSERT_TRUE(config.GetWalConfigRecoveryErrorIgnore(bool_val).ok());
ASSERT_TRUE(bool_val == wal_recovery_ignore);
int64_t wal_buffer_size = 128;
int64_t wal_buffer_size = 128 * 1024 * 1024; // 128 M
ASSERT_TRUE(config.SetWalConfigBufferSize(std::to_string(wal_buffer_size)).ok());
ASSERT_TRUE(config.GetWalConfigBufferSize(int64_val).ok());
ASSERT_TRUE(int64_val == wal_buffer_size);
@ -356,45 +342,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
ASSERT_TRUE(str_val == wal_path);
/* logs config */
std::string logs_level = "debug";
ASSERT_TRUE(config.SetLogsLevel(logs_level).ok());
ASSERT_TRUE(config.GetLogsLevel(str_val).ok());
ASSERT_TRUE(str_val == logs_level);
bool logs_trace_enable = false;
ASSERT_TRUE(config.SetLogsTraceEnable(std::to_string(logs_trace_enable)).ok());
ASSERT_TRUE(config.GetLogsTraceEnable(bool_val).ok());
ASSERT_TRUE(bool_val == logs_trace_enable);
bool logs_debug_enable = false;
ASSERT_TRUE(config.SetLogsDebugEnable(std::to_string(logs_debug_enable)).ok());
ASSERT_TRUE(config.GetLogsDebugEnable(bool_val).ok());
ASSERT_TRUE(bool_val == logs_debug_enable);
bool logs_info_enable = false;
ASSERT_TRUE(config.SetLogsTraceEnable(std::to_string(logs_info_enable)).ok());
ASSERT_TRUE(config.GetLogsTraceEnable(bool_val).ok());
ASSERT_TRUE(bool_val == logs_info_enable);
bool logs_warning_enable = false;
ASSERT_TRUE(config.SetLogsDebugEnable(std::to_string(logs_warning_enable)).ok());
ASSERT_TRUE(config.GetLogsDebugEnable(bool_val).ok());
ASSERT_TRUE(bool_val == logs_warning_enable);
bool logs_error_enable = false;
ASSERT_TRUE(config.SetLogsTraceEnable(std::to_string(logs_error_enable)).ok());
ASSERT_TRUE(config.GetLogsTraceEnable(bool_val).ok());
ASSERT_TRUE(bool_val == logs_error_enable);
bool logs_fatal_enable = false;
ASSERT_TRUE(config.SetLogsDebugEnable(std::to_string(logs_fatal_enable)).ok());
ASSERT_TRUE(config.GetLogsDebugEnable(bool_val).ok());
ASSERT_TRUE(bool_val == logs_fatal_enable);
std::string logs_path = "/tmp/aaa/logs";
ASSERT_TRUE(config.SetLogsPath(logs_path).ok());
ASSERT_TRUE(config.GetLogsPath(str_val).ok());
ASSERT_TRUE(str_val == logs_path);
int64_t logs_max_log_file_size = 1000;
ASSERT_TRUE(config.SetLogsMaxLogFileSize(std::to_string(logs_max_log_file_size)).ok());
std::string logs_max_log_file_size = "1000MB";
auto s = config.SetLogsMaxLogFileSize(logs_max_log_file_size);
ASSERT_TRUE(s.ok()) << s.message();
ASSERT_TRUE(config.GetLogsMaxLogFileSize(int64_val).ok());
ASSERT_TRUE(int64_val == logs_max_log_file_size);
ASSERT_TRUE(int64_val == 1000 * 1024 * 1024); // 1000MB
int64_t logs_log_rotate_num = 100;
ASSERT_TRUE(config.SetLogsLogRotateNum(std::to_string(logs_log_rotate_num)).ok());
@ -421,7 +388,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) {
auto s = config.LoadConfigFile(conf_file);
ASSERT_TRUE(s.ok()) << s.message();
s = config.ResetDefaultConfig();
ASSERT_TRUE(s.ok());
ASSERT_TRUE(s.ok()) << s.message();
std::string get_cmd, set_cmd;
std::string result, dummy;
@ -431,17 +398,17 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) {
/* server config */
std::string server_addr = "192.168.1.155";
get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS);
set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr);
get_cmd = gen_get_command(ms::CONFIG_NETWORK, ms::CONFIG_NETWORK_BIND_ADDRESS);
set_cmd = gen_set_command(ms::CONFIG_NETWORK, ms::CONFIG_NETWORK_BIND_ADDRESS, server_addr);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(s.ok()) << s.message();
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(s.ok()) << s.message();
/* db config */
std::string db_backend_url = "sqlite://milvus:zilliz@:/";
get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL);
set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL, db_backend_url);
get_cmd = gen_get_command(ms::CONFIG_GENERAL, ms::CONFIG_GENERAL_METAURI);
set_cmd = gen_set_command(ms::CONFIG_GENERAL, ms::CONFIG_GENERAL_METAURI, db_backend_url);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
@ -458,13 +425,23 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) {
/* storage config */
std::string storage_primary_path = "/tmp/milvus1";
get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH);
set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH, storage_primary_path);
get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PATH);
set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PATH, storage_primary_path);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
std::string storage_auto_flush_interval = "42";
get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_AUTO_FLUSH_INTERVAL);
set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_AUTO_FLUSH_INTERVAL, storage_auto_flush_interval);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
/* cache config */
std::string cache_cpu_cache_capacity = "1";
get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY);
@ -532,8 +509,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) {
#ifdef MILVUS_GPU_VERSION
std::string engine_gpu_search_threshold = "800";
get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold);
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD,
engine_gpu_search_threshold);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
@ -620,42 +598,40 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
ASSERT_FALSE(config.LoadConfigFile(config_path + "dummy.yaml").ok());
/* server config */
ASSERT_FALSE(config.SetServerConfigAddress("0.0.0").ok());
ASSERT_FALSE(config.SetServerConfigAddress("0.0.0.256").ok());
ASSERT_FALSE(config.SetNetworkConfigBindAddress("0.0.0").ok());
ASSERT_FALSE(config.SetNetworkConfigBindAddress("0.0.0.256").ok());
ASSERT_FALSE(config.SetServerConfigPort("a").ok());
ASSERT_FALSE(config.SetServerConfigPort("99999").ok());
ASSERT_FALSE(config.SetNetworkConfigBindPort("a").ok());
ASSERT_FALSE(config.SetNetworkConfigBindPort("99999").ok());
ASSERT_FALSE(config.SetServerConfigWebPort("a").ok());
ASSERT_FALSE(config.SetServerConfigWebPort("99999").ok());
ASSERT_FALSE(config.SetServerConfigWebPort("-1").ok());
ASSERT_FALSE(config.SetNetworkConfigHTTPPort("a").ok());
ASSERT_FALSE(config.SetNetworkConfigHTTPPort("99999").ok());
ASSERT_FALSE(config.SetNetworkConfigHTTPPort("-1").ok());
ASSERT_FALSE(config.SetServerConfigDeployMode("cluster").ok());
ASSERT_FALSE(config.SetClusterConfigRole("cluster").ok());
ASSERT_FALSE(config.SetServerConfigTimeZone("GM").ok());
ASSERT_FALSE(config.SetServerConfigTimeZone("GMT8").ok());
ASSERT_FALSE(config.SetServerConfigTimeZone("UTCA").ok());
ASSERT_FALSE(config.SetGeneralConfigTimezone("GM").ok());
ASSERT_FALSE(config.SetGeneralConfigTimezone("GMT8").ok());
ASSERT_FALSE(config.SetGeneralConfigTimezone("UTCA").ok());
/* db config */
ASSERT_FALSE(config.SetDBConfigBackendUrl("http://www.google.com").ok());
ASSERT_FALSE(config.SetDBConfigBackendUrl("sqlite://:@:").ok());
ASSERT_FALSE(config.SetDBConfigBackendUrl("mysql://root:123456@127.0.0.1/milvus").ok());
ASSERT_FALSE(config.SetGeneralConfigMetaURI("http://www.google.com").ok());
ASSERT_FALSE(config.SetGeneralConfigMetaURI("sqlite://:@:").ok());
ASSERT_FALSE(config.SetGeneralConfigMetaURI("mysql://root:123456@127.0.0.1/milvus").ok());
ASSERT_FALSE(config.SetDBConfigArchiveDiskThreshold("0x10").ok());
ASSERT_FALSE(config.SetDBConfigArchiveDaysThreshold("0x10").ok());
ASSERT_FALSE(config.SetDBConfigAutoFlushInterval("0.1").ok());
/* storage config */
ASSERT_FALSE(config.SetStorageConfigPrimaryPath("").ok());
ASSERT_FALSE(config.SetStorageConfigPrimaryPath("./milvus").ok());
ASSERT_FALSE(config.SetStorageConfigPrimaryPath("../milvus").ok());
ASSERT_FALSE(config.SetStorageConfigPrimaryPath("/**milvus").ok());
ASSERT_FALSE(config.SetStorageConfigPrimaryPath("/milvus--/path").ok());
ASSERT_FALSE(config.SetStorageConfigPath("").ok());
ASSERT_FALSE(config.SetStorageConfigPath("./milvus").ok());
ASSERT_FALSE(config.SetStorageConfigPath("../milvus").ok());
ASSERT_FALSE(config.SetStorageConfigPath("/**milvus").ok());
ASSERT_FALSE(config.SetStorageConfigPath("/milvus--/path").ok());
ASSERT_FALSE(config.SetStorageConfigSecondaryPath("../milvus,./zilliz").ok());
ASSERT_FALSE(config.SetStorageConfigSecondaryPath("/home/^^__^^,/zilliz").ok());
ASSERT_FALSE(config.SetStorageConfigAutoFlushInterval("0.1").ok());
// ASSERT_FALSE(config.SetStorageConfigS3Enable("10").ok());
//
@ -680,7 +656,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
/* cache config */
ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("a").ok());
ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("0").ok());
ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("2048").ok());
ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("2048G").ok());
ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("-1").ok());
ASSERT_FALSE(config.SetCacheConfigCpuCacheThreshold("a").ok());
@ -689,7 +665,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("a").ok());
ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("0").ok());
ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("2048").ok());
ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("2048GB").ok());
ASSERT_FALSE(config.SetCacheConfigInsertBufferSize("-1").ok());
ASSERT_FALSE(config.SetCacheConfigCacheInsertData("N").ok());
@ -704,7 +680,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
ASSERT_FALSE(config.SetEngineConfigSimdType("None").ok());
#ifdef MILVUS_GPU_VERSION
ASSERT_FALSE(config.SetEngineConfigGpuSearchThreshold("-1").ok());
ASSERT_FALSE(config.SetGpuResourceConfigGpuSearchThreshold("-1").ok());
#endif
/* gpu resource config */
@ -712,7 +688,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
ASSERT_FALSE(config.SetGpuResourceConfigEnable("ok").ok());
ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("a").ok());
ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("128").ok());
ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("128GB").ok());
ASSERT_FALSE(config.SetGpuResourceConfigCacheCapacity("-1").ok());
ASSERT_FALSE(config.SetGpuResourceConfigCacheThreshold("a").ok());
@ -734,15 +710,11 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
ASSERT_FALSE(config.SetWalConfigBufferSize("a").ok());
/* wal config */
ASSERT_FALSE(config.SetLogsLevel("invalid").ok());
ASSERT_FALSE(config.SetLogsTraceEnable("invalid").ok());
ASSERT_FALSE(config.SetLogsDebugEnable("invalid").ok());
ASSERT_FALSE(config.SetLogsInfoEnable("invalid").ok());
ASSERT_FALSE(config.SetLogsWarningEnable("invalid").ok());
ASSERT_FALSE(config.SetLogsErrorEnable("invalid").ok());
ASSERT_FALSE(config.SetLogsFatalEnable("invalid").ok());
ASSERT_FALSE(config.SetLogsPath("").ok());
ASSERT_FALSE(config.SetLogsMaxLogFileSize("-1").ok());
ASSERT_FALSE(config.SetLogsMaxLogFileSize("511").ok());
ASSERT_FALSE(config.SetLogsMaxLogFileSize("511MB").ok());
ASSERT_FALSE(config.SetLogsLogRotateNum("-1").ok());
ASSERT_FALSE(config.SetLogsLogRotateNum("1025").ok());
}
@ -759,7 +731,8 @@ TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
config.GetConfigJsonStr(config_json_str);
std::cout << config_json_str << std::endl;
ASSERT_TRUE(config.ResetDefaultConfig().ok());
auto s = config.ResetDefaultConfig();
ASSERT_TRUE(s.ok()) << s.message();
}
TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) {
@ -776,41 +749,36 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) {
fiu_disable("check_config_version_fail");
/* server config */
fiu_enable("check_config_address_fail", 1, NULL, 0);
fiu_enable("check_config_bind_address_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_address_fail");
fiu_disable("check_config_bind_address_fail");
fiu_enable("check_config_port_fail", 1, NULL, 0);
fiu_enable("check_config_bind_port_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_port_fail");
fiu_disable("check_config_bind_port_fail");
fiu_enable("check_config_deploy_mode_fail", 1, NULL, 0);
fiu_enable("check_config_cluster_role_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_deploy_mode_fail");
fiu_disable("check_config_cluster_role_fail");
fiu_enable("check_config_time_zone_fail", 1, NULL, 0);
fiu_enable("check_config_timezone_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_time_zone_fail");
fiu_disable("check_config_timezone_fail");
/* db config */
fiu_enable("check_config_primary_path_fail", 1, NULL, 0);
fiu_enable("check_config_path_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_primary_path_fail");
fiu_disable("check_config_path_fail");
fiu_enable("check_config_secondary_path_fail", 1, NULL, 0);
fiu_enable("check_config_meta_uri_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_secondary_path_fail");
fiu_enable("check_config_backend_url_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_backend_url_fail");
fiu_disable("check_config_meta_uri_fail");
fiu_enable("check_config_archive_disk_threshold_fail", 1, NULL, 0);
s = config.ValidateConfig();
@ -835,10 +803,10 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) {
fiu_disable("check_config_enable_monitor_fail");
/* cache config */
fiu_enable("check_config_cpu_cache_capacity_fail", 1, NULL, 0);
fiu_enable("check_config_cache_size_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_cpu_cache_capacity_fail");
fiu_disable("check_config_cache_size_fail");
fiu_enable("check_config_cpu_cache_threshold_fail", 1, NULL, 0);
s = config.ValidateConfig();
@ -877,25 +845,25 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) {
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_enable_fail");
fiu_enable("check_gpu_resource_config_cache_capacity_fail", 1, NULL, 0);
fiu_enable("check_gpu_cache_size_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_cache_capacity_fail");
fiu_disable("check_gpu_cache_size_fail");
fiu_enable("check_config_gpu_resource_cache_threshold_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_cache_threshold_fail");
fiu_enable("check_gpu_resource_config_search_fail", 1, NULL, 0);
fiu_enable("check_gpu_search_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_search_fail");
fiu_disable("check_gpu_search_fail");
fiu_enable("check_gpu_resource_config_build_index_fail", 1, NULL, 0);
fiu_enable("check_gpu_build_index_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_build_index_fail");
fiu_disable("check_gpu_build_index_fail");
#endif
fiu_enable("get_config_json_config_path_fail", 1, NULL, 0);
@ -992,36 +960,16 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) {
fiu_disable("check_wal_path_fail");
/* logs config */
fiu_enable("check_logs_level_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_level_fail");
fiu_enable("check_logs_trace_enable_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_trace_enable_fail");
fiu_enable("check_logs_debug_enable_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_debug_enable_fail");
fiu_enable("check_logs_info_enable_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_info_enable_fail");
fiu_enable("check_logs_warning_enable_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_warning_enable_fail");
fiu_enable("check_logs_error_enable_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_error_enable_fail");
fiu_enable("check_logs_fatal_enable_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_fatal_enable_fail");
fiu_enable("check_logs_path_fail", 1, NULL, 0);
s = config.ValidateConfig();
ASSERT_FALSE(s.ok());
@ -1050,41 +998,36 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) {
ASSERT_TRUE(s.ok());
/* server config */
fiu_enable("check_config_address_fail", 1, NULL, 0);
fiu_enable("check_config_bind_address_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_address_fail");
fiu_disable("check_config_bind_address_fail");
fiu_enable("check_config_port_fail", 1, NULL, 0);
fiu_enable("check_config_bind_port_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_port_fail");
fiu_disable("check_config_bind_port_fail");
fiu_enable("check_config_deploy_mode_fail", 1, NULL, 0);
fiu_enable("check_config_cluster_role_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_deploy_mode_fail");
fiu_disable("check_config_cluster_role_fail");
fiu_enable("check_config_time_zone_fail", 1, NULL, 0);
fiu_enable("check_config_timezone_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_time_zone_fail");
fiu_disable("check_config_timezone_fail");
/* db config */
fiu_enable("check_config_primary_path_fail", 1, NULL, 0);
fiu_enable("check_config_path_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_primary_path_fail");
fiu_disable("check_config_path_fail");
fiu_enable("check_config_secondary_path_fail", 1, NULL, 0);
fiu_enable("check_config_meta_uri_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_secondary_path_fail");
fiu_enable("check_config_backend_url_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_backend_url_fail");
fiu_disable("check_config_meta_uri_fail");
fiu_enable("check_config_preload_collection_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
@ -1119,10 +1062,10 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) {
fiu_disable("check_config_enable_monitor_fail");
/* cache config */
fiu_enable("check_config_cpu_cache_capacity_fail", 1, NULL, 0);
fiu_enable("check_config_cache_size_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_cpu_cache_capacity_fail");
fiu_disable("check_config_cache_size_fail");
fiu_enable("check_config_cpu_cache_threshold_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
@ -1161,25 +1104,25 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) {
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_enable_fail");
fiu_enable("check_gpu_resource_config_cache_capacity_fail", 1, NULL, 0);
fiu_enable("check_gpu_cache_size_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_cache_capacity_fail");
fiu_disable("check_gpu_cache_size_fail");
fiu_enable("check_config_gpu_resource_cache_threshold_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_gpu_resource_cache_threshold_fail");
fiu_enable("check_gpu_resource_config_search_fail", 1, NULL, 0);
fiu_enable("check_gpu_search_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_search_fail");
fiu_disable("check_gpu_search_fail");
fiu_enable("check_gpu_resource_config_build_index_fail", 1, NULL, 0);
fiu_enable("check_gpu_build_index_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_gpu_resource_config_build_index_fail");
fiu_disable("check_gpu_build_index_fail");
#endif
s = config.ResetDefaultConfig();
@ -1207,36 +1150,16 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) {
fiu_disable("check_wal_path_fail");
/* logs config */
fiu_enable("check_logs_level_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_level_fail");
fiu_enable("check_logs_trace_enable_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_trace_enable_fail");
fiu_enable("check_logs_debug_enable_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_debug_enable_fail");
fiu_enable("check_logs_info_enable_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_info_enable_fail");
fiu_enable("check_logs_warning_enable_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_warning_enable_fail");
fiu_enable("check_logs_error_enable_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_error_enable_fail");
fiu_enable("check_logs_fatal_enable_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_logs_fatal_enable_fail");
fiu_enable("check_logs_path_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
@ -1358,18 +1281,12 @@ TEST_F(ConfigTest, SERVER_CONFIG_UPDATE_TEST) {
ASSERT_EQ("false", yaml_value);
// test path
cmd_set = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH, "/tmp/milvus_config_unittest");
cmd_set = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PATH, "/tmp/milvus_config_unittest");
ASSERT_TRUE(config.ProcessConfigCli(reply_set, cmd_set).ok());
ASSERT_TRUE(lambda(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH,
ms::CONFIG_STORAGE_PRIMARY_PATH_DEFAULT, yaml_value).ok());
ASSERT_TRUE(lambda(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PATH,
ms::CONFIG_STORAGE_PATH_DEFAULT, yaml_value).ok());
ASSERT_EQ("/tmp/milvus_config_unittest", yaml_value);
cmd_set = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_SECONDARY_PATH, "/home/zilliz,/home/milvus");
ASSERT_TRUE(config.ProcessConfigCli(reply_set, cmd_set).ok());
ASSERT_TRUE(lambda(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_SECONDARY_PATH,
ms::CONFIG_STORAGE_SECONDARY_PATH_DEFAULT, yaml_value).ok());
ASSERT_EQ("/home/zilliz,/home/milvus", yaml_value);
#ifdef MILVUS_GPU_VERSION
cmd_set = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, "gpu0");
ASSERT_TRUE(config.ProcessConfigCli(reply_set, cmd_set).ok());

7
core/unittest/server/test_rpc.cpp
View File

@ -84,14 +84,13 @@ class RpcHandlerTest : public testing::Test {
milvus::engine::DBOptions opt;
milvus::server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/");
milvus::server::Config::GetInstance().SetGeneralConfigMetaURI("sqlite://:@:/");
milvus::server::Config::GetInstance().SetDBConfigArchiveDiskThreshold("");
milvus::server::Config::GetInstance().SetDBConfigArchiveDaysThreshold("");
milvus::server::Config::GetInstance().SetStorageConfigPrimaryPath("/tmp/milvus_test");
milvus::server::Config::GetInstance().SetStorageConfigSecondaryPath("");
milvus::server::Config::GetInstance().SetStorageConfigPath("/tmp/milvus_test");
milvus::server::Config::GetInstance().SetCacheConfigCacheInsertData("");
milvus::server::Config::GetInstance().SetEngineConfigOmpThreadNum("");
milvus::server::Config::GetInstance().SetServerConfigPort("19531");
milvus::server::Config::GetInstance().SetNetworkConfigBindPort("19531");
// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster");
// DBWrapper::GetInstance().GetInstance().StartService();

12
core/unittest/server/test_util.cpp
View File

@ -249,12 +249,14 @@ TEST(UtilTest, LOG_TEST) {
fiu_init(0);
fiu_enable("LogUtil.InitLog.set_max_log_size_small_than_min", 1, NULL, 0);
auto status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10);
auto status = milvus::server::InitLog(true, true, true, true, true, true,
"/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB
ASSERT_FALSE(status.ok());
fiu_disable("LogUtil.InitLog.set_max_log_size_small_than_min");
fiu_enable("LogUtil.InitLog.delete_exceeds_small_than_min", 1, NULL, 0);
status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10);
status = milvus::server::InitLog(true, true, true, true, true, true,
"/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB
ASSERT_FALSE(status.ok());
fiu_disable("LogUtil.InitLog.delete_exceeds_small_than_min");
@ -264,7 +266,8 @@ TEST(UtilTest, LOG_TEST) {
fiu_enable("LogUtil.InitLog.trace_enable_to_false", 1, NULL, 0);
fiu_enable("LogUtil.InitLog.error_enable_to_false", 1, NULL, 0);
fiu_enable("LogUtil.InitLog.fatal_enable_to_false", 1, NULL, 0);
status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10);
status = milvus::server::InitLog(true, true, true, true, true, true,
"/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB
ASSERT_TRUE(status.ok()) << status.message();
fiu_disable("LogUtil.InitLog.fatal_enable_to_false");
fiu_disable("LogUtil.InitLog.error_enable_to_false");
@ -273,7 +276,8 @@ TEST(UtilTest, LOG_TEST) {
fiu_disable("LogUtil.InitLog.debug_enable_to_false");
fiu_disable("LogUtil.InitLog.info_enable_to_false");
status = milvus::server::InitLog(true, true, true, true, true, true, "/tmp/test_util", 1024, 10);
status = milvus::server::InitLog(true, true, true, true, true, true,
"/tmp/test_util", 1024 * 1024 * 1024, 10); // 1024 MB
ASSERT_TRUE(status.ok()) << status.message();
EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer));

160
core/unittest/server/test_web.cpp
View File

@ -142,109 +142,64 @@ namespace {
static const char* CONTROLLER_TEST_VALID_CONFIG_STR =
"# Default values are used when you make no changes to the following parameters.\n"
"\n"
"version: 0.4\n"
"version: 0.5\n"
"\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"# Server Config | Description | Type | Default "
" |\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"server_config:\n"
" address: 0.0.0.0\n"
" port: 19530\n"
" deploy_mode: single\n"
" time_zone: UTC+8\n"
" web_enable: true\n"
" web_port: 19121\n"
"cluster:\n"
" enable: false\n"
" role: rw\n"
"\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"# DataBase Config | Description | Type | Default "
" |\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"db_config:\n"
" backend_url: sqlite://:@:/\n"
"general:\n"
" timezone: UTC+8\n"
" meta_uri: sqlite://:@:/\n"
"\n"
"network:\n"
" bind.address: 0.0.0.0\n"
" bind.port: 19530\n"
" http.enable: true\n"
" http.port: 19121\n"
"\n"
"storage:\n"
" path: /tmp/milvus\n"
" auto_flush_interval: 1\n"
"\n"
"wal:\n"
" enable: true\n"
" recovery_error_ignore: false\n"
" buffer_size: 256MB\n"
" path: /tmp/milvus/wal\n"
"\n"
"cache:\n"
" cache_size: 4GB\n"
" insert_buffer_size: 1GB\n"
" preload_collection:\n"
"\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"# Storage Config | Description | Type | Default "
" |\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"storage_config:\n"
" primary_path: /tmp/milvus\n"
" secondary_path:\n"
" s3_enable: false\n"
" s3_address: 127.0.0.1\n"
" s3_port: 9000\n"
" s3_access_key: minioadmin\n"
" s3_secret_key: minioadmin\n"
" s3_bucket: milvus-bucket\n"
"gpu:\n"
" enable: true\n"
" cache_size: 1GB\n"
" gpu_search_threshold: 1000\n"
" search_devices:\n"
" - gpu0\n"
" build_index_devices:\n"
" - gpu0\n"
"\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"# Metric Config | Description | Type | Default "
" |\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"metric_config:\n"
" enable_monitor: false\n"
"logs:\n"
" level: debug\n"
" trace.enable: true\n"
" path: /tmp/milvus/logs\n"
" max_log_file_size: 1024MB\n"
" log_rotate_num: 0\n"
"\n"
"metric:\n"
" enable: false\n"
" address: 127.0.0.1\n"
" port: 9091\n"
"\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"# Cache Config | Description | Type | Default "
" |\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"cache_config:\n"
" cpu_cache_capacity: 4\n"
" insert_buffer_size: 1\n"
" cache_insert_data: false\n"
"\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"# Engine Config | Description | Type | Default "
" |\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"engine_config:\n"
" use_blas_threshold: 1100\n"
#ifdef MILVUS_GPU_VERSION
" gpu_search_threshold: 1000\n"
"\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"# GPU Resource Config | Description | Type | Default "
" |\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"gpu_resource_config:\n"
" enable: true\n"
" cache_capacity: 1\n"
" search_resources:\n"
" - gpu0\n"
" build_index_resources:\n"
" - gpu0\n"
#endif
"\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"# Tracing Config | Description | Type | Default "
" |\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
"tracing_config:\n"
" json_config_path:\n"
"";
"\n";
} // namespace
static const char* CONTROLLER_TEST_COLLECTION_NAME = "controller_unit_test";
static const char* CONTROLLER_TEST_CONFIG_DIR = "/tmp/milvus_web_controller_test/";
static const char* CONTROLLER_TEST_CONFIG_WAL_DIR = "/tmp/milvus_web_controller_test/wal";
static const char* CONTROLLER_TEST_CONFIG_FILE = "config.yaml";
class TestClient : public oatpp::web::client::ApiClient {
@ -382,13 +337,14 @@ class WebControllerTest : public ::testing::Test {
milvus::engine::DBOptions opt;
milvus::server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/");
milvus::server::Config::GetInstance().SetGeneralConfigMetaURI("sqlite://:@:/");
boost::filesystem::remove_all(CONTROLLER_TEST_CONFIG_DIR);
milvus::server::Config::GetInstance().SetStorageConfigPrimaryPath(CONTROLLER_TEST_CONFIG_DIR);
milvus::server::Config::GetInstance().SetStorageConfigPath(CONTROLLER_TEST_CONFIG_DIR);
milvus::server::Config::GetInstance().SetWalConfigWalPath(CONTROLLER_TEST_CONFIG_WAL_DIR);
milvus::server::DBWrapper::GetInstance().StartService();
milvus::server::Config::GetInstance().SetServerConfigWebPort("29999");
milvus::server::Config::GetInstance().SetNetworkConfigHTTPPort("29999");
milvus::server::web::WebServer::GetInstance().Start();
@ -1421,11 +1377,11 @@ TEST_F(WebControllerTest, CONFIG) {
OString collection_name_s = "milvus_test_webcontroller_test_preload_collection_s";
GenCollection(client_ptr, conncetion_ptr, collection_name_s, 16, 10, "L2");
OString body_str = "{\"db_config\": {\"preload_collection\": \"" + collection_name + "\"}}";
OString body_str = "{\"cache\": {\"preload_collection\": \"" + collection_name + "\"}}";
response = client_ptr->op("config", body_str, conncetion_ptr);
ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str();
body_str = "{\"db_config\": {\"preload_collection\": \"" + collection_name + "," + collection_name_s + "\"}}";
body_str = "{\"cache\": {\"preload_collection\": \"" + collection_name + "," + collection_name_s + "\"}}";
response = client_ptr->op("config", body_str, conncetion_ptr);
ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str();
auto set_result_json = nlohmann::json::parse(response->readBodyToString()->c_str());
@ -1467,7 +1423,7 @@ TEST_F(WebControllerTest, ADVANCED_CONFIG) {
auto config_dto = milvus::server::web::AdvancedConfigDto::createShared();
response = client_ptr->setAdvanced(config_dto, conncetion_ptr);
ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode());
ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str();
config_dto->cpu_cache_capacity = 3;
response = client_ptr->setAdvanced(config_dto, conncetion_ptr);
@ -1489,7 +1445,7 @@ TEST_F(WebControllerTest, ADVANCED_CONFIG) {
// test fault
// cpu cache capacity exceed total memory
config_dto->cpu_cache_capacity = 10000000;
config_dto->cpu_cache_capacity = 10000L * (1024L * 1024 * 1024); // 10000 GB
response = client_ptr->setAdvanced(config_dto, conncetion_ptr);
ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode());
}
@ -1543,10 +1499,10 @@ TEST_F(WebControllerTest, GPU_CONFIG) {
ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode());
//// test fault config
// cache capacity exceed GPU mem size
gpu_config_dto->cache_capacity = 100000;
// cache capacity exceed GPU mem size (GiB)
gpu_config_dto->cache_capacity = 100000L * 1024 * 1024 * 1024; // 100000 GiB
response = client_ptr->setGPUConfig(gpu_config_dto, conncetion_ptr);
ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode());
ASSERT_EQ(OStatus::CODE_400.code, response->getStatusCode()) << response->readBodyToString()->c_str();
gpu_config_dto->cache_capacity = 1;
// duplicate resources

59
core/unittest/server/utils.cpp
View File

@ -19,6 +19,64 @@
namespace {
static const char* VALID_CONFIG_STR =
"# Default values are used when you make no changes to the following parameters.\n"
"\n"
"version: 0.5\n"
"\n"
"cluster:\n"
" enable: false\n"
" role: rw\n"
"\n"
"general:\n"
" timezone: UTC+8\n"
" meta_uri: sqlite://:@:/\n"
"\n"
"network:\n"
" bind.address: 0.0.0.0\n"
" bind.port: 19530\n"
" http.enable: true\n"
" http.port: 19121\n"
"\n"
"storage:\n"
" path: /tmp/milvus\n"
" auto_flush_interval: 1\n"
"\n"
"wal:\n"
" enable: true\n"
" recovery_error_ignore: false\n"
" buffer_size: 256MB\n"
" path: /tmp/milvus/wal\n"
"\n"
"cache:\n"
" cache_size: 4GB\n"
" insert_buffer_size: 1GB\n"
" preload_collection:\n"
"\n"
"gpu:\n"
" enable: true\n"
" cache_size: 1GB\n"
" gpu_search_threshold: 1000\n"
" search_devices:\n"
" - gpu0\n"
" build_index_devices:\n"
" - gpu0\n"
"\n"
"logs:\n"
" level: debug\n"
" trace.enable: true\n"
" path: /tmp/milvus/logs\n"
" max_log_file_size: 1024MB\n"
" log_rotate_num: 0\n"
"\n"
"metric:\n"
" enable: false\n"
" address: 127.0.0.1\n"
" port: 9091\n"
"\n";
/*
static const char* VALID_CONFIG_STR =
"# Default values are used when you make no changes to the following parameters.\n"
"\n"
@ -84,6 +142,7 @@ static const char* VALID_CONFIG_STR =
" max_log_file_size: 256\n"
" delete_exceeds: 10\n"
"";
*/
static const char* INVALID_CONFIG_STR = "*INVALID*";

1
tests/milvus_python_test/test_compact.py
View File

@ -378,6 +378,7 @@ class TestCompactBase:
assert status.OK()
@pytest.mark.timeout(COMPACT_TIMEOUT)
@pytest.mark.repeat(10)
def test_index_creation_after_compact(self, connect, collection, get_simple_index):
'''
target: test index creation after compact

1067
tests/milvus_python_test/test_config.py
View File

File diff suppressed because it is too large Load Diff

8
tests/milvus_python_test/utils.py
View File

@ -38,15 +38,15 @@ def get_milvus(host, port, uri=None, handler=None, **kwargs):
def disable_flush(connect):
status, reply = connect.set_config("db_config", "auto_flush_interval", big_flush_interval)
status, reply = connect.set_config("storage", "auto_flush_interval", big_flush_interval)
assert status.OK()
def enable_flush(connect):
# reset auto_flush_interval=1
status, reply = connect.set_config("db_config", "auto_flush_interval", default_flush_interval)
status, reply = connect.set_config("storage", "auto_flush_interval", default_flush_interval)
assert status.OK()
status, config_value = connect.get_config("db_config", "auto_flush_interval")
status, config_value = connect.get_config("storage", "auto_flush_interval")
assert status.OK()
assert config_value == str(default_flush_interval)
@ -470,7 +470,7 @@ def gen_invalid_cache_config():
return invalid_configs
def gen_invalid_engine_config():
def gen_invalid_gpu_config():
invalid_configs = [
-1,
[1,2,3],