test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-30 19:08:30 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

enhance: Add precheck when chunk manager init (#28330)

Browse Source 
#28329

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
zhagnlu 2023-11-23 19:56:32 +08:00 committed by GitHub
parent da339535d5
commit 0d9d098186
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 57 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
internal/core/src/storage/ChunkManager.cpp
View File

@ -97,6 +97,8 @@ AwsChunkManager::AwsChunkManager(const StorageConfig& storage_config) {
BuildAccessKeyClient(storage_config, config);
}
PreCheck(storage_config);
LOG_SEGCORE_INFO_ << "init AwsChunkManager with parameter[endpoint: '"
<< storage_config.address << "', default_bucket_name:'"
<< storage_config.bucket_name << "', root_path:'"
@ -131,6 +133,8 @@ GcpChunkManager::GcpChunkManager(const StorageConfig& storage_config) {
BuildAccessKeyClient(storage_config, config);
}
PreCheck(storage_config);
LOG_SEGCORE_INFO_ << "init GcpChunkManager with parameter[endpoint: '"
<< storage_config.address << "', default_bucket_name:'"
<< storage_config.bucket_name << "', root_path:'"
@ -169,6 +173,8 @@ AliyunChunkManager::AliyunChunkManager(const StorageConfig& storage_config) {
BuildAccessKeyClient(mutable_config, config);
}
PreCheck(storage_config);
LOG_SEGCORE_INFO_ << "init AliyunChunkManager with parameter[endpoint: '"
<< storage_config.address << "', default_bucket_name:'"
<< storage_config.bucket_name << "', root_path:'"

23
internal/core/src/storage/MinioChunkManager.cpp
View File

@ -217,6 +217,27 @@ MinioChunkManager::BuildS3Client(
}
}
void
MinioChunkManager::PreCheck(const StorageConfig& config) {
LOG_SEGCORE_INFO_ << "start to precheck chunk manager with configuration:"
<< config.ToString();
try {
// Just test connection not check real list, avoid cost resource.
ListWithPrefix("justforconnectioncheck");
} catch (SegcoreError& e) {
auto err_message = fmt::format(
"precheck chunk manager client failed, "
"error:{}, "
"configuration:{}",
e.what(),
config.ToString());
LOG_SEGCORE_ERROR_ << err_message;
throw SegcoreError(S3Error, err_message);
} catch (std::exception& e) {
throw e;
}
};
void
MinioChunkManager::BuildAccessKeyClient(
const StorageConfig& storage_config,
@ -322,6 +343,8 @@ MinioChunkManager::MinioChunkManager(const StorageConfig& storage_config)
BuildGoogleCloudClient(storage_config, config);
}
PreCheck(storage_config);
LOG_SEGCORE_INFO_ << "init MinioChunkManager with parameter[endpoint: '"
<< storage_config.address << "', default_bucket_name:'"
<< storage_config.bucket_name << "', root_path:'"

7
internal/core/src/storage/MinioChunkManager.h
View File

@ -133,7 +133,7 @@ class MinioChunkManager : public ChunkManager {
Write(const std::string& filepath, void* buf, uint64_t len);
virtual std::vector<std::string>
ListWithPrefix(const std::string& filepath);
ListWithPrefix(const std::string& filepath = "");
virtual void
Remove(const std::string& filepath);
@ -200,6 +200,11 @@ class MinioChunkManager : public ChunkManager {
InitSDKAPI(RemoteStorageType type,
bool useIAM,
const std::string& log_level);
// Precheck whether client is configure ready or not.
void
PreCheck(const StorageConfig& storage_config);
void
ShutdownSDKAPI();
void

15
internal/core/src/storage/Types.h
View File

@ -96,6 +96,21 @@ struct StorageConfig {
bool useIAM = false;
bool useVirtualHost = false;
int64_t requestTimeoutMs = 3000;
std::string
ToString() const {
std::stringstream ss;
ss << "[address=" << address << ", bucket_name=" << bucket_name
<< ", root_path=" << root_path << ", storage_type=" << storage_type
<< ", cloud_provider=" << cloud_provider
<< ", iam_endpoint=" << iam_endpoint << ", log_level=" << log_level
<< ", region=" << region << ", useSSL=" << std::boolalpha << useSSL
<< ", useIAM=" << std::boolalpha << useIAM
<< ", useVirtualHost=" << std::boolalpha << useVirtualHost
<< ", requestTimeoutMs=" << requestTimeoutMs << "]";
return ss.str();
}
};
} // namespace milvus::storage

7
internal/core/unittest/test_minio_chunk_manager.cpp
View File

@ -83,6 +83,13 @@ class MinioChunkManagerTest : public testing::Test {
// MinioChunkManagerPtr chunk_manager_;
//};
TEST_F(MinioChunkManagerTest, InitFailed) {
auto configs = StorageConfig{};
// wrong address
configs.address = "1.2.3.4:9000";
EXPECT_THROW(std::make_unique<MinioChunkManager>(configs), SegcoreError);
}
TEST_F(MinioChunkManagerTest, BucketPositive) {
string testBucketName = "test-bucket";
chunk_manager_->SetBucketName(testBucketName);