test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-29 18:38:44 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

fix: wrong path spelling when use rootpath in segcore (#37453)

Browse Source 
#36532

Signed-off-by: lixinguo <xinguo.li@zilliz.com>
Co-authored-by: lixinguo <xinguo.li@zilliz.com>
This commit is contained in:
smellthemoon 2024-11-07 11:22:25 +08:00 committed by GitHub
parent b4c749dcd5
commit 9b6dd23f8e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 72 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
internal/core/src/clustering/KmeansClustering.h
View File

@ -17,9 +17,11 @@
#pragma once
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "boost/filesystem/path.hpp"
#include "storage/MemFileManagerImpl.h"
#include "pb/clustering.pb.h"
#include "knowhere/cluster/cluster_factory.h"
@ -61,27 +63,32 @@ class KmeansClustering {
GetRemoteCentroidsObjectPrefix() const {
auto index_meta_ = file_manager_->GetIndexMeta();
auto field_meta_ = file_manager_->GetFieldDataMeta();
return file_manager_->GetChunkManager()->GetRootPath() + "/" +
std::string(ANALYZE_ROOT_PATH) + "/" +
std::to_string(index_meta_.build_id) + "/" +
std::to_string(index_meta_.index_version) + "/" +
std::to_string(field_meta_.collection_id) + "/" +
std::to_string(field_meta_.partition_id) + "/" +
std::to_string(field_meta_.field_id);
boost::filesystem::path prefix =
file_manager_->GetChunkManager()->GetRootPath();
boost::filesystem::path path =
std::to_string(index_meta_.build_id) + "/" +
std::to_string(index_meta_.index_version) + "/" +
std::to_string(field_meta_.collection_id) + "/" +
std::to_string(field_meta_.partition_id) + "/" +
std::to_string(field_meta_.field_id);
return (prefix / path).string();
}
inline std::string
GetRemoteCentroidIdMappingObjectPrefix(int64_t segment_id) const {
auto index_meta_ = file_manager_->GetIndexMeta();
auto field_meta_ = file_manager_->GetFieldDataMeta();
return file_manager_->GetChunkManager()->GetRootPath() + "/" +
std::string(ANALYZE_ROOT_PATH) + "/" +
std::to_string(index_meta_.build_id) + "/" +
std::to_string(index_meta_.index_version) + "/" +
std::to_string(field_meta_.collection_id) + "/" +
std::to_string(field_meta_.partition_id) + "/" +
std::to_string(field_meta_.field_id) + "/" +
std::to_string(segment_id);
boost::filesystem::path prefix =
file_manager_->GetChunkManager()->GetRootPath();
boost::filesystem::path path = std::string(ANALYZE_ROOT_PATH);
boost::filesystem::path path1 =
std::to_string(index_meta_.build_id) + "/" +
std::to_string(index_meta_.index_version) + "/" +
std::to_string(field_meta_.collection_id) + "/" +
std::to_string(field_meta_.partition_id) + "/" +
std::to_string(field_meta_.field_id) + "/" +
std::to_string(segment_id);
return (prefix / path / path1).string();
}
~KmeansClustering() = default;

14
internal/core/src/index/InvertedIndexTantivy.cpp
View File

@ -188,15 +188,15 @@ InvertedIndexTantivy<T>::Load(milvus::tracer::TraceContext ctx,
}),
files_value.end());
auto index_valid_data_file =
mem_file_manager_->GetRemoteIndexObjectPrefix() +
std::string("/index_null_offset");
auto it = std::find(
files_value.begin(), files_value.end(), index_valid_data_file);
auto it = std::find_if(
files_value.begin(), files_value.end(), [](const std::string& file) {
return file.substr(file.find_last_of('/') + 1) ==
"index_null_offset";
});
if (it != files_value.end()) {
files_value.erase(it);
std::vector<std::string> file;
file.push_back(index_valid_data_file);
file.push_back(*it);
files_value.erase(it);
auto index_datas = mem_file_manager_->LoadIndexToMemory(file);
AssembleIndexDatas(index_datas);
BinarySet binary_set;

1
internal/core/src/storage/ChunkManager.h
View File

@ -117,6 +117,7 @@ class ChunkManager {
/**
* @brief Get the Root Path
* @return std::string
* Note: when join path, please check the training '/'
*/
virtual std::string
GetRootPath() const = 0;

31
internal/core/src/storage/FileManager.h
View File

@ -21,6 +21,7 @@
#include <memory>
#include "common/Consts.h"
#include "boost/filesystem/path.hpp"
#include "knowhere/file_manager.h"
#include "log/Log.h"
#include "storage/ChunkManager.h"
@ -129,11 +130,14 @@ class FileManagerImpl : public knowhere::FileManager {
virtual std::string
GetRemoteIndexObjectPrefix() const {
return rcm_->GetRootPath() + "/" + std::string(INDEX_ROOT_PATH) + "/" +
std::to_string(index_meta_.build_id) + "/" +
std::to_string(index_meta_.index_version) + "/" +
std::to_string(field_meta_.partition_id) + "/" +
std::to_string(field_meta_.segment_id);
boost::filesystem::path prefix = rcm_->GetRootPath();
boost::filesystem::path path = std::string(INDEX_ROOT_PATH);
boost::filesystem::path path1 =
std::to_string(index_meta_.build_id) + "/" +
std::to_string(index_meta_.index_version) + "/" +
std::to_string(field_meta_.partition_id) + "/" +
std::to_string(field_meta_.segment_id);
return (prefix / path / path1).string();
}
virtual std::string
@ -147,13 +151,16 @@ class FileManagerImpl : public knowhere::FileManager {
virtual std::string
GetRemoteTextLogPrefix() const {
return rcm_->GetRootPath() + "/" + std::string(TEXT_LOG_ROOT_PATH) +
"/" + std::to_string(index_meta_.build_id) + "/" +
std::to_string(index_meta_.index_version) + "/" +
std::to_string(field_meta_.collection_id) + "/" +
std::to_string(field_meta_.partition_id) + "/" +
std::to_string(field_meta_.segment_id) + "/" +
std::to_string(field_meta_.field_id);
boost::filesystem::path prefix = rcm_->GetRootPath();
boost::filesystem::path path = std::string(TEXT_LOG_ROOT_PATH);
boost::filesystem::path path1 =
std::to_string(index_meta_.build_id) + "/" +
std::to_string(index_meta_.index_version) + "/" +
std::to_string(field_meta_.collection_id) + "/" +
std::to_string(field_meta_.partition_id) + "/" +
std::to_string(field_meta_.segment_id) + "/" +
std::to_string(field_meta_.field_id);
return (prefix / path / path1).string();
}
protected:

34
internal/core/src/storage/Util.cpp
View File

@ -502,8 +502,11 @@ std::string
GenIndexPathPrefix(ChunkManagerPtr cm,
int64_t build_id,
int64_t index_version) {
return cm->GetRootPath() + "/" + std::string(INDEX_ROOT_PATH) + "/" +
GenIndexPathIdentifier(build_id, index_version);
boost::filesystem::path prefix = cm->GetRootPath();
boost::filesystem::path path = std::string(INDEX_ROOT_PATH);
boost::filesystem::path path1 =
GenIndexPathIdentifier(build_id, index_version);
return (prefix / path / path1).string();
}
std::string
@ -512,29 +515,38 @@ GenTextIndexPathPrefix(ChunkManagerPtr cm,
int64_t index_version,
int64_t segment_id,
int64_t field_id) {
return cm->GetRootPath() + "/" + std::string(TEXT_LOG_ROOT_PATH) + "/" +
GenTextIndexPathIdentifier(
build_id, index_version, segment_id, field_id);
boost::filesystem::path prefix = cm->GetRootPath();
boost::filesystem::path path = std::string(TEXT_LOG_ROOT_PATH);
boost::filesystem::path path1 = GenTextIndexPathIdentifier(
build_id, index_version, segment_id, field_id);
return (prefix / path / path1).string();
}
std::string
GetIndexPathPrefixWithBuildID(ChunkManagerPtr cm, int64_t build_id) {
return cm->GetRootPath() + "/" + std::string(INDEX_ROOT_PATH) + "/" +
std::to_string(build_id);
boost::filesystem::path prefix = cm->GetRootPath();
boost::filesystem::path path = std::string(INDEX_ROOT_PATH);
boost::filesystem::path path1 = std::to_string(build_id);
return (prefix / path / path1).string();
}
std::string
GenFieldRawDataPathPrefix(ChunkManagerPtr cm,
int64_t segment_id,
int64_t field_id) {
return cm->GetRootPath() + "/" + std::string(RAWDATA_ROOT_PATH) + "/" +
std::to_string(segment_id) + "/" + std::to_string(field_id) + "/";
boost::filesystem::path prefix = cm->GetRootPath();
boost::filesystem::path path = std::string(RAWDATA_ROOT_PATH);
boost::filesystem::path path1 =
std::to_string(segment_id) + "/" + std::to_string(field_id) + "/";
return (prefix / path / path1).string();
}
std::string
GetSegmentRawDataPathPrefix(ChunkManagerPtr cm, int64_t segment_id) {
return cm->GetRootPath() + "/" + std::string(RAWDATA_ROOT_PATH) + "/" +
std::to_string(segment_id);
boost::filesystem::path prefix = cm->GetRootPath();
boost::filesystem::path path = std::string(RAWDATA_ROOT_PATH);
boost::filesystem::path path1 = std::to_string(segment_id);
return (prefix / path / path1).string();
}
std::unique_ptr<DataCodec>