test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-03 12:29:36 +08:00
Code Issues Actions 13 Packages Projects Releases Wiki Activity

enhance: [2.4]reduce mmap-rss after warmup (#35965)

Browse Source 
related pr: https://github.com/milvus-io/milvus/pull/35974

Signed-off-by: cqy123456 <qianya.cheng@zilliz.com>
This commit is contained in:
cqy123456 2024-09-05 17:41:04 +08:00 committed by GitHub
parent b578064869
commit 9de431991f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 24 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -152,8 +152,9 @@ SegmentSealedImpl::WarmupChunkCache(const FieldId field_id) {
auto field_info = it->second;
auto cc = storage::MmapManager::GetInstance().GetChunkCache();
const bool mmap_rss_not_need = true;
for (const auto& data_path : field_info.insert_files) {
auto column = cc->Read(data_path, mmap_descriptor_);
auto column = cc->Read(data_path, mmap_descriptor_, mmap_rss_not_need);
}
}

21
internal/core/src/storage/ChunkCache.cpp
View File

@ -22,7 +22,8 @@
namespace milvus::storage {
std::shared_ptr<ColumnBase>
ChunkCache::Read(const std::string& filepath,
const MmapChunkDescriptorPtr& descriptor) {
const MmapChunkDescriptorPtr& descriptor,
const bool mmap_rss_not_need) {
// use rlock to get future
{
std::shared_lock lck(mutex_);
@ -61,6 +62,22 @@ ChunkCache::Read(const std::string& filepath,
try {
field_data = DownloadAndDecodeRemoteFile(cm_.get(), filepath);
column = Mmap(field_data->GetFieldData(), descriptor);
if (mmap_rss_not_need) {
auto ok = madvise(reinterpret_cast<void*>(
const_cast<char*>(column->MmappedData())),
column->ByteSize(),
ReadAheadPolicy_Map["dontneed"]);
if (ok != 0) {
LOG_WARN(
"failed to madvise to the data file {}, addr {}, size {}, "
"err: "
"{}",
filepath,
static_cast<const void*>(column->MmappedData()),
column->ByteSize(),
strerror(errno));
}
}
allocate_success = true;
} catch (const SegcoreError& e) {
err_code = e.get_error_code();
@ -110,7 +127,7 @@ ChunkCache::Prefetch(const std::string& filepath) {
LOG_WARN(
"failed to madvise to the data file {}, addr {}, size {}, err: {}",
filepath,
column->MmappedData(),
static_cast<const void*>(column->MmappedData()),
column->ByteSize(),
strerror(errno));
}

4
internal/core/src/storage/ChunkCache.h
View File

@ -45,7 +45,9 @@ class ChunkCache {
public:
std::shared_ptr<ColumnBase>
Read(const std::string& filepath, const MmapChunkDescriptorPtr& descriptor);
Read(const std::string& filepath,
const MmapChunkDescriptorPtr& descriptor,
const bool mmap_rss_not_need = false);
void
Remove(const std::string& filepath);