test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-04 12:59:23 +08:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

fix: [2.4] Write padding into mmap file in case of SIGBUS (#34443) (#34455)

Browse Source 
Cherry-pick from master
pr: #34443
See also #34442

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
This commit is contained in:
congqixia 2024-07-08 10:06:19 +08:00 committed by GitHub
parent 326370c1be
commit 014820e9d2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 45 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
internal/core/src/mmap/Column.h
View File

@ -129,7 +129,9 @@ class ColumnBase {
size_ = size; size_ = size;
cap_size_ = size; cap_size_ = size;
size_t mapped_size = cap_size_ + padding_; // use exactly same size of file, padding shall be written in file already
// see also https://github.com/milvus-io/milvus/issues/34442
size_t mapped_size = cap_size_;
data_ = static_cast<char*>(mmap( data_ = static_cast<char*>(mmap(
nullptr, mapped_size, PROT_READ, MAP_SHARED, file.Descriptor(), 0)); nullptr, mapped_size, PROT_READ, MAP_SHARED, file.Descriptor(), 0));
AssertInfo(data_ != MAP_FAILED, AssertInfo(data_ != MAP_FAILED,
@ -156,7 +158,9 @@ class ColumnBase {
mapping_type_(MappingType::MAP_WITH_FILE) { mapping_type_(MappingType::MAP_WITH_FILE) {
SetPaddingSize(data_type); SetPaddingSize(data_type);
size_t mapped_size = cap_size_ + padding_; // use exact same size of file, padding shall be written in file already
// see also https://github.com/milvus-io/milvus/issues/34442
size_t mapped_size = cap_size_;
data_ = static_cast<char*>(mmap( data_ = static_cast<char*>(mmap(
nullptr, mapped_size, PROT_READ, MAP_SHARED, file.Descriptor(), 0)); nullptr, mapped_size, PROT_READ, MAP_SHARED, file.Descriptor(), 0));
AssertInfo(data_ != MAP_FAILED, AssertInfo(data_ != MAP_FAILED,

39
internal/core/src/mmap/Utils.h
View File

@ -39,6 +39,33 @@ namespace milvus {
file.Path(), \ file.Path(), \
strerror(errno))); strerror(errno)));
/*
* If string field's value all empty, need a string padding to avoid
* mmap failing because size_ is zero which causing invalid arguement
* array has the same problem
* TODO: remove it when support NULL value
*/
constexpr size_t FILE_STRING_PADDING = 1;
constexpr size_t FILE_ARRAY_PADDING = 1;
inline size_t
PaddingSize(const DataType& type) {
switch (type) {
case DataType::JSON:
// simdjson requires a padding following the json data
return simdjson::SIMDJSON_PADDING;
case DataType::VARCHAR:
case DataType::STRING:
return FILE_STRING_PADDING;
break;
case DataType::ARRAY:
return FILE_ARRAY_PADDING;
default:
break;
}
return 0;
}
inline void inline void
WriteFieldData(File& file, WriteFieldData(File& file,
DataType data_type, DataType data_type,
@ -128,5 +155,17 @@ WriteFieldData(File& file,
total_written += data->Size(i); total_written += data->Size(i);
} }
} }
// write padding 0 in file content directly
// see also https://github.com/milvus-io/milvus/issues/34442
auto padding_size = PaddingSize(data_type);
if (padding_size > 0 ) {
std::vector<char> padding(padding_size, 0);
ssize_t written = file.Write(padding.data(), padding_size);
if (written < padding_size) {
THROW_FILE_WRITE_ERROR
}
total_written += written;
}
} }
} // namespace milvus } // namespace milvus