Skip to content

Commit

Permalink
disable mlock (qdrant#1885)
Browse files Browse the repository at this point in the history
  • Loading branch information
generall authored May 12, 2023
1 parent e72e1ed commit ce2763f
Showing 1 changed file with 18 additions and 2 deletions.
20 changes: 18 additions & 2 deletions lib/segment/src/vector_storage/mmap_vectors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,15 @@ impl MmapVectors {
max_threads: usize,
stopped: &AtomicBool,
) -> OperationResult<()> {
self.lock_deleted_flags();
// In theory, we can lock deleted flags here, as we assume that it is the hottest data.
// Uncomment the following code if you want to do that:
//
// self.lock_deleted_flags();
//
// But it seems, that mlock functionality is not working properly if qdrant is running
// inside docker container with limited memory.
// Additionally, the speedup is not measured explicitly.

let vector_data_iterator = (0..self.num_vectors as u32).map(|i| {
let offset = self.data_offset(i as PointOffsetType).unwrap_or_default();
self.raw_vector_offset(offset)
Expand All @@ -107,7 +115,14 @@ impl MmapVectors {
distance: Distance,
) -> OperationResult<()> {
if QuantizedVectors::config_exists(data_path) {
self.lock_deleted_flags();
// In theory, we can lock deleted flags here, as we assume that it is the hottest data.
// Uncomment the following code if you want to do that:
//
// self.lock_deleted_flags();
//
// But it seems, that mlock functionality is not working properly if qdrant is running
// inside docker container with limited memory.
// Additionally, the speedup is not measured explicitly.
self.quantized_vectors = Some(QuantizedVectors::load(data_path, true, distance)?);
}
Ok(())
Expand Down Expand Up @@ -169,6 +184,7 @@ impl MmapVectors {
/// huge. Calling this will lock the deleted flags in memory to prevent this.
///
/// This is only supported on Unix.
#[allow(unused)]
fn lock_deleted_flags(&self) {
#[cfg(unix)]
if let Err(err) = self.deleted.mlock() {
Expand Down

0 comments on commit ce2763f

Please sign in to comment.