optimize sort by date and docid (#3849)

* add pre-sorting of splits and some infrastructure for incremential merge

* implement incremental collector

* rewrite leaf_search to allow for incremental merge

* use incremental collector in leaf search

* extract launching split search into a separate function

quickwit/quickwit-common/src/binary_heap.rs

@@ -93,6 +93,8 @@ where
         .collect();
     resulting_top_k
 }
+
+#[derive(Clone)]
 struct OrderItemPair<O: Ord, T> {
     order: O,
     item: T,
@@ -124,6 +126,7 @@ pub trait SortKeyMapper<Value> {
 }
 
 /// Progressively compute top-k.
+#[derive(Clone)]
 pub struct TopK<T, O: Ord, S> {
     heap: BinaryHeap<Reverse<OrderItemPair<O, T>>>,
     sort_key_mapper: S,

quickwit/quickwit-common/src/lib.rs

@@ -176,6 +176,17 @@ where T: Debug
     }
 }
 
+#[inline]
+pub const fn div_ceil(lhs: i64, rhs: i64) -> i64 {
+    let d = lhs / rhs;
+    let r = lhs % rhs;
+    if (r > 0 && rhs > 0) || (r < 0 && rhs < 0) {
+        d + 1
+    } else {
+        d
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::io::ErrorKind;
@@ -231,4 +242,23 @@ mod tests {
         let pretty_sample = PrettySample::new(&[1, 2, 3, 4], 2);
         assert_eq!(format!("{pretty_sample:?}"), "[1, 2, and 2 more]");
     }
+
+    #[test]
+    fn test_div_ceil() {
+        assert_eq!(div_ceil(5, 1), 5);
+        assert_eq!(div_ceil(5, 2), 3);
+        assert_eq!(div_ceil(6, 2), 3);
+
+        assert_eq!(div_ceil(-5, 1), -5);
+        assert_eq!(div_ceil(-5, 2), -2);
+        assert_eq!(div_ceil(-6, 2), -3);
+
+        assert_eq!(div_ceil(5, -1), -5);
+        assert_eq!(div_ceil(5, -2), -2);
+        assert_eq!(div_ceil(6, -2), -3);
+
+        assert_eq!(div_ceil(-5, -1), 5);
+        assert_eq!(div_ceil(-5, -2), 3);
+        assert_eq!(div_ceil(-6, -2), 3);
+    }
 }

quickwit/quickwit-proto/src/search/mod.rs

@@ -117,3 +117,14 @@ impl PartialOrd for SortValue {
         Some(self.cmp(other))
     }
 }
+
+impl PartialHit {
+    /// Helper to get access to the 1st sort value
+    pub fn sort_value(&self) -> Option<SortValue> {
+        if let Some(sort_value) = self.sort_value {
+            sort_value.sort_value
+        } else {
+            None
+        }
+    }
+}