Fix runtime information and live update for lazy join of two index scans (#2480)

Since #2256, the details of the runtime information of the index scans of a lazy join of two index scans were not properly updated anymore. In particular, `num-blocks-read` and `num-elements-read` were always zero, even after completion of the query. A live update of the runtime information of index scans in this scenario never worked so far. This is all fixed now. Together with #2473, live updates now fully work for any cascade of lazy joins. Here is simple example query, where we can now see live updates for all operations (the predicate must be large enough so that the joins are actually lazy):
```sparql
SELECT * {
  ?s a ?o1, ?o2, ?o3
}
```

Co-authored-by: RobinTF <[email protected]>

Author: hannahbast

src/engine/IndexScan.cpp

@@ -459,7 +459,8 @@ IndexScan::lazyScanForJoinOfColumnWithScan(
 }
 
 // _____________________________________________________________________________
-void IndexScan::updateRuntimeInfoForLazyScan(const LazyScanMetadata& metadata) {
+void IndexScan::updateRuntimeInfoForLazyScan(const LazyScanMetadata& metadata,
+                                             bool signalUpdate) {
   auto& rti = runtimeInfo();
   rti.status_ = RuntimeInformation::Status::lazilyMaterialized;
   rti.numRows_ = metadata.numElementsYielded_;
@@ -479,7 +480,9 @@ void IndexScan::updateRuntimeInfoForLazyScan(const LazyScanMetadata& metadata) {
   updateIfPositive(metadata.numBlocksPostprocessed_,
                    "num-blocks-postprocessed");
   updateIfPositive(metadata.numBlocksWithUpdate_, "num-blocks-with-update");
-  signalQueryUpdate();
+  if (signalUpdate) {
+    signalQueryUpdate();
+  }
 }
 
 // Store a Generator and its corresponding iterator as well as unconsumed values

src/engine/IndexScan.h

@@ -181,7 +181,8 @@ class IndexScan final : public Operation {
   // Set the runtime info of the `scanTree` when it was lazily executed during a
   // join.
   void updateRuntimeInfoForLazyScan(
-      const CompressedRelationReader::LazyScanMetadata& metadata);
+      const CompressedRelationReader::LazyScanMetadata& metadata,
+      bool signalUpdate = true);
 
   bool columnOriginatesFromGraphOrUndef(
       const Variable& variable) const override;

src/engine/Join.cpp

@@ -563,11 +563,44 @@ void Join::addCombinedRowToIdTable(const ROW_A& rowA, const ROW_B& rowB,
   }
 }
 
+// _____________________________________________________________________________
+namespace {
+// Type alias for the general InputRangeTypeErased with specific types.
+using IteratorWithSingleCol = InputRangeTypeErased<IdTableAndFirstCol<IdTable>>;
+
+// Convert a `CompressedRelationReader::IdTableGeneratorInputRange` to a
+// `InputRangeTypeErased<IdTableAndFirstCol<IdTable>>` for more efficient access
+// in the join columns below. This also makes sure the runtime information of
+// the passed `IndexScan` is updated properly as the range is consumed.
+IteratorWithSingleCol convertGenerator(
+    CompressedRelationReader::IdTableGeneratorInputRange gen, IndexScan& scan,
+    bool postUpdates) {
+  // Store the generator in a wrapper so we can access its details after moving
+  auto generatorStorage =
+      std::make_shared<CompressedRelationReader::IdTableGeneratorInputRange>(
+          std::move(gen));
+
+  auto range = CachingTransformInputRange(
+      *generatorStorage, [generatorStorage, &scan, postUpdates,
+                          first = true](auto& table) mutable {
+        scan.updateRuntimeInfoForLazyScan(generatorStorage->details(),
+                                          first || postUpdates);
+        first = false;
+        // IndexScans don't have a local vocabulary, so we can just use an empty
+        // one.
+        return IdTableAndFirstCol{std::move(table), LocalVocab{}};
+      });
+
+  return IteratorWithSingleCol{std::move(range)};
+}
+}  // namespace
+
 // ______________________________________________________________________________________________________
 Result Join::computeResultForTwoIndexScans(bool requestLaziness) const {
   return createResult(
       requestLaziness,
-      [this](std::function<void(IdTable&, LocalVocab&)> yieldTable) {
+      [this,
+       requestLaziness](std::function<void(IdTable&, LocalVocab&)> yieldTable) {
         auto leftScan =
             std::dynamic_pointer_cast<IndexScan>(_left->getRootOperation());
         auto rightScan =
@@ -585,15 +618,17 @@ Result Join::computeResultForTwoIndexScans(bool requestLaziness) const {
             IndexScan::lazyScanForJoinOfTwoScans(*leftScan, *rightScan);
         runtimeInfo().addDetail("time-for-filtering-blocks", timer.msecs());
 
-        auto leftBlocks = convertGenerator(std::move(leftBlocksInternal));
-        auto rightBlocks = convertGenerator(std::move(rightBlocksInternal));
+        // If requestLaziness, we don't need to serialize json for every update
+        // of the child. If we serialize it whenever the join operation yields a
+        // table that's frequent enough and reduces the overhead.
+        auto leftBlocks = convertGenerator(std::move(leftBlocksInternal),
+                                           *leftScan, !requestLaziness);
+        auto rightBlocks = convertGenerator(std::move(rightBlocksInternal),
+                                            *rightScan, !requestLaziness);
 
         ad_utility::zipperJoinForBlocksWithoutUndef(leftBlocks, rightBlocks,
                                                     std::less{}, rowAdder);
 
-        leftScan->updateRuntimeInfoForLazyScan(leftBlocks.details());
-        rightScan->updateRuntimeInfoForLazyScan(rightBlocks.details());
-
         auto localVocab = std::move(rowAdder.localVocab());
         return Result::IdTableVocabPair{std::move(rowAdder).resultTable(),
                                         std::move(localVocab)};
@@ -611,7 +646,7 @@ Result Join::computeResultForIndexScanAndIdTable(
   auto resultPermutation = joinColMap.permutationResult();
   return createResult(
       requestLaziness,
-      [this, scan = std::move(scan),
+      [this, requestLaziness, scan = std::move(scan),
        resultWithIdTable = std::move(resultWithIdTable),
        joinColMap = std::move(joinColMap)](
           std::function<void(IdTable&, LocalVocab&)> yieldTable) {
@@ -632,9 +667,9 @@ Result Join::computeResultForIndexScanAndIdTable(
                 .isUndefined();
         std::optional<std::shared_ptr<const Result>> indexScanResult =
             std::nullopt;
-        auto rightBlocks = [&scan, idTableHasUndef, &permutationIdTable,
-                            &indexScanResult]()
-            -> std::variant<LazyInputView, GeneratorWithDetails> {
+        auto rightBlocks = [requestLaziness, &scan, idTableHasUndef,
+                            &permutationIdTable,
+                            &indexScanResult]() -> LazyInputView {
           if (idTableHasUndef) {
             indexScanResult =
                 scan->getResult(false, ComputationMode::LAZY_IF_SUPPORTED);
@@ -644,7 +679,8 @@ Result Join::computeResultForIndexScanAndIdTable(
           } else {
             auto rightBlocksInternal =
                 scan->lazyScanForJoinOfColumnWithScan(permutationIdTable.col());
-            return convertGenerator(std::move(rightBlocksInternal));
+            return convertGenerator(std::move(rightBlocksInternal), *scan,
+                                    !requestLaziness);
           }
         }();
 
@@ -657,19 +693,10 @@ Result Join::computeResultForIndexScanAndIdTable(
               left, right, std::less{}, rowAdder);
         };
         auto blockForIdTable = std::array{std::move(permutationIdTable)};
-        std::visit(
-            [&doJoin, &blockForIdTable](auto& blocks) {
-              if constexpr (idTableIsRightInput) {
-                doJoin(blocks, blockForIdTable);
-              } else {
-                doJoin(blockForIdTable, blocks);
-              }
-            },
-            rightBlocks);
-
-        if (std::holds_alternative<GeneratorWithDetails>(rightBlocks)) {
-          scan->updateRuntimeInfoForLazyScan(
-              std::get<GeneratorWithDetails>(rightBlocks).details());
+        if constexpr (idTableIsRightInput) {
+          doJoin(rightBlocks, blockForIdTable);
+        } else {
+          doJoin(blockForIdTable, rightBlocks);
         }
 
         auto localVocab = std::move(rowAdder.localVocab());

src/engine/JoinHelpers.h

@@ -1,6 +1,6 @@
-//   Copyright 2025, University of Freiburg,
-//   Chair of Algorithms and Data Structures.
-//   Author: Robin Textor-Falconi <[email protected]>
+// Copyright 2025, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Author: Robin Textor-Falconi <[email protected]>
 //
 // Copyright 2025, Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
 
@@ -83,33 +83,6 @@ inline std::variant<LazyInputView, MaterializedInputView> resultToView(
   return convertGenerator(result.idTables(), permutation);
 }
 
-// Type alias for the general InputRangeTypeErasedWithDetails with specific
-// types
-using GeneratorWithDetails =
-    InputRangeTypeErasedWithDetails<IdTableAndFirstCol<IdTable>,
-                                    CompressedRelationReader::LazyScanMetadata>;
-
-// Convert a `generator<IdTable` to a `generator<IdTableAndFirstCol>` for more
-// efficient access in the join columns below.
-inline GeneratorWithDetails convertGenerator(
-    CompressedRelationReader::IdTableGeneratorInputRange gen) {
-  // Store the generator in a wrapper so we can access its details after moving
-  auto generatorStorage =
-      std::make_shared<CompressedRelationReader::IdTableGeneratorInputRange>(
-          std::move(gen));
-
-  // Create the range with a pointer to the generator's details
-  auto range = InputRangeTypeErased<IdTableAndFirstCol<IdTable>>(
-      CachingTransformInputRange(
-          *generatorStorage, [generatorStorage](auto& table) {
-            (void)generatorStorage;  // Only captured for lifetime reasons.
-            // IndexScans don't have a local vocabulary, so we can just use an
-            return IdTableAndFirstCol{std::move(table), LocalVocab{}};
-          }));
-
-  return GeneratorWithDetails{std::move(range), generatorStorage->details()};
-}
-
 // Part of the implementation of `createResult`. This function is called when
 // the result should be yielded lazily.
 // Action is a lambda that itself runs the join operation in a blocking

src/util/Iterators.h

@@ -538,63 +538,6 @@ template <typename ValueType, typename DetailsType = NoDetails>
 InputRangeTypeErased(std::unique_ptr<InputRangeFromGet<ValueType, DetailsType>>)
     -> InputRangeTypeErased<ValueType, DetailsType>;
 
-// A general type-erased input range with details. This combines an
-// InputRangeTypeErased with additional metadata/details of arbitrary type.
-template <typename ValueType, typename DetailsType>
-class InputRangeTypeErasedWithDetails {
- private:
-  InputRangeTypeErased<ValueType> range_;
-  // Use variant to support both owned details and external details pointer
-  std::variant<DetailsType, const DetailsType*> details_;
-
- public:
-  // Constructor that takes the range and owned details
-  template <typename Range>
-  explicit InputRangeTypeErasedWithDetails(Range range, DetailsType details)
-      : range_(std::move(range)), details_(std::move(details)) {}
-
-  // Constructor that takes the range and a pointer to external details
-  template <typename Range>
-  explicit InputRangeTypeErasedWithDetails(Range range,
-                                           const DetailsType* detailsPtr)
-      : range_(std::move(range)), details_(detailsPtr) {}
-
-  // Delegate iterator methods to the underlying range
-  auto begin() { return range_.begin(); }
-  auto end() { return range_.end(); }
-
-  // Provide access to the details
-  const DetailsType& details() const {
-    return std::visit(
-        [](const auto& d) -> const DetailsType& {
-          if constexpr (std::is_same_v<std::decay_t<decltype(d)>,
-                                       DetailsType>) {
-            return d;
-          } else {
-            return *d;
-          }
-        },
-        details_);
-  }
-
-  // Note: Mutable access only available for owned details
-  DetailsType& details() {
-    AD_CONTRACT_CHECK(std::holds_alternative<DetailsType>(details_),
-                      "Cannot get mutable reference to external details");
-    return std::get<DetailsType>(details_);
-  }
-
-  // Additional type aliases for compatibility
-  using value_type = ValueType;
-  using iterator = typename InputRangeTypeErased<ValueType>::iterator;
-};
-
-// Deduction guide
-template <typename Range, typename DetailsType>
-InputRangeTypeErasedWithDetails(Range, DetailsType)
-    -> InputRangeTypeErasedWithDetails<ql::ranges::range_value_t<Range>,
-                                       DetailsType>;
-
 // A view that takes an iterator and a sentinel (similar to
 // `ql::ranges::subrange`, but yields the iterators instead of the values when
 // being iterated over. Currently, the iterators must be random-access and the

test/IteratorTest.cpp

@@ -261,100 +261,3 @@ TEST(Iterator, IteratorRange) {
   EXPECT_EQ(beg[3], v.begin() + 3);
   EXPECT_EQ(*beg[3], 7);
 }
-
-//_____________________________________________________________________________
-TEST(Iterator, InputRangeTypeErasedWithDetails) {
-  using namespace ad_utility;
-
-  // Define a simple details type for testing
-  struct TestDetails {
-    std::string name;
-    int count = 0;
-  };
-
-  // Create a simple input range
-  std::vector<int> values{1, 2, 3, 4, 5};
-  TestDetails details{"test_range", 42};
-
-  // Test construction and basic functionality
-  InputRangeTypeErasedWithDetails<int, TestDetails> rangeWithDetails{values,
-                                                                     details};
-
-  // Test details access
-  EXPECT_EQ(rangeWithDetails.details().name, "test_range");
-  EXPECT_EQ(rangeWithDetails.details().count, 42);
-
-  // Test that the range functionality works
-  std::vector<int> result;
-  for (auto& value : rangeWithDetails) {
-    result.push_back(value);
-  }
-  EXPECT_EQ(result, values);
-
-  // Test modifying details
-  rangeWithDetails.details().count = 100;
-  EXPECT_EQ(rangeWithDetails.details().count, 100);
-
-  // Test with a custom InputRangeFromGet implementation
-  struct CountingRange : InputRangeFromGet<int> {
-    int current_ = 0;
-    int max_;
-    explicit CountingRange(int max) : max_(max) {}
-    std::optional<int> get() override {
-      if (current_ >= max_) return std::nullopt;
-      return current_++;
-    }
-  };
-
-  TestDetails countingDetails{"counting", 999};
-  InputRangeTypeErasedWithDetails<int, TestDetails> countingRangeWithDetails{
-      CountingRange{3}, countingDetails};
-
-  std::vector<int> countingResult;
-  for (auto& value : countingRangeWithDetails) {
-    countingResult.push_back(value);
-  }
-  EXPECT_EQ(countingResult, (std::vector<int>{0, 1, 2}));
-  EXPECT_EQ(countingRangeWithDetails.details().name, "counting");
-  EXPECT_EQ(countingRangeWithDetails.details().count, 999);
-
-  // Test deduction guide
-  auto deducedRange =
-      InputRangeTypeErasedWithDetails{values, TestDetails{"deduced", 123}};
-  static_assert(
-      std::is_same_v<decltype(deducedRange),
-                     InputRangeTypeErasedWithDetails<int, TestDetails>>);
-  EXPECT_EQ(deducedRange.details().name, "deduced");
-  EXPECT_EQ(deducedRange.details().count, 123);
-
-  // Test external details (pointer-based) scenario
-  TestDetails externalDetails{"external", 456};
-  DISABLE_FREE_NONHEAP_WARNINGS
-  InputRangeTypeErasedWithDetails<int, TestDetails> rangeWithExternalDetails{
-      values, &externalDetails};
-  GCC_REENABLE_WARNINGS
-
-  // Test that external details are accessible (const access)
-  const auto& constRangeRef = rangeWithExternalDetails;
-  EXPECT_EQ(constRangeRef.details().name, "external");
-  EXPECT_EQ(constRangeRef.details().count, 456);
-
-  // Test that modifying the external details affects the range
-  externalDetails.count = 789;
-  EXPECT_EQ(constRangeRef.details().count, 789);
-
-  // Test that mutable access throws for external details
-  EXPECT_THROW(
-      {
-        auto& mutableDetails = rangeWithExternalDetails.details();
-        mutableDetails.count = 999;  // This should throw
-      },
-      ad_utility::Exception);
-
-  // Test range functionality with external details
-  std::vector<int> externalResult;
-  for (auto& value : rangeWithExternalDetails) {
-    externalResult.push_back(value);
-  }
-  EXPECT_EQ(externalResult, values);
-}