Completely refactor the fulltext operations

e2e/scientists_queries.yaml

@@ -52,7 +52,7 @@ queries:
           ?x <is-a> <Scientist> .
           ?x <Place_of_birth> <Ulm> .
           ?t ql:contains-entity ?x .
-          ?t ql:contains-word "relat* physikalische rela*"
+          ?t ql:contains-word "RElaT* phySIKalische rela*"
       }
     checks:
       - num_cols: 5

src/engine/QueryPlanner.cpp

@@ -683,7 +683,7 @@ QueryPlanner::TripleGraph QueryPlanner::createTripleGraph(
       // Add one node for each word
       for (const auto& term :
            absl::StrSplit(sv.substr(1, sv.size() - 2), ' ')) {
-        std::string s{term};
+        std::string s{ad_utility::utf8ToLower(term)};
         potentialTermsForCvar[t._s.getVariable()].push_back(s);
         addNodeToTripleGraph(
             TripleGraph::Node(tg._nodeStorage.size(), t._s.getVariable(), s, t),
@@ -720,11 +720,15 @@ QueryPlanner::TripleGraph QueryPlanner::createTripleGraph(
   return tg;
 }
 
-void QueryPlanner::handleSingleVariableCase(
-    const TripleGraph::Node& node, std::function<void(SubtreePlan)> pushPlan,
-    std::function<void(Permutation::Enum)> addIndexScan) {
+// _____________________________________________________________________________
+template <typename PushPlanFunction, typename AddedIndexScanFunction>
+void QueryPlanner::indexScanSingleVarCase(
+    const TripleGraph::Node& node, const PushPlanFunction& pushPlan,
+    const AddedIndexScanFunction& addIndexScan) {
   using enum Permutation::Enum;
 
+  // TODO: The case where the same variable appears in subject + predicate or
+  // object + predicate is missing here and leads to an assertion failure.
   if (isVariable(node.triple_._s) && isVariable(node.triple_._o) &&
       node.triple_._s == node.triple_._o) {
     if (isVariable(node.triple_._p._iri)) {
@@ -758,11 +762,15 @@ void QueryPlanner::handleSingleVariableCase(
   }
 }
 
-void QueryPlanner::handleTwoVariablesCase(
+// _____________________________________________________________________________
+template <typename AddedIndexScanFunction>
+void QueryPlanner::indexScanTwoVarsCase(
     const TripleGraph::Node& node,
-    std::function<void(Permutation::Enum)> addIndexScan) {
+    const AddedIndexScanFunction& addIndexScan) const {
   using enum Permutation::Enum;
 
+  // TODO: The case that the same variable appears in more than one position
+  // leads (as in indexScanSingleVarCase) to an assertion.
   if (!isVariable(node.triple_._p._iri)) {
     addIndexScan(PSO);
     addIndexScan(POS);
@@ -775,9 +783,11 @@ void QueryPlanner::handleTwoVariablesCase(
   }
 }
 
-void QueryPlanner::handleThreeVariablesCase(
+// _____________________________________________________________________________
+template <typename AddedIndexScanFunction>
+void QueryPlanner::indexScanThreeVarsCase(
     const TripleGraph::Node& node,
-    std::function<void(Permutation::Enum)> addIndexScan) {
+    const AddedIndexScanFunction& addIndexScan) const {
   using enum Permutation::Enum;
 
   if (!_qec || _qec->getIndex().hasAllPermutations()) {
@@ -797,6 +807,20 @@ void QueryPlanner::handleThreeVariablesCase(
   }
 }
 
+// _____________________________________________________________________________
+template <typename PushPlanFunction, typename AddedIndexScanFunction>
+void QueryPlanner::seedFromOrdinaryTriple(
+    const TripleGraph::Node& node, const PushPlanFunction& pushPlan,
+    const AddedIndexScanFunction& addIndexScan) {
+  if (node._variables.size() == 1) {
+    indexScanSingleVarCase(node, pushPlan, addIndexScan);
+  } else if (node._variables.size() == 2) {
+    indexScanTwoVarsCase(node, addIndexScan);
+  } else {
+    indexScanThreeVarsCase(node, addIndexScan);
+  }
+}
+
 // _____________________________________________________________________________
 vector<QueryPlanner::SubtreePlan> QueryPlanner::seedWithScansAndText(
     const QueryPlanner::TripleGraph& tg,
@@ -817,12 +841,12 @@ vector<QueryPlanner::SubtreePlan> QueryPlanner::seedWithScansAndText(
   for (size_t i = 0; i < tg._nodeMap.size(); ++i) {
     const TripleGraph::Node& node = *tg._nodeMap.find(i)->second;
 
-    auto pushPlan = [&](SubtreePlan plan) {
+    auto pushPlan = [&seeds, i](SubtreePlan plan) {
       plan._idsOfIncludedNodes = (uint64_t(1) << i);
       seeds.push_back(std::move(plan));
     };
 
-    auto addIndexScan = [&](Permutation::Enum permutation) {
+    auto addIndexScan = [this, pushPlan, node](Permutation::Enum permutation) {
       pushPlan(makeSubtreePlan<IndexScan>(_qec, permutation, node.triple_));
     };
 
@@ -862,13 +886,7 @@ vector<QueryPlanner::SubtreePlan> QueryPlanner::seedWithScansAndText(
       continue;
     }
 
-    if (node._variables.size() == 1) {
-      handleSingleVariableCase(node, pushPlan, addIndexScan);
-    } else if (node._variables.size() == 2) {
-      handleTwoVariablesCase(node, addIndexScan);
-    } else {
-      handleThreeVariablesCase(node, addIndexScan);
-    }
+    seedFromOrdinaryTriple(node, pushPlan, addIndexScan);
   }
   return seeds;
 }
@@ -1373,21 +1391,6 @@ bool QueryPlanner::TripleGraph::isTextNode(size_t i) const {
           _nodeMap.find(i)->second->triple_._p._iri == CONTAINS_WORD_PREDICATE);
 }
 
-// _____________________________________________________________________________
-ad_utility::HashMap<Variable, vector<size_t>>
-QueryPlanner::TripleGraph::identifyTextCliques() const {
-  ad_utility::HashMap<Variable, vector<size_t>> contextVarToTextNodesIds;
-  // Fill contextVar -> triples map
-  for (size_t i = 0; i < _adjLists.size(); ++i) {
-    if (isTextNode(i)) {
-      auto& triple = _nodeMap.find(i)->second->triple_;
-      auto& cvar = triple._s;
-      contextVarToTextNodesIds[cvar.getVariable()].push_back(i);
-    }
-  }
-  return contextVarToTextNodesIds;
-}
-
 // _____________________________________________________________________________
 vector<pair<QueryPlanner::TripleGraph, vector<SparqlFilter>>>
 QueryPlanner::TripleGraph::splitAtContextVars(

src/engine/QueryPlanner.h

@@ -103,8 +103,6 @@ class QueryPlanner {
     ad_utility::HashMap<size_t, Node*> _nodeMap;
     std::list<TripleGraph::Node> _nodeStorage;
 
-    ad_utility::HashMap<Variable, vector<size_t>> identifyTextCliques() const;
-
     vector<size_t> bfsLeaveOut(size_t startNode,
                                ad_utility::HashSet<size_t> leaveOut) const;
 
@@ -226,17 +224,27 @@ class QueryPlanner {
   [[nodiscard]] std::vector<QueryPlanner::SubtreePlan> optimize(
       ParsedQuery::GraphPattern* rootPattern);
 
-  void handleSingleVariableCase(
-      const TripleGraph::Node& node, std::function<void(SubtreePlan)> pushPlan,
-      std::function<void(Permutation::Enum)> addIndexScan);
-
-  void handleTwoVariablesCase(
-      const TripleGraph::Node& node,
-      std::function<void(Permutation::Enum)> addIndexScan);
-
-  void handleThreeVariablesCase(
-      const TripleGraph::Node& node,
-      std::function<void(Permutation::Enum)> addIndexScan);
+  // Helper function used by the seedFromOrdinaryTriple function
+  template <typename PushPlanFunction, typename AddedIndexScanFunction>
+  void indexScanSingleVarCase(const TripleGraph::Node& node,
+                              const PushPlanFunction& pushPlan,
+                              const AddedIndexScanFunction& addIndexScan);
+
+  // Helper function used by the seedFromOrdinaryTriple function
+  template <typename AddedIndexScanFunction>
+  void indexScanTwoVarsCase(const TripleGraph::Node& node,
+                            const AddedIndexScanFunction& addIndexScan) const;
+
+  // Helper function used by the seedFromOrdinaryTriple function
+  template <typename AddedIndexScanFunction>
+  void indexScanThreeVarsCase(const TripleGraph::Node& node,
+                              const AddedIndexScanFunction& addIndexScan) const;
+
+  // Helper function used by the seedFromScansAndText function
+  template <typename PushPlanFunction, typename AddedIndexScanFunction>
+  void seedFromOrdinaryTriple(const TripleGraph::Node& node,
+                              const PushPlanFunction& pushPlan,
+                              const AddedIndexScanFunction& addIndexScan);
 
   /**
    * @brief Fills children with all operations that are associated with a single

src/engine/TextIndexScanForEntity.cpp

@@ -1,12 +1,16 @@
+//  Copyright 2023, University of Freiburg,
+//                  Chair of Algorithms and Data Structures.
+//  Author: Nick Göckel <[email protected]>
+
 #include "engine/TextIndexScanForEntity.h"
 
 // _____________________________________________________________________________
 TextIndexScanForEntity::TextIndexScanForEntity(
     QueryExecutionContext* qec, Variable textRecordVar,
-    const std::variant<Variable, std::string>& entity, string word)
+    std::variant<Variable, std::string> entity, string word)
     : Operation(qec),
       textRecordVar_(std::move(textRecordVar)),
-      entity_(VarOrFixedEntity(qec, entity)),
+      varOrFixed_(VarOrFixedEntity(qec, std::move(entity))),
       word_(std::move(word)) {}
 
 // _____________________________________________________________________________
@@ -15,14 +19,21 @@ ResultTable TextIndexScanForEntity::computeResult() {
       word_, getExecutionContext()->getAllocator());
 
   if (hasFixedEntity()) {
-    auto beginErase = std::ranges::remove_if(
-        idTable.begin(), idTable.end(), [this](const auto& row) {
-          return row[1].getVocabIndex() != entity_.index_;
-        });
+    auto beginErase = std::ranges::remove_if(idTable, [this](const auto& row) {
+      return row[1].getVocabIndex() != getVocabIndexOfFixedEntity();
+    });
     idTable.erase(beginErase.begin(), idTable.end());
     idTable.setColumnSubset(std::vector<ColumnIndex>{0, 2});
   }
 
+  // Add details to the runtimeInfo. This is has no effect on the result.
+  if (hasFixedEntity()) {
+    runtimeInfo().addDetail("fixed entity: ", getFixedEntity());
+  } else {
+    runtimeInfo().addDetail("entity var: ", getEntityVariable().name());
+  }
+  runtimeInfo().addDetail("word: ", word_);
+
   return {std::move(idTable), resultSortedOn(), LocalVocab{}};
 }
 
@@ -36,11 +47,10 @@ VariableToColumnMap TextIndexScanForEntity::computeVariableToColumnMap() const {
   };
   addDefinedVar(textRecordVar_);
   if (hasFixedEntity()) {
-    addDefinedVar(
-        textRecordVar_.getScoreVariable(entity_.fixedEntity_.value()));
+    addDefinedVar(textRecordVar_.getScoreVariable(getFixedEntity()));
   } else {
-    addDefinedVar(entity_.entityVar_.value());
-    addDefinedVar(textRecordVar_.getScoreVariable(entity_.entityVar_.value()));
+    addDefinedVar(getEntityVariable());
+    addDefinedVar(textRecordVar_.getScoreVariable(getEntityVariable()));
   }
   return vcmap;
 }
@@ -53,19 +63,24 @@ size_t TextIndexScanForEntity::getResultWidth() const {
 // _____________________________________________________________________________
 size_t TextIndexScanForEntity::getCostEstimate() {
   if (hasFixedEntity()) {
-    return 2 * getExecutionContext()->getIndex().getEntitySizeEstimate(word_);
+    // We currently have to first materialize and then filter the complete list
+    // for the fixed entity
+    return 2 * getExecutionContext()->getIndex().getSizeOfTextBlockForEntities(
+                   word_);
   } else {
-    return getExecutionContext()->getIndex().getEntitySizeEstimate(word_);
+    return getExecutionContext()->getIndex().getSizeOfTextBlockForEntities(
+        word_);
   }
 }
 
 // _____________________________________________________________________________
 uint64_t TextIndexScanForEntity::getSizeEstimateBeforeLimit() {
   if (hasFixedEntity()) {
-    return uint64_t(
+    return static_cast<uint64_t>(
         getExecutionContext()->getIndex().getAverageNofEntityContexts());
   } else {
-    return getExecutionContext()->getIndex().getEntitySizeEstimate(word_);
+    return getExecutionContext()->getIndex().getSizeOfTextBlockForEntities(
+        word_);
   }
 }
 
@@ -89,7 +104,6 @@ string TextIndexScanForEntity::getCacheKeyImpl() const {
   std::ostringstream os;
   os << "ENTITY INDEX SCAN FOR WORD: "
      << " with word: \"" << word_ << "\" and fixed-entity: \""
-     << (hasFixedEntity() ? entity_.fixedEntity_.value() : "no fixed-entity")
-     << " \"";
+     << (hasFixedEntity() ? getFixedEntity() : "no fixed-entity") << " \"";
   return std::move(os).str();
 }