Rewrite single chain joins to scan on materialized view if possible #2649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

ullingerc wants to merge 84 commits into ad-freiburg:master from ullingerc:matview-detect-join

src/ServerMain.cpp

-Original file line number
+Diff line change
@@ Expand Up / @@ -182,6 +182,11 @@ int main(int argc, char** argv) { @@
               ->multitoken(),
           "The names of materialized views to be loaded automatically on server "
           "start (this option takes an arbitrary number of arguments).");
+      add("enable-materialized-view-query-rewrite",
+          optionFactory.getProgramOption<
+              &RuntimeParameters::enableMaterializedViewQueryRewrite_>(),
+          "If set to true, loaded materialized views will be considered as "
+          "alternative query plans for certain supported query patterns.");
       po::variables_map optionsMap;
       try {
@@ Expand Down @@

src/engine/CMakeLists.txt

-Original file line number
+Diff line change
@@ Expand Up / @@ -18,7 +18,8 @@ add_library(engine @@
             QueryExecutionContext.cpp ExistsJoin.cpp SparqlProtocol.cpp ParsedRequestBuilder.cpp
             NeutralOptional.cpp Load.cpp StripColumns.cpp NamedResultCache.cpp
             ExplicitIdTableOperation.cpp StringMapping.cpp MaterializedViews.cpp
-            PermutationSelector.cpp ConstructQueryEvaluator.cpp ConstructTripleGenerator.cpp)
+            PermutationSelector.cpp ConstructQueryEvaluator.cpp ConstructTripleGenerator.cpp
+            MaterializedViewsQueryAnalysis.cpp)
     qlever_target_link_libraries(engine util index parser global sparqlExpressions SortPerformanceEstimator Boost::iostreams s2 spatialjoin-dev pb_util pb_util_geo)
@@ Expand Down @@

src/engine/MaterializedViews.cpp

-Original file line number
+Diff line change
@@ Expand Up / @@ -14,6 +14,8 @@ @@
     #include <stdexcept>
     #include "engine/IndexScan.h"
+    #include "engine/Join.h"
+    #include "engine/MaterializedViewsQueryAnalysis.h"
     #include "engine/QueryExecutionContext.h"
     #include "engine/QueryExecutionTree.h"
     #include "engine/VariableToColumnMap.h"
@@ Expand Down Expand Up @@
       // Column information for the columns selected by the user's query.
       auto existingCols = ::ranges::to<std::vector<ColumnNameAndIndex>>(
           targetVarsAndCols | ql::views::transform([](const auto& opt) {
-            AD_CONTRACT_CHECK(opt.has_value());
+            AD_CONTRACT_CHECK(
+                opt.has_value(),
+                "Please ensure that all variables in your SELECT are bound.");
             return ColumnNameAndIndex{opt.value().variable_,
                                       opt.value().columnIndex_};
           }));
@@ Expand Down Expand Up @@
     // _____________________________________________________________________________
     void MaterializedViewsManager::loadView(const std::string& name) const {
       auto lock = loadedViews_.wlock();
+      auto patternLock = queryPatternCache_.wlock();
       if (lock->contains(name)) {
         return;
       }
-      lock->insert({name, std::make_shared<MaterializedView>(onDiskBase_, name)});
-    };
+      auto view = std::make_shared<MaterializedView>(onDiskBase_, name);
+      lock->insert({name, view});
+      // Analyzing the view when loading instead of (de)serializing an analysis
+      // result has the benefit that query analysis can be extended without needing
+      // to rewrite views.
+      if (patternLock->analyzeView(view)) {
+        AD_LOG_INFO << "The materialized view '" << name
+                    << "' was added to the query pattern cache." << std::endl;
+      }
+    }
     // _____________________________________________________________________________
     void MaterializedViewsManager::unloadViewIfLoaded(
         const std::string& name) const {
       // `HashMap::erase` is a no-op for nonexisting keys.
       loadedViews_.wlock()->erase(name);
+      // TODO<ullingerc> Query pattern cache unload.
     }
     // _____________________________________________________________________________
@@ Expand Down Expand Up @@
           viewQuery.getVarsToKeep());
     }
+    // _____________________________________________________________________________
+    std::vector<MaterializedViewJoinReplacement>
+    MaterializedViewsManager::makeJoinReplacementIndexScans(
+        QueryExecutionContext* qec,
+        const parsedQuery::BasicGraphPattern& triples) const {
+      return queryPatternCache_.rlock()->makeJoinReplacementIndexScans(qec,
+                                                                       triples);
+    }
     // _____________________________________________________________________________
     std::shared_ptr<IndexScan> MaterializedViewsManager::makeIndexScan(
         QueryExecutionContext* qec,
@@ Expand Down @@

src/engine/MaterializedViews.h

-Original file line number
+Diff line change
@@ Expand Up / @@ -7,16 +7,19 @@ @@
     #ifndef QLEVER_SRC_ENGINE_MATERIALIZEDVIEWS_H_
     #define QLEVER_SRC_ENGINE_MATERIALIZEDVIEWS_H_
+    #include "engine/MaterializedViewsQueryAnalysis.h"
     #include "engine/VariableToColumnMap.h"
     #include "engine/idTable/CompressedExternalIdTable.h"
     #include "index/DeltaTriples.h"
     #include "index/ExternalSortFunctors.h"
     #include "index/Permutation.h"
     #include "libqlever/QleverTypes.h"
+    #include "parser/GraphPatternOperation.h"
     #include "parser/MaterializedViewQuery.h"
     #include "parser/ParsedQuery.h"
     #include "parser/SparqlTriple.h"
     #include "util/HashMap.h"
+    #include "util/Synchronized.h"
     // Forward declarations
     class QueryExecutionContext;
@@ Expand Down Expand Up / @@ -223,6 +226,9 @@ class MaterializedView { @@
           const parsedQuery::MaterializedViewQuery& viewQuery) const;
     };
+    // Shorthand for query rewriting helper class.
+    using materializedViewsQueryAnalysis::MaterializedViewJoinReplacement;
     // The `MaterializedViewsManager` is part of the `QueryExecutionContext` and is
     // used to manage the currently loaded `MaterializedViews` in a `Server` or
     // `Qlever` instance.
@@ Expand All / @@ -232,6 +238,9 @@ class MaterializedViewsManager { @@
       mutable ad_utility::Synchronized<
           ad_utility::HashMap<std::string, std::shared_ptr<MaterializedView>>>
           loadedViews_;
+      mutable ad_utility::Synchronized<
+          materializedViewsQueryAnalysis::QueryPatternCache>
+          queryPatternCache_;
      public:
       MaterializedViewsManager() = default;
@@ Expand Down Expand Up / @@ -266,6 +275,14 @@ class MaterializedViewsManager { @@
           QueryExecutionContext* qec,
           const parsedQuery::MaterializedViewQuery& viewQuery) const;
+      // Given a set of triples, check if some join operations that would be
+      // required when evaluating them can be replaced by scans on materialized
+      // views that are currently loaded. This is implemented using the
+      // `queryPatternCache_`.
+      std::vector<MaterializedViewJoinReplacement> makeJoinReplacementIndexScans(
+          QueryExecutionContext* qec,
+          const parsedQuery::BasicGraphPattern& triples) const;
       // Write a `MaterializedView` given a valid `name` (consisting only of
       // alphanumerics and hyphens) and a `queryPlan` to be executed. The query's
       // result is written to the view.
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Rewrite single chain joins to scan on materialized view if possible #2649

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Rewrite single chain joins to scan on materialized view if possible #2649

Are you sure you want to change the base?

Uh oh!

Rewrite single chain joins to scan on materialized view if possible #2649

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!