From 02a27b217d7707c19f51cf3443e3fd310187a005 Mon Sep 17 00:00:00 2001 From: ullingerc Date: Tue, 3 Feb 2026 11:49:09 +0100 Subject: [PATCH 1/6] write materialized views in qlever-index --- src/index/IndexBuilderMain.cpp | 42 ++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/index/IndexBuilderMain.cpp b/src/index/IndexBuilderMain.cpp index 5b27bf3d72..c929b6c04e 100644 --- a/src/index/IndexBuilderMain.cpp +++ b/src/index/IndexBuilderMain.cpp @@ -20,6 +20,7 @@ #include "libqlever/Qlever.h" #include "util/ProgramOptionsHelpers.h" #include "util/ReadableNumberFacet.h" +#include "util/json.h" using std::string; @@ -159,6 +160,7 @@ int main(int argc, char** argv) { std::vector inputFile; std::vector defaultGraphs; std::vector parseParallel; + std::string materializedViewsJson; boost::program_options::options_description boostOptions( "Options for qlever-index"); @@ -251,6 +253,10 @@ int main(int argc, char** argv) { "large enough to hold a single input triple. Default: 10 MB."); add("keep-temporary-files,k", po::bool_switch(&config.keepTemporaryFiles_), "Do not delete temporary files from index creation for debugging."); + add("write-materialized-views", po::value(&materializedViewsJson), + "Write materialized views after index building. Takes a JSON object " + "mapping view names to SPARQL queries, e.g., " + R"({"view1":"SELECT ?s ?p WHERE { ?s ?p ?o }", "view2":"SELECT ..."})"); // Process command line arguments. po::variables_map optionsMap; @@ -283,6 +289,42 @@ int main(int argc, char** argv) { config.validate(); qlever::Qlever::buildIndex(config); + // TODO move into libqlever + // Write materialized views if specified. + if (!materializedViewsJson.empty()) { + AD_LOG_INFO << "Writing materialized views..." << std::endl; + try { + auto viewsJson = nlohmann::json::parse(materializedViewsJson); + if (!viewsJson.is_object()) { + throw std::runtime_error( + "The --write-materialized-views option must be a JSON object " + "mapping view names to SPARQL queries."); + } + + // Create a Qlever instance to write the materialized views. + qlever::EngineConfig engineConfig(config); + qlever::Qlever qlever(engineConfig); + + // Write each materialized view. + for (auto& [viewName, query] : viewsJson.items()) { + if (!query.is_string()) { + throw std::runtime_error(absl::StrCat( + "Query for materialized view '", viewName, + "' must be a string, but got type: ", jsonToTypeString(query))); + } + AD_LOG_INFO << "Writing materialized view: " << viewName << std::endl; + qlever.writeMaterializedView(viewName, query.get()); + AD_LOG_INFO << "Successfully wrote materialized view: " << viewName + << std::endl; + } + AD_LOG_INFO << "All materialized views written successfully." + << std::endl; + } catch (const nlohmann::json::exception& e) { + throw std::runtime_error(absl::StrCat( + "Failed to parse materialized views JSON: ", e.what())); + } + } + } catch (std::exception& e) { AD_LOG_ERROR << "Creating the index for QLever failed with the following " "exception: " From c06346b30f213a3cbf285690ac4b5307dcc52e6d Mon Sep 17 00:00:00 2001 From: ullingerc Date: Tue, 3 Feb 2026 14:11:24 +0100 Subject: [PATCH 2/6] refactor, nice code --- src/index/IndexBuilderMain.cpp | 69 ++++++++++++++++------------------ src/libqlever/Qlever.cpp | 12 ++++++ src/libqlever/Qlever.h | 5 +++ 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/src/index/IndexBuilderMain.cpp b/src/index/IndexBuilderMain.cpp index c929b6c04e..5b990f8419 100644 --- a/src/index/IndexBuilderMain.cpp +++ b/src/index/IndexBuilderMain.cpp @@ -143,6 +143,36 @@ auto getFileSpecifications = [](const auto& filetype, auto& inputFile, return fileSpecs; }; +// Helper to convert the JSON given for writing materialized views to a proper +// `WriteMaterializedViews` vector. +qlever::IndexBuilderConfig::WriteMaterializedViews parseMaterializedViewsJson( + std::string materializedViewsJson) { + qlever::IndexBuilderConfig::WriteMaterializedViews views; + if (!materializedViewsJson.empty()) { + AD_LOG_INFO << "Writing materialized views..." << std::endl; + try { + auto viewsJson = nlohmann::json::parse(materializedViewsJson); + if (!viewsJson.is_object()) { + throw std::runtime_error( + "The --write-materialized-views option must be a JSON object " + "mapping view names to SPARQL queries."); + } + for (auto& [viewName, query] : viewsJson.items()) { + if (!query.is_string()) { + throw std::runtime_error(absl::StrCat( + "Query for materialized view '", viewName, + "' must be a string, but got type: ", jsonToTypeString(query))); + } + views.push_back({std::move(viewName), query.get()}); + } + } catch (const nlohmann::json::exception& e) { + throw std::runtime_error( + absl::StrCat("Failed to parse materialized views JSON: ", e.what())); + } + } + return views; +} + // Main function. int main(int argc, char** argv) { // Copy the git hash and datetime of compilation (which require relinking) @@ -286,45 +316,10 @@ int main(int argc, char** argv) { try { config.inputFiles_ = getFileSpecifications(filetype, inputFile, defaultGraphs, parseParallel); + config.writeMaterializedViews_ = + parseMaterializedViewsJson(materializedViewsJson); config.validate(); qlever::Qlever::buildIndex(config); - - // TODO move into libqlever - // Write materialized views if specified. - if (!materializedViewsJson.empty()) { - AD_LOG_INFO << "Writing materialized views..." << std::endl; - try { - auto viewsJson = nlohmann::json::parse(materializedViewsJson); - if (!viewsJson.is_object()) { - throw std::runtime_error( - "The --write-materialized-views option must be a JSON object " - "mapping view names to SPARQL queries."); - } - - // Create a Qlever instance to write the materialized views. - qlever::EngineConfig engineConfig(config); - qlever::Qlever qlever(engineConfig); - - // Write each materialized view. - for (auto& [viewName, query] : viewsJson.items()) { - if (!query.is_string()) { - throw std::runtime_error(absl::StrCat( - "Query for materialized view '", viewName, - "' must be a string, but got type: ", jsonToTypeString(query))); - } - AD_LOG_INFO << "Writing materialized view: " << viewName << std::endl; - qlever.writeMaterializedView(viewName, query.get()); - AD_LOG_INFO << "Successfully wrote materialized view: " << viewName - << std::endl; - } - AD_LOG_INFO << "All materialized views written successfully." - << std::endl; - } catch (const nlohmann::json::exception& e) { - throw std::runtime_error(absl::StrCat( - "Failed to parse materialized views JSON: ", e.what())); - } - } - } catch (std::exception& e) { AD_LOG_ERROR << "Creating the index for QLever failed with the following " "exception: " diff --git a/src/libqlever/Qlever.cpp b/src/libqlever/Qlever.cpp index a29fb9bcfa..7cf2b21eea 100644 --- a/src/libqlever/Qlever.cpp +++ b/src/libqlever/Qlever.cpp @@ -105,6 +105,18 @@ void Qlever::buildIndex(IndexBuilderConfig config) { "version of QLever"); #endif } + + // Build materialized views if requested. + if (!config.writeMaterializedViews_.empty()) { + AD_LOG_INFO + << "Loading the new index to execute materialized view write queries..." + << std::endl; + Qlever engine{EngineConfig{config}}; + for (auto& [viewName, query] : config.writeMaterializedViews_) { + engine.writeMaterializedView(viewName, query); + } + AD_LOG_INFO << "All materialized views written successfully." << std::endl; + } } // ___________________________________________________________________________ diff --git a/src/libqlever/Qlever.h b/src/libqlever/Qlever.h index 91207a97ca..3a2b63cb2c 100644 --- a/src/libqlever/Qlever.h +++ b/src/libqlever/Qlever.h @@ -137,6 +137,11 @@ struct IndexBuilderConfig : CommonConfig { float bScoringParam_ = 0.75; float kScoringParam_ = 1.75; + // Materialized views to be written after normal index build is complete. + using WriteMaterializedViews = + std::vector>; + WriteMaterializedViews writeMaterializedViews_; + // Assert that the given configuration is valid. void validate() const; From 5df3fc3f6c3bed278c1644df4d35ed4aeb3b30f9 Mon Sep 17 00:00:00 2001 From: ullingerc Date: Tue, 3 Feb 2026 14:14:24 +0100 Subject: [PATCH 3/6] Unit test --- test/libqlever/QleverTest.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/libqlever/QleverTest.cpp b/test/libqlever/QleverTest.cpp index 57fa4f797a..a54d81f64f 100644 --- a/test/libqlever/QleverTest.cpp +++ b/test/libqlever/QleverTest.cpp @@ -37,6 +37,10 @@ TEST(LibQlever, buildIndexAndRunQuery) { c.parserBufferSize_ = std::nullopt; EXPECT_NO_THROW(Qlever::buildIndex(c)); + + // Test materialized views to be written at index build time. + c.writeMaterializedViews_ = {{"demoView", "SELECT ?s { ?s

}"}}; + { EngineConfig ec{c}; Qlever engine{ec}; @@ -78,6 +82,9 @@ TEST(LibQlever, buildIndexAndRunQuery) { engine.clearNamedResultCache(); AD_EXPECT_THROW_WITH_MESSAGE(engine.query(serviceQuery), notPinned); AD_EXPECT_THROW_WITH_MESSAGE(engine.query(serviceQuery2), notPinned); + + // Test that the requested materialized view exists. + engine.loadMaterializedView("demoView"); } #ifndef QLEVER_REDUCED_FEATURE_SET_FOR_CPP17 From 1309ec9fcd68d7925a475e60a5cd09292bd489f1 Mon Sep 17 00:00:00 2001 From: ullingerc Date: Tue, 3 Feb 2026 14:24:05 +0100 Subject: [PATCH 4/6] improve help text --- src/index/IndexBuilderMain.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index/IndexBuilderMain.cpp b/src/index/IndexBuilderMain.cpp index 5b990f8419..ab51169c75 100644 --- a/src/index/IndexBuilderMain.cpp +++ b/src/index/IndexBuilderMain.cpp @@ -285,8 +285,8 @@ int main(int argc, char** argv) { "Do not delete temporary files from index creation for debugging."); add("write-materialized-views", po::value(&materializedViewsJson), "Write materialized views after index building. Takes a JSON object " - "mapping view names to SPARQL queries, e.g., " - R"({"view1":"SELECT ?s ?p WHERE { ?s ?p ?o }", "view2":"SELECT ..."})"); + "mapping view names to SELECT queries for writing the view, for example: " + R"({"view1": "SELECT ...", "view2": "SELECT ..."})"); // Process command line arguments. po::variables_map optionsMap; From d099e099a0ca902e6c38053bd2fa80ec8de99ac7 Mon Sep 17 00:00:00 2001 From: ullingerc Date: Tue, 3 Feb 2026 14:25:47 +0100 Subject: [PATCH 5/6] move --- src/index/IndexBuilderMain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index/IndexBuilderMain.cpp b/src/index/IndexBuilderMain.cpp index ab51169c75..25def72bdb 100644 --- a/src/index/IndexBuilderMain.cpp +++ b/src/index/IndexBuilderMain.cpp @@ -317,7 +317,7 @@ int main(int argc, char** argv) { config.inputFiles_ = getFileSpecifications(filetype, inputFile, defaultGraphs, parseParallel); config.writeMaterializedViews_ = - parseMaterializedViewsJson(materializedViewsJson); + parseMaterializedViewsJson(std::move(materializedViewsJson)); config.validate(); qlever::Qlever::buildIndex(config); } catch (std::exception& e) { From de4425e5813b85a8f5c9b325bb90e30ed0fb43f3 Mon Sep 17 00:00:00 2001 From: ullingerc Date: Tue, 3 Feb 2026 15:03:54 +0100 Subject: [PATCH 6/6] fix unit test --- test/libqlever/QleverTest.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/libqlever/QleverTest.cpp b/test/libqlever/QleverTest.cpp index a54d81f64f..4f2b2a8ece 100644 --- a/test/libqlever/QleverTest.cpp +++ b/test/libqlever/QleverTest.cpp @@ -36,11 +36,12 @@ TEST(LibQlever, buildIndexAndRunQuery) { ::testing::HasSubstr("buffer size")); c.parserBufferSize_ = std::nullopt; - EXPECT_NO_THROW(Qlever::buildIndex(c)); // Test materialized views to be written at index build time. c.writeMaterializedViews_ = {{"demoView", "SELECT ?s { ?s

}"}}; + EXPECT_NO_THROW(Qlever::buildIndex(c)); + { EngineConfig ec{c}; Qlever engine{ec}; @@ -84,7 +85,7 @@ TEST(LibQlever, buildIndexAndRunQuery) { AD_EXPECT_THROW_WITH_MESSAGE(engine.query(serviceQuery2), notPinned); // Test that the requested materialized view exists. - engine.loadMaterializedView("demoView"); + EXPECT_NO_THROW(engine.loadMaterializedView("demoView")); } #ifndef QLEVER_REDUCED_FEATURE_SET_FOR_CPP17