Merge upstream/master into improveConstructQueryResultToTriples

Author: marvin7122

.github/workflows/native-build-with-conan-and-emscripten.yml

@@ -0,0 +1,57 @@
+name: "Native build to WebAssembly using Conan and Emscripten"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-24.04
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: 'recursive'
+    - name: Install conan package manager
+      run: |
+        sudo apt-get install pipx
+        pipx ensurepath
+        source ~/.bashrc
+        pipx install conan
+      shell: bash
+
+    - name: Set up nodejs v25.
+      uses: actions/setup-node@v6
+      with:
+        node-version: 25
+
+    - name: Get build and host profiles for conan
+      working-directory: ${{github.workspace}}
+      run: |
+        conan profile detect
+
+    - name: Cache for conan
+      uses: actions/cache@v3
+      env:
+        cache-name: cache-conan-with-emscripten-modules-ubuntu-24.04
+      with:
+        path: ~/.conan2
+        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('conanfile.txt', 'conanprofiles/emscripten.profile')}}
+
+    - name: Install and run conan. Also create a build directory.
+      working-directory:  ${{github.workspace}}
+      run: conan install . -pr:b default -pr:h conanprofiles/emscripten.profile -s build_type=Release -b missing -of build
+
+    - name: Configure CMake
+      working-directory: ${{github.workspace}}/build
+      run: cmake -S .. -B . -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DLOGLEVEL=INFO -DUSE_PARALLEL=true -D_NO_TIMING_TESTS=ON -D_EMSCRIPTEN_NO_INDEXBUILDER_AND_SERVER=ON -D_NO_BENCHMARK=ON -D_DISABLE_EMSCRIPTEN_PROBLEMATIC_TESTS=ON -GNinja
+
+    - name: Build
+      working-directory: ${{github.workspace}}/build
+
+      run: cmake --build .
+

CMakeLists.txt

@@ -478,13 +478,17 @@ add_subdirectory(test)
 add_library(compilationInfo ${CMAKE_CURRENT_BINARY_DIR}/CompilationInfo.cpp)
 qlever_target_link_libraries(compilationInfo)
 
-add_executable(qlever-index src/index/IndexBuilderMain.cpp)
-qlever_target_link_libraries(qlever-index qlever index ${CMAKE_THREAD_LIBS_INIT} Boost::program_options compilationInfo global)
+OPTION(_EMSCRIPTEN_NO_INDEXBUILDER_AND_SERVER "Don't build some targets when compiling with Emscripten." OFF)
 
-add_executable(qlever-server src/ServerMain.cpp)
-qlever_target_link_libraries(qlever-server engine server ${CMAKE_THREAD_LIBS_INIT} Boost::program_options compilationInfo global)
-if (USE_PRECOMPILED_HEADERS)
-    target_precompile_headers(qlever-server REUSE_FROM engine)
+if (NOT _EMSCRIPTEN_NO_INDEXBUILDER_AND_SERVER)
+    add_executable(qlever-index src/index/IndexBuilderMain.cpp)
+    qlever_target_link_libraries(qlever-index qlever index ${CMAKE_THREAD_LIBS_INIT} Boost::program_options compilationInfo global)
+
+    add_executable(qlever-server src/ServerMain.cpp)
+    qlever_target_link_libraries(qlever-server engine server ${CMAKE_THREAD_LIBS_INIT} Boost::program_options compilationInfo global)
+    if (USE_PRECOMPILED_HEADERS)
+        target_precompile_headers(qlever-server REUSE_FROM engine)
+    endif()
 endif()
 
 add_executable(LibQLeverExample src/libqlever/LibQLeverExample.cpp)
@@ -496,11 +500,13 @@ qlever_target_link_libraries(VocabularyMergerMain index parser ${CMAKE_THREAD_LI
 add_executable(PrintIndexVersionMain src/PrintIndexVersionMain.cpp)
 qlever_target_link_libraries(PrintIndexVersionMain util)
 
-install(TARGETS
-    qlever-index
-    qlever-server
-    RUNTIME DESTINATION bin
-)
+if (NOT _EMSCRIPTEN_NO_INDEXBUILDER_AND_SERVER)
+    install(TARGETS
+        qlever-index
+        qlever-server
+        RUNTIME DESTINATION bin
+    )
+endif()
 
 ###############################################################
 # CPack packaging

benchmark/CMakeLists.txt

@@ -1,34 +1,39 @@
 # Re-compiling all the infrastructure all the time is a waste of time, so we
 # pre-compile that part as a library.
 
-# This part compiles 'just' the benchmark infrastructure and doesn't provide a main function.
-add_library(benchmark infrastructure/Benchmark.cpp infrastructure/BenchmarkMeasurementContainer.cpp
-    infrastructure/BenchmarkToJson.cpp infrastructure/BenchmarkToString.cpp)
-qlever_target_link_libraries(benchmark Boost::program_options absl::str_format parser util configManager)
+OPTION(_NO_BENCHMARK "Disable benchmark targets, for example when building with Emscripten." OFF)
 
-# This provides the library of the benchmark infrastructure together with its default main function.
-add_library(benchmarkWithMain infrastructure/BenchmarkMain.cpp)
-qlever_target_link_libraries(benchmarkWithMain benchmark)
+if (NOT _NO_BENCHMARK)
+    # This part compiles 'just' the benchmark infrastructure and doesn't provide a main function.
+    add_library(benchmark infrastructure/Benchmark.cpp infrastructure/BenchmarkMeasurementContainer.cpp
+        infrastructure/BenchmarkToJson.cpp infrastructure/BenchmarkToString.cpp)
+    qlever_target_link_libraries(benchmark Boost::program_options absl::str_format parser util configManager)
 
-# Link binary ${basename} against the benchmark library,
-# and all libraries that are specified as additional arguments.
-function (linkBenchmark basename)
-    qlever_target_link_libraries(${basename} benchmarkWithMain ${ARGN})
-endfunction()
+    # This provides the library of the benchmark infrastructure together with its default main function.
+    add_library(benchmarkWithMain infrastructure/BenchmarkMain.cpp)
+    qlever_target_link_libraries(benchmarkWithMain benchmark)
 
-# Compile the benchmark and link it.
-# Usage: addAndLinkBenchmark(baseName, [additionalLibraries...])
-#   - baseName: The name of the benchmark file without the .cpp ending.
-function(addAndLinkBenchmark baseName)
-    add_executable(${baseName}  "${baseName}.cpp")
-    linkBenchmark(${baseName} ${ARGN})
-endfunction()
+    # Link binary ${basename} against the benchmark library,
+    # and all libraries that are specified as additional arguments.
+    function (linkBenchmark basename)
+        qlever_target_link_libraries(${basename} benchmarkWithMain ${ARGN})
+    endfunction()
 
-# Add benchmarks after here.
-addAndLinkBenchmark(BenchmarkExamples)
+    # Compile the benchmark and link it.
+    # Usage: addAndLinkBenchmark(baseName, [additionalLibraries...])
+    #   - baseName: The name of the benchmark file without the .cpp ending.
+    function(addAndLinkBenchmark baseName)
+        add_executable(${baseName}  "${baseName}.cpp")
+        linkBenchmark(${baseName} ${ARGN})
+    endfunction()
 
-addAndLinkBenchmark(JoinAlgorithmBenchmark testUtil memorySize)
+    # Add benchmarks after here.
+    addAndLinkBenchmark(BenchmarkExamples)
 
-addAndLinkBenchmark(ParallelMergeBenchmark testUtil)
+    addAndLinkBenchmark(JoinAlgorithmBenchmark testUtil memorySize)
 
-addAndLinkBenchmark(GroupByHashMapBenchmark engine testUtil gtest gmock)
+    addAndLinkBenchmark(ParallelMergeBenchmark testUtil)
+
+    addAndLinkBenchmark(GroupByHashMapBenchmark engine testUtil gtest gmock)
+
+endif()

conanprofiles/emscripten.profile

@@ -0,0 +1,47 @@
+[settings]
+arch=wasm64
+build_type=Release
+compiler=emcc
+compiler.cppstd=20
+compiler.libcxx=libc++
+# emsdk 3.1.73 is the latest version provided by the Conan package manager.
+compiler.version=3.1.73
+os=Emscripten
+compiler.threads=posix
+
+[tool_requires]
+emsdk/3.1.73
+
+[options]
+icu/*:with_icuio=False
+
+boost/*:without_atomic=True
+boost/*:without_charconv=True
+boost/*:without_chrono=True
+boost/*:without_context=True
+boost/*:without_contract=True
+boost/*:without_coroutine=True
+boost/*:without_date_time=True
+boost/*:without_exception=True
+boost/*:without_fiber=True
+boost/*:without_filesystem=True
+boost/*:without_graph=True
+boost/*:without_json=True
+boost/*:without_locale=True
+boost/*:without_log=True
+boost/*:without_math=True
+boost/*:without_nowide=True
+boost/*:without_process=True
+boost/*:without_serialization=True
+boost/*:without_stacktrace=True
+boost/*:without_test=True
+boost/*:without_thread=True
+boost/*:without_timer=True
+boost/*:without_type_erasure=True
+boost/*:without_wave=True
+
+[conf]
+tools.cmake.cmaketoolchain:generator=Ninja
+tools.build:exelinkflags=['-sALLOW_MEMORY_GROWTH=1', '-sMAXIMUM_MEMORY=4GB', '-sINITIAL_MEMORY=64MB', '-sMEMORY64=1', '-sUSE_ICU=1', '-sUSE_BOOST_HEADERS=1', '-sUSE_ZLIB=1', '-sUSE_BZIP2=1', '-fexceptions']
+tools.build:sharedlinkflags=['-sALLOW_MEMORY_GROWTH=1', '-sMAXIMUM_MEMORY=4GB', '-sINITIAL_MEMORY=64MB', '-sMEMORY64=1', '-sUSE_ICU=1', '-sUSE_BOOST_HEADERS=1', '-sUSE_ZLIB=1', '-sUSE_BZIP2=1', '-fexceptions']
+boost/*:tools.build:cxxflags=['-sMEMORY64=1']

src/ServerMain.cpp

@@ -53,6 +53,7 @@ int main(int argc, char** argv) {
   bool noPatterns;
   bool onlyPsoAndPosPermutations;
   bool persistUpdates;
+  std::vector<std::string> preloadMaterializedViews;
 
   ad_utility::MemorySize memoryMaxSize;
 
@@ -176,6 +177,11 @@ int main(int argc, char** argv) {
           &RuntimeParameters::materializedViewWriterMemory_>(),
       "Memory limit for sorting rows during the writing of materialized "
       "views.");
+  add("preload-materialized-views,l",
+      po::value<std::vector<std::string>>(&preloadMaterializedViews)
+          ->multitoken(),
+      "The names of materialized views to be loaded automatically on server "
+      "start (this option takes an arbitrary number of arguments).");
   po::variables_map optionsMap;
 
   try {
@@ -203,7 +209,7 @@ int main(int argc, char** argv) {
     Server server(port, numSimultaneousQueries, memoryMaxSize,
                   std::move(accessToken), noAccessCheck, !noPatterns);
     server.run(indexBasename, text, !noPatterns, !onlyPsoAndPosPermutations,
-               persistUpdates);
+               persistUpdates, preloadMaterializedViews);
   } catch (const std::exception& e) {
     // This code should never be reached as all exceptions should be handled
     // within server.run()

src/engine/CartesianProductJoin.cpp

@@ -1,6 +1,12 @@
-//  Copyright 2023, University of Freiburg,
-//                  Chair of Algorithms and Data Structures.
-//  Author: Johannes Kalmbach <[email protected]>
+// Copyright 2024 - 2026 The QLever Authors, in particular:
+//
+// 2023 - 2025 Johannes Kalmbach <[email protected]>, UFR
+// 2026        Hannah Bast <[email protected]>, UFR
+//
+// UFR = University of Freiburg, Chair of Algorithms and Data Structures
+
+// You may not use this file except in compliance with the Apache 2.0 License,
+// which can be found in the `LICENSE` file at the root of the QLever project.
 
 #include "engine/CartesianProductJoin.h"
 
@@ -360,7 +366,7 @@ CPP_template_def(typename R)(requires ql::ranges::range<R>) Result::LazyResult
 // _____________________________________________________________________________
 Result::LazyResult CartesianProductJoin::createLazyConsumer(
     LocalVocab staticMergedVocab,
-    ql::span<const std::shared_ptr<const Result>> subresults,
+    std::vector<std::shared_ptr<const Result>> subresults,
     std::shared_ptr<const Result> lazyResult) const {
   AD_CONTRACT_CHECK(lazyResult);
   std::vector<std::reference_wrapper<const IdTable>> idTables;
@@ -371,7 +377,8 @@ Result::LazyResult CartesianProductJoin::createLazyConsumer(
   auto get = [self = this, staticMergedVocab = std::move(staticMergedVocab),
               limit = getLimitOffset().limitOrDefault(),
               offset = getLimitOffset()._offset, idTables = std::move(idTables),
-              lastTableOffset = size_t{0}, producedTableSize = size_t{0},
+              subresults = std::move(subresults), lastTableOffset = size_t{0},
+              producedTableSize = size_t{0},
               idTableOpt = std::optional<Result::IdTableVocabPair>{}](
                  auto& idTableVocabPair) mutable {
     // These things have to be done after handling a single input, so we do them

src/engine/CartesianProductJoin.h

@@ -1,6 +1,12 @@
-//  Copyright 2023, University of Freiburg,
-//                  Chair of Algorithms and Data Structures.
-//  Author: Johannes Kalmbach <[email protected]>
+// Copyright 2024 - 2026 The QLever Authors, in particular:
+//
+// 2023 - 2025 Johannes Kalmbach <[email protected]>, UFR
+// 2026        Hannah Bast <[email protected]>, UFR
+//
+// UFR = University of Freiburg, Chair of Algorithms and Data Structures
+
+// You may not use this file except in compliance with the Apache 2.0 License,
+// which can be found in the `LICENSE` file at the root of the QLever project.
 
 #ifndef QLEVER_SRC_ENGINE_CARTESIANPRODUCTJOIN_H
 #define QLEVER_SRC_ENGINE_CARTESIANPRODUCTJOIN_H
@@ -132,7 +138,7 @@ class CartesianProductJoin : public Operation {
   // Similar to `produceTablesLazily` but can handle a single lazy result.
   Result::LazyResult createLazyConsumer(
       LocalVocab staticMergedVocab,
-      ql::span<const std::shared_ptr<const Result>> subresults,
+      std::vector<std::shared_ptr<const Result>> subresults,
       std::shared_ptr<const Result> lazyResult) const;
 };
 

src/engine/ExportQueryExecutionTrees.cpp

@@ -270,6 +270,22 @@ auto ExportQueryExecutionTrees::constructQueryResultToTriples(
       std::move(cancellationHandle));
 }
 
+// _____________________________________________________________________________
+template <>
+STREAMABLE_GENERATOR_TYPE ExportQueryExecutionTrees::
+    constructQueryResultToStream<ad_utility::MediaType::turtle>(
+        const QueryExecutionTree& qet,
+        const ad_utility::sparql_types::Triples& constructTriples,
+        LimitOffsetClause limitAndOffset, std::shared_ptr<const Result> result,
+        CancellationHandle cancellationHandle,
+        [[maybe_unused]] STREAMABLE_YIELDER_TYPE streamableYielder) {
+  result->logResultSize();
+  [[maybe_unused]] uint64_t resultSize = 0;
+  auto generator = constructQueryResultToTriples(
+      qet, constructTriples, limitAndOffset, result, resultSize,
+      std::move(cancellationHandle));
+}
+
 // _____________________________________________________________________________
 InputRangeTypeErased<std::string>
 ExportQueryExecutionTrees::constructQueryResultBindingsToQLeverJSON(

src/engine/ExportQueryExecutionTrees.h

@@ -251,6 +251,28 @@ class ExportQueryExecutionTrees {
   static ad_utility::InputRangeTypeErased<TableConstRefWithVocab> getIdTables(
       const Result& result);
 
+  // Generate the result in "blocks" and, when iterating over the generator
+  // from beginning to end, return the total number of rows in the result
+  // in `totalResultSize`.
+  //
+  // Blocks, where all rows are before OFFSET, are requested (and hence
+  // computed), but skipped.
+  //
+  // Blocks, where at least one row is after OFFSET but before the effective
+  // export limit (minimum of the LIMIT and the value of the `send` parameter),
+  // are requested and yielded (together with the corresponding `LocalVocab`
+  // and the range from that `IdTable` that belongs to the result).
+  //
+  // Blocks after the effective export limit until the LIMIT are requested, and
+  // counted towards the `totalResultSize`, but not yielded.
+  //
+  // Blocks after the LIMIT are not even requested.
+ public:
+  static ad_utility::InputRangeTypeErased<TableWithRange> getRowIndices(
+      const LimitOffsetClause& limitOffset, const Result& result,
+      uint64_t& resutSizeTotal, uint64_t resultSizeMultiplicator = 1);
+
+ private:
   FRIEND_TEST(ExportQueryExecutionTrees, getIdTablesReturnsSingletonIterator);
   FRIEND_TEST(ExportQueryExecutionTrees, getIdTablesMirrorsGenerator);
   FRIEND_TEST(ExportQueryExecutionTrees, ensureCorrectSlicingOfSingleIdTable);

src/engine/MaterializedViews.cpp

@@ -244,11 +244,9 @@ MaterializedViewWriter::RangeOfIdTables MaterializedViewWriter::getSortedBlocks(
 IndexMetaDataMmap MaterializedViewWriter::writePermutation(
     RangeOfIdTables sortedBlocksSPO) const {
   std::string spoFilename = getFilenameBase() + ".index.spo";
-  CompressedRelationWriter spoWriter{
-      numCols(),
-      ad_utility::File{spoFilename, "w"},
-      UNCOMPRESSED_BLOCKSIZE_COMPRESSED_METADATA_PER_COLUMN,
-  };
+  auto spoWriter = std::make_unique<CompressedRelationWriter>(
+      numCols(), ad_utility::File{spoFilename, "w"},
+      UNCOMPRESSED_BLOCKSIZE_COMPRESSED_METADATA_PER_COLUMN);
 
   qlever::KeyOrder spoKeyOrder{0, 1, 2, 3};
   IndexMetaDataMmap spoMetaData;
@@ -262,7 +260,7 @@ IndexMetaDataMmap MaterializedViewWriter::writePermutation(
 
   auto [numDistinctPredicates, blockData] =
       CompressedRelationWriter::createPermutation(
-          {spoWriter, spoCallback},
+          {std::move(spoWriter), spoCallback},
           ad_utility::InputRangeTypeErased{std::move(sortedBlocksSPO)},
           spoKeyOrder, {});