Rebuild index based on current data, including updates #2408

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

RobinTF wants to merge 70 commits into ad-freiburg:master from RobinTF:rebuild-index-and-vocab

src/engine/Server.cpp

-Original file line number
+Diff line change
@@ Expand Up / @@ -28,6 +28,7 @@ @@
     #include "engine/SparqlProtocol.h"
     #include "global/RuntimeParameters.h"
     #include "index/IndexImpl.h"
+    #include "index/IndexRebuilder.h"
     #include "parser/SparqlParser.h"
     #include "util/AsioHelpers.h"
     #include "util/Exception.h"
@@ Expand Down Expand Up / @@ -474,6 +475,38 @@ CPP_template_def(typename RequestT, typename ResponseT)( @@
           json[nlohmann::json(key)] = std::move(value);
         }
         response = createJsonResponse(json, request);
+      } else if (auto cmd = checkParameter("cmd", "rebuild-index")) {
+        requireValidAccessToken("rebuild-index");
+        if (rebuildInProgress_.exchange(true)) {
+          response = createHttpResponseFromString(
+              "Another rebuild is currently in progress!",
+              http::status::too_many_requests, request, MediaType::textPlain);
+        } else {
+          absl::Cleanup cleanup{[this]() { rebuildInProgress_.store(false); }};
+          logCommand(cmd, "rebuilding index");
+          auto fileName =
+              checkParameter("index-name", std::nullopt).value_or("new_index");
+          // There is no mechanism to actually cancel the handle.
+          auto handle = std::make_shared<ad_utility::CancellationHandle<>>();
+          // We don't directly `co_await` because of lifetime issues (bugs) in the
+          // Conan setup.
+          auto coroutine = computeInNewThread(
+              queryThreadPool_,
+              [this, &handle, fileName = std::move(fileName)] {
+                auto logFileName = fileName + ".rebuild-index-log.txt";
+                auto [currentSnapshot, localVocabCopy, ownedBlocks] =
+                    index_.deltaTriplesManager()
+                        .getCurrentLocatedTriplesSharedStateWithVocab();
+                qlever::materializeToIndex(index_.getImpl(), fileName,
+                                           currentSnapshot, localVocabCopy,
+                                           ownedBlocks, handle, logFileName);
+              },
+              handle);
+          co_await std::move(coroutine);
+          response =
+              createOkResponse("Done writing", request, MediaType::textPlain);
+        }
       } else if (auto cmd = checkParameter("cmd", "write-materialized-view")) {
         requireValidAccessToken("write-materialized-view");
         logCommand(cmd, "write materialized view");
@@ Expand Down @@

src/engine/Server.h

-Original file line number
+Diff line change
@@ Expand Up / @@ -113,6 +113,10 @@ class Server { @@
       /// Executor with a single thread that is used to run timers asynchronously.
       boost::asio::static_thread_pool timerExecutor_{1};
+      // Indicates if an index rebuild is currently in progress so that we prevent
+      // triggering this twice.
+      std::atomic_bool rebuildInProgress_{false};
       template <typename T>
       using Awaitable = boost::asio::awaitable<T>;
@@ Expand Down @@

src/index/CMakeLists.txt

-Original file line number
+Diff line change
@@ Expand Up / @@ -7,5 +7,5 @@ add_library(index @@
             PrefixHeuristic.cpp CompressedRelation.cpp
             PatternCreator.cpp ScanSpecification.cpp
             DeltaTriples.cpp LocalVocabEntry.cpp TextScoring.cpp TextScoringEnum.cpp TextIndexReadWrite.cpp
-            TextIndexBuilder.cpp GraphFilter.cpp)
+            TextIndexBuilder.cpp GraphFilter.cpp IndexRebuilder.cpp)
     qlever_target_link_libraries(index util parser vocabulary global)

src/index/IndexImpl.cpp

-Original file line number
+Diff line change
@@ Expand Up / @@ -49,9 +49,12 @@ constexpr std::string_view BLANK_NODE_ALLOCATION_START = @@
         "num-blank-nodes-total";
     // _____________________________________________________________________________
-    IndexImpl::IndexImpl(ad_utility::AllocatorWithLimit<Id> allocator)
+    IndexImpl::IndexImpl(ad_utility::AllocatorWithLimit<Id> allocator,
+                         bool registerSingleton)
         : allocator_{std::move(allocator)} {
-      globalSingletonIndex_ = this;
+      if (registerSingleton) {
+        globalSingletonIndex_ = this;
+      }
       deltaTriples_.emplace(*this);
     }
@@ Expand Down Expand Up / @@ -1138,6 +1141,9 @@ void IndexImpl::setKeepTempFiles(bool keepTempFiles) { @@
     // _____________________________________________________________________________
     bool& IndexImpl::usePatterns() { return usePatterns_; }
+    // _____________________________________________________________________________
+    bool IndexImpl::usePatterns() const { return usePatterns_; }
     // _____________________________________________________________________________
     bool& IndexImpl::loadAllPermutations() { return loadAllPermutations_; }
@@ Expand Down Expand Up / @@ -1900,6 +1906,22 @@ void IndexImpl::writePatternsToFile() const { @@
                                           statistics);
     }
+    // _____________________________________________________________________________
+    void IndexImpl::loadConfigFromOldIndex(const std::string& newName,
+                                           const IndexImpl& other,
+                                           const nlohmann::json& newStats) {
+      setOnDiskBase(newName);
+      setKbName(other.getKbName());
+      blocksizePermutationPerColumn() = other.blocksizePermutationPerColumn();
+      configurationJson_ = newStats;
+      numTriples_ = static_cast<NumNormalAndInternal>(newStats["num-triples"]);
+      numPredicates_ =
+          static_cast<NumNormalAndInternal>(newStats["num-predicates"]);
+      numSubjects_ = static_cast<NumNormalAndInternal>(newStats["num-subjects"]);
+      numObjects_ = static_cast<NumNormalAndInternal>(newStats["num-objects"]);
+      writeConfiguration();
+    }
     // _____________________________________________________________________________
     void IndexImpl::countDistinct(std::optional<Id>& lastId, size_t& counter,
                                   const IdTable& table) {
@@ Expand Down @@

src/index/IndexImpl.h

-Original file line number
+Diff line change
@@ Expand Up / @@ -208,7 +208,8 @@ class IndexImpl { @@
       std::optional<DeltaTriplesManager> deltaTriples_;
      public:
-      explicit IndexImpl(ad_utility::AllocatorWithLimit<Id> allocator);
+      explicit IndexImpl(ad_utility::AllocatorWithLimit<Id> allocator,
+                         bool registerSingleton = true);
       // Forbid copying.
       IndexImpl& operator=(const IndexImpl&) = delete;
@@ Expand Down Expand Up / @@ -273,6 +274,10 @@ class IndexImpl { @@
       const auto& getScoreData() const { return scoreData_; }
+      const ad_utility::AllocatorWithLimit<Id>& allocator() const {
+        return allocator_;
+      };
       ad_utility::BlankNodeManager* getBlankNodeManager() const;
       DeltaTriplesManager& deltaTriplesManager() { return deltaTriples_.value(); }
@@ Expand Down Expand Up / @@ -442,6 +447,8 @@ class IndexImpl { @@
       bool& usePatterns();
+      bool usePatterns() const;
       bool& loadAllPermutations();
       bool& doNotLoadPermutations();
@@ Expand All / @@ -464,6 +471,10 @@ class IndexImpl { @@
         return blocksizePermutationPerColumn_;
       }
+      const ad_utility::MemorySize& blocksizePermutationPerColumn() const {
+        return blocksizePermutationPerColumn_;
+      }
       void setOnDiskBase(const std::string& onDiskBase);
       void setSettingsFile(const std::string& filename);
@@ Expand Down Expand Up / @@ -698,6 +709,7 @@ class IndexImpl { @@
       friend class CreatePatternsFixture_createPatterns_Test;
       FRIEND_TEST(IndexImpl, recomputeStatistics);
       FRIEND_TEST(IndexImpl, writePatternsToFile);
+      FRIEND_TEST(IndexImpl, loadConfigFromOldIndex);
       bool isLiteral(std::string_view object) const;
@@ Expand Down Expand Up / @@ -862,6 +874,13 @@ class IndexImpl { @@
       void storeTextScoringParamsInConfiguration(TextScoringMetric scoringMetric,
                                                  float b, float k);
+      // Overwrite the config of this instance of `IndexImpl` with the config of
+      // `other`, adjusting the name to `newName` and the statistics to
+      // `newStats`.
+      void loadConfigFromOldIndex(const std::string& newName,
+                                  const IndexImpl& other,
+                                  const nlohmann::json& newStats);
       // Write the stored in-memory patterns to a pattern file.
       void writePatternsToFile() const;
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Rebuild index based on current data, including updates #2408

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Rebuild index based on current data, including updates #2408

Are you sure you want to change the base?

Uh oh!

Rebuild index based on current data, including updates #2408

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!