From cb2eb4e603342b3b94a49a608024a93c4a7d9129 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Wed, 17 Dec 2025 16:50:30 +0100 Subject: [PATCH 01/17] wip # Conflicts: # src/engine/HasPredicateScan.cpp # src/index/IndexImpl.cpp # src/index/Permutation.cpp --- src/engine/CountAvailablePredicates.cpp | 3 +- src/engine/GroupByImpl.cpp | 6 ++- src/engine/IndexScan.cpp | 28 ++++++----- src/engine/IndexScan.h | 10 ++-- src/index/DeltaTriples.h | 10 +++- src/index/IndexImpl.cpp | 15 ++++-- src/index/Permutation.cpp | 63 ++++++++++--------------- src/index/Permutation.h | 23 +++++---- 8 files changed, 83 insertions(+), 75 deletions(-) diff --git a/src/engine/CountAvailablePredicates.cpp b/src/engine/CountAvailablePredicates.cpp index dc270d6943..45b1649344 100644 --- a/src/engine/CountAvailablePredicates.cpp +++ b/src/engine/CountAvailablePredicates.cpp @@ -169,7 +169,8 @@ void CountAvailablePredicates::computePatternTrickAllEntities( std::nullopt} .toScanSpecification(index); const auto& perm = index.getPermutation(Permutation::Enum::PSO); - const auto& locatedTriple = locatedTriplesSnapshot(); + const auto& locatedTriple = + perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot()); auto fullHasPattern = perm.lazyScan(perm.getScanSpecAndBlocks(scanSpec, locatedTriple), std::nullopt, {}, cancellationHandle_, locatedTriple); diff --git a/src/engine/GroupByImpl.cpp b/src/engine/GroupByImpl.cpp index bbe9a46c9b..a4b89dfd10 100644 --- a/src/engine/GroupByImpl.cpp +++ b/src/engine/GroupByImpl.cpp @@ -841,7 +841,8 @@ std::optional GroupByImpl::computeGroupByObjectWithCount() const { // do the index scan, but something smarter). const auto& permutation = indexScan->permutation(); auto result = permutation.getDistinctCol1IdsAndCounts( - col0Id.value(), cancellationHandle_, locatedTriplesSnapshot(), + col0Id.value(), cancellationHandle_, + permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot()), indexScan->getLimitOffset()); indexScan->updateRuntimeInformationWhenOptimizedOut({}); @@ -900,7 +901,8 @@ std::optional GroupByImpl::computeGroupByForFullIndexScan() const { getExecutionContext()->getIndex().getPimpl().getPermutation( permutationEnum.value()); auto table = permutation.getDistinctCol0IdsAndCounts( - cancellationHandle_, locatedTriplesSnapshot(), + cancellationHandle_, + permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot()), indexScan->getLimitOffset()); if (numCounts == 0) { table.setColumnSubset(std::array{ColumnIndex{0}}); diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index c6840aa7b9..e1a7fb0de5 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -32,7 +32,7 @@ static size_t getNumberOfVariables(const TripleComponent& subject, // _____________________________________________________________________________ IndexScan::IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, - LocatedTriplesSnapshotPtr locatedTriplesSnapshot, + LocatedTriplesPerBlockPtr locatedTriplesSnapshot, const SparqlTripleSimple& triple, Graphs graphsToFilter, std::optional scanSpecAndBlocks, VarsToKeep varsToKeep) @@ -78,14 +78,16 @@ IndexScan::IndexScan(QueryExecutionContext* qec, Permutation::Enum permutationType, const SparqlTripleSimple& triple, Graphs graphsToFilter, std::optional scanSpecAndBlocks) - : IndexScan(qec, - qec->getIndex().getImpl().getPermutationPtr(permutationType), - qec->sharedLocatedTriplesSnapshot(), triple, - std::move(graphsToFilter), std::move(scanSpecAndBlocks)) {} + : IndexScan( + qec, qec->getIndex().getImpl().getPermutationPtr(permutationType), + // TODO: choose the right snapshot (internal/external) + qec->sharedLocatedTriplesSnapshot().getLocatedTriplesForPermutation( + permutationType), + triple, std::move(graphsToFilter), std::move(scanSpecAndBlocks)) {} // _____________________________________________________________________________ IndexScan::IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, - LocatedTriplesSnapshotPtr locatedTriplesSnapshot, + LocatedTriplesPerBlockPtr locatedTriplesSnapshot, const TripleComponent& s, const TripleComponent& p, const TripleComponent& o, std::vector additionalColumns, @@ -277,7 +279,7 @@ Result::LazyResult IndexScan::chunkedIndexScan() const { IdTable IndexScan::materializedIndexScan() const { IdTable idTable = permutation().scan( scanSpecAndBlocks_, additionalColumns(), cancellationHandle_, - locatedTriplesSnapshot(), getLimitOffset()); + locatedTriplesPerBlock(), getLimitOffset()); AD_LOG_DEBUG << "IndexScan result computation done.\n"; checkCancellation(); idTable = makeApplyColumnSubset()(std::move(idTable)); @@ -301,7 +303,7 @@ const Permutation& IndexScan::permutation() const { } // _____________________________________________________________________________ -const LocatedTriplesSnapshot& IndexScan::locatedTriplesSnapshot() const { +const LocatedTriplesPerBlock& IndexScan::locatedTriplesPerBlock() const { AD_CONTRACT_CHECK(locatedTriplesSnapshot_ != nullptr); return *locatedTriplesSnapshot_; } @@ -310,7 +312,7 @@ const LocatedTriplesSnapshot& IndexScan::locatedTriplesSnapshot() const { std::pair IndexScan::computeSizeEstimate() const { AD_CORRECTNESS_CHECK(_executionContext); auto [lower, upper] = permutation().getSizeEstimateForScan( - scanSpecAndBlocks_, locatedTriplesSnapshot()); + scanSpecAndBlocks_, locatedTriplesPerBlock()); // NOTE: Starting from C++20 we could use `std::midpoint` here return {lower == upper, lower + (upper - lower) / 2}; } @@ -319,7 +321,7 @@ std::pair IndexScan::computeSizeEstimate() const { size_t IndexScan::getExactSize() const { AD_CORRECTNESS_CHECK(_executionContext); return permutation().getResultSizeOfScan(scanSpecAndBlocks_, - locatedTriplesSnapshot()); + locatedTriplesPerBlock()); } // _____________________________________________________________________________ @@ -397,7 +399,7 @@ IndexScan::getSortedVariableAndMetadataColumnIndexForPrefiltering() const { // ___________________________________________________________________________ Permutation::ScanSpecAndBlocks IndexScan::getScanSpecAndBlocks() const { return permutation().getScanSpecAndBlocks(getScanSpecification(), - locatedTriplesSnapshot()); + locatedTriplesPerBlock()); } // _____________________________________________________________________________ @@ -411,7 +413,7 @@ CompressedRelationReader::IdTableGeneratorInputRange IndexScan::getLazyScan( getLimitOffset().isUnconstrained() ? std::move(blocks) : std::nullopt; auto lazyScanAllCols = permutation().lazyScan( scanSpecAndBlocks_, filteredBlocks, additionalColumns(), - cancellationHandle_, locatedTriplesSnapshot(), getLimitOffset()); + cancellationHandle_, locatedTriplesPerBlock(), getLimitOffset()); return CompressedRelationReader::IdTableGeneratorInputRange{ ad_utility::CachingTransformInputRange< @@ -425,7 +427,7 @@ CompressedRelationReader::IdTableGeneratorInputRange IndexScan::getLazyScan( std::optional IndexScan::getMetadataForScan() const { return permutation().getMetadataAndBlocks(scanSpecAndBlocks_, - locatedTriplesSnapshot()); + locatedTriplesPerBlock()); }; // _____________________________________________________________________________ diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index d25ebeaf26..90abcb8a9f 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -19,13 +19,15 @@ class IndexScan final : public Operation { using PermutationPtr = std::shared_ptr; using LocatedTriplesSnapshotPtr = std::shared_ptr; + using LocatedTriplesPerBlockPtr = + std::shared_ptr; private: using ScanSpecAndBlocks = Permutation::ScanSpecAndBlocks; private: PermutationPtr permutation_; - LocatedTriplesSnapshotPtr locatedTriplesSnapshot_; + LocatedTriplesPerBlockPtr locatedTriplesSnapshot_; TripleComponent subject_; TripleComponent predicate_; TripleComponent object_; @@ -51,7 +53,7 @@ class IndexScan final : public Operation { public: IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, - LocatedTriplesSnapshotPtr locatedTriplesSnapshot, + LocatedTriplesPerBlockPtr locatedTriplesSnapshot, const SparqlTripleSimple& triple, Graphs graphsToFilter = Graphs::All(), std::optional scanSpecAndBlocks = std::nullopt, @@ -67,7 +69,7 @@ class IndexScan final : public Operation { // Constructor to simplify copy creation of an `IndexScan`. IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, - LocatedTriplesSnapshotPtr locatedTriplesSnapshot, + LocatedTriplesPerBlockPtr locatedTriplesSnapshot, const TripleComponent& s, const TripleComponent& p, const TripleComponent& o, std::vector additionalColumns, @@ -188,7 +190,7 @@ class IndexScan final : public Operation { // class, which accesses the one stored in the `QueryExecutionContext`, use // the `LocatedTriplesSnapshot` held in this object. This might be a different // one if a custom permutation is used. - const LocatedTriplesSnapshot& locatedTriplesSnapshot() const override; + const LocatedTriplesPerBlock& locatedTriplesPerBlock() const; // Return the stored triple in the order that corresponds to the // `permutation_`. For example if `permutation_ == PSO` then the result is diff --git a/src/index/DeltaTriples.h b/src/index/DeltaTriples.h index c46eedf922..3569c59d37 100644 --- a/src/index/DeltaTriples.h +++ b/src/index/DeltaTriples.h @@ -52,7 +52,15 @@ struct LocatedTriplesSnapshot { // A shared pointer to a constant `LocatedTriplesSnapshot`, but as an explicit // class, such that it can be forward-declared. class SharedLocatedTriplesSnapshot - : public std::shared_ptr {}; + : public std::shared_ptr { + public: + std::shared_ptr getLocatedTriplesForPermutation( + Permutation::Enum permutation) const { + return {*this, &this->get()->getLocatedTriplesForPermutation(permutation)}; + } + std::shared_ptr + getInternalLocatedTriplesForPermutation(Permutation::Enum permutation) const; +}; // A class for keeping track of the number of triples of the `DeltaTriples`. struct DeltaTriplesCount { diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index d585e8d287..79f3cf3b5c 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -1568,8 +1568,9 @@ Index::NumNormalAndInternal IndexImpl::numDistinctCol0( size_t IndexImpl::getCardinality( Id id, Permutation::Enum permutation, const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { - if (const auto& meta = - getPermutation(permutation).getMetadata(id, locatedTriplesSnapshot); + const auto& perm = getPermutation(permutation); + if (const auto& meta = perm.getMetadata( + id, perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot)); meta.has_value()) { return meta.value().numRows_; } @@ -1618,7 +1619,9 @@ std::vector IndexImpl::getMultiplicities( const TripleComponent& key, const Permutation& permutation, const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { if (auto keyId = key.toValueId(getVocab(), encodedIriManager())) { - auto meta = permutation.getMetadata(keyId.value(), locatedTriplesSnapshot); + auto meta = permutation.getMetadata( + keyId.value(), + permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot)); if (meta.has_value()) { return {meta.value().getCol1Multiplicity(), meta.value().getCol2Multiplicity()}; @@ -1644,9 +1647,11 @@ size_t IndexImpl::getResultSizeOfScan( const Permutation::Enum& permutation, const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { const auto& perm = getPermutation(permutation); + const auto& locatedTriples = + perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot); return perm.getResultSizeOfScan( - perm.getScanSpecAndBlocks(scanSpecification, locatedTriplesSnapshot), - locatedTriplesSnapshot); + perm.getScanSpecAndBlocks(scanSpecification, locatedTriples), + locatedTriples); } // _____________________________________________________________________________ diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp index 3b41e44f0d..e552ccbdf2 100644 --- a/src/index/Permutation.cpp +++ b/src/index/Permutation.cpp @@ -21,9 +21,9 @@ Permutation::Permutation(Enum permutation, Allocator allocator) // _____________________________________________________________________ CompressedRelationReader::ScanSpecAndBlocks Permutation::getScanSpecAndBlocks( const ScanSpecification& scanSpec, - const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { - return {scanSpec, BlockMetadataRanges(getAugmentedMetadataForPermutation( - locatedTriplesSnapshot))}; + const LocatedTriplesPerBlock& locatedTriples) const { + return {scanSpec, BlockMetadataRanges( + getAugmentedMetadataForPermutation(locatedTriples))}; } // _____________________________________________________________________ @@ -73,58 +73,51 @@ void Permutation::setOriginalMetadataForDeltaTriples( IdTable Permutation::scan(const ScanSpecAndBlocks& scanSpecAndBlocks, ColumnIndicesRef additionalColumns, const CancellationHandle& cancellationHandle, - const LocatedTriplesSnapshot& locatedTriplesSnapshot, + const LocatedTriplesPerBlock& locatedTriples, const LimitOffsetClause& limitOffset) const { if (!isLoaded_) { throw std::runtime_error("This query requires the permutation " + readableName_ + ", which was not loaded"); } return reader().scan(scanSpecAndBlocks, additionalColumns, cancellationHandle, - getLocatedTriplesForPermutation(locatedTriplesSnapshot), - limitOffset); + locatedTriples, limitOffset); } // _____________________________________________________________________ size_t Permutation::getResultSizeOfScan( const ScanSpecAndBlocks& scanSpecAndBlocks, - const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { - return reader().getResultSizeOfScan( - scanSpecAndBlocks, - getLocatedTriplesForPermutation(locatedTriplesSnapshot)); + const LocatedTriplesPerBlock& locatedTriples) const { + return reader().getResultSizeOfScan(scanSpecAndBlocks, locatedTriples); } // _____________________________________________________________________ std::pair Permutation::getSizeEstimateForScan( const ScanSpecAndBlocks& scanSpecAndBlocks, - const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { - return reader().getSizeEstimateForScan( - scanSpecAndBlocks, - getLocatedTriplesForPermutation(locatedTriplesSnapshot)); + const LocatedTriplesPerBlock& locatedTriples) const { + return reader().getSizeEstimateForScan(scanSpecAndBlocks, locatedTriples); } // ____________________________________________________________________________ IdTable Permutation::getDistinctCol1IdsAndCounts( Id col0Id, const CancellationHandle& cancellationHandle, - const LocatedTriplesSnapshot& locatedTriplesSnapshot, + const LocatedTriplesPerBlock& locatedTriples, const LimitOffsetClause& limitOffset) const { return reader().getDistinctCol1IdsAndCounts( getScanSpecAndBlocks( ScanSpecification{col0Id, std::nullopt, std::nullopt}, - locatedTriplesSnapshot), - cancellationHandle, - getLocatedTriplesForPermutation(locatedTriplesSnapshot), limitOffset); + locatedTriples), + cancellationHandle, locatedTriples, limitOffset); } // ____________________________________________________________________________ IdTable Permutation::getDistinctCol0IdsAndCounts( const CancellationHandle& cancellationHandle, - const LocatedTriplesSnapshot& locatedTriplesSnapshot, + const LocatedTriplesPerBlock& locatedTriples, const LimitOffsetClause& limitOffset) const { ScanSpecification scanSpec{std::nullopt, std::nullopt, std::nullopt}; return reader().getDistinctCol0IdsAndCounts( - getScanSpecAndBlocks(scanSpec, locatedTriplesSnapshot), - cancellationHandle, - getLocatedTriplesForPermutation(locatedTriplesSnapshot), limitOffset); + getScanSpecAndBlocks(scanSpec, locatedTriples), cancellationHandle, + locatedTriples, limitOffset); } // _____________________________________________________________________ @@ -169,24 +162,23 @@ std::string_view Permutation::toString(Permutation::Enum permutation) { // _____________________________________________________________________ std::optional Permutation::getMetadata( - Id col0Id, const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { + Id col0Id, const LocatedTriplesPerBlock& locatedTriples) const { if (meta_.col0IdExists(col0Id)) { return meta_.getMetaData(col0Id); } return reader().getMetadataForSmallRelation( getScanSpecAndBlocks( ScanSpecification{col0Id, std::nullopt, std::nullopt}, - locatedTriplesSnapshot), - col0Id, getLocatedTriplesForPermutation(locatedTriplesSnapshot)); + locatedTriples), + col0Id, locatedTriples); } // _____________________________________________________________________ std::optional Permutation::getMetadataAndBlocks( const ScanSpecAndBlocks& scanSpecAndBlocks, - const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { + const LocatedTriplesPerBlock& locatedTriples) const { auto firstAndLastTriple = reader().getFirstAndLastTripleIgnoringGraph( - scanSpecAndBlocks, - getLocatedTriplesForPermutation(locatedTriplesSnapshot)); + scanSpecAndBlocks, locatedTriples); if (!firstAndLastTriple.has_value()) { return std::nullopt; } @@ -200,17 +192,16 @@ CompressedRelationReader::IdTableGeneratorInputRange Permutation::lazyScan( std::optional> optBlocks, ColumnIndicesRef additionalColumns, const CancellationHandle& cancellationHandle, - const LocatedTriplesSnapshot& locatedTriplesSnapshot, + const LocatedTriplesPerBlock& locatedTriples, const LimitOffsetClause& limitOffset) const { ColumnIndices columns{additionalColumns.begin(), additionalColumns.end()}; if (!optBlocks.has_value()) { optBlocks = CompressedRelationReader::convertBlockMetadataRangesToVector( scanSpecAndBlocks.blockMetadata_); } - return reader().lazyScan( - scanSpecAndBlocks.scanSpec_, std::move(optBlocks.value()), - std::move(columns), cancellationHandle, - getLocatedTriplesForPermutation(locatedTriplesSnapshot), limitOffset); + return reader().lazyScan(scanSpecAndBlocks.scanSpec_, + std::move(optBlocks.value()), std::move(columns), + cancellationHandle, locatedTriples, limitOffset); } // ______________________________________________________________________ @@ -225,10 +216,8 @@ const LocatedTriplesPerBlock& Permutation::getLocatedTriplesForPermutation( // ______________________________________________________________________ BlockMetadataRanges Permutation::getAugmentedMetadataForPermutation( - const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { - BlockMetadataSpan blocks( - getLocatedTriplesForPermutation(locatedTriplesSnapshot) - .getAugmentedMetadata()); + const LocatedTriplesPerBlock& locatedTriples) const { + BlockMetadataSpan blocks(locatedTriples.getAugmentedMetadata()); return {{blocks.begin(), blocks.end()}}; } diff --git a/src/index/Permutation.h b/src/index/Permutation.h index bacf418855..27f11fce3f 100644 --- a/src/index/Permutation.h +++ b/src/index/Permutation.h @@ -84,7 +84,7 @@ class Permutation { IdTable scan(const ScanSpecAndBlocks& scanSpecAndBlocks, ColumnIndicesRef additionalColumns, const CancellationHandle& cancellationHandle, - const LocatedTriplesSnapshot& locatedTriplesSnapshot, + const LocatedTriplesPerBlock& locatedTriples, const LimitOffsetClause& limitOffset = {}) const; // For a given relation, determine the `col1Id`s and their counts. This is @@ -92,12 +92,12 @@ class Permutation { // in `meta_`. IdTable getDistinctCol1IdsAndCounts( Id col0Id, const CancellationHandle& cancellationHandle, - const LocatedTriplesSnapshot& locatedTriplesSnapshot, + const LocatedTriplesPerBlock& locatedTriples, const LimitOffsetClause& limitOffset) const; IdTable getDistinctCol0IdsAndCounts( const CancellationHandle& cancellationHandle, - const LocatedTriplesSnapshot& locatedTriplesSnapshot, + const LocatedTriplesPerBlock& locatedTriples, const LimitOffsetClause& limitOffset) const; // Typedef to propagate the `MetadataAndblocks` and `IdTableGenerator` type. @@ -126,17 +126,17 @@ class Permutation { std::optional> optBlocks, ColumnIndicesRef additionalColumns, const CancellationHandle& cancellationHandle, - const LocatedTriplesSnapshot& locatedTriplesSnapshot, + const LocatedTriplesPerBlock& locatedTriples, const LimitOffsetClause& limitOffset = {}) const; // Returns the corresponding `CompressedRelationReader::ScanSpecAndBlocks` // with relevant `BlockMetadataRanges`. ScanSpecAndBlocks getScanSpecAndBlocks( const ScanSpecification& scanSpec, - const LocatedTriplesSnapshot& locatedTriplesSnapshot) const; + const LocatedTriplesPerBlock& locatedTriples) const; std::optional getMetadata( - Id col0Id, const LocatedTriplesSnapshot& locatedTriplesSnapshot) const; + Id col0Id, const LocatedTriplesPerBlock& locatedTriples) const; // Return the metadata for the scan specified by the `scanSpecification` // along with the metadata for all the blocks that are relevant for this @@ -144,21 +144,21 @@ class Permutation { // be empty) return `nullopt`. std::optional getMetadataAndBlocks( const ScanSpecAndBlocks& scanSpecAndBlocks, - const LocatedTriplesSnapshot& locatedTriplesSnapshot) const; + const LocatedTriplesPerBlock& locatedTriples) const; // Get the exact size of the result of a scan, taking into account the // given located triples. This requires an exact location of the delta // triples within the respective blocks. size_t getResultSizeOfScan( const ScanSpecAndBlocks& scanSpecAndBlocks, - const LocatedTriplesSnapshot& locatedTriplesSnapshot) const; + const LocatedTriplesPerBlock& locatedTriples) const; // Get a lower and upper bound for the size of the result of a scan, taking // into account the given `deltaTriples`. For this call, it is enough that // each delta triple know to which block it belongs. std::pair getSizeEstimateForScan( const ScanSpecAndBlocks& scanSpecAndBlocks, - const LocatedTriplesSnapshot& locatedTriplesSnapshot) const; + const LocatedTriplesPerBlock& locatedTriples) const; // _______________________________________________________ void setKbName(const std::string& name) { meta_.setName(name); } @@ -188,10 +188,9 @@ class Permutation { const LocatedTriplesPerBlock& getLocatedTriplesForPermutation( const LocatedTriplesSnapshot& locatedTriplesSnapshot) const; - // From the given snapshot, get the augmented block metadata for this - // permutation. + // From the given `LocatedTriplesPerBlock`, get the augmented block metadata. BlockMetadataRanges getAugmentedMetadataForPermutation( - const LocatedTriplesSnapshot& locatedTriplesSnapshot) const; + const LocatedTriplesPerBlock& locatedTriples) const; const CompressedRelationReader& reader() const { return reader_.value(); } From 182b23e661628bfe1db549c4bebbdd84857d43d7 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Thu, 18 Dec 2025 12:59:09 +0100 Subject: [PATCH 02/17] wip --- src/engine/CountAvailablePredicates.cpp | 3 +- src/engine/GroupByImpl.cpp | 9 +- src/engine/IndexScan.cpp | 1 - src/index/DeltaTriples.h | 5 +- src/index/IndexImpl.cpp | 9 +- src/index/Permutation.cpp | 11 +++ src/index/Permutation.h | 2 + test/CompressedRelationsTest.cpp | 9 +- test/IndexTest.cpp | 108 +++++++++++------------- test/util/IndexTestHelpers.cpp | 8 +- 10 files changed, 92 insertions(+), 73 deletions(-) diff --git a/src/engine/CountAvailablePredicates.cpp b/src/engine/CountAvailablePredicates.cpp index 45b1649344..d837fa4aa9 100644 --- a/src/engine/CountAvailablePredicates.cpp +++ b/src/engine/CountAvailablePredicates.cpp @@ -170,7 +170,8 @@ void CountAvailablePredicates::computePatternTrickAllEntities( .toScanSpecification(index); const auto& perm = index.getPermutation(Permutation::Enum::PSO); const auto& locatedTriple = - perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot()); + perm.getActualPermutation(scanSpec).getLocatedTriplesForPermutation( + locatedTriplesSnapshot()); auto fullHasPattern = perm.lazyScan(perm.getScanSpecAndBlocks(scanSpec, locatedTriple), std::nullopt, {}, cancellationHandle_, locatedTriple); diff --git a/src/engine/GroupByImpl.cpp b/src/engine/GroupByImpl.cpp index a4b89dfd10..1cbeca85c6 100644 --- a/src/engine/GroupByImpl.cpp +++ b/src/engine/GroupByImpl.cpp @@ -430,7 +430,7 @@ void GroupByImpl::processGroup( evaluationContext._previousResultsFromSameGroup.at(resultColumn) = sparqlExpression::copyExpressionResult(expressionResult); - auto visitor = CPP_template_lambda_mut(&)(typename T)(T && singleResult)( + auto visitor = CPP_template_lambda_mut (&)(typename T)(T && singleResult)( requires sparqlExpression::SingleExpressionResult) { constexpr static bool isStrongId = std::is_same_v; if constexpr (isStrongId) { @@ -842,7 +842,8 @@ std::optional GroupByImpl::computeGroupByObjectWithCount() const { const auto& permutation = indexScan->permutation(); auto result = permutation.getDistinctCol1IdsAndCounts( col0Id.value(), cancellationHandle_, - permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot()), + permutation.getActualPermutation(col0Id.value()) + .getLocatedTriplesForPermutation(locatedTriplesSnapshot()), indexScan->getLimitOffset()); indexScan->updateRuntimeInformationWhenOptimizedOut({}); @@ -902,7 +903,9 @@ std::optional GroupByImpl::computeGroupByForFullIndexScan() const { permutationEnum.value()); auto table = permutation.getDistinctCol0IdsAndCounts( cancellationHandle_, - permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot()), + permutation + .getActualPermutation({std::nullopt, std::nullopt, std::nullopt}) + .getLocatedTriplesForPermutation(locatedTriplesSnapshot()), indexScan->getLimitOffset()); if (numCounts == 0) { table.setColumnSubset(std::array{ColumnIndex{0}}); diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index e1a7fb0de5..aa345dc081 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -80,7 +80,6 @@ IndexScan::IndexScan(QueryExecutionContext* qec, std::optional scanSpecAndBlocks) : IndexScan( qec, qec->getIndex().getImpl().getPermutationPtr(permutationType), - // TODO: choose the right snapshot (internal/external) qec->sharedLocatedTriplesSnapshot().getLocatedTriplesForPermutation( permutationType), triple, std::move(graphsToFilter), std::move(scanSpecAndBlocks)) {} diff --git a/src/index/DeltaTriples.h b/src/index/DeltaTriples.h index 3569c59d37..8baa2e2667 100644 --- a/src/index/DeltaTriples.h +++ b/src/index/DeltaTriples.h @@ -59,7 +59,10 @@ class SharedLocatedTriplesSnapshot return {*this, &this->get()->getLocatedTriplesForPermutation(permutation)}; } std::shared_ptr - getInternalLocatedTriplesForPermutation(Permutation::Enum permutation) const; + getInternalLocatedTriplesForPermutation(Permutation::Enum permutation) const { + return {*this, + &this->get()->getInternalLocatedTriplesForPermutation(permutation)}; + } }; // A class for keeping track of the number of triples of the `DeltaTriples`. diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index 79f3cf3b5c..22a4aae3ac 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -1570,7 +1570,8 @@ size_t IndexImpl::getCardinality( const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { const auto& perm = getPermutation(permutation); if (const auto& meta = perm.getMetadata( - id, perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot)); + id, perm.getActualPermutation(id).getLocatedTriplesForPermutation( + locatedTriplesSnapshot)); meta.has_value()) { return meta.value().numRows_; } @@ -1621,7 +1622,8 @@ std::vector IndexImpl::getMultiplicities( if (auto keyId = key.toValueId(getVocab(), encodedIriManager())) { auto meta = permutation.getMetadata( keyId.value(), - permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot)); + permutation.getActualPermutation(keyId.value()) + .getLocatedTriplesForPermutation(locatedTriplesSnapshot)); if (meta.has_value()) { return {meta.value().getCol1Multiplicity(), meta.value().getCol2Multiplicity()}; @@ -1648,7 +1650,8 @@ size_t IndexImpl::getResultSizeOfScan( const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { const auto& perm = getPermutation(permutation); const auto& locatedTriples = - perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot); + perm.getActualPermutation(scanSpecification) + .getLocatedTriplesForPermutation(locatedTriplesSnapshot); return perm.getResultSizeOfScan( perm.getScanSpecAndBlocks(scanSpecification, locatedTriples), locatedTriples); diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp index e552ccbdf2..085ce756ab 100644 --- a/src/index/Permutation.cpp +++ b/src/index/Permutation.cpp @@ -214,6 +214,17 @@ const LocatedTriplesPerBlock& Permutation::getLocatedTriplesForPermutation( permutation_); } +// ______________________________________________________________________ +std::shared_ptr +Permutation::getLocatedTriplesForPermutation( + SharedLocatedTriplesSnapshot locatedTriplesSnapshot) const { + return isInternalPermutation_ + ? locatedTriplesSnapshot.getInternalLocatedTriplesForPermutation( + permutation_) + : locatedTriplesSnapshot.getLocatedTriplesForPermutation( + permutation_); +} + // ______________________________________________________________________ BlockMetadataRanges Permutation::getAugmentedMetadataForPermutation( const LocatedTriplesPerBlock& locatedTriples) const { diff --git a/src/index/Permutation.h b/src/index/Permutation.h index 27f11fce3f..7fde21ccf0 100644 --- a/src/index/Permutation.h +++ b/src/index/Permutation.h @@ -187,6 +187,8 @@ class Permutation { // From the given snapshot, get the located triples for this permutation. const LocatedTriplesPerBlock& getLocatedTriplesForPermutation( const LocatedTriplesSnapshot& locatedTriplesSnapshot) const; + std::shared_ptr getLocatedTriplesForPermutation( + SharedLocatedTriplesSnapshot locatedTriplesSnapshot) const; // From the given `LocatedTriplesPerBlock`, get the augmented block metadata. BlockMetadataRanges getAugmentedMetadataForPermutation( diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp index c874e4c3df..f858ff6973 100644 --- a/test/CompressedRelationsTest.cpp +++ b/test/CompressedRelationsTest.cpp @@ -981,8 +981,9 @@ TEST(CompressedRelationReader, getResultSizeImpl) { auto loc = generateLocationTrace(sourceLocation); auto& perm = impl.getPermutation(p); auto& reader = perm.reader(); - auto augmentedBlocks = - perm.getAugmentedMetadataForPermutation(locatedTriplesSnapshot); + auto augmentedBlocks = perm.getAugmentedMetadataForPermutation( + perm.getActualPermutation(scanSpec).getLocatedTriplesForPermutation( + locatedTriplesSnapshot)); auto& ltpb = locatedTriplesSnapshot.getLocatedTriplesForPermutation( perm.permutation()); auto [actual_lower, actual_upper] = reader.getSizeEstimateForScan( @@ -1066,7 +1067,9 @@ TEST(CompressedRelationReader, getFirstAndLastTripleIgnoringGraph) { : ScanSpecification::GraphFilter::All()}; CompressedRelationReader::ScanSpecAndBlocks metadataAndBlocks{ std::move(scanSpecification), - permutation.getAugmentedMetadataForPermutation(*currentSnapshot)}; + permutation.getAugmentedMetadataForPermutation( + permutation.getActualPermutation(scanSpecification) + .getLocatedTriplesForPermutation(*currentSnapshot))}; const auto& reader = index.getImpl().getPermutation(permutationEnum).reader(); return reader.getFirstAndLastTripleIgnoringGraph(metadataAndBlocks, diff --git a/test/IndexTest.cpp b/test/IndexTest.cpp index 8ec5b89fc7..0a6c06fc6b 100644 --- a/test/IndexTest.cpp +++ b/test/IndexTest.cpp @@ -129,49 +129,33 @@ TEST(IndexTest, createFromTurtleTest) { Id c2 = getId(""); // TODO We could also test the multiplicities here. - ASSERT_TRUE( - index.PSO().getMetadata(b, locatedTriplesSnapshot).has_value()); - ASSERT_TRUE( - index.PSO().getMetadata(b2, locatedTriplesSnapshot).has_value()); + const auto& pso = index.PSO(); + const auto& psoLTPB = + pso.getLocatedTriplesForPermutation(locatedTriplesSnapshot); + ASSERT_TRUE(pso.getMetadata(b, psoLTPB).has_value()); + ASSERT_TRUE(pso.getMetadata(b2, psoLTPB).has_value()); + ASSERT_FALSE(pso.getMetadata(a2, psoLTPB).has_value()); + ASSERT_FALSE(pso.getMetadata(c, psoLTPB).has_value()); ASSERT_FALSE( - index.PSO().getMetadata(a2, locatedTriplesSnapshot).has_value()); - ASSERT_FALSE( - index.PSO().getMetadata(c, locatedTriplesSnapshot).has_value()); - ASSERT_FALSE( - index.PSO() - .getMetadata(Id::makeFromVocabIndex(VocabIndex::make(735)), - locatedTriplesSnapshot) + pso.getMetadata(Id::makeFromVocabIndex(VocabIndex::make(735)), + psoLTPB) .has_value()); - ASSERT_FALSE(index.PSO() - .getMetadata(b, locatedTriplesSnapshot) - .value() - .isFunctional()); - ASSERT_TRUE(index.PSO() - .getMetadata(b2, locatedTriplesSnapshot) - .value() - .isFunctional()); - - ASSERT_TRUE( - index.POS().getMetadata(b, locatedTriplesSnapshot).has_value()); - ASSERT_TRUE( - index.POS().getMetadata(b2, locatedTriplesSnapshot).has_value()); - ASSERT_FALSE( - index.POS().getMetadata(a2, locatedTriplesSnapshot).has_value()); + ASSERT_FALSE(pso.getMetadata(b, psoLTPB).value().isFunctional()); + ASSERT_TRUE(pso.getMetadata(b2, psoLTPB).value().isFunctional()); + + const auto& pos = index.POS(); + const auto& posLTPB = + pos.getLocatedTriplesForPermutation(locatedTriplesSnapshot); + ASSERT_TRUE(pos.getMetadata(b, posLTPB).has_value()); + ASSERT_TRUE(pos.getMetadata(b2, posLTPB).has_value()); + ASSERT_FALSE(pos.getMetadata(a2, posLTPB).has_value()); + ASSERT_FALSE(pos.getMetadata(c, posLTPB).has_value()); ASSERT_FALSE( - index.POS().getMetadata(c, locatedTriplesSnapshot).has_value()); - ASSERT_FALSE( - index.POS() - .getMetadata(Id::makeFromVocabIndex(VocabIndex::make(735)), - locatedTriplesSnapshot) + pos.getMetadata(Id::makeFromVocabIndex(VocabIndex::make(735)), + posLTPB) .has_value()); - ASSERT_TRUE(index.POS() - .getMetadata(b, locatedTriplesSnapshot) - .value() - .isFunctional()); - ASSERT_TRUE(index.POS() - .getMetadata(b2, locatedTriplesSnapshot) - .value() - .isFunctional()); + ASSERT_TRUE(pos.getMetadata(b, posLTPB).value().isFunctional()); + ASSERT_TRUE(pos.getMetadata(b2, posLTPB).value().isFunctional()); // Relation b // Pair index @@ -221,15 +205,17 @@ TEST(IndexTest, createFromTurtleTest) { Id c = getId(""); Id isA = getId(""); - ASSERT_TRUE(index.PSO().getMetadata(isA, deltaTriples).has_value()); - ASSERT_FALSE(index.PSO().getMetadata(a, deltaTriples).has_value()); + const auto& pso = index.PSO(); + const auto& psoLTPB = pso.getLocatedTriplesForPermutation(deltaTriples); + ASSERT_TRUE(pso.getMetadata(isA, psoLTPB).has_value()); + ASSERT_FALSE(pso.getMetadata(a, psoLTPB).has_value()); - ASSERT_FALSE( - index.PSO().getMetadata(isA, deltaTriples).value().isFunctional()); + ASSERT_FALSE(pso.getMetadata(isA, psoLTPB).value().isFunctional()); - ASSERT_TRUE(index.POS().getMetadata(isA, deltaTriples).has_value()); - ASSERT_FALSE( - index.POS().getMetadata(isA, deltaTriples).value().isFunctional()); + const auto& pos = index.POS(); + const auto& posLTPB = pos.getLocatedTriplesForPermutation(deltaTriples); + ASSERT_TRUE(pos.getMetadata(isA, posLTPB).has_value()); + ASSERT_FALSE(pos.getMetadata(isA, posLTPB).value().isFunctional()); auto testTwo = makeTestScanWidthTwo(index, qec); testTwo(iri(""), Permutation::PSO, @@ -274,19 +260,23 @@ TEST(IndexTest, createFromOnDiskIndexTest) { Id a = getId(""); Id c = getId(""); - ASSERT_TRUE(index.PSO().getMetadata(b, deltaTriples).has_value()); - ASSERT_TRUE(index.PSO().getMetadata(b2, deltaTriples).has_value()); - ASSERT_FALSE(index.PSO().getMetadata(a, deltaTriples).has_value()); - ASSERT_FALSE(index.PSO().getMetadata(c, deltaTriples).has_value()); - ASSERT_FALSE(index.PSO().getMetadata(b, deltaTriples).value().isFunctional()); - ASSERT_TRUE(index.PSO().getMetadata(b2, deltaTriples).value().isFunctional()); - - ASSERT_TRUE(index.POS().getMetadata(b, deltaTriples).has_value()); - ASSERT_TRUE(index.POS().getMetadata(b2, deltaTriples).has_value()); - ASSERT_FALSE(index.POS().getMetadata(a, deltaTriples).has_value()); - ASSERT_FALSE(index.POS().getMetadata(c, deltaTriples).has_value()); - ASSERT_TRUE(index.POS().getMetadata(b, deltaTriples).value().isFunctional()); - ASSERT_TRUE(index.POS().getMetadata(b2, deltaTriples).value().isFunctional()); + const auto& pso = index.PSO(); + const auto& psoLTPB = pso.getLocatedTriplesForPermutation(deltaTriples); + ASSERT_TRUE(pso.getMetadata(b, psoLTPB).has_value()); + ASSERT_TRUE(pso.getMetadata(b2, psoLTPB).has_value()); + ASSERT_FALSE(pso.getMetadata(a, psoLTPB).has_value()); + ASSERT_FALSE(pso.getMetadata(c, psoLTPB).has_value()); + ASSERT_FALSE(pso.getMetadata(b, psoLTPB).value().isFunctional()); + ASSERT_TRUE(pso.getMetadata(b2, psoLTPB).value().isFunctional()); + + const auto& pos = index.POS(); + const auto& posLTPB = pos.getLocatedTriplesForPermutation(deltaTriples); + ASSERT_TRUE(pos.getMetadata(b, posLTPB).has_value()); + ASSERT_TRUE(pos.getMetadata(b2, posLTPB).has_value()); + ASSERT_FALSE(pos.getMetadata(a, posLTPB).has_value()); + ASSERT_FALSE(pos.getMetadata(c, posLTPB).has_value()); + ASSERT_TRUE(pos.getMetadata(b, posLTPB).value().isFunctional()); + ASSERT_TRUE(pos.getMetadata(b2, posLTPB).value().isFunctional()); }; TEST(IndexTest, indexIdAndGitHash) { diff --git a/test/util/IndexTestHelpers.cpp b/test/util/IndexTestHelpers.cpp index 61c4d54c35..07d07a9b1c 100644 --- a/test/util/IndexTestHelpers.cpp +++ b/test/util/IndexTestHelpers.cpp @@ -138,12 +138,16 @@ void checkConsistencyBetweenPatternPredicateAndAdditionalColumn( }; auto predicates = index.getImpl().PSO().getDistinctCol0IdsAndCounts( - cancellationDummy, locatedTriplesSnapshot, {}); + cancellationDummy, + locatedTriplesSnapshot.getLocatedTriplesForPermutation(Permutation::PSO), + {}); for (const auto& predicate : predicates.getColumn(0)) { checkConsistencyForPredicate(predicate); } auto objects = index.getImpl().OSP().getDistinctCol0IdsAndCounts( - cancellationDummy, locatedTriplesSnapshot, {}); + cancellationDummy, + locatedTriplesSnapshot.getLocatedTriplesForPermutation(Permutation::OSP), + {}); for (const auto& object : objects.getColumn(0)) { checkConsistencyForObject(object); } From c25381d9b78444cdf40e6d120d3c4025aed4e45e Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Thu, 18 Dec 2025 13:39:57 +0100 Subject: [PATCH 03/17] fixes from merge --- src/engine/CountAvailablePredicates.cpp | 3 +-- src/engine/GroupByImpl.cpp | 7 ++----- src/engine/QueryPlanner.cpp | 8 +++++--- src/index/IndexImpl.cpp | 9 +++------ test/CompressedRelationsTest.cpp | 6 ++---- test/IndexTest.cpp | 15 ++++++++------- test/util/IndexTestHelpers.cpp | 12 ++++++++---- 7 files changed, 29 insertions(+), 31 deletions(-) diff --git a/src/engine/CountAvailablePredicates.cpp b/src/engine/CountAvailablePredicates.cpp index d837fa4aa9..45b1649344 100644 --- a/src/engine/CountAvailablePredicates.cpp +++ b/src/engine/CountAvailablePredicates.cpp @@ -170,8 +170,7 @@ void CountAvailablePredicates::computePatternTrickAllEntities( .toScanSpecification(index); const auto& perm = index.getPermutation(Permutation::Enum::PSO); const auto& locatedTriple = - perm.getActualPermutation(scanSpec).getLocatedTriplesForPermutation( - locatedTriplesSnapshot()); + perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot()); auto fullHasPattern = perm.lazyScan(perm.getScanSpecAndBlocks(scanSpec, locatedTriple), std::nullopt, {}, cancellationHandle_, locatedTriple); diff --git a/src/engine/GroupByImpl.cpp b/src/engine/GroupByImpl.cpp index 1cbeca85c6..b6aac67c6e 100644 --- a/src/engine/GroupByImpl.cpp +++ b/src/engine/GroupByImpl.cpp @@ -842,8 +842,7 @@ std::optional GroupByImpl::computeGroupByObjectWithCount() const { const auto& permutation = indexScan->permutation(); auto result = permutation.getDistinctCol1IdsAndCounts( col0Id.value(), cancellationHandle_, - permutation.getActualPermutation(col0Id.value()) - .getLocatedTriplesForPermutation(locatedTriplesSnapshot()), + permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot()), indexScan->getLimitOffset()); indexScan->updateRuntimeInformationWhenOptimizedOut({}); @@ -903,9 +902,7 @@ std::optional GroupByImpl::computeGroupByForFullIndexScan() const { permutationEnum.value()); auto table = permutation.getDistinctCol0IdsAndCounts( cancellationHandle_, - permutation - .getActualPermutation({std::nullopt, std::nullopt, std::nullopt}) - .getLocatedTriplesForPermutation(locatedTriplesSnapshot()), + permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot()), indexScan->getLimitOffset()); if (numCounts == 0) { table.setColumnSubset(std::array{ColumnIndex{0}}); diff --git a/src/engine/QueryPlanner.cpp b/src/engine/QueryPlanner.cpp index 6f0ac45dda..c85e1570f0 100644 --- a/src/engine/QueryPlanner.cpp +++ b/src/engine/QueryPlanner.cpp @@ -925,9 +925,11 @@ auto QueryPlanner::seedWithScansAndText( auto actualPermutation = qlever::getPermutationForTriple( permutation, _qec->getIndex(), triple); - pushPlan(makeSubtreePlan(_qec, std::move(actualPermutation), - _qec->sharedLocatedTriplesSnapshot(), - std::move(triple), relevantGraphs)); + pushPlan(makeSubtreePlan( + _qec, actualPermutation, + actualPermutation->getLocatedTriplesForPermutation( + _qec->sharedLocatedTriplesSnapshot()), + std::move(triple), relevantGraphs)); }; seedFromOrdinaryTriple(node, addIndexScan, addFilter); } diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index 22a4aae3ac..79f3cf3b5c 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -1570,8 +1570,7 @@ size_t IndexImpl::getCardinality( const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { const auto& perm = getPermutation(permutation); if (const auto& meta = perm.getMetadata( - id, perm.getActualPermutation(id).getLocatedTriplesForPermutation( - locatedTriplesSnapshot)); + id, perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot)); meta.has_value()) { return meta.value().numRows_; } @@ -1622,8 +1621,7 @@ std::vector IndexImpl::getMultiplicities( if (auto keyId = key.toValueId(getVocab(), encodedIriManager())) { auto meta = permutation.getMetadata( keyId.value(), - permutation.getActualPermutation(keyId.value()) - .getLocatedTriplesForPermutation(locatedTriplesSnapshot)); + permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot)); if (meta.has_value()) { return {meta.value().getCol1Multiplicity(), meta.value().getCol2Multiplicity()}; @@ -1650,8 +1648,7 @@ size_t IndexImpl::getResultSizeOfScan( const LocatedTriplesSnapshot& locatedTriplesSnapshot) const { const auto& perm = getPermutation(permutation); const auto& locatedTriples = - perm.getActualPermutation(scanSpecification) - .getLocatedTriplesForPermutation(locatedTriplesSnapshot); + perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot); return perm.getResultSizeOfScan( perm.getScanSpecAndBlocks(scanSpecification, locatedTriples), locatedTriples); diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp index f858ff6973..a300968061 100644 --- a/test/CompressedRelationsTest.cpp +++ b/test/CompressedRelationsTest.cpp @@ -982,8 +982,7 @@ TEST(CompressedRelationReader, getResultSizeImpl) { auto& perm = impl.getPermutation(p); auto& reader = perm.reader(); auto augmentedBlocks = perm.getAugmentedMetadataForPermutation( - perm.getActualPermutation(scanSpec).getLocatedTriplesForPermutation( - locatedTriplesSnapshot)); + perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot)); auto& ltpb = locatedTriplesSnapshot.getLocatedTriplesForPermutation( perm.permutation()); auto [actual_lower, actual_upper] = reader.getSizeEstimateForScan( @@ -1068,8 +1067,7 @@ TEST(CompressedRelationReader, getFirstAndLastTripleIgnoringGraph) { CompressedRelationReader::ScanSpecAndBlocks metadataAndBlocks{ std::move(scanSpecification), permutation.getAugmentedMetadataForPermutation( - permutation.getActualPermutation(scanSpecification) - .getLocatedTriplesForPermutation(*currentSnapshot))}; + permutation.getLocatedTriplesForPermutation(*currentSnapshot))}; const auto& reader = index.getImpl().getPermutation(permutationEnum).reader(); return reader.getFirstAndLastTripleIgnoringGraph(metadataAndBlocks, diff --git a/test/IndexTest.cpp b/test/IndexTest.cpp index 0a6c06fc6b..0d64c01ea3 100644 --- a/test/IndexTest.cpp +++ b/test/IndexTest.cpp @@ -57,14 +57,15 @@ auto makeTestScanWidthOne = [](const IndexImpl& index, ad_utility::source_location l = AD_CURRENT_SOURCE_LOC()) { auto t = generateLocationTrace(l); const auto& actualPermutation = index.getPermutation(permutation); - auto locatedTriplesSnapshot = qec.locatedTriplesSnapshot(); + auto locatedTriples = actualPermutation.getLocatedTriplesForPermutation( + qec.locatedTriplesSnapshot()); IdTable result = actualPermutation.scan( actualPermutation.getScanSpecAndBlocks( ScanSpecificationAsTripleComponent{c0, c1, std::nullopt} .toScanSpecification(index), - locatedTriplesSnapshot), + locatedTriples), additionalColumns, std::make_shared>(), - locatedTriplesSnapshot); + locatedTriples); ASSERT_EQ(result.numColumns(), 1 + additionalColumns.size()); ASSERT_EQ(result, makeIdTableFromVector(expected)); }; @@ -81,15 +82,15 @@ auto makeTestScanWidthTwo = [](const IndexImpl& index, ad_utility::source_location l = AD_CURRENT_SOURCE_LOC()) { auto t = generateLocationTrace(l); const auto& actualPermutation = index.getPermutation(permutation); - auto locatedTriplesSnapshot = qec.locatedTriplesSnapshot(); + auto locatedTriples = actualPermutation.getLocatedTriplesForPermutation( + qec.locatedTriplesSnapshot()); IdTable wol = actualPermutation.scan( actualPermutation.getScanSpecAndBlocks( ScanSpecificationAsTripleComponent{c0, std::nullopt, std::nullopt} .toScanSpecification(index), - locatedTriplesSnapshot), + locatedTriples), Permutation::ColumnIndicesRef{}, - std::make_shared>(), - locatedTriplesSnapshot); + std::make_shared>(), locatedTriples); ASSERT_EQ(wol, makeIdTableFromVector(expected)); }; }; diff --git a/test/util/IndexTestHelpers.cpp b/test/util/IndexTestHelpers.cpp index 07d07a9b1c..e4690eba21 100644 --- a/test/util/IndexTestHelpers.cpp +++ b/test/util/IndexTestHelpers.cpp @@ -82,13 +82,15 @@ void checkConsistencyBetweenPatternPredicateAndAdditionalColumn( &indexImpl](size_t patternIdx, Id id) { const auto& permutation = indexImpl.getPermutation(Permutation::Enum::PSO).internalPermutation(); + const auto& locatedTriples = + permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot); auto scanResultHasPattern = permutation.scan(permutation.getScanSpecAndBlocks( ScanSpecificationAsTripleComponent{ iriOfHasPattern, id, std::nullopt} .toScanSpecification(indexImpl), - locatedTriplesSnapshot), - {}, cancellationDummy, locatedTriplesSnapshot); + locatedTriples), + {}, cancellationDummy, locatedTriples); // Each ID has at most one pattern, it can have none if it doesn't // appear as a subject in the knowledge graph. AD_CORRECTNESS_CHECK(scanResultHasPattern.numRows() <= 1); @@ -104,13 +106,15 @@ void checkConsistencyBetweenPatternPredicateAndAdditionalColumn( auto checkConsistencyForCol0IdAndPermutation = [&](Id col0Id, const Permutation& permutation, size_t subjectColIdx, size_t objectColIdx) { + const auto& locatedTriples = + permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot); auto scanResult = permutation.scan( permutation.getScanSpecAndBlocks( ScanSpecification{col0Id, std::nullopt, std::nullopt}, - locatedTriplesSnapshot), + locatedTriples), std::array{ColumnIndex{ADDITIONAL_COLUMN_INDEX_SUBJECT_PATTERN}, ColumnIndex{ADDITIONAL_COLUMN_INDEX_OBJECT_PATTERN}}, - cancellationDummy, locatedTriplesSnapshot); + cancellationDummy, locatedTriples); ASSERT_EQ(scanResult.numColumns(), 4u); for (const auto& row : scanResult) { auto patternIdx = row[2].getInt(); From 67a4859bc47596502a520b0a1e17f913caf63b8f Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Thu, 18 Dec 2025 14:00:09 +0100 Subject: [PATCH 04/17] format --- src/engine/IndexScan.cpp | 9 +++++++-- src/engine/IndexScan.h | 1 + src/index/Permutation.h | 1 + test/util/IndexTestHelpers.cpp | 13 ++++++------- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index aa345dc081..a37ad001b6 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -80,8 +80,13 @@ IndexScan::IndexScan(QueryExecutionContext* qec, std::optional scanSpecAndBlocks) : IndexScan( qec, qec->getIndex().getImpl().getPermutationPtr(permutationType), - qec->sharedLocatedTriplesSnapshot().getLocatedTriplesForPermutation( - permutationType), + // TODO this is ugly, but this constructor is only used in + // tests + qec->getIndex() + .getImpl() + .getPermutationPtr(permutationType) + ->getLocatedTriplesForPermutation( + qec->sharedLocatedTriplesSnapshot()), triple, std::move(graphsToFilter), std::move(scanSpecAndBlocks)) {} // _____________________________________________________________________________ diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index 90abcb8a9f..7c282a3183 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -186,6 +186,7 @@ class IndexScan final : public Operation { // An index scan can directly and efficiently support LIMIT and OFFSET [[nodiscard]] bool supportsLimitOffset() const override { return true; } + // TODO: rewrite comment // Instead of using the `LocatedTriplesSnapshot` of the `Operation` base // class, which accesses the one stored in the `QueryExecutionContext`, use // the `LocatedTriplesSnapshot` held in this object. This might be a different diff --git a/src/index/Permutation.h b/src/index/Permutation.h index 7fde21ccf0..55db5ee4d6 100644 --- a/src/index/Permutation.h +++ b/src/index/Permutation.h @@ -187,6 +187,7 @@ class Permutation { // From the given snapshot, get the located triples for this permutation. const LocatedTriplesPerBlock& getLocatedTriplesForPermutation( const LocatedTriplesSnapshot& locatedTriplesSnapshot) const; + // TODO: comment std::shared_ptr getLocatedTriplesForPermutation( SharedLocatedTriplesSnapshot locatedTriplesSnapshot) const; diff --git a/test/util/IndexTestHelpers.cpp b/test/util/IndexTestHelpers.cpp index e4690eba21..ceb68755c3 100644 --- a/test/util/IndexTestHelpers.cpp +++ b/test/util/IndexTestHelpers.cpp @@ -140,18 +140,17 @@ void checkConsistencyBetweenPatternPredicateAndAdditionalColumn( checkConsistencyForCol0IdAndPermutation( objectId, indexImpl.getPermutation(OSP), 0, col0IdTag); }; - - auto predicates = index.getImpl().PSO().getDistinctCol0IdsAndCounts( + const auto& pso = index.getImpl().PSO(); + auto predicates = pso.getDistinctCol0IdsAndCounts( cancellationDummy, - locatedTriplesSnapshot.getLocatedTriplesForPermutation(Permutation::PSO), - {}); + pso.getLocatedTriplesForPermutation(locatedTriplesSnapshot), {}); for (const auto& predicate : predicates.getColumn(0)) { checkConsistencyForPredicate(predicate); } - auto objects = index.getImpl().OSP().getDistinctCol0IdsAndCounts( + const auto& osp = index.getImpl().OSP(); + auto objects = osp.getDistinctCol0IdsAndCounts( cancellationDummy, - locatedTriplesSnapshot.getLocatedTriplesForPermutation(Permutation::OSP), - {}); + osp.getLocatedTriplesForPermutation(locatedTriplesSnapshot), {}); for (const auto& object : objects.getColumn(0)) { checkConsistencyForObject(object); } From bcdbeca650f2301705249c8b4308b58699cd7f56 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Thu, 18 Dec 2025 14:47:13 +0100 Subject: [PATCH 05/17] cleanup --- src/engine/IndexScan.cpp | 27 ++++++++++++++++++++------- src/engine/IndexScan.h | 5 +++++ src/engine/QueryPlanner.cpp | 11 ++++++----- src/index/DeltaTriples.h | 13 +------------ src/index/Permutation.cpp | 11 ----------- src/index/Permutation.h | 3 --- 6 files changed, 32 insertions(+), 38 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index a37ad001b6..80e5709026 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -80,13 +80,12 @@ IndexScan::IndexScan(QueryExecutionContext* qec, std::optional scanSpecAndBlocks) : IndexScan( qec, qec->getIndex().getImpl().getPermutationPtr(permutationType), - // TODO this is ugly, but this constructor is only used in - // tests - qec->getIndex() - .getImpl() - .getPermutationPtr(permutationType) - ->getLocatedTriplesForPermutation( - qec->sharedLocatedTriplesSnapshot()), + // TODO this constructor is only used in tests, can it be + // removed? + std::shared_ptr{ + qec->sharedLocatedTriplesSnapshot(), + &qec->sharedLocatedTriplesSnapshot() + ->getLocatedTriplesForPermutation(permutationType)}, triple, std::move(graphsToFilter), std::move(scanSpecAndBlocks)) {} // _____________________________________________________________________________ @@ -805,3 +804,17 @@ std::vector IndexScan::getSubsetForStrippedColumns() const { } return result; } + +// _____________________________________________________________________________ +std::shared_ptr +IndexScan::getLocatedTriplesPerBlockForTriple( + Permutation::Enum permutation, LocatedTriplesSnapshotPtr snapshot, + const SparqlTripleSimple& triple) { + // Create alias shared pointer of internal the right `LocatedTriplesPerBlock`. + const auto& locatedTriples = + containsInternalIri(triple) + ? snapshot->getInternalLocatedTriplesForPermutation(permutation) + : snapshot->getLocatedTriplesForPermutation(permutation); + return std::shared_ptr{std::move(snapshot), + &locatedTriples}; +} diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index 7c282a3183..c0e5cb215f 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -295,6 +295,11 @@ class IndexScan final : public Operation { std::optional> makeTreeWithStrippedColumns( const std::set& variables) const override; + + static std::shared_ptr + getLocatedTriplesPerBlockForTriple(Permutation::Enum permutation, + LocatedTriplesSnapshotPtr snapshot, + const SparqlTripleSimple& triple); }; #endif // QLEVER_SRC_ENGINE_INDEXSCAN_H diff --git a/src/engine/QueryPlanner.cpp b/src/engine/QueryPlanner.cpp index c85e1570f0..655d8e0abb 100644 --- a/src/engine/QueryPlanner.cpp +++ b/src/engine/QueryPlanner.cpp @@ -924,12 +924,13 @@ auto QueryPlanner::seedWithScansAndText( auto actualPermutation = qlever::getPermutationForTriple( permutation, _qec->getIndex(), triple); + const auto& locatedTriples = + IndexScan::getLocatedTriplesPerBlockForTriple( + permutation, _qec->sharedLocatedTriplesSnapshot(), triple); - pushPlan(makeSubtreePlan( - _qec, actualPermutation, - actualPermutation->getLocatedTriplesForPermutation( - _qec->sharedLocatedTriplesSnapshot()), - std::move(triple), relevantGraphs)); + pushPlan(makeSubtreePlan(_qec, actualPermutation, + locatedTriples, std::move(triple), + relevantGraphs)); }; seedFromOrdinaryTriple(node, addIndexScan, addFilter); } diff --git a/src/index/DeltaTriples.h b/src/index/DeltaTriples.h index 8baa2e2667..c46eedf922 100644 --- a/src/index/DeltaTriples.h +++ b/src/index/DeltaTriples.h @@ -52,18 +52,7 @@ struct LocatedTriplesSnapshot { // A shared pointer to a constant `LocatedTriplesSnapshot`, but as an explicit // class, such that it can be forward-declared. class SharedLocatedTriplesSnapshot - : public std::shared_ptr { - public: - std::shared_ptr getLocatedTriplesForPermutation( - Permutation::Enum permutation) const { - return {*this, &this->get()->getLocatedTriplesForPermutation(permutation)}; - } - std::shared_ptr - getInternalLocatedTriplesForPermutation(Permutation::Enum permutation) const { - return {*this, - &this->get()->getInternalLocatedTriplesForPermutation(permutation)}; - } -}; + : public std::shared_ptr {}; // A class for keeping track of the number of triples of the `DeltaTriples`. struct DeltaTriplesCount { diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp index 085ce756ab..e552ccbdf2 100644 --- a/src/index/Permutation.cpp +++ b/src/index/Permutation.cpp @@ -214,17 +214,6 @@ const LocatedTriplesPerBlock& Permutation::getLocatedTriplesForPermutation( permutation_); } -// ______________________________________________________________________ -std::shared_ptr -Permutation::getLocatedTriplesForPermutation( - SharedLocatedTriplesSnapshot locatedTriplesSnapshot) const { - return isInternalPermutation_ - ? locatedTriplesSnapshot.getInternalLocatedTriplesForPermutation( - permutation_) - : locatedTriplesSnapshot.getLocatedTriplesForPermutation( - permutation_); -} - // ______________________________________________________________________ BlockMetadataRanges Permutation::getAugmentedMetadataForPermutation( const LocatedTriplesPerBlock& locatedTriples) const { diff --git a/src/index/Permutation.h b/src/index/Permutation.h index 55db5ee4d6..27f11fce3f 100644 --- a/src/index/Permutation.h +++ b/src/index/Permutation.h @@ -187,9 +187,6 @@ class Permutation { // From the given snapshot, get the located triples for this permutation. const LocatedTriplesPerBlock& getLocatedTriplesForPermutation( const LocatedTriplesSnapshot& locatedTriplesSnapshot) const; - // TODO: comment - std::shared_ptr getLocatedTriplesForPermutation( - SharedLocatedTriplesSnapshot locatedTriplesSnapshot) const; // From the given `LocatedTriplesPerBlock`, get the augmented block metadata. BlockMetadataRanges getAugmentedMetadataForPermutation( From 2be00579e7967683bfe3c597f0c9c8bc7dd59050 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Thu, 18 Dec 2025 16:00:39 +0100 Subject: [PATCH 06/17] and a bit more --- src/engine/IndexScan.cpp | 22 +++++++++++----------- src/engine/IndexScan.h | 6 +++--- src/index/Permutation.cpp | 5 ++--- src/index/Permutation.h | 2 +- test/CompressedRelationsTest.cpp | 4 ++-- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 80e5709026..bbbf000b74 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -32,13 +32,13 @@ static size_t getNumberOfVariables(const TripleComponent& subject, // _____________________________________________________________________________ IndexScan::IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, - LocatedTriplesPerBlockPtr locatedTriplesSnapshot, + LocatedTriplesPerBlockPtr locatedTriplesPerBlock, const SparqlTripleSimple& triple, Graphs graphsToFilter, std::optional scanSpecAndBlocks, VarsToKeep varsToKeep) : Operation(qec), permutation_(permutation), - locatedTriplesSnapshot_(locatedTriplesSnapshot), + locatedTriplesPerBlock_(locatedTriplesPerBlock), subject_(triple.s_), predicate_(triple.p_), object_(triple.o_), @@ -49,7 +49,7 @@ IndexScan::IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, numVariables_(getNumberOfVariables(subject_, predicate_, object_)), varsToKeep_(std::move(varsToKeep)) { AD_CONTRACT_CHECK(permutation_ != nullptr); - AD_CONTRACT_CHECK(locatedTriplesSnapshot_ != nullptr); + AD_CONTRACT_CHECK(locatedTriplesPerBlock_ != nullptr); // We previously had `nullptr`s here in unit tests. This is no longer // necessary nor allowed. @@ -90,7 +90,7 @@ IndexScan::IndexScan(QueryExecutionContext* qec, // _____________________________________________________________________________ IndexScan::IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, - LocatedTriplesPerBlockPtr locatedTriplesSnapshot, + LocatedTriplesPerBlockPtr locatedTriplesPerBlock, const TripleComponent& s, const TripleComponent& p, const TripleComponent& o, std::vector additionalColumns, @@ -99,7 +99,7 @@ IndexScan::IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, bool scanSpecAndBlocksIsPrefiltered, VarsToKeep varsToKeep) : Operation(qec), permutation_(permutation), - locatedTriplesSnapshot_(locatedTriplesSnapshot), + locatedTriplesPerBlock_(locatedTriplesPerBlock), subject_(s), predicate_(p), object_(o), @@ -112,7 +112,7 @@ IndexScan::IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, varsToKeep_{std::move(varsToKeep)} { AD_CONTRACT_CHECK(qec != nullptr); AD_CONTRACT_CHECK(permutation_ != nullptr); - AD_CONTRACT_CHECK(locatedTriplesSnapshot_ != nullptr); + AD_CONTRACT_CHECK(locatedTriplesPerBlock_ != nullptr); std::tie(sizeEstimateIsExact_, sizeEstimate_) = computeSizeEstimate(); determineMultiplicities(); } @@ -265,7 +265,7 @@ std::shared_ptr IndexScan::makeCopyWithPrefilteredScanSpecAndBlocks( ScanSpecAndBlocks scanSpecAndBlocks) const { return ad_utility::makeExecutionTree( - getExecutionContext(), permutation_, locatedTriplesSnapshot_, subject_, + getExecutionContext(), permutation_, locatedTriplesPerBlock_, subject_, predicate_, object_, additionalColumns_, additionalVariables_, graphsToFilter_, std::move(scanSpecAndBlocks), true, varsToKeep_); } @@ -307,8 +307,8 @@ const Permutation& IndexScan::permutation() const { // _____________________________________________________________________________ const LocatedTriplesPerBlock& IndexScan::locatedTriplesPerBlock() const { - AD_CONTRACT_CHECK(locatedTriplesSnapshot_ != nullptr); - return *locatedTriplesSnapshot_; + AD_CONTRACT_CHECK(locatedTriplesPerBlock_ != nullptr); + return *locatedTriplesPerBlock_; } // _____________________________________________________________________________ @@ -751,7 +751,7 @@ std::pair IndexScan::prefilterTables( // _____________________________________________________________________________ std::unique_ptr IndexScan::cloneImpl() const { return std::make_unique( - _executionContext, permutation_, locatedTriplesSnapshot_, subject_, + _executionContext, permutation_, locatedTriplesPerBlock_, subject_, predicate_, object_, additionalColumns_, additionalVariables_, graphsToFilter_, scanSpecAndBlocks_, scanSpecAndBlocksIsPrefiltered_, varsToKeep_); @@ -776,7 +776,7 @@ IndexScan::makeTreeWithStrippedColumns( } return ad_utility::makeExecutionTree( - _executionContext, permutation_, locatedTriplesSnapshot_, subject_, + _executionContext, permutation_, locatedTriplesPerBlock_, subject_, predicate_, object_, additionalColumns_, additionalVariables_, graphsToFilter_, scanSpecAndBlocks_, scanSpecAndBlocksIsPrefiltered_, VarsToKeep{std::move(newVariables)}); diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index c0e5cb215f..bb3454398d 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -27,7 +27,7 @@ class IndexScan final : public Operation { private: PermutationPtr permutation_; - LocatedTriplesPerBlockPtr locatedTriplesSnapshot_; + LocatedTriplesPerBlockPtr locatedTriplesPerBlock_; TripleComponent subject_; TripleComponent predicate_; TripleComponent object_; @@ -53,7 +53,7 @@ class IndexScan final : public Operation { public: IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, - LocatedTriplesPerBlockPtr locatedTriplesSnapshot, + LocatedTriplesPerBlockPtr locatedTriplesPerBlock, const SparqlTripleSimple& triple, Graphs graphsToFilter = Graphs::All(), std::optional scanSpecAndBlocks = std::nullopt, @@ -69,7 +69,7 @@ class IndexScan final : public Operation { // Constructor to simplify copy creation of an `IndexScan`. IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, - LocatedTriplesPerBlockPtr locatedTriplesSnapshot, + LocatedTriplesPerBlockPtr locatedTriplesPerBlock, const TripleComponent& s, const TripleComponent& p, const TripleComponent& o, std::vector additionalColumns, diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp index e552ccbdf2..3092f1b587 100644 --- a/src/index/Permutation.cpp +++ b/src/index/Permutation.cpp @@ -22,8 +22,7 @@ Permutation::Permutation(Enum permutation, Allocator allocator) CompressedRelationReader::ScanSpecAndBlocks Permutation::getScanSpecAndBlocks( const ScanSpecification& scanSpec, const LocatedTriplesPerBlock& locatedTriples) const { - return {scanSpec, BlockMetadataRanges( - getAugmentedMetadataForPermutation(locatedTriples))}; + return {scanSpec, getAugmentedMetadata(locatedTriples)}; } // _____________________________________________________________________ @@ -215,7 +214,7 @@ const LocatedTriplesPerBlock& Permutation::getLocatedTriplesForPermutation( } // ______________________________________________________________________ -BlockMetadataRanges Permutation::getAugmentedMetadataForPermutation( +BlockMetadataRanges Permutation::getAugmentedMetadata( const LocatedTriplesPerBlock& locatedTriples) const { BlockMetadataSpan blocks(locatedTriples.getAugmentedMetadata()); return {{blocks.begin(), blocks.end()}}; diff --git a/src/index/Permutation.h b/src/index/Permutation.h index 27f11fce3f..8e5103398f 100644 --- a/src/index/Permutation.h +++ b/src/index/Permutation.h @@ -189,7 +189,7 @@ class Permutation { const LocatedTriplesSnapshot& locatedTriplesSnapshot) const; // From the given `LocatedTriplesPerBlock`, get the augmented block metadata. - BlockMetadataRanges getAugmentedMetadataForPermutation( + BlockMetadataRanges getAugmentedMetadata( const LocatedTriplesPerBlock& locatedTriples) const; const CompressedRelationReader& reader() const { return reader_.value(); } diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp index a300968061..840b09c507 100644 --- a/test/CompressedRelationsTest.cpp +++ b/test/CompressedRelationsTest.cpp @@ -981,7 +981,7 @@ TEST(CompressedRelationReader, getResultSizeImpl) { auto loc = generateLocationTrace(sourceLocation); auto& perm = impl.getPermutation(p); auto& reader = perm.reader(); - auto augmentedBlocks = perm.getAugmentedMetadataForPermutation( + auto augmentedBlocks = perm.getAugmentedMetadata( perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot)); auto& ltpb = locatedTriplesSnapshot.getLocatedTriplesForPermutation( perm.permutation()); @@ -1066,7 +1066,7 @@ TEST(CompressedRelationReader, getFirstAndLastTripleIgnoringGraph) { : ScanSpecification::GraphFilter::All()}; CompressedRelationReader::ScanSpecAndBlocks metadataAndBlocks{ std::move(scanSpecification), - permutation.getAugmentedMetadataForPermutation( + permutation.getAugmentedMetadata( permutation.getLocatedTriplesForPermutation(*currentSnapshot))}; const auto& reader = index.getImpl().getPermutation(permutationEnum).reader(); From 56e6f01127b3253b79c7fbe82f02d0a466a54dbb Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Mon, 22 Dec 2025 20:36:03 +0100 Subject: [PATCH 07/17] adapt to changes --- src/engine/HasPredicateScan.cpp | 4 +++- src/engine/IndexScan.cpp | 14 -------------- src/engine/IndexScan.h | 5 ----- src/engine/MaterializedViews.cpp | 26 ++++++++++---------------- src/engine/MaterializedViews.h | 7 ++++--- src/engine/PermutationSelector.cpp | 14 ++++++++++++++ src/engine/PermutationSelector.h | 7 +++++++ src/engine/QueryPlanner.cpp | 5 ++--- test/MaterializedViewsTest.cpp | 2 +- 9 files changed, 41 insertions(+), 43 deletions(-) diff --git a/src/engine/HasPredicateScan.cpp b/src/engine/HasPredicateScan.cpp index cc8313772c..51036b7833 100644 --- a/src/engine/HasPredicateScan.cpp +++ b/src/engine/HasPredicateScan.cpp @@ -424,5 +424,7 @@ std::shared_ptr HasPredicateScan::makePatternScan( qec, qlever::getPermutationForTriple(Permutation::Enum::PSO, qec->getIndex(), triple), - qec->sharedLocatedTriplesSnapshot(), triple); + qlever::getLocatedTriplesPerBlockForTriple( + Permutation::Enum::PSO, qec->sharedLocatedTriplesSnapshot(), triple), + triple); } diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index bbbf000b74..235d1a97c1 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -804,17 +804,3 @@ std::vector IndexScan::getSubsetForStrippedColumns() const { } return result; } - -// _____________________________________________________________________________ -std::shared_ptr -IndexScan::getLocatedTriplesPerBlockForTriple( - Permutation::Enum permutation, LocatedTriplesSnapshotPtr snapshot, - const SparqlTripleSimple& triple) { - // Create alias shared pointer of internal the right `LocatedTriplesPerBlock`. - const auto& locatedTriples = - containsInternalIri(triple) - ? snapshot->getInternalLocatedTriplesForPermutation(permutation) - : snapshot->getLocatedTriplesForPermutation(permutation); - return std::shared_ptr{std::move(snapshot), - &locatedTriples}; -} diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index bb3454398d..f2d142d371 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -295,11 +295,6 @@ class IndexScan final : public Operation { std::optional> makeTreeWithStrippedColumns( const std::set& variables) const override; - - static std::shared_ptr - getLocatedTriplesPerBlockForTriple(Permutation::Enum permutation, - LocatedTriplesSnapshotPtr snapshot, - const SparqlTripleSimple& triple); }; #endif // QLEVER_SRC_ENGINE_INDEXSCAN_H diff --git a/src/engine/MaterializedViews.cpp b/src/engine/MaterializedViews.cpp index c35a634852..a273faa563 100644 --- a/src/engine/MaterializedViews.cpp +++ b/src/engine/MaterializedViews.cpp @@ -296,7 +296,7 @@ void MaterializedViewWriter::computeResultAndWritePermutation() const { MaterializedView::MaterializedView(std::string onDiskBase, std::string name) : onDiskBase_{std::move(onDiskBase)}, name_{std::move(name)}, - locatedTriplesSnapshot_{makeEmptyLocatedTriplesSnapshot()} { + locatedTriplesPerBlock_{makeEmptyLocatedTriplesPerBlock()} { AD_CORRECTNESS_CHECK(onDiskBase_ != "", "The index base filename was not set."); throwIfInvalidName(name_); @@ -517,23 +517,17 @@ void MaterializedViewsManager::setOnDiskBase(const std::string& onDiskBase) { } // _____________________________________________________________________________ -std::shared_ptr -MaterializedView::locatedTriplesSnapshot() const { - return locatedTriplesSnapshot_; +std::shared_ptr +MaterializedView::locatedTriplesPerBlock() const { + return locatedTriplesPerBlock_; } // _____________________________________________________________________________ -std::shared_ptr -MaterializedView::makeEmptyLocatedTriplesSnapshot() const { - LocatedTriplesPerBlockAllPermutations emptyLocatedTriples; - emptyLocatedTriples[static_cast(permutation_->permutation())] - .setOriginalMetadata(permutation_->metaData().blockDataShared()); - LocatedTriplesPerBlockAllPermutations emptyInternalLocatedTriples; - LocalVocab emptyVocab; - - return std::make_shared( - emptyLocatedTriples, emptyInternalLocatedTriples, - emptyVocab.getLifetimeExtender(), 0); +std::shared_ptr +MaterializedView::makeEmptyLocatedTriplesPerBlock() const { + auto ltpb = std::make_shared(); + ltpb->setOriginalMetadata(permutation_->metaData().blockDataShared()); + return ltpb; } // _____________________________________________________________________________ @@ -550,7 +544,7 @@ std::shared_ptr MaterializedView::makeIndexScan( // query. auto scanTriple = makeScanConfig(viewQuery); return std::make_shared( - qec, permutation_, locatedTriplesSnapshot_, std::move(scanTriple), + qec, permutation_, locatedTriplesPerBlock_, std::move(scanTriple), IndexScan::Graphs::All(), std::nullopt, viewQuery.getVarsToKeep()); } diff --git a/src/engine/MaterializedViews.h b/src/engine/MaterializedViews.h index fe089a93ed..321e56aec8 100644 --- a/src/engine/MaterializedViews.h +++ b/src/engine/MaterializedViews.h @@ -148,13 +148,14 @@ class MaterializedView { std::shared_ptr permutation_{std::make_shared( Permutation::Enum::SPO, ad_utility::makeUnlimitedAllocator())}; VariableToColumnMap varToColMap_; - std::shared_ptr locatedTriplesSnapshot_; + std::shared_ptr locatedTriplesPerBlock_; using AdditionalScanColumns = SparqlTripleSimple::AdditionalScanColumns; // Helper to create an empty `LocatedTriplesSnapshot` for `IndexScan`s as // materialized views do not support updates yet. - std::shared_ptr makeEmptyLocatedTriplesSnapshot() + // TODO: is trivial now -> inline? + std::shared_ptr makeEmptyLocatedTriplesPerBlock() const; public: @@ -185,7 +186,7 @@ class MaterializedView { // Return a reference to the `LocatedTriplesSnapshot` for the permutation. For // now this is always an empty snapshot but with the correct permutation // metadata. - std::shared_ptr locatedTriplesSnapshot() const; + std::shared_ptr locatedTriplesPerBlock() const; // Checks if the given name is allowed for a materialized view. Currently only // alphanumerics and hyphens are allowed. This is relevant for safe filenames diff --git a/src/engine/PermutationSelector.cpp b/src/engine/PermutationSelector.cpp index 0949cbb21a..594024b556 100644 --- a/src/engine/PermutationSelector.cpp +++ b/src/engine/PermutationSelector.cpp @@ -47,4 +47,18 @@ std::shared_ptr getPermutationForTriple( } return actualPermutation; } + +// _____________________________________________________________________________ +std::shared_ptr +getLocatedTriplesPerBlockForTriple(Permutation::Enum permutation, + LocatedTriplesSnapshotPtr snapshot, + const SparqlTripleSimple& triple) { + // Create alias shared pointer of internal the right `LocatedTriplesPerBlock`. + const auto& locatedTriples = + containsInternalIri(triple) + ? snapshot->getInternalLocatedTriplesForPermutation(permutation) + : snapshot->getLocatedTriplesForPermutation(permutation); + return std::shared_ptr{std::move(snapshot), + &locatedTriples}; +} } // namespace qlever diff --git a/src/engine/PermutationSelector.h b/src/engine/PermutationSelector.h index 53389c1519..38d42608e5 100644 --- a/src/engine/PermutationSelector.h +++ b/src/engine/PermutationSelector.h @@ -23,6 +23,13 @@ namespace qlever { std::shared_ptr getPermutationForTriple( Permutation::Enum permutation, const Index& index, const SparqlTripleSimple& triple); + +// TODO: is this good here or should it go somewhere else? +using LocatedTriplesSnapshotPtr = std::shared_ptr; +std::shared_ptr +getLocatedTriplesPerBlockForTriple(Permutation::Enum permutation, + LocatedTriplesSnapshotPtr snapshot, + const SparqlTripleSimple& triple); } // namespace qlever #endif // QLEVER_SRC_ENGINE_PERMUTATIONSELECTOR_H diff --git a/src/engine/QueryPlanner.cpp b/src/engine/QueryPlanner.cpp index 655d8e0abb..52b839f27d 100644 --- a/src/engine/QueryPlanner.cpp +++ b/src/engine/QueryPlanner.cpp @@ -924,9 +924,8 @@ auto QueryPlanner::seedWithScansAndText( auto actualPermutation = qlever::getPermutationForTriple( permutation, _qec->getIndex(), triple); - const auto& locatedTriples = - IndexScan::getLocatedTriplesPerBlockForTriple( - permutation, _qec->sharedLocatedTriplesSnapshot(), triple); + const auto& locatedTriples = qlever::getLocatedTriplesPerBlockForTriple( + permutation, _qec->sharedLocatedTriplesSnapshot(), triple); pushPlan(makeSubtreePlan(_qec, actualPermutation, locatedTriples, std::move(triple), diff --git a/test/MaterializedViewsTest.cpp b/test/MaterializedViewsTest.cpp index 7169b88e1b..49dce577a9 100644 --- a/test/MaterializedViewsTest.cpp +++ b/test/MaterializedViewsTest.cpp @@ -318,7 +318,7 @@ TEST_F(MaterializedViewsTest, ManualConfigurations) { ASSERT_TRUE(view != nullptr); EXPECT_EQ(view->name(), "testView1"); EXPECT_EQ(view->permutation()->permutation(), Permutation::Enum::SPO); - EXPECT_NE(view->locatedTriplesSnapshot(), nullptr); + EXPECT_NE(view->locatedTriplesPerBlock(), nullptr); MaterializedViewsManager managerNoBaseName; AD_EXPECT_THROW_WITH_MESSAGE( From 6868cefa005055f9b7cf675577d5fc56852ab292 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Mon, 22 Dec 2025 20:48:40 +0100 Subject: [PATCH 08/17] format --- src/engine/GroupByImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/GroupByImpl.cpp b/src/engine/GroupByImpl.cpp index b6aac67c6e..a4b89dfd10 100644 --- a/src/engine/GroupByImpl.cpp +++ b/src/engine/GroupByImpl.cpp @@ -430,7 +430,7 @@ void GroupByImpl::processGroup( evaluationContext._previousResultsFromSameGroup.at(resultColumn) = sparqlExpression::copyExpressionResult(expressionResult); - auto visitor = CPP_template_lambda_mut (&)(typename T)(T && singleResult)( + auto visitor = CPP_template_lambda_mut(&)(typename T)(T && singleResult)( requires sparqlExpression::SingleExpressionResult) { constexpr static bool isStrongId = std::is_same_v; if constexpr (isStrongId) { From 3b4c74047754f3a22397501cfae6586b50bdf25a Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Mon, 22 Dec 2025 20:57:11 +0100 Subject: [PATCH 09/17] move shared_ptrs --- src/engine/IndexScan.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 235d1a97c1..278c21ac49 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -37,8 +37,8 @@ IndexScan::IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, std::optional scanSpecAndBlocks, VarsToKeep varsToKeep) : Operation(qec), - permutation_(permutation), - locatedTriplesPerBlock_(locatedTriplesPerBlock), + permutation_(std::move(permutation)), + locatedTriplesPerBlock_(std::move(locatedTriplesPerBlock)), subject_(triple.s_), predicate_(triple.p_), object_(triple.o_), @@ -98,8 +98,8 @@ IndexScan::IndexScan(QueryExecutionContext* qec, PermutationPtr permutation, Graphs graphsToFilter, ScanSpecAndBlocks scanSpecAndBlocks, bool scanSpecAndBlocksIsPrefiltered, VarsToKeep varsToKeep) : Operation(qec), - permutation_(permutation), - locatedTriplesPerBlock_(locatedTriplesPerBlock), + permutation_(std::move(permutation)), + locatedTriplesPerBlock_(std::move(locatedTriplesPerBlock)), subject_(s), predicate_(p), object_(o), From a05f4fcf78879abf842ee77786af2be3e5140b83 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Mon, 22 Dec 2025 22:04:21 +0100 Subject: [PATCH 10/17] fixes --- src/engine/MaterializedViews.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/engine/MaterializedViews.h b/src/engine/MaterializedViews.h index 321e56aec8..4a870812cf 100644 --- a/src/engine/MaterializedViews.h +++ b/src/engine/MaterializedViews.h @@ -152,9 +152,8 @@ class MaterializedView { using AdditionalScanColumns = SparqlTripleSimple::AdditionalScanColumns; - // Helper to create an empty `LocatedTriplesSnapshot` for `IndexScan`s as + // Helper to create an empty `LocatedTriplesPerBlock` for `IndexScan`s as // materialized views do not support updates yet. - // TODO: is trivial now -> inline? std::shared_ptr makeEmptyLocatedTriplesPerBlock() const; @@ -183,7 +182,7 @@ class MaterializedView { // `nullptr`. std::shared_ptr permutation() const; - // Return a reference to the `LocatedTriplesSnapshot` for the permutation. For + // Return a reference to the `LocatedTriplesPerBlock` for the permutation. For // now this is always an empty snapshot but with the correct permutation // metadata. std::shared_ptr locatedTriplesPerBlock() const; From 9a99b3af4fcd4ffc4ce05b439877560dfff8c3db Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Thu, 25 Dec 2025 11:37:36 +0100 Subject: [PATCH 11/17] address review comments --- src/engine/CountAvailablePredicates.cpp | 3 +- src/engine/HasPredicateScan.cpp | 14 +++--- src/engine/IndexScan.cpp | 4 +- src/engine/IndexScan.h | 7 +-- src/engine/PermutationSelector.cpp | 28 +++++++---- src/engine/PermutationSelector.h | 42 +++++++++++------ src/engine/QueryPlanner.cpp | 15 +++--- src/index/CompressedRelation.cpp | 8 ++++ src/index/CompressedRelation.h | 6 +++ src/index/IndexImpl.cpp | 3 +- src/index/Permutation.cpp | 23 ++------- src/index/Permutation.h | 14 +----- test/CompressedRelationsTest.cpp | 19 ++++---- test/IndexTest.cpp | 14 +++--- test/engine/PermutationSelectorTest.cpp | 63 ++++++++++++++++++------- test/util/IndexTestHelpers.cpp | 16 +++---- 16 files changed, 162 insertions(+), 117 deletions(-) diff --git a/src/engine/CountAvailablePredicates.cpp b/src/engine/CountAvailablePredicates.cpp index 45b1649344..0c34a7f679 100644 --- a/src/engine/CountAvailablePredicates.cpp +++ b/src/engine/CountAvailablePredicates.cpp @@ -172,7 +172,8 @@ void CountAvailablePredicates::computePatternTrickAllEntities( const auto& locatedTriple = perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot()); auto fullHasPattern = - perm.lazyScan(perm.getScanSpecAndBlocks(scanSpec, locatedTriple), + perm.lazyScan(CompressedRelationReader::ScanSpecAndBlocks::withUpdates( + scanSpec, locatedTriple), std::nullopt, {}, cancellationHandle_, locatedTriple); for (const auto& idTable : fullHasPattern) { for (const auto& patternId : idTable.getColumn(1)) { diff --git a/src/engine/HasPredicateScan.cpp b/src/engine/HasPredicateScan.cpp index 51036b7833..7c9cbfb5b9 100644 --- a/src/engine/HasPredicateScan.cpp +++ b/src/engine/HasPredicateScan.cpp @@ -420,11 +420,11 @@ std::shared_ptr HasPredicateScan::makePatternScan( std::move(subject), ad_utility::triple_component::Iri::fromIriref(HAS_PATTERN_PREDICATE), TripleComponent{std::move(object)}}; - return ad_utility::makeExecutionTree( - qec, - qlever::getPermutationForTriple(Permutation::Enum::PSO, qec->getIndex(), - triple), - qlever::getLocatedTriplesPerBlockForTriple( - Permutation::Enum::PSO, qec->sharedLocatedTriplesSnapshot(), triple), - triple); + auto [permutation, locatedTriples] = + qlever::getPermutationAndLocatedTriplesPerBlockForTriple( + Permutation::Enum::PSO, qec->getIndex(), + qec->sharedLocatedTriplesSnapshot(), triple); + return ad_utility::makeExecutionTree(qec, std::move(permutation), + std::move(locatedTriples), + std::move(triple)); } diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 278c21ac49..324643f990 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -401,8 +401,8 @@ IndexScan::getSortedVariableAndMetadataColumnIndexForPrefiltering() const { // ___________________________________________________________________________ Permutation::ScanSpecAndBlocks IndexScan::getScanSpecAndBlocks() const { - return permutation().getScanSpecAndBlocks(getScanSpecification(), - locatedTriplesPerBlock()); + return CompressedRelationReader::ScanSpecAndBlocks::withUpdates( + getScanSpecification(), locatedTriplesPerBlock()); } // _____________________________________________________________________________ diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index f2d142d371..657b56b2ec 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -16,6 +16,7 @@ class SparqlTripleSimple; class IndexScan final : public Operation { public: using Graphs = ScanSpecificationAsTripleComponent::GraphFilter; + // TODO: get rid of them, especially the dubling with `PermutationSelector` using PermutationPtr = std::shared_ptr; using LocatedTriplesSnapshotPtr = std::shared_ptr; @@ -186,11 +187,11 @@ class IndexScan final : public Operation { // An index scan can directly and efficiently support LIMIT and OFFSET [[nodiscard]] bool supportsLimitOffset() const override { return true; } - // TODO: rewrite comment // Instead of using the `LocatedTriplesSnapshot` of the `Operation` base // class, which accesses the one stored in the `QueryExecutionContext`, use - // the `LocatedTriplesSnapshot` held in this object. This might be a different - // one if a custom permutation is used. + // the `LocatedTriplesPerBlock` held in this object. This already is exactly + // the located triples for the permutation of the index scan and should be + // used wherever possible. const LocatedTriplesPerBlock& locatedTriplesPerBlock() const; // Return the stored triple in the order that corresponds to the diff --git a/src/engine/PermutationSelector.cpp b/src/engine/PermutationSelector.cpp index 594024b556..272cf71502 100644 --- a/src/engine/PermutationSelector.cpp +++ b/src/engine/PermutationSelector.cpp @@ -4,6 +4,8 @@ #include "engine/PermutationSelector.h" +#include + #include "index/IndexImpl.h" namespace { @@ -34,9 +36,9 @@ bool containsInternalIri(const SparqlTripleSimple& triple) { namespace qlever { // _____________________________________________________________________________ -std::shared_ptr getPermutationForTriple( - Permutation::Enum permutation, const Index& index, - const SparqlTripleSimple& triple) { +PermutationPtr getPermutationForTriple(Permutation::Enum permutation, + const Index& index, + const SparqlTripleSimple& triple) { auto actualPermutation = index.getImpl().getPermutationPtr(permutation); if (containsInternalIri(triple)) { @@ -49,16 +51,24 @@ std::shared_ptr getPermutationForTriple( } // _____________________________________________________________________________ -std::shared_ptr -getLocatedTriplesPerBlockForTriple(Permutation::Enum permutation, - LocatedTriplesSnapshotPtr snapshot, - const SparqlTripleSimple& triple) { +LocatedTriplesPerBlockPtr getLocatedTriplesPerBlockForTriple( + Permutation::Enum permutation, LocatedTriplesSnapshotPtr snapshot, + const SparqlTripleSimple& triple) { // Create alias shared pointer of internal the right `LocatedTriplesPerBlock`. const auto& locatedTriples = containsInternalIri(triple) ? snapshot->getInternalLocatedTriplesForPermutation(permutation) : snapshot->getLocatedTriplesForPermutation(permutation); - return std::shared_ptr{std::move(snapshot), - &locatedTriples}; + return LocatedTriplesPerBlockPtr{std::move(snapshot), &locatedTriples}; +} + +// _____________________________________________________________________________ +std::pair +getPermutationAndLocatedTriplesPerBlockForTriple( + Permutation::Enum permutation, const Index& index, + LocatedTriplesSnapshotPtr snapshot, const SparqlTripleSimple& triple) { + return {getPermutationForTriple(permutation, index, triple), + getLocatedTriplesPerBlockForTriple(permutation, std::move(snapshot), + triple)}; } } // namespace qlever diff --git a/src/engine/PermutationSelector.h b/src/engine/PermutationSelector.h index 38d42608e5..46351e2f52 100644 --- a/src/engine/PermutationSelector.h +++ b/src/engine/PermutationSelector.h @@ -13,23 +13,37 @@ namespace qlever { +// TODO: get rid of them +using LocatedTriplesSnapshotPtr = std::shared_ptr; +using LocatedTriplesPerBlockPtr = std::shared_ptr; +using PermutationPtr = std::shared_ptr; + // Return a shared pointer to the correct permutation in `index` based on the -// `permutation` enum and the values in `triple`. In particular return the -// associated internal permutation if the passed `triple` contains an internal -// IRI at any position. If no internal permutation is available for the passed -// `permutation` enum, throw instead.Internal IRIs include language-tagged IRIs. -// like `@en@rdfs:label` for example, or ones that start with -// ` getPermutationForTriple( - Permutation::Enum permutation, const Index& index, +// `permutation` enum and the values in `triple`. See +// `getPermutationAndLocatedTriplesPerBlockForTriple` for details. +PermutationPtr getPermutationForTriple(Permutation::Enum permutation, + const Index& index, + const SparqlTripleSimple& triple); + +// Return a shared pointer to the correct `LocatedTriplesPerBlock` in `snapshot` +// based on the `permutation` enum and the values in `triple`. See +// `getPermutationAndLocatedTriplesPerBlockForTriple` for details. +LocatedTriplesPerBlockPtr getLocatedTriplesPerBlockForTriple( + Permutation::Enum permutation, LocatedTriplesSnapshotPtr snapshot, const SparqlTripleSimple& triple); -// TODO: is this good here or should it go somewhere else? -using LocatedTriplesSnapshotPtr = std::shared_ptr; -std::shared_ptr -getLocatedTriplesPerBlockForTriple(Permutation::Enum permutation, - LocatedTriplesSnapshotPtr snapshot, - const SparqlTripleSimple& triple); +// Return a shared pointer to the correct permutation in `index` and +// `LocatedTriplesPerBlock` in `snapshot ` based on the `permutation` enum and +// the values in `triple`. In particular return the associated internal +// permutation if the passed `triple` contains an internal IRI at any position. +// If no internal permutation is available for the passed `permutation` enum, +// throw instead. Internal IRIs include language-tagged IRIs. like +// `@en@rdfs:label` for example, or ones that start with +// ` +getPermutationAndLocatedTriplesPerBlockForTriple( + Permutation::Enum permutation, const Index& index, + LocatedTriplesSnapshotPtr snapshot, const SparqlTripleSimple& triple); } // namespace qlever #endif // QLEVER_SRC_ENGINE_PERMUTATIONSELECTOR_H diff --git a/src/engine/QueryPlanner.cpp b/src/engine/QueryPlanner.cpp index 52b839f27d..91824f1675 100644 --- a/src/engine/QueryPlanner.cpp +++ b/src/engine/QueryPlanner.cpp @@ -922,14 +922,13 @@ auto QueryPlanner::seedWithScansAndText( std::move(internalVariable)); } - auto actualPermutation = qlever::getPermutationForTriple( - permutation, _qec->getIndex(), triple); - const auto& locatedTriples = qlever::getLocatedTriplesPerBlockForTriple( - permutation, _qec->sharedLocatedTriplesSnapshot(), triple); - - pushPlan(makeSubtreePlan(_qec, actualPermutation, - locatedTriples, std::move(triple), - relevantGraphs)); + auto [actualPermutation, locatedTriples] = + qlever::getPermutationAndLocatedTriplesPerBlockForTriple( + permutation, _qec->getIndex(), + _qec->sharedLocatedTriplesSnapshot(), triple); + pushPlan(makeSubtreePlan(_qec, std::move(actualPermutation), + std::move(locatedTriples), + std::move(triple), relevantGraphs)); }; seedFromOrdinaryTriple(node, addIndexScan, addFilter); } diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index a245bc1480..c1b14d1fd2 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -1817,6 +1817,14 @@ CompressedRelationReader::ScanSpecAndBlocks::ScanSpecAndBlocks( sizeBlockMetadata_ = getNumberOfBlockMetadataValues(blockMetadata_); } +// _____________________________________________________________________________ +CompressedRelationReader::ScanSpecAndBlocks +CompressedRelationReader::ScanSpecAndBlocks::withUpdates( + ScanSpecification scanSpec, const LocatedTriplesPerBlock& locatedTriples) { + BlockMetadataSpan blocks(locatedTriples.getAugmentedMetadata()); + return {std::move(scanSpec), {{blocks.begin(), blocks.end()}}}; +} + // _____________________________________________________________________________ ql::span CompressedRelationReader::ScanSpecAndBlocks::getBlockMetadataSpan() const { diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h index 10feb7570f..2f6422def3 100644 --- a/src/index/CompressedRelation.h +++ b/src/index/CompressedRelation.h @@ -592,6 +592,12 @@ class CompressedRelationReader { ScanSpecAndBlocks(ScanSpecification scanSpec, const BlockMetadataRanges& blockMetadataRanges); + // Create a new `ScanSpecAndBlocks` with the current block metadata changed + // through updates. + static ScanSpecAndBlocks withUpdates( + ScanSpecification scanSpec, + const LocatedTriplesPerBlock& locatedTriples); + // Direct view access via `ql::views::join` over all // `CompressedBlockMetadata` values contained in `BlockMetadatatRanges // blockMetadata_`. diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index 79f3cf3b5c..7be45f40b9 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -1650,7 +1650,8 @@ size_t IndexImpl::getResultSizeOfScan( const auto& locatedTriples = perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot); return perm.getResultSizeOfScan( - perm.getScanSpecAndBlocks(scanSpecification, locatedTriples), + CompressedRelationReader::ScanSpecAndBlocks::withUpdates( + scanSpecification, locatedTriples), locatedTriples); } diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp index 3092f1b587..3fa43d5817 100644 --- a/src/index/Permutation.cpp +++ b/src/index/Permutation.cpp @@ -18,13 +18,6 @@ Permutation::Permutation(Enum permutation, Allocator allocator) allocator_{std::move(allocator)}, permutation_{permutation} {} -// _____________________________________________________________________ -CompressedRelationReader::ScanSpecAndBlocks Permutation::getScanSpecAndBlocks( - const ScanSpecification& scanSpec, - const LocatedTriplesPerBlock& locatedTriples) const { - return {scanSpec, getAugmentedMetadata(locatedTriples)}; -} - // _____________________________________________________________________ void Permutation::loadFromDisk(const std::string& onDiskBase, bool loadInternalPermutation) { @@ -102,7 +95,7 @@ IdTable Permutation::getDistinctCol1IdsAndCounts( const LocatedTriplesPerBlock& locatedTriples, const LimitOffsetClause& limitOffset) const { return reader().getDistinctCol1IdsAndCounts( - getScanSpecAndBlocks( + CompressedRelationReader::ScanSpecAndBlocks::withUpdates( ScanSpecification{col0Id, std::nullopt, std::nullopt}, locatedTriples), cancellationHandle, locatedTriples, limitOffset); @@ -115,8 +108,9 @@ IdTable Permutation::getDistinctCol0IdsAndCounts( const LimitOffsetClause& limitOffset) const { ScanSpecification scanSpec{std::nullopt, std::nullopt, std::nullopt}; return reader().getDistinctCol0IdsAndCounts( - getScanSpecAndBlocks(scanSpec, locatedTriples), cancellationHandle, - locatedTriples, limitOffset); + CompressedRelationReader::ScanSpecAndBlocks::withUpdates(scanSpec, + locatedTriples), + cancellationHandle, locatedTriples, limitOffset); } // _____________________________________________________________________ @@ -166,7 +160,7 @@ std::optional Permutation::getMetadata( return meta_.getMetaData(col0Id); } return reader().getMetadataForSmallRelation( - getScanSpecAndBlocks( + CompressedRelationReader::ScanSpecAndBlocks::withUpdates( ScanSpecification{col0Id, std::nullopt, std::nullopt}, locatedTriples), col0Id, locatedTriples); @@ -213,13 +207,6 @@ const LocatedTriplesPerBlock& Permutation::getLocatedTriplesForPermutation( permutation_); } -// ______________________________________________________________________ -BlockMetadataRanges Permutation::getAugmentedMetadata( - const LocatedTriplesPerBlock& locatedTriples) const { - BlockMetadataSpan blocks(locatedTriples.getAugmentedMetadata()); - return {{blocks.begin(), blocks.end()}}; -} - // ______________________________________________________________________ const Permutation& Permutation::internalPermutation() const { AD_CONTRACT_CHECK(internalPermutation_ != nullptr); diff --git a/src/index/Permutation.h b/src/index/Permutation.h index 8e5103398f..93050cadc4 100644 --- a/src/index/Permutation.h +++ b/src/index/Permutation.h @@ -19,9 +19,9 @@ // Forward declaration of `IdTable` class IdTable; -// Forward declaration of `LocatedTriplesPerBlock` +// TODO: get rid of them +// Forward declaration of `LocatedTriplesPerBlock` and `LocatedTriplesSnapshot` class LocatedTriplesPerBlock; -class SharedLocatedTriplesSnapshot; struct LocatedTriplesSnapshot; class DeltaTriples; @@ -129,12 +129,6 @@ class Permutation { const LocatedTriplesPerBlock& locatedTriples, const LimitOffsetClause& limitOffset = {}) const; - // Returns the corresponding `CompressedRelationReader::ScanSpecAndBlocks` - // with relevant `BlockMetadataRanges`. - ScanSpecAndBlocks getScanSpecAndBlocks( - const ScanSpecification& scanSpec, - const LocatedTriplesPerBlock& locatedTriples) const; - std::optional getMetadata( Id col0Id, const LocatedTriplesPerBlock& locatedTriples) const; @@ -188,10 +182,6 @@ class Permutation { const LocatedTriplesPerBlock& getLocatedTriplesForPermutation( const LocatedTriplesSnapshot& locatedTriplesSnapshot) const; - // From the given `LocatedTriplesPerBlock`, get the augmented block metadata. - BlockMetadataRanges getAugmentedMetadata( - const LocatedTriplesPerBlock& locatedTriples) const; - const CompressedRelationReader& reader() const { return reader_.value(); } Enum permutation() const { return permutation_; } diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp index 840b09c507..b7bee1c9e1 100644 --- a/test/CompressedRelationsTest.cpp +++ b/test/CompressedRelationsTest.cpp @@ -981,16 +981,15 @@ TEST(CompressedRelationReader, getResultSizeImpl) { auto loc = generateLocationTrace(sourceLocation); auto& perm = impl.getPermutation(p); auto& reader = perm.reader(); - auto augmentedBlocks = perm.getAugmentedMetadata( - perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot)); + auto scanSpecAndBlocks = ScanSpecAndBlocks::withUpdates( + scanSpec, perm.getLocatedTriplesForPermutation(locatedTriplesSnapshot)); auto& ltpb = locatedTriplesSnapshot.getLocatedTriplesForPermutation( perm.permutation()); - auto [actual_lower, actual_upper] = reader.getSizeEstimateForScan( - ScanSpecAndBlocks{scanSpec, augmentedBlocks}, ltpb); + auto [actual_lower, actual_upper] = + reader.getSizeEstimateForScan(scanSpecAndBlocks, ltpb); EXPECT_THAT(actual_lower, testing::Eq(lower)); EXPECT_THAT(actual_upper, testing::Eq(upper)); - auto actual_exact = reader.getResultSizeOfScan( - ScanSpecAndBlocks{scanSpec, augmentedBlocks}, ltpb); + auto actual_exact = reader.getResultSizeOfScan(scanSpecAndBlocks, ltpb); EXPECT_THAT(actual_exact, testing::Eq(exact)); }; // The Scans request all triples of the one and only block. @@ -1064,10 +1063,10 @@ TEST(CompressedRelationReader, getFirstAndLastTripleIgnoringGraph) { graphId.has_value() ? ScanSpecification::GraphFilter::Whitelist( {std::move(graphId).value()}) : ScanSpecification::GraphFilter::All()}; - CompressedRelationReader::ScanSpecAndBlocks metadataAndBlocks{ - std::move(scanSpecification), - permutation.getAugmentedMetadata( - permutation.getLocatedTriplesForPermutation(*currentSnapshot))}; + CompressedRelationReader::ScanSpecAndBlocks metadataAndBlocks = + CompressedRelationReader::ScanSpecAndBlocks::withUpdates( + std::move(scanSpecification), + permutation.getLocatedTriplesForPermutation(*currentSnapshot)); const auto& reader = index.getImpl().getPermutation(permutationEnum).reader(); return reader.getFirstAndLastTripleIgnoringGraph(metadataAndBlocks, diff --git a/test/IndexTest.cpp b/test/IndexTest.cpp index 0d64c01ea3..f6b6ba7bc4 100644 --- a/test/IndexTest.cpp +++ b/test/IndexTest.cpp @@ -57,10 +57,11 @@ auto makeTestScanWidthOne = [](const IndexImpl& index, ad_utility::source_location l = AD_CURRENT_SOURCE_LOC()) { auto t = generateLocationTrace(l); const auto& actualPermutation = index.getPermutation(permutation); - auto locatedTriples = actualPermutation.getLocatedTriplesForPermutation( - qec.locatedTriplesSnapshot()); + const auto& locatedTriples = + actualPermutation.getLocatedTriplesForPermutation( + qec.locatedTriplesSnapshot()); IdTable result = actualPermutation.scan( - actualPermutation.getScanSpecAndBlocks( + CompressedRelationReader::ScanSpecAndBlocks::withUpdates( ScanSpecificationAsTripleComponent{c0, c1, std::nullopt} .toScanSpecification(index), locatedTriples), @@ -82,10 +83,11 @@ auto makeTestScanWidthTwo = [](const IndexImpl& index, ad_utility::source_location l = AD_CURRENT_SOURCE_LOC()) { auto t = generateLocationTrace(l); const auto& actualPermutation = index.getPermutation(permutation); - auto locatedTriples = actualPermutation.getLocatedTriplesForPermutation( - qec.locatedTriplesSnapshot()); + const auto& locatedTriples = + actualPermutation.getLocatedTriplesForPermutation( + qec.locatedTriplesSnapshot()); IdTable wol = actualPermutation.scan( - actualPermutation.getScanSpecAndBlocks( + CompressedRelationReader::ScanSpecAndBlocks::withUpdates( ScanSpecificationAsTripleComponent{c0, std::nullopt, std::nullopt} .toScanSpecification(index), locatedTriples), diff --git a/test/engine/PermutationSelectorTest.cpp b/test/engine/PermutationSelectorTest.cpp index 80fae6c2dd..177451090a 100644 --- a/test/engine/PermutationSelectorTest.cpp +++ b/test/engine/PermutationSelectorTest.cpp @@ -8,10 +8,8 @@ #include "engine/PermutationSelector.h" #include "index/IndexImpl.h" -// _____________________________________________________________________________ -TEST(PermutationSelectorTest, internalPrefixIsCorrectlyChosen) { - auto* qec = ad_utility::testing::getQec(); - const auto& index = qec->getIndex(); +auto testPermutationSelection = [](auto retrivalFunction, auto getExpected, + auto getExpectedInternal) { TripleComponent internalIri{ad_utility::triple_component::Iri::fromIriref( makeQleverInternalIri("something"))}; TripleComponent languageTaggedIri{ @@ -21,20 +19,17 @@ TEST(PermutationSelectorTest, internalPrefixIsCorrectlyChosen) { TripleComponent regularLiteral{1}; for (auto permutation : Permutation::ALL) { - const auto* permutationPtr = &index.getImpl().getPermutation(permutation); + const auto* expectedPtr = getExpected(permutation); for (const auto& triple : {SparqlTripleSimple{regularIri, regularIri, regularIri}, SparqlTripleSimple{regularLiteral, regularLiteral, regularLiteral}, SparqlTripleSimple{regularLiteral, regularIri, regularLiteral}}) { - EXPECT_EQ( - qlever::getPermutationForTriple(permutation, index, triple).get(), - permutationPtr); + EXPECT_EQ(retrivalFunction(permutation, triple).get(), expectedPtr); } } for (auto permutation : Permutation::INTERNAL) { - const auto* permutationPtr = - &index.getImpl().getPermutation(permutation).internalPermutation(); + const auto* expectedPtr = getExpectedInternal(permutation); for (const auto& triple : {SparqlTripleSimple{internalIri, regularIri, regularIri}, SparqlTripleSimple{regularIri, internalIri, regularIri}, @@ -42,19 +37,51 @@ TEST(PermutationSelectorTest, internalPrefixIsCorrectlyChosen) { SparqlTripleSimple{languageTaggedIri, regularIri, regularIri}, SparqlTripleSimple{regularIri, languageTaggedIri, regularIri}, SparqlTripleSimple{regularIri, regularIri, languageTaggedIri}}) { - EXPECT_EQ( - qlever::getPermutationForTriple(permutation, index, triple).get(), - permutationPtr); + EXPECT_EQ(retrivalFunction(permutation, triple).get(), expectedPtr); } } using enum Permutation::Enum; // Unsupported configurations. for (auto permutation : {OPS, OSP, SOP, SPO}) { - EXPECT_THROW(qlever::getPermutationForTriple( - permutation, index, - SparqlTripleSimple{languageTaggedIri, internalIri, - languageTaggedIri}), - ad_utility::Exception); + EXPECT_THROW( + retrivalFunction(permutation, + SparqlTripleSimple{languageTaggedIri, internalIri, + languageTaggedIri}), + ad_utility::Exception); } +}; + +// _____________________________________________________________________________ +TEST(PermutationSelectorTest, internalPrefixIsCorrectlyChosen) { + auto* qec = ad_utility::testing::getQec(); + const auto& index = qec->getIndex(); + + testPermutationSelection( + [&index](auto p, const auto& triple) { + return qlever::getPermutationForTriple(p, index, triple); + }, + + [&index](const auto& p) { return &index.getImpl().getPermutation(p); }, + [&index](const auto& p) { + return &index.getImpl().getPermutation(p).internalPermutation(); + }); +} + +// _____________________________________________________________________________ +TEST(PermutationSelectorTest, getLocatedTriplesPerBlockForTriple) { + const auto* qec = ad_utility::testing::getQec(); + const auto& locatedTriples = qec->sharedLocatedTriplesSnapshot(); + + testPermutationSelection( + [&locatedTriples](auto p, const auto& triple) { + return qlever::getLocatedTriplesPerBlockForTriple(p, locatedTriples, + triple); + }, + [&locatedTriples](const auto& p) { + return &locatedTriples->getLocatedTriplesForPermutation(p); + }, + [&locatedTriples](const auto& p) { + return &locatedTriples->getInternalLocatedTriplesForPermutation(p); + }); } diff --git a/test/util/IndexTestHelpers.cpp b/test/util/IndexTestHelpers.cpp index ceb68755c3..78ebf435d2 100644 --- a/test/util/IndexTestHelpers.cpp +++ b/test/util/IndexTestHelpers.cpp @@ -84,13 +84,13 @@ void checkConsistencyBetweenPatternPredicateAndAdditionalColumn( indexImpl.getPermutation(Permutation::Enum::PSO).internalPermutation(); const auto& locatedTriples = permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot); - auto scanResultHasPattern = - permutation.scan(permutation.getScanSpecAndBlocks( - ScanSpecificationAsTripleComponent{ - iriOfHasPattern, id, std::nullopt} - .toScanSpecification(indexImpl), - locatedTriples), - {}, cancellationDummy, locatedTriples); + auto scanResultHasPattern = permutation.scan( + CompressedRelationReader::ScanSpecAndBlocks::withUpdates( + ScanSpecificationAsTripleComponent{iriOfHasPattern, id, + std::nullopt} + .toScanSpecification(indexImpl), + locatedTriples), + {}, cancellationDummy, locatedTriples); // Each ID has at most one pattern, it can have none if it doesn't // appear as a subject in the knowledge graph. AD_CORRECTNESS_CHECK(scanResultHasPattern.numRows() <= 1); @@ -109,7 +109,7 @@ void checkConsistencyBetweenPatternPredicateAndAdditionalColumn( const auto& locatedTriples = permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot); auto scanResult = permutation.scan( - permutation.getScanSpecAndBlocks( + CompressedRelationReader::ScanSpecAndBlocks::withUpdates( ScanSpecification{col0Id, std::nullopt, std::nullopt}, locatedTriples), std::array{ColumnIndex{ADDITIONAL_COLUMN_INDEX_SUBJECT_PATTERN}, From 2b3fa80df964de11aa4ebea4b098ac582877bf2f Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Fri, 26 Dec 2025 16:43:07 +0100 Subject: [PATCH 12/17] it is indeed not only used in tests --- src/engine/IndexScan.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 324643f990..fa39bc6f60 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -80,8 +80,6 @@ IndexScan::IndexScan(QueryExecutionContext* qec, std::optional scanSpecAndBlocks) : IndexScan( qec, qec->getIndex().getImpl().getPermutationPtr(permutationType), - // TODO this constructor is only used in tests, can it be - // removed? std::shared_ptr{ qec->sharedLocatedTriplesSnapshot(), &qec->sharedLocatedTriplesSnapshot() From 8c551363a92db337dcf9b63ab59c249733e74a65 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Mon, 12 Jan 2026 13:01:14 +0100 Subject: [PATCH 13/17] resolve some todos --- src/engine/IndexScan.cpp | 1 - src/engine/IndexScan.h | 3 --- src/engine/PermutationSelector.cpp | 4 ++-- src/engine/PermutationSelector.h | 9 ++------- src/index/Permutation.h | 5 ++++- 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index acf2ef4eb0..604b7d402d 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -79,7 +79,6 @@ IndexScan::IndexScan(QueryExecutionContext* qec, Permutation::Enum permutationType, const SparqlTripleSimple& triple, Graphs graphsToFilter, std::optional scanSpecAndBlocks) - // TODO<2618>: do the internal selection correctly : IndexScan( qec, qec->getIndex().getImpl().getPermutationPtr(permutationType), std::shared_ptr{ diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index 374ee1eb14..00f6bb4a93 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -17,10 +17,7 @@ class SparqlTripleSimple; class IndexScan final : public Operation { public: using Graphs = ScanSpecificationAsTripleComponent::GraphFilter; - // TODO<2618>: get rid of them, especially the dubling with - // `PermutationSelector` using PermutationPtr = std::shared_ptr; - using LocatedTriplesStatePtr = std::shared_ptr; using LocatedTriplesPerBlockPtr = std::shared_ptr; diff --git a/src/engine/PermutationSelector.cpp b/src/engine/PermutationSelector.cpp index 5b051feea0..0b6ee906f7 100644 --- a/src/engine/PermutationSelector.cpp +++ b/src/engine/PermutationSelector.cpp @@ -52,7 +52,7 @@ PermutationPtr getPermutationForTriple(Permutation::Enum permutation, // _____________________________________________________________________________ LocatedTriplesPerBlockPtr getLocatedTriplesPerBlockForTriple( - Permutation::Enum permutation, LocatedTriplesStatePtr snapshot, + Permutation::Enum permutation, LocatedTriplesSharedState snapshot, const SparqlTripleSimple& triple) { // Create alias shared pointer of internal the right `LocatedTriplesPerBlock`. const auto& locatedTriples = @@ -66,7 +66,7 @@ LocatedTriplesPerBlockPtr getLocatedTriplesPerBlockForTriple( std::pair getPermutationAndLocatedTriplesPerBlockForTriple( Permutation::Enum permutation, const Index& index, - LocatedTriplesStatePtr snapshot, const SparqlTripleSimple& triple) { + LocatedTriplesSharedState snapshot, const SparqlTripleSimple& triple) { return {getPermutationForTriple(permutation, index, triple), getLocatedTriplesPerBlockForTriple(permutation, std::move(snapshot), triple)}; diff --git a/src/engine/PermutationSelector.h b/src/engine/PermutationSelector.h index c65a7fb23c..fcf787475f 100644 --- a/src/engine/PermutationSelector.h +++ b/src/engine/PermutationSelector.h @@ -5,16 +5,11 @@ #ifndef QLEVER_SRC_ENGINE_PERMUTATIONSELECTOR_H #define QLEVER_SRC_ENGINE_PERMUTATIONSELECTOR_H -#include - -#include "index/Index.h" #include "index/Permutation.h" #include "parser/SparqlTriple.h" namespace qlever { -// TODO<2618>: get rid of them -using LocatedTriplesStatePtr = std::shared_ptr; using LocatedTriplesPerBlockPtr = std::shared_ptr; using PermutationPtr = std::shared_ptr; @@ -29,7 +24,7 @@ PermutationPtr getPermutationForTriple(Permutation::Enum permutation, // based on the `permutation` enum and the values in `triple`. See // `getPermutationAndLocatedTriplesPerBlockForTriple` for details. LocatedTriplesPerBlockPtr getLocatedTriplesPerBlockForTriple( - Permutation::Enum permutation, LocatedTriplesStatePtr snapshot, + Permutation::Enum permutation, LocatedTriplesSharedState snapshot, const SparqlTripleSimple& triple); // Return a shared pointer to the correct permutation in `index` and @@ -43,7 +38,7 @@ LocatedTriplesPerBlockPtr getLocatedTriplesPerBlockForTriple( std::pair getPermutationAndLocatedTriplesPerBlockForTriple( Permutation::Enum permutation, const Index& index, - LocatedTriplesStatePtr snapshot, const SparqlTripleSimple& triple); + LocatedTriplesSharedState snapshot, const SparqlTripleSimple& triple); } // namespace qlever #endif // QLEVER_SRC_ENGINE_PERMUTATIONSELECTOR_H diff --git a/src/index/Permutation.h b/src/index/Permutation.h index dc2a2f810a..37f21b46bb 100644 --- a/src/index/Permutation.h +++ b/src/index/Permutation.h @@ -19,7 +19,6 @@ // Forward declaration of `IdTable` class IdTable; -// TODO<2618>: get rid of them // Forward declaration of `LocatedTriplesPerBlock` and `LocatedTriplesSnapshot` class LocatedTriplesPerBlock; struct LocatedTriplesState; @@ -216,4 +215,8 @@ class Permutation { bool isInternalPermutation_ = false; }; +namespace qlever { +using PermutationPtr = std::shared_ptr; +} + #endif // QLEVER_SRC_INDEX_PERMUTATION_H From edd6a90167a3d80a7143800946db31c833c2a57d Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Mon, 12 Jan 2026 13:22:14 +0100 Subject: [PATCH 14/17] resolve todos --- src/engine/HasPredicateScan.cpp | 5 ++--- src/engine/IndexScan.cpp | 10 +++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/engine/HasPredicateScan.cpp b/src/engine/HasPredicateScan.cpp index 13f1eaf75f..4097b64b21 100644 --- a/src/engine/HasPredicateScan.cpp +++ b/src/engine/HasPredicateScan.cpp @@ -424,7 +424,6 @@ std::shared_ptr HasPredicateScan::makePatternScan( qlever::getPermutationAndLocatedTriplesPerBlockForTriple( Permutation::Enum::PSO, qec->getIndex(), qec->locatedTriplesSharedState(), triple); - return ad_utility::makeExecutionTree(qec, std::move(permutation), - std::move(locatedTriples), - std::move(triple)); + return ad_utility::makeExecutionTree( + qec, std::move(permutation), std::move(locatedTriples), triple); } diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 604b7d402d..77038d1eb1 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -11,6 +11,7 @@ #include #include +#include "PermutationSelector.h" #include "engine/QueryExecutionTree.h" #include "index/IndexImpl.h" #include "parser/ParsedQuery.h" @@ -81,10 +82,8 @@ IndexScan::IndexScan(QueryExecutionContext* qec, std::optional scanSpecAndBlocks) : IndexScan( qec, qec->getIndex().getImpl().getPermutationPtr(permutationType), - std::shared_ptr{ - qec->locatedTriplesSharedState(), - &qec->locatedTriplesSharedState() - ->getLocatedTriplesForPermutation(permutationType)}, + qlever::getLocatedTriplesPerBlockForTriple( + permutationType, qec->locatedTriplesSharedState(), triple), triple, std::move(graphsToFilter), std::move(scanSpecAndBlocks)) {} // _____________________________________________________________________________ @@ -399,7 +398,8 @@ IndexScan::getSortedVariableAndMetadataColumnIndexForPrefiltering() const { } // ___________________________________________________________________________ -Permutation::ScanSpecAndBlocks IndexScan::getScanSpecAndBlocks() const { +CompressedRelationReader::ScanSpecAndBlocks IndexScan::getScanSpecAndBlocks() + const { return CompressedRelationReader::ScanSpecAndBlocks::withUpdates( getScanSpecification(), locatedTriplesPerBlock()); } From e850238c1d1da42b17a10008217714515937679f Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Mon, 12 Jan 2026 14:37:40 +0100 Subject: [PATCH 15/17] test function inlining --- src/engine/CountAvailablePredicates.cpp | 14 ++++++------- src/engine/GroupByImpl.cpp | 7 ++++--- src/index/IndexImpl.cpp | 5 +++-- test/CompressedRelationsTest.cpp | 27 ++++++++++++------------ test/IndexTest.cpp | 28 +++++++++++++++---------- test/util/IndexTestHelpers.cpp | 17 +++++++++------ 6 files changed, 55 insertions(+), 43 deletions(-) diff --git a/src/engine/CountAvailablePredicates.cpp b/src/engine/CountAvailablePredicates.cpp index 81451adffa..02799b2b0a 100644 --- a/src/engine/CountAvailablePredicates.cpp +++ b/src/engine/CountAvailablePredicates.cpp @@ -168,13 +168,13 @@ void CountAvailablePredicates::computePatternTrickAllEntities( TripleComponent::Iri::fromIriref(HAS_PATTERN_PREDICATE), std::nullopt, std::nullopt} .toScanSpecification(index); - const auto& perm = index.getPermutation(Permutation::Enum::PSO); - const auto& locatedTriple = - perm.getLocatedTriplesForPermutation(locatedTriplesState()); - auto fullHasPattern = - perm.lazyScan(CompressedRelationReader::ScanSpecAndBlocks::withUpdates( - scanSpec, locatedTriple), - std::nullopt, {}, cancellationHandle_, locatedTriple); + const auto perm = Permutation::Enum::PSO; + const auto& locatedTriples = + locatedTriplesState().getLocatedTriplesForPermutation(perm); + auto fullHasPattern = index.getPermutation(perm).lazyScan( + CompressedRelationReader::ScanSpecAndBlocks::withUpdates(scanSpec, + locatedTriples), + std::nullopt, {}, cancellationHandle_, locatedTriples); for (const auto& idTable : fullHasPattern) { for (const auto& patternId : idTable.getColumn(1)) { AD_CORRECTNESS_CHECK(patternId.getDatatype() == Datatype::Int); diff --git a/src/engine/GroupByImpl.cpp b/src/engine/GroupByImpl.cpp index 0b3bef333f..68e198fa55 100644 --- a/src/engine/GroupByImpl.cpp +++ b/src/engine/GroupByImpl.cpp @@ -840,9 +840,9 @@ std::optional GroupByImpl::computeGroupByObjectWithCount() const { // Compute the result and update the runtime information (we don't actually // do the index scan, but something smarter). const auto& permutation = indexScan->permutation(); + const auto& locatedTriples = indexScan->locatedTriplesPerBlock(); auto result = permutation.getDistinctCol1IdsAndCounts( - col0Id.value(), cancellationHandle_, - permutation.getLocatedTriplesForPermutation(locatedTriplesState()), + col0Id.value(), cancellationHandle_, locatedTriples, indexScan->getLimitOffset()); indexScan->updateRuntimeInformationWhenOptimizedOut({}); @@ -902,7 +902,8 @@ std::optional GroupByImpl::computeGroupByForFullIndexScan() const { permutationEnum.value()); auto table = permutation.getDistinctCol0IdsAndCounts( cancellationHandle_, - permutation.getLocatedTriplesForPermutation(locatedTriplesState()), + locatedTriplesState().getLocatedTriplesForPermutation( + permutationEnum.value()), indexScan->getLimitOffset()); if (numCounts == 0) { table.setColumnSubset(std::array{ColumnIndex{0}}); diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index f61be29b36..bd1e68090e 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -1565,7 +1565,8 @@ size_t IndexImpl::getCardinality( const LocatedTriplesState& locatedTriplesState) const { const auto& perm = getPermutation(permutation); if (const auto& meta = perm.getMetadata( - id, perm.getLocatedTriplesForPermutation(locatedTriplesState)); + id, locatedTriplesState.getLocatedTriplesForPermutation( + permutation)); meta.has_value()) { return meta.value().numRows_; } @@ -1643,7 +1644,7 @@ size_t IndexImpl::getResultSizeOfScan( const LocatedTriplesState& locatedTriplesState) const { const auto& perm = getPermutation(permutation); const auto& locatedTriples = - perm.getLocatedTriplesForPermutation(locatedTriplesState); + locatedTriplesState.getLocatedTriplesForPermutation(permutation); return perm.getResultSizeOfScan( CompressedRelationReader::ScanSpecAndBlocks::withUpdates( scanSpecification, locatedTriples), diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp index 422178af6f..0ea1a0443b 100644 --- a/test/CompressedRelationsTest.cpp +++ b/test/CompressedRelationsTest.cpp @@ -985,7 +985,8 @@ TEST(CompressedRelationReader, getResultSizeImpl) { auto& perm = impl.getPermutation(p); auto& reader = perm.reader(); auto scanSpecAndBlocks = ScanSpecAndBlocks::withUpdates( - scanSpec, perm.getLocatedTriplesForPermutation(locatedTriplesState)); + scanSpec, + locatedTriplesState.getLocatedTriplesForPermutation(p)); auto& ltpb = locatedTriplesState.getLocatedTriplesForPermutation( perm.permutation()); auto [actual_lower, actual_upper] = @@ -1071,8 +1072,8 @@ TEST(CompressedRelationReader, getFirstAndLastTripleIgnoringGraph) { CompressedRelationReader::ScanSpecAndBlocks metadataAndBlocks = CompressedRelationReader::ScanSpecAndBlocks::withUpdates( std::move(scanSpecification), - permutation.getLocatedTriplesForPermutation( - *currentLocatedTriplesSharedState)); + currentLocatedTriplesSharedState + ->getLocatedTriplesForPermutation(permutationEnum)); const auto& reader = index.getImpl().getPermutation(permutationEnum).reader(); return reader.getFirstAndLastTripleIgnoringGraph(metadataAndBlocks, @@ -1129,15 +1130,15 @@ TEST(CompressedRelationReader, ensureDummyBlockWith6ColumnsDoesntCauseIssues) { for (bool usePatternPermutation : {false, true}) { auto permutationEnum = usePatternPermutation ? Permutation::Enum::PSO : Permutation::Enum::SPO; - const auto& permutation = index.getImpl().getPermutation(permutationEnum); + const auto& locatedTriples = + locatedTriplesSharedState->getLocatedTriplesForPermutation( + permutationEnum); ScanSpecification scanSpecification{std::nullopt, std::nullopt, std::nullopt}; auto metadataAndBlocks = CompressedRelationReader::ScanSpecAndBlocks::withUpdates( - std::move(scanSpecification), - permutation.getLocatedTriplesForPermutation( - *locatedTriplesSharedState)); + std::move(scanSpecification), locatedTriples); std::vector additionalColumns{ADDITIONAL_COLUMN_GRAPH_ID}; if (usePatternPermutation) { @@ -1146,13 +1147,11 @@ TEST(CompressedRelationReader, ensureDummyBlockWith6ColumnsDoesntCauseIssues) { } while (!additionalColumns.empty()) { - auto blocks = index.getImpl() - .getPermutation(permutationEnum) - .lazyScan(metadataAndBlocks, std::nullopt, - additionalColumns, cancellationHandle, - locatedTriplesSharedState - ->getLocatedTriplesForPermutation( - permutationEnum)); + auto blocks = + index.getImpl() + .getPermutation(permutationEnum) + .lazyScan(metadataAndBlocks, std::nullopt, additionalColumns, + cancellationHandle, locatedTriples); for (const IdTable& block : blocks) { EXPECT_EQ(block.numColumns(), 3 + additionalColumns.size()); } diff --git a/test/IndexTest.cpp b/test/IndexTest.cpp index 5d55d310d9..15a2c81b28 100644 --- a/test/IndexTest.cpp +++ b/test/IndexTest.cpp @@ -58,8 +58,8 @@ auto makeTestScanWidthOne = [](const IndexImpl& index, auto t = generateLocationTrace(l); const auto& actualPermutation = index.getPermutation(permutation); const auto& locatedTriples = - actualPermutation.getLocatedTriplesForPermutation( - qec.locatedTriplesState()); + qec.locatedTriplesState().getLocatedTriplesForPermutation( + permutation); IdTable result = actualPermutation.scan( CompressedRelationReader::ScanSpecAndBlocks::withUpdates( ScanSpecificationAsTripleComponent{c0, c1, std::nullopt} @@ -84,8 +84,8 @@ auto makeTestScanWidthTwo = [](const IndexImpl& index, auto t = generateLocationTrace(l); const auto& actualPermutation = index.getPermutation(permutation); const auto& locatedTriples = - actualPermutation.getLocatedTriplesForPermutation( - qec.locatedTriplesState()); + qec.locatedTriplesState().getLocatedTriplesForPermutation( + permutation); IdTable wol = actualPermutation.scan( CompressedRelationReader::ScanSpecAndBlocks::withUpdates( ScanSpecificationAsTripleComponent{c0, std::nullopt, std::nullopt} @@ -121,7 +121,7 @@ TEST(IndexTest, createFromTurtleTest) { return; } const auto& [index, qec] = getIndex(); - const auto& locatedTriplesSnapshot = qec.locatedTriplesState(); + const auto& locatedTriplesState = qec.locatedTriplesState(); auto getId = makeGetId(getQec(kb)->getIndex()); Id a = getId(""); @@ -134,7 +134,8 @@ TEST(IndexTest, createFromTurtleTest) { // TODO We could also test the multiplicities here. const auto& pso = index.PSO(); const auto& psoLTPB = - pso.getLocatedTriplesForPermutation(locatedTriplesSnapshot); + qec.locatedTriplesState().getLocatedTriplesForPermutation( + Permutation::PSO); ASSERT_TRUE(pso.getMetadata(b, psoLTPB).has_value()); ASSERT_TRUE(pso.getMetadata(b2, psoLTPB).has_value()); ASSERT_FALSE(pso.getMetadata(a2, psoLTPB).has_value()); @@ -148,7 +149,8 @@ TEST(IndexTest, createFromTurtleTest) { const auto& pos = index.POS(); const auto& posLTPB = - pos.getLocatedTriplesForPermutation(locatedTriplesSnapshot); + locatedTriplesState.getLocatedTriplesForPermutation( + Permutation::POS); ASSERT_TRUE(pos.getMetadata(b, posLTPB).has_value()); ASSERT_TRUE(pos.getMetadata(b2, posLTPB).has_value()); ASSERT_FALSE(pos.getMetadata(a2, posLTPB).has_value()); @@ -209,14 +211,16 @@ TEST(IndexTest, createFromTurtleTest) { Id isA = getId(""); const auto& pso = index.PSO(); - const auto& psoLTPB = pso.getLocatedTriplesForPermutation(deltaTriples); + const auto& psoLTPB = deltaTriples.getLocatedTriplesForPermutation( + Permutation::Enum::PSO); ASSERT_TRUE(pso.getMetadata(isA, psoLTPB).has_value()); ASSERT_FALSE(pso.getMetadata(a, psoLTPB).has_value()); ASSERT_FALSE(pso.getMetadata(isA, psoLTPB).value().isFunctional()); const auto& pos = index.POS(); - const auto& posLTPB = pos.getLocatedTriplesForPermutation(deltaTriples); + const auto& posLTPB = deltaTriples.getLocatedTriplesForPermutation( + Permutation::Enum::POS); ASSERT_TRUE(pos.getMetadata(isA, posLTPB).has_value()); ASSERT_FALSE(pos.getMetadata(isA, posLTPB).value().isFunctional()); @@ -264,7 +268,8 @@ TEST(IndexTest, createFromOnDiskIndexTest) { Id c = getId(""); const auto& pso = index.PSO(); - const auto& psoLTPB = pso.getLocatedTriplesForPermutation(deltaTriples); + const auto& psoLTPB = deltaTriples.getLocatedTriplesForPermutation( + Permutation::Enum::PSO); ASSERT_TRUE(pso.getMetadata(b, psoLTPB).has_value()); ASSERT_TRUE(pso.getMetadata(b2, psoLTPB).has_value()); ASSERT_FALSE(pso.getMetadata(a, psoLTPB).has_value()); @@ -273,7 +278,8 @@ TEST(IndexTest, createFromOnDiskIndexTest) { ASSERT_TRUE(pso.getMetadata(b2, psoLTPB).value().isFunctional()); const auto& pos = index.POS(); - const auto& posLTPB = pos.getLocatedTriplesForPermutation(deltaTriples); + const auto& posLTPB = deltaTriples.getLocatedTriplesForPermutation( + Permutation::Enum::POS); ASSERT_TRUE(pos.getMetadata(b, posLTPB).has_value()); ASSERT_TRUE(pos.getMetadata(b2, posLTPB).has_value()); ASSERT_FALSE(pos.getMetadata(a, posLTPB).has_value()); diff --git a/test/util/IndexTestHelpers.cpp b/test/util/IndexTestHelpers.cpp index 6b0622bcbd..753e3247a3 100644 --- a/test/util/IndexTestHelpers.cpp +++ b/test/util/IndexTestHelpers.cpp @@ -82,11 +82,12 @@ void checkConsistencyBetweenPatternPredicateAndAdditionalColumn( auto checkSingleElement = [&cancellationDummy, &iriOfHasPattern, &locatedTriplesSnapshot, &indexImpl](size_t patternIdx, Id id) { - const auto& permutation = - indexImpl.getPermutation(Permutation::Enum::PSO).internalPermutation(); + const auto perm = Permutation::Enum::PSO; + const auto& actualPermutation = + indexImpl.getPermutation(perm).internalPermutation(); const auto& locatedTriples = - permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot); - auto scanResultHasPattern = permutation.scan( + locatedTriplesSnapshot.getLocatedTriplesForPermutation(perm); + auto scanResultHasPattern = actualPermutation.scan( CompressedRelationReader::ScanSpecAndBlocks::withUpdates( ScanSpecificationAsTripleComponent{iriOfHasPattern, id, std::nullopt} @@ -145,14 +146,18 @@ void checkConsistencyBetweenPatternPredicateAndAdditionalColumn( const auto& pso = index.getImpl().PSO(); auto predicates = pso.getDistinctCol0IdsAndCounts( cancellationDummy, - pso.getLocatedTriplesForPermutation(locatedTriplesSnapshot), {}); + locatedTriplesSnapshot.getLocatedTriplesForPermutation( + Permutation::Enum::PSO), + {}); for (const auto& predicate : predicates.getColumn(0)) { checkConsistencyForPredicate(predicate); } const auto& osp = index.getImpl().OSP(); auto objects = osp.getDistinctCol0IdsAndCounts( cancellationDummy, - osp.getLocatedTriplesForPermutation(locatedTriplesSnapshot), {}); + locatedTriplesSnapshot.getLocatedTriplesForPermutation( + Permutation::Enum::OSP), + {}); for (const auto& object : objects.getColumn(0)) { checkConsistencyForObject(object); } From 2e1516a9a9e7dc724739896ed2ccad1bf6be676a Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Mon, 12 Jan 2026 15:55:41 +0100 Subject: [PATCH 16/17] code review --- src/engine/IndexScan.cpp | 20 +++++++++++++++++--- src/engine/IndexScan.h | 8 ++++++-- src/engine/PermutationSelector.h | 4 ++++ src/index/Index.cpp | 7 ------- src/index/Index.h | 5 ----- src/index/IndexImpl.cpp | 17 +---------------- src/index/IndexImpl.h | 5 ----- src/index/Permutation.cpp | 10 ---------- src/index/Permutation.h | 4 ---- test/IndexTest.cpp | 6 ------ test/engine/IndexScanTest.cpp | 10 ++++++++++ test/engine/PermutationSelectorTest.cpp | 19 ++++++++++++------- test/util/IndexTestHelpers.cpp | 13 +++++++------ 13 files changed, 57 insertions(+), 71 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 77038d1eb1..a53cfdcf2e 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -11,7 +11,7 @@ #include #include -#include "PermutationSelector.h" +#include "engine/PermutationSelector.h" #include "engine/QueryExecutionTree.h" #include "index/IndexImpl.h" #include "parser/ParsedQuery.h" @@ -342,8 +342,7 @@ void IndexScan::determineMultiplicities() { // There are no duplicate triples in RDF and two elements are fixed. return {1.0f}; } else if (numVariables_ == 2) { - return idx.getMultiplicities(*getPermutedTriple()[0], permutation(), - locatedTriplesState()); + return getMultiplicities(*getPermutedTriple()[0]); } else { AD_CORRECTNESS_CHECK(numVariables_ == 3); return idx.getMultiplicities(permutation()); @@ -803,3 +802,18 @@ std::vector IndexScan::getSubsetForStrippedColumns() const { } return result; } + +// _____________________________________________________________________________ +std::vector IndexScan::getMultiplicities( + const TripleComponent& key) const { + const auto& idx = getIndex(); + if (auto keyId = key.toValueId(idx.getVocab(), idx.encodedIriManager())) { + auto meta = + permutation().getMetadata(keyId.value(), locatedTriplesPerBlock()); + if (meta.has_value()) { + return {meta.value().getCol1Multiplicity(), + meta.value().getCol2Multiplicity()}; + } + } + return {1.0f, 1.0f}; +} diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index 00f6bb4a93..48dda9e978 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -15,6 +15,8 @@ class SparqlTriple; class SparqlTripleSimple; class IndexScan final : public Operation { + FRIEND_TEST(IndexScanTest, getMultiplicities); + public: using Graphs = ScanSpecificationAsTripleComponent::GraphFilter; using PermutationPtr = std::shared_ptr; @@ -188,8 +190,8 @@ class IndexScan final : public Operation { // Instead of using the `LocatedTriplesSnapshot` of the `Operation` base // class, which accesses the one stored in the `QueryExecutionContext`, use // the `LocatedTriplesPerBlock` held in this object. This already is exactly - // the located triples for the permutation of the index scan and should be - // used wherever possible. + // the located triples for the permutation of the index scan. + // `locatedTriplesState` should not be used in `IndexScan`. const LocatedTriplesPerBlock& locatedTriplesPerBlock() const; // Return the stored triple in the order that corresponds to the @@ -290,6 +292,8 @@ class IndexScan final : public Operation { }; } + std::vector getMultiplicities(const TripleComponent& key) const; + public: std::optional> makeTreeWithStrippedColumns( diff --git a/src/engine/PermutationSelector.h b/src/engine/PermutationSelector.h index fcf787475f..df353a2034 100644 --- a/src/engine/PermutationSelector.h +++ b/src/engine/PermutationSelector.h @@ -5,6 +5,10 @@ #ifndef QLEVER_SRC_ENGINE_PERMUTATIONSELECTOR_H #define QLEVER_SRC_ENGINE_PERMUTATIONSELECTOR_H +#include + +#include "index/DeltaTriples.h" +#include "index/Index.h" #include "index/Permutation.h" #include "parser/SparqlTriple.h" diff --git a/src/index/Index.cpp b/src/index/Index.cpp index 8a8a03def7..32c19d002f 100644 --- a/src/index/Index.cpp +++ b/src/index/Index.cpp @@ -267,13 +267,6 @@ std::vector Index::getMultiplicities( return pimpl_->getMultiplicities(permutation); } -// ____________________________________________________________________________ -std::vector Index::getMultiplicities( - const TripleComponent& key, const Permutation& p, - const LocatedTriplesState& locatedTriplesState) const { - return pimpl_->getMultiplicities(key, p, locatedTriplesState); -} - // ____________________________________________________________________________ size_t Index::getResultSizeOfScan( const ScanSpecification& scanSpecification, diff --git a/src/index/Index.h b/src/index/Index.h index 985e922e59..de0ad509a3 100644 --- a/src/index/Index.h +++ b/src/index/Index.h @@ -221,11 +221,6 @@ class Index { bool hasAllPermutations() const; - // ___________________________________________________________________________ - std::vector getMultiplicities( - const TripleComponent& key, const Permutation& permutation, - const LocatedTriplesState& locatedTriplesState) const; - // ___________________________________________________________________________ std::vector getMultiplicities(const Permutation& permutation) const; diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index bd1e68090e..2924c4e505 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -1610,25 +1610,10 @@ Index::Vocab::PrefixRanges IndexImpl::prefixRanges( return vocab_.prefixRanges(prefix); } -// _____________________________________________________________________________ -std::vector IndexImpl::getMultiplicities( - const TripleComponent& key, const Permutation& permutation, - const LocatedTriplesState& locatedTriplesState) const { - if (auto keyId = key.toValueId(getVocab(), encodedIriManager())) { - auto meta = permutation.getMetadata( - keyId.value(), - permutation.getLocatedTriplesForPermutation(locatedTriplesState)); - if (meta.has_value()) { - return {meta.value().getCol1Multiplicity(), - meta.value().getCol2Multiplicity()}; - } - } - return {1.0f, 1.0f}; -} - // _____________________________________________________________________________ std::vector IndexImpl::getMultiplicities( const Permutation& permutation) const { + // TODO: these numbers are wrong if there are updates auto numTriples = static_cast(this->numTriples().normal); std::array multiplicities{numTriples / numDistinctSubjects().normal, numTriples / numDistinctPredicates().normal, diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h index 9723fd0524..c402301098 100644 --- a/src/index/IndexImpl.h +++ b/src/index/IndexImpl.h @@ -480,11 +480,6 @@ class IndexImpl { bool hasAllPermutations() const { return SPO().isLoaded(); } - // ___________________________________________________________________________ - std::vector getMultiplicities( - const TripleComponent& key, const Permutation& permutation, - const LocatedTriplesState& locatedTriplesState) const; - // ___________________________________________________________________________ std::vector getMultiplicities(const Permutation& permutation) const; diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp index 7ba9bc0c5a..a95513f3db 100644 --- a/src/index/Permutation.cpp +++ b/src/index/Permutation.cpp @@ -197,16 +197,6 @@ CompressedRelationReader::IdTableGeneratorInputRange Permutation::lazyScan( cancellationHandle, locatedTriples, limitOffset); } -// ______________________________________________________________________ -const LocatedTriplesPerBlock& Permutation::getLocatedTriplesForPermutation( - const LocatedTriplesState& locatedTriplesState) const { - return isInternalPermutation_ - ? locatedTriplesState.getLocatedTriplesForPermutation( - permutation_) - : locatedTriplesState.getLocatedTriplesForPermutation( - permutation_); -} - // ______________________________________________________________________ const Permutation& Permutation::internalPermutation() const { AD_CONTRACT_CHECK(internalPermutation_ != nullptr); diff --git a/src/index/Permutation.h b/src/index/Permutation.h index 37f21b46bb..92d489c47d 100644 --- a/src/index/Permutation.h +++ b/src/index/Permutation.h @@ -177,10 +177,6 @@ class Permutation { // _______________________________________________________ const MetaData& metaData() const { return meta_; } - // From the given snapshot, get the located triples for this permutation. - const LocatedTriplesPerBlock& getLocatedTriplesForPermutation( - const LocatedTriplesState& locatedTriplesSnapshot) const; - const CompressedRelationReader& reader() const { return reader_.value(); } Enum permutation() const { return permutation_; } diff --git a/test/IndexTest.cpp b/test/IndexTest.cpp index 15a2c81b28..7a81bb0788 100644 --- a/test/IndexTest.cpp +++ b/test/IndexTest.cpp @@ -511,12 +511,6 @@ TEST(IndexTest, NumDistinctEntities) { EXPECT_FLOAT_EQ(multiplicities[0], 7.0 / 3.0); EXPECT_FLOAT_EQ(multiplicities[1], 7.0 / 2.0); EXPECT_FLOAT_EQ(multiplicities[2], 7.0 / 7.0); - - multiplicities = index.getMultiplicities( - iri(""), index.getPermutation(Permutation::SPO), - qec.locatedTriplesState()); - EXPECT_FLOAT_EQ(multiplicities[0], 2.5); - EXPECT_FLOAT_EQ(multiplicities[1], 1); } TEST(IndexTest, NumDistinctEntitiesCornerCases) { diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp index 4a92121d9a..5c41ec3355 100644 --- a/test/engine/IndexScanTest.cpp +++ b/test/engine/IndexScanTest.cpp @@ -1823,3 +1823,13 @@ TEST(IndexScanTest, StripColumnsWithPrefiltering) { << varsToKeep.size() << " variables"; } } + +TEST(IndexScanTest, getMultiplicities) { + auto* qec = getQec(); + auto s = Tc::Iri::fromIriref(""); + IndexScan scan{qec, Permutation::SPO, + SparqlTripleSimple{s, Var{"?y"}, Var{"?z"}}}; + auto multiplicities = scan.getMultiplicities(s); + EXPECT_FLOAT_EQ(multiplicities[0], 2.5); + EXPECT_FLOAT_EQ(multiplicities[1], 1); +} diff --git a/test/engine/PermutationSelectorTest.cpp b/test/engine/PermutationSelectorTest.cpp index 33f09973e4..1e3c077bba 100644 --- a/test/engine/PermutationSelectorTest.cpp +++ b/test/engine/PermutationSelectorTest.cpp @@ -4,12 +4,17 @@ #include +#include "../util/GTestHelpers.h" #include "../util/IndexTestHelpers.h" #include "engine/PermutationSelector.h" #include "index/IndexImpl.h" -auto testPermutationSelection = [](auto retrivalFunction, auto getExpected, - auto getExpectedInternal) { +auto testPermutationSelection = [](auto retrievalFunction, auto getExpected, + auto getExpectedInternal, + ad_utility::source_location sourceLocation = + AD_CURRENT_SOURCE_LOC()) { + auto l = generateLocationTrace(sourceLocation); + TripleComponent internalIri{ad_utility::triple_component::Iri::fromIriref( makeQleverInternalIri("something"))}; TripleComponent languageTaggedIri{ @@ -24,7 +29,7 @@ auto testPermutationSelection = [](auto retrivalFunction, auto getExpected, {SparqlTripleSimple{regularIri, regularIri, regularIri}, SparqlTripleSimple{regularLiteral, regularLiteral, regularLiteral}, SparqlTripleSimple{regularLiteral, regularIri, regularLiteral}}) { - EXPECT_EQ(retrivalFunction(permutation, triple).get(), expectedPtr); + EXPECT_EQ(retrievalFunction(permutation, triple).get(), expectedPtr); } } @@ -37,7 +42,7 @@ auto testPermutationSelection = [](auto retrivalFunction, auto getExpected, SparqlTripleSimple{languageTaggedIri, regularIri, regularIri}, SparqlTripleSimple{regularIri, languageTaggedIri, regularIri}, SparqlTripleSimple{regularIri, regularIri, languageTaggedIri}}) { - EXPECT_EQ(retrivalFunction(permutation, triple).get(), expectedPtr); + EXPECT_EQ(retrievalFunction(permutation, triple).get(), expectedPtr); } } @@ -45,9 +50,9 @@ auto testPermutationSelection = [](auto retrivalFunction, auto getExpected, // Unsupported configurations. for (auto permutation : {OPS, OSP, SOP, SPO}) { EXPECT_THROW( - retrivalFunction(permutation, - SparqlTripleSimple{languageTaggedIri, internalIri, - languageTaggedIri}), + retrievalFunction(permutation, + SparqlTripleSimple{languageTaggedIri, internalIri, + languageTaggedIri}), ad_utility::Exception); } }; diff --git a/test/util/IndexTestHelpers.cpp b/test/util/IndexTestHelpers.cpp index 753e3247a3..d27f17e044 100644 --- a/test/util/IndexTestHelpers.cpp +++ b/test/util/IndexTestHelpers.cpp @@ -106,11 +106,12 @@ void checkConsistencyBetweenPatternPredicateAndAdditionalColumn( } }; - auto checkConsistencyForCol0IdAndPermutation = + auto checkConsistencyForCol0IdAndExternalPermutation = [&](Id col0Id, const Permutation& permutation, size_t subjectColIdx, size_t objectColIdx) { const auto& locatedTriples = - permutation.getLocatedTriplesForPermutation(locatedTriplesSnapshot); + locatedTriplesSnapshot.getLocatedTriplesForPermutation( + permutation.permutation()); auto scanResult = permutation.scan( CompressedRelationReader::ScanSpecAndBlocks::withUpdates( ScanSpecification{col0Id, std::nullopt, std::nullopt}, @@ -131,16 +132,16 @@ void checkConsistencyBetweenPatternPredicateAndAdditionalColumn( auto checkConsistencyForPredicate = [&](Id predicateId) { using enum Permutation::Enum; - checkConsistencyForCol0IdAndPermutation( + checkConsistencyForCol0IdAndExternalPermutation( predicateId, indexImpl.getPermutation(PSO), 0, 1); - checkConsistencyForCol0IdAndPermutation( + checkConsistencyForCol0IdAndExternalPermutation( predicateId, indexImpl.getPermutation(POS), 1, 0); }; auto checkConsistencyForObject = [&](Id objectId) { using enum Permutation::Enum; - checkConsistencyForCol0IdAndPermutation( + checkConsistencyForCol0IdAndExternalPermutation( objectId, indexImpl.getPermutation(OPS), 1, col0IdTag); - checkConsistencyForCol0IdAndPermutation( + checkConsistencyForCol0IdAndExternalPermutation( objectId, indexImpl.getPermutation(OSP), 0, col0IdTag); }; const auto& pso = index.getImpl().PSO(); From d4590a5f0de31beb92e62dcdc64d00ab2c2367c7 Mon Sep 17 00:00:00 2001 From: Julian Mundhahs Date: Wed, 14 Jan 2026 14:29:47 +0100 Subject: [PATCH 17/17] remove unused capture # Conflicts: # src/engine/CMakeLists.txt # src/engine/Server.cpp --- test/CompressedRelationsTest.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp index 0ea1a0443b..782d9f0016 100644 --- a/test/CompressedRelationsTest.cpp +++ b/test/CompressedRelationsTest.cpp @@ -1038,7 +1038,6 @@ TEST(CompressedRelationReader, getFirstAndLastTripleIgnoringGraph) { auto currentLocatedTriplesSharedState = index.deltaTriplesManager().getCurrentLocatedTriplesSharedState(); auto permutationEnum = Permutation::Enum::SPO; - const auto& permutation = index.getImpl().getPermutation(permutationEnum); const auto& locatedTriplesPerBlock = currentLocatedTriplesSharedState->getLocatedTriplesForPermutation( permutationEnum); @@ -1057,10 +1056,10 @@ TEST(CompressedRelationReader, getFirstAndLastTripleIgnoringGraph) { auto g2 = getId(""); auto g3 = getId(""); - auto getTriples = [&permutation, ¤tLocatedTriplesSharedState, - permutationEnum, &index, &locatedTriplesPerBlock]( - std::optional col0, std::optional col1, - std::optional graphId) { + auto getTriples = [¤tLocatedTriplesSharedState, permutationEnum, &index, + &locatedTriplesPerBlock](std::optional col0, + std::optional col1, + std::optional graphId) { ScanSpecification scanSpecification{ std::move(col0), std::move(col1),