Apply binary search filter expressions directly on the block metadata of Index Scans

src/engine/Filter.cpp

@@ -48,8 +48,8 @@ string Filter::getDescriptor() const {
 void Filter::setPrefilterExpressionForChildren() {
   std::vector<PrefilterVariablePair> prefilterPairs =
       _expression.getPrefilterExpressionForMetadata();
-  auto optNewSubTree = _subtree->setPrefilterGetUpdatedQueryExecutionTree(
-      std::move(prefilterPairs));
+  auto optNewSubTree =
+      _subtree->setPrefilterGetUpdatedQueryExecutionTree(prefilterPairs);
   if (optNewSubTree.has_value()) {
     _subtree = std::move(optNewSubTree.value());
   }

src/engine/Filter.h

@@ -61,10 +61,10 @@ class Filter : public Operation {
 
   // The method is directly invoked with the construction of this `Filter`
   // object. Its implementation retrieves <PrefilterExpression, Variable> pairs
-  // from the corresponding `SparqlExpression` and calls
-  // `QueryExecutionTree::setPrefilterGetUpdatedQueryExecutionTree()` on those
-  // pair values. If necessary the `QueryExecutionTree` for this entity will be
-  // updated.
+  // from the corresponding `SparqlExpression` and uses them to call
+  // `QueryExecutionTree::setPrefilterGetUpdatedQueryExecutionTree()` on the
+  // `subtree_`. If necessary the `QueryExecutionTree` for this
+  // entity will be updated.
   void setPrefilterExpressionForChildren();
 
   ProtoResult computeResult(bool requestLaziness) override;

src/engine/IndexScan.cpp

@@ -157,12 +157,12 @@
 // _____________________________________________________________________________
 std::optional<std::shared_ptr<QueryExecutionTree>>
 IndexScan::setPrefilterGetUpdatedQueryExecutionTree(
-    std::vector<PrefilterVariablePair> prefilterVariablePairs) {
+    const std::vector<PrefilterVariablePair>& prefilterVariablePairs) const {
   auto optSortedVarColIdxPair =
       getSortedVariableAndMetadataColumnIndexForPrefiltering();
   if (!optSortedVarColIdxPair.has_value()) {
     return std::nullopt;
   }
   const auto& [sortedVar, colIdx] = optSortedVarColIdxPair.value();
   auto it =
       std::ranges::find(prefilterVariablePairs, sortedVar, ad_utility::second);
@@ -315,8 +315,8 @@
 std::optional<std::pair<Variable, ColumnIndex>>
 IndexScan::getSortedVariableAndMetadataColumnIndexForPrefiltering() const {
   if (numVariables_ < 1) {
     return std::nullopt;
   }
   const auto& permutedTriple = getPermutedTriple();
   size_t colIdx = 3 - numVariables_;
   const auto& tripleComp = permutedTriple.at(colIdx);
@@ -342,15 +342,15 @@
   if (optBlockSpan.has_value()) {
     const auto& blockSpan = optBlockSpan.value();
     optBlocks = {blockSpan.begin(), blockSpan.end()};
-    applyPefilterIfPossible(optBlocks.value());
+    applyPrefilterIfPossible(optBlocks.value());
   }
   return optBlocks;
 }
 
 // _____________________________________________________________________________
-void IndexScan::applyPefilterIfPossible(
+void IndexScan::applyPrefilterIfPossible(
     std::vector<CompressedBlockMetadata>& blocks) const {
-  if (prefilter_.has_value() && getLimit().isUnconstrained()) {
+  if (prefilter_.has_value()) {
     // Apply the prefilter on given blocks.
     auto& [prefilterExpr, columnIndex] = prefilter_.value();
     blocks = prefilterExpr->evaluate(blocks, columnIndex);
@@ -360,14 +360,20 @@
 // _____________________________________________________________________________
 Permutation::IdTableGenerator IndexScan::getLazyScan(
     std::vector<CompressedBlockMetadata> blocks) const {
-  applyPefilterIfPossible(blocks);
   // If there is a LIMIT or OFFSET clause that constrains the scan
   // (which can happen with an explicit subquery), we cannot use the prefiltered
   // blocks, as we currently have no mechanism to include limits and offsets
   // into the prefiltering (`std::nullopt` means `scan all blocks`).
   auto filteredBlocks = getLimit().isUnconstrained()
                             ? std::optional(std::move(blocks))
                             : std::nullopt;
+  if (filteredBlocks.has_value()) {
+    // Note: The prefilter expression applied with applyPrefilterIfPossible()
+    // is not related to the prefilter procedure mentioned in the comment above.
+    // If this IndexScan owns a <PrefilterExpression, ColumnIdx> pair, it can
+    // be applied.
+    applyPrefilterIfPossible(filteredBlocks.value());
+  }
   return getScanPermutation().lazyScan(getScanSpecification(), filteredBlocks,
                                        additionalColumns(), cancellationHandle_,
                                        locatedTriplesSnapshot(), getLimit());

src/engine/IndexScan.h

@@ -76,7 +76,8 @@ class IndexScan final : public Operation {
   // if necessary.
   std::optional<std::shared_ptr<QueryExecutionTree>>
   setPrefilterGetUpdatedQueryExecutionTree(
-      std::vector<PrefilterVariablePair> prefilterVariablePairs) override;
+      const std::vector<PrefilterVariablePair>& prefilterVariablePairs)
+      const override;
 
   size_t numVariables() const { return numVariables_; }
 
@@ -195,7 +196,7 @@ class IndexScan final : public Operation {
   // If `isUnconstrained()` yields true, return the blocks as given or the
   // prefiltered blocks (if `prefilter_` has value). If `isUnconstrained()` is
   // false, return `std::nullopt`.
-  void applyPefilterIfPossible(
+  void applyPrefilterIfPossible(
       std::vector<CompressedBlockMetadata>& blocks) const;
 
   // Helper functions for the public `getLazyScanFor...` methods and

src/engine/Operation.h

@@ -93,7 +93,8 @@ class Operation {
   // `IndexScan` in their subtree.
   virtual std::optional<std::shared_ptr<QueryExecutionTree>>
   setPrefilterGetUpdatedQueryExecutionTree(
-      [[maybe_unused]] std::vector<PrefilterVariablePair> prefilterPairs) {
+      [[maybe_unused]] const std::vector<PrefilterVariablePair>& prefilterPairs)
+      const {
     return std::nullopt;
   };
 

src/engine/QueryExecutionTree.cpp

@@ -107,7 +107,7 @@ size_t QueryExecutionTree::getSizeEstimate() {
 //_____________________________________________________________________________
 std::optional<std::shared_ptr<QueryExecutionTree>>
 QueryExecutionTree::setPrefilterGetUpdatedQueryExecutionTree(
-    std::vector<Operation::PrefilterVariablePair> prefilterPairs) const {
+    std::vector<Operation::PrefilterVariablePair>& prefilterPairs) const {
   AD_CONTRACT_CHECK(rootOperation_);
   VariableToColumnMap varToColMap = getVariableColumns();
   std::erase_if(prefilterPairs, [&varToColMap](const auto& pair) {
@@ -118,7 +118,7 @@ QueryExecutionTree::setPrefilterGetUpdatedQueryExecutionTree(
     return std::nullopt;
   } else {
     return rootOperation_->setPrefilterGetUpdatedQueryExecutionTree(
-        std::move(prefilterPairs));
+        prefilterPairs);
   }
 }
 

src/engine/QueryExecutionTree.h

@@ -100,7 +100,7 @@ class QueryExecutionTree {
   // the corresponding `Variable` is visible in the `VariableToColumnMap`.
   std::optional<std::shared_ptr<QueryExecutionTree>>
   setPrefilterGetUpdatedQueryExecutionTree(
-      std::vector<Operation::PrefilterVariablePair> prefilterPairs) const;
+      std::vector<Operation::PrefilterVariablePair>& prefilterPairs) const;
 
   size_t getDistinctEstimate(size_t col) const {
     return static_cast<size_t>(rootOperation_->getSizeEstimate() /

src/index/Permutation.cpp

@@ -67,9 +67,12 @@ IdTable Permutation::scan(
   const auto& p = getActualPermutation(scanSpec);
 
   return p.reader().scan(
-      scanSpec, blocks.has_value() ? blocks.value() : p.meta_.blockData(),
+      scanSpec,
+      blocks.has_value()
+          ? blocks.value()
+          : p.getAugmentedMetadataForPermutation(locatedTriplesSnapshot),
       additionalColumns, cancellationHandle,
-      getLocatedTriplesForPermutation(locatedTriplesSnapshot), limitOffset);
+      p.getLocatedTriplesForPermutation(locatedTriplesSnapshot), limitOffset);
 }
 
 // _____________________________________________________________________
@@ -79,8 +82,11 @@ size_t Permutation::getResultSizeOfScan(
     std::optional<std::vector<CompressedBlockMetadata>> blocks) const {
   const auto& p = getActualPermutation(scanSpec);
   return p.reader().getResultSizeOfScan(
-      scanSpec, blocks.has_value() ? blocks.value() : p.meta_.blockData(),
-      getLocatedTriplesForPermutation(locatedTriplesSnapshot));
+      scanSpec,
+      blocks.has_value()
+          ? blocks.value()
+          : p.getAugmentedMetadataForPermutation(locatedTriplesSnapshot),
+      p.getLocatedTriplesForPermutation(locatedTriplesSnapshot));
 }
 
 // _____________________________________________________________________
@@ -90,8 +96,11 @@ std::pair<size_t, size_t> Permutation::getSizeEstimateForScan(
     std::optional<std::vector<CompressedBlockMetadata>> blocks) const {
   const auto& p = getActualPermutation(scanSpec);
   return p.reader().getSizeEstimateForScan(
-      scanSpec, blocks.has_value() ? blocks.value() : p.meta_.blockData(),
-      getLocatedTriplesForPermutation(locatedTriplesSnapshot));
+      scanSpec,
+      blocks.has_value()
+          ? blocks.value()
+          : p.getAugmentedMetadataForPermutation(locatedTriplesSnapshot),
+      p.getLocatedTriplesForPermutation(locatedTriplesSnapshot));
 }
 
 // ____________________________________________________________________________

test/engine/IndexScanTest.cpp

@@ -182,10 +182,12 @@ void testLazyScanWithColumnThrows(
 const auto testSetAndMakeScanWithPrefilterExpr =
     [](const std::string& kg, const SparqlTriple& triple,
        const Permutation::Enum permutation, IndexScan::PrefilterVariablePair pr,
-       const IdTable& idTableWithExpectedColumn, bool prefilterCanBeSet = true,
+       const std::vector<ValueId>& expectedIdsOnFilterColumn,
+       bool prefilterCanBeSet = true,
        source_location l = source_location::current()) {
       auto t = generateLocationTrace(l);
       IndexScan scan{getQec(kg), permutation, triple};
+      auto variable = pr.second;
       auto optUpdatedQet = scan.setPrefilterGetUpdatedQueryExecutionTree(
           makeFilterExpression::filterHelper::makePrefilterVec(std::move(pr)));
       if (optUpdatedQet.has_value()) {
@@ -194,15 +196,15 @@ const auto testSetAndMakeScanWithPrefilterExpr =
         // Check that the prefiltering procedure yields the correct result given
         // that the <PrefilterExpression, Variable> pair is correctly assigned
         // to the IndexScan.
-        const IdTable& idTableFiltered = updatedQet->getRootOperation()
-                                             ->computeResultOnlyForTesting()
-                                             .idTable();
-        auto isColumnIdSpan = idTableFiltered.getColumn(0);
-        auto expectedColumnIdSpan = idTableWithExpectedColumn.getColumn(0);
+        IdTable idTableFiltered = updatedQet->getRootOperation()
+                                      ->computeResultOnlyForTesting()
+                                      .idTable()
+                                      .clone();
+        auto isColumnIdSpan =
+            idTableFiltered.getColumn(updatedQet->getVariableColumn(variable));
         ASSERT_EQ(
             (std::vector<Id>{isColumnIdSpan.begin(), isColumnIdSpan.end()}),
-            (std::vector<Id>{expectedColumnIdSpan.begin(),
-                             expectedColumnIdSpan.end()}));
+            expectedIdsOnFilterColumn);
       } else {
         // Check our prediction that the prefilter with the given
         // <PrefilterExpression, Variable> pair is not applicable (no updated
@@ -695,7 +697,7 @@ TEST(IndexScan, SetPrefilterVariablePairAndCheckCacheKey) {
 TEST(IndexScan, checkEvaluationWithPrefiltering) {
   using namespace makeFilterExpression;
   using namespace filterHelper;
-  using I = TripleComponent::Iri;
+  auto I = ad_utility::testing::IntId;
   std::string kg =
       "<P1> <price_tag> 10 . <P2> <price_tag> 12 . <P3> <price_tag> "
       "18 . <P4> <price_tag> 22 . <P5> <price_tag> 25 . <P6> "
@@ -706,57 +708,54 @@ TEST(IndexScan, checkEvaluationWithPrefiltering) {
 
   // For the following tests, the <PrefilterExpression, Variable> pair is set
   // and applied for the respective IndexScan.
-  testSetAndMakeScanWithPrefilterExpr(
-      kg, triple, Permutation::POS, pr(ge(IntId(10)), Variable{"?price"}),
-      makeIdTableFromVector(
-          {{10}, {12}, {18}, {22}, {25}, {147}, {174}, {174}, {189}, {194}},
-          IntId));
+  testSetAndMakeScanWithPrefilterExpr(kg, triple, Permutation::POS,
+                                      pr(ge(IntId(10)), Variable{"?price"}),
+                                      {I(10), I(12), I(18), I(22), I(25),
+                                       I(147), I(174), I(174), I(189), I(194)});
   testSetAndMakeScanWithPrefilterExpr(
       kg, triple, Permutation::POS,
       pr(lt(DoubleId(147.32)), Variable{"?price"}),
-      makeIdTableFromVector({{10}, {12}, {18}, {22}, {25}, {147}}, IntId));
+      {I(10), I(12), I(18), I(22), I(25), I(147)});
   testSetAndMakeScanWithPrefilterExpr(
       kg, triple, Permutation::POS,
       pr(andExpr(gt(DoubleId(12.00)), le(IntId(174))), Variable{"?price"}),
-      makeIdTableFromVector({{18}, {22}, {25}, {147}, {174}, {174}}, IntId));
+      {I(18), I(22), I(25), I(147), I(174), I(174)});
 
   // For the following test, the Variable value doesn't match any of the scan
   // triple Variable values. We expect that the prefilter is not applicable (=>
   // set bool flag to false).
   testSetAndMakeScanWithPrefilterExpr(
       kg, triple, Permutation::POS,
-      pr(andExpr(gt(DoubleId(12.00)), le(IntId(174))), Variable{"?y"}),
-      makeIdTableFromVector({{0}}, IntId), false);
+      pr(andExpr(gt(DoubleId(12.00)), le(IntId(174))), Variable{"?y"}), {},
+      false);
 
   // For the following tests, the first sorted column given the permutation
   // doesn't match with the corresponding column for the Variable of the
   // <PrefilterExpression, Variable> pair. We expect that the provided prefilter
   // is not applicable (and can't be set).
   testSetAndMakeScanWithPrefilterExpr(
       kg, triple, Permutation::PSO,
-      pr(andExpr(gt(DoubleId(12.00)), le(IntId(174))), Variable{"?price"}),
-      makeIdTableFromVector({{0}}, IntId), false);
+      pr(andExpr(gt(DoubleId(12.00)), le(IntId(174))), Variable{"?price"}), {},
+      false);
   testSetAndMakeScanWithPrefilterExpr(
       kg, triple, Permutation::POS,
-      pr(andExpr(gt(VocabId(0)), lt(VocabId(100))), Variable{"?x"}),
-      makeIdTableFromVector({{0}}, IntId), false);
+      pr(andExpr(gt(VocabId(0)), lt(VocabId(100))), Variable{"?x"}), {}, false);
 
   // This knowledge graph yields an incomplete first and last block.
   std::string kgFirstAndLastIncomplete =
       "<a> <price_tag> 10 . <b> <price_tag> 12 . <b> <price_tag> "
       "18 . <b> <price_tag> 22 . <b> <price_tag> 25 . <b> "
       "<price_tag> 147 . <b> <price_tag> 189 . <c> <price_tag> 194 "
       ".";
-  triple = {I::fromIriref("<b>"), "<price_tag>", Tc{Variable{"?price"}}};
   // The following test verifies that the prefilter procedure is successfully
   // applicable under the condition that the first and last block are
   // potentially incomplete.
   testSetAndMakeScanWithPrefilterExpr(
-      kgFirstAndLastIncomplete, triple, Permutation::PSO,
+      kgFirstAndLastIncomplete, triple, Permutation::POS,
       pr(orExpr(gt(IntId(100)), le(IntId(10))), Variable{"?price"}),
-      makeIdTableFromVector({{12}, {25}, {147}, {189}}, IntId));
+      {I(10), I(12), I(25), I(147), I(189), I(194)});
   testSetAndMakeScanWithPrefilterExpr(
-      kgFirstAndLastIncomplete, triple, Permutation::SPO,
+      kgFirstAndLastIncomplete, triple, Permutation::POS,
       pr(andExpr(gt(IntId(10)), lt(IntId(194))), Variable{"?price"}),
-      makeIdTableFromVector({{12}, {18}, {22}, {25}, {147}, {189}}, IntId));
+      {I(10), I(12), I(18), I(22), I(25), I(147), I(189), I(194)});
 }