Apply binary search filter expressions directly on the block metadata of Index Scans

src/engine/Filter.cpp

@@ -27,7 +27,9 @@
                sparqlExpression::SparqlExpressionPimpl expression)
     : Operation(qec),
       _subtree(std::move(subtree)),
-      _expression{std::move(expression)} {}
+      _expression{std::move(expression)} {
+  setPrefilterExpressionForDirectIndexScanChild();
+}
 
 // _____________________________________________________________________________
 string Filter::getCacheKeyImpl() const {
@@ -37,10 +39,22 @@
   return std::move(os).str();
 }
 
+//______________________________________________________________________________
 string Filter::getDescriptor() const {
   return absl::StrCat("Filter ", _expression.getDescriptor());
 }
 
+//______________________________________________________________________________
+void Filter::setPrefilterExpressionForDirectIndexScanChild() {
+  std::vector<PrefilterVariablePair> prefilterPairs =
+      _expression.getPrefilterExpressionForMetadata();
+  auto optNewSubTree =
+      _subtree->setPrefilterExprGetUpdatedQetPtr(std::move(prefilterPairs));
+  if (optNewSubTree.has_value()) {
+    _subtree = std::move(optNewSubTree.value());
+  }
+}
+
 // _____________________________________________________________________________
 ProtoResult Filter::computeResult(bool requestLaziness) {
   LOG(DEBUG) << "Getting sub-result for Filter result computation..." << endl;

src/engine/Filter.h

@@ -10,10 +10,11 @@
 
 #include "engine/Operation.h"
 #include "engine/QueryExecutionTree.h"
-#include "engine/sparqlExpressions/SparqlExpressionPimpl.h"
 #include "parser/ParsedQuery.h"
 
 class Filter : public Operation {
+  using PrefilterVariablePair = sparqlExpression::PrefilterExprVariablePair;
+
  private:
   std::shared_ptr<QueryExecutionTree> _subtree;
   sparqlExpression::SparqlExpressionPimpl _expression;
@@ -58,6 +59,12 @@ class Filter : public Operation {
     return _subtree->getVariableColumns();
   }
 
+  // This method is directly called by the constructor.
+  // It sets the appropriate `<PrefilterExpression, Variable>` pair for each
+  // `IndexScan` child by invoking `setPrefilterExpression` on all descendants
+  // in the `QueryExecutionTree`.
+  void setPrefilterExpressionForDirectIndexScanChild();
+
   ProtoResult computeResult(bool requestLaziness) override;
 
   // Perform the actual filter operation of the data provided.

src/engine/IndexScan.cpp

@@ -17,13 +17,15 @@
 
 // _____________________________________________________________________________
 IndexScan::IndexScan(QueryExecutionContext* qec, Permutation::Enum permutation,
-                     const SparqlTripleSimple& triple, Graphs graphsToFilter)
+                     const SparqlTripleSimple& triple, Graphs graphsToFilter,
+                     PrefilterIndexPair prefilter)
     : Operation(qec),
       permutation_(permutation),
       subject_(triple.s_),
       predicate_(triple.p_),
       object_(triple.o_),
       graphsToFilter_{std::move(graphsToFilter)},
+      prefilter_{std::move(prefilter)},
       numVariables_(static_cast<size_t>(subject_.isVariable()) +
                     static_cast<size_t>(predicate_.isVariable()) +
                     static_cast<size_t>(object_.isVariable())) {
@@ -51,9 +53,27 @@
 
 // _____________________________________________________________________________
 IndexScan::IndexScan(QueryExecutionContext* qec, Permutation::Enum permutation,
-                     const SparqlTriple& triple, Graphs graphsToFilter)
-    : IndexScan(qec, permutation, triple.getSimple(),
-                std::move(graphsToFilter)) {}
+                     const SparqlTriple& triple, Graphs graphsToFilter,
+                     PrefilterIndexPair prefilter)
+    : IndexScan(qec, permutation, triple.getSimple(), std::move(graphsToFilter),
+                std::move(prefilter)) {}
+
+// _____________________________________________________________________________
+IndexScan::IndexScan(QueryExecutionContext* qec, Permutation::Enum permutation,
+                     const TripleComponent& s, const TripleComponent& p,
+                     const TripleComponent& o,
+                     std::vector<ColumnIndex> additionalColumns,
+                     std::vector<Variable> additionalVariables,
+                     Graphs graphsToFilter, PrefilterIndexPair prefilter)
+    : Operation(qec),
+      permutation_(permutation),
+      subject_(s),
+      predicate_(p),
+      object_(o),
+      graphsToFilter_(std::move(graphsToFilter)),
+      prefilter_(std::move(prefilter)),
+      additionalColumns_(std::move(additionalColumns)),
+      additionalVariables_(std::move(additionalVariables)) {}
 
 // _____________________________________________________________________________
 string IndexScan::getCacheKeyImpl() const {
@@ -89,6 +109,12 @@
     os << "\nFiltered by Graphs:";
     os << absl::StrJoin(graphIdVec, " ");
   }
+  if (prefilter_.has_value()) {
+    auto& [prefilterExpr, columnIdx] = prefilter_.value();
+    os << "Added PrefiterExpression: \n";
+    os << *prefilterExpr;
+    os << "\n Applied on: " << columnIdx << ".";
+  }
   return std::move(os).str();
 }
 
@@ -115,6 +141,24 @@
   return result;
 }
 
+// _____________________________________________________________________________
+std::optional<std::shared_ptr<QueryExecutionTree>>
+IndexScan::setPrefilterExprGetUpdatedQetPtr(
+    std::vector<PrefilterVariablePair> prefilterVariablePairs) {
+  if (numVariables_ < 1) {
+    return std::nullopt;
+  }
+
+  auto map = getVariableToSortedIndexMap();
+  for (auto& [prefilterExpr, variable] : prefilterVariablePairs) {
+    if (map.find(variable) != map.end()) {
+      return makeCopyWithAddedPrefilters(
+          std::make_pair(prefilterExpr->clone(), map[variable]));
+    }
+  }
+  return std::nullopt;
+}
+
 // _____________________________________________________________________________
 VariableToColumnMap IndexScan::computeVariableToColumnMap() const {
   VariableToColumnMap variableToColumnMap;
@@ -134,40 +178,60 @@
   return variableToColumnMap;
 }
 
+//______________________________________________________________________________
+std::shared_ptr<QueryExecutionTree> IndexScan::makeCopyWithAddedPrefilters(
+    PrefilterIndexPair prefilter) const {
+  return ad_utility::makeExecutionTree<IndexScan>(
+      getExecutionContext(), permutation_, subject_, predicate_, object_,
+      additionalColumns_, additionalVariables_, graphsToFilter_,
+      std::move(prefilter));
+}
+
 // _____________________________________________________________________________
-Result::Generator IndexScan::scanInChunks() const {
-  auto metadata = getMetadataForScan();
-  if (!metadata.has_value()) {
+Result::Generator IndexScan::chunkedIndexScan() const {
+  auto optBlockSpan = getOptionalBlockSpan();
+  if (!optBlockSpan.has_value()) {
     co_return;
   }
-  auto blocksSpan =
-      CompressedRelationReader::getBlocksFromMetadata(metadata.value());
-  std::vector<CompressedBlockMetadata> blocks{blocksSpan.begin(),
-                                              blocksSpan.end()};
-  for (IdTable& idTable : getLazyScan(std::move(blocks))) {
+  const auto& blockSpan = optBlockSpan.value();
+  for (IdTable& idTable : getLazyScan({blockSpan.begin(), blockSpan.end()})) {
     co_yield {std::move(idTable), LocalVocab{}};
   }
 }
 
 // _____________________________________________________________________________
-ProtoResult IndexScan::computeResult(bool requestLaziness) {
-  LOG(DEBUG) << "IndexScan result computation...\n";
-  if (requestLaziness) {
-    return {scanInChunks(), resultSortedOn()};
+IdTable IndexScan::materializedIndexScan() const {
+  // Get the blocks.
+  auto optBlockSpan = getOptionalBlockSpan();
+  std::optional<std::vector<CompressedBlockMetadata>> optFilteredBlocks = {};
+  if (optBlockSpan.has_value()) {
+    const auto& blockSpan = optBlockSpan.value();
+    optFilteredBlocks =
+        getOptionalPrefilteredBlocks({blockSpan.begin(), blockSpan.end()});
   }
-  IdTable idTable{getExecutionContext()->getAllocator()};
-
+  // Create the IdTable and fill it with content by performing scan().
   using enum Permutation::Enum;
+  IdTable idTable{getExecutionContext()->getAllocator()};
   idTable.setNumColumns(numVariables_);
   const auto& index = _executionContext->getIndex();
-  idTable =
-      index.scan(getScanSpecification(), permutation_, additionalColumns(),
-                 cancellationHandle_, locatedTriplesSnapshot(), getLimit());
+  idTable = index.getImpl()
+                .getPermutation(permutation_)
+                .scan(getScanSpecification(), additionalColumns(),
+                      cancellationHandle_, locatedTriplesSnapshot(), getLimit(),
+                      optFilteredBlocks);
   AD_CORRECTNESS_CHECK(idTable.numColumns() == getResultWidth());
   LOG(DEBUG) << "IndexScan result computation done.\n";
   checkCancellation();
+  return idTable;
+}
 
-  return {std::move(idTable), resultSortedOn(), LocalVocab{}};
+// _____________________________________________________________________________
+ProtoResult IndexScan::computeResult(bool requestLaziness) {
+  LOG(DEBUG) << "IndexScan result computation...\n";
+  if (requestLaziness) {
+    return {chunkedIndexScan(), resultSortedOn()};
+  }
+  return {materializedIndexScan(), getResultSortedOn(), LocalVocab{}};
 }
 
 // _____________________________________________________________________________
@@ -219,7 +283,7 @@
   AD_CONTRACT_CHECK(multiplicity_.size() == getResultWidth());
 }
 
-// ___________________________________________________________________________
+// _____________________________________________________________________________
 std::array<const TripleComponent* const, 3> IndexScan::getPermutedTriple()
     const {
   std::array triple{&subject_, &predicate_, &object_};
@@ -228,30 +292,69 @@
           triple[permutation[2]]};
 }
 
-// ___________________________________________________________________________
+// _____________________________________________________________________________
 ScanSpecification IndexScan::getScanSpecification() const {
   const IndexImpl& index = getIndex().getImpl();
   return getScanSpecificationTc().toScanSpecification(index);
 }
 
-// ___________________________________________________________________________
+// _____________________________________________________________________________
 ScanSpecificationAsTripleComponent IndexScan::getScanSpecificationTc() const {
   auto permutedTriple = getPermutedTriple();
   return {*permutedTriple[0], *permutedTriple[1], *permutedTriple[2],
           graphsToFilter_};
 }
 
-// ___________________________________________________________________________
+// _____________________________________________________________________________
+ad_utility::HashMap<Variable, ColumnIndex>
+IndexScan::getVariableToSortedIndexMap() const {
+  ad_utility::HashMap<Variable, ColumnIndex> map;
+  ColumnIndex idx = 0;
+  for (const TripleComponent* const ptr : getPermutedTriple()) {
+    if (!ptr->isVariable()) {
+      break;
+    }
+    map[ptr->getVariable()] = idx;
+    idx++;
+  }
+  return map;
+}
+
+// _____________________________________________________________________________
+std::optional<std::span<const CompressedBlockMetadata>>
+IndexScan::getOptionalBlockSpan() const {
+  auto metadata = getMetadataForScan();
+  if (metadata.has_value()) {
+    return CompressedRelationReader::getBlocksFromMetadata(metadata.value());
+  }
+  return std::nullopt;
+}
+
+// _____________________________________________________________________________
+std::optional<std::vector<CompressedBlockMetadata>>
+IndexScan::getOptionalPrefilteredBlocks(
+    std::vector<CompressedBlockMetadata> blocks) const {
+  if (!getLimit().isUnconstrained()) {
+    return std::nullopt;
+  }
+
+  if (prefilter_.has_value()) {
+    // Apply the prefilter on given blocks.
+    auto& [prefilterExpr, columnIndex] = prefilter_.value();
+    return prefilterExpr->evaluate(blocks, columnIndex);
+  } else {
+    return blocks;
+  }
+}
+
+// _____________________________________________________________________________
 Permutation::IdTableGenerator IndexScan::getLazyScan(
     std::vector<CompressedBlockMetadata> blocks) const {
   // If there is a LIMIT or OFFSET clause that constrains the scan
   // (which can happen with an explicit subquery), we cannot use the prefiltered
   // blocks, as we currently have no mechanism to include limits and offsets
   // into the prefiltering (`std::nullopt` means `scan all blocks`).
-  auto actualBlocks = getLimit().isUnconstrained()
-                          ? std::optional{std::move(blocks)}
-                          : std::nullopt;
-
+  auto actualBlocks = getOptionalPrefilteredBlocks(std::move(blocks));
   return getIndex()
       .getImpl()
       .getPermutation(permutation())
@@ -260,15 +363,15 @@
                 locatedTriplesSnapshot(), getLimit());
 };
 
-// ________________________________________________________________
+// _____________________________________________________________________________
 std::optional<Permutation::MetadataAndBlocks> IndexScan::getMetadataForScan()
     const {
   const auto& index = getExecutionContext()->getIndex().getImpl();
   return index.getPermutation(permutation())
       .getMetadataAndBlocks(getScanSpecification(), locatedTriplesSnapshot());
 };
 
-// ________________________________________________________________
+// _____________________________________________________________________________
 std::array<Permutation::IdTableGenerator, 2>
 IndexScan::lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2) {
   AD_CONTRACT_CHECK(s1.numVariables_ <= 3 && s2.numVariables_ <= 3);
@@ -317,7 +420,7 @@
   return result;
 }
 
-// ________________________________________________________________
+// _____________________________________________________________________________
 Permutation::IdTableGenerator IndexScan::lazyScanForJoinOfColumnWithScan(
     std::span<const Id> joinColumn) const {
   AD_EXPENSIVE_CHECK(std::ranges::is_sorted(joinColumn));