Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions src/basic_memory/repository/sqlite_search_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -391,15 +391,13 @@ def compile_fts_filter(
for idx, filt in enumerate(parsed_filters):
path_param = f"meta_path_{idx}"
extract_expr = None
use_tags_column = False

if filt.path_parts == ["status"] and "frontmatter_status" in entity_columns:
extract_expr = "entity.frontmatter_status"
elif filt.path_parts == ["type"] and "frontmatter_type" in entity_columns:
extract_expr = "entity.frontmatter_type"
elif filt.path_parts == ["tags"] and "tags_json" in entity_columns:
extract_expr = "entity.tags_json"
use_tags_column = True

if extract_expr is None:
params[path_param] = build_sqlite_json_path(filt.path_parts)
Expand Down Expand Up @@ -433,6 +431,12 @@ def compile_fts_filter(
continue

if filt.op == "contains":
# Membership walks the original metadata document, not tags_json.
# The generated column holds json_extract's result, which is plain
# text for a scalar such as `tags: ""`, and json_each() raises
# "malformed JSON" on it. json_each(doc, path) treats a scalar as
# one value and simply finds no match.
params[path_param] = build_sqlite_json_path(filt.path_parts)
tag_conditions: list[str] = []
for j, val in enumerate(filt.value):
value_param = f"meta_val_{idx}_{j}"
Expand All @@ -445,14 +449,10 @@ def compile_fts_filter(
param_prefix=value_param,
params=params,
)
json_each_expr = (
"json_each(entity.tags_json)"
if use_tags_column
else f"json_each(entity.entity_metadata, :{path_param})"
)
tag_conditions.append(
"("
f"EXISTS (SELECT 1 FROM {json_each_expr} WHERE value = :{value_param}) "
"EXISTS (SELECT 1 FROM json_each(entity.entity_metadata, "
f":{path_param}) WHERE value = :{value_param}) "
f"OR {like_condition}"
")"
)
Expand Down Expand Up @@ -548,7 +548,8 @@ def __init__(self, session_maker: async_sessionmaker[AsyncSession]) -> None:
async def _entity_column_names(self) -> frozenset[str]:
if self._entity_columns is None:
async with db.scoped_session(self._session_maker) as session:
result = await session.execute(text("PRAGMA table_info(entity)"))
# table_info omits generated columns; table_xinfo lists them.
result = await session.execute(text("PRAGMA table_xinfo(entity)"))
Comment thread
phernandez marked this conversation as resolved.
Resolved
self._entity_columns = frozenset(row[1] for row in result.fetchall())
return self._entity_columns

Expand Down
5 changes: 4 additions & 1 deletion src/basic_memory/repository/sqlite_vec_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -497,7 +497,10 @@ def _candidates(
"c.source_row_id, vector_matches.distance "
"FROM vector_matches "
"JOIN search_vector_chunks c ON c.id = vector_matches.rowid "
"AND c.source_hash = vector_matches.source_hash "
# SQLite before 3.38 can push this join term into the vec0 scan despite
# MATERIALIZED, and vec0 rejects any constraint on an auxiliary column
# in a KNN query (#2065). The unary plus keeps the term out of the scan.
"AND c.source_hash = +vector_matches.source_hash "
"WHERE c.vector_index = 'sqlite-vec' "
"AND c.embedding_status = 'ready' "
"AND c.embedding_model = :embedding_identity)"
Expand Down
65 changes: 65 additions & 0 deletions tests/repository/test_metadata_filters_edge_cases.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,16 @@
"""

from datetime import datetime, timezone
from typing import Any

import pytest
from sqlalchemy import text

from basic_memory import db
from basic_memory.file_types import file_entity_metadata
from basic_memory.models import Entity
from basic_memory.repository.search_index_row import SearchIndexRow
from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository
from basic_memory.schemas.search import SearchItemType


Expand Down Expand Up @@ -532,3 +535,65 @@ async def test_contains_operator_matches_like_the_bare_list(search_repository, s

results = await search_repository.search(metadata_filters={"tags": {"contains": "security"}})
assert [r.title for r in results] == ["Tagged"]


# --- Generated frontmatter columns on a migrated SQLite database (#2065) ---

# The DDL migration d7e8f9a0b1c2 runs on SQLite. Test databases come from
# create_all, which has no generated columns, so this test adds them the same way.
GENERATED_FRONTMATTER_COLUMNS = (
"ALTER TABLE entity ADD COLUMN tags_json TEXT "
"GENERATED ALWAYS AS (json_extract(entity_metadata, '$.tags')) VIRTUAL",
"ALTER TABLE entity ADD COLUMN frontmatter_status TEXT "
"GENERATED ALWAYS AS (json_extract(entity_metadata, '$.status')) VIRTUAL",
"ALTER TABLE entity ADD COLUMN frontmatter_type TEXT "
"GENERATED ALWAYS AS (json_extract(entity_metadata, '$.type')) VIRTUAL",
)


@pytest.mark.asyncio
async def test_sqlite_filters_use_the_generated_frontmatter_columns(
search_repository, session_maker
):
"""A migrated SQLite database answers status, type and tags from its generated columns.

PRAGMA table_info leaves generated columns out, so the columns were never found
and every filter fell back to json_extract. table_xinfo lists them.
"""
if not isinstance(search_repository, SQLiteSearchRepository):
pytest.skip("generated frontmatter columns exist only on SQLite")
async with db.scoped_session(session_maker) as session:
for statement in GENERATED_FRONTMATTER_COLUMNS:
await session.execute(text(statement))

tagged = await _index_entity_with_metadata(
search_repository,
session_maker,
"Generated Columns Tagged",
{"status": "active", "type": "spec", "tags": ["security", "auth"]},
)
untagged = await _index_entity_with_metadata(
search_repository,
session_maker,
"Generated Columns Untagged",
{"status": "draft", "type": "note"},
)
# tags_json holds plain text for a scalar tag value, which is not JSON.
await _index_entity_with_metadata(
search_repository,
session_maker,
"Generated Columns Empty Tag",
{"status": "draft", "type": "note", "tags": ""},
)

assert {"tags_json", "frontmatter_status", "frontmatter_type"} <= (
await search_repository._fts._entity_column_names()
)

async def matching(filters: dict[str, Any]) -> set[int]:
return {r.id for r in await search_repository.search(metadata_filters=filters)}

assert await matching({"status": "active"}) == {tagged.id}
assert await matching({"type": "note", "tags": None}) == {untagged.id}
assert await matching({"tags": ["security"]}) == {tagged.id}
assert await matching({"tags": None}) == {untagged.id}
19 changes: 19 additions & 0 deletions tests/repository/test_sqlite_vec_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import pytest
from sqlalchemy.exc import OperationalError as SAOperationalError

from basic_memory.repository.search_scope import ProjectScope
from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError
from basic_memory.repository.semantic_vector_index import VectorIndexScope
from basic_memory.repository.sqlite_vec_index import SQLiteVecIndex
Expand Down Expand Up @@ -39,3 +40,21 @@ async def test_missing_extension_support_is_a_dependency_error() -> None:

with pytest.raises(SemanticDependenciesMissingError, match="extension loading"):
await index._ensure_loaded(session)


def test_knn_join_keeps_the_auxiliary_column_out_of_the_vec0_scan() -> None:
"""SQLite before 3.38 pushes a plain join term on source_hash into the vec0 KNN.

vec0 then fails every vector search with "An illegal WHERE constraint was
provided on a vec0 auxiliary column in a KNN query" (#2065). The test suite's
SQLite is newer and never pushes the term down, so this pins the unary plus
that keeps it out of the scan.
"""
index = SQLiteVecIndex(
MagicMock(),
VectorIndexScope(namespace="test", embedding_identity="test", dimensions=4),
)

sql = index._candidates([0.0, 0.0, 0.0, 1.0], {}, ProjectScope.single(1), None, vector_k=3)

assert "c.source_hash = +vector_matches.source_hash" in sql
Loading