Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions src/basic_memory/index/note_content_materialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -685,6 +685,30 @@ async def settle_local_project_change(
)


async def settle_interrupted_note_deletes(
*,
session_maker: async_sessionmaker[AsyncSession],
project_id: int,
) -> int:
"""Settle `deleted` journal rows that a lost local process never settled.

A local delete commits its journal row, runs the file cleanup inline, then
settles the row. A crash or cancellation between the commit and the settle
leaves the row unsettled, and the wiki projector defers on it forever (#1625).
recover_stuck_materializations cannot reach it, because note_content cascaded
away with the entity. Meant to run once per project at local startup: the local
runtime has no durable cleanup queue, so no other job will settle these rows.
The entity delete is final whatever happened to the file; a file left on disk
is re-indexed as a new note.
"""
async with db.scoped_session(session_maker) as session:
return await ProjectRepository().settle_unmaterialized_deletes(
session,
project_id,
materialized_at=datetime.now(tz=UTC),
)


@dataclass(frozen=True, slots=True)
class LocalNoteContentMaterializationProvider:
"""Run accepted-note materialization inline for the local runtime."""
Expand Down
221 changes: 209 additions & 12 deletions src/basic_memory/indexing/directory_delete_runner.py

Large diffs are not rendered by default.

173 changes: 147 additions & 26 deletions src/basic_memory/repository/project_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,46 @@


from loguru import logger
from sqlalchemy import Executable, inspect as sa_inspect, select, text, update
from sqlalchemy import Executable, exists, insert, inspect as sa_inspect, select, text, update
from sqlalchemy.exc import NoResultFound, OperationalError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import aliased

from basic_memory.models.project import AcceptedProjectNoteChange, Project
from basic_memory.repository.repository import Repository
from basic_memory.runtime.project_partition import RuntimeAcceptedProjectNoteChange
from basic_memory.runtime.project_partition import (
RuntimeAcceptedProjectNoteChange,
RuntimeProjectNoteOperation,
)

# Bound each IN list well under SQLite's and Postgres's bind-parameter limits, so a
# folder of thousands of notes settles in a few statements without hitting either.
_SETTLE_POSITION_CHUNK_SIZE = 500


def _accepted_note_change_values(change: RuntimeAcceptedProjectNoteChange) -> dict[str, Any]:
"""Map runtime journal evidence onto accepted_project_note_change columns."""
return {
"project_id": change.project_id,
"project_external_id": change.project_external_id,
"partition_position": change.partition_position,
"entity_id": change.entity_id,
"note_external_id": change.note_external_id,
"permalink": change.permalink,
"title": change.title,
"operation": change.operation.value,
"file_path": change.file_path,
"previous_file_path": change.previous_file_path,
"accepted_at": change.accepted_at,
"source": change.source,
"db_version": change.db_version,
"db_checksum": change.db_checksum,
"actor_user_profile_id": (
str(change.actor_user_profile_id) if change.actor_user_profile_id is not None else None
),
"actor_kind": change.actor_kind,
"actor_name": change.actor_name,
}


async def _load_sqlite_vec_on_session(session) -> bool:
Expand Down Expand Up @@ -173,38 +206,53 @@ async def advance_partition_position(
raise RuntimeError(f"Project partition is missing for project_id={project_id}")
return int(position)

async def advance_partition_position_by(
self,
session: AsyncSession,
project_id: int,
count: int,
) -> int:
"""Atomically claim ``count`` consecutive positions and return the last one.

The claimed positions are ``head - count + 1`` through ``head``. One UPDATE
holds the project row lock once, however many changes a bulk mutation records.
"""
if count < 1:
raise ValueError(f"Partition position claim count must be positive, got {count}")
statement = (
update(Project)
.where(Project.id == project_id)
.values(partition_position=Project.partition_position + count)
.returning(Project.partition_position)
.execution_options(synchronize_session=False)
)
position = (await session.execute(statement)).scalar_one_or_none()
if position is None:
raise RuntimeError(f"Project partition is missing for project_id={project_id}")
return int(position)

async def record_accepted_note_change(
self,
session: AsyncSession,
change: RuntimeAcceptedProjectNoteChange,
) -> None:
"""Persist replayable accepted evidence in the mutation transaction."""
persisted = AcceptedProjectNoteChange(
project_id=change.project_id,
project_external_id=change.project_external_id,
partition_position=change.partition_position,
entity_id=change.entity_id,
note_external_id=change.note_external_id,
permalink=change.permalink,
title=change.title,
operation=change.operation.value,
file_path=change.file_path,
previous_file_path=change.previous_file_path,
accepted_at=change.accepted_at,
source=change.source,
db_version=change.db_version,
db_checksum=change.db_checksum,
actor_user_profile_id=(
str(change.actor_user_profile_id)
if change.actor_user_profile_id is not None
else None
),
actor_kind=change.actor_kind,
actor_name=change.actor_name,
)
session.add(persisted)
session.add(AcceptedProjectNoteChange(**_accepted_note_change_values(change)))
await session.flush()

async def record_accepted_note_changes(
self,
session: AsyncSession,
changes: Sequence[RuntimeAcceptedProjectNoteChange],
) -> None:
"""Persist a batch of accepted evidence rows with one bulk insert."""
if not changes:
return
await session.execute(
insert(AcceptedProjectNoteChange),
[_accepted_note_change_values(change) for change in changes],
)

async def get_partition_position(
self,
session: AsyncSession,
Expand Down Expand Up @@ -283,6 +331,79 @@ async def mark_accepted_note_change_materialized(
)
return bool((await session.execute(statement)).scalars().all())

async def mark_accepted_note_changes_materialized(
self,
session: AsyncSession,
project_id: int,
partition_positions: Sequence[int],
*,
materialized_at: datetime,
) -> int:
"""Settle several journal positions at once; return how many rows were stamped.

Same rule as ``mark_accepted_note_change_materialized`` for each position:
the target row and every older unsettled row of the same note are retired.
"""
unique_positions = sorted(set(partition_positions))
settled = 0
for start in range(0, len(unique_positions), _SETTLE_POSITION_CHUNK_SIZE):
chunk = unique_positions[start : start + _SETTLE_POSITION_CHUNK_SIZE]
target = aliased(AcceptedProjectNoteChange)
target_covers_row = exists().where(
target.project_id == project_id,
target.partition_position.in_(chunk),
target.note_external_id == AcceptedProjectNoteChange.note_external_id,
target.partition_position >= AcceptedProjectNoteChange.partition_position,
)
statement = (
update(AcceptedProjectNoteChange)
.where(
AcceptedProjectNoteChange.project_id == project_id,
AcceptedProjectNoteChange.materialized_at.is_(None),
target_covers_row,
)
.values(materialized_at=materialized_at)
.returning(AcceptedProjectNoteChange.id)
.execution_options(synchronize_session=False)
)
settled += len((await session.execute(statement)).scalars().all())
return settled

async def settle_unmaterialized_deletes(
self,
session: AsyncSession,
project_id: int,
*,
materialized_at: datetime,
) -> int:
"""Settle every unsettled `deleted` row of a project; return how many rows were stamped.

Each row is settled by the ``mark_accepted_note_change_materialized`` rule, so
older unsettled rows of the same deleted note are retired with it.
"""
positions = (
(
await session.execute(
select(AcceptedProjectNoteChange.partition_position).where(
AcceptedProjectNoteChange.project_id == project_id,
AcceptedProjectNoteChange.operation
== RuntimeProjectNoteOperation.deleted.value,
AcceptedProjectNoteChange.materialized_at.is_(None),
)
)
)
.scalars()
.all()
)
if not positions:
return 0
return await self.mark_accepted_note_changes_materialized(
session,
project_id,
positions,
materialized_at=materialized_at,
)

async def mark_note_changes_materialized_through_version(
self,
session: AsyncSession,
Expand Down
14 changes: 12 additions & 2 deletions src/basic_memory/runtime/cleanup.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,13 +153,23 @@ class RuntimeDirectoryFileSnapshot:
last_modified_at: float | None = None
size: int | None = None

def to_pending_note_file_delete(self, *, project_id: ProjectId) -> RuntimePendingNoteFileDelete:
"""Return the note-file cleanup work represented by this accepted snapshot."""
def to_pending_note_file_delete(
self,
*,
project_id: ProjectId,
project_change: RuntimeAcceptedProjectNoteChange | None = None,
) -> RuntimePendingNoteFileDelete:
"""Return the note-file cleanup work represented by this accepted snapshot.

``project_change`` is the note's `deleted` journal row; the cleanup job that
receives it settles the row once the file work is done.
"""
return RuntimePendingNoteFileDelete(
project_id=project_id,
entity_id=self.entity_id,
file_path=self.file_path,
file_checksum=self.file_checksum,
project_change=project_change,
)


Expand Down
9 changes: 9 additions & 0 deletions src/basic_memory/services/directory_deletes.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
accept_directory_delete,
finish_directory_delete_acceptance,
normalize_directory_delete_path,
settle_directory_delete_project_changes,
)
from basic_memory.read_cache import (
ReadCacheInvalidator,
Expand Down Expand Up @@ -105,6 +106,14 @@ async def delete_directory(
enqueuer=self.runtime.file_delete_enqueuer,
)

# Trigger: a file's cleanup ran inline (local) or could not be queued.
# Why: no queued job will settle its journal row, and one unsettled row
# defers every later wiki projection for the project (GitHub #1625).
# Outcome: settle those rows now; queued cleanups (cloud) settle their own.
if result.project_changes_to_settle:
async with db.scoped_session(self.session_maker) as session:
await settle_directory_delete_project_changes(session, result)
Comment on lines +113 to +115

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Recover unsettled directory deletes after interrupted cleanup

If a local folder delete is cancelled during inline cleanup (for example, Ctrl-C during a large deletion), the entity deletions and journal rows have already committed, but settlement is skipped because it runs only after the entire cleanup loop returns. Even notes whose files were successfully deleted retain materialized_at=NULL, permanently blocking subsequent wiki projections in plan_wiki_projection. Startup recovery cannot repair these rows: recover_stuck_materializations enumerates NoteContent, which this delete has cascaded away; retrying the folder delete also finds no entities for the removed files. Add a recovery path for these committed delete rows so interruption cannot leave the project's wiki projection permanently stalled.

AGENTS.md reference: AGENTS.md:L185-L187

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 9f72e87. Local startup recovery (recover_project_materializations) now settles every unsettled deleted journal row through settle_interrupted_note_deletes, so an interrupted cleanup can no longer stall wiki projection. Details: #2120 (comment)


# Trigger: notes outside the deleted directory linked into it.
# Why: the delete cascaded their relation rows away, but those sources own
# matching search_index relation rows that now dangle; without a reindex
Expand Down
28 changes: 25 additions & 3 deletions src/basic_memory/services/initialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,17 +62,39 @@ async def recover_project_materializations(
between accepting the DB snapshot and writing the file leaves note_content
stuck in writing/pending (a transient write error leaves it failed) and the
source-of-truth file missing. A moved note can also synchronize its destination
before the in-memory source cleanup is lost. Runs once at startup, before the
initial index, so both durable states converge. Non-fatal: a recovery failure
must not block startup, so it is logged and startup continues.
before the in-memory source cleanup is lost. A delete's journal row can also be
left unsettled when the process exits between the delete's commit and its inline
file cleanup. Runs once at startup, before the initial index, so these durable
states converge. Non-fatal: a recovery failure must not block startup, so it is
logged and startup continues.
"""
from basic_memory.index.note_content_materialization import (
recover_move_vacates,
recover_stuck_materializations,
settle_interrupted_note_deletes,
)
from basic_memory.read_cache import invalidate_cache
from basic_memory.services.file_service import FileService

# Trigger: a delete committed but the process exited before it settled the row.
# Why: one unsettled row defers every later wiki projection (GitHub #1625), and
# the local runtime has no queued cleanup job that would settle it later.
# Outcome: settle those rows; a failure here is logged and recovery continues.
try:
settled_deletes = await settle_interrupted_note_deletes(
session_maker=session_maker,
project_id=project.id,
)
except Exception as e: # pragma: no cover - defensive startup guard
logger.error(f"Error settling interrupted deletes for project {project.name}: {e}")
else:
if settled_deletes:
logger.info(
"Settled interrupted note deletes on startup",
project=project.name,
settled_journal_rows=settled_deletes,
)

# FileService needs only base_path to write the accepted markdown bytes;
# the markdown_processor/app_config are unused on the materialization path.
file_service = FileService(Path(project.path))
Expand Down
2 changes: 2 additions & 0 deletions test-int/read_cache/test_runtime_invalidation.py
Original file line number Diff line number Diff line change
Expand Up @@ -1272,6 +1272,7 @@ async def test_directory_delete_invalidates_before_and_after_cleanup_in_real_red
title="Deleted During Cleanup",
note_type="note",
content_type="text/markdown",
permalink="delete-with-cache/note",
file_path="delete-with-cache/note.md",
checksum="directory-delete-checksum",
created_at=datetime.now(UTC),
Expand Down Expand Up @@ -1325,6 +1326,7 @@ async def test_cancelled_committed_directory_delete_finishes_real_redis_invalida
title=title,
note_type="note",
content_type="text/markdown",
permalink="cancelled-delete/note",
file_path="cancelled-delete/note.md",
checksum="cancelled-directory-delete-checksum",
created_at=datetime.now(UTC),
Expand Down
1 change: 1 addition & 0 deletions test-int/test_relation_generation_concurrency.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ async def delete_directory() -> None:
await store.delete_directory_entities(
session,
project_id=test_project.id,
project_external_id=test_project.external_id,
directory="directory-race",
entity_ids=[source_id, sibling_id],
)
Expand Down
3 changes: 3 additions & 0 deletions tests/cloud/test_cloud_services.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ async def delete_directory_entities(
session: AsyncSession,
*,
project_id: int,
project_external_id: str,
directory: str,
entity_ids,
) -> DirectoryEntityDeleteResult:
Expand Down Expand Up @@ -1121,12 +1122,14 @@ async def delete_directory_entities(
session: AsyncSession,
*,
project_id: int,
project_external_id: str,
directory: str,
entity_ids,
) -> DirectoryEntityDeleteResult:
await super().delete_directory_entities(
session,
project_id=project_id,
project_external_id=project_external_id,
directory=directory,
entity_ids=entity_ids,
)
Expand Down
Loading
Loading