From 65564d8541dfcbfd75fb787d9bf651341b2d6c9b Mon Sep 17 00:00:00 2001 From: h Date: Tue, 13 Oct 2026 01:22:36 +0200 Subject: [PATCH] fix(vault,conversations): the envelope diff drops query blocks and frontmatter, history hands the envelope apart from the user's words --- src/beaver_gateway/backends/transcript.py | 10 +++-- src/beaver_gateway/conversations/rows.py | 27 ++++++++++- src/beaver_gateway/conversations/texts.py | 11 +++++ src/beaver_gateway/vault/watch.py | 46 +++++++++++++++++-- tests/test_envelope.py | 55 ++++++++++++++++++++++- 5 files changed, 139 insertions(+), 10 deletions(-) diff --git a/src/beaver_gateway/backends/transcript.py b/src/beaver_gateway/backends/transcript.py index c127c7e..f955692 100644 --- a/src/beaver_gateway/backends/transcript.py +++ b/src/beaver_gateway/backends/transcript.py @@ -164,9 +164,9 @@ def build_entries( def messages_from_entries( - entries: Iterable[Mapping[str, Any]], *, with_ts: bool = False + entries: Iterable[Mapping[str, Any]], *, stamped: bool = False ) -> list[dict[str, Any]]: - """Messages as the model saw them; ``with_ts`` adds when each one was written.""" + """Messages as the model saw them; ``stamped`` adds each one's entry id and time.""" out: list[dict[str, Any]] = [] last_message_id: str | None = None for entry in entries: @@ -189,7 +189,8 @@ def messages_from_entries( "role": "user", "content": _user_content(content), } - if with_ts: + if stamped: + message_out["id"] = entry.get("uuid") message_out["ts"] = entry.get("timestamp") out.append(message_out) last_message_id = None @@ -205,7 +206,8 @@ def messages_from_entries( out[-1]["content"].extend(blocks) else: message_out: dict[str, Any] = {"role": "assistant", "content": blocks} - if with_ts: + if stamped: + message_out["id"] = entry.get("uuid") message_out["ts"] = entry.get("timestamp") out.append(message_out) last_message_id = message_id diff --git a/src/beaver_gateway/conversations/rows.py b/src/beaver_gateway/conversations/rows.py index a18439f..c77826c 100644 --- a/src/beaver_gateway/conversations/rows.py +++ b/src/beaver_gateway/conversations/rows.py @@ -19,6 +19,7 @@ from beaver_gateway.backends.transcript import ( ) from beaver_gateway.conversations.kinds import KINDS, Kind from beaver_gateway.conversations.state import State, iso +from beaver_gateway.conversations.texts import split_service from beaver_gateway.frontends.markdown.history import load_messages from beaver_gateway.storage.models import ( Conversation, @@ -482,9 +483,31 @@ class Rows(State): return await load_messages( session, conversation_id=cast("int", conv.id) ) - return messages_from_entries( - cast("Any", await self.entries(conv)), with_ts=True + messages = messages_from_entries( + cast("Any", await self.entries(conv)), stamped=True ) + return [self._unwrapped(message) for message in messages] + + def _unwrapped(self, message: dict[str, Any]) -> dict[str, Any]: + if message["role"] != "user": + return message + content = message["content"] + if isinstance(content, str): + text, service = split_service(content, self._texts) + return ( + message + if service is None + else {**message, "content": text, "service": service} + ) + for index, block in enumerate(content): + if block.get("type") != "text": + continue + text, service = split_service(str(block.get("text") or ""), self._texts) + if service is not None: + blocks = [*content] + blocks[index] = {**block, "text": text} + return {**message, "content": blocks, "service": service} + return message async def entries(self, conv: Conversation, *, subpath: str = "") -> list[Any]: if conv.session_id is None: diff --git a/src/beaver_gateway/conversations/texts.py b/src/beaver_gateway/conversations/texts.py index cc77d63..7a13e0b 100644 --- a/src/beaver_gateway/conversations/texts.py +++ b/src/beaver_gateway/conversations/texts.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from dataclasses import dataclass, field from typing import TYPE_CHECKING @@ -22,6 +23,7 @@ __all__ = [ "NewDayContext", "SeedContext", "UserSaid", + "split_service", ] @@ -127,3 +129,12 @@ class ConversationTexts: "; {fixed} open tool calls received tool_result «{interrupted}»" ) envelope: EnvelopeTexts = field(default_factory=EnvelopeTexts) + + +def split_service(text: str, texts: ConversationTexts) -> tuple[str, str | None]: + bundle = re.escape(texts.bundle_header).replace(r"\{since\}", ".+") + heads = f"{re.escape(texts.envelope.header)}|{bundle}" + match = re.search(rf"(?:\A|\n\n)(?=(?:{heads})$)", text, re.MULTILINE) + if match is None: + return text, None + return text[: match.start()], text[match.end() :] diff --git a/src/beaver_gateway/vault/watch.py b/src/beaver_gateway/vault/watch.py index 778e435..de71e10 100644 --- a/src/beaver_gateway/vault/watch.py +++ b/src/beaver_gateway/vault/watch.py @@ -13,6 +13,7 @@ import difflib import fnmatch import logging import os +import re from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path @@ -30,6 +31,8 @@ __all__ = ["Change", "VaultFilter", "VaultWatch", "WatchRules"] _log = logging.getLogger("beaver_gateway.vault.watch") +_FENCE = re.compile(r"^\s*(`{3,}|~{3,})\s*(.*?)\s*$") + @dataclass(frozen=True, slots=True) class WatchRules: @@ -44,6 +47,8 @@ class WatchRules: names: tuple[str, ...] = () ignore: tuple[str, ...] = () suffixes: tuple[str, ...] = (".md",) + skip_fences: tuple[str, ...] = ("tasks", "dataview", "dataviewjs") + """Fenced blocks of these languages are queries: ``full`` files drop them.""" def kind(self, rel: str, *, today: str) -> str | None: """The most specific matching pattern wins, whichever list it is in.""" @@ -140,7 +145,7 @@ class VaultWatch: if kind is None: continue with contextlib.suppress(OSError): - lines = _read_lines(path) + lines = self._lines(path, full=kind == "full") self._seen[rel] = _Seen( lines=lines if kind == "full" else None, count=len(lines) ) @@ -158,7 +163,7 @@ class VaultWatch: self._pending.pop(rel, None) return None stat = path.stat() - lines = _read_lines(path) + lines = self._lines(path, full=kind == "full") seen = self._seen.get(rel) change = Change( path=rel, @@ -186,12 +191,16 @@ class VaultWatch: for change in changes: path = self.root / change.path with contextlib.suppress(OSError): - lines = _read_lines(path) + lines = self._lines(path, full=change.full) self._seen[change.path] = _Seen( lines=lines if change.full else None, count=len(lines) ) return changes + def _lines(self, path: Path, *, full: bool) -> list[str]: + lines = _read_lines(path) + return _prose(lines, self.rules.skip_fences) if full else lines + def _rel(self, path: Path) -> str: return path.resolve().relative_to(self.root.resolve()).as_posix() @@ -216,6 +225,37 @@ def _read_lines(path: Path) -> list[str]: return path.read_text(encoding="utf-8", errors="replace").splitlines() +def _prose(lines: list[str], skip_fences: tuple[str, ...]) -> list[str]: + start = 0 + if lines and lines[0].strip() == "---": + closing = next( + (i for i, line in enumerate(lines[1:], 1) if line.strip() == "---"), None + ) + start = closing + 1 if closing is not None else 0 + out: list[str] = [] + fence: str | None = None + skipping = False + for line in lines[start:]: + match = _FENCE.match(line) + if fence is None and match: + fence = match.group(1) + language = match.group(2).split(maxsplit=1)[0] if match.group(2) else "" + skipping = language.lower() in skip_fences + elif fence is not None and match and _closes(match, fence): + fence = None + if skipping: + skipping = False + continue + if not skipping and line.strip(): + out.append(line) + return out + + +def _closes(match: re.Match[str], fence: str) -> bool: + mark = match.group(1) + return mark[0] == fence[0] and len(mark) >= len(fence) and not match.group(2) + + def _added(old: list[str], new: list[str]) -> list[str]: matcher = difflib.SequenceMatcher(a=old, b=new, autojunk=False) added: list[str] = [] diff --git a/tests/test_envelope.py b/tests/test_envelope.py index 804264b..e891aa6 100644 --- a/tests/test_envelope.py +++ b/tests/test_envelope.py @@ -7,7 +7,11 @@ from test_conversations import ScriptedClient, World, world from beaver_gateway.conversations.service import UserSaid from beaver_gateway.conversations.envelope import Envelope, RecallContext, render -from beaver_gateway.conversations.texts import EnvelopeTexts +from beaver_gateway.conversations.texts import ( + ConversationTexts, + EnvelopeTexts, + split_service, +) from beaver_gateway.vault.watch import Change, VaultWatch, WatchRules __all__ = ["world"] @@ -77,6 +81,37 @@ def test_watch_reports_only_added_lines_with_mtime() -> None: assert only.added == ["- 14:00 снова"] +def test_a_templated_note_shows_its_text_without_queries_or_frontmatter() -> None: + root, watch = vault() + rel = f"дни/{watch.today()}.md" + diary = root / rel + watch._seen.pop(rel) + diary.write_text( + "---\ntags:\n - дни\n---\n## сводка\n- \n\n\n---\n\n# планы\nпросрочены:\n" + "```tasks\nnot done\ndue before 2026-10-13\n```\n" + "обязательно сегодня:\n~~~dataview\nlist\n~~~\n" + "```python\nprint(1)\n```\n06:04 - спать\n" + ) + watch.note(diary) + (change,) = watch.take() + assert change.added == [ + "## сводка", + "- ", + "---", + "# планы", + "просрочены:", + "обязательно сегодня:", + "```python", + "print(1)", + "```", + "06:04 - спать", + ] + append(diary, "```tasks\ndone on 2026-10-13\n```\n07:10 - встал\n") + watch.note(diary) + (later,) = watch.take() + assert later.added == ["07:10 - встал"] + + def test_envelope_respects_ceilings_and_names_only_window() -> None: root, watch = vault() today = watch.today() @@ -129,6 +164,24 @@ def test_render_hits_total_ceiling() -> None: assert lines[1] == "time: 2026-08-26 13:04 (Warsaw)" +def test_the_users_words_part_from_what_the_gateway_put_under_them() -> None: + texts = ConversationTexts() + envelope = f"{HEADER}\ntime: 2026-10-13 10:00 (UTC)" + bundle = texts.bundle_header.format(since="2026-10-13T10:00:00+00:00") + injects = f"{bundle}\n- [cron] tick" + assert split_service(f"hello\n\n{envelope}", texts) == ("hello", envelope) + assert split_service(f"hello\n\n{envelope}\n\n{injects}", texts) == ( + "hello", + f"{envelope}\n\n{injects}", + ) + assert split_service(f"hello\n\n{injects}", texts) == ("hello", injects) + assert split_service(f"\n\n{envelope}", texts) == ("", envelope) + assert split_service("hello\n\n[a note of mine]\nmore", texts) == ( + "hello\n\n[a note of mine]\nmore", + None, + ) + + async def test_master_turn_gets_envelope_after_text_and_before_injects( world: World, ) -> None: