fix(vault,conversations): the envelope diff drops query blocks and frontmatter, history hands the envelope apart from the user's words

src/beaver_gateway/backends/transcript.py

@@ -164,9 +164,9 @@ def build_entries(
 
 
 def messages_from_entries(
-    entries: Iterable[Mapping[str, Any]], *, with_ts: bool = False
+    entries: Iterable[Mapping[str, Any]], *, stamped: bool = False
 ) -> list[dict[str, Any]]:
-    """Messages as the model saw them; ``with_ts`` adds when each one was written."""
+    """Messages as the model saw them; ``stamped`` adds each one's entry id and time."""
     out: list[dict[str, Any]] = []
     last_message_id: str | None = None
     for entry in entries:
@@ -189,7 +189,8 @@ def messages_from_entries(
                     "role": "user",
                     "content": _user_content(content),
                 }
-                if with_ts:
+                if stamped:
+                    message_out["id"] = entry.get("uuid")
                     message_out["ts"] = entry.get("timestamp")
                 out.append(message_out)
             last_message_id = None
@@ -205,7 +206,8 @@ def messages_from_entries(
             out[-1]["content"].extend(blocks)
         else:
             message_out: dict[str, Any] = {"role": "assistant", "content": blocks}
-            if with_ts:
+            if stamped:
+                message_out["id"] = entry.get("uuid")
                 message_out["ts"] = entry.get("timestamp")
             out.append(message_out)
         last_message_id = message_id

src/beaver_gateway/conversations/rows.py

@@ -19,6 +19,7 @@ from beaver_gateway.backends.transcript import (
 )
 from beaver_gateway.conversations.kinds import KINDS, Kind
 from beaver_gateway.conversations.state import State, iso
+from beaver_gateway.conversations.texts import split_service
 from beaver_gateway.frontends.markdown.history import load_messages
 from beaver_gateway.storage.models import (
     Conversation,
@@ -482,9 +483,31 @@ class Rows(State):
                 return await load_messages(
                     session, conversation_id=cast("int", conv.id)
                 )
-        return messages_from_entries(
-            cast("Any", await self.entries(conv)), with_ts=True
+        messages = messages_from_entries(
+            cast("Any", await self.entries(conv)), stamped=True
         )
+        return [self._unwrapped(message) for message in messages]
+
+    def _unwrapped(self, message: dict[str, Any]) -> dict[str, Any]:
+        if message["role"] != "user":
+            return message
+        content = message["content"]
+        if isinstance(content, str):
+            text, service = split_service(content, self._texts)
+            return (
+                message
+                if service is None
+                else {**message, "content": text, "service": service}
+            )
+        for index, block in enumerate(content):
+            if block.get("type") != "text":
+                continue
+            text, service = split_service(str(block.get("text") or ""), self._texts)
+            if service is not None:
+                blocks = [*content]
+                blocks[index] = {**block, "text": text}
+                return {**message, "content": blocks, "service": service}
+        return message
 
     async def entries(self, conv: Conversation, *, subpath: str = "") -> list[Any]:
         if conv.session_id is None:

src/beaver_gateway/conversations/texts.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import re
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
@@ -22,6 +23,7 @@ __all__ = [
     "NewDayContext",
     "SeedContext",
     "UserSaid",
+    "split_service",
 ]
 
 
@@ -127,3 +129,12 @@ class ConversationTexts:
         "; {fixed} open tool calls received tool_result «{interrupted}»"
     )
     envelope: EnvelopeTexts = field(default_factory=EnvelopeTexts)
+
+
+def split_service(text: str, texts: ConversationTexts) -> tuple[str, str | None]:
+    bundle = re.escape(texts.bundle_header).replace(r"\{since\}", ".+")
+    heads = f"{re.escape(texts.envelope.header)}|{bundle}"
+    match = re.search(rf"(?:\A|\n\n)(?=(?:{heads})$)", text, re.MULTILINE)
+    if match is None:
+        return text, None
+    return text[: match.start()], text[match.end() :]

src/beaver_gateway/vault/watch.py

@@ -13,6 +13,7 @@ import difflib
 import fnmatch
 import logging
 import os
+import re
 from dataclasses import dataclass, field
 from datetime import UTC, datetime
 from pathlib import Path
@@ -30,6 +31,8 @@ __all__ = ["Change", "VaultFilter", "VaultWatch", "WatchRules"]
 
 _log = logging.getLogger("beaver_gateway.vault.watch")
 
+_FENCE = re.compile(r"^\s*(`{3,}|~{3,})\s*(.*?)\s*$")
+
 
 @dataclass(frozen=True, slots=True)
 class WatchRules:
@@ -44,6 +47,8 @@ class WatchRules:
     names: tuple[str, ...] = ()
     ignore: tuple[str, ...] = ()
     suffixes: tuple[str, ...] = (".md",)
+    skip_fences: tuple[str, ...] = ("tasks", "dataview", "dataviewjs")
+    """Fenced blocks of these languages are queries: ``full`` files drop them."""
 
     def kind(self, rel: str, *, today: str) -> str | None:
         """The most specific matching pattern wins, whichever list it is in."""
@@ -140,7 +145,7 @@ class VaultWatch:
             if kind is None:
                 continue
             with contextlib.suppress(OSError):
-                lines = _read_lines(path)
+                lines = self._lines(path, full=kind == "full")
                 self._seen[rel] = _Seen(
                     lines=lines if kind == "full" else None, count=len(lines)
                 )
@@ -158,7 +163,7 @@ class VaultWatch:
             self._pending.pop(rel, None)
             return None
         stat = path.stat()
-        lines = _read_lines(path)
+        lines = self._lines(path, full=kind == "full")
         seen = self._seen.get(rel)
         change = Change(
             path=rel,
@@ -186,12 +191,16 @@ class VaultWatch:
         for change in changes:
             path = self.root / change.path
             with contextlib.suppress(OSError):
-                lines = _read_lines(path)
+                lines = self._lines(path, full=change.full)
                 self._seen[change.path] = _Seen(
                     lines=lines if change.full else None, count=len(lines)
                 )
         return changes
 
+    def _lines(self, path: Path, *, full: bool) -> list[str]:
+        lines = _read_lines(path)
+        return _prose(lines, self.rules.skip_fences) if full else lines
+
     def _rel(self, path: Path) -> str:
         return path.resolve().relative_to(self.root.resolve()).as_posix()
 
@@ -216,6 +225,37 @@ def _read_lines(path: Path) -> list[str]:
     return path.read_text(encoding="utf-8", errors="replace").splitlines()
 
 
+def _prose(lines: list[str], skip_fences: tuple[str, ...]) -> list[str]:
+    start = 0
+    if lines and lines[0].strip() == "---":
+        closing = next(
+            (i for i, line in enumerate(lines[1:], 1) if line.strip() == "---"), None
+        )
+        start = closing + 1 if closing is not None else 0
+    out: list[str] = []
+    fence: str | None = None
+    skipping = False
+    for line in lines[start:]:
+        match = _FENCE.match(line)
+        if fence is None and match:
+            fence = match.group(1)
+            language = match.group(2).split(maxsplit=1)[0] if match.group(2) else ""
+            skipping = language.lower() in skip_fences
+        elif fence is not None and match and _closes(match, fence):
+            fence = None
+            if skipping:
+                skipping = False
+                continue
+        if not skipping and line.strip():
+            out.append(line)
+    return out
+
+
+def _closes(match: re.Match[str], fence: str) -> bool:
+    mark = match.group(1)
+    return mark[0] == fence[0] and len(mark) >= len(fence) and not match.group(2)
+
+
 def _added(old: list[str], new: list[str]) -> list[str]:
     matcher = difflib.SequenceMatcher(a=old, b=new, autojunk=False)
     added: list[str] = []