feat: add stateful conversation storage

src/beaver_gateway/frontends/markdown/parser.py

@@ -27,7 +27,41 @@ if TYPE_CHECKING:
     from anthropic.types import MessageParam
 
 
-__all__ = ["ParsedFile", "last_role", "parse", "resolve_agent"]
+__all__ = [
+    "AssistantSegment",
+    "ParsedFile",
+    "ParsedTurn",
+    "TextSegment",
+    "ToolSegment",
+    "last_role",
+    "parse",
+    "parse_assistant_structure",
+    "resolve_agent",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class TextSegment:
+    """A run of plain text inside an assistant turn (between callouts)."""
+
+    text: str
+
+
+@dataclass(frozen=True, slots=True)
+class ToolSegment:
+    """A ``> [!tool]- <name>`` callout placeholder.
+
+    Only the tool ``name`` is captured — the " · summary" suffix on the
+    callout title and the JSON body inside the quote block are
+    decorative for the human reader; the canonical tool_use block lives
+    in the DB and is keyed by *position+name* against the structure
+    parsed here.
+    """
+
+    name: str
+
+
+AssistantSegment = TextSegment | ToolSegment
 
 
 # Turn marker — must be exactly ``### User:`` or ``### Assistant:`` on
@@ -42,6 +76,35 @@ _TURN_RE = re.compile(r"^###\s+(User|Assistant):\s*$", re.MULTILINE)
 # need to drop the whole quoted block.
 _CALLOUT_START_RE = re.compile(r"^>\s+\[!(thinking|tool)\]")
 
+# Tool-callout title line: ``> [!tool]- <name>`` or ``> [!tool]- <name> · <summary>``.
+# We only need the ``<name>`` part for skeleton matching; the summary is
+# decorative (built by ``renderer.summarize_tool_input`` from inputs the
+# user can edit visually without semantic consequence).
+_TOOL_TITLE_RE = re.compile(r"^>\s+\[!tool\]-\s*(.*?)\s*$")
+# Renderer joins name + summary with " · " (U+00B7) — see
+# ``renderer.summarize_tool_input``. We split on it to recover the
+# bare tool name.
+_TOOL_TITLE_SEP = " · "
+
+
+@dataclass(frozen=True, slots=True)
+class ParsedTurn:
+    """One turn extracted from the chat file.
+
+    ``role`` is ``"user"`` or ``"assistant"``. ``text`` is the spoken
+    content with callouts stripped and HRs dropped — used both as the
+    backend's ``MessageParam.content`` (back-compat with the existing
+    parser shape) and as the diff key against stored turns.
+    ``structure`` is non-empty only for assistant turns: an ordered
+    list of ``TextSegment`` / ``ToolSegment`` reflecting the visible
+    layout of the assistant block, used by the conversation store to
+    align with the canonical tool_use blocks held in DB.
+    """
+
+    role: str
+    text: str
+    structure: tuple[TextSegment | ToolSegment, ...] = ()
+
 
 @dataclass(frozen=True, slots=True)
 class ParsedFile:
@@ -49,48 +112,159 @@ class ParsedFile:
 
     ``metadata`` is the YAML frontmatter as a plain dict (empty if the
     file has none). ``messages`` is the conversation history shaped for
-    ``Backend.complete`` — assistant turns are text-only. ``body`` is the
-    raw markdown content *after* the frontmatter is stripped; the
-    renderer needs it when it appends a new assistant turn so it can
-    preserve whatever the human typed verbatim (including any callouts
-    or HRs they added).
+    ``Backend.complete`` — assistant turns are text-only. ``turns`` is
+    1:1 with ``messages`` and carries the per-turn structure (for
+    assistant turns) that the conversation store needs to detect
+    text-only edits vs. structural forks. ``body`` is the raw markdown
+    content *after* the frontmatter is stripped; the renderer needs it
+    when it appends a new assistant turn so it can preserve whatever
+    the human typed verbatim (including any callouts or HRs they
+    added).
     """
 
     metadata: dict[str, Any]
     body: str
     messages: list[MessageParam]
+    turns: list[ParsedTurn]
 
 
 def parse(text: str) -> ParsedFile:
-    """Parse a chat ``.md`` into ``(metadata, body, messages)``.
+    """Parse a chat ``.md`` into ``(metadata, body, messages, turns)``.
 
     A file with no turn markers but non-empty body is treated as a
     single user turn — the friendly path for "user types into a new
     file and hits send" before any turn markers exist.
+
+    Assistant turns that have *only* tool callouts (no spoken text) are
+    preserved here even though their ``MessageParam.content`` is empty
+    — the structure carries tool-segment information the conversation
+    store needs for skeleton matching. The renderer in practice always
+    emits at least a trailing text block, so this branch is defensive.
     """
     parsed = frontmatter.loads(text)
     metadata = dict(parsed.metadata)
     body = parsed.content
 
     messages: list[MessageParam] = []
-    turns = _split_turns(body)
-    if not turns:
+    parsed_turns: list[ParsedTurn] = []
+    raw_turns = _split_turns(body)
+    if not raw_turns:
         stripped = body.strip()
         if stripped:
             messages.append({"role": "user", "content": stripped})
-        return ParsedFile(metadata=metadata, body=body, messages=messages)
+            parsed_turns.append(ParsedTurn(role="user", text=stripped))
+        return ParsedFile(
+            metadata=metadata, body=body, messages=messages, turns=parsed_turns
+        )
 
-    for role, raw in turns:
+    for role, raw in raw_turns:
         if role == "user":
             text_content = _strip_hrs(raw).strip()
             if text_content:
                 messages.append({"role": "user", "content": text_content})
+                parsed_turns.append(ParsedTurn(role="user", text=text_content))
         else:
-            text_content = _extract_assistant_text(raw)
+            structure = parse_assistant_structure(raw)
+            text_content = _segments_to_spoken_text(structure)
+            has_tools = any(isinstance(s, ToolSegment) for s in structure)
             if text_content:
                 messages.append({"role": "assistant", "content": text_content})
+                parsed_turns.append(
+                    ParsedTurn(
+                        role="assistant", text=text_content, structure=tuple(structure)
+                    )
+                )
+            elif has_tools:
+                # Tool-only assistant turn: nothing to feed the backend
+                # as ``content`` (it'd reject an empty string), but the
+                # structure must survive so the store can align it
+                # against stored tool_use blocks. We synthesize a
+                # single-space text content for backend round-trip; the
+                # conversation store will replace this payload with the
+                # canonical stored blocks before the backend ever sees
+                # it on a continuation.
+                messages.append({"role": "assistant", "content": " "})
+                parsed_turns.append(
+                    ParsedTurn(role="assistant", text="", structure=tuple(structure))
+                )
 
-    return ParsedFile(metadata=metadata, body=body, messages=messages)
+    return ParsedFile(
+        metadata=metadata, body=body, messages=messages, turns=parsed_turns
+    )
+
+
+def parse_assistant_structure(raw: str) -> list[TextSegment | ToolSegment]:
+    """Walk an assistant turn body, return its ordered text/tool segments.
+
+    Tool callouts become :class:`ToolSegment` with just the tool name —
+    the title's optional ``" · summary"`` suffix and the JSON body
+    inside the quote block are decorative; the canonical tool_use
+    block is held in the conversation store. Thinking callouts are
+    stripped entirely (they were never round-trippable through the
+    file — signatures expire). HR separator lines drop out.
+
+    Empty / whitespace-only text segments at the boundaries (start,
+    end, between adjacent tool callouts) are dropped so the skeleton
+    is robust against renderer whitespace choices; a non-empty text
+    segment with surrounding whitespace is trimmed on both ends but
+    preserved.
+    """
+    segments: list[TextSegment | ToolSegment] = []
+    pending_text: list[str] = []
+
+    def _flush_text() -> None:
+        if not pending_text:
+            return
+        joined = "\n".join(pending_text)
+        # Collapse runs of >2 blank lines (created when we stripped a
+        # mid-block callout) into one so the diff against a re-render
+        # is stable.
+        cleaned = re.sub(r"\n{3,}", "\n\n", joined).strip()
+        pending_text.clear()
+        if cleaned:
+            segments.append(TextSegment(text=cleaned))
+
+    lines = raw.splitlines()
+    i = 0
+    while i < len(lines):
+        line = lines[i]
+        callout_match = _CALLOUT_START_RE.match(line)
+        if callout_match:
+            kind = callout_match.group(1)
+            # Capture tool name *before* advancing past the block.
+            if kind == "tool":
+                title_match = _TOOL_TITLE_RE.match(line)
+                title = title_match.group(1) if title_match else ""
+                name = title.split(_TOOL_TITLE_SEP, 1)[0].strip()
+                _flush_text()
+                segments.append(ToolSegment(name=name))
+            else:
+                # Thinking callout — drop the whole block, emit nothing.
+                _flush_text()
+            # Skip the rest of the quote block.
+            while i < len(lines) and lines[i].lstrip().startswith(">"):
+                i += 1
+            continue
+        if line.strip() == "---":
+            i += 1
+            continue
+        pending_text.append(line)
+        i += 1
+    _flush_text()
+    return segments
+
+
+def _segments_to_spoken_text(segments: list[TextSegment | ToolSegment]) -> str:
+    r"""Reduce a structure list to the spoken-text view the backend sees.
+
+    Concatenates :class:`TextSegment` contents with ``\n\n`` between
+    them, dropping :class:`ToolSegment` entries. Equivalent to what
+    the pre-Conversation-store parser did — we keep that behavior so
+    existing fingerprints (frontmatter ``fingerprint`` field) stay
+    valid.
+    """
+    chunks = [s.text for s in segments if isinstance(s, TextSegment)]
+    return "\n\n".join(c for c in chunks if c).strip()
 
 
 def last_role(messages: list[MessageParam]) -> str | None:
@@ -148,40 +322,3 @@ def _strip_hrs(raw: str) -> str:
     lines = raw.splitlines()
     kept = [ln for ln in lines if ln.strip() != "---"]
     return "\n".join(kept)
-
-
-def _extract_assistant_text(raw: str) -> str:
-    """Strip thinking/tool callouts from an assistant turn, return spoken text.
-
-    Walks line by line. When we see a callout-start line (``> [!thinking]-``
-    or ``> [!tool]- ...``), we skip the entire contiguous quote block
-    (lines beginning with ``>`` or blank-then-`>` continuations don't
-    happen in Obsidian callouts — a blank line ends the callout). HR
-    lines (``---``) are dropped. Everything else is kept and joined,
-    then collapsed to a clean trim.
-    """
-    lines = raw.splitlines()
-    out_lines: list[str] = []
-    i = 0
-    while i < len(lines):
-        line = lines[i]
-        if _CALLOUT_START_RE.match(line):
-            # Skip the whole quote block (consecutive lines starting
-            # with ``>``). Stop at first non-``>`` line, leaving it for
-            # the next iteration. Blank lines do not end the block — a
-            # callout body with a blank line uses ``> `` (quote-space)
-            # too — but in practice Obsidian's quote block ends on the
-            # first line that doesn't start with ``>``.
-            while i < len(lines) and lines[i].lstrip().startswith(">"):
-                i += 1
-            continue
-        if line.strip() == "---":
-            i += 1
-            continue
-        out_lines.append(line)
-        i += 1
-    # Collapse runs of blank lines that callout-stripping creates
-    # (two newlines around a stripped block fold into one).
-    text_joined = "\n".join(out_lines)
-    text_joined = re.sub(r"\n{3,}", "\n\n", text_joined)
-    return text_joined.strip()