refactor: add markdown frontend
This commit is contained in:
@@ -0,0 +1,302 @@
|
||||
"""Cross-frontend chat logger.
|
||||
|
||||
When ``MarkdownFrontend(log_all_chats=True)`` is configured, every turn
|
||||
completed by any other frontend (currently the Anthropic Messages
|
||||
frontend) is mirrored into the vault as a ``.md`` file. Subsequent
|
||||
turns of the same conversation append to the same file — matched by a
|
||||
content-hash fingerprint stored in YAML frontmatter.
|
||||
|
||||
The fingerprint hashes the message history *before* the new assistant
|
||||
reply. So the next request's input history (which now includes the
|
||||
prior assistant reply) hashes to the value we just persisted —
|
||||
``hash(prev_input + [assistant_reply])`` — and the lookup hits the
|
||||
same file. New conversations (no prior fingerprint match) get a fresh
|
||||
file under ``{vault_path}/{logged_subdir}/{agent_name}/``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import frontmatter
|
||||
|
||||
from beaver_gateway.frontends.markdown import renderer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Iterable
|
||||
from pathlib import Path
|
||||
|
||||
from anthropic.types import MessageParam
|
||||
|
||||
from beaver_gateway.core.turn_record import TurnRecord
|
||||
|
||||
|
||||
_log = logging.getLogger("beaver_gateway.frontends.markdown.crossfront")
|
||||
|
||||
|
||||
# User hook: take a turn + vault root, return where the new file should
|
||||
# live. Returning a relative ``Path`` is treated as relative to the
|
||||
# vault. ``None`` (the default) keeps the built-in
|
||||
# ``{vault}/{logged_subdir}/{agent}/{YYYY-MM-DD}_{hex8}.md`` layout.
|
||||
LogPathFn = "Callable[[TurnRecord, Path], Path]"
|
||||
|
||||
|
||||
__all__ = ["CrossFrontendLogger", "LogPathFn", "fingerprint_messages"]
|
||||
|
||||
|
||||
def fingerprint_messages(messages: Iterable[MessageParam]) -> str:
|
||||
"""Stable, short hex hash of a conversation prefix.
|
||||
|
||||
Built from ``(role, normalized_content)`` pairs only — so the
|
||||
Markdown frontend's parser-shaped messages (text-only) and the
|
||||
Anthropic frontend's raw ``messages`` payload (which may also be
|
||||
string-only at v1) hash compatibly when they represent the same
|
||||
conversation. Tool blocks / images would diverge, but those aren't
|
||||
in the v1 ingest path.
|
||||
"""
|
||||
h = hashlib.sha1(usedforsecurity=False)
|
||||
for msg in messages:
|
||||
role = str(msg.get("role", ""))
|
||||
content = msg.get("content", "")
|
||||
if isinstance(content, str):
|
||||
text = content
|
||||
else:
|
||||
parts: list[str] = []
|
||||
for blk in content:
|
||||
if not isinstance(blk, dict):
|
||||
continue
|
||||
btype = blk.get("type")
|
||||
if btype == "text":
|
||||
parts.append(str(blk.get("text", "")))
|
||||
text = "\n".join(parts)
|
||||
h.update(role.encode("utf-8"))
|
||||
h.update(b"\x00")
|
||||
h.update(text.strip().encode("utf-8"))
|
||||
h.update(b"\x01")
|
||||
return h.hexdigest()[:16]
|
||||
|
||||
|
||||
class CrossFrontendLogger:
|
||||
"""Maintains the fingerprint→file map and writes turns to disk.
|
||||
|
||||
The map is in-process; on startup ``warm_index`` rebuilds it from
|
||||
YAML frontmatter of every file under ``logged_subdir``. A miss
|
||||
creates a new file, a hit appends to the existing one. All disk
|
||||
work funnels through one ``asyncio.Lock`` because the writes are
|
||||
cheap and serializing them sidesteps a class of races we don't need
|
||||
to think about.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vault_path: Path,
|
||||
logged_subdir: str,
|
||||
log_path: Callable[[TurnRecord, Path], Path] | None = None,
|
||||
) -> None:
|
||||
self._vault = vault_path
|
||||
self._root = vault_path / logged_subdir
|
||||
self._index: dict[str, Path] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
self._log_path_fn = log_path
|
||||
# When the user supplies a custom path function, files can land
|
||||
# anywhere in the vault — so we have to scan the whole vault on
|
||||
# startup to rebuild the fingerprint→path map. With the default
|
||||
# layout we can bound the scan to ``_logs/``.
|
||||
self._scan_root = vault_path if log_path is not None else self._root
|
||||
|
||||
def warm_index(self) -> None:
|
||||
"""Scan logged files synchronously, populating the fingerprint map.
|
||||
|
||||
Called from ``MarkdownFrontend.configure`` so the map is ready
|
||||
before any cross-frontend turn arrives. ``frontmatter.load``
|
||||
reads only enough of the file to parse the YAML head, so the
|
||||
scan is cheap even on large vaults — but a custom ``log_path``
|
||||
forces a full-vault walk; mention that in the constructor doc.
|
||||
"""
|
||||
if not self._scan_root.exists():
|
||||
return
|
||||
for path in self._scan_root.rglob("*.md"):
|
||||
try:
|
||||
post = frontmatter.load(str(path))
|
||||
except Exception: # noqa: BLE001
|
||||
_log.warning("could not read frontmatter from %s — skipping", path)
|
||||
continue
|
||||
fp = post.metadata.get("fingerprint")
|
||||
if isinstance(fp, str) and fp:
|
||||
self._index[fp] = path
|
||||
_log.info(
|
||||
"crossfront index warmed: %d logged file(s) under %s",
|
||||
len(self._index),
|
||||
self._scan_root,
|
||||
)
|
||||
|
||||
async def handle(self, record: TurnRecord) -> None:
|
||||
"""Append or create a logged file for ``record``.
|
||||
|
||||
Records that the markdown frontend itself produced
|
||||
(``source=="markdown"``) are skipped — those already live in the
|
||||
user's hand-written file and shouldn't be duplicated into the
|
||||
``_logs`` shadow tree.
|
||||
"""
|
||||
if record.source == "markdown":
|
||||
return
|
||||
|
||||
async with self._lock:
|
||||
# ``input_messages`` is the *full* history sent to the backend
|
||||
# (last entry is the new user turn). Match against the prefix
|
||||
# that excludes the new user turn — that's what the previous
|
||||
# write stored as its fingerprint. Empty prefix is the
|
||||
# well-known "brand new chat" sentinel.
|
||||
prefix = record.input_messages[:-1]
|
||||
prev_fp = fingerprint_messages(prefix) if prefix else None
|
||||
target = self._index.get(prev_fp) if prev_fp else None
|
||||
if target is None:
|
||||
target = self._new_file_path(record)
|
||||
|
||||
# Build the full history including the assistant reply; the
|
||||
# new fingerprint matches *that* prefix, so the next user
|
||||
# turn (history grows by one user msg) will hit this file.
|
||||
assistant_msg: MessageParam = {
|
||||
"role": "assistant",
|
||||
"content": _flatten_text(record.output_message),
|
||||
}
|
||||
full_history = [*record.input_messages, assistant_msg]
|
||||
new_fp = fingerprint_messages(full_history)
|
||||
|
||||
if target.exists():
|
||||
existing = target.read_text(encoding="utf-8")
|
||||
parsed = frontmatter.loads(existing)
|
||||
body = _strip_trailing_user_scaffold(parsed.content)
|
||||
# We append only the *new* user turn (the last one in
|
||||
# input_messages, since prior turns are already on disk)
|
||||
# plus the assistant reply.
|
||||
new_user = record.input_messages[-1]
|
||||
new_block = renderer.render_user_param(new_user)
|
||||
new_block = renderer.append_to_body(
|
||||
new_block, renderer.render_assistant_message(record.output_message)
|
||||
)
|
||||
new_body = renderer.append_to_body(body, new_block)
|
||||
metadata = dict(parsed.metadata)
|
||||
else:
|
||||
# Materialize the whole conversation from scratch.
|
||||
new_body = _render_full_history(
|
||||
record.input_messages, record.output_message
|
||||
)
|
||||
metadata = {}
|
||||
|
||||
new_body = renderer.append_to_body(new_body, renderer.USER_SCAFFOLD)
|
||||
|
||||
metadata["agent"] = record.agent_name
|
||||
metadata["fingerprint"] = new_fp
|
||||
metadata["source"] = record.source
|
||||
self._write(target, metadata, new_body)
|
||||
# Maintain the index: drop the old fp (it's stale once we
|
||||
# write the new turn), add the new one.
|
||||
if prev_fp:
|
||||
self._index.pop(prev_fp, None)
|
||||
self._index[new_fp] = target
|
||||
|
||||
# ---- internals -----------------------------------------------------
|
||||
|
||||
def _new_file_path(self, record: TurnRecord) -> Path:
|
||||
"""Pick a fresh filename for a brand-new conversation.
|
||||
|
||||
With a user-supplied ``log_path`` we delegate to it (joining a
|
||||
relative result with the vault root). Without one, we fall back
|
||||
to ``{logged_subdir}/{agent}/{date}_{hex8}.md`` and ensure the
|
||||
``.md`` suffix in case the user picks a non-md extension by hand.
|
||||
"""
|
||||
if self._log_path_fn is not None:
|
||||
result = self._log_path_fn(record, self._vault)
|
||||
if not result.is_absolute():
|
||||
result = self._vault / result
|
||||
if result.suffix != ".md":
|
||||
result = result.with_suffix(".md")
|
||||
result.parent.mkdir(parents=True, exist_ok=True)
|
||||
return result
|
||||
day = datetime.now(UTC).strftime("%Y-%m-%d")
|
||||
# Short hex from the input hash so two same-day chats sort
|
||||
# stably and don't collide.
|
||||
salt = fingerprint_messages(record.input_messages)[:8]
|
||||
agent_dir = self._root / record.agent_name
|
||||
agent_dir.mkdir(parents=True, exist_ok=True)
|
||||
return agent_dir / f"{day}_{salt}.md"
|
||||
|
||||
def _write(self, path: Path, metadata: dict[str, Any], body: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if metadata:
|
||||
post = frontmatter.Post(content=body, **metadata)
|
||||
text = frontmatter.dumps(post) + "\n"
|
||||
else:
|
||||
text = body if body.endswith("\n") else body + "\n"
|
||||
# Sync write inside the lock — keeps the implementation tiny;
|
||||
# individual logged turns are small enough that the blocking
|
||||
# write doesn't matter at human conversation rates.
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def _flatten_text(message: Any) -> str:
|
||||
"""Same as ``frontend._flatten_assistant_text`` but local to break a cycle."""
|
||||
chunks = [
|
||||
getattr(block, "text", "") or ""
|
||||
for block in getattr(message, "content", ())
|
||||
if getattr(block, "type", None) == "text"
|
||||
]
|
||||
return "\n\n".join(c for c in chunks if c)
|
||||
|
||||
|
||||
def _render_full_history(messages: list[MessageParam], assistant: Any) -> str:
|
||||
"""Render an entire conversation (used when materializing a new logged file)."""
|
||||
blocks: list[str] = []
|
||||
for msg in messages:
|
||||
role = msg.get("role")
|
||||
if role == "user":
|
||||
blocks.append(renderer.render_user_param(msg))
|
||||
elif role == "assistant":
|
||||
content = msg.get("content", "")
|
||||
text = content if isinstance(content, str) else _content_to_text(content)
|
||||
blocks.append(f"### Assistant:\n\n{text.strip()}\n" if text.strip() else "")
|
||||
blocks.append(renderer.render_assistant_message(assistant))
|
||||
body = ""
|
||||
for block in blocks:
|
||||
if not block:
|
||||
continue
|
||||
body = renderer.append_to_body(body, block)
|
||||
return body
|
||||
|
||||
|
||||
def _strip_trailing_user_scaffold(body: str) -> str:
|
||||
"""Drop a trailing empty ``### User:`` block if present.
|
||||
|
||||
Cross-frontend turns aren't typed into the file by the human — they
|
||||
arrive whole from another frontend. If we leave the previous run's
|
||||
scaffold in place, we'd write the new user turn right after an
|
||||
empty marker (visual noise, two ``### User:`` headers in a row).
|
||||
Trim it and let the append flow add a fresh scaffold at the end.
|
||||
"""
|
||||
stripped = body.rstrip()
|
||||
marker = "### User:"
|
||||
if not stripped.endswith(marker):
|
||||
return body
|
||||
# Walk back: the scaffold is the marker preceded by either start-of-file
|
||||
# or an HR/blank line. Find the last newline before the marker, cut.
|
||||
head = stripped[: -len(marker)].rstrip()
|
||||
if head.endswith("---"):
|
||||
head = head[: -len("---")].rstrip()
|
||||
return head
|
||||
|
||||
|
||||
def _content_to_text(content: Any) -> str:
|
||||
if not isinstance(content, list):
|
||||
return ""
|
||||
chunks = [
|
||||
str(blk.get("text", ""))
|
||||
for blk in content
|
||||
if isinstance(blk, dict) and blk.get("type") == "text"
|
||||
]
|
||||
return "\n\n".join(chunks)
|
||||
Reference in New Issue
Block a user