307 lines
12 KiB
Python
307 lines
12 KiB
Python
"""Cross-frontend chat logger.
|
|
|
|
When ``MarkdownFrontend(log_all_chats=True)`` is configured, every turn
|
|
completed by any other frontend (currently the Anthropic Messages
|
|
frontend) is mirrored into the vault as a ``.md`` file. Subsequent
|
|
turns of the same conversation append to the same file — matched by a
|
|
content-hash fingerprint stored in YAML frontmatter.
|
|
|
|
The fingerprint hashes the message history *before* the new assistant
|
|
reply. So the next request's input history (which now includes the
|
|
prior assistant reply) hashes to the value we just persisted —
|
|
``hash(prev_input + [assistant_reply])`` — and the lookup hits the
|
|
same file. New conversations (no prior fingerprint match) get a fresh
|
|
file under ``{vault_path}/{logged_subdir}/{agent_name}/``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import logging
|
|
from datetime import UTC, datetime
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
import frontmatter
|
|
|
|
from beaver_gateway.frontends.markdown import renderer
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Callable, Iterable
|
|
from pathlib import Path
|
|
|
|
from anthropic.types import MessageParam
|
|
|
|
from beaver_gateway.core.turn_record import TurnRecord
|
|
|
|
|
|
_log = logging.getLogger("beaver_gateway.frontends.markdown.crossfront")
|
|
|
|
|
|
__all__ = [
|
|
"ChatPathFn",
|
|
"CrossFrontendLogger",
|
|
"fingerprint_messages",
|
|
"strip_trailing_user_scaffold",
|
|
]
|
|
|
|
ChatPathFn = "Callable[[str, str, Path], Path]"
|
|
"""``(title, agent, vault) -> path`` of a new chat file; relative = under the
|
|
vault. ``None`` keeps ``{vault}/{logged_subdir}/{agent}/{YYYY-MM-DD}_{slug}.md``."""
|
|
|
|
|
|
def fingerprint_messages(messages: Iterable[MessageParam]) -> str:
|
|
"""Stable, short hex hash of a conversation prefix.
|
|
|
|
Built from ``(role, normalized_content)`` pairs only — so the
|
|
Markdown frontend's parser-shaped messages (text-only) and the
|
|
Anthropic frontend's raw ``messages`` payload (which may also be
|
|
string-only at v1) hash compatibly when they represent the same
|
|
conversation. Tool blocks / images would diverge, but those aren't
|
|
in the v1 ingest path.
|
|
"""
|
|
h = hashlib.sha1(usedforsecurity=False)
|
|
for msg in messages:
|
|
role = str(msg.get("role", ""))
|
|
content = msg.get("content", "")
|
|
if isinstance(content, str):
|
|
text = content
|
|
else:
|
|
parts: list[str] = []
|
|
for blk in content:
|
|
if not isinstance(blk, dict):
|
|
continue
|
|
btype = blk.get("type")
|
|
if btype == "text":
|
|
parts.append(str(blk.get("text", "")))
|
|
text = "\n".join(parts)
|
|
h.update(role.encode("utf-8"))
|
|
h.update(b"\x00")
|
|
h.update(text.strip().encode("utf-8"))
|
|
h.update(b"\x01")
|
|
return h.hexdigest()[:16]
|
|
|
|
|
|
class CrossFrontendLogger:
|
|
"""Maintains the fingerprint→file map and writes turns to disk.
|
|
|
|
The map is in-process; on startup ``warm_index`` rebuilds it from
|
|
YAML frontmatter of every file under ``logged_subdir``. A miss
|
|
creates a new file, a hit appends to the existing one. All disk
|
|
work funnels through one ``asyncio.Lock`` because the writes are
|
|
cheap and serializing them sidesteps a class of races we don't need
|
|
to think about.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
vault_path: Path,
|
|
logged_subdir: str,
|
|
chat_path: Callable[[str, str, Path], Path] | None = None,
|
|
) -> None:
|
|
self._vault = vault_path
|
|
self._root = vault_path / logged_subdir
|
|
self._index: dict[str, Path] = {}
|
|
self._lock = asyncio.Lock()
|
|
self._chat_path_fn = chat_path
|
|
# When the user supplies a custom path function, files can land
|
|
# anywhere in the vault — so we have to scan the whole vault on
|
|
# startup to rebuild the fingerprint→path map. With the default
|
|
# layout we can bound the scan to ``_logs/``.
|
|
self._scan_root = vault_path if chat_path is not None else self._root
|
|
|
|
def warm_index(self) -> None:
|
|
"""Scan logged files synchronously, populating the fingerprint map.
|
|
|
|
Called from ``MarkdownFrontend.configure`` so the map is ready
|
|
before any cross-frontend turn arrives. ``frontmatter.load``
|
|
reads only enough of the file to parse the YAML head, so the
|
|
scan is cheap even on large vaults — but a custom ``log_path``
|
|
forces a full-vault walk; mention that in the constructor doc.
|
|
"""
|
|
if not self._scan_root.exists():
|
|
return
|
|
for path in self._scan_root.rglob("*.md"):
|
|
try:
|
|
post = frontmatter.load(str(path))
|
|
except Exception: # noqa: BLE001
|
|
_log.warning("could not read frontmatter from %s — skipping", path)
|
|
continue
|
|
fp = post.metadata.get("fingerprint")
|
|
if isinstance(fp, str) and fp:
|
|
self._index[fp] = path
|
|
_log.info(
|
|
"crossfront index warmed: %d logged file(s) under %s",
|
|
len(self._index),
|
|
self._scan_root,
|
|
)
|
|
|
|
async def handle(self, record: TurnRecord) -> None:
|
|
"""Append or create a logged file for ``record``.
|
|
|
|
Records that the markdown frontend itself produced
|
|
(``source=="markdown"``) are skipped — those already live in the
|
|
user's hand-written file and shouldn't be duplicated into the
|
|
``_logs`` shadow tree.
|
|
"""
|
|
if record.source == "markdown":
|
|
return
|
|
|
|
async with self._lock:
|
|
# ``input_messages`` is the *full* history sent to the backend
|
|
# (last entry is the new user turn). Match against the prefix
|
|
# that excludes the new user turn — that's what the previous
|
|
# write stored as its fingerprint. Empty prefix is the
|
|
# well-known "brand new chat" sentinel.
|
|
prefix = record.input_messages[:-1]
|
|
prev_fp = fingerprint_messages(prefix) if prefix else None
|
|
target = self._index.get(prev_fp) if prev_fp else None
|
|
if target is None:
|
|
target = self._new_file_path(record)
|
|
|
|
# Build the full history including the assistant reply; the
|
|
# new fingerprint matches *that* prefix, so the next user
|
|
# turn (history grows by one user msg) will hit this file.
|
|
assistant_msg: MessageParam = {
|
|
"role": "assistant",
|
|
"content": _flatten_text(record.output_message),
|
|
}
|
|
full_history = [*record.input_messages, assistant_msg]
|
|
new_fp = fingerprint_messages(full_history)
|
|
|
|
if target.exists():
|
|
existing = target.read_text(encoding="utf-8")
|
|
parsed = frontmatter.loads(existing)
|
|
body = strip_trailing_user_scaffold(parsed.content)
|
|
# We append only the *new* user turn (the last one in
|
|
# input_messages, since prior turns are already on disk)
|
|
# plus the assistant reply.
|
|
new_user = record.input_messages[-1]
|
|
new_block = renderer.render_user_param(new_user)
|
|
new_block = renderer.append_to_body(
|
|
new_block, renderer.render_assistant_message(record.output_message)
|
|
)
|
|
new_body = renderer.append_to_body(body, new_block)
|
|
metadata = dict(parsed.metadata)
|
|
else:
|
|
# Materialize the whole conversation from scratch.
|
|
new_body = _render_full_history(
|
|
record.input_messages, record.output_message
|
|
)
|
|
metadata = {}
|
|
|
|
new_body = renderer.append_to_body(new_body, renderer.USER_SCAFFOLD)
|
|
|
|
metadata["agent"] = record.agent_name
|
|
metadata["fingerprint"] = new_fp
|
|
metadata["source"] = record.source
|
|
self._write(target, metadata, new_body)
|
|
# Maintain the index: drop the old fp (it's stale once we
|
|
# write the new turn), add the new one.
|
|
if prev_fp:
|
|
self._index.pop(prev_fp, None)
|
|
self._index[new_fp] = target
|
|
|
|
# ---- internals -----------------------------------------------------
|
|
|
|
def _new_file_path(self, record: TurnRecord) -> Path:
|
|
"""Pick a fresh filename for a brand-new conversation.
|
|
|
|
With a user-supplied ``chat_path`` we delegate to it (joining a
|
|
relative result with the vault root). Without one, we fall back
|
|
to ``{logged_subdir}/{agent}/{date}_{hex8}.md`` and ensure the
|
|
``.md`` suffix in case the user picks a non-md extension by hand.
|
|
"""
|
|
if self._chat_path_fn is not None:
|
|
result = self._chat_path_fn(
|
|
record.first_user_text, record.agent_name, self._vault
|
|
)
|
|
if not result.is_absolute():
|
|
result = self._vault / result
|
|
if result.suffix != ".md":
|
|
result = result.with_suffix(".md")
|
|
result.parent.mkdir(parents=True, exist_ok=True)
|
|
return result
|
|
day = datetime.now(UTC).strftime("%Y-%m-%d")
|
|
# Short hex from the input hash so two same-day chats sort
|
|
# stably and don't collide.
|
|
salt = fingerprint_messages(record.input_messages)[:8]
|
|
agent_dir = self._root / record.agent_name
|
|
agent_dir.mkdir(parents=True, exist_ok=True)
|
|
return agent_dir / f"{day}_{salt}.md"
|
|
|
|
def _write(self, path: Path, metadata: dict[str, Any], body: str) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
if metadata:
|
|
post = frontmatter.Post(content=body, **metadata)
|
|
text = frontmatter.dumps(post) + "\n"
|
|
else:
|
|
text = body if body.endswith("\n") else body + "\n"
|
|
# Sync write inside the lock — keeps the implementation tiny;
|
|
# individual logged turns are small enough that the blocking
|
|
# write doesn't matter at human conversation rates.
|
|
path.write_text(text, encoding="utf-8")
|
|
|
|
|
|
def _flatten_text(message: Any) -> str:
|
|
"""Same as ``frontend._flatten_assistant_text`` but local to break a cycle."""
|
|
chunks = [
|
|
getattr(block, "text", "") or ""
|
|
for block in getattr(message, "content", ())
|
|
if getattr(block, "type", None) == "text"
|
|
]
|
|
return "\n\n".join(c for c in chunks if c)
|
|
|
|
|
|
def _render_full_history(messages: list[MessageParam], assistant: Any) -> str:
|
|
"""Render an entire conversation (used when materializing a new logged file)."""
|
|
blocks: list[str] = []
|
|
for msg in messages:
|
|
role = msg.get("role")
|
|
if role == "user":
|
|
blocks.append(renderer.render_user_param(msg))
|
|
elif role == "assistant":
|
|
content = msg.get("content", "")
|
|
text = content if isinstance(content, str) else _content_to_text(content)
|
|
blocks.append(f"### Assistant:\n\n{text.strip()}\n" if text.strip() else "")
|
|
blocks.append(renderer.render_assistant_message(assistant))
|
|
body = ""
|
|
for block in blocks:
|
|
if not block:
|
|
continue
|
|
body = renderer.append_to_body(body, block)
|
|
return body
|
|
|
|
|
|
def strip_trailing_user_scaffold(body: str) -> str:
|
|
"""Drop a trailing empty ``### User:`` block if present.
|
|
|
|
Cross-frontend turns aren't typed into the file by the human — they
|
|
arrive whole from another frontend. If we leave the previous run's
|
|
scaffold in place, we'd write the new user turn right after an
|
|
empty marker (visual noise, two ``### User:`` headers in a row).
|
|
Trim it and let the append flow add a fresh scaffold at the end.
|
|
"""
|
|
stripped = body.rstrip()
|
|
marker = "### User:"
|
|
if not stripped.endswith(marker):
|
|
return body
|
|
# Walk back: the scaffold is the marker preceded by either start-of-file
|
|
# or an HR/blank line. Find the last newline before the marker, cut.
|
|
head = stripped[: -len(marker)].rstrip()
|
|
if head.endswith("---"):
|
|
head = head[: -len("---")].rstrip()
|
|
return head
|
|
|
|
|
|
def _content_to_text(content: Any) -> str:
|
|
if not isinstance(content, list):
|
|
return ""
|
|
chunks = [
|
|
str(blk.get("text", ""))
|
|
for blk in content
|
|
if isinstance(blk, dict) and blk.get("type") == "text"
|
|
]
|
|
return "\n\n".join(chunks)
|