refactor: no comments left - one-line module docstrings, contracts on public fields only; jobs/job.py; example config and README
This commit is contained in:
@@ -1,17 +1,7 @@
|
||||
"""Cross-frontend chat logger.
|
||||
|
||||
When ``MarkdownFrontend(log_all_chats=True)`` is configured, every turn
|
||||
completed by any other frontend (currently the Anthropic Messages
|
||||
frontend) is mirrored into the vault as a ``.md`` file. Subsequent
|
||||
turns of the same conversation append to the same file — matched by a
|
||||
content-hash fingerprint stored in YAML frontmatter.
|
||||
|
||||
The fingerprint hashes the message history *before* the new assistant
|
||||
reply. So the next request's input history (which now includes the
|
||||
prior assistant reply) hashes to the value we just persisted —
|
||||
``hash(prev_input + [assistant_reply])`` — and the lookup hits the
|
||||
same file. New conversations (no prior fingerprint match) get a fresh
|
||||
file under ``{vault_path}/{logged_subdir}/{agent_name}/``.
|
||||
Mirrors turns completed by other frontends into the vault as ``.md``
|
||||
files, matching a conversation's continuation by content-hash fingerprint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -53,12 +43,8 @@ vault. ``None`` keeps ``{vault}/{logged_subdir}/{agent}/{YYYY-MM-DD}_{slug}.md``
|
||||
def fingerprint_messages(messages: Iterable[MessageParam]) -> str:
|
||||
"""Stable, short hex hash of a conversation prefix.
|
||||
|
||||
Built from ``(role, normalized_content)`` pairs only — so the
|
||||
Markdown frontend's parser-shaped messages (text-only) and the
|
||||
Anthropic frontend's raw ``messages`` payload (which may also be
|
||||
string-only at v1) hash compatibly when they represent the same
|
||||
conversation. Tool blocks / images would diverge, but those aren't
|
||||
in the v1 ingest path.
|
||||
Hashes ``(role, text-only content)`` pairs so differently-shaped message
|
||||
histories that carry the same text still fingerprint identically.
|
||||
"""
|
||||
h = hashlib.sha1(usedforsecurity=False)
|
||||
for msg in messages:
|
||||
@@ -85,12 +71,8 @@ def fingerprint_messages(messages: Iterable[MessageParam]) -> str:
|
||||
class CrossFrontendLogger:
|
||||
"""Maintains the fingerprint→file map and writes turns to disk.
|
||||
|
||||
The map is in-process; on startup ``warm_index`` rebuilds it from
|
||||
YAML frontmatter of every file under ``logged_subdir``. A miss
|
||||
creates a new file, a hit appends to the existing one. All disk
|
||||
work funnels through one ``asyncio.Lock`` because the writes are
|
||||
cheap and serializing them sidesteps a class of races we don't need
|
||||
to think about.
|
||||
The map is in-process, rebuilt by ``warm_index`` on startup; all disk
|
||||
writes funnel through one lock to sidestep races.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -105,20 +87,13 @@ class CrossFrontendLogger:
|
||||
self._index: dict[str, Path] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
self._chat_path_fn = chat_path
|
||||
# When the user supplies a custom path function, files can land
|
||||
# anywhere in the vault — so we have to scan the whole vault on
|
||||
# startup to rebuild the fingerprint→path map. With the default
|
||||
# layout we can bound the scan to ``_logs/``.
|
||||
self._scan_root = vault_path if chat_path is not None else self._root
|
||||
|
||||
def warm_index(self) -> None:
|
||||
"""Scan logged files synchronously, populating the fingerprint map.
|
||||
|
||||
Called from ``MarkdownFrontend.configure`` so the map is ready
|
||||
before any cross-frontend turn arrives. ``frontmatter.load``
|
||||
reads only enough of the file to parse the YAML head, so the
|
||||
scan is cheap even on large vaults — but a custom ``log_path``
|
||||
forces a full-vault walk; mention that in the constructor doc.
|
||||
Called before any cross-frontend turn arrives; a custom ``chat_path``
|
||||
forces a full-vault walk instead of scanning just ``logged_subdir``.
|
||||
"""
|
||||
if not self._scan_root.exists():
|
||||
return
|
||||
@@ -140,29 +115,19 @@ class CrossFrontendLogger:
|
||||
async def handle(self, record: TurnRecord) -> None:
|
||||
"""Append or create a logged file for ``record``.
|
||||
|
||||
Records that the markdown frontend itself produced
|
||||
(``source=="markdown"``) are skipped — those already live in the
|
||||
user's hand-written file and shouldn't be duplicated into the
|
||||
``_logs`` shadow tree.
|
||||
Skips ``source == "markdown"`` records (already on disk); matches
|
||||
the target file by fingerprinting ``input_messages`` sans the new turn.
|
||||
"""
|
||||
if record.source == "markdown":
|
||||
return
|
||||
|
||||
async with self._lock:
|
||||
# ``input_messages`` is the *full* history sent to the backend
|
||||
# (last entry is the new user turn). Match against the prefix
|
||||
# that excludes the new user turn — that's what the previous
|
||||
# write stored as its fingerprint. Empty prefix is the
|
||||
# well-known "brand new chat" sentinel.
|
||||
prefix = record.input_messages[:-1]
|
||||
prev_fp = fingerprint_messages(prefix) if prefix else None
|
||||
target = self._index.get(prev_fp) if prev_fp else None
|
||||
if target is None:
|
||||
target = self._new_file_path(record)
|
||||
|
||||
# Build the full history including the assistant reply; the
|
||||
# new fingerprint matches *that* prefix, so the next user
|
||||
# turn (history grows by one user msg) will hit this file.
|
||||
assistant_msg: MessageParam = {
|
||||
"role": "assistant",
|
||||
"content": _flatten_text(record.output_message),
|
||||
@@ -174,9 +139,6 @@ class CrossFrontendLogger:
|
||||
existing = target.read_text(encoding="utf-8")
|
||||
parsed = frontmatter.loads(existing)
|
||||
body = strip_trailing_user_scaffold(parsed.content)
|
||||
# We append only the *new* user turn (the last one in
|
||||
# input_messages, since prior turns are already on disk)
|
||||
# plus the assistant reply.
|
||||
new_user = record.input_messages[-1]
|
||||
new_block = renderer.render_user_param(new_user)
|
||||
new_block = renderer.append_to_body(
|
||||
@@ -185,7 +147,6 @@ class CrossFrontendLogger:
|
||||
new_body = renderer.append_to_body(body, new_block)
|
||||
metadata = dict(parsed.metadata)
|
||||
else:
|
||||
# Materialize the whole conversation from scratch.
|
||||
new_body = _render_full_history(
|
||||
record.input_messages, record.output_message
|
||||
)
|
||||
@@ -197,21 +158,15 @@ class CrossFrontendLogger:
|
||||
metadata["fingerprint"] = new_fp
|
||||
metadata["source"] = record.source
|
||||
self._write(target, metadata, new_body)
|
||||
# Maintain the index: drop the old fp (it's stale once we
|
||||
# write the new turn), add the new one.
|
||||
if prev_fp:
|
||||
self._index.pop(prev_fp, None)
|
||||
self._index[new_fp] = target
|
||||
|
||||
# ---- internals -----------------------------------------------------
|
||||
|
||||
def _new_file_path(self, record: TurnRecord) -> Path:
|
||||
"""Pick a fresh filename for a brand-new conversation.
|
||||
|
||||
With a user-supplied ``chat_path`` we delegate to it (joining a
|
||||
relative result with the vault root). Without one, we fall back
|
||||
to ``{logged_subdir}/{agent}/{date}_{hex8}.md`` and ensure the
|
||||
``.md`` suffix in case the user picks a non-md extension by hand.
|
||||
Delegates to ``chat_path`` if set; otherwise
|
||||
``{logged_subdir}/{agent}/{date}_{hex8}.md``.
|
||||
"""
|
||||
if self._chat_path_fn is not None:
|
||||
result = self._chat_path_fn(
|
||||
@@ -224,8 +179,6 @@ class CrossFrontendLogger:
|
||||
result.parent.mkdir(parents=True, exist_ok=True)
|
||||
return result
|
||||
day = datetime.now(UTC).strftime("%Y-%m-%d")
|
||||
# Short hex from the input hash so two same-day chats sort
|
||||
# stably and don't collide.
|
||||
salt = fingerprint_messages(record.input_messages)[:8]
|
||||
agent_dir = self._root / record.agent_name
|
||||
agent_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -238,9 +191,6 @@ class CrossFrontendLogger:
|
||||
text = frontmatter.dumps(post) + "\n"
|
||||
else:
|
||||
text = body if body.endswith("\n") else body + "\n"
|
||||
# Sync write inside the lock — keeps the implementation tiny;
|
||||
# individual logged turns are small enough that the blocking
|
||||
# write doesn't matter at human conversation rates.
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
@@ -277,18 +227,13 @@ def _render_full_history(messages: list[MessageParam], assistant: Any) -> str:
|
||||
def strip_trailing_user_scaffold(body: str) -> str:
|
||||
"""Drop a trailing empty ``### User:`` block if present.
|
||||
|
||||
Cross-frontend turns aren't typed into the file by the human — they
|
||||
arrive whole from another frontend. If we leave the previous run's
|
||||
scaffold in place, we'd write the new user turn right after an
|
||||
empty marker (visual noise, two ``### User:`` headers in a row).
|
||||
Trim it and let the append flow add a fresh scaffold at the end.
|
||||
Avoids leaving two ``### User:`` headers in a row when appending a
|
||||
turn that wasn't typed into the file by hand.
|
||||
"""
|
||||
stripped = body.rstrip()
|
||||
marker = "### User:"
|
||||
if not stripped.endswith(marker):
|
||||
return body
|
||||
# Walk back: the scaffold is the marker preceded by either start-of-file
|
||||
# or an HR/blank line. Find the last newline before the marker, cut.
|
||||
head = stripped[: -len(marker)].rstrip()
|
||||
if head.endswith("---"):
|
||||
head = head[: -len("---")].rstrip()
|
||||
|
||||
Reference in New Issue
Block a user