beaver/beaver-gateway
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: no comments left - one-line module docstrings, contracts on public fields only; jobs/job.py; example config and README

Browse Source
This commit is contained in:
hh committed 2026-09-02 00:33:04 +02:00
1 parent 90d0fb0f00
commit c9c2e94954
65 files changed
+795 -1891
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/beaver_gateway/frontends/markdown/frontend.py
+35 -131
View File
@@ -1,34 +1,7 @@
"""``MarkdownFrontend`` — chat-via-markdown-files frontend.
Wires:
* ``POST /chat {filename, content?, agent?}`` — bearer-authenticated
trigger. The plugin in Obsidian fires this after the user edits a
``.md`` and the file gets synced (or with ``content`` to short-circuit
the sync delay). We parse the file, check the last turn — assistant
→ no-op, user → run the agent and append.
* ``GET /healthz`` — liveness.
Concurrency model: an in-memory ``set[Path]`` of files currently in
flight. Two concurrent requests for the same file → the second gets
409. The set is single-process (one gateway instance) — that's by
design; the markdown frontend is the only writer in its vault from
the gateway side. The turn itself runs through ``core/conversations``
(one turn per conversation, ``running_turn`` in the DB), so a message
posted to the same conversation via ``/api`` waits its turn.
A chat file is a ``deep`` conversation bound as
``(markdown, <vault-relative path>)``; frontmatter carries only ``agent``
and ``conversation_id`` (§3.10), tool calls are never rendered. The
frontend is the home of ``deep``: ``materialize`` gives a conversation
spawned elsewhere its file, and :class:`.mirror.ChatMirror` keeps that
file in step with replies produced outside ``/chat``.
Cross-frontend logging: when ``log_all_chats=True``, ``configure()``
registers a handler on ``runtime.turn_log_handlers`` so every other
frontend's completed turns also land in the vault. The handler logic
lives in :mod:`.crossfront` so this module stays focused on the HTTP
shape.
``POST /chat`` (and SSE ``/chat/stream``) parses the vault file, runs
the agent on the trailing user turn, and appends the reply.
"""
from __future__ import annotations
@@ -85,23 +58,17 @@ _log = logging.getLogger("beaver_gateway.frontends.markdown")
__all__ = ["MarkdownFrontend"]
# How often we re-render the assistant turn into the .md file while the
# backend stream is still open. Trades responsiveness (faster updates to
# Obsidian sync / Raycast tailers) against write amplification. Each
# ``RawContentBlockStopEvent`` also forces a flush regardless of the
# timer, so block boundaries always land in the file.
_STREAM_FLUSH_DEBOUNCE = 0.4
# Debounce for the SSE ``/chat/stream`` path. Network IO is cheaper than
# atomic file rewrites, so we send updates more frequently — the client
# wants the lowest possible latency and we control the renderer on the
# other end (the Obsidian plugin splices deltas into the editor, no
# disk round-trip).
_SSE_FLUSH_DEBOUNCE = 0.1
class MarkdownFrontend(Frontend):
"""FastAPI app behind ``POST /chat`` driven by Obsidian-vault files."""
"""FastAPI app behind ``POST /chat`` driven by Obsidian-vault files.
``_busy`` tracks in-flight files; check-and-add must stay atomic (no
``await`` between them) so concurrent requests reliably lose to 409.
"""
name = FRONTEND
kinds = ("deep",)
@@ -116,6 +83,11 @@ class MarkdownFrontend(Frontend):
logged_subdir: str = "_logs",
chat_path: Callable[[str, str, Path], Path] | None = None,
) -> None:
"""Configure the vault-backed frontend.
``chat_path``, if given, overrides where new chat files are created;
``logged_subdir`` holds cross-frontend logs when ``log_all_chats`` is set.
"""
self.vault_path = Path(vault_path).expanduser().resolve()
self.default_agent = default_agent
self.log_all_chats = log_all_chats
@@ -123,10 +95,6 @@ class MarkdownFrontend(Frontend):
self.chat_path = chat_path
self._runtime: GatewayRuntime | None = None
self._app: FastAPI | None = None
# Files currently being processed by an in-flight ``POST /chat``.
# Checked-and-added atomically in the request handler (no
# ``await`` between the check and the insert) so a concurrent
# request reliably loses the race to 409.
self._busy: set[Path] = set()
self._crossfront: CrossFrontendLogger | None = None
self._mirror: ChatMirror | None = None
@@ -149,9 +117,6 @@ class MarkdownFrontend(Frontend):
logged_subdir=self.logged_subdir,
chat_path=self.chat_path,
)
# Scan the existing logged files synchronously here so the
# fingerprint→path map is populated before the first
# cross-frontend turn arrives. Cheap: frontmatter-only read.
self._crossfront.warm_index()
runtime.turn_log_handlers.append(self._crossfront.handle)
self._app = self._build_app(runtime)
@@ -175,16 +140,10 @@ class MarkdownFrontend(Frontend):
async def serve(self) -> None:
await self.mirror.run()
# ---- app builder ---------------------------------------------------
def _build_app(self, runtime: GatewayRuntime) -> FastAPI:
"""CORS is wide open here since auth is bearer-token, not cookie-based."""
app = FastAPI(title="beaver-gateway / Markdown")
# ``/chat/stream`` is consumed via ``fetch`` from the Obsidian
# plugin (``requestUrl`` can't read a body incrementally), and
# ``fetch`` is subject to CORS. Auth is bearer-token so we don't
# need credentialed mode; allow any origin and the standard
# methods/headers. The other endpoints are happy to ride along.
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
@@ -226,8 +185,6 @@ class MarkdownFrontend(Frontend):
file_path = self._resolve_path(filename)
# Atomic check-and-claim: both ops run between awaits, so a
# second request can't slip into the same file slot.
if file_path in self._busy:
return JSONResponse(
status_code=status.HTTP_409_CONFLICT,
@@ -248,12 +205,10 @@ class MarkdownFrontend(Frontend):
@app.post("/chat/stream")
async def chat_stream(request: Request) -> Any:
# Same contract as ``/chat`` (bearer auth, identical body),
# but the response is ``text/event-stream`` and intermediate
# rendered states are pushed as ``delta`` events. The
# gateway-side disk write only happens once, at end of turn,
# so streaming consumers (Obsidian plugin) and Obsidian Sync
# don't fight over the same file mid-stream.
"""SSE variant of ``/chat``: ``delta`` events, one disk write at the end.
The 409-conflict response is still plain JSON — the stream hasn't started.
"""
token_name = await require_token(request, runtime, scope="messages")
try:
body = await request.json()
@@ -276,8 +231,6 @@ class MarkdownFrontend(Frontend):
file_path = self._resolve_path(filename)
# 409 path stays JSON — the stream hasn't started yet, so
# the caller can read it the same way as on ``/chat``.
if file_path in self._busy:
return JSONResponse(
status_code=status.HTTP_409_CONFLICT,
@@ -305,8 +258,6 @@ class MarkdownFrontend(Frontend):
return app
# ---- dispatch ------------------------------------------------------
async def _handle_chat(
self,
*,
@@ -317,6 +268,7 @@ class MarkdownFrontend(Frontend):
content_override: Any,
agent_override: str | None,
) -> Any:
"""Non-streaming ``/chat`` handler: parse, align, run, persist."""
write_disk = content_override is None
if isinstance(content_override, str):
file_text = content_override
@@ -340,8 +292,6 @@ class MarkdownFrontend(Frontend):
"or configure `default_agent`",
)
# When the parser produced no messages (file empty / only
# frontmatter), there's nothing to dispatch.
if not parsed.messages:
return {
"status": "nothing_to_do",
@@ -384,9 +334,6 @@ class MarkdownFrontend(Frontend):
msgs=len(parsed.messages),
)
# Resolve / mint the conversation row, align incoming against
# stored history, and feed the aligned messages to the backend
# - see ``core/conversation_store.py`` for the full rationale.
conv, conv_external_id, stored_msgs = await self._resolve_conversation(
runtime=runtime,
metadata=parsed.metadata,
@@ -437,9 +384,6 @@ class MarkdownFrontend(Frontend):
message=message,
)
# Broadcast our own turn so other handlers (none today, but the
# symmetry is worth keeping) see what happened. ``source`` marks
# the origin so ``CrossFrontendLogger`` can skip its own files.
record = TurnRecord(
agent_name=agent.name,
input_messages=list(parsed.messages),
@@ -460,9 +404,7 @@ class MarkdownFrontend(Frontend):
"new_content": new_content,
}
# ---- streaming dispatch (SSE) --------------------------------------
async def _handle_chat_streaming( # noqa: PLR0915 — mirrors _handle_chat, splitting only doubles read cost
async def _handle_chat_streaming( # noqa: PLR0915
self,
*,
runtime: GatewayRuntime,
@@ -474,21 +416,9 @@ class MarkdownFrontend(Frontend):
) -> AsyncIterator[bytes]:
"""SSE counterpart of :meth:`_handle_chat`.
Mirrors the same pipeline (resolve file → parse → resolve agent →
run backend → persist), but emits ``event: delta`` frames as the
rendered turn grows and a single terminal ``event: done`` /
``event: error``. Errors that ``_handle_chat`` would surface as
``HTTPException`` go out as ``error`` frames here (the HTTP
envelope is already 200 by the time the stream starts).
Intermediate disk writes are deliberately skipped — only the
post-stream :meth:`_write_assistant_reply` lands on disk, so the
gateway-side vault and the plugin-side editor are the only
writers in their respective halves of Obsidian Sync. Final
content is identical on both sides, so Sync no-ops.
Errors surface as ``error`` frames (HTTP is already 200 by then); disk
is only written once at the end, avoiding a write race with Obsidian Sync.
"""
# With ``content`` the plugin is the only writer of the file
# (§3.10): the gateway never touches disk in that case.
write_disk = content_override is None
if isinstance(content_override, str):
file_text = content_override
@@ -653,18 +583,12 @@ class MarkdownFrontend(Frontend):
or (now - last_flush) >= _SSE_FLUSH_DEBOUNCE
):
payload = snapshot()
# Skip duplicate snapshots — e.g. tool_use blocks
# render to the same prefix as before they closed
# (we don't surface the tool-call args in markdown).
if payload is not None and payload != last_payload:
yield sse_pack("delta", {"new_content": payload})
last_payload = payload
last_flush = now
except Exception as exc: # noqa: BLE001 — wire any backend failure as an SSE error frame
except Exception as exc: # noqa: BLE001
_log.exception("backend failed for %s", filename)
# Mirror the legacy path: write the last partial + an error
# callout to disk so other consumers (logs, file watchers)
# see what arrived. The client gets a clean SSE ``error``.
partial = acc.finalize(model=model)
new_body = parsed.body
if partial.content:
@@ -728,8 +652,6 @@ class MarkdownFrontend(Frontend):
},
)
# ---- helpers -------------------------------------------------------
async def _stream_to_file(
self,
*,
@@ -742,17 +664,8 @@ class MarkdownFrontend(Frontend):
) -> Any:
"""Drain ``events`` into a ``Message``, flushing partials to disk.
Flushes happen on each ``RawContentBlockStopEvent`` (natural
block boundary, content is markdown-consistent) and on the
``_STREAM_FLUSH_DEBOUNCE`` timer between events. The partial
write keeps the as-parsed frontmatter; the post-stream final
write in ``_write_assistant_reply`` is what stamps the refreshed
fingerprint / agent / conversation_id.
On backend exception we still flush the last partial and append
an error callout, so the human sees both what arrived and why it
stopped. The exception propagates so ``_handle_chat`` can map it
to a 500.
Flushes on each block boundary and on a debounce timer; on backend
failure it still flushes a partial + error callout, then re-raises.
"""
acc = StreamAccumulator()
@@ -829,11 +742,8 @@ class MarkdownFrontend(Frontend):
) -> tuple[Conversation, str, list[dict[str, Any]]]:
"""Resolve the ``deep`` conversation for this file + its stored messages.
Frontmatter ``conversation_id`` wins; a file that lost it is found
by its visible ``(markdown, path)`` binding, then by adopting the
one unbound pre-SDK conversation that starts with the same prompt;
otherwise a new conversation is created. The binding follows the
file: a moved chat re-binds to its new path on the next turn.
Precedence: frontmatter ``conversation_id`` > existing path binding >
an adopted unbound conversation with the same first prompt > a new one.
"""
conversations = runtime.conversations
rel = file_path.relative_to(self.vault_path).as_posix()
@@ -882,9 +792,8 @@ class MarkdownFrontend(Frontend):
) -> None:
"""Stamp the DB with the post-turn canonical Anthropic-shape history.
Combines the matched/spliced prior state, the new user prompt,
and the synthesized assistant/tool cycle from the backend (or a
text-only fallback for backends that left ``capture`` empty).
Combines prior state + new user prompt + the backend's synthesized
cycle, falling back to text-only if ``capture`` is empty.
"""
new_user_msg = {"role": "user", "content": new_user_text}
synthesized = capture.synthesized_messages or _fallback_synthesized(message)
@@ -906,12 +815,11 @@ class MarkdownFrontend(Frontend):
)
def _resolve_path(self, filename: str) -> Path:
"""Resolve ``filename`` under the vault; reject escapes."""
# ``filename`` may be relative or absolute; we always anchor
# under ``vault_path`` so absolute paths from outside the vault
# don't sneak through. ``Path("/foo/bar")`` combined with a
# vault path keeps the absolute side; we strip leading slashes
# to coerce the rooted form into a relative path before joining.
"""Resolve ``filename`` under the vault; reject escapes.
Leading slashes are stripped first — ``Path.__truediv__`` would
otherwise discard ``vault_path`` for an absolute ``filename``.
"""
rel = filename.lstrip("/")
if not rel.endswith(".md"):
rel = rel + ".md"
@@ -928,12 +836,8 @@ class MarkdownFrontend(Frontend):
def _fallback_synthesized(message: Any) -> list[dict[str, Any]]:
"""Build a single-assistant ``synthesized_messages`` list from a raw ``Message``.
For backends that don't populate a :class:`TurnCapture` (anthropic
HTTP, raycast, …) we don't have access to per-tool-cycle
granularity, so the assistant reply lands in the DB as one
canonical-block message. Tool memory across cache misses would
degrade in that case, but those backends don't have the cache-miss
re-seed problem to begin with — they manage history client-side.
Fallback for backends that don't populate :class:`TurnCapture` (anthropic
HTTP, raycast, …); the reply lands as one canonical-block message.
"""
content: list[dict[str, Any]] = []
for block in getattr(message, "content", ()):