beaver/beaver-gateway
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: no comments left - one-line module docstrings, contracts on public fields only; jobs/job.py; example config and README

Browse Source
This commit is contained in:
hh committed 2026-09-02 00:33:04 +02:00
1 parent 3915ab48f9
commit 0e07e8409d
65 files changed
+795 -1891
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/beaver_gateway/security/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Bearer tokens, the audit log and credential redaction."""
src/beaver_gateway/security/audit.py
+11 -39
View File
@@ -1,32 +1,7 @@
"""Single entry point for writing :class:`AuditLog` rows.
Every frontend ends up needing the same three-line pattern — open a DB
session, append a row, swallow failures so the user-visible request
still succeeds. Phase 4.3 inlined that pattern in the admin frontend
under a private ``_audit()`` helper; Phase 4.4 lifts it here so the
Messages and MCP frontends can call the same function and so the
swallow-and-log policy lives in one place.
The contract:
* ``log(runtime, actor=..., kind=...)`` is fire-and-forget. It awaits
the DB write (so callers can ``await`` it before responding and get
ordering), but never raises — if the audit insert fails, the function
emits an ``exception`` log line and returns.
* ``actor`` is a free-form string. By convention: ``"token:<name>"``
for bearer-authenticated traffic, ``"admin:<user>"`` for admin-UI
actions, ``"anon"`` for failed-auth paths we still want to record.
* ``kind`` is a short tag — see :data:`KNOWN_KINDS` for the set the
current frontends emit; new tags don't need a code change here, the
column is free-form.
* ``**detail`` is JSON-serialised by :func:`append_audit`. Keep it
small: paths, methods, status codes — not request bodies. Anything
passed here lands in ``AuditLog.detail_json`` verbatim.
Why a thin wrapper rather than ``append_audit`` directly: callers want
"write if you can, otherwise carry on", and pulling the try/except into
every frontend was already starting to drift (admin had it, bearer
frontends would have copy-pasted). One module, one policy.
``log()`` is fire-and-forget: it awaits the DB write so callers get
ordering, but never raises — failures are logged and swallowed.
"""
from __future__ import annotations
@@ -43,15 +18,11 @@ if TYPE_CHECKING:
_log = logging.getLogger("beaver_gateway.audit")
# Tags currently emitted by the gateway. The set is informational —
# ``AuditLog.kind`` is free-form so new code can introduce new tags
# without touching this list — but listing them here gives the admin UI
# and any downstream log consumers one canonical reference.
KNOWN_KINDS: frozenset[str] = frozenset(
{
"messages", # POST /v1/messages accepted
"mcp_call", # /mcp/<ns>/... proxied
"tool_call", # a model's tool call seen by the PreToolUse hook
"messages",
"mcp_call",
"tool_call",
"login_ok",
"login_failed",
"logout",
@@ -59,6 +30,7 @@ KNOWN_KINDS: frozenset[str] = frozenset(
"token_revoke",
}
)
"""Kinds the gateway currently emits; ``AuditLog.kind`` stays free-form."""
async def log(
@@ -69,12 +41,12 @@ async def log(
agent_name: str | None = None,
**detail: Any,
) -> None:
"""Best-effort audit insert. Never raises.
"""Best-effort audit insert. Never raises; DB failures are logged.
Opens its own short-lived :class:`AsyncSession` so callers don't
have to thread one through. If the DB hiccups (table missing,
disk full, connection drop), we log and move on — the audit trail
is observability, not a hard precondition for serving the request.
``actor`` is free-form (``"token:<name>"``, ``"admin:<user>"``,
``"anon"``); ``kind`` is a short tag (see :data:`KNOWN_KINDS`); and
``**detail`` is JSON-serialised into ``AuditLog.detail_json`` — keep
it small, not full request bodies.
"""
try:
async with runtime.db.session() as session:
src/beaver_gateway/security/auth.py
+36 -83
View File
@@ -1,32 +1,7 @@
"""Bearer-token verification (Phase 4.2 — DB-backed with in-memory cache).
"""Bearer-token verification: a DB-backed store with an in-memory cache.
The store is fed by two sources:
1. **DB** (``Token`` table from Phase 4.1) — the primary source. Rows
carry Argon2id hashes; the admin UI (Phase 4.3) will be the only
writer at steady state.
2. **`BOOTSTRAP_TOKENS`** env — a name→plaintext map kept around for
first-run, disaster-recovery, and ``examples/`` smoke tests. These
entries live alongside DB rows in the cache and are never persisted.
Hot path is in-memory: at :meth:`start` we pull every non-revoked DB
row and stash it in a list; subsequent :meth:`verify` calls re-load
when the cache is older than ``ttl_seconds``. ``last_used_at`` updates
are coalesced into a small dict and flushed by a background task every
``flush_interval`` seconds — one transaction per flush rather than one
per request.
We can't index DB rows by a derived plaintext key because Argon2 salts
are random — so verify does a linear scan over candidates, calling
``argon2.PasswordHasher.verify`` on each. N is small by design (single
operator, ~10 tokens at most); the cost is irrelevant. The scan runs
through ``asyncio.to_thread`` to keep the event loop free of the ~50ms
KDF block.
The module knows nothing about HTTP frameworks. It takes a raw token
(or a verbatim ``Authorization`` header value) and returns a
:class:`TokenIdentity` (name + scope + db-id), or ``None`` for a miss.
Frontends own the 401 response shape.
Optional ``BOOTSTRAP_TOKENS`` env entries are checked first. Verify takes
a raw token or ``Authorization`` header and returns a :class:`TokenIdentity`.
"""
from __future__ import annotations
@@ -61,7 +36,7 @@ class TokenStoreError(ValueError):
VALID_SCOPES: frozenset[str] = frozenset({"*", "messages", "mcp", "admin", "api"})
"""The scopes a ``Token.scope`` may hold (Phase 4.3 admin UI enforces).
"""The scopes a ``Token.scope`` may hold.
* ``*`` — wildcard, may use any frontend
* ``messages`` — Anthropic Messages frontend only
@@ -69,7 +44,7 @@ VALID_SCOPES: frozenset[str] = frozenset({"*", "messages", "mcp", "admin", "api"
* ``api`` — conversations API (``/api``) and its SSE
* ``admin`` — reserved for programmatic admin access; the AdminFrontend
itself authenticates via session cookies, not bearer tokens, so this
scope is unused today and kept for forward compatibility.
scope is unused today.
"""
@@ -108,33 +83,23 @@ class _CachedToken:
hashed_value: str
# Default Argon2id parameters from ``argon2-cffi`` are fine for our scope.
# They target ~50ms on a modern CPU — enough to make a stolen-hash brute
# force expensive, cheap enough to verify a handful per request.
_HASHER = PasswordHasher()
def hash_token(plaintext: str) -> str:
"""Return an Argon2id hash for ``plaintext`` (admin / seed-only path).
Phase 4.3 will call this when the admin creates a token; Phase 4.2
exposes it so smoke scripts can seed the DB without re-implementing
the same line.
"""
"""Return an Argon2id hash for ``plaintext``, for admin/seed use."""
return _HASHER.hash(plaintext)
class TokenStore:
"""DB-backed verifier with in-memory cache + TTL + batched touches.
"""DB-backed verifier with an in-memory cache, TTL, and batched touches.
Construct in ``cli.main`` after :class:`Database` is up, then
``await store.start()`` to prime the cache and spin up the flusher
task. ``await store.stop()`` on shutdown drains the touch queue.
Call ``await store.start()`` after construction to prime the cache
and start the flusher; ``await store.stop()`` drains the touch queue.
Bootstrap entries (from ``BOOTSTRAP_TOKENS``) sit alongside DB rows
in the same lookup path; we check them first, in constant time, so
they remain usable even if the DB is unreachable. They never appear
in ``last_used_at`` flushes because they have no DB row.
Bootstrap entries (``BOOTSTRAP_TOKENS``) are checked first and stay
usable even if the DB is unreachable; they never get flushed to
``last_used_at`` since they have no DB row.
"""
__slots__ = (
@@ -159,8 +124,11 @@ class TokenStore:
ttl_seconds: float = 30.0,
flush_interval: float = 5.0,
) -> None:
# Bootstrap is keyed by value internally so verify is O(1) over
# plaintext. Each value also keeps its name for audit lines.
"""Index bootstrap tokens for O(1) verify lookup.
Each one's scope defaults to ``"*"`` unless narrowed via
``name:value:scope``.
"""
by_value: dict[str, str] = {}
for name, value in (bootstrap or {}).items():
if not name or not value:
@@ -174,9 +142,6 @@ class TokenStore:
raise TokenStoreError(msg)
by_value[value] = name
self._bootstrap_by_value: dict[str, str] = by_value
# Bootstrap entries are ``"*"`` unless the env narrows them
# (``name:value:scope``) - a webhook token in a URL should not be
# an admin token.
self._bootstrap_scopes: dict[str, str] = {
name: (bootstrap_scopes or {}).get(name, _BOOTSTRAP_SCOPE)
for name in by_value.values()
@@ -192,8 +157,6 @@ class TokenStore:
self._touch_queue: dict[int, datetime] = {}
self._flusher_task: asyncio.Task[None] | None = None
# ---- bootstrap parsing (kept for `cli` / tests) ---------------------
@staticmethod
def parse_bootstrap(raw: str) -> dict[str, str]:
"""Parse ``name1:value1,name2:value2[:scope]`` (``BOOTSTRAP_TOKENS``)."""
@@ -210,15 +173,9 @@ class TokenStore:
@classmethod
def from_env(cls, raw: str, db: Database | None = None) -> TokenStore:
"""Legacy entrypoint: bootstrap-only (or bootstrap + db).
Phase 1.3 call sites still expect a one-liner; we keep the
classmethod so they don't have to learn the new constructor.
"""
"""Bootstrap-only (or bootstrap + db) convenience constructor."""
return cls(db, bootstrap=cls.parse_bootstrap(raw))
# ---- lifecycle ------------------------------------------------------
async def start(self) -> None:
"""Prime the cache and (if a DB is attached) start the flusher loop."""
await self._refresh()
@@ -242,19 +199,19 @@ class TokenStore:
self._bootstrap_scopes[name] = scope
async def invalidate(self) -> None:
"""Force the next verify to re-read from DB (Phase 4.3 admin hook)."""
"""Force the next verify to re-read from DB."""
self._loaded_at = 0.0
# ---- verify path ----------------------------------------------------
async def verify(self, token: str | None) -> TokenIdentity | None:
"""Return the matching identity, or ``None`` for unknown/empty tokens."""
"""Return the matching identity, or ``None`` for unknown/empty tokens.
Bootstrap entries are checked first via constant-time compare;
the DB cache list is swapped rather than mutated on refresh, so
a concurrent reload can't affect an in-progress scan.
"""
if not token:
return None
# Bootstrap first: constant-time compare per entry, never hits DB.
# `compare_digest` is overkill for a name→value lookup but cheap
# and removes one timing variable for free.
for value, name in self._bootstrap_by_value.items():
if hmac.compare_digest(token, value):
return TokenIdentity(
@@ -268,9 +225,6 @@ class TokenStore:
await self._ensure_fresh()
# Snapshot the cache reference so a refresh mid-scan doesn't
# surprise us. List itself is immutable per refresh (we swap,
# not mutate).
cache = self._cache
for entry in cache:
try:
@@ -306,17 +260,18 @@ class TokenStore:
def __bool__(self) -> bool:
return bool(self._cache) or bool(self._bootstrap_by_value)
# ---- internals ------------------------------------------------------
async def _ensure_fresh(self) -> None:
"""Reload the cache when stale.
Double-checked under the lock so concurrent callers don't all
trigger a reload.
"""
if self._db is None:
return
now = time.monotonic()
if now - self._loaded_at <= self._ttl:
return
async with self._lock:
# Re-check under the lock — first arrival reloaded, others
# should fall through.
now = time.monotonic()
if now - self._loaded_at <= self._ttl:
return
@@ -331,8 +286,6 @@ class TokenStore:
next_cache: list[_CachedToken] = []
for row in rows:
if row.id is None:
# Defensive: SQLModel will assign an id on insert; a
# None here would mean someone handed us an unsaved row.
continue
next_cache.append(
_CachedToken(
@@ -353,20 +306,20 @@ class TokenStore:
await self._flush_now()
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001 — never let the flusher die silently
except Exception: # noqa: BLE001
_log.exception("token flusher crashed; touches will stop")
async def _flush_now(self) -> None:
"""Flush queued touches in one transaction.
Detaches the queue first so concurrent ``verify()`` writes land
in the next flush.
"""
if self._db is None or not self._touch_queue:
return
# Detach the queue so concurrent verify() writes don't bleed
# into the in-flight transaction.
pending, self._touch_queue = self._touch_queue, {}
async with self._db.session() as session:
for token_id in pending:
# We don't pass the timestamp through — `touch_token`
# stamps `now` itself, and we'd rather have one source
# of truth than reconcile clocks.
await touch_token(session, token_id=token_id)
_log.debug("flushed %d token touch(es)", len(pending))
src/beaver_gateway/security/redact.py
+16 -67
View File
@@ -1,33 +1,7 @@
"""Recognise credentials in free text and mask them.
``docker logs`` is not a vault: the stack's stdout lands in an unrotated
json file on the host, and the model itself can page through it (the
komodo tool has ``logs`` / ``search_logs``). Nobody has to write a
careless log call for a secret to end up there — two habits do it on
their own:
* ``httpx`` logs ``HTTP Request: POST <full url>`` at ``INFO``, so every
request to an upstream that keeps its credential *in* the URL prints
the credential once per call;
* a transport error carries that same URL through the traceback, which
``_log.exception`` writes out in full.
Two of our upstreams are exactly that shape: a Google Calendar
``.../private-<token>/basic.ics`` feed and a USOS ``?key=<token>`` ical
feed, both handed to the calendar MCP as query parameters.
So: silence the HTTP clients' per-request chatter (nothing here reads
it), and run one redaction pass over every formatted record as a second
line of defence.
The same pass guards the wider channel — what an MCP tool hands back to
the model (see :mod:`beaver_gateway.mcp.redacting`). :func:`redact`
recognises three things: literal values of the secret-looking
environment variables this process was started with, credentials that
live in URLs and headers, and ``NAME: value`` assignments whose *name*
says the value is a credential. The last one is what covers a config
dump of a stack whose secrets this process never held — and it keeps
the name, masking only the value, so the dump still says what is set.
Used to filter log output and to redact what MCP tools hand back to
the model (see :mod:`beaver_gateway.mcp.redacting`).
"""
from __future__ import annotations
@@ -43,39 +17,29 @@ if TYPE_CHECKING:
MASK = "<…>"
# Loggers that print request URLs at INFO. We keep their warnings.
CHATTY: tuple[str, ...] = ("httpx", "httpcore", "aiohttp.client", "urllib3")
"""Loggers that print request URLs at ``INFO``; only their warnings are kept."""
# Loggers that bring their own handlers (uvicorn re-runs ``dictConfig``
# when a server starts), so wrapping the root formatter misses them.
# ``dictConfig`` drops a logger's handlers but keeps its filters.
OWN_HANDLERS: tuple[str, ...] = ("uvicorn", "uvicorn.access", "uvicorn.error")
"""Loggers whose own handlers survive ``dictConfig``, unreached by the
root formatter."""
# Names whose *value* is a credential — matched as a substring, so
# ``FIREFLY_PAT`` and ``T3_MAC_TOKEN`` both qualify. Used twice: to pick
# which environment variables contribute literal values, and to decide
# whether a ``NAME: value`` line in a config dump should keep its value.
# ``MCP`` is in here because every MCP URL in this stack carries its
# credential in the query string.
_SECRET_NAME = re.compile(
r"TOKEN|SECRET|KEY|PASSWORD|PASS\b|PAT\b|BEARER|CREDENTIAL"
r"|MCPS?\b|AUTH|PRIVATE|SESSION|DSN",
re.IGNORECASE,
)
"""Substring match for names whose *value* is a credential."""
_MIN_SECRET = 8
# A query parameter may sit behind a plain ``?``/``&`` or behind their
# percent-encoded twins when a whole URL is nested in another one.
_LEFT = r"(?:(?<![A-Za-z0-9])|(?<=%26)|(?<=%3F))"
# ...and runs until the next separator, encoded ``&`` included.
_VALUE = r"(?:(?!%26)[^&\s\"'<>,;)\]}])+"
"""Query-parameter boundaries: a value may follow a plain or
percent-encoded ``?``/``&`` and runs to the next separator."""
# Names worth masking where they sit in a dump next to their password,
# but not worth hunting for as literals across every log line: a
# username is short and ordinary, and ``CLAUDE_RUNNER_USER=beaver-runner``
# turned into a global literal would mask half the entrypoint script.
_SECRET_IN_DUMP = re.compile(r"USER\b|LOGIN\b", re.IGNORECASE)
"""Masked only when paired with a password in a dump, not hunted as a global literal."""
def is_secret_name(name: str) -> bool:
@@ -102,13 +66,10 @@ def _mask_named_value(match: re.Match[str]) -> str:
type _Repl = str | Callable[[re.Match[str]], str]
_RULES: tuple[tuple[re.Pattern[str], _Repl], ...] = (
# Google Calendar's secret path segment; the ``/basic.ics`` after it
# survives, encoded or not, so the line still says what it fetched.
(
re.compile(r"(private-)(?:(?!%2F)[A-Za-z0-9_%-]){8,}", re.IGNORECASE),
r"\1" + MASK,
),
# ``key=…``, ``api_key=…``, ``token=…``, … in a query string.
(
re.compile(
_LEFT
@@ -120,22 +81,13 @@ _RULES: tuple[tuple[re.Pattern[str], _Repl], ...] = (
),
r"\1\2" + MASK,
),
# ``Authorization: Bearer …`` in a header dump; scheme goes too.
(
re.compile(
r"(authorization[\"']?\s*[:=]\s*[\"']?)(?:\S+\s+)?\S+", re.IGNORECASE
),
r"\1" + MASK,
),
# A bare ``Bearer <token>`` anywhere else.
(re.compile(r"\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}", re.IGNORECASE), r"\1 " + MASK),
# ``NAME: value`` / ``NAME=value`` at the start of a line — the shape
# of a compose ``environment:`` block, an ``.env`` file, a printed
# settings object. Anchored to the line so that an ``actor=token:foo``
# in the middle of a log line doesn't swallow the rest of it, and
# ``=`` must be tight (``NAME=value``, the way env files write it) so
# that ``SESSION_SECRET = os.environ[…]`` in source the dispatcher is
# reading through t3code stays readable.
(
re.compile(
r"""(?m)^(?P<lead>[ \t]*(?:-[ \t]+)?["']?)
@@ -146,9 +98,6 @@ _RULES: tuple[tuple[re.Pattern[str], _Repl], ...] = (
),
_mask_named_value,
),
# The same at the start of a line in lower case, but only for names
# that are unambiguously a credential — ``_SECRET_NAME`` is too broad
# here, it would read the ``mcp:`` prefix of a log line as one.
(
re.compile(
r"""(?m)^(?P<lead>[ \t]*(?:-[ \t]+)?["']?)
@@ -160,12 +109,7 @@ _RULES: tuple[tuple[re.Pattern[str], _Repl], ...] = (
),
_mask_named_value,
),
# ``scheme://user:password@host`` — the credential a connection
# string carries. Host and database stay, so the line still locates
# the service it failed to reach.
(re.compile(r"(://[^/\s:@]+:)[^@\s/]+(@)"), r"\1" + MASK + r"\2"),
# ``"name": "value"`` anywhere — the same idea for one-line JSON,
# where the quotes make the end of the value unambiguous.
(
re.compile(
r'(?P<lead>")(?P<name>[^"\n]{1,64})(?P<sep>"\s*:\s*")'
@@ -174,6 +118,10 @@ _RULES: tuple[tuple[re.Pattern[str], _Repl], ...] = (
_mask_named_value,
),
)
"""Redaction rules applied in order: calendar tokens, query-string secrets,
``Authorization`` headers, bare bearer tokens, ``NAME: value``/``NAME=value``
lines (upper- then lower-case names), connection-string passwords, and
one-line JSON ``"name": "value"`` pairs."""
_ENV_SECRETS: list[str] = []
"""Literal secret values to mask, longest first. Filled by :func:`install`."""
@@ -255,9 +203,10 @@ class RedactFilter(logging.Filter):
"""
def filter(self, record: logging.LogRecord) -> bool:
"""Let a record through unmasked if its template is malformed."""
try:
message = record.getMessage()
except (TypeError, ValueError): # a broken template is not ours to fix
except (TypeError, ValueError):
return True
masked = redact(message)
if masked != message: