refactor: split flat core into capability packages, layer the conversations service, English defaults for every model-facing text
This commit is contained in:
hh committed
1 parent
253bde1b11
commit
9aaddbed75
77 files changed
+2987
-2944
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,94 @@
|
||||
"""Single entry point for writing :class:`AuditLog` rows.
|
||||
|
||||
Every frontend ends up needing the same three-line pattern — open a DB
|
||||
session, append a row, swallow failures so the user-visible request
|
||||
still succeeds. Phase 4.3 inlined that pattern in the admin frontend
|
||||
under a private ``_audit()`` helper; Phase 4.4 lifts it here so the
|
||||
Messages and MCP frontends can call the same function and so the
|
||||
swallow-and-log policy lives in one place.
|
||||
|
||||
The contract:
|
||||
|
||||
* ``log(runtime, actor=..., kind=...)`` is fire-and-forget. It awaits
|
||||
the DB write (so callers can ``await`` it before responding and get
|
||||
ordering), but never raises — if the audit insert fails, the function
|
||||
emits an ``exception`` log line and returns.
|
||||
* ``actor`` is a free-form string. By convention: ``"token:<name>"``
|
||||
for bearer-authenticated traffic, ``"admin:<user>"`` for admin-UI
|
||||
actions, ``"anon"`` for failed-auth paths we still want to record.
|
||||
* ``kind`` is a short tag — see :data:`KNOWN_KINDS` for the set the
|
||||
current frontends emit; new tags don't need a code change here, the
|
||||
column is free-form.
|
||||
* ``**detail`` is JSON-serialised by :func:`append_audit`. Keep it
|
||||
small: paths, methods, status codes — not request bodies. Anything
|
||||
passed here lands in ``AuditLog.detail_json`` verbatim.
|
||||
|
||||
Why a thin wrapper rather than ``append_audit`` directly: callers want
|
||||
"write if you can, otherwise carry on", and pulling the try/except into
|
||||
every frontend was already starting to drift (admin had it, bearer
|
||||
frontends would have copy-pasted). One module, one policy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from beaver_gateway.storage import append_audit
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from beaver_gateway.frontends.base import GatewayRuntime
|
||||
|
||||
|
||||
_log = logging.getLogger("beaver_gateway.audit")
|
||||
|
||||
|
||||
# Tags currently emitted by the gateway. The set is informational —
|
||||
# ``AuditLog.kind`` is free-form so new code can introduce new tags
|
||||
# without touching this list — but listing them here gives the admin UI
|
||||
# and any downstream log consumers one canonical reference.
|
||||
KNOWN_KINDS: frozenset[str] = frozenset(
|
||||
{
|
||||
"messages", # POST /v1/messages accepted
|
||||
"mcp_call", # /mcp/<ns>/... proxied
|
||||
"tool_call", # a model's tool call seen by the PreToolUse hook
|
||||
"login_ok",
|
||||
"login_failed",
|
||||
"logout",
|
||||
"token_create",
|
||||
"token_revoke",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def log(
|
||||
runtime: GatewayRuntime,
|
||||
*,
|
||||
actor: str,
|
||||
kind: str,
|
||||
agent_name: str | None = None,
|
||||
**detail: Any,
|
||||
) -> None:
|
||||
"""Best-effort audit insert. Never raises.
|
||||
|
||||
Opens its own short-lived :class:`AsyncSession` so callers don't
|
||||
have to thread one through. If the DB hiccups (table missing,
|
||||
disk full, connection drop), we log and move on — the audit trail
|
||||
is observability, not a hard precondition for serving the request.
|
||||
"""
|
||||
try:
|
||||
async with runtime.db.session() as session:
|
||||
await append_audit(
|
||||
session,
|
||||
actor=actor,
|
||||
kind=kind,
|
||||
agent_name=agent_name,
|
||||
detail=detail or None,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
_log.exception(
|
||||
"audit write failed: actor=%s kind=%s agent=%s", actor, kind, agent_name
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["KNOWN_KINDS", "log"]
|
||||
@@ -0,0 +1,399 @@
|
||||
"""Bearer-token verification (Phase 4.2 — DB-backed with in-memory cache).
|
||||
|
||||
The store is fed by two sources:
|
||||
|
||||
1. **DB** (``Token`` table from Phase 4.1) — the primary source. Rows
|
||||
carry Argon2id hashes; the admin UI (Phase 4.3) will be the only
|
||||
writer at steady state.
|
||||
2. **`BOOTSTRAP_TOKENS`** env — a name→plaintext map kept around for
|
||||
first-run, disaster-recovery, and ``examples/`` smoke tests. These
|
||||
entries live alongside DB rows in the cache and are never persisted.
|
||||
|
||||
Hot path is in-memory: at :meth:`start` we pull every non-revoked DB
|
||||
row and stash it in a list; subsequent :meth:`verify` calls re-load
|
||||
when the cache is older than ``ttl_seconds``. ``last_used_at`` updates
|
||||
are coalesced into a small dict and flushed by a background task every
|
||||
``flush_interval`` seconds — one transaction per flush rather than one
|
||||
per request.
|
||||
|
||||
We can't index DB rows by a derived plaintext key because Argon2 salts
|
||||
are random — so verify does a linear scan over candidates, calling
|
||||
``argon2.PasswordHasher.verify`` on each. N is small by design (single
|
||||
operator, ~10 tokens at most); the cost is irrelevant. The scan runs
|
||||
through ``asyncio.to_thread`` to keep the event loop free of the ~50ms
|
||||
KDF block.
|
||||
|
||||
The module knows nothing about HTTP frameworks. It takes a raw token
|
||||
(or a verbatim ``Authorization`` header value) and returns a
|
||||
:class:`TokenIdentity` (name + scope + db-id), or ``None`` for a miss.
|
||||
Frontends own the 401 response shape.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import hmac
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerifyMismatchError
|
||||
|
||||
from beaver_gateway.storage import list_active_tokens, touch_token
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping
|
||||
|
||||
from beaver_gateway.storage import Database
|
||||
|
||||
|
||||
_log = logging.getLogger("beaver_gateway.auth")
|
||||
|
||||
_BOOTSTRAP_SCOPE = "*"
|
||||
|
||||
|
||||
class TokenStoreError(ValueError):
|
||||
"""Malformed ``BOOTSTRAP_TOKENS`` value or duplicate token."""
|
||||
|
||||
|
||||
VALID_SCOPES: frozenset[str] = frozenset({"*", "messages", "mcp", "admin", "api"})
|
||||
"""The scopes a ``Token.scope`` may hold (Phase 4.3 admin UI enforces).
|
||||
|
||||
* ``*`` — wildcard, may use any frontend
|
||||
* ``messages`` — Anthropic Messages frontend only
|
||||
* ``mcp`` — MCP server frontend only
|
||||
* ``api`` — conversations API (``/api``) and its SSE
|
||||
* ``admin`` — reserved for programmatic admin access; the AdminFrontend
|
||||
itself authenticates via session cookies, not bearer tokens, so this
|
||||
scope is unused today and kept for forward compatibility.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TokenIdentity:
|
||||
"""What :meth:`TokenStore.verify` resolves to on success.
|
||||
|
||||
``token_id`` is the DB row id for persisted tokens, or ``None`` for
|
||||
an env-bootstrap match (those have no DB row to touch). ``scope``
|
||||
gates which frontend the token may hit (see :data:`VALID_SCOPES`);
|
||||
bootstrap tokens implicitly get ``"*"``.
|
||||
"""
|
||||
|
||||
name: str
|
||||
scope: str
|
||||
token_id: int | None
|
||||
|
||||
def allows(self, required: str) -> bool:
|
||||
"""``True`` when this identity may access a route gated by ``required``.
|
||||
|
||||
``"*"`` is the wildcard; an exact match satisfies a single scope.
|
||||
Unknown ``required`` values intentionally fall through to a
|
||||
strict equality check — callers should pass one of
|
||||
:data:`VALID_SCOPES`.
|
||||
"""
|
||||
return self.scope in ("*", required)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _CachedToken:
|
||||
"""One non-revoked row, copied out of the DB into the hot-path cache."""
|
||||
|
||||
id: int
|
||||
name: str
|
||||
scope: str
|
||||
hashed_value: str
|
||||
|
||||
|
||||
# Default Argon2id parameters from ``argon2-cffi`` are fine for our scope.
|
||||
# They target ~50ms on a modern CPU — enough to make a stolen-hash brute
|
||||
# force expensive, cheap enough to verify a handful per request.
|
||||
_HASHER = PasswordHasher()
|
||||
|
||||
|
||||
def hash_token(plaintext: str) -> str:
|
||||
"""Return an Argon2id hash for ``plaintext`` (admin / seed-only path).
|
||||
|
||||
Phase 4.3 will call this when the admin creates a token; Phase 4.2
|
||||
exposes it so smoke scripts can seed the DB without re-implementing
|
||||
the same line.
|
||||
"""
|
||||
return _HASHER.hash(plaintext)
|
||||
|
||||
|
||||
class TokenStore:
|
||||
"""DB-backed verifier with in-memory cache + TTL + batched touches.
|
||||
|
||||
Construct in ``cli.main`` after :class:`Database` is up, then
|
||||
``await store.start()`` to prime the cache and spin up the flusher
|
||||
task. ``await store.stop()`` on shutdown drains the touch queue.
|
||||
|
||||
Bootstrap entries (from ``BOOTSTRAP_TOKENS``) sit alongside DB rows
|
||||
in the same lookup path; we check them first, in constant time, so
|
||||
they remain usable even if the DB is unreachable. They never appear
|
||||
in ``last_used_at`` flushes because they have no DB row.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_bootstrap_by_value",
|
||||
"_bootstrap_scopes",
|
||||
"_cache",
|
||||
"_db",
|
||||
"_flush_interval",
|
||||
"_flusher_task",
|
||||
"_loaded_at",
|
||||
"_lock",
|
||||
"_touch_queue",
|
||||
"_ttl",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db: Database | None = None,
|
||||
*,
|
||||
bootstrap: Mapping[str, str] | None = None,
|
||||
bootstrap_scopes: Mapping[str, str] | None = None,
|
||||
ttl_seconds: float = 30.0,
|
||||
flush_interval: float = 5.0,
|
||||
) -> None:
|
||||
# Bootstrap is keyed by value internally so verify is O(1) over
|
||||
# plaintext. Each value also keeps its name for audit lines.
|
||||
by_value: dict[str, str] = {}
|
||||
for name, value in (bootstrap or {}).items():
|
||||
if not name or not value:
|
||||
msg = f"empty name or value in bootstrap map (name={name!r})"
|
||||
raise TokenStoreError(msg)
|
||||
if value in by_value:
|
||||
msg = (
|
||||
f"duplicate bootstrap token value for names "
|
||||
f"{by_value[value]!r} and {name!r}"
|
||||
)
|
||||
raise TokenStoreError(msg)
|
||||
by_value[value] = name
|
||||
self._bootstrap_by_value: dict[str, str] = by_value
|
||||
# Bootstrap entries are ``"*"`` unless the env narrows them
|
||||
# (``name:value:scope``) - a webhook token in a URL should not be
|
||||
# an admin token.
|
||||
self._bootstrap_scopes: dict[str, str] = {
|
||||
name: (bootstrap_scopes or {}).get(name, _BOOTSTRAP_SCOPE)
|
||||
for name in by_value.values()
|
||||
}
|
||||
|
||||
self._db = db
|
||||
self._ttl = ttl_seconds
|
||||
self._flush_interval = flush_interval
|
||||
|
||||
self._cache: list[_CachedToken] = []
|
||||
self._loaded_at: float = 0.0
|
||||
self._lock = asyncio.Lock()
|
||||
self._touch_queue: dict[int, datetime] = {}
|
||||
self._flusher_task: asyncio.Task[None] | None = None
|
||||
|
||||
# ---- bootstrap parsing (kept for `cli` / tests) ---------------------
|
||||
|
||||
@staticmethod
|
||||
def parse_bootstrap(raw: str) -> dict[str, str]:
|
||||
"""Parse ``name1:value1,name2:value2[:scope]`` (``BOOTSTRAP_TOKENS``)."""
|
||||
return {name: value for name, (value, _scope) in _parse_entries(raw).items()}
|
||||
|
||||
@staticmethod
|
||||
def parse_bootstrap_scopes(raw: str) -> dict[str, str]:
|
||||
"""Scopes given as a third field in ``BOOTSTRAP_TOKENS``; the rest get ``*``."""
|
||||
return {
|
||||
name: scope
|
||||
for name, (_value, scope) in _parse_entries(raw).items()
|
||||
if scope is not None
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, raw: str, db: Database | None = None) -> TokenStore:
|
||||
"""Legacy entrypoint: bootstrap-only (or bootstrap + db).
|
||||
|
||||
Phase 1.3 call sites still expect a one-liner; we keep the
|
||||
classmethod so they don't have to learn the new constructor.
|
||||
"""
|
||||
return cls(db, bootstrap=cls.parse_bootstrap(raw))
|
||||
|
||||
# ---- lifecycle ------------------------------------------------------
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Prime the cache and (if a DB is attached) start the flusher loop."""
|
||||
await self._refresh()
|
||||
if self._db is not None:
|
||||
self._flusher_task = asyncio.create_task(
|
||||
self._flusher_loop(), name="beaver-gateway.token-flusher"
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Cancel the flusher and run one final drain."""
|
||||
if self._flusher_task is not None:
|
||||
self._flusher_task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await self._flusher_task
|
||||
self._flusher_task = None
|
||||
await self._flush_now()
|
||||
|
||||
def grant(self, name: str, value: str, *, scope: str = _BOOTSTRAP_SCOPE) -> None:
|
||||
"""Add an in-memory token for the process lifetime (the admin UI's bearer)."""
|
||||
self._bootstrap_by_value[value] = name
|
||||
self._bootstrap_scopes[name] = scope
|
||||
|
||||
async def invalidate(self) -> None:
|
||||
"""Force the next verify to re-read from DB (Phase 4.3 admin hook)."""
|
||||
self._loaded_at = 0.0
|
||||
|
||||
# ---- verify path ----------------------------------------------------
|
||||
|
||||
async def verify(self, token: str | None) -> TokenIdentity | None:
|
||||
"""Return the matching identity, or ``None`` for unknown/empty tokens."""
|
||||
if not token:
|
||||
return None
|
||||
|
||||
# Bootstrap first: constant-time compare per entry, never hits DB.
|
||||
# `compare_digest` is overkill for a name→value lookup but cheap
|
||||
# and removes one timing variable for free.
|
||||
for value, name in self._bootstrap_by_value.items():
|
||||
if hmac.compare_digest(token, value):
|
||||
return TokenIdentity(
|
||||
name=name,
|
||||
scope=self._bootstrap_scopes.get(name, _BOOTSTRAP_SCOPE),
|
||||
token_id=None,
|
||||
)
|
||||
|
||||
if self._db is None:
|
||||
return None
|
||||
|
||||
await self._ensure_fresh()
|
||||
|
||||
# Snapshot the cache reference so a refresh mid-scan doesn't
|
||||
# surprise us. List itself is immutable per refresh (we swap,
|
||||
# not mutate).
|
||||
cache = self._cache
|
||||
for entry in cache:
|
||||
try:
|
||||
await asyncio.to_thread(_HASHER.verify, entry.hashed_value, token)
|
||||
except VerifyMismatchError:
|
||||
continue
|
||||
except InvalidHashError:
|
||||
_log.warning(
|
||||
"token row %d has an unparseable hash — skipping", entry.id
|
||||
)
|
||||
continue
|
||||
self._touch_queue[entry.id] = datetime.now(UTC)
|
||||
return TokenIdentity(name=entry.name, scope=entry.scope, token_id=entry.id)
|
||||
return None
|
||||
|
||||
async def verify_bearer(self, authorization: str | None) -> TokenIdentity | None:
|
||||
"""Strip the ``Bearer`` prefix (case-insensitive) then verify.
|
||||
|
||||
Accepts a bare token too — Cursor's MCP transport sometimes
|
||||
passes the raw value via ``?token=`` and reuses the same
|
||||
verifier; treating an unprefixed header as a bare token keeps
|
||||
both call sites on one method.
|
||||
"""
|
||||
if not authorization:
|
||||
return None
|
||||
head, sep, rest = authorization.partition(" ")
|
||||
token = rest.strip() if sep and head.lower() == "bearer" else authorization
|
||||
return await self.verify(token)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._cache) + len(self._bootstrap_by_value)
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
return bool(self._cache) or bool(self._bootstrap_by_value)
|
||||
|
||||
# ---- internals ------------------------------------------------------
|
||||
|
||||
async def _ensure_fresh(self) -> None:
|
||||
if self._db is None:
|
||||
return
|
||||
now = time.monotonic()
|
||||
if now - self._loaded_at <= self._ttl:
|
||||
return
|
||||
async with self._lock:
|
||||
# Re-check under the lock — first arrival reloaded, others
|
||||
# should fall through.
|
||||
now = time.monotonic()
|
||||
if now - self._loaded_at <= self._ttl:
|
||||
return
|
||||
await self._refresh()
|
||||
|
||||
async def _refresh(self) -> None:
|
||||
if self._db is None:
|
||||
self._loaded_at = time.monotonic()
|
||||
return
|
||||
async with self._db.session() as session:
|
||||
rows = await list_active_tokens(session)
|
||||
next_cache: list[_CachedToken] = []
|
||||
for row in rows:
|
||||
if row.id is None:
|
||||
# Defensive: SQLModel will assign an id on insert; a
|
||||
# None here would mean someone handed us an unsaved row.
|
||||
continue
|
||||
next_cache.append(
|
||||
_CachedToken(
|
||||
id=row.id,
|
||||
name=row.name,
|
||||
scope=row.scope,
|
||||
hashed_value=row.hashed_value,
|
||||
)
|
||||
)
|
||||
self._cache = next_cache
|
||||
self._loaded_at = time.monotonic()
|
||||
_log.debug("token cache refreshed: %d active row(s)", len(next_cache))
|
||||
|
||||
async def _flusher_loop(self) -> None:
|
||||
try:
|
||||
while True:
|
||||
await asyncio.sleep(self._flush_interval)
|
||||
await self._flush_now()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 — never let the flusher die silently
|
||||
_log.exception("token flusher crashed; touches will stop")
|
||||
|
||||
async def _flush_now(self) -> None:
|
||||
if self._db is None or not self._touch_queue:
|
||||
return
|
||||
# Detach the queue so concurrent verify() writes don't bleed
|
||||
# into the in-flight transaction.
|
||||
pending, self._touch_queue = self._touch_queue, {}
|
||||
async with self._db.session() as session:
|
||||
for token_id in pending:
|
||||
# We don't pass the timestamp through — `touch_token`
|
||||
# stamps `now` itself, and we'd rather have one source
|
||||
# of truth than reconcile clocks.
|
||||
await touch_token(session, token_id=token_id)
|
||||
_log.debug("flushed %d token touch(es)", len(pending))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"VALID_SCOPES",
|
||||
"TokenIdentity",
|
||||
"TokenStore",
|
||||
"TokenStoreError",
|
||||
"hash_token",
|
||||
]
|
||||
|
||||
|
||||
def _parse_entries(raw: str) -> dict[str, tuple[str, str | None]]:
|
||||
entries: dict[str, tuple[str, str | None]] = {}
|
||||
for chunk in raw.split(","):
|
||||
entry = chunk.strip()
|
||||
if not entry:
|
||||
continue
|
||||
name, sep, rest = entry.partition(":")
|
||||
if not sep:
|
||||
msg = f"token entry missing ':' separator: {entry!r}"
|
||||
raise TokenStoreError(msg)
|
||||
value, sep, scope = rest.partition(":")
|
||||
name, value, scope = name.strip(), value.strip(), scope.strip()
|
||||
if name in entries:
|
||||
msg = f"duplicate token name: {name!r}"
|
||||
raise TokenStoreError(msg)
|
||||
entries[name] = (value, scope if sep and scope else None)
|
||||
return entries
|
||||
@@ -0,0 +1,286 @@
|
||||
"""Recognise credentials in free text and mask them.
|
||||
|
||||
``docker logs`` is not a vault: the stack's stdout lands in an unrotated
|
||||
json file on the host, and the model itself can page through it (the
|
||||
komodo tool has ``logs`` / ``search_logs``). Nobody has to write a
|
||||
careless log call for a secret to end up there — two habits do it on
|
||||
their own:
|
||||
|
||||
* ``httpx`` logs ``HTTP Request: POST <full url>`` at ``INFO``, so every
|
||||
request to an upstream that keeps its credential *in* the URL prints
|
||||
the credential once per call;
|
||||
* a transport error carries that same URL through the traceback, which
|
||||
``_log.exception`` writes out in full.
|
||||
|
||||
Two of our upstreams are exactly that shape: a Google Calendar
|
||||
``.../private-<token>/basic.ics`` feed and a USOS ``?key=<token>`` ical
|
||||
feed, both handed to the calendar MCP as query parameters.
|
||||
|
||||
So: silence the HTTP clients' per-request chatter (nothing here reads
|
||||
it), and run one redaction pass over every formatted record as a second
|
||||
line of defence.
|
||||
|
||||
The same pass guards the wider channel — what an MCP tool hands back to
|
||||
the model (see :mod:`beaver_gateway.mcp.redacting`). :func:`redact`
|
||||
recognises three things: literal values of the secret-looking
|
||||
environment variables this process was started with, credentials that
|
||||
live in URLs and headers, and ``NAME: value`` assignments whose *name*
|
||||
says the value is a credential. The last one is what covers a config
|
||||
dump of a stack whose secrets this process never held — and it keeps
|
||||
the name, masking only the value, so the dump still says what is set.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Mapping
|
||||
|
||||
MASK = "<…>"
|
||||
|
||||
# Loggers that print request URLs at INFO. We keep their warnings.
|
||||
CHATTY: tuple[str, ...] = ("httpx", "httpcore", "aiohttp.client", "urllib3")
|
||||
|
||||
# Loggers that bring their own handlers (uvicorn re-runs ``dictConfig``
|
||||
# when a server starts), so wrapping the root formatter misses them.
|
||||
# ``dictConfig`` drops a logger's handlers but keeps its filters.
|
||||
OWN_HANDLERS: tuple[str, ...] = ("uvicorn", "uvicorn.access", "uvicorn.error")
|
||||
|
||||
# Names whose *value* is a credential — matched as a substring, so
|
||||
# ``FIREFLY_PAT`` and ``T3_MAC_TOKEN`` both qualify. Used twice: to pick
|
||||
# which environment variables contribute literal values, and to decide
|
||||
# whether a ``NAME: value`` line in a config dump should keep its value.
|
||||
# ``MCP`` is in here because every MCP URL in this stack carries its
|
||||
# credential in the query string.
|
||||
_SECRET_NAME = re.compile(
|
||||
r"TOKEN|SECRET|KEY|PASSWORD|PASS\b|PAT\b|BEARER|CREDENTIAL"
|
||||
r"|MCPS?\b|AUTH|PRIVATE|SESSION|DSN",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_MIN_SECRET = 8
|
||||
|
||||
# A query parameter may sit behind a plain ``?``/``&`` or behind their
|
||||
# percent-encoded twins when a whole URL is nested in another one.
|
||||
_LEFT = r"(?:(?<![A-Za-z0-9])|(?<=%26)|(?<=%3F))"
|
||||
# ...and runs until the next separator, encoded ``&`` included.
|
||||
_VALUE = r"(?:(?!%26)[^&\s\"'<>,;)\]}])+"
|
||||
|
||||
|
||||
# Names worth masking where they sit in a dump next to their password,
|
||||
# but not worth hunting for as literals across every log line: a
|
||||
# username is short and ordinary, and ``CLAUDE_RUNNER_USER=beaver-runner``
|
||||
# turned into a global literal would mask half the entrypoint script.
|
||||
_SECRET_IN_DUMP = re.compile(r"USER\b|LOGIN\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def is_secret_name(name: str) -> bool:
|
||||
"""Whether a variable/field of this name holds a credential."""
|
||||
return bool(_SECRET_NAME.search(name) or _SECRET_IN_DUMP.search(name))
|
||||
|
||||
|
||||
def _mask_named_value(match: re.Match[str]) -> str:
|
||||
"""Mask ``value`` in a ``NAME: value`` match, keeping the name.
|
||||
|
||||
A dump is still worth reading when it says *which* variables are
|
||||
set; it is the values that must not travel. An already-masked or
|
||||
empty value is left alone so the pass stays idempotent, and ``lead``
|
||||
/ ``tail`` carry back whatever quoting the surrounding format used.
|
||||
"""
|
||||
groups = match.groupdict()
|
||||
name, value = groups["name"], groups["value"]
|
||||
if not is_secret_name(name) or value.strip(" \t\"'") in ("", MASK):
|
||||
return match[0]
|
||||
lead, tail = groups.get("lead") or "", groups.get("tail") or ""
|
||||
return f"{lead}{name}{groups['sep']}{MASK}{tail}"
|
||||
|
||||
|
||||
type _Repl = str | Callable[[re.Match[str]], str]
|
||||
|
||||
_RULES: tuple[tuple[re.Pattern[str], _Repl], ...] = (
|
||||
# Google Calendar's secret path segment; the ``/basic.ics`` after it
|
||||
# survives, encoded or not, so the line still says what it fetched.
|
||||
(
|
||||
re.compile(r"(private-)(?:(?!%2F)[A-Za-z0-9_%-]){8,}", re.IGNORECASE),
|
||||
r"\1" + MASK,
|
||||
),
|
||||
# ``key=…``, ``api_key=…``, ``token=…``, … in a query string.
|
||||
(
|
||||
re.compile(
|
||||
_LEFT
|
||||
+ r"(api[-_]?key|access[-_]?token|auth[-_]?token"
|
||||
+ r"|token|key|secret|password|passwd|pat|signature|sig)"
|
||||
+ r"(=|%3D)"
|
||||
+ _VALUE,
|
||||
re.IGNORECASE,
|
||||
),
|
||||
r"\1\2" + MASK,
|
||||
),
|
||||
# ``Authorization: Bearer …`` in a header dump; scheme goes too.
|
||||
(
|
||||
re.compile(
|
||||
r"(authorization[\"']?\s*[:=]\s*[\"']?)(?:\S+\s+)?\S+", re.IGNORECASE
|
||||
),
|
||||
r"\1" + MASK,
|
||||
),
|
||||
# A bare ``Bearer <token>`` anywhere else.
|
||||
(re.compile(r"\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}", re.IGNORECASE), r"\1 " + MASK),
|
||||
# ``NAME: value`` / ``NAME=value`` at the start of a line — the shape
|
||||
# of a compose ``environment:`` block, an ``.env`` file, a printed
|
||||
# settings object. Anchored to the line so that an ``actor=token:foo``
|
||||
# in the middle of a log line doesn't swallow the rest of it, and
|
||||
# ``=`` must be tight (``NAME=value``, the way env files write it) so
|
||||
# that ``SESSION_SECRET = os.environ[…]`` in source the dispatcher is
|
||||
# reading through t3code stays readable.
|
||||
(
|
||||
re.compile(
|
||||
r"""(?m)^(?P<lead>[ \t]*(?:-[ \t]+)?["']?)
|
||||
(?P<name>[A-Z][A-Z0-9_]{2,})
|
||||
(?P<sep>["']?(?::[ \t]*|=))
|
||||
(?P<value>"[^"\n]*"|'[^'\n]*'|[^\n]*)""",
|
||||
re.VERBOSE,
|
||||
),
|
||||
_mask_named_value,
|
||||
),
|
||||
# The same at the start of a line in lower case, but only for names
|
||||
# that are unambiguously a credential — ``_SECRET_NAME`` is too broad
|
||||
# here, it would read the ``mcp:`` prefix of a log line as one.
|
||||
(
|
||||
re.compile(
|
||||
r"""(?m)^(?P<lead>[ \t]*(?:-[ \t]+)?["']?)
|
||||
(?P<name>password|passwd|secret|token|api[-_]?key|apikey
|
||||
|access[-_]?token|auth[-_]?token|private[-_]?key|credentials?)
|
||||
(?P<sep>["']?(?::[ \t]*|=))
|
||||
(?P<value>"[^"\n]*"|'[^'\n]*'|[^\n]*)""",
|
||||
re.VERBOSE | re.IGNORECASE,
|
||||
),
|
||||
_mask_named_value,
|
||||
),
|
||||
# ``scheme://user:password@host`` — the credential a connection
|
||||
# string carries. Host and database stay, so the line still locates
|
||||
# the service it failed to reach.
|
||||
(re.compile(r"(://[^/\s:@]+:)[^@\s/]+(@)"), r"\1" + MASK + r"\2"),
|
||||
# ``"name": "value"`` anywhere — the same idea for one-line JSON,
|
||||
# where the quotes make the end of the value unambiguous.
|
||||
(
|
||||
re.compile(
|
||||
r'(?P<lead>")(?P<name>[^"\n]{1,64})(?P<sep>"\s*:\s*")'
|
||||
r'(?P<value>[^"\n]*)(?P<tail>")'
|
||||
),
|
||||
_mask_named_value,
|
||||
),
|
||||
)
|
||||
|
||||
_ENV_SECRETS: list[str] = []
|
||||
"""Literal secret values to mask, longest first. Filled by :func:`install`."""
|
||||
|
||||
|
||||
def env_secrets(env: Mapping[str, str]) -> list[str]:
|
||||
"""Literal values worth masking, longest first.
|
||||
|
||||
Each secret-looking variable contributes its value plus the
|
||||
percent-encoded and percent-decoded spellings of it, because a
|
||||
credential that travels inside another URL is logged encoded.
|
||||
"""
|
||||
found: set[str] = set()
|
||||
for name, value in env.items():
|
||||
if not _SECRET_NAME.search(name):
|
||||
continue
|
||||
for form in (value, quote(value, safe=""), unquote(value)):
|
||||
if len(form.strip()) >= _MIN_SECRET:
|
||||
found.add(form.strip())
|
||||
ordered = list(found)
|
||||
ordered.sort(key=len, reverse=True)
|
||||
return ordered
|
||||
|
||||
|
||||
def redact(text: str) -> str:
|
||||
"""Mask every credential we can recognise in ``text``."""
|
||||
for value in _ENV_SECRETS:
|
||||
if value in text:
|
||||
text = text.replace(value, MASK)
|
||||
for pattern, repl in _RULES:
|
||||
text = pattern.sub(repl, text)
|
||||
return text
|
||||
|
||||
|
||||
def redact_data(value: object) -> object:
|
||||
"""Mask every string inside a decoded JSON tree.
|
||||
|
||||
A field name is the only context a bare value has: ``"0d1e…"`` on
|
||||
its own looks like nothing, ``{"ADMIN_PASS": "0d1e…"}`` is a
|
||||
password. So a secret-looking key masks its whole subtree — the
|
||||
structured twin of the ``NAME: value`` rule above.
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
return redact(value)
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: MASK if is_secret_name(str(key)) and item else redact_data(item)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [redact_data(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
class RedactingFormatter(logging.Formatter):
|
||||
"""Wrap another formatter and redact whatever it produced.
|
||||
|
||||
Formatting first is the point: it catches the message, its ``%``
|
||||
arguments and the traceback of an ``exception()`` call in one pass,
|
||||
which a record-level filter cannot do.
|
||||
"""
|
||||
|
||||
def __init__(self, inner: logging.Formatter | None = None) -> None:
|
||||
super().__init__()
|
||||
self._inner = inner if inner is not None else logging.Formatter()
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
return redact(self._inner.format(record))
|
||||
|
||||
|
||||
class RedactFilter(logging.Filter):
|
||||
"""Redact at record level, for loggers whose handlers aren't ours.
|
||||
|
||||
Interpolates first and replaces the record with the result, because
|
||||
a secret is often split across the template and its arguments
|
||||
(``"key=%s", token``) and neither half looks like a credential on
|
||||
its own. The arguments are dropped only when something was actually
|
||||
masked, so untouched records stay structured.
|
||||
"""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
try:
|
||||
message = record.getMessage()
|
||||
except (TypeError, ValueError): # a broken template is not ours to fix
|
||||
return True
|
||||
masked = redact(message)
|
||||
if masked != message:
|
||||
record.msg = masked
|
||||
record.args = ()
|
||||
return True
|
||||
|
||||
|
||||
def load_secrets(env: Mapping[str, str] | None = None) -> None:
|
||||
"""(Re)read the literal values to mask.
|
||||
|
||||
Called once at startup and again after ``.env`` is loaded, because
|
||||
in dev the process environment only fills up at that point.
|
||||
"""
|
||||
_ENV_SECRETS[:] = env_secrets(os.environ if env is None else env)
|
||||
|
||||
|
||||
def install(env: Mapping[str, str] | None = None) -> None:
|
||||
"""Wire redaction into the root logger. Call right after ``basicConfig``."""
|
||||
load_secrets(env)
|
||||
for handler in logging.getLogger().handlers:
|
||||
handler.setFormatter(RedactingFormatter(handler.formatter))
|
||||
for name in CHATTY:
|
||||
logging.getLogger(name).setLevel(logging.WARNING)
|
||||
for name in OWN_HANDLERS:
|
||||
logging.getLogger(name).addFilter(RedactFilter())
|
||||
Reference in new issue
Block a user