feat(keystore,api,ui,docker): encrypted secrets store with a write-only admin page and a secret run helper for the model
This commit is contained in:
1 parent
2a47c6771d
commit
7b36cb2156
22 files changed
+1569
-5
No files matched your search
@@ -33,6 +33,8 @@ from beaver_gateway.frontends.base import GatewayRuntime
|
||||
from beaver_gateway.frontends.bearer import require_token
|
||||
from beaver_gateway.frontends.root import build_root_app
|
||||
from beaver_gateway.jobs.scheduler import Scheduler
|
||||
from beaver_gateway.keystore import SecretBox, SecretStore
|
||||
from beaver_gateway.keystore import serve as serve_secrets
|
||||
from beaver_gateway.mcp.internal_app import build_internal_app
|
||||
from beaver_gateway.security.auth import TokenStore, scopes_with
|
||||
from beaver_gateway.storage import (
|
||||
@@ -44,7 +46,7 @@ from beaver_gateway.storage import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Iterator
|
||||
from collections.abc import Iterable, Iterator, Sequence
|
||||
|
||||
from claude_agent_sdk import McpSdkServerConfig
|
||||
from fastmcp import FastMCP
|
||||
@@ -120,6 +122,7 @@ async def run(gateway: Gateway, settings: Settings) -> None:
|
||||
mcps = McpRegistry(gateway.mcps)
|
||||
db = Database(settings.database_url)
|
||||
await db.create_all()
|
||||
secrets = SecretStore(db, _secret_box(settings))
|
||||
token_store = TokenStore(
|
||||
db,
|
||||
bootstrap=TokenStore.parse_bootstrap(settings.bootstrap_tokens),
|
||||
@@ -197,6 +200,7 @@ async def run(gateway: Gateway, settings: Settings) -> None:
|
||||
public_url=gateway.public_url.rstrip("/") if gateway.public_url else None,
|
||||
scopes=scopes_with(fe.scope for fe in gateway.frontends),
|
||||
send_file_roots=_roots(settings.send_file_roots),
|
||||
secrets=secrets,
|
||||
)
|
||||
for fe in gateway.frontends:
|
||||
fe.configure(runtime)
|
||||
@@ -224,10 +228,43 @@ async def run(gateway: Gateway, settings: Settings) -> None:
|
||||
tg.create_task(_serve_root(gateway, extra={"/hooks": hooks}))
|
||||
if gateway.watch is not None:
|
||||
tg.create_task(gateway.watch.run())
|
||||
if secrets.enabled:
|
||||
tg.create_task(_serve_secrets(secrets, settings=settings, db=db))
|
||||
for fe in gateway.frontends:
|
||||
tg.create_task(fe.serve())
|
||||
|
||||
|
||||
def _secret_box(settings: Settings) -> SecretBox | None:
|
||||
if not settings.secrets_key:
|
||||
return None
|
||||
try:
|
||||
return SecretBox.from_hex(settings.secrets_key)
|
||||
except ValueError as exc:
|
||||
_log.error("secrets are off: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
async def _serve_secrets(
|
||||
store: SecretStore, *, settings: Settings, db: Database
|
||||
) -> None:
|
||||
async def used(names: Sequence[str]) -> None:
|
||||
try:
|
||||
async with db.session() as session:
|
||||
await append_audit(
|
||||
session,
|
||||
actor="runner",
|
||||
kind="secret_use",
|
||||
detail={"names": list(names)},
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
_log.exception("secret audit write failed")
|
||||
|
||||
try:
|
||||
await serve_secrets(store, settings.secrets_socket, on_use=used)
|
||||
except OSError:
|
||||
_log.exception("secrets socket %s did not open", settings.secrets_socket)
|
||||
|
||||
|
||||
async def _authorize_hook(
|
||||
request: Request, *, runtime: GatewayRuntime, scope: str
|
||||
) -> str:
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
"""``ApiFrontend`` - ``/api/*``: conversations, SSE, sessions, usage, limits.
|
||||
|
||||
Bearer scope ``api``; token and audit management need ``admin``. Every
|
||||
Bearer scope ``api``; tokens, secrets and audit need ``admin``. Every
|
||||
write goes through conversations/service.py; the frontend only shapes JSON.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
@@ -34,6 +36,7 @@ from beaver_gateway.frontends.sse import (
|
||||
sse_pack,
|
||||
)
|
||||
from beaver_gateway.frontends.urls import frontend_url
|
||||
from beaver_gateway.keystore import SecretsDisabledError
|
||||
from beaver_gateway.security import audit
|
||||
from beaver_gateway.security.auth import hash_token
|
||||
from beaver_gateway.storage import (
|
||||
@@ -46,6 +49,7 @@ from beaver_gateway.storage.models import (
|
||||
Conversation,
|
||||
InjectQueueItem,
|
||||
RateLimit,
|
||||
Secret,
|
||||
Token,
|
||||
Usage,
|
||||
)
|
||||
@@ -978,6 +982,58 @@ def build_app( # noqa: PLR0915
|
||||
)
|
||||
return {"id": token_id, "revoked": True}
|
||||
|
||||
@app.get("/secrets")
|
||||
async def secrets_list(request: Request) -> dict[str, Any]:
|
||||
await require_token(request, runtime, scope=ADMIN_SCOPE)
|
||||
store = runtime.secrets
|
||||
rows = await store.rows() if store is not None else []
|
||||
return {
|
||||
"enabled": store is not None and store.enabled,
|
||||
"secrets": [_secret_public(r) for r in rows],
|
||||
}
|
||||
|
||||
@app.put("/secrets/{name}")
|
||||
async def secret_put(name: str, request: Request) -> dict[str, Any]:
|
||||
actor = await require_token(request, runtime, scope=ADMIN_SCOPE)
|
||||
store = runtime.secrets
|
||||
if store is None or not store.enabled:
|
||||
raise HTTPException(
|
||||
status.HTTP_503_SERVICE_UNAVAILABLE, "SECRETS_KEY is not set"
|
||||
)
|
||||
data = await body_of(request)
|
||||
description = data.get("description")
|
||||
payload, filename = _secret_payload(data)
|
||||
try:
|
||||
row = await store.put(
|
||||
name,
|
||||
description=str(description) if description is not None else None,
|
||||
data=payload,
|
||||
filename=filename,
|
||||
)
|
||||
except SecretsDisabledError as exc:
|
||||
raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc
|
||||
await audit.log(
|
||||
runtime,
|
||||
actor=f"token:{actor}",
|
||||
kind="secret_set",
|
||||
name=name,
|
||||
replaced=payload is not None,
|
||||
)
|
||||
return {"secret": _secret_public(row)}
|
||||
|
||||
@app.delete("/secrets/{name}")
|
||||
async def secret_delete(name: str, request: Request) -> dict[str, Any]:
|
||||
actor = await require_token(request, runtime, scope=ADMIN_SCOPE)
|
||||
store = runtime.secrets
|
||||
if store is None or not await store.delete(name):
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, f"no secret {name}")
|
||||
await audit.log(
|
||||
runtime, actor=f"token:{actor}", kind="secret_delete", name=name
|
||||
)
|
||||
return {"name": name, "deleted": True}
|
||||
|
||||
@app.get("/audit")
|
||||
async def audit_list(request: Request) -> dict[str, Any]:
|
||||
await require_token(request, runtime, scope=ADMIN_SCOPE)
|
||||
@@ -1254,6 +1310,46 @@ def _token_public(row: Token) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _secret_public(row: Secret) -> dict[str, Any]:
|
||||
return {
|
||||
"name": row.name,
|
||||
"description": row.description,
|
||||
"kind": "file" if row.filename else "value",
|
||||
"filename": row.filename,
|
||||
"size": row.size,
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
"used_at": _iso(row.used_at),
|
||||
}
|
||||
|
||||
|
||||
def _secret_payload(data: dict[str, Any]) -> tuple[bytes | None, str | None]:
|
||||
value, upload = data.get("value"), data.get("file")
|
||||
if value is not None and upload is not None:
|
||||
raise HTTPException(
|
||||
status.HTTP_400_BAD_REQUEST, "send `value` or `file`, not both"
|
||||
)
|
||||
if isinstance(value, str):
|
||||
if not value:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, "`value` is empty")
|
||||
return value.encode(), None
|
||||
if isinstance(upload, dict):
|
||||
filename, content = upload.get("name"), upload.get("content")
|
||||
if not isinstance(filename, str) or not isinstance(content, str):
|
||||
raise HTTPException(
|
||||
status.HTTP_400_BAD_REQUEST, "`file` needs `name` and base64 `content`"
|
||||
)
|
||||
try:
|
||||
return base64.b64decode(content, validate=True), filename
|
||||
except binascii.Error as exc:
|
||||
raise HTTPException(
|
||||
status.HTTP_400_BAD_REQUEST, "`file.content` is not base64"
|
||||
) from exc
|
||||
if value is not None or upload is not None:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, "`value` must be a string")
|
||||
return None, None
|
||||
|
||||
|
||||
def _detail(raw: str) -> Any:
|
||||
try:
|
||||
return json.loads(raw) if raw else {}
|
||||
|
||||
@@ -23,6 +23,7 @@ if TYPE_CHECKING:
|
||||
from beaver_gateway.backends.base import Backend
|
||||
from beaver_gateway.conversations.kinds import Kind
|
||||
from beaver_gateway.frontends.turn_record import TurnRecord
|
||||
from beaver_gateway.keystore import SecretStore
|
||||
from beaver_gateway.security.auth import TokenStore
|
||||
from beaver_gateway.storage import Database
|
||||
from beaver_gateway.storage.models import Conversation, ConversationBinding
|
||||
@@ -62,6 +63,8 @@ class GatewayRuntime:
|
||||
"""Every token scope this gateway knows: the builtins plus each frontend's."""
|
||||
send_file_roots: tuple[Path, ...] = ()
|
||||
"""Directories ``send_file`` may read, from ``SEND_FILE_ROOTS``."""
|
||||
secrets: SecretStore | None = None
|
||||
"""Sealed secrets for ``secret run``; ``None`` in setups without them."""
|
||||
|
||||
|
||||
class Frontend(ABC):
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Secrets the model runs commands with: sealed in the database, handed out to root."""
|
||||
|
||||
from beaver_gateway.keystore.server import serve
|
||||
from beaver_gateway.keystore.store import (
|
||||
NAME,
|
||||
SecretBox,
|
||||
SecretsDisabledError,
|
||||
SecretStore,
|
||||
UnknownSecretError,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"NAME",
|
||||
"SecretBox",
|
||||
"SecretStore",
|
||||
"SecretsDisabledError",
|
||||
"UnknownSecretError",
|
||||
"serve",
|
||||
]
|
||||
@@ -0,0 +1,289 @@
|
||||
"""``secret ls`` and ``secret run NAME... -- command``: stdlib only, run by path.
|
||||
|
||||
The model's uid calls this file; it asks the root half (the same file
|
||||
through ``sudo``) for the values, which reads them from the gateway's
|
||||
root-only socket. The command then runs as the model's uid with each
|
||||
value in its own variable, files in a private tmpfs directory, and the
|
||||
values masked in what it prints.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import IO, TYPE_CHECKING, Any
|
||||
from urllib.parse import quote
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from io import BufferedReader
|
||||
|
||||
SOCKET = Path("/run/beaver-secrets/gateway.sock")
|
||||
ROOT_HALF = ("sudo", "-n", "/usr/local/libexec/beaver-secret")
|
||||
SHM = Path("/dev/shm") # noqa: S108
|
||||
PREFIX = "beaver-secret-"
|
||||
NAME = re.compile(r"[A-Z][A-Z0-9_]{0,63}")
|
||||
MIN_MASKED = 6
|
||||
HOLD = 1 << 16
|
||||
CHUNK = 1 << 16
|
||||
DRAIN = 2.0
|
||||
|
||||
USAGE = """usage:
|
||||
secret ls names and descriptions
|
||||
secret run NAME [NAME...] -- CMD run CMD with each secret in $NAME
|
||||
|
||||
A value secret arrives as its text; a file secret as the path to a
|
||||
private copy that is removed when CMD exits. Values are masked in
|
||||
CMD's output as <secret:NAME>.
|
||||
"""
|
||||
|
||||
|
||||
class SecretError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def request(payload: dict[str, Any], path: Path = SOCKET) -> dict[str, Any]:
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as conn:
|
||||
try:
|
||||
conn.connect(str(path))
|
||||
except OSError as exc:
|
||||
msg = f"the gateway's secrets socket is not reachable ({exc.strerror})"
|
||||
raise SecretError(msg) from exc
|
||||
conn.sendall(json.dumps(payload).encode() + b"\n")
|
||||
chunks = []
|
||||
while chunk := conn.recv(CHUNK):
|
||||
chunks.append(chunk)
|
||||
reply = json.loads(b"".join(chunks) or b"{}")
|
||||
if not reply.get("ok"):
|
||||
raise SecretError(reply.get("error") or "the gateway gave no answer")
|
||||
return reply
|
||||
|
||||
|
||||
def root_half(argv: list[str]) -> int:
|
||||
if argv[:1] == ["ls"] and len(argv) == 1:
|
||||
payload: dict[str, Any] = {"op": "ls"}
|
||||
elif (
|
||||
argv[:1] == ["get"]
|
||||
and len(argv) > 1
|
||||
and all(NAME.fullmatch(a) for a in argv[1:])
|
||||
):
|
||||
payload = {"op": "get", "names": argv[1:]}
|
||||
else:
|
||||
print("beaver-secret: ls | get NAME...", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
reply = request(payload)
|
||||
except SecretError as exc:
|
||||
reply = {"ok": False, "error": str(exc)}
|
||||
sys.stdout.write(json.dumps(reply))
|
||||
return 0 if reply.get("ok") else 1
|
||||
|
||||
|
||||
def ask(argv: list[str]) -> dict[str, Any]:
|
||||
if os.geteuid() == 0:
|
||||
return request(
|
||||
{"op": "ls"} if argv == ["ls"] else {"op": "get", "names": argv[1:]}
|
||||
)
|
||||
proc = subprocess.run( # noqa: S603
|
||||
[*ROOT_HALF, *argv], capture_output=True, check=False
|
||||
)
|
||||
try:
|
||||
reply = json.loads(proc.stdout)
|
||||
except json.JSONDecodeError:
|
||||
err = proc.stderr.decode(errors="replace").strip()
|
||||
msg = f"the root helper failed: {err or f'exit {proc.returncode}'}"
|
||||
raise SecretError(msg) from None
|
||||
if not reply.get("ok"):
|
||||
raise SecretError(reply.get("error") or "the root helper gave no answer")
|
||||
return reply
|
||||
|
||||
|
||||
def list_secrets() -> int:
|
||||
rows = ask(["ls"])["secrets"]
|
||||
if not rows:
|
||||
print("no secrets yet; a human adds them in the admin, System -> Secrets")
|
||||
return 0
|
||||
width = max(len(r["name"]) for r in rows)
|
||||
for r in rows:
|
||||
kind = f"file {r['filename']}" if r.get("filename") else "value"
|
||||
line = f"{r['name']:<{width}} {kind}"
|
||||
if r.get("description"):
|
||||
line += f" - {r['description']}"
|
||||
print(line)
|
||||
return 0
|
||||
|
||||
|
||||
class Masker:
|
||||
"""Streams bytes through, replacing every needle; never splits one."""
|
||||
|
||||
def __init__(self, needles: dict[bytes, bytes]) -> None:
|
||||
self._needles = needles
|
||||
self._longest = max((len(n) for n in needles), default=0)
|
||||
ordered = sorted(needles, key=lambda n: -len(n))
|
||||
self._pattern = (
|
||||
re.compile(b"|".join(re.escape(n) for n in ordered)) if ordered else None
|
||||
)
|
||||
self._pending = b""
|
||||
|
||||
def feed(self, data: bytes) -> bytes:
|
||||
if self._pattern is None:
|
||||
return data
|
||||
self._pending += data
|
||||
cut = self._pending.rfind(b"\n") + 1
|
||||
if len(self._pending) - cut > HOLD:
|
||||
cut = max(cut, len(self._pending) - self._longest + 1)
|
||||
for match in self._pattern.finditer(self._pending):
|
||||
if match.start() < cut < match.end():
|
||||
cut = match.end()
|
||||
head, self._pending = self._pending[:cut], self._pending[cut:]
|
||||
return self._mask(head)
|
||||
|
||||
def flush(self) -> bytes:
|
||||
tail, self._pending = self._pending, b""
|
||||
return self._mask(tail)
|
||||
|
||||
def _mask(self, data: bytes) -> bytes:
|
||||
if self._pattern is None or not data:
|
||||
return data
|
||||
return self._pattern.sub(lambda m: self._needles[m[0]], data)
|
||||
|
||||
|
||||
def needles_for(values: dict[str, bytes]) -> dict[bytes, bytes]:
|
||||
out: dict[bytes, bytes] = {}
|
||||
for name, data in values.items():
|
||||
mask = f"<secret:{name}>".encode()
|
||||
try:
|
||||
text = data.decode()
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
for part in text.splitlines() or [text]:
|
||||
for form in {part.strip(), quote(part.strip(), safe="")}:
|
||||
if len(form) >= MIN_MASKED:
|
||||
out[form.encode()] = mask
|
||||
return out
|
||||
|
||||
|
||||
def sweep(root: Path) -> None:
|
||||
for stale in root.glob(f"{PREFIX}*"):
|
||||
pid = stale.name.removeprefix(PREFIX).split("-", 1)[0]
|
||||
if not pid.isdigit() or _alive(int(pid)):
|
||||
continue
|
||||
shutil.rmtree(stale, ignore_errors=True)
|
||||
|
||||
|
||||
def _alive(pid: int) -> bool:
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def _pump(src: BufferedReader, dst: IO[bytes], masker: Masker) -> None:
|
||||
while chunk := src.read1(CHUNK):
|
||||
dst.write(masker.feed(chunk))
|
||||
dst.flush()
|
||||
dst.write(masker.flush())
|
||||
dst.flush()
|
||||
|
||||
|
||||
def run(names: list[str], command: list[str], *, root: Path | None = None) -> int:
|
||||
secrets = ask(["get", *names])["secrets"]
|
||||
base = root or (SHM if SHM.is_dir() else Path(tempfile.gettempdir()))
|
||||
sweep(base)
|
||||
workdir = Path(tempfile.mkdtemp(prefix=f"{PREFIX}{os.getpid()}-", dir=base))
|
||||
env = dict(os.environ)
|
||||
values: dict[str, bytes] = {}
|
||||
try:
|
||||
for name in names:
|
||||
item = secrets[name]
|
||||
data = base64.b64decode(item["data"])
|
||||
if item.get("filename"):
|
||||
path = workdir / name / item["filename"]
|
||||
path.parent.mkdir(mode=0o700)
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(fd, "wb") as fh:
|
||||
fh.write(data)
|
||||
env[name] = str(path)
|
||||
else:
|
||||
env[name] = data.decode(errors="surrogateescape")
|
||||
values[name] = data
|
||||
return _spawn(command, env, needles_for(values))
|
||||
finally:
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
|
||||
|
||||
def _spawn(command: list[str], env: dict[str, str], needles: dict[bytes, bytes]) -> int:
|
||||
try:
|
||||
child = subprocess.Popen( # noqa: S603
|
||||
command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
||||
)
|
||||
except (OSError, ValueError) as exc:
|
||||
reason = exc.strerror if isinstance(exc, OSError) else str(exc)
|
||||
print(f"secret: cannot run {command[0]}: {reason}", file=sys.stderr)
|
||||
return 127
|
||||
forwarded = (
|
||||
(signal.SIGINT, signal.SIGTERM, signal.SIGHUP)
|
||||
if threading.current_thread() is threading.main_thread()
|
||||
else ()
|
||||
)
|
||||
previous = {
|
||||
sig: signal.signal(sig, lambda s, _f: child.send_signal(s)) for sig in forwarded
|
||||
}
|
||||
pumps = [
|
||||
threading.Thread(target=_pump, args=(src, dst, Masker(needles)), daemon=True)
|
||||
for src, dst in (
|
||||
(child.stdout, sys.stdout.buffer),
|
||||
(child.stderr, sys.stderr.buffer),
|
||||
)
|
||||
]
|
||||
for pump in pumps:
|
||||
pump.start()
|
||||
try:
|
||||
code = child.wait()
|
||||
for pump in pumps:
|
||||
pump.join(DRAIN)
|
||||
finally:
|
||||
for sig, handler in previous.items():
|
||||
signal.signal(sig, handler)
|
||||
return 128 - code if code < 0 else code
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = sys.argv[1:] if argv is None else argv
|
||||
if args[:1] == ["--root"]:
|
||||
return root_half(args[1:])
|
||||
try:
|
||||
if args == ["ls"]:
|
||||
return list_secrets()
|
||||
if args[:1] == ["run"] and "--" in args:
|
||||
split = args.index("--")
|
||||
names, command = args[1:split], args[split + 1 :]
|
||||
bad = [n for n in names if not NAME.fullmatch(n)]
|
||||
if names and command and not bad:
|
||||
return run(names, command)
|
||||
if bad:
|
||||
print(f"secret: invalid name {bad[0]!r}", file=sys.stderr)
|
||||
return 2
|
||||
except SecretError as exc:
|
||||
print(f"secret: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(USAGE, end="", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with contextlib.suppress(KeyboardInterrupt):
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Root-only unix socket that hands sealed secrets to the ``secret`` helper."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from beaver_gateway.keystore.store import (
|
||||
SecretsDisabledError,
|
||||
UnknownSecretError,
|
||||
check_name,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from pathlib import Path
|
||||
|
||||
from beaver_gateway.keystore.store import SecretStore
|
||||
|
||||
_log = logging.getLogger("beaver_gateway.keystore")
|
||||
|
||||
REQUEST_LIMIT = 1 << 16
|
||||
|
||||
|
||||
async def serve(
|
||||
store: SecretStore,
|
||||
path: Path,
|
||||
*,
|
||||
on_use: Callable[[Sequence[str]], Awaitable[None]] | None = None,
|
||||
) -> None:
|
||||
"""Listen on ``path`` (mode 0600 in a 0700 directory) until cancelled."""
|
||||
_clear(path)
|
||||
|
||||
async def handle(
|
||||
reader: asyncio.StreamReader, writer: asyncio.StreamWriter
|
||||
) -> None:
|
||||
try:
|
||||
line = await reader.readline()
|
||||
reply = await answer(store, line, on_use=on_use)
|
||||
writer.write(json.dumps(reply).encode() + b"\n")
|
||||
await writer.drain()
|
||||
except Exception: # noqa: BLE001
|
||||
_log.exception("secret request failed")
|
||||
finally:
|
||||
writer.close()
|
||||
with contextlib.suppress(Exception):
|
||||
await writer.wait_closed()
|
||||
|
||||
server = await asyncio.start_unix_server(
|
||||
handle, path=str(path), limit=REQUEST_LIMIT
|
||||
)
|
||||
_lock(path)
|
||||
_log.info("secrets socket on %s", path)
|
||||
async with server:
|
||||
await server.serve_forever()
|
||||
|
||||
|
||||
async def answer(
|
||||
store: SecretStore,
|
||||
line: bytes,
|
||||
*,
|
||||
on_use: Callable[[Sequence[str]], Awaitable[None]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
request = json.loads(line)
|
||||
op = request.get("op") if isinstance(request, dict) else None
|
||||
if op == "ls":
|
||||
return {"ok": True, "secrets": [_listed(r) for r in await store.rows()]}
|
||||
if op == "get":
|
||||
names = [check_name(str(n)) for n in request.get("names") or ()]
|
||||
if not names:
|
||||
return {"ok": False, "error": "no secret names given"}
|
||||
revealed = await store.reveal(names)
|
||||
if on_use is not None:
|
||||
await on_use(names)
|
||||
return {
|
||||
"ok": True,
|
||||
"secrets": {
|
||||
r.name: {
|
||||
"data": base64.b64encode(r.data).decode(),
|
||||
"filename": r.filename,
|
||||
}
|
||||
for r in revealed
|
||||
},
|
||||
}
|
||||
except UnknownSecretError as exc:
|
||||
return {"ok": False, "error": f"unknown secret: {exc}"}
|
||||
except (SecretsDisabledError, ValueError) as exc:
|
||||
return {"ok": False, "error": str(exc)}
|
||||
return {"ok": False, "error": "op must be `ls` or `get`"}
|
||||
|
||||
|
||||
def _clear(path: Path) -> None:
|
||||
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
path.parent.chmod(0o700)
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _lock(path: Path) -> None:
|
||||
path.chmod(0o600)
|
||||
|
||||
|
||||
def _listed(row: Any) -> dict[str, Any]:
|
||||
return {"name": row.name, "description": row.description, "filename": row.filename}
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Seal, list, replace and reveal secrets; the key never leaves this module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from cryptography.exceptions import InvalidTag
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
from sqlmodel import col, select
|
||||
|
||||
from beaver_gateway.storage.models import Secret
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Sequence
|
||||
|
||||
from beaver_gateway.storage import Database
|
||||
|
||||
NAME = re.compile(r"[A-Z][A-Z0-9_]{0,63}")
|
||||
"""A secret's name is the environment variable a command receives it under."""
|
||||
|
||||
KEY_BYTES = 32
|
||||
NONCE_BYTES = 12
|
||||
MAX_SIZE = 1 << 20
|
||||
"""Largest value or file the store accepts, in bytes."""
|
||||
|
||||
|
||||
class SecretsDisabledError(RuntimeError):
|
||||
"""``SECRETS_KEY`` is unset or unusable, so nothing can be sealed or opened."""
|
||||
|
||||
|
||||
class UnknownSecretError(LookupError):
|
||||
"""A name that is not in the store."""
|
||||
|
||||
|
||||
class SecretBox:
|
||||
"""AES-256-GCM with the secret's name as associated data."""
|
||||
|
||||
__slots__ = ("_aead",)
|
||||
|
||||
def __init__(self, key: bytes) -> None:
|
||||
if len(key) != KEY_BYTES:
|
||||
msg = f"SECRETS_KEY must be {KEY_BYTES} bytes, got {len(key)}"
|
||||
raise ValueError(msg)
|
||||
self._aead = AESGCM(key)
|
||||
|
||||
@classmethod
|
||||
def from_hex(cls, text: str) -> SecretBox:
|
||||
try:
|
||||
key = bytes.fromhex(text.strip())
|
||||
except ValueError as exc:
|
||||
msg = "SECRETS_KEY must be hex, e.g. `openssl rand -hex 32`"
|
||||
raise ValueError(msg) from exc
|
||||
return cls(key)
|
||||
|
||||
def seal(self, name: str, data: bytes) -> bytes:
|
||||
nonce = os.urandom(NONCE_BYTES)
|
||||
return nonce + self._aead.encrypt(nonce, data, name.encode())
|
||||
|
||||
def open(self, name: str, sealed: bytes) -> bytes:
|
||||
nonce, body = sealed[:NONCE_BYTES], sealed[NONCE_BYTES:]
|
||||
return self._aead.decrypt(nonce, body, name.encode())
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Revealed:
|
||||
name: str
|
||||
data: bytes
|
||||
filename: str | None
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def check_name(name: str) -> str:
|
||||
if not NAME.fullmatch(name):
|
||||
msg = (
|
||||
f"invalid secret name {name!r}: use A-Z, 0-9 and _, starting with a letter"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
return name
|
||||
|
||||
|
||||
def clean_filename(filename: str) -> str:
|
||||
base = os.path.basename(filename.replace("\\", "/")).strip() # noqa: PTH119
|
||||
if base in ("", ".", ".."):
|
||||
msg = f"invalid file name {filename!r}"
|
||||
raise ValueError(msg)
|
||||
return base
|
||||
|
||||
|
||||
class SecretStore:
|
||||
"""Rows of :class:`Secret`; ``box`` is ``None`` when no key is configured."""
|
||||
|
||||
__slots__ = ("_box", "_db")
|
||||
|
||||
def __init__(self, db: Database, box: SecretBox | None) -> None:
|
||||
self._db = db
|
||||
self._box = box
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self._box is not None
|
||||
|
||||
def _require_box(self) -> SecretBox:
|
||||
if self._box is None:
|
||||
msg = "SECRETS_KEY is not set"
|
||||
raise SecretsDisabledError(msg)
|
||||
return self._box
|
||||
|
||||
async def rows(self) -> Sequence[Secret]:
|
||||
async with self._db.session() as session:
|
||||
result = await session.exec(select(Secret).order_by(col(Secret.name)))
|
||||
return result.all()
|
||||
|
||||
async def put(
|
||||
self,
|
||||
name: str,
|
||||
*,
|
||||
description: str | None = None,
|
||||
data: bytes | None = None,
|
||||
filename: str | None = None,
|
||||
) -> Secret:
|
||||
"""Create or replace; ``data=None`` keeps the value, sets the description."""
|
||||
check_name(name)
|
||||
if data is not None:
|
||||
if len(data) > MAX_SIZE:
|
||||
msg = f"secret {name} is {len(data)} bytes, the limit is {MAX_SIZE}"
|
||||
raise ValueError(msg)
|
||||
sealed = self._require_box().seal(name, data)
|
||||
async with self._db.session() as session:
|
||||
row = await session.get(Secret, name)
|
||||
if row is None:
|
||||
if data is None:
|
||||
msg = f"secret {name} needs a value or a file"
|
||||
raise ValueError(msg)
|
||||
row = Secret(name=name, sealed=sealed)
|
||||
if data is not None:
|
||||
row.sealed = sealed
|
||||
row.size = len(data)
|
||||
row.filename = clean_filename(filename) if filename else None
|
||||
if description is not None:
|
||||
row.description = description.strip()
|
||||
row.updated_at = _now()
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return row
|
||||
|
||||
async def delete(self, name: str) -> bool:
|
||||
async with self._db.session() as session:
|
||||
row = await session.get(Secret, name)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def reveal(self, names: Iterable[str]) -> list[Revealed]:
|
||||
box = self._require_box()
|
||||
wanted = list(dict.fromkeys(names))
|
||||
async with self._db.session() as session:
|
||||
rows = {
|
||||
r.name: r
|
||||
for r in (
|
||||
await session.exec(
|
||||
select(Secret).where(col(Secret.name).in_(wanted))
|
||||
)
|
||||
).all()
|
||||
}
|
||||
missing = [n for n in wanted if n not in rows]
|
||||
if missing:
|
||||
raise UnknownSecretError(", ".join(missing))
|
||||
out = [_open(box, rows[n]) for n in wanted]
|
||||
now = _now()
|
||||
for row in rows.values():
|
||||
row.used_at = now
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
|
||||
def _open(box: SecretBox, row: Secret) -> Revealed:
|
||||
try:
|
||||
data = box.open(row.name, row.sealed)
|
||||
except InvalidTag as exc:
|
||||
msg = f"secret {row.name} does not open: SECRETS_KEY changed since it was set"
|
||||
raise SecretsDisabledError(msg) from exc
|
||||
return Revealed(name=row.name, data=data, filename=row.filename)
|
||||
@@ -28,6 +28,9 @@ KNOWN_KINDS: frozenset[str] = frozenset(
|
||||
"logout",
|
||||
"token_create",
|
||||
"token_revoke",
|
||||
"secret_set",
|
||||
"secret_delete",
|
||||
"secret_use",
|
||||
}
|
||||
)
|
||||
"""Kinds the gateway currently emits; ``AuditLog.kind`` stays free-form."""
|
||||
|
||||
@@ -61,6 +61,13 @@ class Settings(BaseSettings):
|
||||
"""Directories the ``send_file`` tool may read, separated by ``:``. A
|
||||
frontend always adds the inbox its own attachments land in."""
|
||||
|
||||
secrets_key: str | None = None
|
||||
"""Hex AES-256 key (``openssl rand -hex 32``) sealing the secrets table;
|
||||
unset leaves secrets off. Never reaches the model's environment."""
|
||||
|
||||
secrets_socket: Path = Path("/run/beaver-secrets/gateway.sock")
|
||||
"""Root-only socket the ``secret`` helper reads values from."""
|
||||
|
||||
bootstrap_tokens: str = ""
|
||||
"""Out-of-band token seed: ``name1:value1,name2:value2``. Layers
|
||||
alongside DB-issued tokens; used for first-run setup and recovery."""
|
||||
@@ -10,7 +10,7 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import JSON, Column, Index, UniqueConstraint, text
|
||||
from sqlalchemy import JSON, Column, Index, LargeBinary, UniqueConstraint, text
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlmodel import Field, SQLModel
|
||||
|
||||
@@ -374,6 +374,25 @@ class JobRunRecord(SQLModel, table=True):
|
||||
)
|
||||
|
||||
|
||||
class Secret(SQLModel, table=True):
|
||||
"""A value or file the model may run a command with, sealed by ``SECRETS_KEY``.
|
||||
|
||||
``sealed`` is nonce + AES-GCM ciphertext bound to ``name``; nothing
|
||||
here decrypts without the key, so a dump of the table stays closed.
|
||||
"""
|
||||
|
||||
__tablename__ = "secrets"
|
||||
|
||||
name: str = Field(primary_key=True)
|
||||
description: str = Field(default="")
|
||||
filename: str | None = Field(default=None)
|
||||
size: int = Field(default=0)
|
||||
sealed: bytes = Field(sa_column=Column(LargeBinary, nullable=False))
|
||||
created_at: datetime = Field(default_factory=_utcnow)
|
||||
updated_at: datetime = Field(default_factory=_utcnow)
|
||||
used_at: datetime | None = Field(default=None)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AuditLog",
|
||||
"Conversation",
|
||||
@@ -383,6 +402,7 @@ __all__ = [
|
||||
"InjectQueueItem",
|
||||
"JobRunRecord",
|
||||
"RateLimit",
|
||||
"Secret",
|
||||
"TelegramUpdate",
|
||||
"Token",
|
||||
"TranscriptEntry",
|
||||
|
||||
Reference in new issue
Block a user