feat(backends,api,ui): process trees with cpu and stderr per session, interrupt, close and kill from the system page

src/beaver_gateway/backends/claude_sdk.py

@@ -20,6 +20,7 @@ import tempfile
 import time
 import uuid
 import warnings
+from collections import deque
 from collections.abc import Mapping
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -49,7 +50,12 @@ from claude_agent_sdk import (
 from beaver_gateway.agents import policy as policy_mod
 from beaver_gateway.agents import prompts as prompt_assembly
 from beaver_gateway.backends.capture import TurnCapture, TurnUsage
-from beaver_gateway.backends.sessions import Session, SessionClient, SessionPool
+from beaver_gateway.backends.sessions import (
+    STDERR_LINES,
+    Session,
+    SessionClient,
+    SessionPool,
+)
 from beaver_gateway.backends.transcript import (
     build_entries,
     close_open_tool_uses,
@@ -71,6 +77,7 @@ from beaver_gateway.events.stream import (
     build_thinking_delta,
     build_tool_use_block_start,
 )
+from beaver_gateway.security.redact import redact
 
 if TYPE_CHECKING:
     from collections.abc import (
@@ -262,12 +269,7 @@ class ClaudeSdkBackend:
         await self._pool.close(key)
 
     async def interrupt(self, key: str) -> bool:
-        live = self._pool.get(key)
-        if live is None or not live.busy:
-            return False
-        live.interrupt_requested = True
-        await live.client.interrupt()
-        return True
+        return await self._pool.interrupt(key)
 
     def live(self, key: str) -> Session | None:
         return self._pool.get(key)
@@ -674,7 +676,8 @@ class ClaudeSdkBackend:
     async def _spawn(
         self, resume: str | None, *, key: str, spec: _SessionSpec
     ) -> Session:
-        options = self._build_options(resume, key=key, spec=spec)
+        stderr: deque[str] = deque(maxlen=STDERR_LINES)
+        options = self._build_options(resume, key=key, spec=spec, stderr=stderr)
         client = self._factory(options)
         await client.connect()
         _log.info(
@@ -693,6 +696,7 @@ class ClaudeSdkBackend:
             session_id=resume,
             resumed=resume is not None,
             pinned=spec.pinned,
+            stderr=stderr,
         )
         live.reader = asyncio.create_task(self._read(live))
         return live
@@ -701,7 +705,12 @@ class ClaudeSdkBackend:
         return _RunnerClient(options, uid=self._uid)
 
     def _build_options(
-        self, resume: str | None, *, key: str, spec: _SessionSpec
+        self,
+        resume: str | None,
+        *,
+        key: str,
+        spec: _SessionSpec,
+        stderr: deque[str] | None = None,
     ) -> ClaudeAgentOptions:
         agent = self._agent
         opt = agent.options
@@ -747,11 +756,14 @@ class ClaudeSdkBackend:
             plugins=cast("Any", plugins),
             skills="all" if plugins else None,
             max_turns=opt.max_turns,
-            stderr=lambda line: _log.warning(
-                "claude[%s]: %s", agent.name, line.rstrip()
-            ),
+            stderr=lambda line: self._stderr(line.rstrip(), stderr),
         )
 
+    def _stderr(self, line: str, tail: deque[str] | None) -> None:
+        _log.warning("claude[%s]: %s", self._agent.name, line)
+        if tail is not None:
+            tail.append(redact(line))
+
     def _can_use_tool(
         self, key: str
     ) -> Callable[

src/beaver_gateway/backends/processes.py

@@ -0,0 +1,118 @@
+"""What runs under the gateway: the process tree with memory and CPU, and the kill."""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import psutil
+
+from beaver_gateway.security.redact import redact
+
+__all__ = ["Processes", "subtree"]
+
+_VALUE_HIDDEN = frozenset(
+    {
+        "--system-prompt",
+        "--append-system-prompt",
+        "--mcp-config",
+        "--settings",
+        "--agents",
+    }
+)
+_ARG_CLIP = 80
+_COMMAND_CLIP = 240
+
+
+@dataclass(slots=True)
+class Processes:
+    root: int = field(default_factory=os.getpid)
+    _cpu_seen: dict[int, tuple[float, float]] = field(default_factory=dict)
+
+    def tree_parents_first(self) -> list[dict[str, Any]]:
+        try:
+            found = psutil.Process(self.root).children(recursive=True)
+        except psutil.Error:
+            return []
+        now = time.monotonic()
+        rows: dict[int, dict[str, Any]] = {}
+        for process in found:
+            with contextlib.suppress(psutil.Error), process.oneshot():
+                spent = process.cpu_times()
+                rows[process.pid] = {
+                    "pid": process.pid,
+                    "parent": process.ppid(),
+                    "name": process.name(),
+                    "command": _command(process.cmdline()),
+                    "rss": process.memory_info().rss,
+                    "cpu": self._rate(process.pid, spent.user + spent.system, now),
+                    "age_seconds": round(time.time() - process.create_time(), 1),
+                }
+        self._cpu_seen = {p: seen for p, seen in self._cpu_seen.items() if p in rows}
+        children: dict[int, list[int]] = {}
+        for pid, row in rows.items():
+            parent = row["parent"] if row["parent"] in rows else self.root
+            children.setdefault(parent, []).append(pid)
+        ordered: list[dict[str, Any]] = []
+
+        def walk(parent: int, depth: int) -> None:
+            for pid in sorted(children.get(parent, ())):
+                ordered.append({**rows[pid], "depth": depth})
+                walk(pid, depth + 1)
+
+        walk(self.root, 0)
+        return ordered
+
+    def kill_own_descendant(self, pid: int) -> bool:
+        try:
+            owned = psutil.Process(self.root).children(recursive=True)
+            target = next((p for p in owned if p.pid == pid), None)
+            if target is None:
+                return False
+            doomed = [*target.children(recursive=True), target]
+        except psutil.Error:
+            return False
+        for process in doomed:
+            with contextlib.suppress(psutil.Error):
+                process.kill()
+        return True
+
+    def _rate(self, pid: int, spent: float, now: float) -> float | None:
+        before = self._cpu_seen.get(pid)
+        self._cpu_seen[pid] = (spent, now)
+        if before is None or now <= before[1]:
+            return None
+        return round(100 * max(spent - before[0], 0.0) / (now - before[1]), 1)
+
+
+def subtree(rows: list[dict[str, Any]], pid: int | None) -> list[dict[str, Any]]:
+    start = next((i for i, row in enumerate(rows) if row["pid"] == pid), None)
+    if start is None:
+        return []
+    base = rows[start]["depth"]
+    out = [{**rows[start], "depth": 0}]
+    for row in rows[start + 1 :]:
+        if row["depth"] <= base:
+            break
+        out.append({**row, "depth": row["depth"] - base})
+    return out
+
+
+def _command(argv: list[str]) -> str:
+    if not argv:
+        return ""
+    parts = [Path(argv[0]).name]
+    hide = False
+    for arg in argv[1:]:
+        if hide:
+            parts.append("…")
+            hide = False
+            continue
+        hide = arg in _VALUE_HIDDEN
+        parts.append(arg if len(arg) <= _ARG_CLIP else f"{arg[:_ARG_CLIP]}…")
+    command = redact(" ".join(parts))
+    return command if len(command) <= _COMMAND_CLIP else f"{command[:_COMMAND_CLIP]}…"

src/beaver_gateway/backends/sessions.py

@@ -13,12 +13,15 @@ import contextlib
 import logging
 import os
 import time
+from collections import deque
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Protocol
 
 import psutil
 
+from beaver_gateway.backends.processes import Processes, subtree
+
 if TYPE_CHECKING:
     from collections.abc import (
         AsyncIterable,
@@ -43,6 +46,7 @@ DEFAULT_TTL: Mapping[str, float | None] = {
 
 _EVICT_ORDER = {"fork": 0, "job": 0, "deep": 1, "branch": 2, "master": 3}
 _RSS_HEADROOM = 0.8
+STDERR_LINES = 200
 
 
 class SessionClient(Protocol):
@@ -86,6 +90,7 @@ class Session:
     """Summary of the last task notification, the pending turn's prompt."""
     observer: Callable[[Any], None] | None = None
     """The last turn's observer; sees subagent traffic between turns."""
+    stderr: deque[str] = field(default_factory=lambda: deque(maxlen=STDERR_LINES))
 
     @property
     def busy(self) -> bool:
@@ -121,6 +126,7 @@ class SessionPool:
         self._max_live = max_live
         self._ttl = dict(ttl)
         self._reap_interval = reap_interval
+        self._processes = Processes()
 
     def __len__(self) -> int:
         return len(self._sessions)
@@ -168,19 +174,6 @@ class SessionPool:
                 total += child.memory_info().rss
         return total
 
-    @staticmethod
-    def rss_of(session: Session) -> int | None:
-        pid = session.pid
-        if pid is None:
-            return None
-        try:
-            process = psutil.Process(pid)
-            return process.memory_info().rss + sum(
-                c.memory_info().rss for c in process.children(recursive=True)
-            )
-        except psutil.Error:
-            return None
-
     def over_limit(self) -> bool:
         if self._rss_limit is not None:
             return self.rss() > self._rss_limit * _RSS_HEADROOM
@@ -214,11 +207,21 @@ class SessionPool:
         _log.info("closing session %s (%s, %s)", session.session_id, session.kind, key)
         if session.reader is not None and session.reader is not asyncio.current_task():
             session.reader.cancel()
+        if session.attached:
+            session.inbox.put_nowait(RuntimeError(f"session {key} was closed"))
         try:
             await session.client.disconnect()
         except Exception:  # noqa: BLE001
             _log.exception("disconnect failed for session %s", session.session_id)
 
+    async def interrupt(self, key: str) -> bool:
+        session = self._sessions.get(key)
+        if session is None or not session.busy:
+            return False
+        session.interrupt_requested = True
+        await session.client.interrupt()
+        return True
+
     async def close_all(self, *, agent: str | None = None) -> None:
         for session in list(self._sessions.values()):
             if agent is None or session.agent == agent:
@@ -246,27 +249,57 @@ class SessionPool:
                 _log.exception("session reaper failed")
 
     def snapshot(self) -> list[dict[str, Any]]:
+        return self.census()["sessions"]
+
+    def census(self) -> dict[str, Any]:
         now = time.monotonic()
-        return [
-            {
-                "key": s.key,
-                "agent": s.agent,
-                "kind": s.kind,
-                "session_id": s.session_id,
-                "pid": s.pid,
-                "rss": self.rss_of(s),
-                "idle_seconds": round(now - s.last_used, 1),
-                "age_seconds": round(now - s.created_at, 1),
-                "turns": s.turns,
-                "busy": s.busy,
-                "running_turn": s.running_turn,
-                "pending": len(s.pending),
-                "pending_question": s.pending_question,
-                "pinned": s.pinned,
-                "dirty": s.dirty,
-            }
-            for s in self._sessions.values()
-        ]
+        rows = self._processes.tree_parents_first()
+        owned: set[int] = set()
+        sessions: list[dict[str, Any]] = []
+        for s in self._sessions.values():
+            tree = subtree(rows, s.pid)
+            owned.update(row["pid"] for row in tree)
+            sessions.append(
+                {
+                    "key": s.key,
+                    "agent": s.agent,
+                    "kind": s.kind,
+                    "session_id": s.session_id,
+                    "pid": s.pid,
+                    "rss": sum(row["rss"] for row in tree) if tree else None,
+                    "cpu": _cpu(tree),
+                    "idle_seconds": round(now - s.last_used, 1),
+                    "age_seconds": round(now - s.created_at, 1),
+                    "turns": s.turns,
+                    "busy": s.busy,
+                    "running_turn": s.running_turn,
+                    "pending": len(s.pending),
+                    "pending_question": s.pending_question,
+                    "pinned": s.pinned,
+                    "dirty": s.dirty,
+                    "processes": tree,
+                }
+            )
+        return {
+            "rss": sum(row["rss"] for row in rows),
+            "rss_limit": self._rss_limit,
+            "cpu": _cpu(rows),
+            "sessions": sessions,
+            "others": [row for row in rows if row["pid"] not in owned],
+        }
+
+    async def kill(self, pid: int) -> bool:
+        if not self._processes.kill_own_descendant(pid):
+            return False
+        for session in list(self._sessions.values()):
+            if session.pid == pid:
+                await self.close(session.key)
+        return True
+
+
+def _cpu(rows: list[dict[str, Any]]) -> float | None:
+    rates = [row["cpu"] for row in rows if row["cpu"] is not None]
+    return round(sum(rates), 1) if rates else None
 
 
 def cgroup_limit() -> int | None:

src/beaver_gateway/frontends/api/frontend.py

@@ -55,6 +55,7 @@ if TYPE_CHECKING:
     from collections.abc import AsyncIterator, Iterable, Sequence
     from pathlib import Path
 
+    from beaver_gateway.backends.sessions import Session, SessionPool
     from beaver_gateway.conversations.service import Conversations
     from beaver_gateway.frontends.base import GatewayRuntime
     from beaver_gateway.jobs.scheduler import Job, Scheduler
@@ -633,15 +634,74 @@ def build_app(  # noqa: PLR0915
         await require_token(request, runtime, scope=SCOPE)
         return _sse(runtime, conversation_id=None)
 
+    def pool_of() -> SessionPool:
+        if runtime.pool is None:
+            raise HTTPException(status.HTTP_404_NOT_FOUND, "no session pool")
+        return runtime.pool
+
+    def session_of(key: str) -> Session:
+        session = pool_of().get(key)
+        if session is None:
+            raise HTTPException(status.HTTP_404_NOT_FOUND, f"no live session {key}")
+        return session
+
     @app.get("/sessions")
     async def sessions(request: Request) -> dict[str, Any]:
         await require_token(request, runtime, scope=SCOPE)
-        pool = runtime.pool
-        return {
-            "rss": pool.rss() if pool is not None else None,
-            "rss_limit": pool.rss_limit if pool is not None else None,
-            "sessions": pool.snapshot() if pool is not None else [],
-        }
+        if runtime.pool is None:
+            return {
+                "rss": None,
+                "rss_limit": None,
+                "cpu": None,
+                "sessions": [],
+                "others": [],
+            }
+        return runtime.pool.census()
+
+    @app.get("/sessions/{key}/stderr")
+    async def session_stderr(key: str, request: Request) -> dict[str, Any]:
+        await require_token(request, runtime, scope=ADMIN_SCOPE)
+        return {"key": key, "lines": list(session_of(key).stderr)}
+
+    @app.post("/sessions/{key}/interrupt")
+    async def session_interrupt(key: str, request: Request) -> dict[str, Any]:
+        token = await require_token(request, runtime, scope=SCOPE)
+        session = session_of(key)
+        interrupted = await pool_of().interrupt(key)
+        await audit.log(
+            runtime,
+            actor=f"token:{token}",
+            kind="session_interrupt",
+            agent_name=session.agent,
+            session=key,
+            interrupted=interrupted,
+        )
+        return {"key": key, "interrupted": interrupted}
+
+    @app.post("/sessions/{key}/close")
+    async def session_close(key: str, request: Request) -> dict[str, Any]:
+        token = await require_token(request, runtime, scope=ADMIN_SCOPE)
+        session = session_of(key)
+        await pool_of().close(key)
+        await audit.log(
+            runtime,
+            actor=f"token:{token}",
+            kind="session_close",
+            agent_name=session.agent,
+            session=key,
+            busy=session.busy,
+        )
+        return {"key": key, "closed": True}
+
+    @app.post("/processes/{pid}/kill")
+    async def process_kill(pid: int, request: Request) -> dict[str, Any]:
+        token = await require_token(request, runtime, scope=ADMIN_SCOPE)
+        if not await pool_of().kill(pid):
+            raise HTTPException(
+                status.HTTP_404_NOT_FOUND, f"no process {pid} under the gateway"
+            )
+        await audit.log(runtime, actor=f"token:{token}", kind="process_kill", pid=pid)
+        return {"pid": pid, "killed": True}
 
     @app.get("/schedules")
     async def schedules(request: Request) -> dict[str, Any]: