feat(backends,api,ui): process trees with cpu and stderr per session, interrupt, close and kill from the system page

src/beaver_gateway/backends/claude_sdk.py

@@ -20,6 +20,7 @@ import tempfile
 import time
 import uuid
 import warnings
+from collections import deque
 from collections.abc import Mapping
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -49,7 +50,12 @@ from claude_agent_sdk import (
 from beaver_gateway.agents import policy as policy_mod
 from beaver_gateway.agents import prompts as prompt_assembly
 from beaver_gateway.backends.capture import TurnCapture, TurnUsage
-from beaver_gateway.backends.sessions import Session, SessionClient, SessionPool
+from beaver_gateway.backends.sessions import (
+    STDERR_LINES,
+    Session,
+    SessionClient,
+    SessionPool,
+)
 from beaver_gateway.backends.transcript import (
     build_entries,
     close_open_tool_uses,
@@ -71,6 +77,7 @@ from beaver_gateway.events.stream import (
     build_thinking_delta,
     build_tool_use_block_start,
 )
+from beaver_gateway.security.redact import redact
 
 if TYPE_CHECKING:
     from collections.abc import (
@@ -262,12 +269,7 @@ class ClaudeSdkBackend:
         await self._pool.close(key)
 
     async def interrupt(self, key: str) -> bool:
-        live = self._pool.get(key)
+        return await self._pool.interrupt(key)
-        if live is None or not live.busy:
-            return False
-        live.interrupt_requested = True
-        await live.client.interrupt()
-        return True
 
     def live(self, key: str) -> Session | None:
         return self._pool.get(key)
@@ -674,7 +676,8 @@ class ClaudeSdkBackend:
     async def _spawn(
         self, resume: str | None, *, key: str, spec: _SessionSpec
     ) -> Session:
-        options = self._build_options(resume, key=key, spec=spec)
+        stderr: deque[str] = deque(maxlen=STDERR_LINES)
+        options = self._build_options(resume, key=key, spec=spec, stderr=stderr)
         client = self._factory(options)
         await client.connect()
         _log.info(
@@ -693,6 +696,7 @@ class ClaudeSdkBackend:
             session_id=resume,
             resumed=resume is not None,
             pinned=spec.pinned,
+            stderr=stderr,
         )
         live.reader = asyncio.create_task(self._read(live))
         return live
@@ -701,7 +705,12 @@ class ClaudeSdkBackend:
         return _RunnerClient(options, uid=self._uid)
 
     def _build_options(
-        self, resume: str | None, *, key: str, spec: _SessionSpec
+        self,
+        resume: str | None,
+        *,
+        key: str,
+        spec: _SessionSpec,
+        stderr: deque[str] | None = None,
     ) -> ClaudeAgentOptions:
         agent = self._agent
         opt = agent.options
@@ -747,11 +756,14 @@ class ClaudeSdkBackend:
             plugins=cast("Any", plugins),
             skills="all" if plugins else None,
             max_turns=opt.max_turns,
-            stderr=lambda line: _log.warning(
+            stderr=lambda line: self._stderr(line.rstrip(), stderr),
-                "claude[%s]: %s", agent.name, line.rstrip()
-            ),
         )
 
+    def _stderr(self, line: str, tail: deque[str] | None) -> None:
+        _log.warning("claude[%s]: %s", self._agent.name, line)
+        if tail is not None:
+            tail.append(redact(line))
+
     def _can_use_tool(
         self, key: str
     ) -> Callable[

src/beaver_gateway/backends/processes.py

@@ -0,0 +1,118 @@
+"""What runs under the gateway: the process tree with memory and CPU, and the kill."""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import psutil
+
+from beaver_gateway.security.redact import redact
+
+__all__ = ["Processes", "subtree"]
+
+_VALUE_HIDDEN = frozenset(
+    {
+        "--system-prompt",
+        "--append-system-prompt",
+        "--mcp-config",
+        "--settings",
+        "--agents",
+    }
+)
+_ARG_CLIP = 80
+_COMMAND_CLIP = 240
+
+
+@dataclass(slots=True)
+class Processes:
+    root: int = field(default_factory=os.getpid)
+    _cpu_seen: dict[int, tuple[float, float]] = field(default_factory=dict)
+
+    def tree_parents_first(self) -> list[dict[str, Any]]:
+        try:
+            found = psutil.Process(self.root).children(recursive=True)
+        except psutil.Error:
+            return []
+        now = time.monotonic()
+        rows: dict[int, dict[str, Any]] = {}
+        for process in found:
+            with contextlib.suppress(psutil.Error), process.oneshot():
+                spent = process.cpu_times()
+                rows[process.pid] = {
+                    "pid": process.pid,
+                    "parent": process.ppid(),
+                    "name": process.name(),
+                    "command": _command(process.cmdline()),
+                    "rss": process.memory_info().rss,
+                    "cpu": self._rate(process.pid, spent.user + spent.system, now),
+                    "age_seconds": round(time.time() - process.create_time(), 1),
+                }
+        self._cpu_seen = {p: seen for p, seen in self._cpu_seen.items() if p in rows}
+        children: dict[int, list[int]] = {}
+        for pid, row in rows.items():
+            parent = row["parent"] if row["parent"] in rows else self.root
+            children.setdefault(parent, []).append(pid)
+        ordered: list[dict[str, Any]] = []
+
+        def walk(parent: int, depth: int) -> None:
+            for pid in sorted(children.get(parent, ())):
+                ordered.append({**rows[pid], "depth": depth})
+                walk(pid, depth + 1)
+
+        walk(self.root, 0)
+        return ordered
+
+    def kill_own_descendant(self, pid: int) -> bool:
+        try:
+            owned = psutil.Process(self.root).children(recursive=True)
+            target = next((p for p in owned if p.pid == pid), None)
+            if target is None:
+                return False
+            doomed = [*target.children(recursive=True), target]
+        except psutil.Error:
+            return False
+        for process in doomed:
+            with contextlib.suppress(psutil.Error):
+                process.kill()
+        return True
+
+    def _rate(self, pid: int, spent: float, now: float) -> float | None:
+        before = self._cpu_seen.get(pid)
+        self._cpu_seen[pid] = (spent, now)
+        if before is None or now <= before[1]:
+            return None
+        return round(100 * max(spent - before[0], 0.0) / (now - before[1]), 1)
+
+
+def subtree(rows: list[dict[str, Any]], pid: int | None) -> list[dict[str, Any]]:
+    start = next((i for i, row in enumerate(rows) if row["pid"] == pid), None)
+    if start is None:
+        return []
+    base = rows[start]["depth"]
+    out = [{**rows[start], "depth": 0}]
+    for row in rows[start + 1 :]:
+        if row["depth"] <= base:
+            break
+        out.append({**row, "depth": row["depth"] - base})
+    return out
+
+
+def _command(argv: list[str]) -> str:
+    if not argv:
+        return ""
+    parts = [Path(argv[0]).name]
+    hide = False
+    for arg in argv[1:]:
+        if hide:
+            parts.append("…")
+            hide = False
+            continue
+        hide = arg in _VALUE_HIDDEN
+        parts.append(arg if len(arg) <= _ARG_CLIP else f"{arg[:_ARG_CLIP]}…")
+    command = redact(" ".join(parts))
+    return command if len(command) <= _COMMAND_CLIP else f"{command[:_COMMAND_CLIP]}…"

src/beaver_gateway/backends/sessions.py

@@ -13,12 +13,15 @@ import contextlib
 import logging
 import os
 import time
+from collections import deque
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Protocol
 
 import psutil
 
+from beaver_gateway.backends.processes import Processes, subtree
+
 if TYPE_CHECKING:
     from collections.abc import (
         AsyncIterable,
@@ -43,6 +46,7 @@ DEFAULT_TTL: Mapping[str, float | None] = {
 
 _EVICT_ORDER = {"fork": 0, "job": 0, "deep": 1, "branch": 2, "master": 3}
 _RSS_HEADROOM = 0.8
+STDERR_LINES = 200
 
 
 class SessionClient(Protocol):
@@ -86,6 +90,7 @@ class Session:
     """Summary of the last task notification, the pending turn's prompt."""
     observer: Callable[[Any], None] | None = None
     """The last turn's observer; sees subagent traffic between turns."""
+    stderr: deque[str] = field(default_factory=lambda: deque(maxlen=STDERR_LINES))
 
     @property
     def busy(self) -> bool:
@@ -121,6 +126,7 @@ class SessionPool:
         self._max_live = max_live
         self._ttl = dict(ttl)
         self._reap_interval = reap_interval
+        self._processes = Processes()
 
     def __len__(self) -> int:
         return len(self._sessions)
@@ -168,19 +174,6 @@ class SessionPool:
                 total += child.memory_info().rss
         return total
 
-    @staticmethod
-    def rss_of(session: Session) -> int | None:
-        pid = session.pid
-        if pid is None:
-            return None
-        try:
-            process = psutil.Process(pid)
-            return process.memory_info().rss + sum(
-                c.memory_info().rss for c in process.children(recursive=True)
-            )
-        except psutil.Error:
-            return None
-
     def over_limit(self) -> bool:
         if self._rss_limit is not None:
             return self.rss() > self._rss_limit * _RSS_HEADROOM
@@ -214,11 +207,21 @@ class SessionPool:
         _log.info("closing session %s (%s, %s)", session.session_id, session.kind, key)
         if session.reader is not None and session.reader is not asyncio.current_task():
             session.reader.cancel()
+        if session.attached:
+            session.inbox.put_nowait(RuntimeError(f"session {key} was closed"))
         try:
             await session.client.disconnect()
         except Exception:  # noqa: BLE001
             _log.exception("disconnect failed for session %s", session.session_id)
 
+    async def interrupt(self, key: str) -> bool:
+        session = self._sessions.get(key)
+        if session is None or not session.busy:
+            return False
+        session.interrupt_requested = True
+        await session.client.interrupt()
+        return True
+
     async def close_all(self, *, agent: str | None = None) -> None:
         for session in list(self._sessions.values()):
             if agent is None or session.agent == agent:
@@ -246,27 +249,57 @@ class SessionPool:
                 _log.exception("session reaper failed")
 
     def snapshot(self) -> list[dict[str, Any]]:
+        return self.census()["sessions"]
+
+    def census(self) -> dict[str, Any]:
         now = time.monotonic()
-        return [
+        rows = self._processes.tree_parents_first()
-            {
+        owned: set[int] = set()
-                "key": s.key,
+        sessions: list[dict[str, Any]] = []
-                "agent": s.agent,
+        for s in self._sessions.values():
-                "kind": s.kind,
+            tree = subtree(rows, s.pid)
-                "session_id": s.session_id,
+            owned.update(row["pid"] for row in tree)
-                "pid": s.pid,
+            sessions.append(
-                "rss": self.rss_of(s),
+                {
-                "idle_seconds": round(now - s.last_used, 1),
+                    "key": s.key,
-                "age_seconds": round(now - s.created_at, 1),
+                    "agent": s.agent,
-                "turns": s.turns,
+                    "kind": s.kind,
-                "busy": s.busy,
+                    "session_id": s.session_id,
-                "running_turn": s.running_turn,
+                    "pid": s.pid,
-                "pending": len(s.pending),
+                    "rss": sum(row["rss"] for row in tree) if tree else None,
-                "pending_question": s.pending_question,
+                    "cpu": _cpu(tree),
-                "pinned": s.pinned,
+                    "idle_seconds": round(now - s.last_used, 1),
-                "dirty": s.dirty,
+                    "age_seconds": round(now - s.created_at, 1),
-            }
+                    "turns": s.turns,
-            for s in self._sessions.values()
+                    "busy": s.busy,
-        ]
+                    "running_turn": s.running_turn,
+                    "pending": len(s.pending),
+                    "pending_question": s.pending_question,
+                    "pinned": s.pinned,
+                    "dirty": s.dirty,
+                    "processes": tree,
+                }
+            )
+        return {
+            "rss": sum(row["rss"] for row in rows),
+            "rss_limit": self._rss_limit,
+            "cpu": _cpu(rows),
+            "sessions": sessions,
+            "others": [row for row in rows if row["pid"] not in owned],
+        }
+
+    async def kill(self, pid: int) -> bool:
+        if not self._processes.kill_own_descendant(pid):
+            return False
+        for session in list(self._sessions.values()):
+            if session.pid == pid:
+                await self.close(session.key)
+        return True
+
+
+def _cpu(rows: list[dict[str, Any]]) -> float | None:
+    rates = [row["cpu"] for row in rows if row["cpu"] is not None]
+    return round(sum(rates), 1) if rates else None
 
 
 def cgroup_limit() -> int | None:

src/beaver_gateway/frontends/api/frontend.py

@@ -55,6 +55,7 @@ if TYPE_CHECKING:
     from collections.abc import AsyncIterator, Iterable, Sequence
     from pathlib import Path
 
+    from beaver_gateway.backends.sessions import Session, SessionPool
     from beaver_gateway.conversations.service import Conversations
     from beaver_gateway.frontends.base import GatewayRuntime
     from beaver_gateway.jobs.scheduler import Job, Scheduler
@@ -633,15 +634,74 @@ def build_app(  # noqa: PLR0915
         await require_token(request, runtime, scope=SCOPE)
         return _sse(runtime, conversation_id=None)
 
+    def pool_of() -> SessionPool:
+        if runtime.pool is None:
+            raise HTTPException(status.HTTP_404_NOT_FOUND, "no session pool")
+        return runtime.pool
+
+    def session_of(key: str) -> Session:
+        session = pool_of().get(key)
+        if session is None:
+            raise HTTPException(status.HTTP_404_NOT_FOUND, f"no live session {key}")
+        return session
+
     @app.get("/sessions")
     async def sessions(request: Request) -> dict[str, Any]:
         await require_token(request, runtime, scope=SCOPE)
-        pool = runtime.pool
+        if runtime.pool is None:
-        return {
+            return {
-            "rss": pool.rss() if pool is not None else None,
+                "rss": None,
-            "rss_limit": pool.rss_limit if pool is not None else None,
+                "rss_limit": None,
-            "sessions": pool.snapshot() if pool is not None else [],
+                "cpu": None,
-        }
+                "sessions": [],
+                "others": [],
+            }
+        return runtime.pool.census()
+
+    @app.get("/sessions/{key}/stderr")
+    async def session_stderr(key: str, request: Request) -> dict[str, Any]:
+        await require_token(request, runtime, scope=ADMIN_SCOPE)
+        return {"key": key, "lines": list(session_of(key).stderr)}
+
+    @app.post("/sessions/{key}/interrupt")
+    async def session_interrupt(key: str, request: Request) -> dict[str, Any]:
+        token = await require_token(request, runtime, scope=SCOPE)
+        session = session_of(key)
+        interrupted = await pool_of().interrupt(key)
+        await audit.log(
+            runtime,
+            actor=f"token:{token}",
+            kind="session_interrupt",
+            agent_name=session.agent,
+            session=key,
+            interrupted=interrupted,
+        )
+        return {"key": key, "interrupted": interrupted}
+
+    @app.post("/sessions/{key}/close")
+    async def session_close(key: str, request: Request) -> dict[str, Any]:
+        token = await require_token(request, runtime, scope=ADMIN_SCOPE)
+        session = session_of(key)
+        await pool_of().close(key)
+        await audit.log(
+            runtime,
+            actor=f"token:{token}",
+            kind="session_close",
+            agent_name=session.agent,
+            session=key,
+            busy=session.busy,
+        )
+        return {"key": key, "closed": True}
+
+    @app.post("/processes/{pid}/kill")
+    async def process_kill(pid: int, request: Request) -> dict[str, Any]:
+        token = await require_token(request, runtime, scope=ADMIN_SCOPE)
+        if not await pool_of().kill(pid):
+            raise HTTPException(
+                status.HTTP_404_NOT_FOUND, f"no process {pid} under the gateway"
+            )
+        await audit.log(runtime, actor=f"token:{token}", kind="process_kill", pid=pid)
+        return {"pid": pid, "killed": True}
 
     @app.get("/schedules")
     async def schedules(request: Request) -> dict[str, Any]:

tests/test_api.py

@@ -396,6 +396,54 @@ async def test_describe_snapshots_open_tools_and_records_rate_limit(
     assert page["offset"] == 0 and len(page["entries"]) == 1
 
 
+async def test_sessions_show_their_processes_and_take_orders(world: World) -> None:
+    world.backend._factory = ToolClient  # noqa: SLF001
+    api = Api(world)
+    api.store.grant("api-only", "weak", scope="api")
+    weak = {"Authorization": "Bearer weak"}
+    conv = await world.conversations.create(kind="master", agent="a", origin="test")
+    key = conv.external_id
+    ScriptedClient.hold = asyncio.Event()
+    await world.conversations.post(conv, "go")
+    await asyncio.sleep(0.3)
+    census = await api.get("/sessions")
+    (row,) = census["sessions"]
+    assert row["key"] == key and row["busy"]
+    assert row["processes"] == [] and row["cpu"] is None
+    assert isinstance(census["others"], list)
+    live = world.pool.get(key)
+    assert live is not None
+    live.stderr.append("warning: something")
+    stderr = await api.get(f"/sessions/{key}/stderr")
+    assert stderr["lines"] == ["warning: something"]
+    for path in (f"/sessions/{key}/close", "/processes/1/kill"):
+        assert (await api.http.post(path, headers=weak)).status_code == 403
+    assert (
+        await api.http.get(f"/sessions/{key}/stderr", headers=weak)
+    ).status_code == 403
+
+    closed = await api.http.post(f"/sessions/{key}/close", headers=HEADERS)
+    assert closed.json() == {"key": key, "closed": True}
+    assert world.pool.get(key) is None
+    deadline = asyncio.get_running_loop().time() + 5
+    while ("user", "failed") not in await world.statuses(conv):
+        assert asyncio.get_running_loop().time() < deadline, "the cut turn hangs"
+        await asyncio.sleep(0.02)
+    again = await api.http.post(f"/sessions/{key}/close", headers=HEADERS)
+    assert again.status_code == 404
+    gone = await api.http.post("/processes/999999999/kill", headers=HEADERS)
+    assert gone.status_code == 404
+
+    ScriptedClient.hold = asyncio.Event()
+    await world.conversations.post(conv, "once more")
+    await asyncio.sleep(0.3)
+    interrupted = await api.http.post(f"/sessions/{key}/interrupt", headers=weak)
+    assert interrupted.json() == {"key": key, "interrupted": True}
+    assert ScriptedClient.instances[-1].interrupted
+    kinds = [r["kind"] for r in (await api.get("/audit"))["records"]]
+    assert kinds == ["session_interrupt", "session_close"]
+
+
 async def test_tokens_and_audit_need_admin_scope(world: World) -> None:
     api = Api(world)
     api.store.grant("api-only", "weak", scope="api")

tests/test_processes.py

@@ -0,0 +1,92 @@
+import os
+import subprocess
+import sys
+import time
+
+from beaver_gateway.backends.processes import Processes, _command, subtree
+
+SLEEP = "import time; time.sleep(60)"
+NEST = (
+    "import subprocess, sys, time; "
+    f"subprocess.Popen([sys.executable, '-c', {SLEEP!r}]); time.sleep(60)"
+)
+
+
+def spawn(code: str) -> subprocess.Popen[bytes]:
+    return subprocess.Popen([sys.executable, "-c", code])
+
+
+def rows_under(processes: Processes, pid: int, count: int) -> list[dict]:
+    deadline = time.monotonic() + 5
+    while time.monotonic() < deadline:
+        rows = subtree(processes.tree_parents_first(), pid)
+        if len(rows) >= count:
+            return rows
+        time.sleep(0.05)
+    msg = f"{pid} never grew to {count} processes"
+    raise AssertionError(msg)
+
+
+def test_the_tree_shows_what_the_gateway_started_and_what_it_costs() -> None:
+    child = spawn(SLEEP)
+    try:
+        processes = Processes()
+        (row,) = rows_under(processes, child.pid, 1)
+        assert row["parent"] == os.getpid()
+        assert row["depth"] == 0
+        assert row["rss"] > 0
+        assert row["cpu"] is None
+        assert "time.sleep" in row["command"]
+        (again,) = rows_under(processes, child.pid, 1)
+        assert again["cpu"] is not None
+    finally:
+        child.kill()
+        child.wait()
+
+
+def test_kill_takes_the_branch_and_nothing_the_gateway_did_not_start() -> None:
+    parent = spawn(NEST)
+    try:
+        processes = Processes()
+        top, nested = rows_under(processes, parent.pid, 2)
+        assert (top["depth"], nested["depth"]) == (0, 1)
+        assert not processes.kill_own_descendant(os.getpid())
+        assert not processes.kill_own_descendant(os.getppid())
+        assert processes.kill_own_descendant(parent.pid)
+        parent.wait(timeout=5)
+        alive = {row["pid"] for row in processes.tree_parents_first()}
+        assert not alive & {top["pid"], nested["pid"]}
+    finally:
+        parent.kill()
+        parent.wait()
+
+
+def test_a_subtree_ends_at_the_next_sibling() -> None:
+    rows = [
+        {"pid": 1, "depth": 0},
+        {"pid": 2, "depth": 1},
+        {"pid": 3, "depth": 2},
+        {"pid": 4, "depth": 1},
+        {"pid": 5, "depth": 0},
+    ]
+    assert [(r["pid"], r["depth"]) for r in subtree(rows, 2)] == [(2, 0), (3, 1)]
+    assert [r["pid"] for r in subtree(rows, 1)] == [1, 2, 3, 4]
+    assert subtree(rows, 9) == []
+    assert subtree(rows, None) == []
+
+
+def test_a_command_line_keeps_prompts_and_configs_to_itself() -> None:
+    argv = [
+        "/opt/claude/claude",
+        "--model",
+        "m",
+        "--system-prompt",
+        "you are a beaver",
+        "--mcp-config",
+        '{"gateway": {"headers": {"Authorization": "Bearer abc"}}}',
+        "x" * 100,
+    ]
+    assert _command(argv) == (
+        "claude --model m --system-prompt … --mcp-config … " + "x" * 80 + "…"
+    )
+    assert _command([]) == ""

ui/src/lib/api/client.ts

@@ -313,6 +313,22 @@ export class ApiClient {
     return this.get("/api/sessions");
   }
 
+  sessionStderr(key: string): Promise<{ key: string; lines: string[] }> {
+    return this.get(`/api/sessions/${encodeURIComponent(key)}/stderr`);
+  }
+
+  interruptSession(key: string): Promise<{ interrupted: boolean }> {
+    return this.post(`/api/sessions/${encodeURIComponent(key)}/interrupt`);
+  }
+
+  closeSession(key: string): Promise<{ closed: boolean }> {
+    return this.post(`/api/sessions/${encodeURIComponent(key)}/close`);
+  }
+
+  killProcess(pid: number): Promise<{ killed: boolean }> {
+    return this.post(`/api/processes/${pid}/kill`);
+  }
+
   jobs(): Promise<JobsResponse> {
     return this.get("/api/jobs");
   }

ui/src/lib/api/types.ts

@@ -106,7 +106,9 @@ export interface ContentBlock {
 
 export interface HistoryMessage {
   content: string | ContentBlock[];
+  id?: string | null;
   role: "user" | "assistant";
+  service?: string;
   // When the first block of this message was written; absent for old rows.
   ts?: string | null;
 }
@@ -144,17 +146,31 @@ export interface AgentsResponse {
   mcps: { name: string; kind: string }[];
 }
 
+export interface ProcessRow {
+  age_seconds: number;
+  command: string;
+  cpu: number | null;
+  depth: number;
+  name: string;
+  parent: number;
+  pid: number;
+  rss: number;
+}
+
 export interface SessionRow {
   age_seconds: number;
   agent: string;
   busy: boolean;
+  cpu: number | null;
   dirty: boolean;
   idle_seconds: number;
   key: string;
   kind: string;
+  pending: number;
   pending_question: boolean;
   pid: number | null;
   pinned: boolean;
+  processes: ProcessRow[];
   rss: number | null;
   running_turn: string | null;
   session_id: string | null;
@@ -162,6 +178,8 @@ export interface SessionRow {
 }
 
 export interface SessionsResponse {
+  cpu: number | null;
+  others: ProcessRow[];
   rss: number | null;
   rss_limit: number | null;
   sessions: SessionRow[];

ui/src/lib/format.ts

@@ -68,6 +68,12 @@ export function fmtSeconds(seconds: number | null | undefined): string {
     : fmtDuration(seconds * THOUSAND);
 }
 
+export function fmtCpu(percent: number | null | undefined): string {
+  return percent === null || percent === undefined
+    ? "–"
+    : `${Math.round(percent)}%`;
+}
+
 export function fmtBytes(bytes: number | null | undefined): string {
   if (bytes === null || bytes === undefined) {
     return "–";

ui/src/lib/system/process-table.svelte

@@ -0,0 +1,68 @@
+<script lang="ts">
+  import type { ProcessRow } from "$lib/api/types";
+  import { Button } from "$lib/components/ui/button";
+  import { fmtBytes, fmtCpu, fmtSeconds } from "$lib/format";
+
+  let {
+    rows,
+    onKill,
+  }: { rows: ProcessRow[]; onKill: (row: ProcessRow) => void } = $props();
+
+  const INDENT_REM = 0.875;
+</script>
+
+<div class="overflow-x-auto">
+  <table class="w-full text-sm">
+    <thead class="text-muted-foreground text-xs">
+      <tr class="border-b text-left">
+        <th class="py-1.5 pr-3 font-medium">pid</th>
+        <th class="w-full py-1.5 pr-3 font-medium">command</th>
+        <th class="py-1.5 pr-3 text-right font-medium">cpu</th>
+        <th class="py-1.5 pr-3 text-right font-medium">rss</th>
+        <th class="hidden py-1.5 pr-3 text-right font-medium sm:table-cell">
+          age
+        </th>
+        <th class="py-1.5"><span class="sr-only">actions</span></th>
+      </tr>
+    </thead>
+    <tbody>
+      {#each rows as row (row.pid)}
+        <tr class="border-b last:border-b-0">
+          <td class="tabular py-1 pr-3 text-muted-foreground text-xs">
+            {row.pid}
+          </td>
+          <td class="max-w-0 py-1 pr-3">
+            <span
+              class="block truncate font-mono text-xs"
+              title={row.command}
+              style:padding-left="{row.depth * INDENT_REM}rem"
+            >
+              {row.command || row.name}
+            </span>
+          </td>
+          <td class="tabular py-1 pr-3 text-right text-xs">
+            {fmtCpu(row.cpu)}
+          </td>
+          <td class="tabular whitespace-nowrap py-1 pr-3 text-right text-xs">
+            {fmtBytes(row.rss)}
+          </td>
+          <td
+            class="tabular hidden whitespace-nowrap py-1 pr-3 text-right text-muted-foreground text-xs sm:table-cell"
+          >
+            {fmtSeconds(row.age_seconds)}
+          </td>
+          <td class="py-0.5 text-right">
+            <Button
+              class="text-muted-foreground hover:text-destructive max-sm:h-10"
+              onclick={() => onKill(row)}
+              size="xs"
+              variant="ghost"
+            >
+              Kill
+            </Button>
+          </td>
+        </tr>
+      {/each}
+    </tbody>
+  </table>
+</div>

ui/src/lib/system/session-detail.svelte

@@ -0,0 +1,150 @@
+<script lang="ts">
+  import { onMount, untrack } from "svelte";
+  import { base } from "$app/paths";
+  import type { ApiClient } from "$lib/api/client";
+  import type { ProcessRow, SessionRow } from "$lib/api/types";
+  import { Button } from "$lib/components/ui/button";
+  import { shortId } from "$lib/format";
+  import { ConversationFeed } from "$lib/panel/conversation.svelte";
+  import TurnCard from "$lib/panel/turn-card.svelte";
+  import ProcessTable from "./process-table.svelte";
+
+  let {
+    client,
+    row,
+    busy,
+    note,
+    onInterrupt,
+    onClose,
+    onKill,
+  }: {
+    client: ApiClient;
+    row: SessionRow;
+    busy: boolean;
+    note: string;
+    onInterrupt: (row: SessionRow) => void;
+    onClose: (row: SessionRow) => void;
+    onKill: (process: ProcessRow) => void;
+  } = $props();
+
+  const TICK_MS = 1000;
+  const feed = new ConversationFeed(
+    () => client,
+    untrack(() => row.key)
+  );
+  let stderr = $state<string[] | null>(null);
+  let now = $state(Date.now());
+
+  const turn = $derived(feed.model.running);
+
+  onMount(() => {
+    feed.start();
+    const timer = setInterval(() => {
+      now = Date.now();
+    }, TICK_MS);
+    return () => {
+      feed.stop();
+      clearInterval(timer);
+    };
+  });
+
+  $effect(() => {
+    const { key, idle_seconds: polled } = row;
+    if (polled >= 0) {
+      client
+        .sessionStderr(key)
+        .then((tail) => {
+          stderr = tail.lines;
+        })
+        .catch(() => undefined);
+    }
+  });
+</script>
+
+<div class="flex flex-col gap-4 px-3 pt-2 pb-4">
+  <div class="flex flex-wrap items-center gap-x-4 gap-y-2">
+    <p class="tabular min-w-0 flex-1 text-muted-foreground text-xs">
+      session
+      {shortId(row.session_id)}
+      {#if row.pid}
+        · pid {row.pid}
+      {/if}
+      {#if row.pending > 0}
+        · {row.pending}
+        buffered
+      {/if}
+    </p>
+    <div class="flex flex-wrap items-center gap-2">
+      {#if note}
+        <span class="text-muted-foreground text-xs" role="status">{note}</span>
+      {/if}
+      {#if !feed.error}
+        <Button
+          class="max-sm:h-10"
+          href="{base}/conversations/{row.key}"
+          size="sm"
+          variant="ghost"
+        >
+          Open thread
+        </Button>
+      {/if}
+      <Button
+        class="max-sm:h-10"
+        disabled={busy || !row.busy}
+        onclick={() => onInterrupt(row)}
+        size="sm"
+        variant="outline"
+      >
+        Interrupt turn
+      </Button>
+      <Button
+        class="max-sm:h-10"
+        disabled={busy}
+        onclick={() => onClose(row)}
+        size="sm"
+        variant="destructive"
+      >
+        Close session
+      </Button>
+    </div>
+  </div>
+
+  <section class="flex flex-col gap-1">
+    <h4 class="label-quiet">Now</h4>
+    {#if turn}
+      <TurnCard {now} {turn} />
+    {:else}
+      <p class="text-muted-foreground text-sm">
+        {row.busy
+          ? "A turn is running; its events have not reached this page yet."
+          : "Nothing in flight."}
+      </p>
+    {/if}
+  </section>
+
+  <section class="flex flex-col gap-1">
+    <h4 class="label-quiet">Process tree</h4>
+    {#if row.processes.length === 0}
+      <p class="text-muted-foreground text-sm">
+        The process is not visible to the gateway.
+      </p>
+    {:else}
+      <ProcessTable {onKill} rows={row.processes} />
+    {/if}
+  </section>
+
+  <section class="flex flex-col gap-1">
+    <h4 class="label-quiet">stderr</h4>
+    {#if stderr?.length === 0}
+      <p class="text-muted-foreground text-sm">
+        Nothing on stderr since this process started.
+      </p>
+    {:else if stderr}
+      <pre
+        class="max-h-64 overflow-auto whitespace-pre-wrap break-all rounded-md bg-muted/40 p-2 text-xs"
+      >{stderr.join(
+          "\n"
+        )}</pre>
+    {/if}
+  </section>
+</div>

ui/src/routes/system/+page.svelte

@@ -1,16 +1,39 @@
 <script lang="ts">
+  import ChevronRightIcon from "@lucide/svelte/icons/chevron-right";
   import { onMount } from "svelte";
-  import { base } from "$app/paths";
+  import type {
-  import type { SessionsResponse } from "$lib/api/types";
+    ProcessRow,
+    SessionRow,
+    SessionsResponse,
+  } from "$lib/api/types";
   import ErrorNote from "$lib/components/error-note.svelte";
-  import { fmtBytes, fmtSeconds, shortId } from "$lib/format";
+  import { Button } from "$lib/components/ui/button";
+  import * as Dialog from "$lib/components/ui/dialog";
+  import { fmtBytes, fmtCpu, fmtSeconds, shortId } from "$lib/format";
+  import { gateway } from "$lib/gateway.svelte";
   import { session } from "$lib/session.svelte";
   import KindMark from "$lib/shell/kind-mark.svelte";
+  import ProcessTable from "$lib/system/process-table.svelte";
+  import SessionDetail from "$lib/system/session-detail.svelte";
   import { cn } from "$lib/utils";
 
-  const POLL_MS = 10_000;
+  interface Order {
+    confirm: string;
+    lines: string[];
+    run: () => Promise<unknown>;
+    title: string;
+  }
+
+  const POLL_MS = 3000;
+  const SECOND_LOOK_FOR_CPU_MS = 1000;
+  const COLUMNS = 8;
   let sessions = $state<SessionsResponse | null>(null);
   let failure = $state<string | null>(null);
+  let unfolded = $state<Set<string>>(new Set());
+  let order = $state<Order | null>(null);
+  let orderFailure = $state<string | null>(null);
+  let notes = $state<Record<string, string>>({});
+  let busy = $state(false);
 
   async function load() {
     const { client } = session;
@@ -25,90 +48,315 @@
     }
   }
 
+  function toggle(key: string) {
+    const next = new Set(unfolded);
+    if (!next.delete(key)) {
+      next.add(key);
+    }
+    unfolded = next;
+  }
+
+  function ask(next: Order) {
+    orderFailure = null;
+    order = next;
+  }
+
+  async function carryOut(pending: Order) {
+    busy = true;
+    try {
+      await pending.run();
+      order = null;
+      await load();
+    } catch (cause) {
+      orderFailure = cause instanceof Error ? cause.message : String(cause);
+    } finally {
+      busy = false;
+    }
+  }
+
+  async function interrupt(row: SessionRow) {
+    const { client } = session;
+    if (!client) {
+      return;
+    }
+    busy = true;
+    try {
+      const { interrupted } = await client.interruptSession(row.key);
+      notes = {
+        ...notes,
+        [row.key]: interrupted ? "" : "Nothing was running.",
+      };
+      await load();
+    } catch (cause) {
+      const message = cause instanceof Error ? cause.message : String(cause);
+      notes = { ...notes, [row.key]: message };
+    } finally {
+      busy = false;
+    }
+  }
+
+  function askClose(row: SessionRow) {
+    const { client } = session;
+    if (!client) {
+      return;
+    }
+    const lines = [
+      "The claude process exits. The conversation stays and resumes from its transcript on the next message.",
+    ];
+    if (row.busy) {
+      lines.push("A turn is running: it is cut and its reply is lost.");
+    }
+    if (row.pinned) {
+      lines.push("This session is pinned: the gateway never closes it itself.");
+    }
+    ask({
+      confirm: "Close session",
+      lines,
+      run: () => client.closeSession(row.key),
+      title: `Close the ${row.kind} session?`,
+    });
+  }
+
+  function askKill(process: ProcessRow) {
+    const { client } = session;
+    if (!client) {
+      return;
+    }
+    const owner = sessions?.sessions.find((row) => row.pid === process.pid);
+    const lines = [
+      `SIGKILL to ${process.name} and everything it started. Nothing gets to clean up.`,
+    ];
+    if (owner) {
+      lines.push(
+        "It is a session's claude: the session is dropped, and the conversation resumes from its transcript on the next message."
+      );
+    }
+    ask({
+      confirm: "Kill",
+      lines,
+      run: () => client.killProcess(process.pid),
+      title: `Kill process ${process.pid}?`,
+    });
+  }
+
+  const titles = $derived(
+    new Map(gateway.conversations.map((row) => [row.id, row.title]))
+  );
+
   onMount(() => {
     load();
+    const second = setTimeout(load, SECOND_LOOK_FOR_CPU_MS);
     const poll = setInterval(load, POLL_MS);
-    return () => clearInterval(poll);
+    return () => {
+      clearTimeout(second);
+      clearInterval(poll);
+    };
   });
 </script>
 
-<section class="flex flex-col gap-3">
+{#snippet state(s: SessionRow)}
-  <div class="flex items-baseline gap-3">
+  <span class={cn(s.busy ? "text-signal" : "text-muted-foreground")}>
-    <h2 class="font-medium text-sm">Live claude processes</h2>
+    {s.busy ? "busy" : "idle"}
-    {#if sessions}
+  </span>
-      <span class="tabular text-muted-foreground text-xs">
+  {#if s.pinned}
-        {sessions.sessions.length}
+    <span class="ml-1 text-muted-foreground">pinned</span>
-        · {fmtBytes(sessions.rss)}
+  {/if}
-        {#if sessions.rss_limit}
+  {#if s.dirty}
-          of {fmtBytes(sessions.rss_limit)}
+    <span class="ml-1 text-warn">dirty</span>
-        {/if}
+  {/if}
-      </span>
+  {#if s.pending_question}
-    {/if}
+    <span class="ml-1 text-attention-foreground">question</span>
-  </div>
+  {/if}
+{/snippet}
+
+<div class="flex flex-wrap items-baseline gap-3">
+  <h2 class="font-medium text-sm">Live claude processes</h2>
+  {#if sessions}
+    <span class="tabular text-muted-foreground text-xs">
+      {sessions.sessions.length}
+      {sessions.sessions.length === 1 ? "session" : "sessions"}
+      · {fmtBytes(sessions.rss)}
+      {#if sessions.rss_limit}
+        of {fmtBytes(sessions.rss_limit)}
+      {/if}
+      · cpu {fmtCpu(sessions.cpu)}
+    </span>
+  {/if}
+</div>
+
+<div class="flex flex-col gap-6">
   {#if failure}
     <ErrorNote message={failure} retry={load} />
   {:else if !sessions}
     <p class="text-muted-foreground text-sm">Loading…</p>
-  {:else if sessions.sessions.length === 0}
-    <p class="text-muted-foreground text-sm">
-      No claude processes alive. Sessions spawn on the first turn and are reaped
-      by idle time or memory pressure.
-    </p>
   {:else}
-    <div class="overflow-x-auto rounded-lg border bg-strip">
+    <section class="flex flex-col gap-2">
-      <table class="w-full text-sm">
+      <h3 class="label-quiet">Sessions</h3>
-        <thead class="text-muted-foreground text-xs">
+      {#if sessions.sessions.length === 0}
-          <tr class="border-b text-left">
+        <p class="text-muted-foreground text-sm">
-            <th class="px-3 py-2 font-medium">conversation</th>
+          No claude processes alive. Sessions spawn on the first turn and are
-            <th class="px-3 py-2 font-medium">agent</th>
+          reaped by idle time or memory pressure.
-            <th class="px-3 py-2 font-medium">state</th>
+        </p>
-            <th class="px-3 py-2 text-right font-medium">rss</th>
+      {:else}
-            <th class="px-3 py-2 text-right font-medium">idle</th>
+        <div class="overflow-x-auto rounded-lg border bg-strip">
-            <th class="px-3 py-2 text-right font-medium">age</th>
+          <table class="w-full text-sm">
-            <th class="px-3 py-2 text-right font-medium">turns</th>
+            <thead class="text-muted-foreground text-xs">
-          </tr>
+              <tr class="border-b text-left">
-        </thead>
+                <th class="px-3 py-2 font-medium">conversation</th>
-        <tbody>
+                <th class="hidden px-3 py-2 font-medium sm:table-cell">
-          {#each sessions.sessions as s (s.key)}
+                  agent
-            <tr class="row-hover border-b last:border-b-0">
+                </th>
-              <td class="px-3 py-1.5">
+                <th class="hidden px-3 py-2 font-medium sm:table-cell">
-                <a
+                  state
-                  class="flex items-center gap-2 hover:underline"
+                </th>
-                  href="{base}/conversations/{s.key}"
+                <th class="px-3 py-2 text-right font-medium">cpu</th>
+                <th class="px-3 py-2 text-right font-medium">rss</th>
+                <th
+                  class="hidden px-3 py-2 text-right font-medium sm:table-cell"
                 >
-                  <KindMark kind={s.kind} />
+                  idle
-                  <span class="tabular text-xs">{shortId(s.key)}</span>
+                </th>
-                </a>
+                <th
-              </td>
+                  class="hidden px-3 py-2 text-right font-medium sm:table-cell"
-              <td class="px-3 py-1.5">{s.agent}</td>
-              <td class="px-3 py-1.5 text-xs">
-                <span
-                  class={cn(s.busy ? "text-signal" : "text-muted-foreground")}
                 >
-                  {s.busy ? "busy" : "idle"}
+                  age
-                </span>
+                </th>
-                {#if s.pinned}
+                <th
-                  <span class="ml-1 text-muted-foreground">pinned</span>
+                  class="hidden px-3 py-2 text-right font-medium sm:table-cell"
+                >
+                  turns
+                </th>
+              </tr>
+            </thead>
+            <tbody>
+              {#each sessions.sessions as s (s.key)}
+                {@const expanded = unfolded.has(s.key)}
+                <tr class={cn("border-b", !expanded && "last:border-b-0")}>
+                  <td class="max-w-0 px-1.5 py-1 sm:w-2/5">
+                    <button
+                      aria-expanded={expanded}
+                      class="row-hover flex max-w-full items-center gap-2 rounded-md px-1.5 py-1 text-left max-sm:min-h-10"
+                      onclick={() => toggle(s.key)}
+                      type="button"
+                    >
+                      <ChevronRightIcon
+                        class={cn(
+                          "size-3.5 text-icon transition-transform duration-150",
+                          expanded && "rotate-90"
+                        )}
+                      />
+                      <KindMark kind={s.kind} />
+                      <span class="flex min-w-0 flex-col">
+                        <span class="truncate font-medium">
+                          {titles.get(s.key) ?? shortId(s.key)}
+                        </span>
+                        <span class="text-xs sm:hidden">
+                          {@render state(s)}
+                        </span>
+                      </span>
+                    </button>
+                  </td>
+                  <td class="hidden px-3 py-1.5 sm:table-cell">{s.agent}</td>
+                  <td
+                    class="hidden whitespace-nowrap px-3 py-1.5 text-xs sm:table-cell"
+                  >
+                    {@render state(s)}
+                  </td>
+                  <td class="tabular px-3 py-1.5 text-right">
+                    {fmtCpu(s.cpu)}
+                  </td>
+                  <td class="tabular whitespace-nowrap px-3 py-1.5 text-right">
+                    {fmtBytes(s.rss)}
+                  </td>
+                  <td
+                    class="tabular hidden px-3 py-1.5 text-right sm:table-cell"
+                  >
+                    {fmtSeconds(s.idle_seconds)}
+                  </td>
+                  <td
+                    class="tabular hidden px-3 py-1.5 text-right sm:table-cell"
+                  >
+                    {fmtSeconds(s.age_seconds)}
+                  </td>
+                  <td
+                    class="tabular hidden px-3 py-1.5 text-right sm:table-cell"
+                  >
+                    {s.turns}
+                  </td>
+                </tr>
+                {#if expanded && session.client}
+                  <tr class="border-b last:border-b-0">
+                    <td class="bg-background/60" colspan={COLUMNS}>
+                      <SessionDetail
+                        {busy}
+                        client={session.client}
+                        note={notes[s.key] ?? ""}
+                        onClose={askClose}
+                        onInterrupt={interrupt}
+                        onKill={askKill}
+                        row={s}
+                      />
+                    </td>
+                  </tr>
                 {/if}
-                {#if s.dirty}
+              {/each}
-                  <span class="ml-1 text-warn">dirty</span>
+            </tbody>
-                {/if}
+          </table>
-                {#if s.pending_question}
+        </div>
-                  <span class="ml-1 text-attention-foreground">question</span>
+      {/if}
-                {/if}
+    </section>
-              </td>
+
-              <td class="tabular px-3 py-1.5 text-right">{fmtBytes(s.rss)}</td>
+    <section class="flex flex-col gap-2">
-              <td class="tabular px-3 py-1.5 text-right">
+      <h3 class="label-quiet">Outside any session</h3>
-                {fmtSeconds(s.idle_seconds)}
+      {#if sessions.others.length === 0}
-              </td>
+        <p class="text-muted-foreground text-sm">
-              <td class="tabular px-3 py-1.5 text-right">
+          Nothing else runs under the gateway.
-                {fmtSeconds(s.age_seconds)}
+        </p>
-              </td>
+      {:else}
-              <td class="tabular px-3 py-1.5 text-right">{s.turns}</td>
+        <ProcessTable onKill={askKill} rows={sessions.others} />
-            </tr>
+      {/if}
-          {/each}
+    </section>
-        </tbody>
-      </table>
-    </div>
   {/if}
-</section>
+</div>
+
+<Dialog.Root
+  onOpenChange={(next) => {
+    if (!next) {
+      order = null;
+    }
+  }}
+  open={order !== null}
+>
+  <Dialog.Content>
+    {#if order}
+      {@const pending = order}
+      <Dialog.Header>
+        <Dialog.Title>{pending.title}</Dialog.Title>
+        {#each pending.lines as line (line)}
+          <Dialog.Description>{line}</Dialog.Description>
+        {/each}
+        {#if orderFailure}
+          <p class="text-destructive text-sm" role="alert">{orderFailure}</p>
+        {/if}
+      </Dialog.Header>
+      <Dialog.Footer>
+        <Button
+          disabled={busy}
+          onclick={() => {
+            order = null;
+          }}
+          variant="ghost"
+        >
+          Cancel
+        </Button>
+        <Button
+          disabled={busy}
+          onclick={() => carryOut(pending)}
+          variant="destructive"
+        >
+          {pending.confirm}
+        </Button>
+      </Dialog.Footer>
+    {/if}
+  </Dialog.Content>
+</Dialog.Root>