feat(ui,api): strip board redesign - island, rail by day, context view, server search, vault graph

src/beaver_gateway/conversations/context.py

@@ -0,0 +1,201 @@
+"""What a conversation's model holds: prompt granules, skills, tools, files touched."""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Mapping
+
+    from beaver_gateway.agents.claude import ClaudeAgent
+    from beaver_gateway.conversations.kinds import Kind
+
+__all__ = ["compose", "files_touched", "granules", "skill_sets"]
+
+CHARS_PER_TOKEN = 3.2
+READ_TOOLS = frozenset({"Read", "NotebookRead"})
+WRITE_TOOLS = frozenset({"Write", "Edit", "MultiEdit", "NotebookEdit"})
+SEARCH_TOOLS = frozenset({"Glob", "Grep"})
+PATH_KEYS = ("file_path", "notebook_path", "path", "file", "filename")
+_FRONTMATTER = re.compile(r"^---\s*\n(.*?)\n---", re.DOTALL)
+
+
+def _tokens(chars: int) -> int:
+    return round(chars / CHARS_PER_TOKEN)
+
+
+def _relative(path: str, cwd: Path | None) -> str:
+    if cwd is None:
+        return path
+    try:
+        return str(Path(path).relative_to(cwd))
+    except ValueError:
+        return path
+
+
+def _tool_uses(entries: Iterable[Mapping[str, Any]]) -> Iterable[tuple[str, Any, str]]:
+    for entry in entries:
+        message = entry.get("message")
+        if entry.get("type") != "assistant" or not isinstance(message, dict):
+            continue
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        stamp = str(entry.get("timestamp") or "")
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "tool_use":
+                yield str(block.get("name") or "?"), block.get("input"), stamp
+
+
+def files_touched(
+    entries: Iterable[Mapping[str, Any]], *, cwd: Path | None = None
+) -> tuple[list[dict[str, Any]], dict[str, int]]:
+    """Files named in tool inputs, with how they were touched, plus tool counts."""
+    files: dict[str, dict[str, Any]] = {}
+    counts: Counter[str] = Counter()
+    for name, tool_input, stamp in _tool_uses(entries):
+        counts[name] += 1
+        if not isinstance(tool_input, dict):
+            continue
+        raw = next(
+            (v for k in PATH_KEYS if isinstance(v := tool_input.get(k), str) and v),
+            None,
+        )
+        if raw is None or (name in SEARCH_TOOLS and "file_path" not in tool_input):
+            continue
+        key = _relative(raw, cwd)
+        row = files.setdefault(
+            key, {"path": key, "reads": 0, "writes": 0, "other": 0, "last_at": stamp}
+        )
+        if name in READ_TOOLS:
+            row["reads"] += 1
+        elif name in WRITE_TOOLS:
+            row["writes"] += 1
+        else:
+            row["other"] += 1
+        row["last_at"] = max(row["last_at"], stamp)
+    ordered = sorted(files.values(), key=lambda r: r["last_at"], reverse=True)
+    return ordered, dict(counts)
+
+
+def granules(agent: ClaudeAgent, kind: Kind) -> dict[str, Any]:
+    """The system prompt as its sources: tag, path, size, token estimate."""
+    sources = agent.prompt_for(kind) if agent.serves(kind) else None
+    rows: list[dict[str, Any]] = []
+    if sources is None:
+        chars = len(agent.system_prompt)
+        if chars:
+            rows.append(
+                {
+                    "tag": None,
+                    "path": None,
+                    "bytes": chars,
+                    "tokens_est": _tokens(chars),
+                }
+            )
+    else:
+        for source in sources:
+            tag, raw = source if isinstance(source, tuple) else (None, source)
+            path = Path(raw)
+            try:
+                chars = len(path.read_text(encoding="utf-8"))
+            except OSError:
+                chars = 0
+            rows.append(
+                {
+                    "tag": tag,
+                    "path": _relative(str(path), agent.cwd),
+                    "bytes": chars,
+                    "tokens_est": _tokens(chars),
+                }
+            )
+    total = sum(r["bytes"] for r in rows)
+    return {"bytes": total, "tokens_est": _tokens(total), "granules": rows}
+
+
+def _skill_meta(skill_md: Path) -> dict[str, str]:
+    try:
+        text = skill_md.read_text(encoding="utf-8")
+    except OSError:
+        return {}
+    match = _FRONTMATTER.match(text)
+    meta: dict[str, str] = {}
+    if match:
+        for line in match.group(1).splitlines():
+            key, sep, value = line.partition(":")
+            if sep:
+                meta[key.strip()] = value.strip()
+    return meta
+
+
+def skill_sets(agent: ClaudeAgent, kind: Kind) -> list[dict[str, Any]]:
+    """Skill-set directories the kind loads, with each skill's name and description."""
+    out: list[dict[str, Any]] = []
+    for directory in agent.skills_for(kind):
+        skills = []
+        try:
+            children = sorted(p for p in directory.iterdir() if p.is_dir())
+        except OSError:
+            children = []
+        for child in children:
+            skill_md = child / "SKILL.md"
+            if skill_md.is_file():
+                meta = _skill_meta(skill_md)
+                skills.append(
+                    {
+                        "name": meta.get("name") or child.name,
+                        "description": meta.get("description", ""),
+                        "path": _relative(str(child), agent.cwd),
+                    }
+                )
+        out.append(
+            {
+                "set": directory.name,
+                "path": _relative(str(directory), agent.cwd),
+                "skills": skills,
+            }
+        )
+    return out
+
+
+def compose(
+    agent: ClaudeAgent,
+    kind: Kind,
+    entries: Iterable[Mapping[str, Any]],
+    *,
+    context_tokens: int,
+) -> dict[str, Any]:
+    files, counts = files_touched(entries, cwd=agent.cwd)
+    prompt = granules(agent, kind)
+    skills = skill_sets(agent, kind)
+    skills_chars = sum(
+        len(s["name"]) + len(s["description"])
+        for group in skills
+        for s in group["skills"]
+    )
+    return {
+        "agent": {
+            "name": agent.name,
+            "model": agent.model,
+            "effort": agent.options.effort,
+        },
+        "kind": kind,
+        "context_tokens": context_tokens,
+        "prompt": prompt,
+        "skills": skills,
+        "skills_tokens_est": _tokens(skills_chars),
+        "tools": {
+            "allowed": list(agent.options.tools) if agent.options.tools else None,
+            "disallowed": list(agent.options.disallowed_tools),
+            "gateway": list(agent.gateway_tools),
+            "mcps": [m.name for m in agent.expose_mcps],
+        },
+        "files": files,
+        "tool_counts": counts,
+        "history_tokens_est": max(
+            0, context_tokens - prompt["tokens_est"] - _tokens(skills_chars)
+        ),
+    }

src/beaver_gateway/conversations/rows.py

@@ -8,6 +8,8 @@ from datetime import UTC, datetime
 from typing import TYPE_CHECKING, Any, cast
 from uuid import uuid4
 
+from sqlalchemy import String, func
+from sqlalchemy import cast as sa_cast
 from sqlmodel import col, select
 
 from beaver_gateway.backends.transcript import (
@@ -23,6 +25,7 @@ from beaver_gateway.storage.models import (
     ConversationBinding,
     ConversationMessage,
     RateLimit,
+    TranscriptEntry,
     Usage,
 )
 
@@ -132,6 +135,30 @@ class Rows(State):
         async with self._db.session() as session:
             return list((await session.exec(stmt)).all())
 
+    async def search(self, query: str, *, limit: int = 20) -> list[Conversation]:
+        needle = f"%{query}%"
+        by_title = select(Conversation.id).where(col(Conversation.title).ilike(needle))
+        by_first = select(ConversationMessage.conversation_id).where(
+            ConversationMessage.seq == 0,
+            col(ConversationMessage.content_json).ilike(needle),
+        )
+        by_entry = (
+            select(Conversation.id)
+            .join(
+                TranscriptEntry,
+                col(TranscriptEntry.session_id) == col(Conversation.session_id),
+            )
+            .where(sa_cast(col(TranscriptEntry.entry), String).ilike(needle))
+        )
+        stmt = (
+            select(Conversation)
+            .where(col(Conversation.id).in_(by_title.union(by_first, by_entry)))
+            .order_by(col(Conversation.id).desc())
+            .limit(limit)
+        )
+        async with self._db.session() as session:
+            return list((await session.exec(stmt)).all())
+
     async def bindings(self, conv: Conversation) -> list[ConversationBinding]:
         async with self._db.session() as session:
             result = await session.exec(
@@ -317,6 +344,7 @@ class Rows(State):
     async def describe(self, conv: Conversation) -> dict[str, Any]:
         out = self.public(conv)
         out["title"] = await self.implied_title(conv)
+        out["context_tokens"] = await self.context_tokens(conv)
         parent = await self.get_row(conv.parent_id) if conv.parent_id else None
         out["parent"] = parent.external_id if parent is not None else None
         out["bindings"] = [
@@ -359,6 +387,25 @@ class Rows(State):
             ).first()
         return context_of(row)
 
+    async def context_tokens_by_id(self, ids: Iterable[str]) -> dict[str, int]:
+        wanted = list(ids)
+        if not wanted:
+            return {}
+        latest = (
+            select(func.max(Usage.id))
+            .where(col(Usage.conversation_id).in_(wanted))
+            .group_by(Usage.conversation_id)
+        )
+        async with self._db.session() as session:
+            rows = (
+                await session.exec(select(Usage).where(col(Usage.id).in_(latest)))
+            ).all()
+        return {
+            cast("str", row.conversation_id): context_of(row)
+            for row in rows
+            if row.conversation_id
+        }
+
     async def usage_tokens(self, since: datetime) -> int:
         async with self._db.session() as session:
             rows = (

src/beaver_gateway/frontends/api/frontend.py

@@ -18,6 +18,10 @@ from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse, StreamingResponse
 from sqlmodel import col, select
 
+from beaver_gateway.agents.claude import ClaudeAgent
+from beaver_gateway.agents.prompts import assemble
+from beaver_gateway.backends.transcript import prompt_count
+from beaver_gateway.conversations.context import compose
 from beaver_gateway.conversations.injects import URGENCY
 from beaver_gateway.conversations.kinds import Kind, as_kind
 from beaver_gateway.conversations.service import SEEDS, implied_title
@@ -45,6 +49,7 @@ from beaver_gateway.storage.models import (
     Token,
     Usage,
 )
+from beaver_gateway.vault.links import LinkIndex
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator, Iterable, Sequence
@@ -70,6 +75,10 @@ WINDOWS: dict[str, timedelta] = {
 MEMORY_MAX_DEPTH = 12
 MEMORY_MAX_FILE = 2_000_000
 MEMORY_MAX_ENTRIES = 5000
+SEARCH_MIN = 2
+GRAPH_LIMIT = 120
+SEARCH_LIMIT = 20
+SNIPPET_MAX = 160
 
 
 class ApiFrontend(Frontend):
@@ -85,12 +94,14 @@ class ApiFrontend(Frontend):
         deep_agent: str | None = None,
         job_agent: str | None = None,
         memory_root: Path | None = None,
+        vault_root: Path | None = None,
     ) -> None:
         self.master_agent = master_agent
         self.branch_agent = branch_agent
         self.deep_agent = deep_agent
         self.job_agent = job_agent
         self.memory_root = memory_root.resolve() if memory_root is not None else None
+        self.vault_root = vault_root.resolve() if vault_root is not None else None
         self._app: FastAPI | None = None
 
     def agent_for(self, kind: Kind) -> str | None:
@@ -105,14 +116,22 @@ class ApiFrontend(Frontend):
         if runtime.conversations is None or runtime.bus is None:
             msg = "ApiFrontend needs runtime.conversations and runtime.bus"
             raise RuntimeError(msg)
-        self._app = build_app(runtime, memory_root=self.memory_root)
+        self._app = build_app(
+            runtime, memory_root=self.memory_root, vault_root=self.vault_root
+        )
 
     def app(self) -> FastAPI | None:
         return self._app
 
 
-def build_app(runtime: GatewayRuntime, *, memory_root: Path | None = None) -> FastAPI:  # noqa: PLR0915
+def build_app(  # noqa: PLR0915
+    runtime: GatewayRuntime,
+    *,
+    memory_root: Path | None = None,
+    vault_root: Path | None = None,
+) -> FastAPI:
     app = FastAPI(title="beaver-gateway / API")
+    links = LinkIndex(vault_root) if vault_root is not None else None
     app.add_middleware(
         CORSMiddleware,
         allow_origins=["*"],
@@ -218,17 +237,49 @@ def build_app(runtime: GatewayRuntime, *, memory_root: Path | None = None) -> Fa
         firsts = await conversations.first_user_texts(
             cast("int", r.id) for r in rows if not r.title
         )
+        contexts = await conversations.context_tokens_by_id(r.external_id for r in rows)
+        public_ids = {r.id: r.external_id for r in rows}
         return {
             "conversations": [
                 {
                     **conversations.public(r),
                     "title": r.title or implied_title(firsts.get(cast("int", r.id))),
                     "last_item": _queue_item(latest.get(cast("int", r.id))),
+                    "context_tokens": contexts.get(r.external_id, 0),
+                    "parent": public_ids.get(r.parent_id) if r.parent_id else None,
                 }
                 for r in rows
             ]
         }
 
+    @app.get("/search")
+    async def search(request: Request) -> dict[str, Any]:
+        await require_token(request, runtime, scope=SCOPE)
+        query = (request.query_params.get("q") or "").strip()
+        if len(query) < SEARCH_MIN:
+            raise HTTPException(
+                status.HTTP_400_BAD_REQUEST, f"q must be at least {SEARCH_MIN} chars"
+            )
+        limit = min(query_int(request, "limit", SEARCH_LIMIT), SEARCH_LIMIT)
+        rows = await conversations.search(query, limit=limit)
+        firsts = await conversations.first_user_texts(
+            cast("int", r.id) for r in rows if not r.title
+        )
+        contexts = await conversations.context_tokens_by_id(r.external_id for r in rows)
+        files = _grep_memory(memory_root, query, limit=limit) if memory_root else []
+        return {
+            "query": query,
+            "conversations": [
+                {
+                    **conversations.public(r),
+                    "title": r.title or implied_title(firsts.get(cast("int", r.id))),
+                    "context_tokens": contexts.get(r.external_id, 0),
+                }
+                for r in rows
+            ],
+            "files": files,
+        }
+
     @app.post("/conversations", status_code=status.HTTP_201_CREATED)
     async def create_conversation(request: Request) -> dict[str, Any]:
         token = await require_token(request, runtime, scope=SCOPE)
@@ -301,6 +352,45 @@ def build_app(runtime: GatewayRuntime, *, memory_root: Path | None = None) -> Fa
         conv = await conv_of(public_id)
         return {"id": conv.external_id, "messages": await conversations.history(conv)}
 
+    @app.get("/conversations/{public_id}/context")
+    async def get_context(public_id: str, request: Request) -> dict[str, Any]:
+        await require_token(request, runtime, scope=SCOPE)
+        conv = await conv_of(public_id)
+        agent = runtime.agents.get(conv.agent_name)
+        if not isinstance(agent, ClaudeAgent):
+            raise HTTPException(
+                status.HTTP_404_NOT_FOUND, f"no claude agent behind {public_id}"
+            )
+        entries = list(await conversations.entries(conv))
+        for sub in await conversations.subpaths(conv):
+            entries.extend(await conversations.entries(conv, subpath=sub))
+        out = compose(
+            agent,
+            as_kind(conv.kind),
+            entries,
+            context_tokens=await conversations.context_tokens(conv),
+        )
+        out["id"] = conv.external_id
+        out["turns"] = prompt_count(entries)
+        if request.query_params.get("prompt"):
+            sources = agent.prompt_for(as_kind(conv.kind))
+            out["prompt_text"] = (
+                assemble(sources) if sources is not None else agent.system_prompt
+            )
+        return out
+
+    @app.get("/vault/graph")
+    async def vault_graph(request: Request) -> dict[str, Any]:
+        await require_token(request, runtime, scope=SCOPE)
+        if links is None:
+            raise HTTPException(
+                status.HTTP_404_NOT_FOUND,
+                "no vault root: set ApiFrontend(vault_root=...)",
+            )
+        paths = [p for p in request.query_params.getlist("path") if p]
+        limit = min(query_int(request, "limit", GRAPH_LIMIT), GRAPH_LIMIT)
+        return {"root": str(links.root), **links.neighbours(paths, limit=limit)}
+
     @app.get("/conversations/{public_id}/entries")
     async def get_entries(public_id: str, request: Request) -> dict[str, Any]:
         await require_token(request, runtime, scope=SCOPE)
@@ -993,6 +1083,35 @@ def _memory_path(root: Path, raw: str) -> Path:
     return target
 
 
+def _grep_memory(root: Path, query: str, *, limit: int) -> list[dict[str, Any]]:
+    needle = query.casefold()
+    hits: list[dict[str, Any]] = []
+    for path in sorted(root.rglob("*")):
+        if len(hits) >= limit:
+            break
+        if not path.is_file() or any(part.startswith(".") for part in path.parts):
+            continue
+        if path.suffix not in (".md", ".txt", ".json", ".yaml", ".yml"):
+            continue
+        try:
+            if path.stat().st_size > MEMORY_MAX_FILE:
+                continue
+            text = path.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            continue
+        rel = str(path.relative_to(root))
+        if needle in rel.casefold():
+            hits.append({"path": rel, "line": 0, "snippet": ""})
+            continue
+        for number, line in enumerate(text.splitlines(), 1):
+            if needle in line.casefold():
+                hits.append(
+                    {"path": rel, "line": number, "snippet": line.strip()[:SNIPPET_MAX]}
+                )
+                break
+    return hits
+
+
 def _tree(root: Path, directory: Path, *, depth: int) -> list[dict[str, Any]]:
     if depth > MEMORY_MAX_DEPTH:
         return []

src/beaver_gateway/vault/links.py

@@ -0,0 +1,136 @@
+"""Wikilink index over a vault: which note links to which, refreshed by mtime."""
+
+from __future__ import annotations
+
+import os
+import re
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+__all__ = ["LinkIndex"]
+
+_WIKILINK = re.compile(r"\[\[([^\]|#\n]+)(?:#[^\]|\n]*)?(?:\|[^\]\n]*)?\]\]")
+_MD = ".md"
+
+
+@dataclass(slots=True)
+class _Note:
+    mtime: float
+    links: set[str] = field(default_factory=set)
+
+
+class LinkIndex:
+    """Forward and backward links between the vault's markdown notes."""
+
+    def __init__(self, root: Path, *, max_age: float = 20.0) -> None:
+        self.root = root.resolve()
+        self.max_age = max_age
+        self._notes: dict[str, _Note] = {}
+        self._raw: dict[str, set[str]] = {}
+        self._scanned_at = 0.0
+
+    def refresh(self, *, force: bool = False) -> None:
+        if not force and time.monotonic() - self._scanned_at < self.max_age:
+            return
+        seen: set[str] = set()
+        for dirpath, dirnames, filenames in os.walk(self.root):
+            dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+            for name in filenames:
+                if not name.endswith(_MD):
+                    continue
+                path = Path(dirpath) / name
+                rel = str(path.relative_to(self.root))
+                seen.add(rel)
+                try:
+                    mtime = path.stat().st_mtime
+                except OSError:
+                    continue
+                note = self._notes.get(rel)
+                if note is None or note.mtime != mtime:
+                    self._notes[rel] = _Note(mtime=mtime)
+                    self._raw[rel] = self._targets(path)
+        for rel in list(self._notes):
+            if rel not in seen:
+                self._notes.pop(rel, None)
+                self._raw.pop(rel, None)
+        self._resolve()
+        self._scanned_at = time.monotonic()
+
+    @staticmethod
+    def _targets(path: Path) -> set[str]:
+        try:
+            text = path.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            return set()
+        return {m.group(1).strip() for m in _WIKILINK.finditer(text)}
+
+    def _resolve(self) -> None:
+        by_stem: dict[str, str] = {}
+        for rel in sorted(self._notes):
+            by_stem.setdefault(Path(rel).stem.casefold(), rel)
+        by_path = {rel.casefold(): rel for rel in self._notes}
+        for rel, targets in self._raw.items():
+            links: set[str] = set()
+            for target in targets:
+                candidate = target if target.endswith(_MD) else f"{target}{_MD}"
+                found = by_path.get(candidate.casefold()) or by_stem.get(
+                    Path(target).stem.casefold()
+                )
+                if found and found != rel:
+                    links.add(found)
+            self._notes[rel].links = links
+
+    def resolve(self, raw: str) -> str:
+        """A path as a tool named it (absolute or relative) as an index key."""
+        path = Path(raw)
+        if path.is_absolute():
+            try:
+                return str(path.resolve().relative_to(self.root))
+            except (ValueError, OSError):
+                return raw
+        return raw
+
+    def neighbours(self, paths: list[str], *, limit: int = 80) -> dict[str, Any]:
+        """Nodes for ``paths`` plus every note one link away, and their edges."""
+        self.refresh()
+        touched = [self.resolve(p) for p in paths]
+        backlinks: dict[str, set[str]] = {}
+        for rel, note in self._notes.items():
+            for target in note.links:
+                backlinks.setdefault(target, set()).add(rel)
+        nodes: dict[str, dict[str, Any]] = {}
+        for rel in touched:
+            nodes[rel] = {
+                "path": rel,
+                "title": Path(rel).stem,
+                "touched": True,
+                "exists": rel in self._notes,
+            }
+        for rel in touched:
+            note = self._notes.get(rel)
+            around = (note.links if note else set()) | backlinks.get(rel, set())
+            for other in sorted(around):
+                if len(nodes) >= limit:
+                    break
+                nodes.setdefault(
+                    other,
+                    {
+                        "path": other,
+                        "title": Path(other).stem,
+                        "touched": False,
+                        "exists": True,
+                    },
+                )
+        edges: list[dict[str, str]] = []
+        for rel in nodes:
+            note = self._notes.get(rel)
+            if note is None:
+                continue
+            edges.extend(
+                {"from": rel, "to": target}
+                for target in sorted(note.links)
+                if target in nodes
+            )
+        return {"nodes": list(nodes.values()), "edges": edges}