feat(frontends,conversations,telegram): the agent can hand a file to the human

src/beaver_gateway/app.py

@@ -6,6 +6,7 @@ import asyncio
 import functools
 import logging
 from contextlib import AsyncExitStack
+from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 import psycopg
@@ -195,6 +196,7 @@ async def run(gateway: Gateway, settings: Settings) -> None:
             scheduler=scheduler,
             public_url=gateway.public_url.rstrip("/") if gateway.public_url else None,
             scopes=scopes_with(fe.scope for fe in gateway.frontends),
+            send_file_roots=_roots(settings.send_file_roots),
         )
         for fe in gateway.frontends:
             fe.configure(runtime)
@@ -297,6 +299,10 @@ async def _serve_internal_mcp(app: Starlette, *, settings: Settings) -> None:
     await uvicorn.Server(config).serve()
 
 
+def _roots(value: str) -> tuple[Path, ...]:
+    return tuple(Path(p) for p in value.split(":") if p.strip())
+
+
 class _LateConversations:
     conversations: Conversations | None = None
 

src/beaver_gateway/conversations/messaging.py

@@ -1,4 +1,4 @@
-"""Putting words into a conversation: a message, an inject, ``say``, ``schedule``."""
+"""Putting words into a conversation: a message, an inject, ``say``, a file."""
 
 from __future__ import annotations
 
@@ -6,6 +6,7 @@ import logging
 from typing import TYPE_CHECKING, Any, cast
 
 from beaver_gateway.conversations.turns import Turns
+from beaver_gateway.frontends.files import SendFileError
 
 if TYPE_CHECKING:
     from datetime import datetime
@@ -110,6 +111,31 @@ class Messaging(Turns):
             turn_id=runner.turn_id if runner is not None else None,
         )
 
+    async def send_file(
+        self, conv: Conversation, path: str, *, caption: str = "", method: str = "auto"
+    ) -> str:
+        """Hand a file to the human: this conversation's window, else the master's."""
+        for target in (conv, await self.open_master()):
+            if target is None:
+                continue
+            for fe in self._frontends:
+                if target.kind in fe.kinds and fe.sends_files:
+                    note = await fe.send_file(
+                        target, path, caption=caption, method=method
+                    )
+                    _log.info("send_file[%s]: %s", target.external_id, note)
+                    return note
+        msg = (
+            "nowhere to send: no frontend that sends files shows this "
+            "conversation, and there is no open master"
+        )
+        raise SendFileError(msg)
+
+    def file_note(self) -> str:
+        """What the ``send_file`` tool tells the model about roots and size."""
+        sender = next((f for f in self._frontends if f.sends_files), None)
+        return sender.file_note() if sender else "no frontend here sends files"
+
     async def schedule(
         self,
         conv: Conversation,

src/beaver_gateway/conversations/tools.py

@@ -14,6 +14,7 @@ from claude_agent_sdk import create_sdk_mcp_server, tool
 
 from beaver_gateway.conversations.injects import URGENCY
 from beaver_gateway.conversations.kinds import as_kind
+from beaver_gateway.frontends.files import AUTO, METHODS, SendFileError
 from beaver_gateway.security.redact import redact_data
 
 URGENCY_HELP = (
@@ -42,7 +43,23 @@ SAY_IN_USER_TURN = (
     "reply text reaches them by itself - put what you wanted to say into the "
     "reply instead of repeating it here"
 )
-TOOL_NAMES = ("read_conversation", "spawn", "say", "schedule", "inject", "close_chat")
+TOOL_NAMES = (
+    "read_conversation",
+    "spawn",
+    "say",
+    "schedule",
+    "inject",
+    "close_chat",
+    "send_file",
+)
+SEND_FILE_HELP = (
+    "Send a file from disk to the human, into this conversation's window (the "
+    "master's if it has none): {note}. `method` is how Telegram renders it; "
+    "`auto` reads the extension - image is a photo, mp3/flac/m4a audio, ogg "
+    "voice, mp4/mov video, anything else a document. Photo and video are "
+    "recompressed, so pass `document` when the bytes must arrive untouched. "
+    "`caption` takes the same markdown as your replies."
+)
 
 
 def build_tool_server(
@@ -232,6 +249,33 @@ def _tools(conversations: Conversations, key: str) -> list[SdkMcpTool[Any]]:
         )
         return _text(f"queued #{item.id}")
 
+    @tool(
+        "send_file",
+        SEND_FILE_HELP.format(note=conversations.file_note()),
+        {
+            "type": "object",
+            "properties": {
+                "path": {"type": "string", "description": "absolute path on disk"},
+                "caption": {"type": "string"},
+                "method": {"type": "string", "enum": [AUTO, *METHODS], "default": AUTO},
+            },
+            "required": ["path"],
+        },
+    )
+    async def send_file(args: dict[str, Any]) -> dict[str, Any]:
+        conv = await current()
+        try:
+            return _text(
+                await conversations.send_file(
+                    conv,
+                    str(args["path"]),
+                    caption=str(args.get("caption") or ""),
+                    method=str(args.get("method") or AUTO),
+                )
+            )
+        except SendFileError as exc:
+            return _error(str(exc))
+
     @tool(
         "close_chat",
         "Close this deep chat once the current reply is finished: the "
@@ -248,7 +292,7 @@ def _tools(conversations: Conversations, key: str) -> list[SdkMcpTool[Any]]:
             return _error(str(exc))
         return _text("ok: the chat closes after this reply")
 
-    return [read_conversation, spawn, say, schedule, inject, close_chat]
+    return [read_conversation, spawn, say, schedule, inject, send_file, close_chat]
 
 
 def _text(text: str) -> dict[str, Any]:

src/beaver_gateway/frontends/base.py

@@ -10,10 +10,12 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any
 
+from beaver_gateway.frontends.files import SendFileError
 from beaver_gateway.security.auth import BUILTIN_SCOPES
 
 if TYPE_CHECKING:
     from collections.abc import Awaitable, Callable, Mapping, Sequence
+    from pathlib import Path
 
     from starlette.types import ASGIApp
 
@@ -58,6 +60,8 @@ class GatewayRuntime:
     """``Gateway.public_url``; ``None`` derives the origin from the request."""
     scopes: frozenset[str] = BUILTIN_SCOPES
     """Every token scope this gateway knows: the builtins plus each frontend's."""
+    send_file_roots: tuple[Path, ...] = ()
+    """Directories ``send_file`` may read, from ``SEND_FILE_ROOTS``."""
 
 
 class Frontend(ABC):
@@ -85,6 +89,8 @@ class Frontend(ABC):
     path: str | None = None
     landing: bool = False
     scope: str | None = None
+    sends_files: bool = False
+    """Whether ``send_file`` reaches a human through this frontend."""
 
     @abstractmethod
     def configure(self, runtime: GatewayRuntime) -> None: ...
@@ -104,3 +110,19 @@ class Frontend(ABC):
     async def mark_closed(self, conv: Conversation) -> bool:  # noqa: ARG002
         """Show in the window that the conversation is over (a renamed topic)."""
         return False
+
+    async def send_file(
+        self,
+        conv: Conversation,  # noqa: ARG002
+        path: str,  # noqa: ARG002
+        *,
+        caption: str = "",  # noqa: ARG002
+        method: str = "auto",  # noqa: ARG002
+    ) -> str:
+        """Put a file in this conversation's window; the answer goes to the model."""
+        msg = f"frontend {self.name!r} cannot send files"
+        raise SendFileError(msg)
+
+    def file_note(self) -> str:
+        """Roots and ceiling, for the tool description the model reads."""
+        return ""

src/beaver_gateway/frontends/files.py

@@ -0,0 +1,95 @@
+"""What an agent may hand to a human: an allowed path, a ceiling, a method.
+
+The ceiling and the method names belong to the frontend's API; the check
+lives here so every frontend refuses the same way, in words the model reads.
+"""
+
+from __future__ import annotations
+
+import mimetypes
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+__all__ = ["AUTO", "DEFAULTS", "METHODS", "SendFileError", "SendFiles", "method_for"]
+
+AUTO = "auto"
+METHODS = ("document", "photo", "audio", "voice", "video", "video_note", "animation")
+
+_BY_SUFFIX = {
+    ".jpg": "photo",
+    ".jpeg": "photo",
+    ".png": "photo",
+    ".webp": "photo",
+    ".gif": "animation",
+    ".mp3": "audio",
+    ".flac": "audio",
+    ".m4a": "audio",
+    ".wav": "audio",
+    ".aac": "audio",
+    ".ogg": "voice",
+    ".oga": "voice",
+    ".opus": "voice",
+    ".mp4": "video",
+    ".m4v": "video",
+    ".mov": "video",
+}
+_BY_TYPE = {"image": "photo", "audio": "audio", "video": "video"}
+
+
+class SendFileError(Exception):
+    """Why a file was not sent, worded for the model that asked."""
+
+
+@dataclass(frozen=True, slots=True)
+class SendFiles:
+    """Which files a frontend accepts from an agent and how big they may be."""
+
+    roots: tuple[Path, ...] = ()
+    """Directories a path must resolve inside; nothing outside is sendable."""
+    max_bytes: int = 50 * 1024 * 1024
+    """Upload ceiling of the API behind the frontend; the Bot API gives 50 MB."""
+
+    def with_roots(self, *roots: Path) -> SendFiles:
+        return replace(self, roots=tuple(dict.fromkeys((*self.roots, *roots))))
+
+    @property
+    def note(self) -> str:
+        """Roots and ceiling as one line for the tool description."""
+        where = ", ".join(str(r) for r in self.roots) or "none, so nothing is sendable"
+        return f"paths under {where}, up to {mb(self.max_bytes)} MB"
+
+    def resolve(self, raw: str) -> Path:
+        path = Path(raw.strip()).expanduser().resolve()
+        if not any(path.is_relative_to(r.expanduser().resolve()) for r in self.roots):
+            msg = f"{path} is outside the allowed roots ({self.note}); copy it there"
+            raise SendFileError(msg)
+        if not path.is_file():
+            msg = f"{path} is not a file"
+            raise SendFileError(msg)
+        size = path.stat().st_size
+        if size > self.max_bytes:
+            msg = (
+                f"{path.name} is {mb(size)} MB, over the {mb(self.max_bytes)} MB limit"
+            )
+            raise SendFileError(msg)
+        return path
+
+
+DEFAULTS = SendFiles()
+
+
+def method_for(path: Path, method: str = AUTO) -> str:
+    if method and method != AUTO:
+        if method not in METHODS:
+            msg = f"unknown method {method!r}; one of: {', '.join((AUTO, *METHODS))}"
+            raise SendFileError(msg)
+        return method
+    guessed = _BY_SUFFIX.get(path.suffix.lower())
+    if guessed:
+        return guessed
+    mime, _ = mimetypes.guess_type(path.name)
+    return _BY_TYPE.get(mime.split("/")[0], "document") if mime else "document"
+
+
+def mb(size: int) -> str:
+    return f"{round(size / 1024 / 1024, 2):g}"

src/beaver_gateway/frontends/telegram/frontend.py

@@ -21,10 +21,13 @@ from zoneinfo import ZoneInfo
 
 from aiogram import Bot
 from aiogram.client.default import DefaultBotProperties
-from aiogram.exceptions import TelegramAPIError
+from aiogram.client.session.aiohttp import AiohttpSession
+from aiogram.client.telegram import TelegramAPIServer
+from aiogram.exceptions import TelegramAPIError, TelegramNetworkError
 from aiogram.types import (
     BotCommand,
     CallbackQuery,
+    FSInputFile,
     InlineKeyboardButton,
     InlineKeyboardMarkup,
     Message,
@@ -33,10 +36,17 @@ from aiogram.types import (
 )
 
 from beaver_gateway.frontends.base import Frontend
+from beaver_gateway.frontends.files import (
+    DEFAULTS,
+    SendFileError,
+    SendFiles,
+    mb,
+    method_for,
+)
 from beaver_gateway.frontends.telegram.drafts import Draft
 from beaver_gateway.frontends.telegram.inbox import Inbox
 from beaver_gateway.frontends.telegram.outbox import Outbox
-from beaver_gateway.frontends.telegram.render import chunks, status_label
+from beaver_gateway.frontends.telegram.render import chunks, status_label, to_html
 from beaver_gateway.frontends.telegram.texts import TelegramTexts
 
 if TYPE_CHECKING:
@@ -53,6 +63,8 @@ _log = logging.getLogger("beaver_gateway.frontends.telegram")
 FRONTEND = "telegram"
 _DONE = "done"
 _COMMANDS = ("merge", "new", "chat", "status", "help", "start")
+_CAPTION = 1024
+_NO_CAPTION = ("video_note",)
 
 
 @dataclass(frozen=True, slots=True)
@@ -134,6 +146,7 @@ class _Album:
 class TelegramFrontend(Frontend):
     name = FRONTEND
     kinds = ("master", "branch")
+    sends_files = True
 
     def __init__(
         self,
@@ -144,6 +157,8 @@ class TelegramFrontend(Frontend):
         branch_agent: str | None = None,
         chat_id: int | None = None,
         attachments: Attachments = EPHEMERAL,
+        send_files: SendFiles = DEFAULTS,
+        api_base_url: str | None = None,
         draft_interval: float = 0.7,
         master_topic: str = "🦫 General",
         queued_reaction: str = "👀",
@@ -159,6 +174,8 @@ class TelegramFrontend(Frontend):
         self.master_agent = master_agent
         self.branch_agent = branch_agent
         self.attachments = attachments
+        self.send_files = send_files
+        self.api_base_url = api_base_url
         self.master_topic = master_topic
         self.draft_interval = draft_interval
         self.queued_reaction = queued_reaction
@@ -187,7 +204,14 @@ class TelegramFrontend(Frontend):
             raise RuntimeError(msg)
         self._runtime = runtime
         if self._bot is None:
-            self._bot = Bot(self._token, default=DefaultBotProperties(parse_mode=None))
+            self._bot = Bot(
+                self._token,
+                session=self._session(),
+                default=DefaultBotProperties(parse_mode=None),
+            )
+        self.send_files = self.send_files.with_roots(
+            *runtime.send_file_roots, self.attachments.root
+        )
         self._inbox = Inbox(
             runtime.db, self._bot, handler=self._handle, poll_timeout=self.poll_timeout
         )
@@ -195,6 +219,16 @@ class TelegramFrontend(Frontend):
             runtime.db, self._bot, bus=runtime.bus, backoff=self.outbox_backoff
         )
 
+    def _session(self) -> AiohttpSession | None:
+        """A local Bot API server instead of api.telegram.org (2 GB uploads)."""
+        if not self.api_base_url:
+            return None
+        return AiohttpSession(
+            api=TelegramAPIServer.from_base(
+                self.api_base_url.rstrip("/"), is_local=True
+            )
+        )
+
     async def serve(self) -> None:
         me = await self.bot.get_me()
         _log.info(
@@ -251,6 +285,44 @@ class TelegramFrontend(Frontend):
         self._topic_names[target[1]] = f"{prefix}{name}"
         return True
 
+    def file_note(self) -> str:
+        return self.send_files.note
+
+    async def send_file(
+        self, conv: Conversation, path: str, *, caption: str = "", method: str = "auto"
+    ) -> str:
+        """Upload the file into this conversation's topic, caption and all."""
+        target = await self._target_of(conv)
+        if target is None:
+            msg = f"conversation {conv.external_id} has no Telegram window"
+            raise SendFileError(msg)
+        file = self.send_files.resolve(path)
+        how = method_for(file, method)
+        rest = caption.strip()
+        payload: dict[str, Any] = {
+            "message_thread_id": target[1],
+            how: FSInputFile(file),
+        }
+        if how == "audio":
+            payload["title"] = file.stem
+        if rest and how not in _NO_CAPTION and len(rest) <= _CAPTION:
+            payload |= {"caption": to_html(rest), "parse_mode": "HTML"}
+            rest = ""
+        try:
+            await getattr(self.bot, f"send_{how}")(target[0], **payload)
+        except TelegramNetworkError as exc:
+            msg = f"Telegram did not answer while sending {file.name}: {exc}"
+            raise SendFileError(msg) from exc
+        except TelegramAPIError as exc:
+            msg = f"Telegram refused {file.name} as {how}: {exc}"
+            raise SendFileError(msg) from exc
+        except (OSError, TimeoutError) as exc:
+            msg = f"sending {file.name} failed: {exc}"
+            raise SendFileError(msg) from exc
+        if rest:
+            await self._deliver(conv, rest)
+        return f"sent {file.name} as {how}, {mb(file.stat().st_size)} MB"
+
     @property
     def bot(self) -> Bot:
         if self._bot is None:

src/beaver_gateway/frontends/telegram/texts.py

@@ -26,6 +26,7 @@ TOOL_LABELS: dict[str, str] = {
     "mcp__gateway__say": "speaking",
     "mcp__gateway__schedule": "setting a reminder",
     "mcp__gateway__inject": "passing to another conversation",
+    "mcp__gateway__send_file": "sending a file",
 }
 
 

src/beaver_gateway/settings.py

@@ -57,6 +57,10 @@ class Settings(BaseSettings):
     """``Accept-Language`` sent to the Raycast API and the default locale for
     auto ``UserPreferences``. One value per gateway (one shared Client)."""
 
+    send_file_roots: str = "/mnt/hole/shared"
+    """Directories the ``send_file`` tool may read, separated by ``:``. A
+    frontend always adds the inbox its own attachments land in."""
+
     bootstrap_tokens: str = ""
     """Out-of-band token seed: ``name1:value1,name2:value2``. Layers
     alongside DB-issued tokens; used for first-run setup and recovery."""