feat(transcribe,telegram,config): voice notes, round videos and audio get a transcript from pluggable providers, an openai-compatible one built in

src/beaver_gateway/app.py

@@ -201,6 +201,7 @@ async def run(gateway: Gateway, settings: Settings) -> None:
             scopes=scopes_with(fe.scope for fe in gateway.frontends),
             send_file_roots=_roots(settings.send_file_roots),
             secrets=secrets,
+            transcribers=tuple(gateway.transcribers),
         )
         for fe in gateway.frontends:
             fe.configure(runtime)

src/beaver_gateway/config.py

@@ -16,6 +16,7 @@ from beaver_gateway.conversations.texts import ConversationTexts
 from beaver_gateway.frontends.base import Frontend
 from beaver_gateway.jobs.scheduler import Job
 from beaver_gateway.mcp.types import HttpMcp, McpServer, PythonToolMcp, StdioMcp
+from beaver_gateway.transcribe import Audio, OpenAITranscriber
 
 if TYPE_CHECKING:
     from collections.abc import Awaitable, Callable
@@ -27,6 +28,7 @@ if TYPE_CHECKING:
     from beaver_gateway.conversations.texts import UserSaid
     from beaver_gateway.jobs.scheduler import Budget
     from beaver_gateway.mcp.types import McpServerT
+    from beaver_gateway.transcribe import Transcriber
     from beaver_gateway.vault.watch import VaultWatch
 
 __all__ = ["ConfigError", "Gateway", "load"]
@@ -51,6 +53,8 @@ class Gateway:
     """Envelope lookup on the user's text: pointers the setup derives from its files."""
     user_sink: Callable[[UserSaid], Awaitable[None] | None] | None = None
     """Sees every user message entering a master or branch turn."""
+    transcribers: list[Transcriber] = field(default_factory=list)
+    """Speech to text for voice and audio a window receives; the first text wins."""
     budget: Budget | None = None
     """Subscription utilisation past which non-critical jobs wait."""
     distiller: Distiller | None = None
@@ -75,6 +79,8 @@ _PUBLIC_NAMES: dict[str, Any] = {
     "ExposedMcp": ExposedMcp,
     "Gateway": Gateway,
     "ConversationTexts": ConversationTexts,
+    "Audio": Audio,
+    "OpenAITranscriber": OpenAITranscriber,
 }
 
 _McpInstance = StdioMcp | HttpMcp | PythonToolMcp
@@ -144,6 +150,13 @@ def _validate(gw: Gateway, path: Path) -> None:
                 f"a frontend scope is a lowercase name like 'voice', never '*'"
             )
             raise ConfigError(msg)
+    for i, t in enumerate(gw.transcribers):
+        if not callable(t):
+            msg = (
+                f"{path}: gateway.transcribers[{i}] must be an async callable "
+                f"taking an Audio, got {type(t).__name__}"
+            )
+            raise ConfigError(msg)
     names: set[str] = set()
     for i, j in enumerate(gw.jobs):
         if not isinstance(j, Job):

src/beaver_gateway/frontends/base.py

@@ -27,6 +27,7 @@ if TYPE_CHECKING:
     from beaver_gateway.security.auth import TokenStore
     from beaver_gateway.storage import Database
     from beaver_gateway.storage.models import Conversation, ConversationBinding
+    from beaver_gateway.transcribe import Transcriber
 
     TurnLogHandler = Callable[[TurnRecord], Awaitable[None]]
 
@@ -65,6 +66,8 @@ class GatewayRuntime:
     """Directories ``send_file`` may read, from ``SEND_FILE_ROOTS``."""
     secrets: SecretStore | None = None
     """Sealed secrets for ``secret run``; ``None`` in setups without them."""
+    transcribers: Sequence[Transcriber] = ()
+    """``Gateway.transcribers``: speech to text a window runs on audio it receives."""
 
 
 class Frontend(ABC):

src/beaver_gateway/frontends/telegram/frontend.py

@@ -48,13 +48,17 @@ from beaver_gateway.frontends.telegram.inbox import Inbox
 from beaver_gateway.frontends.telegram.outbox import Outbox
 from beaver_gateway.frontends.telegram.render import chunks, status_label, to_html
 from beaver_gateway.frontends.telegram.texts import TelegramTexts
+from beaver_gateway.transcribe import Audio, transcribe
 
 if TYPE_CHECKING:
+    from collections.abc import Sequence
+
     from beaver_gateway.conversations.kinds import Kind
     from beaver_gateway.conversations.service import Conversations
     from beaver_gateway.events.bus import Event, EventBus
     from beaver_gateway.frontends.base import GatewayRuntime
     from beaver_gateway.storage.models import Conversation, ConversationBinding
+    from beaver_gateway.transcribe import Transcriber
 
 __all__ = ["FRONTEND", "Attachments", "TelegramFrontend"]
 
@@ -65,6 +69,7 @@ _DONE = "done"
 _COMMANDS = ("merge", "new", "chat", "status", "help", "start")
 _CAPTION = 1024
 _NO_CAPTION = ("video_note",)
+_HEARD = ("voice", "audio", "video_note")
 
 
 @dataclass(frozen=True, slots=True)
@@ -132,6 +137,16 @@ class _Ask:
         )
 
 
+@dataclass(frozen=True, slots=True)
+class _Media:
+    file_id: str
+    kind: str
+    size: int | None
+    name: str
+    media_type: str | None = None
+    duration: int | None = None
+
+
 @dataclass
 class _Album:
     message: Message
@@ -166,6 +181,7 @@ class TelegramFrontend(Frontend):
         outbox_backoff: float = 2.0,
         texts: TelegramTexts | None = None,
         album_delay: float = 1.0,
+        transcribe_timeout: float = 90.0,
     ) -> None:
         self._token = token
         self.texts = texts or TelegramTexts()
@@ -190,6 +206,7 @@ class TelegramFrontend(Frontend):
         self._topic_names: dict[int, str] = {}
         self._drafts: dict[str, Draft] = {}
         self.album_delay = album_delay
+        self.transcribe_timeout = transcribe_timeout
         self._asks: dict[str, _Ask] = {}
         self._albums: dict[str, _Album] = {}
         self._reactions: dict[int, tuple[int, int]] = {}
@@ -687,10 +704,11 @@ class TelegramFrontend(Frontend):
             return texts.attachment_sticker, None
         if message.animation:
             return texts.attachment_animation, None
-        file_id, kind, size, name, media_type = _media(message)
-        if file_id is None or name is None:
+        media = _media(message)
+        if media is None:
             return None, None
-        kind = texts.attachment_kinds.get(kind, kind)
+        kind = texts.attachment_kinds.get(media.kind, media.kind)
+        size, name = media.size, media.name
         if size and size > 20 * 1024 * 1024:
             return (
                 texts.attachment_too_big.format(
@@ -706,7 +724,7 @@ class TelegramFrontend(Frontend):
             folder.chmod(0o755)
         path = folder / f"{int(now)}-{Path(name).name}"
         try:
-            await self.bot.download(file_id, destination=path)
+            await self.bot.download(media.file_id, destination=path)
             path.chmod(0o644)
         except (TelegramAPIError, OSError) as exc:
             _log.warning("attachment download failed: %s", exc)
@@ -714,16 +732,38 @@ class TelegramFrontend(Frontend):
         shown = texts.attachment_size.format(kb=size // 1024) if size else ""
         keep = self.attachments.keep_days
         seen = (
-            {"path": str(path), "media_type": media_type}
-            if media_type and media_type.startswith("image/")
+            {"path": str(path), "media_type": media.media_type}
+            if media.media_type and media.media_type.startswith("image/")
             else None
         )
-        if keep is None:
-            return texts.attachment.format(kind=kind, path=path, size=shown), seen
-        return (
-            texts.attachment_kept.format(kind=kind, path=path, size=shown, days=keep),
-            seen,
+        note = (
+            texts.attachment.format(kind=kind, path=path, size=shown)
+            if keep is None
+            else texts.attachment_kept.format(
+                kind=kind, path=path, size=shown, days=keep
+            )
         )
+        transcribers = cast("GatewayRuntime", self._runtime).transcribers
+        if media.kind in _HEARD and transcribers:
+            audio = Audio(path, media.media_type, media.duration)
+            heard = await self._transcribe(transcribers, audio)
+            if heard:
+                note = texts.attachment_transcript.format(note=note, text=heard)
+        return note, seen
+
+    async def _transcribe(
+        self, transcribers: Sequence[Transcriber], audio: Audio
+    ) -> str | None:
+        try:
+            async with asyncio.timeout(self.transcribe_timeout):
+                return await transcribe(transcribers, audio)
+        except TimeoutError:
+            _log.warning(
+                "transcription of %s took over %.0fs, the note goes without it",
+                audio.path.name,
+                self.transcribe_timeout,
+            )
+            return None
 
     async def _sweep_loop(self, interval: float = 6 * 3600) -> None:
         while True:
@@ -1190,13 +1230,10 @@ class TelegramFrontend(Frontend):
                 )
 
 
-def _media(
-    message: Message,
-) -> tuple[str | None, str, int | None, str | None, str | None]:
-    """``(file_id, kind, size, name, media_type)`` of the message's media, if any."""
+def _media(message: Message) -> _Media | None:
     if message.photo:
         photo = message.photo[-1]
-        return (
+        return _Media(
             photo.file_id,
             "photo",
             photo.file_size,
@@ -1206,32 +1243,43 @@ def _media(
     if message.document:
         doc = message.document
         name = doc.file_name or f"{doc.file_unique_id}.bin"
-        return doc.file_id, "document", doc.file_size, name, doc.mime_type
+        return _Media(doc.file_id, "document", doc.file_size, name, doc.mime_type)
     if message.voice:
         v = message.voice
-        return v.file_id, "voice", v.file_size, f"{v.file_unique_id}.ogg", None
+        return _Media(
+            v.file_id,
+            "voice",
+            v.file_size,
+            f"{v.file_unique_id}.ogg",
+            v.mime_type or "audio/ogg",
+            v.duration,
+        )
     if message.audio:
         a = message.audio
-        return (
+        return _Media(
             a.file_id,
             "audio",
             a.file_size,
             a.file_name or f"{a.file_unique_id}.mp3",
-            None,
+            a.mime_type,
+            a.duration,
         )
     if message.video:
         v = message.video
-        return (
-            v.file_id,
-            "video",
-            v.file_size,
-            v.file_name or f"{v.file_unique_id}.mp4",
-            None,
+        return _Media(
+            v.file_id, "video", v.file_size, v.file_name or f"{v.file_unique_id}.mp4"
         )
     if message.video_note:
         n = message.video_note
-        return n.file_id, "video_note", n.file_size, f"{n.file_unique_id}.mp4", None
-    return None, "", None, None, None
+        return _Media(
+            n.file_id,
+            "video_note",
+            n.file_size,
+            f"{n.file_unique_id}.mp4",
+            "video/mp4",
+            n.duration,
+        )
+    return None
 
 
 def _question_html(question: dict[str, Any]) -> str:

src/beaver_gateway/frontends/telegram/texts.py

@@ -71,6 +71,7 @@ class TelegramTexts:
         "[attachment: {kind} {path}{size}; kept {days} days, move it into the "
         "vault if it matters]"
     )
+    attachment_transcript: str = "{note}\ntranscript: {text}"
     merge_nothing: str = "nothing to merge: this is not an open branch"
     merging: str = "🔀 merging into the master"
     merged: str = "✅ merged into the master"

src/beaver_gateway/transcribe/__init__.py

@@ -0,0 +1,6 @@
+"""Speech to text for audio a window receives: a chain of pluggable providers."""
+
+from beaver_gateway.transcribe.base import Audio, Transcriber, transcribe
+from beaver_gateway.transcribe.openai import OpenAITranscriber
+
+__all__ = ["Audio", "OpenAITranscriber", "Transcriber", "transcribe"]

src/beaver_gateway/transcribe/base.py

@@ -0,0 +1,52 @@
+"""What a transcription provider is handed and the chain that tries them in order."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+    from pathlib import Path
+
+__all__ = ["Audio", "Transcriber", "transcribe"]
+
+_log = logging.getLogger("beaver_gateway.transcribe")
+
+
+@dataclass(frozen=True, slots=True)
+class Audio:
+    path: Path
+    """The downloaded file, readable for as long as the call lasts."""
+    media_type: str | None = None
+    """MIME type as the window reported it, e.g. ``audio/ogg``; ``None`` if unknown."""
+    duration: float | None = None
+    """Length in seconds, when the window knows it."""
+
+
+class Transcriber(Protocol):
+    async def __call__(self, audio: Audio) -> str | None: ...
+
+
+async def transcribe(transcribers: Sequence[Transcriber], audio: Audio) -> str | None:
+    """The first non-empty text from ``transcribers``, a failing one skipped."""
+    for transcriber in transcribers:
+        try:
+            text = await transcriber(audio)
+        except Exception as exc:  # noqa: BLE001
+            _log.warning(
+                "transcriber %s failed on %s: %s: %s",
+                _name(transcriber),
+                audio.path.name,
+                type(exc).__name__,
+                exc,
+            )
+            continue
+        if text and text.strip():
+            return text.strip()
+    return None
+
+
+def _name(transcriber: Transcriber) -> str:
+    return getattr(transcriber, "__qualname__", None) or type(transcriber).__name__

src/beaver_gateway/transcribe/openai.py

@@ -0,0 +1,57 @@
+"""A provider for any OpenAI-compatible ``/audio/transcriptions`` endpoint."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+import aiohttp
+
+if TYPE_CHECKING:
+    from beaver_gateway.transcribe.base import Audio
+
+__all__ = ["OpenAITranscriber"]
+
+
+@dataclass(frozen=True, slots=True)
+class OpenAITranscriber:
+    """OpenAI, Groq, faster-whisper-server, a whisper.cpp server: one wire."""
+
+    base_url: str = "https://api.openai.com/v1"
+    """API root; ``/audio/transcriptions`` is appended."""
+    api_key: str | None = None
+    """Sent as a bearer token; ``None`` for a local server without auth."""
+    model: str = "whisper-1"
+    """Model name as the server knows it, e.g. ``whisper-large-v3`` on Groq."""
+    language: str | None = None
+    """ISO-639-1 hint such as ``ru``; ``None`` lets the model detect it."""
+    timeout: float = 120.0
+    """Seconds for the whole request, upload included."""
+
+    async def __call__(self, audio: Audio) -> str | None:
+        headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
+        form = aiohttp.FormData()
+        form.add_field("model", self.model)
+        form.add_field("response_format", "json")
+        if self.language:
+            form.add_field("language", self.language)
+        with audio.path.open("rb") as file:
+            form.add_field(
+                "file",
+                file,
+                filename=audio.path.name,
+                content_type=audio.media_type or "application/octet-stream",
+            )
+            async with (
+                aiohttp.ClientSession(
+                    timeout=aiohttp.ClientTimeout(total=self.timeout)
+                ) as http,
+                http.post(
+                    f"{self.base_url.rstrip('/')}/audio/transcriptions",
+                    data=form,
+                    headers=headers,
+                ) as response,
+            ):
+                response.raise_for_status()
+                body = await response.json(content_type=None)
+        return str(body.get("text") or "") or None