feat(transcribe,telegram,config): voice notes, round videos and audio get a transcript from pluggable providers, an openai-compatible one built in

This commit is contained in:
hh committed 2026-10-28 02:09:26 +00:00
1 parent 944b68e275
commit 42a7058f75
12 files changed
+454 -28

No files matched your search

+43
View File
@@ -38,6 +38,9 @@ is a full setup built on them.
write-only from the admin, handed out only through `secret run NAME -- write-only from the admin, handed out only through `secret run NAME --
cmd` (one `sudo` rule for `beaver-runner`, a root-only socket, files in cmd` (one `sudo` rule for `beaver-runner`, a root-only socket, files in
`/dev/shm`, values masked in the command's output). `/dev/shm`, values masked in the command's output).
- **transcribe/** - speech to text for the voice notes, round videos and
audio a window receives: providers tried in order, `OpenAITranscriber`
for any OpenAI-compatible endpoint built in.
- **events/** - the in-process bus every frontend taps, and the stream - **events/** - the in-process bus every frontend taps, and the stream
event protocol backends emit. event protocol backends emit.
- **storage/** - SQLModel tables and the Postgres session store. - **storage/** - SQLModel tables and the Postgres session store.
@@ -67,6 +70,46 @@ file next to it adds Postgres. Without Docker: `uv sync`, build the SPA with
`DATABASE_URL` and `CONFIG_PATH` set. A setup's own secrets come from `.env` `DATABASE_URL` and `CONFIG_PATH` set. A setup's own secrets come from `.env`
next to its config (`load_dotenv` runs before the config is executed). next to its config (`load_dotenv` runs before the config is executed).
## Transcription
A voice note, a round video or an audio file sent to the Telegram window is
downloaded as any attachment; with `Gateway.transcribers` set, the frontend
runs them in order and the first non-empty text rides under the attachment
note (`TelegramTexts.attachment_transcript`). A provider that raises is
logged and the next one tried; the whole chain gives up after the
frontend's `transcribe_timeout` (90 s) and the note goes without the text.
No transcribers, nothing changes.
```python
import os
from beaver_gateway.transcribe import Audio, OpenAITranscriber
async def voice_only(audio: Audio) -> str | None:
if audio.media_type != "audio/ogg":
return None
return await my_service.transcribe(audio.path, seconds=audio.duration)
gateway = Gateway(
transcribers=[
voice_only,
OpenAITranscriber(
base_url="https://api.groq.com/openai/v1",
api_key=os.environ["GROQ_API_KEY"],
model="whisper-large-v3",
language="ru",
),
],
)
```
A provider is any `async (Audio) -> str | None`; `None` passes the audio
on. `OpenAITranscriber` posts the file to `{base_url}/audio/transcriptions`,
the wire of OpenAI, Groq, faster-whisper-server and a whisper.cpp server;
`api_key=None` for a local one without auth.
## Develop ## Develop
```sh ```sh
+1
View File
@@ -201,6 +201,7 @@ async def run(gateway: Gateway, settings: Settings) -> None:
scopes=scopes_with(fe.scope for fe in gateway.frontends), scopes=scopes_with(fe.scope for fe in gateway.frontends),
send_file_roots=_roots(settings.send_file_roots), send_file_roots=_roots(settings.send_file_roots),
secrets=secrets, secrets=secrets,
transcribers=tuple(gateway.transcribers),
) )
for fe in gateway.frontends: for fe in gateway.frontends:
fe.configure(runtime) fe.configure(runtime)
+13
View File
@@ -16,6 +16,7 @@ from beaver_gateway.conversations.texts import ConversationTexts
from beaver_gateway.frontends.base import Frontend from beaver_gateway.frontends.base import Frontend
from beaver_gateway.jobs.scheduler import Job from beaver_gateway.jobs.scheduler import Job
from beaver_gateway.mcp.types import HttpMcp, McpServer, PythonToolMcp, StdioMcp from beaver_gateway.mcp.types import HttpMcp, McpServer, PythonToolMcp, StdioMcp
from beaver_gateway.transcribe import Audio, OpenAITranscriber
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Awaitable, Callable from collections.abc import Awaitable, Callable
@@ -27,6 +28,7 @@ if TYPE_CHECKING:
from beaver_gateway.conversations.texts import UserSaid from beaver_gateway.conversations.texts import UserSaid
from beaver_gateway.jobs.scheduler import Budget from beaver_gateway.jobs.scheduler import Budget
from beaver_gateway.mcp.types import McpServerT from beaver_gateway.mcp.types import McpServerT
from beaver_gateway.transcribe import Transcriber
from beaver_gateway.vault.watch import VaultWatch from beaver_gateway.vault.watch import VaultWatch
__all__ = ["ConfigError", "Gateway", "load"] __all__ = ["ConfigError", "Gateway", "load"]
@@ -51,6 +53,8 @@ class Gateway:
"""Envelope lookup on the user's text: pointers the setup derives from its files.""" """Envelope lookup on the user's text: pointers the setup derives from its files."""
user_sink: Callable[[UserSaid], Awaitable[None] | None] | None = None user_sink: Callable[[UserSaid], Awaitable[None] | None] | None = None
"""Sees every user message entering a master or branch turn.""" """Sees every user message entering a master or branch turn."""
transcribers: list[Transcriber] = field(default_factory=list)
"""Speech to text for voice and audio a window receives; the first text wins."""
budget: Budget | None = None budget: Budget | None = None
"""Subscription utilisation past which non-critical jobs wait.""" """Subscription utilisation past which non-critical jobs wait."""
distiller: Distiller | None = None distiller: Distiller | None = None
@@ -75,6 +79,8 @@ _PUBLIC_NAMES: dict[str, Any] = {
"ExposedMcp": ExposedMcp, "ExposedMcp": ExposedMcp,
"Gateway": Gateway, "Gateway": Gateway,
"ConversationTexts": ConversationTexts, "ConversationTexts": ConversationTexts,
"Audio": Audio,
"OpenAITranscriber": OpenAITranscriber,
} }
_McpInstance = StdioMcp | HttpMcp | PythonToolMcp _McpInstance = StdioMcp | HttpMcp | PythonToolMcp
@@ -144,6 +150,13 @@ def _validate(gw: Gateway, path: Path) -> None:
f"a frontend scope is a lowercase name like 'voice', never '*'" f"a frontend scope is a lowercase name like 'voice', never '*'"
) )
raise ConfigError(msg) raise ConfigError(msg)
for i, t in enumerate(gw.transcribers):
if not callable(t):
msg = (
f"{path}: gateway.transcribers[{i}] must be an async callable "
f"taking an Audio, got {type(t).__name__}"
)
raise ConfigError(msg)
names: set[str] = set() names: set[str] = set()
for i, j in enumerate(gw.jobs): for i, j in enumerate(gw.jobs):
if not isinstance(j, Job): if not isinstance(j, Job):
+3
View File
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
from beaver_gateway.security.auth import TokenStore from beaver_gateway.security.auth import TokenStore
from beaver_gateway.storage import Database from beaver_gateway.storage import Database
from beaver_gateway.storage.models import Conversation, ConversationBinding from beaver_gateway.storage.models import Conversation, ConversationBinding
from beaver_gateway.transcribe import Transcriber
TurnLogHandler = Callable[[TurnRecord], Awaitable[None]] TurnLogHandler = Callable[[TurnRecord], Awaitable[None]]
@@ -65,6 +66,8 @@ class GatewayRuntime:
"""Directories ``send_file`` may read, from ``SEND_FILE_ROOTS``.""" """Directories ``send_file`` may read, from ``SEND_FILE_ROOTS``."""
secrets: SecretStore | None = None secrets: SecretStore | None = None
"""Sealed secrets for ``secret run``; ``None`` in setups without them.""" """Sealed secrets for ``secret run``; ``None`` in setups without them."""
transcribers: Sequence[Transcriber] = ()
"""``Gateway.transcribers``: speech to text a window runs on audio it receives."""
class Frontend(ABC): class Frontend(ABC):
@@ -48,13 +48,17 @@ from beaver_gateway.frontends.telegram.inbox import Inbox
from beaver_gateway.frontends.telegram.outbox import Outbox from beaver_gateway.frontends.telegram.outbox import Outbox
from beaver_gateway.frontends.telegram.render import chunks, status_label, to_html from beaver_gateway.frontends.telegram.render import chunks, status_label, to_html
from beaver_gateway.frontends.telegram.texts import TelegramTexts from beaver_gateway.frontends.telegram.texts import TelegramTexts
from beaver_gateway.transcribe import Audio, transcribe
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Sequence
from beaver_gateway.conversations.kinds import Kind from beaver_gateway.conversations.kinds import Kind
from beaver_gateway.conversations.service import Conversations from beaver_gateway.conversations.service import Conversations
from beaver_gateway.events.bus import Event, EventBus from beaver_gateway.events.bus import Event, EventBus
from beaver_gateway.frontends.base import GatewayRuntime from beaver_gateway.frontends.base import GatewayRuntime
from beaver_gateway.storage.models import Conversation, ConversationBinding from beaver_gateway.storage.models import Conversation, ConversationBinding
from beaver_gateway.transcribe import Transcriber
__all__ = ["FRONTEND", "Attachments", "TelegramFrontend"] __all__ = ["FRONTEND", "Attachments", "TelegramFrontend"]
@@ -65,6 +69,7 @@ _DONE = "done"
_COMMANDS = ("merge", "new", "chat", "status", "help", "start") _COMMANDS = ("merge", "new", "chat", "status", "help", "start")
_CAPTION = 1024 _CAPTION = 1024
_NO_CAPTION = ("video_note",) _NO_CAPTION = ("video_note",)
_HEARD = ("voice", "audio", "video_note")
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
@@ -132,6 +137,16 @@ class _Ask:
) )
@dataclass(frozen=True, slots=True)
class _Media:
file_id: str
kind: str
size: int | None
name: str
media_type: str | None = None
duration: int | None = None
@dataclass @dataclass
class _Album: class _Album:
message: Message message: Message
@@ -166,6 +181,7 @@ class TelegramFrontend(Frontend):
outbox_backoff: float = 2.0, outbox_backoff: float = 2.0,
texts: TelegramTexts | None = None, texts: TelegramTexts | None = None,
album_delay: float = 1.0, album_delay: float = 1.0,
transcribe_timeout: float = 90.0,
) -> None: ) -> None:
self._token = token self._token = token
self.texts = texts or TelegramTexts() self.texts = texts or TelegramTexts()
@@ -190,6 +206,7 @@ class TelegramFrontend(Frontend):
self._topic_names: dict[int, str] = {} self._topic_names: dict[int, str] = {}
self._drafts: dict[str, Draft] = {} self._drafts: dict[str, Draft] = {}
self.album_delay = album_delay self.album_delay = album_delay
self.transcribe_timeout = transcribe_timeout
self._asks: dict[str, _Ask] = {} self._asks: dict[str, _Ask] = {}
self._albums: dict[str, _Album] = {} self._albums: dict[str, _Album] = {}
self._reactions: dict[int, tuple[int, int]] = {} self._reactions: dict[int, tuple[int, int]] = {}
@@ -687,10 +704,11 @@ class TelegramFrontend(Frontend):
return texts.attachment_sticker, None return texts.attachment_sticker, None
if message.animation: if message.animation:
return texts.attachment_animation, None return texts.attachment_animation, None
file_id, kind, size, name, media_type = _media(message) media = _media(message)
if file_id is None or name is None: if media is None:
return None, None return None, None
kind = texts.attachment_kinds.get(kind, kind) kind = texts.attachment_kinds.get(media.kind, media.kind)
size, name = media.size, media.name
if size and size > 20 * 1024 * 1024: if size and size > 20 * 1024 * 1024:
return ( return (
texts.attachment_too_big.format( texts.attachment_too_big.format(
@@ -706,7 +724,7 @@ class TelegramFrontend(Frontend):
folder.chmod(0o755) folder.chmod(0o755)
path = folder / f"{int(now)}-{Path(name).name}" path = folder / f"{int(now)}-{Path(name).name}"
try: try:
await self.bot.download(file_id, destination=path) await self.bot.download(media.file_id, destination=path)
path.chmod(0o644) path.chmod(0o644)
except (TelegramAPIError, OSError) as exc: except (TelegramAPIError, OSError) as exc:
_log.warning("attachment download failed: %s", exc) _log.warning("attachment download failed: %s", exc)
@@ -714,16 +732,38 @@ class TelegramFrontend(Frontend):
shown = texts.attachment_size.format(kb=size // 1024) if size else "" shown = texts.attachment_size.format(kb=size // 1024) if size else ""
keep = self.attachments.keep_days keep = self.attachments.keep_days
seen = ( seen = (
{"path": str(path), "media_type": media_type} {"path": str(path), "media_type": media.media_type}
if media_type and media_type.startswith("image/") if media.media_type and media.media_type.startswith("image/")
else None else None
) )
if keep is None: note = (
return texts.attachment.format(kind=kind, path=path, size=shown), seen texts.attachment.format(kind=kind, path=path, size=shown)
return ( if keep is None
texts.attachment_kept.format(kind=kind, path=path, size=shown, days=keep), else texts.attachment_kept.format(
seen, kind=kind, path=path, size=shown, days=keep
)
) )
transcribers = cast("GatewayRuntime", self._runtime).transcribers
if media.kind in _HEARD and transcribers:
audio = Audio(path, media.media_type, media.duration)
heard = await self._transcribe(transcribers, audio)
if heard:
note = texts.attachment_transcript.format(note=note, text=heard)
return note, seen
async def _transcribe(
self, transcribers: Sequence[Transcriber], audio: Audio
) -> str | None:
try:
async with asyncio.timeout(self.transcribe_timeout):
return await transcribe(transcribers, audio)
except TimeoutError:
_log.warning(
"transcription of %s took over %.0fs, the note goes without it",
audio.path.name,
self.transcribe_timeout,
)
return None
async def _sweep_loop(self, interval: float = 6 * 3600) -> None: async def _sweep_loop(self, interval: float = 6 * 3600) -> None:
while True: while True:
@@ -1190,13 +1230,10 @@ class TelegramFrontend(Frontend):
) )
def _media( def _media(message: Message) -> _Media | None:
message: Message,
) -> tuple[str | None, str, int | None, str | None, str | None]:
"""``(file_id, kind, size, name, media_type)`` of the message's media, if any."""
if message.photo: if message.photo:
photo = message.photo[-1] photo = message.photo[-1]
return ( return _Media(
photo.file_id, photo.file_id,
"photo", "photo",
photo.file_size, photo.file_size,
@@ -1206,32 +1243,43 @@ def _media(
if message.document: if message.document:
doc = message.document doc = message.document
name = doc.file_name or f"{doc.file_unique_id}.bin" name = doc.file_name or f"{doc.file_unique_id}.bin"
return doc.file_id, "document", doc.file_size, name, doc.mime_type return _Media(doc.file_id, "document", doc.file_size, name, doc.mime_type)
if message.voice: if message.voice:
v = message.voice v = message.voice
return v.file_id, "voice", v.file_size, f"{v.file_unique_id}.ogg", None return _Media(
v.file_id,
"voice",
v.file_size,
f"{v.file_unique_id}.ogg",
v.mime_type or "audio/ogg",
v.duration,
)
if message.audio: if message.audio:
a = message.audio a = message.audio
return ( return _Media(
a.file_id, a.file_id,
"audio", "audio",
a.file_size, a.file_size,
a.file_name or f"{a.file_unique_id}.mp3", a.file_name or f"{a.file_unique_id}.mp3",
None, a.mime_type,
a.duration,
) )
if message.video: if message.video:
v = message.video v = message.video
return ( return _Media(
v.file_id, v.file_id, "video", v.file_size, v.file_name or f"{v.file_unique_id}.mp4"
"video",
v.file_size,
v.file_name or f"{v.file_unique_id}.mp4",
None,
) )
if message.video_note: if message.video_note:
n = message.video_note n = message.video_note
return n.file_id, "video_note", n.file_size, f"{n.file_unique_id}.mp4", None return _Media(
return None, "", None, None, None n.file_id,
"video_note",
n.file_size,
f"{n.file_unique_id}.mp4",
"video/mp4",
n.duration,
)
return None
def _question_html(question: dict[str, Any]) -> str: def _question_html(question: dict[str, Any]) -> str:
@@ -71,6 +71,7 @@ class TelegramTexts:
"[attachment: {kind} {path}{size}; kept {days} days, move it into the " "[attachment: {kind} {path}{size}; kept {days} days, move it into the "
"vault if it matters]" "vault if it matters]"
) )
attachment_transcript: str = "{note}\ntranscript: {text}"
merge_nothing: str = "nothing to merge: this is not an open branch" merge_nothing: str = "nothing to merge: this is not an open branch"
merging: str = "🔀 merging into the master" merging: str = "🔀 merging into the master"
merged: str = "✅ merged into the master" merged: str = "✅ merged into the master"
@@ -0,0 +1,6 @@
"""Speech to text for audio a window receives: a chain of pluggable providers."""
from beaver_gateway.transcribe.base import Audio, Transcriber, transcribe
from beaver_gateway.transcribe.openai import OpenAITranscriber
__all__ = ["Audio", "OpenAITranscriber", "Transcriber", "transcribe"]
+52
View File
@@ -0,0 +1,52 @@
"""What a transcription provider is handed and the chain that tries them in order."""
from __future__ import annotations
import logging
from dataclasses import dataclass
from typing import TYPE_CHECKING, Protocol
if TYPE_CHECKING:
from collections.abc import Sequence
from pathlib import Path
__all__ = ["Audio", "Transcriber", "transcribe"]
_log = logging.getLogger("beaver_gateway.transcribe")
@dataclass(frozen=True, slots=True)
class Audio:
path: Path
"""The downloaded file, readable for as long as the call lasts."""
media_type: str | None = None
"""MIME type as the window reported it, e.g. ``audio/ogg``; ``None`` if unknown."""
duration: float | None = None
"""Length in seconds, when the window knows it."""
class Transcriber(Protocol):
async def __call__(self, audio: Audio) -> str | None: ...
async def transcribe(transcribers: Sequence[Transcriber], audio: Audio) -> str | None:
"""The first non-empty text from ``transcribers``, a failing one skipped."""
for transcriber in transcribers:
try:
text = await transcriber(audio)
except Exception as exc: # noqa: BLE001
_log.warning(
"transcriber %s failed on %s: %s: %s",
_name(transcriber),
audio.path.name,
type(exc).__name__,
exc,
)
continue
if text and text.strip():
return text.strip()
return None
def _name(transcriber: Transcriber) -> str:
return getattr(transcriber, "__qualname__", None) or type(transcriber).__name__
+57
View File
@@ -0,0 +1,57 @@
"""A provider for any OpenAI-compatible ``/audio/transcriptions`` endpoint."""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING
import aiohttp
if TYPE_CHECKING:
from beaver_gateway.transcribe.base import Audio
__all__ = ["OpenAITranscriber"]
@dataclass(frozen=True, slots=True)
class OpenAITranscriber:
"""OpenAI, Groq, faster-whisper-server, a whisper.cpp server: one wire."""
base_url: str = "https://api.openai.com/v1"
"""API root; ``/audio/transcriptions`` is appended."""
api_key: str | None = None
"""Sent as a bearer token; ``None`` for a local server without auth."""
model: str = "whisper-1"
"""Model name as the server knows it, e.g. ``whisper-large-v3`` on Groq."""
language: str | None = None
"""ISO-639-1 hint such as ``ru``; ``None`` lets the model detect it."""
timeout: float = 120.0
"""Seconds for the whole request, upload included."""
async def __call__(self, audio: Audio) -> str | None:
headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
form = aiohttp.FormData()
form.add_field("model", self.model)
form.add_field("response_format", "json")
if self.language:
form.add_field("language", self.language)
with audio.path.open("rb") as file:
form.add_field(
"file",
file,
filename=audio.path.name,
content_type=audio.media_type or "application/octet-stream",
)
async with (
aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=self.timeout)
) as http,
http.post(
f"{self.base_url.rstrip('/')}/audio/transcriptions",
data=form,
headers=headers,
) as response,
):
response.raise_for_status()
body = await response.json(content_type=None)
return str(body.get("text") or "") or None
+14
View File
@@ -1,6 +1,8 @@
import tempfile import tempfile
from pathlib import Path from pathlib import Path
import pytest
from beaver_gateway import config from beaver_gateway import config
@@ -14,3 +16,15 @@ def test_config_imports_sibling_modules():
) )
gw = config.load(root / "config.py") gw = config.load(root / "config.py")
assert gw.agents == [] assert gw.agents == []
def test_config_takes_transcribers_and_refuses_what_cannot_be_called():
root = Path(tempfile.mkdtemp(prefix="beaver-cfg-"))
(root / "config.py").write_text(
"gateway = Gateway(transcribers=[OpenAITranscriber(language='ru')])\n"
)
(gw_stt,) = config.load(root / "config.py").transcribers
assert gw_stt.language == "ru" and gw_stt.model == "whisper-1"
(root / "config.py").write_text("gateway = Gateway(transcribers=['whisper'])\n")
with pytest.raises(config.ConfigError, match=r"transcribers\[0\]"):
config.load(root / "config.py")
+87
View File
@@ -1,6 +1,7 @@
import asyncio import asyncio
import base64 import base64
import contextlib import contextlib
import dataclasses
import functools import functools
import tempfile import tempfile
from pathlib import Path from pathlib import Path
@@ -29,6 +30,7 @@ from beaver_gateway.frontends.telegram.render import (
to_html_tail, to_html_tail,
) )
from beaver_gateway.storage.models import ConversationBinding, Delivery, TelegramUpdate from beaver_gateway.storage.models import ConversationBinding, Delivery, TelegramUpdate
from beaver_gateway.transcribe import Audio
from sqlmodel import select from sqlmodel import select
from test_conversations import ScriptedClient, StubFrontend, World from test_conversations import ScriptedClient, StubFrontend, World
@@ -871,6 +873,91 @@ def test_sweep_drops_old_files_and_empty_day_folders() -> None:
assert not old.parent.exists() and fresh.exists() assert not old.parent.exists() and fresh.exists()
def _voice(caption: str) -> dict[str, Any]:
return {
"caption": caption,
"voice": {
"file_id": "v1",
"file_unique_id": "vu1",
"duration": 3,
"mime_type": "audio/ogg",
"file_size": 4096,
},
}
def _prompt_with(needle: str) -> str:
return next(p for c in ScriptedClient.instances for p in c.prompts if needle in p)
async def test_voice_without_transcribers_keeps_the_plain_note(stack: Stack) -> None:
root = Path(tempfile.mkdtemp(prefix="beaver-voice-"))
stack.tg.attachments = Attachments(dir=root, keep_days=3)
stack.bot.media(_voice("слушай"))
await stack.until(lambda: stack.sent_with("слушай"), what="reply")
(saved,) = [p for p in root.rglob("*") if p.is_file()]
assert saved.name.endswith("-vu1.ogg")
assert _prompt_with("слушай").endswith(
f"слушай\n\n[attachment: voice message {saved}, 4 KB; kept 3 days, "
"move it into the vault if it matters]"
)
async def test_voice_and_video_note_carry_the_transcript(stack: Stack) -> None:
heard: list[Audio] = []
async def stt(audio: Audio) -> str | None:
heard.append(audio)
return f"сказано {len(heard)}"
root = Path(tempfile.mkdtemp(prefix="beaver-voice-"))
stack.tg.attachments = Attachments(dir=root, keep_days=None)
stack.tg._runtime = dataclasses.replace( # noqa: SLF001
stack.tg._runtime, # noqa: SLF001
transcribers=(stt,),
)
stack.bot.media(_voice("раз"))
await stack.until(lambda: stack.sent_with("раз"), what="voice reply")
(voice,) = heard
assert voice.media_type == "audio/ogg" and voice.duration == 3
assert voice.path.name.endswith("-vu1.ogg") and voice.path.read_bytes() == b"data"
assert _prompt_with("раз").endswith(
f"раз\n\n[attachment: voice message {voice.path}, 4 KB]\ntranscript: сказано 1"
)
stack.bot.media(
{
"video_note": {
"file_id": "n1",
"file_unique_id": "nu1",
"length": 240,
"duration": 5,
}
}
)
await stack.until(lambda: stack.sent_with("сказано 2"), what="video note reply")
assert (heard[1].media_type, heard[1].duration) == ("video/mp4", 5)
stack.bot.media(_photo("фото"))
await stack.until(lambda: stack.sent_with("фото"), what="photo reply")
assert len(heard) == 2 and "transcript" not in _prompt_with("фото")
async def test_a_hung_transcriber_does_not_hold_the_turn(stack: Stack) -> None:
async def hung(_: Audio) -> str:
await asyncio.sleep(10)
return "поздно"
stack.tg.transcribe_timeout = 0.05
stack.tg._runtime = dataclasses.replace( # noqa: SLF001
stack.tg._runtime, # noqa: SLF001
transcribers=(hung,),
)
stack.bot.media(_voice("жду"))
await stack.until(lambda: stack.sent_with("жду"), what="reply")
prompt = _prompt_with("жду")
assert prompt.endswith(" KB; kept 7 days, move it into the vault if it matters]")
assert "transcript" not in prompt
async def test_topic_rename_updates_the_title(stack: Stack) -> None: async def test_topic_rename_updates_the_title(stack: Stack) -> None:
stack.bot.topic_created("план", 7) stack.bot.topic_created("план", 7)
stack.bot.message("hello", thread=7) stack.bot.message("hello", thread=7)
+101
View File
@@ -0,0 +1,101 @@
import tempfile
from pathlib import Path
from typing import Any
import pytest
from aioresponses import CallbackResult, aioresponses
from beaver_gateway.transcribe import Audio, OpenAITranscriber, transcribe
URL = "http://whisper.local/v1/audio/transcriptions"
@pytest.fixture
def audio() -> Audio:
path = Path(tempfile.mkdtemp(prefix="beaver-stt-")) / "u1.ogg"
path.write_bytes(b"OggS")
return Audio(path, "audio/ogg", 3)
def saying(text: str | None, calls: list[str] | None = None):
async def transcriber(audio: Audio) -> str | None:
if calls is not None:
calls.append(text or "")
return text
return transcriber
async def broken(audio: Audio) -> str | None:
msg = "provider down"
raise ConnectionError(msg)
async def test_the_first_non_empty_text_wins(audio: Audio) -> None:
calls: list[str] = []
chain = [saying(None, calls), saying(" \n", calls), saying(" привет ", calls)]
chain.append(saying("never", calls))
assert await transcribe(chain, audio) == "привет"
assert calls == ["", " \n", " привет "]
async def test_a_raising_provider_is_skipped(audio: Audio, caplog) -> None:
assert await transcribe([broken, saying("ok")], audio) == "ok"
(record,) = [r for r in caplog.records if "provider down" in r.getMessage()]
assert record.levelname == "WARNING" and record.exc_info is None
async def test_nothing_comes_back_as_none(audio: Audio) -> None:
assert await transcribe([], audio) is None
assert await transcribe([broken, saying("")], audio) is None
async def test_openai_transcriber_posts_the_file_and_reads_the_text(
audio: Audio,
) -> None:
seen: dict[str, Any] = {}
def answer(url, **kwargs: Any) -> CallbackResult:
fields = kwargs["data"]._fields
seen["headers"] = kwargs["headers"]
seen["fields"] = {f[0]["name"]: f[2] for f in fields}
seen["file"] = next(f for f in fields if f[0]["name"] == "file")
return CallbackResult(payload={"text": " привет "})
stt = OpenAITranscriber(
base_url="http://whisper.local/v1/", api_key="sk", model="m", language="ru"
)
with aioresponses() as http:
http.post(URL, callback=answer)
assert await stt(audio) == " привет "
assert seen["headers"] == {"Authorization": "Bearer sk"}
assert seen["fields"]["model"] == "m"
assert seen["fields"]["language"] == "ru"
assert seen["fields"]["response_format"] == "json"
options, headers, _ = seen["file"]
assert options["filename"] == "u1.ogg"
assert headers["Content-Type"] == "audio/ogg"
async def test_openai_transcriber_without_a_key_sends_no_auth(audio: Audio) -> None:
seen: dict[str, Any] = {}
def answer(url, **kwargs: Any) -> CallbackResult:
seen["headers"] = kwargs["headers"]
seen["fields"] = [f[0]["name"] for f in kwargs["data"]._fields]
return CallbackResult(payload={"text": ""})
with aioresponses() as http:
http.post(URL, callback=answer)
assert (
await OpenAITranscriber(base_url="http://whisper.local/v1")(audio) is None
)
assert seen["headers"] == {}
assert "language" not in seen["fields"]
async def test_openai_transcriber_error_reaches_the_chain(audio: Audio) -> None:
stt = OpenAITranscriber(base_url="http://whisper.local/v1")
with aioresponses() as http:
http.post(URL, status=401, payload={"error": "bad key"})
assert await transcribe([stt, saying("fallback")], audio) == "fallback"