feat(transcribe,telegram,config): voice notes, round videos and audio get a transcript from pluggable providers, an openai-compatible one built in
This commit is contained in:
1 parent
944b68e275
commit
42a7058f75
12 files changed
+454
-28
No files matched your search
@@ -38,6 +38,9 @@ is a full setup built on them.
|
||||
write-only from the admin, handed out only through `secret run NAME --
|
||||
cmd` (one `sudo` rule for `beaver-runner`, a root-only socket, files in
|
||||
`/dev/shm`, values masked in the command's output).
|
||||
- **transcribe/** - speech to text for the voice notes, round videos and
|
||||
audio a window receives: providers tried in order, `OpenAITranscriber`
|
||||
for any OpenAI-compatible endpoint built in.
|
||||
- **events/** - the in-process bus every frontend taps, and the stream
|
||||
event protocol backends emit.
|
||||
- **storage/** - SQLModel tables and the Postgres session store.
|
||||
@@ -67,6 +70,46 @@ file next to it adds Postgres. Without Docker: `uv sync`, build the SPA with
|
||||
`DATABASE_URL` and `CONFIG_PATH` set. A setup's own secrets come from `.env`
|
||||
next to its config (`load_dotenv` runs before the config is executed).
|
||||
|
||||
## Transcription
|
||||
|
||||
A voice note, a round video or an audio file sent to the Telegram window is
|
||||
downloaded as any attachment; with `Gateway.transcribers` set, the frontend
|
||||
runs them in order and the first non-empty text rides under the attachment
|
||||
note (`TelegramTexts.attachment_transcript`). A provider that raises is
|
||||
logged and the next one tried; the whole chain gives up after the
|
||||
frontend's `transcribe_timeout` (90 s) and the note goes without the text.
|
||||
No transcribers, nothing changes.
|
||||
|
||||
```python
|
||||
import os
|
||||
|
||||
from beaver_gateway.transcribe import Audio, OpenAITranscriber
|
||||
|
||||
|
||||
async def voice_only(audio: Audio) -> str | None:
|
||||
if audio.media_type != "audio/ogg":
|
||||
return None
|
||||
return await my_service.transcribe(audio.path, seconds=audio.duration)
|
||||
|
||||
|
||||
gateway = Gateway(
|
||||
transcribers=[
|
||||
voice_only,
|
||||
OpenAITranscriber(
|
||||
base_url="https://api.groq.com/openai/v1",
|
||||
api_key=os.environ["GROQ_API_KEY"],
|
||||
model="whisper-large-v3",
|
||||
language="ru",
|
||||
),
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
A provider is any `async (Audio) -> str | None`; `None` passes the audio
|
||||
on. `OpenAITranscriber` posts the file to `{base_url}/audio/transcriptions`,
|
||||
the wire of OpenAI, Groq, faster-whisper-server and a whisper.cpp server;
|
||||
`api_key=None` for a local one without auth.
|
||||
|
||||
## Develop
|
||||
|
||||
```sh
|
||||
|
||||
@@ -201,6 +201,7 @@ async def run(gateway: Gateway, settings: Settings) -> None:
|
||||
scopes=scopes_with(fe.scope for fe in gateway.frontends),
|
||||
send_file_roots=_roots(settings.send_file_roots),
|
||||
secrets=secrets,
|
||||
transcribers=tuple(gateway.transcribers),
|
||||
)
|
||||
for fe in gateway.frontends:
|
||||
fe.configure(runtime)
|
||||
|
||||
@@ -16,6 +16,7 @@ from beaver_gateway.conversations.texts import ConversationTexts
|
||||
from beaver_gateway.frontends.base import Frontend
|
||||
from beaver_gateway.jobs.scheduler import Job
|
||||
from beaver_gateway.mcp.types import HttpMcp, McpServer, PythonToolMcp, StdioMcp
|
||||
from beaver_gateway.transcribe import Audio, OpenAITranscriber
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Awaitable, Callable
|
||||
@@ -27,6 +28,7 @@ if TYPE_CHECKING:
|
||||
from beaver_gateway.conversations.texts import UserSaid
|
||||
from beaver_gateway.jobs.scheduler import Budget
|
||||
from beaver_gateway.mcp.types import McpServerT
|
||||
from beaver_gateway.transcribe import Transcriber
|
||||
from beaver_gateway.vault.watch import VaultWatch
|
||||
|
||||
__all__ = ["ConfigError", "Gateway", "load"]
|
||||
@@ -51,6 +53,8 @@ class Gateway:
|
||||
"""Envelope lookup on the user's text: pointers the setup derives from its files."""
|
||||
user_sink: Callable[[UserSaid], Awaitable[None] | None] | None = None
|
||||
"""Sees every user message entering a master or branch turn."""
|
||||
transcribers: list[Transcriber] = field(default_factory=list)
|
||||
"""Speech to text for voice and audio a window receives; the first text wins."""
|
||||
budget: Budget | None = None
|
||||
"""Subscription utilisation past which non-critical jobs wait."""
|
||||
distiller: Distiller | None = None
|
||||
@@ -75,6 +79,8 @@ _PUBLIC_NAMES: dict[str, Any] = {
|
||||
"ExposedMcp": ExposedMcp,
|
||||
"Gateway": Gateway,
|
||||
"ConversationTexts": ConversationTexts,
|
||||
"Audio": Audio,
|
||||
"OpenAITranscriber": OpenAITranscriber,
|
||||
}
|
||||
|
||||
_McpInstance = StdioMcp | HttpMcp | PythonToolMcp
|
||||
@@ -144,6 +150,13 @@ def _validate(gw: Gateway, path: Path) -> None:
|
||||
f"a frontend scope is a lowercase name like 'voice', never '*'"
|
||||
)
|
||||
raise ConfigError(msg)
|
||||
for i, t in enumerate(gw.transcribers):
|
||||
if not callable(t):
|
||||
msg = (
|
||||
f"{path}: gateway.transcribers[{i}] must be an async callable "
|
||||
f"taking an Audio, got {type(t).__name__}"
|
||||
)
|
||||
raise ConfigError(msg)
|
||||
names: set[str] = set()
|
||||
for i, j in enumerate(gw.jobs):
|
||||
if not isinstance(j, Job):
|
||||
|
||||
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
|
||||
from beaver_gateway.security.auth import TokenStore
|
||||
from beaver_gateway.storage import Database
|
||||
from beaver_gateway.storage.models import Conversation, ConversationBinding
|
||||
from beaver_gateway.transcribe import Transcriber
|
||||
|
||||
TurnLogHandler = Callable[[TurnRecord], Awaitable[None]]
|
||||
|
||||
@@ -65,6 +66,8 @@ class GatewayRuntime:
|
||||
"""Directories ``send_file`` may read, from ``SEND_FILE_ROOTS``."""
|
||||
secrets: SecretStore | None = None
|
||||
"""Sealed secrets for ``secret run``; ``None`` in setups without them."""
|
||||
transcribers: Sequence[Transcriber] = ()
|
||||
"""``Gateway.transcribers``: speech to text a window runs on audio it receives."""
|
||||
|
||||
|
||||
class Frontend(ABC):
|
||||
|
||||
@@ -48,13 +48,17 @@ from beaver_gateway.frontends.telegram.inbox import Inbox
|
||||
from beaver_gateway.frontends.telegram.outbox import Outbox
|
||||
from beaver_gateway.frontends.telegram.render import chunks, status_label, to_html
|
||||
from beaver_gateway.frontends.telegram.texts import TelegramTexts
|
||||
from beaver_gateway.transcribe import Audio, transcribe
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
from beaver_gateway.conversations.kinds import Kind
|
||||
from beaver_gateway.conversations.service import Conversations
|
||||
from beaver_gateway.events.bus import Event, EventBus
|
||||
from beaver_gateway.frontends.base import GatewayRuntime
|
||||
from beaver_gateway.storage.models import Conversation, ConversationBinding
|
||||
from beaver_gateway.transcribe import Transcriber
|
||||
|
||||
__all__ = ["FRONTEND", "Attachments", "TelegramFrontend"]
|
||||
|
||||
@@ -65,6 +69,7 @@ _DONE = "done"
|
||||
_COMMANDS = ("merge", "new", "chat", "status", "help", "start")
|
||||
_CAPTION = 1024
|
||||
_NO_CAPTION = ("video_note",)
|
||||
_HEARD = ("voice", "audio", "video_note")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -132,6 +137,16 @@ class _Ask:
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Media:
|
||||
file_id: str
|
||||
kind: str
|
||||
size: int | None
|
||||
name: str
|
||||
media_type: str | None = None
|
||||
duration: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Album:
|
||||
message: Message
|
||||
@@ -166,6 +181,7 @@ class TelegramFrontend(Frontend):
|
||||
outbox_backoff: float = 2.0,
|
||||
texts: TelegramTexts | None = None,
|
||||
album_delay: float = 1.0,
|
||||
transcribe_timeout: float = 90.0,
|
||||
) -> None:
|
||||
self._token = token
|
||||
self.texts = texts or TelegramTexts()
|
||||
@@ -190,6 +206,7 @@ class TelegramFrontend(Frontend):
|
||||
self._topic_names: dict[int, str] = {}
|
||||
self._drafts: dict[str, Draft] = {}
|
||||
self.album_delay = album_delay
|
||||
self.transcribe_timeout = transcribe_timeout
|
||||
self._asks: dict[str, _Ask] = {}
|
||||
self._albums: dict[str, _Album] = {}
|
||||
self._reactions: dict[int, tuple[int, int]] = {}
|
||||
@@ -687,10 +704,11 @@ class TelegramFrontend(Frontend):
|
||||
return texts.attachment_sticker, None
|
||||
if message.animation:
|
||||
return texts.attachment_animation, None
|
||||
file_id, kind, size, name, media_type = _media(message)
|
||||
if file_id is None or name is None:
|
||||
media = _media(message)
|
||||
if media is None:
|
||||
return None, None
|
||||
kind = texts.attachment_kinds.get(kind, kind)
|
||||
kind = texts.attachment_kinds.get(media.kind, media.kind)
|
||||
size, name = media.size, media.name
|
||||
if size and size > 20 * 1024 * 1024:
|
||||
return (
|
||||
texts.attachment_too_big.format(
|
||||
@@ -706,7 +724,7 @@ class TelegramFrontend(Frontend):
|
||||
folder.chmod(0o755)
|
||||
path = folder / f"{int(now)}-{Path(name).name}"
|
||||
try:
|
||||
await self.bot.download(file_id, destination=path)
|
||||
await self.bot.download(media.file_id, destination=path)
|
||||
path.chmod(0o644)
|
||||
except (TelegramAPIError, OSError) as exc:
|
||||
_log.warning("attachment download failed: %s", exc)
|
||||
@@ -714,16 +732,38 @@ class TelegramFrontend(Frontend):
|
||||
shown = texts.attachment_size.format(kb=size // 1024) if size else ""
|
||||
keep = self.attachments.keep_days
|
||||
seen = (
|
||||
{"path": str(path), "media_type": media_type}
|
||||
if media_type and media_type.startswith("image/")
|
||||
{"path": str(path), "media_type": media.media_type}
|
||||
if media.media_type and media.media_type.startswith("image/")
|
||||
else None
|
||||
)
|
||||
if keep is None:
|
||||
return texts.attachment.format(kind=kind, path=path, size=shown), seen
|
||||
return (
|
||||
texts.attachment_kept.format(kind=kind, path=path, size=shown, days=keep),
|
||||
seen,
|
||||
note = (
|
||||
texts.attachment.format(kind=kind, path=path, size=shown)
|
||||
if keep is None
|
||||
else texts.attachment_kept.format(
|
||||
kind=kind, path=path, size=shown, days=keep
|
||||
)
|
||||
)
|
||||
transcribers = cast("GatewayRuntime", self._runtime).transcribers
|
||||
if media.kind in _HEARD and transcribers:
|
||||
audio = Audio(path, media.media_type, media.duration)
|
||||
heard = await self._transcribe(transcribers, audio)
|
||||
if heard:
|
||||
note = texts.attachment_transcript.format(note=note, text=heard)
|
||||
return note, seen
|
||||
|
||||
async def _transcribe(
|
||||
self, transcribers: Sequence[Transcriber], audio: Audio
|
||||
) -> str | None:
|
||||
try:
|
||||
async with asyncio.timeout(self.transcribe_timeout):
|
||||
return await transcribe(transcribers, audio)
|
||||
except TimeoutError:
|
||||
_log.warning(
|
||||
"transcription of %s took over %.0fs, the note goes without it",
|
||||
audio.path.name,
|
||||
self.transcribe_timeout,
|
||||
)
|
||||
return None
|
||||
|
||||
async def _sweep_loop(self, interval: float = 6 * 3600) -> None:
|
||||
while True:
|
||||
@@ -1190,13 +1230,10 @@ class TelegramFrontend(Frontend):
|
||||
)
|
||||
|
||||
|
||||
def _media(
|
||||
message: Message,
|
||||
) -> tuple[str | None, str, int | None, str | None, str | None]:
|
||||
"""``(file_id, kind, size, name, media_type)`` of the message's media, if any."""
|
||||
def _media(message: Message) -> _Media | None:
|
||||
if message.photo:
|
||||
photo = message.photo[-1]
|
||||
return (
|
||||
return _Media(
|
||||
photo.file_id,
|
||||
"photo",
|
||||
photo.file_size,
|
||||
@@ -1206,32 +1243,43 @@ def _media(
|
||||
if message.document:
|
||||
doc = message.document
|
||||
name = doc.file_name or f"{doc.file_unique_id}.bin"
|
||||
return doc.file_id, "document", doc.file_size, name, doc.mime_type
|
||||
return _Media(doc.file_id, "document", doc.file_size, name, doc.mime_type)
|
||||
if message.voice:
|
||||
v = message.voice
|
||||
return v.file_id, "voice", v.file_size, f"{v.file_unique_id}.ogg", None
|
||||
return _Media(
|
||||
v.file_id,
|
||||
"voice",
|
||||
v.file_size,
|
||||
f"{v.file_unique_id}.ogg",
|
||||
v.mime_type or "audio/ogg",
|
||||
v.duration,
|
||||
)
|
||||
if message.audio:
|
||||
a = message.audio
|
||||
return (
|
||||
return _Media(
|
||||
a.file_id,
|
||||
"audio",
|
||||
a.file_size,
|
||||
a.file_name or f"{a.file_unique_id}.mp3",
|
||||
None,
|
||||
a.mime_type,
|
||||
a.duration,
|
||||
)
|
||||
if message.video:
|
||||
v = message.video
|
||||
return (
|
||||
v.file_id,
|
||||
"video",
|
||||
v.file_size,
|
||||
v.file_name or f"{v.file_unique_id}.mp4",
|
||||
None,
|
||||
return _Media(
|
||||
v.file_id, "video", v.file_size, v.file_name or f"{v.file_unique_id}.mp4"
|
||||
)
|
||||
if message.video_note:
|
||||
n = message.video_note
|
||||
return n.file_id, "video_note", n.file_size, f"{n.file_unique_id}.mp4", None
|
||||
return None, "", None, None, None
|
||||
return _Media(
|
||||
n.file_id,
|
||||
"video_note",
|
||||
n.file_size,
|
||||
f"{n.file_unique_id}.mp4",
|
||||
"video/mp4",
|
||||
n.duration,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _question_html(question: dict[str, Any]) -> str:
|
||||
|
||||
@@ -71,6 +71,7 @@ class TelegramTexts:
|
||||
"[attachment: {kind} {path}{size}; kept {days} days, move it into the "
|
||||
"vault if it matters]"
|
||||
)
|
||||
attachment_transcript: str = "{note}\ntranscript: {text}"
|
||||
merge_nothing: str = "nothing to merge: this is not an open branch"
|
||||
merging: str = "🔀 merging into the master"
|
||||
merged: str = "✅ merged into the master"
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Speech to text for audio a window receives: a chain of pluggable providers."""
|
||||
|
||||
from beaver_gateway.transcribe.base import Audio, Transcriber, transcribe
|
||||
from beaver_gateway.transcribe.openai import OpenAITranscriber
|
||||
|
||||
__all__ = ["Audio", "OpenAITranscriber", "Transcriber", "transcribe"]
|
||||
@@ -0,0 +1,52 @@
|
||||
"""What a transcription provider is handed and the chain that tries them in order."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
__all__ = ["Audio", "Transcriber", "transcribe"]
|
||||
|
||||
_log = logging.getLogger("beaver_gateway.transcribe")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Audio:
|
||||
path: Path
|
||||
"""The downloaded file, readable for as long as the call lasts."""
|
||||
media_type: str | None = None
|
||||
"""MIME type as the window reported it, e.g. ``audio/ogg``; ``None`` if unknown."""
|
||||
duration: float | None = None
|
||||
"""Length in seconds, when the window knows it."""
|
||||
|
||||
|
||||
class Transcriber(Protocol):
|
||||
async def __call__(self, audio: Audio) -> str | None: ...
|
||||
|
||||
|
||||
async def transcribe(transcribers: Sequence[Transcriber], audio: Audio) -> str | None:
|
||||
"""The first non-empty text from ``transcribers``, a failing one skipped."""
|
||||
for transcriber in transcribers:
|
||||
try:
|
||||
text = await transcriber(audio)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_log.warning(
|
||||
"transcriber %s failed on %s: %s: %s",
|
||||
_name(transcriber),
|
||||
audio.path.name,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
if text and text.strip():
|
||||
return text.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _name(transcriber: Transcriber) -> str:
|
||||
return getattr(transcriber, "__qualname__", None) or type(transcriber).__name__
|
||||
@@ -0,0 +1,57 @@
|
||||
"""A provider for any OpenAI-compatible ``/audio/transcriptions`` endpoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import aiohttp
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from beaver_gateway.transcribe.base import Audio
|
||||
|
||||
__all__ = ["OpenAITranscriber"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OpenAITranscriber:
|
||||
"""OpenAI, Groq, faster-whisper-server, a whisper.cpp server: one wire."""
|
||||
|
||||
base_url: str = "https://api.openai.com/v1"
|
||||
"""API root; ``/audio/transcriptions`` is appended."""
|
||||
api_key: str | None = None
|
||||
"""Sent as a bearer token; ``None`` for a local server without auth."""
|
||||
model: str = "whisper-1"
|
||||
"""Model name as the server knows it, e.g. ``whisper-large-v3`` on Groq."""
|
||||
language: str | None = None
|
||||
"""ISO-639-1 hint such as ``ru``; ``None`` lets the model detect it."""
|
||||
timeout: float = 120.0
|
||||
"""Seconds for the whole request, upload included."""
|
||||
|
||||
async def __call__(self, audio: Audio) -> str | None:
|
||||
headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
|
||||
form = aiohttp.FormData()
|
||||
form.add_field("model", self.model)
|
||||
form.add_field("response_format", "json")
|
||||
if self.language:
|
||||
form.add_field("language", self.language)
|
||||
with audio.path.open("rb") as file:
|
||||
form.add_field(
|
||||
"file",
|
||||
file,
|
||||
filename=audio.path.name,
|
||||
content_type=audio.media_type or "application/octet-stream",
|
||||
)
|
||||
async with (
|
||||
aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=self.timeout)
|
||||
) as http,
|
||||
http.post(
|
||||
f"{self.base_url.rstrip('/')}/audio/transcriptions",
|
||||
data=form,
|
||||
headers=headers,
|
||||
) as response,
|
||||
):
|
||||
response.raise_for_status()
|
||||
body = await response.json(content_type=None)
|
||||
return str(body.get("text") or "") or None
|
||||
@@ -1,6 +1,8 @@
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from beaver_gateway import config
|
||||
|
||||
|
||||
@@ -14,3 +16,15 @@ def test_config_imports_sibling_modules():
|
||||
)
|
||||
gw = config.load(root / "config.py")
|
||||
assert gw.agents == []
|
||||
|
||||
|
||||
def test_config_takes_transcribers_and_refuses_what_cannot_be_called():
|
||||
root = Path(tempfile.mkdtemp(prefix="beaver-cfg-"))
|
||||
(root / "config.py").write_text(
|
||||
"gateway = Gateway(transcribers=[OpenAITranscriber(language='ru')])\n"
|
||||
)
|
||||
(gw_stt,) = config.load(root / "config.py").transcribers
|
||||
assert gw_stt.language == "ru" and gw_stt.model == "whisper-1"
|
||||
(root / "config.py").write_text("gateway = Gateway(transcribers=['whisper'])\n")
|
||||
with pytest.raises(config.ConfigError, match=r"transcribers\[0\]"):
|
||||
config.load(root / "config.py")
|
||||
@@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import dataclasses
|
||||
import functools
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
@@ -29,6 +30,7 @@ from beaver_gateway.frontends.telegram.render import (
|
||||
to_html_tail,
|
||||
)
|
||||
from beaver_gateway.storage.models import ConversationBinding, Delivery, TelegramUpdate
|
||||
from beaver_gateway.transcribe import Audio
|
||||
from sqlmodel import select
|
||||
from test_conversations import ScriptedClient, StubFrontend, World
|
||||
|
||||
@@ -871,6 +873,91 @@ def test_sweep_drops_old_files_and_empty_day_folders() -> None:
|
||||
assert not old.parent.exists() and fresh.exists()
|
||||
|
||||
|
||||
def _voice(caption: str) -> dict[str, Any]:
|
||||
return {
|
||||
"caption": caption,
|
||||
"voice": {
|
||||
"file_id": "v1",
|
||||
"file_unique_id": "vu1",
|
||||
"duration": 3,
|
||||
"mime_type": "audio/ogg",
|
||||
"file_size": 4096,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _prompt_with(needle: str) -> str:
|
||||
return next(p for c in ScriptedClient.instances for p in c.prompts if needle in p)
|
||||
|
||||
|
||||
async def test_voice_without_transcribers_keeps_the_plain_note(stack: Stack) -> None:
|
||||
root = Path(tempfile.mkdtemp(prefix="beaver-voice-"))
|
||||
stack.tg.attachments = Attachments(dir=root, keep_days=3)
|
||||
stack.bot.media(_voice("слушай"))
|
||||
await stack.until(lambda: stack.sent_with("слушай"), what="reply")
|
||||
(saved,) = [p for p in root.rglob("*") if p.is_file()]
|
||||
assert saved.name.endswith("-vu1.ogg")
|
||||
assert _prompt_with("слушай").endswith(
|
||||
f"слушай\n\n[attachment: voice message {saved}, 4 KB; kept 3 days, "
|
||||
"move it into the vault if it matters]"
|
||||
)
|
||||
|
||||
|
||||
async def test_voice_and_video_note_carry_the_transcript(stack: Stack) -> None:
|
||||
heard: list[Audio] = []
|
||||
|
||||
async def stt(audio: Audio) -> str | None:
|
||||
heard.append(audio)
|
||||
return f"сказано {len(heard)}"
|
||||
|
||||
root = Path(tempfile.mkdtemp(prefix="beaver-voice-"))
|
||||
stack.tg.attachments = Attachments(dir=root, keep_days=None)
|
||||
stack.tg._runtime = dataclasses.replace( # noqa: SLF001
|
||||
stack.tg._runtime, # noqa: SLF001
|
||||
transcribers=(stt,),
|
||||
)
|
||||
stack.bot.media(_voice("раз"))
|
||||
await stack.until(lambda: stack.sent_with("раз"), what="voice reply")
|
||||
(voice,) = heard
|
||||
assert voice.media_type == "audio/ogg" and voice.duration == 3
|
||||
assert voice.path.name.endswith("-vu1.ogg") and voice.path.read_bytes() == b"data"
|
||||
assert _prompt_with("раз").endswith(
|
||||
f"раз\n\n[attachment: voice message {voice.path}, 4 KB]\ntranscript: сказано 1"
|
||||
)
|
||||
stack.bot.media(
|
||||
{
|
||||
"video_note": {
|
||||
"file_id": "n1",
|
||||
"file_unique_id": "nu1",
|
||||
"length": 240,
|
||||
"duration": 5,
|
||||
}
|
||||
}
|
||||
)
|
||||
await stack.until(lambda: stack.sent_with("сказано 2"), what="video note reply")
|
||||
assert (heard[1].media_type, heard[1].duration) == ("video/mp4", 5)
|
||||
stack.bot.media(_photo("фото"))
|
||||
await stack.until(lambda: stack.sent_with("фото"), what="photo reply")
|
||||
assert len(heard) == 2 and "transcript" not in _prompt_with("фото")
|
||||
|
||||
|
||||
async def test_a_hung_transcriber_does_not_hold_the_turn(stack: Stack) -> None:
|
||||
async def hung(_: Audio) -> str:
|
||||
await asyncio.sleep(10)
|
||||
return "поздно"
|
||||
|
||||
stack.tg.transcribe_timeout = 0.05
|
||||
stack.tg._runtime = dataclasses.replace( # noqa: SLF001
|
||||
stack.tg._runtime, # noqa: SLF001
|
||||
transcribers=(hung,),
|
||||
)
|
||||
stack.bot.media(_voice("жду"))
|
||||
await stack.until(lambda: stack.sent_with("жду"), what="reply")
|
||||
prompt = _prompt_with("жду")
|
||||
assert prompt.endswith(" KB; kept 7 days, move it into the vault if it matters]")
|
||||
assert "transcript" not in prompt
|
||||
|
||||
|
||||
async def test_topic_rename_updates_the_title(stack: Stack) -> None:
|
||||
stack.bot.topic_created("план", 7)
|
||||
stack.bot.message("hello", thread=7)
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from aioresponses import CallbackResult, aioresponses
|
||||
|
||||
from beaver_gateway.transcribe import Audio, OpenAITranscriber, transcribe
|
||||
|
||||
URL = "http://whisper.local/v1/audio/transcriptions"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def audio() -> Audio:
|
||||
path = Path(tempfile.mkdtemp(prefix="beaver-stt-")) / "u1.ogg"
|
||||
path.write_bytes(b"OggS")
|
||||
return Audio(path, "audio/ogg", 3)
|
||||
|
||||
|
||||
def saying(text: str | None, calls: list[str] | None = None):
|
||||
async def transcriber(audio: Audio) -> str | None:
|
||||
if calls is not None:
|
||||
calls.append(text or "")
|
||||
return text
|
||||
|
||||
return transcriber
|
||||
|
||||
|
||||
async def broken(audio: Audio) -> str | None:
|
||||
msg = "provider down"
|
||||
raise ConnectionError(msg)
|
||||
|
||||
|
||||
async def test_the_first_non_empty_text_wins(audio: Audio) -> None:
|
||||
calls: list[str] = []
|
||||
chain = [saying(None, calls), saying(" \n", calls), saying(" привет ", calls)]
|
||||
chain.append(saying("never", calls))
|
||||
assert await transcribe(chain, audio) == "привет"
|
||||
assert calls == ["", " \n", " привет "]
|
||||
|
||||
|
||||
async def test_a_raising_provider_is_skipped(audio: Audio, caplog) -> None:
|
||||
assert await transcribe([broken, saying("ok")], audio) == "ok"
|
||||
(record,) = [r for r in caplog.records if "provider down" in r.getMessage()]
|
||||
assert record.levelname == "WARNING" and record.exc_info is None
|
||||
|
||||
|
||||
async def test_nothing_comes_back_as_none(audio: Audio) -> None:
|
||||
assert await transcribe([], audio) is None
|
||||
assert await transcribe([broken, saying("")], audio) is None
|
||||
|
||||
|
||||
async def test_openai_transcriber_posts_the_file_and_reads_the_text(
|
||||
audio: Audio,
|
||||
) -> None:
|
||||
seen: dict[str, Any] = {}
|
||||
|
||||
def answer(url, **kwargs: Any) -> CallbackResult:
|
||||
fields = kwargs["data"]._fields
|
||||
seen["headers"] = kwargs["headers"]
|
||||
seen["fields"] = {f[0]["name"]: f[2] for f in fields}
|
||||
seen["file"] = next(f for f in fields if f[0]["name"] == "file")
|
||||
return CallbackResult(payload={"text": " привет "})
|
||||
|
||||
stt = OpenAITranscriber(
|
||||
base_url="http://whisper.local/v1/", api_key="sk", model="m", language="ru"
|
||||
)
|
||||
with aioresponses() as http:
|
||||
http.post(URL, callback=answer)
|
||||
assert await stt(audio) == " привет "
|
||||
assert seen["headers"] == {"Authorization": "Bearer sk"}
|
||||
assert seen["fields"]["model"] == "m"
|
||||
assert seen["fields"]["language"] == "ru"
|
||||
assert seen["fields"]["response_format"] == "json"
|
||||
options, headers, _ = seen["file"]
|
||||
assert options["filename"] == "u1.ogg"
|
||||
assert headers["Content-Type"] == "audio/ogg"
|
||||
|
||||
|
||||
async def test_openai_transcriber_without_a_key_sends_no_auth(audio: Audio) -> None:
|
||||
seen: dict[str, Any] = {}
|
||||
|
||||
def answer(url, **kwargs: Any) -> CallbackResult:
|
||||
seen["headers"] = kwargs["headers"]
|
||||
seen["fields"] = [f[0]["name"] for f in kwargs["data"]._fields]
|
||||
return CallbackResult(payload={"text": ""})
|
||||
|
||||
with aioresponses() as http:
|
||||
http.post(URL, callback=answer)
|
||||
assert (
|
||||
await OpenAITranscriber(base_url="http://whisper.local/v1")(audio) is None
|
||||
)
|
||||
assert seen["headers"] == {}
|
||||
assert "language" not in seen["fields"]
|
||||
|
||||
|
||||
async def test_openai_transcriber_error_reaches_the_chain(audio: Audio) -> None:
|
||||
stt = OpenAITranscriber(base_url="http://whisper.local/v1")
|
||||
with aioresponses() as http:
|
||||
http.post(URL, status=401, payload={"error": "bad key"})
|
||||
assert await transcribe([stt, saying("fallback")], audio) == "fallback"
|
||||
Reference in new issue
Block a user