feat(transcribe,telegram,config): voice notes, round videos and audio get a transcript from pluggable providers, an openai-compatible one built in
This commit is contained in:
1 parent
944b68e275
commit
42a7058f75
12 files changed
+454
-28
No files matched your search
@@ -38,6 +38,9 @@ is a full setup built on them.
|
|||||||
write-only from the admin, handed out only through `secret run NAME --
|
write-only from the admin, handed out only through `secret run NAME --
|
||||||
cmd` (one `sudo` rule for `beaver-runner`, a root-only socket, files in
|
cmd` (one `sudo` rule for `beaver-runner`, a root-only socket, files in
|
||||||
`/dev/shm`, values masked in the command's output).
|
`/dev/shm`, values masked in the command's output).
|
||||||
|
- **transcribe/** - speech to text for the voice notes, round videos and
|
||||||
|
audio a window receives: providers tried in order, `OpenAITranscriber`
|
||||||
|
for any OpenAI-compatible endpoint built in.
|
||||||
- **events/** - the in-process bus every frontend taps, and the stream
|
- **events/** - the in-process bus every frontend taps, and the stream
|
||||||
event protocol backends emit.
|
event protocol backends emit.
|
||||||
- **storage/** - SQLModel tables and the Postgres session store.
|
- **storage/** - SQLModel tables and the Postgres session store.
|
||||||
@@ -67,6 +70,46 @@ file next to it adds Postgres. Without Docker: `uv sync`, build the SPA with
|
|||||||
`DATABASE_URL` and `CONFIG_PATH` set. A setup's own secrets come from `.env`
|
`DATABASE_URL` and `CONFIG_PATH` set. A setup's own secrets come from `.env`
|
||||||
next to its config (`load_dotenv` runs before the config is executed).
|
next to its config (`load_dotenv` runs before the config is executed).
|
||||||
|
|
||||||
|
## Transcription
|
||||||
|
|
||||||
|
A voice note, a round video or an audio file sent to the Telegram window is
|
||||||
|
downloaded as any attachment; with `Gateway.transcribers` set, the frontend
|
||||||
|
runs them in order and the first non-empty text rides under the attachment
|
||||||
|
note (`TelegramTexts.attachment_transcript`). A provider that raises is
|
||||||
|
logged and the next one tried; the whole chain gives up after the
|
||||||
|
frontend's `transcribe_timeout` (90 s) and the note goes without the text.
|
||||||
|
No transcribers, nothing changes.
|
||||||
|
|
||||||
|
```python
|
||||||
|
import os
|
||||||
|
|
||||||
|
from beaver_gateway.transcribe import Audio, OpenAITranscriber
|
||||||
|
|
||||||
|
|
||||||
|
async def voice_only(audio: Audio) -> str | None:
|
||||||
|
if audio.media_type != "audio/ogg":
|
||||||
|
return None
|
||||||
|
return await my_service.transcribe(audio.path, seconds=audio.duration)
|
||||||
|
|
||||||
|
|
||||||
|
gateway = Gateway(
|
||||||
|
transcribers=[
|
||||||
|
voice_only,
|
||||||
|
OpenAITranscriber(
|
||||||
|
base_url="https://api.groq.com/openai/v1",
|
||||||
|
api_key=os.environ["GROQ_API_KEY"],
|
||||||
|
model="whisper-large-v3",
|
||||||
|
language="ru",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
A provider is any `async (Audio) -> str | None`; `None` passes the audio
|
||||||
|
on. `OpenAITranscriber` posts the file to `{base_url}/audio/transcriptions`,
|
||||||
|
the wire of OpenAI, Groq, faster-whisper-server and a whisper.cpp server;
|
||||||
|
`api_key=None` for a local one without auth.
|
||||||
|
|
||||||
## Develop
|
## Develop
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
|
|||||||
@@ -201,6 +201,7 @@ async def run(gateway: Gateway, settings: Settings) -> None:
|
|||||||
scopes=scopes_with(fe.scope for fe in gateway.frontends),
|
scopes=scopes_with(fe.scope for fe in gateway.frontends),
|
||||||
send_file_roots=_roots(settings.send_file_roots),
|
send_file_roots=_roots(settings.send_file_roots),
|
||||||
secrets=secrets,
|
secrets=secrets,
|
||||||
|
transcribers=tuple(gateway.transcribers),
|
||||||
)
|
)
|
||||||
for fe in gateway.frontends:
|
for fe in gateway.frontends:
|
||||||
fe.configure(runtime)
|
fe.configure(runtime)
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from beaver_gateway.conversations.texts import ConversationTexts
|
|||||||
from beaver_gateway.frontends.base import Frontend
|
from beaver_gateway.frontends.base import Frontend
|
||||||
from beaver_gateway.jobs.scheduler import Job
|
from beaver_gateway.jobs.scheduler import Job
|
||||||
from beaver_gateway.mcp.types import HttpMcp, McpServer, PythonToolMcp, StdioMcp
|
from beaver_gateway.mcp.types import HttpMcp, McpServer, PythonToolMcp, StdioMcp
|
||||||
|
from beaver_gateway.transcribe import Audio, OpenAITranscriber
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import Awaitable, Callable
|
from collections.abc import Awaitable, Callable
|
||||||
@@ -27,6 +28,7 @@ if TYPE_CHECKING:
|
|||||||
from beaver_gateway.conversations.texts import UserSaid
|
from beaver_gateway.conversations.texts import UserSaid
|
||||||
from beaver_gateway.jobs.scheduler import Budget
|
from beaver_gateway.jobs.scheduler import Budget
|
||||||
from beaver_gateway.mcp.types import McpServerT
|
from beaver_gateway.mcp.types import McpServerT
|
||||||
|
from beaver_gateway.transcribe import Transcriber
|
||||||
from beaver_gateway.vault.watch import VaultWatch
|
from beaver_gateway.vault.watch import VaultWatch
|
||||||
|
|
||||||
__all__ = ["ConfigError", "Gateway", "load"]
|
__all__ = ["ConfigError", "Gateway", "load"]
|
||||||
@@ -51,6 +53,8 @@ class Gateway:
|
|||||||
"""Envelope lookup on the user's text: pointers the setup derives from its files."""
|
"""Envelope lookup on the user's text: pointers the setup derives from its files."""
|
||||||
user_sink: Callable[[UserSaid], Awaitable[None] | None] | None = None
|
user_sink: Callable[[UserSaid], Awaitable[None] | None] | None = None
|
||||||
"""Sees every user message entering a master or branch turn."""
|
"""Sees every user message entering a master or branch turn."""
|
||||||
|
transcribers: list[Transcriber] = field(default_factory=list)
|
||||||
|
"""Speech to text for voice and audio a window receives; the first text wins."""
|
||||||
budget: Budget | None = None
|
budget: Budget | None = None
|
||||||
"""Subscription utilisation past which non-critical jobs wait."""
|
"""Subscription utilisation past which non-critical jobs wait."""
|
||||||
distiller: Distiller | None = None
|
distiller: Distiller | None = None
|
||||||
@@ -75,6 +79,8 @@ _PUBLIC_NAMES: dict[str, Any] = {
|
|||||||
"ExposedMcp": ExposedMcp,
|
"ExposedMcp": ExposedMcp,
|
||||||
"Gateway": Gateway,
|
"Gateway": Gateway,
|
||||||
"ConversationTexts": ConversationTexts,
|
"ConversationTexts": ConversationTexts,
|
||||||
|
"Audio": Audio,
|
||||||
|
"OpenAITranscriber": OpenAITranscriber,
|
||||||
}
|
}
|
||||||
|
|
||||||
_McpInstance = StdioMcp | HttpMcp | PythonToolMcp
|
_McpInstance = StdioMcp | HttpMcp | PythonToolMcp
|
||||||
@@ -144,6 +150,13 @@ def _validate(gw: Gateway, path: Path) -> None:
|
|||||||
f"a frontend scope is a lowercase name like 'voice', never '*'"
|
f"a frontend scope is a lowercase name like 'voice', never '*'"
|
||||||
)
|
)
|
||||||
raise ConfigError(msg)
|
raise ConfigError(msg)
|
||||||
|
for i, t in enumerate(gw.transcribers):
|
||||||
|
if not callable(t):
|
||||||
|
msg = (
|
||||||
|
f"{path}: gateway.transcribers[{i}] must be an async callable "
|
||||||
|
f"taking an Audio, got {type(t).__name__}"
|
||||||
|
)
|
||||||
|
raise ConfigError(msg)
|
||||||
names: set[str] = set()
|
names: set[str] = set()
|
||||||
for i, j in enumerate(gw.jobs):
|
for i, j in enumerate(gw.jobs):
|
||||||
if not isinstance(j, Job):
|
if not isinstance(j, Job):
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
|
|||||||
from beaver_gateway.security.auth import TokenStore
|
from beaver_gateway.security.auth import TokenStore
|
||||||
from beaver_gateway.storage import Database
|
from beaver_gateway.storage import Database
|
||||||
from beaver_gateway.storage.models import Conversation, ConversationBinding
|
from beaver_gateway.storage.models import Conversation, ConversationBinding
|
||||||
|
from beaver_gateway.transcribe import Transcriber
|
||||||
|
|
||||||
TurnLogHandler = Callable[[TurnRecord], Awaitable[None]]
|
TurnLogHandler = Callable[[TurnRecord], Awaitable[None]]
|
||||||
|
|
||||||
@@ -65,6 +66,8 @@ class GatewayRuntime:
|
|||||||
"""Directories ``send_file`` may read, from ``SEND_FILE_ROOTS``."""
|
"""Directories ``send_file`` may read, from ``SEND_FILE_ROOTS``."""
|
||||||
secrets: SecretStore | None = None
|
secrets: SecretStore | None = None
|
||||||
"""Sealed secrets for ``secret run``; ``None`` in setups without them."""
|
"""Sealed secrets for ``secret run``; ``None`` in setups without them."""
|
||||||
|
transcribers: Sequence[Transcriber] = ()
|
||||||
|
"""``Gateway.transcribers``: speech to text a window runs on audio it receives."""
|
||||||
|
|
||||||
|
|
||||||
class Frontend(ABC):
|
class Frontend(ABC):
|
||||||
|
|||||||
@@ -48,13 +48,17 @@ from beaver_gateway.frontends.telegram.inbox import Inbox
|
|||||||
from beaver_gateway.frontends.telegram.outbox import Outbox
|
from beaver_gateway.frontends.telegram.outbox import Outbox
|
||||||
from beaver_gateway.frontends.telegram.render import chunks, status_label, to_html
|
from beaver_gateway.frontends.telegram.render import chunks, status_label, to_html
|
||||||
from beaver_gateway.frontends.telegram.texts import TelegramTexts
|
from beaver_gateway.frontends.telegram.texts import TelegramTexts
|
||||||
|
from beaver_gateway.transcribe import Audio, transcribe
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
from beaver_gateway.conversations.kinds import Kind
|
from beaver_gateway.conversations.kinds import Kind
|
||||||
from beaver_gateway.conversations.service import Conversations
|
from beaver_gateway.conversations.service import Conversations
|
||||||
from beaver_gateway.events.bus import Event, EventBus
|
from beaver_gateway.events.bus import Event, EventBus
|
||||||
from beaver_gateway.frontends.base import GatewayRuntime
|
from beaver_gateway.frontends.base import GatewayRuntime
|
||||||
from beaver_gateway.storage.models import Conversation, ConversationBinding
|
from beaver_gateway.storage.models import Conversation, ConversationBinding
|
||||||
|
from beaver_gateway.transcribe import Transcriber
|
||||||
|
|
||||||
__all__ = ["FRONTEND", "Attachments", "TelegramFrontend"]
|
__all__ = ["FRONTEND", "Attachments", "TelegramFrontend"]
|
||||||
|
|
||||||
@@ -65,6 +69,7 @@ _DONE = "done"
|
|||||||
_COMMANDS = ("merge", "new", "chat", "status", "help", "start")
|
_COMMANDS = ("merge", "new", "chat", "status", "help", "start")
|
||||||
_CAPTION = 1024
|
_CAPTION = 1024
|
||||||
_NO_CAPTION = ("video_note",)
|
_NO_CAPTION = ("video_note",)
|
||||||
|
_HEARD = ("voice", "audio", "video_note")
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
@@ -132,6 +137,16 @@ class _Ask:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _Media:
|
||||||
|
file_id: str
|
||||||
|
kind: str
|
||||||
|
size: int | None
|
||||||
|
name: str
|
||||||
|
media_type: str | None = None
|
||||||
|
duration: int | None = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class _Album:
|
class _Album:
|
||||||
message: Message
|
message: Message
|
||||||
@@ -166,6 +181,7 @@ class TelegramFrontend(Frontend):
|
|||||||
outbox_backoff: float = 2.0,
|
outbox_backoff: float = 2.0,
|
||||||
texts: TelegramTexts | None = None,
|
texts: TelegramTexts | None = None,
|
||||||
album_delay: float = 1.0,
|
album_delay: float = 1.0,
|
||||||
|
transcribe_timeout: float = 90.0,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._token = token
|
self._token = token
|
||||||
self.texts = texts or TelegramTexts()
|
self.texts = texts or TelegramTexts()
|
||||||
@@ -190,6 +206,7 @@ class TelegramFrontend(Frontend):
|
|||||||
self._topic_names: dict[int, str] = {}
|
self._topic_names: dict[int, str] = {}
|
||||||
self._drafts: dict[str, Draft] = {}
|
self._drafts: dict[str, Draft] = {}
|
||||||
self.album_delay = album_delay
|
self.album_delay = album_delay
|
||||||
|
self.transcribe_timeout = transcribe_timeout
|
||||||
self._asks: dict[str, _Ask] = {}
|
self._asks: dict[str, _Ask] = {}
|
||||||
self._albums: dict[str, _Album] = {}
|
self._albums: dict[str, _Album] = {}
|
||||||
self._reactions: dict[int, tuple[int, int]] = {}
|
self._reactions: dict[int, tuple[int, int]] = {}
|
||||||
@@ -687,10 +704,11 @@ class TelegramFrontend(Frontend):
|
|||||||
return texts.attachment_sticker, None
|
return texts.attachment_sticker, None
|
||||||
if message.animation:
|
if message.animation:
|
||||||
return texts.attachment_animation, None
|
return texts.attachment_animation, None
|
||||||
file_id, kind, size, name, media_type = _media(message)
|
media = _media(message)
|
||||||
if file_id is None or name is None:
|
if media is None:
|
||||||
return None, None
|
return None, None
|
||||||
kind = texts.attachment_kinds.get(kind, kind)
|
kind = texts.attachment_kinds.get(media.kind, media.kind)
|
||||||
|
size, name = media.size, media.name
|
||||||
if size and size > 20 * 1024 * 1024:
|
if size and size > 20 * 1024 * 1024:
|
||||||
return (
|
return (
|
||||||
texts.attachment_too_big.format(
|
texts.attachment_too_big.format(
|
||||||
@@ -706,7 +724,7 @@ class TelegramFrontend(Frontend):
|
|||||||
folder.chmod(0o755)
|
folder.chmod(0o755)
|
||||||
path = folder / f"{int(now)}-{Path(name).name}"
|
path = folder / f"{int(now)}-{Path(name).name}"
|
||||||
try:
|
try:
|
||||||
await self.bot.download(file_id, destination=path)
|
await self.bot.download(media.file_id, destination=path)
|
||||||
path.chmod(0o644)
|
path.chmod(0o644)
|
||||||
except (TelegramAPIError, OSError) as exc:
|
except (TelegramAPIError, OSError) as exc:
|
||||||
_log.warning("attachment download failed: %s", exc)
|
_log.warning("attachment download failed: %s", exc)
|
||||||
@@ -714,16 +732,38 @@ class TelegramFrontend(Frontend):
|
|||||||
shown = texts.attachment_size.format(kb=size // 1024) if size else ""
|
shown = texts.attachment_size.format(kb=size // 1024) if size else ""
|
||||||
keep = self.attachments.keep_days
|
keep = self.attachments.keep_days
|
||||||
seen = (
|
seen = (
|
||||||
{"path": str(path), "media_type": media_type}
|
{"path": str(path), "media_type": media.media_type}
|
||||||
if media_type and media_type.startswith("image/")
|
if media.media_type and media.media_type.startswith("image/")
|
||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
if keep is None:
|
note = (
|
||||||
return texts.attachment.format(kind=kind, path=path, size=shown), seen
|
texts.attachment.format(kind=kind, path=path, size=shown)
|
||||||
return (
|
if keep is None
|
||||||
texts.attachment_kept.format(kind=kind, path=path, size=shown, days=keep),
|
else texts.attachment_kept.format(
|
||||||
seen,
|
kind=kind, path=path, size=shown, days=keep
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
transcribers = cast("GatewayRuntime", self._runtime).transcribers
|
||||||
|
if media.kind in _HEARD and transcribers:
|
||||||
|
audio = Audio(path, media.media_type, media.duration)
|
||||||
|
heard = await self._transcribe(transcribers, audio)
|
||||||
|
if heard:
|
||||||
|
note = texts.attachment_transcript.format(note=note, text=heard)
|
||||||
|
return note, seen
|
||||||
|
|
||||||
|
async def _transcribe(
|
||||||
|
self, transcribers: Sequence[Transcriber], audio: Audio
|
||||||
|
) -> str | None:
|
||||||
|
try:
|
||||||
|
async with asyncio.timeout(self.transcribe_timeout):
|
||||||
|
return await transcribe(transcribers, audio)
|
||||||
|
except TimeoutError:
|
||||||
|
_log.warning(
|
||||||
|
"transcription of %s took over %.0fs, the note goes without it",
|
||||||
|
audio.path.name,
|
||||||
|
self.transcribe_timeout,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
async def _sweep_loop(self, interval: float = 6 * 3600) -> None:
|
async def _sweep_loop(self, interval: float = 6 * 3600) -> None:
|
||||||
while True:
|
while True:
|
||||||
@@ -1190,13 +1230,10 @@ class TelegramFrontend(Frontend):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _media(
|
def _media(message: Message) -> _Media | None:
|
||||||
message: Message,
|
|
||||||
) -> tuple[str | None, str, int | None, str | None, str | None]:
|
|
||||||
"""``(file_id, kind, size, name, media_type)`` of the message's media, if any."""
|
|
||||||
if message.photo:
|
if message.photo:
|
||||||
photo = message.photo[-1]
|
photo = message.photo[-1]
|
||||||
return (
|
return _Media(
|
||||||
photo.file_id,
|
photo.file_id,
|
||||||
"photo",
|
"photo",
|
||||||
photo.file_size,
|
photo.file_size,
|
||||||
@@ -1206,32 +1243,43 @@ def _media(
|
|||||||
if message.document:
|
if message.document:
|
||||||
doc = message.document
|
doc = message.document
|
||||||
name = doc.file_name or f"{doc.file_unique_id}.bin"
|
name = doc.file_name or f"{doc.file_unique_id}.bin"
|
||||||
return doc.file_id, "document", doc.file_size, name, doc.mime_type
|
return _Media(doc.file_id, "document", doc.file_size, name, doc.mime_type)
|
||||||
if message.voice:
|
if message.voice:
|
||||||
v = message.voice
|
v = message.voice
|
||||||
return v.file_id, "voice", v.file_size, f"{v.file_unique_id}.ogg", None
|
return _Media(
|
||||||
|
v.file_id,
|
||||||
|
"voice",
|
||||||
|
v.file_size,
|
||||||
|
f"{v.file_unique_id}.ogg",
|
||||||
|
v.mime_type or "audio/ogg",
|
||||||
|
v.duration,
|
||||||
|
)
|
||||||
if message.audio:
|
if message.audio:
|
||||||
a = message.audio
|
a = message.audio
|
||||||
return (
|
return _Media(
|
||||||
a.file_id,
|
a.file_id,
|
||||||
"audio",
|
"audio",
|
||||||
a.file_size,
|
a.file_size,
|
||||||
a.file_name or f"{a.file_unique_id}.mp3",
|
a.file_name or f"{a.file_unique_id}.mp3",
|
||||||
None,
|
a.mime_type,
|
||||||
|
a.duration,
|
||||||
)
|
)
|
||||||
if message.video:
|
if message.video:
|
||||||
v = message.video
|
v = message.video
|
||||||
return (
|
return _Media(
|
||||||
v.file_id,
|
v.file_id, "video", v.file_size, v.file_name or f"{v.file_unique_id}.mp4"
|
||||||
"video",
|
|
||||||
v.file_size,
|
|
||||||
v.file_name or f"{v.file_unique_id}.mp4",
|
|
||||||
None,
|
|
||||||
)
|
)
|
||||||
if message.video_note:
|
if message.video_note:
|
||||||
n = message.video_note
|
n = message.video_note
|
||||||
return n.file_id, "video_note", n.file_size, f"{n.file_unique_id}.mp4", None
|
return _Media(
|
||||||
return None, "", None, None, None
|
n.file_id,
|
||||||
|
"video_note",
|
||||||
|
n.file_size,
|
||||||
|
f"{n.file_unique_id}.mp4",
|
||||||
|
"video/mp4",
|
||||||
|
n.duration,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _question_html(question: dict[str, Any]) -> str:
|
def _question_html(question: dict[str, Any]) -> str:
|
||||||
|
|||||||
@@ -71,6 +71,7 @@ class TelegramTexts:
|
|||||||
"[attachment: {kind} {path}{size}; kept {days} days, move it into the "
|
"[attachment: {kind} {path}{size}; kept {days} days, move it into the "
|
||||||
"vault if it matters]"
|
"vault if it matters]"
|
||||||
)
|
)
|
||||||
|
attachment_transcript: str = "{note}\ntranscript: {text}"
|
||||||
merge_nothing: str = "nothing to merge: this is not an open branch"
|
merge_nothing: str = "nothing to merge: this is not an open branch"
|
||||||
merging: str = "🔀 merging into the master"
|
merging: str = "🔀 merging into the master"
|
||||||
merged: str = "✅ merged into the master"
|
merged: str = "✅ merged into the master"
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
"""Speech to text for audio a window receives: a chain of pluggable providers."""
|
||||||
|
|
||||||
|
from beaver_gateway.transcribe.base import Audio, Transcriber, transcribe
|
||||||
|
from beaver_gateway.transcribe.openai import OpenAITranscriber
|
||||||
|
|
||||||
|
__all__ = ["Audio", "OpenAITranscriber", "Transcriber", "transcribe"]
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
"""What a transcription provider is handed and the chain that tries them in order."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING, Protocol
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
__all__ = ["Audio", "Transcriber", "transcribe"]
|
||||||
|
|
||||||
|
_log = logging.getLogger("beaver_gateway.transcribe")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Audio:
|
||||||
|
path: Path
|
||||||
|
"""The downloaded file, readable for as long as the call lasts."""
|
||||||
|
media_type: str | None = None
|
||||||
|
"""MIME type as the window reported it, e.g. ``audio/ogg``; ``None`` if unknown."""
|
||||||
|
duration: float | None = None
|
||||||
|
"""Length in seconds, when the window knows it."""
|
||||||
|
|
||||||
|
|
||||||
|
class Transcriber(Protocol):
|
||||||
|
async def __call__(self, audio: Audio) -> str | None: ...
|
||||||
|
|
||||||
|
|
||||||
|
async def transcribe(transcribers: Sequence[Transcriber], audio: Audio) -> str | None:
|
||||||
|
"""The first non-empty text from ``transcribers``, a failing one skipped."""
|
||||||
|
for transcriber in transcribers:
|
||||||
|
try:
|
||||||
|
text = await transcriber(audio)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
_log.warning(
|
||||||
|
"transcriber %s failed on %s: %s: %s",
|
||||||
|
_name(transcriber),
|
||||||
|
audio.path.name,
|
||||||
|
type(exc).__name__,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
if text and text.strip():
|
||||||
|
return text.strip()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _name(transcriber: Transcriber) -> str:
|
||||||
|
return getattr(transcriber, "__qualname__", None) or type(transcriber).__name__
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
"""A provider for any OpenAI-compatible ``/audio/transcriptions`` endpoint."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import aiohttp
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from beaver_gateway.transcribe.base import Audio
|
||||||
|
|
||||||
|
__all__ = ["OpenAITranscriber"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class OpenAITranscriber:
|
||||||
|
"""OpenAI, Groq, faster-whisper-server, a whisper.cpp server: one wire."""
|
||||||
|
|
||||||
|
base_url: str = "https://api.openai.com/v1"
|
||||||
|
"""API root; ``/audio/transcriptions`` is appended."""
|
||||||
|
api_key: str | None = None
|
||||||
|
"""Sent as a bearer token; ``None`` for a local server without auth."""
|
||||||
|
model: str = "whisper-1"
|
||||||
|
"""Model name as the server knows it, e.g. ``whisper-large-v3`` on Groq."""
|
||||||
|
language: str | None = None
|
||||||
|
"""ISO-639-1 hint such as ``ru``; ``None`` lets the model detect it."""
|
||||||
|
timeout: float = 120.0
|
||||||
|
"""Seconds for the whole request, upload included."""
|
||||||
|
|
||||||
|
async def __call__(self, audio: Audio) -> str | None:
|
||||||
|
headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
|
||||||
|
form = aiohttp.FormData()
|
||||||
|
form.add_field("model", self.model)
|
||||||
|
form.add_field("response_format", "json")
|
||||||
|
if self.language:
|
||||||
|
form.add_field("language", self.language)
|
||||||
|
with audio.path.open("rb") as file:
|
||||||
|
form.add_field(
|
||||||
|
"file",
|
||||||
|
file,
|
||||||
|
filename=audio.path.name,
|
||||||
|
content_type=audio.media_type or "application/octet-stream",
|
||||||
|
)
|
||||||
|
async with (
|
||||||
|
aiohttp.ClientSession(
|
||||||
|
timeout=aiohttp.ClientTimeout(total=self.timeout)
|
||||||
|
) as http,
|
||||||
|
http.post(
|
||||||
|
f"{self.base_url.rstrip('/')}/audio/transcriptions",
|
||||||
|
data=form,
|
||||||
|
headers=headers,
|
||||||
|
) as response,
|
||||||
|
):
|
||||||
|
response.raise_for_status()
|
||||||
|
body = await response.json(content_type=None)
|
||||||
|
return str(body.get("text") or "") or None
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
from beaver_gateway import config
|
from beaver_gateway import config
|
||||||
|
|
||||||
|
|
||||||
@@ -14,3 +16,15 @@ def test_config_imports_sibling_modules():
|
|||||||
)
|
)
|
||||||
gw = config.load(root / "config.py")
|
gw = config.load(root / "config.py")
|
||||||
assert gw.agents == []
|
assert gw.agents == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_config_takes_transcribers_and_refuses_what_cannot_be_called():
|
||||||
|
root = Path(tempfile.mkdtemp(prefix="beaver-cfg-"))
|
||||||
|
(root / "config.py").write_text(
|
||||||
|
"gateway = Gateway(transcribers=[OpenAITranscriber(language='ru')])\n"
|
||||||
|
)
|
||||||
|
(gw_stt,) = config.load(root / "config.py").transcribers
|
||||||
|
assert gw_stt.language == "ru" and gw_stt.model == "whisper-1"
|
||||||
|
(root / "config.py").write_text("gateway = Gateway(transcribers=['whisper'])\n")
|
||||||
|
with pytest.raises(config.ConfigError, match=r"transcribers\[0\]"):
|
||||||
|
config.load(root / "config.py")
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import base64
|
import base64
|
||||||
import contextlib
|
import contextlib
|
||||||
|
import dataclasses
|
||||||
import functools
|
import functools
|
||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -29,6 +30,7 @@ from beaver_gateway.frontends.telegram.render import (
|
|||||||
to_html_tail,
|
to_html_tail,
|
||||||
)
|
)
|
||||||
from beaver_gateway.storage.models import ConversationBinding, Delivery, TelegramUpdate
|
from beaver_gateway.storage.models import ConversationBinding, Delivery, TelegramUpdate
|
||||||
|
from beaver_gateway.transcribe import Audio
|
||||||
from sqlmodel import select
|
from sqlmodel import select
|
||||||
from test_conversations import ScriptedClient, StubFrontend, World
|
from test_conversations import ScriptedClient, StubFrontend, World
|
||||||
|
|
||||||
@@ -871,6 +873,91 @@ def test_sweep_drops_old_files_and_empty_day_folders() -> None:
|
|||||||
assert not old.parent.exists() and fresh.exists()
|
assert not old.parent.exists() and fresh.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def _voice(caption: str) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"caption": caption,
|
||||||
|
"voice": {
|
||||||
|
"file_id": "v1",
|
||||||
|
"file_unique_id": "vu1",
|
||||||
|
"duration": 3,
|
||||||
|
"mime_type": "audio/ogg",
|
||||||
|
"file_size": 4096,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _prompt_with(needle: str) -> str:
|
||||||
|
return next(p for c in ScriptedClient.instances for p in c.prompts if needle in p)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_voice_without_transcribers_keeps_the_plain_note(stack: Stack) -> None:
|
||||||
|
root = Path(tempfile.mkdtemp(prefix="beaver-voice-"))
|
||||||
|
stack.tg.attachments = Attachments(dir=root, keep_days=3)
|
||||||
|
stack.bot.media(_voice("слушай"))
|
||||||
|
await stack.until(lambda: stack.sent_with("слушай"), what="reply")
|
||||||
|
(saved,) = [p for p in root.rglob("*") if p.is_file()]
|
||||||
|
assert saved.name.endswith("-vu1.ogg")
|
||||||
|
assert _prompt_with("слушай").endswith(
|
||||||
|
f"слушай\n\n[attachment: voice message {saved}, 4 KB; kept 3 days, "
|
||||||
|
"move it into the vault if it matters]"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_voice_and_video_note_carry_the_transcript(stack: Stack) -> None:
|
||||||
|
heard: list[Audio] = []
|
||||||
|
|
||||||
|
async def stt(audio: Audio) -> str | None:
|
||||||
|
heard.append(audio)
|
||||||
|
return f"сказано {len(heard)}"
|
||||||
|
|
||||||
|
root = Path(tempfile.mkdtemp(prefix="beaver-voice-"))
|
||||||
|
stack.tg.attachments = Attachments(dir=root, keep_days=None)
|
||||||
|
stack.tg._runtime = dataclasses.replace( # noqa: SLF001
|
||||||
|
stack.tg._runtime, # noqa: SLF001
|
||||||
|
transcribers=(stt,),
|
||||||
|
)
|
||||||
|
stack.bot.media(_voice("раз"))
|
||||||
|
await stack.until(lambda: stack.sent_with("раз"), what="voice reply")
|
||||||
|
(voice,) = heard
|
||||||
|
assert voice.media_type == "audio/ogg" and voice.duration == 3
|
||||||
|
assert voice.path.name.endswith("-vu1.ogg") and voice.path.read_bytes() == b"data"
|
||||||
|
assert _prompt_with("раз").endswith(
|
||||||
|
f"раз\n\n[attachment: voice message {voice.path}, 4 KB]\ntranscript: сказано 1"
|
||||||
|
)
|
||||||
|
stack.bot.media(
|
||||||
|
{
|
||||||
|
"video_note": {
|
||||||
|
"file_id": "n1",
|
||||||
|
"file_unique_id": "nu1",
|
||||||
|
"length": 240,
|
||||||
|
"duration": 5,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
await stack.until(lambda: stack.sent_with("сказано 2"), what="video note reply")
|
||||||
|
assert (heard[1].media_type, heard[1].duration) == ("video/mp4", 5)
|
||||||
|
stack.bot.media(_photo("фото"))
|
||||||
|
await stack.until(lambda: stack.sent_with("фото"), what="photo reply")
|
||||||
|
assert len(heard) == 2 and "transcript" not in _prompt_with("фото")
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_hung_transcriber_does_not_hold_the_turn(stack: Stack) -> None:
|
||||||
|
async def hung(_: Audio) -> str:
|
||||||
|
await asyncio.sleep(10)
|
||||||
|
return "поздно"
|
||||||
|
|
||||||
|
stack.tg.transcribe_timeout = 0.05
|
||||||
|
stack.tg._runtime = dataclasses.replace( # noqa: SLF001
|
||||||
|
stack.tg._runtime, # noqa: SLF001
|
||||||
|
transcribers=(hung,),
|
||||||
|
)
|
||||||
|
stack.bot.media(_voice("жду"))
|
||||||
|
await stack.until(lambda: stack.sent_with("жду"), what="reply")
|
||||||
|
prompt = _prompt_with("жду")
|
||||||
|
assert prompt.endswith(" KB; kept 7 days, move it into the vault if it matters]")
|
||||||
|
assert "transcript" not in prompt
|
||||||
|
|
||||||
|
|
||||||
async def test_topic_rename_updates_the_title(stack: Stack) -> None:
|
async def test_topic_rename_updates_the_title(stack: Stack) -> None:
|
||||||
stack.bot.topic_created("план", 7)
|
stack.bot.topic_created("план", 7)
|
||||||
stack.bot.message("hello", thread=7)
|
stack.bot.message("hello", thread=7)
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from aioresponses import CallbackResult, aioresponses
|
||||||
|
|
||||||
|
from beaver_gateway.transcribe import Audio, OpenAITranscriber, transcribe
|
||||||
|
|
||||||
|
URL = "http://whisper.local/v1/audio/transcriptions"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def audio() -> Audio:
|
||||||
|
path = Path(tempfile.mkdtemp(prefix="beaver-stt-")) / "u1.ogg"
|
||||||
|
path.write_bytes(b"OggS")
|
||||||
|
return Audio(path, "audio/ogg", 3)
|
||||||
|
|
||||||
|
|
||||||
|
def saying(text: str | None, calls: list[str] | None = None):
|
||||||
|
async def transcriber(audio: Audio) -> str | None:
|
||||||
|
if calls is not None:
|
||||||
|
calls.append(text or "")
|
||||||
|
return text
|
||||||
|
|
||||||
|
return transcriber
|
||||||
|
|
||||||
|
|
||||||
|
async def broken(audio: Audio) -> str | None:
|
||||||
|
msg = "provider down"
|
||||||
|
raise ConnectionError(msg)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_the_first_non_empty_text_wins(audio: Audio) -> None:
|
||||||
|
calls: list[str] = []
|
||||||
|
chain = [saying(None, calls), saying(" \n", calls), saying(" привет ", calls)]
|
||||||
|
chain.append(saying("never", calls))
|
||||||
|
assert await transcribe(chain, audio) == "привет"
|
||||||
|
assert calls == ["", " \n", " привет "]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_raising_provider_is_skipped(audio: Audio, caplog) -> None:
|
||||||
|
assert await transcribe([broken, saying("ok")], audio) == "ok"
|
||||||
|
(record,) = [r for r in caplog.records if "provider down" in r.getMessage()]
|
||||||
|
assert record.levelname == "WARNING" and record.exc_info is None
|
||||||
|
|
||||||
|
|
||||||
|
async def test_nothing_comes_back_as_none(audio: Audio) -> None:
|
||||||
|
assert await transcribe([], audio) is None
|
||||||
|
assert await transcribe([broken, saying("")], audio) is None
|
||||||
|
|
||||||
|
|
||||||
|
async def test_openai_transcriber_posts_the_file_and_reads_the_text(
|
||||||
|
audio: Audio,
|
||||||
|
) -> None:
|
||||||
|
seen: dict[str, Any] = {}
|
||||||
|
|
||||||
|
def answer(url, **kwargs: Any) -> CallbackResult:
|
||||||
|
fields = kwargs["data"]._fields
|
||||||
|
seen["headers"] = kwargs["headers"]
|
||||||
|
seen["fields"] = {f[0]["name"]: f[2] for f in fields}
|
||||||
|
seen["file"] = next(f for f in fields if f[0]["name"] == "file")
|
||||||
|
return CallbackResult(payload={"text": " привет "})
|
||||||
|
|
||||||
|
stt = OpenAITranscriber(
|
||||||
|
base_url="http://whisper.local/v1/", api_key="sk", model="m", language="ru"
|
||||||
|
)
|
||||||
|
with aioresponses() as http:
|
||||||
|
http.post(URL, callback=answer)
|
||||||
|
assert await stt(audio) == " привет "
|
||||||
|
assert seen["headers"] == {"Authorization": "Bearer sk"}
|
||||||
|
assert seen["fields"]["model"] == "m"
|
||||||
|
assert seen["fields"]["language"] == "ru"
|
||||||
|
assert seen["fields"]["response_format"] == "json"
|
||||||
|
options, headers, _ = seen["file"]
|
||||||
|
assert options["filename"] == "u1.ogg"
|
||||||
|
assert headers["Content-Type"] == "audio/ogg"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_openai_transcriber_without_a_key_sends_no_auth(audio: Audio) -> None:
|
||||||
|
seen: dict[str, Any] = {}
|
||||||
|
|
||||||
|
def answer(url, **kwargs: Any) -> CallbackResult:
|
||||||
|
seen["headers"] = kwargs["headers"]
|
||||||
|
seen["fields"] = [f[0]["name"] for f in kwargs["data"]._fields]
|
||||||
|
return CallbackResult(payload={"text": ""})
|
||||||
|
|
||||||
|
with aioresponses() as http:
|
||||||
|
http.post(URL, callback=answer)
|
||||||
|
assert (
|
||||||
|
await OpenAITranscriber(base_url="http://whisper.local/v1")(audio) is None
|
||||||
|
)
|
||||||
|
assert seen["headers"] == {}
|
||||||
|
assert "language" not in seen["fields"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_openai_transcriber_error_reaches_the_chain(audio: Audio) -> None:
|
||||||
|
stt = OpenAITranscriber(base_url="http://whisper.local/v1")
|
||||||
|
with aioresponses() as http:
|
||||||
|
http.post(URL, status=401, payload={"error": "bad key"})
|
||||||
|
assert await transcribe([stt, saying("fallback")], audio) == "fallback"
|
||||||
Reference in new issue
Block a user