feat(backend,frontend,mcp): any voice or round video can be transcribed on request, media is found by its bytes

frontend/src/lib/api/endpoints.ts

@@ -575,15 +575,23 @@ export function fetchMedia(
   });
 }
 
+export interface TranscribeResult {
+  error: string | null;
+  status: "done" | "failed" | "pending";
+  text: string | null;
+}
+
 export function transcribeMedia(
   chatId: number,
-  messageId: number
-): Promise<{ job_id: number }> {
-  return request<{ job_id: number }>("/media/transcribe", {
+  messageId: number,
+  wait: number
+): Promise<TranscribeResult> {
+  return request<TranscribeResult>("/media/transcribe", {
     body: {
       account_id: accounts.selectedId,
       chat_id: chatId,
       message_id: messageId,
+      wait,
     },
     method: "POST",
   });

frontend/src/lib/api/media.ts

@@ -3,8 +3,8 @@ import { getMessageMedia, transcribeMedia } from "#lib/api/endpoints.js";
 import type { MediaRef } from "#lib/api/types.js";
 import { accounts } from "#lib/stores/accounts.svelte.js";
 
-const TRANSCRIBE_TRIES = 10;
-const TRANSCRIBE_DELAY = 2000;
+const TRANSCRIBE_WAIT = 25;
+const TRANSCRIBE_TRIES = 3;
 
 export type InlineMedia =
   | {
@@ -15,7 +15,12 @@ export type InlineMedia =
       url: string;
       transcript: string | null;
     }
-  | { state: "not-downloaded"; mediaId: number; kind: string }
+  | {
+      state: "not-downloaded";
+      mediaId: number;
+      kind: string;
+      transcript: string | null;
+    }
   | { state: "missing" };
 
 export interface ViewerItem {
@@ -173,7 +178,12 @@ async function resolve(
     return { state: "missing" };
   }
   if (!meta.downloaded) {
-    return { kind: meta.kind, mediaId: meta.id, state: "not-downloaded" };
+    return {
+      kind: meta.kind,
+      mediaId: meta.id,
+      state: "not-downloaded",
+      transcript: meta.extracted_text,
+    };
   }
   const blob = await requestMedia(meta.id);
   if (blob.state === "ready") {
@@ -187,7 +197,12 @@ async function resolve(
     };
   }
   if (blob.state === "not-downloaded") {
-    return { kind: meta.kind, mediaId: meta.id, state: "not-downloaded" };
+    return {
+      kind: meta.kind,
+      mediaId: meta.id,
+      state: "not-downloaded",
+      transcript: meta.extracted_text,
+    };
   }
   return { state: "missing" };
 }
@@ -200,7 +215,12 @@ async function resolveById(media: MediaRef): Promise<InlineMedia> {
     return { state: "missing" };
   }
   if (!media.downloaded) {
-    return { kind: media.kind, mediaId: media.id, state: "not-downloaded" };
+    return {
+      kind: media.kind,
+      mediaId: media.id,
+      state: "not-downloaded",
+      transcript: null,
+    };
   }
   const blob = await requestMedia(media.id);
   if (blob.state === "ready") {
@@ -214,7 +234,12 @@ async function resolveById(media: MediaRef): Promise<InlineMedia> {
     };
   }
   if (blob.state === "not-downloaded") {
-    return { kind: media.kind, mediaId: media.id, state: "not-downloaded" };
+    return {
+      kind: media.kind,
+      mediaId: media.id,
+      state: "not-downloaded",
+      transcript: null,
+    };
   }
   return { state: "missing" };
 }
@@ -257,32 +282,29 @@ function patchTranscript(
     return;
   }
   const cached = ready.get(cacheKey(account, chatId, messageId));
-  if (cached?.state === "ready") {
+  if (cached && cached.state !== "missing") {
     cached.transcript = text;
   }
 }
 
+export type TranscriptOutcome = { text: string } | { error: string };
+
 export async function requestTranscription(
   chatId: number,
   messageId: number
-): Promise<string | null> {
-  await transcribeMedia(chatId, messageId);
+): Promise<TranscriptOutcome> {
   for (let i = 0; i < TRANSCRIBE_TRIES; i += 1) {
-    // biome-ignore lint/performance/noAwaitInLoops: polling waits between tries on purpose
-    await new Promise((done) => {
-      setTimeout(done, TRANSCRIBE_DELAY);
-    });
-    try {
-      const meta = await getMessageMedia(chatId, messageId);
-      if (meta.extracted_text) {
-        patchTranscript(chatId, messageId, meta.extracted_text);
-        return meta.extracted_text;
-      }
-    } catch {
-      return null;
+    // biome-ignore lint/performance/noAwaitInLoops: each try waits on the server
+    const result = await transcribeMedia(chatId, messageId, TRANSCRIBE_WAIT);
+    if (result.status === "done" && result.text) {
+      patchTranscript(chatId, messageId, result.text);
+      return { text: result.text };
+    }
+    if (result.status === "failed") {
+      return { error: result.error ?? "failed" };
     }
   }
-  return null;
+  return { error: "pending" };
 }
 
 export function loadInlineMedia(