feat(backend,frontend,mcp): any voice or round video can be transcribed on request, media is found by its bytes

frontend/src/lib/api/endpoints.ts

@@ -575,15 +575,23 @@ export function fetchMedia(
   });
 }
 
+export interface TranscribeResult {
+  error: string | null;
+  status: "done" | "failed" | "pending";
+  text: string | null;
+}
+
 export function transcribeMedia(
   chatId: number,
-  messageId: number
-): Promise<{ job_id: number }> {
-  return request<{ job_id: number }>("/media/transcribe", {
+  messageId: number,
+  wait: number
+): Promise<TranscribeResult> {
+  return request<TranscribeResult>("/media/transcribe", {
     body: {
       account_id: accounts.selectedId,
       chat_id: chatId,
       message_id: messageId,
+      wait,
     },
     method: "POST",
   });

frontend/src/lib/api/media.ts

@@ -3,8 +3,8 @@ import { getMessageMedia, transcribeMedia } from "#lib/api/endpoints.js";
 import type { MediaRef } from "#lib/api/types.js";
 import { accounts } from "#lib/stores/accounts.svelte.js";
 
-const TRANSCRIBE_TRIES = 10;
-const TRANSCRIBE_DELAY = 2000;
+const TRANSCRIBE_WAIT = 25;
+const TRANSCRIBE_TRIES = 3;
 
 export type InlineMedia =
   | {
@@ -15,7 +15,12 @@ export type InlineMedia =
       url: string;
       transcript: string | null;
     }
-  | { state: "not-downloaded"; mediaId: number; kind: string }
+  | {
+      state: "not-downloaded";
+      mediaId: number;
+      kind: string;
+      transcript: string | null;
+    }
   | { state: "missing" };
 
 export interface ViewerItem {
@@ -173,7 +178,12 @@ async function resolve(
     return { state: "missing" };
   }
   if (!meta.downloaded) {
-    return { kind: meta.kind, mediaId: meta.id, state: "not-downloaded" };
+    return {
+      kind: meta.kind,
+      mediaId: meta.id,
+      state: "not-downloaded",
+      transcript: meta.extracted_text,
+    };
   }
   const blob = await requestMedia(meta.id);
   if (blob.state === "ready") {
@@ -187,7 +197,12 @@ async function resolve(
     };
   }
   if (blob.state === "not-downloaded") {
-    return { kind: meta.kind, mediaId: meta.id, state: "not-downloaded" };
+    return {
+      kind: meta.kind,
+      mediaId: meta.id,
+      state: "not-downloaded",
+      transcript: meta.extracted_text,
+    };
   }
   return { state: "missing" };
 }
@@ -200,7 +215,12 @@ async function resolveById(media: MediaRef): Promise<InlineMedia> {
     return { state: "missing" };
   }
   if (!media.downloaded) {
-    return { kind: media.kind, mediaId: media.id, state: "not-downloaded" };
+    return {
+      kind: media.kind,
+      mediaId: media.id,
+      state: "not-downloaded",
+      transcript: null,
+    };
   }
   const blob = await requestMedia(media.id);
   if (blob.state === "ready") {
@@ -214,7 +234,12 @@ async function resolveById(media: MediaRef): Promise<InlineMedia> {
     };
   }
   if (blob.state === "not-downloaded") {
-    return { kind: media.kind, mediaId: media.id, state: "not-downloaded" };
+    return {
+      kind: media.kind,
+      mediaId: media.id,
+      state: "not-downloaded",
+      transcript: null,
+    };
   }
   return { state: "missing" };
 }
@@ -257,32 +282,29 @@ function patchTranscript(
     return;
   }
   const cached = ready.get(cacheKey(account, chatId, messageId));
-  if (cached?.state === "ready") {
+  if (cached && cached.state !== "missing") {
     cached.transcript = text;
   }
 }
 
+export type TranscriptOutcome = { text: string } | { error: string };
+
 export async function requestTranscription(
   chatId: number,
   messageId: number
-): Promise<string | null> {
-  await transcribeMedia(chatId, messageId);
+): Promise<TranscriptOutcome> {
   for (let i = 0; i < TRANSCRIBE_TRIES; i += 1) {
-    // biome-ignore lint/performance/noAwaitInLoops: polling waits between tries on purpose
-    await new Promise((done) => {
-      setTimeout(done, TRANSCRIBE_DELAY);
-    });
-    try {
-      const meta = await getMessageMedia(chatId, messageId);
-      if (meta.extracted_text) {
-        patchTranscript(chatId, messageId, meta.extracted_text);
-        return meta.extracted_text;
-      }
-    } catch {
-      return null;
+    // biome-ignore lint/performance/noAwaitInLoops: each try waits on the server
+    const result = await transcribeMedia(chatId, messageId, TRANSCRIBE_WAIT);
+    if (result.status === "done" && result.text) {
+      patchTranscript(chatId, messageId, result.text);
+      return { text: result.text };
+    }
+    if (result.status === "failed") {
+      return { error: result.error ?? "failed" };
     }
   }
-  return null;
+  return { error: "pending" };
 }
 
 export function loadInlineMedia(

frontend/src/lib/components/MessageMedia.svelte

@@ -167,7 +167,12 @@
           url={ready.url}
         />
       {:else if ready && kind === "video_note"}
-        <VideoNote transcript={ready.transcript} url={ready.url} />
+        <VideoNote
+          chatId={message.chat_id}
+          messageId={message.message_id}
+          transcript={ready.transcript}
+          url={ready.url}
+        />
       {:else if ready && kind === "audio"}
         <AudioFile {own} title={fileName} url={ready.url} />
       {:else if ready && isImage}
@@ -210,6 +215,27 @@
           <Icon name="document" size="1.25rem" />
           <span>{label}</span>
         </button>
+      {:else if media?.state === "not-downloaded" && kind === "voice"}
+        <VoiceMessage
+          chatId={message.chat_id}
+          length={ref?.duration}
+          messageId={message.message_id}
+          ondownload={queue}
+          {own}
+          transcript={media.transcript}
+          url={null}
+          {waiting}
+        />
+      {:else if media?.state === "not-downloaded" && kind === "video_note"}
+        <VideoNote
+          chatId={message.chat_id}
+          length={ref?.duration}
+          messageId={message.message_id}
+          ondownload={queue}
+          transcript={media.transcript}
+          url={null}
+          {waiting}
+        />
       {:else if media?.state === "not-downloaded" && vk !== "other"}
         <button class="media-placeholder" onclick={queue} type="button">
           <Icon name={waiting ? "timer" : "download"} size="1.5rem" />

frontend/src/lib/components/media/VideoNote.svelte

@@ -1,18 +1,37 @@
 <script lang="ts">
   import Icon from "#lib/components/ui/Icon.svelte";
+  import Spinner from "#lib/components/ui/Spinner.svelte";
   import { formatDuration } from "#lib/format/duration.js";
   import { t } from "#lib/i18n/locale.svelte.js";
   import { claimPlayback, releasePlayback } from "#lib/media/playback.js";
   import { POSTER_TIME, poster } from "#lib/media/poster.js";
+  import { Transcript } from "#lib/media/transcript.svelte.js";
 
   interface Props {
+    chatId: number;
+    length?: number | null;
+    messageId: number;
+    ondownload?: () => void;
     transcript?: string | null;
-    url: string;
+    url: string | null;
+    waiting?: boolean;
   }
 
-  let { url, transcript = null }: Props = $props();
+  let {
+    url,
+    transcript = null,
+    chatId,
+    messageId,
+    length = null,
+    ondownload,
+    waiting = false,
+  }: Props = $props();
 
-  let showTranscript = $state(false);
+  const stt = new Transcript(
+    () => chatId,
+    () => messageId,
+    () => transcript
+  );
 
   const RADIUS = 94;
   const CIRCUMFERENCE = 2 * Math.PI * RADIUS;
@@ -24,8 +43,21 @@
 
   const progress = $derived(duration > 0 ? currentTime / duration : 0);
   const remaining = $derived(Math.max(0, duration - currentTime));
+  const shownTime = $derived(
+    paused && currentTime <= POSTER_TIME ? duration || (length ?? 0) : remaining
+  );
+  const toggleLabel = $derived.by(() => {
+    if (!url) {
+      return t("media.downloadToArchive");
+    }
+    return paused ? t("media.playRound") : t("ui.pause");
+  });
 
   function toggle() {
+    if (!url) {
+      ondownload?.();
+      return;
+    }
     if (!element) {
       return;
     }
@@ -39,60 +71,69 @@
 
 <div class="RoundVideoWrap">
   <button
-    aria-label={paused ? t("media.playRound") : t("ui.pause")}
+    aria-label={toggleLabel}
     class="RoundVideo"
     onclick={toggle}
     type="button"
   >
-    <!-- svelte-ignore a11y_media_has_caption -->
-    <!-- biome-ignore lint/a11y/useMediaCaption: archived round video has no captions -->
-    <video
-      onended={() => element && releasePlayback(element)}
-      onplay={() => element && claimPlayback(element)}
-      playsinline
-      preload="auto"
-      src={url}
-      bind:this={element}
-      bind:currentTime
-      bind:duration
-      bind:paused
-      use:poster
-    ></video>
-    <svg aria-hidden="true" class="RoundVideoRing" viewBox="0 0 200 200">
-      <circle
-        class="RoundVideoProgress"
-        cx="100"
-        cy="100"
-        r={RADIUS}
-        stroke-dasharray={CIRCUMFERENCE}
-        stroke-dashoffset={CIRCUMFERENCE * (1 - progress)}
-      />
-    </svg>
-    {#if paused}
-      <span class="play"><Icon name="large-play" size="2.75rem" /></span>
+    {#if url}
+      <!-- svelte-ignore a11y_media_has_caption -->
+      <!-- biome-ignore lint/a11y/useMediaCaption: archived round video has no captions -->
+      <video
+        onended={() => element && releasePlayback(element)}
+        onplay={() => element && claimPlayback(element)}
+        playsinline
+        preload="auto"
+        src={url}
+        bind:this={element}
+        bind:currentTime
+        bind:duration
+        bind:paused
+        use:poster
+      ></video>
+      <svg aria-hidden="true" class="RoundVideoRing" viewBox="0 0 200 200">
+        <circle
+          class="RoundVideoProgress"
+          cx="100"
+          cy="100"
+          r={RADIUS}
+          stroke-dasharray={CIRCUMFERENCE}
+          stroke-dashoffset={CIRCUMFERENCE * (1 - progress)}
+        />
+      </svg>
+      {#if paused}
+        <span class="play"><Icon name="large-play" size="2.75rem" /></span>
+      {/if}
+    {:else}
+      <span class="blank"></span>
+      <span class="play">
+        <Icon name={waiting ? "timer" : "download"} size="2.25rem" />
+      </span>
     {/if}
     <span class="badge">
       <Icon name="microphone" size="0.875rem" />
-      {formatDuration(
-        paused && currentTime <= POSTER_TIME ? duration : remaining
-      )}
+      {formatDuration(shownTime)}
     </span>
   </button>
-  {#if transcript}
-    <button
-      class="transcribe"
-      onclick={() => {
-        showTranscript = !showTranscript;
-      }}
-      type="button"
-      class:active={showTranscript}
-    >
+  <button
+    aria-label={stt.open
+      ? t("media.hideTranscription")
+      : t("media.showTranscription")}
+    class="transcribe"
+    disabled={stt.pending}
+    onclick={() => stt.toggle()}
+    type="button"
+    class:active={stt.open}
+  >
+    {#if stt.pending}
+      <Spinner color="gray" size="1rem" />
+    {:else}
       <Icon name="transcribe" size="1rem" />
-      {t("media.transcription")}
-    </button>
-    {#if showTranscript}
-      <div class="transcript">{transcript}</div>
     {/if}
+    {t("media.transcription")}
+  </button>
+  {#if stt.open}
+    <div class="transcript">{stt.text}</div>
   {/if}
 </div>
 
@@ -123,6 +164,10 @@
     &.active {
       color: var(--color-primary);
     }
+
+    &:disabled {
+      cursor: default;
+    }
   }
 
   .transcript {
@@ -150,6 +195,16 @@
     -webkit-tap-highlight-color: transparent;
   }
 
+  .blank {
+    display: block;
+
+    width: 100%;
+    height: 100%;
+    border-radius: 50%;
+
+    background-color: var(--color-default-shadow);
+  }
+
   video {
     display: block;
 

frontend/src/lib/components/media/VoiceMessage.svelte

@@ -1,52 +1,39 @@
 <script lang="ts">
-  import { requestTranscription } from "#lib/api/media.js";
   import Icon from "#lib/components/ui/Icon.svelte";
   import Spinner from "#lib/components/ui/Spinner.svelte";
   import { formatDuration } from "#lib/format/duration.js";
   import { t } from "#lib/i18n/locale.svelte.js";
   import { claimPlayback, releasePlayback } from "#lib/media/playback.js";
+  import { Transcript } from "#lib/media/transcript.svelte.js";
   import { computeWaveform, flatWaveform } from "#lib/media/waveform.js";
-  import { toasts } from "#lib/stores/toasts.svelte.js";
 
   interface Props {
     chatId: number;
+    length?: number | null;
     messageId: number;
+    ondownload?: () => void;
     own: boolean;
     transcript?: string | null;
-    url: string;
+    url: string | null;
+    waiting?: boolean;
   }
 
-  let { url, own, transcript = null, chatId, messageId }: Props = $props();
+  let {
+    url,
+    own,
+    transcript = null,
+    chatId,
+    messageId,
+    length = null,
+    ondownload,
+    waiting = false,
+  }: Props = $props();
 
-  let fetched = $state<string | null>(null);
-  let pending = $state(false);
-  const text = $derived(fetched ?? transcript);
-
-  let showTranscript = $state(false);
-
-  async function onTranscribe() {
-    if (text) {
-      showTranscript = !showTranscript;
-      return;
-    }
-    if (pending) {
-      return;
-    }
-    pending = true;
-    try {
-      const result = await requestTranscription(chatId, messageId);
-      if (result) {
-        fetched = result;
-        showTranscript = true;
-      } else {
-        toasts.error(t("media.transcribeFailed"));
-      }
-    } catch {
-      toasts.error(t("media.transcribeFailed"));
-    } finally {
-      pending = false;
-    }
-  }
+  const stt = new Transcript(
+    () => chatId,
+    () => messageId,
+    () => transcript
+  );
   let element = $state<HTMLAudioElement>();
   let peaks = $state<number[]>(flatWaveform());
   let currentTime = $state(0);
@@ -54,11 +41,20 @@
   let paused = $state(true);
 
   const progress = $derived(duration > 0 ? currentTime / duration : 0);
+  const toggleLabel = $derived.by(() => {
+    if (!url) {
+      return t("media.downloadToArchive");
+    }
+    return paused ? t("media.playVoice") : t("ui.pause");
+  });
   const elapsed = $derived(
-    paused && currentTime === 0 ? duration : currentTime
+    paused && currentTime === 0 ? duration || (length ?? 0) : currentTime
   );
 
   $effect(() => {
+    if (!url) {
+      return;
+    }
     let active = true;
     computeWaveform(url)
       .then((result) => {
@@ -73,6 +69,10 @@
   });
 
   function toggle() {
+    if (!url) {
+      ondownload?.();
+      return;
+    }
     if (!element) {
       return;
     }
@@ -97,16 +97,20 @@
 <div class="VoiceWrap" class:own>
   <div class="Voice">
     <button
-      aria-label={paused ? t("media.playVoice") : t("ui.pause")}
+      aria-label={toggleLabel}
       class="toggle"
       onclick={toggle}
       type="button"
     >
-      <Icon
-        class={paused ? "nudge" : ""}
-        name={paused ? "play" : "pause"}
-        size="1.5rem"
-      />
+      {#if !url}
+        <Icon name={waiting ? "timer" : "download"} size="1.5rem" />
+      {:else}
+        <Icon
+          class={paused ? "nudge" : ""}
+          name={paused ? "play" : "pause"}
+          size="1.5rem"
+        />
+      {/if}
     </button>
     <div class="body">
       <!-- svelte-ignore a11y_no_static_element_interactions -->
@@ -122,35 +126,37 @@
       <div class="time">{formatDuration(elapsed)}</div>
     </div>
     <button
-      aria-label={showTranscript && text
+      aria-label={stt.open
         ? t("media.hideTranscription")
         : t("media.showTranscription")}
       class="transcribe"
-      disabled={pending}
-      onclick={onTranscribe}
+      disabled={stt.pending}
+      onclick={() => stt.toggle()}
       type="button"
-      class:active={showTranscript && Boolean(text)}
+      class:active={stt.open}
     >
-      {#if pending}
+      {#if stt.pending}
         <Spinner color={own ? "white" : "gray"} size="1rem" />
       {:else}
         <Icon name="transcribe" size="1.125rem" />
       {/if}
     </button>
-    <!-- biome-ignore lint/a11y/useMediaCaption: archived voice note has no captions -->
-    <audio
-      onended={() => element && releasePlayback(element)}
-      onplay={() => element && claimPlayback(element)}
-      preload="metadata"
-      src={url}
-      bind:this={element}
-      bind:currentTime
-      bind:duration
-      bind:paused
-    ></audio>
+    {#if url}
+      <!-- biome-ignore lint/a11y/useMediaCaption: archived voice note has no captions -->
+      <audio
+        onended={() => element && releasePlayback(element)}
+        onplay={() => element && claimPlayback(element)}
+        preload="metadata"
+        src={url}
+        bind:this={element}
+        bind:currentTime
+        bind:duration
+        bind:paused
+      ></audio>
+    {/if}
   </div>
-  {#if text && showTranscript}
-    <div class="transcript">{text}</div>
+  {#if stt.open}
+    <div class="transcript">{stt.text}</div>
   {/if}
 </div>
 

frontend/src/lib/components/profile/SharedMedia.svelte

@@ -131,6 +131,7 @@
           kind: item.kind,
           mediaId: item.id,
           state: "not-downloaded",
+          transcript: null,
         };
         loadMediaItem({
           downloaded: item.downloaded,

frontend/src/lib/i18n/messages/media.ts

@@ -35,10 +35,26 @@ export const media = {
     en: "Show transcription",
     ru: "Показать расшифровку",
   },
+  "media.transcribeEmpty": {
+    en: "No speech in the recording",
+    ru: "В записи не слышно речи",
+  },
   "media.transcribeFailed": {
     en: "Could not transcribe",
     ru: "Не удалось расшифровать",
   },
+  "media.transcribeNoPremium": {
+    en: "Transcription needs Telegram Premium",
+    ru: "Для расшифровки нужен Telegram Premium",
+  },
+  "media.transcribePending": {
+    en: "Telegram is still transcribing, try again shortly",
+    ru: "Telegram ещё расшифровывает, нажмите чуть позже",
+  },
+  "media.transcribeTooLong": {
+    en: "Too long for Telegram to transcribe",
+    ru: "Слишком длинное для расшифровки в Telegram",
+  },
   "media.transcription": { en: "Transcription", ru: "Расшифровка" },
   "media.unit.B": { en: "B", ru: "Б" },
   "media.unit.GB": { en: "GB", ru: "ГБ" },

frontend/src/lib/media/transcript.svelte.ts

@@ -0,0 +1,65 @@
+import { requestTranscription } from "#lib/api/media.js";
+import { t } from "#lib/i18n/locale.svelte.js";
+import type { MessageKey } from "#lib/i18n/messages.js";
+import { toasts } from "#lib/stores/toasts.svelte.js";
+
+const REFUSALS: Record<string, MessageKey> = {
+  empty: "media.transcribeEmpty",
+  no_premium: "media.transcribeNoPremium",
+  pending: "media.transcribePending",
+  too_long: "media.transcribeTooLong",
+};
+
+export class Transcript {
+  readonly #chatId: () => number;
+  readonly #messageId: () => number;
+  readonly #initial: () => string | null;
+  fetched = $state<string | null>(null);
+  pending = $state(false);
+  shown = $state(false);
+
+  constructor(
+    chatId: () => number,
+    messageId: () => number,
+    initial: () => string | null
+  ) {
+    this.#chatId = chatId;
+    this.#messageId = messageId;
+    this.#initial = initial;
+  }
+
+  get text(): string | null {
+    return this.fetched ?? this.#initial();
+  }
+
+  get open(): boolean {
+    return this.shown && Boolean(this.text);
+  }
+
+  async toggle(): Promise<void> {
+    if (this.text) {
+      this.shown = !this.shown;
+      return;
+    }
+    if (this.pending) {
+      return;
+    }
+    this.pending = true;
+    try {
+      const outcome = await requestTranscription(
+        this.#chatId(),
+        this.#messageId()
+      );
+      if ("text" in outcome) {
+        this.fetched = outcome.text;
+        this.shown = true;
+      } else {
+        toasts.error(t(REFUSALS[outcome.error] ?? "media.transcribeFailed"));
+      }
+    } catch {
+      toasts.error(t("media.transcribeFailed"));
+    } finally {
+      this.pending = false;
+    }
+  }
+}