feat(backend,frontend,mcp): any voice or round video can be transcribed on request, media is found by its bytes

frontend/src/lib/components/MessageMedia.svelte

@@ -167,7 +167,12 @@
           url={ready.url}
         />
       {:else if ready && kind === "video_note"}
-        <VideoNote transcript={ready.transcript} url={ready.url} />
+        <VideoNote
+          chatId={message.chat_id}
+          messageId={message.message_id}
+          transcript={ready.transcript}
+          url={ready.url}
+        />
       {:else if ready && kind === "audio"}
         <AudioFile {own} title={fileName} url={ready.url} />
       {:else if ready && isImage}
@@ -210,6 +215,27 @@
           <Icon name="document" size="1.25rem" />
           <span>{label}</span>
         </button>
+      {:else if media?.state === "not-downloaded" && kind === "voice"}
+        <VoiceMessage
+          chatId={message.chat_id}
+          length={ref?.duration}
+          messageId={message.message_id}
+          ondownload={queue}
+          {own}
+          transcript={media.transcript}
+          url={null}
+          {waiting}
+        />
+      {:else if media?.state === "not-downloaded" && kind === "video_note"}
+        <VideoNote
+          chatId={message.chat_id}
+          length={ref?.duration}
+          messageId={message.message_id}
+          ondownload={queue}
+          transcript={media.transcript}
+          url={null}
+          {waiting}
+        />
       {:else if media?.state === "not-downloaded" && vk !== "other"}
         <button class="media-placeholder" onclick={queue} type="button">
           <Icon name={waiting ? "timer" : "download"} size="1.5rem" />

frontend/src/lib/components/media/VideoNote.svelte

@@ -1,18 +1,37 @@
 <script lang="ts">
   import Icon from "#lib/components/ui/Icon.svelte";
+  import Spinner from "#lib/components/ui/Spinner.svelte";
   import { formatDuration } from "#lib/format/duration.js";
   import { t } from "#lib/i18n/locale.svelte.js";
   import { claimPlayback, releasePlayback } from "#lib/media/playback.js";
   import { POSTER_TIME, poster } from "#lib/media/poster.js";
+  import { Transcript } from "#lib/media/transcript.svelte.js";
 
   interface Props {
+    chatId: number;
+    length?: number | null;
+    messageId: number;
+    ondownload?: () => void;
     transcript?: string | null;
-    url: string;
+    url: string | null;
+    waiting?: boolean;
   }
 
-  let { url, transcript = null }: Props = $props();
+  let {
+    url,
+    transcript = null,
+    chatId,
+    messageId,
+    length = null,
+    ondownload,
+    waiting = false,
+  }: Props = $props();
 
-  let showTranscript = $state(false);
+  const stt = new Transcript(
+    () => chatId,
+    () => messageId,
+    () => transcript
+  );
 
   const RADIUS = 94;
   const CIRCUMFERENCE = 2 * Math.PI * RADIUS;
@@ -24,8 +43,21 @@
 
   const progress = $derived(duration > 0 ? currentTime / duration : 0);
   const remaining = $derived(Math.max(0, duration - currentTime));
+  const shownTime = $derived(
+    paused && currentTime <= POSTER_TIME ? duration || (length ?? 0) : remaining
+  );
+  const toggleLabel = $derived.by(() => {
+    if (!url) {
+      return t("media.downloadToArchive");
+    }
+    return paused ? t("media.playRound") : t("ui.pause");
+  });
 
   function toggle() {
+    if (!url) {
+      ondownload?.();
+      return;
+    }
     if (!element) {
       return;
     }
@@ -39,60 +71,69 @@
 
 <div class="RoundVideoWrap">
   <button
-    aria-label={paused ? t("media.playRound") : t("ui.pause")}
+    aria-label={toggleLabel}
     class="RoundVideo"
     onclick={toggle}
     type="button"
   >
-    <!-- svelte-ignore a11y_media_has_caption -->
-    <!-- biome-ignore lint/a11y/useMediaCaption: archived round video has no captions -->
-    <video
-      onended={() => element && releasePlayback(element)}
-      onplay={() => element && claimPlayback(element)}
-      playsinline
-      preload="auto"
-      src={url}
-      bind:this={element}
-      bind:currentTime
-      bind:duration
-      bind:paused
-      use:poster
-    ></video>
-    <svg aria-hidden="true" class="RoundVideoRing" viewBox="0 0 200 200">
-      <circle
-        class="RoundVideoProgress"
-        cx="100"
-        cy="100"
-        r={RADIUS}
-        stroke-dasharray={CIRCUMFERENCE}
-        stroke-dashoffset={CIRCUMFERENCE * (1 - progress)}
-      />
-    </svg>
-    {#if paused}
-      <span class="play"><Icon name="large-play" size="2.75rem" /></span>
+    {#if url}
+      <!-- svelte-ignore a11y_media_has_caption -->
+      <!-- biome-ignore lint/a11y/useMediaCaption: archived round video has no captions -->
+      <video
+        onended={() => element && releasePlayback(element)}
+        onplay={() => element && claimPlayback(element)}
+        playsinline
+        preload="auto"
+        src={url}
+        bind:this={element}
+        bind:currentTime
+        bind:duration
+        bind:paused
+        use:poster
+      ></video>
+      <svg aria-hidden="true" class="RoundVideoRing" viewBox="0 0 200 200">
+        <circle
+          class="RoundVideoProgress"
+          cx="100"
+          cy="100"
+          r={RADIUS}
+          stroke-dasharray={CIRCUMFERENCE}
+          stroke-dashoffset={CIRCUMFERENCE * (1 - progress)}
+        />
+      </svg>
+      {#if paused}
+        <span class="play"><Icon name="large-play" size="2.75rem" /></span>
+      {/if}
+    {:else}
+      <span class="blank"></span>
+      <span class="play">
+        <Icon name={waiting ? "timer" : "download"} size="2.25rem" />
+      </span>
     {/if}
     <span class="badge">
       <Icon name="microphone" size="0.875rem" />
-      {formatDuration(
-        paused && currentTime <= POSTER_TIME ? duration : remaining
-      )}
+      {formatDuration(shownTime)}
     </span>
   </button>
-  {#if transcript}
-    <button
-      class="transcribe"
-      onclick={() => {
-        showTranscript = !showTranscript;
-      }}
-      type="button"
-      class:active={showTranscript}
-    >
+  <button
+    aria-label={stt.open
+      ? t("media.hideTranscription")
+      : t("media.showTranscription")}
+    class="transcribe"
+    disabled={stt.pending}
+    onclick={() => stt.toggle()}
+    type="button"
+    class:active={stt.open}
+  >
+    {#if stt.pending}
+      <Spinner color="gray" size="1rem" />
+    {:else}
       <Icon name="transcribe" size="1rem" />
-      {t("media.transcription")}
-    </button>
-    {#if showTranscript}
-      <div class="transcript">{transcript}</div>
     {/if}
+    {t("media.transcription")}
+  </button>
+  {#if stt.open}
+    <div class="transcript">{stt.text}</div>
   {/if}
 </div>
 
@@ -123,6 +164,10 @@
     &.active {
       color: var(--color-primary);
     }
+
+    &:disabled {
+      cursor: default;
+    }
   }
 
   .transcript {
@@ -150,6 +195,16 @@
     -webkit-tap-highlight-color: transparent;
   }
 
+  .blank {
+    display: block;
+
+    width: 100%;
+    height: 100%;
+    border-radius: 50%;
+
+    background-color: var(--color-default-shadow);
+  }
+
   video {
     display: block;
 

frontend/src/lib/components/media/VoiceMessage.svelte

@@ -1,52 +1,39 @@
 <script lang="ts">
-  import { requestTranscription } from "#lib/api/media.js";
   import Icon from "#lib/components/ui/Icon.svelte";
   import Spinner from "#lib/components/ui/Spinner.svelte";
   import { formatDuration } from "#lib/format/duration.js";
   import { t } from "#lib/i18n/locale.svelte.js";
   import { claimPlayback, releasePlayback } from "#lib/media/playback.js";
+  import { Transcript } from "#lib/media/transcript.svelte.js";
   import { computeWaveform, flatWaveform } from "#lib/media/waveform.js";
-  import { toasts } from "#lib/stores/toasts.svelte.js";
 
   interface Props {
     chatId: number;
+    length?: number | null;
     messageId: number;
+    ondownload?: () => void;
     own: boolean;
     transcript?: string | null;
-    url: string;
+    url: string | null;
+    waiting?: boolean;
   }
 
-  let { url, own, transcript = null, chatId, messageId }: Props = $props();
+  let {
+    url,
+    own,
+    transcript = null,
+    chatId,
+    messageId,
+    length = null,
+    ondownload,
+    waiting = false,
+  }: Props = $props();
 
-  let fetched = $state<string | null>(null);
-  let pending = $state(false);
-  const text = $derived(fetched ?? transcript);
-
-  let showTranscript = $state(false);
-
-  async function onTranscribe() {
-    if (text) {
-      showTranscript = !showTranscript;
-      return;
-    }
-    if (pending) {
-      return;
-    }
-    pending = true;
-    try {
-      const result = await requestTranscription(chatId, messageId);
-      if (result) {
-        fetched = result;
-        showTranscript = true;
-      } else {
-        toasts.error(t("media.transcribeFailed"));
-      }
-    } catch {
-      toasts.error(t("media.transcribeFailed"));
-    } finally {
-      pending = false;
-    }
-  }
+  const stt = new Transcript(
+    () => chatId,
+    () => messageId,
+    () => transcript
+  );
   let element = $state<HTMLAudioElement>();
   let peaks = $state<number[]>(flatWaveform());
   let currentTime = $state(0);
@@ -54,11 +41,20 @@
   let paused = $state(true);
 
   const progress = $derived(duration > 0 ? currentTime / duration : 0);
+  const toggleLabel = $derived.by(() => {
+    if (!url) {
+      return t("media.downloadToArchive");
+    }
+    return paused ? t("media.playVoice") : t("ui.pause");
+  });
   const elapsed = $derived(
-    paused && currentTime === 0 ? duration : currentTime
+    paused && currentTime === 0 ? duration || (length ?? 0) : currentTime
   );
 
   $effect(() => {
+    if (!url) {
+      return;
+    }
     let active = true;
     computeWaveform(url)
       .then((result) => {
@@ -73,6 +69,10 @@
   });
 
   function toggle() {
+    if (!url) {
+      ondownload?.();
+      return;
+    }
     if (!element) {
       return;
     }
@@ -97,16 +97,20 @@
 <div class="VoiceWrap" class:own>
   <div class="Voice">
     <button
-      aria-label={paused ? t("media.playVoice") : t("ui.pause")}
+      aria-label={toggleLabel}
       class="toggle"
       onclick={toggle}
       type="button"
     >
-      <Icon
-        class={paused ? "nudge" : ""}
-        name={paused ? "play" : "pause"}
-        size="1.5rem"
-      />
+      {#if !url}
+        <Icon name={waiting ? "timer" : "download"} size="1.5rem" />
+      {:else}
+        <Icon
+          class={paused ? "nudge" : ""}
+          name={paused ? "play" : "pause"}
+          size="1.5rem"
+        />
+      {/if}
     </button>
     <div class="body">
       <!-- svelte-ignore a11y_no_static_element_interactions -->
@@ -122,35 +126,37 @@
       <div class="time">{formatDuration(elapsed)}</div>
     </div>
     <button
-      aria-label={showTranscript && text
+      aria-label={stt.open
         ? t("media.hideTranscription")
         : t("media.showTranscription")}
       class="transcribe"
-      disabled={pending}
-      onclick={onTranscribe}
+      disabled={stt.pending}
+      onclick={() => stt.toggle()}
       type="button"
-      class:active={showTranscript && Boolean(text)}
+      class:active={stt.open}
     >
-      {#if pending}
+      {#if stt.pending}
         <Spinner color={own ? "white" : "gray"} size="1rem" />
       {:else}
         <Icon name="transcribe" size="1.125rem" />
       {/if}
     </button>
-    <!-- biome-ignore lint/a11y/useMediaCaption: archived voice note has no captions -->
-    <audio
-      onended={() => element && releasePlayback(element)}
-      onplay={() => element && claimPlayback(element)}
-      preload="metadata"
-      src={url}
-      bind:this={element}
-      bind:currentTime
-      bind:duration
-      bind:paused
-    ></audio>
+    {#if url}
+      <!-- biome-ignore lint/a11y/useMediaCaption: archived voice note has no captions -->
+      <audio
+        onended={() => element && releasePlayback(element)}
+        onplay={() => element && claimPlayback(element)}
+        preload="metadata"
+        src={url}
+        bind:this={element}
+        bind:currentTime
+        bind:duration
+        bind:paused
+      ></audio>
+    {/if}
   </div>
-  {#if text && showTranscript}
-    <div class="transcript">{text}</div>
+  {#if stt.open}
+    <div class="transcript">{stt.text}</div>
   {/if}
 </div>
 

frontend/src/lib/components/profile/SharedMedia.svelte

@@ -131,6 +131,7 @@
           kind: item.kind,
           mediaId: item.id,
           state: "not-downloaded",
+          transcript: null,
         };
         loadMediaItem({
           downloaded: item.downloaded,