beaver/beavergram
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(userbot,api,frontend): backfill stories and unbreak all-stories list

Browse Source
This commit is contained in:
hh
2026-08-06 14:08:10 +02:00
parent 1e143c7573
commit 12cc7d57e3
11 changed files with 326 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/api/routers/backfill.py
+15
View File
@@ -19,6 +19,11 @@ class BackfillRequest(BaseModel):
full: bool = False
class StoriesBackfillRequest(BaseModel):
account_id: int
peer_id: int
class FetchMediaRequest(BaseModel):
account_id: int
chat_id: int
@@ -76,6 +81,16 @@ async def enqueue_backfill(
return EnqueueResponse(job_id=job_id)
@router.post("/stories/backfill", status_code=201)
async def enqueue_stories_backfill(
pool: FromDishka[asyncpg.Pool], body: StoriesBackfillRequest
) -> EnqueueResponse:
job_id = await enqueue(
pool, body.account_id, "backfill_stories", {"peer_id": body.peer_id}
)
return EnqueueResponse(job_id=job_id)
@router.post("/media/fetch", status_code=201)
async def enqueue_fetch_media(
pool: FromDishka[asyncpg.Pool], body: FetchMediaRequest
backend/src/userbot/handlers/stories.py
+3 -41
View File
@@ -1,52 +1,14 @@
from io import BytesIO
from pyrogram.types import Story
from userbot import PyroClient
from userbot.modules.stories import repository
def _peer_id(story: Story) -> int:
if story.chat is not None:
return story.chat.id or 0
if story.from_user is not None:
return story.from_user.id or 0
return 0
from userbot.modules.stories.service import save_story
@PyroClient.on_story()
async def on_story(client: PyroClient, story: Story) -> None:
ctx = client.capture
if ctx is None:
if client.capture is None:
return
media_kind = story.media.name.lower() if story.media else None
storage_key: str | None = None
file_size: int | None = None
downloaded = False
if not story.deleted and story.media is not None:
buffer = await client.download_media(story, in_memory=True)
if isinstance(buffer, BytesIO):
data = buffer.getvalue()
storage_key = ctx.storage.put(data)
file_size = len(data)
downloaded = True
await repository.upsert_story(
ctx.pool,
ctx.account_id,
_peer_id(story),
story.id,
story.date,
story.expire_date,
story.caption,
media_kind,
storage_key,
file_size,
story.views,
str(story.raw),
pinned=bool(story.pinned),
deleted=bool(story.deleted),
downloaded=downloaded,
)
await save_story(client, client.capture, story)
handlers = on_story.handlers
backend/src/userbot/modules/jobs/handlers/__init__.py
+2
View File
@@ -1,5 +1,6 @@
from userbot.modules.jobs.handlers import (
backfill,
backfill_stories,
enrich_chat,
fetch_avatar,
fetch_custom_emoji,
@@ -12,6 +13,7 @@ from userbot.modules.jobs.handlers import (
__all__ = [
"backfill",
"backfill_stories",
"enrich_chat",
"fetch_avatar",
"fetch_custom_emoji",
backend/src/userbot/modules/jobs/handlers/backfill_stories.py
+70
View File
@@ -0,0 +1,70 @@
from collections.abc import AsyncIterator, Callable
from pyrogram import Client
from pyrogram.errors import FloodPremiumWait, FloodWait, RPCError
from pyrogram.types import Story
from userbot.modules.capture.context import CaptureContext
from userbot.modules.jobs.context import JobContext
from userbot.modules.jobs.registry import register
from userbot.modules.stories.service import save_story
SAVE_EVERY = 10
StorySource = Callable[[], AsyncIterator[Story]]
def _sources(client: Client, peer_id: int, *, own: bool) -> dict[str, StorySource]:
sources: dict[str, StorySource] = {
"active": lambda: client.get_chat_stories(peer_id),
"pinned": lambda: client.get_pinned_stories(peer_id),
}
if own:
sources["archived"] = lambda: client.get_archived_stories(peer_id)
return sources
async def _drain(
ctx: JobContext, capture: CaptureContext, name: str, source: StorySource
) -> int:
client = ctx.client
if client is None:
return 0
saved = 0
async for story in source():
try:
await save_story(client, capture, story)
except (FloodWait, FloodPremiumWait):
raise
except RPCError:
continue
saved += 1
if saved % SAVE_EVERY == 0:
await ctx.report_progress({"saved": saved, "source": name})
if await ctx.is_canceled():
break
return saved
@register("backfill_stories")
async def backfill_stories(ctx: JobContext) -> None:
client = ctx.client
if client is None:
return
capture = getattr(client, "capture", None)
if capture is None:
return
peer_id = ctx.job.params["peer_id"]
own = client.me is not None and client.me.id == peer_id
saved = 0
errors: dict[str, str] = {}
for name, source in _sources(client, peer_id, own=own).items():
try:
saved += await _drain(ctx, capture, name, source)
except (FloodWait, FloodPremiumWait):
raise
except RPCError as exc:
errors[name] = type(exc).__name__
if await ctx.is_canceled():
break
await ctx.report_progress({"saved": saved, "done": True, "errors": errors})
backend/src/userbot/modules/stories/repository.py
+14
View File
@@ -22,6 +22,20 @@ ON CONFLICT (account_id, peer_id, story_id) DO UPDATE SET
"""
async def is_downloaded(
pool: asyncpg.Pool, account_id: int, peer_id: int, story_id: int
) -> bool:
return bool(
await pool.fetchval(
"SELECT downloaded FROM stories "
"WHERE account_id = $1 AND peer_id = $2 AND story_id = $3",
account_id,
peer_id,
story_id,
)
)
async def upsert_story( # noqa: PLR0913
pool: asyncpg.Pool,
account_id: int,
backend/src/userbot/modules/stories/service.py
+49
View File
@@ -0,0 +1,49 @@
from io import BytesIO
from pyrogram import Client
from pyrogram.types import Story
from userbot.modules.capture.context import CaptureContext
from userbot.modules.stories import repository
def story_peer_id(story: Story) -> int:
if story.chat is not None:
return story.chat.id or 0
if story.from_user is not None:
return story.from_user.id or 0
return 0
async def save_story(client: Client, capture: CaptureContext, story: Story) -> None:
peer_id = story_peer_id(story)
storage_key: str | None = None
file_size: int | None = None
downloaded = False
stored = await repository.is_downloaded(
capture.pool, capture.account_id, peer_id, story.id
)
if not (stored or story.deleted or story.media is None):
buffer = await client.download_media(story, in_memory=True)
if isinstance(buffer, BytesIO):
data = buffer.getvalue()
storage_key = capture.storage.put(data)
file_size = len(data)
downloaded = True
await repository.upsert_story(
capture.pool,
capture.account_id,
peer_id,
story.id,
story.date,
story.expire_date,
story.caption,
story.media.name.lower() if story.media else None,
storage_key,
file_size,
story.views,
str(story.raw),
pinned=bool(story.pinned),
deleted=bool(story.deleted),
downloaded=downloaded,
)
frontend/src/lib/api/endpoints.ts
+10 -1
View File
@@ -315,7 +315,7 @@ export function getMessageAt(chatId: number, date: string): Promise<MessageAt> {
}
export function getStories(
peerId: number,
peerId: number | null,
page: Page = {}
): Promise<StoryView[]> {
return request<StoryView[]>("/stories", {
@@ -367,6 +367,15 @@ export function enqueueBackfill(
});
}
export function enqueueStoriesBackfill(
peerId: number
): Promise<{ job_id: number }> {
return request<{ job_id: number }>("/stories/backfill", {
method: "POST",
body: { account_id: accounts.selectedId, peer_id: peerId },
});
}
export function discoverPeers(
query: string,
remote = false
frontend/src/lib/components/jobs/JobList.svelte
+3 -2
View File
@@ -16,6 +16,7 @@
const KIND_LABELS: Record<string, string> = {
backfill: "Бэкфилл",
backfill_stories: "Бэкфилл сторис",
fetch_media: "Докачка медиа",
fetch_avatar: "Аватар",
fetch_custom_emoji: "Кастом-эмодзи",
@@ -88,12 +89,12 @@
}
function processed(job: JobView): number | null {
const value = job.progress.processed;
const value = job.progress.processed ?? job.progress.saved;
return typeof value === "number" ? value : null;
}
function chatId(job: JobView): number | null {
const value = job.params.chat_id;
const value = job.params.chat_id ?? job.params.peer_id;
return typeof value === "number" ? value : null;
}
frontend/src/lib/components/profile/ProfilePanel.svelte
+5 -1
View File
@@ -6,19 +6,21 @@
import ProfileInfo from "$lib/components/profile/ProfileInfo.svelte";
import SharedLinks from "$lib/components/profile/SharedLinks.svelte";
import SharedMedia from "$lib/components/profile/SharedMedia.svelte";
import StoriesArchive from "$lib/components/stories/StoriesArchive.svelte";
import Avatar from "$lib/components/ui/Avatar.svelte";
import EmptyState from "$lib/components/ui/EmptyState.svelte";
import Icon from "$lib/components/ui/Icon.svelte";
import { peerName } from "$lib/format/peer";
import { chats } from "$lib/stores/chats.svelte";
type Tab = "info" | "media" | "files" | "links" | "calendar";
type Tab = "info" | "media" | "files" | "links" | "stories" | "calendar";
const TABS: { id: Tab; icon: string; label: string }[] = [
{ id: "info", icon: "info", label: "Инфо" },
{ id: "media", icon: "photo", label: "Медиа" },
{ id: "files", icon: "document", label: "Файлы" },
{ id: "links", icon: "link", label: "Ссылки" },
{ id: "stories", icon: "play-story", label: "Сторис" },
{ id: "calendar", icon: "calendar", label: "Календарь" },
];
@@ -114,6 +116,8 @@
<SharedMedia {chatId} kinds={FILE_KINDS} layout="list" />
{:else if tab === "links"}
<SharedLinks {chatId} />
{:else if tab === "stories"}
<StoriesArchive {chatId} />
{:else if tab === "calendar"}
<ChatCalendar {chatId} />
{/if}
frontend/src/lib/components/stories/AllStoriesArchive.svelte
+35 -2
View File
@@ -1,16 +1,23 @@
<script lang="ts">
import { untrack } from "svelte";
import { getChat, getPeers, getStories } from "$lib/api/endpoints";
import {
enqueueStoriesBackfill,
getChat,
getPeers,
getStories,
} from "$lib/api/endpoints";
import { loadStoryMedia } from "$lib/api/stories";
import type { StoryView } from "$lib/api/types";
import StoryViewer from "$lib/components/stories/StoryViewer.svelte";
import Avatar from "$lib/components/ui/Avatar.svelte";
import Button from "$lib/components/ui/Button.svelte";
import EmptyState from "$lib/components/ui/EmptyState.svelte";
import Icon from "$lib/components/ui/Icon.svelte";
import Spinner from "$lib/components/ui/Spinner.svelte";
import { peerName } from "$lib/format/peer";
import { poster } from "$lib/media/poster";
import { accounts } from "$lib/stores/accounts.svelte";
import { toasts } from "$lib/stores/toasts.svelte";
interface Group {
hasAvatar: boolean;
@@ -31,12 +38,13 @@
let viewerIndex = $state(0);
let viewerItems = $state<StoryView[]>([]);
let viewerPeerId = $state(0);
let backfilling = $state<number | null>(null);
async function load() {
const current = token;
loading = true;
try {
const stories = await getStories(0, { limit: FETCH_LIMIT });
const stories = await getStories(null, { limit: FETCH_LIMIT });
if (current !== token) {
return;
}
@@ -127,6 +135,21 @@
};
});
async function backfill(peerId: number) {
if (backfilling !== null) {
return;
}
backfilling = peerId;
try {
await enqueueStoriesBackfill(peerId);
toasts.success("Загружаем старые сторис");
} catch {
toasts.error("Не удалось запустить загрузку сторис");
} finally {
backfilling = null;
}
}
function openViewer(group: Group, index: number) {
viewerItems = group.stories;
viewerPeerId = group.peerId;
@@ -154,6 +177,16 @@
/>
<span class="group-name">{group.name}</span>
<span class="group-count">{group.stories.length}</span>
<Button
variant="translucent"
round
smaller
loading={backfilling === group.peerId}
onclick={() => backfill(group.peerId)}
aria-label="Загрузить старые сторис"
>
<Icon name="cloud-download" />
</Button>
</header>
<div class="grid">
{#each group.stories as item, index (item.story_id)}
frontend/src/lib/components/stories/StoriesArchive.svelte
+70 -5
View File
@@ -1,22 +1,32 @@
<script lang="ts">
import { untrack } from "svelte";
import { page } from "$app/state";
import { getStories } from "$lib/api/endpoints";
import { enqueueStoriesBackfill, getStories } from "$lib/api/endpoints";
import { loadStoryMedia } from "$lib/api/stories";
import type { StoryView } from "$lib/api/types";
import StoryViewer from "$lib/components/stories/StoryViewer.svelte";
import Button from "$lib/components/ui/Button.svelte";
import EmptyState from "$lib/components/ui/EmptyState.svelte";
import Icon from "$lib/components/ui/Icon.svelte";
import Spinner from "$lib/components/ui/Spinner.svelte";
import { poster } from "$lib/media/poster";
import { accounts } from "$lib/stores/accounts.svelte";
import { toasts } from "$lib/stores/toasts.svelte";
interface Props {
chatId?: number | null;
}
let { chatId = null }: Props = $props();
const PAGE = 60;
const peerId = $derived(
page.params.chatId ? Number(page.params.chatId) : null
chatId ?? (page.params.chatId ? Number(page.params.chatId) : null)
);
let backfilling = $state(false);
let items = $state<StoryView[]>([]);
let loading = $state(false);
let done = $state(false);
@@ -90,15 +100,63 @@
viewerIndex = index;
viewerOpen = true;
}
async function backfill(id: number) {
if (backfilling) {
return;
}
backfilling = true;
try {
await enqueueStoriesBackfill(id);
toasts.success("Загружаем старые сторис");
} catch {
toasts.error("Не удалось запустить загрузку сторис");
} finally {
backfilling = false;
}
}
async function reload(id: number) {
token++;
items = [];
done = false;
loading = false;
await loadMore(id);
}
</script>
{#if peerId === null}
<EmptyState title="Сторис" description="Откройте чат" />
{:else if items.length === 0}
{:else}
<div class="toolbar">
<Button
variant="secondary"
pill
smaller
loading={backfilling}
onclick={() => peerId !== null && backfill(peerId)}
>
<Icon name="cloud-download" />Загрузить старые
</Button>
<Button
variant="translucent"
round
smaller
onclick={() => peerId !== null && reload(peerId).catch(() => undefined)}
aria-label="Обновить"
>
<Icon name="reload" />
</Button>
</div>
{#if items.length === 0}
{#if loading}
<div class="center"><Spinner /></div>
{:else}
<EmptyState title="Нет сторис" />
<EmptyState
title="Нет сторис"
description="Нажмите «Загрузить старые», чтобы забрать архив"
/>
{/if}
{:else}
<div class="grid">
@@ -150,7 +208,6 @@
</button>
{/if}
{#if peerId !== null}
<StoryViewer
{peerId}
{items}
@@ -167,6 +224,14 @@
padding: 2rem 0;
}
.toolbar {
display: flex;
align-items: center;
gap: 0.5rem;
padding: 0.625rem 0.75rem;
border-bottom: 1px solid var(--color-borders);
}
.grid {
display: grid;
grid-template-columns: repeat(3, 1fr);