feat(watcher,api,frontend): watched bots become accounts and drain the update backlog

backend/src/api/mcp/server.py

@@ -13,9 +13,9 @@ from pydantic import BaseModel
 from api.mcp.format import build_transcript, load_notes, load_photos, resolve_names
 from dependencies.container import container
 from utils.jobs import enqueue
-from utils.read import annotations, bots, chats, media, peers, presence, social, watches
+from utils.read import annotations, chats, media, peers, presence, social, watches
 from utils.read.accounts import self_user_id, unknown_account_hint
-from utils.read.models import DEFAULT_LIMIT, MAX_LIMIT, Page
+from utils.read.models import DEFAULT_LIMIT, Page
 from utils.search.models import SearchFilters
 from utils.search.repository import search_messages
 from utils.storage import ContentAddressedStorage
@@ -58,11 +58,10 @@ Typical flows:
 - Monitoring: `set_watch` creates a local rule, `list_watches` lists rules, and
   `list_alerts` reads what those rules fired.
 
-Watched bots (a separate axis, `account_id` does not apply):
-- beavergram can also hold bot tokens of arbitrary bots and archive who writes
-  to them over MTProto. `list_watched_bots` names them, `get_bot_stats` gives
-  the aggregate picture and `get_bot_updates` the raw incoming stream. These
-  take `bot_id` (the bot's Telegram id), never `account_id`.
+Watched bots:
+- a bot whose token the user added is just another archived account: it has its
+  own `account_id` and its chats, messages and media read through the same
+  tools as a user account. Nothing bot-specific to call.
 
 Writes: everything is read-only except `set_watch`, the only allowed write.
 
@@ -362,41 +361,3 @@ async def list_alerts(
             await _pool(), account_id, Page(limit=limit, offset=offset), seen=seen
         )
     )
-
-
-@mcp.tool
-async def list_watched_bots() -> list[dict[str, Any]]:
-    """List bots this beavergram watches, with their status and counters."""
-    return _dump(await bots.list_bots(await _pool()))
-
-
-@mcp.tool
-async def get_bot_stats(
-    bot_id: int, days: int = 90, senders: int = 20
-) -> dict[str, Any] | None:
-    """Who writes to a watched bot: totals, daily volume, hours, top senders."""
-    result = await bots.stats(await _pool(), bot_id, days=days, senders=senders)
-    return result.model_dump(mode="json") if result else None
-
-
-@mcp.tool
-async def get_bot_updates(
-    bot_id: int,
-    sender_id: int | None = None,
-    chat_id: int | None = None,
-    incoming_only: bool = True,
-    limit: int = DEFAULT_LIMIT,
-    offset: int = 0,
-) -> list[dict[str, Any]]:
-    """Read what people sent to a watched bot, newest first."""
-    return _dump(
-        await bots.list_updates(
-            await _pool(),
-            bot_id,
-            sender_id=sender_id,
-            chat_id=chat_id,
-            incoming_only=incoming_only,
-            limit=min(limit, MAX_LIMIT),
-            offset=offset,
-        )
-    )

backend/src/api/routers/bots.py

@@ -1,12 +1,10 @@
-from typing import Annotated
-
 import asyncpg
 from dishka.integrations.fastapi import DishkaRoute, FromDishka
-from fastapi import APIRouter, HTTPException, Query
+from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 
 from utils.read import bots
-from utils.read.models import DEFAULT_LIMIT, BotStats, BotUpdateView, BotView, Page
+from utils.read.models import BotView
 from watcher import tokens
 
 router = APIRouter(prefix="/api/bots", tags=["bots"], route_class=DishkaRoute)
@@ -42,38 +40,3 @@ async def remove_bot(pool: FromDishka[asyncpg.Pool], bot_id: int) -> None:
     if not await bots.remove_bot(pool, bot_id):
         raise HTTPException(status_code=404, detail="unknown bot")
     await bots.notify_bots_changed(pool)
-
-
-@router.get("/{bot_id}/stats")
-async def stats(
-    pool: FromDishka[asyncpg.Pool],
-    bot_id: int,
-    days: Annotated[int, Query()] = 90,
-    senders: Annotated[int, Query()] = 20,
-) -> BotStats:
-    result = await bots.stats(pool, bot_id, days=days, senders=senders)
-    if result is None:
-        raise HTTPException(status_code=404, detail="unknown bot")
-    return result
-
-
-@router.get("/{bot_id}/updates")
-async def list_updates(
-    pool: FromDishka[asyncpg.Pool],
-    bot_id: int,
-    sender_id: Annotated[int | None, Query()] = None,
-    chat_id: Annotated[int | None, Query()] = None,
-    incoming_only: Annotated[bool, Query()] = True,
-    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
-    offset: Annotated[int, Query()] = 0,
-) -> list[BotUpdateView]:
-    page = Page(limit=limit, offset=offset)
-    return await bots.list_updates(
-        pool,
-        bot_id,
-        sender_id=sender_id,
-        chat_id=chat_id,
-        incoming_only=incoming_only,
-        limit=page.capped_limit,
-        offset=page.offset,
-    )

backend/src/userbot/modules/client.py

@@ -13,19 +13,23 @@ DEVICE_MODEL_LIMIT = 32
 
 
 class PyroClient(Client):
-    def __init__(
+    def __init__(  # noqa: PLR0913
         self,
         name: str,
         *,
         workdir: str = "sessions",
         device_model: str | None = None,
         load_handlers: bool = True,
+        bot_token: str | None = None,
+        skip_updates: bool = True,
     ) -> None:
         super().__init__(
             name,
             workdir=workdir,
             api_id=API_ID,
             api_hash=API_HASH,
+            bot_token=bot_token,
+            skip_updates=skip_updates,
             device_model=device_model or DEVICE_MODEL,
             system_version="Windows 11 x64",
             app_version="7.0.8 x64",

backend/src/utils/read/accounts.py

@@ -7,17 +7,18 @@ from utils.read.models import AccountView
 
 ACCOUNTS_CHANGED_CHANNEL = "accounts_changed"
 
-_ACCOUNT_COLS = "account_id, label, phone, tg_user_id, is_active, device_model"
+_ACCOUNT_COLS = "account_id, label, phone, tg_user_id, is_active, is_bot, device_model"
 
 _UPSERT_ACCOUNT = """
 INSERT INTO accounts
-    (tg_user_id, label, phone, session_name, is_active, raw, updated_at)
-VALUES ($1, $2, $3, $4, TRUE, $5::jsonb, now())
+    (tg_user_id, label, phone, session_name, is_active, is_bot, raw, updated_at)
+VALUES ($1, $2, $3, $4, TRUE, $6, $5::jsonb, now())
 ON CONFLICT (tg_user_id) DO UPDATE SET
     label = EXCLUDED.label,
     phone = EXCLUDED.phone,
     session_name = EXCLUDED.session_name,
     is_active = TRUE,
+    is_bot = EXCLUDED.is_bot,
     raw = EXCLUDED.raw,
     updated_at = now()
 RETURNING account_id
@@ -89,7 +90,9 @@ async def get_account(pool: asyncpg.Pool, account_id: int) -> AccountView | None
     return AccountView(**dict(row)) if row else None
 
 
-async def sync_account(pool: asyncpg.Pool, me: User, session_name: str) -> int:
+async def sync_account(
+    pool: asyncpg.Pool, me: User, session_name: str, *, is_bot: bool = False
+) -> int:
     raw = json.dumps(
         {
             "id": me.id,
@@ -101,7 +104,7 @@ async def sync_account(pool: asyncpg.Pool, me: User, session_name: str) -> int:
     )
     label = " ".join(filter(None, [me.first_name, me.last_name])) or me.username
     return await pool.fetchval(
-        _UPSERT_ACCOUNT, me.id, label, me.phone_number, session_name, raw
+        _UPSERT_ACCOUNT, me.id, label, me.phone_number, session_name, raw, is_bot
     )
 
 

backend/src/utils/read/bots.py

@@ -1,17 +1,13 @@
-from datetime import UTC, datetime, timedelta
+"""Registry of bot tokens the watcher holds.
+
+Only session bookkeeping lives here: what the archive itself keeps about a
+bot is an ordinary account row and ordinary messages, read through
+``utils.read.accounts`` and ``utils.read.chats`` like any other account.
+"""
 
 import asyncpg
 
-from utils.read.models import (
-    BotBucket,
-    BotCount,
-    BotHour,
-    BotSender,
-    BotStats,
-    BotSummary,
-    BotUpdateView,
-    BotView,
-)
+from utils.read.models import BotView
 
 BOTS_CHANGED_CHANNEL = "watched_bots_changed"
 
@@ -22,17 +18,9 @@ STATUS_ERROR = "error"
 STATUS_REMOVED = "removed"
 
 _BOT_COLS = """
-SELECT b.bot_id, b.username, b.title, b.enabled, b.status, b.status_detail,
-       b.backfilled_at, b.checked_at, b.added_at,
-       coalesce(u.incoming, 0) AS incoming,
-       coalesce(u.senders, 0) AS senders,
-       u.last_ts
-FROM watched_bots b
-LEFT JOIN LATERAL (
-    SELECT count(*) AS incoming, count(DISTINCT sender_id) AS senders,
-           max(ts) AS last_ts
-    FROM bot_updates WHERE bot_id = b.bot_id AND NOT outgoing
-) u ON true
+SELECT bot_id, account_id, username, title, enabled, status, status_detail,
+       backfilled_at, checked_at, added_at
+FROM watched_bots
 """
 
 _REGISTER = """
@@ -58,91 +46,18 @@ UPDATE watched_bots SET
 WHERE bot_id = $1
 """
 
-_STORE = """
-INSERT INTO bot_updates
-    (bot_id, kind, chat_id, item_id, ts, source, outgoing, sender_id,
-     sender_username, sender_name, sender_is_bot, chat_type, chat_title,
-     text, media_kind, edited_at, raw)
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15,
-        $16, $17::jsonb)
-ON CONFLICT (bot_id, kind, chat_id, item_id, ts) DO UPDATE SET
-    text = EXCLUDED.text,
-    media_kind = EXCLUDED.media_kind,
-    chat_title = coalesce(EXCLUDED.chat_title, bot_updates.chat_title),
-    sender_username = coalesce(EXCLUDED.sender_username,
-                               bot_updates.sender_username),
-    sender_name = coalesce(EXCLUDED.sender_name, bot_updates.sender_name),
-    edited_at = coalesce(EXCLUDED.edited_at, bot_updates.edited_at),
-    raw = EXCLUDED.raw
-"""
-
-_INCOMING = "bot_id = $1 AND NOT outgoing AND ts >= $2"
-
-_SUMMARY = """
-SELECT count(*) FILTER (WHERE NOT outgoing) AS incoming,
-       count(*) FILTER (WHERE outgoing) AS outgoing,
-       count(DISTINCT sender_id) FILTER (WHERE NOT outgoing) AS senders,
-       count(DISTINCT chat_id) AS chats,
-       min(ts) AS first_ts, max(ts) AS last_ts
-FROM bot_updates WHERE bot_id = $1 AND ts >= $2
-"""
-
-_TIMELINE = f"""
-SELECT time_bucket('1 day', ts) AS bucket, count(*) AS incoming,
-       count(DISTINCT sender_id) AS senders
-FROM bot_updates WHERE {_INCOMING}
-GROUP BY bucket ORDER BY bucket
-"""  # noqa: S608
-
-_KINDS = f"""
-SELECT kind AS key, count(*) AS incoming
-FROM bot_updates WHERE {_INCOMING}
-GROUP BY kind ORDER BY incoming DESC
-"""  # noqa: S608
-
-_MEDIA = f"""
-SELECT media_kind AS key, count(*) AS incoming
-FROM bot_updates WHERE {_INCOMING} AND media_kind IS NOT NULL
-GROUP BY media_kind ORDER BY incoming DESC
-"""  # noqa: S608
-
-_HOURS = f"""
-SELECT EXTRACT(HOUR FROM ts)::int AS hour, count(*) AS incoming
-FROM bot_updates WHERE {_INCOMING}
-GROUP BY hour ORDER BY hour
-"""  # noqa: S608
-
-_SENDERS = f"""
-SELECT sender_id,
-       (array_agg(sender_username ORDER BY ts DESC) FILTER (
-           WHERE sender_username IS NOT NULL))[1] AS username,
-       (array_agg(sender_name ORDER BY ts DESC) FILTER (
-           WHERE sender_name IS NOT NULL))[1] AS name,
-       bool_or(sender_is_bot) AS is_bot,
-       count(*) AS incoming,
-       (array_agg(chat_id ORDER BY ts DESC))[1] AS chat_id,
-       min(ts) AS first_ts, max(ts) AS last_ts
-FROM bot_updates WHERE {_INCOMING} AND sender_id IS NOT NULL
-GROUP BY sender_id ORDER BY incoming DESC LIMIT $3
-"""  # noqa: S608
-
-_VIEW_COLS = (
-    "kind, chat_id, item_id, ts, source, outgoing, sender_id, sender_username, "
-    "sender_name, sender_is_bot, chat_type, chat_title, text, media_kind, edited_at"
-)
-
 
 async def notify_bots_changed(pool: asyncpg.Pool) -> None:
     await pool.execute("SELECT pg_notify($1, '')", BOTS_CHANGED_CHANNEL)
 
 
 async def list_bots(pool: asyncpg.Pool) -> list[BotView]:
-    rows = await pool.fetch(f"{_BOT_COLS} ORDER BY u.last_ts DESC NULLS LAST")
+    rows = await pool.fetch(f"{_BOT_COLS} ORDER BY added_at")
     return [BotView(**dict(row)) for row in rows]
 
 
 async def get_bot(pool: asyncpg.Pool, bot_id: int) -> BotView | None:
-    row = await pool.fetchrow(f"{_BOT_COLS} WHERE b.bot_id = $1", bot_id)
+    row = await pool.fetchrow(f"{_BOT_COLS} WHERE bot_id = $1", bot_id)
     return BotView(**dict(row)) if row else None
 
 
@@ -154,6 +69,19 @@ async def enabled_bot_ids(pool: asyncpg.Pool) -> set[int]:
     return {row["bot_id"] for row in rows}
 
 
+async def removed_bot_ids(pool: asyncpg.Pool) -> set[int]:
+    rows = await pool.fetch(
+        "SELECT bot_id FROM watched_bots WHERE NOT enabled AND status = $1",
+        STATUS_REMOVED,
+    )
+    return {row["bot_id"] for row in rows}
+
+
+async def known_bot_ids(pool: asyncpg.Pool) -> set[int]:
+    rows = await pool.fetch("SELECT bot_id FROM watched_bots")
+    return {row["bot_id"] for row in rows}
+
+
 async def register_bot(pool: asyncpg.Pool, bot_id: int) -> None:
     await pool.execute(_REGISTER, bot_id)
 
@@ -162,6 +90,14 @@ async def remove_bot(pool: asyncpg.Pool, bot_id: int) -> bool:
     return await pool.execute(_REMOVE, bot_id) != "UPDATE 0"
 
 
+async def link_account(pool: asyncpg.Pool, bot_id: int, account_id: int) -> None:
+    await pool.execute(
+        "UPDATE watched_bots SET account_id = $2, updated_at = now() WHERE bot_id = $1",
+        bot_id,
+        account_id,
+    )
+
+
 async def set_status(  # noqa: PLR0913
     pool: asyncpg.Pool,
     bot_id: int,
@@ -186,76 +122,3 @@ async def mark_backfilled(pool: asyncpg.Pool, bot_id: int) -> None:
         "WHERE bot_id = $1",
         bot_id,
     )
-
-
-async def store_update(pool: asyncpg.Pool, values: tuple) -> None:
-    await pool.execute(_STORE, *values)
-
-
-async def store_updates(pool: asyncpg.Pool, values: list[tuple]) -> None:
-    if values:
-        await pool.executemany(_STORE, values)
-
-
-async def stats(
-    pool: asyncpg.Pool, bot_id: int, *, days: int = 90, senders: int = 20
-) -> BotStats | None:
-    bot = await get_bot(pool, bot_id)
-    if bot is None:
-        return None
-    since = datetime.now(UTC) - timedelta(days=days)
-    summary = await pool.fetchrow(_SUMMARY, bot_id, since)
-    timeline = await pool.fetch(_TIMELINE, bot_id, since)
-    kinds = await pool.fetch(_KINDS, bot_id, since)
-    media = await pool.fetch(_MEDIA, bot_id, since)
-    hours = await pool.fetch(_HOURS, bot_id, since)
-    sender_rows = await pool.fetch(_SENDERS, bot_id, since, senders)
-    return BotStats(
-        bot=bot,
-        summary=BotSummary(**dict(summary)),
-        timeline=[BotBucket(**dict(row)) for row in timeline],
-        kinds=[BotCount(**dict(row)) for row in kinds],
-        media=[BotCount(**dict(row)) for row in media],
-        hours=[BotHour(**dict(row)) for row in hours],
-        senders=[BotSender(**dict(row)) for row in sender_rows],
-    )
-
-
-async def list_updates(  # noqa: PLR0913
-    pool: asyncpg.Pool,
-    bot_id: int,
-    *,
-    sender_id: int | None = None,
-    chat_id: int | None = None,
-    incoming_only: bool = True,
-    limit: int = 50,
-    offset: int = 0,
-) -> list[BotUpdateView]:
-    rows = await pool.fetch(
-        f"SELECT {_VIEW_COLS} FROM bot_updates "  # noqa: S608
-        "WHERE bot_id = $1 "
-        "AND ($2::bigint IS NULL OR sender_id = $2) "
-        "AND ($3::bigint IS NULL OR chat_id = $3) "
-        "AND (NOT $4 OR NOT outgoing) "
-        "ORDER BY ts DESC LIMIT $5 OFFSET $6",
-        bot_id,
-        sender_id,
-        chat_id,
-        incoming_only,
-        limit,
-        offset,
-    )
-    return [BotUpdateView(**dict(row)) for row in rows]
-
-
-async def removed_bot_ids(pool: asyncpg.Pool) -> set[int]:
-    rows = await pool.fetch(
-        "SELECT bot_id FROM watched_bots WHERE NOT enabled AND status = $1",
-        STATUS_REMOVED,
-    )
-    return {row["bot_id"] for row in rows}
-
-
-async def known_bot_ids(pool: asyncpg.Pool) -> set[int]:
-    rows = await pool.fetch("SELECT bot_id FROM watched_bots")
-    return {row["bot_id"] for row in rows}

backend/src/utils/read/models.py

@@ -22,6 +22,7 @@ class AccountView(BaseModel):
     phone: str | None
     tg_user_id: int | None
     is_active: bool
+    is_bot: bool
     device_model: str | None
 
 
@@ -427,6 +428,7 @@ class FileShareHitView(BaseModel):
 
 class BotView(BaseModel):
     bot_id: int
+    account_id: int | None
     username: str | None
     title: str | None
     enabled: bool
@@ -435,70 +437,3 @@ class BotView(BaseModel):
     backfilled_at: datetime | None
     checked_at: datetime | None
     added_at: datetime
-    incoming: int
-    senders: int
-    last_ts: datetime | None
-
-
-class BotSummary(BaseModel):
-    incoming: int
-    outgoing: int
-    senders: int
-    chats: int
-    first_ts: datetime | None
-    last_ts: datetime | None
-
-
-class BotSender(BaseModel):
-    sender_id: int
-    username: str | None
-    name: str | None
-    is_bot: bool | None
-    incoming: int
-    chat_id: int | None
-    first_ts: datetime
-    last_ts: datetime
-
-
-class BotBucket(BaseModel):
-    bucket: datetime
-    incoming: int
-    senders: int
-
-
-class BotCount(BaseModel):
-    key: str
-    incoming: int
-
-
-class BotHour(BaseModel):
-    hour: int
-    incoming: int
-
-
-class BotStats(BaseModel):
-    bot: BotView
-    summary: BotSummary
-    timeline: list[BotBucket]
-    kinds: list[BotCount]
-    media: list[BotCount]
-    hours: list[BotHour]
-    senders: list[BotSender]
-
-
-class BotUpdateView(BaseModel):
-    kind: str
-    chat_id: int
-    item_id: int
-    ts: datetime
-    source: str
-    outgoing: bool
-    sender_id: int | None
-    sender_username: str | None
-    sender_name: str | None
-    sender_is_bot: bool | None
-    chat_type: str | None
-    chat_title: str | None
-    text: str | None
-    media_kind: str | None
-    edited_at: datetime | None

backend/src/watcher/backfill.py

@@ -1,71 +0,0 @@
-import asyncio
-
-import asyncpg
-from pyrogram import Client
-from pyrogram.errors import FloodWait, RPCError
-
-from utils.env import env
-from utils.logging import logger
-from utils.read import bots
-from watcher.capture import SOURCE_HISTORY, Row, message_row, store_many
-
-BATCH = 200
-MAX_FLOOD_WAIT = 300
-HISTORY_UNAVAILABLE = "История недоступна: {}"
-
-
-async def _flush(pool: asyncpg.Pool, rows: list[Row]) -> int:
-    await store_many(pool, rows)
-    stored = len(rows)
-    rows.clear()
-    return stored
-
-
-async def _chat_history(
-    client: Client, pool: asyncpg.Pool, bot_id: int, chat_id: int
-) -> int:
-    rows: list[Row] = []
-    stored = 0
-    async for message in client.get_chat_history(
-        chat_id, limit=env.bots.backfill_per_chat
-    ):
-        if message.empty or message.chat is None or message.date is None:
-            continue
-        rows.append(message_row(bot_id, message, source=SOURCE_HISTORY))
-        if len(rows) >= BATCH:
-            stored += await _flush(pool, rows)
-    return stored + await _flush(pool, rows)
-
-
-async def backfill(client: Client, pool: asyncpg.Pool, bot_id: int) -> None:
-    try:
-        dialogs = [
-            dialog.chat.id
-            async for dialog in client.get_dialogs(limit=env.bots.backfill_chats)
-            if dialog.chat is not None and dialog.chat.id is not None
-        ]
-    except RPCError as exc:
-        logger.warning(f"[yellow]Bot {bot_id}: dialogs unavailable:[/] {exc}")
-        await bots.set_status(
-            pool, bot_id, bots.STATUS_OK, detail=HISTORY_UNAVAILABLE.format(exc)
-        )
-        return
-
-    stored = 0
-    for chat_id in dialogs:
-        try:
-            stored += await _chat_history(client, pool, bot_id, chat_id)
-        except FloodWait as exc:
-            wait = exc.value if isinstance(exc.value, int) else MAX_FLOOD_WAIT + 1
-            if wait > MAX_FLOOD_WAIT:
-                logger.warning(f"[yellow]Bot {bot_id}: flood wait {wait}s, stop[/]")
-                break
-            await asyncio.sleep(wait)
-        except RPCError as exc:
-            logger.warning(f"[yellow]Bot {bot_id}: chat {chat_id} skipped:[/] {exc}")
-
-    await bots.mark_backfilled(pool, bot_id)
-    logger.info(
-        f"[green]Bot {bot_id}: backfilled[/] {stored} messages "
-        f"from {len(dialogs)} chats"
-    )

backend/src/watcher/capture.py

@@ -1,146 +0,0 @@
-from datetime import UTC, datetime
-
-import asyncpg
-from pyrogram.types import CallbackQuery, ChatMemberUpdated, Message, User
-
-from userbot.modules.raw_json import raw_json
-from utils.read import bots
-
-SOURCE_HISTORY = "history"
-SOURCE_LIVE = "live"
-
-KIND_MESSAGE = "message"
-KIND_CALLBACK = "callback_query"
-KIND_MEMBER = "member"
-
-MEDIA_KINDS = (
-    "photo",
-    "video",
-    "animation",
-    "audio",
-    "voice",
-    "video_note",
-    "document",
-    "sticker",
-    "contact",
-    "location",
-    "venue",
-    "poll",
-    "dice",
-    "story",
-    "game",
-)
-
-Row = tuple
-
-
-def _media_kind(message: Message) -> str | None:
-    return next(
-        (kind for kind in MEDIA_KINDS if getattr(message, kind, None) is not None), None
-    )
-
-
-def _full_name(user: User | None) -> str | None:
-    if user is None:
-        return None
-    parts = [user.first_name, user.last_name]
-    return " ".join(part for part in parts if part) or None
-
-
-def _chat_title(message: Message) -> str | None:
-    chat = message.chat
-    if chat is None:
-        return None
-    parts = [chat.first_name, chat.last_name]
-    return chat.title or " ".join(part for part in parts if part) or None
-
-
-def _chat_type(message: Message) -> str | None:
-    kind = getattr(message.chat, "type", None)
-    return kind.value if kind is not None else None
-
-
-def message_row(bot_id: int, message: Message, *, source: str) -> Row:
-    user = message.from_user
-    chat = message.chat
-    sender = user.id if user is not None else None
-    if sender is None and message.sender_chat is not None:
-        sender = message.sender_chat.id
-    return (
-        bot_id,
-        KIND_MESSAGE,
-        chat.id if chat is not None else 0,
-        message.id,
-        message.date,
-        source,
-        bool(message.outgoing),
-        sender,
-        user.username if user is not None else None,
-        _full_name(user),
-        user.is_bot if user is not None else None,
-        _chat_type(message),
-        _chat_title(message),
-        message.text or message.caption,
-        _media_kind(message),
-        message.edit_date,
-        raw_json(message),
-    )
-
-
-def callback_row(bot_id: int, query: CallbackQuery) -> Row:
-    user = query.from_user
-    message = query.message
-    chat = message.chat if message is not None else None
-    return (
-        bot_id,
-        KIND_CALLBACK,
-        chat.id if chat is not None else 0,
-        message.id if message is not None else 0,
-        datetime.now(UTC),
-        SOURCE_LIVE,
-        False,
-        user.id if user is not None else None,
-        user.username if user is not None else None,
-        _full_name(user),
-        user.is_bot if user is not None else None,
-        chat.type.value if chat is not None and chat.type is not None else None,
-        chat.title if chat is not None else None,
-        query.data,
-        None,
-        None,
-        raw_json(query),
-    )
-
-
-def member_row(bot_id: int, update: ChatMemberUpdated) -> Row:
-    member = update.new_chat_member
-    user = member.user if member is not None else None
-    chat = update.chat
-    status = member.status.value if member is not None else None
-    return (
-        bot_id,
-        KIND_MEMBER,
-        chat.id if chat is not None else 0,
-        user.id if user is not None else 0,
-        update.date or datetime.now(UTC),
-        SOURCE_LIVE,
-        False,
-        update.from_user.id if update.from_user is not None else None,
-        update.from_user.username if update.from_user is not None else None,
-        _full_name(update.from_user),
-        update.from_user.is_bot if update.from_user is not None else None,
-        chat.type.value if chat is not None and chat.type is not None else None,
-        chat.title if chat is not None else None,
-        status,
-        None,
-        None,
-        raw_json(update),
-    )
-
-
-async def store(pool: asyncpg.Pool, row: Row) -> None:
-    await bots.store_update(pool, row)
-
-
-async def store_many(pool: asyncpg.Pool, rows: list[Row]) -> None:
-    await bots.store_updates(pool, rows)

backend/src/watcher/context.py

@@ -0,0 +1,33 @@
+"""A capture context for a bot session.
+
+Same context the userbot capture path expects, minus the two caches a bot
+may not fill: ``messages.getDialogFilters`` and ``contacts.getContacts`` are
+both refused for bots, so folders and contacts simply stay empty and policy
+falls back to the per-account defaults.
+"""
+
+import asyncpg
+from pyrogram import Client
+
+from userbot.modules.capture.context import CaptureContext
+from userbot.modules.contacts import ContactCache
+from userbot.modules.folders import FolderCache
+from utils.storage import ContentAddressedStorage
+
+
+async def build_bot_capture_context(
+    client: Client,
+    pool: asyncpg.Pool,
+    storage: ContentAddressedStorage,
+    account_id: int,
+) -> CaptureContext:
+    ctx = CaptureContext(
+        account_id,
+        pool,
+        storage,
+        FolderCache(client, pool, account_id),
+        ContactCache(client),
+    )
+    await ctx.reload_policies()
+    await ctx.watches.refresh()
+    return ctx

backend/src/watcher/drain.py

@@ -0,0 +1,96 @@
+"""Pull the updates Telegram still holds for a bot that nobody consumed.
+
+A bot's undelivered updates outlive the session they were meant for: the
+same queue Bot API ``getUpdates`` drains is reachable over MTProto through
+``updates.getDifference``. A fresh login normally starts from
+``updates.getState``, which throws that queue away - this walks it instead.
+"""
+
+from pyrogram import raw, types
+from pyrogram.errors import RPCError
+
+from userbot import PyroClient
+from userbot.modules.capture import capture_message
+from userbot.modules.capture.chat_meta import meta_from_chat
+from userbot.modules.capture.context import CaptureContext
+from utils.logging import logger
+
+EARLIEST_PTS = 1
+EARLIEST_QTS = 1
+MAX_SLICES = 50
+
+
+async def _store(
+    client: PyroClient, ctx: CaptureContext, message: types.Message
+) -> int:
+    if message.empty or message.chat is None or message.date is None:
+        return 0
+    meta = meta_from_chat(message.chat, ctx.contacts.ids)
+    toggles = ctx.resolve(meta)
+    if not toggles.messages:
+        return 0
+    await capture_message(client, message, ctx, toggles)
+    return 1
+
+
+async def _parse_all(
+    client: PyroClient, new_messages: list, users: list, chats: list
+) -> list[types.Message]:
+    user_map = {u.id: u for u in users}
+    chat_map = {c.id: c for c in chats}
+    parsed = []
+    for raw_message in new_messages:
+        if isinstance(raw_message, raw.types.MessageEmpty):
+            continue
+        parsed.append(
+            await types.Message._parse(client, raw_message, user_map, chat_map)  # noqa: SLF001
+        )
+    return parsed
+
+
+async def drain_backlog(client: PyroClient, account_id: int) -> int:
+    """Walk the bot's pending update queue into the archive. Returns stored count."""
+    ctx = client.capture
+    if ctx is None:
+        return 0
+    pts, qts, date = EARLIEST_PTS, EARLIEST_QTS, 0
+    stored = 0
+    for _ in range(MAX_SLICES):
+        try:
+            diff = await client.invoke(
+                raw.functions.updates.GetDifference(pts=pts, date=date, qts=qts)
+            )
+        except RPCError as exc:
+            logger.warning(
+                f"[yellow]Account {account_id}: getDifference failed:[/] {exc}"
+            )
+            return stored
+
+        if isinstance(diff, raw.types.updates.DifferenceEmpty):
+            logger.info(f"Account {account_id}: backlog empty at pts={pts}")
+            return stored
+        if isinstance(diff, raw.types.updates.DifferenceTooLong):
+            logger.info(
+                f"Account {account_id}: backlog too long, server pts={diff.pts}; "
+                f"nothing older is retrievable"
+            )
+            return stored
+
+        for message in await _parse_all(
+            client, diff.new_messages, diff.users, diff.chats
+        ):
+            stored += await _store(client, ctx, message)
+
+        if isinstance(diff, raw.types.updates.Difference):
+            logger.info(
+                f"[green]Account {account_id}: backlog drained,[/] {stored} messages"
+            )
+            return stored
+
+        pts, qts, date = (
+            diff.intermediate_state.pts,
+            diff.intermediate_state.qts,
+            diff.intermediate_state.date,
+        )
+    logger.warning(f"[yellow]Account {account_id}: backlog walk hit the slice cap[/]")
+    return stored

backend/src/watcher/registry.py

@@ -3,31 +3,37 @@ import contextlib
 from dataclasses import dataclass
 
 import asyncpg
-from pyrogram import Client
+from pyrogram import Client, enums
 from pyrogram.errors import RPCError
 from pyrogram.handlers import (
-    CallbackQueryHandler,
     ChatMemberUpdatedHandler,
     EditedMessageHandler,
     MessageHandler,
 )
 
+from userbot import PyroClient
+from userbot.modules.capture import capture_message
+from userbot.modules.capture.chat_meta import meta_from_chat
+from userbot.modules.jobs import JobConsumer
+from utils.events import notify_bg_event
+from utils.jobs import enqueue
 from utils.logging import logger
-from utils.read import bots
-from utils.telegram import API_HASH, API_ID
+from utils.read import accounts, bots
+from utils.storage import ContentAddressedStorage
 from watcher import tokens
-from watcher.backfill import backfill
-from watcher.capture import SOURCE_LIVE, callback_row, member_row, message_row, store
+from watcher.context import build_bot_capture_context
+from watcher.drain import drain_backlog
 
-DEVICE_MODEL = "Beavergram Watcher"
+DEVICE_MODEL = "Beavergram Bot"
 TOKEN_MISSING = "Токен не найден на диске"  # noqa: S105
 INVALID_TOKEN_ERRORS = ("ACCESS_TOKEN_INVALID", "ACCESS_TOKEN_EXPIRED")
 
 
 @dataclass
 class RunningBot:
-    client: Client
-    backfill_task: asyncio.Task | None
+    client: PyroClient
+    account_id: int
+    tasks: list[asyncio.Task]
 
 
 async def _cancel(task: asyncio.Task) -> None:
@@ -36,22 +42,40 @@ async def _cancel(task: asyncio.Task) -> None:
         await task
 
 
-def _handlers(pool: asyncpg.Pool, bot_id: int) -> tuple:
-    async def on_message(_client: Client, message) -> None:  # noqa: ANN001
-        if message.empty or message.chat is None or message.date is None:
+def _handlers(pool: asyncpg.Pool) -> tuple:
+    async def on_message(client: Client, message) -> None:  # noqa: ANN001
+        ctx = getattr(client, "capture", None)
+        if ctx is None or message.empty or message.chat is None or message.date is None:
             return
-        await store(pool, message_row(bot_id, message, source=SOURCE_LIVE))
+        meta = meta_from_chat(message.chat, ctx.contacts.ids)
+        await ctx.watches.on_text(
+            meta.chat_id, message.id, message.text or message.caption
+        )
+        toggles = ctx.resolve(meta)
+        if not toggles.messages:
+            return
+        await capture_message(client, message, ctx, toggles)
+        await notify_bg_event(
+            ctx.pool,
+            "message",
+            ctx.account_id,
+            chat_id=meta.chat_id,
+            message_id=message.id,
+        )
+        if message.chat.type is not enums.ChatType.PRIVATE:
+            await enqueue(
+                pool, ctx.account_id, "enrich_chat", {"chat_id": meta.chat_id}
+            )
 
-    async def on_callback(_client: Client, query) -> None:  # noqa: ANN001
-        await store(pool, callback_row(bot_id, query))
-
-    async def on_member(_client: Client, update) -> None:  # noqa: ANN001
-        await store(pool, member_row(bot_id, update))
+    async def on_member(client: Client, update) -> None:  # noqa: ANN001
+        ctx = getattr(client, "capture", None)
+        if ctx is None or update.chat is None:
+            return
+        await enqueue(pool, ctx.account_id, "enrich_chat", {"chat_id": update.chat.id})
 
     return (
         MessageHandler(on_message),
         EditedMessageHandler(on_message),
-        CallbackQueryHandler(on_callback),
         ChatMemberUpdatedHandler(on_member),
     )
 
@@ -61,8 +85,9 @@ def _is_invalid_token(exc: RPCError) -> bool:
 
 
 class BotRegistry:
-    def __init__(self, pool: asyncpg.Pool) -> None:
+    def __init__(self, pool: asyncpg.Pool, storage: ContentAddressedStorage) -> None:
         self._pool = pool
+        self._storage = storage
         self._running: dict[int, RunningBot] = {}
         self._lock = asyncio.Lock()
 
@@ -91,6 +116,16 @@ class BotRegistry:
             await bots.register_bot(self._pool, bot_id)
             logger.info(f"[green]Bot token adopted from disk:[/] {bot_id}")
 
+    def _client(self, bot_id: int, token: str) -> PyroClient:
+        return PyroClient(
+            str(bot_id),
+            workdir=str(tokens.sessions_dir()),
+            device_model=DEVICE_MODEL,
+            load_handlers=False,
+            bot_token=token,
+            skip_updates=False,
+        )
+
     async def _start(self, bot_id: int) -> None:
         token = tokens.load(bot_id)
         if token is None:
@@ -98,15 +133,8 @@ class BotRegistry:
                 self._pool, bot_id, bots.STATUS_UNAUTHORIZED, detail=TOKEN_MISSING
             )
             return
-        client = Client(
-            str(bot_id),
-            api_id=API_ID,
-            api_hash=API_HASH,
-            bot_token=token,
-            workdir=str(tokens.sessions_dir()),
-            device_model=DEVICE_MODEL,
-        )
-        for handler in _handlers(self._pool, bot_id):
+        client = self._client(bot_id, token)
+        for handler in _handlers(self._pool):
             client.add_handler(handler)
         try:
             await client.start()
@@ -134,23 +162,39 @@ class BotRegistry:
                 await client.stop()
             return
 
+        account_id = await accounts.sync_account(
+            self._pool, me, str(bot_id), is_bot=True
+        )
+        client.capture = await build_bot_capture_context(
+            client, self._pool, self._storage, account_id
+        )
+        await bots.link_account(self._pool, bot_id, account_id)
         await bots.set_status(
             self._pool, bot_id, bots.STATUS_OK, username=me.username, title=me.full_name
         )
-        task = (
-            asyncio.create_task(backfill(client, self._pool, bot_id))
-            if await bots.needs_backfill(self._pool, bot_id)
-            else None
+        await accounts.notify_accounts_changed(self._pool)
+
+        consumer = JobConsumer(client, self._pool, account_id)
+        tasks = [asyncio.create_task(consumer.run())]
+        if await bots.needs_backfill(self._pool, bot_id):
+            tasks.append(asyncio.create_task(self._backlog(client, bot_id, account_id)))
+        self._running[bot_id] = RunningBot(client, account_id, tasks)
+        logger.info(
+            f"[green]Bot account {account_id} watched:[/] @{me.username} ({bot_id})"
         )
-        self._running[bot_id] = RunningBot(client, task)
-        logger.info(f"[green]Bot watched:[/] @{me.username} ({bot_id})")
+
+    async def _backlog(self, client: PyroClient, bot_id: int, account_id: int) -> None:
+        stored = await drain_backlog(client, account_id)
+        await bots.mark_backfilled(self._pool, bot_id)
+        if stored:
+            await accounts.notify_accounts_changed(self._pool)
 
     async def _stop(self, bot_id: int) -> None:
         running = self._running.pop(bot_id, None)
         if running is None:
             return
-        if running.backfill_task is not None:
-            await _cancel(running.backfill_task)
+        for task in running.tasks:
+            await _cancel(task)
         with contextlib.suppress(Exception):
             await running.client.stop()
         logger.info(f"[yellow]Bot unwatched:[/] {bot_id}")

backend/src/watcher/runner.py

@@ -9,6 +9,7 @@ from dependencies.container import container
 from utils.env import env
 from utils.logging import logger, setup_logging
 from utils.read.bots import BOTS_CHANGED_CHANNEL
+from utils.storage import ContentAddressedStorage
 from watcher.registry import BotRegistry
 
 setup_logging()
@@ -40,7 +41,8 @@ async def _listen_changes(
 
 async def runner() -> None:
     pool = await container.get(asyncpg.Pool)
-    registry = BotRegistry(pool)
+    storage = await container.get(ContentAddressedStorage)
+    registry = BotRegistry(pool, storage)
     tasks: set[asyncio.Task] = set()
     listen_conn: asyncpg.Connection | None = None
     stop = asyncio.Event()