feat(watcher,api,frontend): archive who writes to watched bots over mtproto

backend/src/api/app.py

@@ -18,6 +18,7 @@ from api.routers import (
     annotations,
     avatars,
     backfill,
+    bots,
     chats,
     custom_emoji,
     discover,
@@ -73,6 +74,7 @@ async def health(pool: FromDishka[asyncpg.Pool]) -> dict[str, bool]:
 
 app.include_router(accounts.router)
 app.include_router(analytics.router)
+app.include_router(bots.router)
 app.include_router(policy.router)
 app.include_router(folders.router)
 app.include_router(backfill.router)

backend/src/api/mcp/server.py

@@ -13,9 +13,9 @@ from pydantic import BaseModel
 from api.mcp.format import build_transcript, load_notes, load_photos, resolve_names
 from dependencies.container import container
 from utils.jobs import enqueue
-from utils.read import annotations, chats, media, peers, presence, social, watches
+from utils.read import annotations, bots, chats, media, peers, presence, social, watches
 from utils.read.accounts import self_user_id, unknown_account_hint
-from utils.read.models import DEFAULT_LIMIT, Page
+from utils.read.models import DEFAULT_LIMIT, MAX_LIMIT, Page
 from utils.search.models import SearchFilters
 from utils.search.repository import search_messages
 from utils.storage import ContentAddressedStorage
@@ -58,6 +58,12 @@ Typical flows:
 - Monitoring: `set_watch` creates a local rule, `list_watches` lists rules, and
   `list_alerts` reads what those rules fired.
 
+Watched bots (a separate axis, `account_id` does not apply):
+- beavergram can also hold bot tokens of arbitrary bots and archive who writes
+  to them over MTProto. `list_watched_bots` names them, `get_bot_stats` gives
+  the aggregate picture and `get_bot_updates` the raw incoming stream. These
+  take `bot_id` (the bot's Telegram id), never `account_id`.
+
 Writes: everything is read-only except `set_watch`, the only allowed write.
 
 Paging: list/search tools accept `limit` and `offset`.
@@ -356,3 +362,41 @@ async def list_alerts(
             await _pool(), account_id, Page(limit=limit, offset=offset), seen=seen
         )
     )
+
+
+@mcp.tool
+async def list_watched_bots() -> list[dict[str, Any]]:
+    """List bots this beavergram watches, with their status and counters."""
+    return _dump(await bots.list_bots(await _pool()))
+
+
+@mcp.tool
+async def get_bot_stats(
+    bot_id: int, days: int = 90, senders: int = 20
+) -> dict[str, Any] | None:
+    """Who writes to a watched bot: totals, daily volume, hours, top senders."""
+    result = await bots.stats(await _pool(), bot_id, days=days, senders=senders)
+    return result.model_dump(mode="json") if result else None
+
+
+@mcp.tool
+async def get_bot_updates(
+    bot_id: int,
+    sender_id: int | None = None,
+    chat_id: int | None = None,
+    incoming_only: bool = True,
+    limit: int = DEFAULT_LIMIT,
+    offset: int = 0,
+) -> list[dict[str, Any]]:
+    """Read what people sent to a watched bot, newest first."""
+    return _dump(
+        await bots.list_updates(
+            await _pool(),
+            bot_id,
+            sender_id=sender_id,
+            chat_id=chat_id,
+            incoming_only=incoming_only,
+            limit=min(limit, MAX_LIMIT),
+            offset=offset,
+        )
+    )

backend/src/api/routers/bots.py

@@ -0,0 +1,79 @@
+from typing import Annotated
+
+import asyncpg
+from dishka.integrations.fastapi import DishkaRoute, FromDishka
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel
+
+from utils.read import bots
+from utils.read.models import DEFAULT_LIMIT, BotStats, BotUpdateView, BotView, Page
+from watcher import tokens
+
+router = APIRouter(prefix="/api/bots", tags=["bots"], route_class=DishkaRoute)
+
+INVALID_TOKEN = "token must look like 123456:ABC-DEF..."  # noqa: S105
+
+
+class BotAdd(BaseModel):
+    token: str
+
+
+@router.get("")
+async def list_bots(pool: FromDishka[asyncpg.Pool]) -> list[BotView]:
+    return await bots.list_bots(pool)
+
+
+@router.post("", status_code=201)
+async def add_bot(pool: FromDishka[asyncpg.Pool], body: BotAdd) -> BotView:
+    bot_id = tokens.bot_id_of(body.token)
+    if bot_id is None:
+        raise HTTPException(status_code=422, detail=INVALID_TOKEN)
+    tokens.save(bot_id, body.token)
+    await bots.register_bot(pool, bot_id)
+    await bots.notify_bots_changed(pool)
+    bot = await bots.get_bot(pool, bot_id)
+    if bot is None:
+        raise HTTPException(status_code=500, detail="bot was not registered")
+    return bot
+
+
+@router.delete("/{bot_id}", status_code=204)
+async def remove_bot(pool: FromDishka[asyncpg.Pool], bot_id: int) -> None:
+    if not await bots.remove_bot(pool, bot_id):
+        raise HTTPException(status_code=404, detail="unknown bot")
+    await bots.notify_bots_changed(pool)
+
+
+@router.get("/{bot_id}/stats")
+async def stats(
+    pool: FromDishka[asyncpg.Pool],
+    bot_id: int,
+    days: Annotated[int, Query()] = 90,
+    senders: Annotated[int, Query()] = 20,
+) -> BotStats:
+    result = await bots.stats(pool, bot_id, days=days, senders=senders)
+    if result is None:
+        raise HTTPException(status_code=404, detail="unknown bot")
+    return result
+
+
+@router.get("/{bot_id}/updates")
+async def list_updates(
+    pool: FromDishka[asyncpg.Pool],
+    bot_id: int,
+    sender_id: Annotated[int | None, Query()] = None,
+    chat_id: Annotated[int | None, Query()] = None,
+    incoming_only: Annotated[bool, Query()] = True,
+    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
+    offset: Annotated[int, Query()] = 0,
+) -> list[BotUpdateView]:
+    page = Page(limit=limit, offset=offset)
+    return await bots.list_updates(
+        pool,
+        bot_id,
+        sender_id=sender_id,
+        chat_id=chat_id,
+        incoming_only=incoming_only,
+        limit=page.capped_limit,
+        offset=page.offset,
+    )

backend/src/userbot/modules/client.py

@@ -2,6 +2,8 @@ from typing import TYPE_CHECKING
 
 from pyrogram import Client, enums
 
+from utils.telegram import API_HASH, API_ID
+
 if TYPE_CHECKING:
     from userbot.modules.capture import CaptureContext
 
@@ -22,8 +24,8 @@ class PyroClient(Client):
         super().__init__(
             name,
             workdir=workdir,
-            api_id=2040,
-            api_hash="b18441a1ff607e10a989891a5462e627",
+            api_id=API_ID,
+            api_hash=API_HASH,
             device_model=device_model or DEVICE_MODEL,
             system_version="Windows 11 x64",
             app_version="7.0.8 x64",

backend/src/utils/env.py

@@ -31,6 +31,12 @@ class TelegramSettings(BaseSettings):
     sessions_dir: str = "sessions"
 
 
+class BotsSettings(BaseSettings):
+    sessions_dir: str = "sessions/bots"
+    backfill_per_chat: int = 0
+    backfill_chats: int = 0
+
+
 class ApiSettings(BaseSettings):
     host: str = "0.0.0.0"  # noqa: S104
     port: int = 8080
@@ -56,6 +62,7 @@ class LogSettings(BaseSettings):
 class Settings(BaseSettings):
     db: DatabaseSettings = Field(default_factory=DatabaseSettings)
     tg: TelegramSettings = Field(default_factory=TelegramSettings)
+    bots: BotsSettings = Field(default_factory=BotsSettings)
     api: ApiSettings = Field(default_factory=ApiSettings)
     auth: AuthSettings = Field(default_factory=AuthSettings)
     storage: StorageSettings = Field(default_factory=StorageSettings)

backend/src/utils/read/bots.py

@@ -0,0 +1,261 @@
+from datetime import UTC, datetime, timedelta
+
+import asyncpg
+
+from utils.read.models import (
+    BotBucket,
+    BotCount,
+    BotHour,
+    BotSender,
+    BotStats,
+    BotSummary,
+    BotUpdateView,
+    BotView,
+)
+
+BOTS_CHANGED_CHANNEL = "watched_bots_changed"
+
+STATUS_PENDING = "pending"
+STATUS_OK = "ok"
+STATUS_UNAUTHORIZED = "unauthorized"
+STATUS_ERROR = "error"
+STATUS_REMOVED = "removed"
+
+_BOT_COLS = """
+SELECT b.bot_id, b.username, b.title, b.enabled, b.status, b.status_detail,
+       b.backfilled_at, b.checked_at, b.added_at,
+       coalesce(u.incoming, 0) AS incoming,
+       coalesce(u.senders, 0) AS senders,
+       u.last_ts
+FROM watched_bots b
+LEFT JOIN LATERAL (
+    SELECT count(*) AS incoming, count(DISTINCT sender_id) AS senders,
+           max(ts) AS last_ts
+    FROM bot_updates WHERE bot_id = b.bot_id AND NOT outgoing
+) u ON true
+"""
+
+_REGISTER = """
+INSERT INTO watched_bots (bot_id, status) VALUES ($1, 'pending')
+ON CONFLICT (bot_id) DO UPDATE SET
+    enabled = true, status = 'pending', status_detail = NULL, updated_at = now()
+"""
+
+_REMOVE = """
+UPDATE watched_bots
+SET enabled = false, status = 'removed', status_detail = NULL, updated_at = now()
+WHERE bot_id = $1 AND enabled
+"""
+
+_SET_STATUS = """
+UPDATE watched_bots SET
+    status = $2,
+    status_detail = $3,
+    username = coalesce($4, username),
+    title = coalesce($5, title),
+    checked_at = now(),
+    updated_at = now()
+WHERE bot_id = $1
+"""
+
+_STORE = """
+INSERT INTO bot_updates
+    (bot_id, kind, chat_id, item_id, ts, source, outgoing, sender_id,
+     sender_username, sender_name, sender_is_bot, chat_type, chat_title,
+     text, media_kind, edited_at, raw)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15,
+        $16, $17::jsonb)
+ON CONFLICT (bot_id, kind, chat_id, item_id, ts) DO UPDATE SET
+    text = EXCLUDED.text,
+    media_kind = EXCLUDED.media_kind,
+    chat_title = coalesce(EXCLUDED.chat_title, bot_updates.chat_title),
+    sender_username = coalesce(EXCLUDED.sender_username,
+                               bot_updates.sender_username),
+    sender_name = coalesce(EXCLUDED.sender_name, bot_updates.sender_name),
+    edited_at = coalesce(EXCLUDED.edited_at, bot_updates.edited_at),
+    raw = EXCLUDED.raw
+"""
+
+_INCOMING = "bot_id = $1 AND NOT outgoing AND ts >= $2"
+
+_SUMMARY = """
+SELECT count(*) FILTER (WHERE NOT outgoing) AS incoming,
+       count(*) FILTER (WHERE outgoing) AS outgoing,
+       count(DISTINCT sender_id) FILTER (WHERE NOT outgoing) AS senders,
+       count(DISTINCT chat_id) AS chats,
+       min(ts) AS first_ts, max(ts) AS last_ts
+FROM bot_updates WHERE bot_id = $1 AND ts >= $2
+"""
+
+_TIMELINE = f"""
+SELECT time_bucket('1 day', ts) AS bucket, count(*) AS incoming,
+       count(DISTINCT sender_id) AS senders
+FROM bot_updates WHERE {_INCOMING}
+GROUP BY bucket ORDER BY bucket
+"""  # noqa: S608
+
+_KINDS = f"""
+SELECT kind AS key, count(*) AS incoming
+FROM bot_updates WHERE {_INCOMING}
+GROUP BY kind ORDER BY incoming DESC
+"""  # noqa: S608
+
+_MEDIA = f"""
+SELECT media_kind AS key, count(*) AS incoming
+FROM bot_updates WHERE {_INCOMING} AND media_kind IS NOT NULL
+GROUP BY media_kind ORDER BY incoming DESC
+"""  # noqa: S608
+
+_HOURS = f"""
+SELECT EXTRACT(HOUR FROM ts)::int AS hour, count(*) AS incoming
+FROM bot_updates WHERE {_INCOMING}
+GROUP BY hour ORDER BY hour
+"""  # noqa: S608
+
+_SENDERS = f"""
+SELECT sender_id,
+       (array_agg(sender_username ORDER BY ts DESC) FILTER (
+           WHERE sender_username IS NOT NULL))[1] AS username,
+       (array_agg(sender_name ORDER BY ts DESC) FILTER (
+           WHERE sender_name IS NOT NULL))[1] AS name,
+       bool_or(sender_is_bot) AS is_bot,
+       count(*) AS incoming,
+       (array_agg(chat_id ORDER BY ts DESC))[1] AS chat_id,
+       min(ts) AS first_ts, max(ts) AS last_ts
+FROM bot_updates WHERE {_INCOMING} AND sender_id IS NOT NULL
+GROUP BY sender_id ORDER BY incoming DESC LIMIT $3
+"""  # noqa: S608
+
+_VIEW_COLS = (
+    "kind, chat_id, item_id, ts, source, outgoing, sender_id, sender_username, "
+    "sender_name, sender_is_bot, chat_type, chat_title, text, media_kind, edited_at"
+)
+
+
+async def notify_bots_changed(pool: asyncpg.Pool) -> None:
+    await pool.execute("SELECT pg_notify($1, '')", BOTS_CHANGED_CHANNEL)
+
+
+async def list_bots(pool: asyncpg.Pool) -> list[BotView]:
+    rows = await pool.fetch(f"{_BOT_COLS} ORDER BY u.last_ts DESC NULLS LAST")
+    return [BotView(**dict(row)) for row in rows]
+
+
+async def get_bot(pool: asyncpg.Pool, bot_id: int) -> BotView | None:
+    row = await pool.fetchrow(f"{_BOT_COLS} WHERE b.bot_id = $1", bot_id)
+    return BotView(**dict(row)) if row else None
+
+
+async def enabled_bot_ids(pool: asyncpg.Pool) -> set[int]:
+    rows = await pool.fetch(
+        "SELECT bot_id FROM watched_bots WHERE enabled AND status <> $1",
+        STATUS_UNAUTHORIZED,
+    )
+    return {row["bot_id"] for row in rows}
+
+
+async def register_bot(pool: asyncpg.Pool, bot_id: int) -> None:
+    await pool.execute(_REGISTER, bot_id)
+
+
+async def remove_bot(pool: asyncpg.Pool, bot_id: int) -> bool:
+    return await pool.execute(_REMOVE, bot_id) != "UPDATE 0"
+
+
+async def set_status(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    bot_id: int,
+    status: str,
+    *,
+    detail: str | None = None,
+    username: str | None = None,
+    title: str | None = None,
+) -> None:
+    await pool.execute(_SET_STATUS, bot_id, status, detail, username, title)
+
+
+async def needs_backfill(pool: asyncpg.Pool, bot_id: int) -> bool:
+    return not await pool.fetchval(
+        "SELECT backfilled_at IS NOT NULL FROM watched_bots WHERE bot_id = $1", bot_id
+    )
+
+
+async def mark_backfilled(pool: asyncpg.Pool, bot_id: int) -> None:
+    await pool.execute(
+        "UPDATE watched_bots SET backfilled_at = now(), updated_at = now() "
+        "WHERE bot_id = $1",
+        bot_id,
+    )
+
+
+async def store_update(pool: asyncpg.Pool, values: tuple) -> None:
+    await pool.execute(_STORE, *values)
+
+
+async def store_updates(pool: asyncpg.Pool, values: list[tuple]) -> None:
+    if values:
+        await pool.executemany(_STORE, values)
+
+
+async def stats(
+    pool: asyncpg.Pool, bot_id: int, *, days: int = 90, senders: int = 20
+) -> BotStats | None:
+    bot = await get_bot(pool, bot_id)
+    if bot is None:
+        return None
+    since = datetime.now(UTC) - timedelta(days=days)
+    summary = await pool.fetchrow(_SUMMARY, bot_id, since)
+    timeline = await pool.fetch(_TIMELINE, bot_id, since)
+    kinds = await pool.fetch(_KINDS, bot_id, since)
+    media = await pool.fetch(_MEDIA, bot_id, since)
+    hours = await pool.fetch(_HOURS, bot_id, since)
+    sender_rows = await pool.fetch(_SENDERS, bot_id, since, senders)
+    return BotStats(
+        bot=bot,
+        summary=BotSummary(**dict(summary)),
+        timeline=[BotBucket(**dict(row)) for row in timeline],
+        kinds=[BotCount(**dict(row)) for row in kinds],
+        media=[BotCount(**dict(row)) for row in media],
+        hours=[BotHour(**dict(row)) for row in hours],
+        senders=[BotSender(**dict(row)) for row in sender_rows],
+    )
+
+
+async def list_updates(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    bot_id: int,
+    *,
+    sender_id: int | None = None,
+    chat_id: int | None = None,
+    incoming_only: bool = True,
+    limit: int = 50,
+    offset: int = 0,
+) -> list[BotUpdateView]:
+    rows = await pool.fetch(
+        f"SELECT {_VIEW_COLS} FROM bot_updates "  # noqa: S608
+        "WHERE bot_id = $1 "
+        "AND ($2::bigint IS NULL OR sender_id = $2) "
+        "AND ($3::bigint IS NULL OR chat_id = $3) "
+        "AND (NOT $4 OR NOT outgoing) "
+        "ORDER BY ts DESC LIMIT $5 OFFSET $6",
+        bot_id,
+        sender_id,
+        chat_id,
+        incoming_only,
+        limit,
+        offset,
+    )
+    return [BotUpdateView(**dict(row)) for row in rows]
+
+
+async def removed_bot_ids(pool: asyncpg.Pool) -> set[int]:
+    rows = await pool.fetch(
+        "SELECT bot_id FROM watched_bots WHERE NOT enabled AND status = $1",
+        STATUS_REMOVED,
+    )
+    return {row["bot_id"] for row in rows}
+
+
+async def known_bot_ids(pool: asyncpg.Pool) -> set[int]:
+    rows = await pool.fetch("SELECT bot_id FROM watched_bots")
+    return {row["bot_id"] for row in rows}

backend/src/utils/read/models.py

@@ -423,3 +423,82 @@ class FileShareHitView(BaseModel):
     ip: str | None
     user_agent: str | None
     counted: bool
+
+
+class BotView(BaseModel):
+    bot_id: int
+    username: str | None
+    title: str | None
+    enabled: bool
+    status: str
+    status_detail: str | None
+    backfilled_at: datetime | None
+    checked_at: datetime | None
+    added_at: datetime
+    incoming: int
+    senders: int
+    last_ts: datetime | None
+
+
+class BotSummary(BaseModel):
+    incoming: int
+    outgoing: int
+    senders: int
+    chats: int
+    first_ts: datetime | None
+    last_ts: datetime | None
+
+
+class BotSender(BaseModel):
+    sender_id: int
+    username: str | None
+    name: str | None
+    is_bot: bool | None
+    incoming: int
+    chat_id: int | None
+    first_ts: datetime
+    last_ts: datetime
+
+
+class BotBucket(BaseModel):
+    bucket: datetime
+    incoming: int
+    senders: int
+
+
+class BotCount(BaseModel):
+    key: str
+    incoming: int
+
+
+class BotHour(BaseModel):
+    hour: int
+    incoming: int
+
+
+class BotStats(BaseModel):
+    bot: BotView
+    summary: BotSummary
+    timeline: list[BotBucket]
+    kinds: list[BotCount]
+    media: list[BotCount]
+    hours: list[BotHour]
+    senders: list[BotSender]
+
+
+class BotUpdateView(BaseModel):
+    kind: str
+    chat_id: int
+    item_id: int
+    ts: datetime
+    source: str
+    outgoing: bool
+    sender_id: int | None
+    sender_username: str | None
+    sender_name: str | None
+    sender_is_bot: bool | None
+    chat_type: str | None
+    chat_title: str | None
+    text: str | None
+    media_kind: str | None
+    edited_at: datetime | None

backend/src/utils/telegram.py

@@ -0,0 +1,2 @@
+API_ID = 2040
+API_HASH = "b18441a1ff607e10a989891a5462e627"

backend/src/watcher/__main__.py

@@ -0,0 +1,4 @@
+from .runner import main
+
+if __name__ == "__main__":
+    main()

backend/src/watcher/backfill.py

@@ -0,0 +1,71 @@
+import asyncio
+
+import asyncpg
+from pyrogram import Client
+from pyrogram.errors import FloodWait, RPCError
+
+from utils.env import env
+from utils.logging import logger
+from utils.read import bots
+from watcher.capture import SOURCE_HISTORY, Row, message_row, store_many
+
+BATCH = 200
+MAX_FLOOD_WAIT = 300
+HISTORY_UNAVAILABLE = "История недоступна: {}"
+
+
+async def _flush(pool: asyncpg.Pool, rows: list[Row]) -> int:
+    await store_many(pool, rows)
+    stored = len(rows)
+    rows.clear()
+    return stored
+
+
+async def _chat_history(
+    client: Client, pool: asyncpg.Pool, bot_id: int, chat_id: int
+) -> int:
+    rows: list[Row] = []
+    stored = 0
+    async for message in client.get_chat_history(
+        chat_id, limit=env.bots.backfill_per_chat
+    ):
+        if message.empty or message.chat is None or message.date is None:
+            continue
+        rows.append(message_row(bot_id, message, source=SOURCE_HISTORY))
+        if len(rows) >= BATCH:
+            stored += await _flush(pool, rows)
+    return stored + await _flush(pool, rows)
+
+
+async def backfill(client: Client, pool: asyncpg.Pool, bot_id: int) -> None:
+    try:
+        dialogs = [
+            dialog.chat.id
+            async for dialog in client.get_dialogs(limit=env.bots.backfill_chats)
+            if dialog.chat is not None and dialog.chat.id is not None
+        ]
+    except RPCError as exc:
+        logger.warning(f"[yellow]Bot {bot_id}: dialogs unavailable:[/] {exc}")
+        await bots.set_status(
+            pool, bot_id, bots.STATUS_OK, detail=HISTORY_UNAVAILABLE.format(exc)
+        )
+        return
+
+    stored = 0
+    for chat_id in dialogs:
+        try:
+            stored += await _chat_history(client, pool, bot_id, chat_id)
+        except FloodWait as exc:
+            wait = exc.value if isinstance(exc.value, int) else MAX_FLOOD_WAIT + 1
+            if wait > MAX_FLOOD_WAIT:
+                logger.warning(f"[yellow]Bot {bot_id}: flood wait {wait}s, stop[/]")
+                break
+            await asyncio.sleep(wait)
+        except RPCError as exc:
+            logger.warning(f"[yellow]Bot {bot_id}: chat {chat_id} skipped:[/] {exc}")
+
+    await bots.mark_backfilled(pool, bot_id)
+    logger.info(
+        f"[green]Bot {bot_id}: backfilled[/] {stored} messages "
+        f"from {len(dialogs)} chats"
+    )

backend/src/watcher/capture.py

@@ -0,0 +1,146 @@
+from datetime import UTC, datetime
+
+import asyncpg
+from pyrogram.types import CallbackQuery, ChatMemberUpdated, Message, User
+
+from userbot.modules.raw_json import raw_json
+from utils.read import bots
+
+SOURCE_HISTORY = "history"
+SOURCE_LIVE = "live"
+
+KIND_MESSAGE = "message"
+KIND_CALLBACK = "callback_query"
+KIND_MEMBER = "member"
+
+MEDIA_KINDS = (
+    "photo",
+    "video",
+    "animation",
+    "audio",
+    "voice",
+    "video_note",
+    "document",
+    "sticker",
+    "contact",
+    "location",
+    "venue",
+    "poll",
+    "dice",
+    "story",
+    "game",
+)
+
+Row = tuple
+
+
+def _media_kind(message: Message) -> str | None:
+    return next(
+        (kind for kind in MEDIA_KINDS if getattr(message, kind, None) is not None), None
+    )
+
+
+def _full_name(user: User | None) -> str | None:
+    if user is None:
+        return None
+    parts = [user.first_name, user.last_name]
+    return " ".join(part for part in parts if part) or None
+
+
+def _chat_title(message: Message) -> str | None:
+    chat = message.chat
+    if chat is None:
+        return None
+    parts = [chat.first_name, chat.last_name]
+    return chat.title or " ".join(part for part in parts if part) or None
+
+
+def _chat_type(message: Message) -> str | None:
+    kind = getattr(message.chat, "type", None)
+    return kind.value if kind is not None else None
+
+
+def message_row(bot_id: int, message: Message, *, source: str) -> Row:
+    user = message.from_user
+    chat = message.chat
+    sender = user.id if user is not None else None
+    if sender is None and message.sender_chat is not None:
+        sender = message.sender_chat.id
+    return (
+        bot_id,
+        KIND_MESSAGE,
+        chat.id if chat is not None else 0,
+        message.id,
+        message.date,
+        source,
+        bool(message.outgoing),
+        sender,
+        user.username if user is not None else None,
+        _full_name(user),
+        user.is_bot if user is not None else None,
+        _chat_type(message),
+        _chat_title(message),
+        message.text or message.caption,
+        _media_kind(message),
+        message.edit_date,
+        raw_json(message),
+    )
+
+
+def callback_row(bot_id: int, query: CallbackQuery) -> Row:
+    user = query.from_user
+    message = query.message
+    chat = message.chat if message is not None else None
+    return (
+        bot_id,
+        KIND_CALLBACK,
+        chat.id if chat is not None else 0,
+        message.id if message is not None else 0,
+        datetime.now(UTC),
+        SOURCE_LIVE,
+        False,
+        user.id if user is not None else None,
+        user.username if user is not None else None,
+        _full_name(user),
+        user.is_bot if user is not None else None,
+        chat.type.value if chat is not None and chat.type is not None else None,
+        chat.title if chat is not None else None,
+        query.data,
+        None,
+        None,
+        raw_json(query),
+    )
+
+
+def member_row(bot_id: int, update: ChatMemberUpdated) -> Row:
+    member = update.new_chat_member
+    user = member.user if member is not None else None
+    chat = update.chat
+    status = member.status.value if member is not None else None
+    return (
+        bot_id,
+        KIND_MEMBER,
+        chat.id if chat is not None else 0,
+        user.id if user is not None else 0,
+        update.date or datetime.now(UTC),
+        SOURCE_LIVE,
+        False,
+        update.from_user.id if update.from_user is not None else None,
+        update.from_user.username if update.from_user is not None else None,
+        _full_name(update.from_user),
+        update.from_user.is_bot if update.from_user is not None else None,
+        chat.type.value if chat is not None and chat.type is not None else None,
+        chat.title if chat is not None else None,
+        status,
+        None,
+        None,
+        raw_json(update),
+    )
+
+
+async def store(pool: asyncpg.Pool, row: Row) -> None:
+    await bots.store_update(pool, row)
+
+
+async def store_many(pool: asyncpg.Pool, rows: list[Row]) -> None:
+    await bots.store_updates(pool, rows)

backend/src/watcher/registry.py

@@ -0,0 +1,156 @@
+import asyncio
+import contextlib
+from dataclasses import dataclass
+
+import asyncpg
+from pyrogram import Client
+from pyrogram.errors import RPCError
+from pyrogram.handlers import (
+    CallbackQueryHandler,
+    ChatMemberUpdatedHandler,
+    EditedMessageHandler,
+    MessageHandler,
+)
+
+from utils.logging import logger
+from utils.read import bots
+from utils.telegram import API_HASH, API_ID
+from watcher import tokens
+from watcher.backfill import backfill
+from watcher.capture import SOURCE_LIVE, callback_row, member_row, message_row, store
+
+DEVICE_MODEL = "Beavergram Watcher"
+TOKEN_MISSING = "Токен не найден на диске"  # noqa: S105
+INVALID_TOKEN_ERRORS = ("ACCESS_TOKEN_INVALID", "ACCESS_TOKEN_EXPIRED")
+
+
+@dataclass
+class RunningBot:
+    client: Client
+    backfill_task: asyncio.Task | None
+
+
+async def _cancel(task: asyncio.Task) -> None:
+    task.cancel()
+    with contextlib.suppress(asyncio.CancelledError):
+        await task
+
+
+def _handlers(pool: asyncpg.Pool, bot_id: int) -> tuple:
+    async def on_message(_client: Client, message) -> None:  # noqa: ANN001
+        if message.empty or message.chat is None or message.date is None:
+            return
+        await store(pool, message_row(bot_id, message, source=SOURCE_LIVE))
+
+    async def on_callback(_client: Client, query) -> None:  # noqa: ANN001
+        await store(pool, callback_row(bot_id, query))
+
+    async def on_member(_client: Client, update) -> None:  # noqa: ANN001
+        await store(pool, member_row(bot_id, update))
+
+    return (
+        MessageHandler(on_message),
+        EditedMessageHandler(on_message),
+        CallbackQueryHandler(on_callback),
+        ChatMemberUpdatedHandler(on_member),
+    )
+
+
+def _is_invalid_token(exc: RPCError) -> bool:
+    return any(marker in str(exc) for marker in INVALID_TOKEN_ERRORS)
+
+
+class BotRegistry:
+    def __init__(self, pool: asyncpg.Pool) -> None:
+        self._pool = pool
+        self._running: dict[int, RunningBot] = {}
+        self._lock = asyncio.Lock()
+
+    @property
+    def running(self) -> list[int]:
+        return list(self._running)
+
+    async def sync(self) -> None:
+        async with self._lock:
+            await self._adopt_dropped_in()
+            enabled = await bots.enabled_bot_ids(self._pool)
+            for bot_id in set(self._running) - enabled:
+                await self._stop(bot_id)
+            for bot_id in enabled - set(self._running):
+                await self._start(bot_id)
+            for bot_id in await bots.removed_bot_ids(self._pool):
+                tokens.drop(bot_id)
+
+    async def close(self) -> None:
+        for bot_id in list(self._running):
+            await self._stop(bot_id)
+
+    async def _adopt_dropped_in(self) -> None:
+        unknown = tokens.stored_bot_ids() - await bots.known_bot_ids(self._pool)
+        for bot_id in unknown:
+            await bots.register_bot(self._pool, bot_id)
+            logger.info(f"[green]Bot token adopted from disk:[/] {bot_id}")
+
+    async def _start(self, bot_id: int) -> None:
+        token = tokens.load(bot_id)
+        if token is None:
+            await bots.set_status(
+                self._pool, bot_id, bots.STATUS_UNAUTHORIZED, detail=TOKEN_MISSING
+            )
+            return
+        client = Client(
+            str(bot_id),
+            api_id=API_ID,
+            api_hash=API_HASH,
+            bot_token=token,
+            workdir=str(tokens.sessions_dir()),
+            device_model=DEVICE_MODEL,
+        )
+        for handler in _handlers(self._pool, bot_id):
+            client.add_handler(handler)
+        try:
+            await client.start()
+            me = client.me
+            if me is None:
+                msg = "bot session is not authorized"
+                raise RuntimeError(msg)
+        except RPCError as exc:
+            status = (
+                bots.STATUS_UNAUTHORIZED
+                if _is_invalid_token(exc)
+                else bots.STATUS_ERROR
+            )
+            await bots.set_status(self._pool, bot_id, status, detail=str(exc))
+            logger.warning(f"[yellow]Bot {bot_id} failed to start:[/] {exc}")
+            with contextlib.suppress(Exception):
+                await client.stop()
+            return
+        except Exception as exc:
+            await bots.set_status(
+                self._pool, bot_id, bots.STATUS_ERROR, detail=str(exc)
+            )
+            logger.exception(f"[red]Bot {bot_id} failed to start[/]")
+            with contextlib.suppress(Exception):
+                await client.stop()
+            return
+
+        await bots.set_status(
+            self._pool, bot_id, bots.STATUS_OK, username=me.username, title=me.full_name
+        )
+        task = (
+            asyncio.create_task(backfill(client, self._pool, bot_id))
+            if await bots.needs_backfill(self._pool, bot_id)
+            else None
+        )
+        self._running[bot_id] = RunningBot(client, task)
+        logger.info(f"[green]Bot watched:[/] @{me.username} ({bot_id})")
+
+    async def _stop(self, bot_id: int) -> None:
+        running = self._running.pop(bot_id, None)
+        if running is None:
+            return
+        if running.backfill_task is not None:
+            await _cancel(running.backfill_task)
+        with contextlib.suppress(Exception):
+            await running.client.stop()
+        logger.info(f"[yellow]Bot unwatched:[/] {bot_id}")

backend/src/watcher/runner.py

@@ -0,0 +1,72 @@
+import asyncio
+import contextlib
+import signal
+
+import asyncpg
+import uvloop
+
+from dependencies.container import container
+from utils.env import env
+from utils.logging import logger, setup_logging
+from utils.read.bots import BOTS_CHANGED_CHANNEL
+from watcher.registry import BotRegistry
+
+setup_logging()
+
+RETRY_INTERVAL = 300
+
+
+async def _retry_loop(registry: BotRegistry) -> None:
+    while True:
+        await asyncio.sleep(RETRY_INTERVAL)
+        with contextlib.suppress(Exception):
+            await registry.sync()
+
+
+async def _listen_changes(
+    registry: BotRegistry, tasks: set[asyncio.Task]
+) -> asyncpg.Connection:
+    def on_change(
+        _conn: asyncpg.Connection, _pid: int, _channel: str, _payload: str
+    ) -> None:
+        task = asyncio.create_task(registry.sync())
+        tasks.add(task)
+        task.add_done_callback(tasks.discard)
+
+    conn = await asyncpg.connect(dsn=env.db.connection_url)
+    await conn.add_listener(BOTS_CHANGED_CHANNEL, on_change)
+    return conn
+
+
+async def runner() -> None:
+    pool = await container.get(asyncpg.Pool)
+    registry = BotRegistry(pool)
+    tasks: set[asyncio.Task] = set()
+    listen_conn: asyncpg.Connection | None = None
+    stop = asyncio.Event()
+    asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, stop.set)
+    retry = asyncio.create_task(_retry_loop(registry))
+    try:
+        await registry.sync()
+        if not registry.running:
+            logger.warning("[yellow]No bots yet. Add a bot token in the web UI.[/]")
+        listen_conn = await _listen_changes(registry, tasks)
+        logger.info("[green]Bot watcher running.[/]")
+        await stop.wait()
+    finally:
+        retry.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await retry
+        if listen_conn is not None:
+            with contextlib.suppress(Exception):
+                await listen_conn.close()
+        await registry.close()
+        await container.close()
+
+
+def main() -> None:
+    uvloop.install()
+    logger.info("Starting bot watcher...")
+    with contextlib.suppress(KeyboardInterrupt):
+        asyncio.run(runner())
+    logger.info("[red]Bot watcher stopped.[/]")

backend/src/watcher/tokens.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+from utils.env import env
+
+TOKEN_SUFFIX = ".token"  # noqa: S105
+SESSION_SUFFIX = ".session"
+TOKEN_MODE = 0o600
+
+
+def sessions_dir() -> Path:
+    path = Path(env.bots.sessions_dir)
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def bot_id_of(token: str) -> int | None:
+    head, _, secret = token.strip().partition(":")
+    return int(head) if head.isdigit() and secret else None
+
+
+def save(bot_id: int, token: str) -> None:
+    path = sessions_dir() / f"{bot_id}{TOKEN_SUFFIX}"
+    path.write_text(token.strip(), encoding="utf-8")
+    path.chmod(TOKEN_MODE)
+
+
+def load(bot_id: int) -> str | None:
+    path = sessions_dir() / f"{bot_id}{TOKEN_SUFFIX}"
+    return path.read_text(encoding="utf-8").strip() if path.is_file() else None
+
+
+def drop(bot_id: int) -> None:
+    for suffix in (TOKEN_SUFFIX, SESSION_SUFFIX):
+        (sessions_dir() / f"{bot_id}{suffix}").unlink(missing_ok=True)
+
+
+def stored_bot_ids() -> set[int]:
+    return {
+        int(path.stem)
+        for path in sessions_dir().glob(f"*{TOKEN_SUFFIX}")
+        if path.stem.isdigit()
+    }