feat: web UI chat render, panels, presence + analytics

backend/src/userbot/handlers/deletes.py

@@ -3,6 +3,7 @@ from pyrogram.types import Message
 from userbot import PyroClient
 from userbot.modules.capture import repository
 from userbot.modules.capture.repository import CHANNEL_ID_THRESHOLD
+from utils.events import notify_bg_event
 
 
 @PyroClient.on_deleted_messages()
@@ -21,8 +22,12 @@ async def on_deleted_messages(client: PyroClient, messages: list[Message]) -> No
             channels.setdefault(chat_id, []).append(message.id)
     if box:
         await repository.mark_deleted_box(ctx.pool, ctx.account_id, box)
+        await notify_bg_event(ctx.pool, "delete", ctx.account_id, message_ids=box)
     for chat_id, ids in channels.items():
         await repository.mark_deleted_channel(ctx.pool, ctx.account_id, chat_id, ids)
+        await notify_bg_event(
+            ctx.pool, "delete", ctx.account_id, chat_id=chat_id, message_ids=ids
+        )
 
 
 handlers = on_deleted_messages.handlers

backend/src/userbot/handlers/edits.py

@@ -5,6 +5,7 @@ from userbot.modules.capture import repository
 from userbot.modules.capture.chat_meta import meta_from_chat
 from userbot.modules.capture.message import sender_id
 from userbot.modules.media import capture_media, media_unique_id, self_destruct_ttl
+from utils.events import notify_bg_event
 
 
 @PyroClient.on_edited_message()
@@ -32,8 +33,13 @@ async def on_edited_message(client: PyroClient, message: Message) -> None:
         has_media=message.media is not None,
         is_self_destruct=self_destruct_ttl(message) is not None,
     )
-    if changed and message.media is not None:
+    if not changed:
+        return
+    if message.media is not None:
         await capture_media(client, message, ctx, chat_id, message.id, toggles)
+    await notify_bg_event(
+        ctx.pool, "edit", ctx.account_id, chat_id=chat_id, message_id=message.id
+    )
 
 
 handlers = on_edited_message.handlers

backend/src/userbot/handlers/messages.py

@@ -5,6 +5,7 @@ from userbot.modules.capture import capture_message
 from userbot.modules.capture.chat_meta import meta_from_chat
 from userbot.modules.stt import is_transcribable
 from userbot.modules.stt.gate import safe_transcribe
+from utils.events import notify_bg_event
 
 
 @PyroClient.on_message()
@@ -18,6 +19,9 @@ async def on_message(client: PyroClient, message: Message) -> None:
     if not toggles.messages:
         return
     await capture_message(client, message, ctx, toggles)
+    await notify_bg_event(
+        ctx.pool, "message", ctx.account_id, chat_id=meta.chat_id, message_id=message.id
+    )
     if (
         toggles.stt
         and is_transcribable(message)

backend/src/userbot/handlers/presence.py

@@ -4,6 +4,7 @@ from pyrogram.types import User
 
 from userbot import PyroClient
 from userbot.modules.presence import repository
+from utils.events import notify_bg_event
 
 
 @PyroClient.on_user_status()
@@ -22,6 +23,7 @@ async def on_user_status(client: PyroClient, user: User) -> None:
         str(user.raw),
     )
     await ctx.watches.on_status(user.id, is_online=user.status.name.lower() == "online")
+    await notify_bg_event(ctx.pool, "presence", ctx.account_id, chat_id=user.id)
 
 
 handlers = on_user_status.handlers

backend/src/userbot/handlers/raw/reactions.py

@@ -3,6 +3,7 @@ from pyrogram import raw, utils
 from userbot import PyroClient
 from userbot.modules.capture import repository
 from userbot.modules.capture.chat_meta import meta_from_peer
+from utils.events import notify_bg_event
 
 HANDLES = (raw.types.UpdateMessageReactions,)
 
@@ -47,3 +48,10 @@ async def handle(
     await repository.sync_reactions(
         ctx.pool, ctx.account_id, meta.chat_id, update.msg_id, current
     )
+    await notify_bg_event(
+        ctx.pool,
+        "reaction",
+        ctx.account_id,
+        chat_id=meta.chat_id,
+        message_id=update.msg_id,
+    )

backend/src/userbot/handlers/raw/read_receipts.py

@@ -4,6 +4,7 @@ from pyrogram import raw, utils
 
 from userbot import PyroClient
 from userbot.modules.read_receipts import repository
+from utils.events import notify_bg_event
 
 HANDLES = (raw.types.UpdateReadHistoryOutbox,)
 
@@ -24,3 +25,6 @@ async def handle(
         update.max_id,
         str(update),
     )
+    await notify_bg_event(
+        ctx.pool, "receipt", ctx.account_id, chat_id=chat_id, message_id=update.max_id
+    )

backend/src/userbot/modules/custom_emoji/repository.py

@@ -0,0 +1,34 @@
+import asyncpg
+
+_GET = """
+SELECT downloaded FROM custom_emoji WHERE custom_emoji_id = $1
+"""
+
+_UPSERT_DOWNLOADED = """
+INSERT INTO custom_emoji
+    (custom_emoji_id, storage_key, file_size, mime, kind, downloaded, raw)
+VALUES ($1, $2, $3, $4, $5, true, '{}'::jsonb)
+ON CONFLICT (custom_emoji_id) DO UPDATE SET
+    storage_key = EXCLUDED.storage_key,
+    file_size = EXCLUDED.file_size,
+    mime = EXCLUDED.mime,
+    kind = EXCLUDED.kind,
+    downloaded = true
+"""
+
+
+async def is_downloaded(pool: asyncpg.Pool, custom_emoji_id: int) -> bool:
+    return bool(await pool.fetchval(_GET, custom_emoji_id))
+
+
+async def upsert_downloaded(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    custom_emoji_id: int,
+    storage_key: str,
+    file_size: int | None,
+    mime: str | None,
+    kind: str,
+) -> None:
+    await pool.execute(
+        _UPSERT_DOWNLOADED, custom_emoji_id, storage_key, file_size, mime, kind
+    )

backend/src/userbot/modules/jobs/handlers/__init__.py

@@ -2,8 +2,18 @@ from userbot.modules.jobs.handlers import (
     backfill,
     enrich_chat,
     fetch_avatar,
+    fetch_custom_emoji,
     fetch_media,
+    sync_dialogs,
     transcribe,
 )
 
-__all__ = ["backfill", "enrich_chat", "fetch_avatar", "fetch_media", "transcribe"]
+__all__ = [
+    "backfill",
+    "enrich_chat",
+    "fetch_avatar",
+    "fetch_custom_emoji",
+    "fetch_media",
+    "sync_dialogs",
+    "transcribe",
+]

backend/src/userbot/modules/jobs/handlers/backfill.py

@@ -1,3 +1,5 @@
+from pyrogram.errors import PeerIdInvalid
+
 from userbot.modules.capture import capture_message
 from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import register
@@ -23,11 +25,15 @@ async def backfill(ctx: JobContext) -> None:
     max_id = (ctx.job.cursor or {}).get("max_id", 0)
     processed = ctx.job.progress.get("processed", 0)
     kwargs = {"max_id": max_id} if max_id else {}
-    async for message in client.get_chat_history(chat_id, **kwargs):
-        await capture_message(client, message, capture, toggles)
-        processed += 1
-        if processed % SAVE_EVERY == 0:
-            next_max = message.id - 1
-            await ctx.save_cursor({"max_id": next_max})
-            await ctx.report_progress({"processed": processed, "max_id": next_max})
+    try:
+        async for message in client.get_chat_history(chat_id, **kwargs):
+            await capture_message(client, message, capture, toggles)
+            processed += 1
+            if processed % SAVE_EVERY == 0:
+                next_max = message.id - 1
+                await ctx.save_cursor({"max_id": next_max})
+                await ctx.report_progress({"processed": processed, "max_id": next_max})
+    except PeerIdInvalid:
+        await ctx.report_progress({"processed": processed, "error": "peer_id_invalid"})
+        return
     await ctx.report_progress({"processed": processed, "done": True})

backend/src/userbot/modules/jobs/handlers/fetch_custom_emoji.py

@@ -0,0 +1,45 @@
+from io import BytesIO
+
+from pyrogram.types import Sticker
+
+from userbot.modules.custom_emoji.repository import is_downloaded, upsert_downloaded
+from userbot.modules.jobs.context import JobContext
+from userbot.modules.jobs.registry import register
+
+
+def _kind(sticker: Sticker) -> str:
+    if sticker.is_animated:
+        return "animated"
+    if sticker.is_video:
+        return "video"
+    return "static"
+
+
+@register("fetch_custom_emoji")
+async def fetch_custom_emoji(ctx: JobContext) -> None:
+    client = ctx.client
+    if client is None:
+        return
+    capture = getattr(client, "capture", None)
+    if capture is None:
+        return
+    custom_emoji_id = int(ctx.job.params["custom_emoji_id"])
+    if await is_downloaded(ctx.pool, custom_emoji_id):
+        return
+    stickers = await client.get_custom_emoji_stickers([str(custom_emoji_id)])
+    if not stickers:
+        return
+    sticker = stickers[0]
+    buffer = await client.download_media(sticker.file_id, in_memory=True)
+    if not isinstance(buffer, BytesIO):
+        return
+    data = buffer.getvalue()
+    storage_key = capture.storage.put(data)
+    await upsert_downloaded(
+        ctx.pool,
+        custom_emoji_id,
+        storage_key,
+        len(data),
+        sticker.mime_type,
+        _kind(sticker),
+    )

backend/src/userbot/modules/jobs/handlers/sync_dialogs.py

@@ -0,0 +1,108 @@
+from datetime import UTC, datetime
+
+from pyrogram import Client
+from pyrogram.errors import BadRequest, Forbidden
+from pyrogram.types import Chat, User
+
+from userbot.modules.avatars import note_avatar
+from userbot.modules.capture.context import CaptureContext
+from userbot.modules.groups.repository import insert_chat_history
+from userbot.modules.jobs.context import JobContext
+from userbot.modules.jobs.registry import register
+from userbot.modules.profiles.parse import snapshot_from_chat, snapshot_from_high_level
+from userbot.modules.profiles.repository import write_profile
+
+SAVE_EVERY = 100
+USERS_BATCH = 200
+
+_UPSERT_DIALOG = """
+INSERT INTO dialogs (account_id, chat_id) VALUES ($1, $2)
+ON CONFLICT (account_id, chat_id) DO UPDATE SET updated_at = now()
+"""
+
+
+async def _save_private(ctx: CaptureContext, chat: Chat, chat_id: int) -> bool:
+    fields, photo_file_id, photo_unique_id = snapshot_from_chat(chat)
+    await write_profile(ctx.pool, ctx.account_id, chat_id, fields, str(chat))
+    if photo_file_id and photo_unique_id:
+        await note_avatar(
+            ctx.pool, ctx.account_id, chat_id, "peer", photo_unique_id, photo_file_id
+        )
+    return bool(fields.first_name or fields.last_name or fields.username)
+
+
+async def _enrich_users(client: Client, ctx: CaptureContext, ids: list[int]) -> None:
+    for start in range(0, len(ids), USERS_BATCH):
+        batch = ids[start : start + USERS_BATCH]
+        try:
+            result = await client.get_users(batch)
+        except (BadRequest, Forbidden):
+            continue
+        users = result if isinstance(result, list) else [result]
+        for user in users:
+            if not isinstance(user, User):
+                continue
+            fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
+            await write_profile(ctx.pool, ctx.account_id, user.id, fields, str(user))
+            if photo_file_id and photo_unique_id:
+                await note_avatar(
+                    ctx.pool,
+                    ctx.account_id,
+                    user.id,
+                    "peer",
+                    photo_unique_id,
+                    photo_file_id,
+                )
+
+
+async def _save_group(ctx: CaptureContext, chat: Chat, chat_id: int) -> None:
+    photo = chat.photo
+    photo_unique_id = photo.big_photo_unique_id if photo else None
+    photo_file_id = photo.big_file_id if photo else None
+    await insert_chat_history(
+        ctx.pool,
+        ctx.account_id,
+        chat_id,
+        0,
+        "meta",
+        chat.title,
+        photo_unique_id,
+        None,
+        datetime.now(UTC),
+        str(chat),
+    )
+    if photo_file_id and photo_unique_id:
+        await note_avatar(
+            ctx.pool, ctx.account_id, chat_id, "chat", photo_unique_id, photo_file_id
+        )
+
+
+@register("sync_dialogs")
+async def sync_dialogs(ctx: JobContext) -> None:
+    client = ctx.client
+    if client is None:
+        return
+    capture = getattr(client, "capture", None)
+    if capture is None:
+        return
+    processed = ctx.job.progress.get("processed", 0)
+    nameless: list[int] = []
+    async for dialog in client.get_dialogs():
+        chat = dialog.chat
+        if chat is None or chat.id is None:
+            continue
+        chat_id = chat.id
+        try:
+            if chat_id > 0:
+                if not await _save_private(capture, chat, chat_id):
+                    nameless.append(chat_id)
+            else:
+                await _save_group(capture, chat, chat_id)
+        except (BadRequest, Forbidden):
+            pass
+        await ctx.pool.execute(_UPSERT_DIALOG, ctx.account_id, chat_id)
+        processed += 1
+        if processed % SAVE_EVERY == 0:
+            await ctx.report_progress({"processed": processed})
+    await _enrich_users(client, capture, nameless)
+    await ctx.report_progress({"processed": processed, "done": True})

backend/src/userbot/modules/media/downloader.py

@@ -20,11 +20,20 @@ _MEDIA_ATTRS = (
 )
 
 
+_WEB_PAGE_ATTRS = ("photo", "video", "animation", "document", "audio")
+
+
 def media_object(message: Message) -> tuple[str | None, Any]:
     for attr in _MEDIA_ATTRS:
         obj = getattr(message, attr, None)
         if obj is not None:
             return attr, obj
+    web_page = getattr(message, "web_page", None)
+    if web_page is not None:
+        for attr in _WEB_PAGE_ATTRS:
+            obj = getattr(web_page, attr, None)
+            if obj is not None:
+                return attr, obj
     return None, None
 
 
@@ -70,7 +79,8 @@ async def capture_media(  # noqa: PLR0913
             file_size = existing["file_size"]
             downloaded = True
         else:
-            buffer = await client.download_media(message, in_memory=True)
+            target = message if getattr(message, kind or "", None) is obj else obj
+            buffer = await client.download_media(target, in_memory=True)
             if isinstance(buffer, BytesIO):
                 data = buffer.getvalue()
                 storage_key = ctx.storage.put(data)

backend/src/userbot/modules/profiles/parse.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 
 from pyrogram import Client, raw
-from pyrogram.types import User
+from pyrogram.types import Chat, User
 
 
 @dataclass(frozen=True)
@@ -54,3 +54,18 @@ def snapshot_from_high_level(
         is_deleted_account=bool(user.is_deleted),
     )
     return fields, photo_file_id, photo_unique_id
+
+
+def snapshot_from_chat(chat: Chat) -> tuple[ProfileFields, str | None, str | None]:
+    photo = chat.photo
+    photo_unique_id = photo.big_photo_unique_id if photo else None
+    photo_file_id = photo.big_file_id if photo else None
+    fields = ProfileFields(
+        first_name=chat.first_name,
+        last_name=chat.last_name,
+        username=chat.username,
+        phone=None,
+        photo_unique_id=photo_unique_id,
+        is_deleted_account=False,
+    )
+    return fields, photo_file_id, photo_unique_id

backend/src/userbot/runner.py

@@ -13,6 +13,7 @@ from userbot import PyroClient
 from userbot.modules.capture import CaptureContext, build_capture_context
 from userbot.modules.jobs import JobConsumer
 from utils.env import env
+from utils.jobs import enqueue
 from utils.logging import logger, setup_logging
 from utils.read.watches import WATCHES_CHANGED_CHANNEL
 from utils.storage import ContentAddressedStorage
@@ -72,6 +73,17 @@ async def _setup_capture(
     logger.info("[green]Capture context ready.[/]")
 
 
+async def _enqueue_sync_dialogs(pool: asyncpg.Pool, account_id: int) -> None:
+    existing = await pool.fetchval(
+        "SELECT 1 FROM jobs WHERE account_id = $1 AND kind = 'sync_dialogs' "
+        "AND status IN ('pending', 'running') LIMIT 1",
+        account_id,
+    )
+    if existing is None:
+        await enqueue(pool, account_id, "sync_dialogs", {})
+        logger.info("[green]Queued sync_dialogs.[/]")
+
+
 async def _listen_changes(
     clients: list[PyroClient], tasks: set[asyncio.Task]
 ) -> asyncpg.Connection:
@@ -134,6 +146,7 @@ async def runner() -> None:
                 await _setup_capture(pool, client, account_id, storage)
                 consumer = JobConsumer(client, pool, account_id)
                 consumer_tasks.append(asyncio.create_task(consumer.run()))
+                await _enqueue_sync_dialogs(pool, account_id)
 
         if clients:
             listen_conn = await _listen_changes(clients, reload_tasks)