feat: add backfills logic

backend/src/userbot/modules/capture/__init__.py

@@ -1,3 +1,4 @@
 from userbot.modules.capture.context import CaptureContext, build_capture_context
+from userbot.modules.capture.message import capture_message
 
-__all__ = ["CaptureContext", "build_capture_context"]
+__all__ = ["CaptureContext", "build_capture_context", "capture_message"]

backend/src/userbot/modules/capture/message.py

@@ -0,0 +1,57 @@
+from pyrogram import Client
+from pyrogram.types import Message
+
+from userbot.modules.capture import repository
+from userbot.modules.capture.context import CaptureContext
+from userbot.modules.media import capture_media, self_destruct_ttl
+from utils.policy.models import CaptureToggles
+
+
+def sender_id(message: Message) -> int | None:
+    if message.from_user is not None:
+        return message.from_user.id
+    if message.sender_chat is not None:
+        return message.sender_chat.id
+    return None
+
+
+def callbacks(message: Message) -> list[tuple[int, str | None, bytes | None]]:
+    rows = getattr(message.reply_markup, "inline_keyboard", None)
+    if not rows:
+        return []
+    buttons: list[tuple[int, str | None, bytes | None]] = []
+    position = 0
+    for row in rows:
+        for button in row:
+            data = button.callback_data
+            if data is not None:
+                encoded = data.encode() if isinstance(data, str) else data
+                buttons.append((position, button.text, encoded))
+            position += 1
+    return buttons
+
+
+async def capture_message(
+    client: Client, message: Message, ctx: CaptureContext, toggles: CaptureToggles
+) -> None:
+    if message.empty or message.chat is None or message.date is None:
+        return
+    chat_id = message.chat.id or 0
+    await repository.upsert_message(
+        ctx.pool,
+        ctx.account_id,
+        chat_id,
+        message.id,
+        message.date,
+        sender_id(message),
+        message.text or message.caption,
+        str(message),
+        has_media=message.media is not None,
+        is_self_destruct=self_destruct_ttl(message) is not None,
+    )
+    await capture_media(client, message, ctx, chat_id, message.id, toggles)
+    buttons = callbacks(message)
+    if buttons:
+        await repository.insert_callbacks(
+            ctx.pool, ctx.account_id, chat_id, message.id, buttons
+        )

backend/src/userbot/modules/jobs/__init__.py

@@ -1,3 +1,4 @@
+from userbot.modules.jobs import handlers
 from userbot.modules.jobs.consumer import JOBS_CHANGED_CHANNEL, JobConsumer
 from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import JOB_HANDLERS, JobHandler, register
@@ -8,5 +9,6 @@ __all__ = [
     "JobConsumer",
     "JobContext",
     "JobHandler",
+    "handlers",
     "register",
 ]

backend/src/userbot/modules/jobs/handlers/__init__.py

@@ -0,0 +1,3 @@
+from userbot.modules.jobs.handlers import backfill, fetch_media
+
+__all__ = ["backfill", "fetch_media"]

backend/src/userbot/modules/jobs/handlers/backfill.py

@@ -0,0 +1,33 @@
+from userbot.modules.capture import capture_message
+from userbot.modules.jobs.context import JobContext
+from userbot.modules.jobs.registry import register
+from utils.policy.models import CaptureToggles
+
+SAVE_EVERY = 100
+
+
+@register("backfill")
+async def backfill(ctx: JobContext) -> None:
+    client = ctx.client
+    if client is None:
+        return
+    capture = getattr(client, "capture", None)
+    if capture is None:
+        return
+    chat_id = ctx.job.params["chat_id"]
+    toggles = CaptureToggles(
+        messages=True,
+        media=bool(ctx.job.params.get("media")),
+        self_destruct_media=False,
+    )
+    max_id = (ctx.job.cursor or {}).get("max_id", 0)
+    processed = ctx.job.progress.get("processed", 0)
+    kwargs = {"max_id": max_id} if max_id else {}
+    async for message in client.get_chat_history(chat_id, **kwargs):
+        await capture_message(client, message, capture, toggles)
+        processed += 1
+        if processed % SAVE_EVERY == 0:
+            next_max = message.id - 1
+            await ctx.save_cursor({"max_id": next_max})
+            await ctx.report_progress({"processed": processed, "max_id": next_max})
+    await ctx.report_progress({"processed": processed, "done": True})

backend/src/userbot/modules/jobs/handlers/fetch_media.py

@@ -0,0 +1,23 @@
+from userbot.modules.jobs.context import JobContext
+from userbot.modules.jobs.registry import register
+from userbot.modules.media import capture_media
+from utils.policy.models import CaptureToggles
+
+
+@register("fetch_media")
+async def fetch_media(ctx: JobContext) -> None:
+    client = ctx.client
+    if client is None:
+        return
+    capture = getattr(client, "capture", None)
+    if capture is None:
+        return
+    chat_id = ctx.job.params["chat_id"]
+    message_id = ctx.job.params["message_id"]
+    message = await client.get_messages(chat_id, message_id)
+    if isinstance(message, list):
+        message = message[0] if message else None
+    if message is None or message.empty:
+        return
+    toggles = CaptureToggles(media=True, self_destruct_media=True)
+    await capture_media(client, message, capture, chat_id, message_id, toggles)