feat(backend): read time of my messages from the archive, exact dates and group readers asked from telegram on demand

backend/src/userbot/modules/jobs/handlers/__init__.py

@@ -5,6 +5,7 @@ from userbot.modules.jobs.handlers import (
     fetch_avatar,
     fetch_custom_emoji,
     fetch_media,
+    fetch_read_info,
     search_peers,
     sync_contacts,
     sync_dialogs,
@@ -18,6 +19,7 @@ __all__ = [
     "fetch_avatar",
     "fetch_custom_emoji",
     "fetch_media",
+    "fetch_read_info",
     "search_peers",
     "sync_contacts",
     "sync_dialogs",

backend/src/userbot/modules/jobs/handlers/fetch_read_info.py

@@ -0,0 +1,124 @@
+from datetime import UTC, datetime, timedelta
+
+from pyrogram import Client, raw
+from pyrogram.errors import (
+    ChannelInvalid,
+    ChannelPrivate,
+    ChatTooBig,
+    FloodPremiumWait,
+    FloodWait,
+    MessageIdInvalid,
+    MessageNotReadYet,
+    MessageTooOld,
+    MsgIdInvalid,
+    MsgTooOld,
+    PeerIdInvalid,
+    RPCError,
+    UserPrivacyRestricted,
+    YourPrivacyRestricted,
+)
+
+from userbot.modules.jobs.context import JobContext
+from userbot.modules.jobs.registry import register
+from userbot.modules.read_receipts.repository import save_fetch, save_reads
+from utils.jobs import enqueue
+from utils.logging import logger
+from utils.policy.models import ChatKind
+from utils.read.peers import get_peers
+from utils.read.read_receipts import read_info
+
+RETRY = timedelta(seconds=60)
+LATER = timedelta(days=1)
+
+Refusal = tuple[tuple[type[Exception], ...], str, timedelta]
+
+REFUSALS: tuple[Refusal, ...] = (
+    ((MessageNotReadYet,), "not_read_yet", RETRY),
+    ((UserPrivacyRestricted,), "user_privacy", RETRY),
+    ((YourPrivacyRestricted,), "your_privacy", LATER),
+    ((ChatTooBig,), "chat_too_big", LATER),
+    ((MsgTooOld, MessageTooOld), "too_old", LATER),
+    (
+        (
+            PeerIdInvalid,
+            ChannelInvalid,
+            ChannelPrivate,
+            MessageIdInvalid,
+            MsgIdInvalid,
+            KeyError,
+            ValueError,
+        ),
+        "unavailable",
+        RETRY,
+    ),
+)
+
+
+def _refusal(exc: Exception) -> tuple[str, timedelta]:
+    for kinds, outcome, retry in REFUSALS:
+        if isinstance(exc, kinds):
+            return outcome, retry
+    return "error", RETRY
+
+
+def _at(timestamp: int) -> datetime:
+    return datetime.fromtimestamp(timestamp, UTC)
+
+
+async def ask_telegram(
+    client: Client, chat_id: int, message_id: int, *, group: bool
+) -> list[tuple[int, datetime]]:
+    peer = await client.resolve_peer(chat_id)
+    if peer is None:
+        msg = f"unknown peer {chat_id}"
+        raise ValueError(msg)
+    if group:
+        participants = await client.invoke(
+            raw.functions.messages.GetMessageReadParticipants(
+                peer=peer, msg_id=message_id
+            )
+        )
+        return [(item.user_id, _at(item.date)) for item in participants]
+    outbox = await client.invoke(
+        raw.functions.messages.GetOutboxReadDate(peer=peer, msg_id=message_id)
+    )
+    return [(chat_id, _at(outbox.date))]
+
+
+async def _enrich_unknown(ctx: JobContext, chat_id: int, reader_ids: list[int]) -> None:
+    known = {
+        peer.peer_id for peer in await get_peers(ctx.pool, ctx.account_id, reader_ids)
+    }
+    if set(reader_ids) - known:
+        await enqueue(ctx.pool, ctx.account_id, "enrich_chat", {"chat_id": chat_id})
+
+
+@register("fetch_read_info")
+async def fetch_read_info(ctx: JobContext) -> None:
+    client = ctx.client
+    if client is None:
+        return
+    chat_id = ctx.job.params["chat_id"]
+    message_id = ctx.job.params["message_id"]
+    info = await read_info(ctx.pool, ctx.account_id, chat_id, message_id)
+    if info is None or not info.can_fetch:
+        return
+    group = info.chat_kind == ChatKind.GROUP
+    try:
+        reads = await ask_telegram(client, chat_id, message_id, group=group)
+    except (FloodWait, FloodPremiumWait) as exc:
+        wait = exc.value if isinstance(exc.value, int) else 0
+        retry = max(RETRY, timedelta(seconds=wait))
+        await save_fetch(ctx.pool, ctx.account_id, chat_id, message_id, "flood", retry)
+        return
+    except (RPCError, KeyError, ValueError) as exc:
+        outcome, retry = _refusal(exc)
+        if outcome == "error":
+            logger.warning(f"[yellow]Read info {chat_id}/{message_id}: {exc!r}[/]")
+        await save_fetch(ctx.pool, ctx.account_id, chat_id, message_id, outcome, retry)
+        return
+    source = "participants" if group else "outbox_date"
+    await save_reads(ctx.pool, ctx.account_id, chat_id, message_id, source, reads)
+    await save_fetch(ctx.pool, ctx.account_id, chat_id, message_id, "ok", RETRY)
+    if group and reads:
+        await _enrich_unknown(ctx, chat_id, [reader_id for reader_id, _ in reads])

backend/src/userbot/modules/read_receipts/repository.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import asyncpg
 
@@ -9,6 +9,26 @@ VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
 ON CONFLICT (account_id, chat_id, message_id, ts) DO NOTHING
 """
 
+_UPSERT_READ = """
+INSERT INTO message_reads
+    (account_id, chat_id, message_id, reader_id, read_at, source, fetched_at)
+VALUES ($1, $2, $3, $4, $5, $6, now())
+ON CONFLICT (account_id, chat_id, message_id, reader_id) DO UPDATE SET
+    read_at = EXCLUDED.read_at,
+    source = EXCLUDED.source,
+    fetched_at = EXCLUDED.fetched_at
+"""
+
+_UPSERT_FETCH = """
+INSERT INTO read_fetches
+    (account_id, chat_id, message_id, outcome, fetched_at, retry_at)
+VALUES ($1, $2, $3, $4, now(), now() + $5::interval)
+ON CONFLICT (account_id, chat_id, message_id) DO UPDATE SET
+    outcome = EXCLUDED.outcome,
+    fetched_at = EXCLUDED.fetched_at,
+    retry_at = EXCLUDED.retry_at
+"""
+
 
 async def insert_read(  # noqa: PLR0913
     pool: asyncpg.Pool,
@@ -34,3 +54,33 @@ async def insert_listened(  # noqa: PLR0913
     await pool.execute(
         _INSERT, account_id, chat_id, message_id, ts, reader_id, "listened", raw
     )
+
+
+async def save_reads(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    account_id: int,
+    chat_id: int,
+    message_id: int,
+    source: str,
+    reads: list[tuple[int, datetime]],
+) -> None:
+    await pool.executemany(
+        _UPSERT_READ,
+        [
+            (account_id, chat_id, message_id, reader_id, read_at, source)
+            for reader_id, read_at in reads
+        ],
+    )
+
+
+async def save_fetch(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    account_id: int,
+    chat_id: int,
+    message_id: int,
+    outcome: str,
+    retry_after: timedelta,
+) -> None:
+    await pool.execute(
+        _UPSERT_FETCH, account_id, chat_id, message_id, outcome, retry_after
+    )