fix(userbot): encode absent pyrogram raw payloads as empty json

backend/src/userbot/modules/capture/identity.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
 from userbot.modules.capture.chat_meta import chat_kind
 from userbot.modules.profiles.parse import ProfileFields, snapshot_from_high_level
 from userbot.modules.profiles.repository import get_peer, write_profile
+from userbot.modules.raw_json import raw_json
 from utils.policy.models import ChatKind
 
 if TYPE_CHECKING:
@@ -59,7 +60,7 @@ async def _capture_peer(message: Message, ctx: CaptureContext) -> None:
     fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
     if not await ctx.peer_identity.changed(ctx.pool, ctx.account_id, user.id, fields):
         return
-    await write_profile(ctx.pool, ctx.account_id, user.id, fields, str(user))
+    await write_profile(ctx.pool, ctx.account_id, user.id, fields, raw_json(user))
     if photo_file_id and photo_unique_id:
         from userbot.modules.avatars import note_avatar  # noqa: PLC0415
 
@@ -95,7 +96,7 @@ async def _capture_chat(message: Message, ctx: CaptureContext) -> None:
         photo_unique_id,
         None,
         message.date,
-        str(message),
+        raw_json(message),
     )
     if photo_file_id and photo_unique_id:
         from userbot.modules.avatars import note_avatar  # noqa: PLC0415

backend/src/userbot/modules/capture/message.py

@@ -6,6 +6,7 @@ from userbot.modules.capture.context import CaptureContext
 from userbot.modules.links import extract_links
 from userbot.modules.links import repository as links_repository
 from userbot.modules.media import capture_media, self_destruct_ttl
+from userbot.modules.raw_json import raw_json
 from utils.policy.models import CaptureToggles
 
 
@@ -47,7 +48,7 @@ async def capture_message(
         message.date,
         sender_id(message),
         message.text or message.caption,
-        str(message),
+        raw_json(message),
         has_media=message.media is not None,
         is_self_destruct=self_destruct_ttl(message) is not None,
     )
@@ -68,5 +69,5 @@ async def capture_message(
     links = extract_links(message)
     if links:
         await links_repository.insert_links(
-            ctx.pool, ctx.account_id, chat_id, message.id, links, str(message)
+            ctx.pool, ctx.account_id, chat_id, message.id, links, raw_json(message)
         )

backend/src/userbot/modules/groups/service.py

@@ -5,6 +5,7 @@ from userbot.modules.avatars import capture_avatar
 from userbot.modules.capture.context import CaptureContext
 from userbot.modules.capture.message import sender_id
 from userbot.modules.groups import repository
+from userbot.modules.raw_json import raw_json
 
 
 async def capture_service(
@@ -28,7 +29,7 @@ async def capture_service(
             None,
             actor,
             ts,
-            str(message),
+            raw_json(message),
         )
     elif service is enums.MessageServiceType.NEW_CHAT_PHOTO:
         photo = message.new_chat_photo
@@ -43,11 +44,17 @@ async def capture_service(
             unique_id,
             actor,
             ts,
-            str(message),
+            raw_json(message),
         )
         if photo is not None and unique_id is not None:
             await capture_avatar(
-                client, ctx, chat_id, "chat", photo.file_id, unique_id, str(message)
+                client,
+                ctx,
+                chat_id,
+                "chat",
+                photo.file_id,
+                unique_id,
+                raw_json(message),
             )
     elif service is enums.MessageServiceType.DELETE_CHAT_PHOTO:
         await repository.insert_chat_history(
@@ -60,7 +67,7 @@ async def capture_service(
             None,
             actor,
             ts,
-            str(message),
+            raw_json(message),
         )
     elif service is enums.MessageServiceType.NEW_CHAT_MEMBERS:
         for member in message.new_chat_members or []:
@@ -73,7 +80,7 @@ async def capture_service(
                 "join",
                 actor,
                 ts,
-                str(message),
+                raw_json(message),
             )
     elif service is enums.MessageServiceType.LEFT_CHAT_MEMBER:
         member = message.left_chat_member
@@ -87,5 +94,5 @@ async def capture_service(
                 "leave",
                 actor,
                 ts,
-                str(message),
+                raw_json(message),
             )

backend/src/userbot/modules/jobs/handlers/backfill_stories.py

@@ -8,6 +8,7 @@ from userbot.modules.capture.context import CaptureContext
 from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import register
 from userbot.modules.stories.service import save_story
+from utils.logging import logger
 
 SAVE_EVERY = 10
 
@@ -38,6 +39,9 @@ async def _drain(
             raise
         except RPCError:
             continue
+        except Exception:
+            logger.exception(f"Failed to save story {story.id} from {name}")
+            continue
         saved += 1
         if saved % SAVE_EVERY == 0:
             await ctx.report_progress({"saved": saved, "source": name})

backend/src/userbot/modules/jobs/handlers/enrich_chat.py

@@ -11,6 +11,7 @@ from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import register
 from userbot.modules.profiles.parse import snapshot_from_high_level
 from userbot.modules.profiles.repository import write_profile
+from userbot.modules.raw_json import raw_json
 
 MEMBER_CAP = 200
 
@@ -26,7 +27,7 @@ LIMIT 100
 
 async def _save_user(ctx: CaptureContext, user: User) -> None:
     fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
-    await write_profile(ctx.pool, ctx.account_id, user.id, fields, str(user))
+    await write_profile(ctx.pool, ctx.account_id, user.id, fields, raw_json(user))
     if photo_file_id and photo_unique_id:
         await note_avatar(
             ctx.pool, ctx.account_id, user.id, "peer", photo_unique_id, photo_file_id
@@ -48,7 +49,7 @@ async def _enrich_chat_meta(client: Client, ctx: CaptureContext, chat_id: int) -
         photo_unique_id,
         None,
         datetime.now(UTC),
-        str(chat),
+        raw_json(chat),
     )
     if photo_file_id and photo_unique_id:
         await note_avatar(

backend/src/userbot/modules/jobs/handlers/sync_contacts.py

@@ -5,6 +5,7 @@ from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import register
 from userbot.modules.profiles.parse import snapshot_from_high_level
 from userbot.modules.profiles.repository import write_profile
+from userbot.modules.raw_json import raw_json
 
 
 @register("sync_contacts")
@@ -21,7 +22,7 @@ async def sync_contacts(ctx: JobContext) -> None:
         if not isinstance(user, User):
             continue
         fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
-        await write_profile(ctx.pool, ctx.account_id, user.id, fields, str(user))
+        await write_profile(ctx.pool, ctx.account_id, user.id, fields, raw_json(user))
         if photo_file_id and photo_unique_id:
             await note_avatar(
                 ctx.pool,

backend/src/userbot/modules/jobs/handlers/sync_dialogs.py

@@ -9,6 +9,8 @@ from userbot.modules.jobs.registry import register
 from userbot.modules.profiles.parse import snapshot_from_high_level
 from userbot.modules.profiles.repository import write_profile
 from userbot.modules.profiles.snapshots import save_group, save_private
+from userbot.modules.raw_json import raw_json
+from utils.logging import logger
 
 SAVE_EVERY = 100
 USERS_BATCH = 200
@@ -31,7 +33,9 @@ async def _enrich_users(client: Client, ctx: CaptureContext, ids: list[int]) ->
             if not isinstance(user, User):
                 continue
             fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
-            await write_profile(ctx.pool, ctx.account_id, user.id, fields, str(user))
+            await write_profile(
+                ctx.pool, ctx.account_id, user.id, fields, raw_json(user)
+            )
             if photo_file_id and photo_unique_id:
                 await note_avatar(
                     ctx.pool,
@@ -66,6 +70,8 @@ async def sync_dialogs(ctx: JobContext) -> None:
                 await save_group(capture, chat)
         except (BadRequest, Forbidden):
             pass
+        except Exception:
+            logger.exception(f"Failed to snapshot dialog {chat_id}")
         await ctx.pool.execute(_UPSERT_DIALOG, ctx.account_id, chat_id)
         processed += 1
         if processed % SAVE_EVERY == 0:

backend/src/userbot/modules/profiles/snapshots.py

@@ -7,12 +7,13 @@ from userbot.modules.capture.context import CaptureContext
 from userbot.modules.groups.repository import insert_chat_history
 from userbot.modules.profiles.parse import snapshot_from_chat
 from userbot.modules.profiles.repository import write_profile
+from userbot.modules.raw_json import raw_json
 
 
 async def save_private(ctx: CaptureContext, chat: Chat) -> bool:
     chat_id = chat.id or 0
     fields, photo_file_id, photo_unique_id = snapshot_from_chat(chat)
-    await write_profile(ctx.pool, ctx.account_id, chat_id, fields, str(chat))
+    await write_profile(ctx.pool, ctx.account_id, chat_id, fields, raw_json(chat))
     if photo_file_id and photo_unique_id:
         await note_avatar(
             ctx.pool, ctx.account_id, chat_id, "peer", photo_unique_id, photo_file_id
@@ -35,7 +36,7 @@ async def save_group(ctx: CaptureContext, chat: Chat) -> None:
         photo_unique_id,
         None,
         datetime.now(UTC),
-        str(chat),
+        raw_json(chat),
     )
     if photo_file_id and photo_unique_id:
         await note_avatar(

backend/src/userbot/modules/raw_json.py

@@ -0,0 +1,8 @@
+from pyrogram.raw.core import TLObject
+from pyrogram.types import Object
+
+EMPTY_JSON = "{}"
+
+
+def raw_json(obj: Object | TLObject | None) -> str:
+    return EMPTY_JSON if obj is None else str(obj)

backend/src/userbot/modules/stories/service.py

@@ -3,6 +3,7 @@ from pyrogram.types import Story
 
 from userbot.modules.capture.context import CaptureContext
 from userbot.modules.download import download_bytes
+from userbot.modules.raw_json import raw_json
 from userbot.modules.stories import repository
 
 
@@ -40,7 +41,7 @@ async def save_story(client: Client, capture: CaptureContext, story: Story) -> N
         storage_key,
         file_size,
         story.views,
-        str(story.raw),
+        raw_json(story.raw),
         pinned=bool(story.pinned),
         deleted=bool(story.deleted),
         downloaded=downloaded,