feat: 1-to-1 message render + web data-lake backend

This commit is contained in:
h
2026-05-31 01:27:40 +02:00
parent f0afb7ec5b
commit 75425d1bee
110 changed files with 10199 additions and 54 deletions
@@ -0,0 +1,27 @@
"""media unique_id for edit detection
Revision ID: a9c3e7f1d2b4
Revises: f7a2c9e1b3d5
Create Date: 2026-05-30 04:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "a9c3e7f1d2b4"
down_revision: str | None = "f7a2c9e1b3d5"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.add_column("media", sa.Column("unique_id", sa.String(), nullable=True))
op.add_column("media_versions", sa.Column("unique_id", sa.String(), nullable=True))
def downgrade() -> None:
op.drop_column("media_versions", "unique_id")
op.drop_column("media", "unique_id")
@@ -0,0 +1,56 @@
"""media versions (keep every edited media)
Revision ID: f7a2c9e1b3d5
Revises: a3f1c8e94d72
Create Date: 2026-05-30 03:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "f7a2c9e1b3d5"
down_revision: str | None = "a3f1c8e94d72"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"media_versions",
sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False),
sa.Column("account_id", sa.Integer(), nullable=False),
sa.Column("chat_id", sa.BigInteger(), nullable=False),
sa.Column("message_id", sa.BigInteger(), nullable=False),
sa.Column(
"observed_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.Column("kind", sa.String(), nullable=False),
sa.Column("storage_key", sa.String(), nullable=False),
sa.Column("file_size", sa.BigInteger(), nullable=True),
sa.Column("mime", sa.String(), nullable=True),
sa.Column("ttl_seconds", sa.Integer(), nullable=True),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint(
"account_id",
"chat_id",
"message_id",
"storage_key",
name="uq_media_versions_content",
),
)
op.create_index(
"ix_media_versions_message",
"media_versions",
["account_id", "chat_id", "message_id", "observed_at"],
)
def downgrade() -> None:
op.drop_index("ix_media_versions_message", table_name="media_versions")
op.drop_table("media_versions")
+18
View File
@@ -1,16 +1,20 @@
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from pathlib import Path
import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka, setup_dishka
from fastapi import FastAPI
from fastapi.responses import FileResponse
from fastmcp.utilities.lifespan import combine_lifespans
from starlette.applications import Starlette
from api.auth import BearerAuthMiddleware
from api.mcp.server import mcp
from api.routers import (
accounts,
annotations,
avatars,
backfill,
chats,
folders,
@@ -54,12 +58,14 @@ async def health(pool: FromDishka[asyncpg.Pool]) -> dict[str, bool]:
return {"db": db_ok, "timescaledb": bool(timescale_ok)}
app.include_router(accounts.router)
app.include_router(policy.router)
app.include_router(folders.router)
app.include_router(backfill.router)
app.include_router(search.router)
app.include_router(chats.router)
app.include_router(media.router)
app.include_router(avatars.router)
app.include_router(social.router)
app.include_router(presence.router)
app.include_router(peers.router)
@@ -68,6 +74,18 @@ app.include_router(watches.router)
app.mount("/mcp", mcp_app)
_spa_dir = Path(env.api.static_dir).resolve()
if _spa_dir.is_dir():
_spa_index = _spa_dir / "index.html"
@app.get("/{spa_path:path}")
async def serve_spa(spa_path: str) -> FileResponse:
candidate = (_spa_dir / spa_path).resolve()
if spa_path and candidate.is_relative_to(_spa_dir) and candidate.is_file():
return FileResponse(candidate)
return FileResponse(_spa_index)
app.add_middleware(BearerAuthMiddleware, token=_token)
setup_dishka(container, app)
+13
View File
@@ -0,0 +1,13 @@
import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter
from utils.read import accounts
from utils.read.models import AccountView
router = APIRouter(prefix="/api", tags=["accounts"], route_class=DishkaRoute)
@router.get("/accounts")
async def list_accounts(pool: FromDishka[asyncpg.Pool]) -> list[AccountView]:
return await accounts.list_accounts(pool)
+40
View File
@@ -0,0 +1,40 @@
from typing import Annotated
import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, HTTPException, Query
from fastapi.responses import FileResponse
from utils.jobs import enqueue
from utils.read.avatars import current_avatar
from utils.storage import ContentAddressedStorage
router = APIRouter(prefix="/api/avatars", tags=["avatars"], route_class=DishkaRoute)
@router.get("/{owner_kind}/{owner_id}")
async def serve_avatar(
pool: FromDishka[asyncpg.Pool],
storage: FromDishka[ContentAddressedStorage],
owner_kind: str,
owner_id: int,
account_id: Annotated[int, Query()],
) -> FileResponse:
avatar = await current_avatar(pool, account_id, owner_kind, owner_id)
if avatar is None:
raise HTTPException(status_code=404, detail="avatar not found")
if not avatar.downloaded or avatar.storage_key is None:
await enqueue(
pool,
account_id,
"fetch_avatar",
{
"owner_kind": owner_kind,
"owner_id": owner_id,
"unique_id": avatar.unique_id,
},
)
raise HTTPException(status_code=409, detail="avatar not downloaded; fetching")
return FileResponse(
storage.url(avatar.storage_key), media_type=avatar.mime or "image/jpeg"
)
+15
View File
@@ -3,7 +3,9 @@ from typing import Annotated
import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, Query
from pydantic import BaseModel
from utils.jobs import enqueue
from utils.read import chats
from utils.read.models import (
DEFAULT_LIMIT,
@@ -15,6 +17,11 @@ from utils.read.models import (
router = APIRouter(prefix="/api", tags=["chats"], route_class=DishkaRoute)
class EnrichRequest(BaseModel):
account_id: int
AccountId = Annotated[int, Query()]
Limit = Annotated[int, Query()]
Offset = Annotated[int, Query()]
@@ -48,6 +55,14 @@ async def chat_history(
)
@router.post("/chats/{chat_id}/enrich")
async def enrich_chat(
pool: FromDishka[asyncpg.Pool], chat_id: int, body: EnrichRequest
) -> dict[str, int]:
job_id = await enqueue(pool, body.account_id, "enrich_chat", {"chat_id": chat_id})
return {"job_id": job_id}
@router.get("/chats/{chat_id}/messages/{message_id}/versions")
async def message_versions(
pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, account_id: AccountId
+48 -3
View File
@@ -1,10 +1,17 @@
from typing import Annotated
import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, HTTPException
from fastapi import APIRouter, HTTPException, Query
from fastapi.responses import FileResponse
from utils.read.media import get_media
from utils.read.models import MediaView
from utils.read.media import (
get_media,
get_media_version,
get_media_versions,
get_message_media,
)
from utils.read.models import MediaVersionView, MediaView
from utils.storage import ContentAddressedStorage
router = APIRouter(prefix="/api/media", tags=["media"], route_class=DishkaRoute)
@@ -18,6 +25,44 @@ async def media_meta(pool: FromDishka[asyncpg.Pool], media_id: int) -> MediaView
return media
@router.get("/versions/{chat_id}/{message_id}")
async def message_media_versions(
pool: FromDishka[asyncpg.Pool],
chat_id: int,
message_id: int,
account_id: Annotated[int, Query()],
) -> list[MediaVersionView]:
return await get_media_versions(pool, account_id, chat_id, message_id)
@router.get("/version/{version_id}")
async def serve_media_version(
pool: FromDishka[asyncpg.Pool],
storage: FromDishka[ContentAddressedStorage],
version_id: int,
) -> FileResponse:
version = await get_media_version(pool, version_id)
if version is None:
raise HTTPException(status_code=404, detail="media version not found")
return FileResponse(
storage.url(version.storage_key),
media_type=version.mime or "application/octet-stream",
)
@router.get("/message/{chat_id}/{message_id}")
async def message_media(
pool: FromDishka[asyncpg.Pool],
chat_id: int,
message_id: int,
account_id: Annotated[int, Query()],
) -> MediaView:
media = await get_message_media(pool, account_id, chat_id, message_id)
if media is None:
raise HTTPException(status_code=404, detail="media not found")
return media
@router.get("/{media_id}")
async def serve_media(
pool: FromDishka[asyncpg.Pool],
+8
View File
@@ -12,6 +12,14 @@ router = APIRouter(prefix="/api", tags=["peers"], route_class=DishkaRoute)
AccountId = Annotated[int, Query()]
@router.get("/peers/batch")
async def get_peers(
pool: FromDishka[asyncpg.Pool], account_id: AccountId, ids: Annotated[str, Query()]
) -> list[PeerView]:
parsed = [int(part) for part in ids.split(",") if part.strip()]
return await peers.get_peers(pool, account_id, parsed)
@router.get("/peers/{peer_id}")
async def get_peer(
pool: FromDishka[asyncpg.Pool], peer_id: int, account_id: AccountId
+5 -2
View File
@@ -4,7 +4,7 @@ from userbot import PyroClient
from userbot.modules.capture import repository
from userbot.modules.capture.chat_meta import meta_from_chat
from userbot.modules.capture.message import sender_id
from userbot.modules.media import self_destruct_ttl
from userbot.modules.media import capture_media, media_unique_id, self_destruct_ttl
@PyroClient.on_edited_message()
@@ -18,7 +18,7 @@ async def on_edited_message(client: PyroClient, message: Message) -> None:
toggles = ctx.resolve(meta)
if not toggles.track_edits_deletes:
return
await repository.add_version(
changed = await repository.add_version(
ctx.pool,
ctx.account_id,
chat_id,
@@ -28,9 +28,12 @@ async def on_edited_message(client: PyroClient, message: Message) -> None:
message.text or message.caption,
str(message),
message.edit_date,
media_unique_id(message),
has_media=message.media is not None,
is_self_destruct=self_destruct_ttl(message) is not None,
)
if changed and message.media is not None:
await capture_media(client, message, ctx, chat_id, message.id, toggles)
handlers = on_edited_message.handlers
@@ -1,3 +1,4 @@
from userbot.modules.avatars.downloader import capture_avatar
from userbot.modules.avatars.repository import note_avatar
__all__ = ["capture_avatar"]
__all__ = ["capture_avatar", "note_avatar"]
@@ -1,3 +1,5 @@
import json
import asyncpg
_INSERT_AVATAR = """
@@ -13,6 +15,16 @@ SELECT 1 FROM avatars
WHERE account_id = $1 AND owner_id = $2 AND unique_id = $3
"""
_GET_FILE = """
SELECT raw ->> 'file_id' AS file_id, downloaded FROM avatars
WHERE account_id = $1 AND owner_kind = $2 AND owner_id = $3 AND unique_id = $4
"""
_MARK_DOWNLOADED = """
UPDATE avatars SET downloaded = true, storage_key = $5, file_size = $6
WHERE account_id = $1 AND owner_kind = $2 AND owner_id = $3 AND unique_id = $4
"""
async def avatar_exists(
pool: asyncpg.Pool, account_id: int, owner_id: int, unique_id: str
@@ -46,3 +58,54 @@ async def insert_avatar( # noqa: PLR0913
downloaded,
raw,
)
async def note_avatar( # noqa: PLR0913
pool: asyncpg.Pool,
account_id: int,
owner_id: int,
owner_kind: str,
unique_id: str,
file_id: str,
) -> None:
await insert_avatar(
pool,
account_id,
owner_id,
owner_kind,
unique_id,
None,
None,
None,
json.dumps({"file_id": file_id}),
downloaded=False,
)
async def get_avatar_file(
pool: asyncpg.Pool, account_id: int, owner_kind: str, owner_id: int, unique_id: str
) -> tuple[str | None, bool] | None:
row = await pool.fetchrow(_GET_FILE, account_id, owner_kind, owner_id, unique_id)
if row is None:
return None
return row["file_id"], row["downloaded"]
async def mark_avatar_downloaded( # noqa: PLR0913
pool: asyncpg.Pool,
account_id: int,
owner_kind: str,
owner_id: int,
unique_id: str,
storage_key: str,
file_size: int,
) -> None:
await pool.execute(
_MARK_DOWNLOADED,
account_id,
owner_kind,
owner_id,
unique_id,
storage_key,
file_size,
)
@@ -1,6 +1,7 @@
import asyncpg
from pyrogram import Client
from userbot.modules.capture.identity import ChatMetaCache, PeerIdentityCache
from userbot.modules.contacts import ContactCache
from userbot.modules.folders import FolderCache
from userbot.modules.watches import WatchCache
@@ -25,6 +26,8 @@ class CaptureContext:
self.folders = folders
self.contacts = contacts
self.watches = WatchCache(pool, account_id)
self.peer_identity = PeerIdentityCache()
self.chat_meta = ChatMetaCache()
self.policies = PolicySet()
async def reload_policies(self) -> None:
@@ -0,0 +1,110 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from userbot.modules.capture.chat_meta import chat_kind
from userbot.modules.profiles.parse import ProfileFields, snapshot_from_high_level
from userbot.modules.profiles.repository import get_peer, write_profile
from utils.policy.models import ChatKind
if TYPE_CHECKING:
import asyncpg
from pyrogram.types import Message
from userbot.modules.capture.context import CaptureContext
class PeerIdentityCache:
def __init__(self) -> None:
self._cache: dict[int, ProfileFields | None] = {}
async def changed(
self, pool: asyncpg.Pool, account_id: int, peer_id: int, fields: ProfileFields
) -> bool:
if peer_id not in self._cache:
self._cache[peer_id] = await get_peer(pool, account_id, peer_id)
if self._cache[peer_id] == fields:
return False
self._cache[peer_id] = fields
return True
class ChatMetaCache:
def __init__(self) -> None:
self._cache: dict[int, tuple[str | None, str | None]] = {}
async def changed(
self,
pool: asyncpg.Pool,
account_id: int,
chat_id: int,
meta: tuple[str | None, str | None],
) -> bool:
from userbot.modules.groups.repository import ( # noqa: PLC0415
get_latest_chat_meta,
)
if chat_id not in self._cache:
self._cache[chat_id] = await get_latest_chat_meta(pool, account_id, chat_id)
if self._cache[chat_id] == meta:
return False
self._cache[chat_id] = meta
return True
async def _capture_peer(message: Message, ctx: CaptureContext) -> None:
user = message.from_user
if user is None:
return
fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
if not await ctx.peer_identity.changed(ctx.pool, ctx.account_id, user.id, fields):
return
await write_profile(ctx.pool, ctx.account_id, user.id, fields, str(user))
if photo_file_id and photo_unique_id:
from userbot.modules.avatars import note_avatar # noqa: PLC0415
await note_avatar(
ctx.pool, ctx.account_id, user.id, "peer", photo_unique_id, photo_file_id
)
async def _capture_chat(message: Message, ctx: CaptureContext) -> None:
chat = message.chat
if (
chat is None
or chat.id is None
or message.date is None
or chat_kind(chat.type) is ChatKind.DM
):
return
photo = chat.photo
photo_unique_id = photo.big_photo_unique_id if photo else None
photo_file_id = photo.big_file_id if photo else None
meta = (chat.title, photo_unique_id)
if not await ctx.chat_meta.changed(ctx.pool, ctx.account_id, chat.id, meta):
return
from userbot.modules.groups.repository import insert_chat_history # noqa: PLC0415
await insert_chat_history(
ctx.pool,
ctx.account_id,
chat.id,
message.id,
"meta",
chat.title,
photo_unique_id,
None,
message.date,
str(message),
)
if photo_file_id and photo_unique_id:
from userbot.modules.avatars import note_avatar # noqa: PLC0415
await note_avatar(
ctx.pool, ctx.account_id, chat.id, "chat", photo_unique_id, photo_file_id
)
async def capture_identity(message: Message, ctx: CaptureContext) -> None:
await _capture_peer(message, ctx)
await _capture_chat(message, ctx)
@@ -51,6 +51,9 @@ async def capture_message(
has_media=message.media is not None,
is_self_destruct=self_destruct_ttl(message) is not None,
)
from userbot.modules.capture.identity import capture_identity # noqa: PLC0415
await capture_identity(message, ctx)
await capture_media(client, message, ctx, chat_id, message.id, toggles)
buttons = callbacks(message)
if buttons:
@@ -23,28 +23,71 @@ INSERT INTO messages
has_media, is_self_destruct, edited_at)
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8, $9, now())
ON CONFLICT (account_id, chat_id, message_id, date) DO UPDATE SET
text = EXCLUDED.text,
raw = EXCLUDED.raw,
has_media = EXCLUDED.has_media,
is_self_destruct = EXCLUDED.is_self_destruct,
edited_at = now()
"""
_INSERT_VERSION = """
INSERT INTO message_versions
(account_id, chat_id, message_id, observed_at, edit_date, text, raw)
VALUES ($1, $2, $3, now(), $4, $5, $6::jsonb)
VALUES ($1, $2, $3, clock_timestamp(), $4, $5, $6::jsonb)
ON CONFLICT DO NOTHING
"""
_SNAPSHOT_ORIGINAL = """
INSERT INTO message_versions
(account_id, chat_id, message_id, observed_at, edit_date, text, raw)
SELECT account_id, chat_id, message_id, clock_timestamp(), NULL, text, raw
FROM messages m
WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3
AND NOT EXISTS (
SELECT 1 FROM message_versions v
WHERE v.account_id = m.account_id AND v.chat_id = m.chat_id
AND v.message_id = m.message_id
)
ON CONFLICT DO NOTHING
"""
_CURRENT_CONTENT = """
SELECT
m.text AS text,
(SELECT d.unique_id FROM media d
WHERE d.account_id = m.account_id AND d.chat_id = m.chat_id
AND d.message_id = m.message_id) AS media_unique_id
FROM messages m
WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3
ORDER BY m.date DESC LIMIT 1
"""
_CURRENT_MEDIA = """
SELECT unique_id, storage_key, file_size, downloaded FROM media
WHERE account_id = $1 AND chat_id = $2 AND message_id = $3
"""
_INSERT_MEDIA_VERSION = """
INSERT INTO media_versions
(account_id, chat_id, message_id, observed_at, kind, storage_key,
file_size, mime, ttl_seconds, unique_id)
VALUES ($1, $2, $3, clock_timestamp(), $4, $5, $6, $7, $8, $9)
ON CONFLICT (account_id, chat_id, message_id, storage_key) DO NOTHING
"""
_INSERT_MEDIA = """
INSERT INTO media
(account_id, chat_id, message_id, kind, storage_key, file_size, mime,
ttl_seconds, downloaded)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ttl_seconds, downloaded, unique_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (account_id, chat_id, message_id) DO UPDATE SET
kind = EXCLUDED.kind,
storage_key = EXCLUDED.storage_key,
file_size = EXCLUDED.file_size,
mime = EXCLUDED.mime,
ttl_seconds = EXCLUDED.ttl_seconds,
downloaded = EXCLUDED.downloaded
downloaded = EXCLUDED.downloaded,
unique_id = EXCLUDED.unique_id
"""
@@ -111,11 +154,20 @@ async def add_version( # noqa: PLR0913
text: str | None,
raw: str,
edit_date: datetime | None,
media_unique_id: str | None,
*,
has_media: bool,
is_self_destruct: bool,
) -> None:
) -> bool:
async with pool.acquire() as conn, conn.transaction():
current = await conn.fetchrow(_CURRENT_CONTENT, account_id, chat_id, message_id)
text_changed = current is None or current["text"] != text
media_changed = (
current is not None and current["media_unique_id"] != media_unique_id
)
if not (text_changed or media_changed):
return False
await conn.execute(_SNAPSHOT_ORIGINAL, account_id, chat_id, message_id)
await conn.execute(
_TOUCH_EDITED,
account_id,
@@ -131,6 +183,13 @@ async def add_version( # noqa: PLR0913
await conn.execute(
_INSERT_VERSION, account_id, chat_id, message_id, edit_date, text, raw
)
return True
async def current_media(
pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
) -> asyncpg.Record | None:
return await pool.fetchrow(_CURRENT_MEDIA, account_id, chat_id, message_id)
async def insert_media( # noqa: PLR0913
@@ -143,21 +202,37 @@ async def insert_media( # noqa: PLR0913
file_size: int | None,
mime: str | None,
ttl_seconds: int | None,
unique_id: str | None,
*,
downloaded: bool,
) -> None:
await pool.execute(
_INSERT_MEDIA,
account_id,
chat_id,
message_id,
kind,
storage_key,
file_size,
mime,
ttl_seconds,
downloaded,
)
async with pool.acquire() as conn, conn.transaction():
await conn.execute(
_INSERT_MEDIA,
account_id,
chat_id,
message_id,
kind,
storage_key,
file_size,
mime,
ttl_seconds,
downloaded,
unique_id,
)
if storage_key is not None:
await conn.execute(
_INSERT_MEDIA_VERSION,
account_id,
chat_id,
message_id,
kind,
storage_key,
file_size,
mime,
ttl_seconds,
unique_id,
)
async def insert_callbacks(
@@ -17,6 +17,26 @@ VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb)
ON CONFLICT (account_id, chat_id, message_id, user_id) DO NOTHING
"""
_LATEST_TITLE = """
SELECT title FROM chat_history
WHERE account_id = $1 AND chat_id = $2 AND title IS NOT NULL
ORDER BY ts DESC LIMIT 1
"""
_LATEST_PHOTO = """
SELECT photo_unique_id FROM chat_history
WHERE account_id = $1 AND chat_id = $2 AND photo_unique_id IS NOT NULL
ORDER BY ts DESC LIMIT 1
"""
async def get_latest_chat_meta(
pool: asyncpg.Pool, account_id: int, chat_id: int
) -> tuple[str | None, str | None]:
title = await pool.fetchval(_LATEST_TITLE, account_id, chat_id)
photo_unique_id = await pool.fetchval(_LATEST_PHOTO, account_id, chat_id)
return title, photo_unique_id
async def insert_chat_history( # noqa: PLR0913
pool: asyncpg.Pool,
@@ -1,3 +1,9 @@
from userbot.modules.jobs.handlers import backfill, fetch_media, transcribe
from userbot.modules.jobs.handlers import (
backfill,
enrich_chat,
fetch_avatar,
fetch_media,
transcribe,
)
__all__ = ["backfill", "fetch_media", "transcribe"]
__all__ = ["backfill", "enrich_chat", "fetch_avatar", "fetch_media", "transcribe"]
@@ -0,0 +1,91 @@
from datetime import UTC, datetime
from pyrogram import Client
from pyrogram.errors import BadRequest, Forbidden
from pyrogram.types import User
from userbot.modules.avatars import note_avatar
from userbot.modules.capture.context import CaptureContext
from userbot.modules.groups.repository import insert_chat_history
from userbot.modules.jobs.context import JobContext
from userbot.modules.jobs.registry import register
from userbot.modules.profiles.parse import snapshot_from_high_level
from userbot.modules.profiles.repository import write_profile
MEMBER_CAP = 200
_MISSING_SENDERS = """
SELECT DISTINCT sender_id FROM messages
WHERE account_id = $1 AND chat_id = $2 AND sender_id > 0
AND NOT EXISTS (
SELECT 1 FROM peers p WHERE p.account_id = $1 AND p.peer_id = messages.sender_id
)
LIMIT 100
"""
async def _save_user(ctx: CaptureContext, user: User) -> None:
fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
await write_profile(ctx.pool, ctx.account_id, user.id, fields, str(user))
if photo_file_id and photo_unique_id:
await note_avatar(
ctx.pool, ctx.account_id, user.id, "peer", photo_unique_id, photo_file_id
)
async def _enrich_chat_meta(client: Client, ctx: CaptureContext, chat_id: int) -> None:
chat = await client.get_chat(chat_id)
photo = chat.photo
photo_unique_id = photo.big_photo_unique_id if photo else None
photo_file_id = photo.big_file_id if photo else None
await insert_chat_history(
ctx.pool,
ctx.account_id,
chat_id,
0,
"meta",
chat.title,
photo_unique_id,
None,
datetime.now(UTC),
str(chat),
)
if photo_file_id and photo_unique_id:
await note_avatar(
ctx.pool, ctx.account_id, chat_id, "chat", photo_unique_id, photo_file_id
)
async def _enrich_members(client: Client, ctx: CaptureContext, chat_id: int) -> None:
try:
async for member in client.get_chat_members(chat_id, limit=MEMBER_CAP):
if isinstance(member.user, User):
await _save_user(ctx, member.user)
except (BadRequest, Forbidden):
return
async def _enrich_senders(client: Client, ctx: CaptureContext, chat_id: int) -> None:
rows = await ctx.pool.fetch(_MISSING_SENDERS, ctx.account_id, chat_id)
ids = [row["sender_id"] for row in rows]
for sender_id in ids:
try:
user = await client.get_users(sender_id)
except BadRequest:
continue
if isinstance(user, User):
await _save_user(ctx, user)
@register("enrich_chat")
async def enrich_chat(ctx: JobContext) -> None:
client = ctx.client
if client is None:
return
capture = getattr(client, "capture", None)
if capture is None:
return
chat_id = ctx.job.params["chat_id"]
await _enrich_chat_meta(client, capture, chat_id)
await _enrich_members(client, capture, chat_id)
await _enrich_senders(client, capture, chat_id)
@@ -0,0 +1,40 @@
from io import BytesIO
from userbot.modules.avatars.repository import get_avatar_file, mark_avatar_downloaded
from userbot.modules.jobs.context import JobContext
from userbot.modules.jobs.registry import register
@register("fetch_avatar")
async def fetch_avatar(ctx: JobContext) -> None:
client = ctx.client
if client is None:
return
capture = getattr(client, "capture", None)
if capture is None:
return
owner_kind = ctx.job.params["owner_kind"]
owner_id = ctx.job.params["owner_id"]
unique_id = ctx.job.params["unique_id"]
found = await get_avatar_file(
ctx.pool, ctx.account_id, owner_kind, owner_id, unique_id
)
if found is None:
return
file_id, downloaded = found
if downloaded or file_id is None:
return
buffer = await client.download_media(file_id, in_memory=True)
if not isinstance(buffer, BytesIO):
return
data = buffer.getvalue()
storage_key = capture.storage.put(data)
await mark_avatar_downloaded(
ctx.pool,
ctx.account_id,
owner_kind,
owner_id,
unique_id,
storage_key,
len(data),
)
@@ -1,3 +1,7 @@
from userbot.modules.media.downloader import capture_media, self_destruct_ttl
from userbot.modules.media.downloader import (
capture_media,
media_unique_id,
self_destruct_ttl,
)
__all__ = ["capture_media", "self_destruct_ttl"]
__all__ = ["capture_media", "media_unique_id", "self_destruct_ttl"]
@@ -33,6 +33,11 @@ def self_destruct_ttl(message: Message) -> int | None:
return getattr(obj, "ttl_seconds", None) if obj is not None else None
def media_unique_id(message: Message) -> str | None:
_, obj = media_object(message)
return getattr(obj, "file_unique_id", None) if obj is not None else None
async def capture_media( # noqa: PLR0913
client: Client,
message: Message,
@@ -44,6 +49,7 @@ async def capture_media( # noqa: PLR0913
kind, obj = media_object(message)
if obj is None:
return
unique_id = getattr(obj, "file_unique_id", None)
ttl = getattr(obj, "ttl_seconds", None)
want = toggles.self_destruct_media if ttl else toggles.media
file_size = getattr(obj, "file_size", None)
@@ -51,12 +57,25 @@ async def capture_media( # noqa: PLR0913
storage_key: str | None = None
downloaded = False
if want:
buffer = await client.download_media(message, in_memory=True)
if isinstance(buffer, BytesIO):
data = buffer.getvalue()
storage_key = ctx.storage.put(data)
file_size = len(data)
existing = await repository.current_media(
ctx.pool, ctx.account_id, chat_id, message_id
)
if (
existing is not None
and existing["downloaded"]
and existing["unique_id"] == unique_id
and existing["storage_key"] is not None
):
storage_key = existing["storage_key"]
file_size = existing["file_size"]
downloaded = True
else:
buffer = await client.download_media(message, in_memory=True)
if isinstance(buffer, BytesIO):
data = buffer.getvalue()
storage_key = ctx.storage.put(data)
file_size = len(data)
downloaded = True
await repository.insert_media(
ctx.pool,
ctx.account_id,
@@ -67,5 +86,6 @@ async def capture_media( # noqa: PLR0913
file_size,
mime,
ttl,
unique_id,
downloaded=downloaded,
)
@@ -1,7 +1,13 @@
from userbot.modules.profiles.parse import (
ProfileFields,
active_username,
snapshot_from_high_level,
snapshot_from_user,
)
__all__ = ["ProfileFields", "active_username", "snapshot_from_user"]
__all__ = [
"ProfileFields",
"active_username",
"snapshot_from_high_level",
"snapshot_from_user",
]
@@ -37,3 +37,20 @@ def snapshot_from_user(
is_deleted_account=bool(getattr(raw_user, "deleted", False)),
)
return fields, photo_file_id, photo_unique_id
def snapshot_from_high_level(
user: User,
) -> tuple[ProfileFields, str | None, str | None]:
photo = user.photo
photo_unique_id = photo.big_photo_unique_id if photo else None
photo_file_id = photo.big_file_id if photo else None
fields = ProfileFields(
first_name=user.first_name,
last_name=user.last_name,
username=user.username,
phone=user.phone_number,
photo_unique_id=photo_unique_id,
is_deleted_account=bool(user.is_deleted),
)
return fields, photo_file_id, photo_unique_id
+1
View File
@@ -34,6 +34,7 @@ class TelegramSettings(BaseSettings):
class ApiSettings(BaseSettings):
host: str = "0.0.0.0" # noqa: S104
port: int = 8080
static_dir: str = "static"
class AuthSettings(BaseSettings):
+11
View File
@@ -0,0 +1,11 @@
import asyncpg
from utils.read.models import AccountView
async def list_accounts(pool: asyncpg.Pool) -> list[AccountView]:
rows = await pool.fetch(
"SELECT account_id, label, phone, tg_user_id, is_active FROM accounts "
"ORDER BY account_id"
)
return [AccountView(**dict(row)) for row in rows]
+30
View File
@@ -0,0 +1,30 @@
import asyncpg
from utils.read.models import AvatarRef
_PEER_UNIQUE_ID = """
SELECT photo_unique_id FROM peers
WHERE account_id = $1 AND peer_id = $2
"""
_CHAT_UNIQUE_ID = """
SELECT photo_unique_id FROM chat_history
WHERE account_id = $1 AND chat_id = $2 AND photo_unique_id IS NOT NULL
ORDER BY ts DESC LIMIT 1
"""
_AVATAR = """
SELECT unique_id, storage_key, downloaded, mime FROM avatars
WHERE account_id = $1 AND owner_id = $2 AND unique_id = $3
"""
async def current_avatar(
pool: asyncpg.Pool, account_id: int, owner_kind: str, owner_id: int
) -> AvatarRef | None:
query = _PEER_UNIQUE_ID if owner_kind == "peer" else _CHAT_UNIQUE_ID
unique_id = await pool.fetchval(query, account_id, owner_id)
if unique_id is None:
return None
row = await pool.fetchrow(_AVATAR, account_id, owner_id, unique_id)
return AvatarRef(**dict(row)) if row else None
+106 -6
View File
@@ -1,13 +1,46 @@
import asyncpg
from utils.read.models import ChatListItem, MessageVersionView, MessageView, Page
from utils.read.message_view import build_message_view, load_raw, media_ref_from
from utils.read.models import (
ChatListItem,
MediaRef,
MessageVersionView,
MessageView,
Page,
)
_MESSAGE_COLS = (
"chat_id, message_id, date, sender_id, text, "
"has_media, is_self_destruct, edited_at, deleted_at"
"chat_id, message_id, date, sender_id, text, has_media, is_self_destruct, "
"edited_at, deleted_at, raw, raw->>'media_group_id' AS media_group_id"
)
async def _media_map(
pool: asyncpg.Pool, account_id: int, rows: list[asyncpg.Record]
) -> dict[tuple[int, int], asyncpg.Record]:
message_ids = list({row["message_id"] for row in rows})
if not message_ids:
return {}
media_rows = await pool.fetch(
"SELECT id, chat_id, message_id, kind, downloaded, mime, file_size, "
"ttl_seconds FROM media "
"WHERE account_id = $1 AND message_id = ANY($2::bigint[])",
account_id,
message_ids,
)
return {(row["chat_id"], row["message_id"]): row for row in media_rows}
def _single_media(
row: asyncpg.Record, raw: dict, media_by_key: dict[tuple[int, int], asyncpg.Record]
) -> list[MediaRef]:
media_row = media_by_key.get((row["chat_id"], row["message_id"]))
if not (row["has_media"] or media_row):
return []
ref = media_ref_from(row["message_id"], raw, media_row)
return [ref] if ref else []
def _peer_title(
first: str | None, last: str | None, username: str | None
) -> str | None:
@@ -28,7 +61,24 @@ async def list_chats(
"WHERE p.account_id = $1 AND p.peer_id = m.chat_id) AS username, "
"(SELECT ch.title FROM chat_history ch "
"WHERE ch.account_id = $1 AND ch.chat_id = m.chat_id "
"AND ch.title IS NOT NULL ORDER BY ch.ts DESC LIMIT 1) AS group_title "
"AND ch.title IS NOT NULL ORDER BY ch.ts DESC LIMIT 1) AS group_title, "
"EXISTS (SELECT 1 FROM avatars a "
"WHERE a.account_id = $1 AND a.owner_id = m.chat_id) AS has_avatar, "
"(SELECT COALESCE((p.raw->>'is_bot')::bool, (p.raw->>'bot')::bool, false) "
"FROM peers p WHERE p.account_id = $1 AND p.peer_id = m.chat_id) AS is_bot, "
"(SELECT COALESCE((p.raw->>'is_contact')::bool, (p.raw->>'contact')::bool, "
"false) FROM peers p "
"WHERE p.account_id = $1 AND p.peer_id = m.chat_id) AS is_contact, "
"(SELECT ch.raw->'chat'->>'type' = 'ChatType.CHANNEL' FROM chat_history ch "
"WHERE ch.account_id = $1 AND ch.chat_id = m.chat_id "
"AND ch.raw->'chat'->>'type' IS NOT NULL "
"ORDER BY ch.ts DESC LIMIT 1) AS is_broadcast, "
"(SELECT lm.text FROM messages lm "
"WHERE lm.account_id = $1 AND lm.chat_id = m.chat_id "
"ORDER BY lm.date DESC, lm.message_id DESC LIMIT 1) AS last_text, "
"(SELECT lm.sender_id FROM messages lm "
"WHERE lm.account_id = $1 AND lm.chat_id = m.chat_id "
"ORDER BY lm.date DESC, lm.message_id DESC LIMIT 1) AS last_sender_id "
"FROM messages m WHERE m.account_id = $1 "
"GROUP BY m.chat_id ORDER BY last_date DESC LIMIT $2 OFFSET $3",
account_id,
@@ -44,8 +94,15 @@ async def list_chats(
ChatListItem(
chat_id=row["chat_id"],
title=title,
kind="private" if row["chat_id"] > 0 else "group",
has_avatar=row["has_avatar"],
is_bot=bool(row["is_bot"]),
is_contact=bool(row["is_contact"]),
is_broadcast=bool(row["is_broadcast"]),
message_count=row["message_count"],
last_date=row["last_date"],
last_text=row["last_text"],
last_sender_id=row["last_sender_id"],
)
)
return items
@@ -70,7 +127,43 @@ async def get_chat_history(
page.capped_limit,
page.offset,
)
return [MessageView(**dict(row)) for row in rows]
media_by_key = await _media_map(pool, account_id, rows)
parsed = [(row, load_raw(row["raw"])) for row in rows]
views: list[MessageView] = []
index = 0
while index < len(parsed):
group_id = parsed[index][0]["media_group_id"]
end = index + 1
if group_id is not None:
while end < len(parsed) and parsed[end][0]["media_group_id"] == group_id:
end += 1
members = parsed[index:end]
if len(members) == 1:
row, raw = members[0]
views.append(
build_message_view(row, raw, _single_media(row, raw, media_by_key))
)
else:
views.append(_build_album(members, media_by_key))
index = end
return views
def _build_album(
members: list[tuple[asyncpg.Record, dict]],
media_by_key: dict[tuple[int, int], asyncpg.Record],
) -> MessageView:
ordered = sorted(members, key=lambda m: m[0]["message_id"])
media: list[MediaRef] = []
for row, raw in ordered:
media_row = media_by_key.get((row["chat_id"], row["message_id"]))
ref = media_ref_from(row["message_id"], raw, media_row)
if ref:
media.append(ref)
primary_row, primary_raw = next(
((row, raw) for row, raw in ordered if row["text"]), ordered[0]
)
return build_message_view(primary_row, primary_raw, media)
async def get_deleted_messages(
@@ -88,7 +181,14 @@ async def get_deleted_messages(
f"ORDER BY deleted_at DESC LIMIT ${len(params) - 1} OFFSET ${len(params)}",
*params,
)
return [MessageView(**dict(row)) for row in rows]
media_by_key = await _media_map(pool, account_id, rows)
views: list[MessageView] = []
for row in rows:
raw = load_raw(row["raw"])
views.append(
build_message_view(row, raw, _single_media(row, raw, media_by_key))
)
return views
async def get_message_versions(
+27 -1
View File
@@ -1,12 +1,14 @@
import asyncpg
from utils.read.models import MediaView
from utils.read.models import MediaVersionView, MediaView
_MEDIA_COLS = (
"id, account_id, chat_id, message_id, kind, storage_key, file_size, "
"mime, ttl_seconds, downloaded, extracted_text, created_at"
)
_VERSION_COLS = "id, kind, storage_key, file_size, mime, observed_at"
async def get_media(pool: asyncpg.Pool, media_id: int) -> MediaView | None:
row = await pool.fetchrow(
@@ -27,3 +29,27 @@ async def get_message_media(
message_id,
)
return MediaView(**dict(row)) if row else None
async def get_media_versions(
pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
) -> list[MediaVersionView]:
rows = await pool.fetch(
f"SELECT {_VERSION_COLS} FROM media_versions " # noqa: S608
"WHERE account_id = $1 AND chat_id = $2 AND message_id = $3 "
"ORDER BY observed_at",
account_id,
chat_id,
message_id,
)
return [MediaVersionView(**dict(row)) for row in rows]
async def get_media_version(
pool: asyncpg.Pool, version_id: int
) -> MediaVersionView | None:
row = await pool.fetchrow(
f"SELECT {_VERSION_COLS} FROM media_versions WHERE id = $1", # noqa: S608
version_id,
)
return MediaVersionView(**dict(row)) if row else None
+404
View File
@@ -0,0 +1,404 @@
import json
from datetime import datetime
from typing import Any
import asyncpg
from pydantic import ValidationError
from utils.read.models import (
ContactView,
EntityView,
ForwardView,
InlineButton,
LocationView,
MediaRef,
MessageView,
PollOption,
PollView,
ReactionCount,
ReplyView,
ServiceView,
StickerView,
WebPageView,
)
_MEDIA_KEYS = (
"photo",
"video",
"animation",
"voice",
"video_note",
"audio",
"document",
"sticker",
)
def load_raw(raw: str | None) -> dict[str, Any]:
if not raw:
return {}
try:
parsed = json.loads(raw)
except (ValueError, TypeError):
return {}
return parsed if isinstance(parsed, dict) else {}
def _enum(value: object) -> str | None:
if not isinstance(value, str):
return None
return value.rsplit(".", 1)[-1].lower()
def _parse_dt(value: object) -> datetime | None:
if not isinstance(value, str):
return None
try:
return datetime.fromisoformat(value)
except ValueError:
return None
def _peer_name(user: dict[str, Any]) -> str | None:
name = " ".join(
part for part in (user.get("first_name"), user.get("last_name")) if part
)
return name or user.get("username")
def _entities(raw: dict[str, Any]) -> list[EntityView]:
source = raw.get("entities") or raw.get("caption_entities") or []
if not isinstance(source, list):
return []
out: list[EntityView] = []
for item in source:
if not isinstance(item, dict):
continue
kind = _enum(item.get("type"))
offset = item.get("offset")
length = item.get("length")
if kind is None or not isinstance(offset, int) or not isinstance(length, int):
continue
custom = item.get("custom_emoji_id")
out.append(
EntityView(
type=kind,
offset=offset,
length=length,
url=item.get("url"),
custom_emoji_id=str(custom) if custom is not None else None,
language=item.get("language"),
)
)
return out
def _media_kind(message: dict[str, Any]) -> str | None:
kind = _enum(message.get("media"))
if kind:
return kind
for key in _MEDIA_KEYS:
if key in message:
return key
return None
def _reply(raw: dict[str, Any]) -> ReplyView | None:
reply = raw.get("reply_to_message")
reply_id = raw.get("reply_to_message_id")
if not isinstance(reply, dict):
return ReplyView(message_id=reply_id) if reply_id else None
sender = reply.get("from_user")
sender_chat = reply.get("sender_chat")
sender_id = None
sender_name = None
if isinstance(sender, dict):
sender_id = sender.get("id")
sender_name = _peer_name(sender)
elif isinstance(sender_chat, dict):
sender_id = sender_chat.get("id")
sender_name = sender_chat.get("title")
return ReplyView(
message_id=reply.get("id") or reply_id,
sender_id=sender_id,
sender_name=sender_name,
text=reply.get("text") or reply.get("caption"),
media_kind=_media_kind(reply),
)
def _forward(raw: dict[str, Any]) -> ForwardView | None:
origin = raw.get("forward_origin")
if not isinstance(origin, dict):
return None
tag = origin.get("_")
date = _parse_dt(origin.get("date"))
if tag == "MessageOriginUser":
user = origin.get("sender_user")
user = user if isinstance(user, dict) else {}
return ForwardView(
kind="user", from_id=user.get("id"), from_name=_peer_name(user), date=date
)
if tag == "MessageOriginChannel":
chat = origin.get("chat")
chat = chat if isinstance(chat, dict) else {}
return ForwardView(
kind="channel",
chat_id=chat.get("id"),
chat_title=chat.get("title"),
message_id=origin.get("message_id"),
signature=origin.get("author_signature"),
date=date,
)
return ForwardView(
kind="hidden", from_name=origin.get("sender_user_name"), date=date
)
def _reactions(raw: dict[str, Any]) -> list[ReactionCount]:
container = raw.get("reactions")
if not isinstance(container, dict):
return []
items = container.get("reactions")
if not isinstance(items, list):
return []
out: list[ReactionCount] = []
for item in items:
if not isinstance(item, dict):
continue
custom = item.get("custom_emoji_id")
out.append(
ReactionCount(
emoji=item.get("emoji"),
custom_emoji_id=str(custom) if custom is not None else None,
count=item.get("count") or 0,
chosen="chosen_order" in item,
)
)
return out
def _button_kind(button: dict[str, Any]) -> str:
if button.get("url"):
return "url"
if button.get("callback_data") is not None:
return "callback"
if "switch_inline_query" in button or "switch_inline_query_current_chat" in button:
return "switch"
return "other"
def _inline_buttons(raw: dict[str, Any]) -> list[list[InlineButton]]:
markup = raw.get("reply_markup")
if not isinstance(markup, dict):
return []
rows = markup.get("inline_keyboard")
if not isinstance(rows, list):
return []
out: list[list[InlineButton]] = []
for row in rows:
if not isinstance(row, list):
continue
buttons: list[InlineButton] = []
for button in row:
if not isinstance(button, dict):
continue
data = button.get("callback_data")
buttons.append(
InlineButton(
text=button.get("text") or "",
kind=_button_kind(button),
url=button.get("url"),
data=data if isinstance(data, str) else None,
)
)
if buttons:
out.append(buttons)
return out
def _web_page(raw: dict[str, Any]) -> WebPageView | None:
page = raw.get("web_page")
if not isinstance(page, dict) or not page.get("url"):
return None
return WebPageView(
url=page["url"],
display_url=page.get("display_url"),
type=page.get("type"),
site_name=page.get("site_name"),
title=page.get("title"),
description=page.get("description"),
has_photo="photo" in page,
)
def _text_of(value: dict[str, Any] | str | None) -> str | None:
if isinstance(value, dict):
text = value.get("text")
return text if isinstance(text, str) else None
return value if isinstance(value, str) else None
def _poll(raw: dict[str, Any]) -> PollView | None:
poll = raw.get("poll")
if not isinstance(poll, dict):
return None
raw_options = poll.get("options")
options: list[PollOption] = []
if isinstance(raw_options, list):
for option in raw_options:
if not isinstance(option, dict):
continue
options.append(
PollOption(
text=_text_of(option.get("text")) or "",
voter_count=option.get("voter_count") or 0,
vote_percentage=option.get("vote_percentage") or 0,
correct=option.get("is_correct"),
)
)
return PollView(
question=_text_of(poll.get("question")) or "",
options=options,
total_voter_count=poll.get("total_voter_count") or 0,
quiz=_enum(poll.get("type")) == "quiz",
closed=bool(poll.get("is_closed")),
multiple=bool(poll.get("allows_multiple_answers")),
anonymous=bool(poll.get("is_anonymous", True)),
)
def _contact(raw: dict[str, Any]) -> ContactView | None:
contact = raw.get("contact")
if not isinstance(contact, dict):
return None
return ContactView(
user_id=contact.get("user_id"),
first_name=contact.get("first_name"),
last_name=contact.get("last_name"),
phone_number=contact.get("phone_number"),
)
def _location(raw: dict[str, Any]) -> LocationView | None:
venue = raw.get("venue")
if isinstance(venue, dict):
point = venue.get("location")
point = point if isinstance(point, dict) else {}
return LocationView(
latitude=point.get("latitude"),
longitude=point.get("longitude"),
title=venue.get("title"),
address=venue.get("address"),
)
point = raw.get("location")
if not isinstance(point, dict):
return None
return LocationView(
latitude=point.get("latitude"), longitude=point.get("longitude")
)
def _service(raw: dict[str, Any]) -> ServiceView | None:
kind = _enum(raw.get("service"))
if kind is None:
return None
members = raw.get("new_chat_members") or raw.get("left_chat_member")
member_ids = None
if isinstance(members, list):
member_ids = [m["id"] for m in members if isinstance(m, dict) and "id" in m]
elif isinstance(members, dict) and "id" in members:
member_ids = [members["id"]]
pinned = raw.get("pinned_message")
call = raw.get("phone_call_ended")
return ServiceView(
kind=kind,
member_ids=member_ids,
pinned_message_id=pinned.get("id") if isinstance(pinned, dict) else None,
duration=call.get("duration") if isinstance(call, dict) else None,
)
def _sticker(raw: dict[str, Any]) -> StickerView | None:
sticker = raw.get("sticker")
if not isinstance(sticker, dict):
return None
return StickerView(
emoji=sticker.get("emoji"),
set_name=sticker.get("set_name"),
width=sticker.get("width"),
height=sticker.get("height"),
mime=sticker.get("mime_type"),
is_animated=bool(sticker.get("is_animated")),
is_video=bool(sticker.get("is_video")),
)
def media_ref_from(
message_id: int, raw: dict[str, Any], media_row: asyncpg.Record | None
) -> MediaRef | None:
kind = (media_row["kind"] if media_row else None) or _media_kind(raw)
if kind is None:
return None
obj = raw.get(kind)
obj = obj if isinstance(obj, dict) else {}
width = obj.get("width") or obj.get("length")
height = obj.get("height") or obj.get("length")
return MediaRef(
message_id=message_id,
id=media_row["id"] if media_row else None,
kind=kind,
downloaded=bool(media_row["downloaded"]) if media_row else False,
width=width,
height=height,
duration=obj.get("duration"),
mime=(media_row["mime"] if media_row else None) or obj.get("mime_type"),
file_size=(media_row["file_size"] if media_row else None)
or obj.get("file_size"),
ttl_seconds=media_row["ttl_seconds"] if media_row else None,
)
def _base_fields(row: asyncpg.Record) -> dict[str, Any]:
return {
"chat_id": row["chat_id"],
"message_id": row["message_id"],
"date": row["date"],
"sender_id": row["sender_id"],
"text": row["text"],
"has_media": row["has_media"],
"is_self_destruct": row["is_self_destruct"],
"edited_at": row["edited_at"],
"deleted_at": row["deleted_at"],
"media_group_id": row["media_group_id"],
}
def build_message_view(
row: asyncpg.Record, raw: dict[str, Any], media: list[MediaRef]
) -> MessageView:
base = _base_fields(row)
via_bot = raw.get("via_bot")
sticker = _sticker(raw)
try:
return MessageView(
**base,
entities=_entities(raw),
quote=_text_of(raw.get("quote")),
reply=_reply(raw),
forward=_forward(raw),
media=media,
reactions=_reactions(raw),
inline_buttons=_inline_buttons(raw),
web_page=_web_page(raw),
poll=_poll(raw),
contact=_contact(raw),
location=_location(raw),
service=_service(raw),
via_bot_id=via_bot.get("id") if isinstance(via_bot, dict) else None,
sticker=sticker,
is_sticker=sticker is not None,
is_animated_emoji=False,
)
except ValidationError:
return MessageView(**base, media=media)
+162
View File
@@ -16,11 +16,139 @@ class Page(BaseModel):
return min(self.limit, MAX_LIMIT)
class AccountView(BaseModel):
account_id: int
label: str | None
phone: str | None
tg_user_id: int | None
is_active: bool
class ChatListItem(BaseModel):
chat_id: int
title: str | None
kind: str
has_avatar: bool
is_bot: bool
is_contact: bool
is_broadcast: bool
message_count: int
last_date: datetime | None
last_text: str | None
last_sender_id: int | None
class EntityView(BaseModel):
type: str
offset: int
length: int
url: str | None = None
custom_emoji_id: str | None = None
language: str | None = None
class ReplyView(BaseModel):
message_id: int | None = None
sender_id: int | None = None
sender_name: str | None = None
text: str | None = None
media_kind: str | None = None
class ForwardView(BaseModel):
kind: str
from_id: int | None = None
from_name: str | None = None
chat_id: int | None = None
chat_title: str | None = None
message_id: int | None = None
date: datetime | None = None
signature: str | None = None
class MediaRef(BaseModel):
message_id: int
id: int | None = None
kind: str
downloaded: bool = False
width: int | None = None
height: int | None = None
duration: float | None = None
mime: str | None = None
file_size: int | None = None
ttl_seconds: int | None = None
class ReactionCount(BaseModel):
emoji: str | None = None
custom_emoji_id: str | None = None
count: int
chosen: bool = False
class InlineButton(BaseModel):
text: str
kind: str
url: str | None = None
data: str | None = None
class WebPageView(BaseModel):
url: str
display_url: str | None = None
type: str | None = None
site_name: str | None = None
title: str | None = None
description: str | None = None
has_photo: bool = False
class PollOption(BaseModel):
text: str
voter_count: int = 0
vote_percentage: int = 0
correct: bool | None = None
class PollView(BaseModel):
question: str
options: list[PollOption] = []
total_voter_count: int = 0
quiz: bool = False
closed: bool = False
multiple: bool = False
anonymous: bool = True
class ContactView(BaseModel):
user_id: int | None = None
first_name: str | None = None
last_name: str | None = None
phone_number: str | None = None
class LocationView(BaseModel):
latitude: float | None = None
longitude: float | None = None
title: str | None = None
address: str | None = None
class ServiceView(BaseModel):
kind: str
member_ids: list[int] | None = None
pinned_message_id: int | None = None
duration: int | None = None
class StickerView(BaseModel):
emoji: str | None = None
set_name: str | None = None
width: int | None = None
height: int | None = None
mime: str | None = None
is_animated: bool = False
is_video: bool = False
class MessageView(BaseModel):
@@ -33,6 +161,23 @@ class MessageView(BaseModel):
is_self_destruct: bool
edited_at: datetime | None
deleted_at: datetime | None
entities: list[EntityView] = []
quote: str | None = None
reply: ReplyView | None = None
forward: ForwardView | None = None
media_group_id: str | None = None
media: list[MediaRef] = []
reactions: list[ReactionCount] = []
inline_buttons: list[list[InlineButton]] = []
web_page: WebPageView | None = None
poll: PollView | None = None
contact: ContactView | None = None
location: LocationView | None = None
service: ServiceView | None = None
via_bot_id: int | None = None
sticker: StickerView | None = None
is_sticker: bool = False
is_animated_emoji: bool = False
class MessageVersionView(BaseModel):
@@ -56,6 +201,22 @@ class MediaView(BaseModel):
created_at: datetime
class MediaVersionView(BaseModel):
id: int
kind: str
storage_key: str
file_size: int | None
mime: str | None
observed_at: datetime
class AvatarRef(BaseModel):
unique_id: str
storage_key: str | None
downloaded: bool
mime: str | None
class CallbackView(BaseModel):
position: int
label: str | None
@@ -103,6 +264,7 @@ class PeerView(BaseModel):
phone: str | None
photo_unique_id: str | None
is_deleted_account: bool
has_avatar: bool
updated_at: datetime
+22 -2
View File
@@ -2,13 +2,19 @@ import asyncpg
from utils.read.models import Page, PeerHistoryView, PeerView, StoryView
_PEER_COLS = (
"peer_id, first_name, last_name, username, phone, photo_unique_id, "
"is_deleted_account, updated_at, "
"EXISTS (SELECT 1 FROM avatars a WHERE a.account_id = peers.account_id "
"AND a.owner_id = peers.peer_id) AS has_avatar"
)
async def get_peer(
pool: asyncpg.Pool, account_id: int, peer_id: int
) -> PeerView | None:
row = await pool.fetchrow(
"SELECT peer_id, first_name, last_name, username, phone, "
"photo_unique_id, is_deleted_account, updated_at FROM peers "
f"SELECT {_PEER_COLS} FROM peers " # noqa: S608
"WHERE account_id = $1 AND peer_id = $2",
account_id,
peer_id,
@@ -16,6 +22,20 @@ async def get_peer(
return PeerView(**dict(row)) if row else None
async def get_peers(
pool: asyncpg.Pool, account_id: int, ids: list[int]
) -> list[PeerView]:
if not ids:
return []
rows = await pool.fetch(
f"SELECT {_PEER_COLS} FROM peers " # noqa: S608
"WHERE account_id = $1 AND peer_id = ANY($2)",
account_id,
ids,
)
return [PeerView(**dict(row)) for row in rows]
async def get_peer_history(
pool: asyncpg.Pool, account_id: int, peer_id: int
) -> list[PeerHistoryView]: