perf(*): bound list queries, batch fetches, harden media downloads

backend/src/utils/jobs.py

@@ -9,12 +9,22 @@ JOBS_CHANGED_CHANNEL = "jobs_changed"
 async def enqueue(
     pool: asyncpg.Pool, account_id: int, kind: str, params: dict[str, Any]
 ) -> int:
+    params_json = json.dumps(params)
+    existing = await pool.fetchval(
+        "SELECT id FROM jobs WHERE account_id = $1 AND kind = $2 "
+        "AND params = $3::jsonb AND status IN ('pending', 'running') LIMIT 1",
+        account_id,
+        kind,
+        params_json,
+    )
+    if existing is not None:
+        return existing
     job_id = await pool.fetchval(
         "INSERT INTO jobs (account_id, kind, params) "
         "VALUES ($1, $2, $3::jsonb) RETURNING id",
         account_id,
         kind,
-        json.dumps(params),
+        params_json,
     )
     await pool.execute(f"NOTIFY {JOBS_CHANGED_CHANNEL}")
     return job_id

backend/src/utils/read/accounts.py

@@ -30,10 +30,19 @@ RETURNING {_ACCOUNT_COLS}
 """  # noqa: S608
 
 
+_self_ids: dict[int, int] = {}
+
+
 async def self_user_id(pool: asyncpg.Pool, account_id: int) -> int | None:
-    return await pool.fetchval(
+    cached = _self_ids.get(account_id)
+    if cached is not None:
+        return cached
+    tg_user_id = await pool.fetchval(
         "SELECT tg_user_id FROM accounts WHERE account_id = $1", account_id
     )
+    if tg_user_id is not None:
+        _self_ids[account_id] = tg_user_id
+    return tg_user_id
 
 
 async def list_accounts(pool: asyncpg.Pool) -> list[AccountView]:

backend/src/utils/read/analytics.py

@@ -33,7 +33,7 @@ async def message_volume(
 
 
 async def response_stats(
-    pool: asyncpg.Pool, account_id: int, chat_id: int
+    pool: asyncpg.Pool, account_id: int, chat_id: int, *, days: int = 365
 ) -> ResponseStats:
     self_id = await self_user_id(pool, account_id)
     rows = await pool.fetch(
@@ -43,6 +43,7 @@ async def response_stats(
         "lag(date) OVER w AS prev_date "
         "FROM messages "
         "WHERE account_id = $1 AND chat_id = $2 AND sender_id IS NOT NULL "
+        "AND date >= $4 "
         "WINDOW w AS (ORDER BY date, message_id)), "
         "resp AS ("
         "SELECT (sender_id = $3) AS is_mine, "
@@ -55,6 +56,7 @@ async def response_stats(
         account_id,
         chat_id,
         self_id,
+        datetime.now(UTC) - timedelta(days=days),
     )
     stats = ResponseStats(
         mine_median_seconds=None, mine_count=0, their_median_seconds=None, their_count=0

backend/src/utils/read/avatars.py

@@ -21,7 +21,7 @@ WHERE account_id = $1 AND owner_id = $2 AND unique_id = $3
 _AVATAR_HISTORY = """
 SELECT unique_id, first_seen_at, downloaded FROM avatars
 WHERE account_id = $1 AND owner_id = $2
-ORDER BY first_seen_at DESC
+ORDER BY first_seen_at DESC LIMIT 200
 """
 
 

backend/src/utils/read/chats.py

@@ -62,6 +62,18 @@ _ONE_ID = """
     WHERE account_id = $1 AND scope_type = 'chat' AND scope_id = $2
 """
 
+_MANY_IDS = """
+    SELECT chat_id FROM chat_stats
+    WHERE account_id = $1 AND chat_id = ANY($2::bigint[])
+    UNION
+    SELECT chat_id FROM dialogs
+    WHERE account_id = $1 AND chat_id = ANY($2::bigint[])
+    UNION
+    SELECT scope_id AS chat_id FROM capture_policy
+    WHERE account_id = $1 AND scope_type = 'chat'
+      AND scope_id = ANY($2::bigint[])
+"""
+
 _CHAT_ROWS = """
 WITH ids AS ({ids})
 SELECT ids.chat_id,
@@ -171,6 +183,15 @@ async def get_chat(
     return _chat_item(row) if row is not None else None
 
 
+async def get_chats(
+    pool: asyncpg.Pool, account_id: int, ids: list[int]
+) -> list[ChatListItem]:
+    if not ids:
+        return []
+    rows = await pool.fetch(_CHAT_ROWS.format(ids=_MANY_IDS), account_id, ids)
+    return [_chat_item(row) for row in rows]
+
+
 async def get_chat_history(  # noqa: PLR0913
     pool: asyncpg.Pool,
     account_id: int,
@@ -308,7 +329,7 @@ async def get_message_versions(
     rows = await pool.fetch(
         "SELECT observed_at, edit_date, text FROM message_versions "
         "WHERE account_id = $1 AND chat_id = $2 AND message_id = $3 "
-        "ORDER BY observed_at",
+        "ORDER BY observed_at LIMIT 500",
         account_id,
         chat_id,
         message_id,

backend/src/utils/read/discover.py

@@ -54,9 +54,8 @@ WITH chat_meta AS (
         bool_or(is_broadcast) AS is_broadcast
     FROM hits GROUP BY chat_id
 ), counts AS (
-    SELECT chat_id, count(*) AS message_count FROM messages
+    SELECT chat_id, message_count FROM chat_stats
     WHERE account_id = $1 AND chat_id IN (SELECT chat_id FROM merged)
-    GROUP BY chat_id
 )
 SELECT m.chat_id, m.title, m.username, m.is_bot, m.is_contact, m.is_broadcast,
     COALESCE(c.message_count, 0) AS message_count,

backend/src/utils/read/media.py

@@ -1,7 +1,8 @@
+import json
+
 import asyncpg
 
 from utils.files import media_file_name
-from utils.read.message_view import load_raw
 from utils.read.models import MediaVersionView, MediaView
 
 MEDIA_COLS = (
@@ -40,15 +41,15 @@ async def _web_page_media_stub(
     pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
 ) -> MediaView | None:
     row = await pool.fetchrow(
-        "SELECT date, raw FROM messages "
+        "SELECT date, raw->'web_page' AS web_page FROM messages "
         "WHERE account_id = $1 AND chat_id = $2 AND message_id = $3",
         account_id,
         chat_id,
         message_id,
     )
-    if row is None:
+    if row is None or row["web_page"] is None:
         return None
-    web_page = load_raw(row["raw"]).get("web_page")
+    web_page = json.loads(row["web_page"])
     if not isinstance(web_page, dict):
         return None
     kind = next(
@@ -105,7 +106,7 @@ async def get_media_versions(
     rows = await pool.fetch(
         f"SELECT {_VERSION_COLS} FROM media_versions "  # noqa: S608
         "WHERE account_id = $1 AND chat_id = $2 AND message_id = $3 "
-        "ORDER BY observed_at",
+        "ORDER BY observed_at LIMIT 500",
         account_id,
         chat_id,
         message_id,

backend/src/utils/read/peers.py

@@ -42,7 +42,8 @@ async def get_peer_history(
     rows = await pool.fetch(
         "SELECT observed_at, first_name, last_name, username, phone, "
         "photo_unique_id, is_deleted_account FROM peer_history "
-        "WHERE account_id = $1 AND peer_id = $2 ORDER BY observed_at DESC",
+        "WHERE account_id = $1 AND peer_id = $2 "
+        "ORDER BY observed_at DESC LIMIT 500",
         account_id,
         peer_id,
     )

backend/src/utils/read/presence.py

@@ -1,9 +1,11 @@
-from datetime import datetime
+from datetime import UTC, datetime, timedelta
 
 import asyncpg
 
 from utils.read.models import Page, PresenceHourly, PresenceSample
 
+HOURLY_DEFAULT_DAYS = 90
+
 
 async def presence_history(  # noqa: PLR0913
     pool: asyncpg.Pool,
@@ -54,6 +56,8 @@ async def presence_hourly(
     date_from: datetime | None = None,
     date_to: datetime | None = None,
 ) -> list[PresenceHourly]:
+    if date_from is None:
+        date_from = datetime.now(UTC) - timedelta(days=HOURLY_DEFAULT_DAYS)
     params: list[object] = [account_id, peer_id]
     where = "account_id = $1 AND peer_id = $2"
     if date_from is not None:

backend/src/utils/read/profile.py

@@ -42,17 +42,27 @@ async def chat_links(
 
 
 async def daily_counts(
-    pool: asyncpg.Pool, account_id: int, chat_id: int
+    pool: asyncpg.Pool,
+    account_id: int,
+    chat_id: int,
+    *,
+    date_from: datetime | None = None,
+    date_to: datetime | None = None,
 ) -> list[DayCount]:
     self_id = await self_user_id(pool, account_id)
+    params: list[object] = [account_id, chat_id, self_id]
+    where = "account_id = $1 AND chat_id = $2"
+    if date_from is not None:
+        params.append(date_from)
+        where += f" AND date >= ${len(params)}"
+    if date_to is not None:
+        params.append(date_to)
+        where += f" AND date < ${len(params)}"
     rows = await pool.fetch(
-        "SELECT date_trunc('day', date) AS day, count(*) AS count, "
+        "SELECT date_trunc('day', date) AS day, count(*) AS count, "  # noqa: S608
         "count(*) FILTER (WHERE sender_id = $3) AS outgoing FROM messages "
-        "WHERE account_id = $1 AND chat_id = $2 "
-        "GROUP BY day ORDER BY day",
-        account_id,
-        chat_id,
-        self_id,
+        f"WHERE {where} GROUP BY day ORDER BY day",
+        *params,
     )
     return [DayCount(**dict(row)) for row in rows]