chore(deps): update ghcr.io/astral-sh/uv:python3.13-trixie-slim docker digest to 331fbb1 (main) (#22)

Co-authored-by: renovate <renovate@hhhhh.dev>

README.md

@@ -11,7 +11,7 @@ Useful on its own: a Telegram where nothing disappears. In the [beaver](https://
 - **Voice** transcribed through Telegram's own transcription (Premium), queued when the quota runs out.
 - **Presence** as a time series (TimescaleDB), read receipts, stories.
 - **Capture policy** from your Telegram folders with per-chat overrides: channels nothing, groups messages only, private chats everything, by default.
-- **Annotations and watches**: your notes on chats and rules that fire alerts, both yours to write in the web client.
+- **Annotations and watches**: your notes on messages, with `#tags` and one list of them all, and rules that fire alerts, both yours to write in the web client.
 - **Watched bots**: bot tokens you add in the web client. Each one logs in over MTProto (`auth.importBotAuthorization`, the same door a bot-login client uses), backfills the history of the bot's dialogs once and then keeps the incoming stream: who writes to the bot, what, and when. A separate authorization, so it does not take `getUpdates` away from whoever runs that bot over the Bot API.
 
 Nothing is ever deleted from the archive.
@@ -19,12 +19,12 @@ Nothing is ever deleted from the archive.
 ## Parts
 
 - `backend/` - `userbot` (kurigram, the capture loop and jobs), `watcher` (kurigram again, one bot-token session per watched bot) and `api` (FastAPI: the web API, the MCP server, and the built frontend as static files).
-- `frontend/` - SvelteKit, `adapter-static`; a chat client over the archive: folders, chat list, thread with media, versions and the deleted messages inline, search, stories, presence graphs.
+- `frontend/` - SvelteKit, `adapter-static`; a chat client over the archive: folders, chat list, thread with media, versions and the deleted messages inline, a feed of what was deleted or edited across chats, search, stories, presence graphs.
 - `docker-compose.yml` - includes `backend/` (Postgres with TimescaleDB, `userbot`, `api`) and `frontend/` (the dev server); profiles `external` and `services`.
 
 ## MCP
 
-Read-only: `list_watched_bots`, `get_bot_stats`, `get_bot_updates`, `search_messages_tool`, `list_chats`, `get_chat_history` (with deleted messages included), `get_deleted_messages`, `get_message_versions`, `get_media`, `get_callbacks`, `presence_history`, `get_peer_history`, `get_stories`, `get_annotations`, `list_watches`, `list_alerts`. The one write is `set_watch`. There are no send, edit or delete tools and there will not be.
+Read-only: `search_messages_tool`, `list_chats`, `get_chat_history` (a readable transcript, deleted messages included) and `get_chat_history_raw` (the same as JSON), `get_deleted_messages`, `get_feed` (deleted and edited messages and profile changes across chats), `get_message_versions`, `get_media`, `get_callbacks`, `get_read_info`, `presence_history`, `presence_stats`, `get_peer_history`, `get_person`, `get_stories`, `get_annotations`, `list_watches`, `list_alerts`. Two tools write: `set_watch`, and `download_media`, which pulls any attachment and leaves a public link to it for a day, visible and revocable in the web client. There are no send, edit or delete tools and there will not be.
 
 ## Run
 

backend/.env.example

@@ -25,6 +25,7 @@ BOTS__BACKFILL_PER_CHAT=0
 
 API__HOST=0.0.0.0
 API__PORT=8080
+API__PUBLIC_URL=
 
 AUTH__TOKEN=
 

backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/astral-sh/uv:python3.13-trixie-slim
+FROM ghcr.io/astral-sh/uv:python3.13-trixie-slim@sha256:331fbb158258608b9391e802305a9a2c9e42d689435c1d98706647122a122ab0
 
 RUN apt-get update && apt-get install -y build-essential && rm -rf /var/lib/apt/lists/*
 

backend/docker-compose.yml

@@ -80,7 +80,7 @@ services:
       database:
 
   postgres:
-    image: timescale/timescaledb:2.30.2-pg17
+    image: timescale/timescaledb:2.30.2-pg17@sha256:b346edcdb51a1fd6020e3965e0bd1c9f3406fa6d5fbce1e28f4852587ef934e2
     profiles: [postgres, external]
     restart: unless-stopped
     environment:

backend/migrations/versions/a2c4e6f8b1d3_edit_versions_everywhere.py

@@ -0,0 +1,32 @@
+"""edit versions everywhere
+
+Revision ID: a2c4e6f8b1d3
+Revises: e8b3f1a6c2d4
+Create Date: 2026-10-19 02:30:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "a2c4e6f8b1d3"
+down_revision: str | None = "e8b3f1a6c2d4"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        "UPDATE capture_policy SET track_edits_deletes = TRUE, updated_at = now() "
+        "WHERE scope_type IN ('default_group', 'default_channel') "
+        "AND NOT track_edits_deletes"
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        "UPDATE capture_policy SET track_edits_deletes = FALSE, updated_at = now() "
+        "WHERE scope_type IN ('default_group', 'default_channel') "
+        "AND track_edits_deletes"
+    )

backend/migrations/versions/b3d5f7a9c2e4_feed.py

@@ -0,0 +1,198 @@
+"""feed of deleted and edited messages
+
+Revision ID: b3d5f7a9c2e4
+Revises: a2c4e6f8b1d3
+Create Date: 2026-10-19 02:40:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "b3d5f7a9c2e4"
+down_revision: str | None = "a2c4e6f8b1d3"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_ITEMS = """
+CREATE TABLE feed_items (
+    id bigserial PRIMARY KEY,
+    account_id integer NOT NULL,
+    kind text NOT NULL,
+    chat_id bigint NOT NULL,
+    chat_kind text NOT NULL,
+    message_id bigint NOT NULL,
+    message_date timestamptz,
+    at timestamptz NOT NULL,
+    since timestamptz NOT NULL,
+    message_count integer NOT NULL DEFAULT 1
+)
+"""
+
+_SEEN = """
+CREATE TABLE feed_seen (
+    account_id integer PRIMARY KEY,
+    seen_at timestamptz NOT NULL
+)
+"""
+
+_CHAT_KIND = """
+CREATE FUNCTION feed_chat_kind(chat_id bigint, chat_type text)
+RETURNS text LANGUAGE sql IMMUTABLE AS $$
+    SELECT CASE
+        WHEN chat_id > 0 THEN 'dm'
+        WHEN chat_type = 'ChatType.CHANNEL' THEN 'channel'
+        ELSE 'group' END
+$$
+"""
+
+_MESSAGE_DATE = """
+CREATE FUNCTION feed_message_date(stamp text)
+RETURNS timestamptz LANGUAGE sql STABLE AS $$
+    SELECT CASE
+        WHEN stamp ~ '^\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?$'
+        THEN stamp::timestamp AT TIME ZONE 'UTC' END
+$$
+"""
+
+_ON_DELETE = """
+CREATE FUNCTION feed_on_delete() RETURNS trigger LANGUAGE plpgsql AS $$
+DECLARE
+    item feed_items%ROWTYPE;
+BEGIN
+    SELECT * INTO item FROM feed_items
+    WHERE account_id = NEW.account_id AND chat_id = NEW.chat_id
+      AND kind = 'deleted'
+    ORDER BY at DESC, id DESC LIMIT 1;
+    IF FOUND AND NEW.deleted_at BETWEEN item.since - interval '5 seconds'
+                                    AND item.at + interval '5 seconds' THEN
+        IF (NEW.date, NEW.message_id) > (item.message_date, item.message_id) THEN
+            item.message_id := NEW.message_id;
+            item.message_date := NEW.date;
+        END IF;
+        UPDATE feed_items SET
+            message_count = message_count + 1,
+            at = greatest(at, NEW.deleted_at),
+            since = least(since, NEW.deleted_at),
+            message_id = item.message_id,
+            message_date = item.message_date
+        WHERE id = item.id;
+    ELSE
+        INSERT INTO feed_items
+            (account_id, kind, chat_id, chat_kind, message_id, message_date,
+             at, since)
+        VALUES (NEW.account_id, 'deleted', NEW.chat_id,
+                feed_chat_kind(NEW.chat_id, NEW.raw->'chat'->>'type'),
+                NEW.message_id, NEW.date, NEW.deleted_at, NEW.deleted_at);
+    END IF;
+    RETURN NULL;
+END;
+$$
+"""
+
+_ON_VERSION = """
+CREATE FUNCTION feed_on_version() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+    IF NEW.edit_date IS NULL AND NOT EXISTS (
+        SELECT 1 FROM message_versions v
+        WHERE v.account_id = NEW.account_id AND v.chat_id = NEW.chat_id
+          AND v.message_id = NEW.message_id AND v.observed_at < NEW.observed_at
+    ) THEN
+        RETURN NULL;
+    END IF;
+    INSERT INTO feed_items
+        (account_id, kind, chat_id, chat_kind, message_id, message_date, at, since)
+    VALUES (NEW.account_id, 'edited', NEW.chat_id,
+            feed_chat_kind(NEW.chat_id, NEW.raw->'chat'->>'type'),
+            NEW.message_id, feed_message_date(NEW.raw->>'date'),
+            NEW.observed_at, NEW.observed_at);
+    RETURN NULL;
+END;
+$$
+"""
+
+_BACKFILL = """
+WITH d AS (
+    SELECT account_id, chat_id, message_id, date, deleted_at,
+           raw->'chat'->>'type' AS chat_type,
+           deleted_at - lag(deleted_at) OVER (
+               PARTITION BY account_id, chat_id ORDER BY deleted_at, message_id
+           ) AS gap
+    FROM messages WHERE deleted_at IS NOT NULL
+), b AS (
+    SELECT *, count(*) FILTER (WHERE gap IS NULL OR gap > interval '5 seconds')
+        OVER (PARTITION BY account_id, chat_id ORDER BY deleted_at, message_id)
+        AS burst
+    FROM d
+), deleted AS (
+    SELECT account_id, 'deleted' AS kind, chat_id,
+           feed_chat_kind(chat_id,
+               (array_agg(chat_type ORDER BY date DESC, message_id DESC))[1])
+               AS chat_kind,
+           (array_agg(message_id ORDER BY date DESC, message_id DESC))[1]
+               AS message_id,
+           max(date) AS message_date, max(deleted_at) AS at,
+           min(deleted_at) AS since, count(*)::integer AS message_count
+    FROM b GROUP BY account_id, chat_id, burst
+), edited AS (
+    SELECT v.account_id, 'edited' AS kind, v.chat_id,
+           feed_chat_kind(v.chat_id, v.raw->'chat'->>'type') AS chat_kind,
+           v.message_id, feed_message_date(v.raw->>'date') AS message_date,
+           v.observed_at AS at, v.observed_at AS since, 1 AS message_count
+    FROM message_versions v
+    WHERE v.edit_date IS NOT NULL OR EXISTS (
+        SELECT 1 FROM message_versions p
+        WHERE p.account_id = v.account_id AND p.chat_id = v.chat_id
+          AND p.message_id = v.message_id AND p.observed_at < v.observed_at
+    )
+)
+INSERT INTO feed_items
+    (account_id, kind, chat_id, chat_kind, message_id, message_date, at, since,
+     message_count)
+SELECT * FROM (SELECT * FROM deleted UNION ALL SELECT * FROM edited) items
+ORDER BY at, chat_id, message_id
+"""
+
+
+def upgrade() -> None:
+    op.execute("SET LOCAL jit = off")
+    op.execute(_ITEMS)
+    op.execute(_SEEN)
+    op.execute(_CHAT_KIND)
+    op.execute(_MESSAGE_DATE)
+    op.execute(_ON_DELETE)
+    op.execute(_ON_VERSION)
+    op.execute(
+        "CREATE TRIGGER messages_feed_deleted AFTER UPDATE OF deleted_at "
+        "ON messages FOR EACH ROW "
+        "WHEN (OLD.deleted_at IS NULL AND NEW.deleted_at IS NOT NULL) "
+        "EXECUTE FUNCTION feed_on_delete()"
+    )
+    op.execute(
+        "CREATE TRIGGER message_versions_feed AFTER INSERT ON message_versions "
+        "FOR EACH ROW EXECUTE FUNCTION feed_on_version()"
+    )
+    op.execute(_BACKFILL)
+    op.execute(
+        "CREATE INDEX ix_feed_items_recent ON feed_items (account_id, at DESC, id DESC)"
+    )
+    op.execute(
+        "CREATE INDEX ix_feed_items_chat ON feed_items "
+        "(account_id, chat_id, at DESC, id DESC)"
+    )
+    op.execute(
+        "INSERT INTO feed_seen (account_id, seen_at) SELECT account_id, now() "
+        "FROM accounts"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP TRIGGER IF EXISTS message_versions_feed ON message_versions")
+    op.execute("DROP TRIGGER IF EXISTS messages_feed_deleted ON messages")
+    op.execute("DROP FUNCTION IF EXISTS feed_on_version()")
+    op.execute("DROP FUNCTION IF EXISTS feed_on_delete()")
+    op.execute("DROP FUNCTION IF EXISTS feed_message_date(text)")
+    op.execute("DROP FUNCTION IF EXISTS feed_chat_kind(bigint, text)")
+    op.execute("DROP TABLE IF EXISTS feed_seen")
+    op.execute("DROP TABLE IF EXISTS feed_items")

backend/migrations/versions/c5e7a9b1d3f6_feed_profiles.py

@@ -0,0 +1,321 @@
+"""feed of profile changes and chat renames
+
+Revision ID: c5e7a9b1d3f6
+Revises: b3d5f7a9c2e4
+Create Date: 2026-10-19 04:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "c5e7a9b1d3f6"
+down_revision: str | None = "b3d5f7a9c2e4"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_PEER_VALUE = """
+CREATE FUNCTION feed_peer_value(
+    field text, first_name text, last_name text, username text, photo text
+) RETURNS text LANGUAGE sql IMMUTABLE AS $$
+    SELECT CASE field
+        WHEN 'name' THEN NULLIF(trim(concat_ws(' ', first_name, last_name)), '')
+        WHEN 'username' THEN NULLIF(username, '')
+        WHEN 'photo' THEN NULLIF(photo, '') END
+$$
+"""
+
+_PEER_SCOPE = """
+CREATE FUNCTION feed_peer_in_scope(account integer, peer bigint)
+RETURNS boolean LANGUAGE sql STABLE AS $$
+    SELECT EXISTS (
+        SELECT 1 FROM dialogs WHERE account_id = account AND chat_id = peer
+    ) OR EXISTS (
+        SELECT 1 FROM peers
+        WHERE account_id = account AND peer_id = peer
+          AND (raw->>'is_contact' = 'true' OR raw->>'contact' = 'true')
+    )
+$$
+"""
+
+_MERGE_DETAIL = """
+CREATE FUNCTION feed_merge_detail(base jsonb, incoming jsonb)
+RETURNS jsonb LANGUAGE sql IMMUTABLE AS $$
+    SELECT coalesce(jsonb_object_agg(key, pair), '{}'::jsonb)
+    FROM (
+        SELECT key, jsonb_build_array(
+                   coalesce(base->key->0, incoming->key->0),
+                   coalesce(incoming->key->1, base->key->1)) AS pair
+        FROM (
+            SELECT jsonb_object_keys(coalesce(base, '{}'::jsonb)) AS key
+            UNION
+            SELECT jsonb_object_keys(incoming)
+        ) keys
+    ) merged
+    WHERE pair->0 IS DISTINCT FROM pair->1
+$$
+"""
+
+_UPSERT = """
+CREATE FUNCTION feed_profile_upsert(
+    account integer, chat bigint, kind_of_chat text, message bigint,
+    changed_at timestamptz, changes jsonb
+) RETURNS void LANGUAGE plpgsql AS $$
+DECLARE
+    item feed_items%ROWTYPE;
+    merged jsonb;
+BEGIN
+    SELECT * INTO item FROM feed_items
+    WHERE account_id = account AND chat_id = chat AND kind = 'profile'
+    ORDER BY at DESC, id DESC LIMIT 1;
+    IF FOUND AND changed_at BETWEEN item.since - interval '1 day'
+                                AND item.at + interval '1 day' THEN
+        merged := feed_merge_detail(item.detail, changes);
+        IF merged = '{}'::jsonb THEN
+            DELETE FROM feed_items WHERE id = item.id;
+        ELSE
+            UPDATE feed_items SET
+                detail = merged,
+                message_count = message_count
+                    + (SELECT count(*) FROM jsonb_object_keys(changes)),
+                at = greatest(at, changed_at),
+                since = least(since, changed_at),
+                message_id = greatest(message_id, message)
+            WHERE id = item.id;
+        END IF;
+    ELSE
+        INSERT INTO feed_items
+            (account_id, kind, chat_id, chat_kind, message_id, at, since,
+             message_count, detail)
+        VALUES (account, 'profile', chat, kind_of_chat, message, changed_at,
+                changed_at,
+                (SELECT count(*) FROM jsonb_object_keys(changes)), changes);
+    END IF;
+END;
+$$
+"""
+
+_ON_PEER = """
+CREATE FUNCTION feed_on_peer_change() RETURNS trigger LANGUAGE plpgsql AS $$
+DECLARE
+    changes jsonb := '{}'::jsonb;
+    field text;
+    fresh text;
+    old text;
+BEGIN
+    IF NEW.peer_id <= 0 OR NOT feed_peer_in_scope(NEW.account_id, NEW.peer_id) THEN
+        RETURN NULL;
+    END IF;
+    FOREACH field IN ARRAY ARRAY['name', 'username', 'photo'] LOOP
+        fresh := feed_peer_value(field, NEW.first_name, NEW.last_name,
+                                 NEW.username, NEW.photo_unique_id);
+        CONTINUE WHEN fresh IS NULL;
+        SELECT value INTO old FROM (
+            SELECT feed_peer_value(field, h.first_name, h.last_name, h.username,
+                                   h.photo_unique_id) AS value, h.observed_at
+            FROM peer_history h
+            WHERE h.account_id = NEW.account_id AND h.peer_id = NEW.peer_id
+              AND h.observed_at < NEW.observed_at
+        ) earlier
+        WHERE value IS NOT NULL
+        ORDER BY observed_at DESC LIMIT 1;
+        CONTINUE WHEN old IS NULL OR old = fresh;
+        CONTINUE WHEN EXISTS (
+            SELECT 1 FROM peer_history h
+            WHERE h.account_id = NEW.account_id AND h.peer_id = NEW.peer_id
+              AND h.observed_at < NEW.observed_at
+              AND feed_peer_value(field, h.first_name, h.last_name, h.username,
+                                  h.photo_unique_id) = fresh
+        );
+        changes := changes || jsonb_build_object(field, jsonb_build_array(old, fresh));
+    END LOOP;
+    IF changes <> '{}'::jsonb THEN
+        PERFORM feed_profile_upsert(NEW.account_id, NEW.peer_id, 'dm', 0,
+                                    NEW.observed_at, changes);
+    END IF;
+    RETURN NULL;
+END;
+$$
+"""
+
+_ON_TITLE = """
+CREATE FUNCTION feed_on_chat_title() RETURNS trigger LANGUAGE plpgsql AS $$
+DECLARE
+    previous text;
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM dialogs
+        WHERE account_id = NEW.account_id AND chat_id = NEW.chat_id
+    ) THEN
+        RETURN NULL;
+    END IF;
+    SELECT title INTO previous FROM chat_history
+    WHERE account_id = NEW.account_id AND chat_id = NEW.chat_id
+      AND event IN ('meta', 'title') AND title IS NOT NULL
+      AND (ts, message_id) < (NEW.ts, NEW.message_id)
+    ORDER BY ts DESC, message_id DESC LIMIT 1;
+    IF previous IS NULL OR previous = NEW.title THEN
+        RETURN NULL;
+    END IF;
+    PERFORM feed_profile_upsert(
+        NEW.account_id, NEW.chat_id,
+        feed_chat_kind(NEW.chat_id, NEW.raw->'chat'->>'type'),
+        CASE WHEN NEW.event = 'title' THEN NEW.message_id ELSE 0 END,
+        NEW.ts, jsonb_build_object('title', jsonb_build_array(previous, NEW.title)));
+    RETURN NULL;
+END;
+$$
+"""
+
+_BACKFILL_PEERS = """
+WITH scope AS (
+    SELECT account_id::integer AS account_id, chat_id AS peer_id
+    FROM dialogs WHERE chat_id > 0
+    UNION
+    SELECT account_id, peer_id FROM peers
+    WHERE raw->>'is_contact' = 'true' OR raw->>'contact' = 'true'
+), firsts AS (
+    SELECT h.account_id, h.peer_id, f.field, f.value,
+           min(h.observed_at) AS at
+    FROM peer_history h
+    JOIN scope s ON s.account_id = h.account_id AND s.peer_id = h.peer_id
+    CROSS JOIN LATERAL (
+        SELECT field, feed_peer_value(field, h.first_name, h.last_name,
+                                      h.username, h.photo_unique_id) AS value
+        FROM unnest(ARRAY['name', 'username', 'photo']) AS field
+    ) f
+    WHERE f.value IS NOT NULL
+    GROUP BY 1, 2, 3, 4
+), ranked AS (
+    SELECT *, row_number() OVER (
+        PARTITION BY account_id, peer_id, field ORDER BY at
+    ) AS position
+    FROM firsts
+), changes AS (
+    SELECT r.account_id, r.peer_id, r.field, r.value AS fresh, r.at, o.value AS old
+    FROM ranked r
+    CROSS JOIN LATERAL (
+        SELECT value FROM (
+            SELECT feed_peer_value(r.field, h.first_name, h.last_name,
+                                   h.username, h.photo_unique_id) AS value,
+                   h.observed_at
+            FROM peer_history h
+            WHERE h.account_id = r.account_id AND h.peer_id = r.peer_id
+              AND h.observed_at < r.at
+        ) earlier
+        WHERE value IS NOT NULL
+        ORDER BY observed_at DESC LIMIT 1
+    ) o
+    WHERE r.position > 1 AND r.at > now() - interval '90 days'
+      AND o.value <> r.value
+), gaps AS (
+    SELECT *, at - lag(at) OVER (PARTITION BY account_id, peer_id ORDER BY at)
+              AS gap
+    FROM changes
+), bursts AS (
+    SELECT *, count(*) FILTER (WHERE gap IS NULL OR gap > interval '1 day')
+              OVER (PARTITION BY account_id, peer_id ORDER BY at) AS burst
+    FROM gaps
+), fields AS (
+    SELECT account_id, peer_id, burst, field,
+           (array_agg(old ORDER BY at))[1] AS old,
+           (array_agg(fresh ORDER BY at DESC))[1] AS fresh,
+           count(*) AS changes, min(at) AS since, max(at) AS at
+    FROM bursts GROUP BY 1, 2, 3, 4
+)
+INSERT INTO feed_items
+    (account_id, kind, chat_id, chat_kind, message_id, at, since,
+     message_count, detail)
+SELECT account_id, 'profile', peer_id, 'dm', 0, max(at), min(since),
+       sum(changes)::integer,
+       jsonb_object_agg(field, jsonb_build_array(old, fresh))
+           FILTER (WHERE old <> fresh)
+FROM fields
+GROUP BY account_id, peer_id, burst
+HAVING bool_or(old <> fresh)
+ORDER BY max(at)
+"""
+
+_BACKFILL_CHATS = """
+WITH titles AS (
+    SELECT account_id, chat_id, message_id, event, ts, title,
+           raw->'chat'->>'type' AS chat_type,
+           lag(title) OVER (
+               PARTITION BY account_id, chat_id ORDER BY ts, message_id
+           ) AS previous
+    FROM chat_history
+    WHERE event IN ('meta', 'title') AND title IS NOT NULL
+), changes AS (
+    SELECT * FROM titles t
+    WHERE previous IS NOT NULL AND previous <> title
+      AND ts > now() - interval '90 days'
+      AND EXISTS (
+          SELECT 1 FROM dialogs d
+          WHERE d.account_id = t.account_id AND d.chat_id = t.chat_id
+      )
+), gaps AS (
+    SELECT *, ts - lag(ts) OVER (PARTITION BY account_id, chat_id ORDER BY ts)
+              AS gap
+    FROM changes
+), bursts AS (
+    SELECT *, count(*) FILTER (WHERE gap IS NULL OR gap > interval '1 day')
+              OVER (PARTITION BY account_id, chat_id ORDER BY ts) AS burst
+    FROM gaps
+), grouped AS (
+    SELECT account_id, chat_id, burst,
+           (array_agg(chat_type ORDER BY ts DESC))[1] AS chat_type,
+           max(CASE WHEN event = 'title' THEN message_id ELSE 0 END) AS message_id,
+           (array_agg(previous ORDER BY ts))[1] AS old,
+           (array_agg(title ORDER BY ts DESC))[1] AS fresh,
+           count(*) AS changes, min(ts) AS since, max(ts) AS at
+    FROM bursts GROUP BY 1, 2, 3
+)
+INSERT INTO feed_items
+    (account_id, kind, chat_id, chat_kind, message_id, at, since,
+     message_count, detail)
+SELECT account_id, 'profile', chat_id, feed_chat_kind(chat_id, chat_type),
+       message_id, at, since, changes::integer,
+       jsonb_build_object('title', jsonb_build_array(old, fresh))
+FROM grouped
+WHERE old <> fresh
+ORDER BY at
+"""
+
+
+def upgrade() -> None:
+    op.execute("SET LOCAL jit = off")
+    op.execute("ALTER TABLE feed_items ADD COLUMN detail jsonb")
+    op.execute(_PEER_VALUE)
+    op.execute(_PEER_SCOPE)
+    op.execute(_MERGE_DETAIL)
+    op.execute(_UPSERT)
+    op.execute(_ON_PEER)
+    op.execute(_ON_TITLE)
+    op.execute(
+        "CREATE TRIGGER peer_history_feed AFTER INSERT ON peer_history "
+        "FOR EACH ROW EXECUTE FUNCTION feed_on_peer_change()"
+    )
+    op.execute(
+        "CREATE TRIGGER chat_history_feed AFTER INSERT ON chat_history "
+        "FOR EACH ROW WHEN (NEW.event IN ('meta', 'title') AND NEW.title IS NOT NULL) "
+        "EXECUTE FUNCTION feed_on_chat_title()"
+    )
+    op.execute(_BACKFILL_PEERS)
+    op.execute(_BACKFILL_CHATS)
+
+
+def downgrade() -> None:
+    op.execute("DROP TRIGGER IF EXISTS chat_history_feed ON chat_history")
+    op.execute("DROP TRIGGER IF EXISTS peer_history_feed ON peer_history")
+    op.execute("DROP FUNCTION IF EXISTS feed_on_chat_title()")
+    op.execute("DROP FUNCTION IF EXISTS feed_on_peer_change()")
+    op.execute(
+        "DROP FUNCTION IF EXISTS "
+        "feed_profile_upsert(integer, bigint, text, bigint, timestamptz, jsonb)"
+    )
+    op.execute("DROP FUNCTION IF EXISTS feed_merge_detail(jsonb, jsonb)")
+    op.execute("DROP FUNCTION IF EXISTS feed_peer_in_scope(integer, bigint)")
+    op.execute("DROP FUNCTION IF EXISTS feed_peer_value(text, text, text, text, text)")
+    op.execute("DELETE FROM feed_items WHERE kind = 'profile'")
+    op.execute("ALTER TABLE feed_items DROP COLUMN detail")

backend/migrations/versions/c9e2a7f4b1d8_media_trash.py

@@ -0,0 +1,57 @@
+"""media trash
+
+Revision ID: c9e2a7f4b1d8
+Revises: b6d1f4a82e57
+Create Date: 2026-10-14 03:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "c9e2a7f4b1d8"
+down_revision: str | None = "b6d1f4a82e57"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_TRASH = """
+CREATE TABLE media_trash (
+    storage_key text PRIMARY KEY,
+    account_id integer NOT NULL,
+    chat_id bigint NOT NULL,
+    message_id bigint NOT NULL,
+    kind text NOT NULL,
+    mime text,
+    file_name text NOT NULL,
+    file_size bigint,
+    trashed_at timestamptz NOT NULL DEFAULT now()
+)
+"""
+
+
+def upgrade() -> None:
+    op.execute(
+        "ALTER TABLE media ADD COLUMN evicted_at timestamptz, "
+        "ADD COLUMN evicted_key text"
+    )
+    op.execute(
+        "CREATE INDEX ix_media_evicted_unique ON media (unique_id) "
+        "WHERE evicted_at IS NOT NULL"
+    )
+    op.execute(
+        "CREATE INDEX ix_media_evicted_key ON media (evicted_key) "
+        "WHERE evicted_key IS NOT NULL"
+    )
+    op.execute(_TRASH)
+    op.execute(
+        "CREATE INDEX ix_media_trash_account ON media_trash "
+        "(account_id, trashed_at DESC)"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS media_trash")
+    op.execute("DROP INDEX IF EXISTS ix_media_evicted_key")
+    op.execute("DROP INDEX IF EXISTS ix_media_evicted_unique")
+    op.execute("ALTER TABLE media DROP COLUMN evicted_key, DROP COLUMN evicted_at")

backend/migrations/versions/d7a3c5e9f2b1_message_reads.py

@@ -0,0 +1,51 @@
+"""message reads
+
+Revision ID: d7a3c5e9f2b1
+Revises: f8c2d6e1a4b7
+Create Date: 2026-10-19 01:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "d7a3c5e9f2b1"
+down_revision: str | None = "f8c2d6e1a4b7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_READS = """
+CREATE TABLE message_reads (
+    account_id integer NOT NULL,
+    chat_id bigint NOT NULL,
+    message_id bigint NOT NULL,
+    reader_id bigint NOT NULL,
+    read_at timestamptz NOT NULL,
+    source text NOT NULL,
+    fetched_at timestamptz NOT NULL DEFAULT now(),
+    PRIMARY KEY (account_id, chat_id, message_id, reader_id)
+)
+"""
+
+_FETCHES = """
+CREATE TABLE read_fetches (
+    account_id integer NOT NULL,
+    chat_id bigint NOT NULL,
+    message_id bigint NOT NULL,
+    outcome text NOT NULL,
+    fetched_at timestamptz NOT NULL DEFAULT now(),
+    retry_at timestamptz NOT NULL,
+    PRIMARY KEY (account_id, chat_id, message_id)
+)
+"""
+
+
+def upgrade() -> None:
+    op.execute(_READS)
+    op.execute(_FETCHES)
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS read_fetches")
+    op.execute("DROP TABLE IF EXISTS message_reads")

backend/migrations/versions/d8e3b5a1c7f4_watch_names_webhooks.py

@@ -0,0 +1,86 @@
+"""watch names and webhooks
+
+Revision ID: d8e3b5a1c7f4
+Revises: d8f4a1c6e2b7
+Create Date: 2026-10-19 03:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "d8e3b5a1c7f4"
+down_revision: str | None = "d8f4a1c6e2b7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_WEBHOOKS = """
+CREATE TABLE webhooks (
+    id serial PRIMARY KEY,
+    account_id integer NOT NULL,
+    url text NOT NULL,
+    secret text NOT NULL,
+    enabled boolean NOT NULL DEFAULT true,
+    watch_ids integer[],
+    created_at timestamptz NOT NULL DEFAULT now(),
+    updated_at timestamptz NOT NULL DEFAULT now()
+)
+"""
+
+_DELIVERIES = """
+CREATE TABLE webhook_deliveries (
+    id bigserial PRIMARY KEY,
+    webhook_id integer NOT NULL REFERENCES webhooks (id) ON DELETE CASCADE,
+    alert_id integer REFERENCES alerts (id) ON DELETE CASCADE,
+    event text NOT NULL,
+    status text NOT NULL DEFAULT 'pending',
+    attempts integer NOT NULL DEFAULT 0,
+    next_attempt_at timestamptz NOT NULL DEFAULT now(),
+    response_code integer,
+    last_error text,
+    body text,
+    created_at timestamptz NOT NULL DEFAULT now(),
+    finished_at timestamptz
+)
+"""
+
+_LEGACY_KEYWORD = """
+UPDATE watches
+SET params = (params - 'keyword') || jsonb_build_object('pattern', params->'keyword')
+WHERE kind = 'keyword' AND params ? 'keyword' AND NOT params ? 'pattern'
+"""
+
+
+def upgrade() -> None:
+    op.execute("ALTER TABLE watches ADD COLUMN name text")
+    op.execute(_LEGACY_KEYWORD)
+    op.execute("ALTER TABLE alerts DROP CONSTRAINT alerts_watch_id_fkey")
+    op.execute(
+        "ALTER TABLE alerts ADD CONSTRAINT alerts_watch_id_fkey "
+        "FOREIGN KEY (watch_id) REFERENCES watches (id) ON DELETE CASCADE"
+    )
+    op.execute("CREATE INDEX ix_alerts_account_id ON alerts (account_id, id DESC)")
+    op.execute(_WEBHOOKS)
+    op.execute("CREATE INDEX ix_webhooks_account ON webhooks (account_id)")
+    op.execute(_DELIVERIES)
+    op.execute(
+        "CREATE INDEX ix_webhook_deliveries_due ON webhook_deliveries "
+        "(next_attempt_at) WHERE status = 'pending'"
+    )
+    op.execute(
+        "CREATE INDEX ix_webhook_deliveries_webhook ON webhook_deliveries "
+        "(webhook_id, id DESC)"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS webhook_deliveries")
+    op.execute("DROP TABLE IF EXISTS webhooks")
+    op.execute("DROP INDEX IF EXISTS ix_alerts_account_id")
+    op.execute("ALTER TABLE alerts DROP CONSTRAINT alerts_watch_id_fkey")
+    op.execute(
+        "ALTER TABLE alerts ADD CONSTRAINT alerts_watch_id_fkey "
+        "FOREIGN KEY (watch_id) REFERENCES watches (id)"
+    )
+    op.execute("ALTER TABLE watches DROP COLUMN name")

backend/migrations/versions/d8f1a6c3e5b7_message_search.py

@@ -0,0 +1,148 @@
+"""message search index outside the hypertable, versions searchable
+
+Revision ID: d8f1a6c3e5b7
+Revises: c5e7a9b1d3f6
+Create Date: 2026-10-19 03:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "d8f1a6c3e5b7"
+down_revision: str | None = "c5e7a9b1d3f6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_VECTOR = """
+CREATE FUNCTION search_vector(body text) RETURNS tsvector
+LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
+    SELECT to_tsvector('russian', translate(coalesce(body, ''), 'Ёё', 'Ее'))
+        || to_tsvector('simple', translate(coalesce(body, ''), 'Ёё', 'Ее'))
+$$
+"""
+
+_TABLE = """
+CREATE TABLE message_search (
+    account_id integer NOT NULL,
+    chat_id bigint NOT NULL,
+    message_id bigint NOT NULL,
+    date timestamptz NOT NULL,
+    sender_id bigint,
+    text text,
+    has_media boolean NOT NULL DEFAULT false,
+    edited_at timestamptz,
+    deleted_at timestamptz,
+    tsv tsvector GENERATED ALWAYS AS (search_vector(text)) STORED,
+    PRIMARY KEY (account_id, chat_id, message_id, date)
+)
+"""
+
+_BACKFILL = """
+INSERT INTO message_search (account_id, chat_id, message_id, date, sender_id,
+                            text, has_media, edited_at, deleted_at)
+SELECT account_id, chat_id, message_id, date, sender_id,
+       text, has_media, edited_at, deleted_at
+FROM messages
+"""
+
+_SYNC = """
+CREATE FUNCTION message_search_sync() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+    IF TG_OP = 'DELETE' THEN
+        DELETE FROM message_search
+        WHERE account_id = OLD.account_id AND chat_id = OLD.chat_id
+          AND message_id = OLD.message_id AND date = OLD.date;
+        RETURN NULL;
+    END IF;
+    INSERT INTO message_search (account_id, chat_id, message_id, date,
+                                sender_id, text, has_media, edited_at,
+                                deleted_at)
+    VALUES (NEW.account_id, NEW.chat_id, NEW.message_id, NEW.date,
+            NEW.sender_id, NEW.text, NEW.has_media, NEW.edited_at,
+            NEW.deleted_at)
+    ON CONFLICT (account_id, chat_id, message_id, date) DO UPDATE SET
+        sender_id = EXCLUDED.sender_id,
+        text = EXCLUDED.text,
+        has_media = EXCLUDED.has_media,
+        edited_at = EXCLUDED.edited_at,
+        deleted_at = EXCLUDED.deleted_at;
+    RETURN NULL;
+END;
+$$
+"""
+
+_CHANGED = (
+    "OLD.text IS DISTINCT FROM NEW.text "
+    "OR OLD.sender_id IS DISTINCT FROM NEW.sender_id "
+    "OR OLD.has_media IS DISTINCT FROM NEW.has_media "
+    "OR OLD.edited_at IS DISTINCT FROM NEW.edited_at "
+    "OR OLD.deleted_at IS DISTINCT FROM NEW.deleted_at"
+)
+
+
+def upgrade() -> None:
+    op.execute(_VECTOR)
+    op.execute(_TABLE)
+    op.execute(_BACKFILL)
+    op.execute("CREATE INDEX ix_message_search_tsv ON message_search USING gin (tsv)")
+    op.execute(
+        "CREATE INDEX ix_message_search_recent ON message_search "
+        "(account_id, date DESC, chat_id DESC, message_id DESC)"
+    )
+    op.execute(
+        "CREATE INDEX ix_message_search_sender ON message_search "
+        "(account_id, sender_id, date DESC)"
+    )
+    for flag in ("deleted", "edited"):
+        op.execute(
+            f"CREATE INDEX ix_message_search_{flag} ON message_search "
+            "(account_id, date DESC, chat_id DESC, message_id DESC) "
+            f"WHERE {flag}_at IS NOT NULL"
+        )
+    op.execute("ALTER TABLE message_search ALTER COLUMN tsv SET STATISTICS 1000")
+    op.execute(_SYNC)
+    op.execute(
+        "CREATE TRIGGER messages_search_insert AFTER INSERT ON messages "
+        "FOR EACH ROW EXECUTE FUNCTION message_search_sync()"
+    )
+    op.execute(
+        "CREATE TRIGGER messages_search_update AFTER UPDATE ON messages "
+        f"FOR EACH ROW WHEN ({_CHANGED}) EXECUTE FUNCTION message_search_sync()"
+    )
+    op.execute(
+        "CREATE TRIGGER messages_search_delete AFTER DELETE ON messages "
+        "FOR EACH ROW EXECUTE FUNCTION message_search_sync()"
+    )
+    op.execute(
+        "CREATE INDEX ix_message_versions_tsv ON message_versions USING gin "
+        "(search_vector(text))"
+    )
+    op.execute("DROP INDEX IF EXISTS ix_media_tsv")
+    op.execute(
+        "CREATE INDEX ix_media_tsv ON media USING gin (search_vector(extracted_text))"
+    )
+    op.execute("DROP INDEX IF EXISTS ix_messages_tsv")
+    for index in ("ix_message_versions_tsv", "ix_media_tsv"):
+        op.execute(f"ALTER INDEX {index} ALTER COLUMN 1 SET STATISTICS 1000")
+    op.execute("ANALYZE message_search, message_versions, media")
+
+
+def downgrade() -> None:
+    op.execute(
+        "CREATE INDEX ix_messages_tsv ON messages USING gin "
+        "(to_tsvector('russian', coalesce(text, '')))"
+    )
+    op.execute("DROP INDEX IF EXISTS ix_media_tsv")
+    op.execute(
+        "CREATE INDEX ix_media_tsv ON media USING gin "
+        "(to_tsvector('russian', coalesce(extracted_text, '')))"
+    )
+    op.execute("DROP INDEX IF EXISTS ix_message_versions_tsv")
+    op.execute("DROP TRIGGER IF EXISTS messages_search_delete ON messages")
+    op.execute("DROP TRIGGER IF EXISTS messages_search_update ON messages")
+    op.execute("DROP TRIGGER IF EXISTS messages_search_insert ON messages")
+    op.execute("DROP FUNCTION IF EXISTS message_search_sync()")
+    op.execute("DROP TABLE IF EXISTS message_search")
+    op.execute("DROP FUNCTION IF EXISTS search_vector(text)")

backend/migrations/versions/d8f4a1c6e2b7_capture_uptime.py

@@ -0,0 +1,50 @@
+"""capture uptime
+
+Revision ID: d8f4a1c6e2b7
+Revises: d7a3c5e9f2b1
+Create Date: 2026-10-19 03:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "d8f4a1c6e2b7"
+down_revision: str | None = "d7a3c5e9f2b1"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_UPTIME = """
+CREATE TABLE capture_uptime (
+    account_id integer NOT NULL,
+    started_at timestamptz NOT NULL,
+    seen_at timestamptz NOT NULL,
+    PRIMARY KEY (account_id, started_at)
+)
+"""
+
+_BACKFILL = """
+INSERT INTO capture_uptime (account_id, started_at, seen_at)
+SELECT account_id, min(ts), max(ts)
+FROM (
+    SELECT account_id, ts,
+           count(*) FILTER (WHERE gap > interval '30 minutes')
+               OVER (PARTITION BY account_id ORDER BY ts) AS stretch
+    FROM (
+        SELECT account_id, ts,
+               ts - lag(ts) OVER (PARTITION BY account_id ORDER BY ts) AS gap
+        FROM presence
+    ) gaps
+) stretches
+GROUP BY account_id, stretch
+"""
+
+
+def upgrade() -> None:
+    op.execute(_UPTIME)
+    op.execute(_BACKFILL)
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS capture_uptime")

backend/migrations/versions/e8b3f1a6c2d4_profile_history_reads.py

@@ -0,0 +1,54 @@
+"""profile history reads
+
+Revision ID: e8b3f1a6c2d4
+Revises: d8e3b5a1c7f4
+Create Date: 2026-10-19 03:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "e8b3f1a6c2d4"
+down_revision: str | None = "d8e3b5a1c7f4"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_PEER_STATE = (
+    "first_name, last_name, username, phone, photo_unique_id, is_deleted_account"
+)
+
+
+def upgrade() -> None:
+    op.execute(
+        "ALTER TABLE peer_history DROP CONSTRAINT peer_history_pkey, "
+        "ADD CONSTRAINT peer_history_pkey "
+        f"PRIMARY KEY (account_id, peer_id, observed_at) INCLUDE ({_PEER_STATE})"
+    )
+    op.execute(
+        "ALTER TABLE chat_history DROP CONSTRAINT chat_history_pkey, "
+        "ADD CONSTRAINT chat_history_pkey "
+        "PRIMARY KEY (account_id, chat_id, message_id, ts)"
+    )
+    op.execute(
+        "CREATE INDEX ix_memberships_user ON memberships (account_id, user_id, ts DESC)"
+    )
+    op.execute(
+        "CREATE INDEX ix_jobs_kind_created ON jobs (account_id, kind, created_at DESC)"
+    )
+    with op.get_context().autocommit_block():
+        op.execute("VACUUM (ANALYZE) peer_history")
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS ix_jobs_kind_created")
+    op.execute("DROP INDEX IF EXISTS ix_memberships_user")
+    op.execute(
+        "ALTER TABLE chat_history DROP CONSTRAINT chat_history_pkey, "
+        "ADD CONSTRAINT chat_history_pkey PRIMARY KEY (account_id, chat_id, message_id)"
+    )
+    op.execute(
+        "ALTER TABLE peer_history DROP CONSTRAINT peer_history_pkey, "
+        "ADD CONSTRAINT peer_history_pkey PRIMARY KEY (account_id, peer_id, observed_at)"
+    )

backend/migrations/versions/f1c4a8e2d6b3_chat_stats_last_deleted.py

@@ -0,0 +1,97 @@
+"""chat stats last deleted
+
+Revision ID: f1c4a8e2d6b3
+Revises: d8f1a6c3e5b7
+Create Date: 2026-10-19 04:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "f1c4a8e2d6b3"
+down_revision: str | None = "d8f1a6c3e5b7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_APPLY = """
+CREATE OR REPLACE FUNCTION chat_stats_apply() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+    IF TG_OP = 'INSERT' THEN
+        INSERT INTO chat_stats AS cs (account_id, chat_id, message_count,
+                                      last_date, last_message_id,
+                                      last_text, last_sender_id{column})
+        VALUES (NEW.account_id, NEW.chat_id, 1,
+                CASE WHEN NEW.date <= now() + interval '1 day'
+                     THEN NEW.date END,
+                CASE WHEN NEW.date <= now() + interval '1 day'
+                     THEN NEW.message_id END,
+                NEW.text, NEW.sender_id{value})
+        ON CONFLICT (account_id, chat_id) DO UPDATE SET
+            message_count = cs.message_count + 1,
+            last_date = CASE WHEN chat_stats_newer(cs, EXCLUDED)
+                        THEN EXCLUDED.last_date ELSE cs.last_date END,
+            last_message_id = CASE WHEN chat_stats_newer(cs, EXCLUDED)
+                              THEN EXCLUDED.last_message_id
+                              ELSE cs.last_message_id END,
+            last_text = CASE WHEN chat_stats_newer(cs, EXCLUDED)
+                        THEN EXCLUDED.last_text ELSE cs.last_text END,
+            last_sender_id = CASE WHEN chat_stats_newer(cs, EXCLUDED)
+                             THEN EXCLUDED.last_sender_id
+                             ELSE cs.last_sender_id END{merge};
+    ELSE
+        UPDATE chat_stats
+        SET last_text = NEW.text, last_sender_id = NEW.sender_id{update}
+        WHERE account_id = NEW.account_id
+          AND chat_id = NEW.chat_id
+          AND last_message_id = NEW.message_id;
+    END IF;
+    RETURN NULL;
+END;
+$$
+"""
+
+_DELETED_PARTS = {
+    "column": ", last_deleted",
+    "value": ", NEW.deleted_at IS NOT NULL",
+    "merge": (
+        ",\n            last_deleted = CASE WHEN chat_stats_newer(cs, EXCLUDED)"
+        " THEN EXCLUDED.last_deleted ELSE cs.last_deleted END"
+    ),
+    "update": ", last_deleted = NEW.deleted_at IS NOT NULL",
+    "deleted": " OR OLD.deleted_at IS DISTINCT FROM NEW.deleted_at",
+}
+_PLAIN_PARTS = dict.fromkeys(_DELETED_PARTS, "")
+
+_UPDATE_TRIGGER = (
+    "CREATE TRIGGER messages_chat_stats_update AFTER UPDATE ON messages "
+    "FOR EACH ROW WHEN (OLD.text IS DISTINCT FROM NEW.text "
+    "OR OLD.sender_id IS DISTINCT FROM NEW.sender_id{deleted}) "
+    "EXECUTE FUNCTION chat_stats_apply()"
+)
+
+_BACKFILL = """
+UPDATE chat_stats cs SET last_deleted = true
+FROM messages m
+WHERE m.account_id = cs.account_id AND m.chat_id = cs.chat_id
+  AND m.message_id = cs.last_message_id AND m.date = cs.last_date
+  AND m.deleted_at IS NOT NULL
+"""
+
+
+def upgrade() -> None:
+    op.execute(
+        "ALTER TABLE chat_stats ADD COLUMN last_deleted boolean NOT NULL DEFAULT false"
+    )
+    op.execute(_BACKFILL)
+    op.execute(_APPLY.format(**_DELETED_PARTS))
+    op.execute("DROP TRIGGER IF EXISTS messages_chat_stats_update ON messages")
+    op.execute(_UPDATE_TRIGGER.format(**_DELETED_PARTS))
+
+
+def downgrade() -> None:
+    op.execute("DROP TRIGGER IF EXISTS messages_chat_stats_update ON messages")
+    op.execute(_UPDATE_TRIGGER.format(**_PLAIN_PARTS))
+    op.execute(_APPLY.format(**_PLAIN_PARTS))
+    op.execute("ALTER TABLE chat_stats DROP COLUMN last_deleted")

backend/migrations/versions/f8c2d6e1a4b7_capture_reliability.py

@@ -0,0 +1,39 @@
+"""capture reliability
+
+Revision ID: f8c2d6e1a4b7
+Revises: c9e2a7f4b1d8
+Create Date: 2026-10-19 03:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "f8c2d6e1a4b7"
+down_revision: str | None = "c9e2a7f4b1d8"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "jobs", sa.Column("run_after", sa.DateTime(timezone=True), nullable=True)
+    )
+    op.execute(
+        "ALTER TABLE accounts "
+        "ADD COLUMN catchup_pts integer, "
+        "ADD COLUMN catchup_qts integer, "
+        "ADD COLUMN catchup_date integer"
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        "ALTER TABLE accounts "
+        "DROP COLUMN IF EXISTS catchup_pts, "
+        "DROP COLUMN IF EXISTS catchup_qts, "
+        "DROP COLUMN IF EXISTS catchup_date"
+    )
+    op.drop_column("jobs", "run_after")

backend/pyproject.toml

@@ -20,10 +20,11 @@ dependencies = [
     "dishka>=1.4.0",
     "greenlet>=3.1.0",
     "fastmcp>=3.3",
+    "httpx2>=2.13.1",
 ]
 
 [build-system]
-requires = ["uv_build>=0.12.13,<0.13.0"]
+requires = ["uv_build>=0.13.0,<0.14.0"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

backend/src/api/access_log.py

@@ -1,19 +1,23 @@
 import logging
+import re
 
 QUERYLESS_PREFIXES = ("/mcp",)
 _PATH_ARG = 2
+_TOKEN = re.compile(r"(?<=[?&]token=)[^&]+")
 
 
-class DropQueryFilter(logging.Filter):
+class ScrubQueryFilter(logging.Filter):
     def filter(self, record: logging.LogRecord) -> bool:
         args = record.args
         if isinstance(args, tuple) and len(args) > _PATH_ARG:
             path = str(args[_PATH_ARG])
             if path.startswith(QUERYLESS_PREFIXES):
-                bare = path.partition("?")[0]
-                record.args = (*args[:_PATH_ARG], bare, *args[_PATH_ARG + 1 :])
+                path = path.partition("?")[0]
+            else:
+                path = _TOKEN.sub("***", path)
+            record.args = (*args[:_PATH_ARG], path, *args[_PATH_ARG + 1 :])
         return True
 
 
-def hide_mcp_query() -> None:
-    logging.getLogger("uvicorn.access").addFilter(DropQueryFilter())
+def scrub_access_log() -> None:
+    logging.getLogger("uvicorn.access").addFilter(ScrubQueryFilter())

backend/src/api/app.py

@@ -1,3 +1,4 @@
+import asyncio
 from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 from pathlib import Path
@@ -9,7 +10,7 @@ from fastapi.responses import FileResponse, RedirectResponse
 from fastmcp.utilities.lifespan import combine_lifespans
 from starlette.applications import Starlette
 
-from api.access_log import hide_mcp_query
+from api.access_log import scrub_access_log
 from api.auth import BearerAuthMiddleware
 from api.mcp.server import mcp
 from api.realtime import hub
@@ -20,10 +21,12 @@ from api.routers import (
     avatars,
     backfill,
     bots,
+    capture,
     chats,
     custom_emoji,
     discover,
     events,
+    feed,
     files,
     folders,
     media,
@@ -34,19 +37,24 @@ from api.routers import (
     search,
     shares,
     social,
+    storage,
     stories,
     watches,
+    webhooks,
 )
+from api.webhooks import outbox
 from dependencies.container import container
 from utils.cache import DAY_HEADERS, IMMUTABLE_HEADERS, NO_STORE_HEADERS
 from utils.env import env
+from utils.storage import ContentAddressedStorage
+from utils.trash import purge_loop
 
 if env.auth.token is None:
     msg = "AUTH__TOKEN is required for the api process"
     raise RuntimeError(msg)
 _token = env.auth.token.get_secret_value()
 
-hide_mcp_query()
+scrub_access_log()
 
 mcp_app = mcp.http_app(path="/")
 
@@ -54,8 +62,15 @@ mcp_app = mcp.http_app(path="/")
 @asynccontextmanager
 async def lifespan(app_: Starlette) -> AsyncGenerator[None]:
     pool = await container.get(asyncpg.Pool)
+    hub.alert_listeners.append(outbox.wake)
     await hub.start(pool)
+    purger = asyncio.create_task(
+        purge_loop(pool, await container.get(ContentAddressedStorage))
+    )
+    deliverer = asyncio.create_task(outbox.run(pool))
     yield
+    deliverer.cancel()
+    purger.cancel()
     await hub.stop()
     await app_.state.dishka_container.close()
 
@@ -83,6 +98,7 @@ app.include_router(folders.router)
 app.include_router(backfill.router)
 app.include_router(search.router)
 app.include_router(chats.router)
+app.include_router(feed.router)
 app.include_router(media.router)
 app.include_router(avatars.router)
 app.include_router(custom_emoji.router)
@@ -95,8 +111,11 @@ app.include_router(peers.router)
 app.include_router(discover.router)
 app.include_router(annotations.router)
 app.include_router(watches.router)
+app.include_router(webhooks.router)
 app.include_router(shares.router)
+app.include_router(storage.router)
 app.include_router(files.router)
+app.include_router(capture.router)
 
 app.mount("/mcp", mcp_app)
 

backend/src/api/mcp/server.py

@@ -1,3 +1,4 @@
+import re
 from collections.abc import Sequence
 from datetime import datetime
 from typing import Any
@@ -20,12 +21,27 @@ from api.mcp.download import (
 from api.mcp.format import build_transcript, load_notes, load_photos, resolve_names
 from dependencies.container import container
 from utils.files import resolve_mime
-from utils.read import annotations, chats, media, peers, presence, social, watches
+from utils.read import (
+    annotations,
+    chats,
+    feed,
+    media,
+    peers,
+    people,
+    presence,
+    presence_stats,
+    social,
+    watches,
+)
 from utils.read.accounts import self_user_id, unknown_account_hint
+from utils.read.feed import CURSOR_PATTERN, FeedChatKind, FeedFilter, FeedKind
 from utils.read.models import DEFAULT_LIMIT, Page
-from utils.search.models import SearchFilters
+from utils.read.period import DEFAULT_TZ, MAX_DAYS, known_timezone
+from utils.read.read_receipts import read_info
+from utils.search.models import ChatKind, SearchFilters
 from utils.search.repository import search_messages
 from utils.storage import ContentAddressedStorage
+from utils.watch_rules import WatchKind, WatchParamsError, clean_name, normalize_params
 
 INSTRUCTIONS = """\
 beavergram archives Telegram data (chats, messages, media, presence, stories,
@@ -56,17 +72,25 @@ Typical flows:
   "📝 [your private note, NOT in Telegram]" are the user's own annotations
   attached locally via the web UI; they never existed in Telegram. Treat them
   as private notes from the user to you, not as chat content.
-- Find something: `search_messages_tool` (full-text over message text and STT
-  transcripts; supports chat/sender/date filters and regex).
-- Forensics: `get_deleted_messages` and `get_message_versions` recover content
-  removed or edited in Telegram but kept in the archive.
+- Find something: `search_messages_tool` (words by prefix over message text,
+  STT transcripts and earlier versions of edited messages; filters by chat,
+  sender, chat kind, media, deleted, edited, dates, regex).
+- People: `get_person` gives one person in full: profile changes, shared groups,
+  joins and leaves, deleted message count, the private chat.
+- Forensics: `get_feed` lists what was deleted or edited in Telegram across
+  all chats, newest first, with the text before and after each edit; a bulk
+  deletion is one item with `message_count`. Its `profile` kind lists name,
+  username and photo changes of people and renames of chats. `get_deleted_messages` and
+  `get_message_versions` recover the content of one chat or one message.
 - Files: `download_media` gets any message attachment (photo, video, voice,
   document, ...). It pulls the file from Telegram if the archive lacks it and
   returns a public link valid 24h that needs no auth (`curl -O <link>`), plus
   the content itself for small images and text files. `get_media` returns
   metadata only.
-- Monitoring: `set_watch` creates a local rule, `list_watches` lists rules, and
-  `list_alerts` reads what those rules fired.
+- Monitoring: `set_watch` creates a local rule (kinds `keyword`,
+  `new_message`, `message_deleted`, `message_edited`, `peer_online`; params in
+  its description), `list_watches` lists rules, and `list_alerts` reads what
+  those rules fired, with chat, message preview and a web client `path`.
 
 Watched bots:
 - a bot whose token the user added is just another archived account: it has its
@@ -76,7 +100,8 @@ Watched bots:
 Writes: everything is read-only except `set_watch` and the share links
 `download_media` creates (visible and revocable in the web UI).
 
-Paging: list/search tools accept `limit` and `offset`.
+Paging: list tools accept `limit` and `offset`; `search_messages_tool` returns
+`next_cursor` to pass back as `cursor`.
 """
 
 mcp: FastMCP = FastMCP("beavergram", instructions=INSTRUCTIONS)
@@ -115,27 +140,49 @@ async def search_messages_tool(
     query: str | None = None,
     chat_id: int | None = None,
     sender_id: int | None = None,
+    kind: ChatKind | None = None,
     has_media: bool | None = None,
+    deleted: bool | None = None,
+    edited: bool | None = None,
     date_from: datetime | None = None,
     date_to: datetime | None = None,
     regex: str | None = None,
+    cursor: str | None = None,
     limit: int = DEFAULT_LIMIT,
-    offset: int = 0,
-) -> list[dict[str, Any]]:
-    """Full-text search over message text and STT transcripts."""
+) -> dict[str, Any]:
+    """Search messages by words, newest first, including deleted ones.
+
+    Words match by prefix (3+ letters) and by stem, Russian and English.
+    `"exact phrase"`, `-word` and `or` work as in web search. Besides the
+    current text it searches voice transcripts and earlier versions of edited
+    messages: a hit with `source="version"` matched only an old text, given in
+    `version_text`. Every hit carries `deleted_at` and `edited_at`.
+
+    Filters: `chat_id`, `sender_id` (peer id), `kind` (private, group,
+    channel), `has_media`, `deleted` / `edited` (true for only those, false to
+    exclude them), `date_from` / `date_to`, `regex` over the current text. With
+    no `query` it lists messages that pass the filters.
+
+    Paging: pass `next_cursor` from the previous result as `cursor`; it is null
+    on the last page.
+    """
     filters = SearchFilters(
         account_id=account_id,
         query=query,
         chat_id=chat_id,
         sender_id=sender_id,
+        kind=kind,
         has_media=has_media,
+        deleted=deleted,
+        edited=edited,
         date_from=date_from,
         date_to=date_to,
         regex=regex,
+        cursor=cursor,
         limit=limit,
-        offset=offset,
     )
-    return _dump(await search_messages(await _pool(), filters))
+    page = await search_messages(await _pool(), filters)
+    return page.model_dump(mode="json", exclude={"terms"})
 
 
 @mcp.tool
@@ -238,6 +285,41 @@ async def get_deleted_messages(
     )
 
 
+@mcp.tool
+async def get_feed(
+    account_id: int,
+    kind: FeedKind = FeedKind.ALL,
+    since: datetime | None = None,
+    chat_id: int | None = None,
+    chat_kind: FeedChatKind | None = None,
+    limit: int = DEFAULT_LIMIT,
+    cursor: str | None = None,
+) -> dict[str, Any]:
+    """Deleted and edited messages and profile changes across chats, newest first.
+
+    `kind` picks deleted, edited, profile, messages (deleted and edited) or all;
+    `since` keeps items at or after that time; `chat_id` or `chat_kind` (dm,
+    group, channel) narrow it down. Edited items carry `before` and `after`
+    texts. A deleted item with `message_count` above 1 is a bulk deletion in
+    one chat; read its messages with `get_deleted_messages` for that chat.
+    Profile items cover people in private chats or contacts and chats in the
+    dialog list: `detail` maps `name`, `username`, `photo` (unique file ids) or
+    `title` to `[old, new]`, changes a day apart or closer are merged. Pass
+    `next_cursor` back as `cursor` for the next page.
+    """
+    if cursor is not None and re.fullmatch(CURSOR_PATTERN, cursor) is None:
+        msg = "cursor must be a next_cursor returned by get_feed"
+        raise ToolError(msg)
+    page = await feed.list_feed(
+        await _pool(),
+        account_id,
+        FeedFilter(kind=kind, chat_id=chat_id, chat_kind=chat_kind, since=since),
+        limit=limit,
+        cursor=cursor,
+    )
+    return page.model_dump(mode="json")
+
+
 @mcp.tool
 async def get_message_versions(
     account_id: int, chat_id: int, message_id: int
@@ -295,6 +377,23 @@ async def get_callbacks(
     return _dump(items)
 
 
+@mcp.tool
+async def get_read_info(
+    account_id: int, chat_id: int, message_id: int
+) -> dict[str, Any] | None:
+    """When the user's own message was read, from the archive only.
+
+    `observed_at` is when the archive saw the chat read up to this message,
+    `exact_at` the read date Telegram reported for a private chat, `readers`
+    who read it in a group and when, `listened_at` when a voice or round
+    video was played. `outcome` is the last Telegram answer: ok, not_read_yet,
+    user_privacy (they hide read time), your_privacy (the user hides theirs),
+    chat_too_big, too_old, unavailable, flood or error.
+    """
+    info = await read_info(await _pool(), account_id, chat_id, message_id)
+    return info.model_dump(mode="json") if info else None
+
+
 @mcp.tool
 async def presence_history(
     account_id: int,
@@ -317,12 +416,58 @@ async def presence_history(
     )
 
 
+@mcp.tool(name="presence_stats")
+async def presence_stats_tool(
+    account_id: int,
+    peer_id: int,
+    days: int = 30,
+    tz: str = DEFAULT_TZ,
+    limit: int = DEFAULT_LIMIT,
+    offset: int = 0,
+) -> dict[str, Any]:
+    """Time a peer spent online, from status pushes turned into online sessions.
+
+    `tz` is an IANA zone name; days and hours are counted in it. `week` holds
+    the average online minutes for each weekday (0 is Monday) and hour, null
+    where the archive has no data for that hour. `days` gives online seconds
+    per day, `gaps` the stretches when capture was down, `sessions` the latest
+    sessions (online stretches with breaks under 5 minutes), paged by
+    `limit`/`offset`. `hidden` means the peer hides their last seen time.
+    """
+    pool = await _pool()
+    if not 1 <= days <= MAX_DAYS:
+        msg = f"days must be between 1 and {MAX_DAYS}"
+        raise ToolError(msg)
+    if not await known_timezone(pool, tz):
+        msg = f"Unknown time zone: {tz}"
+        raise ToolError(msg)
+    stats = await presence_stats.presence_stats(
+        pool, account_id, peer_id, Page(limit=limit, offset=offset), days=days, tz=tz
+    )
+    return stats.model_dump(mode="json")
+
+
 @mcp.tool
 async def get_peer_history(account_id: int, peer_id: int) -> list[dict[str, Any]]:
     """Get name/username/avatar change history of a contact."""
     return _dump(await peers.get_peer_history(await _pool(), account_id, peer_id))
 
 
+@mcp.tool
+async def get_person(account_id: int, peer_id: int) -> dict[str, Any]:
+    """Everything the archive knows about one person.
+
+    Current profile, name/username/phone changes with times (newest first),
+    avatar history, groups shared in the archive with their message counts and
+    last message, joins and leaves with who added or removed them, the number
+    of their deleted messages, and the private chat if there is one.
+    """
+    pool = await _pool()
+    overview = await people.get_overview(pool, account_id, peer_id)
+    groups = await people.get_groups(pool, account_id, peer_id)
+    return {**overview.model_dump(mode="json"), **groups.model_dump(mode="json")}
+
+
 @mcp.tool
 async def get_stories(
     account_id: int,
@@ -361,13 +506,37 @@ async def get_annotations(
 @mcp.tool
 async def set_watch(
     account_id: int,
-    kind: str,
+    kind: WatchKind,
     params: dict[str, Any] | None = None,
+    name: str | None = None,
     enabled: bool = True,
 ) -> dict[str, Any]:
-    """Create a local watch rule (the only local write MCP is allowed)."""
+    """Create a local watch rule (the only local write MCP is allowed).
+
+    A watch fires an alert into the web client (live counter and list) and to
+    the user's webhooks. The user's own messages never fire. Kinds and params:
+
+    - `keyword`: `{"pattern": str, "regex": bool = false, "chat_id"?: int,
+      "sender_id"?: int}`. Case-insensitive substring, or a Python regex when
+      `regex` is true (at most 200 characters, no nested repeats like
+      `(a+)+`). Checked on new messages, on edits (the new text) and on voice
+      transcripts. Without `chat_id` it watches every chat.
+    - `new_message`: `{"chat_id"?: int, "sender_id"?: int}`, at least one: any
+      new message in that chat, or from that person anywhere, or both.
+    - `message_deleted`: `{"chat_id"?: int}`. A stored message deleted in that
+      chat; without `chat_id`, in any private chat.
+    - `message_edited`: `{"chat_id"?: int}`. Same scopes as `message_deleted`.
+    - `peer_online`: `{"peer_id": int}`. The person comes online after at
+      least 15 minutes away.
+
+    `name` is an optional human label shown in the list and in alerts.
+    """
+    try:
+        clean = normalize_params(kind, params or {})
+    except WatchParamsError as exc:
+        raise ToolError(str(exc)) from exc
     watch = await watches.create_watch(
-        await _pool(), account_id, kind, params or {}, enabled=enabled
+        await _pool(), account_id, kind, clean, name=clean_name(name), enabled=enabled
     )
     return watch.model_dump(mode="json")
 
@@ -382,12 +551,19 @@ async def list_watches(account_id: int) -> list[dict[str, Any]]:
 async def list_alerts(
     account_id: int,
     seen: bool | None = None,
+    before: int | None = None,
     limit: int = DEFAULT_LIMIT,
-    offset: int = 0,
-) -> list[dict[str, Any]]:
-    """List fired alerts from watch rules."""
-    return _dump(
-        await watches.list_alerts(
-            await _pool(), account_id, Page(limit=limit, offset=offset), seen=seen
-        )
+) -> dict[str, Any]:
+    """List fired alerts from watch rules, newest first.
+
+    Each alert names its watch (id, kind, name), the event (`message`, `edit`,
+    `transcript`, `delete`, `online`), the chat, the message preview with its
+    sender and deleted/edited flags, the peer for `peer_online`, and `path`,
+    the web client link. `seen=false` keeps unseen ones. Page older with
+    `before` set to `next_before` from the previous call; `unseen` is the
+    account's unseen count.
+    """
+    page = await watches.list_alerts(
+        await _pool(), account_id, seen=seen, before=before, limit=limit
     )
+    return page.model_dump(mode="json")

backend/src/api/realtime.py

@@ -1,14 +1,19 @@
 import asyncio
 import json
 import logging
-from typing import Any
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any
 
 import asyncpg
 
-from utils.env import env
 from utils.events import BG_EVENTS_CHANNEL
+from utils.listen import Listener
 from utils.read import chats as chats_read
 from utils.read import presence as presence_read
+from utils.read import watches as watches_read
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
 
 logger = logging.getLogger(__name__)
 
@@ -26,8 +31,15 @@ class EventHub:
     def __init__(self) -> None:
         self._subscribers: set[Subscriber] = set()
         self._pool: asyncpg.Pool | None = None
-        self._conn: asyncpg.Connection | None = None
+        self._listener = Listener(
+            "realtime",
+            {
+                BG_EVENTS_CHANNEL: self._on_notify,
+                watches_read.ALERTS_CHANGED_CHANNEL: self._on_alert,
+            },
+        )
         self._tasks: set[asyncio.Task] = set()
+        self.alert_listeners: list[Callable[[], None]] = []
 
     def subscribe(self, account_id: int, chat_id: int | None) -> Subscriber:
         sub = Subscriber(account_id, chat_id)
@@ -39,25 +51,51 @@ class EventHub:
 
     async def start(self, pool: asyncpg.Pool) -> None:
         self._pool = pool
-        conn = await asyncpg.connect(dsn=env.db.connection_url)
-        await conn.add_listener(BG_EVENTS_CHANNEL, self._on_notify)
-        self._conn = conn
-        logger.info("Realtime hub listening on %s", BG_EVENTS_CHANNEL)
+        await self._listener.start()
 
     async def stop(self) -> None:
         for task in self._tasks:
             task.cancel()
-        if self._conn is not None:
-            await self._conn.close()
-            self._conn = None
+        await self._listener.stop()
 
-    def _on_notify(
-        self, _conn: asyncpg.Connection, _pid: int, _channel: str, payload: str
-    ) -> None:
+    def _on_notify(self, payload: str) -> None:
         task = asyncio.create_task(self._dispatch(payload))
         self._tasks.add(task)
         task.add_done_callback(self._tasks.discard)
 
+    def _on_alert(self, payload: str) -> None:
+        for listener in self.alert_listeners:
+            listener()
+        task = asyncio.create_task(self._dispatch_alert(payload))
+        self._tasks.add(task)
+        task.add_done_callback(self._tasks.discard)
+
+    async def _dispatch_alert(self, payload: str) -> None:
+        try:
+            event = json.loads(payload)
+        except json.JSONDecodeError:
+            return
+        account_id = event.get("account_id")
+        targets = [
+            sub
+            for sub in self._subscribers
+            if sub.account_id == account_id and sub.chat_id is None
+        ]
+        if not targets or self._pool is None:
+            return
+        alert_id = event.get("alert_id")
+        item = await watches_read.get_alert(self._pool, alert_id) if alert_id else None
+        frame = {
+            "type": "alert",
+            "alert": item.model_dump(mode="json") if item else None,
+            "unseen": await watches_read.unseen_count(self._pool, account_id),
+        }
+        for sub in targets:
+            try:
+                sub.queue.put_nowait(frame)
+            except asyncio.QueueFull:
+                logger.warning("Dropping alert for slow subscriber")
+
     async def _dispatch(self, payload: str) -> None:
         try:
             event = json.loads(payload)
@@ -65,16 +103,12 @@ class EventHub:
             return
         account_id = event.get("account_id")
         chat_id = event.get("chat_id")
-        scoped = event.get("kind") == "presence"
+        account_wide = event.get("kind") == "presence"
         targets = [
             sub
             for sub in self._subscribers
             if sub.account_id == account_id
-            and (
-                sub.chat_id == chat_id
-                if scoped
-                else sub.chat_id is None or sub.chat_id == chat_id
-            )
+            and (account_wide or sub.chat_id is None or sub.chat_id == chat_id)
         ]
         if not targets:
             return
@@ -121,6 +155,7 @@ class EventHub:
                 "type": "receipt",
                 "chat_id": event["chat_id"],
                 "read_up_to": event["message_id"],
+                "read_at": datetime.now(UTC).isoformat(),
             }
         return None
 

backend/src/api/routers/analytics.py

@@ -4,8 +4,10 @@ import asyncpg
 from dishka.integrations.fastapi import DishkaRoute, FromDishka
 from fastapi import APIRouter, Query
 
+from api.timezone import checked_tz
 from utils.read import analytics
 from utils.read.models import ResponseStats, VolumeBucket
+from utils.read.period import DEFAULT_TZ, MAX_DAYS
 
 router = APIRouter(prefix="/api/analytics", tags=["analytics"], route_class=DishkaRoute)
 
@@ -18,9 +20,12 @@ async def volume(
     pool: FromDishka[asyncpg.Pool],
     account_id: AccountId,
     chat_id: ChatId,
-    days: Annotated[int, Query()] = 90,
+    days: Annotated[int, Query(ge=1, le=MAX_DAYS)] = 90,
+    tz: Annotated[str, Query()] = DEFAULT_TZ,
 ) -> list[VolumeBucket]:
-    return await analytics.message_volume(pool, account_id, chat_id, days=days)
+    return await analytics.message_volume(
+        pool, account_id, chat_id, days=days, tz=await checked_tz(pool, tz)
+    )
 
 
 @router.get("/response-time")

backend/src/api/routers/annotations.py

@@ -6,7 +6,7 @@ from fastapi import APIRouter, HTTPException, Query
 from pydantic import BaseModel
 
 from utils.read import annotations
-from utils.read.models import DEFAULT_LIMIT, AnnotationView, Page
+from utils.read.models import DEFAULT_LIMIT, AnnotationView, NoteTag, NoteView, Page
 
 router = APIRouter(
     prefix="/api/annotations", tags=["annotations"], route_class=DishkaRoute
@@ -42,6 +42,26 @@ async def list_annotations(
     )
 
 
+@router.get("/notes")
+async def list_notes(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: Annotated[int, Query()],
+    tag: Annotated[str | None, Query()] = None,
+    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
+    offset: Annotated[int, Query()] = 0,
+) -> list[NoteView]:
+    return await annotations.list_notes(
+        pool, account_id, Page(limit=limit, offset=offset), tag=tag
+    )
+
+
+@router.get("/tags")
+async def list_tags(
+    pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
+) -> list[NoteTag]:
+    return await annotations.list_tags(pool, account_id)
+
+
 @router.post("", status_code=201)
 async def create_annotation(
     pool: FromDishka[asyncpg.Pool], body: AnnotationCreate

backend/src/api/routers/capture.py

@@ -0,0 +1,32 @@
+from datetime import datetime
+from typing import Annotated
+
+import asyncpg
+from dishka.integrations.fastapi import DishkaRoute, FromDishka
+from fastapi import APIRouter, Query
+
+from utils.read import capture
+from utils.read.models import CaptureGap, CaptureStatus
+
+router = APIRouter(prefix="/api/capture", tags=["capture"], route_class=DishkaRoute)
+
+AccountId = Annotated[int, Query()]
+
+
+@router.get("/status")
+async def capture_status(
+    pool: FromDishka[asyncpg.Pool], account_id: AccountId
+) -> CaptureStatus:
+    return await capture.capture_status(pool, account_id)
+
+
+@router.get("/gaps")
+async def capture_gaps(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: AccountId,
+    date_from: Annotated[datetime | None, Query()] = None,
+    date_to: Annotated[datetime | None, Query()] = None,
+) -> list[CaptureGap]:
+    return await capture.capture_gaps(
+        pool, account_id, date_from=date_from, date_to=date_to
+    )

backend/src/api/routers/feed.py

@@ -0,0 +1,70 @@
+from datetime import datetime
+from typing import Annotated
+
+import asyncpg
+from dishka.integrations.fastapi import DishkaRoute, FromDishka
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel
+
+from utils.read import feed
+from utils.read.feed import CURSOR_PATTERN, FeedChatKind, FeedFilter, FeedKind
+from utils.read.models import DEFAULT_LIMIT, FeedMessagesPage, FeedPage, FeedUnseen
+
+router = APIRouter(prefix="/api/feed", tags=["feed"], route_class=DishkaRoute)
+
+AccountId = Annotated[int, Query()]
+Limit = Annotated[int, Query(ge=1)]
+Cursor = Annotated[str | None, Query(pattern=CURSOR_PATTERN)]
+
+
+class SeenRequest(BaseModel):
+    account_id: int
+    at: datetime | None = None
+
+
+@router.get("")
+async def list_feed(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: AccountId,
+    kind: Annotated[FeedKind, Query()] = FeedKind.ALL,
+    chat_id: Annotated[int | None, Query()] = None,
+    chat_kind: Annotated[FeedChatKind | None, Query()] = None,
+    since: Annotated[datetime | None, Query()] = None,
+    cursor: Cursor = None,
+    limit: Limit = DEFAULT_LIMIT,
+) -> FeedPage:
+    return await feed.list_feed(
+        pool,
+        account_id,
+        FeedFilter(kind=kind, chat_id=chat_id, chat_kind=chat_kind, since=since),
+        limit=limit,
+        cursor=cursor,
+    )
+
+
+@router.get("/unseen")
+async def feed_unseen(
+    pool: FromDishka[asyncpg.Pool], account_id: AccountId
+) -> FeedUnseen:
+    return await feed.unseen(pool, account_id)
+
+
+@router.post("/seen")
+async def feed_seen(pool: FromDishka[asyncpg.Pool], body: SeenRequest) -> FeedUnseen:
+    return await feed.mark_seen(pool, body.account_id, body.at)
+
+
+@router.get("/{item_id}/messages")
+async def feed_item_messages(
+    pool: FromDishka[asyncpg.Pool],
+    item_id: int,
+    account_id: AccountId,
+    cursor: Cursor = None,
+    limit: Limit = DEFAULT_LIMIT,
+) -> FeedMessagesPage:
+    page = await feed.item_messages(
+        pool, account_id, item_id, limit=limit, cursor=cursor
+    )
+    if page is None:
+        raise HTTPException(status_code=404, detail="Feed item not found")
+    return page

backend/src/api/routers/media.py

@@ -54,7 +54,7 @@ async def serve_media_version(
     download: Download = False,
 ) -> FileResponse:
     version = await get_media_version(pool, version_id)
-    if version is None:
+    if version is None or not storage.exists(version.storage_key):
         raise HTTPException(status_code=404, detail="media version not found")
     headers = IMMUTABLE_HEADERS
     if download:
@@ -96,6 +96,8 @@ async def serve_media(
             status_code=409,
             detail="media not downloaded; enqueue fetch via POST /api/media/fetch",
         )
+    if not storage.exists(media.storage_key):
+        raise HTTPException(status_code=404, detail="media file is missing")
     headers = DAY_HEADERS
     if download:
         headers = _attachment(headers, media.file_name or f"media_{media_id}")

backend/src/api/routers/peers.py

@@ -4,8 +4,14 @@ import asyncpg
 from dishka.integrations.fastapi import DishkaRoute, FromDishka
 from fastapi import APIRouter, HTTPException, Query
 
-from utils.read import peers
-from utils.read.models import PeerHistoryView, PeerView
+from utils.read import peers, people
+from utils.read.models import (
+    PeerGroups,
+    PeerHistoryView,
+    PeerOverview,
+    PeerView,
+    ProfileChange,
+)
 
 router = APIRouter(prefix="/api", tags=["peers"], route_class=DishkaRoute)
 
@@ -35,3 +41,24 @@ async def peer_history(
     pool: FromDishka[asyncpg.Pool], peer_id: int, account_id: AccountId
 ) -> list[PeerHistoryView]:
     return await peers.get_peer_history(pool, account_id, peer_id)
+
+
+@router.get("/peers/{peer_id}/overview")
+async def peer_overview(
+    pool: FromDishka[asyncpg.Pool], peer_id: int, account_id: AccountId
+) -> PeerOverview:
+    return await people.get_overview(pool, account_id, peer_id)
+
+
+@router.get("/peers/{peer_id}/groups")
+async def peer_groups(
+    pool: FromDishka[asyncpg.Pool], peer_id: int, account_id: AccountId
+) -> PeerGroups:
+    return await people.get_groups(pool, account_id, peer_id)
+
+
+@router.get("/chats/{chat_id}/changes")
+async def chat_changes(
+    pool: FromDishka[asyncpg.Pool], chat_id: int, account_id: AccountId
+) -> list[ProfileChange]:
+    return await people.get_chat_changes(pool, account_id, chat_id)

backend/src/api/routers/policy.py

@@ -5,13 +5,15 @@ from dishka.integrations.fastapi import DishkaRoute, FromDishka
 from fastapi import APIRouter, HTTPException, Query
 from pydantic import BaseModel
 
-from utils.policy import repository
+from utils.policy import overview, repository
 from utils.policy.models import (
     CaptureToggles,
     ChatKind,
     ChatMeta,
     PolicyCreate,
     PolicyRecord,
+    PolicyRuleView,
+    PolicyScopeView,
 )
 from utils.policy.resolver import resolve
 
@@ -50,6 +52,32 @@ async def effective_policy(
     return resolve(chat, folders, policies)
 
 
+@router.get("/overview")
+async def policy_overview(
+    pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
+) -> list[PolicyRuleView]:
+    return await overview.list_rules(pool, account_id)
+
+
+@router.get("/scope")
+async def policy_scope(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: Annotated[int, Query()],
+    chat_id: Annotated[int | None, Query()] = None,
+    folder_id: Annotated[int | None, Query()] = None,
+) -> PolicyScopeView:
+    if chat_id is not None:
+        return await overview.chat_scope(pool, account_id, chat_id)
+    scope = (
+        await overview.folder_scope(pool, account_id, folder_id)
+        if folder_id is not None
+        else None
+    )
+    if scope is None:
+        raise HTTPException(status_code=404, detail="scope not found")
+    return scope
+
+
 @router.post("", status_code=201)
 async def create_policy(
     pool: FromDishka[asyncpg.Pool], body: PolicyCreate

backend/src/api/routers/presence.py

@@ -5,8 +5,18 @@ import asyncpg
 from dishka.integrations.fastapi import DishkaRoute, FromDishka
 from fastapi import APIRouter, Query
 
-from utils.read import presence
-from utils.read.models import DEFAULT_LIMIT, Page, PresenceHourly, PresenceSample
+from api.timezone import checked_tz
+from utils.read import presence, presence_stats
+from utils.read.models import (
+    DEFAULT_LIMIT,
+    MAX_LIMIT,
+    Page,
+    PresenceHourly,
+    PresenceSample,
+    PresenceSession,
+    PresenceStats,
+)
+from utils.read.period import DEFAULT_TZ, MAX_DAYS
 
 router = APIRouter(prefix="/api/presence", tags=["presence"], route_class=DishkaRoute)
 
@@ -14,6 +24,8 @@ AccountId = Annotated[int, Query()]
 PeerId = Annotated[int, Query()]
 DateFrom = Annotated[datetime | None, Query()]
 DateTo = Annotated[datetime | None, Query()]
+Days = Annotated[int, Query(ge=1, le=MAX_DAYS)]
+Tz = Annotated[str, Query()]
 
 
 @router.get("")
@@ -43,6 +55,16 @@ async def current_presence(
     return await presence.current_presence(pool, account_id, peer_id)
 
 
+@router.get("/online")
+async def online_now(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: AccountId,
+    peer_ids: Annotated[str, Query()],
+) -> list[PresenceSample]:
+    parsed = [int(part) for part in peer_ids.split(",") if part.strip()]
+    return await presence.online_now(pool, account_id, parsed[:MAX_LIMIT])
+
+
 @router.get("/hourly")
 async def presence_hourly(
     pool: FromDishka[asyncpg.Pool],
@@ -54,3 +76,43 @@ async def presence_hourly(
     return await presence.presence_hourly(
         pool, account_id, peer_id, date_from=date_from, date_to=date_to
     )
+
+
+@router.get("/stats")
+async def stats(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: AccountId,
+    peer_id: PeerId,
+    days: Days = 30,
+    tz: Tz = DEFAULT_TZ,
+    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
+    offset: Annotated[int, Query()] = 0,
+) -> PresenceStats:
+    return await presence_stats.presence_stats(
+        pool,
+        account_id,
+        peer_id,
+        Page(limit=limit, offset=offset),
+        days=days,
+        tz=await checked_tz(pool, tz),
+    )
+
+
+@router.get("/sessions")
+async def sessions(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: AccountId,
+    peer_id: PeerId,
+    days: Days = 30,
+    tz: Tz = DEFAULT_TZ,
+    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
+    offset: Annotated[int, Query()] = 0,
+) -> list[PresenceSession]:
+    return await presence_stats.presence_sessions(
+        pool,
+        account_id,
+        peer_id,
+        Page(limit=limit, offset=offset),
+        days=days,
+        tz=await checked_tz(pool, tz),
+    )

backend/src/api/routers/search.py

@@ -4,7 +4,7 @@ import asyncpg
 from dishka.integrations.fastapi import DishkaRoute, FromDishka
 from fastapi import APIRouter, Query
 
-from utils.search.models import SearchFilters, SearchHit
+from utils.search.models import SearchFilters, SearchPage
 from utils.search.repository import search_messages
 
 router = APIRouter(prefix="/api/search", tags=["search"], route_class=DishkaRoute)
@@ -13,5 +13,5 @@ router = APIRouter(prefix="/api/search", tags=["search"], route_class=DishkaRout
 @router.get("")
 async def search(
     pool: FromDishka[asyncpg.Pool], filters: Annotated[SearchFilters, Query()]
-) -> list[SearchHit]:
+) -> SearchPage:
     return await search_messages(pool, filters)

backend/src/api/routers/social.py

@@ -2,10 +2,13 @@ from typing import Annotated
 
 import asyncpg
 from dishka.integrations.fastapi import DishkaRoute, FromDishka
-from fastapi import APIRouter, Query
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel
 
+from utils.jobs import enqueue
 from utils.read import social
-from utils.read.models import CallbackView, LinkView, ReactionView
+from utils.read.models import CallbackView, LinkView, ReactionView, ReadInfoView
+from utils.read.read_receipts import read_info
 
 router = APIRouter(
     prefix="/api/messages/{chat_id}/{message_id}",
@@ -16,6 +19,10 @@ router = APIRouter(
 AccountId = Annotated[int, Query()]
 
 
+class RefreshRequest(BaseModel):
+    account_id: int
+
+
 @router.get("/callbacks")
 async def message_callbacks(
     pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, account_id: AccountId
@@ -35,3 +42,29 @@ async def message_links(
     pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, account_id: AccountId
 ) -> list[LinkView]:
     return await social.get_links(pool, account_id, chat_id, message_id)
+
+
+@router.get("/read")
+async def message_read(
+    pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, account_id: AccountId
+) -> ReadInfoView:
+    info = await read_info(pool, account_id, chat_id, message_id)
+    if info is None:
+        raise HTTPException(status_code=404, detail="message not found")
+    return info
+
+
+@router.post("/read/refresh")
+async def refresh_message_read(
+    pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, body: RefreshRequest
+) -> dict[str, int | None]:
+    info = await read_info(pool, body.account_id, chat_id, message_id)
+    if info is None or not info.can_fetch:
+        return {"job_id": None}
+    job_id = await enqueue(
+        pool,
+        body.account_id,
+        "fetch_read_info",
+        {"chat_id": chat_id, "message_id": message_id},
+    )
+    return {"job_id": job_id}

backend/src/api/routers/storage.py

@@ -0,0 +1,166 @@
+from typing import Annotated
+
+import asyncpg
+from dishka.integrations.fastapi import DishkaRoute, FromDishka
+from fastapi import APIRouter, HTTPException, Query
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
+
+from utils import trash
+from utils.cache import NO_STORE_HEADERS
+from utils.files import resolve_mime
+from utils.read import storage as storage_read
+from utils.read.models import (
+    DEFAULT_LIMIT,
+    Page,
+    StorageChatView,
+    StorageFileView,
+    StorageScopeView,
+    StorageSummaryView,
+    TrashItemView,
+    TrashOutcome,
+)
+from utils.storage import ContentAddressedStorage
+
+router = APIRouter(prefix="/api/storage", tags=["storage"], route_class=DishkaRoute)
+
+
+class TrashRequest(BaseModel):
+    account_id: int
+    media_ids: list[int]
+
+
+class RestoreRequest(BaseModel):
+    account_id: int
+    keys: list[str] | None = None
+
+
+class ScopeRequest(BaseModel):
+    account_id: int
+    chat_id: int | None = None
+    folder_id: int | None = None
+
+
+class PurgeRequest(BaseModel):
+    account_id: int
+    keys: list[str] | None = None
+
+
+@router.get("/summary")
+async def summary(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: Annotated[int, Query()],
+    chat_id: Annotated[int | None, Query()] = None,
+) -> StorageSummaryView:
+    return await storage_read.get_summary(pool, account_id, chat_id)
+
+
+@router.get("/files")
+async def files(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: Annotated[int, Query()],
+    group: Annotated[str | None, Query()] = None,
+    chat_id: Annotated[int | None, Query()] = None,
+    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
+    offset: Annotated[int, Query()] = 0,
+) -> list[StorageFileView]:
+    page = Page(limit=limit, offset=offset)
+    return await storage_read.list_files(pool, account_id, group, page, chat_id)
+
+
+@router.get("/chats")
+async def chats(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: Annotated[int, Query()],
+    group: Annotated[str | None, Query()] = None,
+    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
+    offset: Annotated[int, Query()] = 0,
+) -> list[StorageChatView]:
+    page = Page(limit=limit, offset=offset)
+    return await storage_read.list_chats(pool, account_id, group, page)
+
+
+async def _chat_ids(
+    pool: asyncpg.Pool, account_id: int, chat_id: int | None, folder_id: int | None
+) -> list[int]:
+    chat_ids = await storage_read.scope_chat_ids(pool, account_id, chat_id, folder_id)
+    if chat_ids is None:
+        raise HTTPException(status_code=404, detail="scope not found")
+    return chat_ids
+
+
+@router.get("/scope")
+async def scope(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: Annotated[int, Query()],
+    chat_id: Annotated[int | None, Query()] = None,
+    folder_id: Annotated[int | None, Query()] = None,
+) -> StorageScopeView:
+    chat_ids = await _chat_ids(pool, account_id, chat_id, folder_id)
+    return await storage_read.get_scope(pool, account_id, chat_ids)
+
+
+@router.get("/trash")
+async def trash_items(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: Annotated[int, Query()],
+    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
+    offset: Annotated[int, Query()] = 0,
+) -> list[TrashItemView]:
+    page = Page(limit=limit, offset=offset)
+    return await storage_read.list_trash(pool, account_id, page)
+
+
+@router.post("/trash/scope")
+async def trash_scope(
+    pool: FromDishka[asyncpg.Pool],
+    storage: FromDishka[ContentAddressedStorage],
+    body: ScopeRequest,
+) -> TrashOutcome:
+    chat_ids = await _chat_ids(pool, body.account_id, body.chat_id, body.folder_id)
+    blobs = await pool.fetch(storage_read.SCOPE_BLOBS, body.account_id, chat_ids)
+    free = [blob for blob in blobs if blob["free"]]
+    return await trash.trash_blobs(pool, storage, body.account_id, free)
+
+
+@router.post("/trash")
+async def move_to_trash(
+    pool: FromDishka[asyncpg.Pool],
+    storage: FromDishka[ContentAddressedStorage],
+    body: TrashRequest,
+) -> TrashOutcome:
+    return await trash.move_to_trash(pool, storage, body.account_id, body.media_ids)
+
+
+@router.post("/trash/restore")
+async def restore(
+    pool: FromDishka[asyncpg.Pool],
+    storage: FromDishka[ContentAddressedStorage],
+    body: RestoreRequest,
+) -> TrashOutcome:
+    return await trash.restore(pool, storage, body.account_id, body.keys)
+
+
+@router.post("/trash/purge")
+async def purge(
+    pool: FromDishka[asyncpg.Pool],
+    storage: FromDishka[ContentAddressedStorage],
+    body: PurgeRequest,
+) -> TrashOutcome:
+    return await trash.purge(pool, storage, body.account_id, body.keys)
+
+
+@router.get("/trash/{storage_key}/file")
+async def serve_trashed(
+    pool: FromDishka[asyncpg.Pool],
+    storage: FromDishka[ContentAddressedStorage],
+    storage_key: str,
+) -> FileResponse:
+    item = await storage_read.get_trash_item(pool, storage_key)
+    if item is None or not storage.trashed(storage_key):
+        raise HTTPException(status_code=404, detail="file is not in trash")
+    return FileResponse(
+        storage.trash_url(storage_key),
+        media_type=resolve_mime(item["kind"], item["mime"], item["file_name"]),
+        headers=NO_STORE_HEADERS,
+    )

backend/src/api/routers/stories.py

@@ -5,7 +5,7 @@ from dishka.integrations.fastapi import DishkaRoute, FromDishka
 from fastapi import APIRouter, HTTPException, Query
 from fastapi.responses import FileResponse
 
-from utils.files import content_disposition, story_file_name
+from utils.files import content_disposition, sniff_media_kind, story_file_name
 from utils.read import peers
 from utils.read.models import DEFAULT_LIMIT, Page, StoryView
 from utils.storage import ContentAddressedStorage
@@ -17,17 +17,34 @@ AccountId = Annotated[int, Query()]
 _STORY_MIME = {"photo": "image/jpeg", "video": "video/mp4"}
 
 
+async def _with_kind(
+    pool: asyncpg.Pool,
+    storage: ContentAddressedStorage,
+    account_id: int,
+    story: StoryView,
+) -> StoryView:
+    if story.media_kind or not story.storage_key:
+        return story
+    if not storage.exists(story.storage_key):
+        return story
+    kind = sniff_media_kind(storage.head(story.storage_key))
+    await peers.set_story_kind(pool, account_id, story.peer_id, story.story_id, kind)
+    return story.model_copy(update={"media_kind": kind})
+
+
 @router.get("/stories")
 async def list_stories(
     pool: FromDishka[asyncpg.Pool],
+    storage: FromDishka[ContentAddressedStorage],
     account_id: AccountId,
     peer_id: Annotated[int | None, Query()] = None,
     limit: Annotated[int, Query()] = DEFAULT_LIMIT,
     offset: Annotated[int, Query()] = 0,
 ) -> list[StoryView]:
-    return await peers.get_stories(
+    stories = await peers.get_stories(
         pool, account_id, Page(limit=limit, offset=offset), peer_id=peer_id
     )
+    return [await _with_kind(pool, storage, account_id, story) for story in stories]
 
 
 @router.get("/stories/{peer_id}/{story_id}/media")

backend/src/api/routers/watches.py

@@ -6,55 +6,88 @@ from fastapi import APIRouter, HTTPException, Query
 from pydantic import BaseModel
 
 from utils.read import watches
-from utils.read.models import DEFAULT_LIMIT, AlertView, Page, WatchView
+from utils.read.models import AlertPage, WatchItem, WatchView
+from utils.watch_rules import WatchParamsError, clean_name, normalize_params
 
 router = APIRouter(prefix="/api", tags=["watches"], route_class=DishkaRoute)
 
 
-class WatchCreate(BaseModel):
+class WatchUpdate(BaseModel):
+    kind: str | None = None
+    params: dict[str, Any] = {}
+    name: str | None = None
+    enabled: bool = True
+
+
+class WatchCreate(WatchUpdate):
     account_id: int
     kind: str
-    params: dict[str, Any] = {}
-    enabled: bool = True
 
 
-class WatchUpdate(BaseModel):
-    params: dict[str, Any] = {}
-    enabled: bool = True
+class SeenAll(BaseModel):
+    account_id: int
+    up_to: int | None = None
+
+
+def _params(kind: str, params: dict[str, Any]) -> dict[str, Any]:
+    try:
+        return normalize_params(kind, params)
+    except WatchParamsError as exc:
+        raise HTTPException(status_code=422, detail=str(exc)) from exc
+
+
+async def _item(pool: asyncpg.Pool, watch: WatchView) -> WatchItem:
+    return (await watches.describe_watches(pool, watch.account_id, [watch]))[0]
 
 
 @router.get("/watches")
 async def list_watches(
     pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
-) -> list[WatchView]:
-    return await watches.list_watches(pool, account_id)
+) -> list[WatchItem]:
+    rows = await watches.list_watches(pool, account_id)
+    return await watches.describe_watches(pool, account_id, rows)
 
 
 @router.post("/watches", status_code=201)
-async def create_watch(pool: FromDishka[asyncpg.Pool], body: WatchCreate) -> WatchView:
-    return await watches.create_watch(
-        pool, body.account_id, body.kind, body.params, enabled=body.enabled
+async def create_watch(pool: FromDishka[asyncpg.Pool], body: WatchCreate) -> WatchItem:
+    watch = await watches.create_watch(
+        pool,
+        body.account_id,
+        body.kind,
+        _params(body.kind, body.params),
+        name=clean_name(body.name),
+        enabled=body.enabled,
     )
+    return await _item(pool, watch)
 
 
 @router.get("/watches/{watch_id}")
-async def get_watch(pool: FromDishka[asyncpg.Pool], watch_id: int) -> WatchView:
+async def get_watch(pool: FromDishka[asyncpg.Pool], watch_id: int) -> WatchItem:
     watch = await watches.get_watch(pool, watch_id)
     if watch is None:
         raise HTTPException(status_code=404, detail="watch not found")
-    return watch
+    return await _item(pool, watch)
 
 
 @router.put("/watches/{watch_id}")
 async def update_watch(
     pool: FromDishka[asyncpg.Pool], watch_id: int, body: WatchUpdate
-) -> WatchView:
+) -> WatchItem:
+    current = await watches.get_watch(pool, watch_id)
+    if current is None:
+        raise HTTPException(status_code=404, detail="watch not found")
+    kind = body.kind or current.kind
     watch = await watches.update_watch(
-        pool, watch_id, body.params, enabled=body.enabled
+        pool,
+        watch_id,
+        kind,
+        _params(kind, body.params),
+        name=clean_name(body.name),
+        enabled=body.enabled,
     )
     if watch is None:
         raise HTTPException(status_code=404, detail="watch not found")
-    return watch
+    return await _item(pool, watch)
 
 
 @router.delete("/watches/{watch_id}", status_code=204)
@@ -68,14 +101,29 @@ async def list_alerts(
     pool: FromDishka[asyncpg.Pool],
     account_id: Annotated[int, Query()],
     seen: Annotated[bool | None, Query()] = None,
-    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
-    offset: Annotated[int, Query()] = 0,
-) -> list[AlertView]:
+    before: Annotated[int | None, Query()] = None,
+    limit: Annotated[int, Query()] = watches.ALERT_PAGE,
+) -> AlertPage:
     return await watches.list_alerts(
-        pool, account_id, Page(limit=limit, offset=offset), seen=seen
+        pool, account_id, seen=seen, before=before, limit=limit
     )
 
 
+@router.get("/alerts/unseen")
+async def unseen_alerts(
+    pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
+) -> dict[str, int]:
+    return {"unseen": await watches.unseen_count(pool, account_id)}
+
+
+@router.post("/alerts/seen")
+async def mark_all_seen(
+    pool: FromDishka[asyncpg.Pool], body: SeenAll
+) -> dict[str, int]:
+    marked = await watches.mark_all_seen(pool, body.account_id, body.up_to)
+    return {"marked": marked}
+
+
 @router.post("/alerts/{alert_id}/seen", status_code=204)
 async def mark_alert_seen(pool: FromDishka[asyncpg.Pool], alert_id: int) -> None:
     if not await watches.mark_alert_seen(pool, alert_id):

backend/src/api/routers/webhooks.py

@@ -0,0 +1,121 @@
+from typing import Annotated
+from urllib.parse import urlsplit
+
+import asyncpg
+from dishka.integrations.fastapi import DishkaRoute, FromDishka
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel
+
+from api.webhooks import send_test
+from utils.read import webhooks
+from utils.read.models import DeliveryView, WebhookView
+
+router = APIRouter(prefix="/api/webhooks", tags=["webhooks"], route_class=DishkaRoute)
+
+MAX_URL = 2048
+
+
+class WebhookUpdate(BaseModel):
+    url: str
+    watch_ids: list[int] | None = None
+    enabled: bool = True
+
+
+class WebhookCreate(WebhookUpdate):
+    account_id: int
+
+
+def _url(value: str) -> str:
+    url = value.strip()
+    parts = urlsplit(url)
+    if (
+        len(url) > MAX_URL
+        or parts.scheme not in {"http", "https"}
+        or not parts.hostname
+    ):
+        raise HTTPException(
+            status_code=422, detail="url must be an http or https address"
+        )
+    return url
+
+
+def _watch_ids(ids: list[int] | None) -> list[int] | None:
+    return sorted(set(ids)) if ids else None
+
+
+async def _webhook(pool: asyncpg.Pool, webhook_id: int) -> WebhookView:
+    webhook = await webhooks.get_webhook(pool, webhook_id)
+    if webhook is None:
+        raise HTTPException(status_code=404, detail="webhook not found")
+    return webhook
+
+
+@router.get("")
+async def list_webhooks(
+    pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
+) -> list[WebhookView]:
+    return await webhooks.list_webhooks(pool, account_id)
+
+
+@router.post("", status_code=201)
+async def create_webhook(
+    pool: FromDishka[asyncpg.Pool], body: WebhookCreate
+) -> WebhookView:
+    webhook = await webhooks.create_webhook(
+        pool,
+        body.account_id,
+        _url(body.url),
+        _watch_ids(body.watch_ids),
+        enabled=body.enabled,
+    )
+    if webhook is None:
+        raise HTTPException(status_code=500, detail="webhook was not saved")
+    return webhook
+
+
+@router.put("/{webhook_id}")
+async def update_webhook(
+    pool: FromDishka[asyncpg.Pool], webhook_id: int, body: WebhookUpdate
+) -> WebhookView:
+    webhook = await webhooks.update_webhook(
+        pool,
+        webhook_id,
+        _url(body.url),
+        _watch_ids(body.watch_ids),
+        enabled=body.enabled,
+    )
+    if webhook is None:
+        raise HTTPException(status_code=404, detail="webhook not found")
+    return webhook
+
+
+@router.delete("/{webhook_id}", status_code=204)
+async def delete_webhook(pool: FromDishka[asyncpg.Pool], webhook_id: int) -> None:
+    if not await webhooks.delete_webhook(pool, webhook_id):
+        raise HTTPException(status_code=404, detail="webhook not found")
+
+
+@router.post("/{webhook_id}/secret")
+async def rotate_secret(pool: FromDishka[asyncpg.Pool], webhook_id: int) -> WebhookView:
+    webhook = await webhooks.rotate_secret(pool, webhook_id)
+    if webhook is None:
+        raise HTTPException(status_code=404, detail="webhook not found")
+    return webhook
+
+
+@router.post("/{webhook_id}/test")
+async def test_webhook(pool: FromDishka[asyncpg.Pool], webhook_id: int) -> DeliveryView:
+    delivery = await send_test(pool, await _webhook(pool, webhook_id))
+    if delivery is None:
+        raise HTTPException(status_code=500, detail="test delivery was not recorded")
+    return delivery
+
+
+@router.get("/{webhook_id}/deliveries")
+async def list_deliveries(
+    pool: FromDishka[asyncpg.Pool],
+    webhook_id: int,
+    limit: Annotated[int, Query(ge=1, le=100)] = webhooks.DELIVERY_LOG,
+) -> list[DeliveryView]:
+    await _webhook(pool, webhook_id)
+    return await webhooks.list_deliveries(pool, webhook_id, limit)

backend/src/api/timezone.py

@@ -0,0 +1,10 @@
+import asyncpg
+from fastapi import HTTPException
+
+from utils.read.period import known_timezone
+
+
+async def checked_tz(pool: asyncpg.Pool, tz: str) -> str:
+    if not await known_timezone(pool, tz):
+        raise HTTPException(status_code=422, detail=f"Unknown time zone: {tz}")
+    return tz

backend/src/api/webhooks.py

@@ -0,0 +1,191 @@
+import asyncio
+import contextlib
+import hashlib
+import hmac
+import json
+import logging
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+import asyncpg
+import httpx2
+
+from utils.env import env
+from utils.read import watches
+from utils.read import webhooks as webhooks_read
+from utils.read.models import AlertItem, DeliveryView, WebhookView
+
+logger = logging.getLogger(__name__)
+
+TIMEOUT_SECONDS = 10
+SCAN_SECONDS = 30
+BATCH = 20
+RETRY_DELAYS = (30, 90, 270, 810, 2430)
+ERROR_CHARS = 300
+USER_AGENT = "beavergram-webhooks"
+
+
+def sign(secret: str, body: bytes) -> str:
+    digest = hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
+    return f"sha256={digest}"
+
+
+def _encode(body: dict[str, Any]) -> str:
+    return json.dumps(body, ensure_ascii=False, separators=(",", ":"))
+
+
+def _with_url(body: dict[str, Any], path: str) -> dict[str, Any]:
+    if env.api.public_url:
+        body["url"] = env.api.public_url.rstrip("/") + path
+    return body
+
+
+def alert_body(item: AlertItem) -> dict[str, Any]:
+    data = item.model_dump(mode="json", exclude={"id", "seen"})
+    return _with_url({"type": "alert", "alert_id": item.id, **data}, item.path)
+
+
+def test_body(webhook: WebhookView) -> dict[str, Any]:
+    body = {
+        "type": "test",
+        "alert_id": None,
+        "account_id": webhook.account_id,
+        "ts": datetime.now(UTC).isoformat(),
+        "event": "test",
+        "watch": None,
+        "chat": None,
+        "message": None,
+        "peer": None,
+        "path": "/app",
+    }
+    return _with_url(body, "/app")
+
+
+@dataclass(frozen=True, slots=True)
+class Target:
+    delivery_id: int
+    event: str
+    url: str
+    secret: str
+
+
+async def post(
+    client: httpx2.AsyncClient, target: Target, body: str
+) -> tuple[int | None, str | None]:
+    raw = body.encode()
+    headers = {
+        "Content-Type": "application/json",
+        "User-Agent": USER_AGENT,
+        "X-Beavergram-Event": target.event,
+        "X-Beavergram-Delivery": str(target.delivery_id),
+        "X-Beavergram-Signature": sign(target.secret, raw),
+    }
+    try:
+        response = await client.post(target.url, content=raw, headers=headers)
+    except httpx2.HTTPError as exc:
+        return None, (str(exc) or type(exc).__name__)[:ERROR_CHARS]
+    if response.is_success:
+        return response.status_code, None
+    reason = response.text.strip()[:ERROR_CHARS] or response.reason_phrase
+    return response.status_code, reason or f"HTTP {response.status_code}"
+
+
+def new_client() -> httpx2.AsyncClient:
+    return httpx2.AsyncClient(timeout=TIMEOUT_SECONDS, follow_redirects=False)
+
+
+async def _body(pool: asyncpg.Pool, row: asyncpg.Record) -> str | None:
+    if row["body"] is not None:
+        return row["body"]
+    if row["alert_id"] is None:
+        return None
+    item = await watches.get_alert(pool, row["alert_id"])
+    if item is None:
+        return None
+    body = _encode(alert_body(item))
+    await webhooks_read.save_body(pool, row["id"], body)
+    return body
+
+
+async def _deliver(
+    pool: asyncpg.Pool, client: httpx2.AsyncClient, row: asyncpg.Record
+) -> None:
+    body = await _body(pool, row)
+    if body is None:
+        await webhooks_read.finish(
+            pool, row["id"], webhooks_read.STATUS_FAILED, None, "alert is gone"
+        )
+        return
+    target = Target(row["id"], row["event"], row["url"], row["secret"])
+    code, error = await post(client, target, body)
+    if error is None:
+        await webhooks_read.finish(
+            pool, row["id"], webhooks_read.STATUS_DELIVERED, code, None
+        )
+        return
+    attempt = row["attempts"]
+    if attempt > len(RETRY_DELAYS):
+        await webhooks_read.finish(
+            pool, row["id"], webhooks_read.STATUS_FAILED, code, error
+        )
+        return
+    retry_at = datetime.now(UTC) + timedelta(seconds=RETRY_DELAYS[attempt - 1])
+    await webhooks_read.finish(
+        pool, row["id"], webhooks_read.STATUS_PENDING, code, error, retry_at
+    )
+
+
+async def send_test(pool: asyncpg.Pool, webhook: WebhookView) -> DeliveryView | None:
+    body = _encode(test_body(webhook))
+    delivery_id = await webhooks_read.start_test(pool, webhook.id, body)
+    async with new_client() as client:
+        target = Target(delivery_id, "test", webhook.url, webhook.secret)
+        code, error = await post(client, target, body)
+    status = webhooks_read.STATUS_FAILED if error else webhooks_read.STATUS_DELIVERED
+    await webhooks_read.finish(pool, delivery_id, status, code, error)
+    return await webhooks_read.get_delivery(pool, delivery_id)
+
+
+class Outbox:
+    def __init__(self) -> None:
+        self._wake = asyncio.Event()
+
+    def wake(self) -> None:
+        self._wake.set()
+
+    async def run(self, pool: asyncpg.Pool) -> None:
+        released = False
+        async with new_client() as client:
+            while True:
+                self._wake.clear()
+                try:
+                    if not released:
+                        await webhooks_read.release_stuck(pool)
+                        released = True
+                    await self._drain(pool, client)
+                    delay = await self._delay(pool)
+                except Exception:
+                    logger.exception("Webhook delivery failed")
+                    delay = SCAN_SECONDS
+                with contextlib.suppress(TimeoutError):
+                    await asyncio.wait_for(self._wake.wait(), timeout=delay)
+
+    async def _drain(self, pool: asyncpg.Pool, client: httpx2.AsyncClient) -> None:
+        while rows := await webhooks_read.claim_due(pool, BATCH):
+            results = await asyncio.gather(
+                *(_deliver(pool, client, row) for row in rows), return_exceptions=True
+            )
+            for result in results:
+                if isinstance(result, Exception):
+                    logger.error("Webhook delivery crashed: %r", result)
+
+    async def _delay(self, pool: asyncpg.Pool) -> float:
+        due = await webhooks_read.next_due(pool)
+        if due is None:
+            return SCAN_SECONDS
+        wait = (due - datetime.now(UTC)).total_seconds()
+        return min(SCAN_SECONDS, max(wait, 1.0))
+
+
+outbox = Outbox()

backend/src/userbot/handlers/deletes.py

@@ -20,14 +20,19 @@ async def on_deleted_messages(client: PyroClient, messages: list[Message]) -> No
             box.append(message.id)
         else:
             channels.setdefault(chat_id, []).append(message.id)
+    ctx.pending_deletes.remember(None, box)
+    for chat_id, ids in channels.items():
+        ctx.pending_deletes.remember(chat_id, ids)
     if box:
         await repository.mark_deleted_box(ctx.pool, ctx.account_id, box)
+        await ctx.watches.on_deleted(None, box)
         await notify_bg_event(ctx.pool, "delete", ctx.account_id, message_ids=box)
     for chat_id, ids in channels.items():
         await repository.mark_deleted_channel(ctx.pool, ctx.account_id, chat_id, ids)
+        await ctx.watches.on_deleted(chat_id, ids)
         await notify_bg_event(
             ctx.pool, "delete", ctx.account_id, chat_id=chat_id, message_ids=ids
         )
 
 
-handlers = on_deleted_messages.handlers
+handlers = on_deleted_messages.handlers  # ty: ignore[unresolved-attribute]

backend/src/userbot/handlers/edits.py

@@ -2,10 +2,10 @@ from pyrogram.types import Message
 
 from userbot.modules.capture import repository
 from userbot.modules.capture.chat_meta import meta_from_chat
-from userbot.modules.capture.message import sender_id
+from userbot.modules.capture.message import message_row
 from userbot.modules.client import PyroClient
-from userbot.modules.media import capture_media, media_unique_id, self_destruct_ttl
-from userbot.modules.raw_json import raw_json
+from userbot.modules.media import capture_media
+from userbot.modules.watches.seen import seen_message
 from utils.events import notify_bg_event
 
 
@@ -23,22 +23,12 @@ async def on_edited_message(client: PyroClient, message: Message) -> None:
     chat = message.chat
     chat_id = chat.id or 0
     meta = meta_from_chat(chat, ctx.contacts.ids)
+    await ctx.watches.on_edit(seen_message(chat_id, message))
     toggles = ctx.resolve(meta)
-    if not toggles.track_edits_deletes:
+    if not (toggles.messages and toggles.track_edits_deletes):
         return
     changed = await repository.add_version(
-        ctx.pool,
-        ctx.account_id,
-        chat_id,
-        message.id,
-        message.date,
-        sender_id(message),
-        message.text or message.caption,
-        raw_json(message),
-        message.edit_date,
-        media_unique_id(message),
-        has_media=message.media is not None,
-        is_self_destruct=self_destruct_ttl(message) is not None,
+        ctx.pool, ctx.account_id, message_row(message, chat_id, message.date)
     )
     if not changed:
         return
@@ -49,4 +39,4 @@ async def on_edited_message(client: PyroClient, message: Message) -> None:
     )
 
 
-handlers = on_edited_message.handlers
+handlers = on_edited_message.handlers  # ty: ignore[unresolved-attribute]

backend/src/userbot/handlers/messages.py

@@ -5,6 +5,7 @@ from userbot.modules.capture.chat_meta import meta_from_chat
 from userbot.modules.client import PyroClient
 from userbot.modules.stt import is_transcribable
 from userbot.modules.stt.gate import safe_transcribe
+from userbot.modules.watches.seen import seen_message
 from utils.events import notify_bg_event
 
 
@@ -20,7 +21,7 @@ async def on_message(client: PyroClient, message: Message) -> None:
     ):
         return
     meta = meta_from_chat(message.chat, ctx.contacts.ids)
-    await ctx.watches.on_text(meta.chat_id, message.id, message.text or message.caption)
+    await ctx.watches.on_message(seen_message(meta.chat_id, message))
     toggles = ctx.resolve(meta)
     if not toggles.messages:
         return
@@ -36,4 +37,4 @@ async def on_message(client: PyroClient, message: Message) -> None:
         await safe_transcribe(client, ctx, meta.chat_id, message.id)
 
 
-handlers = on_message.handlers
+handlers = on_message.handlers  # ty: ignore[unresolved-attribute]

backend/src/userbot/handlers/presence.py

@@ -13,18 +13,21 @@ async def on_user_status(client: PyroClient, user: User) -> None:
     ctx = client.capture
     if ctx is None or user.status is None:
         return
-    await repository.insert_status(
-        ctx.pool,
-        ctx.account_id,
-        user.id,
-        datetime.now(UTC),
-        user.status.name.lower(),
-        user.last_online_date,
-        user.next_offline_date,
-        raw_json(user.raw),
-    )
-    await ctx.watches.on_status(user.id, is_online=user.status.name.lower() == "online")
-    await notify_bg_event(ctx.pool, "presence", ctx.account_id, chat_id=user.id)
+    at = datetime.now(UTC)
+    status = user.status.name.lower()
+    if ctx.resolve_peer(user.id, is_bot=bool(user.is_bot)).presence:
+        await repository.insert_status(
+            ctx.pool,
+            ctx.account_id,
+            user.id,
+            at,
+            status,
+            user.last_online_date,
+            user.next_offline_date,
+            raw_json(user.raw),
+        )
+        await notify_bg_event(ctx.pool, "presence", ctx.account_id, chat_id=user.id)
+    await ctx.watches.on_status(user.id, status, at, user.last_online_date)
 
 
-handlers = on_user_status.handlers
+handlers = on_user_status.handlers  # ty: ignore[unresolved-attribute]

backend/src/userbot/handlers/raw/dispatcher.py

@@ -24,8 +24,9 @@ _MODULES = (
     read_receipts,
     transcribed,
 )
-_REGISTRY: dict[type, RawHandler] = {
-    update_type: module.handle for module in _MODULES for update_type in module.HANDLES
+_REGISTRY: dict[type, list[RawHandler]] = {
+    update_type: [module.handle for module in _MODULES if update_type in module.HANDLES]
+    for update_type in {kind for module in _MODULES for kind in module.HANDLES}
 }
 
 
@@ -33,9 +34,8 @@ _REGISTRY: dict[type, RawHandler] = {
 async def dispatch(
     client: PyroClient, update: raw.base.Update, users: dict, chats: dict
 ) -> None:
-    handler = _REGISTRY.get(type(update))
-    if handler is not None:
+    for handler in _REGISTRY.get(type(update), ()):
         await handler(client, update, users, chats)
 
 
-handlers = dispatch.handlers
+handlers = dispatch.handlers  # ty: ignore[unresolved-attribute]

backend/src/userbot/handlers/raw/profiles.py

@@ -21,12 +21,18 @@ async def _handle_user(
     raw_user = users.get(update.user_id)
     if raw_user is None:
         return
-    fields, photo_file_id, photo_unique_id = snapshot_from_user(client, raw_user)
+    fields, photo_file_id, photo_unique_id = await snapshot_from_user(client, raw_user)
     current = await get_peer(ctx.pool, ctx.account_id, update.user_id)
     if current == fields:
         return
+    is_bot = bool(getattr(raw_user, "bot", False))
     await write_profile(
-        ctx.pool, ctx.account_id, update.user_id, fields, raw_json(raw_user)
+        ctx.pool,
+        ctx.account_id,
+        update.user_id,
+        fields,
+        raw_json(raw_user),
+        history=ctx.resolve_peer(update.user_id, is_bot=is_bot).profile_history,
     )
     changed_photo = current is None or current.photo_unique_id != photo_unique_id
     if photo_file_id and photo_unique_id and changed_photo:
@@ -51,7 +57,14 @@ async def _merge_partial(
     merged = replace(current or ProfileFields(), **overrides)
     if merged == current:
         return
-    await write_profile(ctx.pool, ctx.account_id, peer_id, merged, raw_str)
+    await write_profile(
+        ctx.pool,
+        ctx.account_id,
+        peer_id,
+        merged,
+        raw_str,
+        history=ctx.resolve_peer(peer_id).profile_history,
+    )
 
 
 async def handle(

backend/src/userbot/handlers/raw/read_receipts.py

@@ -7,25 +7,35 @@ from userbot.modules.raw_json import raw_json
 from userbot.modules.read_receipts import repository
 from utils.events import notify_bg_event
 
-HANDLES = (raw.types.UpdateReadHistoryOutbox,)
+HANDLES = (raw.types.UpdateReadHistoryOutbox, raw.types.UpdateReadChannelOutbox)
+
+
+def _watermark(update: raw.base.Update) -> tuple[int, int] | None:
+    if isinstance(update, raw.types.UpdateReadHistoryOutbox):
+        return utils.get_peer_id(update.peer), update.max_id
+    if isinstance(update, raw.types.UpdateReadChannelOutbox):
+        peer = raw.types.PeerChannel(channel_id=update.channel_id)
+        return utils.get_peer_id(peer), update.max_id
+    return None
 
 
 async def handle(
     client: PyroClient, update: raw.base.Update, _users: dict, _chats: dict
 ) -> None:
     ctx = client.capture
-    if ctx is None or not isinstance(update, raw.types.UpdateReadHistoryOutbox):
+    watermark = _watermark(update)
+    if ctx is None or watermark is None:
         return
-    chat_id = utils.get_peer_id(update.peer)
+    chat_id, max_id = watermark
     await repository.insert_read(
         ctx.pool,
         ctx.account_id,
         chat_id,
         chat_id,
         datetime.now(UTC),
-        update.max_id,
+        max_id,
         raw_json(update),
     )
     await notify_bg_event(
-        ctx.pool, "receipt", ctx.account_id, chat_id=chat_id, message_id=update.max_id
+        ctx.pool, "receipt", ctx.account_id, chat_id=chat_id, message_id=max_id
     )

backend/src/userbot/handlers/raw/transcribed.py

@@ -20,4 +20,4 @@ async def handle(
     await repository.set_extracted_text(
         ctx.pool, ctx.account_id, meta.chat_id, update.msg_id, update.text
     )
-    await ctx.watches.on_text(meta.chat_id, update.msg_id, update.text)
+    await ctx.watches.on_transcript(meta.chat_id, update.msg_id, update.text)

backend/src/userbot/handlers/stories.py

@@ -1,14 +1,19 @@
 from pyrogram.types import Story
 
 from userbot.modules.client import PyroClient
-from userbot.modules.stories.service import save_story
+from userbot.modules.stories.service import save_story, story_peer_id
 
 
 @PyroClient.on_story()
 async def on_story(client: PyroClient, story: Story) -> None:
-    if client.capture is None:
+    ctx = client.capture
+    if ctx is None:
         return
-    await save_story(client, client.capture, story)
+    peer_id = story_peer_id(story)
+    is_bot = bool(story.from_user and story.from_user.is_bot)
+    if peer_id > 0 and not ctx.resolve_peer(peer_id, is_bot=is_bot).stories:
+        return
+    await save_story(client, ctx, story)
 
 
-handlers = on_story.handlers
+handlers = on_story.handlers  # ty: ignore[unresolved-attribute]

backend/src/userbot/modules/capture/context.py

@@ -2,10 +2,11 @@ import asyncpg
 from pyrogram import Client
 
 from userbot.modules.capture.identity import ChatMetaCache, PeerIdentityCache
+from userbot.modules.capture.pending import PendingDeletes
 from userbot.modules.contacts import ContactCache
 from userbot.modules.folders import FolderCache
 from userbot.modules.watches import WatchCache
-from utils.policy.models import CaptureToggles, ChatMeta, PolicySet
+from utils.policy.models import CaptureToggles, ChatKind, ChatMeta, PolicySet
 from utils.policy.repository import load_policy_set
 from utils.policy.resolver import resolve
 from utils.storage import ContentAddressedStorage
@@ -28,6 +29,7 @@ class CaptureContext:
         self.watches = WatchCache(pool, account_id)
         self.peer_identity = PeerIdentityCache()
         self.chat_meta = ChatMetaCache()
+        self.pending_deletes = PendingDeletes()
         self.policies = PolicySet()
 
     async def reload_policies(self) -> None:
@@ -36,6 +38,16 @@ class CaptureContext:
     def resolve(self, chat: ChatMeta) -> CaptureToggles:
         return resolve(chat, self.folders.folders, self.policies)
 
+    def resolve_peer(self, peer_id: int, *, is_bot: bool = False) -> CaptureToggles:
+        return self.resolve(
+            ChatMeta(
+                chat_id=peer_id,
+                kind=ChatKind.DM,
+                is_bot=is_bot,
+                is_contact=peer_id in self.contacts.ids,
+            )
+        )
+
 
 async def build_capture_context(
     client: Client,

backend/src/userbot/modules/capture/identity.py

@@ -58,7 +58,14 @@ async def _capture_peer(message: Message, ctx: CaptureContext) -> None:
     fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
     if not await ctx.peer_identity.changed(ctx.pool, ctx.account_id, user.id, fields):
         return
-    await write_profile(ctx.pool, ctx.account_id, user.id, fields, raw_json(user))
+    await write_profile(
+        ctx.pool,
+        ctx.account_id,
+        user.id,
+        fields,
+        raw_json(user),
+        history=ctx.resolve_peer(user.id, is_bot=bool(user.is_bot)).profile_history,
+    )
     if photo_file_id and photo_unique_id:
         from userbot.modules.avatars import note_avatar  # noqa: PLC0415
 
@@ -82,17 +89,15 @@ async def _capture_chat(message: Message, ctx: CaptureContext) -> None:
     meta = (chat.title, photo_unique_id)
     if not await ctx.chat_meta.changed(ctx.pool, ctx.account_id, chat.id, meta):
         return
-    from userbot.modules.groups.repository import insert_chat_history  # noqa: PLC0415
+    from userbot.modules.groups.repository import record_chat_meta  # noqa: PLC0415
 
-    await insert_chat_history(
+    await record_chat_meta(
         ctx.pool,
         ctx.account_id,
         chat.id,
         message.id,
-        "meta",
         chat.title,
         photo_unique_id,
-        None,
         message.date,
         raw_json(message),
     )

backend/src/userbot/modules/capture/message.py

@@ -1,11 +1,14 @@
+from datetime import datetime
+
 from pyrogram import Client
 from pyrogram.types import Message
 
 from userbot.modules.capture import repository
 from userbot.modules.capture.context import CaptureContext
+from userbot.modules.capture.repository import MessageRow
 from userbot.modules.links import extract_links
 from userbot.modules.links import repository as links_repository
-from userbot.modules.media import capture_media, self_destruct_ttl
+from userbot.modules.media import capture_media, media_unique_id, self_destruct_ttl
 from userbot.modules.raw_json import raw_json
 from utils.policy.models import CaptureToggles
 
@@ -34,27 +37,39 @@ def callbacks(message: Message) -> list[tuple[int, str | None, bytes | None]]:
     return buttons
 
 
+def message_row(message: Message, chat_id: int, date: datetime) -> MessageRow:
+    return MessageRow(
+        chat_id=chat_id,
+        message_id=message.id,
+        date=date,
+        sender_id=sender_id(message),
+        text=message.text or message.caption,
+        raw=raw_json(message),
+        has_media=message.media is not None,
+        is_self_destruct=self_destruct_ttl(message) is not None,
+        edit_date=message.edit_date,
+        media_unique_id=media_unique_id(message),
+    )
+
+
 async def capture_message(
     client: Client, message: Message, ctx: CaptureContext, toggles: CaptureToggles
 ) -> None:
     if message.empty or message.chat is None or message.date is None:
         return
     chat_id = message.chat.id or 0
-    await repository.upsert_message(
-        ctx.pool,
-        ctx.account_id,
-        chat_id,
-        message.id,
-        message.date,
-        sender_id(message),
-        message.text or message.caption,
-        raw_json(message),
-        has_media=message.media is not None,
-        is_self_destruct=self_destruct_ttl(message) is not None,
+    current = await repository.upsert_message(
+        ctx.pool, ctx.account_id, message_row(message, chat_id, message.date)
     )
+    if ctx.pending_deletes.take(chat_id, message.id):
+        await repository.mark_deleted_late(
+            ctx.pool, ctx.account_id, chat_id, message.id, message.date
+        )
     from userbot.modules.capture.identity import capture_identity  # noqa: PLC0415
 
     await capture_identity(message, ctx)
+    if not current:
+        return
     await capture_media(client, message, ctx, chat_id, message.id, toggles)
     buttons = callbacks(message)
     if buttons:

backend/src/userbot/modules/capture/pending.py

@@ -0,0 +1,36 @@
+import time
+
+from userbot.modules.capture.repository import CHANNEL_ID_THRESHOLD
+
+PENDING_DELETE_SECONDS = 120.0
+PENDING_DELETE_LIMIT = 10_000
+
+DeleteKey = tuple[int | None, int]
+
+
+def _key(chat_id: int | None, message_id: int) -> DeleteKey:
+    if chat_id is None or chat_id > CHANNEL_ID_THRESHOLD:
+        return None, message_id
+    return chat_id, message_id
+
+
+class PendingDeletes:
+    def __init__(self) -> None:
+        self._expires: dict[DeleteKey, float] = {}
+
+    def remember(self, chat_id: int | None, message_ids: list[int]) -> None:
+        now = time.monotonic()
+        while self._expires:
+            oldest = next(iter(self._expires))
+            fresh = self._expires[oldest] > now
+            if fresh and len(self._expires) < PENDING_DELETE_LIMIT:
+                break
+            del self._expires[oldest]
+        for message_id in message_ids:
+            key = _key(chat_id, message_id)
+            self._expires.pop(key, None)
+            self._expires[key] = now + PENDING_DELETE_SECONDS
+
+    def take(self, chat_id: int, message_id: int) -> bool:
+        expires = self._expires.pop(_key(chat_id, message_id), None)
+        return expires is not None and expires > time.monotonic()

backend/src/userbot/modules/capture/repository.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from datetime import datetime
 
 import asyncpg
@@ -21,13 +22,19 @@ _TOUCH_EDITED = """
 INSERT INTO messages
     (account_id, chat_id, message_id, date, sender_id, text, raw,
      has_media, is_self_destruct, edited_at)
-VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8, $9, now())
+VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8, $9, CASE WHEN $10 THEN now() END)
 ON CONFLICT (account_id, chat_id, message_id, date) DO UPDATE SET
     text = EXCLUDED.text,
     raw = EXCLUDED.raw,
     has_media = EXCLUDED.has_media,
     is_self_destruct = EXCLUDED.is_self_destruct,
-    edited_at = now()
+    edited_at = coalesce(EXCLUDED.edited_at, messages.edited_at)
+"""
+
+_LOCK_MESSAGE = """
+SELECT pg_advisory_xact_lock(
+    hashtextextended(concat_ws(':', 'message', $1::int, $2::bigint, $3::bigint), 0)
+)
 """
 
 _FIND_DELETED = """
@@ -40,6 +47,12 @@ UPDATE messages SET deleted_at = now()
 WHERE account_id = $1 AND chat_id = $2 AND message_id = $3 AND date = $4
 """
 
+_MARK_DELETED_LATE = """
+UPDATE messages SET deleted_at = now()
+WHERE account_id = $1 AND chat_id = $2 AND message_id = $3 AND date = $4
+  AND deleted_at IS NULL
+"""
+
 _INSERT_VERSION = """
 INSERT INTO message_versions
     (account_id, chat_id, message_id, observed_at, edit_date, text, raw)
@@ -52,7 +65,7 @@ INSERT INTO message_versions
     (account_id, chat_id, message_id, observed_at, edit_date, text, raw)
 SELECT account_id, chat_id, message_id, clock_timestamp(), NULL, text, raw
 FROM messages m
-WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3
+WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3 AND m.date = $4
   AND NOT EXISTS (
     SELECT 1 FROM message_versions v
     WHERE v.account_id = m.account_id AND v.chat_id = m.chat_id
@@ -61,15 +74,37 @@ WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3
 ON CONFLICT DO NOTHING
 """
 
+_INSERT_PAST_VERSION = """
+INSERT INTO message_versions
+    (account_id, chat_id, message_id, observed_at, edit_date, text, raw)
+SELECT $1::int, $2::bigint, $3::bigint,
+    coalesce(
+        (SELECT min(v.observed_at) - interval '1 microsecond'
+         FROM message_versions v
+         WHERE v.account_id = $1 AND v.chat_id = $2 AND v.message_id = $3
+           AND v.edit_date > $4::timestamptz),
+        $4::timestamptz,
+        $5::timestamptz
+    ),
+    $4::timestamptz, $6::text, $7::jsonb
+WHERE NOT EXISTS (
+    SELECT 1 FROM message_versions v
+    WHERE v.account_id = $1 AND v.chat_id = $2 AND v.message_id = $3
+      AND v.edit_date IS NOT DISTINCT FROM $4::timestamptz
+      AND v.text IS NOT DISTINCT FROM $6::text
+)
+ON CONFLICT DO NOTHING
+"""
+
 _CURRENT_CONTENT = """
 SELECT
     m.text AS text,
+    m.raw->>'edit_date' AS edit_date,
     (SELECT d.unique_id FROM media d
      WHERE d.account_id = m.account_id AND d.chat_id = m.chat_id
        AND d.message_id = m.message_id) AS media_unique_id
 FROM messages m
-WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3
-ORDER BY m.date DESC LIMIT 1
+WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3 AND m.date = $4
 """
 
 _CURRENT_MEDIA = """
@@ -88,8 +123,8 @@ ON CONFLICT (account_id, chat_id, message_id, storage_key) DO NOTHING
 _INSERT_MEDIA = """
 INSERT INTO media
     (account_id, chat_id, message_id, kind, storage_key, file_size, mime,
-     ttl_seconds, downloaded, unique_id)
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+     ttl_seconds, downloaded, unique_id, evicted_at)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, CASE WHEN $11 THEN now() END)
 ON CONFLICT (account_id, chat_id, message_id) DO UPDATE SET
     kind = EXCLUDED.kind,
     storage_key = EXCLUDED.storage_key,
@@ -97,37 +132,134 @@ ON CONFLICT (account_id, chat_id, message_id) DO UPDATE SET
     mime = EXCLUDED.mime,
     ttl_seconds = EXCLUDED.ttl_seconds,
     downloaded = EXCLUDED.downloaded,
-    unique_id = EXCLUDED.unique_id
+    unique_id = EXCLUDED.unique_id,
+    evicted_at = CASE
+        WHEN EXCLUDED.downloaded THEN NULL
+        WHEN media.unique_id IS DISTINCT FROM EXCLUDED.unique_id
+            THEN EXCLUDED.evicted_at
+        ELSE coalesce(media.evicted_at, EXCLUDED.evicted_at) END,
+    evicted_key = CASE WHEN EXCLUDED.downloaded
+        OR media.unique_id IS DISTINCT FROM EXCLUDED.unique_id
+        THEN NULL ELSE media.evicted_key END
+"""
+
+_IS_EVICTED = """
+SELECT EXISTS (
+    SELECT 1 FROM media WHERE unique_id = $1 AND evicted_at IS NOT NULL
+)
+"""
+
+_UNEVICT = """
+UPDATE media SET storage_key = $2, file_size = $3, downloaded = true,
+    evicted_at = NULL, evicted_key = NULL
+WHERE evicted_key = $2 OR (unique_id = $1 AND evicted_at IS NOT NULL)
 """
 
 
-async def upsert_message(  # noqa: PLR0913
-    pool: asyncpg.Pool,
-    account_id: int,
-    chat_id: int,
-    message_id: int,
-    date: datetime,
-    sender_id: int | None,
-    text: str | None,
-    raw: str,
-    *,
-    has_media: bool,
-    is_self_destruct: bool,
-) -> None:
-    await pool.execute(
-        _UPSERT_MESSAGE,
-        account_id,
-        chat_id,
-        message_id,
-        date,
-        sender_id,
-        text,
-        raw,
-        has_media,
-        is_self_destruct,
+@dataclass(frozen=True, slots=True)
+class MessageRow:
+    chat_id: int
+    message_id: int
+    date: datetime
+    sender_id: int | None
+    text: str | None
+    raw: str
+    has_media: bool
+    is_self_destruct: bool
+    edit_date: datetime | None
+    media_unique_id: str | None
+
+    @property
+    def columns(self) -> tuple:
+        return (
+            self.chat_id,
+            self.message_id,
+            self.date,
+            self.sender_id,
+            self.text,
+            self.raw,
+            self.has_media,
+            self.is_self_destruct,
+        )
+
+
+async def _lock_current(
+    conn: asyncpg.Connection, account_id: int, row: MessageRow
+) -> asyncpg.Record | None:
+    await conn.execute(_LOCK_MESSAGE, account_id, row.chat_id, row.message_id)
+    return await conn.fetchrow(
+        _CURRENT_CONTENT, account_id, row.chat_id, row.message_id, row.date
     )
 
 
+def _is_stale(current: asyncpg.Record, row: MessageRow) -> bool:
+    stored = current["edit_date"]
+    if stored is None:
+        return False
+    if row.edit_date is None:
+        return True
+    try:
+        return row.edit_date < datetime.fromisoformat(stored)
+    except (TypeError, ValueError):
+        return False
+
+
+async def _record_past(
+    conn: asyncpg.Connection, account_id: int, row: MessageRow
+) -> None:
+    await conn.execute(
+        _INSERT_PAST_VERSION,
+        account_id,
+        row.chat_id,
+        row.message_id,
+        row.edit_date,
+        row.date,
+        row.text,
+        row.raw,
+    )
+
+
+async def _record_edit(
+    conn: asyncpg.Connection, account_id: int, row: MessageRow, *, mark_edited: bool
+) -> None:
+    await conn.execute(
+        _SNAPSHOT_ORIGINAL, account_id, row.chat_id, row.message_id, row.date
+    )
+    await conn.execute(_TOUCH_EDITED, account_id, *row.columns, mark_edited)
+    await conn.execute(
+        _INSERT_VERSION,
+        account_id,
+        row.chat_id,
+        row.message_id,
+        row.edit_date,
+        row.text,
+        row.raw,
+    )
+
+
+async def upsert_message(pool: asyncpg.Pool, account_id: int, row: MessageRow) -> bool:
+    async with pool.acquire() as conn, conn.transaction():
+        current = await _lock_current(conn, account_id, row)
+        if current is None:
+            await conn.execute(_UPSERT_MESSAGE, account_id, *row.columns)
+            return True
+        stored_media = current["media_unique_id"]
+        changed = current["text"] != row.text or (
+            stored_media is not None and stored_media != row.media_unique_id
+        )
+        if _is_stale(current, row):
+            if changed:
+                await _record_past(conn, account_id, row)
+            return False
+        if changed:
+            await _record_edit(
+                conn, account_id, row, mark_edited=row.edit_date is not None
+            )
+        else:
+            await conn.execute(_UPSERT_MESSAGE, account_id, *row.columns)
+    return True
+
+
 async def max_message_id(
     pool: asyncpg.Pool, account_id: int, chat_id: int
 ) -> int | None:
@@ -168,48 +300,29 @@ async def mark_deleted_channel(
     await _mark_deleted(pool, account_id, rows)
 
 
-async def add_version(  # noqa: PLR0913
-    pool: asyncpg.Pool,
-    account_id: int,
-    chat_id: int,
-    message_id: int,
-    date: datetime,
-    sender_id: int | None,
-    text: str | None,
-    raw: str,
-    edit_date: datetime | None,
-    media_unique_id: str | None,
-    *,
-    has_media: bool,
-    is_self_destruct: bool,
-) -> bool:
+async def add_version(pool: asyncpg.Pool, account_id: int, row: MessageRow) -> bool:
     async with pool.acquire() as conn, conn.transaction():
-        current = await conn.fetchrow(_CURRENT_CONTENT, account_id, chat_id, message_id)
-        text_changed = current is None or current["text"] != text
-        media_changed = (
-            current is not None and current["media_unique_id"] != media_unique_id
-        )
-        if not (text_changed or media_changed):
-            return False
-        await conn.execute(_SNAPSHOT_ORIGINAL, account_id, chat_id, message_id)
-        await conn.execute(
-            _TOUCH_EDITED,
-            account_id,
-            chat_id,
-            message_id,
-            date,
-            sender_id,
-            text,
-            raw,
-            has_media,
-            is_self_destruct,
-        )
-        await conn.execute(
-            _INSERT_VERSION, account_id, chat_id, message_id, edit_date, text, raw
-        )
+        current = await _lock_current(conn, account_id, row)
+        if current is not None:
+            changed = (
+                current["text"] != row.text
+                or current["media_unique_id"] != row.media_unique_id
+            )
+            if not changed:
+                return False
+            if _is_stale(current, row):
+                await _record_past(conn, account_id, row)
+                return False
+        await _record_edit(conn, account_id, row, mark_edited=True)
     return True
 
 
+async def mark_deleted_late(
+    pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int, date: datetime
+) -> None:
+    await pool.execute(_MARK_DELETED_LATE, account_id, chat_id, message_id, date)
+
+
 async def current_media(
     pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
 ) -> asyncpg.Record | None:
@@ -229,6 +342,7 @@ async def insert_media(  # noqa: PLR0913
     unique_id: str | None,
     *,
     downloaded: bool,
+    evicted: bool = False,
 ) -> None:
     async with pool.acquire() as conn, conn.transaction():
         await conn.execute(
@@ -243,6 +357,7 @@ async def insert_media(  # noqa: PLR0913
             ttl_seconds,
             downloaded,
             unique_id,
+            evicted,
         )
         if storage_key is not None:
             await conn.execute(
@@ -259,6 +374,22 @@ async def insert_media(  # noqa: PLR0913
             )
 
 
+async def is_evicted(pool: asyncpg.Pool, unique_id: str | None) -> bool:
+    return unique_id is not None and await pool.fetchval(_IS_EVICTED, unique_id)
+
+
+async def unevict(
+    pool: asyncpg.Pool, unique_id: str | None, storage_key: str, file_size: int | None
+) -> bool:
+    async with pool.acquire() as conn, conn.transaction():
+        await conn.execute(_UNEVICT, unique_id, storage_key, file_size)
+        dropped = await conn.fetchval(
+            "DELETE FROM media_trash WHERE storage_key = $1 RETURNING storage_key",
+            storage_key,
+        )
+    return dropped is not None
+
+
 async def insert_callbacks(
     pool: asyncpg.Pool,
     account_id: int,

backend/src/userbot/modules/catchup/__init__.py

@@ -0,0 +1,3 @@
+from userbot.modules.catchup.service import SavedState, catch_up, read_saved_state
+
+__all__ = ["SavedState", "catch_up", "read_saved_state"]

backend/src/userbot/modules/catchup/repository.py

@@ -0,0 +1,41 @@
+from dataclasses import dataclass
+
+import asyncpg
+
+_LOAD = """
+SELECT catchup_pts, catchup_qts, catchup_date FROM accounts WHERE account_id = $1
+"""
+
+_SAVE = """
+UPDATE accounts SET catchup_pts = $2, catchup_qts = $3, catchup_date = $4
+WHERE account_id = $1
+"""
+
+_CLEAR = """
+UPDATE accounts SET catchup_pts = NULL, catchup_qts = NULL, catchup_date = NULL
+WHERE account_id = $1
+"""
+
+
+@dataclass(frozen=True)
+class UpdateCursor:
+    pts: int
+    qts: int
+    date: int
+
+
+async def load_cursor(pool: asyncpg.Pool, account_id: int) -> UpdateCursor | None:
+    row = await pool.fetchrow(_LOAD, account_id)
+    if row is None or row["catchup_pts"] is None:
+        return None
+    return UpdateCursor(row["catchup_pts"], row["catchup_qts"], row["catchup_date"])
+
+
+async def save_cursor(
+    pool: asyncpg.Pool, account_id: int, cursor: UpdateCursor
+) -> None:
+    await pool.execute(_SAVE, account_id, cursor.pts, cursor.qts, cursor.date)
+
+
+async def clear_cursor(pool: asyncpg.Pool, account_id: int) -> None:
+    await pool.execute(_CLEAR, account_id)

backend/src/userbot/modules/catchup/service.py

@@ -0,0 +1,255 @@
+import asyncio
+import sqlite3
+import time
+from contextlib import closing
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from pyrogram import raw, utils
+from pyrogram.errors import FloodPremiumWait, FloodWait, RPCError
+
+from userbot.modules.capture.chat_meta import meta_from_peer
+from userbot.modules.capture.context import CaptureContext
+from userbot.modules.catchup import repository
+from userbot.modules.catchup.repository import UpdateCursor
+from userbot.modules.client import PyroClient
+from utils.logging import logger
+from utils.policy.models import ChatKind
+
+BOX_SLICES = 50
+CHANNEL_SLICES = 10
+CHANNEL_LIMIT = 100
+MAX_CHANNELS = 50
+QUEUE_HIGH_WATER = 200
+QUEUE_WAIT_SECONDS = 0.5
+NO_QTS = -1
+MAX_AGE_SECONDS = 3 * 24 * 3600
+
+
+@dataclass(frozen=True)
+class SavedState:
+    box: UpdateCursor | None = None
+    channels: dict[int, int] = field(default_factory=dict)
+
+
+@dataclass
+class BoxWalk:
+    cursor: UpdateCursor
+    done: bool = False
+    updates: int = 0
+    channels: dict[int, raw.base.Chat] = field(default_factory=dict)
+
+
+def read_saved_state(path: Path) -> SavedState:
+    try:
+        uri = f"{path.resolve().as_uri()}?mode=ro"
+        with closing(sqlite3.connect(uri, uri=True)) as conn:
+            rows = conn.execute(
+                "SELECT id, pts, qts, date FROM update_state"
+            ).fetchall()
+    except Exception as exc:
+        logger.warning(f"[yellow]Session {path.stem}: no saved update state ({exc})[/]")
+        return SavedState()
+    box = None
+    channels: dict[int, int] = {}
+    for state_id, pts, qts, date in rows:
+        if pts is None:
+            continue
+        if state_id == 0:
+            box = UpdateCursor(
+                pts, NO_QTS if qts is None else qts, date or int(time.time())
+            )
+        else:
+            channels[state_id] = pts
+    return SavedState(box, channels)
+
+
+async def _dispatch(
+    client: PyroClient,
+    new_messages: list,
+    other_updates: list,
+    users: list,
+    chats: list,
+) -> int:
+    await client.fetch_peers(users)
+    await client.fetch_peers(chats)
+    user_map = {user.id: user for user in users}
+    chat_map = {chat.id: chat for chat in chats}
+    queue = client.dispatcher.updates_queue
+    for message in new_messages:
+        update = raw.types.UpdateNewMessage(message=message, pts=0, pts_count=0)
+        queue.put_nowait((update, user_map, chat_map))
+    for update in other_updates:
+        queue.put_nowait((update, user_map, chat_map))
+    while queue.qsize() > QUEUE_HIGH_WATER:  # noqa: ASYNC110
+        await asyncio.sleep(QUEUE_WAIT_SECONDS)
+    return len(new_messages) + len(other_updates)
+
+
+def _channel_hints(other_updates: list, chats: list) -> dict[int, raw.base.Chat]:
+    by_id = {chat.id: chat for chat in chats}
+    return {
+        utils.get_channel_id(update.channel_id): by_id[update.channel_id]
+        for update in other_updates
+        if isinstance(update, raw.types.UpdateChannelTooLong)
+        and update.channel_id in by_id
+    }
+
+
+async def _walk_box(
+    client: PyroClient, ctx: CaptureContext, cursor: UpdateCursor
+) -> BoxWalk:
+    walk = BoxWalk(cursor)
+    for _ in range(BOX_SLICES):
+        diff = await client.invoke(
+            raw.functions.updates.GetDifference(
+                pts=walk.cursor.pts, date=walk.cursor.date, qts=walk.cursor.qts
+            )
+        )
+        if isinstance(diff, raw.types.updates.DifferenceEmpty):
+            walk.done = True
+            return walk
+        if isinstance(diff, raw.types.updates.DifferenceTooLong):
+            logger.warning(
+                f"[yellow]Account {ctx.account_id}: too much missed to catch up "
+                f"(pts {walk.cursor.pts} to {diff.pts}), that gap stays[/]"
+            )
+            walk.done = True
+            return walk
+        walk.updates += await _dispatch(
+            client, diff.new_messages, diff.other_updates, diff.users, diff.chats
+        )
+        walk.channels.update(_channel_hints(diff.other_updates, diff.chats))
+        final = isinstance(diff, raw.types.updates.Difference)
+        state = diff.state if final else diff.intermediate_state
+        moved = UpdateCursor(state.pts, state.qts, state.date or walk.cursor.date)
+        if moved == walk.cursor and not final:
+            logger.warning(
+                f"[yellow]Account {ctx.account_id}: catch-up stopped advancing "
+                f"at pts={moved.pts}[/]"
+            )
+            return walk
+        walk.cursor = moved
+        await repository.save_cursor(ctx.pool, ctx.account_id, moved)
+        if final:
+            walk.done = True
+            return walk
+    return walk
+
+
+async def _walk_channel(client: PyroClient, peer_id: int, pts: int) -> int:
+    peer = await client.resolve_peer(peer_id)
+    if not isinstance(peer, raw.types.InputPeerChannel):
+        return 0
+    channel = raw.types.InputChannel(
+        channel_id=peer.channel_id, access_hash=peer.access_hash
+    )
+    updates = 0
+    for _ in range(CHANNEL_SLICES):
+        diff = await client.invoke(
+            raw.functions.updates.GetChannelDifference(
+                channel=channel,
+                filter=raw.types.ChannelMessagesFilterEmpty(),
+                pts=pts,
+                limit=CHANNEL_LIMIT,
+            )
+        )
+        if isinstance(diff, raw.types.updates.ChannelDifferenceEmpty):
+            return updates
+        if isinstance(diff, raw.types.updates.ChannelDifferenceTooLong):
+            logger.info(f"Channel {peer_id}: too much missed to catch up")
+            return updates
+        updates += await _dispatch(
+            client, diff.new_messages, diff.other_updates, diff.users, diff.chats
+        )
+        if diff.final:
+            return updates
+        pts = diff.pts
+    return updates
+
+
+def _captured_channels(
+    ctx: CaptureContext, hints: dict[int, raw.base.Chat], saved: dict[int, int]
+) -> list[int]:
+    wanted: list[tuple[bool, int]] = []
+    for peer_id, chat in hints.items():
+        if peer_id not in saved:
+            continue
+        meta = meta_from_peer(
+            raw.types.PeerChannel(channel_id=chat.id),
+            {chat.id: chat},
+            {},
+            ctx.contacts.ids,
+        )
+        toggles = ctx.resolve(meta)
+        if toggles.messages or toggles.track_edits_deletes:
+            wanted.append((meta.kind is ChatKind.CHANNEL, peer_id))
+    return [peer_id for _, peer_id in sorted(wanted)[:MAX_CHANNELS]]
+
+
+async def _walk_channels(
+    client: PyroClient, ctx: CaptureContext, walk: BoxWalk, saved: dict[int, int]
+) -> int:
+    peer_ids = _captured_channels(ctx, walk.channels, saved)
+    updates = 0
+    for peer_id in peer_ids:
+        try:
+            updates += await _walk_channel(client, peer_id, saved[peer_id])
+        except (FloodWait, FloodPremiumWait) as exc:
+            logger.warning(
+                f"[yellow]Account {ctx.account_id}: channel catch-up stopped "
+                f"by a flood wait:[/] {exc}"
+            )
+            break
+        except RPCError as exc:
+            logger.warning(
+                f"[yellow]Account {ctx.account_id}: channel {peer_id} "
+                f"catch-up failed:[/] {exc}"
+            )
+    skipped = len(walk.channels) - len(peer_ids)
+    if skipped:
+        logger.info(f"Account {ctx.account_id}: {skipped} channel gap(s) left as is")
+    return updates
+
+
+async def catch_up(client: PyroClient, saved: SavedState) -> None:
+    ctx = client.capture
+    if ctx is None:
+        return
+    try:
+        pending = await repository.load_cursor(ctx.pool, ctx.account_id)
+        cursor = pending or saved.box
+        if cursor is None:
+            logger.info(f"Account {ctx.account_id}: no saved update state to catch up")
+            return
+        if time.time() - cursor.date > MAX_AGE_SECONDS:
+            logger.warning(
+                f"[yellow]Account {ctx.account_id}: last update seen too long ago "
+                f"(pts={cursor.pts}), not catching up[/]"
+            )
+            await repository.clear_cursor(ctx.pool, ctx.account_id)
+            return
+        await repository.save_cursor(ctx.pool, ctx.account_id, cursor)
+        logger.info(
+            f"Account {ctx.account_id}: catching up from pts={cursor.pts}"
+            f"{' (resumed)' if pending else ''}"
+        )
+        walk = await _walk_box(client, ctx, cursor)
+        if not walk.done:
+            logger.warning(
+                f"[yellow]Account {ctx.account_id}: catch-up unfinished at "
+                f"pts={walk.cursor.pts},[/] {walk.updates} updates replayed, "
+                f"resumes on the next start"
+            )
+            return
+        from_channels = await _walk_channels(client, ctx, walk, saved.channels)
+        await repository.clear_cursor(ctx.pool, ctx.account_id)
+        logger.info(
+            f"[green]Account {ctx.account_id}: caught up,[/] {walk.updates} updates "
+            f"from the main box, {from_channels} from channels"
+        )
+    except Exception:
+        logger.exception(
+            f"[red]Account {ctx.account_id}: catch-up failed,[/] "
+            f"resumes on the next start"
+        )

backend/src/userbot/modules/groups/repository.py

@@ -7,7 +7,23 @@ INSERT INTO chat_history
     (account_id, chat_id, message_id, event, title, photo_unique_id, actor_id,
      ts, raw)
 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb)
-ON CONFLICT (account_id, chat_id, message_id) DO NOTHING
+ON CONFLICT DO NOTHING
+"""
+
+_INSERT_CHAT_META = """
+INSERT INTO chat_history
+    (account_id, chat_id, message_id, event, title, photo_unique_id, ts, raw)
+SELECT $1, $2, $3, 'meta', $4::text, $5::text, $6, $7::jsonb
+WHERE ($4::text, $5::text) IS DISTINCT FROM (
+    (SELECT title FROM chat_history
+     WHERE account_id = $1 AND chat_id = $2 AND event IN ('meta', 'title')
+     ORDER BY ts DESC LIMIT 1),
+    (SELECT photo_unique_id FROM chat_history
+     WHERE account_id = $1 AND chat_id = $2
+       AND event IN ('meta', 'photo', 'photo_deleted')
+     ORDER BY ts DESC LIMIT 1)
+)
+ON CONFLICT DO NOTHING
 """
 
 _INSERT_MEMBERSHIP = """
@@ -19,13 +35,14 @@ ON CONFLICT (account_id, chat_id, message_id, user_id) DO NOTHING
 
 _LATEST_TITLE = """
 SELECT title FROM chat_history
-WHERE account_id = $1 AND chat_id = $2 AND title IS NOT NULL
+WHERE account_id = $1 AND chat_id = $2 AND event IN ('meta', 'title')
 ORDER BY ts DESC LIMIT 1
 """
 
 _LATEST_PHOTO = """
 SELECT photo_unique_id FROM chat_history
-WHERE account_id = $1 AND chat_id = $2 AND photo_unique_id IS NOT NULL
+WHERE account_id = $1 AND chat_id = $2
+  AND event IN ('meta', 'photo', 'photo_deleted')
 ORDER BY ts DESC LIMIT 1
 """
 
@@ -64,6 +81,29 @@ async def insert_chat_history(  # noqa: PLR0913
     )
 
 
+async def record_chat_meta(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    account_id: int,
+    chat_id: int,
+    message_id: int,
+    title: str | None,
+    photo_unique_id: str | None,
+    ts: datetime,
+    raw: str,
+) -> bool:
+    status = await pool.execute(
+        _INSERT_CHAT_META,
+        account_id,
+        chat_id,
+        message_id,
+        title,
+        photo_unique_id,
+        ts,
+        raw,
+    )
+    return status == "INSERT 0 1"
+
+
 async def insert_membership(  # noqa: PLR0913
     pool: asyncpg.Pool,
     account_id: int,

backend/src/userbot/modules/jobs/consumer.py

@@ -9,11 +9,12 @@ from userbot.modules.jobs import repository
 from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import JOB_HANDLERS, JobHandler
 from utils.db.models import Job, JobStatus
-from utils.env import env
+from utils.listen import Listener
 from utils.logging import logger
 
 JOBS_CHANGED_CHANNEL = "jobs_changed"
 POLL_INTERVAL_SECONDS = 60.0
+RESTART_SECONDS = 10.0
 
 
 class JobConsumer:
@@ -38,25 +39,20 @@ class JobConsumer:
                 self.pool, ctx.job_id, JobStatus.FAILED, f"no handler for {job.kind!r}"
             )
             return
-        while True:
-            try:
-                await handler(ctx)
-            except (FloodWait, FloodPremiumWait) as exc:
-                await repository.bump_flood_wait(self.pool, ctx.job_id)
-                wait = exc.value if isinstance(exc.value, int) else 1
-                logger.warning(
-                    f"[yellow]FloodWait {wait}s on job {ctx.job_id} ({job.kind})[/]"
-                )
-                await asyncio.sleep(wait)
-            except Exception as exc:
-                logger.exception(f"Job {ctx.job_id} ({job.kind}) failed")
-                await repository.finish(
-                    self.pool, ctx.job_id, JobStatus.FAILED, str(exc)
-                )
-                return
-            else:
-                await repository.finish(self.pool, ctx.job_id, JobStatus.DONE)
-                return
+        try:
+            await handler(ctx)
+        except (FloodWait, FloodPremiumWait) as exc:
+            wait = exc.value if isinstance(exc.value, int) else 1
+            await repository.defer_flood_wait(self.pool, ctx.job_id, wait)
+            logger.warning(
+                f"[yellow]FloodWait {wait}s on job {ctx.job_id} ({job.kind}), "
+                f"deferred[/]"
+            )
+        except Exception as exc:
+            logger.exception(f"Job {ctx.job_id} ({job.kind}) failed")
+            await repository.finish(self.pool, ctx.job_id, JobStatus.FAILED, str(exc))
+        else:
+            await repository.finish(self.pool, ctx.job_id, JobStatus.DONE)
 
     async def drain(self) -> None:
         while True:
@@ -66,23 +62,41 @@ class JobConsumer:
             await self.execute(job)
 
     async def run(self) -> None:
+        while True:
+            try:
+                await self._serve()
+            except Exception:
+                logger.exception(
+                    f"[red]Job consumer for account {self.account_id} failed,[/] "
+                    f"restarting in {RESTART_SECONDS:.0f}s"
+                )
+            await asyncio.sleep(RESTART_SECONDS)
+
+    async def _serve(self) -> None:
         requeued = await repository.requeue_running(self.pool, self.account_id)
         if requeued:
             logger.info(f"[yellow]Requeued {requeued} stale running job(s).[/]")
         pruned = await repository.prune_finished(self.pool, self.account_id)
         if pruned:
             logger.info(f"[yellow]Pruned {pruned} finished job(s).[/]")
-        conn = await asyncpg.connect(dsn=env.db.connection_url)
-        await conn.add_listener(JOBS_CHANGED_CHANNEL, lambda *_: self._wake.set())
+        listener = Listener(
+            f"jobs {self.account_id}",
+            {JOBS_CHANGED_CHANNEL: lambda _payload: self._wake.set()},
+            self._wake.set,
+        )
+        await listener.start()
         logger.info(f"[green]Job consumer running for account {self.account_id}.[/]")
         try:
             while True:
-                await self.drain()
                 self._wake.clear()
+                await self.drain()
                 with contextlib.suppress(TimeoutError):
                     await asyncio.wait_for(
-                        self._wake.wait(), timeout=POLL_INTERVAL_SECONDS
+                        self._wake.wait(), timeout=await self._idle_seconds()
                     )
         finally:
-            with contextlib.suppress(Exception):
-                await conn.close()
+            await listener.stop()
+
+    async def _idle_seconds(self) -> float:
+        due = await repository.seconds_to_next_due(self.pool, self.account_id)
+        return POLL_INTERVAL_SECONDS if due is None else min(due, POLL_INTERVAL_SECONDS)

backend/src/userbot/modules/jobs/handlers/__init__.py

@@ -5,6 +5,7 @@ from userbot.modules.jobs.handlers import (
     fetch_avatar,
     fetch_custom_emoji,
     fetch_media,
+    fetch_read_info,
     search_peers,
     sync_contacts,
     sync_dialogs,
@@ -18,6 +19,7 @@ __all__ = [
     "fetch_avatar",
     "fetch_custom_emoji",
     "fetch_media",
+    "fetch_read_info",
     "search_peers",
     "sync_contacts",
     "sync_dialogs",

backend/src/userbot/modules/jobs/handlers/enrich_chat.py

@@ -1,17 +1,11 @@
-from datetime import UTC, datetime
-
 from pyrogram import Client
 from pyrogram.errors import BadRequest, Forbidden
 from pyrogram.types import User
 
-from userbot.modules.avatars import note_avatar
 from userbot.modules.capture.context import CaptureContext
-from userbot.modules.groups.repository import insert_chat_history
 from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import register
-from userbot.modules.profiles.parse import snapshot_from_high_level
-from userbot.modules.profiles.repository import write_profile
-from userbot.modules.raw_json import raw_json
+from userbot.modules.profiles.snapshots import save_group, save_user
 
 MEMBER_CAP = 200
 
@@ -25,43 +19,17 @@ LIMIT 100
 """
 
 
-async def _save_user(ctx: CaptureContext, user: User) -> None:
-    fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
-    await write_profile(ctx.pool, ctx.account_id, user.id, fields, raw_json(user))
-    if photo_file_id and photo_unique_id:
-        await note_avatar(
-            ctx.pool, ctx.account_id, user.id, "peer", photo_unique_id, photo_file_id
-        )
-
-
 async def _enrich_chat_meta(client: Client, ctx: CaptureContext, chat_id: int) -> None:
     chat = await client.get_chat(chat_id)
-    photo = chat.photo
-    photo_unique_id = photo.big_photo_unique_id if photo else None
-    photo_file_id = photo.big_file_id if photo else None
-    await insert_chat_history(
-        ctx.pool,
-        ctx.account_id,
-        chat_id,
-        0,
-        "meta",
-        chat.title,
-        photo_unique_id,
-        None,
-        datetime.now(UTC),
-        raw_json(chat),
-    )
-    if photo_file_id and photo_unique_id:
-        await note_avatar(
-            ctx.pool, ctx.account_id, chat_id, "chat", photo_unique_id, photo_file_id
-        )
+    if chat is not None:
+        await save_group(ctx, chat)
 
 
 async def _enrich_members(client: Client, ctx: CaptureContext, chat_id: int) -> None:
     try:
         async for member in client.get_chat_members(chat_id, limit=MEMBER_CAP):
             if isinstance(member.user, User):
-                await _save_user(ctx, member.user)
+                await save_user(ctx, member.user)
     except (BadRequest, Forbidden):
         return
 
@@ -87,7 +55,7 @@ async def _enrich_senders(client: Client, ctx: CaptureContext, chat_id: int) ->
     if not ids:
         return
     for user in await _fetch_users(client, ids):
-        await _save_user(ctx, user)
+        await save_user(ctx, user)
 
 
 @register("enrich_chat")

backend/src/userbot/modules/jobs/handlers/fetch_avatar.py

@@ -1,6 +1,13 @@
 from pyrogram import Client
-from pyrogram.errors import FileIdInvalid, FileReferenceExpired, FileReferenceInvalid
-from pyrogram.types import Photo
+from pyrogram.errors import (
+    ChannelInvalid,
+    ChannelPrivate,
+    FileIdInvalid,
+    FileReferenceExpired,
+    FileReferenceInvalid,
+    PeerIdInvalid,
+)
+from pyrogram.types import ChatPhoto
 
 from userbot.modules.avatars.repository import get_avatar_file, mark_avatar_downloaded
 from userbot.modules.download import download_bytes
@@ -8,19 +15,20 @@ from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import register
 
 STALE_FILE_ID = (FileIdInvalid, FileReferenceExpired, FileReferenceInvalid)
+UNKNOWN_OWNER = (PeerIdInvalid, ChannelInvalid, ChannelPrivate)
 
 
 async def _fresh_file_id(client: Client, owner_id: int, unique_id: str) -> str | None:
-    chat = await client.get_chat(owner_id)
-    photo = chat.photo
+    try:
+        chat = await client.get_chat(owner_id)
+    except UNKNOWN_OWNER:
+        return None
+    photo = chat.photo if chat else None
     if photo and photo.big_photo_unique_id == unique_id:
         return photo.big_file_id
-    photos = client.get_chat_photos(owner_id)
-    if photos is None:
-        return None
-    async for item in photos:
-        if isinstance(item, Photo) and item.file_unique_id == unique_id:
-            return item.file_id
+    async for item in client.get_chat_photos(owner_id):
+        if isinstance(item, ChatPhoto) and item.big_photo_unique_id == unique_id:
+            return item.big_file_id
     return None
 
 

backend/src/userbot/modules/jobs/handlers/fetch_media.py

@@ -21,7 +21,9 @@ async def fetch_media(ctx: JobContext) -> None:
         msg = f"message {chat_id}/{message_id} is unavailable"
         raise RuntimeError(msg)
     toggles = CaptureToggles(media=True, self_destruct_media=True)
-    ok = await capture_media(client, message, capture, chat_id, message_id, toggles)
+    ok = await capture_media(
+        client, message, capture, chat_id, message_id, toggles, force=True
+    )
     if not ok:
         msg = f"media download failed for {chat_id}/{message_id}"
         raise RuntimeError(msg)

backend/src/userbot/modules/jobs/handlers/fetch_read_info.py

@@ -0,0 +1,124 @@
+from datetime import UTC, datetime, timedelta
+
+from pyrogram import Client, raw
+from pyrogram.errors import (
+    ChannelInvalid,
+    ChannelPrivate,
+    ChatTooBig,
+    FloodPremiumWait,
+    FloodWait,
+    MessageIdInvalid,
+    MessageNotReadYet,
+    MessageTooOld,
+    MsgIdInvalid,
+    MsgTooOld,
+    PeerIdInvalid,
+    RPCError,
+    UserPrivacyRestricted,
+    YourPrivacyRestricted,
+)
+
+from userbot.modules.jobs.context import JobContext
+from userbot.modules.jobs.registry import register
+from userbot.modules.read_receipts.repository import save_fetch, save_reads
+from utils.jobs import enqueue
+from utils.logging import logger
+from utils.policy.models import ChatKind
+from utils.read.peers import get_peers
+from utils.read.read_receipts import read_info
+
+RETRY = timedelta(seconds=60)
+LATER = timedelta(days=1)
+
+Refusal = tuple[tuple[type[Exception], ...], str, timedelta]
+
+REFUSALS: tuple[Refusal, ...] = (
+    ((MessageNotReadYet,), "not_read_yet", RETRY),
+    ((UserPrivacyRestricted,), "user_privacy", RETRY),
+    ((YourPrivacyRestricted,), "your_privacy", LATER),
+    ((ChatTooBig,), "chat_too_big", LATER),
+    ((MsgTooOld, MessageTooOld), "too_old", LATER),
+    (
+        (
+            PeerIdInvalid,
+            ChannelInvalid,
+            ChannelPrivate,
+            MessageIdInvalid,
+            MsgIdInvalid,
+            KeyError,
+            ValueError,
+        ),
+        "unavailable",
+        RETRY,
+    ),
+)
+
+
+def _refusal(exc: Exception) -> tuple[str, timedelta]:
+    for kinds, outcome, retry in REFUSALS:
+        if isinstance(exc, kinds):
+            return outcome, retry
+    return "error", RETRY
+
+
+def _at(timestamp: int) -> datetime:
+    return datetime.fromtimestamp(timestamp, UTC)
+
+
+async def ask_telegram(
+    client: Client, chat_id: int, message_id: int, *, group: bool
+) -> list[tuple[int, datetime]]:
+    peer = await client.resolve_peer(chat_id)
+    if peer is None:
+        msg = f"unknown peer {chat_id}"
+        raise ValueError(msg)
+    if group:
+        participants = await client.invoke(
+            raw.functions.messages.GetMessageReadParticipants(
+                peer=peer, msg_id=message_id
+            )
+        )
+        return [(item.user_id, _at(item.date)) for item in participants]
+    outbox = await client.invoke(
+        raw.functions.messages.GetOutboxReadDate(peer=peer, msg_id=message_id)
+    )
+    return [(chat_id, _at(outbox.date))]
+
+
+async def _enrich_unknown(ctx: JobContext, chat_id: int, reader_ids: list[int]) -> None:
+    known = {
+        peer.peer_id for peer in await get_peers(ctx.pool, ctx.account_id, reader_ids)
+    }
+    if set(reader_ids) - known:
+        await enqueue(ctx.pool, ctx.account_id, "enrich_chat", {"chat_id": chat_id})
+
+
+@register("fetch_read_info")
+async def fetch_read_info(ctx: JobContext) -> None:
+    client = ctx.client
+    if client is None:
+        return
+    chat_id = ctx.job.params["chat_id"]
+    message_id = ctx.job.params["message_id"]
+    info = await read_info(ctx.pool, ctx.account_id, chat_id, message_id)
+    if info is None or not info.can_fetch:
+        return
+    group = info.chat_kind == ChatKind.GROUP
+    try:
+        reads = await ask_telegram(client, chat_id, message_id, group=group)
+    except (FloodWait, FloodPremiumWait) as exc:
+        wait = exc.value if isinstance(exc.value, int) else 0
+        retry = max(RETRY, timedelta(seconds=wait))
+        await save_fetch(ctx.pool, ctx.account_id, chat_id, message_id, "flood", retry)
+        return
+    except (RPCError, KeyError, ValueError) as exc:
+        outcome, retry = _refusal(exc)
+        if outcome == "error":
+            logger.warning(f"[yellow]Read info {chat_id}/{message_id}: {exc!r}[/]")
+        await save_fetch(ctx.pool, ctx.account_id, chat_id, message_id, outcome, retry)
+        return
+    source = "participants" if group else "outbox_date"
+    await save_reads(ctx.pool, ctx.account_id, chat_id, message_id, source, reads)
+    await save_fetch(ctx.pool, ctx.account_id, chat_id, message_id, "ok", RETRY)
+    if group and reads:
+        await _enrich_unknown(ctx, chat_id, [reader_id for reader_id, _ in reads])

backend/src/userbot/modules/jobs/handlers/search_peers.py

@@ -46,7 +46,7 @@ async def _save_found(
     source = _source(peer, users, chats)
     if source is None:
         return None
-    chat = Chat._parse_chat(client, source)  # noqa: SLF001
+    chat = await Chat._parse_chat(client, source)  # noqa: SLF001
     if chat is None or chat.id is None:
         return None
     await save_chat(ctx, chat)

backend/src/userbot/modules/jobs/handlers/sync_contacts.py

@@ -1,11 +1,8 @@
 from pyrogram.types import User
 
-from userbot.modules.avatars import note_avatar
 from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import register
-from userbot.modules.profiles.parse import snapshot_from_high_level
-from userbot.modules.profiles.repository import write_profile
-from userbot.modules.raw_json import raw_json
+from userbot.modules.profiles.snapshots import save_user
 
 
 @register("sync_contacts")
@@ -21,17 +18,7 @@ async def sync_contacts(ctx: JobContext) -> None:
     for user in contacts:
         if not isinstance(user, User):
             continue
-        fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
-        await write_profile(ctx.pool, ctx.account_id, user.id, fields, raw_json(user))
-        if photo_file_id and photo_unique_id:
-            await note_avatar(
-                ctx.pool,
-                ctx.account_id,
-                user.id,
-                "peer",
-                photo_unique_id,
-                photo_file_id,
-            )
+        await save_user(capture, user)
         processed += 1
     await capture.contacts.refresh()
     await ctx.report_progress({"processed": processed, "done": True})

backend/src/userbot/modules/jobs/handlers/sync_dialogs.py

@@ -2,14 +2,10 @@ from pyrogram import Client
 from pyrogram.errors import BadRequest, Forbidden
 from pyrogram.types import User
 
-from userbot.modules.avatars import note_avatar
 from userbot.modules.capture.context import CaptureContext
 from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import register
-from userbot.modules.profiles.parse import snapshot_from_high_level
-from userbot.modules.profiles.repository import write_profile
-from userbot.modules.profiles.snapshots import save_group, save_private
-from userbot.modules.raw_json import raw_json
+from userbot.modules.profiles.snapshots import save_group, save_private, save_user
 from utils.logging import logger
 
 SAVE_EVERY = 100
@@ -30,21 +26,8 @@ async def _enrich_users(client: Client, ctx: CaptureContext, ids: list[int]) ->
             continue
         users = result if isinstance(result, list) else [result]
         for user in users:
-            if not isinstance(user, User):
-                continue
-            fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
-            await write_profile(
-                ctx.pool, ctx.account_id, user.id, fields, raw_json(user)
-            )
-            if photo_file_id and photo_unique_id:
-                await note_avatar(
-                    ctx.pool,
-                    ctx.account_id,
-                    user.id,
-                    "peer",
-                    photo_unique_id,
-                    photo_file_id,
-                )
+            if isinstance(user, User):
+                await save_user(ctx, user)
 
 
 @register("sync_dialogs")

backend/src/userbot/modules/jobs/repository.py

@@ -32,6 +32,7 @@ async def claim_next(pool: asyncpg.Pool, account_id: int) -> Job | None:
         "updated_at = now(), attempts = attempts + 1 "
         "WHERE id = ("
         "    SELECT id FROM jobs WHERE account_id = $1 AND status = 'pending' "
+        "    AND (run_after IS NULL OR run_after <= now()) "
         "    ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1"
         ") RETURNING *",
         account_id,
@@ -76,11 +77,21 @@ async def report_progress(
     )
 
 
-async def bump_flood_wait(pool: asyncpg.Pool, job_id: int) -> None:
+async def defer_flood_wait(pool: asyncpg.Pool, job_id: int, seconds: int) -> None:
     await pool.execute(
-        "UPDATE jobs SET flood_waits = flood_waits + 1, updated_at = now() "
-        "WHERE id = $1",
+        "UPDATE jobs SET status = 'pending', flood_waits = flood_waits + 1, "
+        "run_after = now() + make_interval(secs => $2), updated_at = now() "
+        "WHERE id = $1 AND status = 'running'",
         job_id,
+        seconds,
+    )
+
+
+async def seconds_to_next_due(pool: asyncpg.Pool, account_id: int) -> float | None:
+    return await pool.fetchval(
+        "SELECT extract(epoch FROM min(run_after) - now())::float FROM jobs "
+        "WHERE account_id = $1 AND status = 'pending' AND run_after > now()",
+        account_id,
     )
 
 

backend/src/userbot/modules/links/repository.py

@@ -7,6 +7,14 @@ INSERT INTO links
     (account_id, chat_id, message_id, position, url, kind, web_url, web_title,
      web_description, web_site_name, raw)
 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::jsonb)
+ON CONFLICT (account_id, chat_id, message_id, position) DO UPDATE SET
+    url = EXCLUDED.url,
+    kind = EXCLUDED.kind,
+    web_url = EXCLUDED.web_url,
+    web_title = EXCLUDED.web_title,
+    web_description = EXCLUDED.web_description,
+    web_site_name = EXCLUDED.web_site_name,
+    raw = EXCLUDED.raw
 """
 
 

backend/src/userbot/modules/media/downloader.py

@@ -106,6 +106,8 @@ async def capture_media(  # noqa: PLR0913
     chat_id: int,
     message_id: int,
     toggles: CaptureToggles,
+    *,
+    force: bool = False,
 ) -> bool:
     kind, obj = media_object(message)
     if obj is None:
@@ -117,6 +119,7 @@ async def capture_media(  # noqa: PLR0913
     mime = getattr(obj, "mime_type", None)
     storage_key: str | None = None
     downloaded = False
+    evicted = False
     flood: FloodWait | FloodPremiumWait | None = None
     if want:
         existing = await repository.current_media(
@@ -131,7 +134,7 @@ async def capture_media(  # noqa: PLR0913
             storage_key = existing["storage_key"]
             file_size = existing["file_size"]
             downloaded = True
-        else:
+        elif force or not await repository.is_evicted(ctx.pool, unique_id):
             target = _download_target(message)
             try:
                 data = await _download(client, target, chat_id, message_id)
@@ -142,6 +145,8 @@ async def capture_media(  # noqa: PLR0913
                 storage_key = ctx.storage.put(data)
                 file_size = len(data)
                 downloaded = True
+        else:
+            evicted = True
     await repository.insert_media(
         ctx.pool,
         ctx.account_id,
@@ -154,7 +159,14 @@ async def capture_media(  # noqa: PLR0913
         ttl,
         unique_id,
         downloaded=downloaded,
+        evicted=evicted,
     )
+    if (
+        force
+        and storage_key is not None
+        and await repository.unevict(ctx.pool, unique_id, storage_key, file_size)
+    ):
+        ctx.storage.purge(storage_key)
     if flood is not None:
         raise flood
     return downloaded or not want

backend/src/userbot/modules/profiles/parse.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from pyrogram import Client, raw
 from pyrogram.types import Chat, User
@@ -12,6 +12,11 @@ class ProfileFields:
     phone: str | None = None
     photo_unique_id: str | None = None
     is_deleted_account: bool = False
+    unknown: frozenset[str] = field(default=frozenset(), compare=False)
+
+
+MIN_UNKNOWN = frozenset({"username", "phone"})
+CHAT_UNKNOWN = frozenset({"phone", "is_deleted_account"})
 
 
 def active_username(usernames: list[raw.types.Username]) -> str | None:
@@ -21,10 +26,10 @@ def active_username(usernames: list[raw.types.Username]) -> str | None:
     return usernames[0].username if usernames else None
 
 
-def snapshot_from_user(
+async def snapshot_from_user(
     client: Client, raw_user: raw.types.User
 ) -> tuple[ProfileFields, str | None, str | None]:
-    user = User._parse(client, raw_user)  # noqa: SLF001
+    user = await User._parse(client, raw_user)  # noqa: SLF001
     photo = user.photo if user else None
     photo_unique_id = photo.big_photo_unique_id if photo else None
     photo_file_id = photo.big_file_id if photo else None
@@ -35,6 +40,7 @@ def snapshot_from_user(
         phone=user.phone_number if user else None,
         photo_unique_id=photo_unique_id,
         is_deleted_account=bool(getattr(raw_user, "deleted", False)),
+        unknown=MIN_UNKNOWN if getattr(raw_user, "min", False) else frozenset(),
     )
     return fields, photo_file_id, photo_unique_id
 
@@ -52,6 +58,7 @@ def snapshot_from_high_level(
         phone=user.phone_number,
         photo_unique_id=photo_unique_id,
         is_deleted_account=bool(user.is_deleted),
+        unknown=MIN_UNKNOWN if user.is_min else frozenset(),
     )
     return fields, photo_file_id, photo_unique_id
 
@@ -64,8 +71,7 @@ def snapshot_from_chat(chat: Chat) -> tuple[ProfileFields, str | None, str | Non
         first_name=chat.first_name,
         last_name=chat.last_name,
         username=chat.username,
-        phone=None,
         photo_unique_id=photo_unique_id,
-        is_deleted_account=False,
+        unknown=CHAT_UNKNOWN,
     )
     return fields, photo_file_id, photo_unique_id

backend/src/userbot/modules/profiles/repository.py

@@ -9,26 +9,46 @@ WHERE account_id = $1 AND peer_id = $2
 """
 
 _UPSERT_PEER = """
-INSERT INTO peers
+INSERT INTO peers AS p
     (account_id, peer_id, first_name, last_name, username, phone,
      photo_unique_id, is_deleted_account, raw, updated_at)
 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, now())
 ON CONFLICT (account_id, peer_id) DO UPDATE SET
     first_name = EXCLUDED.first_name,
     last_name = EXCLUDED.last_name,
-    username = EXCLUDED.username,
-    phone = EXCLUDED.phone,
+    username = CASE WHEN 'username' = ANY($10::text[])
+        THEN COALESCE(EXCLUDED.username, p.username) ELSE EXCLUDED.username END,
+    phone = CASE WHEN 'phone' = ANY($10::text[])
+        THEN COALESCE(EXCLUDED.phone, p.phone) ELSE EXCLUDED.phone END,
     photo_unique_id = EXCLUDED.photo_unique_id,
-    is_deleted_account = EXCLUDED.is_deleted_account,
+    is_deleted_account = CASE WHEN 'is_deleted_account' = ANY($10::text[])
+        THEN p.is_deleted_account ELSE EXCLUDED.is_deleted_account END,
     raw = EXCLUDED.raw,
     updated_at = now()
 """
 
-_INSERT_HISTORY = """
+_INSERT_CHANGE = """
 INSERT INTO peer_history
     (account_id, peer_id, observed_at, first_name, last_name, username, phone,
      photo_unique_id, is_deleted_account, raw)
-VALUES ($1, $2, now(), $3, $4, $5, $6, $7, $8, $9::jsonb)
+SELECT account_id, peer_id, clock_timestamp(), first_name, last_name, username,
+       phone, photo_unique_id, is_deleted_account, raw
+FROM peers p
+WHERE account_id = $1 AND peer_id = $2
+  AND NOT EXISTS (
+    SELECT 1 FROM (
+        SELECT first_name, last_name, username, phone, photo_unique_id,
+               is_deleted_account
+        FROM peer_history
+        WHERE account_id = $1 AND peer_id = $2
+        ORDER BY observed_at DESC LIMIT 1
+    ) latest
+    WHERE (latest.first_name, latest.last_name, latest.username, latest.phone,
+           latest.photo_unique_id, latest.is_deleted_account)
+          IS NOT DISTINCT FROM
+          (p.first_name, p.last_name, p.username, p.phone,
+           p.photo_unique_id, p.is_deleted_account)
+  )
 ON CONFLICT DO NOTHING
 """
 
@@ -49,18 +69,30 @@ async def get_peer(
     )
 
 
-async def write_profile(
-    pool: asyncpg.Pool, account_id: int, peer_id: int, fields: ProfileFields, raw: str
-) -> None:
-    args = (
-        fields.first_name,
-        fields.last_name,
-        fields.username,
-        fields.phone,
-        fields.photo_unique_id,
-        fields.is_deleted_account,
-        raw,
-    )
+async def write_profile(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    account_id: int,
+    peer_id: int,
+    fields: ProfileFields,
+    raw: str,
+    *,
+    history: bool,
+) -> bool:
     async with pool.acquire() as conn, conn.transaction():
-        await conn.execute(_UPSERT_PEER, account_id, peer_id, *args)
-        await conn.execute(_INSERT_HISTORY, account_id, peer_id, *args)
+        await conn.execute(
+            _UPSERT_PEER,
+            account_id,
+            peer_id,
+            fields.first_name,
+            fields.last_name,
+            fields.username,
+            fields.phone,
+            fields.photo_unique_id,
+            fields.is_deleted_account,
+            raw,
+            sorted(fields.unknown),
+        )
+        if not history:
+            return False
+        status = await conn.execute(_INSERT_CHANGE, account_id, peer_id)
+    return status == "INSERT 0 1"

backend/src/userbot/modules/profiles/snapshots.py

@@ -1,19 +1,46 @@
 from datetime import UTC, datetime
 
-from pyrogram.types import Chat
+from pyrogram.enums import ChatType
+from pyrogram.types import Chat, User
 
 from userbot.modules.avatars import note_avatar
 from userbot.modules.capture.context import CaptureContext
-from userbot.modules.groups.repository import insert_chat_history
-from userbot.modules.profiles.parse import snapshot_from_chat
+from userbot.modules.groups.repository import record_chat_meta
+from userbot.modules.profiles.parse import snapshot_from_chat, snapshot_from_high_level
 from userbot.modules.profiles.repository import write_profile
 from userbot.modules.raw_json import raw_json
 
 
+async def save_user(ctx: CaptureContext, user: User) -> None:
+    fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
+    if not await ctx.peer_identity.changed(ctx.pool, ctx.account_id, user.id, fields):
+        return
+    await write_profile(
+        ctx.pool,
+        ctx.account_id,
+        user.id,
+        fields,
+        raw_json(user),
+        history=ctx.resolve_peer(user.id, is_bot=bool(user.is_bot)).profile_history,
+    )
+    if photo_file_id and photo_unique_id:
+        await note_avatar(
+            ctx.pool, ctx.account_id, user.id, "peer", photo_unique_id, photo_file_id
+        )
+
+
 async def save_private(ctx: CaptureContext, chat: Chat) -> bool:
     chat_id = chat.id or 0
     fields, photo_file_id, photo_unique_id = snapshot_from_chat(chat)
-    await write_profile(ctx.pool, ctx.account_id, chat_id, fields, raw_json(chat))
+    is_bot = chat.type is ChatType.BOT
+    await write_profile(
+        ctx.pool,
+        ctx.account_id,
+        chat_id,
+        fields,
+        raw_json(chat),
+        history=ctx.resolve_peer(chat_id, is_bot=is_bot).profile_history,
+    )
     if photo_file_id and photo_unique_id:
         await note_avatar(
             ctx.pool, ctx.account_id, chat_id, "peer", photo_unique_id, photo_file_id
@@ -26,15 +53,13 @@ async def save_group(ctx: CaptureContext, chat: Chat) -> None:
     photo = chat.photo
     photo_unique_id = photo.big_photo_unique_id if photo else None
     photo_file_id = photo.big_file_id if photo else None
-    await insert_chat_history(
+    await record_chat_meta(
         ctx.pool,
         ctx.account_id,
         chat_id,
         0,
-        "meta",
         chat.title,
         photo_unique_id,
-        None,
         datetime.now(UTC),
         raw_json(chat),
     )

backend/src/userbot/modules/raw_json.py

@@ -1,8 +1,23 @@
+import json
+from typing import Any
+
 from pyrogram.raw.core import TLObject
-from pyrogram.types import Object
+from pyrogram.types import Contact, Object
 
 EMPTY_JSON = "{}"
+PHONE_KEY = "phone_number"
+
+
+def _default(obj: Any) -> Any:  # noqa: ANN401
+    if isinstance(obj, TLObject):
+        return TLObject.default(obj)
+    data = Object.default(obj)
+    if isinstance(obj, Contact) and isinstance(data, dict) and PHONE_KEY in data:
+        data[PHONE_KEY] = obj.phone_number
+    return data
 
 
 def raw_json(obj: Object | TLObject | None) -> str:
-    return EMPTY_JSON if obj is None else str(obj)
+    if obj is None:
+        return EMPTY_JSON
+    return json.dumps(obj, indent=4, default=_default, ensure_ascii=False)

backend/src/userbot/modules/read_receipts/repository.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import asyncpg
 
@@ -9,6 +9,26 @@ VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
 ON CONFLICT (account_id, chat_id, message_id, ts) DO NOTHING
 """
 
+_UPSERT_READ = """
+INSERT INTO message_reads
+    (account_id, chat_id, message_id, reader_id, read_at, source, fetched_at)
+VALUES ($1, $2, $3, $4, $5, $6, now())
+ON CONFLICT (account_id, chat_id, message_id, reader_id) DO UPDATE SET
+    read_at = EXCLUDED.read_at,
+    source = EXCLUDED.source,
+    fetched_at = EXCLUDED.fetched_at
+"""
+
+_UPSERT_FETCH = """
+INSERT INTO read_fetches
+    (account_id, chat_id, message_id, outcome, fetched_at, retry_at)
+VALUES ($1, $2, $3, $4, now(), now() + $5::interval)
+ON CONFLICT (account_id, chat_id, message_id) DO UPDATE SET
+    outcome = EXCLUDED.outcome,
+    fetched_at = EXCLUDED.fetched_at,
+    retry_at = EXCLUDED.retry_at
+"""
+
 
 async def insert_read(  # noqa: PLR0913
     pool: asyncpg.Pool,
@@ -34,3 +54,33 @@ async def insert_listened(  # noqa: PLR0913
     await pool.execute(
         _INSERT, account_id, chat_id, message_id, ts, reader_id, "listened", raw
     )
+
+
+async def save_reads(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    account_id: int,
+    chat_id: int,
+    message_id: int,
+    source: str,
+    reads: list[tuple[int, datetime]],
+) -> None:
+    await pool.executemany(
+        _UPSERT_READ,
+        [
+            (account_id, chat_id, message_id, reader_id, read_at, source)
+            for reader_id, read_at in reads
+        ],
+    )
+
+
+async def save_fetch(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    account_id: int,
+    chat_id: int,
+    message_id: int,
+    outcome: str,
+    retry_after: timedelta,
+) -> None:
+    await pool.execute(
+        _UPSERT_FETCH, account_id, chat_id, message_id, outcome, retry_after
+    )

backend/src/userbot/modules/stories/repository.py

@@ -6,7 +6,7 @@ _UPSERT_STORY = """
 INSERT INTO stories
     (account_id, peer_id, story_id, date, expire_date, caption, media_kind,
      storage_key, file_size, downloaded, views, pinned, deleted, raw, observed_at)
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14::jsonb, now())
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, false, $13::jsonb, now())
 ON CONFLICT (account_id, peer_id, story_id) DO UPDATE SET
     date = EXCLUDED.date,
     expire_date = EXCLUDED.expire_date,
@@ -17,10 +17,17 @@ ON CONFLICT (account_id, peer_id, story_id) DO UPDATE SET
     downloaded = stories.downloaded OR EXCLUDED.downloaded,
     views = EXCLUDED.views,
     pinned = EXCLUDED.pinned,
-    deleted = EXCLUDED.deleted,
+    deleted = false,
     raw = EXCLUDED.raw
 """
 
+_MARK_DELETED = """
+INSERT INTO stories
+    (account_id, peer_id, story_id, downloaded, pinned, deleted, raw, observed_at)
+VALUES ($1, $2, $3, false, false, true, '{}'::jsonb, now())
+ON CONFLICT (account_id, peer_id, story_id) DO UPDATE SET deleted = true
+"""
+
 
 async def is_downloaded(
     pool: asyncpg.Pool, account_id: int, peer_id: int, story_id: int
@@ -51,7 +58,6 @@ async def upsert_story(  # noqa: PLR0913
     raw: str,
     *,
     pinned: bool,
-    deleted: bool,
     downloaded: bool,
 ) -> None:
     await pool.execute(
@@ -68,6 +74,11 @@ async def upsert_story(  # noqa: PLR0913
         downloaded,
         views,
         pinned,
-        deleted,
         raw,
     )
+
+
+async def mark_deleted(
+    pool: asyncpg.Pool, account_id: int, peer_id: int, story_id: int
+) -> None:
+    await pool.execute(_MARK_DELETED, account_id, peer_id, story_id)

backend/src/userbot/modules/stories/service.py

@@ -17,13 +17,18 @@ def story_peer_id(story: Story) -> int:
 
 async def save_story(client: Client, capture: CaptureContext, story: Story) -> None:
     peer_id = story_peer_id(story)
+    if story.deleted:
+        await repository.mark_deleted(
+            capture.pool, capture.account_id, peer_id, story.id
+        )
+        return
     storage_key: str | None = None
     file_size: int | None = None
     downloaded = False
     stored = await repository.is_downloaded(
         capture.pool, capture.account_id, peer_id, story.id
     )
-    if not (stored or story.deleted or story.media is None):
+    if not (stored or story.media is None):
         data = await download_bytes(client, story)
         if data is not None:
             storage_key = capture.storage.put(data)
@@ -43,6 +48,5 @@ async def save_story(client: Client, capture: CaptureContext, story: Story) -> N
         story.views,
         raw_json(story.raw),
         pinned=bool(story.pinned),
-        deleted=bool(story.deleted),
         downloaded=downloaded,
     )

backend/src/userbot/modules/uptime/repository.py

@@ -0,0 +1,16 @@
+import asyncpg
+
+_HEARTBEAT = """
+WITH extended AS (
+    UPDATE capture_uptime SET seen_at = now()
+    WHERE account_id = $1 AND seen_at >= now() - interval '150 seconds'
+    RETURNING account_id
+)
+INSERT INTO capture_uptime (account_id, started_at, seen_at)
+SELECT $1, now(), now()
+WHERE NOT EXISTS (SELECT 1 FROM extended)
+"""
+
+
+async def heartbeat(pool: asyncpg.Pool, account_id: int) -> None:
+    await pool.execute(_HEARTBEAT, account_id)

backend/src/userbot/modules/watches/cache.py

@@ -1,60 +1,182 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING, Any
 
+from userbot.modules.watches import repository
 from userbot.modules.watches.evaluator import (
-    KIND_KEYWORD,
-    KIND_PEER_ONLINE,
+    Rules,
+    Seen,
+    build_rules,
+    match_chat_scope,
     match_keyword,
-    match_peer_online,
+    match_new_message,
 )
 from utils.logging import logger
 from utils.read import watches as watches_read
+from utils.read.accounts import self_user_id
 
 if TYPE_CHECKING:
+    from collections.abc import Coroutine
+
     import asyncpg
 
-    from utils.read.models import WatchView
+ONLINE_QUIET = timedelta(minutes=15)
+PREVIEW_CHARS = 500
+
+
+def _message_payload(event: str, seen: Seen) -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        "event": event,
+        "chat_id": seen.chat_id,
+        "message_id": seen.message_id,
+        "sender_id": seen.sender_id,
+        "text": (seen.text or "")[:PREVIEW_CHARS] or None,
+    }
+    if seen.date is not None:
+        payload["date"] = seen.date.isoformat()
+    return payload
 
 
 class WatchCache:
     def __init__(self, pool: asyncpg.Pool, account_id: int) -> None:
         self._pool = pool
         self._account_id = account_id
-        self.watches: list[WatchView] = []
-        self._online: set[int] = set()
+        self.rules = Rules()
+        self._self_id: int | None = None
+        self._last_seen: dict[int, datetime] = {}
 
     async def refresh(self) -> None:
         rows = await watches_read.list_watches(self._pool, self._account_id)
-        self.watches = [watch for watch in rows if watch.enabled]
-        logger.info(f"[green]Watches cached:[/] {len(self.watches)}")
+        self.rules = build_rules(rows)
+        self._self_id = await self_user_id(self._pool, self._account_id)
+        logger.info(f"[green]Watches cached:[/] {len(self.rules)}")
 
-    async def on_text(self, chat_id: int, message_id: int, text: str | None) -> None:
-        if not text:
-            return
-        for watch in self.watches:
-            if watch.kind == KIND_KEYWORD and match_keyword(
-                text, chat_id, watch.params
-            ):
-                await watches_read.insert_alert(
-                    self._pool,
-                    self._account_id,
-                    watch.id,
-                    {"chat_id": chat_id, "message_id": message_id},
-                    dedup_key=f"{chat_id}:{message_id}",
+    def _own(self, seen: Seen) -> bool:
+        return seen.outgoing or (
+            self._self_id is not None and seen.sender_id == self._self_id
+        )
+
+    async def _guard(self, work: Coroutine[Any, Any, None]) -> None:
+        try:
+            await work
+        except Exception:
+            logger.exception("[red]Watch evaluation failed[/]")
+
+    async def _fire(self, watch_id: int, payload: dict[str, Any], key: str) -> None:
+        await watches_read.insert_alert(
+            self._pool, self._account_id, watch_id, payload, dedup_key=key
+        )
+
+    async def _keyword(self, event: str, seen: Seen) -> None:
+        for rule in self.rules.keyword:
+            if match_keyword(rule, seen):
+                await self._fire(
+                    rule.watch_id,
+                    _message_payload(event, seen),
+                    f"{seen.chat_id}:{seen.message_id}",
                 )
 
-    async def on_status(self, peer_id: int, *, is_online: bool) -> None:
-        if not is_online:
-            self._online.discard(peer_id)
+    async def on_message(self, seen: Seen) -> None:
+        if self._own(seen):
             return
-        if peer_id in self._online:
-            return
-        self._online.add(peer_id)
-        for watch in self.watches:
-            if watch.kind == KIND_PEER_ONLINE and match_peer_online(
-                peer_id, watch.params
-            ):
-                await watches_read.insert_alert(
-                    self._pool, self._account_id, watch.id, {"peer_id": peer_id}
+        await self._guard(self._on_message(seen))
+
+    async def _on_message(self, seen: Seen) -> None:
+        await self._keyword("message", seen)
+        for watch_id, params in self.rules.new_message:
+            if match_new_message(params, seen):
+                await self._fire(
+                    watch_id,
+                    _message_payload("message", seen),
+                    f"{seen.chat_id}:{seen.message_id}",
                 )
+
+    async def on_edit(self, seen: Seen) -> None:
+        if self._own(seen):
+            return
+        await self._guard(self._on_edit(seen))
+
+    async def _on_edit(self, seen: Seen) -> None:
+        await self._keyword("edit", seen)
+        if seen.edit_date is None:
+            return
+        stamp = int(seen.edit_date.timestamp())
+        for watch_id, params in self.rules.edited:
+            if match_chat_scope(params, seen):
+                await self._fire(
+                    watch_id,
+                    _message_payload("edit", seen),
+                    f"{seen.chat_id}:{seen.message_id}:{stamp}",
+                )
+
+    async def on_transcript(self, chat_id: int, message_id: int, text: str) -> None:
+        if self.rules.keyword:
+            await self._guard(self._on_transcript(chat_id, message_id, text))
+
+    async def _on_transcript(self, chat_id: int, message_id: int, text: str) -> None:
+        sender_id = await repository.message_sender(
+            self._pool, self._account_id, chat_id, message_id
+        )
+        seen = Seen(chat_id, message_id, sender_id, text)
+        if not self._own(seen):
+            await self._keyword("transcript", seen)
+
+    async def on_deleted(self, chat_id: int | None, message_ids: list[int]) -> None:
+        if self.rules.deleted and message_ids:
+            await self._guard(self._on_deleted(chat_id, message_ids))
+
+    async def _on_deleted(self, chat_id: int | None, message_ids: list[int]) -> None:
+        rows = await repository.deleted_messages(
+            self._pool, self._account_id, chat_id, message_ids
+        )
+        for row in rows:
+            seen = Seen(
+                row["chat_id"],
+                row["message_id"],
+                row["sender_id"],
+                row["text"],
+                date=row["date"],
+            )
+            if self._own(seen):
+                continue
+            for watch_id, params in self.rules.deleted:
+                if match_chat_scope(params, seen):
+                    await self._fire(
+                        watch_id,
+                        _message_payload("delete", seen),
+                        f"{seen.chat_id}:{seen.message_id}",
+                    )
+
+    async def on_status(
+        self, peer_id: int, status: str, at: datetime, last_online: datetime | None
+    ) -> None:
+        watch_ids = [
+            watch_id
+            for watch_id, params in self.rules.online
+            if params.peer_id == peer_id
+        ]
+        if not watch_ids:
+            return
+        if status != "online":
+            self._last_seen[peer_id] = (last_online or at).astimezone(UTC)
+            return
+        await self._guard(self._on_online(peer_id, at, watch_ids))
+
+    async def _on_online(
+        self, peer_id: int, at: datetime, watch_ids: list[int]
+    ) -> None:
+        stored = await repository.last_seen_before(
+            self._pool, self._account_id, peer_id, at
+        )
+        known = [value for value in (stored, self._last_seen.get(peer_id)) if value]
+        previous = max(known, default=None)
+        self._last_seen[peer_id] = at
+        if previous is not None and at - previous < ONLINE_QUIET:
+            return
+        session = int((previous or at).timestamp())
+        payload: dict[str, Any] = {"event": "online", "peer_id": peer_id}
+        if previous is not None:
+            payload["since"] = previous.isoformat()
+        for watch_id in watch_ids:
+            await self._fire(watch_id, payload, f"{peer_id}:{session}")

backend/src/userbot/modules/watches/evaluator.py

@@ -1,24 +1,104 @@
 import re
-from typing import Any
+from dataclasses import dataclass, field
+from datetime import datetime
 
-KIND_KEYWORD = "keyword"
-KIND_PEER_ONLINE = "peer_online"
+from utils.logging import logger
+from utils.read.models import WatchView
+from utils.watch_rules import (
+    MAX_TEXT,
+    DeletedParams,
+    EditedParams,
+    KeywordParams,
+    NewMessageParams,
+    PeerOnlineParams,
+    WatchParamsError,
+    compile_pattern,
+    parse_params,
+)
 
 
-def match_keyword(text: str, chat_id: int, params: dict[str, Any]) -> bool:
-    target_chat = params.get("chat_id")
-    if target_chat is not None and target_chat != chat_id:
-        return False
-    pattern = params.get("pattern")
-    if not pattern:
-        return False
-    if params.get("regex"):
+@dataclass(frozen=True, slots=True)
+class Seen:
+    chat_id: int
+    message_id: int
+    sender_id: int | None
+    text: str | None
+    outgoing: bool = False
+    date: datetime | None = None
+    edit_date: datetime | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class KeywordRule:
+    watch_id: int
+    params: KeywordParams
+    regex: re.Pattern[str] | None
+
+
+@dataclass(slots=True)
+class Rules:
+    keyword: list[KeywordRule] = field(default_factory=list)
+    new_message: list[tuple[int, NewMessageParams]] = field(default_factory=list)
+    deleted: list[tuple[int, DeletedParams]] = field(default_factory=list)
+    edited: list[tuple[int, EditedParams]] = field(default_factory=list)
+    online: list[tuple[int, PeerOnlineParams]] = field(default_factory=list)
+
+    def __len__(self) -> int:
+        return (
+            len(self.keyword)
+            + len(self.new_message)
+            + len(self.deleted)
+            + len(self.edited)
+            + len(self.online)
+        )
+
+
+def build_rules(watches: list[WatchView]) -> Rules:
+    rules = Rules()
+    for watch in watches:
+        if not watch.enabled:
+            continue
         try:
-            return re.search(pattern, text) is not None
-        except re.error:
-            return False
-    return pattern.casefold() in text.casefold()
+            params = parse_params(watch.kind, watch.params)
+        except WatchParamsError as exc:
+            logger.warning(f"[yellow]Watch {watch.id} skipped:[/] {exc}")
+            continue
+        match params:
+            case KeywordParams():
+                regex = compile_pattern(params.pattern) if params.regex else None
+                rules.keyword.append(KeywordRule(watch.id, params, regex))
+            case NewMessageParams():
+                rules.new_message.append((watch.id, params))
+            case DeletedParams():
+                rules.deleted.append((watch.id, params))
+            case EditedParams():
+                rules.edited.append((watch.id, params))
+            case PeerOnlineParams():
+                rules.online.append((watch.id, params))
+    return rules
 
 
-def match_peer_online(peer_id: int, params: dict[str, Any]) -> bool:
-    return params.get("peer_id") == peer_id
+def _in_scope(chat_id: int | None, sender_id: int | None, seen: Seen) -> bool:
+    if chat_id is not None and chat_id != seen.chat_id:
+        return False
+    return sender_id is None or sender_id == seen.sender_id
+
+
+def match_keyword(rule: KeywordRule, seen: Seen) -> bool:
+    params = rule.params
+    if not seen.text or not _in_scope(params.chat_id, params.sender_id, seen):
+        return False
+    sample = seen.text[:MAX_TEXT]
+    if rule.regex is not None:
+        return rule.regex.search(sample) is not None
+    return params.pattern.casefold() in sample.casefold()
+
+
+def match_new_message(params: NewMessageParams, seen: Seen) -> bool:
+    return _in_scope(params.chat_id, params.sender_id, seen)
+
+
+def match_chat_scope(params: DeletedParams | EditedParams, seen: Seen) -> bool:
+    if params.chat_id is None:
+        return seen.chat_id > 0
+    return params.chat_id == seen.chat_id

backend/src/userbot/modules/watches/repository.py

@@ -0,0 +1,48 @@
+from datetime import datetime
+
+import asyncpg
+
+from userbot.modules.capture.repository import CHANNEL_ID_THRESHOLD
+
+_DELETED = """
+SELECT DISTINCT ON (chat_id, message_id) chat_id, message_id, sender_id, text, date
+FROM messages
+WHERE account_id = $1 AND message_id = ANY($2::bigint[])
+  AND deleted_at IS NOT NULL AND {scope}
+ORDER BY chat_id, message_id, date DESC
+"""
+
+_SENDER = """
+SELECT sender_id FROM messages
+WHERE account_id = $1 AND chat_id = $2 AND message_id = $3
+ORDER BY date DESC LIMIT 1
+"""
+
+_LAST_SEEN = """
+SELECT CASE WHEN status = 'online' THEN ts ELSE coalesce(last_online_date, ts) END
+FROM presence
+WHERE account_id = $1 AND peer_id = $2 AND ts < $3
+ORDER BY ts DESC LIMIT 1
+"""
+
+
+async def deleted_messages(
+    pool: asyncpg.Pool, account_id: int, chat_id: int | None, message_ids: list[int]
+) -> list[asyncpg.Record]:
+    if chat_id is None:
+        query = _DELETED.format(scope="chat_id > $3")
+        return await pool.fetch(query, account_id, message_ids, CHANNEL_ID_THRESHOLD)
+    query = _DELETED.format(scope="chat_id = $3")
+    return await pool.fetch(query, account_id, message_ids, chat_id)
+
+
+async def message_sender(
+    pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
+) -> int | None:
+    return await pool.fetchval(_SENDER, account_id, chat_id, message_id)
+
+
+async def last_seen_before(
+    pool: asyncpg.Pool, account_id: int, peer_id: int, at: datetime
+) -> datetime | None:
+    return await pool.fetchval(_LAST_SEEN, account_id, peer_id, at)

backend/src/userbot/modules/watches/seen.py

@@ -0,0 +1,22 @@
+from datetime import UTC, datetime
+
+from pyrogram.types import Message
+
+from userbot.modules.capture.message import sender_id
+from userbot.modules.watches.evaluator import Seen
+
+
+def utc(moment: datetime | None) -> datetime | None:
+    return moment.astimezone(UTC) if moment else None
+
+
+def seen_message(chat_id: int, message: Message) -> Seen:
+    return Seen(
+        chat_id=chat_id,
+        message_id=message.id,
+        sender_id=sender_id(message),
+        text=message.text or message.caption,
+        outgoing=bool(message.outgoing),
+        date=utc(message.date),
+        edit_date=utc(message.edit_date),
+    )

backend/src/userbot/runner.py

@@ -11,10 +11,13 @@ import uvloop
 
 from dependencies.container import container
 from userbot.modules.capture import CaptureContext, build_capture_context
+from userbot.modules.catchup import SavedState, catch_up, read_saved_state
 from userbot.modules.client import PyroClient
 from userbot.modules.jobs import JobConsumer
+from userbot.modules.uptime import repository as uptime
 from utils.env import env
 from utils.jobs import enqueue
+from utils.listen import Listener
 from utils.logging import logger, setup_logging
 from utils.read.accounts import (
     ACCOUNTS_CHANGED_CHANNEL,
@@ -28,12 +31,14 @@ from utils.storage import ContentAddressedStorage
 setup_logging()
 
 SESSION_FLUSH_INTERVAL = 60
+SESSION_RETRY_INTERVAL = 60
+UPTIME_INTERVAL = 60
 
 
 @dataclass
 class RunningAccount:
     client: PyroClient
-    consumer_task: asyncio.Task
+    tasks: list[asyncio.Task]
     device_model: str | None
 
 
@@ -54,6 +59,7 @@ class AccountRegistry:
         self._pool = pool
         self._storage = storage
         self._running: dict[str, RunningAccount] = {}
+        self._saved: dict[str, SavedState] = {}
         self._lock = asyncio.Lock()
 
     @property
@@ -86,6 +92,8 @@ class AccountRegistry:
 
     async def _start(self, path: Path, device_model: str | None) -> None:
         session_name = path.stem
+        if session_name not in self._saved:
+            self._saved[session_name] = await asyncio.to_thread(read_saved_state, path)
         client = PyroClient(
             session_name, workdir=str(path.parent), device_model=device_model
         )
@@ -100,14 +108,20 @@ class AccountRegistry:
                 client, self._pool, self._storage, account_id
             )
         except Exception:
-            logger.exception(f"[red]Failed to start session:[/] {session_name}")
+            logger.exception(
+                f"[red]Failed to start session:[/] {session_name}, "
+                f"retrying in {SESSION_RETRY_INTERVAL}s"
+            )
             with contextlib.suppress(Exception):
                 await client.stop()
             return
         consumer = JobConsumer(client, self._pool, account_id)
-        self._running[session_name] = RunningAccount(
-            client, asyncio.create_task(consumer.run()), device_model
-        )
+        saved = self._saved.pop(session_name)
+        tasks = [
+            asyncio.create_task(consumer.run()),
+            asyncio.create_task(catch_up(client, saved)),
+        ]
+        self._running[session_name] = RunningAccount(client, tasks, device_model)
         logger.info(f"[green]Client started:[/] {me.full_name} ({me.id})")
         await enqueue(self._pool, account_id, "sync_dialogs", {})
         await enqueue(self._pool, account_id, "sync_contacts", {})
@@ -116,7 +130,8 @@ class AccountRegistry:
         account = self._running.pop(session_name, None)
         if account is None:
             return
-        await _cancel(account.consumer_task)
+        for task in account.tasks:
+            await _cancel(task)
         with contextlib.suppress(Exception):
             await account.client.stop()
         logger.info(f"[yellow]Client stopped:[/] {session_name}")
@@ -124,7 +139,8 @@ class AccountRegistry:
     async def _log_out(self, path: Path) -> None:
         account = self._running.pop(path.stem, None)
         if account is not None:
-            await _cancel(account.consumer_task)
+            for task in account.tasks:
+                await _cancel(task)
             client = account.client
         else:
             client = PyroClient(
@@ -148,9 +164,25 @@ async def _flush_sessions(registry: AccountRegistry) -> None:
                 await client.storage.save()
 
 
-async def _listen_changes(
-    registry: AccountRegistry, tasks: set[asyncio.Task]
-) -> asyncpg.Connection:
+async def _retry_sessions(registry: AccountRegistry) -> None:
+    while True:
+        await asyncio.sleep(SESSION_RETRY_INTERVAL)
+        try:
+            await registry.sync()
+        except Exception:
+            logger.exception("[red]Session sync failed[/]")
+
+
+async def _keep_uptime(registry: AccountRegistry, pool: asyncpg.Pool) -> None:
+    while True:
+        for client in registry.clients:
+            if client.capture is not None and client.is_connected:
+                with contextlib.suppress(Exception):
+                    await uptime.heartbeat(pool, client.capture.account_id)
+        await asyncio.sleep(UPTIME_INTERVAL)
+
+
+def _change_listener(registry: AccountRegistry, tasks: set[asyncio.Task]) -> Listener:
     def spawn(coro: Coroutine[Any, Any, None]) -> None:
         task = asyncio.create_task(coro)
         tasks.add(task)
@@ -163,26 +195,29 @@ async def _listen_changes(
             if client.capture is not None:
                 spawn(make_coro(client.capture))
 
-    def on_policy(
-        _conn: asyncpg.Connection, _pid: int, _channel: str, _payload: str
-    ) -> None:
+    def on_policy(_payload: str) -> None:
         reload(lambda capture: capture.reload_policies())
 
-    def on_watch(
-        _conn: asyncpg.Connection, _pid: int, _channel: str, _payload: str
-    ) -> None:
+    def on_watch(_payload: str) -> None:
         reload(lambda capture: capture.watches.refresh())
 
-    def on_accounts(
-        _conn: asyncpg.Connection, _pid: int, _channel: str, _payload: str
-    ) -> None:
+    def on_accounts(_payload: str) -> None:
         spawn(registry.sync())
 
-    conn = await asyncpg.connect(dsn=env.db.connection_url)
-    await conn.add_listener("policy_changed", on_policy)
-    await conn.add_listener(WATCHES_CHANGED_CHANNEL, on_watch)
-    await conn.add_listener(ACCOUNTS_CHANGED_CHANNEL, on_accounts)
-    return conn
+    def on_reconnect() -> None:
+        on_policy("")
+        on_watch("")
+        on_accounts("")
+
+    return Listener(
+        "userbot",
+        {
+            "policy_changed": on_policy,
+            WATCHES_CHANGED_CHANNEL: on_watch,
+            ACCOUNTS_CHANGED_CHANNEL: on_accounts,
+        },
+        on_reconnect,
+    )
 
 
 async def runner() -> None:
@@ -191,22 +226,24 @@ async def runner() -> None:
 
     registry = AccountRegistry(pool, storage)
     tasks: set[asyncio.Task] = set()
-    listen_conn: asyncpg.Connection | None = None
+    listener = _change_listener(registry, tasks)
     stop = asyncio.Event()
     asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, stop.set)
     flush = asyncio.create_task(_flush_sessions(registry))
+    retry = asyncio.create_task(_retry_sessions(registry))
+    beat = asyncio.create_task(_keep_uptime(registry, pool))
     try:
         await registry.sync()
         if not registry.clients:
             logger.warning("[yellow]No sessions yet. Add an account in the web UI.[/]")
-        listen_conn = await _listen_changes(registry, tasks)
+        await listener.start()
         logger.info("[green]Userbot running.[/]")
         await stop.wait()
     finally:
         await _cancel(flush)
-        if listen_conn is not None:
-            with contextlib.suppress(Exception):
-                await listen_conn.close()
+        await _cancel(retry)
+        await _cancel(beat)
+        await listener.stop()
         await registry.close()
         await container.close()
 

backend/src/utils/db/models.py

@@ -2,8 +2,17 @@ from datetime import datetime
 from enum import StrEnum
 from typing import Any
 
-from sqlalchemy import BigInteger, Column, DateTime, LargeBinary, Text, func
-from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy import (
+    BigInteger,
+    Column,
+    DateTime,
+    ForeignKey,
+    Integer,
+    LargeBinary,
+    Text,
+    func,
+)
+from sqlalchemy.dialects.postgresql import ARRAY, JSONB
 from sqlmodel import Field, SQLModel
 
 
@@ -203,6 +212,9 @@ class Job(SQLModel, table=True):
     finished_at: datetime | None = Field(
         default=None, sa_column=Column(DateTime(timezone=True))
     )
+    run_after: datetime | None = Field(
+        default=None, sa_column=Column(DateTime(timezone=True))
+    )
 
 
 class CapturePolicy(SQLModel, table=True):
@@ -344,7 +356,7 @@ class ChatHistory(SQLModel, table=True):
     title: str | None = None
     photo_unique_id: str | None = None
     actor_id: int | None = Field(default=None, sa_column=Column(BigInteger))
-    ts: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=False))
+    ts: datetime = Field(sa_column=Column(DateTime(timezone=True), primary_key=True))
     raw: dict[str, Any] = Field(
         default_factory=dict, sa_column=Column(JSONB, nullable=False)
     )
@@ -442,6 +454,7 @@ class Watch(SQLModel, table=True):
     id: int | None = Field(default=None, primary_key=True)
     account_id: int
     kind: str
+    name: str | None = Field(default=None, sa_column=Column(Text, nullable=True))
     params: dict[str, Any] = Field(
         default_factory=dict, sa_column=Column(JSONB, nullable=False)
     )
@@ -466,7 +479,11 @@ class Alert(SQLModel, table=True):
 
     id: int | None = Field(default=None, primary_key=True)
     account_id: int
-    watch_id: int = Field(foreign_key="watches.id")
+    watch_id: int = Field(
+        sa_column=Column(
+            Integer, ForeignKey("watches.id", ondelete="CASCADE"), nullable=False
+        )
+    )
     ts: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=False))
     payload: dict[str, Any] = Field(
         default_factory=dict, sa_column=Column(JSONB, nullable=False)
@@ -543,3 +560,62 @@ class FileShareHit(SQLModel, table=True):
     ip: str | None = None
     user_agent: str | None = None
     counted: bool = False
+
+
+class Webhook(SQLModel, table=True):
+    __tablename__ = "webhooks"
+
+    id: int | None = Field(default=None, primary_key=True)
+    account_id: int
+    url: str = Field(sa_column=Column(Text, nullable=False))
+    secret: str = Field(sa_column=Column(Text, nullable=False))
+    enabled: bool = True
+    watch_ids: list[int] | None = Field(
+        default=None, sa_column=Column(ARRAY(Integer), nullable=True)
+    )
+    created_at: datetime = Field(
+        sa_column=Column(
+            DateTime(timezone=True), nullable=False, server_default=func.now()
+        )
+    )
+    updated_at: datetime = Field(
+        sa_column=Column(
+            DateTime(timezone=True), nullable=False, server_default=func.now()
+        )
+    )
+
+
+class WebhookDelivery(SQLModel, table=True):
+    __tablename__ = "webhook_deliveries"
+
+    id: int | None = Field(default=None, sa_column=Column(BigInteger, primary_key=True))
+    webhook_id: int = Field(
+        sa_column=Column(
+            Integer, ForeignKey("webhooks.id", ondelete="CASCADE"), nullable=False
+        )
+    )
+    alert_id: int | None = Field(
+        default=None,
+        sa_column=Column(
+            Integer, ForeignKey("alerts.id", ondelete="CASCADE"), nullable=True
+        ),
+    )
+    event: str = Field(sa_column=Column(Text, nullable=False))
+    status: str = Field(default="pending", sa_column=Column(Text, nullable=False))
+    attempts: int = 0
+    next_attempt_at: datetime = Field(
+        sa_column=Column(
+            DateTime(timezone=True), nullable=False, server_default=func.now()
+        )
+    )
+    response_code: int | None = None
+    last_error: str | None = Field(default=None, sa_column=Column(Text))
+    body: str | None = Field(default=None, sa_column=Column(Text))
+    created_at: datetime = Field(
+        sa_column=Column(
+            DateTime(timezone=True), nullable=False, server_default=func.now()
+        )
+    )
+    finished_at: datetime | None = Field(
+        default=None, sa_column=Column(DateTime(timezone=True))
+    )

backend/src/utils/env.py

@@ -41,6 +41,7 @@ class ApiSettings(BaseSettings):
     host: str = "0.0.0.0"  # noqa: S104
     port: int = 8080
     static_dir: str = "static"
+    public_url: str | None = None
 
 
 class AuthSettings(BaseSettings):

backend/src/utils/events.py

@@ -4,6 +4,7 @@ from typing import Literal
 import asyncpg
 
 BG_EVENTS_CHANNEL = "bg_events"
+MESSAGE_IDS_PER_EVENT = 500
 
 EventKind = Literal["message", "edit", "delete", "reaction", "presence", "receipt"]
 
@@ -22,8 +23,15 @@ async def notify_bg_event(  # noqa: PLR0913
         payload["chat_id"] = chat_id
     if message_id is not None:
         payload["message_id"] = message_id
-    if message_ids is not None:
-        payload["message_ids"] = message_ids
+    if message_ids is None:
+        await _notify(pool, payload)
+        return
+    for start in range(0, len(message_ids), MESSAGE_IDS_PER_EVENT):
+        chunk = message_ids[start : start + MESSAGE_IDS_PER_EVENT]
+        await _notify(pool, {**payload, "message_ids": chunk})
+
+
+async def _notify(pool: asyncpg.Pool, payload: dict[str, object]) -> None:
     await pool.execute(
         "SELECT pg_notify($1, $2)", BG_EVENTS_CHANNEL, json.dumps(payload)
     )

backend/src/utils/files.py

@@ -191,6 +191,11 @@ def story_file_name(peer_id: int, story_id: int, media_kind: str | None) -> str:
     return f"story_{peer_id}_{story_id}{extension_for(media_kind or 'photo', None)}"
 
 
+def sniff_media_kind(head: bytes) -> str:
+    is_video = head[4:8] == b"ftyp" or head.startswith(b"\x1aE\xdf\xa3")
+    return "video" if is_video else "photo"
+
+
 def resolve_mime(
     kind: str | None, mime: str | None, file_name: str | None = None
 ) -> str:

backend/src/utils/jobs.py

@@ -1,4 +1,5 @@
 import json
+from datetime import timedelta
 from typing import Any
 
 import asyncpg
@@ -7,15 +8,22 @@ JOBS_CHANGED_CHANNEL = "jobs_changed"
 
 
 async def enqueue(
-    pool: asyncpg.Pool, account_id: int, kind: str, params: dict[str, Any]
+    pool: asyncpg.Pool,
+    account_id: int,
+    kind: str,
+    params: dict[str, Any],
+    *,
+    cooldown: timedelta | None = None,
 ) -> int:
     params_json = json.dumps(params)
     existing = await pool.fetchval(
         "SELECT id FROM jobs WHERE account_id = $1 AND kind = $2 "
-        "AND params = $3::jsonb AND status IN ('pending', 'running') LIMIT 1",
+        "AND params = $3::jsonb AND (status IN ('pending', 'running') "
+        "OR created_at > now() - $4::interval) LIMIT 1",
         account_id,
         kind,
         params_json,
+        cooldown,
     )
     if existing is not None:
         return existing

backend/src/utils/listen.py

@@ -0,0 +1,87 @@
+import asyncio
+import contextlib
+from collections.abc import Callable, Mapping
+
+import asyncpg
+
+from utils.env import env
+from utils.logging import logger
+
+RETRY_MIN_SECONDS = 1.0
+RETRY_MAX_SECONDS = 60.0
+PING_INTERVAL_SECONDS = 60.0
+PING_TIMEOUT_SECONDS = 10.0
+CLOSE_TIMEOUT_SECONDS = 5.0
+
+OnNotify = Callable[[str], None]
+
+
+class Listener:
+    def __init__(
+        self,
+        name: str,
+        channels: Mapping[str, OnNotify],
+        on_reconnect: Callable[[], None] | None = None,
+    ) -> None:
+        self._name = name
+        self._channels = channels
+        self._on_reconnect = on_reconnect
+        self._task: asyncio.Task | None = None
+        self._ready = asyncio.Event()
+        self._listening = False
+
+    async def start(self) -> None:
+        self._task = asyncio.create_task(self._run())
+        await self._ready.wait()
+
+    async def stop(self) -> None:
+        if self._task is None:
+            return
+        self._task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await self._task
+        self._task = None
+
+    async def _run(self) -> None:
+        delay = RETRY_MIN_SECONDS
+        while True:
+            try:
+                await self._listen()
+            except Exception as exc:
+                if self._listening:
+                    delay = RETRY_MIN_SECONDS
+                logger.warning(
+                    f"{self._name}: no listen connection ({exc!r}), "
+                    f"reconnecting in {delay:.0f}s"
+                )
+            self._listening = False
+            self._ready.set()
+            await asyncio.sleep(delay)
+            delay = min(delay * 2, RETRY_MAX_SECONDS)
+
+    async def _listen(self) -> None:
+        conn = await asyncpg.connect(dsn=env.db.connection_url)
+        try:
+            lost = asyncio.Event()
+            conn.add_termination_listener(lambda _conn: lost.set())
+            for channel, on_notify in self._channels.items():
+                await conn.add_listener(
+                    channel,
+                    lambda _conn, _pid, _channel, payload, f=on_notify: f(payload),
+                )
+            logger.info(f"{self._name}: listening on {', '.join(self._channels)}")
+            self._listening = True
+            if self._ready.is_set() and self._on_reconnect is not None:
+                self._on_reconnect()
+            self._ready.set()
+            while True:
+                try:
+                    await asyncio.wait_for(lost.wait(), PING_INTERVAL_SECONDS)
+                except TimeoutError:
+                    await conn.fetchval("SELECT 1", timeout=PING_TIMEOUT_SECONDS)
+                else:
+                    msg = "closed by the server"
+                    raise ConnectionError(msg)
+        finally:
+            with contextlib.suppress(Exception):
+                await conn.close(timeout=CLOSE_TIMEOUT_SECONDS)

backend/src/utils/policy/defaults.py

@@ -1,8 +1,8 @@
 from utils.policy.models import CaptureToggles, ChatKind
 
 DEFAULTS: dict[ChatKind, CaptureToggles] = {
-    ChatKind.CHANNEL: CaptureToggles(),
-    ChatKind.GROUP: CaptureToggles(messages=True),
+    ChatKind.CHANNEL: CaptureToggles(track_edits_deletes=True),
+    ChatKind.GROUP: CaptureToggles(messages=True, track_edits_deletes=True),
     ChatKind.DM: CaptureToggles(
         messages=True,
         media=True,

backend/src/utils/policy/models.py

@@ -79,6 +79,24 @@ class FolderSpec:
     is_chatlist: bool = False
 
 
+class PolicyParent(BaseModel):
+    scope_type: ScopeType
+    scope_id: int | None
+    title: str | None = None
+    toggles: CaptureToggles
+
+
+class PolicyRuleView(PolicyRecord):
+    title: str | None = None
+    parent: PolicyParent | None = None
+
+
+class PolicyScopeView(BaseModel):
+    title: str | None
+    rule: PolicyRecord | None
+    parent: PolicyParent | None
+
+
 @dataclass(slots=True)
 class PolicySet:
     chat: dict[int, CaptureToggles] = field(default_factory=dict)

backend/src/utils/policy/overview.py

@@ -0,0 +1,103 @@
+import asyncpg
+
+from utils.policy import repository
+from utils.policy.models import (
+    ChatKind,
+    ChatMeta,
+    FolderSpec,
+    PolicyParent,
+    PolicyRecord,
+    PolicyRuleView,
+    PolicyScopeView,
+    PolicySet,
+    ScopeType,
+)
+from utils.policy.resolver import resolve_parent
+from utils.read import chats
+from utils.read.models import ChatListItem
+
+
+def _meta(chat_id: int, chat: ChatListItem | None) -> ChatMeta:
+    if chat_id > 0:
+        kind = ChatKind.DM
+    elif chat is not None and chat.is_broadcast:
+        kind = ChatKind.CHANNEL
+    else:
+        kind = ChatKind.GROUP
+    return ChatMeta(
+        chat_id=chat_id,
+        kind=kind,
+        is_bot=bool(chat and chat.is_bot),
+        is_contact=chat.is_contact if chat else None,
+    )
+
+
+def _without_chats(policies: PolicySet) -> PolicySet:
+    return PolicySet(folder=policies.folder, defaults=policies.defaults)
+
+
+def _chat_parent(
+    chat_id: int,
+    chat: ChatListItem | None,
+    folders: list[FolderSpec],
+    policies: PolicySet,
+) -> PolicyParent:
+    return resolve_parent(_meta(chat_id, chat), folders, _without_chats(policies))
+
+
+async def list_rules(pool: asyncpg.Pool, account_id: int) -> list[PolicyRuleView]:
+    records = await repository.list_policies(pool, account_id)
+    folders = await repository.list_folders(pool, account_id)
+    policies = await repository.load_policy_set(pool, account_id)
+    chat_ids = [
+        record.scope_id
+        for record in records
+        if record.scope_type is ScopeType.CHAT and record.scope_id is not None
+    ]
+    known = {
+        chat.chat_id: chat for chat in await chats.get_chats(pool, account_id, chat_ids)
+    }
+    titles = {folder.folder_id: folder.title for folder in folders}
+    rules: list[PolicyRuleView] = []
+    for record in records:
+        rule = PolicyRuleView(**record.model_dump())
+        if record.scope_type is ScopeType.FOLDER and record.scope_id is not None:
+            rule.title = titles.get(record.scope_id)
+        if record.scope_type is ScopeType.CHAT and record.scope_id is not None:
+            chat = known.get(record.scope_id)
+            rule.title = chat.title if chat else None
+            rule.parent = _chat_parent(record.scope_id, chat, folders, policies)
+        rules.append(rule)
+    return rules
+
+
+async def _rule(
+    pool: asyncpg.Pool, account_id: int, scope_type: ScopeType, scope_id: int
+) -> PolicyRecord | None:
+    return await repository.find_policy(pool, account_id, scope_type, scope_id)
+
+
+async def chat_scope(
+    pool: asyncpg.Pool, account_id: int, chat_id: int
+) -> PolicyScopeView:
+    folders = await repository.list_folders(pool, account_id)
+    policies = await repository.load_policy_set(pool, account_id)
+    chat = await chats.get_chat(pool, account_id, chat_id)
+    return PolicyScopeView(
+        title=chat.title if chat else None,
+        rule=await _rule(pool, account_id, ScopeType.CHAT, chat_id),
+        parent=_chat_parent(chat_id, chat, folders, policies),
+    )
+
+
+async def folder_scope(
+    pool: asyncpg.Pool, account_id: int, folder_id: int
+) -> PolicyScopeView | None:
+    folder = await repository.get_folder(pool, account_id, folder_id)
+    if folder is None:
+        return None
+    return PolicyScopeView(
+        title=folder.title,
+        rule=await _rule(pool, account_id, ScopeType.FOLDER, folder_id),
+        parent=None,
+    )

backend/src/utils/policy/resolver.py

@@ -1,11 +1,14 @@
 from collections.abc import Iterable
 
 from utils.policy.models import (
+    DEFAULT_SCOPE_BY_KIND,
     CaptureToggles,
     ChatKind,
     ChatMeta,
     FolderSpec,
+    PolicyParent,
     PolicySet,
+    ScopeType,
 )
 
 
@@ -33,14 +36,29 @@ def folder_contains(folder: FolderSpec, chat: ChatMeta) -> bool:
     return _category_match(folder, chat)
 
 
+def resolve_parent(
+    chat: ChatMeta, folders: Iterable[FolderSpec], policies: PolicySet
+) -> PolicyParent:
+    for folder in sorted(folders, key=lambda f: f.order_index):
+        toggles = policies.folder.get(folder.folder_id)
+        if toggles is not None and folder_contains(folder, chat):
+            return PolicyParent(
+                scope_type=ScopeType.FOLDER,
+                scope_id=folder.folder_id,
+                title=folder.title,
+                toggles=toggles,
+            )
+    return PolicyParent(
+        scope_type=DEFAULT_SCOPE_BY_KIND[chat.kind],
+        scope_id=None,
+        toggles=policies.defaults.get(chat.kind, CaptureToggles()),
+    )
+
+
 def resolve(
     chat: ChatMeta, folders: Iterable[FolderSpec], policies: PolicySet
 ) -> CaptureToggles:
     chat_override = policies.chat.get(chat.chat_id)
     if chat_override is not None:
         return chat_override
-    for folder in sorted(folders, key=lambda f: f.order_index):
-        toggles = policies.folder.get(folder.folder_id)
-        if toggles is not None and folder_contains(folder, chat):
-            return toggles
-    return policies.defaults.get(chat.kind, CaptureToggles())
+    return resolve_parent(chat, folders, policies).toggles

backend/src/utils/read/analytics.py

@@ -4,23 +4,31 @@ import asyncpg
 
 from utils.read.accounts import self_user_id
 from utils.read.models import ResponseStats, VolumeBucket
+from utils.read.period import period_bounds
+
+_VOLUME = """
+WITH counts AS (
+    SELECT (date AT TIME ZONE $5)::date AS day, count(*) AS total,
+           count(*) FILTER (WHERE sender_id = $3) AS outgoing
+    FROM messages
+    WHERE account_id = $1 AND chat_id = $2 AND date >= $4
+    GROUP BY 1
+)
+SELECT local_day::date AS bucket, coalesce(total, 0) AS total,
+       coalesce(outgoing, 0) AS outgoing
+FROM generate_series(($4 AT TIME ZONE $5)::date, (now() AT TIME ZONE $5)::date,
+                     interval '1 day') AS local_day
+LEFT JOIN counts ON counts.day = local_day::date
+ORDER BY 1
+"""
 
 
 async def message_volume(
-    pool: asyncpg.Pool, account_id: int, chat_id: int, *, days: int = 90
+    pool: asyncpg.Pool, account_id: int, chat_id: int, *, days: int, tz: str
 ) -> list[VolumeBucket]:
     self_id = await self_user_id(pool, account_id)
-    rows = await pool.fetch(
-        "SELECT date_trunc('day', date) AS bucket, count(*) AS total, "
-        "count(*) FILTER (WHERE sender_id = $3) AS outgoing "
-        "FROM messages "
-        "WHERE account_id = $1 AND chat_id = $2 AND date >= $4 "
-        "GROUP BY bucket ORDER BY bucket",
-        account_id,
-        chat_id,
-        self_id,
-        datetime.now(UTC) - timedelta(days=days),
-    )
+    since, _ = await period_bounds(pool, days, tz)
+    rows = await pool.fetch(_VOLUME, account_id, chat_id, self_id, since, tz)
     return [
         VolumeBucket(
             bucket=row["bucket"],

backend/src/utils/read/annotations.py

@@ -1,9 +1,34 @@
+import re
+from collections import Counter
+
 import asyncpg
 
-from utils.read.models import AnnotationView, Page
+from utils.read.chats import get_chats
+from utils.read.models import AnnotationView, NoteTag, NoteView, Page
 
 _COLS = "id, account_id, chat_id, message_id, text, created_at, updated_at"
 
+TAG = re.compile(r"(?<!\w)#(\w+)")
+WORD = re.compile(r"\w+")
+
+_NOTES = """
+SELECT a.id, a.account_id, a.chat_id, a.message_id, a.text,
+       a.created_at, a.updated_at,
+       m.text AS message_text, m.date AS message_date,
+       COALESCE(m.has_media, false) AS message_has_media,
+       m.deleted_at IS NOT NULL AS message_deleted
+FROM annotations a
+LEFT JOIN LATERAL (
+    SELECT text, date, has_media, deleted_at FROM messages
+    WHERE account_id = a.account_id AND chat_id = a.chat_id
+      AND message_id = a.message_id
+    LIMIT 1
+) m ON true
+WHERE a.account_id = $1 AND ($4::text IS NULL OR a.text ~* $4)
+ORDER BY a.created_at DESC, a.id DESC
+LIMIT $2 OFFSET $3
+"""
+
 
 async def list_annotations(
     pool: asyncpg.Pool,
@@ -70,3 +95,49 @@ async def update_annotation(
 async def delete_annotation(pool: asyncpg.Pool, annotation_id: int) -> bool:
     result = await pool.execute("DELETE FROM annotations WHERE id = $1", annotation_id)
     return result.endswith("1")
+
+
+def _tag_pattern(tag: str | None) -> str | None:
+    if tag is None:
+        return None
+    return f"(^|[^[:alnum:]_])#{tag}($|[^[:alnum:]_])"
+
+
+async def list_notes(
+    pool: asyncpg.Pool, account_id: int, page: Page, *, tag: str | None = None
+) -> list[NoteView]:
+    if tag is not None and not WORD.fullmatch(tag):
+        return []
+    rows = await pool.fetch(
+        _NOTES, account_id, page.capped_limit, page.offset, _tag_pattern(tag)
+    )
+    chats = {
+        chat.chat_id: chat
+        for chat in await get_chats(
+            pool, account_id, list({row["chat_id"] for row in rows})
+        )
+    }
+    notes: list[NoteView] = []
+    for row in rows:
+        chat = chats.get(row["chat_id"])
+        notes.append(
+            NoteView(
+                **dict(row),
+                chat_title=chat.title if chat else None,
+                chat_has_avatar=chat.has_avatar if chat else False,
+            )
+        )
+    return notes
+
+
+async def list_tags(pool: asyncpg.Pool, account_id: int) -> list[NoteTag]:
+    rows = await pool.fetch(
+        "SELECT text FROM annotations WHERE account_id = $1", account_id
+    )
+    counts: Counter[str] = Counter()
+    for row in rows:
+        counts.update({tag.lower() for tag in TAG.findall(row["text"])})
+    return [
+        NoteTag(tag=tag, count=count)
+        for tag, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))
+    ]

backend/src/utils/read/capture.py

@@ -0,0 +1,44 @@
+from datetime import datetime
+
+import asyncpg
+
+from utils.read.models import CaptureGap, CaptureStatus
+
+_STATUS = """
+SELECT started_at, seen_at, seen_at >= now() - interval '150 seconds' AS live
+FROM capture_uptime WHERE account_id = $1
+ORDER BY started_at DESC LIMIT 1
+"""
+
+_GAPS = """
+SELECT seen_at AS start, next_start AS "end"
+FROM (
+    SELECT seen_at,
+           lead(started_at) OVER (ORDER BY started_at) AS next_start
+    FROM capture_uptime WHERE account_id = $1
+) stretches
+WHERE next_start - seen_at > interval '30 minutes'
+  AND ($2::timestamptz IS NULL OR next_start >= $2)
+  AND ($3::timestamptz IS NULL OR seen_at <= $3)
+ORDER BY seen_at
+"""
+
+
+async def capture_status(pool: asyncpg.Pool, account_id: int) -> CaptureStatus:
+    row = await pool.fetchrow(_STATUS, account_id)
+    if row is None:
+        return CaptureStatus(live=False, since=None, last_seen=None)
+    return CaptureStatus(
+        live=row["live"], since=row["started_at"], last_seen=row["seen_at"]
+    )
+
+
+async def capture_gaps(
+    pool: asyncpg.Pool,
+    account_id: int,
+    *,
+    date_from: datetime | None = None,
+    date_to: datetime | None = None,
+) -> list[CaptureGap]:
+    rows = await pool.fetch(_GAPS, account_id, date_from, date_to)
+    return [CaptureGap(**dict(row)) for row in rows]

backend/src/utils/read/chats.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+
 import asyncpg
 
 from utils.policy.models import FolderSpec
@@ -10,7 +12,7 @@ from utils.read.models import (
     MessageView,
     Page,
 )
-from utils.read.read_receipts import read_up_to
+from utils.read.read_receipts import read_times, read_up_to
 
 _MESSAGE_COLS = (
     "chat_id, message_id, date, sender_id, text, has_media, is_self_destruct, "
@@ -78,7 +80,8 @@ _CHAT_ROWS = """
 WITH ids AS ({ids})
 SELECT ids.chat_id,
        COALESCE(cs.message_count, 0) AS message_count,
-       cs.last_date, cs.last_text, cs.last_sender_id,
+       cs.last_date, cs.last_text, cs.last_sender_id, cs.last_message_id,
+       COALESCE(cs.last_deleted, false) AS last_deleted,
        COALESCE(named.title,
                 NULLIF(trim(concat_ws(' ', p.first_name, p.last_name)), ''),
                 p.username) AS title,
@@ -147,6 +150,8 @@ def _chat_item(row: asyncpg.Record) -> ChatListItem:
         last_date=row["last_date"],
         last_text=row["last_text"],
         last_sender_id=row["last_sender_id"],
+        last_message_id=row["last_message_id"],
+        last_deleted=row["last_deleted"],
     )
 
 
@@ -176,6 +181,18 @@ async def list_chats(
     return [_chat_item(row) for row in rows]
 
 
+async def folder_chat_ids(
+    pool: asyncpg.Pool, account_id: int, folder: FolderSpec
+) -> list[int]:
+    rows_sql = _CHAT_ROWS.format(ids=_ALL_IDS)
+    rows = await pool.fetch(
+        f"SELECT chat.chat_id FROM ({rows_sql}) chat WHERE {_folder_filter(1)}",  # noqa: S608
+        account_id,
+        *_folder_params(folder),
+    )
+    return [row["chat_id"] for row in rows]
+
+
 async def get_chat(
     pool: asyncpg.Pool, account_id: int, chat_id: int
 ) -> ChatListItem | None:
@@ -245,6 +262,7 @@ async def get_chat_history(  # noqa: PLR0913
             views.append(_build_album(members, media_by_key))
         index = end
     await _apply_read_status(pool, account_id, chat_id, views)
+    await _apply_notes(pool, account_id, chat_id, views)
     return views
 
 
@@ -264,6 +282,7 @@ async def get_message(
     raw = load_raw(row["raw"])
     view = build_message_view(row, raw, _single_media(row, raw, media_by_key))
     await _apply_read_status(pool, account_id, chat_id, [view])
+    await _apply_notes(pool, account_id, chat_id, [view])
     return view
 
 
@@ -273,12 +292,35 @@ async def _apply_read_status(
     self_id = await self_user_id(pool, account_id)
     if self_id is None:
         return
-    marker = await read_up_to(pool, account_id, chat_id)
-    if marker is None:
+    own = [view for view in views if view.sender_id == self_id]
+    if not own:
         return
+    marker = await read_up_to(pool, account_id, chat_id)
+    times = await read_times(
+        pool, account_id, chat_id, [(view.message_id, view.date) for view in own]
+    )
+    for view in own:
+        view.read_at = times.get(view.message_id)
+        view.read = view.read_at is not None or (
+            marker is not None and view.message_id <= marker
+        )
+
+
+async def _apply_notes(
+    pool: asyncpg.Pool, account_id: int, chat_id: int, views: list[MessageView]
+) -> None:
+    if not views:
+        return
+    rows = await pool.fetch(
+        "SELECT DISTINCT message_id FROM annotations "
+        "WHERE account_id = $1 AND chat_id = $2 AND message_id = ANY($3::bigint[])",
+        account_id,
+        chat_id,
+        [view.message_id for view in views],
+    )
+    noted = {row["message_id"] for row in rows}
     for view in views:
-        if view.sender_id == self_id and view.message_id <= marker:
-            view.read = True
+        view.has_note = view.message_id in noted
 
 
 def _build_album(
@@ -313,6 +355,12 @@ async def get_deleted_messages(
         f"ORDER BY deleted_at DESC LIMIT ${len(params) - 1} OFFSET ${len(params)}",
         *params,
     )
+    return await _single_views(pool, account_id, rows)
+
+
+async def _single_views(
+    pool: asyncpg.Pool, account_id: int, rows: list[asyncpg.Record]
+) -> list[MessageView]:
     media_by_key = await _media_map(pool, account_id, rows)
     views: list[MessageView] = []
     for row in rows:
@@ -323,6 +371,67 @@ async def get_deleted_messages(
     return views
 
 
+async def _rows_at(
+    pool: asyncpg.Pool, account_id: int, keys: list[tuple[int, int, datetime]]
+) -> list[asyncpg.Record]:
+    if not keys:
+        return []
+    params: list[object] = [account_id]
+    clauses: list[str] = []
+    for key in keys:
+        params.extend(key)
+        n = len(params)
+        clauses.append(
+            f"(chat_id = ${n - 2} AND message_id = ${n - 1} AND date = ${n})"
+        )
+    return await pool.fetch(
+        f"SELECT {_MESSAGE_COLS} FROM messages "  # noqa: S608
+        f"WHERE account_id = $1 AND ({' OR '.join(clauses)})",
+        *params,
+    )
+
+
+async def get_messages_at(
+    pool: asyncpg.Pool, account_id: int, keys: list[tuple[int, int, datetime | None]]
+) -> dict[tuple[int, int], MessageView]:
+    dated = list(
+        dict.fromkeys((chat, message, date) for chat, message, date in keys if date)
+    )
+    rows = await _rows_at(pool, account_id, dated)
+    views = {
+        (view.chat_id, view.message_id): view
+        for view in await _single_views(pool, account_id, rows)
+    }
+    for chat_id, message_id, _ in keys:
+        if (chat_id, message_id) not in views:
+            view = await get_message(pool, account_id, chat_id, message_id)
+            if view is not None:
+                views[chat_id, message_id] = view
+    return views
+
+
+async def get_deleted_between(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    account_id: int,
+    chat_id: int,
+    since: datetime,
+    until: datetime,
+    limit: int,
+    before: tuple[datetime, int] | None = None,
+) -> list[MessageView]:
+    params: list[object] = [account_id, chat_id, since, until, limit]
+    where = "account_id = $1 AND chat_id = $2 AND deleted_at BETWEEN $3 AND $4"
+    if before is not None:
+        params.extend(before)
+        where += " AND (date, message_id) < ($6, $7)"
+    rows = await pool.fetch(
+        f"SELECT {_MESSAGE_COLS} FROM messages WHERE {where} "  # noqa: S608
+        "ORDER BY date DESC, message_id DESC LIMIT $5",
+        *params,
+    )
+    return await _single_views(pool, account_id, rows)
+
+
 async def get_message_versions(
     pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
 ) -> list[MessageVersionView]:

backend/src/utils/read/feed.py

@@ -0,0 +1,291 @@
+import json
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from enum import StrEnum
+
+import asyncpg
+
+from utils.read import chats, peers
+from utils.read.accounts import self_user_id
+from utils.read.models import (
+    DEFAULT_LIMIT,
+    MAX_LIMIT,
+    ChatListItem,
+    FeedChat,
+    FeedItem,
+    FeedMessagesPage,
+    FeedPage,
+    FeedUnseen,
+    MessageView,
+    PeerView,
+)
+
+CURSOR_PATTERN = r"^\d+\.\d+$"
+_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
+_MICROSECOND = timedelta(microseconds=1)
+
+_ITEM_COLS = (
+    "id, kind, chat_id, chat_kind, message_id, message_date, at, since, "
+    "message_count, detail"
+)
+
+_VERSIONS = """
+SELECT v.chat_id, v.message_id, v.observed_at, v.edit_date, v.text,
+       p.text AS before, p.observed_at IS NOT NULL AS has_before
+FROM unnest($2::bigint[], $3::bigint[], $4::timestamptz[])
+    AS k(chat_id, message_id, observed_at)
+JOIN message_versions v
+  ON v.account_id = $1 AND v.chat_id = k.chat_id
+ AND v.message_id = k.message_id AND v.observed_at = k.observed_at
+LEFT JOIN LATERAL (
+    SELECT text, observed_at FROM message_versions p
+    WHERE p.account_id = $1 AND p.chat_id = v.chat_id
+      AND p.message_id = v.message_id AND p.observed_at < v.observed_at
+    ORDER BY p.observed_at DESC LIMIT 1
+) p ON true
+"""
+
+_UNSEEN = """
+SELECT count(*) FILTER (WHERE kind = 'deleted') AS deleted,
+       count(*) FILTER (WHERE kind = 'edited') AS edited
+FROM feed_items
+WHERE account_id = $1 AND at > coalesce($2, '-infinity'::timestamptz)
+"""
+
+_MARK_SEEN = """
+INSERT INTO feed_seen (account_id, seen_at) VALUES ($1, coalesce($2, now()))
+ON CONFLICT (account_id) DO UPDATE
+SET seen_at = greatest(feed_seen.seen_at, EXCLUDED.seen_at)
+"""
+
+
+class FeedKind(StrEnum):
+    ALL = "all"
+    MESSAGES = "messages"
+    DELETED = "deleted"
+    EDITED = "edited"
+    PROFILE = "profile"
+
+
+class FeedChatKind(StrEnum):
+    DM = "dm"
+    GROUP = "group"
+    CHANNEL = "channel"
+
+
+@dataclass(slots=True)
+class FeedFilter:
+    kind: FeedKind = FeedKind.ALL
+    chat_id: int | None = None
+    chat_kind: FeedChatKind | None = None
+    since: datetime | None = None
+
+
+def encode_cursor(at: datetime, key: int) -> str:
+    return f"{(at - _EPOCH) // _MICROSECOND}.{key}"
+
+
+def decode_cursor(cursor: str) -> tuple[datetime, int]:
+    micros, _, key = cursor.partition(".")
+    return _EPOCH + int(micros) * _MICROSECOND, int(key)
+
+
+def _where(
+    account_id: int, feed: FeedFilter, cursor: str | None
+) -> tuple[str, list[object]]:
+    params: list[object] = [account_id]
+    clauses = ["account_id = $1"]
+    if feed.kind is FeedKind.MESSAGES:
+        clauses.append("kind <> 'profile'")
+    exact = feed.kind not in {FeedKind.ALL, FeedKind.MESSAGES}
+    conditions = (
+        ("kind = {}", feed.kind.value if exact else None),
+        ("chat_id = {}", feed.chat_id),
+        ("chat_kind = {}", feed.chat_kind.value if feed.chat_kind else None),
+        ("at >= {}", feed.since),
+    )
+    for template, value in conditions:
+        if value is not None:
+            params.append(value)
+            clauses.append(template.format(f"${len(params)}"))
+    if cursor is not None:
+        params.extend(decode_cursor(cursor))
+        clauses.append(f"(at, id) < (${len(params) - 1}, ${len(params)})")
+    return " AND ".join(clauses), params
+
+
+def _peer_name(peer: PeerView) -> str | None:
+    name = " ".join(part for part in (peer.first_name, peer.last_name) if part)
+    return name or peer.username
+
+
+async def _versions(
+    pool: asyncpg.Pool, account_id: int, rows: list[asyncpg.Record]
+) -> dict[tuple[int, int, datetime], asyncpg.Record]:
+    edited = [row for row in rows if row["kind"] == FeedKind.EDITED]
+    if not edited:
+        return {}
+    found = await pool.fetch(
+        _VERSIONS,
+        account_id,
+        [row["chat_id"] for row in edited],
+        [row["message_id"] for row in edited],
+        [row["at"] for row in edited],
+    )
+    return {
+        (row["chat_id"], row["message_id"], row["observed_at"]): row for row in found
+    }
+
+
+async def _sender_names(
+    pool: asyncpg.Pool,
+    account_id: int,
+    messages: dict[tuple[int, int], MessageView],
+    chat_views: dict[int, ChatListItem],
+) -> dict[int, str | None]:
+    ids = {m.sender_id for m in messages.values() if m.sender_id is not None}
+    found = await peers.get_peers(pool, account_id, list(ids))
+    names: dict[int, str | None] = {
+        chat_id: chat.title for chat_id, chat in chat_views.items()
+    }
+    names.update({peer.peer_id: _peer_name(peer) for peer in found})
+    return names
+
+
+async def _build_items(
+    pool: asyncpg.Pool, account_id: int, rows: list[asyncpg.Record]
+) -> list[FeedItem]:
+    if not rows:
+        return []
+    messages = await chats.get_messages_at(
+        pool,
+        account_id,
+        [
+            (row["chat_id"], row["message_id"], row["message_date"])
+            for row in rows
+            if row["kind"] != FeedKind.PROFILE
+        ],
+    )
+    versions = await _versions(pool, account_id, rows)
+    chat_views = {
+        chat.chat_id: chat
+        for chat in await chats.get_chats(
+            pool, account_id, list({row["chat_id"] for row in rows})
+        )
+    }
+    names = await _sender_names(pool, account_id, messages, chat_views)
+    self_id = await self_user_id(pool, account_id)
+    items: list[FeedItem] = []
+    for row in rows:
+        message = messages.get((row["chat_id"], row["message_id"]))
+        chat = chat_views.get(row["chat_id"])
+        version = versions.get((row["chat_id"], row["message_id"], row["at"]))
+        sender_id = message.sender_id if message else None
+        items.append(
+            FeedItem(
+                id=row["id"],
+                kind=row["kind"],
+                at=row["at"],
+                since=row["since"],
+                message_count=row["message_count"],
+                chat=FeedChat(
+                    chat_id=row["chat_id"],
+                    title=chat.title if chat else None,
+                    kind=row["chat_kind"],
+                    has_avatar=chat.has_avatar if chat else False,
+                ),
+                message_id=row["message_id"],
+                message=message,
+                sender_id=sender_id,
+                sender_name=names.get(sender_id) if sender_id is not None else None,
+                outgoing=self_id is not None and sender_id == self_id,
+                before=version["before"] if version else None,
+                after=version["text"] if version else None,
+                has_before=bool(version and version["has_before"]),
+                edit_date=version["edit_date"] if version else None,
+                detail=json.loads(row["detail"]) if row["detail"] else None,
+            )
+        )
+    return items
+
+
+async def list_feed(
+    pool: asyncpg.Pool,
+    account_id: int,
+    feed: FeedFilter,
+    *,
+    limit: int = DEFAULT_LIMIT,
+    cursor: str | None = None,
+) -> FeedPage:
+    size = max(1, min(limit, MAX_LIMIT))
+    where, params = _where(account_id, feed, cursor)
+    params.append(size + 1)
+    rows = await pool.fetch(
+        f"SELECT {_ITEM_COLS} FROM feed_items WHERE {where} "  # noqa: S608
+        f"ORDER BY at DESC, id DESC LIMIT ${len(params)}",
+        *params,
+    )
+    page = rows[:size]
+    more = len(rows) > size
+    return FeedPage(
+        items=await _build_items(pool, account_id, page),
+        next_cursor=encode_cursor(page[-1]["at"], page[-1]["id"]) if more else None,
+    )
+
+
+async def unseen(pool: asyncpg.Pool, account_id: int) -> FeedUnseen:
+    seen_at = await pool.fetchval(
+        "SELECT seen_at FROM feed_seen WHERE account_id = $1", account_id
+    )
+    counts = await pool.fetchrow(_UNSEEN, account_id, seen_at)
+    latest = await list_feed(
+        pool, account_id, FeedFilter(kind=FeedKind.MESSAGES), limit=1
+    )
+    return FeedUnseen(
+        count=counts["deleted"] + counts["edited"],
+        deleted=counts["deleted"],
+        edited=counts["edited"],
+        seen_at=seen_at,
+        latest=latest.items[0] if latest.items else None,
+    )
+
+
+async def mark_seen(
+    pool: asyncpg.Pool, account_id: int, at: datetime | None
+) -> FeedUnseen:
+    await pool.execute(_MARK_SEEN, account_id, at)
+    return await unseen(pool, account_id)
+
+
+async def item_messages(
+    pool: asyncpg.Pool,
+    account_id: int,
+    item_id: int,
+    *,
+    limit: int = DEFAULT_LIMIT,
+    cursor: str | None = None,
+) -> FeedMessagesPage | None:
+    row = await pool.fetchrow(
+        "SELECT chat_id, since, at FROM feed_items "
+        "WHERE id = $1 AND account_id = $2 AND kind = 'deleted'",
+        item_id,
+        account_id,
+    )
+    if row is None:
+        return None
+    size = max(1, min(limit, MAX_LIMIT))
+    views = await chats.get_deleted_between(
+        pool,
+        account_id,
+        row["chat_id"],
+        row["since"],
+        row["at"],
+        size + 1,
+        decode_cursor(cursor) if cursor else None,
+    )
+    page = views[:size]
+    more = len(views) > size
+    return FeedMessagesPage(
+        messages=page,
+        next_cursor=encode_cursor(page[-1].date, page[-1].message_id) if more else None,
+    )

backend/src/utils/read/message_view.py

@@ -22,6 +22,7 @@ from utils.read.models import (
     StickerView,
     WebPageView,
 )
+from utils.read.rich_text import rich_text
 
 _MEDIA_KEYS = (
     "photo",
@@ -382,6 +383,8 @@ def build_message_view(
     row: asyncpg.Record, raw: dict[str, Any], media: list[MediaRef]
 ) -> MessageView:
     base = _base_fields(row)
+    if base["text"] is None:
+        base["text"] = rich_text(raw.get("rich_message"))
     via_bot = raw.get("via_bot")
     sticker = _sticker(raw)
     try:

backend/src/utils/read/models.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import date, datetime
 from typing import Any
 
 from pydantic import BaseModel
@@ -38,6 +38,8 @@ class ChatListItem(BaseModel):
     last_date: datetime | None
     last_text: str | None
     last_sender_id: int | None
+    last_message_id: int | None = None
+    last_deleted: bool = False
 
 
 class DiscoverItem(BaseModel):
@@ -203,6 +205,8 @@ class MessageView(BaseModel):
     is_sticker: bool = False
     is_animated_emoji: bool = False
     read: bool = False
+    read_at: datetime | None = None
+    has_note: bool = False
 
 
 class MessageVersionView(BaseModel):
@@ -211,6 +215,50 @@ class MessageVersionView(BaseModel):
     text: str | None
 
 
+class FeedChat(BaseModel):
+    chat_id: int
+    title: str | None
+    kind: str
+    has_avatar: bool
+
+
+class FeedItem(BaseModel):
+    id: int
+    kind: str
+    at: datetime
+    since: datetime
+    message_count: int
+    chat: FeedChat
+    message_id: int
+    message: MessageView | None
+    sender_id: int | None
+    sender_name: str | None
+    outgoing: bool
+    before: str | None = None
+    after: str | None = None
+    has_before: bool = False
+    edit_date: datetime | None = None
+    detail: dict[str, list[str | None]] | None = None
+
+
+class FeedPage(BaseModel):
+    items: list[FeedItem]
+    next_cursor: str | None
+
+
+class FeedUnseen(BaseModel):
+    count: int
+    deleted: int
+    edited: int
+    seen_at: datetime | None
+    latest: FeedItem | None
+
+
+class FeedMessagesPage(BaseModel):
+    messages: list[MessageView]
+    next_cursor: str | None
+
+
 class MediaView(BaseModel):
     id: int
     account_id: int
@@ -236,6 +284,72 @@ class MediaVersionView(BaseModel):
     observed_at: datetime
 
 
+class StorageGroupView(BaseModel):
+    group: str
+    files: int
+    bytes: int
+
+
+class StorageSummaryView(BaseModel):
+    files: int
+    bytes: int
+    heavy_files: int
+    heavy_bytes: int
+    heavy_threshold: int
+    trash_files: int
+    trash_bytes: int
+    groups: list[StorageGroupView]
+
+
+class StorageChatView(BaseModel):
+    chat_id: int
+    files: int
+    bytes: int
+    groups: list[StorageGroupView]
+
+
+class StorageFileView(BaseModel):
+    media_id: int
+    chat_id: int
+    message_id: int
+    kind: str
+    mime: str | None
+    file_name: str
+    file_size: int
+    date: datetime | None
+    refs: int
+    refetchable: bool
+
+
+class TrashItemView(BaseModel):
+    storage_key: str
+    chat_id: int
+    message_id: int
+    kind: str
+    mime: str | None
+    file_name: str
+    file_size: int | None
+    trashed_at: datetime
+    purge_at: datetime
+    refs: int
+    refetchable: bool
+
+
+class TrashOutcome(BaseModel):
+    files: int = 0
+    bytes: int = 0
+    keys: list[str] = []
+    blocked: list[int] = []
+
+
+class StorageScopeView(BaseModel):
+    chats: int
+    files: int
+    bytes: int
+    kept: int
+    gone: int
+
+
 class AvatarRef(BaseModel):
     unique_id: str
     storage_key: str | None
@@ -295,8 +409,61 @@ class PresenceHourly(BaseModel):
     last_seen: datetime | None
 
 
+class PresenceSession(BaseModel):
+    started: datetime
+    ended: datetime
+    seconds: float
+    online_seconds: float
+    ongoing: bool
+
+
+class PresenceHour(BaseModel):
+    weekday: int
+    hour: int
+    hours: int
+    online_minutes: float | None
+    covered_hours: float
+
+
+class PresenceDay(BaseModel):
+    day: date
+    online_seconds: float
+    covered_seconds: float
+
+
+class CoverageGap(BaseModel):
+    started: datetime
+    ended: datetime
+
+
+class PresenceSummary(BaseModel):
+    online_seconds: float
+    daily_seconds: float | None
+    covered_seconds: float
+    session_count: int
+    median_session_seconds: float | None
+    first_online: int | None
+    last_offline: int | None
+    last_seen: datetime | None
+
+
+class PresenceStats(BaseModel):
+    peer_id: int
+    tz: str
+    since: datetime
+    until: datetime
+    current: PresenceSample | None
+    online: bool
+    hidden: bool
+    summary: PresenceSummary
+    week: list[PresenceHour]
+    days: list[PresenceDay]
+    gaps: list[CoverageGap]
+    sessions: list[PresenceSession]
+
+
 class VolumeBucket(BaseModel):
-    bucket: datetime
+    bucket: date
     total: int
     outgoing: int
     incoming: int
@@ -321,6 +488,29 @@ class PeerView(BaseModel):
     updated_at: datetime
 
 
+class ReaderView(BaseModel):
+    peer_id: int
+    read_at: datetime
+    peer: PeerView | None
+
+
+class ReadInfoView(BaseModel):
+    chat_id: int
+    message_id: int
+    chat_kind: str
+    own: bool
+    voice: bool
+    observed_at: datetime | None
+    listened_at: datetime | None
+    exact_at: datetime | None
+    readers: list[ReaderView]
+    read: bool
+    unread: bool
+    outcome: str | None
+    fetched_at: datetime | None
+    can_fetch: bool
+
+
 class PeerHistoryView(BaseModel):
     observed_at: datetime
     first_name: str | None
@@ -331,6 +521,54 @@ class PeerHistoryView(BaseModel):
     is_deleted_account: bool
 
 
+class ProfileChange(BaseModel):
+    field: str
+    old: str | None
+    new: str | None
+    changed_at: datetime
+    actor_id: int | None = None
+    message_id: int | None = None
+
+
+class MembershipView(BaseModel):
+    chat_id: int
+    chat_title: str | None
+    has_avatar: bool
+    message_id: int
+    event: str
+    actor_id: int | None
+    actor_name: str | None
+    ts: datetime
+
+
+class DirectChatView(BaseModel):
+    chat_id: int
+    message_count: int
+    last_date: datetime | None
+
+
+class PeerOverview(BaseModel):
+    peer: PeerView | None
+    changes: list[ProfileChange]
+    avatars: list[AvatarHistoryView]
+    memberships: list[MembershipView]
+    dm: DirectChatView | None
+
+
+class SharedGroupView(BaseModel):
+    chat_id: int
+    title: str | None
+    has_avatar: bool
+    message_count: int
+    last_date: datetime
+    last_message_id: int
+
+
+class PeerGroups(BaseModel):
+    groups: list[SharedGroupView]
+    deleted_count: int
+
+
 class ChatLinkView(BaseModel):
     message_id: int
     date: datetime | None
@@ -376,16 +614,62 @@ class AnnotationView(BaseModel):
     updated_at: datetime
 
 
+class NoteView(AnnotationView):
+    chat_title: str | None
+    chat_has_avatar: bool
+    message_text: str | None
+    message_date: datetime | None
+    message_has_media: bool
+    message_deleted: bool
+
+
+class NoteTag(BaseModel):
+    tag: str
+    count: int
+
+
+class CaptureStatus(BaseModel):
+    live: bool
+    since: datetime | None
+    last_seen: datetime | None
+
+
+class CaptureGap(BaseModel):
+    start: datetime
+    end: datetime
+
+
 class WatchView(BaseModel):
     id: int
     account_id: int
     kind: str
+    name: str | None
     params: dict[str, Any]
     enabled: bool
     created_at: datetime
     updated_at: datetime
 
 
+class ChatRef(BaseModel):
+    id: int
+    title: str | None
+    kind: str
+    has_avatar: bool
+
+
+class PersonRef(BaseModel):
+    id: int
+    name: str | None
+    has_avatar: bool
+
+
+class WatchItem(WatchView):
+    chat: ChatRef | None
+    person: PersonRef | None
+    alert_count: int
+    last_alert_at: datetime | None
+
+
 class AlertView(BaseModel):
     id: int
     account_id: int
@@ -396,6 +680,68 @@ class AlertView(BaseModel):
     created_at: datetime
 
 
+class AlertWatch(BaseModel):
+    id: int
+    kind: str
+    name: str | None
+    params: dict[str, Any]
+
+
+class AlertMessage(BaseModel):
+    id: int
+    text: str | None
+    sender: PersonRef | None
+    date: datetime | None
+    deleted: bool
+    edited: bool
+
+
+class AlertItem(BaseModel):
+    id: int
+    account_id: int
+    ts: datetime
+    seen: bool
+    event: str
+    watch: AlertWatch
+    chat: ChatRef | None
+    message: AlertMessage | None
+    peer: PersonRef | None
+    away_since: datetime | None
+    path: str
+
+
+class AlertPage(BaseModel):
+    items: list[AlertItem]
+    next_before: int | None
+    unseen: int
+
+
+class DeliveryView(BaseModel):
+    id: int
+    webhook_id: int
+    alert_id: int | None
+    event: str
+    status: str
+    attempts: int
+    next_attempt_at: datetime
+    response_code: int | None
+    last_error: str | None
+    created_at: datetime
+    finished_at: datetime | None
+
+
+class WebhookView(BaseModel):
+    id: int
+    account_id: int
+    url: str
+    secret: str
+    enabled: bool
+    watch_ids: list[int] | None
+    created_at: datetime
+    updated_at: datetime
+    last_delivery: DeliveryView | None
+
+
 class FileShareView(BaseModel):
     id: int
     account_id: int

backend/src/utils/read/peers.py

@@ -2,6 +2,8 @@ import asyncpg
 
 from utils.read.models import Page, PeerHistoryView, PeerView, StoryView
 
+HISTORY_LIMIT = 500
+
 _PEER_COLS = (
     "peer_id, first_name, last_name, username, phone, photo_unique_id, "
     "is_deleted_account, updated_at, "
@@ -9,6 +11,23 @@ _PEER_COLS = (
     "AND a.owner_id = peers.peer_id) AS has_avatar"
 )
 
+_HISTORY = """
+SELECT observed_at, first_name, last_name, username, phone, photo_unique_id,
+       is_deleted_account
+FROM (
+    SELECT observed_at, first_name, last_name, username, phone, photo_unique_id,
+           is_deleted_account,
+           lag((first_name, last_name, username, phone, photo_unique_id,
+                is_deleted_account)) OVER (ORDER BY observed_at) AS previous
+    FROM peer_history
+    WHERE account_id = $1 AND peer_id = $2
+) states
+WHERE previous IS DISTINCT FROM
+      (first_name, last_name, username, phone, photo_unique_id, is_deleted_account)
+ORDER BY observed_at DESC
+LIMIT $3
+"""
+
 
 async def get_peer(
     pool: asyncpg.Pool, account_id: int, peer_id: int
@@ -39,14 +58,7 @@ async def get_peers(
 async def get_peer_history(
     pool: asyncpg.Pool, account_id: int, peer_id: int
 ) -> list[PeerHistoryView]:
-    rows = await pool.fetch(
-        "SELECT observed_at, first_name, last_name, username, phone, "
-        "photo_unique_id, is_deleted_account FROM peer_history "
-        "WHERE account_id = $1 AND peer_id = $2 "
-        "ORDER BY observed_at DESC LIMIT 500",
-        account_id,
-        peer_id,
-    )
+    rows = await pool.fetch(_HISTORY, account_id, peer_id, HISTORY_LIMIT)
     return [PeerHistoryView(**dict(row)) for row in rows]
 
 
@@ -64,7 +76,7 @@ async def get_stories(
         "SELECT peer_id, story_id, date, expire_date, caption, media_kind, "  # noqa: S608
         "storage_key, downloaded, views, pinned, deleted FROM stories "
         f"WHERE {where} "
-        f"ORDER BY date DESC NULLS LAST LIMIT ${len(params) - 1} "
+        f"ORDER BY date DESC NULLS LAST, story_id DESC LIMIT ${len(params) - 1} "
         f"OFFSET ${len(params)}",
         *params,
     )
@@ -83,3 +95,16 @@ async def get_story(
         story_id,
     )
     return StoryView(**dict(row)) if row else None
+
+
+async def set_story_kind(
+    pool: asyncpg.Pool, account_id: int, peer_id: int, story_id: int, kind: str
+) -> None:
+    await pool.execute(
+        "UPDATE stories SET media_kind = $4 "
+        "WHERE account_id = $1 AND peer_id = $2 AND story_id = $3",
+        account_id,
+        peer_id,
+        story_id,
+        kind,
+    )

backend/src/utils/read/people.py

@@ -0,0 +1,194 @@
+from collections.abc import Callable
+from datetime import datetime
+from itertools import pairwise
+
+import asyncpg
+
+from utils.read import chats
+from utils.read.avatars import avatar_history
+from utils.read.models import (
+    DirectChatView,
+    MembershipView,
+    PeerGroups,
+    PeerHistoryView,
+    PeerOverview,
+    ProfileChange,
+    SharedGroupView,
+)
+from utils.read.peers import HISTORY_LIMIT, get_peer, get_peer_history
+
+_CHAT_TITLES = """
+SELECT ts, title, previous, actor_id, message_id
+FROM (
+    SELECT ts, title,
+           CASE WHEN event = 'title' THEN actor_id END AS actor_id,
+           CASE WHEN event = 'title' THEN message_id END AS message_id,
+           lag(title) OVER (ORDER BY ts, message_id) AS previous
+    FROM chat_history
+    WHERE account_id = $1 AND chat_id = $2 AND event IN ('meta', 'title')
+      AND title IS NOT NULL
+) titles
+WHERE previous IS DISTINCT FROM title AND previous IS NOT NULL
+ORDER BY ts DESC
+LIMIT $3
+"""
+
+_MEMBERSHIPS = """
+SELECT m.chat_id, m.message_id, m.event, m.actor_id, m.ts,
+       COALESCE(NULLIF(trim(concat_ws(' ', a.first_name, a.last_name)), ''),
+                a.username) AS actor_name
+FROM memberships m
+LEFT JOIN peers a ON a.account_id = m.account_id AND a.peer_id = m.actor_id
+WHERE m.account_id = $1 AND m.user_id = $2
+ORDER BY m.ts DESC
+LIMIT 200
+"""
+
+_DIRECT_CHAT = """
+SELECT $2::bigint AS chat_id, COALESCE(cs.message_count, 0) AS message_count,
+       cs.last_date
+FROM (SELECT 1) one
+LEFT JOIN chat_stats cs ON cs.account_id = $1 AND cs.chat_id = $2
+WHERE cs.chat_id IS NOT NULL
+   OR EXISTS (SELECT 1 FROM dialogs WHERE account_id = $1 AND chat_id = $2)
+"""
+
+_SENT = """
+SELECT chat_id, count(*) AS message_count, max(date) AS last_date,
+       max(message_id) AS last_message_id,
+       count(*) FILTER (WHERE deleted_at IS NOT NULL) AS deleted_count
+FROM messages
+WHERE account_id = $1 AND sender_id = $2
+GROUP BY chat_id
+"""
+
+type Point = tuple[datetime, str | None]
+
+
+def _full_name(state: PeerHistoryView) -> str | None:
+    parts = [part for part in (state.first_name, state.last_name) if part]
+    return " ".join(parts) or None
+
+
+def _timeline(
+    states: list[PeerHistoryView], value: Callable[[PeerHistoryView], str | None]
+) -> list[Point]:
+    points: list[Point] = []
+    for state in states:
+        current = value(state)
+        if not points or points[-1][1] != current:
+            points.append((state.observed_at, current))
+    return points
+
+
+def _without_gaps(points: list[Point]) -> list[Point]:
+    kept: list[Point] = []
+    for at, current in points:
+        gap = len(kept) > 1 and kept[-1][1] is None and kept[-2][1] == current
+        if gap:
+            kept.pop()
+        elif not kept or kept[-1][1] != current:
+            kept.append((at, current))
+    return kept
+
+
+def _known(points: list[Point]) -> list[Point]:
+    return _without_gaps([point for point in points if point[1] is not None])
+
+
+def _changes(field: str, points: list[Point]) -> list[ProfileChange]:
+    return [
+        ProfileChange(field=field, old=old, new=new, changed_at=at)
+        for (_, old), (at, new) in pairwise(points)
+    ]
+
+
+def profile_changes(states: list[PeerHistoryView]) -> list[ProfileChange]:
+    ordered = sorted(states, key=lambda state: state.observed_at)
+    alive = [state for state in ordered if not state.is_deleted_account]
+    deleted = _timeline(
+        ordered, lambda state: "deleted" if state.is_deleted_account else None
+    )
+    changes = [
+        *_changes("name", _without_gaps(_timeline(alive, _full_name))),
+        *_changes(
+            "username", _without_gaps(_timeline(alive, lambda state: state.username))
+        ),
+        *_changes("phone", _known(_timeline(alive, lambda state: state.phone))),
+        *[change for change in _changes("deleted", deleted) if change.new is not None],
+    ]
+    return sorted(changes, key=lambda change: change.changed_at, reverse=True)
+
+
+async def get_chat_changes(
+    pool: asyncpg.Pool, account_id: int, chat_id: int
+) -> list[ProfileChange]:
+    rows = await pool.fetch(_CHAT_TITLES, account_id, chat_id, HISTORY_LIMIT)
+    return [
+        ProfileChange(
+            field="title",
+            old=row["previous"],
+            new=row["title"],
+            changed_at=row["ts"],
+            actor_id=row["actor_id"],
+            message_id=row["message_id"],
+        )
+        for row in rows
+    ]
+
+
+async def _memberships(
+    pool: asyncpg.Pool, account_id: int, peer_id: int
+) -> list[MembershipView]:
+    rows = await pool.fetch(_MEMBERSHIPS, account_id, peer_id)
+    ids = list({row["chat_id"] for row in rows})
+    known = {
+        chat.chat_id: chat for chat in await chats.get_chats(pool, account_id, ids)
+    }
+    views: list[MembershipView] = []
+    for row in rows:
+        chat = known.get(row["chat_id"])
+        views.append(
+            MembershipView(
+                **dict(row),
+                chat_title=chat.title if chat else None,
+                has_avatar=bool(chat and chat.has_avatar),
+            )
+        )
+    return views
+
+
+async def get_overview(
+    pool: asyncpg.Pool, account_id: int, peer_id: int
+) -> PeerOverview:
+    direct = await pool.fetchrow(_DIRECT_CHAT, account_id, peer_id)
+    return PeerOverview(
+        peer=await get_peer(pool, account_id, peer_id),
+        changes=profile_changes(await get_peer_history(pool, account_id, peer_id)),
+        avatars=await avatar_history(pool, account_id, peer_id),
+        memberships=await _memberships(pool, account_id, peer_id),
+        dm=DirectChatView(**dict(direct)) if direct else None,
+    )
+
+
+async def get_groups(pool: asyncpg.Pool, account_id: int, peer_id: int) -> PeerGroups:
+    async with pool.acquire() as conn, conn.transaction():
+        await conn.execute("SET LOCAL jit = off")
+        rows = await conn.fetch(_SENT, account_id, peer_id)
+    sent = {row["chat_id"]: row for row in rows if row["chat_id"] < 0}
+    found = await chats.get_chats(pool, account_id, list(sent))
+    groups = [
+        SharedGroupView(
+            chat_id=chat.chat_id,
+            title=chat.title,
+            has_avatar=chat.has_avatar,
+            message_count=sent[chat.chat_id]["message_count"],
+            last_date=sent[chat.chat_id]["last_date"],
+            last_message_id=sent[chat.chat_id]["last_message_id"],
+        )
+        for chat in found
+    ]
+    groups.sort(key=lambda group: group.last_date, reverse=True)
+    return PeerGroups(
+        groups=groups, deleted_count=sum(row["deleted_count"] for row in rows)
+    )

backend/src/utils/read/period.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+
+import asyncpg
+
+DEFAULT_TZ = "UTC"
+MAX_DAYS = 365
+
+_timezones: set[str] = set()
+
+
+async def known_timezone(pool: asyncpg.Pool, tz: str) -> bool:
+    if not _timezones:
+        rows = await pool.fetch("SELECT name FROM pg_timezone_names")
+        _timezones.update(row["name"] for row in rows)
+    return tz in _timezones
+
+
+async def period_bounds(
+    pool: asyncpg.Pool, days: int, tz: str
+) -> tuple[datetime, datetime]:
+    row = await pool.fetchrow(
+        "SELECT (date_trunc('day', now() AT TIME ZONE $1) "
+        "- make_interval(days => $2 - 1)) AT TIME ZONE $1 AS since, now() AS until",
+        tz,
+        days,
+    )
+    return row["since"], row["until"]