chore(deps): update ghcr.io/astral-sh/uv:python3.13-trixie-slim docker digest to 331fbb1 (main) (#22)

Co-authored-by: renovate <renovate@hhhhh.dev>

README.md

@@ -11,7 +11,7 @@ Useful on its own: a Telegram where nothing disappears. In the [beaver](https://
 - **Voice** transcribed through Telegram's own transcription (Premium), queued when the quota runs out.
 - **Presence** as a time series (TimescaleDB), read receipts, stories.
 - **Capture policy** from your Telegram folders with per-chat overrides: channels nothing, groups messages only, private chats everything, by default.
-- **Annotations and watches**: your notes on chats and rules that fire alerts, both yours to write in the web client.
+- **Annotations and watches**: your notes on messages, with `#tags` and one list of them all, and rules that fire alerts, both yours to write in the web client.
 - **Watched bots**: bot tokens you add in the web client. Each one logs in over MTProto (`auth.importBotAuthorization`, the same door a bot-login client uses), backfills the history of the bot's dialogs once and then keeps the incoming stream: who writes to the bot, what, and when. A separate authorization, so it does not take `getUpdates` away from whoever runs that bot over the Bot API.
 
 Nothing is ever deleted from the archive.
@@ -19,12 +19,12 @@ Nothing is ever deleted from the archive.
 ## Parts
 
 - `backend/` - `userbot` (kurigram, the capture loop and jobs), `watcher` (kurigram again, one bot-token session per watched bot) and `api` (FastAPI: the web API, the MCP server, and the built frontend as static files).
-- `frontend/` - SvelteKit, `adapter-static`; a chat client over the archive: folders, chat list, thread with media, versions and the deleted messages inline, search, stories, presence graphs.
+- `frontend/` - SvelteKit, `adapter-static`; a chat client over the archive: folders, chat list, thread with media, versions and the deleted messages inline, a feed of what was deleted or edited across chats, search, stories, presence graphs.
 - `docker-compose.yml` - includes `backend/` (Postgres with TimescaleDB, `userbot`, `api`) and `frontend/` (the dev server); profiles `external` and `services`.
 
 ## MCP
 
-Read-only: `list_watched_bots`, `get_bot_stats`, `get_bot_updates`, `search_messages_tool`, `list_chats`, `get_chat_history` (with deleted messages included), `get_deleted_messages`, `get_message_versions`, `get_media`, `get_callbacks`, `presence_history`, `get_peer_history`, `get_stories`, `get_annotations`, `list_watches`, `list_alerts`. The one write is `set_watch`. There are no send, edit or delete tools and there will not be.
+Read-only: `search_messages_tool`, `list_chats`, `get_chat_history` (a readable transcript, deleted messages included) and `get_chat_history_raw` (the same as JSON), `get_deleted_messages`, `get_feed` (deleted and edited messages and profile changes across chats), `get_message_versions`, `get_media`, `get_callbacks`, `get_read_info`, `presence_history`, `presence_stats`, `get_peer_history`, `get_person`, `get_stories`, `get_annotations`, `list_watches`, `list_alerts`. Two tools write: `set_watch`, and `download_media`, which pulls any attachment and leaves a public link to it for a day, visible and revocable in the web client. There are no send, edit or delete tools and there will not be.
 
 ## Run
 

backend/.env.example

@@ -25,6 +25,7 @@ BOTS__BACKFILL_PER_CHAT=0
 
 API__HOST=0.0.0.0
 API__PORT=8080
+API__PUBLIC_URL=
 
 AUTH__TOKEN=
 

backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/astral-sh/uv:python3.13-trixie-slim
+FROM ghcr.io/astral-sh/uv:python3.13-trixie-slim@sha256:331fbb158258608b9391e802305a9a2c9e42d689435c1d98706647122a122ab0
 
 RUN apt-get update && apt-get install -y build-essential && rm -rf /var/lib/apt/lists/*
 

backend/migrations/versions/a2c4e6f8b1d3_edit_versions_everywhere.py

@@ -0,0 +1,32 @@
+"""edit versions everywhere
+
+Revision ID: a2c4e6f8b1d3
+Revises: e8b3f1a6c2d4
+Create Date: 2026-10-19 02:30:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "a2c4e6f8b1d3"
+down_revision: str | None = "e8b3f1a6c2d4"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        "UPDATE capture_policy SET track_edits_deletes = TRUE, updated_at = now() "
+        "WHERE scope_type IN ('default_group', 'default_channel') "
+        "AND NOT track_edits_deletes"
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        "UPDATE capture_policy SET track_edits_deletes = FALSE, updated_at = now() "
+        "WHERE scope_type IN ('default_group', 'default_channel') "
+        "AND track_edits_deletes"
+    )

backend/migrations/versions/b3d5f7a9c2e4_feed.py

@@ -0,0 +1,198 @@
+"""feed of deleted and edited messages
+
+Revision ID: b3d5f7a9c2e4
+Revises: a2c4e6f8b1d3
+Create Date: 2026-10-19 02:40:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "b3d5f7a9c2e4"
+down_revision: str | None = "a2c4e6f8b1d3"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_ITEMS = """
+CREATE TABLE feed_items (
+    id bigserial PRIMARY KEY,
+    account_id integer NOT NULL,
+    kind text NOT NULL,
+    chat_id bigint NOT NULL,
+    chat_kind text NOT NULL,
+    message_id bigint NOT NULL,
+    message_date timestamptz,
+    at timestamptz NOT NULL,
+    since timestamptz NOT NULL,
+    message_count integer NOT NULL DEFAULT 1
+)
+"""
+
+_SEEN = """
+CREATE TABLE feed_seen (
+    account_id integer PRIMARY KEY,
+    seen_at timestamptz NOT NULL
+)
+"""
+
+_CHAT_KIND = """
+CREATE FUNCTION feed_chat_kind(chat_id bigint, chat_type text)
+RETURNS text LANGUAGE sql IMMUTABLE AS $$
+    SELECT CASE
+        WHEN chat_id > 0 THEN 'dm'
+        WHEN chat_type = 'ChatType.CHANNEL' THEN 'channel'
+        ELSE 'group' END
+$$
+"""
+
+_MESSAGE_DATE = """
+CREATE FUNCTION feed_message_date(stamp text)
+RETURNS timestamptz LANGUAGE sql STABLE AS $$
+    SELECT CASE
+        WHEN stamp ~ '^\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?$'
+        THEN stamp::timestamp AT TIME ZONE 'UTC' END
+$$
+"""
+
+_ON_DELETE = """
+CREATE FUNCTION feed_on_delete() RETURNS trigger LANGUAGE plpgsql AS $$
+DECLARE
+    item feed_items%ROWTYPE;
+BEGIN
+    SELECT * INTO item FROM feed_items
+    WHERE account_id = NEW.account_id AND chat_id = NEW.chat_id
+      AND kind = 'deleted'
+    ORDER BY at DESC, id DESC LIMIT 1;
+    IF FOUND AND NEW.deleted_at BETWEEN item.since - interval '5 seconds'
+                                    AND item.at + interval '5 seconds' THEN
+        IF (NEW.date, NEW.message_id) > (item.message_date, item.message_id) THEN
+            item.message_id := NEW.message_id;
+            item.message_date := NEW.date;
+        END IF;
+        UPDATE feed_items SET
+            message_count = message_count + 1,
+            at = greatest(at, NEW.deleted_at),
+            since = least(since, NEW.deleted_at),
+            message_id = item.message_id,
+            message_date = item.message_date
+        WHERE id = item.id;
+    ELSE
+        INSERT INTO feed_items
+            (account_id, kind, chat_id, chat_kind, message_id, message_date,
+             at, since)
+        VALUES (NEW.account_id, 'deleted', NEW.chat_id,
+                feed_chat_kind(NEW.chat_id, NEW.raw->'chat'->>'type'),
+                NEW.message_id, NEW.date, NEW.deleted_at, NEW.deleted_at);
+    END IF;
+    RETURN NULL;
+END;
+$$
+"""
+
+_ON_VERSION = """
+CREATE FUNCTION feed_on_version() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+    IF NEW.edit_date IS NULL AND NOT EXISTS (
+        SELECT 1 FROM message_versions v
+        WHERE v.account_id = NEW.account_id AND v.chat_id = NEW.chat_id
+          AND v.message_id = NEW.message_id AND v.observed_at < NEW.observed_at
+    ) THEN
+        RETURN NULL;
+    END IF;
+    INSERT INTO feed_items
+        (account_id, kind, chat_id, chat_kind, message_id, message_date, at, since)
+    VALUES (NEW.account_id, 'edited', NEW.chat_id,
+            feed_chat_kind(NEW.chat_id, NEW.raw->'chat'->>'type'),
+            NEW.message_id, feed_message_date(NEW.raw->>'date'),
+            NEW.observed_at, NEW.observed_at);
+    RETURN NULL;
+END;
+$$
+"""
+
+_BACKFILL = """
+WITH d AS (
+    SELECT account_id, chat_id, message_id, date, deleted_at,
+           raw->'chat'->>'type' AS chat_type,
+           deleted_at - lag(deleted_at) OVER (
+               PARTITION BY account_id, chat_id ORDER BY deleted_at, message_id
+           ) AS gap
+    FROM messages WHERE deleted_at IS NOT NULL
+), b AS (
+    SELECT *, count(*) FILTER (WHERE gap IS NULL OR gap > interval '5 seconds')
+        OVER (PARTITION BY account_id, chat_id ORDER BY deleted_at, message_id)
+        AS burst
+    FROM d
+), deleted AS (
+    SELECT account_id, 'deleted' AS kind, chat_id,
+           feed_chat_kind(chat_id,
+               (array_agg(chat_type ORDER BY date DESC, message_id DESC))[1])
+               AS chat_kind,
+           (array_agg(message_id ORDER BY date DESC, message_id DESC))[1]
+               AS message_id,
+           max(date) AS message_date, max(deleted_at) AS at,
+           min(deleted_at) AS since, count(*)::integer AS message_count
+    FROM b GROUP BY account_id, chat_id, burst
+), edited AS (
+    SELECT v.account_id, 'edited' AS kind, v.chat_id,
+           feed_chat_kind(v.chat_id, v.raw->'chat'->>'type') AS chat_kind,
+           v.message_id, feed_message_date(v.raw->>'date') AS message_date,
+           v.observed_at AS at, v.observed_at AS since, 1 AS message_count
+    FROM message_versions v
+    WHERE v.edit_date IS NOT NULL OR EXISTS (
+        SELECT 1 FROM message_versions p
+        WHERE p.account_id = v.account_id AND p.chat_id = v.chat_id
+          AND p.message_id = v.message_id AND p.observed_at < v.observed_at
+    )
+)
+INSERT INTO feed_items
+    (account_id, kind, chat_id, chat_kind, message_id, message_date, at, since,
+     message_count)
+SELECT * FROM (SELECT * FROM deleted UNION ALL SELECT * FROM edited) items
+ORDER BY at, chat_id, message_id
+"""
+
+
+def upgrade() -> None:
+    op.execute("SET LOCAL jit = off")
+    op.execute(_ITEMS)
+    op.execute(_SEEN)
+    op.execute(_CHAT_KIND)
+    op.execute(_MESSAGE_DATE)
+    op.execute(_ON_DELETE)
+    op.execute(_ON_VERSION)
+    op.execute(
+        "CREATE TRIGGER messages_feed_deleted AFTER UPDATE OF deleted_at "
+        "ON messages FOR EACH ROW "
+        "WHEN (OLD.deleted_at IS NULL AND NEW.deleted_at IS NOT NULL) "
+        "EXECUTE FUNCTION feed_on_delete()"
+    )
+    op.execute(
+        "CREATE TRIGGER message_versions_feed AFTER INSERT ON message_versions "
+        "FOR EACH ROW EXECUTE FUNCTION feed_on_version()"
+    )
+    op.execute(_BACKFILL)
+    op.execute(
+        "CREATE INDEX ix_feed_items_recent ON feed_items (account_id, at DESC, id DESC)"
+    )
+    op.execute(
+        "CREATE INDEX ix_feed_items_chat ON feed_items "
+        "(account_id, chat_id, at DESC, id DESC)"
+    )
+    op.execute(
+        "INSERT INTO feed_seen (account_id, seen_at) SELECT account_id, now() "
+        "FROM accounts"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP TRIGGER IF EXISTS message_versions_feed ON message_versions")
+    op.execute("DROP TRIGGER IF EXISTS messages_feed_deleted ON messages")
+    op.execute("DROP FUNCTION IF EXISTS feed_on_version()")
+    op.execute("DROP FUNCTION IF EXISTS feed_on_delete()")
+    op.execute("DROP FUNCTION IF EXISTS feed_message_date(text)")
+    op.execute("DROP FUNCTION IF EXISTS feed_chat_kind(bigint, text)")
+    op.execute("DROP TABLE IF EXISTS feed_seen")
+    op.execute("DROP TABLE IF EXISTS feed_items")

backend/migrations/versions/c5e7a9b1d3f6_feed_profiles.py

@@ -0,0 +1,321 @@
+"""feed of profile changes and chat renames
+
+Revision ID: c5e7a9b1d3f6
+Revises: b3d5f7a9c2e4
+Create Date: 2026-10-19 04:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "c5e7a9b1d3f6"
+down_revision: str | None = "b3d5f7a9c2e4"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_PEER_VALUE = """
+CREATE FUNCTION feed_peer_value(
+    field text, first_name text, last_name text, username text, photo text
+) RETURNS text LANGUAGE sql IMMUTABLE AS $$
+    SELECT CASE field
+        WHEN 'name' THEN NULLIF(trim(concat_ws(' ', first_name, last_name)), '')
+        WHEN 'username' THEN NULLIF(username, '')
+        WHEN 'photo' THEN NULLIF(photo, '') END
+$$
+"""
+
+_PEER_SCOPE = """
+CREATE FUNCTION feed_peer_in_scope(account integer, peer bigint)
+RETURNS boolean LANGUAGE sql STABLE AS $$
+    SELECT EXISTS (
+        SELECT 1 FROM dialogs WHERE account_id = account AND chat_id = peer
+    ) OR EXISTS (
+        SELECT 1 FROM peers
+        WHERE account_id = account AND peer_id = peer
+          AND (raw->>'is_contact' = 'true' OR raw->>'contact' = 'true')
+    )
+$$
+"""
+
+_MERGE_DETAIL = """
+CREATE FUNCTION feed_merge_detail(base jsonb, incoming jsonb)
+RETURNS jsonb LANGUAGE sql IMMUTABLE AS $$
+    SELECT coalesce(jsonb_object_agg(key, pair), '{}'::jsonb)
+    FROM (
+        SELECT key, jsonb_build_array(
+                   coalesce(base->key->0, incoming->key->0),
+                   coalesce(incoming->key->1, base->key->1)) AS pair
+        FROM (
+            SELECT jsonb_object_keys(coalesce(base, '{}'::jsonb)) AS key
+            UNION
+            SELECT jsonb_object_keys(incoming)
+        ) keys
+    ) merged
+    WHERE pair->0 IS DISTINCT FROM pair->1
+$$
+"""
+
+_UPSERT = """
+CREATE FUNCTION feed_profile_upsert(
+    account integer, chat bigint, kind_of_chat text, message bigint,
+    changed_at timestamptz, changes jsonb
+) RETURNS void LANGUAGE plpgsql AS $$
+DECLARE
+    item feed_items%ROWTYPE;
+    merged jsonb;
+BEGIN
+    SELECT * INTO item FROM feed_items
+    WHERE account_id = account AND chat_id = chat AND kind = 'profile'
+    ORDER BY at DESC, id DESC LIMIT 1;
+    IF FOUND AND changed_at BETWEEN item.since - interval '1 day'
+                                AND item.at + interval '1 day' THEN
+        merged := feed_merge_detail(item.detail, changes);
+        IF merged = '{}'::jsonb THEN
+            DELETE FROM feed_items WHERE id = item.id;
+        ELSE
+            UPDATE feed_items SET
+                detail = merged,
+                message_count = message_count
+                    + (SELECT count(*) FROM jsonb_object_keys(changes)),
+                at = greatest(at, changed_at),
+                since = least(since, changed_at),
+                message_id = greatest(message_id, message)
+            WHERE id = item.id;
+        END IF;
+    ELSE
+        INSERT INTO feed_items
+            (account_id, kind, chat_id, chat_kind, message_id, at, since,
+             message_count, detail)
+        VALUES (account, 'profile', chat, kind_of_chat, message, changed_at,
+                changed_at,
+                (SELECT count(*) FROM jsonb_object_keys(changes)), changes);
+    END IF;
+END;
+$$
+"""
+
+_ON_PEER = """
+CREATE FUNCTION feed_on_peer_change() RETURNS trigger LANGUAGE plpgsql AS $$
+DECLARE
+    changes jsonb := '{}'::jsonb;
+    field text;
+    fresh text;
+    old text;
+BEGIN
+    IF NEW.peer_id <= 0 OR NOT feed_peer_in_scope(NEW.account_id, NEW.peer_id) THEN
+        RETURN NULL;
+    END IF;
+    FOREACH field IN ARRAY ARRAY['name', 'username', 'photo'] LOOP
+        fresh := feed_peer_value(field, NEW.first_name, NEW.last_name,
+                                 NEW.username, NEW.photo_unique_id);
+        CONTINUE WHEN fresh IS NULL;
+        SELECT value INTO old FROM (
+            SELECT feed_peer_value(field, h.first_name, h.last_name, h.username,
+                                   h.photo_unique_id) AS value, h.observed_at
+            FROM peer_history h
+            WHERE h.account_id = NEW.account_id AND h.peer_id = NEW.peer_id
+              AND h.observed_at < NEW.observed_at
+        ) earlier
+        WHERE value IS NOT NULL
+        ORDER BY observed_at DESC LIMIT 1;
+        CONTINUE WHEN old IS NULL OR old = fresh;
+        CONTINUE WHEN EXISTS (
+            SELECT 1 FROM peer_history h
+            WHERE h.account_id = NEW.account_id AND h.peer_id = NEW.peer_id
+              AND h.observed_at < NEW.observed_at
+              AND feed_peer_value(field, h.first_name, h.last_name, h.username,
+                                  h.photo_unique_id) = fresh
+        );
+        changes := changes || jsonb_build_object(field, jsonb_build_array(old, fresh));
+    END LOOP;
+    IF changes <> '{}'::jsonb THEN
+        PERFORM feed_profile_upsert(NEW.account_id, NEW.peer_id, 'dm', 0,
+                                    NEW.observed_at, changes);
+    END IF;
+    RETURN NULL;
+END;
+$$
+"""
+
+_ON_TITLE = """
+CREATE FUNCTION feed_on_chat_title() RETURNS trigger LANGUAGE plpgsql AS $$
+DECLARE
+    previous text;
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM dialogs
+        WHERE account_id = NEW.account_id AND chat_id = NEW.chat_id
+    ) THEN
+        RETURN NULL;
+    END IF;
+    SELECT title INTO previous FROM chat_history
+    WHERE account_id = NEW.account_id AND chat_id = NEW.chat_id
+      AND event IN ('meta', 'title') AND title IS NOT NULL
+      AND (ts, message_id) < (NEW.ts, NEW.message_id)
+    ORDER BY ts DESC, message_id DESC LIMIT 1;
+    IF previous IS NULL OR previous = NEW.title THEN
+        RETURN NULL;
+    END IF;
+    PERFORM feed_profile_upsert(
+        NEW.account_id, NEW.chat_id,
+        feed_chat_kind(NEW.chat_id, NEW.raw->'chat'->>'type'),
+        CASE WHEN NEW.event = 'title' THEN NEW.message_id ELSE 0 END,
+        NEW.ts, jsonb_build_object('title', jsonb_build_array(previous, NEW.title)));
+    RETURN NULL;
+END;
+$$
+"""
+
+_BACKFILL_PEERS = """
+WITH scope AS (
+    SELECT account_id::integer AS account_id, chat_id AS peer_id
+    FROM dialogs WHERE chat_id > 0
+    UNION
+    SELECT account_id, peer_id FROM peers
+    WHERE raw->>'is_contact' = 'true' OR raw->>'contact' = 'true'
+), firsts AS (
+    SELECT h.account_id, h.peer_id, f.field, f.value,
+           min(h.observed_at) AS at
+    FROM peer_history h
+    JOIN scope s ON s.account_id = h.account_id AND s.peer_id = h.peer_id
+    CROSS JOIN LATERAL (
+        SELECT field, feed_peer_value(field, h.first_name, h.last_name,
+                                      h.username, h.photo_unique_id) AS value
+        FROM unnest(ARRAY['name', 'username', 'photo']) AS field
+    ) f
+    WHERE f.value IS NOT NULL
+    GROUP BY 1, 2, 3, 4
+), ranked AS (
+    SELECT *, row_number() OVER (
+        PARTITION BY account_id, peer_id, field ORDER BY at
+    ) AS position
+    FROM firsts
+), changes AS (
+    SELECT r.account_id, r.peer_id, r.field, r.value AS fresh, r.at, o.value AS old
+    FROM ranked r
+    CROSS JOIN LATERAL (
+        SELECT value FROM (
+            SELECT feed_peer_value(r.field, h.first_name, h.last_name,
+                                   h.username, h.photo_unique_id) AS value,
+                   h.observed_at
+            FROM peer_history h
+            WHERE h.account_id = r.account_id AND h.peer_id = r.peer_id
+              AND h.observed_at < r.at
+        ) earlier
+        WHERE value IS NOT NULL
+        ORDER BY observed_at DESC LIMIT 1
+    ) o
+    WHERE r.position > 1 AND r.at > now() - interval '90 days'
+      AND o.value <> r.value
+), gaps AS (
+    SELECT *, at - lag(at) OVER (PARTITION BY account_id, peer_id ORDER BY at)
+              AS gap
+    FROM changes
+), bursts AS (
+    SELECT *, count(*) FILTER (WHERE gap IS NULL OR gap > interval '1 day')
+              OVER (PARTITION BY account_id, peer_id ORDER BY at) AS burst
+    FROM gaps
+), fields AS (
+    SELECT account_id, peer_id, burst, field,
+           (array_agg(old ORDER BY at))[1] AS old,
+           (array_agg(fresh ORDER BY at DESC))[1] AS fresh,
+           count(*) AS changes, min(at) AS since, max(at) AS at
+    FROM bursts GROUP BY 1, 2, 3, 4
+)
+INSERT INTO feed_items
+    (account_id, kind, chat_id, chat_kind, message_id, at, since,
+     message_count, detail)
+SELECT account_id, 'profile', peer_id, 'dm', 0, max(at), min(since),
+       sum(changes)::integer,
+       jsonb_object_agg(field, jsonb_build_array(old, fresh))
+           FILTER (WHERE old <> fresh)
+FROM fields
+GROUP BY account_id, peer_id, burst
+HAVING bool_or(old <> fresh)
+ORDER BY max(at)
+"""
+
+_BACKFILL_CHATS = """
+WITH titles AS (
+    SELECT account_id, chat_id, message_id, event, ts, title,
+           raw->'chat'->>'type' AS chat_type,
+           lag(title) OVER (
+               PARTITION BY account_id, chat_id ORDER BY ts, message_id
+           ) AS previous
+    FROM chat_history
+    WHERE event IN ('meta', 'title') AND title IS NOT NULL
+), changes AS (
+    SELECT * FROM titles t
+    WHERE previous IS NOT NULL AND previous <> title
+      AND ts > now() - interval '90 days'
+      AND EXISTS (
+          SELECT 1 FROM dialogs d
+          WHERE d.account_id = t.account_id AND d.chat_id = t.chat_id
+      )
+), gaps AS (
+    SELECT *, ts - lag(ts) OVER (PARTITION BY account_id, chat_id ORDER BY ts)
+              AS gap
+    FROM changes
+), bursts AS (
+    SELECT *, count(*) FILTER (WHERE gap IS NULL OR gap > interval '1 day')
+              OVER (PARTITION BY account_id, chat_id ORDER BY ts) AS burst
+    FROM gaps
+), grouped AS (
+    SELECT account_id, chat_id, burst,
+           (array_agg(chat_type ORDER BY ts DESC))[1] AS chat_type,
+           max(CASE WHEN event = 'title' THEN message_id ELSE 0 END) AS message_id,
+           (array_agg(previous ORDER BY ts))[1] AS old,
+           (array_agg(title ORDER BY ts DESC))[1] AS fresh,
+           count(*) AS changes, min(ts) AS since, max(ts) AS at
+    FROM bursts GROUP BY 1, 2, 3
+)
+INSERT INTO feed_items
+    (account_id, kind, chat_id, chat_kind, message_id, at, since,
+     message_count, detail)
+SELECT account_id, 'profile', chat_id, feed_chat_kind(chat_id, chat_type),
+       message_id, at, since, changes::integer,
+       jsonb_build_object('title', jsonb_build_array(old, fresh))
+FROM grouped
+WHERE old <> fresh
+ORDER BY at
+"""
+
+
+def upgrade() -> None:
+    op.execute("SET LOCAL jit = off")
+    op.execute("ALTER TABLE feed_items ADD COLUMN detail jsonb")
+    op.execute(_PEER_VALUE)
+    op.execute(_PEER_SCOPE)
+    op.execute(_MERGE_DETAIL)
+    op.execute(_UPSERT)
+    op.execute(_ON_PEER)
+    op.execute(_ON_TITLE)
+    op.execute(
+        "CREATE TRIGGER peer_history_feed AFTER INSERT ON peer_history "
+        "FOR EACH ROW EXECUTE FUNCTION feed_on_peer_change()"
+    )
+    op.execute(
+        "CREATE TRIGGER chat_history_feed AFTER INSERT ON chat_history "
+        "FOR EACH ROW WHEN (NEW.event IN ('meta', 'title') AND NEW.title IS NOT NULL) "
+        "EXECUTE FUNCTION feed_on_chat_title()"
+    )
+    op.execute(_BACKFILL_PEERS)
+    op.execute(_BACKFILL_CHATS)
+
+
+def downgrade() -> None:
+    op.execute("DROP TRIGGER IF EXISTS chat_history_feed ON chat_history")
+    op.execute("DROP TRIGGER IF EXISTS peer_history_feed ON peer_history")
+    op.execute("DROP FUNCTION IF EXISTS feed_on_chat_title()")
+    op.execute("DROP FUNCTION IF EXISTS feed_on_peer_change()")
+    op.execute(
+        "DROP FUNCTION IF EXISTS "
+        "feed_profile_upsert(integer, bigint, text, bigint, timestamptz, jsonb)"
+    )
+    op.execute("DROP FUNCTION IF EXISTS feed_merge_detail(jsonb, jsonb)")
+    op.execute("DROP FUNCTION IF EXISTS feed_peer_in_scope(integer, bigint)")
+    op.execute("DROP FUNCTION IF EXISTS feed_peer_value(text, text, text, text, text)")
+    op.execute("DELETE FROM feed_items WHERE kind = 'profile'")
+    op.execute("ALTER TABLE feed_items DROP COLUMN detail")

backend/migrations/versions/d7a3c5e9f2b1_message_reads.py

@@ -0,0 +1,51 @@
+"""message reads
+
+Revision ID: d7a3c5e9f2b1
+Revises: f8c2d6e1a4b7
+Create Date: 2026-10-19 01:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "d7a3c5e9f2b1"
+down_revision: str | None = "f8c2d6e1a4b7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_READS = """
+CREATE TABLE message_reads (
+    account_id integer NOT NULL,
+    chat_id bigint NOT NULL,
+    message_id bigint NOT NULL,
+    reader_id bigint NOT NULL,
+    read_at timestamptz NOT NULL,
+    source text NOT NULL,
+    fetched_at timestamptz NOT NULL DEFAULT now(),
+    PRIMARY KEY (account_id, chat_id, message_id, reader_id)
+)
+"""
+
+_FETCHES = """
+CREATE TABLE read_fetches (
+    account_id integer NOT NULL,
+    chat_id bigint NOT NULL,
+    message_id bigint NOT NULL,
+    outcome text NOT NULL,
+    fetched_at timestamptz NOT NULL DEFAULT now(),
+    retry_at timestamptz NOT NULL,
+    PRIMARY KEY (account_id, chat_id, message_id)
+)
+"""
+
+
+def upgrade() -> None:
+    op.execute(_READS)
+    op.execute(_FETCHES)
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS read_fetches")
+    op.execute("DROP TABLE IF EXISTS message_reads")

backend/migrations/versions/d8e3b5a1c7f4_watch_names_webhooks.py

@@ -0,0 +1,86 @@
+"""watch names and webhooks
+
+Revision ID: d8e3b5a1c7f4
+Revises: d8f4a1c6e2b7
+Create Date: 2026-10-19 03:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "d8e3b5a1c7f4"
+down_revision: str | None = "d8f4a1c6e2b7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_WEBHOOKS = """
+CREATE TABLE webhooks (
+    id serial PRIMARY KEY,
+    account_id integer NOT NULL,
+    url text NOT NULL,
+    secret text NOT NULL,
+    enabled boolean NOT NULL DEFAULT true,
+    watch_ids integer[],
+    created_at timestamptz NOT NULL DEFAULT now(),
+    updated_at timestamptz NOT NULL DEFAULT now()
+)
+"""
+
+_DELIVERIES = """
+CREATE TABLE webhook_deliveries (
+    id bigserial PRIMARY KEY,
+    webhook_id integer NOT NULL REFERENCES webhooks (id) ON DELETE CASCADE,
+    alert_id integer REFERENCES alerts (id) ON DELETE CASCADE,
+    event text NOT NULL,
+    status text NOT NULL DEFAULT 'pending',
+    attempts integer NOT NULL DEFAULT 0,
+    next_attempt_at timestamptz NOT NULL DEFAULT now(),
+    response_code integer,
+    last_error text,
+    body text,
+    created_at timestamptz NOT NULL DEFAULT now(),
+    finished_at timestamptz
+)
+"""
+
+_LEGACY_KEYWORD = """
+UPDATE watches
+SET params = (params - 'keyword') || jsonb_build_object('pattern', params->'keyword')
+WHERE kind = 'keyword' AND params ? 'keyword' AND NOT params ? 'pattern'
+"""
+
+
+def upgrade() -> None:
+    op.execute("ALTER TABLE watches ADD COLUMN name text")
+    op.execute(_LEGACY_KEYWORD)
+    op.execute("ALTER TABLE alerts DROP CONSTRAINT alerts_watch_id_fkey")
+    op.execute(
+        "ALTER TABLE alerts ADD CONSTRAINT alerts_watch_id_fkey "
+        "FOREIGN KEY (watch_id) REFERENCES watches (id) ON DELETE CASCADE"
+    )
+    op.execute("CREATE INDEX ix_alerts_account_id ON alerts (account_id, id DESC)")
+    op.execute(_WEBHOOKS)
+    op.execute("CREATE INDEX ix_webhooks_account ON webhooks (account_id)")
+    op.execute(_DELIVERIES)
+    op.execute(
+        "CREATE INDEX ix_webhook_deliveries_due ON webhook_deliveries "
+        "(next_attempt_at) WHERE status = 'pending'"
+    )
+    op.execute(
+        "CREATE INDEX ix_webhook_deliveries_webhook ON webhook_deliveries "
+        "(webhook_id, id DESC)"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS webhook_deliveries")
+    op.execute("DROP TABLE IF EXISTS webhooks")
+    op.execute("DROP INDEX IF EXISTS ix_alerts_account_id")
+    op.execute("ALTER TABLE alerts DROP CONSTRAINT alerts_watch_id_fkey")
+    op.execute(
+        "ALTER TABLE alerts ADD CONSTRAINT alerts_watch_id_fkey "
+        "FOREIGN KEY (watch_id) REFERENCES watches (id)"
+    )
+    op.execute("ALTER TABLE watches DROP COLUMN name")

backend/migrations/versions/d8f1a6c3e5b7_message_search.py

@@ -0,0 +1,148 @@
+"""message search index outside the hypertable, versions searchable
+
+Revision ID: d8f1a6c3e5b7
+Revises: c5e7a9b1d3f6
+Create Date: 2026-10-19 03:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "d8f1a6c3e5b7"
+down_revision: str | None = "c5e7a9b1d3f6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_VECTOR = """
+CREATE FUNCTION search_vector(body text) RETURNS tsvector
+LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
+    SELECT to_tsvector('russian', translate(coalesce(body, ''), 'Ёё', 'Ее'))
+        || to_tsvector('simple', translate(coalesce(body, ''), 'Ёё', 'Ее'))
+$$
+"""
+
+_TABLE = """
+CREATE TABLE message_search (
+    account_id integer NOT NULL,
+    chat_id bigint NOT NULL,
+    message_id bigint NOT NULL,
+    date timestamptz NOT NULL,
+    sender_id bigint,
+    text text,
+    has_media boolean NOT NULL DEFAULT false,
+    edited_at timestamptz,
+    deleted_at timestamptz,
+    tsv tsvector GENERATED ALWAYS AS (search_vector(text)) STORED,
+    PRIMARY KEY (account_id, chat_id, message_id, date)
+)
+"""
+
+_BACKFILL = """
+INSERT INTO message_search (account_id, chat_id, message_id, date, sender_id,
+                            text, has_media, edited_at, deleted_at)
+SELECT account_id, chat_id, message_id, date, sender_id,
+       text, has_media, edited_at, deleted_at
+FROM messages
+"""
+
+_SYNC = """
+CREATE FUNCTION message_search_sync() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+    IF TG_OP = 'DELETE' THEN
+        DELETE FROM message_search
+        WHERE account_id = OLD.account_id AND chat_id = OLD.chat_id
+          AND message_id = OLD.message_id AND date = OLD.date;
+        RETURN NULL;
+    END IF;
+    INSERT INTO message_search (account_id, chat_id, message_id, date,
+                                sender_id, text, has_media, edited_at,
+                                deleted_at)
+    VALUES (NEW.account_id, NEW.chat_id, NEW.message_id, NEW.date,
+            NEW.sender_id, NEW.text, NEW.has_media, NEW.edited_at,
+            NEW.deleted_at)
+    ON CONFLICT (account_id, chat_id, message_id, date) DO UPDATE SET
+        sender_id = EXCLUDED.sender_id,
+        text = EXCLUDED.text,
+        has_media = EXCLUDED.has_media,
+        edited_at = EXCLUDED.edited_at,
+        deleted_at = EXCLUDED.deleted_at;
+    RETURN NULL;
+END;
+$$
+"""
+
+_CHANGED = (
+    "OLD.text IS DISTINCT FROM NEW.text "
+    "OR OLD.sender_id IS DISTINCT FROM NEW.sender_id "
+    "OR OLD.has_media IS DISTINCT FROM NEW.has_media "
+    "OR OLD.edited_at IS DISTINCT FROM NEW.edited_at "
+    "OR OLD.deleted_at IS DISTINCT FROM NEW.deleted_at"
+)
+
+
+def upgrade() -> None:
+    op.execute(_VECTOR)
+    op.execute(_TABLE)
+    op.execute(_BACKFILL)
+    op.execute("CREATE INDEX ix_message_search_tsv ON message_search USING gin (tsv)")
+    op.execute(
+        "CREATE INDEX ix_message_search_recent ON message_search "
+        "(account_id, date DESC, chat_id DESC, message_id DESC)"
+    )
+    op.execute(
+        "CREATE INDEX ix_message_search_sender ON message_search "
+        "(account_id, sender_id, date DESC)"
+    )
+    for flag in ("deleted", "edited"):
+        op.execute(
+            f"CREATE INDEX ix_message_search_{flag} ON message_search "
+            "(account_id, date DESC, chat_id DESC, message_id DESC) "
+            f"WHERE {flag}_at IS NOT NULL"
+        )
+    op.execute("ALTER TABLE message_search ALTER COLUMN tsv SET STATISTICS 1000")
+    op.execute(_SYNC)
+    op.execute(
+        "CREATE TRIGGER messages_search_insert AFTER INSERT ON messages "
+        "FOR EACH ROW EXECUTE FUNCTION message_search_sync()"
+    )
+    op.execute(
+        "CREATE TRIGGER messages_search_update AFTER UPDATE ON messages "
+        f"FOR EACH ROW WHEN ({_CHANGED}) EXECUTE FUNCTION message_search_sync()"
+    )
+    op.execute(
+        "CREATE TRIGGER messages_search_delete AFTER DELETE ON messages "
+        "FOR EACH ROW EXECUTE FUNCTION message_search_sync()"
+    )
+    op.execute(
+        "CREATE INDEX ix_message_versions_tsv ON message_versions USING gin "
+        "(search_vector(text))"
+    )
+    op.execute("DROP INDEX IF EXISTS ix_media_tsv")
+    op.execute(
+        "CREATE INDEX ix_media_tsv ON media USING gin (search_vector(extracted_text))"
+    )
+    op.execute("DROP INDEX IF EXISTS ix_messages_tsv")
+    for index in ("ix_message_versions_tsv", "ix_media_tsv"):
+        op.execute(f"ALTER INDEX {index} ALTER COLUMN 1 SET STATISTICS 1000")
+    op.execute("ANALYZE message_search, message_versions, media")
+
+
+def downgrade() -> None:
+    op.execute(
+        "CREATE INDEX ix_messages_tsv ON messages USING gin "
+        "(to_tsvector('russian', coalesce(text, '')))"
+    )
+    op.execute("DROP INDEX IF EXISTS ix_media_tsv")
+    op.execute(
+        "CREATE INDEX ix_media_tsv ON media USING gin "
+        "(to_tsvector('russian', coalesce(extracted_text, '')))"
+    )
+    op.execute("DROP INDEX IF EXISTS ix_message_versions_tsv")
+    op.execute("DROP TRIGGER IF EXISTS messages_search_delete ON messages")
+    op.execute("DROP TRIGGER IF EXISTS messages_search_update ON messages")
+    op.execute("DROP TRIGGER IF EXISTS messages_search_insert ON messages")
+    op.execute("DROP FUNCTION IF EXISTS message_search_sync()")
+    op.execute("DROP TABLE IF EXISTS message_search")
+    op.execute("DROP FUNCTION IF EXISTS search_vector(text)")

backend/migrations/versions/d8f4a1c6e2b7_capture_uptime.py

@@ -0,0 +1,50 @@
+"""capture uptime
+
+Revision ID: d8f4a1c6e2b7
+Revises: d7a3c5e9f2b1
+Create Date: 2026-10-19 03:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "d8f4a1c6e2b7"
+down_revision: str | None = "d7a3c5e9f2b1"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_UPTIME = """
+CREATE TABLE capture_uptime (
+    account_id integer NOT NULL,
+    started_at timestamptz NOT NULL,
+    seen_at timestamptz NOT NULL,
+    PRIMARY KEY (account_id, started_at)
+)
+"""
+
+_BACKFILL = """
+INSERT INTO capture_uptime (account_id, started_at, seen_at)
+SELECT account_id, min(ts), max(ts)
+FROM (
+    SELECT account_id, ts,
+           count(*) FILTER (WHERE gap > interval '30 minutes')
+               OVER (PARTITION BY account_id ORDER BY ts) AS stretch
+    FROM (
+        SELECT account_id, ts,
+               ts - lag(ts) OVER (PARTITION BY account_id ORDER BY ts) AS gap
+        FROM presence
+    ) gaps
+) stretches
+GROUP BY account_id, stretch
+"""
+
+
+def upgrade() -> None:
+    op.execute(_UPTIME)
+    op.execute(_BACKFILL)
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS capture_uptime")

backend/migrations/versions/e8b3f1a6c2d4_profile_history_reads.py

@@ -0,0 +1,54 @@
+"""profile history reads
+
+Revision ID: e8b3f1a6c2d4
+Revises: d8e3b5a1c7f4
+Create Date: 2026-10-19 03:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "e8b3f1a6c2d4"
+down_revision: str | None = "d8e3b5a1c7f4"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_PEER_STATE = (
+    "first_name, last_name, username, phone, photo_unique_id, is_deleted_account"
+)
+
+
+def upgrade() -> None:
+    op.execute(
+        "ALTER TABLE peer_history DROP CONSTRAINT peer_history_pkey, "
+        "ADD CONSTRAINT peer_history_pkey "
+        f"PRIMARY KEY (account_id, peer_id, observed_at) INCLUDE ({_PEER_STATE})"
+    )
+    op.execute(
+        "ALTER TABLE chat_history DROP CONSTRAINT chat_history_pkey, "
+        "ADD CONSTRAINT chat_history_pkey "
+        "PRIMARY KEY (account_id, chat_id, message_id, ts)"
+    )
+    op.execute(
+        "CREATE INDEX ix_memberships_user ON memberships (account_id, user_id, ts DESC)"
+    )
+    op.execute(
+        "CREATE INDEX ix_jobs_kind_created ON jobs (account_id, kind, created_at DESC)"
+    )
+    with op.get_context().autocommit_block():
+        op.execute("VACUUM (ANALYZE) peer_history")
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS ix_jobs_kind_created")
+    op.execute("DROP INDEX IF EXISTS ix_memberships_user")
+    op.execute(
+        "ALTER TABLE chat_history DROP CONSTRAINT chat_history_pkey, "
+        "ADD CONSTRAINT chat_history_pkey PRIMARY KEY (account_id, chat_id, message_id)"
+    )
+    op.execute(
+        "ALTER TABLE peer_history DROP CONSTRAINT peer_history_pkey, "
+        "ADD CONSTRAINT peer_history_pkey PRIMARY KEY (account_id, peer_id, observed_at)"
+    )

backend/migrations/versions/f1c4a8e2d6b3_chat_stats_last_deleted.py

@@ -0,0 +1,97 @@
+"""chat stats last deleted
+
+Revision ID: f1c4a8e2d6b3
+Revises: d8f1a6c3e5b7
+Create Date: 2026-10-19 04:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "f1c4a8e2d6b3"
+down_revision: str | None = "d8f1a6c3e5b7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_APPLY = """
+CREATE OR REPLACE FUNCTION chat_stats_apply() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+    IF TG_OP = 'INSERT' THEN
+        INSERT INTO chat_stats AS cs (account_id, chat_id, message_count,
+                                      last_date, last_message_id,
+                                      last_text, last_sender_id{column})
+        VALUES (NEW.account_id, NEW.chat_id, 1,
+                CASE WHEN NEW.date <= now() + interval '1 day'
+                     THEN NEW.date END,
+                CASE WHEN NEW.date <= now() + interval '1 day'
+                     THEN NEW.message_id END,
+                NEW.text, NEW.sender_id{value})
+        ON CONFLICT (account_id, chat_id) DO UPDATE SET
+            message_count = cs.message_count + 1,
+            last_date = CASE WHEN chat_stats_newer(cs, EXCLUDED)
+                        THEN EXCLUDED.last_date ELSE cs.last_date END,
+            last_message_id = CASE WHEN chat_stats_newer(cs, EXCLUDED)
+                              THEN EXCLUDED.last_message_id
+                              ELSE cs.last_message_id END,
+            last_text = CASE WHEN chat_stats_newer(cs, EXCLUDED)
+                        THEN EXCLUDED.last_text ELSE cs.last_text END,
+            last_sender_id = CASE WHEN chat_stats_newer(cs, EXCLUDED)
+                             THEN EXCLUDED.last_sender_id
+                             ELSE cs.last_sender_id END{merge};
+    ELSE
+        UPDATE chat_stats
+        SET last_text = NEW.text, last_sender_id = NEW.sender_id{update}
+        WHERE account_id = NEW.account_id
+          AND chat_id = NEW.chat_id
+          AND last_message_id = NEW.message_id;
+    END IF;
+    RETURN NULL;
+END;
+$$
+"""
+
+_DELETED_PARTS = {
+    "column": ", last_deleted",
+    "value": ", NEW.deleted_at IS NOT NULL",
+    "merge": (
+        ",\n            last_deleted = CASE WHEN chat_stats_newer(cs, EXCLUDED)"
+        " THEN EXCLUDED.last_deleted ELSE cs.last_deleted END"
+    ),
+    "update": ", last_deleted = NEW.deleted_at IS NOT NULL",
+    "deleted": " OR OLD.deleted_at IS DISTINCT FROM NEW.deleted_at",
+}
+_PLAIN_PARTS = dict.fromkeys(_DELETED_PARTS, "")
+
+_UPDATE_TRIGGER = (
+    "CREATE TRIGGER messages_chat_stats_update AFTER UPDATE ON messages "
+    "FOR EACH ROW WHEN (OLD.text IS DISTINCT FROM NEW.text "
+    "OR OLD.sender_id IS DISTINCT FROM NEW.sender_id{deleted}) "
+    "EXECUTE FUNCTION chat_stats_apply()"
+)
+
+_BACKFILL = """
+UPDATE chat_stats cs SET last_deleted = true
+FROM messages m
+WHERE m.account_id = cs.account_id AND m.chat_id = cs.chat_id
+  AND m.message_id = cs.last_message_id AND m.date = cs.last_date
+  AND m.deleted_at IS NOT NULL
+"""
+
+
+def upgrade() -> None:
+    op.execute(
+        "ALTER TABLE chat_stats ADD COLUMN last_deleted boolean NOT NULL DEFAULT false"
+    )
+    op.execute(_BACKFILL)
+    op.execute(_APPLY.format(**_DELETED_PARTS))
+    op.execute("DROP TRIGGER IF EXISTS messages_chat_stats_update ON messages")
+    op.execute(_UPDATE_TRIGGER.format(**_DELETED_PARTS))
+
+
+def downgrade() -> None:
+    op.execute("DROP TRIGGER IF EXISTS messages_chat_stats_update ON messages")
+    op.execute(_UPDATE_TRIGGER.format(**_PLAIN_PARTS))
+    op.execute(_APPLY.format(**_PLAIN_PARTS))
+    op.execute("ALTER TABLE chat_stats DROP COLUMN last_deleted")

backend/migrations/versions/f8c2d6e1a4b7_capture_reliability.py

@@ -0,0 +1,39 @@
+"""capture reliability
+
+Revision ID: f8c2d6e1a4b7
+Revises: c9e2a7f4b1d8
+Create Date: 2026-10-19 03:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "f8c2d6e1a4b7"
+down_revision: str | None = "c9e2a7f4b1d8"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "jobs", sa.Column("run_after", sa.DateTime(timezone=True), nullable=True)
+    )
+    op.execute(
+        "ALTER TABLE accounts "
+        "ADD COLUMN catchup_pts integer, "
+        "ADD COLUMN catchup_qts integer, "
+        "ADD COLUMN catchup_date integer"
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        "ALTER TABLE accounts "
+        "DROP COLUMN IF EXISTS catchup_pts, "
+        "DROP COLUMN IF EXISTS catchup_qts, "
+        "DROP COLUMN IF EXISTS catchup_date"
+    )
+    op.drop_column("jobs", "run_after")

backend/pyproject.toml

@@ -20,6 +20,7 @@ dependencies = [
     "dishka>=1.4.0",
     "greenlet>=3.1.0",
     "fastmcp>=3.3",
+    "httpx2>=2.13.1",
 ]
 
 [build-system]

backend/src/api/app.py

@@ -21,10 +21,12 @@ from api.routers import (
     avatars,
     backfill,
     bots,
+    capture,
     chats,
     custom_emoji,
     discover,
     events,
+    feed,
     files,
     folders,
     media,
@@ -38,7 +40,9 @@ from api.routers import (
     storage,
     stories,
     watches,
+    webhooks,
 )
+from api.webhooks import outbox
 from dependencies.container import container
 from utils.cache import DAY_HEADERS, IMMUTABLE_HEADERS, NO_STORE_HEADERS
 from utils.env import env
@@ -58,11 +62,14 @@ mcp_app = mcp.http_app(path="/")
 @asynccontextmanager
 async def lifespan(app_: Starlette) -> AsyncGenerator[None]:
     pool = await container.get(asyncpg.Pool)
+    hub.alert_listeners.append(outbox.wake)
     await hub.start(pool)
     purger = asyncio.create_task(
         purge_loop(pool, await container.get(ContentAddressedStorage))
     )
+    deliverer = asyncio.create_task(outbox.run(pool))
     yield
+    deliverer.cancel()
     purger.cancel()
     await hub.stop()
     await app_.state.dishka_container.close()
@@ -91,6 +98,7 @@ app.include_router(folders.router)
 app.include_router(backfill.router)
 app.include_router(search.router)
 app.include_router(chats.router)
+app.include_router(feed.router)
 app.include_router(media.router)
 app.include_router(avatars.router)
 app.include_router(custom_emoji.router)
@@ -103,9 +111,11 @@ app.include_router(peers.router)
 app.include_router(discover.router)
 app.include_router(annotations.router)
 app.include_router(watches.router)
+app.include_router(webhooks.router)
 app.include_router(shares.router)
 app.include_router(storage.router)
 app.include_router(files.router)
+app.include_router(capture.router)
 
 app.mount("/mcp", mcp_app)
 

backend/src/api/mcp/server.py

@@ -1,3 +1,4 @@
+import re
 from collections.abc import Sequence
 from datetime import datetime
 from typing import Any
@@ -20,12 +21,27 @@ from api.mcp.download import (
 from api.mcp.format import build_transcript, load_notes, load_photos, resolve_names
 from dependencies.container import container
 from utils.files import resolve_mime
-from utils.read import annotations, chats, media, peers, presence, social, watches
+from utils.read import (
+    annotations,
+    chats,
+    feed,
+    media,
+    peers,
+    people,
+    presence,
+    presence_stats,
+    social,
+    watches,
+)
 from utils.read.accounts import self_user_id, unknown_account_hint
+from utils.read.feed import CURSOR_PATTERN, FeedChatKind, FeedFilter, FeedKind
 from utils.read.models import DEFAULT_LIMIT, Page
-from utils.search.models import SearchFilters
+from utils.read.period import DEFAULT_TZ, MAX_DAYS, known_timezone
+from utils.read.read_receipts import read_info
+from utils.search.models import ChatKind, SearchFilters
 from utils.search.repository import search_messages
 from utils.storage import ContentAddressedStorage
+from utils.watch_rules import WatchKind, WatchParamsError, clean_name, normalize_params
 
 INSTRUCTIONS = """\
 beavergram archives Telegram data (chats, messages, media, presence, stories,
@@ -56,17 +72,25 @@ Typical flows:
   "📝 [your private note, NOT in Telegram]" are the user's own annotations
   attached locally via the web UI; they never existed in Telegram. Treat them
   as private notes from the user to you, not as chat content.
-- Find something: `search_messages_tool` (full-text over message text and STT
-  transcripts; supports chat/sender/date filters and regex).
-- Forensics: `get_deleted_messages` and `get_message_versions` recover content
-  removed or edited in Telegram but kept in the archive.
+- Find something: `search_messages_tool` (words by prefix over message text,
+  STT transcripts and earlier versions of edited messages; filters by chat,
+  sender, chat kind, media, deleted, edited, dates, regex).
+- People: `get_person` gives one person in full: profile changes, shared groups,
+  joins and leaves, deleted message count, the private chat.
+- Forensics: `get_feed` lists what was deleted or edited in Telegram across
+  all chats, newest first, with the text before and after each edit; a bulk
+  deletion is one item with `message_count`. Its `profile` kind lists name,
+  username and photo changes of people and renames of chats. `get_deleted_messages` and
+  `get_message_versions` recover the content of one chat or one message.
 - Files: `download_media` gets any message attachment (photo, video, voice,
   document, ...). It pulls the file from Telegram if the archive lacks it and
   returns a public link valid 24h that needs no auth (`curl -O <link>`), plus
   the content itself for small images and text files. `get_media` returns
   metadata only.
-- Monitoring: `set_watch` creates a local rule, `list_watches` lists rules, and
-  `list_alerts` reads what those rules fired.
+- Monitoring: `set_watch` creates a local rule (kinds `keyword`,
+  `new_message`, `message_deleted`, `message_edited`, `peer_online`; params in
+  its description), `list_watches` lists rules, and `list_alerts` reads what
+  those rules fired, with chat, message preview and a web client `path`.
 
 Watched bots:
 - a bot whose token the user added is just another archived account: it has its
@@ -76,7 +100,8 @@ Watched bots:
 Writes: everything is read-only except `set_watch` and the share links
 `download_media` creates (visible and revocable in the web UI).
 
-Paging: list/search tools accept `limit` and `offset`.
+Paging: list tools accept `limit` and `offset`; `search_messages_tool` returns
+`next_cursor` to pass back as `cursor`.
 """
 
 mcp: FastMCP = FastMCP("beavergram", instructions=INSTRUCTIONS)
@@ -115,27 +140,49 @@ async def search_messages_tool(
     query: str | None = None,
     chat_id: int | None = None,
     sender_id: int | None = None,
+    kind: ChatKind | None = None,
     has_media: bool | None = None,
+    deleted: bool | None = None,
+    edited: bool | None = None,
     date_from: datetime | None = None,
     date_to: datetime | None = None,
     regex: str | None = None,
+    cursor: str | None = None,
     limit: int = DEFAULT_LIMIT,
-    offset: int = 0,
-) -> list[dict[str, Any]]:
-    """Full-text search over message text and STT transcripts."""
+) -> dict[str, Any]:
+    """Search messages by words, newest first, including deleted ones.
+
+    Words match by prefix (3+ letters) and by stem, Russian and English.
+    `"exact phrase"`, `-word` and `or` work as in web search. Besides the
+    current text it searches voice transcripts and earlier versions of edited
+    messages: a hit with `source="version"` matched only an old text, given in
+    `version_text`. Every hit carries `deleted_at` and `edited_at`.
+
+    Filters: `chat_id`, `sender_id` (peer id), `kind` (private, group,
+    channel), `has_media`, `deleted` / `edited` (true for only those, false to
+    exclude them), `date_from` / `date_to`, `regex` over the current text. With
+    no `query` it lists messages that pass the filters.
+
+    Paging: pass `next_cursor` from the previous result as `cursor`; it is null
+    on the last page.
+    """
     filters = SearchFilters(
         account_id=account_id,
         query=query,
         chat_id=chat_id,
         sender_id=sender_id,
+        kind=kind,
         has_media=has_media,
+        deleted=deleted,
+        edited=edited,
         date_from=date_from,
         date_to=date_to,
         regex=regex,
+        cursor=cursor,
         limit=limit,
-        offset=offset,
     )
-    return _dump(await search_messages(await _pool(), filters))
+    page = await search_messages(await _pool(), filters)
+    return page.model_dump(mode="json", exclude={"terms"})
 
 
 @mcp.tool
@@ -238,6 +285,41 @@ async def get_deleted_messages(
     )
 
 
+@mcp.tool
+async def get_feed(
+    account_id: int,
+    kind: FeedKind = FeedKind.ALL,
+    since: datetime | None = None,
+    chat_id: int | None = None,
+    chat_kind: FeedChatKind | None = None,
+    limit: int = DEFAULT_LIMIT,
+    cursor: str | None = None,
+) -> dict[str, Any]:
+    """Deleted and edited messages and profile changes across chats, newest first.
+
+    `kind` picks deleted, edited, profile, messages (deleted and edited) or all;
+    `since` keeps items at or after that time; `chat_id` or `chat_kind` (dm,
+    group, channel) narrow it down. Edited items carry `before` and `after`
+    texts. A deleted item with `message_count` above 1 is a bulk deletion in
+    one chat; read its messages with `get_deleted_messages` for that chat.
+    Profile items cover people in private chats or contacts and chats in the
+    dialog list: `detail` maps `name`, `username`, `photo` (unique file ids) or
+    `title` to `[old, new]`, changes a day apart or closer are merged. Pass
+    `next_cursor` back as `cursor` for the next page.
+    """
+    if cursor is not None and re.fullmatch(CURSOR_PATTERN, cursor) is None:
+        msg = "cursor must be a next_cursor returned by get_feed"
+        raise ToolError(msg)
+    page = await feed.list_feed(
+        await _pool(),
+        account_id,
+        FeedFilter(kind=kind, chat_id=chat_id, chat_kind=chat_kind, since=since),
+        limit=limit,
+        cursor=cursor,
+    )
+    return page.model_dump(mode="json")
+
+
 @mcp.tool
 async def get_message_versions(
     account_id: int, chat_id: int, message_id: int
@@ -295,6 +377,23 @@ async def get_callbacks(
     return _dump(items)
 
 
+@mcp.tool
+async def get_read_info(
+    account_id: int, chat_id: int, message_id: int
+) -> dict[str, Any] | None:
+    """When the user's own message was read, from the archive only.
+
+    `observed_at` is when the archive saw the chat read up to this message,
+    `exact_at` the read date Telegram reported for a private chat, `readers`
+    who read it in a group and when, `listened_at` when a voice or round
+    video was played. `outcome` is the last Telegram answer: ok, not_read_yet,
+    user_privacy (they hide read time), your_privacy (the user hides theirs),
+    chat_too_big, too_old, unavailable, flood or error.
+    """
+    info = await read_info(await _pool(), account_id, chat_id, message_id)
+    return info.model_dump(mode="json") if info else None
+
+
 @mcp.tool
 async def presence_history(
     account_id: int,
@@ -317,12 +416,58 @@ async def presence_history(
     )
 
 
+@mcp.tool(name="presence_stats")
+async def presence_stats_tool(
+    account_id: int,
+    peer_id: int,
+    days: int = 30,
+    tz: str = DEFAULT_TZ,
+    limit: int = DEFAULT_LIMIT,
+    offset: int = 0,
+) -> dict[str, Any]:
+    """Time a peer spent online, from status pushes turned into online sessions.
+
+    `tz` is an IANA zone name; days and hours are counted in it. `week` holds
+    the average online minutes for each weekday (0 is Monday) and hour, null
+    where the archive has no data for that hour. `days` gives online seconds
+    per day, `gaps` the stretches when capture was down, `sessions` the latest
+    sessions (online stretches with breaks under 5 minutes), paged by
+    `limit`/`offset`. `hidden` means the peer hides their last seen time.
+    """
+    pool = await _pool()
+    if not 1 <= days <= MAX_DAYS:
+        msg = f"days must be between 1 and {MAX_DAYS}"
+        raise ToolError(msg)
+    if not await known_timezone(pool, tz):
+        msg = f"Unknown time zone: {tz}"
+        raise ToolError(msg)
+    stats = await presence_stats.presence_stats(
+        pool, account_id, peer_id, Page(limit=limit, offset=offset), days=days, tz=tz
+    )
+    return stats.model_dump(mode="json")
+
+
 @mcp.tool
 async def get_peer_history(account_id: int, peer_id: int) -> list[dict[str, Any]]:
     """Get name/username/avatar change history of a contact."""
     return _dump(await peers.get_peer_history(await _pool(), account_id, peer_id))
 
 
+@mcp.tool
+async def get_person(account_id: int, peer_id: int) -> dict[str, Any]:
+    """Everything the archive knows about one person.
+
+    Current profile, name/username/phone changes with times (newest first),
+    avatar history, groups shared in the archive with their message counts and
+    last message, joins and leaves with who added or removed them, the number
+    of their deleted messages, and the private chat if there is one.
+    """
+    pool = await _pool()
+    overview = await people.get_overview(pool, account_id, peer_id)
+    groups = await people.get_groups(pool, account_id, peer_id)
+    return {**overview.model_dump(mode="json"), **groups.model_dump(mode="json")}
+
+
 @mcp.tool
 async def get_stories(
     account_id: int,
@@ -361,13 +506,37 @@ async def get_annotations(
 @mcp.tool
 async def set_watch(
     account_id: int,
-    kind: str,
+    kind: WatchKind,
     params: dict[str, Any] | None = None,
+    name: str | None = None,
     enabled: bool = True,
 ) -> dict[str, Any]:
-    """Create a local watch rule (the only local write MCP is allowed)."""
+    """Create a local watch rule (the only local write MCP is allowed).
+
+    A watch fires an alert into the web client (live counter and list) and to
+    the user's webhooks. The user's own messages never fire. Kinds and params:
+
+    - `keyword`: `{"pattern": str, "regex": bool = false, "chat_id"?: int,
+      "sender_id"?: int}`. Case-insensitive substring, or a Python regex when
+      `regex` is true (at most 200 characters, no nested repeats like
+      `(a+)+`). Checked on new messages, on edits (the new text) and on voice
+      transcripts. Without `chat_id` it watches every chat.
+    - `new_message`: `{"chat_id"?: int, "sender_id"?: int}`, at least one: any
+      new message in that chat, or from that person anywhere, or both.
+    - `message_deleted`: `{"chat_id"?: int}`. A stored message deleted in that
+      chat; without `chat_id`, in any private chat.
+    - `message_edited`: `{"chat_id"?: int}`. Same scopes as `message_deleted`.
+    - `peer_online`: `{"peer_id": int}`. The person comes online after at
+      least 15 minutes away.
+
+    `name` is an optional human label shown in the list and in alerts.
+    """
+    try:
+        clean = normalize_params(kind, params or {})
+    except WatchParamsError as exc:
+        raise ToolError(str(exc)) from exc
     watch = await watches.create_watch(
-        await _pool(), account_id, kind, params or {}, enabled=enabled
+        await _pool(), account_id, kind, clean, name=clean_name(name), enabled=enabled
     )
     return watch.model_dump(mode="json")
 
@@ -382,12 +551,19 @@ async def list_watches(account_id: int) -> list[dict[str, Any]]:
 async def list_alerts(
     account_id: int,
     seen: bool | None = None,
+    before: int | None = None,
     limit: int = DEFAULT_LIMIT,
-    offset: int = 0,
-) -> list[dict[str, Any]]:
-    """List fired alerts from watch rules."""
-    return _dump(
-        await watches.list_alerts(
-            await _pool(), account_id, Page(limit=limit, offset=offset), seen=seen
-        )
+) -> dict[str, Any]:
+    """List fired alerts from watch rules, newest first.
+
+    Each alert names its watch (id, kind, name), the event (`message`, `edit`,
+    `transcript`, `delete`, `online`), the chat, the message preview with its
+    sender and deleted/edited flags, the peer for `peer_online`, and `path`,
+    the web client link. `seen=false` keeps unseen ones. Page older with
+    `before` set to `next_before` from the previous call; `unseen` is the
+    account's unseen count.
+    """
+    page = await watches.list_alerts(
+        await _pool(), account_id, seen=seen, before=before, limit=limit
     )
+    return page.model_dump(mode="json")

backend/src/api/realtime.py

@@ -1,14 +1,19 @@
 import asyncio
 import json
 import logging
-from typing import Any
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any
 
 import asyncpg
 
-from utils.env import env
 from utils.events import BG_EVENTS_CHANNEL
+from utils.listen import Listener
 from utils.read import chats as chats_read
 from utils.read import presence as presence_read
+from utils.read import watches as watches_read
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
 
 logger = logging.getLogger(__name__)
 
@@ -26,8 +31,15 @@ class EventHub:
     def __init__(self) -> None:
         self._subscribers: set[Subscriber] = set()
         self._pool: asyncpg.Pool | None = None
-        self._conn: asyncpg.Connection | None = None
+        self._listener = Listener(
+            "realtime",
+            {
+                BG_EVENTS_CHANNEL: self._on_notify,
+                watches_read.ALERTS_CHANGED_CHANNEL: self._on_alert,
+            },
+        )
         self._tasks: set[asyncio.Task] = set()
+        self.alert_listeners: list[Callable[[], None]] = []
 
     def subscribe(self, account_id: int, chat_id: int | None) -> Subscriber:
         sub = Subscriber(account_id, chat_id)
@@ -39,25 +51,51 @@ class EventHub:
 
     async def start(self, pool: asyncpg.Pool) -> None:
         self._pool = pool
-        conn = await asyncpg.connect(dsn=env.db.connection_url)
-        await conn.add_listener(BG_EVENTS_CHANNEL, self._on_notify)
-        self._conn = conn
-        logger.info("Realtime hub listening on %s", BG_EVENTS_CHANNEL)
+        await self._listener.start()
 
     async def stop(self) -> None:
         for task in self._tasks:
             task.cancel()
-        if self._conn is not None:
-            await self._conn.close()
-            self._conn = None
+        await self._listener.stop()
 
-    def _on_notify(
-        self, _conn: asyncpg.Connection, _pid: int, _channel: str, payload: str
-    ) -> None:
+    def _on_notify(self, payload: str) -> None:
         task = asyncio.create_task(self._dispatch(payload))
         self._tasks.add(task)
         task.add_done_callback(self._tasks.discard)
 
+    def _on_alert(self, payload: str) -> None:
+        for listener in self.alert_listeners:
+            listener()
+        task = asyncio.create_task(self._dispatch_alert(payload))
+        self._tasks.add(task)
+        task.add_done_callback(self._tasks.discard)
+
+    async def _dispatch_alert(self, payload: str) -> None:
+        try:
+            event = json.loads(payload)
+        except json.JSONDecodeError:
+            return
+        account_id = event.get("account_id")
+        targets = [
+            sub
+            for sub in self._subscribers
+            if sub.account_id == account_id and sub.chat_id is None
+        ]
+        if not targets or self._pool is None:
+            return
+        alert_id = event.get("alert_id")
+        item = await watches_read.get_alert(self._pool, alert_id) if alert_id else None
+        frame = {
+            "type": "alert",
+            "alert": item.model_dump(mode="json") if item else None,
+            "unseen": await watches_read.unseen_count(self._pool, account_id),
+        }
+        for sub in targets:
+            try:
+                sub.queue.put_nowait(frame)
+            except asyncio.QueueFull:
+                logger.warning("Dropping alert for slow subscriber")
+
     async def _dispatch(self, payload: str) -> None:
         try:
             event = json.loads(payload)
@@ -65,16 +103,12 @@ class EventHub:
             return
         account_id = event.get("account_id")
         chat_id = event.get("chat_id")
-        scoped = event.get("kind") == "presence"
+        account_wide = event.get("kind") == "presence"
         targets = [
             sub
             for sub in self._subscribers
             if sub.account_id == account_id
-            and (
-                sub.chat_id == chat_id
-                if scoped
-                else sub.chat_id is None or sub.chat_id == chat_id
-            )
+            and (account_wide or sub.chat_id is None or sub.chat_id == chat_id)
         ]
         if not targets:
             return
@@ -121,6 +155,7 @@ class EventHub:
                 "type": "receipt",
                 "chat_id": event["chat_id"],
                 "read_up_to": event["message_id"],
+                "read_at": datetime.now(UTC).isoformat(),
             }
         return None
 

backend/src/api/routers/analytics.py

@@ -4,8 +4,10 @@ import asyncpg
 from dishka.integrations.fastapi import DishkaRoute, FromDishka
 from fastapi import APIRouter, Query
 
+from api.timezone import checked_tz
 from utils.read import analytics
 from utils.read.models import ResponseStats, VolumeBucket
+from utils.read.period import DEFAULT_TZ, MAX_DAYS
 
 router = APIRouter(prefix="/api/analytics", tags=["analytics"], route_class=DishkaRoute)
 
@@ -18,9 +20,12 @@ async def volume(
     pool: FromDishka[asyncpg.Pool],
     account_id: AccountId,
     chat_id: ChatId,
-    days: Annotated[int, Query()] = 90,
+    days: Annotated[int, Query(ge=1, le=MAX_DAYS)] = 90,
+    tz: Annotated[str, Query()] = DEFAULT_TZ,
 ) -> list[VolumeBucket]:
-    return await analytics.message_volume(pool, account_id, chat_id, days=days)
+    return await analytics.message_volume(
+        pool, account_id, chat_id, days=days, tz=await checked_tz(pool, tz)
+    )
 
 
 @router.get("/response-time")

backend/src/api/routers/annotations.py

@@ -6,7 +6,7 @@ from fastapi import APIRouter, HTTPException, Query
 from pydantic import BaseModel
 
 from utils.read import annotations
-from utils.read.models import DEFAULT_LIMIT, AnnotationView, Page
+from utils.read.models import DEFAULT_LIMIT, AnnotationView, NoteTag, NoteView, Page
 
 router = APIRouter(
     prefix="/api/annotations", tags=["annotations"], route_class=DishkaRoute
@@ -42,6 +42,26 @@ async def list_annotations(
     )
 
 
+@router.get("/notes")
+async def list_notes(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: Annotated[int, Query()],
+    tag: Annotated[str | None, Query()] = None,
+    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
+    offset: Annotated[int, Query()] = 0,
+) -> list[NoteView]:
+    return await annotations.list_notes(
+        pool, account_id, Page(limit=limit, offset=offset), tag=tag
+    )
+
+
+@router.get("/tags")
+async def list_tags(
+    pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
+) -> list[NoteTag]:
+    return await annotations.list_tags(pool, account_id)
+
+
 @router.post("", status_code=201)
 async def create_annotation(
     pool: FromDishka[asyncpg.Pool], body: AnnotationCreate

backend/src/api/routers/capture.py

@@ -0,0 +1,32 @@
+from datetime import datetime
+from typing import Annotated
+
+import asyncpg
+from dishka.integrations.fastapi import DishkaRoute, FromDishka
+from fastapi import APIRouter, Query
+
+from utils.read import capture
+from utils.read.models import CaptureGap, CaptureStatus
+
+router = APIRouter(prefix="/api/capture", tags=["capture"], route_class=DishkaRoute)
+
+AccountId = Annotated[int, Query()]
+
+
+@router.get("/status")
+async def capture_status(
+    pool: FromDishka[asyncpg.Pool], account_id: AccountId
+) -> CaptureStatus:
+    return await capture.capture_status(pool, account_id)
+
+
+@router.get("/gaps")
+async def capture_gaps(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: AccountId,
+    date_from: Annotated[datetime | None, Query()] = None,
+    date_to: Annotated[datetime | None, Query()] = None,
+) -> list[CaptureGap]:
+    return await capture.capture_gaps(
+        pool, account_id, date_from=date_from, date_to=date_to
+    )

backend/src/api/routers/feed.py

@@ -0,0 +1,70 @@
+from datetime import datetime
+from typing import Annotated
+
+import asyncpg
+from dishka.integrations.fastapi import DishkaRoute, FromDishka
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel
+
+from utils.read import feed
+from utils.read.feed import CURSOR_PATTERN, FeedChatKind, FeedFilter, FeedKind
+from utils.read.models import DEFAULT_LIMIT, FeedMessagesPage, FeedPage, FeedUnseen
+
+router = APIRouter(prefix="/api/feed", tags=["feed"], route_class=DishkaRoute)
+
+AccountId = Annotated[int, Query()]
+Limit = Annotated[int, Query(ge=1)]
+Cursor = Annotated[str | None, Query(pattern=CURSOR_PATTERN)]
+
+
+class SeenRequest(BaseModel):
+    account_id: int
+    at: datetime | None = None
+
+
+@router.get("")
+async def list_feed(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: AccountId,
+    kind: Annotated[FeedKind, Query()] = FeedKind.ALL,
+    chat_id: Annotated[int | None, Query()] = None,
+    chat_kind: Annotated[FeedChatKind | None, Query()] = None,
+    since: Annotated[datetime | None, Query()] = None,
+    cursor: Cursor = None,
+    limit: Limit = DEFAULT_LIMIT,
+) -> FeedPage:
+    return await feed.list_feed(
+        pool,
+        account_id,
+        FeedFilter(kind=kind, chat_id=chat_id, chat_kind=chat_kind, since=since),
+        limit=limit,
+        cursor=cursor,
+    )
+
+
+@router.get("/unseen")
+async def feed_unseen(
+    pool: FromDishka[asyncpg.Pool], account_id: AccountId
+) -> FeedUnseen:
+    return await feed.unseen(pool, account_id)
+
+
+@router.post("/seen")
+async def feed_seen(pool: FromDishka[asyncpg.Pool], body: SeenRequest) -> FeedUnseen:
+    return await feed.mark_seen(pool, body.account_id, body.at)
+
+
+@router.get("/{item_id}/messages")
+async def feed_item_messages(
+    pool: FromDishka[asyncpg.Pool],
+    item_id: int,
+    account_id: AccountId,
+    cursor: Cursor = None,
+    limit: Limit = DEFAULT_LIMIT,
+) -> FeedMessagesPage:
+    page = await feed.item_messages(
+        pool, account_id, item_id, limit=limit, cursor=cursor
+    )
+    if page is None:
+        raise HTTPException(status_code=404, detail="Feed item not found")
+    return page

backend/src/api/routers/peers.py

@@ -4,8 +4,14 @@ import asyncpg
 from dishka.integrations.fastapi import DishkaRoute, FromDishka
 from fastapi import APIRouter, HTTPException, Query
 
-from utils.read import peers
-from utils.read.models import PeerHistoryView, PeerView
+from utils.read import peers, people
+from utils.read.models import (
+    PeerGroups,
+    PeerHistoryView,
+    PeerOverview,
+    PeerView,
+    ProfileChange,
+)
 
 router = APIRouter(prefix="/api", tags=["peers"], route_class=DishkaRoute)
 
@@ -35,3 +41,24 @@ async def peer_history(
     pool: FromDishka[asyncpg.Pool], peer_id: int, account_id: AccountId
 ) -> list[PeerHistoryView]:
     return await peers.get_peer_history(pool, account_id, peer_id)
+
+
+@router.get("/peers/{peer_id}/overview")
+async def peer_overview(
+    pool: FromDishka[asyncpg.Pool], peer_id: int, account_id: AccountId
+) -> PeerOverview:
+    return await people.get_overview(pool, account_id, peer_id)
+
+
+@router.get("/peers/{peer_id}/groups")
+async def peer_groups(
+    pool: FromDishka[asyncpg.Pool], peer_id: int, account_id: AccountId
+) -> PeerGroups:
+    return await people.get_groups(pool, account_id, peer_id)
+
+
+@router.get("/chats/{chat_id}/changes")
+async def chat_changes(
+    pool: FromDishka[asyncpg.Pool], chat_id: int, account_id: AccountId
+) -> list[ProfileChange]:
+    return await people.get_chat_changes(pool, account_id, chat_id)

backend/src/api/routers/presence.py

@@ -5,8 +5,18 @@ import asyncpg
 from dishka.integrations.fastapi import DishkaRoute, FromDishka
 from fastapi import APIRouter, Query
 
-from utils.read import presence
-from utils.read.models import DEFAULT_LIMIT, Page, PresenceHourly, PresenceSample
+from api.timezone import checked_tz
+from utils.read import presence, presence_stats
+from utils.read.models import (
+    DEFAULT_LIMIT,
+    MAX_LIMIT,
+    Page,
+    PresenceHourly,
+    PresenceSample,
+    PresenceSession,
+    PresenceStats,
+)
+from utils.read.period import DEFAULT_TZ, MAX_DAYS
 
 router = APIRouter(prefix="/api/presence", tags=["presence"], route_class=DishkaRoute)
 
@@ -14,6 +24,8 @@ AccountId = Annotated[int, Query()]
 PeerId = Annotated[int, Query()]
 DateFrom = Annotated[datetime | None, Query()]
 DateTo = Annotated[datetime | None, Query()]
+Days = Annotated[int, Query(ge=1, le=MAX_DAYS)]
+Tz = Annotated[str, Query()]
 
 
 @router.get("")
@@ -43,6 +55,16 @@ async def current_presence(
     return await presence.current_presence(pool, account_id, peer_id)
 
 
+@router.get("/online")
+async def online_now(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: AccountId,
+    peer_ids: Annotated[str, Query()],
+) -> list[PresenceSample]:
+    parsed = [int(part) for part in peer_ids.split(",") if part.strip()]
+    return await presence.online_now(pool, account_id, parsed[:MAX_LIMIT])
+
+
 @router.get("/hourly")
 async def presence_hourly(
     pool: FromDishka[asyncpg.Pool],
@@ -54,3 +76,43 @@ async def presence_hourly(
     return await presence.presence_hourly(
         pool, account_id, peer_id, date_from=date_from, date_to=date_to
     )
+
+
+@router.get("/stats")
+async def stats(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: AccountId,
+    peer_id: PeerId,
+    days: Days = 30,
+    tz: Tz = DEFAULT_TZ,
+    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
+    offset: Annotated[int, Query()] = 0,
+) -> PresenceStats:
+    return await presence_stats.presence_stats(
+        pool,
+        account_id,
+        peer_id,
+        Page(limit=limit, offset=offset),
+        days=days,
+        tz=await checked_tz(pool, tz),
+    )
+
+
+@router.get("/sessions")
+async def sessions(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: AccountId,
+    peer_id: PeerId,
+    days: Days = 30,
+    tz: Tz = DEFAULT_TZ,
+    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
+    offset: Annotated[int, Query()] = 0,
+) -> list[PresenceSession]:
+    return await presence_stats.presence_sessions(
+        pool,
+        account_id,
+        peer_id,
+        Page(limit=limit, offset=offset),
+        days=days,
+        tz=await checked_tz(pool, tz),
+    )

backend/src/api/routers/search.py

@@ -4,7 +4,7 @@ import asyncpg
 from dishka.integrations.fastapi import DishkaRoute, FromDishka
 from fastapi import APIRouter, Query
 
-from utils.search.models import SearchFilters, SearchHit
+from utils.search.models import SearchFilters, SearchPage
 from utils.search.repository import search_messages
 
 router = APIRouter(prefix="/api/search", tags=["search"], route_class=DishkaRoute)
@@ -13,5 +13,5 @@ router = APIRouter(prefix="/api/search", tags=["search"], route_class=DishkaRout
 @router.get("")
 async def search(
     pool: FromDishka[asyncpg.Pool], filters: Annotated[SearchFilters, Query()]
-) -> list[SearchHit]:
+) -> SearchPage:
     return await search_messages(pool, filters)

backend/src/api/routers/social.py

@@ -2,10 +2,13 @@ from typing import Annotated
 
 import asyncpg
 from dishka.integrations.fastapi import DishkaRoute, FromDishka
-from fastapi import APIRouter, Query
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel
 
+from utils.jobs import enqueue
 from utils.read import social
-from utils.read.models import CallbackView, LinkView, ReactionView
+from utils.read.models import CallbackView, LinkView, ReactionView, ReadInfoView
+from utils.read.read_receipts import read_info
 
 router = APIRouter(
     prefix="/api/messages/{chat_id}/{message_id}",
@@ -16,6 +19,10 @@ router = APIRouter(
 AccountId = Annotated[int, Query()]
 
 
+class RefreshRequest(BaseModel):
+    account_id: int
+
+
 @router.get("/callbacks")
 async def message_callbacks(
     pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, account_id: AccountId
@@ -35,3 +42,29 @@ async def message_links(
     pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, account_id: AccountId
 ) -> list[LinkView]:
     return await social.get_links(pool, account_id, chat_id, message_id)
+
+
+@router.get("/read")
+async def message_read(
+    pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, account_id: AccountId
+) -> ReadInfoView:
+    info = await read_info(pool, account_id, chat_id, message_id)
+    if info is None:
+        raise HTTPException(status_code=404, detail="message not found")
+    return info
+
+
+@router.post("/read/refresh")
+async def refresh_message_read(
+    pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, body: RefreshRequest
+) -> dict[str, int | None]:
+    info = await read_info(pool, body.account_id, chat_id, message_id)
+    if info is None or not info.can_fetch:
+        return {"job_id": None}
+    job_id = await enqueue(
+        pool,
+        body.account_id,
+        "fetch_read_info",
+        {"chat_id": chat_id, "message_id": message_id},
+    )
+    return {"job_id": job_id}

backend/src/api/routers/storage.py

@@ -13,6 +13,7 @@ from utils.read import storage as storage_read
 from utils.read.models import (
     DEFAULT_LIMIT,
     Page,
+    StorageChatView,
     StorageFileView,
     StorageScopeView,
     StorageSummaryView,
@@ -47,9 +48,11 @@ class PurgeRequest(BaseModel):
 
 @router.get("/summary")
 async def summary(
-    pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
+    pool: FromDishka[asyncpg.Pool],
+    account_id: Annotated[int, Query()],
+    chat_id: Annotated[int | None, Query()] = None,
 ) -> StorageSummaryView:
-    return await storage_read.get_summary(pool, account_id)
+    return await storage_read.get_summary(pool, account_id, chat_id)
 
 
 @router.get("/files")
@@ -57,11 +60,24 @@ async def files(
     pool: FromDishka[asyncpg.Pool],
     account_id: Annotated[int, Query()],
     group: Annotated[str | None, Query()] = None,
+    chat_id: Annotated[int | None, Query()] = None,
     limit: Annotated[int, Query()] = DEFAULT_LIMIT,
     offset: Annotated[int, Query()] = 0,
 ) -> list[StorageFileView]:
     page = Page(limit=limit, offset=offset)
-    return await storage_read.list_files(pool, account_id, group, page)
+    return await storage_read.list_files(pool, account_id, group, page, chat_id)
+
+
+@router.get("/chats")
+async def chats(
+    pool: FromDishka[asyncpg.Pool],
+    account_id: Annotated[int, Query()],
+    group: Annotated[str | None, Query()] = None,
+    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
+    offset: Annotated[int, Query()] = 0,
+) -> list[StorageChatView]:
+    page = Page(limit=limit, offset=offset)
+    return await storage_read.list_chats(pool, account_id, group, page)
 
 
 async def _chat_ids(

backend/src/api/routers/stories.py

@@ -5,7 +5,7 @@ from dishka.integrations.fastapi import DishkaRoute, FromDishka
 from fastapi import APIRouter, HTTPException, Query
 from fastapi.responses import FileResponse
 
-from utils.files import content_disposition, story_file_name
+from utils.files import content_disposition, sniff_media_kind, story_file_name
 from utils.read import peers
 from utils.read.models import DEFAULT_LIMIT, Page, StoryView
 from utils.storage import ContentAddressedStorage
@@ -17,17 +17,34 @@ AccountId = Annotated[int, Query()]
 _STORY_MIME = {"photo": "image/jpeg", "video": "video/mp4"}
 
 
+async def _with_kind(
+    pool: asyncpg.Pool,
+    storage: ContentAddressedStorage,
+    account_id: int,
+    story: StoryView,
+) -> StoryView:
+    if story.media_kind or not story.storage_key:
+        return story
+    if not storage.exists(story.storage_key):
+        return story
+    kind = sniff_media_kind(storage.head(story.storage_key))
+    await peers.set_story_kind(pool, account_id, story.peer_id, story.story_id, kind)
+    return story.model_copy(update={"media_kind": kind})
+
+
 @router.get("/stories")
 async def list_stories(
     pool: FromDishka[asyncpg.Pool],
+    storage: FromDishka[ContentAddressedStorage],
     account_id: AccountId,
     peer_id: Annotated[int | None, Query()] = None,
     limit: Annotated[int, Query()] = DEFAULT_LIMIT,
     offset: Annotated[int, Query()] = 0,
 ) -> list[StoryView]:
-    return await peers.get_stories(
+    stories = await peers.get_stories(
         pool, account_id, Page(limit=limit, offset=offset), peer_id=peer_id
     )
+    return [await _with_kind(pool, storage, account_id, story) for story in stories]
 
 
 @router.get("/stories/{peer_id}/{story_id}/media")

backend/src/api/routers/watches.py

@@ -6,55 +6,88 @@ from fastapi import APIRouter, HTTPException, Query
 from pydantic import BaseModel
 
 from utils.read import watches
-from utils.read.models import DEFAULT_LIMIT, AlertView, Page, WatchView
+from utils.read.models import AlertPage, WatchItem, WatchView
+from utils.watch_rules import WatchParamsError, clean_name, normalize_params
 
 router = APIRouter(prefix="/api", tags=["watches"], route_class=DishkaRoute)
 
 
-class WatchCreate(BaseModel):
+class WatchUpdate(BaseModel):
+    kind: str | None = None
+    params: dict[str, Any] = {}
+    name: str | None = None
+    enabled: bool = True
+
+
+class WatchCreate(WatchUpdate):
     account_id: int
     kind: str
-    params: dict[str, Any] = {}
-    enabled: bool = True
 
 
-class WatchUpdate(BaseModel):
-    params: dict[str, Any] = {}
-    enabled: bool = True
+class SeenAll(BaseModel):
+    account_id: int
+    up_to: int | None = None
+
+
+def _params(kind: str, params: dict[str, Any]) -> dict[str, Any]:
+    try:
+        return normalize_params(kind, params)
+    except WatchParamsError as exc:
+        raise HTTPException(status_code=422, detail=str(exc)) from exc
+
+
+async def _item(pool: asyncpg.Pool, watch: WatchView) -> WatchItem:
+    return (await watches.describe_watches(pool, watch.account_id, [watch]))[0]
 
 
 @router.get("/watches")
 async def list_watches(
     pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
-) -> list[WatchView]:
-    return await watches.list_watches(pool, account_id)
+) -> list[WatchItem]:
+    rows = await watches.list_watches(pool, account_id)
+    return await watches.describe_watches(pool, account_id, rows)
 
 
 @router.post("/watches", status_code=201)
-async def create_watch(pool: FromDishka[asyncpg.Pool], body: WatchCreate) -> WatchView:
-    return await watches.create_watch(
-        pool, body.account_id, body.kind, body.params, enabled=body.enabled
+async def create_watch(pool: FromDishka[asyncpg.Pool], body: WatchCreate) -> WatchItem:
+    watch = await watches.create_watch(
+        pool,
+        body.account_id,
+        body.kind,
+        _params(body.kind, body.params),
+        name=clean_name(body.name),
+        enabled=body.enabled,
     )
+    return await _item(pool, watch)
 
 
 @router.get("/watches/{watch_id}")
-async def get_watch(pool: FromDishka[asyncpg.Pool], watch_id: int) -> WatchView:
+async def get_watch(pool: FromDishka[asyncpg.Pool], watch_id: int) -> WatchItem:
     watch = await watches.get_watch(pool, watch_id)
     if watch is None:
         raise HTTPException(status_code=404, detail="watch not found")
-    return watch
+    return await _item(pool, watch)
 
 
 @router.put("/watches/{watch_id}")
 async def update_watch(
     pool: FromDishka[asyncpg.Pool], watch_id: int, body: WatchUpdate
-) -> WatchView:
+) -> WatchItem:
+    current = await watches.get_watch(pool, watch_id)
+    if current is None:
+        raise HTTPException(status_code=404, detail="watch not found")
+    kind = body.kind or current.kind
     watch = await watches.update_watch(
-        pool, watch_id, body.params, enabled=body.enabled
+        pool,
+        watch_id,
+        kind,
+        _params(kind, body.params),
+        name=clean_name(body.name),
+        enabled=body.enabled,
     )
     if watch is None:
         raise HTTPException(status_code=404, detail="watch not found")
-    return watch
+    return await _item(pool, watch)
 
 
 @router.delete("/watches/{watch_id}", status_code=204)
@@ -68,14 +101,29 @@ async def list_alerts(
     pool: FromDishka[asyncpg.Pool],
     account_id: Annotated[int, Query()],
     seen: Annotated[bool | None, Query()] = None,
-    limit: Annotated[int, Query()] = DEFAULT_LIMIT,
-    offset: Annotated[int, Query()] = 0,
-) -> list[AlertView]:
+    before: Annotated[int | None, Query()] = None,
+    limit: Annotated[int, Query()] = watches.ALERT_PAGE,
+) -> AlertPage:
     return await watches.list_alerts(
-        pool, account_id, Page(limit=limit, offset=offset), seen=seen
+        pool, account_id, seen=seen, before=before, limit=limit
     )
 
 
+@router.get("/alerts/unseen")
+async def unseen_alerts(
+    pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
+) -> dict[str, int]:
+    return {"unseen": await watches.unseen_count(pool, account_id)}
+
+
+@router.post("/alerts/seen")
+async def mark_all_seen(
+    pool: FromDishka[asyncpg.Pool], body: SeenAll
+) -> dict[str, int]:
+    marked = await watches.mark_all_seen(pool, body.account_id, body.up_to)
+    return {"marked": marked}
+
+
 @router.post("/alerts/{alert_id}/seen", status_code=204)
 async def mark_alert_seen(pool: FromDishka[asyncpg.Pool], alert_id: int) -> None:
     if not await watches.mark_alert_seen(pool, alert_id):

backend/src/api/routers/webhooks.py

@@ -0,0 +1,121 @@
+from typing import Annotated
+from urllib.parse import urlsplit
+
+import asyncpg
+from dishka.integrations.fastapi import DishkaRoute, FromDishka
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel
+
+from api.webhooks import send_test
+from utils.read import webhooks
+from utils.read.models import DeliveryView, WebhookView
+
+router = APIRouter(prefix="/api/webhooks", tags=["webhooks"], route_class=DishkaRoute)
+
+MAX_URL = 2048
+
+
+class WebhookUpdate(BaseModel):
+    url: str
+    watch_ids: list[int] | None = None
+    enabled: bool = True
+
+
+class WebhookCreate(WebhookUpdate):
+    account_id: int
+
+
+def _url(value: str) -> str:
+    url = value.strip()
+    parts = urlsplit(url)
+    if (
+        len(url) > MAX_URL
+        or parts.scheme not in {"http", "https"}
+        or not parts.hostname
+    ):
+        raise HTTPException(
+            status_code=422, detail="url must be an http or https address"
+        )
+    return url
+
+
+def _watch_ids(ids: list[int] | None) -> list[int] | None:
+    return sorted(set(ids)) if ids else None
+
+
+async def _webhook(pool: asyncpg.Pool, webhook_id: int) -> WebhookView:
+    webhook = await webhooks.get_webhook(pool, webhook_id)
+    if webhook is None:
+        raise HTTPException(status_code=404, detail="webhook not found")
+    return webhook
+
+
+@router.get("")
+async def list_webhooks(
+    pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
+) -> list[WebhookView]:
+    return await webhooks.list_webhooks(pool, account_id)
+
+
+@router.post("", status_code=201)
+async def create_webhook(
+    pool: FromDishka[asyncpg.Pool], body: WebhookCreate
+) -> WebhookView:
+    webhook = await webhooks.create_webhook(
+        pool,
+        body.account_id,
+        _url(body.url),
+        _watch_ids(body.watch_ids),
+        enabled=body.enabled,
+    )
+    if webhook is None:
+        raise HTTPException(status_code=500, detail="webhook was not saved")
+    return webhook
+
+
+@router.put("/{webhook_id}")
+async def update_webhook(
+    pool: FromDishka[asyncpg.Pool], webhook_id: int, body: WebhookUpdate
+) -> WebhookView:
+    webhook = await webhooks.update_webhook(
+        pool,
+        webhook_id,
+        _url(body.url),
+        _watch_ids(body.watch_ids),
+        enabled=body.enabled,
+    )
+    if webhook is None:
+        raise HTTPException(status_code=404, detail="webhook not found")
+    return webhook
+
+
+@router.delete("/{webhook_id}", status_code=204)
+async def delete_webhook(pool: FromDishka[asyncpg.Pool], webhook_id: int) -> None:
+    if not await webhooks.delete_webhook(pool, webhook_id):
+        raise HTTPException(status_code=404, detail="webhook not found")
+
+
+@router.post("/{webhook_id}/secret")
+async def rotate_secret(pool: FromDishka[asyncpg.Pool], webhook_id: int) -> WebhookView:
+    webhook = await webhooks.rotate_secret(pool, webhook_id)
+    if webhook is None:
+        raise HTTPException(status_code=404, detail="webhook not found")
+    return webhook
+
+
+@router.post("/{webhook_id}/test")
+async def test_webhook(pool: FromDishka[asyncpg.Pool], webhook_id: int) -> DeliveryView:
+    delivery = await send_test(pool, await _webhook(pool, webhook_id))
+    if delivery is None:
+        raise HTTPException(status_code=500, detail="test delivery was not recorded")
+    return delivery
+
+
+@router.get("/{webhook_id}/deliveries")
+async def list_deliveries(
+    pool: FromDishka[asyncpg.Pool],
+    webhook_id: int,
+    limit: Annotated[int, Query(ge=1, le=100)] = webhooks.DELIVERY_LOG,
+) -> list[DeliveryView]:
+    await _webhook(pool, webhook_id)
+    return await webhooks.list_deliveries(pool, webhook_id, limit)

backend/src/api/timezone.py

@@ -0,0 +1,10 @@
+import asyncpg
+from fastapi import HTTPException
+
+from utils.read.period import known_timezone
+
+
+async def checked_tz(pool: asyncpg.Pool, tz: str) -> str:
+    if not await known_timezone(pool, tz):
+        raise HTTPException(status_code=422, detail=f"Unknown time zone: {tz}")
+    return tz

backend/src/api/webhooks.py

@@ -0,0 +1,191 @@
+import asyncio
+import contextlib
+import hashlib
+import hmac
+import json
+import logging
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+import asyncpg
+import httpx2
+
+from utils.env import env
+from utils.read import watches
+from utils.read import webhooks as webhooks_read
+from utils.read.models import AlertItem, DeliveryView, WebhookView
+
+logger = logging.getLogger(__name__)
+
+TIMEOUT_SECONDS = 10
+SCAN_SECONDS = 30
+BATCH = 20
+RETRY_DELAYS = (30, 90, 270, 810, 2430)
+ERROR_CHARS = 300
+USER_AGENT = "beavergram-webhooks"
+
+
+def sign(secret: str, body: bytes) -> str:
+    digest = hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
+    return f"sha256={digest}"
+
+
+def _encode(body: dict[str, Any]) -> str:
+    return json.dumps(body, ensure_ascii=False, separators=(",", ":"))
+
+
+def _with_url(body: dict[str, Any], path: str) -> dict[str, Any]:
+    if env.api.public_url:
+        body["url"] = env.api.public_url.rstrip("/") + path
+    return body
+
+
+def alert_body(item: AlertItem) -> dict[str, Any]:
+    data = item.model_dump(mode="json", exclude={"id", "seen"})
+    return _with_url({"type": "alert", "alert_id": item.id, **data}, item.path)
+
+
+def test_body(webhook: WebhookView) -> dict[str, Any]:
+    body = {
+        "type": "test",
+        "alert_id": None,
+        "account_id": webhook.account_id,
+        "ts": datetime.now(UTC).isoformat(),
+        "event": "test",
+        "watch": None,
+        "chat": None,
+        "message": None,
+        "peer": None,
+        "path": "/app",
+    }
+    return _with_url(body, "/app")
+
+
+@dataclass(frozen=True, slots=True)
+class Target:
+    delivery_id: int
+    event: str
+    url: str
+    secret: str
+
+
+async def post(
+    client: httpx2.AsyncClient, target: Target, body: str
+) -> tuple[int | None, str | None]:
+    raw = body.encode()
+    headers = {
+        "Content-Type": "application/json",
+        "User-Agent": USER_AGENT,
+        "X-Beavergram-Event": target.event,
+        "X-Beavergram-Delivery": str(target.delivery_id),
+        "X-Beavergram-Signature": sign(target.secret, raw),
+    }
+    try:
+        response = await client.post(target.url, content=raw, headers=headers)
+    except httpx2.HTTPError as exc:
+        return None, (str(exc) or type(exc).__name__)[:ERROR_CHARS]
+    if response.is_success:
+        return response.status_code, None
+    reason = response.text.strip()[:ERROR_CHARS] or response.reason_phrase
+    return response.status_code, reason or f"HTTP {response.status_code}"
+
+
+def new_client() -> httpx2.AsyncClient:
+    return httpx2.AsyncClient(timeout=TIMEOUT_SECONDS, follow_redirects=False)
+
+
+async def _body(pool: asyncpg.Pool, row: asyncpg.Record) -> str | None:
+    if row["body"] is not None:
+        return row["body"]
+    if row["alert_id"] is None:
+        return None
+    item = await watches.get_alert(pool, row["alert_id"])
+    if item is None:
+        return None
+    body = _encode(alert_body(item))
+    await webhooks_read.save_body(pool, row["id"], body)
+    return body
+
+
+async def _deliver(
+    pool: asyncpg.Pool, client: httpx2.AsyncClient, row: asyncpg.Record
+) -> None:
+    body = await _body(pool, row)
+    if body is None:
+        await webhooks_read.finish(
+            pool, row["id"], webhooks_read.STATUS_FAILED, None, "alert is gone"
+        )
+        return
+    target = Target(row["id"], row["event"], row["url"], row["secret"])
+    code, error = await post(client, target, body)
+    if error is None:
+        await webhooks_read.finish(
+            pool, row["id"], webhooks_read.STATUS_DELIVERED, code, None
+        )
+        return
+    attempt = row["attempts"]
+    if attempt > len(RETRY_DELAYS):
+        await webhooks_read.finish(
+            pool, row["id"], webhooks_read.STATUS_FAILED, code, error
+        )
+        return
+    retry_at = datetime.now(UTC) + timedelta(seconds=RETRY_DELAYS[attempt - 1])
+    await webhooks_read.finish(
+        pool, row["id"], webhooks_read.STATUS_PENDING, code, error, retry_at
+    )
+
+
+async def send_test(pool: asyncpg.Pool, webhook: WebhookView) -> DeliveryView | None:
+    body = _encode(test_body(webhook))
+    delivery_id = await webhooks_read.start_test(pool, webhook.id, body)
+    async with new_client() as client:
+        target = Target(delivery_id, "test", webhook.url, webhook.secret)
+        code, error = await post(client, target, body)
+    status = webhooks_read.STATUS_FAILED if error else webhooks_read.STATUS_DELIVERED
+    await webhooks_read.finish(pool, delivery_id, status, code, error)
+    return await webhooks_read.get_delivery(pool, delivery_id)
+
+
+class Outbox:
+    def __init__(self) -> None:
+        self._wake = asyncio.Event()
+
+    def wake(self) -> None:
+        self._wake.set()
+
+    async def run(self, pool: asyncpg.Pool) -> None:
+        released = False
+        async with new_client() as client:
+            while True:
+                self._wake.clear()
+                try:
+                    if not released:
+                        await webhooks_read.release_stuck(pool)
+                        released = True
+                    await self._drain(pool, client)
+                    delay = await self._delay(pool)
+                except Exception:
+                    logger.exception("Webhook delivery failed")
+                    delay = SCAN_SECONDS
+                with contextlib.suppress(TimeoutError):
+                    await asyncio.wait_for(self._wake.wait(), timeout=delay)
+
+    async def _drain(self, pool: asyncpg.Pool, client: httpx2.AsyncClient) -> None:
+        while rows := await webhooks_read.claim_due(pool, BATCH):
+            results = await asyncio.gather(
+                *(_deliver(pool, client, row) for row in rows), return_exceptions=True
+            )
+            for result in results:
+                if isinstance(result, Exception):
+                    logger.error("Webhook delivery crashed: %r", result)
+
+    async def _delay(self, pool: asyncpg.Pool) -> float:
+        due = await webhooks_read.next_due(pool)
+        if due is None:
+            return SCAN_SECONDS
+        wait = (due - datetime.now(UTC)).total_seconds()
+        return min(SCAN_SECONDS, max(wait, 1.0))
+
+
+outbox = Outbox()

backend/src/userbot/handlers/deletes.py

@@ -20,11 +20,16 @@ async def on_deleted_messages(client: PyroClient, messages: list[Message]) -> No
             box.append(message.id)
         else:
             channels.setdefault(chat_id, []).append(message.id)
+    ctx.pending_deletes.remember(None, box)
+    for chat_id, ids in channels.items():
+        ctx.pending_deletes.remember(chat_id, ids)
     if box:
         await repository.mark_deleted_box(ctx.pool, ctx.account_id, box)
+        await ctx.watches.on_deleted(None, box)
         await notify_bg_event(ctx.pool, "delete", ctx.account_id, message_ids=box)
     for chat_id, ids in channels.items():
         await repository.mark_deleted_channel(ctx.pool, ctx.account_id, chat_id, ids)
+        await ctx.watches.on_deleted(chat_id, ids)
         await notify_bg_event(
             ctx.pool, "delete", ctx.account_id, chat_id=chat_id, message_ids=ids
         )

backend/src/userbot/handlers/edits.py

@@ -2,10 +2,10 @@ from pyrogram.types import Message
 
 from userbot.modules.capture import repository
 from userbot.modules.capture.chat_meta import meta_from_chat
-from userbot.modules.capture.message import sender_id
+from userbot.modules.capture.message import message_row
 from userbot.modules.client import PyroClient
-from userbot.modules.media import capture_media, media_unique_id, self_destruct_ttl
-from userbot.modules.raw_json import raw_json
+from userbot.modules.media import capture_media
+from userbot.modules.watches.seen import seen_message
 from utils.events import notify_bg_event
 
 
@@ -23,22 +23,12 @@ async def on_edited_message(client: PyroClient, message: Message) -> None:
     chat = message.chat
     chat_id = chat.id or 0
     meta = meta_from_chat(chat, ctx.contacts.ids)
+    await ctx.watches.on_edit(seen_message(chat_id, message))
     toggles = ctx.resolve(meta)
-    if not toggles.track_edits_deletes:
+    if not (toggles.messages and toggles.track_edits_deletes):
         return
     changed = await repository.add_version(
-        ctx.pool,
-        ctx.account_id,
-        chat_id,
-        message.id,
-        message.date,
-        sender_id(message),
-        message.text or message.caption,
-        raw_json(message),
-        message.edit_date,
-        media_unique_id(message),
-        has_media=message.media is not None,
-        is_self_destruct=self_destruct_ttl(message) is not None,
+        ctx.pool, ctx.account_id, message_row(message, chat_id, message.date)
     )
     if not changed:
         return

backend/src/userbot/handlers/messages.py

@@ -5,6 +5,7 @@ from userbot.modules.capture.chat_meta import meta_from_chat
 from userbot.modules.client import PyroClient
 from userbot.modules.stt import is_transcribable
 from userbot.modules.stt.gate import safe_transcribe
+from userbot.modules.watches.seen import seen_message
 from utils.events import notify_bg_event
 
 
@@ -20,7 +21,7 @@ async def on_message(client: PyroClient, message: Message) -> None:
     ):
         return
     meta = meta_from_chat(message.chat, ctx.contacts.ids)
-    await ctx.watches.on_text(meta.chat_id, message.id, message.text or message.caption)
+    await ctx.watches.on_message(seen_message(meta.chat_id, message))
     toggles = ctx.resolve(meta)
     if not toggles.messages:
         return

backend/src/userbot/handlers/presence.py

@@ -13,18 +13,21 @@ async def on_user_status(client: PyroClient, user: User) -> None:
     ctx = client.capture
     if ctx is None or user.status is None:
         return
-    await repository.insert_status(
-        ctx.pool,
-        ctx.account_id,
-        user.id,
-        datetime.now(UTC),
-        user.status.name.lower(),
-        user.last_online_date,
-        user.next_offline_date,
-        raw_json(user.raw),
-    )
-    await ctx.watches.on_status(user.id, is_online=user.status.name.lower() == "online")
-    await notify_bg_event(ctx.pool, "presence", ctx.account_id, chat_id=user.id)
+    at = datetime.now(UTC)
+    status = user.status.name.lower()
+    if ctx.resolve_peer(user.id, is_bot=bool(user.is_bot)).presence:
+        await repository.insert_status(
+            ctx.pool,
+            ctx.account_id,
+            user.id,
+            at,
+            status,
+            user.last_online_date,
+            user.next_offline_date,
+            raw_json(user.raw),
+        )
+        await notify_bg_event(ctx.pool, "presence", ctx.account_id, chat_id=user.id)
+    await ctx.watches.on_status(user.id, status, at, user.last_online_date)
 
 
 handlers = on_user_status.handlers  # ty: ignore[unresolved-attribute]

backend/src/userbot/handlers/raw/dispatcher.py

@@ -24,8 +24,9 @@ _MODULES = (
     read_receipts,
     transcribed,
 )
-_REGISTRY: dict[type, RawHandler] = {
-    update_type: module.handle for module in _MODULES for update_type in module.HANDLES
+_REGISTRY: dict[type, list[RawHandler]] = {
+    update_type: [module.handle for module in _MODULES if update_type in module.HANDLES]
+    for update_type in {kind for module in _MODULES for kind in module.HANDLES}
 }
 
 
@@ -33,8 +34,7 @@ _REGISTRY: dict[type, RawHandler] = {
 async def dispatch(
     client: PyroClient, update: raw.base.Update, users: dict, chats: dict
 ) -> None:
-    handler = _REGISTRY.get(type(update))
-    if handler is not None:
+    for handler in _REGISTRY.get(type(update), ()):
         await handler(client, update, users, chats)
 
 

backend/src/userbot/handlers/raw/profiles.py

@@ -25,8 +25,14 @@ async def _handle_user(
     current = await get_peer(ctx.pool, ctx.account_id, update.user_id)
     if current == fields:
         return
+    is_bot = bool(getattr(raw_user, "bot", False))
     await write_profile(
-        ctx.pool, ctx.account_id, update.user_id, fields, raw_json(raw_user)
+        ctx.pool,
+        ctx.account_id,
+        update.user_id,
+        fields,
+        raw_json(raw_user),
+        history=ctx.resolve_peer(update.user_id, is_bot=is_bot).profile_history,
     )
     changed_photo = current is None or current.photo_unique_id != photo_unique_id
     if photo_file_id and photo_unique_id and changed_photo:
@@ -51,7 +57,14 @@ async def _merge_partial(
     merged = replace(current or ProfileFields(), **overrides)
     if merged == current:
         return
-    await write_profile(ctx.pool, ctx.account_id, peer_id, merged, raw_str)
+    await write_profile(
+        ctx.pool,
+        ctx.account_id,
+        peer_id,
+        merged,
+        raw_str,
+        history=ctx.resolve_peer(peer_id).profile_history,
+    )
 
 
 async def handle(

backend/src/userbot/handlers/raw/read_receipts.py

@@ -7,25 +7,35 @@ from userbot.modules.raw_json import raw_json
 from userbot.modules.read_receipts import repository
 from utils.events import notify_bg_event
 
-HANDLES = (raw.types.UpdateReadHistoryOutbox,)
+HANDLES = (raw.types.UpdateReadHistoryOutbox, raw.types.UpdateReadChannelOutbox)
+
+
+def _watermark(update: raw.base.Update) -> tuple[int, int] | None:
+    if isinstance(update, raw.types.UpdateReadHistoryOutbox):
+        return utils.get_peer_id(update.peer), update.max_id
+    if isinstance(update, raw.types.UpdateReadChannelOutbox):
+        peer = raw.types.PeerChannel(channel_id=update.channel_id)
+        return utils.get_peer_id(peer), update.max_id
+    return None
 
 
 async def handle(
     client: PyroClient, update: raw.base.Update, _users: dict, _chats: dict
 ) -> None:
     ctx = client.capture
-    if ctx is None or not isinstance(update, raw.types.UpdateReadHistoryOutbox):
+    watermark = _watermark(update)
+    if ctx is None or watermark is None:
         return
-    chat_id = utils.get_peer_id(update.peer)
+    chat_id, max_id = watermark
     await repository.insert_read(
         ctx.pool,
         ctx.account_id,
         chat_id,
         chat_id,
         datetime.now(UTC),
-        update.max_id,
+        max_id,
         raw_json(update),
     )
     await notify_bg_event(
-        ctx.pool, "receipt", ctx.account_id, chat_id=chat_id, message_id=update.max_id
+        ctx.pool, "receipt", ctx.account_id, chat_id=chat_id, message_id=max_id
     )

backend/src/userbot/handlers/raw/transcribed.py

@@ -20,4 +20,4 @@ async def handle(
     await repository.set_extracted_text(
         ctx.pool, ctx.account_id, meta.chat_id, update.msg_id, update.text
     )
-    await ctx.watches.on_text(meta.chat_id, update.msg_id, update.text)
+    await ctx.watches.on_transcript(meta.chat_id, update.msg_id, update.text)

backend/src/userbot/handlers/stories.py

@@ -1,14 +1,19 @@
 from pyrogram.types import Story
 
 from userbot.modules.client import PyroClient
-from userbot.modules.stories.service import save_story
+from userbot.modules.stories.service import save_story, story_peer_id
 
 
 @PyroClient.on_story()
 async def on_story(client: PyroClient, story: Story) -> None:
-    if client.capture is None:
+    ctx = client.capture
+    if ctx is None:
         return
-    await save_story(client, client.capture, story)
+    peer_id = story_peer_id(story)
+    is_bot = bool(story.from_user and story.from_user.is_bot)
+    if peer_id > 0 and not ctx.resolve_peer(peer_id, is_bot=is_bot).stories:
+        return
+    await save_story(client, ctx, story)
 
 
 handlers = on_story.handlers  # ty: ignore[unresolved-attribute]

backend/src/userbot/modules/capture/context.py

@@ -2,10 +2,11 @@ import asyncpg
 from pyrogram import Client
 
 from userbot.modules.capture.identity import ChatMetaCache, PeerIdentityCache
+from userbot.modules.capture.pending import PendingDeletes
 from userbot.modules.contacts import ContactCache
 from userbot.modules.folders import FolderCache
 from userbot.modules.watches import WatchCache
-from utils.policy.models import CaptureToggles, ChatMeta, PolicySet
+from utils.policy.models import CaptureToggles, ChatKind, ChatMeta, PolicySet
 from utils.policy.repository import load_policy_set
 from utils.policy.resolver import resolve
 from utils.storage import ContentAddressedStorage
@@ -28,6 +29,7 @@ class CaptureContext:
         self.watches = WatchCache(pool, account_id)
         self.peer_identity = PeerIdentityCache()
         self.chat_meta = ChatMetaCache()
+        self.pending_deletes = PendingDeletes()
         self.policies = PolicySet()
 
     async def reload_policies(self) -> None:
@@ -36,6 +38,16 @@ class CaptureContext:
     def resolve(self, chat: ChatMeta) -> CaptureToggles:
         return resolve(chat, self.folders.folders, self.policies)
 
+    def resolve_peer(self, peer_id: int, *, is_bot: bool = False) -> CaptureToggles:
+        return self.resolve(
+            ChatMeta(
+                chat_id=peer_id,
+                kind=ChatKind.DM,
+                is_bot=is_bot,
+                is_contact=peer_id in self.contacts.ids,
+            )
+        )
+
 
 async def build_capture_context(
     client: Client,

backend/src/userbot/modules/capture/identity.py

@@ -58,7 +58,14 @@ async def _capture_peer(message: Message, ctx: CaptureContext) -> None:
     fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
     if not await ctx.peer_identity.changed(ctx.pool, ctx.account_id, user.id, fields):
         return
-    await write_profile(ctx.pool, ctx.account_id, user.id, fields, raw_json(user))
+    await write_profile(
+        ctx.pool,
+        ctx.account_id,
+        user.id,
+        fields,
+        raw_json(user),
+        history=ctx.resolve_peer(user.id, is_bot=bool(user.is_bot)).profile_history,
+    )
     if photo_file_id and photo_unique_id:
         from userbot.modules.avatars import note_avatar  # noqa: PLC0415
 
@@ -82,17 +89,15 @@ async def _capture_chat(message: Message, ctx: CaptureContext) -> None:
     meta = (chat.title, photo_unique_id)
     if not await ctx.chat_meta.changed(ctx.pool, ctx.account_id, chat.id, meta):
         return
-    from userbot.modules.groups.repository import insert_chat_history  # noqa: PLC0415
+    from userbot.modules.groups.repository import record_chat_meta  # noqa: PLC0415
 
-    await insert_chat_history(
+    await record_chat_meta(
         ctx.pool,
         ctx.account_id,
         chat.id,
         message.id,
-        "meta",
         chat.title,
         photo_unique_id,
-        None,
         message.date,
         raw_json(message),
     )

backend/src/userbot/modules/capture/message.py

@@ -1,11 +1,14 @@
+from datetime import datetime
+
 from pyrogram import Client
 from pyrogram.types import Message
 
 from userbot.modules.capture import repository
 from userbot.modules.capture.context import CaptureContext
+from userbot.modules.capture.repository import MessageRow
 from userbot.modules.links import extract_links
 from userbot.modules.links import repository as links_repository
-from userbot.modules.media import capture_media, self_destruct_ttl
+from userbot.modules.media import capture_media, media_unique_id, self_destruct_ttl
 from userbot.modules.raw_json import raw_json
 from utils.policy.models import CaptureToggles
 
@@ -34,27 +37,39 @@ def callbacks(message: Message) -> list[tuple[int, str | None, bytes | None]]:
     return buttons
 
 
+def message_row(message: Message, chat_id: int, date: datetime) -> MessageRow:
+    return MessageRow(
+        chat_id=chat_id,
+        message_id=message.id,
+        date=date,
+        sender_id=sender_id(message),
+        text=message.text or message.caption,
+        raw=raw_json(message),
+        has_media=message.media is not None,
+        is_self_destruct=self_destruct_ttl(message) is not None,
+        edit_date=message.edit_date,
+        media_unique_id=media_unique_id(message),
+    )
+
+
 async def capture_message(
     client: Client, message: Message, ctx: CaptureContext, toggles: CaptureToggles
 ) -> None:
     if message.empty or message.chat is None or message.date is None:
         return
     chat_id = message.chat.id or 0
-    await repository.upsert_message(
-        ctx.pool,
-        ctx.account_id,
-        chat_id,
-        message.id,
-        message.date,
-        sender_id(message),
-        message.text or message.caption,
-        raw_json(message),
-        has_media=message.media is not None,
-        is_self_destruct=self_destruct_ttl(message) is not None,
+    current = await repository.upsert_message(
+        ctx.pool, ctx.account_id, message_row(message, chat_id, message.date)
     )
+    if ctx.pending_deletes.take(chat_id, message.id):
+        await repository.mark_deleted_late(
+            ctx.pool, ctx.account_id, chat_id, message.id, message.date
+        )
     from userbot.modules.capture.identity import capture_identity  # noqa: PLC0415
 
     await capture_identity(message, ctx)
+    if not current:
+        return
     await capture_media(client, message, ctx, chat_id, message.id, toggles)
     buttons = callbacks(message)
     if buttons:

backend/src/userbot/modules/capture/pending.py

@@ -0,0 +1,36 @@
+import time
+
+from userbot.modules.capture.repository import CHANNEL_ID_THRESHOLD
+
+PENDING_DELETE_SECONDS = 120.0
+PENDING_DELETE_LIMIT = 10_000
+
+DeleteKey = tuple[int | None, int]
+
+
+def _key(chat_id: int | None, message_id: int) -> DeleteKey:
+    if chat_id is None or chat_id > CHANNEL_ID_THRESHOLD:
+        return None, message_id
+    return chat_id, message_id
+
+
+class PendingDeletes:
+    def __init__(self) -> None:
+        self._expires: dict[DeleteKey, float] = {}
+
+    def remember(self, chat_id: int | None, message_ids: list[int]) -> None:
+        now = time.monotonic()
+        while self._expires:
+            oldest = next(iter(self._expires))
+            fresh = self._expires[oldest] > now
+            if fresh and len(self._expires) < PENDING_DELETE_LIMIT:
+                break
+            del self._expires[oldest]
+        for message_id in message_ids:
+            key = _key(chat_id, message_id)
+            self._expires.pop(key, None)
+            self._expires[key] = now + PENDING_DELETE_SECONDS
+
+    def take(self, chat_id: int, message_id: int) -> bool:
+        expires = self._expires.pop(_key(chat_id, message_id), None)
+        return expires is not None and expires > time.monotonic()

backend/src/userbot/modules/capture/repository.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from datetime import datetime
 
 import asyncpg
@@ -21,13 +22,19 @@ _TOUCH_EDITED = """
 INSERT INTO messages
     (account_id, chat_id, message_id, date, sender_id, text, raw,
      has_media, is_self_destruct, edited_at)
-VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8, $9, now())
+VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8, $9, CASE WHEN $10 THEN now() END)
 ON CONFLICT (account_id, chat_id, message_id, date) DO UPDATE SET
     text = EXCLUDED.text,
     raw = EXCLUDED.raw,
     has_media = EXCLUDED.has_media,
     is_self_destruct = EXCLUDED.is_self_destruct,
-    edited_at = now()
+    edited_at = coalesce(EXCLUDED.edited_at, messages.edited_at)
+"""
+
+_LOCK_MESSAGE = """
+SELECT pg_advisory_xact_lock(
+    hashtextextended(concat_ws(':', 'message', $1::int, $2::bigint, $3::bigint), 0)
+)
 """
 
 _FIND_DELETED = """
@@ -40,6 +47,12 @@ UPDATE messages SET deleted_at = now()
 WHERE account_id = $1 AND chat_id = $2 AND message_id = $3 AND date = $4
 """
 
+_MARK_DELETED_LATE = """
+UPDATE messages SET deleted_at = now()
+WHERE account_id = $1 AND chat_id = $2 AND message_id = $3 AND date = $4
+  AND deleted_at IS NULL
+"""
+
 _INSERT_VERSION = """
 INSERT INTO message_versions
     (account_id, chat_id, message_id, observed_at, edit_date, text, raw)
@@ -52,7 +65,7 @@ INSERT INTO message_versions
     (account_id, chat_id, message_id, observed_at, edit_date, text, raw)
 SELECT account_id, chat_id, message_id, clock_timestamp(), NULL, text, raw
 FROM messages m
-WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3
+WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3 AND m.date = $4
   AND NOT EXISTS (
     SELECT 1 FROM message_versions v
     WHERE v.account_id = m.account_id AND v.chat_id = m.chat_id
@@ -61,15 +74,37 @@ WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3
 ON CONFLICT DO NOTHING
 """
 
+_INSERT_PAST_VERSION = """
+INSERT INTO message_versions
+    (account_id, chat_id, message_id, observed_at, edit_date, text, raw)
+SELECT $1::int, $2::bigint, $3::bigint,
+    coalesce(
+        (SELECT min(v.observed_at) - interval '1 microsecond'
+         FROM message_versions v
+         WHERE v.account_id = $1 AND v.chat_id = $2 AND v.message_id = $3
+           AND v.edit_date > $4::timestamptz),
+        $4::timestamptz,
+        $5::timestamptz
+    ),
+    $4::timestamptz, $6::text, $7::jsonb
+WHERE NOT EXISTS (
+    SELECT 1 FROM message_versions v
+    WHERE v.account_id = $1 AND v.chat_id = $2 AND v.message_id = $3
+      AND v.edit_date IS NOT DISTINCT FROM $4::timestamptz
+      AND v.text IS NOT DISTINCT FROM $6::text
+)
+ON CONFLICT DO NOTHING
+"""
+
 _CURRENT_CONTENT = """
 SELECT
     m.text AS text,
+    m.raw->>'edit_date' AS edit_date,
     (SELECT d.unique_id FROM media d
      WHERE d.account_id = m.account_id AND d.chat_id = m.chat_id
        AND d.message_id = m.message_id) AS media_unique_id
 FROM messages m
-WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3
-ORDER BY m.date DESC LIMIT 1
+WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3 AND m.date = $4
 """
 
 _CURRENT_MEDIA = """
@@ -121,33 +156,110 @@ WHERE evicted_key = $2 OR (unique_id = $1 AND evicted_at IS NOT NULL)
 """
 
 
-async def upsert_message(  # noqa: PLR0913
-    pool: asyncpg.Pool,
-    account_id: int,
-    chat_id: int,
-    message_id: int,
-    date: datetime,
-    sender_id: int | None,
-    text: str | None,
-    raw: str,
-    *,
-    has_media: bool,
-    is_self_destruct: bool,
-) -> None:
-    await pool.execute(
-        _UPSERT_MESSAGE,
-        account_id,
-        chat_id,
-        message_id,
-        date,
-        sender_id,
-        text,
-        raw,
-        has_media,
-        is_self_destruct,
+@dataclass(frozen=True, slots=True)
+class MessageRow:
+    chat_id: int
+    message_id: int
+    date: datetime
+    sender_id: int | None
+    text: str | None
+    raw: str
+    has_media: bool
+    is_self_destruct: bool
+    edit_date: datetime | None
+    media_unique_id: str | None
+
+    @property
+    def columns(self) -> tuple:
+        return (
+            self.chat_id,
+            self.message_id,
+            self.date,
+            self.sender_id,
+            self.text,
+            self.raw,
+            self.has_media,
+            self.is_self_destruct,
+        )
+
+
+async def _lock_current(
+    conn: asyncpg.Connection, account_id: int, row: MessageRow
+) -> asyncpg.Record | None:
+    await conn.execute(_LOCK_MESSAGE, account_id, row.chat_id, row.message_id)
+    return await conn.fetchrow(
+        _CURRENT_CONTENT, account_id, row.chat_id, row.message_id, row.date
     )
 
 
+def _is_stale(current: asyncpg.Record, row: MessageRow) -> bool:
+    stored = current["edit_date"]
+    if stored is None:
+        return False
+    if row.edit_date is None:
+        return True
+    try:
+        return row.edit_date < datetime.fromisoformat(stored)
+    except (TypeError, ValueError):
+        return False
+
+
+async def _record_past(
+    conn: asyncpg.Connection, account_id: int, row: MessageRow
+) -> None:
+    await conn.execute(
+        _INSERT_PAST_VERSION,
+        account_id,
+        row.chat_id,
+        row.message_id,
+        row.edit_date,
+        row.date,
+        row.text,
+        row.raw,
+    )
+
+
+async def _record_edit(
+    conn: asyncpg.Connection, account_id: int, row: MessageRow, *, mark_edited: bool
+) -> None:
+    await conn.execute(
+        _SNAPSHOT_ORIGINAL, account_id, row.chat_id, row.message_id, row.date
+    )
+    await conn.execute(_TOUCH_EDITED, account_id, *row.columns, mark_edited)
+    await conn.execute(
+        _INSERT_VERSION,
+        account_id,
+        row.chat_id,
+        row.message_id,
+        row.edit_date,
+        row.text,
+        row.raw,
+    )
+
+
+async def upsert_message(pool: asyncpg.Pool, account_id: int, row: MessageRow) -> bool:
+    async with pool.acquire() as conn, conn.transaction():
+        current = await _lock_current(conn, account_id, row)
+        if current is None:
+            await conn.execute(_UPSERT_MESSAGE, account_id, *row.columns)
+            return True
+        stored_media = current["media_unique_id"]
+        changed = current["text"] != row.text or (
+            stored_media is not None and stored_media != row.media_unique_id
+        )
+        if _is_stale(current, row):
+            if changed:
+                await _record_past(conn, account_id, row)
+            return False
+        if changed:
+            await _record_edit(
+                conn, account_id, row, mark_edited=row.edit_date is not None
+            )
+        else:
+            await conn.execute(_UPSERT_MESSAGE, account_id, *row.columns)
+    return True
+
+
 async def max_message_id(
     pool: asyncpg.Pool, account_id: int, chat_id: int
 ) -> int | None:
@@ -188,48 +300,29 @@ async def mark_deleted_channel(
     await _mark_deleted(pool, account_id, rows)
 
 
-async def add_version(  # noqa: PLR0913
-    pool: asyncpg.Pool,
-    account_id: int,
-    chat_id: int,
-    message_id: int,
-    date: datetime,
-    sender_id: int | None,
-    text: str | None,
-    raw: str,
-    edit_date: datetime | None,
-    media_unique_id: str | None,
-    *,
-    has_media: bool,
-    is_self_destruct: bool,
-) -> bool:
+async def add_version(pool: asyncpg.Pool, account_id: int, row: MessageRow) -> bool:
     async with pool.acquire() as conn, conn.transaction():
-        current = await conn.fetchrow(_CURRENT_CONTENT, account_id, chat_id, message_id)
-        text_changed = current is None or current["text"] != text
-        media_changed = (
-            current is not None and current["media_unique_id"] != media_unique_id
-        )
-        if not (text_changed or media_changed):
-            return False
-        await conn.execute(_SNAPSHOT_ORIGINAL, account_id, chat_id, message_id)
-        await conn.execute(
-            _TOUCH_EDITED,
-            account_id,
-            chat_id,
-            message_id,
-            date,
-            sender_id,
-            text,
-            raw,
-            has_media,
-            is_self_destruct,
-        )
-        await conn.execute(
-            _INSERT_VERSION, account_id, chat_id, message_id, edit_date, text, raw
-        )
+        current = await _lock_current(conn, account_id, row)
+        if current is not None:
+            changed = (
+                current["text"] != row.text
+                or current["media_unique_id"] != row.media_unique_id
+            )
+            if not changed:
+                return False
+            if _is_stale(current, row):
+                await _record_past(conn, account_id, row)
+                return False
+        await _record_edit(conn, account_id, row, mark_edited=True)
     return True
 
 
+async def mark_deleted_late(
+    pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int, date: datetime
+) -> None:
+    await pool.execute(_MARK_DELETED_LATE, account_id, chat_id, message_id, date)
+
+
 async def current_media(
     pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
 ) -> asyncpg.Record | None:

backend/src/userbot/modules/catchup/__init__.py

@@ -0,0 +1,3 @@
+from userbot.modules.catchup.service import SavedState, catch_up, read_saved_state
+
+__all__ = ["SavedState", "catch_up", "read_saved_state"]

backend/src/userbot/modules/catchup/repository.py

@@ -0,0 +1,41 @@
+from dataclasses import dataclass
+
+import asyncpg
+
+_LOAD = """
+SELECT catchup_pts, catchup_qts, catchup_date FROM accounts WHERE account_id = $1
+"""
+
+_SAVE = """
+UPDATE accounts SET catchup_pts = $2, catchup_qts = $3, catchup_date = $4
+WHERE account_id = $1
+"""
+
+_CLEAR = """
+UPDATE accounts SET catchup_pts = NULL, catchup_qts = NULL, catchup_date = NULL
+WHERE account_id = $1
+"""
+
+
+@dataclass(frozen=True)
+class UpdateCursor:
+    pts: int
+    qts: int
+    date: int
+
+
+async def load_cursor(pool: asyncpg.Pool, account_id: int) -> UpdateCursor | None:
+    row = await pool.fetchrow(_LOAD, account_id)
+    if row is None or row["catchup_pts"] is None:
+        return None
+    return UpdateCursor(row["catchup_pts"], row["catchup_qts"], row["catchup_date"])
+
+
+async def save_cursor(
+    pool: asyncpg.Pool, account_id: int, cursor: UpdateCursor
+) -> None:
+    await pool.execute(_SAVE, account_id, cursor.pts, cursor.qts, cursor.date)
+
+
+async def clear_cursor(pool: asyncpg.Pool, account_id: int) -> None:
+    await pool.execute(_CLEAR, account_id)

backend/src/userbot/modules/catchup/service.py

@@ -0,0 +1,255 @@
+import asyncio
+import sqlite3
+import time
+from contextlib import closing
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from pyrogram import raw, utils
+from pyrogram.errors import FloodPremiumWait, FloodWait, RPCError
+
+from userbot.modules.capture.chat_meta import meta_from_peer
+from userbot.modules.capture.context import CaptureContext
+from userbot.modules.catchup import repository
+from userbot.modules.catchup.repository import UpdateCursor
+from userbot.modules.client import PyroClient
+from utils.logging import logger
+from utils.policy.models import ChatKind
+
+BOX_SLICES = 50
+CHANNEL_SLICES = 10
+CHANNEL_LIMIT = 100
+MAX_CHANNELS = 50
+QUEUE_HIGH_WATER = 200
+QUEUE_WAIT_SECONDS = 0.5
+NO_QTS = -1
+MAX_AGE_SECONDS = 3 * 24 * 3600
+
+
+@dataclass(frozen=True)
+class SavedState:
+    box: UpdateCursor | None = None
+    channels: dict[int, int] = field(default_factory=dict)
+
+
+@dataclass
+class BoxWalk:
+    cursor: UpdateCursor
+    done: bool = False
+    updates: int = 0
+    channels: dict[int, raw.base.Chat] = field(default_factory=dict)
+
+
+def read_saved_state(path: Path) -> SavedState:
+    try:
+        uri = f"{path.resolve().as_uri()}?mode=ro"
+        with closing(sqlite3.connect(uri, uri=True)) as conn:
+            rows = conn.execute(
+                "SELECT id, pts, qts, date FROM update_state"
+            ).fetchall()
+    except Exception as exc:
+        logger.warning(f"[yellow]Session {path.stem}: no saved update state ({exc})[/]")
+        return SavedState()
+    box = None
+    channels: dict[int, int] = {}
+    for state_id, pts, qts, date in rows:
+        if pts is None:
+            continue
+        if state_id == 0:
+            box = UpdateCursor(
+                pts, NO_QTS if qts is None else qts, date or int(time.time())
+            )
+        else:
+            channels[state_id] = pts
+    return SavedState(box, channels)
+
+
+async def _dispatch(
+    client: PyroClient,
+    new_messages: list,
+    other_updates: list,
+    users: list,
+    chats: list,
+) -> int:
+    await client.fetch_peers(users)
+    await client.fetch_peers(chats)
+    user_map = {user.id: user for user in users}
+    chat_map = {chat.id: chat for chat in chats}
+    queue = client.dispatcher.updates_queue
+    for message in new_messages:
+        update = raw.types.UpdateNewMessage(message=message, pts=0, pts_count=0)
+        queue.put_nowait((update, user_map, chat_map))
+    for update in other_updates:
+        queue.put_nowait((update, user_map, chat_map))
+    while queue.qsize() > QUEUE_HIGH_WATER:  # noqa: ASYNC110
+        await asyncio.sleep(QUEUE_WAIT_SECONDS)
+    return len(new_messages) + len(other_updates)
+
+
+def _channel_hints(other_updates: list, chats: list) -> dict[int, raw.base.Chat]:
+    by_id = {chat.id: chat for chat in chats}
+    return {
+        utils.get_channel_id(update.channel_id): by_id[update.channel_id]
+        for update in other_updates
+        if isinstance(update, raw.types.UpdateChannelTooLong)
+        and update.channel_id in by_id
+    }
+
+
+async def _walk_box(
+    client: PyroClient, ctx: CaptureContext, cursor: UpdateCursor
+) -> BoxWalk:
+    walk = BoxWalk(cursor)
+    for _ in range(BOX_SLICES):
+        diff = await client.invoke(
+            raw.functions.updates.GetDifference(
+                pts=walk.cursor.pts, date=walk.cursor.date, qts=walk.cursor.qts
+            )
+        )
+        if isinstance(diff, raw.types.updates.DifferenceEmpty):
+            walk.done = True
+            return walk
+        if isinstance(diff, raw.types.updates.DifferenceTooLong):
+            logger.warning(
+                f"[yellow]Account {ctx.account_id}: too much missed to catch up "
+                f"(pts {walk.cursor.pts} to {diff.pts}), that gap stays[/]"
+            )
+            walk.done = True
+            return walk
+        walk.updates += await _dispatch(
+            client, diff.new_messages, diff.other_updates, diff.users, diff.chats
+        )
+        walk.channels.update(_channel_hints(diff.other_updates, diff.chats))
+        final = isinstance(diff, raw.types.updates.Difference)
+        state = diff.state if final else diff.intermediate_state
+        moved = UpdateCursor(state.pts, state.qts, state.date or walk.cursor.date)
+        if moved == walk.cursor and not final:
+            logger.warning(
+                f"[yellow]Account {ctx.account_id}: catch-up stopped advancing "
+                f"at pts={moved.pts}[/]"
+            )
+            return walk
+        walk.cursor = moved
+        await repository.save_cursor(ctx.pool, ctx.account_id, moved)
+        if final:
+            walk.done = True
+            return walk
+    return walk
+
+
+async def _walk_channel(client: PyroClient, peer_id: int, pts: int) -> int:
+    peer = await client.resolve_peer(peer_id)
+    if not isinstance(peer, raw.types.InputPeerChannel):
+        return 0
+    channel = raw.types.InputChannel(
+        channel_id=peer.channel_id, access_hash=peer.access_hash
+    )
+    updates = 0
+    for _ in range(CHANNEL_SLICES):
+        diff = await client.invoke(
+            raw.functions.updates.GetChannelDifference(
+                channel=channel,
+                filter=raw.types.ChannelMessagesFilterEmpty(),
+                pts=pts,
+                limit=CHANNEL_LIMIT,
+            )
+        )
+        if isinstance(diff, raw.types.updates.ChannelDifferenceEmpty):
+            return updates
+        if isinstance(diff, raw.types.updates.ChannelDifferenceTooLong):
+            logger.info(f"Channel {peer_id}: too much missed to catch up")
+            return updates
+        updates += await _dispatch(
+            client, diff.new_messages, diff.other_updates, diff.users, diff.chats
+        )
+        if diff.final:
+            return updates
+        pts = diff.pts
+    return updates
+
+
+def _captured_channels(
+    ctx: CaptureContext, hints: dict[int, raw.base.Chat], saved: dict[int, int]
+) -> list[int]:
+    wanted: list[tuple[bool, int]] = []
+    for peer_id, chat in hints.items():
+        if peer_id not in saved:
+            continue
+        meta = meta_from_peer(
+            raw.types.PeerChannel(channel_id=chat.id),
+            {chat.id: chat},
+            {},
+            ctx.contacts.ids,
+        )
+        toggles = ctx.resolve(meta)
+        if toggles.messages or toggles.track_edits_deletes:
+            wanted.append((meta.kind is ChatKind.CHANNEL, peer_id))
+    return [peer_id for _, peer_id in sorted(wanted)[:MAX_CHANNELS]]
+
+
+async def _walk_channels(
+    client: PyroClient, ctx: CaptureContext, walk: BoxWalk, saved: dict[int, int]
+) -> int:
+    peer_ids = _captured_channels(ctx, walk.channels, saved)
+    updates = 0
+    for peer_id in peer_ids:
+        try:
+            updates += await _walk_channel(client, peer_id, saved[peer_id])
+        except (FloodWait, FloodPremiumWait) as exc:
+            logger.warning(
+                f"[yellow]Account {ctx.account_id}: channel catch-up stopped "
+                f"by a flood wait:[/] {exc}"
+            )
+            break
+        except RPCError as exc:
+            logger.warning(
+                f"[yellow]Account {ctx.account_id}: channel {peer_id} "
+                f"catch-up failed:[/] {exc}"
+            )
+    skipped = len(walk.channels) - len(peer_ids)
+    if skipped:
+        logger.info(f"Account {ctx.account_id}: {skipped} channel gap(s) left as is")
+    return updates
+
+
+async def catch_up(client: PyroClient, saved: SavedState) -> None:
+    ctx = client.capture
+    if ctx is None:
+        return
+    try:
+        pending = await repository.load_cursor(ctx.pool, ctx.account_id)
+        cursor = pending or saved.box
+        if cursor is None:
+            logger.info(f"Account {ctx.account_id}: no saved update state to catch up")
+            return
+        if time.time() - cursor.date > MAX_AGE_SECONDS:
+            logger.warning(
+                f"[yellow]Account {ctx.account_id}: last update seen too long ago "
+                f"(pts={cursor.pts}), not catching up[/]"
+            )
+            await repository.clear_cursor(ctx.pool, ctx.account_id)
+            return
+        await repository.save_cursor(ctx.pool, ctx.account_id, cursor)
+        logger.info(
+            f"Account {ctx.account_id}: catching up from pts={cursor.pts}"
+            f"{' (resumed)' if pending else ''}"
+        )
+        walk = await _walk_box(client, ctx, cursor)
+        if not walk.done:
+            logger.warning(
+                f"[yellow]Account {ctx.account_id}: catch-up unfinished at "
+                f"pts={walk.cursor.pts},[/] {walk.updates} updates replayed, "
+                f"resumes on the next start"
+            )
+            return
+        from_channels = await _walk_channels(client, ctx, walk, saved.channels)
+        await repository.clear_cursor(ctx.pool, ctx.account_id)
+        logger.info(
+            f"[green]Account {ctx.account_id}: caught up,[/] {walk.updates} updates "
+            f"from the main box, {from_channels} from channels"
+        )
+    except Exception:
+        logger.exception(
+            f"[red]Account {ctx.account_id}: catch-up failed,[/] "
+            f"resumes on the next start"
+        )

backend/src/userbot/modules/groups/repository.py

@@ -7,7 +7,23 @@ INSERT INTO chat_history
     (account_id, chat_id, message_id, event, title, photo_unique_id, actor_id,
      ts, raw)
 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb)
-ON CONFLICT (account_id, chat_id, message_id) DO NOTHING
+ON CONFLICT DO NOTHING
+"""
+
+_INSERT_CHAT_META = """
+INSERT INTO chat_history
+    (account_id, chat_id, message_id, event, title, photo_unique_id, ts, raw)
+SELECT $1, $2, $3, 'meta', $4::text, $5::text, $6, $7::jsonb
+WHERE ($4::text, $5::text) IS DISTINCT FROM (
+    (SELECT title FROM chat_history
+     WHERE account_id = $1 AND chat_id = $2 AND event IN ('meta', 'title')
+     ORDER BY ts DESC LIMIT 1),
+    (SELECT photo_unique_id FROM chat_history
+     WHERE account_id = $1 AND chat_id = $2
+       AND event IN ('meta', 'photo', 'photo_deleted')
+     ORDER BY ts DESC LIMIT 1)
+)
+ON CONFLICT DO NOTHING
 """
 
 _INSERT_MEMBERSHIP = """
@@ -19,13 +35,14 @@ ON CONFLICT (account_id, chat_id, message_id, user_id) DO NOTHING
 
 _LATEST_TITLE = """
 SELECT title FROM chat_history
-WHERE account_id = $1 AND chat_id = $2 AND title IS NOT NULL
+WHERE account_id = $1 AND chat_id = $2 AND event IN ('meta', 'title')
 ORDER BY ts DESC LIMIT 1
 """
 
 _LATEST_PHOTO = """
 SELECT photo_unique_id FROM chat_history
-WHERE account_id = $1 AND chat_id = $2 AND photo_unique_id IS NOT NULL
+WHERE account_id = $1 AND chat_id = $2
+  AND event IN ('meta', 'photo', 'photo_deleted')
 ORDER BY ts DESC LIMIT 1
 """
 
@@ -64,6 +81,29 @@ async def insert_chat_history(  # noqa: PLR0913
     )
 
 
+async def record_chat_meta(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    account_id: int,
+    chat_id: int,
+    message_id: int,
+    title: str | None,
+    photo_unique_id: str | None,
+    ts: datetime,
+    raw: str,
+) -> bool:
+    status = await pool.execute(
+        _INSERT_CHAT_META,
+        account_id,
+        chat_id,
+        message_id,
+        title,
+        photo_unique_id,
+        ts,
+        raw,
+    )
+    return status == "INSERT 0 1"
+
+
 async def insert_membership(  # noqa: PLR0913
     pool: asyncpg.Pool,
     account_id: int,

backend/src/userbot/modules/jobs/consumer.py

@@ -9,11 +9,12 @@ from userbot.modules.jobs import repository
 from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import JOB_HANDLERS, JobHandler
 from utils.db.models import Job, JobStatus
-from utils.env import env
+from utils.listen import Listener
 from utils.logging import logger
 
 JOBS_CHANGED_CHANNEL = "jobs_changed"
 POLL_INTERVAL_SECONDS = 60.0
+RESTART_SECONDS = 10.0
 
 
 class JobConsumer:
@@ -38,25 +39,20 @@ class JobConsumer:
                 self.pool, ctx.job_id, JobStatus.FAILED, f"no handler for {job.kind!r}"
             )
             return
-        while True:
-            try:
-                await handler(ctx)
-            except (FloodWait, FloodPremiumWait) as exc:
-                await repository.bump_flood_wait(self.pool, ctx.job_id)
-                wait = exc.value if isinstance(exc.value, int) else 1
-                logger.warning(
-                    f"[yellow]FloodWait {wait}s on job {ctx.job_id} ({job.kind})[/]"
-                )
-                await asyncio.sleep(wait)
-            except Exception as exc:
-                logger.exception(f"Job {ctx.job_id} ({job.kind}) failed")
-                await repository.finish(
-                    self.pool, ctx.job_id, JobStatus.FAILED, str(exc)
-                )
-                return
-            else:
-                await repository.finish(self.pool, ctx.job_id, JobStatus.DONE)
-                return
+        try:
+            await handler(ctx)
+        except (FloodWait, FloodPremiumWait) as exc:
+            wait = exc.value if isinstance(exc.value, int) else 1
+            await repository.defer_flood_wait(self.pool, ctx.job_id, wait)
+            logger.warning(
+                f"[yellow]FloodWait {wait}s on job {ctx.job_id} ({job.kind}), "
+                f"deferred[/]"
+            )
+        except Exception as exc:
+            logger.exception(f"Job {ctx.job_id} ({job.kind}) failed")
+            await repository.finish(self.pool, ctx.job_id, JobStatus.FAILED, str(exc))
+        else:
+            await repository.finish(self.pool, ctx.job_id, JobStatus.DONE)
 
     async def drain(self) -> None:
         while True:
@@ -66,23 +62,41 @@ class JobConsumer:
             await self.execute(job)
 
     async def run(self) -> None:
+        while True:
+            try:
+                await self._serve()
+            except Exception:
+                logger.exception(
+                    f"[red]Job consumer for account {self.account_id} failed,[/] "
+                    f"restarting in {RESTART_SECONDS:.0f}s"
+                )
+            await asyncio.sleep(RESTART_SECONDS)
+
+    async def _serve(self) -> None:
         requeued = await repository.requeue_running(self.pool, self.account_id)
         if requeued:
             logger.info(f"[yellow]Requeued {requeued} stale running job(s).[/]")
         pruned = await repository.prune_finished(self.pool, self.account_id)
         if pruned:
             logger.info(f"[yellow]Pruned {pruned} finished job(s).[/]")
-        conn = await asyncpg.connect(dsn=env.db.connection_url)
-        await conn.add_listener(JOBS_CHANGED_CHANNEL, lambda *_: self._wake.set())
+        listener = Listener(
+            f"jobs {self.account_id}",
+            {JOBS_CHANGED_CHANNEL: lambda _payload: self._wake.set()},
+            self._wake.set,
+        )
+        await listener.start()
         logger.info(f"[green]Job consumer running for account {self.account_id}.[/]")
         try:
             while True:
-                await self.drain()
                 self._wake.clear()
+                await self.drain()
                 with contextlib.suppress(TimeoutError):
                     await asyncio.wait_for(
-                        self._wake.wait(), timeout=POLL_INTERVAL_SECONDS
+                        self._wake.wait(), timeout=await self._idle_seconds()
                     )
         finally:
-            with contextlib.suppress(Exception):
-                await conn.close()
+            await listener.stop()
+
+    async def _idle_seconds(self) -> float:
+        due = await repository.seconds_to_next_due(self.pool, self.account_id)
+        return POLL_INTERVAL_SECONDS if due is None else min(due, POLL_INTERVAL_SECONDS)

backend/src/userbot/modules/jobs/handlers/__init__.py

@@ -5,6 +5,7 @@ from userbot.modules.jobs.handlers import (
     fetch_avatar,
     fetch_custom_emoji,
     fetch_media,
+    fetch_read_info,
     search_peers,
     sync_contacts,
     sync_dialogs,
@@ -18,6 +19,7 @@ __all__ = [
     "fetch_avatar",
     "fetch_custom_emoji",
     "fetch_media",
+    "fetch_read_info",
     "search_peers",
     "sync_contacts",
     "sync_dialogs",

backend/src/userbot/modules/jobs/handlers/enrich_chat.py

@@ -1,17 +1,11 @@
-from datetime import UTC, datetime
-
 from pyrogram import Client
 from pyrogram.errors import BadRequest, Forbidden
 from pyrogram.types import User
 
-from userbot.modules.avatars import note_avatar
 from userbot.modules.capture.context import CaptureContext
-from userbot.modules.groups.repository import insert_chat_history
 from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import register
-from userbot.modules.profiles.parse import snapshot_from_high_level
-from userbot.modules.profiles.repository import write_profile
-from userbot.modules.raw_json import raw_json
+from userbot.modules.profiles.snapshots import save_group, save_user
 
 MEMBER_CAP = 200
 
@@ -25,45 +19,17 @@ LIMIT 100
 """
 
 
-async def _save_user(ctx: CaptureContext, user: User) -> None:
-    fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
-    await write_profile(ctx.pool, ctx.account_id, user.id, fields, raw_json(user))
-    if photo_file_id and photo_unique_id:
-        await note_avatar(
-            ctx.pool, ctx.account_id, user.id, "peer", photo_unique_id, photo_file_id
-        )
-
-
 async def _enrich_chat_meta(client: Client, ctx: CaptureContext, chat_id: int) -> None:
     chat = await client.get_chat(chat_id)
-    if chat is None:
-        return
-    photo = chat.photo
-    photo_unique_id = photo.big_photo_unique_id if photo else None
-    photo_file_id = photo.big_file_id if photo else None
-    await insert_chat_history(
-        ctx.pool,
-        ctx.account_id,
-        chat_id,
-        0,
-        "meta",
-        chat.title,
-        photo_unique_id,
-        None,
-        datetime.now(UTC),
-        raw_json(chat),
-    )
-    if photo_file_id and photo_unique_id:
-        await note_avatar(
-            ctx.pool, ctx.account_id, chat_id, "chat", photo_unique_id, photo_file_id
-        )
+    if chat is not None:
+        await save_group(ctx, chat)
 
 
 async def _enrich_members(client: Client, ctx: CaptureContext, chat_id: int) -> None:
     try:
         async for member in client.get_chat_members(chat_id, limit=MEMBER_CAP):
             if isinstance(member.user, User):
-                await _save_user(ctx, member.user)
+                await save_user(ctx, member.user)
     except (BadRequest, Forbidden):
         return
 
@@ -89,7 +55,7 @@ async def _enrich_senders(client: Client, ctx: CaptureContext, chat_id: int) ->
     if not ids:
         return
     for user in await _fetch_users(client, ids):
-        await _save_user(ctx, user)
+        await save_user(ctx, user)
 
 
 @register("enrich_chat")

backend/src/userbot/modules/jobs/handlers/fetch_avatar.py

@@ -1,6 +1,13 @@
 from pyrogram import Client
-from pyrogram.errors import FileIdInvalid, FileReferenceExpired, FileReferenceInvalid
-from pyrogram.types import Photo
+from pyrogram.errors import (
+    ChannelInvalid,
+    ChannelPrivate,
+    FileIdInvalid,
+    FileReferenceExpired,
+    FileReferenceInvalid,
+    PeerIdInvalid,
+)
+from pyrogram.types import ChatPhoto
 
 from userbot.modules.avatars.repository import get_avatar_file, mark_avatar_downloaded
 from userbot.modules.download import download_bytes
@@ -8,19 +15,20 @@ from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import register
 
 STALE_FILE_ID = (FileIdInvalid, FileReferenceExpired, FileReferenceInvalid)
+UNKNOWN_OWNER = (PeerIdInvalid, ChannelInvalid, ChannelPrivate)
 
 
 async def _fresh_file_id(client: Client, owner_id: int, unique_id: str) -> str | None:
-    chat = await client.get_chat(owner_id)
+    try:
+        chat = await client.get_chat(owner_id)
+    except UNKNOWN_OWNER:
+        return None
     photo = chat.photo if chat else None
     if photo and photo.big_photo_unique_id == unique_id:
         return photo.big_file_id
-    photos = client.get_chat_photos(owner_id)
-    if photos is None:
-        return None
-    async for item in photos:
-        if isinstance(item, Photo) and item.file_unique_id == unique_id:
-            return item.file_id
+    async for item in client.get_chat_photos(owner_id):
+        if isinstance(item, ChatPhoto) and item.big_photo_unique_id == unique_id:
+            return item.big_file_id
     return None
 
 

backend/src/userbot/modules/jobs/handlers/fetch_read_info.py

@@ -0,0 +1,124 @@
+from datetime import UTC, datetime, timedelta
+
+from pyrogram import Client, raw
+from pyrogram.errors import (
+    ChannelInvalid,
+    ChannelPrivate,
+    ChatTooBig,
+    FloodPremiumWait,
+    FloodWait,
+    MessageIdInvalid,
+    MessageNotReadYet,
+    MessageTooOld,
+    MsgIdInvalid,
+    MsgTooOld,
+    PeerIdInvalid,
+    RPCError,
+    UserPrivacyRestricted,
+    YourPrivacyRestricted,
+)
+
+from userbot.modules.jobs.context import JobContext
+from userbot.modules.jobs.registry import register
+from userbot.modules.read_receipts.repository import save_fetch, save_reads
+from utils.jobs import enqueue
+from utils.logging import logger
+from utils.policy.models import ChatKind
+from utils.read.peers import get_peers
+from utils.read.read_receipts import read_info
+
+RETRY = timedelta(seconds=60)
+LATER = timedelta(days=1)
+
+Refusal = tuple[tuple[type[Exception], ...], str, timedelta]
+
+REFUSALS: tuple[Refusal, ...] = (
+    ((MessageNotReadYet,), "not_read_yet", RETRY),
+    ((UserPrivacyRestricted,), "user_privacy", RETRY),
+    ((YourPrivacyRestricted,), "your_privacy", LATER),
+    ((ChatTooBig,), "chat_too_big", LATER),
+    ((MsgTooOld, MessageTooOld), "too_old", LATER),
+    (
+        (
+            PeerIdInvalid,
+            ChannelInvalid,
+            ChannelPrivate,
+            MessageIdInvalid,
+            MsgIdInvalid,
+            KeyError,
+            ValueError,
+        ),
+        "unavailable",
+        RETRY,
+    ),
+)
+
+
+def _refusal(exc: Exception) -> tuple[str, timedelta]:
+    for kinds, outcome, retry in REFUSALS:
+        if isinstance(exc, kinds):
+            return outcome, retry
+    return "error", RETRY
+
+
+def _at(timestamp: int) -> datetime:
+    return datetime.fromtimestamp(timestamp, UTC)
+
+
+async def ask_telegram(
+    client: Client, chat_id: int, message_id: int, *, group: bool
+) -> list[tuple[int, datetime]]:
+    peer = await client.resolve_peer(chat_id)
+    if peer is None:
+        msg = f"unknown peer {chat_id}"
+        raise ValueError(msg)
+    if group:
+        participants = await client.invoke(
+            raw.functions.messages.GetMessageReadParticipants(
+                peer=peer, msg_id=message_id
+            )
+        )
+        return [(item.user_id, _at(item.date)) for item in participants]
+    outbox = await client.invoke(
+        raw.functions.messages.GetOutboxReadDate(peer=peer, msg_id=message_id)
+    )
+    return [(chat_id, _at(outbox.date))]
+
+
+async def _enrich_unknown(ctx: JobContext, chat_id: int, reader_ids: list[int]) -> None:
+    known = {
+        peer.peer_id for peer in await get_peers(ctx.pool, ctx.account_id, reader_ids)
+    }
+    if set(reader_ids) - known:
+        await enqueue(ctx.pool, ctx.account_id, "enrich_chat", {"chat_id": chat_id})
+
+
+@register("fetch_read_info")
+async def fetch_read_info(ctx: JobContext) -> None:
+    client = ctx.client
+    if client is None:
+        return
+    chat_id = ctx.job.params["chat_id"]
+    message_id = ctx.job.params["message_id"]
+    info = await read_info(ctx.pool, ctx.account_id, chat_id, message_id)
+    if info is None or not info.can_fetch:
+        return
+    group = info.chat_kind == ChatKind.GROUP
+    try:
+        reads = await ask_telegram(client, chat_id, message_id, group=group)
+    except (FloodWait, FloodPremiumWait) as exc:
+        wait = exc.value if isinstance(exc.value, int) else 0
+        retry = max(RETRY, timedelta(seconds=wait))
+        await save_fetch(ctx.pool, ctx.account_id, chat_id, message_id, "flood", retry)
+        return
+    except (RPCError, KeyError, ValueError) as exc:
+        outcome, retry = _refusal(exc)
+        if outcome == "error":
+            logger.warning(f"[yellow]Read info {chat_id}/{message_id}: {exc!r}[/]")
+        await save_fetch(ctx.pool, ctx.account_id, chat_id, message_id, outcome, retry)
+        return
+    source = "participants" if group else "outbox_date"
+    await save_reads(ctx.pool, ctx.account_id, chat_id, message_id, source, reads)
+    await save_fetch(ctx.pool, ctx.account_id, chat_id, message_id, "ok", RETRY)
+    if group and reads:
+        await _enrich_unknown(ctx, chat_id, [reader_id for reader_id, _ in reads])

backend/src/userbot/modules/jobs/handlers/sync_contacts.py

@@ -1,11 +1,8 @@
 from pyrogram.types import User
 
-from userbot.modules.avatars import note_avatar
 from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import register
-from userbot.modules.profiles.parse import snapshot_from_high_level
-from userbot.modules.profiles.repository import write_profile
-from userbot.modules.raw_json import raw_json
+from userbot.modules.profiles.snapshots import save_user
 
 
 @register("sync_contacts")
@@ -21,17 +18,7 @@ async def sync_contacts(ctx: JobContext) -> None:
     for user in contacts:
         if not isinstance(user, User):
             continue
-        fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
-        await write_profile(ctx.pool, ctx.account_id, user.id, fields, raw_json(user))
-        if photo_file_id and photo_unique_id:
-            await note_avatar(
-                ctx.pool,
-                ctx.account_id,
-                user.id,
-                "peer",
-                photo_unique_id,
-                photo_file_id,
-            )
+        await save_user(capture, user)
         processed += 1
     await capture.contacts.refresh()
     await ctx.report_progress({"processed": processed, "done": True})

backend/src/userbot/modules/jobs/handlers/sync_dialogs.py

@@ -2,14 +2,10 @@ from pyrogram import Client
 from pyrogram.errors import BadRequest, Forbidden
 from pyrogram.types import User
 
-from userbot.modules.avatars import note_avatar
 from userbot.modules.capture.context import CaptureContext
 from userbot.modules.jobs.context import JobContext
 from userbot.modules.jobs.registry import register
-from userbot.modules.profiles.parse import snapshot_from_high_level
-from userbot.modules.profiles.repository import write_profile
-from userbot.modules.profiles.snapshots import save_group, save_private
-from userbot.modules.raw_json import raw_json
+from userbot.modules.profiles.snapshots import save_group, save_private, save_user
 from utils.logging import logger
 
 SAVE_EVERY = 100
@@ -30,21 +26,8 @@ async def _enrich_users(client: Client, ctx: CaptureContext, ids: list[int]) ->
             continue
         users = result if isinstance(result, list) else [result]
         for user in users:
-            if not isinstance(user, User):
-                continue
-            fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
-            await write_profile(
-                ctx.pool, ctx.account_id, user.id, fields, raw_json(user)
-            )
-            if photo_file_id and photo_unique_id:
-                await note_avatar(
-                    ctx.pool,
-                    ctx.account_id,
-                    user.id,
-                    "peer",
-                    photo_unique_id,
-                    photo_file_id,
-                )
+            if isinstance(user, User):
+                await save_user(ctx, user)
 
 
 @register("sync_dialogs")

backend/src/userbot/modules/jobs/repository.py

@@ -32,6 +32,7 @@ async def claim_next(pool: asyncpg.Pool, account_id: int) -> Job | None:
         "updated_at = now(), attempts = attempts + 1 "
         "WHERE id = ("
         "    SELECT id FROM jobs WHERE account_id = $1 AND status = 'pending' "
+        "    AND (run_after IS NULL OR run_after <= now()) "
         "    ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1"
         ") RETURNING *",
         account_id,
@@ -76,11 +77,21 @@ async def report_progress(
     )
 
 
-async def bump_flood_wait(pool: asyncpg.Pool, job_id: int) -> None:
+async def defer_flood_wait(pool: asyncpg.Pool, job_id: int, seconds: int) -> None:
     await pool.execute(
-        "UPDATE jobs SET flood_waits = flood_waits + 1, updated_at = now() "
-        "WHERE id = $1",
+        "UPDATE jobs SET status = 'pending', flood_waits = flood_waits + 1, "
+        "run_after = now() + make_interval(secs => $2), updated_at = now() "
+        "WHERE id = $1 AND status = 'running'",
         job_id,
+        seconds,
+    )
+
+
+async def seconds_to_next_due(pool: asyncpg.Pool, account_id: int) -> float | None:
+    return await pool.fetchval(
+        "SELECT extract(epoch FROM min(run_after) - now())::float FROM jobs "
+        "WHERE account_id = $1 AND status = 'pending' AND run_after > now()",
+        account_id,
     )
 
 

backend/src/userbot/modules/links/repository.py

@@ -7,6 +7,14 @@ INSERT INTO links
     (account_id, chat_id, message_id, position, url, kind, web_url, web_title,
      web_description, web_site_name, raw)
 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::jsonb)
+ON CONFLICT (account_id, chat_id, message_id, position) DO UPDATE SET
+    url = EXCLUDED.url,
+    kind = EXCLUDED.kind,
+    web_url = EXCLUDED.web_url,
+    web_title = EXCLUDED.web_title,
+    web_description = EXCLUDED.web_description,
+    web_site_name = EXCLUDED.web_site_name,
+    raw = EXCLUDED.raw
 """
 
 

backend/src/userbot/modules/profiles/parse.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from pyrogram import Client, raw
 from pyrogram.types import Chat, User
@@ -12,6 +12,11 @@ class ProfileFields:
     phone: str | None = None
     photo_unique_id: str | None = None
     is_deleted_account: bool = False
+    unknown: frozenset[str] = field(default=frozenset(), compare=False)
+
+
+MIN_UNKNOWN = frozenset({"username", "phone"})
+CHAT_UNKNOWN = frozenset({"phone", "is_deleted_account"})
 
 
 def active_username(usernames: list[raw.types.Username]) -> str | None:
@@ -35,6 +40,7 @@ async def snapshot_from_user(
         phone=user.phone_number if user else None,
         photo_unique_id=photo_unique_id,
         is_deleted_account=bool(getattr(raw_user, "deleted", False)),
+        unknown=MIN_UNKNOWN if getattr(raw_user, "min", False) else frozenset(),
     )
     return fields, photo_file_id, photo_unique_id
 
@@ -52,6 +58,7 @@ def snapshot_from_high_level(
         phone=user.phone_number,
         photo_unique_id=photo_unique_id,
         is_deleted_account=bool(user.is_deleted),
+        unknown=MIN_UNKNOWN if user.is_min else frozenset(),
     )
     return fields, photo_file_id, photo_unique_id
 
@@ -64,8 +71,7 @@ def snapshot_from_chat(chat: Chat) -> tuple[ProfileFields, str | None, str | Non
         first_name=chat.first_name,
         last_name=chat.last_name,
         username=chat.username,
-        phone=None,
         photo_unique_id=photo_unique_id,
-        is_deleted_account=False,
+        unknown=CHAT_UNKNOWN,
     )
     return fields, photo_file_id, photo_unique_id

backend/src/userbot/modules/profiles/repository.py

@@ -9,26 +9,46 @@ WHERE account_id = $1 AND peer_id = $2
 """
 
 _UPSERT_PEER = """
-INSERT INTO peers
+INSERT INTO peers AS p
     (account_id, peer_id, first_name, last_name, username, phone,
      photo_unique_id, is_deleted_account, raw, updated_at)
 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, now())
 ON CONFLICT (account_id, peer_id) DO UPDATE SET
     first_name = EXCLUDED.first_name,
     last_name = EXCLUDED.last_name,
-    username = EXCLUDED.username,
-    phone = EXCLUDED.phone,
+    username = CASE WHEN 'username' = ANY($10::text[])
+        THEN COALESCE(EXCLUDED.username, p.username) ELSE EXCLUDED.username END,
+    phone = CASE WHEN 'phone' = ANY($10::text[])
+        THEN COALESCE(EXCLUDED.phone, p.phone) ELSE EXCLUDED.phone END,
     photo_unique_id = EXCLUDED.photo_unique_id,
-    is_deleted_account = EXCLUDED.is_deleted_account,
+    is_deleted_account = CASE WHEN 'is_deleted_account' = ANY($10::text[])
+        THEN p.is_deleted_account ELSE EXCLUDED.is_deleted_account END,
     raw = EXCLUDED.raw,
     updated_at = now()
 """
 
-_INSERT_HISTORY = """
+_INSERT_CHANGE = """
 INSERT INTO peer_history
     (account_id, peer_id, observed_at, first_name, last_name, username, phone,
      photo_unique_id, is_deleted_account, raw)
-VALUES ($1, $2, now(), $3, $4, $5, $6, $7, $8, $9::jsonb)
+SELECT account_id, peer_id, clock_timestamp(), first_name, last_name, username,
+       phone, photo_unique_id, is_deleted_account, raw
+FROM peers p
+WHERE account_id = $1 AND peer_id = $2
+  AND NOT EXISTS (
+    SELECT 1 FROM (
+        SELECT first_name, last_name, username, phone, photo_unique_id,
+               is_deleted_account
+        FROM peer_history
+        WHERE account_id = $1 AND peer_id = $2
+        ORDER BY observed_at DESC LIMIT 1
+    ) latest
+    WHERE (latest.first_name, latest.last_name, latest.username, latest.phone,
+           latest.photo_unique_id, latest.is_deleted_account)
+          IS NOT DISTINCT FROM
+          (p.first_name, p.last_name, p.username, p.phone,
+           p.photo_unique_id, p.is_deleted_account)
+  )
 ON CONFLICT DO NOTHING
 """
 
@@ -49,18 +69,30 @@ async def get_peer(
     )
 
 
-async def write_profile(
-    pool: asyncpg.Pool, account_id: int, peer_id: int, fields: ProfileFields, raw: str
-) -> None:
-    args = (
-        fields.first_name,
-        fields.last_name,
-        fields.username,
-        fields.phone,
-        fields.photo_unique_id,
-        fields.is_deleted_account,
-        raw,
-    )
+async def write_profile(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    account_id: int,
+    peer_id: int,
+    fields: ProfileFields,
+    raw: str,
+    *,
+    history: bool,
+) -> bool:
     async with pool.acquire() as conn, conn.transaction():
-        await conn.execute(_UPSERT_PEER, account_id, peer_id, *args)
-        await conn.execute(_INSERT_HISTORY, account_id, peer_id, *args)
+        await conn.execute(
+            _UPSERT_PEER,
+            account_id,
+            peer_id,
+            fields.first_name,
+            fields.last_name,
+            fields.username,
+            fields.phone,
+            fields.photo_unique_id,
+            fields.is_deleted_account,
+            raw,
+            sorted(fields.unknown),
+        )
+        if not history:
+            return False
+        status = await conn.execute(_INSERT_CHANGE, account_id, peer_id)
+    return status == "INSERT 0 1"

backend/src/userbot/modules/profiles/snapshots.py

@@ -1,19 +1,46 @@
 from datetime import UTC, datetime
 
-from pyrogram.types import Chat
+from pyrogram.enums import ChatType
+from pyrogram.types import Chat, User
 
 from userbot.modules.avatars import note_avatar
 from userbot.modules.capture.context import CaptureContext
-from userbot.modules.groups.repository import insert_chat_history
-from userbot.modules.profiles.parse import snapshot_from_chat
+from userbot.modules.groups.repository import record_chat_meta
+from userbot.modules.profiles.parse import snapshot_from_chat, snapshot_from_high_level
 from userbot.modules.profiles.repository import write_profile
 from userbot.modules.raw_json import raw_json
 
 
+async def save_user(ctx: CaptureContext, user: User) -> None:
+    fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
+    if not await ctx.peer_identity.changed(ctx.pool, ctx.account_id, user.id, fields):
+        return
+    await write_profile(
+        ctx.pool,
+        ctx.account_id,
+        user.id,
+        fields,
+        raw_json(user),
+        history=ctx.resolve_peer(user.id, is_bot=bool(user.is_bot)).profile_history,
+    )
+    if photo_file_id and photo_unique_id:
+        await note_avatar(
+            ctx.pool, ctx.account_id, user.id, "peer", photo_unique_id, photo_file_id
+        )
+
+
 async def save_private(ctx: CaptureContext, chat: Chat) -> bool:
     chat_id = chat.id or 0
     fields, photo_file_id, photo_unique_id = snapshot_from_chat(chat)
-    await write_profile(ctx.pool, ctx.account_id, chat_id, fields, raw_json(chat))
+    is_bot = chat.type is ChatType.BOT
+    await write_profile(
+        ctx.pool,
+        ctx.account_id,
+        chat_id,
+        fields,
+        raw_json(chat),
+        history=ctx.resolve_peer(chat_id, is_bot=is_bot).profile_history,
+    )
     if photo_file_id and photo_unique_id:
         await note_avatar(
             ctx.pool, ctx.account_id, chat_id, "peer", photo_unique_id, photo_file_id
@@ -26,15 +53,13 @@ async def save_group(ctx: CaptureContext, chat: Chat) -> None:
     photo = chat.photo
     photo_unique_id = photo.big_photo_unique_id if photo else None
     photo_file_id = photo.big_file_id if photo else None
-    await insert_chat_history(
+    await record_chat_meta(
         ctx.pool,
         ctx.account_id,
         chat_id,
         0,
-        "meta",
         chat.title,
         photo_unique_id,
-        None,
         datetime.now(UTC),
         raw_json(chat),
     )

backend/src/userbot/modules/raw_json.py

@@ -1,8 +1,23 @@
+import json
+from typing import Any
+
 from pyrogram.raw.core import TLObject
-from pyrogram.types import Object
+from pyrogram.types import Contact, Object
 
 EMPTY_JSON = "{}"
+PHONE_KEY = "phone_number"
+
+
+def _default(obj: Any) -> Any:  # noqa: ANN401
+    if isinstance(obj, TLObject):
+        return TLObject.default(obj)
+    data = Object.default(obj)
+    if isinstance(obj, Contact) and isinstance(data, dict) and PHONE_KEY in data:
+        data[PHONE_KEY] = obj.phone_number
+    return data
 
 
 def raw_json(obj: Object | TLObject | None) -> str:
-    return EMPTY_JSON if obj is None else str(obj)
+    if obj is None:
+        return EMPTY_JSON
+    return json.dumps(obj, indent=4, default=_default, ensure_ascii=False)

backend/src/userbot/modules/read_receipts/repository.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import asyncpg
 
@@ -9,6 +9,26 @@ VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
 ON CONFLICT (account_id, chat_id, message_id, ts) DO NOTHING
 """
 
+_UPSERT_READ = """
+INSERT INTO message_reads
+    (account_id, chat_id, message_id, reader_id, read_at, source, fetched_at)
+VALUES ($1, $2, $3, $4, $5, $6, now())
+ON CONFLICT (account_id, chat_id, message_id, reader_id) DO UPDATE SET
+    read_at = EXCLUDED.read_at,
+    source = EXCLUDED.source,
+    fetched_at = EXCLUDED.fetched_at
+"""
+
+_UPSERT_FETCH = """
+INSERT INTO read_fetches
+    (account_id, chat_id, message_id, outcome, fetched_at, retry_at)
+VALUES ($1, $2, $3, $4, now(), now() + $5::interval)
+ON CONFLICT (account_id, chat_id, message_id) DO UPDATE SET
+    outcome = EXCLUDED.outcome,
+    fetched_at = EXCLUDED.fetched_at,
+    retry_at = EXCLUDED.retry_at
+"""
+
 
 async def insert_read(  # noqa: PLR0913
     pool: asyncpg.Pool,
@@ -34,3 +54,33 @@ async def insert_listened(  # noqa: PLR0913
     await pool.execute(
         _INSERT, account_id, chat_id, message_id, ts, reader_id, "listened", raw
     )
+
+
+async def save_reads(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    account_id: int,
+    chat_id: int,
+    message_id: int,
+    source: str,
+    reads: list[tuple[int, datetime]],
+) -> None:
+    await pool.executemany(
+        _UPSERT_READ,
+        [
+            (account_id, chat_id, message_id, reader_id, read_at, source)
+            for reader_id, read_at in reads
+        ],
+    )
+
+
+async def save_fetch(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    account_id: int,
+    chat_id: int,
+    message_id: int,
+    outcome: str,
+    retry_after: timedelta,
+) -> None:
+    await pool.execute(
+        _UPSERT_FETCH, account_id, chat_id, message_id, outcome, retry_after
+    )

backend/src/userbot/modules/stories/repository.py

@@ -6,7 +6,7 @@ _UPSERT_STORY = """
 INSERT INTO stories
     (account_id, peer_id, story_id, date, expire_date, caption, media_kind,
      storage_key, file_size, downloaded, views, pinned, deleted, raw, observed_at)
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14::jsonb, now())
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, false, $13::jsonb, now())
 ON CONFLICT (account_id, peer_id, story_id) DO UPDATE SET
     date = EXCLUDED.date,
     expire_date = EXCLUDED.expire_date,
@@ -17,10 +17,17 @@ ON CONFLICT (account_id, peer_id, story_id) DO UPDATE SET
     downloaded = stories.downloaded OR EXCLUDED.downloaded,
     views = EXCLUDED.views,
     pinned = EXCLUDED.pinned,
-    deleted = EXCLUDED.deleted,
+    deleted = false,
     raw = EXCLUDED.raw
 """
 
+_MARK_DELETED = """
+INSERT INTO stories
+    (account_id, peer_id, story_id, downloaded, pinned, deleted, raw, observed_at)
+VALUES ($1, $2, $3, false, false, true, '{}'::jsonb, now())
+ON CONFLICT (account_id, peer_id, story_id) DO UPDATE SET deleted = true
+"""
+
 
 async def is_downloaded(
     pool: asyncpg.Pool, account_id: int, peer_id: int, story_id: int
@@ -51,7 +58,6 @@ async def upsert_story(  # noqa: PLR0913
     raw: str,
     *,
     pinned: bool,
-    deleted: bool,
     downloaded: bool,
 ) -> None:
     await pool.execute(
@@ -68,6 +74,11 @@ async def upsert_story(  # noqa: PLR0913
         downloaded,
         views,
         pinned,
-        deleted,
         raw,
     )
+
+
+async def mark_deleted(
+    pool: asyncpg.Pool, account_id: int, peer_id: int, story_id: int
+) -> None:
+    await pool.execute(_MARK_DELETED, account_id, peer_id, story_id)

backend/src/userbot/modules/stories/service.py

@@ -17,13 +17,18 @@ def story_peer_id(story: Story) -> int:
 
 async def save_story(client: Client, capture: CaptureContext, story: Story) -> None:
     peer_id = story_peer_id(story)
+    if story.deleted:
+        await repository.mark_deleted(
+            capture.pool, capture.account_id, peer_id, story.id
+        )
+        return
     storage_key: str | None = None
     file_size: int | None = None
     downloaded = False
     stored = await repository.is_downloaded(
         capture.pool, capture.account_id, peer_id, story.id
     )
-    if not (stored or story.deleted or story.media is None):
+    if not (stored or story.media is None):
         data = await download_bytes(client, story)
         if data is not None:
             storage_key = capture.storage.put(data)
@@ -43,6 +48,5 @@ async def save_story(client: Client, capture: CaptureContext, story: Story) -> N
         story.views,
         raw_json(story.raw),
         pinned=bool(story.pinned),
-        deleted=bool(story.deleted),
         downloaded=downloaded,
     )

backend/src/userbot/modules/uptime/repository.py

@@ -0,0 +1,16 @@
+import asyncpg
+
+_HEARTBEAT = """
+WITH extended AS (
+    UPDATE capture_uptime SET seen_at = now()
+    WHERE account_id = $1 AND seen_at >= now() - interval '150 seconds'
+    RETURNING account_id
+)
+INSERT INTO capture_uptime (account_id, started_at, seen_at)
+SELECT $1, now(), now()
+WHERE NOT EXISTS (SELECT 1 FROM extended)
+"""
+
+
+async def heartbeat(pool: asyncpg.Pool, account_id: int) -> None:
+    await pool.execute(_HEARTBEAT, account_id)

backend/src/userbot/modules/watches/cache.py

@@ -1,60 +1,182 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING, Any
 
+from userbot.modules.watches import repository
 from userbot.modules.watches.evaluator import (
-    KIND_KEYWORD,
-    KIND_PEER_ONLINE,
+    Rules,
+    Seen,
+    build_rules,
+    match_chat_scope,
     match_keyword,
-    match_peer_online,
+    match_new_message,
 )
 from utils.logging import logger
 from utils.read import watches as watches_read
+from utils.read.accounts import self_user_id
 
 if TYPE_CHECKING:
+    from collections.abc import Coroutine
+
     import asyncpg
 
-    from utils.read.models import WatchView
+ONLINE_QUIET = timedelta(minutes=15)
+PREVIEW_CHARS = 500
+
+
+def _message_payload(event: str, seen: Seen) -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        "event": event,
+        "chat_id": seen.chat_id,
+        "message_id": seen.message_id,
+        "sender_id": seen.sender_id,
+        "text": (seen.text or "")[:PREVIEW_CHARS] or None,
+    }
+    if seen.date is not None:
+        payload["date"] = seen.date.isoformat()
+    return payload
 
 
 class WatchCache:
     def __init__(self, pool: asyncpg.Pool, account_id: int) -> None:
         self._pool = pool
         self._account_id = account_id
-        self.watches: list[WatchView] = []
-        self._online: set[int] = set()
+        self.rules = Rules()
+        self._self_id: int | None = None
+        self._last_seen: dict[int, datetime] = {}
 
     async def refresh(self) -> None:
         rows = await watches_read.list_watches(self._pool, self._account_id)
-        self.watches = [watch for watch in rows if watch.enabled]
-        logger.info(f"[green]Watches cached:[/] {len(self.watches)}")
+        self.rules = build_rules(rows)
+        self._self_id = await self_user_id(self._pool, self._account_id)
+        logger.info(f"[green]Watches cached:[/] {len(self.rules)}")
 
-    async def on_text(self, chat_id: int, message_id: int, text: str | None) -> None:
-        if not text:
-            return
-        for watch in self.watches:
-            if watch.kind == KIND_KEYWORD and match_keyword(
-                text, chat_id, watch.params
-            ):
-                await watches_read.insert_alert(
-                    self._pool,
-                    self._account_id,
-                    watch.id,
-                    {"chat_id": chat_id, "message_id": message_id},
-                    dedup_key=f"{chat_id}:{message_id}",
+    def _own(self, seen: Seen) -> bool:
+        return seen.outgoing or (
+            self._self_id is not None and seen.sender_id == self._self_id
+        )
+
+    async def _guard(self, work: Coroutine[Any, Any, None]) -> None:
+        try:
+            await work
+        except Exception:
+            logger.exception("[red]Watch evaluation failed[/]")
+
+    async def _fire(self, watch_id: int, payload: dict[str, Any], key: str) -> None:
+        await watches_read.insert_alert(
+            self._pool, self._account_id, watch_id, payload, dedup_key=key
+        )
+
+    async def _keyword(self, event: str, seen: Seen) -> None:
+        for rule in self.rules.keyword:
+            if match_keyword(rule, seen):
+                await self._fire(
+                    rule.watch_id,
+                    _message_payload(event, seen),
+                    f"{seen.chat_id}:{seen.message_id}",
                 )
 
-    async def on_status(self, peer_id: int, *, is_online: bool) -> None:
-        if not is_online:
-            self._online.discard(peer_id)
+    async def on_message(self, seen: Seen) -> None:
+        if self._own(seen):
             return
-        if peer_id in self._online:
-            return
-        self._online.add(peer_id)
-        for watch in self.watches:
-            if watch.kind == KIND_PEER_ONLINE and match_peer_online(
-                peer_id, watch.params
-            ):
-                await watches_read.insert_alert(
-                    self._pool, self._account_id, watch.id, {"peer_id": peer_id}
+        await self._guard(self._on_message(seen))
+
+    async def _on_message(self, seen: Seen) -> None:
+        await self._keyword("message", seen)
+        for watch_id, params in self.rules.new_message:
+            if match_new_message(params, seen):
+                await self._fire(
+                    watch_id,
+                    _message_payload("message", seen),
+                    f"{seen.chat_id}:{seen.message_id}",
                 )
+
+    async def on_edit(self, seen: Seen) -> None:
+        if self._own(seen):
+            return
+        await self._guard(self._on_edit(seen))
+
+    async def _on_edit(self, seen: Seen) -> None:
+        await self._keyword("edit", seen)
+        if seen.edit_date is None:
+            return
+        stamp = int(seen.edit_date.timestamp())
+        for watch_id, params in self.rules.edited:
+            if match_chat_scope(params, seen):
+                await self._fire(
+                    watch_id,
+                    _message_payload("edit", seen),
+                    f"{seen.chat_id}:{seen.message_id}:{stamp}",
+                )
+
+    async def on_transcript(self, chat_id: int, message_id: int, text: str) -> None:
+        if self.rules.keyword:
+            await self._guard(self._on_transcript(chat_id, message_id, text))
+
+    async def _on_transcript(self, chat_id: int, message_id: int, text: str) -> None:
+        sender_id = await repository.message_sender(
+            self._pool, self._account_id, chat_id, message_id
+        )
+        seen = Seen(chat_id, message_id, sender_id, text)
+        if not self._own(seen):
+            await self._keyword("transcript", seen)
+
+    async def on_deleted(self, chat_id: int | None, message_ids: list[int]) -> None:
+        if self.rules.deleted and message_ids:
+            await self._guard(self._on_deleted(chat_id, message_ids))
+
+    async def _on_deleted(self, chat_id: int | None, message_ids: list[int]) -> None:
+        rows = await repository.deleted_messages(
+            self._pool, self._account_id, chat_id, message_ids
+        )
+        for row in rows:
+            seen = Seen(
+                row["chat_id"],
+                row["message_id"],
+                row["sender_id"],
+                row["text"],
+                date=row["date"],
+            )
+            if self._own(seen):
+                continue
+            for watch_id, params in self.rules.deleted:
+                if match_chat_scope(params, seen):
+                    await self._fire(
+                        watch_id,
+                        _message_payload("delete", seen),
+                        f"{seen.chat_id}:{seen.message_id}",
+                    )
+
+    async def on_status(
+        self, peer_id: int, status: str, at: datetime, last_online: datetime | None
+    ) -> None:
+        watch_ids = [
+            watch_id
+            for watch_id, params in self.rules.online
+            if params.peer_id == peer_id
+        ]
+        if not watch_ids:
+            return
+        if status != "online":
+            self._last_seen[peer_id] = (last_online or at).astimezone(UTC)
+            return
+        await self._guard(self._on_online(peer_id, at, watch_ids))
+
+    async def _on_online(
+        self, peer_id: int, at: datetime, watch_ids: list[int]
+    ) -> None:
+        stored = await repository.last_seen_before(
+            self._pool, self._account_id, peer_id, at
+        )
+        known = [value for value in (stored, self._last_seen.get(peer_id)) if value]
+        previous = max(known, default=None)
+        self._last_seen[peer_id] = at
+        if previous is not None and at - previous < ONLINE_QUIET:
+            return
+        session = int((previous or at).timestamp())
+        payload: dict[str, Any] = {"event": "online", "peer_id": peer_id}
+        if previous is not None:
+            payload["since"] = previous.isoformat()
+        for watch_id in watch_ids:
+            await self._fire(watch_id, payload, f"{peer_id}:{session}")

backend/src/userbot/modules/watches/evaluator.py

@@ -1,24 +1,104 @@
 import re
-from typing import Any
+from dataclasses import dataclass, field
+from datetime import datetime
 
-KIND_KEYWORD = "keyword"
-KIND_PEER_ONLINE = "peer_online"
+from utils.logging import logger
+from utils.read.models import WatchView
+from utils.watch_rules import (
+    MAX_TEXT,
+    DeletedParams,
+    EditedParams,
+    KeywordParams,
+    NewMessageParams,
+    PeerOnlineParams,
+    WatchParamsError,
+    compile_pattern,
+    parse_params,
+)
 
 
-def match_keyword(text: str, chat_id: int, params: dict[str, Any]) -> bool:
-    target_chat = params.get("chat_id")
-    if target_chat is not None and target_chat != chat_id:
-        return False
-    pattern = params.get("pattern")
-    if not pattern:
-        return False
-    if params.get("regex"):
+@dataclass(frozen=True, slots=True)
+class Seen:
+    chat_id: int
+    message_id: int
+    sender_id: int | None
+    text: str | None
+    outgoing: bool = False
+    date: datetime | None = None
+    edit_date: datetime | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class KeywordRule:
+    watch_id: int
+    params: KeywordParams
+    regex: re.Pattern[str] | None
+
+
+@dataclass(slots=True)
+class Rules:
+    keyword: list[KeywordRule] = field(default_factory=list)
+    new_message: list[tuple[int, NewMessageParams]] = field(default_factory=list)
+    deleted: list[tuple[int, DeletedParams]] = field(default_factory=list)
+    edited: list[tuple[int, EditedParams]] = field(default_factory=list)
+    online: list[tuple[int, PeerOnlineParams]] = field(default_factory=list)
+
+    def __len__(self) -> int:
+        return (
+            len(self.keyword)
+            + len(self.new_message)
+            + len(self.deleted)
+            + len(self.edited)
+            + len(self.online)
+        )
+
+
+def build_rules(watches: list[WatchView]) -> Rules:
+    rules = Rules()
+    for watch in watches:
+        if not watch.enabled:
+            continue
         try:
-            return re.search(pattern, text) is not None
-        except re.error:
-            return False
-    return pattern.casefold() in text.casefold()
+            params = parse_params(watch.kind, watch.params)
+        except WatchParamsError as exc:
+            logger.warning(f"[yellow]Watch {watch.id} skipped:[/] {exc}")
+            continue
+        match params:
+            case KeywordParams():
+                regex = compile_pattern(params.pattern) if params.regex else None
+                rules.keyword.append(KeywordRule(watch.id, params, regex))
+            case NewMessageParams():
+                rules.new_message.append((watch.id, params))
+            case DeletedParams():
+                rules.deleted.append((watch.id, params))
+            case EditedParams():
+                rules.edited.append((watch.id, params))
+            case PeerOnlineParams():
+                rules.online.append((watch.id, params))
+    return rules
 
 
-def match_peer_online(peer_id: int, params: dict[str, Any]) -> bool:
-    return params.get("peer_id") == peer_id
+def _in_scope(chat_id: int | None, sender_id: int | None, seen: Seen) -> bool:
+    if chat_id is not None and chat_id != seen.chat_id:
+        return False
+    return sender_id is None or sender_id == seen.sender_id
+
+
+def match_keyword(rule: KeywordRule, seen: Seen) -> bool:
+    params = rule.params
+    if not seen.text or not _in_scope(params.chat_id, params.sender_id, seen):
+        return False
+    sample = seen.text[:MAX_TEXT]
+    if rule.regex is not None:
+        return rule.regex.search(sample) is not None
+    return params.pattern.casefold() in sample.casefold()
+
+
+def match_new_message(params: NewMessageParams, seen: Seen) -> bool:
+    return _in_scope(params.chat_id, params.sender_id, seen)
+
+
+def match_chat_scope(params: DeletedParams | EditedParams, seen: Seen) -> bool:
+    if params.chat_id is None:
+        return seen.chat_id > 0
+    return params.chat_id == seen.chat_id

backend/src/userbot/modules/watches/repository.py

@@ -0,0 +1,48 @@
+from datetime import datetime
+
+import asyncpg
+
+from userbot.modules.capture.repository import CHANNEL_ID_THRESHOLD
+
+_DELETED = """
+SELECT DISTINCT ON (chat_id, message_id) chat_id, message_id, sender_id, text, date
+FROM messages
+WHERE account_id = $1 AND message_id = ANY($2::bigint[])
+  AND deleted_at IS NOT NULL AND {scope}
+ORDER BY chat_id, message_id, date DESC
+"""
+
+_SENDER = """
+SELECT sender_id FROM messages
+WHERE account_id = $1 AND chat_id = $2 AND message_id = $3
+ORDER BY date DESC LIMIT 1
+"""
+
+_LAST_SEEN = """
+SELECT CASE WHEN status = 'online' THEN ts ELSE coalesce(last_online_date, ts) END
+FROM presence
+WHERE account_id = $1 AND peer_id = $2 AND ts < $3
+ORDER BY ts DESC LIMIT 1
+"""
+
+
+async def deleted_messages(
+    pool: asyncpg.Pool, account_id: int, chat_id: int | None, message_ids: list[int]
+) -> list[asyncpg.Record]:
+    if chat_id is None:
+        query = _DELETED.format(scope="chat_id > $3")
+        return await pool.fetch(query, account_id, message_ids, CHANNEL_ID_THRESHOLD)
+    query = _DELETED.format(scope="chat_id = $3")
+    return await pool.fetch(query, account_id, message_ids, chat_id)
+
+
+async def message_sender(
+    pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
+) -> int | None:
+    return await pool.fetchval(_SENDER, account_id, chat_id, message_id)
+
+
+async def last_seen_before(
+    pool: asyncpg.Pool, account_id: int, peer_id: int, at: datetime
+) -> datetime | None:
+    return await pool.fetchval(_LAST_SEEN, account_id, peer_id, at)

backend/src/userbot/modules/watches/seen.py

@@ -0,0 +1,22 @@
+from datetime import UTC, datetime
+
+from pyrogram.types import Message
+
+from userbot.modules.capture.message import sender_id
+from userbot.modules.watches.evaluator import Seen
+
+
+def utc(moment: datetime | None) -> datetime | None:
+    return moment.astimezone(UTC) if moment else None
+
+
+def seen_message(chat_id: int, message: Message) -> Seen:
+    return Seen(
+        chat_id=chat_id,
+        message_id=message.id,
+        sender_id=sender_id(message),
+        text=message.text or message.caption,
+        outgoing=bool(message.outgoing),
+        date=utc(message.date),
+        edit_date=utc(message.edit_date),
+    )

backend/src/userbot/runner.py

@@ -11,10 +11,13 @@ import uvloop
 
 from dependencies.container import container
 from userbot.modules.capture import CaptureContext, build_capture_context
+from userbot.modules.catchup import SavedState, catch_up, read_saved_state
 from userbot.modules.client import PyroClient
 from userbot.modules.jobs import JobConsumer
+from userbot.modules.uptime import repository as uptime
 from utils.env import env
 from utils.jobs import enqueue
+from utils.listen import Listener
 from utils.logging import logger, setup_logging
 from utils.read.accounts import (
     ACCOUNTS_CHANGED_CHANNEL,
@@ -28,12 +31,14 @@ from utils.storage import ContentAddressedStorage
 setup_logging()
 
 SESSION_FLUSH_INTERVAL = 60
+SESSION_RETRY_INTERVAL = 60
+UPTIME_INTERVAL = 60
 
 
 @dataclass
 class RunningAccount:
     client: PyroClient
-    consumer_task: asyncio.Task
+    tasks: list[asyncio.Task]
     device_model: str | None
 
 
@@ -54,6 +59,7 @@ class AccountRegistry:
         self._pool = pool
         self._storage = storage
         self._running: dict[str, RunningAccount] = {}
+        self._saved: dict[str, SavedState] = {}
         self._lock = asyncio.Lock()
 
     @property
@@ -86,6 +92,8 @@ class AccountRegistry:
 
     async def _start(self, path: Path, device_model: str | None) -> None:
         session_name = path.stem
+        if session_name not in self._saved:
+            self._saved[session_name] = await asyncio.to_thread(read_saved_state, path)
         client = PyroClient(
             session_name, workdir=str(path.parent), device_model=device_model
         )
@@ -100,14 +108,20 @@ class AccountRegistry:
                 client, self._pool, self._storage, account_id
             )
         except Exception:
-            logger.exception(f"[red]Failed to start session:[/] {session_name}")
+            logger.exception(
+                f"[red]Failed to start session:[/] {session_name}, "
+                f"retrying in {SESSION_RETRY_INTERVAL}s"
+            )
             with contextlib.suppress(Exception):
                 await client.stop()
             return
         consumer = JobConsumer(client, self._pool, account_id)
-        self._running[session_name] = RunningAccount(
-            client, asyncio.create_task(consumer.run()), device_model
-        )
+        saved = self._saved.pop(session_name)
+        tasks = [
+            asyncio.create_task(consumer.run()),
+            asyncio.create_task(catch_up(client, saved)),
+        ]
+        self._running[session_name] = RunningAccount(client, tasks, device_model)
         logger.info(f"[green]Client started:[/] {me.full_name} ({me.id})")
         await enqueue(self._pool, account_id, "sync_dialogs", {})
         await enqueue(self._pool, account_id, "sync_contacts", {})
@@ -116,7 +130,8 @@ class AccountRegistry:
         account = self._running.pop(session_name, None)
         if account is None:
             return
-        await _cancel(account.consumer_task)
+        for task in account.tasks:
+            await _cancel(task)
         with contextlib.suppress(Exception):
             await account.client.stop()
         logger.info(f"[yellow]Client stopped:[/] {session_name}")
@@ -124,7 +139,8 @@ class AccountRegistry:
     async def _log_out(self, path: Path) -> None:
         account = self._running.pop(path.stem, None)
         if account is not None:
-            await _cancel(account.consumer_task)
+            for task in account.tasks:
+                await _cancel(task)
             client = account.client
         else:
             client = PyroClient(
@@ -148,9 +164,25 @@ async def _flush_sessions(registry: AccountRegistry) -> None:
                 await client.storage.save()
 
 
-async def _listen_changes(
-    registry: AccountRegistry, tasks: set[asyncio.Task]
-) -> asyncpg.Connection:
+async def _retry_sessions(registry: AccountRegistry) -> None:
+    while True:
+        await asyncio.sleep(SESSION_RETRY_INTERVAL)
+        try:
+            await registry.sync()
+        except Exception:
+            logger.exception("[red]Session sync failed[/]")
+
+
+async def _keep_uptime(registry: AccountRegistry, pool: asyncpg.Pool) -> None:
+    while True:
+        for client in registry.clients:
+            if client.capture is not None and client.is_connected:
+                with contextlib.suppress(Exception):
+                    await uptime.heartbeat(pool, client.capture.account_id)
+        await asyncio.sleep(UPTIME_INTERVAL)
+
+
+def _change_listener(registry: AccountRegistry, tasks: set[asyncio.Task]) -> Listener:
     def spawn(coro: Coroutine[Any, Any, None]) -> None:
         task = asyncio.create_task(coro)
         tasks.add(task)
@@ -163,26 +195,29 @@ async def _listen_changes(
             if client.capture is not None:
                 spawn(make_coro(client.capture))
 
-    def on_policy(
-        _conn: asyncpg.Connection, _pid: int, _channel: str, _payload: str
-    ) -> None:
+    def on_policy(_payload: str) -> None:
         reload(lambda capture: capture.reload_policies())
 
-    def on_watch(
-        _conn: asyncpg.Connection, _pid: int, _channel: str, _payload: str
-    ) -> None:
+    def on_watch(_payload: str) -> None:
         reload(lambda capture: capture.watches.refresh())
 
-    def on_accounts(
-        _conn: asyncpg.Connection, _pid: int, _channel: str, _payload: str
-    ) -> None:
+    def on_accounts(_payload: str) -> None:
         spawn(registry.sync())
 
-    conn = await asyncpg.connect(dsn=env.db.connection_url)
-    await conn.add_listener("policy_changed", on_policy)
-    await conn.add_listener(WATCHES_CHANGED_CHANNEL, on_watch)
-    await conn.add_listener(ACCOUNTS_CHANGED_CHANNEL, on_accounts)
-    return conn
+    def on_reconnect() -> None:
+        on_policy("")
+        on_watch("")
+        on_accounts("")
+
+    return Listener(
+        "userbot",
+        {
+            "policy_changed": on_policy,
+            WATCHES_CHANGED_CHANNEL: on_watch,
+            ACCOUNTS_CHANGED_CHANNEL: on_accounts,
+        },
+        on_reconnect,
+    )
 
 
 async def runner() -> None:
@@ -191,22 +226,24 @@ async def runner() -> None:
 
     registry = AccountRegistry(pool, storage)
     tasks: set[asyncio.Task] = set()
-    listen_conn: asyncpg.Connection | None = None
+    listener = _change_listener(registry, tasks)
     stop = asyncio.Event()
     asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, stop.set)
     flush = asyncio.create_task(_flush_sessions(registry))
+    retry = asyncio.create_task(_retry_sessions(registry))
+    beat = asyncio.create_task(_keep_uptime(registry, pool))
     try:
         await registry.sync()
         if not registry.clients:
             logger.warning("[yellow]No sessions yet. Add an account in the web UI.[/]")
-        listen_conn = await _listen_changes(registry, tasks)
+        await listener.start()
         logger.info("[green]Userbot running.[/]")
         await stop.wait()
     finally:
         await _cancel(flush)
-        if listen_conn is not None:
-            with contextlib.suppress(Exception):
-                await listen_conn.close()
+        await _cancel(retry)
+        await _cancel(beat)
+        await listener.stop()
         await registry.close()
         await container.close()
 

backend/src/utils/db/models.py

@@ -2,8 +2,17 @@ from datetime import datetime
 from enum import StrEnum
 from typing import Any
 
-from sqlalchemy import BigInteger, Column, DateTime, LargeBinary, Text, func
-from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy import (
+    BigInteger,
+    Column,
+    DateTime,
+    ForeignKey,
+    Integer,
+    LargeBinary,
+    Text,
+    func,
+)
+from sqlalchemy.dialects.postgresql import ARRAY, JSONB
 from sqlmodel import Field, SQLModel
 
 
@@ -203,6 +212,9 @@ class Job(SQLModel, table=True):
     finished_at: datetime | None = Field(
         default=None, sa_column=Column(DateTime(timezone=True))
     )
+    run_after: datetime | None = Field(
+        default=None, sa_column=Column(DateTime(timezone=True))
+    )
 
 
 class CapturePolicy(SQLModel, table=True):
@@ -344,7 +356,7 @@ class ChatHistory(SQLModel, table=True):
     title: str | None = None
     photo_unique_id: str | None = None
     actor_id: int | None = Field(default=None, sa_column=Column(BigInteger))
-    ts: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=False))
+    ts: datetime = Field(sa_column=Column(DateTime(timezone=True), primary_key=True))
     raw: dict[str, Any] = Field(
         default_factory=dict, sa_column=Column(JSONB, nullable=False)
     )
@@ -442,6 +454,7 @@ class Watch(SQLModel, table=True):
     id: int | None = Field(default=None, primary_key=True)
     account_id: int
     kind: str
+    name: str | None = Field(default=None, sa_column=Column(Text, nullable=True))
     params: dict[str, Any] = Field(
         default_factory=dict, sa_column=Column(JSONB, nullable=False)
     )
@@ -466,7 +479,11 @@ class Alert(SQLModel, table=True):
 
     id: int | None = Field(default=None, primary_key=True)
     account_id: int
-    watch_id: int = Field(foreign_key="watches.id")
+    watch_id: int = Field(
+        sa_column=Column(
+            Integer, ForeignKey("watches.id", ondelete="CASCADE"), nullable=False
+        )
+    )
     ts: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=False))
     payload: dict[str, Any] = Field(
         default_factory=dict, sa_column=Column(JSONB, nullable=False)
@@ -543,3 +560,62 @@ class FileShareHit(SQLModel, table=True):
     ip: str | None = None
     user_agent: str | None = None
     counted: bool = False
+
+
+class Webhook(SQLModel, table=True):
+    __tablename__ = "webhooks"
+
+    id: int | None = Field(default=None, primary_key=True)
+    account_id: int
+    url: str = Field(sa_column=Column(Text, nullable=False))
+    secret: str = Field(sa_column=Column(Text, nullable=False))
+    enabled: bool = True
+    watch_ids: list[int] | None = Field(
+        default=None, sa_column=Column(ARRAY(Integer), nullable=True)
+    )
+    created_at: datetime = Field(
+        sa_column=Column(
+            DateTime(timezone=True), nullable=False, server_default=func.now()
+        )
+    )
+    updated_at: datetime = Field(
+        sa_column=Column(
+            DateTime(timezone=True), nullable=False, server_default=func.now()
+        )
+    )
+
+
+class WebhookDelivery(SQLModel, table=True):
+    __tablename__ = "webhook_deliveries"
+
+    id: int | None = Field(default=None, sa_column=Column(BigInteger, primary_key=True))
+    webhook_id: int = Field(
+        sa_column=Column(
+            Integer, ForeignKey("webhooks.id", ondelete="CASCADE"), nullable=False
+        )
+    )
+    alert_id: int | None = Field(
+        default=None,
+        sa_column=Column(
+            Integer, ForeignKey("alerts.id", ondelete="CASCADE"), nullable=True
+        ),
+    )
+    event: str = Field(sa_column=Column(Text, nullable=False))
+    status: str = Field(default="pending", sa_column=Column(Text, nullable=False))
+    attempts: int = 0
+    next_attempt_at: datetime = Field(
+        sa_column=Column(
+            DateTime(timezone=True), nullable=False, server_default=func.now()
+        )
+    )
+    response_code: int | None = None
+    last_error: str | None = Field(default=None, sa_column=Column(Text))
+    body: str | None = Field(default=None, sa_column=Column(Text))
+    created_at: datetime = Field(
+        sa_column=Column(
+            DateTime(timezone=True), nullable=False, server_default=func.now()
+        )
+    )
+    finished_at: datetime | None = Field(
+        default=None, sa_column=Column(DateTime(timezone=True))
+    )

backend/src/utils/env.py

@@ -41,6 +41,7 @@ class ApiSettings(BaseSettings):
     host: str = "0.0.0.0"  # noqa: S104
     port: int = 8080
     static_dir: str = "static"
+    public_url: str | None = None
 
 
 class AuthSettings(BaseSettings):

backend/src/utils/events.py

@@ -4,6 +4,7 @@ from typing import Literal
 import asyncpg
 
 BG_EVENTS_CHANNEL = "bg_events"
+MESSAGE_IDS_PER_EVENT = 500
 
 EventKind = Literal["message", "edit", "delete", "reaction", "presence", "receipt"]
 
@@ -22,8 +23,15 @@ async def notify_bg_event(  # noqa: PLR0913
         payload["chat_id"] = chat_id
     if message_id is not None:
         payload["message_id"] = message_id
-    if message_ids is not None:
-        payload["message_ids"] = message_ids
+    if message_ids is None:
+        await _notify(pool, payload)
+        return
+    for start in range(0, len(message_ids), MESSAGE_IDS_PER_EVENT):
+        chunk = message_ids[start : start + MESSAGE_IDS_PER_EVENT]
+        await _notify(pool, {**payload, "message_ids": chunk})
+
+
+async def _notify(pool: asyncpg.Pool, payload: dict[str, object]) -> None:
     await pool.execute(
         "SELECT pg_notify($1, $2)", BG_EVENTS_CHANNEL, json.dumps(payload)
     )

backend/src/utils/files.py

@@ -191,6 +191,11 @@ def story_file_name(peer_id: int, story_id: int, media_kind: str | None) -> str:
     return f"story_{peer_id}_{story_id}{extension_for(media_kind or 'photo', None)}"
 
 
+def sniff_media_kind(head: bytes) -> str:
+    is_video = head[4:8] == b"ftyp" or head.startswith(b"\x1aE\xdf\xa3")
+    return "video" if is_video else "photo"
+
+
 def resolve_mime(
     kind: str | None, mime: str | None, file_name: str | None = None
 ) -> str:

backend/src/utils/jobs.py

@@ -1,4 +1,5 @@
 import json
+from datetime import timedelta
 from typing import Any
 
 import asyncpg
@@ -7,15 +8,22 @@ JOBS_CHANGED_CHANNEL = "jobs_changed"
 
 
 async def enqueue(
-    pool: asyncpg.Pool, account_id: int, kind: str, params: dict[str, Any]
+    pool: asyncpg.Pool,
+    account_id: int,
+    kind: str,
+    params: dict[str, Any],
+    *,
+    cooldown: timedelta | None = None,
 ) -> int:
     params_json = json.dumps(params)
     existing = await pool.fetchval(
         "SELECT id FROM jobs WHERE account_id = $1 AND kind = $2 "
-        "AND params = $3::jsonb AND status IN ('pending', 'running') LIMIT 1",
+        "AND params = $3::jsonb AND (status IN ('pending', 'running') "
+        "OR created_at > now() - $4::interval) LIMIT 1",
         account_id,
         kind,
         params_json,
+        cooldown,
     )
     if existing is not None:
         return existing

backend/src/utils/listen.py

@@ -0,0 +1,87 @@
+import asyncio
+import contextlib
+from collections.abc import Callable, Mapping
+
+import asyncpg
+
+from utils.env import env
+from utils.logging import logger
+
+RETRY_MIN_SECONDS = 1.0
+RETRY_MAX_SECONDS = 60.0
+PING_INTERVAL_SECONDS = 60.0
+PING_TIMEOUT_SECONDS = 10.0
+CLOSE_TIMEOUT_SECONDS = 5.0
+
+OnNotify = Callable[[str], None]
+
+
+class Listener:
+    def __init__(
+        self,
+        name: str,
+        channels: Mapping[str, OnNotify],
+        on_reconnect: Callable[[], None] | None = None,
+    ) -> None:
+        self._name = name
+        self._channels = channels
+        self._on_reconnect = on_reconnect
+        self._task: asyncio.Task | None = None
+        self._ready = asyncio.Event()
+        self._listening = False
+
+    async def start(self) -> None:
+        self._task = asyncio.create_task(self._run())
+        await self._ready.wait()
+
+    async def stop(self) -> None:
+        if self._task is None:
+            return
+        self._task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await self._task
+        self._task = None
+
+    async def _run(self) -> None:
+        delay = RETRY_MIN_SECONDS
+        while True:
+            try:
+                await self._listen()
+            except Exception as exc:
+                if self._listening:
+                    delay = RETRY_MIN_SECONDS
+                logger.warning(
+                    f"{self._name}: no listen connection ({exc!r}), "
+                    f"reconnecting in {delay:.0f}s"
+                )
+            self._listening = False
+            self._ready.set()
+            await asyncio.sleep(delay)
+            delay = min(delay * 2, RETRY_MAX_SECONDS)
+
+    async def _listen(self) -> None:
+        conn = await asyncpg.connect(dsn=env.db.connection_url)
+        try:
+            lost = asyncio.Event()
+            conn.add_termination_listener(lambda _conn: lost.set())
+            for channel, on_notify in self._channels.items():
+                await conn.add_listener(
+                    channel,
+                    lambda _conn, _pid, _channel, payload, f=on_notify: f(payload),
+                )
+            logger.info(f"{self._name}: listening on {', '.join(self._channels)}")
+            self._listening = True
+            if self._ready.is_set() and self._on_reconnect is not None:
+                self._on_reconnect()
+            self._ready.set()
+            while True:
+                try:
+                    await asyncio.wait_for(lost.wait(), PING_INTERVAL_SECONDS)
+                except TimeoutError:
+                    await conn.fetchval("SELECT 1", timeout=PING_TIMEOUT_SECONDS)
+                else:
+                    msg = "closed by the server"
+                    raise ConnectionError(msg)
+        finally:
+            with contextlib.suppress(Exception):
+                await conn.close(timeout=CLOSE_TIMEOUT_SECONDS)

backend/src/utils/policy/defaults.py

@@ -1,8 +1,8 @@
 from utils.policy.models import CaptureToggles, ChatKind
 
 DEFAULTS: dict[ChatKind, CaptureToggles] = {
-    ChatKind.CHANNEL: CaptureToggles(),
-    ChatKind.GROUP: CaptureToggles(messages=True),
+    ChatKind.CHANNEL: CaptureToggles(track_edits_deletes=True),
+    ChatKind.GROUP: CaptureToggles(messages=True, track_edits_deletes=True),
     ChatKind.DM: CaptureToggles(
         messages=True,
         media=True,

backend/src/utils/read/analytics.py

@@ -4,23 +4,31 @@ import asyncpg
 
 from utils.read.accounts import self_user_id
 from utils.read.models import ResponseStats, VolumeBucket
+from utils.read.period import period_bounds
+
+_VOLUME = """
+WITH counts AS (
+    SELECT (date AT TIME ZONE $5)::date AS day, count(*) AS total,
+           count(*) FILTER (WHERE sender_id = $3) AS outgoing
+    FROM messages
+    WHERE account_id = $1 AND chat_id = $2 AND date >= $4
+    GROUP BY 1
+)
+SELECT local_day::date AS bucket, coalesce(total, 0) AS total,
+       coalesce(outgoing, 0) AS outgoing
+FROM generate_series(($4 AT TIME ZONE $5)::date, (now() AT TIME ZONE $5)::date,
+                     interval '1 day') AS local_day
+LEFT JOIN counts ON counts.day = local_day::date
+ORDER BY 1
+"""
 
 
 async def message_volume(
-    pool: asyncpg.Pool, account_id: int, chat_id: int, *, days: int = 90
+    pool: asyncpg.Pool, account_id: int, chat_id: int, *, days: int, tz: str
 ) -> list[VolumeBucket]:
     self_id = await self_user_id(pool, account_id)
-    rows = await pool.fetch(
-        "SELECT date_trunc('day', date) AS bucket, count(*) AS total, "
-        "count(*) FILTER (WHERE sender_id = $3) AS outgoing "
-        "FROM messages "
-        "WHERE account_id = $1 AND chat_id = $2 AND date >= $4 "
-        "GROUP BY bucket ORDER BY bucket",
-        account_id,
-        chat_id,
-        self_id,
-        datetime.now(UTC) - timedelta(days=days),
-    )
+    since, _ = await period_bounds(pool, days, tz)
+    rows = await pool.fetch(_VOLUME, account_id, chat_id, self_id, since, tz)
     return [
         VolumeBucket(
             bucket=row["bucket"],

backend/src/utils/read/annotations.py

@@ -1,9 +1,34 @@
+import re
+from collections import Counter
+
 import asyncpg
 
-from utils.read.models import AnnotationView, Page
+from utils.read.chats import get_chats
+from utils.read.models import AnnotationView, NoteTag, NoteView, Page
 
 _COLS = "id, account_id, chat_id, message_id, text, created_at, updated_at"
 
+TAG = re.compile(r"(?<!\w)#(\w+)")
+WORD = re.compile(r"\w+")
+
+_NOTES = """
+SELECT a.id, a.account_id, a.chat_id, a.message_id, a.text,
+       a.created_at, a.updated_at,
+       m.text AS message_text, m.date AS message_date,
+       COALESCE(m.has_media, false) AS message_has_media,
+       m.deleted_at IS NOT NULL AS message_deleted
+FROM annotations a
+LEFT JOIN LATERAL (
+    SELECT text, date, has_media, deleted_at FROM messages
+    WHERE account_id = a.account_id AND chat_id = a.chat_id
+      AND message_id = a.message_id
+    LIMIT 1
+) m ON true
+WHERE a.account_id = $1 AND ($4::text IS NULL OR a.text ~* $4)
+ORDER BY a.created_at DESC, a.id DESC
+LIMIT $2 OFFSET $3
+"""
+
 
 async def list_annotations(
     pool: asyncpg.Pool,
@@ -70,3 +95,49 @@ async def update_annotation(
 async def delete_annotation(pool: asyncpg.Pool, annotation_id: int) -> bool:
     result = await pool.execute("DELETE FROM annotations WHERE id = $1", annotation_id)
     return result.endswith("1")
+
+
+def _tag_pattern(tag: str | None) -> str | None:
+    if tag is None:
+        return None
+    return f"(^|[^[:alnum:]_])#{tag}($|[^[:alnum:]_])"
+
+
+async def list_notes(
+    pool: asyncpg.Pool, account_id: int, page: Page, *, tag: str | None = None
+) -> list[NoteView]:
+    if tag is not None and not WORD.fullmatch(tag):
+        return []
+    rows = await pool.fetch(
+        _NOTES, account_id, page.capped_limit, page.offset, _tag_pattern(tag)
+    )
+    chats = {
+        chat.chat_id: chat
+        for chat in await get_chats(
+            pool, account_id, list({row["chat_id"] for row in rows})
+        )
+    }
+    notes: list[NoteView] = []
+    for row in rows:
+        chat = chats.get(row["chat_id"])
+        notes.append(
+            NoteView(
+                **dict(row),
+                chat_title=chat.title if chat else None,
+                chat_has_avatar=chat.has_avatar if chat else False,
+            )
+        )
+    return notes
+
+
+async def list_tags(pool: asyncpg.Pool, account_id: int) -> list[NoteTag]:
+    rows = await pool.fetch(
+        "SELECT text FROM annotations WHERE account_id = $1", account_id
+    )
+    counts: Counter[str] = Counter()
+    for row in rows:
+        counts.update({tag.lower() for tag in TAG.findall(row["text"])})
+    return [
+        NoteTag(tag=tag, count=count)
+        for tag, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))
+    ]

backend/src/utils/read/capture.py

@@ -0,0 +1,44 @@
+from datetime import datetime
+
+import asyncpg
+
+from utils.read.models import CaptureGap, CaptureStatus
+
+_STATUS = """
+SELECT started_at, seen_at, seen_at >= now() - interval '150 seconds' AS live
+FROM capture_uptime WHERE account_id = $1
+ORDER BY started_at DESC LIMIT 1
+"""
+
+_GAPS = """
+SELECT seen_at AS start, next_start AS "end"
+FROM (
+    SELECT seen_at,
+           lead(started_at) OVER (ORDER BY started_at) AS next_start
+    FROM capture_uptime WHERE account_id = $1
+) stretches
+WHERE next_start - seen_at > interval '30 minutes'
+  AND ($2::timestamptz IS NULL OR next_start >= $2)
+  AND ($3::timestamptz IS NULL OR seen_at <= $3)
+ORDER BY seen_at
+"""
+
+
+async def capture_status(pool: asyncpg.Pool, account_id: int) -> CaptureStatus:
+    row = await pool.fetchrow(_STATUS, account_id)
+    if row is None:
+        return CaptureStatus(live=False, since=None, last_seen=None)
+    return CaptureStatus(
+        live=row["live"], since=row["started_at"], last_seen=row["seen_at"]
+    )
+
+
+async def capture_gaps(
+    pool: asyncpg.Pool,
+    account_id: int,
+    *,
+    date_from: datetime | None = None,
+    date_to: datetime | None = None,
+) -> list[CaptureGap]:
+    rows = await pool.fetch(_GAPS, account_id, date_from, date_to)
+    return [CaptureGap(**dict(row)) for row in rows]

backend/src/utils/read/chats.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+
 import asyncpg
 
 from utils.policy.models import FolderSpec
@@ -10,7 +12,7 @@ from utils.read.models import (
     MessageView,
     Page,
 )
-from utils.read.read_receipts import read_up_to
+from utils.read.read_receipts import read_times, read_up_to
 
 _MESSAGE_COLS = (
     "chat_id, message_id, date, sender_id, text, has_media, is_self_destruct, "
@@ -78,7 +80,8 @@ _CHAT_ROWS = """
 WITH ids AS ({ids})
 SELECT ids.chat_id,
        COALESCE(cs.message_count, 0) AS message_count,
-       cs.last_date, cs.last_text, cs.last_sender_id,
+       cs.last_date, cs.last_text, cs.last_sender_id, cs.last_message_id,
+       COALESCE(cs.last_deleted, false) AS last_deleted,
        COALESCE(named.title,
                 NULLIF(trim(concat_ws(' ', p.first_name, p.last_name)), ''),
                 p.username) AS title,
@@ -147,6 +150,8 @@ def _chat_item(row: asyncpg.Record) -> ChatListItem:
         last_date=row["last_date"],
         last_text=row["last_text"],
         last_sender_id=row["last_sender_id"],
+        last_message_id=row["last_message_id"],
+        last_deleted=row["last_deleted"],
     )
 
 
@@ -257,6 +262,7 @@ async def get_chat_history(  # noqa: PLR0913
             views.append(_build_album(members, media_by_key))
         index = end
     await _apply_read_status(pool, account_id, chat_id, views)
+    await _apply_notes(pool, account_id, chat_id, views)
     return views
 
 
@@ -276,6 +282,7 @@ async def get_message(
     raw = load_raw(row["raw"])
     view = build_message_view(row, raw, _single_media(row, raw, media_by_key))
     await _apply_read_status(pool, account_id, chat_id, [view])
+    await _apply_notes(pool, account_id, chat_id, [view])
     return view
 
 
@@ -285,12 +292,35 @@ async def _apply_read_status(
     self_id = await self_user_id(pool, account_id)
     if self_id is None:
         return
-    marker = await read_up_to(pool, account_id, chat_id)
-    if marker is None:
+    own = [view for view in views if view.sender_id == self_id]
+    if not own:
         return
+    marker = await read_up_to(pool, account_id, chat_id)
+    times = await read_times(
+        pool, account_id, chat_id, [(view.message_id, view.date) for view in own]
+    )
+    for view in own:
+        view.read_at = times.get(view.message_id)
+        view.read = view.read_at is not None or (
+            marker is not None and view.message_id <= marker
+        )
+
+
+async def _apply_notes(
+    pool: asyncpg.Pool, account_id: int, chat_id: int, views: list[MessageView]
+) -> None:
+    if not views:
+        return
+    rows = await pool.fetch(
+        "SELECT DISTINCT message_id FROM annotations "
+        "WHERE account_id = $1 AND chat_id = $2 AND message_id = ANY($3::bigint[])",
+        account_id,
+        chat_id,
+        [view.message_id for view in views],
+    )
+    noted = {row["message_id"] for row in rows}
     for view in views:
-        if view.sender_id == self_id and view.message_id <= marker:
-            view.read = True
+        view.has_note = view.message_id in noted
 
 
 def _build_album(
@@ -325,6 +355,12 @@ async def get_deleted_messages(
         f"ORDER BY deleted_at DESC LIMIT ${len(params) - 1} OFFSET ${len(params)}",
         *params,
     )
+    return await _single_views(pool, account_id, rows)
+
+
+async def _single_views(
+    pool: asyncpg.Pool, account_id: int, rows: list[asyncpg.Record]
+) -> list[MessageView]:
     media_by_key = await _media_map(pool, account_id, rows)
     views: list[MessageView] = []
     for row in rows:
@@ -335,6 +371,67 @@ async def get_deleted_messages(
     return views
 
 
+async def _rows_at(
+    pool: asyncpg.Pool, account_id: int, keys: list[tuple[int, int, datetime]]
+) -> list[asyncpg.Record]:
+    if not keys:
+        return []
+    params: list[object] = [account_id]
+    clauses: list[str] = []
+    for key in keys:
+        params.extend(key)
+        n = len(params)
+        clauses.append(
+            f"(chat_id = ${n - 2} AND message_id = ${n - 1} AND date = ${n})"
+        )
+    return await pool.fetch(
+        f"SELECT {_MESSAGE_COLS} FROM messages "  # noqa: S608
+        f"WHERE account_id = $1 AND ({' OR '.join(clauses)})",
+        *params,
+    )
+
+
+async def get_messages_at(
+    pool: asyncpg.Pool, account_id: int, keys: list[tuple[int, int, datetime | None]]
+) -> dict[tuple[int, int], MessageView]:
+    dated = list(
+        dict.fromkeys((chat, message, date) for chat, message, date in keys if date)
+    )
+    rows = await _rows_at(pool, account_id, dated)
+    views = {
+        (view.chat_id, view.message_id): view
+        for view in await _single_views(pool, account_id, rows)
+    }
+    for chat_id, message_id, _ in keys:
+        if (chat_id, message_id) not in views:
+            view = await get_message(pool, account_id, chat_id, message_id)
+            if view is not None:
+                views[chat_id, message_id] = view
+    return views
+
+
+async def get_deleted_between(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    account_id: int,
+    chat_id: int,
+    since: datetime,
+    until: datetime,
+    limit: int,
+    before: tuple[datetime, int] | None = None,
+) -> list[MessageView]:
+    params: list[object] = [account_id, chat_id, since, until, limit]
+    where = "account_id = $1 AND chat_id = $2 AND deleted_at BETWEEN $3 AND $4"
+    if before is not None:
+        params.extend(before)
+        where += " AND (date, message_id) < ($6, $7)"
+    rows = await pool.fetch(
+        f"SELECT {_MESSAGE_COLS} FROM messages WHERE {where} "  # noqa: S608
+        "ORDER BY date DESC, message_id DESC LIMIT $5",
+        *params,
+    )
+    return await _single_views(pool, account_id, rows)
+
+
 async def get_message_versions(
     pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
 ) -> list[MessageVersionView]:

backend/src/utils/read/feed.py

@@ -0,0 +1,291 @@
+import json
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from enum import StrEnum
+
+import asyncpg
+
+from utils.read import chats, peers
+from utils.read.accounts import self_user_id
+from utils.read.models import (
+    DEFAULT_LIMIT,
+    MAX_LIMIT,
+    ChatListItem,
+    FeedChat,
+    FeedItem,
+    FeedMessagesPage,
+    FeedPage,
+    FeedUnseen,
+    MessageView,
+    PeerView,
+)
+
+CURSOR_PATTERN = r"^\d+\.\d+$"
+_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
+_MICROSECOND = timedelta(microseconds=1)
+
+_ITEM_COLS = (
+    "id, kind, chat_id, chat_kind, message_id, message_date, at, since, "
+    "message_count, detail"
+)
+
+_VERSIONS = """
+SELECT v.chat_id, v.message_id, v.observed_at, v.edit_date, v.text,
+       p.text AS before, p.observed_at IS NOT NULL AS has_before
+FROM unnest($2::bigint[], $3::bigint[], $4::timestamptz[])
+    AS k(chat_id, message_id, observed_at)
+JOIN message_versions v
+  ON v.account_id = $1 AND v.chat_id = k.chat_id
+ AND v.message_id = k.message_id AND v.observed_at = k.observed_at
+LEFT JOIN LATERAL (
+    SELECT text, observed_at FROM message_versions p
+    WHERE p.account_id = $1 AND p.chat_id = v.chat_id
+      AND p.message_id = v.message_id AND p.observed_at < v.observed_at
+    ORDER BY p.observed_at DESC LIMIT 1
+) p ON true
+"""
+
+_UNSEEN = """
+SELECT count(*) FILTER (WHERE kind = 'deleted') AS deleted,
+       count(*) FILTER (WHERE kind = 'edited') AS edited
+FROM feed_items
+WHERE account_id = $1 AND at > coalesce($2, '-infinity'::timestamptz)
+"""
+
+_MARK_SEEN = """
+INSERT INTO feed_seen (account_id, seen_at) VALUES ($1, coalesce($2, now()))
+ON CONFLICT (account_id) DO UPDATE
+SET seen_at = greatest(feed_seen.seen_at, EXCLUDED.seen_at)
+"""
+
+
+class FeedKind(StrEnum):
+    ALL = "all"
+    MESSAGES = "messages"
+    DELETED = "deleted"
+    EDITED = "edited"
+    PROFILE = "profile"
+
+
+class FeedChatKind(StrEnum):
+    DM = "dm"
+    GROUP = "group"
+    CHANNEL = "channel"
+
+
+@dataclass(slots=True)
+class FeedFilter:
+    kind: FeedKind = FeedKind.ALL
+    chat_id: int | None = None
+    chat_kind: FeedChatKind | None = None
+    since: datetime | None = None
+
+
+def encode_cursor(at: datetime, key: int) -> str:
+    return f"{(at - _EPOCH) // _MICROSECOND}.{key}"
+
+
+def decode_cursor(cursor: str) -> tuple[datetime, int]:
+    micros, _, key = cursor.partition(".")
+    return _EPOCH + int(micros) * _MICROSECOND, int(key)
+
+
+def _where(
+    account_id: int, feed: FeedFilter, cursor: str | None
+) -> tuple[str, list[object]]:
+    params: list[object] = [account_id]
+    clauses = ["account_id = $1"]
+    if feed.kind is FeedKind.MESSAGES:
+        clauses.append("kind <> 'profile'")
+    exact = feed.kind not in {FeedKind.ALL, FeedKind.MESSAGES}
+    conditions = (
+        ("kind = {}", feed.kind.value if exact else None),
+        ("chat_id = {}", feed.chat_id),
+        ("chat_kind = {}", feed.chat_kind.value if feed.chat_kind else None),
+        ("at >= {}", feed.since),
+    )
+    for template, value in conditions:
+        if value is not None:
+            params.append(value)
+            clauses.append(template.format(f"${len(params)}"))
+    if cursor is not None:
+        params.extend(decode_cursor(cursor))
+        clauses.append(f"(at, id) < (${len(params) - 1}, ${len(params)})")
+    return " AND ".join(clauses), params
+
+
+def _peer_name(peer: PeerView) -> str | None:
+    name = " ".join(part for part in (peer.first_name, peer.last_name) if part)
+    return name or peer.username
+
+
+async def _versions(
+    pool: asyncpg.Pool, account_id: int, rows: list[asyncpg.Record]
+) -> dict[tuple[int, int, datetime], asyncpg.Record]:
+    edited = [row for row in rows if row["kind"] == FeedKind.EDITED]
+    if not edited:
+        return {}
+    found = await pool.fetch(
+        _VERSIONS,
+        account_id,
+        [row["chat_id"] for row in edited],
+        [row["message_id"] for row in edited],
+        [row["at"] for row in edited],
+    )
+    return {
+        (row["chat_id"], row["message_id"], row["observed_at"]): row for row in found
+    }
+
+
+async def _sender_names(
+    pool: asyncpg.Pool,
+    account_id: int,
+    messages: dict[tuple[int, int], MessageView],
+    chat_views: dict[int, ChatListItem],
+) -> dict[int, str | None]:
+    ids = {m.sender_id for m in messages.values() if m.sender_id is not None}
+    found = await peers.get_peers(pool, account_id, list(ids))
+    names: dict[int, str | None] = {
+        chat_id: chat.title for chat_id, chat in chat_views.items()
+    }
+    names.update({peer.peer_id: _peer_name(peer) for peer in found})
+    return names
+
+
+async def _build_items(
+    pool: asyncpg.Pool, account_id: int, rows: list[asyncpg.Record]
+) -> list[FeedItem]:
+    if not rows:
+        return []
+    messages = await chats.get_messages_at(
+        pool,
+        account_id,
+        [
+            (row["chat_id"], row["message_id"], row["message_date"])
+            for row in rows
+            if row["kind"] != FeedKind.PROFILE
+        ],
+    )
+    versions = await _versions(pool, account_id, rows)
+    chat_views = {
+        chat.chat_id: chat
+        for chat in await chats.get_chats(
+            pool, account_id, list({row["chat_id"] for row in rows})
+        )
+    }
+    names = await _sender_names(pool, account_id, messages, chat_views)
+    self_id = await self_user_id(pool, account_id)
+    items: list[FeedItem] = []
+    for row in rows:
+        message = messages.get((row["chat_id"], row["message_id"]))
+        chat = chat_views.get(row["chat_id"])
+        version = versions.get((row["chat_id"], row["message_id"], row["at"]))
+        sender_id = message.sender_id if message else None
+        items.append(
+            FeedItem(
+                id=row["id"],
+                kind=row["kind"],
+                at=row["at"],
+                since=row["since"],
+                message_count=row["message_count"],
+                chat=FeedChat(
+                    chat_id=row["chat_id"],
+                    title=chat.title if chat else None,
+                    kind=row["chat_kind"],
+                    has_avatar=chat.has_avatar if chat else False,
+                ),
+                message_id=row["message_id"],
+                message=message,
+                sender_id=sender_id,
+                sender_name=names.get(sender_id) if sender_id is not None else None,
+                outgoing=self_id is not None and sender_id == self_id,
+                before=version["before"] if version else None,
+                after=version["text"] if version else None,
+                has_before=bool(version and version["has_before"]),
+                edit_date=version["edit_date"] if version else None,
+                detail=json.loads(row["detail"]) if row["detail"] else None,
+            )
+        )
+    return items
+
+
+async def list_feed(
+    pool: asyncpg.Pool,
+    account_id: int,
+    feed: FeedFilter,
+    *,
+    limit: int = DEFAULT_LIMIT,
+    cursor: str | None = None,
+) -> FeedPage:
+    size = max(1, min(limit, MAX_LIMIT))
+    where, params = _where(account_id, feed, cursor)
+    params.append(size + 1)
+    rows = await pool.fetch(
+        f"SELECT {_ITEM_COLS} FROM feed_items WHERE {where} "  # noqa: S608
+        f"ORDER BY at DESC, id DESC LIMIT ${len(params)}",
+        *params,
+    )
+    page = rows[:size]
+    more = len(rows) > size
+    return FeedPage(
+        items=await _build_items(pool, account_id, page),
+        next_cursor=encode_cursor(page[-1]["at"], page[-1]["id"]) if more else None,
+    )
+
+
+async def unseen(pool: asyncpg.Pool, account_id: int) -> FeedUnseen:
+    seen_at = await pool.fetchval(
+        "SELECT seen_at FROM feed_seen WHERE account_id = $1", account_id
+    )
+    counts = await pool.fetchrow(_UNSEEN, account_id, seen_at)
+    latest = await list_feed(
+        pool, account_id, FeedFilter(kind=FeedKind.MESSAGES), limit=1
+    )
+    return FeedUnseen(
+        count=counts["deleted"] + counts["edited"],
+        deleted=counts["deleted"],
+        edited=counts["edited"],
+        seen_at=seen_at,
+        latest=latest.items[0] if latest.items else None,
+    )
+
+
+async def mark_seen(
+    pool: asyncpg.Pool, account_id: int, at: datetime | None
+) -> FeedUnseen:
+    await pool.execute(_MARK_SEEN, account_id, at)
+    return await unseen(pool, account_id)
+
+
+async def item_messages(
+    pool: asyncpg.Pool,
+    account_id: int,
+    item_id: int,
+    *,
+    limit: int = DEFAULT_LIMIT,
+    cursor: str | None = None,
+) -> FeedMessagesPage | None:
+    row = await pool.fetchrow(
+        "SELECT chat_id, since, at FROM feed_items "
+        "WHERE id = $1 AND account_id = $2 AND kind = 'deleted'",
+        item_id,
+        account_id,
+    )
+    if row is None:
+        return None
+    size = max(1, min(limit, MAX_LIMIT))
+    views = await chats.get_deleted_between(
+        pool,
+        account_id,
+        row["chat_id"],
+        row["since"],
+        row["at"],
+        size + 1,
+        decode_cursor(cursor) if cursor else None,
+    )
+    page = views[:size]
+    more = len(views) > size
+    return FeedMessagesPage(
+        messages=page,
+        next_cursor=encode_cursor(page[-1].date, page[-1].message_id) if more else None,
+    )

backend/src/utils/read/message_view.py

@@ -22,6 +22,7 @@ from utils.read.models import (
     StickerView,
     WebPageView,
 )
+from utils.read.rich_text import rich_text
 
 _MEDIA_KEYS = (
     "photo",
@@ -382,6 +383,8 @@ def build_message_view(
     row: asyncpg.Record, raw: dict[str, Any], media: list[MediaRef]
 ) -> MessageView:
     base = _base_fields(row)
+    if base["text"] is None:
+        base["text"] = rich_text(raw.get("rich_message"))
     via_bot = raw.get("via_bot")
     sticker = _sticker(raw)
     try:

backend/src/utils/read/models.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import date, datetime
 from typing import Any
 
 from pydantic import BaseModel
@@ -38,6 +38,8 @@ class ChatListItem(BaseModel):
     last_date: datetime | None
     last_text: str | None
     last_sender_id: int | None
+    last_message_id: int | None = None
+    last_deleted: bool = False
 
 
 class DiscoverItem(BaseModel):
@@ -203,6 +205,8 @@ class MessageView(BaseModel):
     is_sticker: bool = False
     is_animated_emoji: bool = False
     read: bool = False
+    read_at: datetime | None = None
+    has_note: bool = False
 
 
 class MessageVersionView(BaseModel):
@@ -211,6 +215,50 @@ class MessageVersionView(BaseModel):
     text: str | None
 
 
+class FeedChat(BaseModel):
+    chat_id: int
+    title: str | None
+    kind: str
+    has_avatar: bool
+
+
+class FeedItem(BaseModel):
+    id: int
+    kind: str
+    at: datetime
+    since: datetime
+    message_count: int
+    chat: FeedChat
+    message_id: int
+    message: MessageView | None
+    sender_id: int | None
+    sender_name: str | None
+    outgoing: bool
+    before: str | None = None
+    after: str | None = None
+    has_before: bool = False
+    edit_date: datetime | None = None
+    detail: dict[str, list[str | None]] | None = None
+
+
+class FeedPage(BaseModel):
+    items: list[FeedItem]
+    next_cursor: str | None
+
+
+class FeedUnseen(BaseModel):
+    count: int
+    deleted: int
+    edited: int
+    seen_at: datetime | None
+    latest: FeedItem | None
+
+
+class FeedMessagesPage(BaseModel):
+    messages: list[MessageView]
+    next_cursor: str | None
+
+
 class MediaView(BaseModel):
     id: int
     account_id: int
@@ -253,6 +301,13 @@ class StorageSummaryView(BaseModel):
     groups: list[StorageGroupView]
 
 
+class StorageChatView(BaseModel):
+    chat_id: int
+    files: int
+    bytes: int
+    groups: list[StorageGroupView]
+
+
 class StorageFileView(BaseModel):
     media_id: int
     chat_id: int
@@ -354,8 +409,61 @@ class PresenceHourly(BaseModel):
     last_seen: datetime | None
 
 
+class PresenceSession(BaseModel):
+    started: datetime
+    ended: datetime
+    seconds: float
+    online_seconds: float
+    ongoing: bool
+
+
+class PresenceHour(BaseModel):
+    weekday: int
+    hour: int
+    hours: int
+    online_minutes: float | None
+    covered_hours: float
+
+
+class PresenceDay(BaseModel):
+    day: date
+    online_seconds: float
+    covered_seconds: float
+
+
+class CoverageGap(BaseModel):
+    started: datetime
+    ended: datetime
+
+
+class PresenceSummary(BaseModel):
+    online_seconds: float
+    daily_seconds: float | None
+    covered_seconds: float
+    session_count: int
+    median_session_seconds: float | None
+    first_online: int | None
+    last_offline: int | None
+    last_seen: datetime | None
+
+
+class PresenceStats(BaseModel):
+    peer_id: int
+    tz: str
+    since: datetime
+    until: datetime
+    current: PresenceSample | None
+    online: bool
+    hidden: bool
+    summary: PresenceSummary
+    week: list[PresenceHour]
+    days: list[PresenceDay]
+    gaps: list[CoverageGap]
+    sessions: list[PresenceSession]
+
+
 class VolumeBucket(BaseModel):
-    bucket: datetime
+    bucket: date
     total: int
     outgoing: int
     incoming: int
@@ -380,6 +488,29 @@ class PeerView(BaseModel):
     updated_at: datetime
 
 
+class ReaderView(BaseModel):
+    peer_id: int
+    read_at: datetime
+    peer: PeerView | None
+
+
+class ReadInfoView(BaseModel):
+    chat_id: int
+    message_id: int
+    chat_kind: str
+    own: bool
+    voice: bool
+    observed_at: datetime | None
+    listened_at: datetime | None
+    exact_at: datetime | None
+    readers: list[ReaderView]
+    read: bool
+    unread: bool
+    outcome: str | None
+    fetched_at: datetime | None
+    can_fetch: bool
+
+
 class PeerHistoryView(BaseModel):
     observed_at: datetime
     first_name: str | None
@@ -390,6 +521,54 @@ class PeerHistoryView(BaseModel):
     is_deleted_account: bool
 
 
+class ProfileChange(BaseModel):
+    field: str
+    old: str | None
+    new: str | None
+    changed_at: datetime
+    actor_id: int | None = None
+    message_id: int | None = None
+
+
+class MembershipView(BaseModel):
+    chat_id: int
+    chat_title: str | None
+    has_avatar: bool
+    message_id: int
+    event: str
+    actor_id: int | None
+    actor_name: str | None
+    ts: datetime
+
+
+class DirectChatView(BaseModel):
+    chat_id: int
+    message_count: int
+    last_date: datetime | None
+
+
+class PeerOverview(BaseModel):
+    peer: PeerView | None
+    changes: list[ProfileChange]
+    avatars: list[AvatarHistoryView]
+    memberships: list[MembershipView]
+    dm: DirectChatView | None
+
+
+class SharedGroupView(BaseModel):
+    chat_id: int
+    title: str | None
+    has_avatar: bool
+    message_count: int
+    last_date: datetime
+    last_message_id: int
+
+
+class PeerGroups(BaseModel):
+    groups: list[SharedGroupView]
+    deleted_count: int
+
+
 class ChatLinkView(BaseModel):
     message_id: int
     date: datetime | None
@@ -435,16 +614,62 @@ class AnnotationView(BaseModel):
     updated_at: datetime
 
 
+class NoteView(AnnotationView):
+    chat_title: str | None
+    chat_has_avatar: bool
+    message_text: str | None
+    message_date: datetime | None
+    message_has_media: bool
+    message_deleted: bool
+
+
+class NoteTag(BaseModel):
+    tag: str
+    count: int
+
+
+class CaptureStatus(BaseModel):
+    live: bool
+    since: datetime | None
+    last_seen: datetime | None
+
+
+class CaptureGap(BaseModel):
+    start: datetime
+    end: datetime
+
+
 class WatchView(BaseModel):
     id: int
     account_id: int
     kind: str
+    name: str | None
     params: dict[str, Any]
     enabled: bool
     created_at: datetime
     updated_at: datetime
 
 
+class ChatRef(BaseModel):
+    id: int
+    title: str | None
+    kind: str
+    has_avatar: bool
+
+
+class PersonRef(BaseModel):
+    id: int
+    name: str | None
+    has_avatar: bool
+
+
+class WatchItem(WatchView):
+    chat: ChatRef | None
+    person: PersonRef | None
+    alert_count: int
+    last_alert_at: datetime | None
+
+
 class AlertView(BaseModel):
     id: int
     account_id: int
@@ -455,6 +680,68 @@ class AlertView(BaseModel):
     created_at: datetime
 
 
+class AlertWatch(BaseModel):
+    id: int
+    kind: str
+    name: str | None
+    params: dict[str, Any]
+
+
+class AlertMessage(BaseModel):
+    id: int
+    text: str | None
+    sender: PersonRef | None
+    date: datetime | None
+    deleted: bool
+    edited: bool
+
+
+class AlertItem(BaseModel):
+    id: int
+    account_id: int
+    ts: datetime
+    seen: bool
+    event: str
+    watch: AlertWatch
+    chat: ChatRef | None
+    message: AlertMessage | None
+    peer: PersonRef | None
+    away_since: datetime | None
+    path: str
+
+
+class AlertPage(BaseModel):
+    items: list[AlertItem]
+    next_before: int | None
+    unseen: int
+
+
+class DeliveryView(BaseModel):
+    id: int
+    webhook_id: int
+    alert_id: int | None
+    event: str
+    status: str
+    attempts: int
+    next_attempt_at: datetime
+    response_code: int | None
+    last_error: str | None
+    created_at: datetime
+    finished_at: datetime | None
+
+
+class WebhookView(BaseModel):
+    id: int
+    account_id: int
+    url: str
+    secret: str
+    enabled: bool
+    watch_ids: list[int] | None
+    created_at: datetime
+    updated_at: datetime
+    last_delivery: DeliveryView | None
+
+
 class FileShareView(BaseModel):
     id: int
     account_id: int

backend/src/utils/read/peers.py

@@ -2,6 +2,8 @@ import asyncpg
 
 from utils.read.models import Page, PeerHistoryView, PeerView, StoryView
 
+HISTORY_LIMIT = 500
+
 _PEER_COLS = (
     "peer_id, first_name, last_name, username, phone, photo_unique_id, "
     "is_deleted_account, updated_at, "
@@ -9,6 +11,23 @@ _PEER_COLS = (
     "AND a.owner_id = peers.peer_id) AS has_avatar"
 )
 
+_HISTORY = """
+SELECT observed_at, first_name, last_name, username, phone, photo_unique_id,
+       is_deleted_account
+FROM (
+    SELECT observed_at, first_name, last_name, username, phone, photo_unique_id,
+           is_deleted_account,
+           lag((first_name, last_name, username, phone, photo_unique_id,
+                is_deleted_account)) OVER (ORDER BY observed_at) AS previous
+    FROM peer_history
+    WHERE account_id = $1 AND peer_id = $2
+) states
+WHERE previous IS DISTINCT FROM
+      (first_name, last_name, username, phone, photo_unique_id, is_deleted_account)
+ORDER BY observed_at DESC
+LIMIT $3
+"""
+
 
 async def get_peer(
     pool: asyncpg.Pool, account_id: int, peer_id: int
@@ -39,14 +58,7 @@ async def get_peers(
 async def get_peer_history(
     pool: asyncpg.Pool, account_id: int, peer_id: int
 ) -> list[PeerHistoryView]:
-    rows = await pool.fetch(
-        "SELECT observed_at, first_name, last_name, username, phone, "
-        "photo_unique_id, is_deleted_account FROM peer_history "
-        "WHERE account_id = $1 AND peer_id = $2 "
-        "ORDER BY observed_at DESC LIMIT 500",
-        account_id,
-        peer_id,
-    )
+    rows = await pool.fetch(_HISTORY, account_id, peer_id, HISTORY_LIMIT)
     return [PeerHistoryView(**dict(row)) for row in rows]
 
 
@@ -64,7 +76,7 @@ async def get_stories(
         "SELECT peer_id, story_id, date, expire_date, caption, media_kind, "  # noqa: S608
         "storage_key, downloaded, views, pinned, deleted FROM stories "
         f"WHERE {where} "
-        f"ORDER BY date DESC NULLS LAST LIMIT ${len(params) - 1} "
+        f"ORDER BY date DESC NULLS LAST, story_id DESC LIMIT ${len(params) - 1} "
         f"OFFSET ${len(params)}",
         *params,
     )
@@ -83,3 +95,16 @@ async def get_story(
         story_id,
     )
     return StoryView(**dict(row)) if row else None
+
+
+async def set_story_kind(
+    pool: asyncpg.Pool, account_id: int, peer_id: int, story_id: int, kind: str
+) -> None:
+    await pool.execute(
+        "UPDATE stories SET media_kind = $4 "
+        "WHERE account_id = $1 AND peer_id = $2 AND story_id = $3",
+        account_id,
+        peer_id,
+        story_id,
+        kind,
+    )

backend/src/utils/read/people.py

@@ -0,0 +1,194 @@
+from collections.abc import Callable
+from datetime import datetime
+from itertools import pairwise
+
+import asyncpg
+
+from utils.read import chats
+from utils.read.avatars import avatar_history
+from utils.read.models import (
+    DirectChatView,
+    MembershipView,
+    PeerGroups,
+    PeerHistoryView,
+    PeerOverview,
+    ProfileChange,
+    SharedGroupView,
+)
+from utils.read.peers import HISTORY_LIMIT, get_peer, get_peer_history
+
+_CHAT_TITLES = """
+SELECT ts, title, previous, actor_id, message_id
+FROM (
+    SELECT ts, title,
+           CASE WHEN event = 'title' THEN actor_id END AS actor_id,
+           CASE WHEN event = 'title' THEN message_id END AS message_id,
+           lag(title) OVER (ORDER BY ts, message_id) AS previous
+    FROM chat_history
+    WHERE account_id = $1 AND chat_id = $2 AND event IN ('meta', 'title')
+      AND title IS NOT NULL
+) titles
+WHERE previous IS DISTINCT FROM title AND previous IS NOT NULL
+ORDER BY ts DESC
+LIMIT $3
+"""
+
+_MEMBERSHIPS = """
+SELECT m.chat_id, m.message_id, m.event, m.actor_id, m.ts,
+       COALESCE(NULLIF(trim(concat_ws(' ', a.first_name, a.last_name)), ''),
+                a.username) AS actor_name
+FROM memberships m
+LEFT JOIN peers a ON a.account_id = m.account_id AND a.peer_id = m.actor_id
+WHERE m.account_id = $1 AND m.user_id = $2
+ORDER BY m.ts DESC
+LIMIT 200
+"""
+
+_DIRECT_CHAT = """
+SELECT $2::bigint AS chat_id, COALESCE(cs.message_count, 0) AS message_count,
+       cs.last_date
+FROM (SELECT 1) one
+LEFT JOIN chat_stats cs ON cs.account_id = $1 AND cs.chat_id = $2
+WHERE cs.chat_id IS NOT NULL
+   OR EXISTS (SELECT 1 FROM dialogs WHERE account_id = $1 AND chat_id = $2)
+"""
+
+_SENT = """
+SELECT chat_id, count(*) AS message_count, max(date) AS last_date,
+       max(message_id) AS last_message_id,
+       count(*) FILTER (WHERE deleted_at IS NOT NULL) AS deleted_count
+FROM messages
+WHERE account_id = $1 AND sender_id = $2
+GROUP BY chat_id
+"""
+
+type Point = tuple[datetime, str | None]
+
+
+def _full_name(state: PeerHistoryView) -> str | None:
+    parts = [part for part in (state.first_name, state.last_name) if part]
+    return " ".join(parts) or None
+
+
+def _timeline(
+    states: list[PeerHistoryView], value: Callable[[PeerHistoryView], str | None]
+) -> list[Point]:
+    points: list[Point] = []
+    for state in states:
+        current = value(state)
+        if not points or points[-1][1] != current:
+            points.append((state.observed_at, current))
+    return points
+
+
+def _without_gaps(points: list[Point]) -> list[Point]:
+    kept: list[Point] = []
+    for at, current in points:
+        gap = len(kept) > 1 and kept[-1][1] is None and kept[-2][1] == current
+        if gap:
+            kept.pop()
+        elif not kept or kept[-1][1] != current:
+            kept.append((at, current))
+    return kept
+
+
+def _known(points: list[Point]) -> list[Point]:
+    return _without_gaps([point for point in points if point[1] is not None])
+
+
+def _changes(field: str, points: list[Point]) -> list[ProfileChange]:
+    return [
+        ProfileChange(field=field, old=old, new=new, changed_at=at)
+        for (_, old), (at, new) in pairwise(points)
+    ]
+
+
+def profile_changes(states: list[PeerHistoryView]) -> list[ProfileChange]:
+    ordered = sorted(states, key=lambda state: state.observed_at)
+    alive = [state for state in ordered if not state.is_deleted_account]
+    deleted = _timeline(
+        ordered, lambda state: "deleted" if state.is_deleted_account else None
+    )
+    changes = [
+        *_changes("name", _without_gaps(_timeline(alive, _full_name))),
+        *_changes(
+            "username", _without_gaps(_timeline(alive, lambda state: state.username))
+        ),
+        *_changes("phone", _known(_timeline(alive, lambda state: state.phone))),
+        *[change for change in _changes("deleted", deleted) if change.new is not None],
+    ]
+    return sorted(changes, key=lambda change: change.changed_at, reverse=True)
+
+
+async def get_chat_changes(
+    pool: asyncpg.Pool, account_id: int, chat_id: int
+) -> list[ProfileChange]:
+    rows = await pool.fetch(_CHAT_TITLES, account_id, chat_id, HISTORY_LIMIT)
+    return [
+        ProfileChange(
+            field="title",
+            old=row["previous"],
+            new=row["title"],
+            changed_at=row["ts"],
+            actor_id=row["actor_id"],
+            message_id=row["message_id"],
+        )
+        for row in rows
+    ]
+
+
+async def _memberships(
+    pool: asyncpg.Pool, account_id: int, peer_id: int
+) -> list[MembershipView]:
+    rows = await pool.fetch(_MEMBERSHIPS, account_id, peer_id)
+    ids = list({row["chat_id"] for row in rows})
+    known = {
+        chat.chat_id: chat for chat in await chats.get_chats(pool, account_id, ids)
+    }
+    views: list[MembershipView] = []
+    for row in rows:
+        chat = known.get(row["chat_id"])
+        views.append(
+            MembershipView(
+                **dict(row),
+                chat_title=chat.title if chat else None,
+                has_avatar=bool(chat and chat.has_avatar),
+            )
+        )
+    return views
+
+
+async def get_overview(
+    pool: asyncpg.Pool, account_id: int, peer_id: int
+) -> PeerOverview:
+    direct = await pool.fetchrow(_DIRECT_CHAT, account_id, peer_id)
+    return PeerOverview(
+        peer=await get_peer(pool, account_id, peer_id),
+        changes=profile_changes(await get_peer_history(pool, account_id, peer_id)),
+        avatars=await avatar_history(pool, account_id, peer_id),
+        memberships=await _memberships(pool, account_id, peer_id),
+        dm=DirectChatView(**dict(direct)) if direct else None,
+    )
+
+
+async def get_groups(pool: asyncpg.Pool, account_id: int, peer_id: int) -> PeerGroups:
+    async with pool.acquire() as conn, conn.transaction():
+        await conn.execute("SET LOCAL jit = off")
+        rows = await conn.fetch(_SENT, account_id, peer_id)
+    sent = {row["chat_id"]: row for row in rows if row["chat_id"] < 0}
+    found = await chats.get_chats(pool, account_id, list(sent))
+    groups = [
+        SharedGroupView(
+            chat_id=chat.chat_id,
+            title=chat.title,
+            has_avatar=chat.has_avatar,
+            message_count=sent[chat.chat_id]["message_count"],
+            last_date=sent[chat.chat_id]["last_date"],
+            last_message_id=sent[chat.chat_id]["last_message_id"],
+        )
+        for chat in found
+    ]
+    groups.sort(key=lambda group: group.last_date, reverse=True)
+    return PeerGroups(
+        groups=groups, deleted_count=sum(row["deleted_count"] for row in rows)
+    )

backend/src/utils/read/period.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+
+import asyncpg
+
+DEFAULT_TZ = "UTC"
+MAX_DAYS = 365
+
+_timezones: set[str] = set()
+
+
+async def known_timezone(pool: asyncpg.Pool, tz: str) -> bool:
+    if not _timezones:
+        rows = await pool.fetch("SELECT name FROM pg_timezone_names")
+        _timezones.update(row["name"] for row in rows)
+    return tz in _timezones
+
+
+async def period_bounds(
+    pool: asyncpg.Pool, days: int, tz: str
+) -> tuple[datetime, datetime]:
+    row = await pool.fetchrow(
+        "SELECT (date_trunc('day', now() AT TIME ZONE $1) "
+        "- make_interval(days => $2 - 1)) AT TIME ZONE $1 AS since, now() AS until",
+        tz,
+        days,
+    )
+    return row["since"], row["until"]

backend/src/utils/read/presence.py

@@ -48,6 +48,29 @@ async def current_presence(
     return PresenceSample(**dict(row)) if row is not None else None
 
 
+_ONLINE = """
+SELECT peer_id, ts, status, last_online_date, next_offline_date
+FROM (
+    SELECT DISTINCT ON (peer_id)
+           peer_id, ts, status, last_online_date, next_offline_date
+    FROM presence
+    WHERE account_id = $1 AND peer_id = ANY($2::bigint[])
+      AND ts > now() - interval '1 hour'
+    ORDER BY peer_id, ts DESC
+) latest
+WHERE status = 'online' AND next_offline_date > now()
+"""
+
+
+async def online_now(
+    pool: asyncpg.Pool, account_id: int, peer_ids: list[int]
+) -> list[PresenceSample]:
+    if not peer_ids:
+        return []
+    rows = await pool.fetch(_ONLINE, account_id, peer_ids)
+    return [PresenceSample(**dict(row)) for row in rows]
+
+
 async def presence_hourly(
     pool: asyncpg.Pool,
     account_id: int,

backend/src/utils/read/presence_stats.py

@@ -0,0 +1,289 @@
+import json
+
+import asyncpg
+
+from utils.read.models import (
+    CoverageGap,
+    Page,
+    PresenceDay,
+    PresenceHour,
+    PresenceSession,
+    PresenceStats,
+    PresenceSummary,
+)
+from utils.read.period import period_bounds
+from utils.read.presence import current_presence
+
+SECONDS_IN_HOUR = 3600
+SECONDS_IN_DAY = 86_400
+MIN_CELL_COVERAGE = 1800
+MIN_CELL_SHARE = 0.5
+STATS_WORK_MEM = "32MB"
+FUZZY_STATUSES = {"recently", "last_week", "last_month", "long_time_ago"}
+
+_SESSIONS = """
+marks AS (
+    SELECT ts, status,
+           least(next_offline_date, ts + interval '5 minutes',
+                 lead(coalesce(last_online_date, ts)) OVER w, $4) AS ended,
+           lag(CASE WHEN status = 'online'
+                    THEN least(next_offline_date, ts + interval '5 minutes')
+                    ELSE coalesce(last_online_date, ts) END) OVER w AS previous
+    FROM presence
+    WHERE account_id = $1 AND peer_id = $2
+      AND ts >= $3::timestamptz - interval '5 minutes' AND ts <= $4
+    WINDOW w AS (ORDER BY ts)
+),
+online AS (
+    SELECT greatest(ts, $3) AS started, ended,
+           count(*) FILTER (WHERE previous IS NULL
+                                OR ts > previous + interval '5 minutes')
+               OVER (ORDER BY ts) AS visit
+    FROM marks
+    WHERE status = 'online' AND ended > greatest(ts, $3)
+),
+sessions AS (
+    SELECT min(started) AS started, max(ended) AS ended,
+           extract(epoch FROM sum(ended - started))::float8 AS online_seconds
+    FROM online
+    GROUP BY visit
+)
+"""
+
+_SESSION_FIELDS = """
+started, ended, extract(epoch FROM ended - started)::float8 AS seconds,
+online_seconds, ended >= $4 AS ongoing
+"""
+
+_SESSION_PAGE = f"""
+WITH {_SESSIONS}
+SELECT {_SESSION_FIELDS}
+FROM sessions
+ORDER BY started DESC
+LIMIT $5 OFFSET $6
+"""  # noqa: S608
+
+_COVERED_HOURS = """
+    SELECT local_hour,
+           extract(epoch FROM sum(
+               least(ended, (local_hour + interval '1 hour') AT TIME ZONE $5)
+               - greatest(started, local_hour AT TIME ZONE $5)))::float8 AS seconds
+    FROM uptime,
+         generate_series(date_trunc('hour', started AT TIME ZONE $5),
+                         date_trunc('hour', ended AT TIME ZONE $5),
+                         interval '1 hour') AS local_hour
+    GROUP BY local_hour
+"""
+
+_STATS = f"""
+WITH {_SESSIONS},
+uptime AS (
+    SELECT greatest(started_at, $3) AS started,
+           least(seen_at + interval '90 seconds', $4) AS ended
+    FROM capture_uptime
+    WHERE account_id = $1 AND started_at < $4
+      AND seen_at + interval '90 seconds' > $3
+),
+quarters AS (
+    SELECT quarter, sum(ended - started - spill) AS inside, sum(spill) AS spill
+    FROM (SELECT started, ended, quarter,
+                 greatest(ended - quarter - interval '15 minutes',
+                          interval '0') AS spill
+          FROM (SELECT started, ended,
+                       date_bin('15 minutes', started, timestamptz 'epoch') AS quarter
+                FROM online) binned) split
+    GROUP BY quarter
+),
+online_hours AS (
+    SELECT date_trunc('hour', quarter AT TIME ZONE $5) AS local_hour,
+           extract(epoch FROM sum(share))::float8 AS seconds
+    FROM (SELECT quarter, inside AS share FROM quarters
+          UNION ALL
+          SELECT quarter + interval '15 minutes', spill FROM quarters) shares
+    GROUP BY 1
+),
+covered_hours AS ({_COVERED_HOURS}),
+hours AS (
+    SELECT local_hour,
+           extract(epoch FROM
+               least((local_hour + interval '1 hour') AT TIME ZONE $5, $4)
+               - greatest(local_hour AT TIME ZONE $5, $3))::float8 AS span,
+           coalesce(online.seconds, 0) AS online,
+           least(3600, greatest(coalesce(covered.seconds, 0),
+                                coalesce(online.seconds, 0))) AS covered
+    FROM generate_series(date_trunc('hour', $3 AT TIME ZONE $5),
+                         date_trunc('hour', $4 AT TIME ZONE $5),
+                         interval '1 hour') AS local_hour
+    LEFT JOIN online_hours AS online USING (local_hour)
+    LEFT JOIN covered_hours AS covered USING (local_hour)
+),
+week AS (
+    SELECT extract(isodow FROM local_hour)::int - 1 AS weekday,
+           extract(hour FROM local_hour)::int AS hour,
+           count(*) AS hours, sum(span) AS span,
+           sum(online) AS online, sum(covered) AS covered
+    FROM hours
+    GROUP BY 1, 2
+),
+days AS (
+    SELECT local_hour::date AS day, sum(online) AS online, sum(covered) AS covered
+    FROM hours
+    GROUP BY 1
+),
+uptime_reach AS (
+    SELECT started,
+           coalesce(max(ended) OVER (ORDER BY started
+                                     ROWS BETWEEN UNBOUNDED PRECEDING
+                                     AND 1 PRECEDING), $3) AS reached
+    FROM uptime
+),
+gaps AS (
+    SELECT reached AS started, started AS ended FROM uptime_reach
+    UNION ALL
+    SELECT coalesce(max(ended), $3), $4 FROM uptime
+),
+quiet AS (
+    SELECT hour
+    FROM week
+    GROUP BY hour
+    ORDER BY sum(online) / nullif(sum(covered), 0) NULLS LAST, hour
+    LIMIT 1
+),
+activity_days AS (
+    SELECT (sessions.started AT TIME ZONE $5) - make_interval(hours => quiet.hour)
+               AS started,
+           (sessions.ended AT TIME ZONE $5) - make_interval(hours => quiet.hour)
+               AS ended,
+           (($4 AT TIME ZONE $5) - make_interval(hours => quiet.hour))::date AS today
+    FROM sessions, quiet
+),
+daily AS (
+    SELECT extract(epoch FROM min(started - started::date))::float8 AS first_online,
+           extract(epoch FROM max(ended - started::date))::float8 AS last_offline
+    FROM activity_days
+    WHERE started::date < today
+    GROUP BY started::date
+)
+SELECT
+    (SELECT count(*) FROM sessions) AS session_count,
+    (SELECT coalesce(sum(online_seconds), 0) FROM sessions) AS online_seconds,
+    (SELECT sum(covered) FROM hours) AS covered_seconds,
+    (SELECT percentile_cont(0.5) WITHIN GROUP (
+         ORDER BY extract(epoch FROM ended - started)) FROM sessions) AS median_session,
+    (SELECT max(ended) FROM sessions) AS last_seen,
+    (SELECT hour FROM quiet) AS day_start,
+    (SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY first_online)
+     FROM daily) AS first_online,
+    (SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY last_offline)
+     FROM daily) AS last_offline,
+    (SELECT json_agg(json_build_object(
+         'weekday', weekday, 'hour', hour, 'hours', hours, 'span', span,
+         'online', online, 'covered', covered))
+     FROM week) AS week,
+    (SELECT json_agg(json_build_object(
+         'day', day, 'online_seconds', online, 'covered_seconds', covered)
+         ORDER BY day)
+     FROM days) AS days,
+    (SELECT json_agg(json_build_object('started', started, 'ended', ended)
+         ORDER BY started)
+     FROM gaps WHERE ended - started > interval '5 minutes') AS gaps,
+    (SELECT json_agg(row_to_json(page) ORDER BY page.started DESC)
+     FROM (SELECT {_SESSION_FIELDS} FROM sessions
+           ORDER BY started DESC LIMIT $6 OFFSET $7) page) AS sessions
+"""  # noqa: S608
+
+
+def _time_of_day(shifted: float | None, day_start: int | None) -> int | None:
+    if shifted is None or day_start is None:
+        return None
+    return round(shifted + day_start * SECONDS_IN_HOUR) % SECONDS_IN_DAY
+
+
+def _week(rows: list[dict]) -> list[PresenceHour]:
+    cells = {(row["weekday"], row["hour"]): row for row in rows}
+    week: list[PresenceHour] = []
+    for weekday in range(7):
+        for hour in range(24):
+            row = cells.get(
+                (weekday, hour), {"hours": 0, "span": 0, "online": 0, "covered": 0}
+            )
+            covered = row["covered"]
+            minutes = (
+                min(60.0, row["online"] / covered * 60)
+                if covered > 0
+                and covered >= min(MIN_CELL_COVERAGE, row["span"] * MIN_CELL_SHARE)
+                else None
+            )
+            week.append(
+                PresenceHour(
+                    weekday=weekday,
+                    hour=hour,
+                    hours=row["hours"],
+                    online_minutes=minutes,
+                    covered_hours=covered / SECONDS_IN_HOUR,
+                )
+            )
+    return week
+
+
+async def presence_stats(  # noqa: PLR0913
+    pool: asyncpg.Pool, account_id: int, peer_id: int, page: Page, *, days: int, tz: str
+) -> PresenceStats:
+    since, until = await period_bounds(pool, days, tz)
+    async with pool.acquire() as conn, conn.transaction():
+        await conn.execute(f"SET LOCAL work_mem = '{STATS_WORK_MEM}'")
+        row = await conn.fetchrow(
+            _STATS,
+            account_id,
+            peer_id,
+            since,
+            until,
+            tz,
+            page.capped_limit,
+            page.offset,
+        )
+    current = await current_presence(pool, account_id, peer_id)
+    sessions = [PresenceSession(**item) for item in json.loads(row["sessions"] or "[]")]
+    covered = row["covered_seconds"] or 0
+    online = row["online_seconds"]
+    return PresenceStats(
+        peer_id=peer_id,
+        tz=tz,
+        since=since,
+        until=until,
+        current=current,
+        online=current is not None
+        and current.status == "online"
+        and current.next_offline_date is not None
+        and current.next_offline_date > until,
+        hidden=row["session_count"] == 0
+        and current is not None
+        and current.status in FUZZY_STATUSES,
+        summary=PresenceSummary(
+            online_seconds=online,
+            daily_seconds=online / covered * SECONDS_IN_DAY
+            if covered >= SECONDS_IN_HOUR
+            else None,
+            covered_seconds=covered,
+            session_count=row["session_count"],
+            median_session_seconds=row["median_session"],
+            first_online=_time_of_day(row["first_online"], row["day_start"]),
+            last_offline=_time_of_day(row["last_offline"], row["day_start"]),
+            last_seen=row["last_seen"]
+            or (current.last_online_date if current else None),
+        ),
+        week=_week(json.loads(row["week"] or "[]")),
+        days=[PresenceDay(**item) for item in json.loads(row["days"] or "[]")],
+        gaps=[CoverageGap(**item) for item in json.loads(row["gaps"] or "[]")],
+        sessions=sessions,
+    )
+
+
+async def presence_sessions(  # noqa: PLR0913
+    pool: asyncpg.Pool, account_id: int, peer_id: int, page: Page, *, days: int, tz: str
+) -> list[PresenceSession]:
+    since, until = await period_bounds(pool, days, tz)
+    rows = await pool.fetch(
+        _SESSION_PAGE, account_id, peer_id, since, until, page.capped_limit, page.offset
+    )
+    return [PresenceSession(**dict(row)) for row in rows]

backend/src/utils/read/read_receipts.py

@@ -1,5 +1,89 @@
+from datetime import UTC, datetime, timedelta
+
 import asyncpg
 
+from utils.policy.models import ChatKind
+from utils.read.discover import chat_kind
+from utils.read.models import ReaderView, ReadInfoView
+from utils.read.peers import get_peers
+
+FETCH_WINDOW = timedelta(days=7)
+CLOCK_SKEW = timedelta(minutes=1)
+VOICE_KINDS = ["voice", "video_note"]
+
+_READ_TIMES = """
+SELECT m.id,
+    COALESCE(
+        (SELECT min(mr.read_at) FROM message_reads mr
+         WHERE mr.account_id = $1 AND mr.chat_id = $2 AND mr.message_id = m.id),
+        CASE WHEN m.date >= $5 THEN (
+            SELECT r.ts FROM read_receipts r
+            WHERE r.account_id = $1 AND r.chat_id = $2 AND r.kind = 'read'
+              AND r.message_id >= m.id AND r.ts >= m.date - $6::interval
+            ORDER BY r.ts LIMIT 1
+        ) END
+    ) AS read_at
+FROM unnest($3::bigint[], $4::timestamptz[]) AS m(id, date)
+"""
+
+_MESSAGE = """
+SELECT m.sender_id, m.date, a.tg_user_id, a.is_bot,
+    EXISTS (
+        SELECT 1 FROM media md
+        WHERE md.account_id = m.account_id AND md.chat_id = m.chat_id
+          AND md.message_id = m.message_id AND md.kind = ANY($4::text[])
+    ) AS voice
+FROM messages m
+JOIN accounts a ON a.account_id = m.account_id
+WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3
+LIMIT 1
+"""
+
+_RECEIPTS = """
+SELECT
+    (SELECT ts FROM read_receipts
+     WHERE account_id = $1 AND chat_id = $2 AND kind = 'read'
+       AND message_id >= $3 AND ts >= $4
+     ORDER BY ts LIMIT 1) AS observed_at,
+    (SELECT min(ts) FROM read_receipts
+     WHERE account_id = $1 AND chat_id = $2 AND kind = 'listened'
+       AND message_id = $3) AS listened_at,
+    (SELECT max(message_id) FROM read_receipts
+     WHERE account_id = $1 AND chat_id = $2 AND kind = 'read') AS watermark
+"""
+
+_READS = """
+SELECT reader_id, read_at, source FROM message_reads
+WHERE account_id = $1 AND chat_id = $2 AND message_id = $3
+ORDER BY read_at
+"""
+
+_FETCH = """
+SELECT outcome, fetched_at, retry_at FROM read_fetches
+WHERE account_id = $1 AND chat_id = $2 AND message_id = $3
+"""
+
+_INHERITED = """
+SELECT outcome FROM read_fetches
+WHERE account_id = $1 AND retry_at > now()
+  AND ((chat_id = $2 AND outcome = 'chat_too_big') OR outcome = 'your_privacy')
+LIMIT 1
+"""
+
+_capture_starts: dict[int, datetime] = {}
+
+
+async def capture_start(pool: asyncpg.Pool, account_id: int) -> datetime | None:
+    cached = _capture_starts.get(account_id)
+    if cached is not None:
+        return cached
+    start = await pool.fetchval(
+        "SELECT min(ts) FROM read_receipts WHERE account_id = $1", account_id
+    )
+    if start is not None:
+        _capture_starts[account_id] = start
+    return start
+
 
 async def read_up_to(pool: asyncpg.Pool, account_id: int, chat_id: int) -> int | None:
     return await pool.fetchval(
@@ -8,3 +92,97 @@ async def read_up_to(pool: asyncpg.Pool, account_id: int, chat_id: int) -> int |
         account_id,
         chat_id,
     )
+
+
+async def read_times(
+    pool: asyncpg.Pool,
+    account_id: int,
+    chat_id: int,
+    messages: list[tuple[int, datetime]],
+) -> dict[int, datetime]:
+    if not messages:
+        return {}
+    start = await capture_start(pool, account_id) or datetime.max.replace(tzinfo=UTC)
+    rows = await pool.fetch(
+        _READ_TIMES,
+        account_id,
+        chat_id,
+        [message_id for message_id, _ in messages],
+        [date for _, date in messages],
+        start,
+        CLOCK_SKEW,
+    )
+    return {row["id"]: row["read_at"] for row in rows if row["read_at"] is not None}
+
+
+async def read_info(
+    pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
+) -> ReadInfoView | None:
+    message = await pool.fetchrow(
+        _MESSAGE, account_id, chat_id, message_id, VOICE_KINDS
+    )
+    if message is None:
+        return None
+    kind = await chat_kind(pool, account_id, chat_id)
+    date: datetime = message["date"]
+    receipts = await pool.fetchrow(
+        _RECEIPTS, account_id, chat_id, message_id, date - CLOCK_SKEW
+    )
+    reads = await pool.fetch(_READS, account_id, chat_id, message_id)
+    fetch = await pool.fetchrow(_FETCH, account_id, chat_id, message_id)
+    inherited = await pool.fetchval(_INHERITED, account_id, chat_id)
+    start = await capture_start(pool, account_id)
+
+    observed_at = receipts["observed_at"] if start and date >= start else None
+    exact_at = next(
+        (row["read_at"] for row in reads if row["source"] == "outbox_date"), None
+    )
+    reader_rows = [row for row in reads if row["source"] == "participants"]
+    known = {
+        peer.peer_id: peer
+        for peer in await get_peers(
+            pool, account_id, [row["reader_id"] for row in reader_rows]
+        )
+    }
+    readers = [
+        ReaderView(
+            peer_id=row["reader_id"],
+            read_at=row["read_at"],
+            peer=known.get(row["reader_id"]),
+        )
+        for row in reader_rows
+    ]
+    outcome = fetch["outcome"] if fetch else inherited
+
+    now = datetime.now(UTC)
+    own = (
+        message["sender_id"] is not None
+        and message["sender_id"] == (message["tg_user_id"])
+    )
+    applicable = own and kind != ChatKind.CHANNEL and chat_id != message["tg_user_id"]
+    watermark = receipts["watermark"]
+    covered = watermark is not None and message_id <= watermark
+    seen = covered or any((exact_at, readers, receipts["listened_at"]))
+    evidence = watermark is not None or outcome in {"ok", "not_read_yet"}
+    due = fetch is None or fetch["retry_at"] <= now
+    return ReadInfoView(
+        chat_id=chat_id,
+        message_id=message_id,
+        chat_kind=kind.value,
+        own=own,
+        voice=message["voice"],
+        observed_at=observed_at,
+        listened_at=receipts["listened_at"],
+        exact_at=exact_at,
+        readers=readers,
+        read=applicable and seen,
+        unread=applicable and not seen and evidence,
+        outcome=outcome,
+        fetched_at=fetch["fetched_at"] if fetch else None,
+        can_fetch=applicable
+        and not message["is_bot"]
+        and now - date < FETCH_WINDOW
+        and exact_at is None
+        and inherited is None
+        and due,
+    )

backend/src/utils/read/rich_text.py

@@ -0,0 +1,61 @@
+from typing import Any
+
+CELL_GAP = " | "
+_TEXT_KEYS = ("summary", "text", "expression", "credit")
+_INLINE_KEYS = ("text", "alternative_text", "expression")
+
+
+def _button(button: dict[str, Any]) -> str:
+    label = _inline(button.get("text"))
+    url = button.get("url")
+    return f"{label} ({url})" if isinstance(url, str) else label
+
+
+def _inline(value: object) -> str:
+    if isinstance(value, str):
+        return value
+    if isinstance(value, list):
+        return "".join(_inline(item) for item in value)
+    if not isinstance(value, dict):
+        return ""
+    button = value.get("button")
+    if isinstance(button, dict):
+        return _button(button)
+    return next((_inline(value[key]) for key in _INLINE_KEYS if key in value), "")
+
+
+def _children(block: dict[str, Any], key: str) -> list[Any]:
+    value = block.get(key)
+    return value if isinstance(value, list) else []
+
+
+def _row(row: object) -> str:
+    cells = row if isinstance(row, list) else []
+    return CELL_GAP.join(
+        _inline(cell.get("text")) for cell in cells if isinstance(cell, dict)
+    )
+
+
+def _block(block: object) -> list[str]:
+    if not isinstance(block, dict):
+        return []
+    lines = [_inline(block[key]) for key in _TEXT_KEYS if key in block]
+    lines += [_row(row) for row in _children(block, "cells")]
+    lines += [
+        _button(button)
+        for button in _children(block, "buttons")
+        if isinstance(button, dict)
+    ]
+    for key in ("items", "blocks"):
+        for child in _children(block, key):
+            lines += _block(child)
+    lines += _block(block.get("caption"))
+    lines = [line for line in lines if line]
+    label = block.get("label")
+    if isinstance(label, str) and lines:
+        lines[0] = f"{label} {lines[0]}"
+    return lines
+
+
+def rich_text(rich: object) -> str | None:
+    return "\n".join(_block(rich)) or None

backend/src/utils/read/storage.py

@@ -1,5 +1,7 @@
 import asyncio
 from collections import defaultdict
+from itertools import groupby
+from operator import itemgetter
 
 import asyncpg
 
@@ -8,6 +10,7 @@ from utils.policy import repository as policy_repository
 from utils.read import chats
 from utils.read.models import (
     Page,
+    StorageChatView,
     StorageFileView,
     StorageGroupView,
     StorageScopeView,
@@ -34,6 +37,7 @@ WITH blobs AS (
     SELECT storage_key, max(file_size) AS size, min(kind) AS kind
     FROM media
     WHERE account_id = $1 AND downloaded AND storage_key IS NOT NULL
+      AND ($3::bigint IS NULL OR chat_id = $3)
     GROUP BY storage_key
 )
 SELECT kind, count(*) AS files, coalesce(sum(size), 0)::bigint AS bytes,
@@ -49,24 +53,49 @@ FROM media_trash WHERE account_id = $1
 
 _FILES = """
 WITH top AS MATERIALIZED (
-    SELECT storage_key, max(file_size) AS file_size, count(*) AS refs
+    SELECT storage_key, max(file_size) AS file_size
     FROM media
     WHERE account_id = $1 AND downloaded AND storage_key IS NOT NULL
       AND ($2::text[] IS NULL OR (kind = ANY($2)) = $3)
+      AND ($7::bigint IS NULL OR chat_id = $7)
     GROUP BY storage_key
     ORDER BY 2 DESC NULLS LAST, 1
     LIMIT $4 OFFSET $5
 )
 SELECT * FROM (
-    SELECT t.storage_key, t.file_size, t.refs, m.id, m.chat_id, m.message_id,
+    SELECT t.storage_key, t.file_size, m.id, m.chat_id, m.message_id,
         m.kind, m.mime, m.ttl_seconds,
-        row_number() OVER (PARTITION BY t.storage_key ORDER BY m.id) AS n
+        count(*) OVER (PARTITION BY t.storage_key) AS refs,
+        row_number() OVER (
+            PARTITION BY t.storage_key
+            ORDER BY (m.chat_id = $7) DESC NULLS LAST, m.id
+        ) AS n
     FROM top t
     JOIN media m ON m.storage_key = t.storage_key
         AND m.account_id = $1 AND m.downloaded
 ) ranked
 WHERE n <= $6
-ORDER BY file_size DESC NULLS LAST, storage_key, id
+ORDER BY file_size DESC NULLS LAST, storage_key, n
+"""
+
+_CHATS = """
+WITH blobs AS (
+    SELECT chat_id, storage_key, max(file_size) AS size, min(kind) AS kind
+    FROM media
+    WHERE account_id = $1 AND downloaded AND storage_key IS NOT NULL
+      AND ($2::text[] IS NULL OR (kind = ANY($2)) = $3)
+    GROUP BY chat_id, storage_key
+), top AS (
+    SELECT chat_id, coalesce(sum(size), 0)::bigint AS total
+    FROM blobs GROUP BY chat_id
+    ORDER BY 2 DESC, 1
+    LIMIT $4 OFFSET $5
+)
+SELECT t.chat_id, t.total, b.kind, count(*) AS files,
+    coalesce(sum(b.size), 0)::bigint AS bytes
+FROM top t JOIN blobs b USING (chat_id)
+GROUP BY t.chat_id, t.total, b.kind
+ORDER BY t.total DESC, t.chat_id
 """
 
 _TRASH = """
@@ -143,9 +172,7 @@ def _group_filter(group: str | None) -> tuple[list[str] | None, bool]:
     return list(GROUP_KINDS.get(group, ())), True
 
 
-async def get_summary(pool: asyncpg.Pool, account_id: int) -> StorageSummaryView:
-    rows = await pool.fetch(_SUMMARY, account_id, HEAVY_BYTES)
-    trash = await pool.fetchrow(_TRASH_TOTAL, account_id)
+def _groups(rows: list[asyncpg.Record]) -> list[StorageGroupView]:
     groups = {
         name: StorageGroupView(group=name, files=0, bytes=0)
         for name in (*GROUP_KINDS, OTHER_GROUP)
@@ -154,6 +181,14 @@ async def get_summary(pool: asyncpg.Pool, account_id: int) -> StorageSummaryView
         group = groups[_KIND_GROUP.get(row["kind"], OTHER_GROUP)]
         group.files += row["files"]
         group.bytes += row["bytes"]
+    return list(groups.values())
+
+
+async def get_summary(
+    pool: asyncpg.Pool, account_id: int, chat_id: int | None = None
+) -> StorageSummaryView:
+    rows = await pool.fetch(_SUMMARY, account_id, HEAVY_BYTES, chat_id)
+    trash = await pool.fetchrow(_TRASH_TOTAL, account_id)
     return StorageSummaryView(
         files=sum(row["files"] for row in rows),
         bytes=sum(row["bytes"] for row in rows),
@@ -162,10 +197,31 @@ async def get_summary(pool: asyncpg.Pool, account_id: int) -> StorageSummaryView
         heavy_threshold=HEAVY_BYTES,
         trash_files=trash["files"],
         trash_bytes=trash["bytes"],
-        groups=list(groups.values()),
+        groups=_groups(rows),
     )
 
 
+async def list_chats(
+    pool: asyncpg.Pool, account_id: int, group: str | None, page: Page
+) -> list[StorageChatView]:
+    kinds, include = _group_filter(group)
+    rows = await pool.fetch(
+        _CHATS, account_id, kinds, include, page.capped_limit, page.offset
+    )
+    return [
+        StorageChatView(
+            chat_id=chat_id,
+            files=sum(row["files"] for row in items),
+            bytes=items[0]["total"],
+            groups=_groups(items),
+        )
+        for chat_id, items in (
+            (chat_id, list(chunk))
+            for chat_id, chunk in groupby(rows, key=itemgetter("chat_id"))
+        )
+    ]
+
+
 async def _messages(
     pool: asyncpg.Pool, account_id: int, refs: list[asyncpg.Record]
 ) -> list[asyncpg.Record | None]:
@@ -189,18 +245,31 @@ def _alive(ref: asyncpg.Record, message: asyncpg.Record | None) -> bool:
 
 
 async def list_files(
-    pool: asyncpg.Pool, account_id: int, group: str | None, page: Page
+    pool: asyncpg.Pool,
+    account_id: int,
+    group: str | None,
+    page: Page,
+    chat_id: int | None = None,
 ) -> list[StorageFileView]:
     kinds, include = _group_filter(group)
     refs = await pool.fetch(
-        _FILES, account_id, kinds, include, page.capped_limit, page.offset, REFS_CHECKED
+        _FILES,
+        account_id,
+        kinds,
+        include,
+        page.capped_limit,
+        page.offset,
+        REFS_CHECKED,
+        chat_id,
     )
     messages = await _messages(pool, account_id, refs)
     files: dict[str, StorageFileView] = {}
     for ref, message in zip(refs, messages, strict=True):
         alive = _alive(ref, message)
         known = files.get(ref["storage_key"])
-        if known is not None and (known.refetchable or not alive):
+        inside = chat_id is None or ref["chat_id"] == chat_id
+        if known is not None and not (alive and inside and not known.refetchable):
+            known.refetchable = known.refetchable or alive
             continue
         files[ref["storage_key"]] = StorageFileView(
             media_id=ref["id"],

backend/src/utils/read/watches.py

@@ -4,13 +4,70 @@ from typing import Any
 
 import asyncpg
 
-from utils.read.models import AlertView, Page, WatchView
+from utils.read import chats as chats_read
+from utils.read import peers as peers_read
+from utils.read.models import (
+    AlertItem,
+    AlertMessage,
+    AlertPage,
+    AlertView,
+    AlertWatch,
+    ChatListItem,
+    ChatRef,
+    PeerView,
+    PersonRef,
+    WatchItem,
+    WatchView,
+)
 
 WATCHES_CHANGED_CHANNEL = "watches_changed"
 ALERTS_CHANGED_CHANNEL = "alerts_changed"
+ALERT_PAGE = 50
+MAX_ALERT_PAGE = 200
 
-_WATCH_COLS = "id, account_id, kind, params, enabled, created_at, updated_at"
+_WATCH_COLS = "id, account_id, kind, name, params, enabled, created_at, updated_at"
 _ALERT_COLS = "id, account_id, watch_id, ts, payload, seen, created_at"
+_LEGACY_EVENTS = {"keyword": "message", "peer_online": "online"}
+
+_INSERT_ALERT = f"""
+WITH alert AS (
+    INSERT INTO alerts (account_id, watch_id, ts, payload, seen, dedup_key)
+    VALUES ($1, $2, $3, $4::jsonb, false, $5)
+    ON CONFLICT (watch_id, dedup_key) WHERE dedup_key IS NOT NULL DO NOTHING
+    RETURNING {_ALERT_COLS}
+), hooks AS (
+    INSERT INTO webhook_deliveries (webhook_id, alert_id, event)
+    SELECT w.id, alert.id, 'alert' FROM alert
+    JOIN webhooks w ON w.account_id = alert.account_id AND w.enabled
+        AND (w.watch_ids IS NULL OR alert.watch_id = ANY(w.watch_ids))
+)
+SELECT * FROM alert
+"""  # noqa: S608
+
+_WATCH_STATS = """
+SELECT watch_id, count(*) AS count, max(ts) AS last FROM alerts
+WHERE watch_id = ANY($1::int[]) GROUP BY watch_id
+"""
+
+_ALERT_ROWS = """
+SELECT a.id, a.account_id, a.ts, a.seen, a.payload,
+       w.id AS watch_id, w.kind AS watch_kind, w.name AS watch_name,
+       w.params AS watch_params,
+       m.text AS stored_text,
+       m.sender_id AS stored_sender, m.date AS stored_date,
+       m.deleted_at IS NOT NULL AS stored_deleted,
+       m.edited_at IS NOT NULL AS stored_edited
+FROM alerts a
+JOIN watches w ON w.id = a.watch_id
+LEFT JOIN LATERAL (
+    SELECT text, sender_id, date, deleted_at, edited_at
+    FROM messages
+    WHERE account_id = a.account_id
+      AND chat_id = (a.payload->>'chat_id')::bigint
+      AND message_id = (a.payload->>'message_id')::bigint
+    ORDER BY date DESC LIMIT 1
+) m ON true
+"""
 
 
 def _to_watch(row: asyncpg.Record) -> WatchView:
@@ -25,6 +82,10 @@ def _to_alert(row: asyncpg.Record) -> AlertView:
     return AlertView(**data)
 
 
+async def _notify(pool: asyncpg.Pool, channel: str, payload: dict[str, Any]) -> None:
+    await pool.execute("SELECT pg_notify($1, $2)", channel, json.dumps(payload))
+
+
 async def list_watches(pool: asyncpg.Pool, account_id: int) -> list[WatchView]:
     rows = await pool.fetch(
         f"SELECT {_WATCH_COLS} FROM watches WHERE account_id = $1 "  # noqa: S608
@@ -42,19 +103,21 @@ async def get_watch(pool: asyncpg.Pool, watch_id: int) -> WatchView | None:
     return _to_watch(row) if row else None
 
 
-async def create_watch(
+async def create_watch(  # noqa: PLR0913
     pool: asyncpg.Pool,
     account_id: int,
     kind: str,
     params: dict[str, Any],
     *,
+    name: str | None = None,
     enabled: bool = True,
 ) -> WatchView:
     row = await pool.fetchrow(
-        "INSERT INTO watches (account_id, kind, params, enabled) "  # noqa: S608
-        f"VALUES ($1, $2, $3::jsonb, $4) RETURNING {_WATCH_COLS}",
+        "INSERT INTO watches (account_id, kind, name, params, enabled) "  # noqa: S608
+        f"VALUES ($1, $2, $3, $4::jsonb, $5) RETURNING {_WATCH_COLS}",
         account_id,
         kind,
+        name,
         json.dumps(params),
         enabled,
     )
@@ -62,13 +125,22 @@ async def create_watch(
     return _to_watch(row)
 
 
-async def update_watch(
-    pool: asyncpg.Pool, watch_id: int, params: dict[str, Any], *, enabled: bool
+async def update_watch(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    watch_id: int,
+    kind: str,
+    params: dict[str, Any],
+    *,
+    name: str | None,
+    enabled: bool,
 ) -> WatchView | None:
     row = await pool.fetchrow(
-        "UPDATE watches SET params = $2::jsonb, enabled = $3, updated_at = now() "  # noqa: S608
+        "UPDATE watches SET kind = $2, name = $3, params = $4::jsonb, "  # noqa: S608
+        "enabled = $5, updated_at = now() "
         f"WHERE id = $1 RETURNING {_WATCH_COLS}",
         watch_id,
+        kind,
+        name,
         json.dumps(params),
         enabled,
     )
@@ -79,29 +151,210 @@ async def update_watch(
 
 
 async def delete_watch(pool: asyncpg.Pool, watch_id: int) -> bool:
-    result = await pool.execute("DELETE FROM watches WHERE id = $1", watch_id)
-    if not result.endswith("1"):
+    account_id = await pool.fetchval(
+        "DELETE FROM watches WHERE id = $1 RETURNING account_id", watch_id
+    )
+    if account_id is None:
         return False
     await pool.execute(f"NOTIFY {WATCHES_CHANGED_CHANNEL}")
+    await _notify(pool, ALERTS_CHANGED_CHANNEL, {"account_id": account_id})
     return True
 
 
+def _chat_ref(chat_id: int, chat: ChatListItem | None, name: str | None) -> ChatRef:
+    if chat is None:
+        kind = "private" if chat_id > 0 else "group"
+        return ChatRef(id=chat_id, title=name, kind=kind, has_avatar=False)
+    kind = "channel" if chat.is_broadcast else chat.kind
+    return ChatRef(
+        id=chat_id, title=chat.title or name, kind=kind, has_avatar=chat.has_avatar
+    )
+
+
+def _peer_name(peer: PeerView) -> str | None:
+    full = " ".join(part for part in (peer.first_name, peer.last_name) if part)
+    return full or peer.username
+
+
+class Refs:
+    def __init__(self, chats: list[ChatListItem], people: list[PeerView]) -> None:
+        self._chats = {chat.chat_id: chat for chat in chats}
+        self._people = {peer.peer_id: peer for peer in people}
+
+    def chat(self, chat_id: int | None) -> ChatRef | None:
+        if chat_id is None:
+            return None
+        peer = self._people.get(chat_id)
+        name = _peer_name(peer) if peer else None
+        return _chat_ref(chat_id, self._chats.get(chat_id), name)
+
+    def person(self, person_id: int | None) -> PersonRef | None:
+        if person_id is None:
+            return None
+        peer = self._people.get(person_id)
+        chat = self._chats.get(person_id)
+        name = (_peer_name(peer) if peer else None) or (chat.title if chat else None)
+        has_avatar = (peer.has_avatar if peer else False) or bool(
+            chat and chat.has_avatar
+        )
+        return PersonRef(id=person_id, name=name, has_avatar=has_avatar)
+
+
+async def load_refs(
+    pool: asyncpg.Pool, account_id: int, chat_ids: set[int], person_ids: set[int]
+) -> Refs:
+    chats = await chats_read.get_chats(pool, account_id, sorted(chat_ids | person_ids))
+    people = await peers_read.get_peers(
+        pool, account_id, sorted(person_ids | {i for i in chat_ids if i > 0})
+    )
+    return Refs(chats, people)
+
+
+def _int(value: object) -> int | None:
+    return value if isinstance(value, int) and not isinstance(value, bool) else None
+
+
+def _watch_person(params: dict[str, Any]) -> int | None:
+    return _int(params.get("sender_id")) or _int(params.get("peer_id"))
+
+
+async def describe_watches(
+    pool: asyncpg.Pool, account_id: int, watches: list[WatchView]
+) -> list[WatchItem]:
+    targets = [
+        (_int(watch.params.get("chat_id")), _watch_person(watch.params))
+        for watch in watches
+    ]
+    refs = await load_refs(
+        pool,
+        account_id,
+        {chat for chat, _ in targets if chat is not None},
+        {person for _, person in targets if person is not None},
+    )
+    stats = {
+        row["watch_id"]: row
+        for row in await pool.fetch(_WATCH_STATS, [watch.id for watch in watches])
+    }
+    return [
+        WatchItem(
+            **watch.model_dump(),
+            chat=refs.chat(chat),
+            person=refs.person(person),
+            alert_count=stats[watch.id]["count"] if watch.id in stats else 0,
+            last_alert_at=stats[watch.id]["last"] if watch.id in stats else None,
+        )
+        for watch, (chat, person) in zip(watches, targets, strict=True)
+    ]
+
+
+def _path(chat_id: int | None, message_id: int | None, peer_id: int | None) -> str:
+    if chat_id is not None and message_id is not None:
+        return f"/app/{chat_id}?message={message_id}"
+    if peer_id is not None:
+        return f"/app/{peer_id}"
+    return "/app"
+
+
+def _alert_item(row: asyncpg.Record, refs: Refs) -> AlertItem:
+    payload = json.loads(row["payload"])
+    event = payload.get("event") or _LEGACY_EVENTS.get(row["watch_kind"], "message")
+    chat_id = _int(payload.get("chat_id"))
+    message_id = _int(payload.get("message_id"))
+    peer_id = _int(payload.get("peer_id"))
+    message = None
+    if message_id is not None:
+        date = row["stored_date"] or (
+            datetime.fromisoformat(payload["date"]) if payload.get("date") else None
+        )
+        message = AlertMessage(
+            id=message_id,
+            text=payload.get("text") or row["stored_text"],
+            sender=refs.person(_int(payload.get("sender_id")) or row["stored_sender"]),
+            date=date,
+            deleted=bool(row["stored_deleted"]) or event == "delete",
+            edited=bool(row["stored_edited"]) or event == "edit",
+        )
+    return AlertItem(
+        id=row["id"],
+        account_id=row["account_id"],
+        ts=row["ts"],
+        seen=row["seen"],
+        event=event,
+        watch=AlertWatch(
+            id=row["watch_id"],
+            kind=row["watch_kind"],
+            name=row["watch_name"],
+            params=json.loads(row["watch_params"]),
+        ),
+        chat=refs.chat(chat_id),
+        message=message,
+        peer=refs.person(peer_id),
+        away_since=payload.get("since"),
+        path=_path(chat_id, message_id, peer_id),
+    )
+
+
+async def _alert_items(
+    pool: asyncpg.Pool, account_id: int, rows: list[asyncpg.Record]
+) -> list[AlertItem]:
+    chat_ids: set[int] = set()
+    person_ids: set[int] = set()
+    for row in rows:
+        payload = json.loads(row["payload"])
+        for key, bucket in (
+            ("chat_id", chat_ids),
+            ("sender_id", person_ids),
+            ("peer_id", person_ids),
+        ):
+            value = _int(payload.get(key))
+            if value is not None:
+                bucket.add(value)
+        if row["stored_sender"] is not None:
+            person_ids.add(row["stored_sender"])
+    refs = await load_refs(pool, account_id, chat_ids, person_ids)
+    return [_alert_item(row, refs) for row in rows]
+
+
 async def list_alerts(
-    pool: asyncpg.Pool, account_id: int, page: Page, *, seen: bool | None = None
-) -> list[AlertView]:
+    pool: asyncpg.Pool,
+    account_id: int,
+    *,
+    seen: bool | None = None,
+    before: int | None = None,
+    limit: int = ALERT_PAGE,
+) -> AlertPage:
+    size = max(1, min(limit, MAX_ALERT_PAGE))
     params: list[object] = [account_id]
-    where = "account_id = $1"
+    where = "a.account_id = $1"
     if seen is not None:
         params.append(seen)
-        where += f" AND seen = ${len(params)}"
-    params.append(page.capped_limit)
-    params.append(page.offset)
+        where += f" AND a.seen = ${len(params)}"
+    if before is not None:
+        params.append(before)
+        where += f" AND a.id < ${len(params)}"
+    params.append(size + 1)
     rows = await pool.fetch(
-        f"SELECT {_ALERT_COLS} FROM alerts WHERE {where} "  # noqa: S608
-        f"ORDER BY ts DESC LIMIT ${len(params) - 1} OFFSET ${len(params)}",
-        *params,
+        f"{_ALERT_ROWS} WHERE {where} ORDER BY a.id DESC LIMIT ${len(params)}", *params
+    )
+    items = await _alert_items(pool, account_id, rows[:size])
+    return AlertPage(
+        items=items,
+        next_before=items[-1].id if len(rows) > size else None,
+        unseen=await unseen_count(pool, account_id),
+    )
+
+
+async def get_alert(pool: asyncpg.Pool, alert_id: int) -> AlertItem | None:
+    row = await pool.fetchrow(f"{_ALERT_ROWS} WHERE a.id = $1", alert_id)
+    if row is None:
+        return None
+    return (await _alert_items(pool, row["account_id"], [row]))[0]
+
+
+async def unseen_count(pool: asyncpg.Pool, account_id: int) -> int:
+    return await pool.fetchval(
+        "SELECT count(*) FROM alerts WHERE account_id = $1 AND NOT seen", account_id
     )
-    return [_to_alert(row) for row in rows]
 
 
 async def insert_alert(
@@ -113,10 +366,7 @@ async def insert_alert(
     dedup_key: str | None = None,
 ) -> AlertView | None:
     row = await pool.fetchrow(
-        "INSERT INTO alerts (account_id, watch_id, ts, payload, seen, dedup_key) "  # noqa: S608
-        "VALUES ($1, $2, $3, $4::jsonb, false, $5) "
-        "ON CONFLICT (watch_id, dedup_key) WHERE dedup_key IS NOT NULL "
-        f"DO NOTHING RETURNING {_ALERT_COLS}",
+        _INSERT_ALERT,
         account_id,
         watch_id,
         datetime.now(UTC),
@@ -125,10 +375,32 @@ async def insert_alert(
     )
     if row is None:
         return None
-    await pool.execute(f"NOTIFY {ALERTS_CHANGED_CHANNEL}")
+    await _notify(
+        pool, ALERTS_CHANGED_CHANNEL, {"account_id": account_id, "alert_id": row["id"]}
+    )
     return _to_alert(row)
 
 
 async def mark_alert_seen(pool: asyncpg.Pool, alert_id: int) -> bool:
-    result = await pool.execute("UPDATE alerts SET seen = true WHERE id = $1", alert_id)
-    return result.endswith("1")
+    account_id = await pool.fetchval(
+        "UPDATE alerts SET seen = true WHERE id = $1 RETURNING account_id", alert_id
+    )
+    if account_id is None:
+        return False
+    await _notify(pool, ALERTS_CHANGED_CHANNEL, {"account_id": account_id})
+    return True
+
+
+async def mark_all_seen(
+    pool: asyncpg.Pool, account_id: int, up_to: int | None = None
+) -> int:
+    result = await pool.execute(
+        "UPDATE alerts SET seen = true WHERE account_id = $1 AND NOT seen "
+        "AND ($2::int IS NULL OR id <= $2)",
+        account_id,
+        up_to,
+    )
+    marked = int(result.split()[-1])
+    if marked:
+        await _notify(pool, ALERTS_CHANGED_CHANNEL, {"account_id": account_id})
+    return marked

backend/src/utils/read/webhooks.py

@@ -0,0 +1,198 @@
+import json
+import secrets
+from datetime import datetime
+
+import asyncpg
+
+from utils.read.models import DeliveryView, WebhookView
+
+STATUS_PENDING = "pending"
+STATUS_SENDING = "sending"
+STATUS_DELIVERED = "delivered"
+STATUS_FAILED = "failed"
+DELIVERY_LOG = 20
+
+_DELIVERY_COLS = (
+    "id, webhook_id, alert_id, event, status, attempts, next_attempt_at, "
+    "response_code, last_error, created_at, finished_at"
+)
+
+_HOOKS = f"""
+SELECT w.id, w.account_id, w.url, w.secret, w.enabled, w.watch_ids,
+       w.created_at, w.updated_at, to_jsonb(d) AS last_delivery
+FROM webhooks w
+LEFT JOIN LATERAL (
+    SELECT {_DELIVERY_COLS} FROM webhook_deliveries
+    WHERE webhook_id = w.id ORDER BY id DESC LIMIT 1
+) d ON true
+"""  # noqa: S608
+
+_CLAIM = """
+UPDATE webhook_deliveries d SET status = 'sending', attempts = d.attempts + 1
+FROM webhooks w
+WHERE w.id = d.webhook_id AND d.id IN (
+    SELECT id FROM webhook_deliveries
+    WHERE status = 'pending' AND next_attempt_at <= now()
+    ORDER BY next_attempt_at LIMIT $1
+    FOR UPDATE SKIP LOCKED
+)
+RETURNING d.id, d.alert_id, d.event, d.attempts, d.body,
+          w.id AS webhook_id, w.account_id, w.url, w.secret
+"""
+
+_FINISH = """
+UPDATE webhook_deliveries SET
+    status = $2, response_code = $3, last_error = $4,
+    next_attempt_at = coalesce($5, next_attempt_at),
+    finished_at = CASE WHEN $2 = 'pending' THEN NULL ELSE now() END
+WHERE id = $1
+"""
+
+
+def new_secret() -> str:
+    return secrets.token_hex(32)
+
+
+def _to_webhook(row: asyncpg.Record) -> WebhookView:
+    data = dict(row)
+    last = data.pop("last_delivery")
+    return WebhookView(
+        **data, last_delivery=DeliveryView(**json.loads(last)) if last else None
+    )
+
+
+async def list_webhooks(pool: asyncpg.Pool, account_id: int) -> list[WebhookView]:
+    rows = await pool.fetch(
+        f"{_HOOKS} WHERE w.account_id = $1 ORDER BY w.id", account_id
+    )
+    return [_to_webhook(row) for row in rows]
+
+
+async def get_webhook(pool: asyncpg.Pool, webhook_id: int) -> WebhookView | None:
+    row = await pool.fetchrow(f"{_HOOKS} WHERE w.id = $1", webhook_id)
+    return _to_webhook(row) if row else None
+
+
+async def create_webhook(
+    pool: asyncpg.Pool,
+    account_id: int,
+    url: str,
+    watch_ids: list[int] | None,
+    *,
+    enabled: bool,
+) -> WebhookView | None:
+    webhook_id = await pool.fetchval(
+        "INSERT INTO webhooks (account_id, url, secret, enabled, watch_ids) "
+        "VALUES ($1, $2, $3, $4, $5) RETURNING id",
+        account_id,
+        url,
+        new_secret(),
+        enabled,
+        watch_ids,
+    )
+    return await get_webhook(pool, webhook_id)
+
+
+async def update_webhook(
+    pool: asyncpg.Pool,
+    webhook_id: int,
+    url: str,
+    watch_ids: list[int] | None,
+    *,
+    enabled: bool,
+) -> WebhookView | None:
+    async with pool.acquire() as conn, conn.transaction():
+        found = await conn.fetchval(
+            "UPDATE webhooks SET url = $2, enabled = $3, watch_ids = $4, "
+            "updated_at = now() WHERE id = $1 RETURNING id",
+            webhook_id,
+            url,
+            enabled,
+            watch_ids,
+        )
+        if found is not None and not enabled:
+            await conn.execute(
+                "UPDATE webhook_deliveries SET status = 'failed', "
+                "last_error = 'endpoint disabled', finished_at = now() "
+                "WHERE webhook_id = $1 AND status = 'pending'",
+                webhook_id,
+            )
+    return await get_webhook(pool, webhook_id)
+
+
+async def rotate_secret(pool: asyncpg.Pool, webhook_id: int) -> WebhookView | None:
+    await pool.execute(
+        "UPDATE webhooks SET secret = $2, updated_at = now() WHERE id = $1",
+        webhook_id,
+        new_secret(),
+    )
+    return await get_webhook(pool, webhook_id)
+
+
+async def delete_webhook(pool: asyncpg.Pool, webhook_id: int) -> bool:
+    result = await pool.execute("DELETE FROM webhooks WHERE id = $1", webhook_id)
+    return result.endswith("1")
+
+
+async def list_deliveries(
+    pool: asyncpg.Pool, webhook_id: int, limit: int = DELIVERY_LOG
+) -> list[DeliveryView]:
+    rows = await pool.fetch(
+        f"SELECT {_DELIVERY_COLS} FROM webhook_deliveries "  # noqa: S608
+        "WHERE webhook_id = $1 ORDER BY id DESC LIMIT $2",
+        webhook_id,
+        limit,
+    )
+    return [DeliveryView(**dict(row)) for row in rows]
+
+
+async def get_delivery(pool: asyncpg.Pool, delivery_id: int) -> DeliveryView | None:
+    row = await pool.fetchrow(
+        f"SELECT {_DELIVERY_COLS} FROM webhook_deliveries WHERE id = $1",  # noqa: S608
+        delivery_id,
+    )
+    return DeliveryView(**dict(row)) if row else None
+
+
+async def start_test(pool: asyncpg.Pool, webhook_id: int, body: str) -> int:
+    return await pool.fetchval(
+        "INSERT INTO webhook_deliveries (webhook_id, event, status, attempts, body) "
+        "VALUES ($1, 'test', 'sending', 1, $2) RETURNING id",
+        webhook_id,
+        body,
+    )
+
+
+async def release_stuck(pool: asyncpg.Pool) -> None:
+    await pool.execute(
+        "UPDATE webhook_deliveries SET status = 'pending' WHERE status = 'sending'"
+    )
+
+
+async def claim_due(pool: asyncpg.Pool, limit: int) -> list[asyncpg.Record]:
+    return await pool.fetch(_CLAIM, limit)
+
+
+async def save_body(pool: asyncpg.Pool, delivery_id: int, body: str) -> None:
+    await pool.execute(
+        "UPDATE webhook_deliveries SET body = $2 WHERE id = $1", delivery_id, body
+    )
+
+
+async def finish(  # noqa: PLR0913
+    pool: asyncpg.Pool,
+    delivery_id: int,
+    status: str,
+    response_code: int | None,
+    error: str | None,
+    next_attempt_at: datetime | None = None,
+) -> None:
+    await pool.execute(
+        _FINISH, delivery_id, status, response_code, error, next_attempt_at
+    )
+
+
+async def next_due(pool: asyncpg.Pool) -> datetime | None:
+    return await pool.fetchval(
+        "SELECT min(next_attempt_at) FROM webhook_deliveries WHERE status = 'pending'"
+    )

backend/src/utils/search/found.py

@@ -0,0 +1,65 @@
+import re
+from collections.abc import Iterator
+from typing import Any
+
+from utils.search.models import Found, FoundWhere
+
+HIDDEN_KEYS = ("chat", "from_user", "sender_chat")
+SKIPPED_KEYS = ("_", "file_id", "file_unique_id")
+ENUM = re.compile(r"^[A-Z][A-Za-z]+\.[A-Z_]+$")
+
+_HEADS: dict[str, FoundWhere] = {
+    "text": "text",
+    "caption": "text",
+    "web_page": "preview",
+}
+_LEAVES: dict[str, FoundWhere] = {"url": "link", "file_name": "file"}
+
+type Path = tuple[str | int, ...]
+
+
+def _scalars(node: object, path: Path = ()) -> Iterator[tuple[Path, str, object]]:
+    if isinstance(node, dict):
+        items = ((k, v) for k, v in node.items() if k not in SKIPPED_KEYS)
+    elif isinstance(node, list):
+        items = enumerate(node)
+    else:
+        return
+    for key, value in items:
+        if isinstance(value, dict | list):
+            yield from _scalars(value, (*path, key))
+        elif isinstance(value, str | int | float) and not isinstance(value, bool):
+            yield (*path, key), str(value), node
+
+
+def _where(path: Path) -> FoundWhere:
+    head, leaf = str(path[0]), str(path[-1])
+    if head == "reply_markup":
+        return "button"
+    if leaf in _LEAVES:
+        return _LEAVES[leaf]
+    if head.startswith("forward"):
+        return "forward"
+    if head.startswith("reply_to"):
+        return "reply"
+    return _HEADS.get(head, "data")
+
+
+def find(raw: dict[str, Any], needle: str) -> Found | None:
+    folded = needle.casefold()
+    fallback = None
+    for path, value, owner in _scalars(raw):
+        if folded not in value.casefold() or ENUM.match(value):
+            continue
+        where = _where(path)
+        button = owner.get("text") if isinstance(owner, dict) else None
+        found = Found(
+            where=where,
+            path=".".join(map(str, path)),
+            value=value,
+            button=button if where == "button" else None,
+        )
+        if where != "text":
+            return found
+        fallback = fallback or found
+    return fallback

backend/src/utils/search/models.py

@@ -1,23 +1,59 @@
-from datetime import datetime
+from datetime import UTC, datetime, timedelta
 from typing import Literal
 
-from pydantic import BaseModel
+from pydantic import BaseModel, field_validator
 
 DEFAULT_LIMIT = 50
 MAX_LIMIT = 500
 
+_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
+_MICROSECOND = timedelta(microseconds=1)
+
+ChatKind = Literal["private", "group", "channel"]
+HitSource = Literal["text", "stt", "version", "data"]
+FoundWhere = Literal[
+    "text", "button", "link", "forward", "reply", "file", "preview", "data"
+]
+
+
+def encode_cursor(date: datetime, chat_id: int, message_id: int) -> str:
+    return f"{(date - _EPOCH) // _MICROSECOND}_{chat_id}_{message_id}"
+
+
+def decode_cursor(cursor: str) -> tuple[datetime, int, int]:
+    micros, chat_id, message_id = (int(part) for part in cursor.split("_"))
+    return _EPOCH + micros * _MICROSECOND, chat_id, message_id
+
 
 class SearchFilters(BaseModel):
     account_id: int
     query: str | None = None
     chat_id: int | None = None
     sender_id: int | None = None
+    kind: ChatKind | None = None
     has_media: bool | None = None
+    deleted: bool | None = None
+    edited: bool | None = None
     date_from: datetime | None = None
     date_to: datetime | None = None
     regex: str | None = None
+    everywhere: bool = False
+    cursor: str | None = None
     limit: int = DEFAULT_LIMIT
-    offset: int = 0
+
+    @field_validator("cursor")
+    @classmethod
+    def _valid_cursor(cls, value: str | None) -> str | None:
+        if value is not None:
+            decode_cursor(value)
+        return value
+
+
+class Found(BaseModel):
+    where: FoundWhere
+    path: str
+    value: str
+    button: str | None = None
 
 
 class SearchHit(BaseModel):
@@ -26,5 +62,19 @@ class SearchHit(BaseModel):
     date: datetime
     sender_id: int | None
     text: str | None
-    source: Literal["text", "stt"]
-    extracted_text: str | None
+    has_media: bool
+    media_kind: str | None
+    edited_at: datetime | None
+    deleted_at: datetime | None
+    source: HitSource
+    extracted_text: str | None = None
+    version_text: str | None = None
+    version_at: datetime | None = None
+    found: Found | None = None
+
+
+class SearchPage(BaseModel):
+    hits: list[SearchHit]
+    next_cursor: str | None
+    terms: list[str]
+    scanned_to: datetime | None = None

backend/src/utils/search/repository.py

@@ -1,88 +1,279 @@
+import json
+import re
+from datetime import UTC, datetime, timedelta
+from time import monotonic
+
 import asyncpg
 
-from utils.search.models import MAX_LIMIT, SearchFilters, SearchHit
-
-_TEXT_TSV = "to_tsvector('russian', coalesce(m.text, ''))"
-_STT_TSV = "to_tsvector('russian', coalesce(md.extracted_text, ''))"
-_TSQUERY = "websearch_to_tsquery('russian', ${})"
-
-_HITS = "{cols} FROM messages m WHERE {where}"  # text branch
-_STT_HITS = (
-    "{cols} FROM messages m "
-    "JOIN media md ON md.account_id = m.account_id "
-    "AND md.chat_id = m.chat_id AND md.message_id = m.message_id "
-    "WHERE {where}"
+from utils.read.message_view import load_raw
+from utils.search.found import ENUM, HIDDEN_KEYS, SKIPPED_KEYS, find
+from utils.search.models import (
+    MAX_LIMIT,
+    ChatKind,
+    SearchFilters,
+    SearchHit,
+    SearchPage,
+    decode_cursor,
+    encode_cursor,
 )
-_TEXT_COLS = (
-    "SELECT m.chat_id, m.message_id, m.date, m.sender_id, m.text, "
-    "'text' AS source, NULL::text AS extracted_text"
+
+MIN_NEEDLE = 3
+_WINDOW = timedelta(days=7)
+_BUDGET_SECONDS = 2.5
+_BOTTOM = -(2**63)
+_LIKE_SPECIAL = re.compile(r"([\\%_])")
+
+_PARSE = """
+SELECT (regexp_replace(websearch_to_tsquery('russian', q)::text,
+                       '''((?:[^'']|''''){3,})''', '''\\1'':*', 'g')::tsquery
+     || regexp_replace(websearch_to_tsquery('simple', q)::text,
+                       '''((?:[^'']|''''){3,})''', '''\\1'':*', 'g')::tsquery
+       )::text AS tsquery,
+       tsvector_to_array(search_vector(q)) AS terms
+FROM translate($1, 'Ёё', 'Ее') AS q
+"""
+
+_CHANNELS = """
+SELECT coalesce(array_agg(chat_id), '{}'::bigint[]) FROM (
+    SELECT DISTINCT ON (chat_id) chat_id,
+           coalesce(raw->'chat'->>'type', raw->>'type') AS type
+    FROM chat_history
+    WHERE account_id = $1
+      AND coalesce(raw->'chat'->>'type', raw->>'type') IS NOT NULL
+    ORDER BY chat_id, ts DESC
+) typed WHERE type = 'ChatType.CHANNEL'
+"""
+
+_KEY_ORDER = "s.date DESC, s.chat_id DESC, s.message_id DESC"
+
+_COLS = (
+    "s.chat_id, s.message_id, s.date, s.sender_id, s.text, s.has_media, "
+    "s.edited_at, s.deleted_at"
 )
-_STT_COLS = (
-    "SELECT m.chat_id, m.message_id, m.date, m.sender_id, m.text, "
-    "'stt' AS source, md.extracted_text"
+
+_TEXT = (
+    f"SELECT {_COLS}, 0 AS rank, NULL::text AS extracted_text, "  # noqa: S608
+    "NULL::text AS version_text, NULL::timestamptz AS version_at "
+    "FROM message_search s WHERE {where}"
+)
+
+_PROBE = (
+    "CROSS JOIN LATERAL (SELECT * FROM message_search s "
+    "WHERE s.chat_id = {source}.chat_id AND s.message_id = {source}.message_id "
+    "AND {where} LIMIT 1) s "
+)
+
+_STT = (
+    f"SELECT {_COLS}, 1 AS rank, md.extracted_text, NULL::text, "  # noqa: S608
+    "NULL::timestamptz FROM media md "
+    f"{_PROBE.replace('{source}', 'md')}"
+    "WHERE md.account_id = {account} "
+    "AND search_vector(md.extracted_text) @@ {tsquery}"
+)
+
+_VERSION = (
+    "SELECT DISTINCT ON (s.date, s.chat_id, s.message_id) "  # noqa: S608
+    f"{_COLS}, 2 AS rank, NULL::text, v.text, v.observed_at "
+    "FROM message_versions v "
+    f"{_PROBE.replace('{source}', 'v')}"
+    "WHERE v.account_id = {account} "
+    "AND search_vector(v.text) @@ {tsquery}"
+)
+
+_WITH_MEDIA = (
+    "SELECT h.*, md.kind AS media_kind FROM ({hits}) h "
+    "LEFT JOIN media md ON md.account_id = {account} "
+    "AND md.chat_id = h.chat_id AND md.message_id = h.message_id "
+    "ORDER BY h.date DESC, h.chat_id DESC, h.message_id DESC"
+)
+
+_SOURCES = ("text", "stt", "version")
+
+_OLDEST = "SELECT min(date) FROM message_search WHERE account_id = $1"
+
+_VALUE_MATCH = (
+    'strict $.** ? (@.type() == "object").keyvalue() ? ('
+    + "".join(f'@.key != "{key}" && ' for key in SKIPPED_KEYS)
+    + '(@.value.type() == "string" || @.value.type() == "number") && '
+    f"!(@.value.string() like_regex {json.dumps(ENUM.pattern)}) && "
+    '@.value.string() like_regex {needle} flag "iq")'
+)
+
+_EVERYWHERE = (
+    f"SELECT {_COLS}, s.raw - {{hidden}}::text[] AS raw "  # noqa: S608
+    "FROM messages s WHERE {where} "
+    "AND CASE WHEN s.raw::text ILIKE {like} "
+    "THEN jsonb_path_exists(s.raw - {hidden}::text[], {path}::jsonpath) "
+    f"ELSE false END ORDER BY {_KEY_ORDER} LIMIT {{limit}}"
 )
 
 
-def _common_conditions(filters: SearchFilters, params: list[object]) -> list[str]:
-    params.append(filters.account_id)
-    conds = [f"m.account_id = ${len(params)}"]
-    if filters.chat_id is not None:
-        params.append(filters.chat_id)
-        conds.append(f"m.chat_id = ${len(params)}")
-    if filters.sender_id is not None:
-        params.append(filters.sender_id)
-        conds.append(f"m.sender_id = ${len(params)}")
-    if filters.has_media is not None:
-        params.append(filters.has_media)
-        conds.append(f"m.has_media = ${len(params)}")
-    if filters.date_from is not None:
-        params.append(filters.date_from)
-        conds.append(f"m.date >= ${len(params)}")
-    if filters.date_to is not None:
-        params.append(filters.date_to)
-        conds.append(f"m.date <= ${len(params)}")
+class _Binder:
+    def __init__(self) -> None:
+        self.params: list[object] = []
+
+    def __call__(self, value: object) -> str:
+        self.params.append(value)
+        return f"${len(self.params)}"
+
+
+def _kind(kind: ChatKind, channels: list[int], bind: _Binder) -> str:
+    if kind == "private":
+        return "s.chat_id > 0"
+    member = f"s.chat_id = ANY({bind(channels)}::bigint[])"
+    return member if kind == "channel" else f"s.chat_id < 0 AND NOT {member}"
+
+
+def _conditions(
+    filters: SearchFilters, channels: list[int], bind: _Binder
+) -> list[str]:
+    conds = [f"s.account_id = {bind(filters.account_id)}"]
+    compared = (
+        ("s.chat_id =", filters.chat_id),
+        ("s.sender_id =", filters.sender_id),
+        ("s.has_media =", filters.has_media),
+        ("s.date >=", filters.date_from),
+        ("s.date <=", filters.date_to),
+        ("s.text ~", filters.regex),
+    )
+    conds += [f"{left} {bind(value)}" for left, value in compared if value is not None]
+    flags = (("s.deleted_at", filters.deleted), ("s.edited_at", filters.edited))
+    conds += [
+        f"{column} IS {'NOT ' if flag else ''}NULL"
+        for column, flag in flags
+        if flag is not None
+    ]
+    if filters.kind is not None:
+        conds.append(_kind(filters.kind, channels, bind))
+    if filters.cursor is not None:
+        date, chat_id, message_id = decode_cursor(filters.cursor)
+        conds.append(
+            "(s.date, s.chat_id, s.message_id) < "
+            f"({bind(date)}, {bind(chat_id)}, {bind(message_id)})"
+        )
     return conds
 
 
-async def search_messages(
-    pool: asyncpg.Pool, filters: SearchFilters
-) -> list[SearchHit]:
-    query = filters.query.strip() if filters.query else None
-    params: list[object] = []
-    common = _common_conditions(filters, params)
-    if filters.regex is not None:
-        params.append(filters.regex)
-        common.append(f"m.text ~ ${len(params)}")
+def _branch(sql: str, limit: str, order: str = _KEY_ORDER, **parts: str) -> str:
+    return f"({sql.format(**parts)} ORDER BY {order} LIMIT {limit})"
 
-    q_idx: int | None = None
-    if query is not None:
-        params.append(query)
-        q_idx = len(params)
 
-    text_conds = [*common]
-    if q_idx is not None:
-        text_conds.append(f"{_TEXT_TSV} @@ {_TSQUERY.format(q_idx)}")
+async def search_messages(pool: asyncpg.Pool, filters: SearchFilters) -> SearchPage:
+    async with pool.acquire() as conn, conn.transaction():
+        await conn.execute("SET LOCAL jit = off")
+        if filters.everywhere:
+            return await _everywhere(conn, filters)
+        return await _search(conn, filters)
 
-    branches = [_HITS.format(cols=_TEXT_COLS, where=" AND ".join(text_conds))]
-    if q_idx is not None:
-        stt_conds = [*common, f"{_STT_TSV} @@ {_TSQUERY.format(q_idx)}"]
-        branches.append(_STT_HITS.format(cols=_STT_COLS, where=" AND ".join(stt_conds)))
 
-    params.append(min(filters.limit, MAX_LIMIT))
-    limit_idx = len(params)
-    params.append(filters.offset)
-    offset_idx = len(params)
-    page = f"ORDER BY date DESC LIMIT ${limit_idx} OFFSET ${offset_idx}"
+async def _channels(conn: asyncpg.Connection, filters: SearchFilters) -> list[int]:
+    if filters.kind in {"group", "channel"}:
+        return await conn.fetchval(_CHANNELS, filters.account_id)
+    return []
 
-    if len(branches) == 1:
-        sql = f"SELECT * FROM ({branches[0]}) hits {page}"  # noqa: S608
-    else:
-        union = " UNION ALL ".join(f"({b})" for b in branches)
-        dedup = (
-            "SELECT DISTINCT ON (chat_id, message_id) * "  # noqa: S608
-            f"FROM ({union}) hits ORDER BY chat_id, message_id, source DESC"
+
+def _data_hit(row: asyncpg.Record, needle: str) -> SearchHit:
+    data = dict(row)
+    raw = load_raw(data.pop("raw"))
+    return SearchHit(**data, source="data", found=find(raw, needle))
+
+
+async def _everywhere(conn: asyncpg.Connection, filters: SearchFilters) -> SearchPage:
+    needle = (filters.query or "").strip()
+    floor = filters.date_from or await conn.fetchval(_OLDEST, filters.account_id)
+    if len(needle) < MIN_NEEDLE or floor is None:
+        return SearchPage(hits=[], next_cursor=None, terms=[])
+    channels = await _channels(conn, filters)
+    quoted = json.dumps(needle, ensure_ascii=False)
+    like = _LIKE_SPECIAL.sub(r"\\\1", quoted[1:-1])
+    path = _VALUE_MATCH.format(needle=quoted)
+    size = min(filters.limit, MAX_LIMIT)
+    cursor = filters.cursor
+    top = decode_cursor(cursor)[0] if cursor else None
+    hits: list[SearchHit] = []
+    started = monotonic()
+    while True:
+        lower = max((top or filters.date_to or datetime.now(UTC)) - _WINDOW, floor)
+        bind = _Binder()
+        conds = _conditions(
+            filters.model_copy(update={"cursor": cursor}), channels, bind
         )
-        sql = f"SELECT * FROM ({dedup}) d {page}"  # noqa: S608
+        conds.append(f"s.date >= {bind(lower)}")
+        if top is not None:
+            conds.append(f"s.date <= {bind(top)}")
+        limit = size - len(hits)
+        sql = _EVERYWHERE.format(
+            where=" AND ".join(conds),
+            hidden=bind(list(HIDDEN_KEYS)),
+            like=bind(f"%{like}%"),
+            path=bind(path),
+            limit=bind(limit),
+        )
+        account = bind(filters.account_id)
+        rows = await conn.fetch(
+            _WITH_MEDIA.format(hits=sql, account=account), *bind.params
+        )
+        hits += [_data_hit(row, needle) for row in rows]
+        if len(rows) >= limit:
+            last = rows[-1]
+            top = last["date"]
+            cursor = encode_cursor(top, last["chat_id"], last["message_id"])
+        elif lower > floor:
+            top = lower
+            cursor = encode_cursor(lower, _BOTTOM, _BOTTOM)
+        else:
+            return SearchPage(hits=hits, next_cursor=None, terms=[], scanned_to=floor)
+        if len(hits) >= size or monotonic() - started > _BUDGET_SECONDS:
+            return SearchPage(hits=hits, next_cursor=cursor, terms=[], scanned_to=top)
 
-    rows = await pool.fetch(sql, *params)
-    return [SearchHit(**dict(row)) for row in rows]
+
+async def _search(conn: asyncpg.Connection, filters: SearchFilters) -> SearchPage:
+    query = (filters.query or "").strip()
+    terms: list[str] = []
+    tsquery = None
+    if query:
+        parsed = await conn.fetchrow(_PARSE, query)
+        terms = list(parsed["terms"])
+        if not parsed["tsquery"]:
+            return SearchPage(hits=[], next_cursor=None, terms=terms)
+        tsquery = parsed["tsquery"]
+
+    channels = await _channels(conn, filters)
+    bind = _Binder()
+    where = " AND ".join(_conditions(filters, channels, bind))
+    size = min(filters.limit, MAX_LIMIT)
+    limit = bind(size + 1)
+
+    account = bind(filters.account_id)
+    if tsquery is None:
+        sql = _branch(_TEXT, limit, where=where)
+    else:
+        match = f"{bind(tsquery)}::tsquery"
+        branches = (
+            _branch(_TEXT, limit, where=f"{where} AND s.tsv @@ {match}"),
+            _branch(_STT, limit, where=where, account=account, tsquery=match),
+            _branch(
+                _VERSION,
+                limit,
+                f"{_KEY_ORDER}, v.observed_at DESC",
+                where=where,
+                account=account,
+                tsquery=match,
+            ),
+        )
+        sql = (
+            "SELECT DISTINCT ON (date, chat_id, message_id) * FROM ("  # noqa: S608
+            f"{' UNION ALL '.join(branches)}) s "
+            f"ORDER BY {_KEY_ORDER}, rank LIMIT {limit}"
+        )
+
+    rows = await conn.fetch(_WITH_MEDIA.format(hits=sql, account=account), *bind.params)
+    hits = [
+        SearchHit(**{**dict(row), "source": _SOURCES[row["rank"]]})
+        for row in rows[:size]
+    ]
+    next_cursor = None
+    if len(rows) > size:
+        last = hits[-1]
+        next_cursor = encode_cursor(last.date, last.chat_id, last.message_id)
+    return SearchPage(hits=hits, next_cursor=next_cursor, terms=terms)

backend/src/utils/storage.py

@@ -24,6 +24,10 @@ class ContentAddressedStorage:
     def get(self, key: str) -> bytes:
         return self._path(key).read_bytes()
 
+    def head(self, key: str, size: int = 16) -> bytes:
+        with self._path(key).open("rb") as file:
+            return file.read(size)
+
     def exists(self, key: str) -> bool:
         return self._path(key).exists()