beaver/beavergram
2
0 Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Compare commits

base: beaver/beavergram:e5a5edcaae53beb27717e7186ad5167988455b15
beaver/beavergram:main beaver/beavergram:dev beaver/beavergram:templates/update
..
compare: beaver/beavergram:templates/update
beaver/beavergram:main beaver/beavergram:dev beaver/beavergram:templates/update
3 Commits
e5a5edcaae .. templates/update
Author SHA1 Message Date
renovate Bot ec5979437d chore(frontend): update template to v0.8.0 2026-10-20 05:00:56 +00:00
renovate Bot 59f171a62d chore(backend): update template to v0.7.1 2026-10-20 05:00:50 +00:00
renovate Bot e3559da2a7 chore(deps): update code dependencies (dev) (#21) 
Co-authored-by: renovate <renovate@hhhhh.dev>
 2026-10-18 04:02:20 +00:00
189 changed files with 2262 additions and 16724 deletions
Expand all files Collapse all files

No files matched your search

.copier/backend.yml
+1 -1
View File
@@ -1,4 +1,4 @@
_commit: v0.6.0
_commit: v0.7.1
_src_path: https://git.kotikot.com/templates/backend-python.git
author_email: h@kotikot.com
author_name: h
.copier/frontend.yml
+1 -1
View File
@@ -1,4 +1,4 @@
_commit: v0.7.0
_commit: v0.8.0
_src_path: https://git.kotikot.com/templates/frontend-svelte.git
install_deps: false
layout: part
README.md
+2 -2
View File
@@ -19,12 +19,12 @@ Nothing is ever deleted from the archive.
## Parts
- `backend/` - `userbot` (kurigram, the capture loop and jobs), `watcher` (kurigram again, one bot-token session per watched bot) and `api` (FastAPI: the web API, the MCP server, and the built frontend as static files).
- `frontend/` - SvelteKit, `adapter-static`; a chat client over the archive: folders, chat list, thread with media, versions and the deleted messages inline, a feed of what was deleted or edited across chats, search, stories, presence graphs.
- `frontend/` - SvelteKit, `adapter-static`; a chat client over the archive: folders, chat list, thread with media, versions and the deleted messages inline, search, stories, presence graphs.
- `docker-compose.yml` - includes `backend/` (Postgres with TimescaleDB, `userbot`, `api`) and `frontend/` (the dev server); profiles `external` and `services`.
## MCP
Read-only: `list_watched_bots`, `get_bot_stats`, `get_bot_updates`, `search_messages_tool`, `list_chats`, `get_chat_history` (with deleted messages included), `get_deleted_messages`, `get_feed`, `get_message_versions`, `get_media`, `get_callbacks`, `presence_history`, `presence_stats`, `get_peer_history`, `get_person`, `get_stories`, `get_annotations`, `list_watches`, `list_alerts`. The one write is `set_watch`. There are no send, edit or delete tools and there will not be.
Read-only: `list_watched_bots`, `get_bot_stats`, `get_bot_updates`, `search_messages_tool`, `list_chats`, `get_chat_history` (with deleted messages included), `get_deleted_messages`, `get_message_versions`, `get_media`, `get_callbacks`, `presence_history`, `get_peer_history`, `get_stories`, `get_annotations`, `list_watches`, `list_alerts`. The one write is `set_watch`. There are no send, edit or delete tools and there will not be.
## Run
backend/.env.example
-1
View File
@@ -25,7 +25,6 @@ BOTS__BACKFILL_PER_CHAT=0
API__HOST=0.0.0.0
API__PORT=8080
API__PUBLIC_URL=
AUTH__TOKEN=
backend/Dockerfile
+1 -1
View File
@@ -1,4 +1,4 @@
FROM ghcr.io/astral-sh/uv:python3.13-trixie-slim@sha256:7dd72af08bdef012fdf433082190d2b5133a96b6a43aca0bb71c2aa793e47e20
FROM ghcr.io/astral-sh/uv:python3.13-trixie-slim
RUN apt-get update && apt-get install -y build-essential && rm -rf /var/lib/apt/lists/*
backend/migrations/versions/a2c4e6f8b1d3_edit_versions_everywhere.py
-32
View File
@@ -1,32 +0,0 @@
"""edit versions everywhere
Revision ID: a2c4e6f8b1d3
Revises: e8b3f1a6c2d4
Create Date: 2026-10-19 02:30:00.000000
"""
from collections.abc import Sequence
from alembic import op
revision: str = "a2c4e6f8b1d3"
down_revision: str | None = "e8b3f1a6c2d4"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute(
"UPDATE capture_policy SET track_edits_deletes = TRUE, updated_at = now() "
"WHERE scope_type IN ('default_group', 'default_channel') "
"AND NOT track_edits_deletes"
)
def downgrade() -> None:
op.execute(
"UPDATE capture_policy SET track_edits_deletes = FALSE, updated_at = now() "
"WHERE scope_type IN ('default_group', 'default_channel') "
"AND track_edits_deletes"
)
backend/migrations/versions/b3d5f7a9c2e4_feed.py
-198
View File
@@ -1,198 +0,0 @@
"""feed of deleted and edited messages
Revision ID: b3d5f7a9c2e4
Revises: a2c4e6f8b1d3
Create Date: 2026-10-19 02:40:00.000000
"""
from collections.abc import Sequence
from alembic import op
revision: str = "b3d5f7a9c2e4"
down_revision: str | None = "a2c4e6f8b1d3"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_ITEMS = """
CREATE TABLE feed_items (
id bigserial PRIMARY KEY,
account_id integer NOT NULL,
kind text NOT NULL,
chat_id bigint NOT NULL,
chat_kind text NOT NULL,
message_id bigint NOT NULL,
message_date timestamptz,
at timestamptz NOT NULL,
since timestamptz NOT NULL,
message_count integer NOT NULL DEFAULT 1
)
"""
_SEEN = """
CREATE TABLE feed_seen (
account_id integer PRIMARY KEY,
seen_at timestamptz NOT NULL
)
"""
_CHAT_KIND = """
CREATE FUNCTION feed_chat_kind(chat_id bigint, chat_type text)
RETURNS text LANGUAGE sql IMMUTABLE AS $$
SELECT CASE
WHEN chat_id > 0 THEN 'dm'
WHEN chat_type = 'ChatType.CHANNEL' THEN 'channel'
ELSE 'group' END
$$
"""
_MESSAGE_DATE = """
CREATE FUNCTION feed_message_date(stamp text)
RETURNS timestamptz LANGUAGE sql STABLE AS $$
SELECT CASE
WHEN stamp ~ '^\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?$'
THEN stamp::timestamp AT TIME ZONE 'UTC' END
$$
"""
_ON_DELETE = """
CREATE FUNCTION feed_on_delete() RETURNS trigger LANGUAGE plpgsql AS $$
DECLARE
item feed_items%ROWTYPE;
BEGIN
SELECT * INTO item FROM feed_items
WHERE account_id = NEW.account_id AND chat_id = NEW.chat_id
AND kind = 'deleted'
ORDER BY at DESC, id DESC LIMIT 1;
IF FOUND AND NEW.deleted_at BETWEEN item.since - interval '5 seconds'
AND item.at + interval '5 seconds' THEN
IF (NEW.date, NEW.message_id) > (item.message_date, item.message_id) THEN
item.message_id := NEW.message_id;
item.message_date := NEW.date;
END IF;
UPDATE feed_items SET
message_count = message_count + 1,
at = greatest(at, NEW.deleted_at),
since = least(since, NEW.deleted_at),
message_id = item.message_id,
message_date = item.message_date
WHERE id = item.id;
ELSE
INSERT INTO feed_items
(account_id, kind, chat_id, chat_kind, message_id, message_date,
at, since)
VALUES (NEW.account_id, 'deleted', NEW.chat_id,
feed_chat_kind(NEW.chat_id, NEW.raw->'chat'->>'type'),
NEW.message_id, NEW.date, NEW.deleted_at, NEW.deleted_at);
END IF;
RETURN NULL;
END;
$$
"""
_ON_VERSION = """
CREATE FUNCTION feed_on_version() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NEW.edit_date IS NULL AND NOT EXISTS (
SELECT 1 FROM message_versions v
WHERE v.account_id = NEW.account_id AND v.chat_id = NEW.chat_id
AND v.message_id = NEW.message_id AND v.observed_at < NEW.observed_at
) THEN
RETURN NULL;
END IF;
INSERT INTO feed_items
(account_id, kind, chat_id, chat_kind, message_id, message_date, at, since)
VALUES (NEW.account_id, 'edited', NEW.chat_id,
feed_chat_kind(NEW.chat_id, NEW.raw->'chat'->>'type'),
NEW.message_id, feed_message_date(NEW.raw->>'date'),
NEW.observed_at, NEW.observed_at);
RETURN NULL;
END;
$$
"""
_BACKFILL = """
WITH d AS (
SELECT account_id, chat_id, message_id, date, deleted_at,
raw->'chat'->>'type' AS chat_type,
deleted_at - lag(deleted_at) OVER (
PARTITION BY account_id, chat_id ORDER BY deleted_at, message_id
) AS gap
FROM messages WHERE deleted_at IS NOT NULL
), b AS (
SELECT *, count(*) FILTER (WHERE gap IS NULL OR gap > interval '5 seconds')
OVER (PARTITION BY account_id, chat_id ORDER BY deleted_at, message_id)
AS burst
FROM d
), deleted AS (
SELECT account_id, 'deleted' AS kind, chat_id,
feed_chat_kind(chat_id,
(array_agg(chat_type ORDER BY date DESC, message_id DESC))[1])
AS chat_kind,
(array_agg(message_id ORDER BY date DESC, message_id DESC))[1]
AS message_id,
max(date) AS message_date, max(deleted_at) AS at,
min(deleted_at) AS since, count(*)::integer AS message_count
FROM b GROUP BY account_id, chat_id, burst
), edited AS (
SELECT v.account_id, 'edited' AS kind, v.chat_id,
feed_chat_kind(v.chat_id, v.raw->'chat'->>'type') AS chat_kind,
v.message_id, feed_message_date(v.raw->>'date') AS message_date,
v.observed_at AS at, v.observed_at AS since, 1 AS message_count
FROM message_versions v
WHERE v.edit_date IS NOT NULL OR EXISTS (
SELECT 1 FROM message_versions p
WHERE p.account_id = v.account_id AND p.chat_id = v.chat_id
AND p.message_id = v.message_id AND p.observed_at < v.observed_at
)
)
INSERT INTO feed_items
(account_id, kind, chat_id, chat_kind, message_id, message_date, at, since,
message_count)
SELECT * FROM (SELECT * FROM deleted UNION ALL SELECT * FROM edited) items
ORDER BY at, chat_id, message_id
"""
def upgrade() -> None:
op.execute("SET LOCAL jit = off")
op.execute(_ITEMS)
op.execute(_SEEN)
op.execute(_CHAT_KIND)
op.execute(_MESSAGE_DATE)
op.execute(_ON_DELETE)
op.execute(_ON_VERSION)
op.execute(
"CREATE TRIGGER messages_feed_deleted AFTER UPDATE OF deleted_at "
"ON messages FOR EACH ROW "
"WHEN (OLD.deleted_at IS NULL AND NEW.deleted_at IS NOT NULL) "
"EXECUTE FUNCTION feed_on_delete()"
)
op.execute(
"CREATE TRIGGER message_versions_feed AFTER INSERT ON message_versions "
"FOR EACH ROW EXECUTE FUNCTION feed_on_version()"
)
op.execute(_BACKFILL)
op.execute(
"CREATE INDEX ix_feed_items_recent ON feed_items (account_id, at DESC, id DESC)"
)
op.execute(
"CREATE INDEX ix_feed_items_chat ON feed_items "
"(account_id, chat_id, at DESC, id DESC)"
)
op.execute(
"INSERT INTO feed_seen (account_id, seen_at) SELECT account_id, now() "
"FROM accounts"
)
def downgrade() -> None:
op.execute("DROP TRIGGER IF EXISTS message_versions_feed ON message_versions")
op.execute("DROP TRIGGER IF EXISTS messages_feed_deleted ON messages")
op.execute("DROP FUNCTION IF EXISTS feed_on_version()")
op.execute("DROP FUNCTION IF EXISTS feed_on_delete()")
op.execute("DROP FUNCTION IF EXISTS feed_message_date(text)")
op.execute("DROP FUNCTION IF EXISTS feed_chat_kind(bigint, text)")
op.execute("DROP TABLE IF EXISTS feed_seen")
op.execute("DROP TABLE IF EXISTS feed_items")
backend/migrations/versions/c5e7a9b1d3f6_feed_profiles.py
-321
View File
@@ -1,321 +0,0 @@
"""feed of profile changes and chat renames
Revision ID: c5e7a9b1d3f6
Revises: b3d5f7a9c2e4
Create Date: 2026-10-19 04:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
revision: str = "c5e7a9b1d3f6"
down_revision: str | None = "b3d5f7a9c2e4"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_PEER_VALUE = """
CREATE FUNCTION feed_peer_value(
field text, first_name text, last_name text, username text, photo text
) RETURNS text LANGUAGE sql IMMUTABLE AS $$
SELECT CASE field
WHEN 'name' THEN NULLIF(trim(concat_ws(' ', first_name, last_name)), '')
WHEN 'username' THEN NULLIF(username, '')
WHEN 'photo' THEN NULLIF(photo, '') END
$$
"""
_PEER_SCOPE = """
CREATE FUNCTION feed_peer_in_scope(account integer, peer bigint)
RETURNS boolean LANGUAGE sql STABLE AS $$
SELECT EXISTS (
SELECT 1 FROM dialogs WHERE account_id = account AND chat_id = peer
) OR EXISTS (
SELECT 1 FROM peers
WHERE account_id = account AND peer_id = peer
AND (raw->>'is_contact' = 'true' OR raw->>'contact' = 'true')
)
$$
"""
_MERGE_DETAIL = """
CREATE FUNCTION feed_merge_detail(base jsonb, incoming jsonb)
RETURNS jsonb LANGUAGE sql IMMUTABLE AS $$
SELECT coalesce(jsonb_object_agg(key, pair), '{}'::jsonb)
FROM (
SELECT key, jsonb_build_array(
coalesce(base->key->0, incoming->key->0),
coalesce(incoming->key->1, base->key->1)) AS pair
FROM (
SELECT jsonb_object_keys(coalesce(base, '{}'::jsonb)) AS key
UNION
SELECT jsonb_object_keys(incoming)
) keys
) merged
WHERE pair->0 IS DISTINCT FROM pair->1
$$
"""
_UPSERT = """
CREATE FUNCTION feed_profile_upsert(
account integer, chat bigint, kind_of_chat text, message bigint,
changed_at timestamptz, changes jsonb
) RETURNS void LANGUAGE plpgsql AS $$
DECLARE
item feed_items%ROWTYPE;
merged jsonb;
BEGIN
SELECT * INTO item FROM feed_items
WHERE account_id = account AND chat_id = chat AND kind = 'profile'
ORDER BY at DESC, id DESC LIMIT 1;
IF FOUND AND changed_at BETWEEN item.since - interval '1 day'
AND item.at + interval '1 day' THEN
merged := feed_merge_detail(item.detail, changes);
IF merged = '{}'::jsonb THEN
DELETE FROM feed_items WHERE id = item.id;
ELSE
UPDATE feed_items SET
detail = merged,
message_count = message_count
+ (SELECT count(*) FROM jsonb_object_keys(changes)),
at = greatest(at, changed_at),
since = least(since, changed_at),
message_id = greatest(message_id, message)
WHERE id = item.id;
END IF;
ELSE
INSERT INTO feed_items
(account_id, kind, chat_id, chat_kind, message_id, at, since,
message_count, detail)
VALUES (account, 'profile', chat, kind_of_chat, message, changed_at,
changed_at,
(SELECT count(*) FROM jsonb_object_keys(changes)), changes);
END IF;
END;
$$
"""
_ON_PEER = """
CREATE FUNCTION feed_on_peer_change() RETURNS trigger LANGUAGE plpgsql AS $$
DECLARE
changes jsonb := '{}'::jsonb;
field text;
fresh text;
old text;
BEGIN
IF NEW.peer_id <= 0 OR NOT feed_peer_in_scope(NEW.account_id, NEW.peer_id) THEN
RETURN NULL;
END IF;
FOREACH field IN ARRAY ARRAY['name', 'username', 'photo'] LOOP
fresh := feed_peer_value(field, NEW.first_name, NEW.last_name,
NEW.username, NEW.photo_unique_id);
CONTINUE WHEN fresh IS NULL;
SELECT value INTO old FROM (
SELECT feed_peer_value(field, h.first_name, h.last_name, h.username,
h.photo_unique_id) AS value, h.observed_at
FROM peer_history h
WHERE h.account_id = NEW.account_id AND h.peer_id = NEW.peer_id
AND h.observed_at < NEW.observed_at
) earlier
WHERE value IS NOT NULL
ORDER BY observed_at DESC LIMIT 1;
CONTINUE WHEN old IS NULL OR old = fresh;
CONTINUE WHEN EXISTS (
SELECT 1 FROM peer_history h
WHERE h.account_id = NEW.account_id AND h.peer_id = NEW.peer_id
AND h.observed_at < NEW.observed_at
AND feed_peer_value(field, h.first_name, h.last_name, h.username,
h.photo_unique_id) = fresh
);
changes := changes || jsonb_build_object(field, jsonb_build_array(old, fresh));
END LOOP;
IF changes <> '{}'::jsonb THEN
PERFORM feed_profile_upsert(NEW.account_id, NEW.peer_id, 'dm', 0,
NEW.observed_at, changes);
END IF;
RETURN NULL;
END;
$$
"""
_ON_TITLE = """
CREATE FUNCTION feed_on_chat_title() RETURNS trigger LANGUAGE plpgsql AS $$
DECLARE
previous text;
BEGIN
IF NOT EXISTS (
SELECT 1 FROM dialogs
WHERE account_id = NEW.account_id AND chat_id = NEW.chat_id
) THEN
RETURN NULL;
END IF;
SELECT title INTO previous FROM chat_history
WHERE account_id = NEW.account_id AND chat_id = NEW.chat_id
AND event IN ('meta', 'title') AND title IS NOT NULL
AND (ts, message_id) < (NEW.ts, NEW.message_id)
ORDER BY ts DESC, message_id DESC LIMIT 1;
IF previous IS NULL OR previous = NEW.title THEN
RETURN NULL;
END IF;
PERFORM feed_profile_upsert(
NEW.account_id, NEW.chat_id,
feed_chat_kind(NEW.chat_id, NEW.raw->'chat'->>'type'),
CASE WHEN NEW.event = 'title' THEN NEW.message_id ELSE 0 END,
NEW.ts, jsonb_build_object('title', jsonb_build_array(previous, NEW.title)));
RETURN NULL;
END;
$$
"""
_BACKFILL_PEERS = """
WITH scope AS (
SELECT account_id::integer AS account_id, chat_id AS peer_id
FROM dialogs WHERE chat_id > 0
UNION
SELECT account_id, peer_id FROM peers
WHERE raw->>'is_contact' = 'true' OR raw->>'contact' = 'true'
), firsts AS (
SELECT h.account_id, h.peer_id, f.field, f.value,
min(h.observed_at) AS at
FROM peer_history h
JOIN scope s ON s.account_id = h.account_id AND s.peer_id = h.peer_id
CROSS JOIN LATERAL (
SELECT field, feed_peer_value(field, h.first_name, h.last_name,
h.username, h.photo_unique_id) AS value
FROM unnest(ARRAY['name', 'username', 'photo']) AS field
) f
WHERE f.value IS NOT NULL
GROUP BY 1, 2, 3, 4
), ranked AS (
SELECT *, row_number() OVER (
PARTITION BY account_id, peer_id, field ORDER BY at
) AS position
FROM firsts
), changes AS (
SELECT r.account_id, r.peer_id, r.field, r.value AS fresh, r.at, o.value AS old
FROM ranked r
CROSS JOIN LATERAL (
SELECT value FROM (
SELECT feed_peer_value(r.field, h.first_name, h.last_name,
h.username, h.photo_unique_id) AS value,
h.observed_at
FROM peer_history h
WHERE h.account_id = r.account_id AND h.peer_id = r.peer_id
AND h.observed_at < r.at
) earlier
WHERE value IS NOT NULL
ORDER BY observed_at DESC LIMIT 1
) o
WHERE r.position > 1 AND r.at > now() - interval '90 days'
AND o.value <> r.value
), gaps AS (
SELECT *, at - lag(at) OVER (PARTITION BY account_id, peer_id ORDER BY at)
AS gap
FROM changes
), bursts AS (
SELECT *, count(*) FILTER (WHERE gap IS NULL OR gap > interval '1 day')
OVER (PARTITION BY account_id, peer_id ORDER BY at) AS burst
FROM gaps
), fields AS (
SELECT account_id, peer_id, burst, field,
(array_agg(old ORDER BY at))[1] AS old,
(array_agg(fresh ORDER BY at DESC))[1] AS fresh,
count(*) AS changes, min(at) AS since, max(at) AS at
FROM bursts GROUP BY 1, 2, 3, 4
)
INSERT INTO feed_items
(account_id, kind, chat_id, chat_kind, message_id, at, since,
message_count, detail)
SELECT account_id, 'profile', peer_id, 'dm', 0, max(at), min(since),
sum(changes)::integer,
jsonb_object_agg(field, jsonb_build_array(old, fresh))
FILTER (WHERE old <> fresh)
FROM fields
GROUP BY account_id, peer_id, burst
HAVING bool_or(old <> fresh)
ORDER BY max(at)
"""
_BACKFILL_CHATS = """
WITH titles AS (
SELECT account_id, chat_id, message_id, event, ts, title,
raw->'chat'->>'type' AS chat_type,
lag(title) OVER (
PARTITION BY account_id, chat_id ORDER BY ts, message_id
) AS previous
FROM chat_history
WHERE event IN ('meta', 'title') AND title IS NOT NULL
), changes AS (
SELECT * FROM titles t
WHERE previous IS NOT NULL AND previous <> title
AND ts > now() - interval '90 days'
AND EXISTS (
SELECT 1 FROM dialogs d
WHERE d.account_id = t.account_id AND d.chat_id = t.chat_id
)
), gaps AS (
SELECT *, ts - lag(ts) OVER (PARTITION BY account_id, chat_id ORDER BY ts)
AS gap
FROM changes
), bursts AS (
SELECT *, count(*) FILTER (WHERE gap IS NULL OR gap > interval '1 day')
OVER (PARTITION BY account_id, chat_id ORDER BY ts) AS burst
FROM gaps
), grouped AS (
SELECT account_id, chat_id, burst,
(array_agg(chat_type ORDER BY ts DESC))[1] AS chat_type,
max(CASE WHEN event = 'title' THEN message_id ELSE 0 END) AS message_id,
(array_agg(previous ORDER BY ts))[1] AS old,
(array_agg(title ORDER BY ts DESC))[1] AS fresh,
count(*) AS changes, min(ts) AS since, max(ts) AS at
FROM bursts GROUP BY 1, 2, 3
)
INSERT INTO feed_items
(account_id, kind, chat_id, chat_kind, message_id, at, since,
message_count, detail)
SELECT account_id, 'profile', chat_id, feed_chat_kind(chat_id, chat_type),
message_id, at, since, changes::integer,
jsonb_build_object('title', jsonb_build_array(old, fresh))
FROM grouped
WHERE old <> fresh
ORDER BY at
"""
def upgrade() -> None:
op.execute("SET LOCAL jit = off")
op.execute("ALTER TABLE feed_items ADD COLUMN detail jsonb")
op.execute(_PEER_VALUE)
op.execute(_PEER_SCOPE)
op.execute(_MERGE_DETAIL)
op.execute(_UPSERT)
op.execute(_ON_PEER)
op.execute(_ON_TITLE)
op.execute(
"CREATE TRIGGER peer_history_feed AFTER INSERT ON peer_history "
"FOR EACH ROW EXECUTE FUNCTION feed_on_peer_change()"
)
op.execute(
"CREATE TRIGGER chat_history_feed AFTER INSERT ON chat_history "
"FOR EACH ROW WHEN (NEW.event IN ('meta', 'title') AND NEW.title IS NOT NULL) "
"EXECUTE FUNCTION feed_on_chat_title()"
)
op.execute(_BACKFILL_PEERS)
op.execute(_BACKFILL_CHATS)
def downgrade() -> None:
op.execute("DROP TRIGGER IF EXISTS chat_history_feed ON chat_history")
op.execute("DROP TRIGGER IF EXISTS peer_history_feed ON peer_history")
op.execute("DROP FUNCTION IF EXISTS feed_on_chat_title()")
op.execute("DROP FUNCTION IF EXISTS feed_on_peer_change()")
op.execute(
"DROP FUNCTION IF EXISTS "
"feed_profile_upsert(integer, bigint, text, bigint, timestamptz, jsonb)"
)
op.execute("DROP FUNCTION IF EXISTS feed_merge_detail(jsonb, jsonb)")
op.execute("DROP FUNCTION IF EXISTS feed_peer_in_scope(integer, bigint)")
op.execute("DROP FUNCTION IF EXISTS feed_peer_value(text, text, text, text, text)")
op.execute("DELETE FROM feed_items WHERE kind = 'profile'")
op.execute("ALTER TABLE feed_items DROP COLUMN detail")
backend/migrations/versions/d7a3c5e9f2b1_message_reads.py
-51
View File
@@ -1,51 +0,0 @@
"""message reads
Revision ID: d7a3c5e9f2b1
Revises: f8c2d6e1a4b7
Create Date: 2026-10-19 01:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
revision: str = "d7a3c5e9f2b1"
down_revision: str | None = "f8c2d6e1a4b7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_READS = """
CREATE TABLE message_reads (
account_id integer NOT NULL,
chat_id bigint NOT NULL,
message_id bigint NOT NULL,
reader_id bigint NOT NULL,
read_at timestamptz NOT NULL,
source text NOT NULL,
fetched_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (account_id, chat_id, message_id, reader_id)
)
"""
_FETCHES = """
CREATE TABLE read_fetches (
account_id integer NOT NULL,
chat_id bigint NOT NULL,
message_id bigint NOT NULL,
outcome text NOT NULL,
fetched_at timestamptz NOT NULL DEFAULT now(),
retry_at timestamptz NOT NULL,
PRIMARY KEY (account_id, chat_id, message_id)
)
"""
def upgrade() -> None:
op.execute(_READS)
op.execute(_FETCHES)
def downgrade() -> None:
op.execute("DROP TABLE IF EXISTS read_fetches")
op.execute("DROP TABLE IF EXISTS message_reads")
backend/migrations/versions/d8e3b5a1c7f4_watch_names_webhooks.py
-86
View File
@@ -1,86 +0,0 @@
"""watch names and webhooks
Revision ID: d8e3b5a1c7f4
Revises: d8f4a1c6e2b7
Create Date: 2026-10-19 03:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
revision: str = "d8e3b5a1c7f4"
down_revision: str | None = "d8f4a1c6e2b7"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_WEBHOOKS = """
CREATE TABLE webhooks (
id serial PRIMARY KEY,
account_id integer NOT NULL,
url text NOT NULL,
secret text NOT NULL,
enabled boolean NOT NULL DEFAULT true,
watch_ids integer[],
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
)
"""
_DELIVERIES = """
CREATE TABLE webhook_deliveries (
id bigserial PRIMARY KEY,
webhook_id integer NOT NULL REFERENCES webhooks (id) ON DELETE CASCADE,
alert_id integer REFERENCES alerts (id) ON DELETE CASCADE,
event text NOT NULL,
status text NOT NULL DEFAULT 'pending',
attempts integer NOT NULL DEFAULT 0,
next_attempt_at timestamptz NOT NULL DEFAULT now(),
response_code integer,
last_error text,
body text,
created_at timestamptz NOT NULL DEFAULT now(),
finished_at timestamptz
)
"""
_LEGACY_KEYWORD = """
UPDATE watches
SET params = (params - 'keyword') || jsonb_build_object('pattern', params->'keyword')
WHERE kind = 'keyword' AND params ? 'keyword' AND NOT params ? 'pattern'
"""
def upgrade() -> None:
op.execute("ALTER TABLE watches ADD COLUMN name text")
op.execute(_LEGACY_KEYWORD)
op.execute("ALTER TABLE alerts DROP CONSTRAINT alerts_watch_id_fkey")
op.execute(
"ALTER TABLE alerts ADD CONSTRAINT alerts_watch_id_fkey "
"FOREIGN KEY (watch_id) REFERENCES watches (id) ON DELETE CASCADE"
)
op.execute("CREATE INDEX ix_alerts_account_id ON alerts (account_id, id DESC)")
op.execute(_WEBHOOKS)
op.execute("CREATE INDEX ix_webhooks_account ON webhooks (account_id)")
op.execute(_DELIVERIES)
op.execute(
"CREATE INDEX ix_webhook_deliveries_due ON webhook_deliveries "
"(next_attempt_at) WHERE status = 'pending'"
)
op.execute(
"CREATE INDEX ix_webhook_deliveries_webhook ON webhook_deliveries "
"(webhook_id, id DESC)"
)
def downgrade() -> None:
op.execute("DROP TABLE IF EXISTS webhook_deliveries")
op.execute("DROP TABLE IF EXISTS webhooks")
op.execute("DROP INDEX IF EXISTS ix_alerts_account_id")
op.execute("ALTER TABLE alerts DROP CONSTRAINT alerts_watch_id_fkey")
op.execute(
"ALTER TABLE alerts ADD CONSTRAINT alerts_watch_id_fkey "
"FOREIGN KEY (watch_id) REFERENCES watches (id)"
)
op.execute("ALTER TABLE watches DROP COLUMN name")
backend/migrations/versions/d8f1a6c3e5b7_message_search.py
-148
View File
@@ -1,148 +0,0 @@
"""message search index outside the hypertable, versions searchable
Revision ID: d8f1a6c3e5b7
Revises: c5e7a9b1d3f6
Create Date: 2026-10-19 03:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
revision: str = "d8f1a6c3e5b7"
down_revision: str | None = "c5e7a9b1d3f6"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_VECTOR = """
CREATE FUNCTION search_vector(body text) RETURNS tsvector
LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$
SELECT to_tsvector('russian', translate(coalesce(body, ''), 'Ёё', 'Ее'))
|| to_tsvector('simple', translate(coalesce(body, ''), 'Ёё', 'Ее'))
$$
"""
_TABLE = """
CREATE TABLE message_search (
account_id integer NOT NULL,
chat_id bigint NOT NULL,
message_id bigint NOT NULL,
date timestamptz NOT NULL,
sender_id bigint,
text text,
has_media boolean NOT NULL DEFAULT false,
edited_at timestamptz,
deleted_at timestamptz,
tsv tsvector GENERATED ALWAYS AS (search_vector(text)) STORED,
PRIMARY KEY (account_id, chat_id, message_id, date)
)
"""
_BACKFILL = """
INSERT INTO message_search (account_id, chat_id, message_id, date, sender_id,
text, has_media, edited_at, deleted_at)
SELECT account_id, chat_id, message_id, date, sender_id,
text, has_media, edited_at, deleted_at
FROM messages
"""
_SYNC = """
CREATE FUNCTION message_search_sync() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF TG_OP = 'DELETE' THEN
DELETE FROM message_search
WHERE account_id = OLD.account_id AND chat_id = OLD.chat_id
AND message_id = OLD.message_id AND date = OLD.date;
RETURN NULL;
END IF;
INSERT INTO message_search (account_id, chat_id, message_id, date,
sender_id, text, has_media, edited_at,
deleted_at)
VALUES (NEW.account_id, NEW.chat_id, NEW.message_id, NEW.date,
NEW.sender_id, NEW.text, NEW.has_media, NEW.edited_at,
NEW.deleted_at)
ON CONFLICT (account_id, chat_id, message_id, date) DO UPDATE SET
sender_id = EXCLUDED.sender_id,
text = EXCLUDED.text,
has_media = EXCLUDED.has_media,
edited_at = EXCLUDED.edited_at,
deleted_at = EXCLUDED.deleted_at;
RETURN NULL;
END;
$$
"""
_CHANGED = (
"OLD.text IS DISTINCT FROM NEW.text "
"OR OLD.sender_id IS DISTINCT FROM NEW.sender_id "
"OR OLD.has_media IS DISTINCT FROM NEW.has_media "
"OR OLD.edited_at IS DISTINCT FROM NEW.edited_at "
"OR OLD.deleted_at IS DISTINCT FROM NEW.deleted_at"
)
def upgrade() -> None:
op.execute(_VECTOR)
op.execute(_TABLE)
op.execute(_BACKFILL)
op.execute("CREATE INDEX ix_message_search_tsv ON message_search USING gin (tsv)")
op.execute(
"CREATE INDEX ix_message_search_recent ON message_search "
"(account_id, date DESC, chat_id DESC, message_id DESC)"
)
op.execute(
"CREATE INDEX ix_message_search_sender ON message_search "
"(account_id, sender_id, date DESC)"
)
for flag in ("deleted", "edited"):
op.execute(
f"CREATE INDEX ix_message_search_{flag} ON message_search "
"(account_id, date DESC, chat_id DESC, message_id DESC) "
f"WHERE {flag}_at IS NOT NULL"
)
op.execute("ALTER TABLE message_search ALTER COLUMN tsv SET STATISTICS 1000")
op.execute(_SYNC)
op.execute(
"CREATE TRIGGER messages_search_insert AFTER INSERT ON messages "
"FOR EACH ROW EXECUTE FUNCTION message_search_sync()"
)
op.execute(
"CREATE TRIGGER messages_search_update AFTER UPDATE ON messages "
f"FOR EACH ROW WHEN ({_CHANGED}) EXECUTE FUNCTION message_search_sync()"
)
op.execute(
"CREATE TRIGGER messages_search_delete AFTER DELETE ON messages "
"FOR EACH ROW EXECUTE FUNCTION message_search_sync()"
)
op.execute(
"CREATE INDEX ix_message_versions_tsv ON message_versions USING gin "
"(search_vector(text))"
)
op.execute("DROP INDEX IF EXISTS ix_media_tsv")
op.execute(
"CREATE INDEX ix_media_tsv ON media USING gin (search_vector(extracted_text))"
)
op.execute("DROP INDEX IF EXISTS ix_messages_tsv")
for index in ("ix_message_versions_tsv", "ix_media_tsv"):
op.execute(f"ALTER INDEX {index} ALTER COLUMN 1 SET STATISTICS 1000")
op.execute("ANALYZE message_search, message_versions, media")
def downgrade() -> None:
op.execute(
"CREATE INDEX ix_messages_tsv ON messages USING gin "
"(to_tsvector('russian', coalesce(text, '')))"
)
op.execute("DROP INDEX IF EXISTS ix_media_tsv")
op.execute(
"CREATE INDEX ix_media_tsv ON media USING gin "
"(to_tsvector('russian', coalesce(extracted_text, '')))"
)
op.execute("DROP INDEX IF EXISTS ix_message_versions_tsv")
op.execute("DROP TRIGGER IF EXISTS messages_search_delete ON messages")
op.execute("DROP TRIGGER IF EXISTS messages_search_update ON messages")
op.execute("DROP TRIGGER IF EXISTS messages_search_insert ON messages")
op.execute("DROP FUNCTION IF EXISTS message_search_sync()")
op.execute("DROP TABLE IF EXISTS message_search")
op.execute("DROP FUNCTION IF EXISTS search_vector(text)")
backend/migrations/versions/d8f4a1c6e2b7_capture_uptime.py
-50
View File
@@ -1,50 +0,0 @@
"""capture uptime
Revision ID: d8f4a1c6e2b7
Revises: d7a3c5e9f2b1
Create Date: 2026-10-19 03:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
revision: str = "d8f4a1c6e2b7"
down_revision: str | None = "d7a3c5e9f2b1"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_UPTIME = """
CREATE TABLE capture_uptime (
account_id integer NOT NULL,
started_at timestamptz NOT NULL,
seen_at timestamptz NOT NULL,
PRIMARY KEY (account_id, started_at)
)
"""
_BACKFILL = """
INSERT INTO capture_uptime (account_id, started_at, seen_at)
SELECT account_id, min(ts), max(ts)
FROM (
SELECT account_id, ts,
count(*) FILTER (WHERE gap > interval '30 minutes')
OVER (PARTITION BY account_id ORDER BY ts) AS stretch
FROM (
SELECT account_id, ts,
ts - lag(ts) OVER (PARTITION BY account_id ORDER BY ts) AS gap
FROM presence
) gaps
) stretches
GROUP BY account_id, stretch
"""
def upgrade() -> None:
op.execute(_UPTIME)
op.execute(_BACKFILL)
def downgrade() -> None:
op.execute("DROP TABLE IF EXISTS capture_uptime")
backend/migrations/versions/e8b3f1a6c2d4_profile_history_reads.py
-54
View File
@@ -1,54 +0,0 @@
"""profile history reads
Revision ID: e8b3f1a6c2d4
Revises: d8e3b5a1c7f4
Create Date: 2026-10-19 03:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
revision: str = "e8b3f1a6c2d4"
down_revision: str | None = "d8e3b5a1c7f4"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_PEER_STATE = (
"first_name, last_name, username, phone, photo_unique_id, is_deleted_account"
)
def upgrade() -> None:
op.execute(
"ALTER TABLE peer_history DROP CONSTRAINT peer_history_pkey, "
"ADD CONSTRAINT peer_history_pkey "
f"PRIMARY KEY (account_id, peer_id, observed_at) INCLUDE ({_PEER_STATE})"
)
op.execute(
"ALTER TABLE chat_history DROP CONSTRAINT chat_history_pkey, "
"ADD CONSTRAINT chat_history_pkey "
"PRIMARY KEY (account_id, chat_id, message_id, ts)"
)
op.execute(
"CREATE INDEX ix_memberships_user ON memberships (account_id, user_id, ts DESC)"
)
op.execute(
"CREATE INDEX ix_jobs_kind_created ON jobs (account_id, kind, created_at DESC)"
)
with op.get_context().autocommit_block():
op.execute("VACUUM (ANALYZE) peer_history")
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_jobs_kind_created")
op.execute("DROP INDEX IF EXISTS ix_memberships_user")
op.execute(
"ALTER TABLE chat_history DROP CONSTRAINT chat_history_pkey, "
"ADD CONSTRAINT chat_history_pkey PRIMARY KEY (account_id, chat_id, message_id)"
)
op.execute(
"ALTER TABLE peer_history DROP CONSTRAINT peer_history_pkey, "
"ADD CONSTRAINT peer_history_pkey PRIMARY KEY (account_id, peer_id, observed_at)"
)
backend/migrations/versions/f8c2d6e1a4b7_capture_reliability.py
-39
View File
@@ -1,39 +0,0 @@
"""capture reliability
Revision ID: f8c2d6e1a4b7
Revises: c9e2a7f4b1d8
Create Date: 2026-10-19 03:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "f8c2d6e1a4b7"
down_revision: str | None = "c9e2a7f4b1d8"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.add_column(
"jobs", sa.Column("run_after", sa.DateTime(timezone=True), nullable=True)
)
op.execute(
"ALTER TABLE accounts "
"ADD COLUMN catchup_pts integer, "
"ADD COLUMN catchup_qts integer, "
"ADD COLUMN catchup_date integer"
)
def downgrade() -> None:
op.execute(
"ALTER TABLE accounts "
"DROP COLUMN IF EXISTS catchup_pts, "
"DROP COLUMN IF EXISTS catchup_qts, "
"DROP COLUMN IF EXISTS catchup_date"
)
op.drop_column("jobs", "run_after")
backend/pyproject.toml
-1
View File
@@ -20,7 +20,6 @@ dependencies = [
"dishka>=1.4.0",
"greenlet>=3.1.0",
"fastmcp>=3.3",
"httpx2>=2.13.1",
]
[build-system]
backend/src/api/app.py
-8
View File
@@ -25,7 +25,6 @@ from api.routers import (
custom_emoji,
discover,
events,
feed,
files,
folders,
media,
@@ -39,9 +38,7 @@ from api.routers import (
storage,
stories,
watches,
webhooks,
)
from api.webhooks import outbox
from dependencies.container import container
from utils.cache import DAY_HEADERS, IMMUTABLE_HEADERS, NO_STORE_HEADERS
from utils.env import env
@@ -61,14 +58,11 @@ mcp_app = mcp.http_app(path="/")
@asynccontextmanager
async def lifespan(app_: Starlette) -> AsyncGenerator[None]:
pool = await container.get(asyncpg.Pool)
hub.alert_listeners.append(outbox.wake)
await hub.start(pool)
purger = asyncio.create_task(
purge_loop(pool, await container.get(ContentAddressedStorage))
)
deliverer = asyncio.create_task(outbox.run(pool))
yield
deliverer.cancel()
purger.cancel()
await hub.stop()
await app_.state.dishka_container.close()
@@ -97,7 +91,6 @@ app.include_router(folders.router)
app.include_router(backfill.router)
app.include_router(search.router)
app.include_router(chats.router)
app.include_router(feed.router)
app.include_router(media.router)
app.include_router(avatars.router)
app.include_router(custom_emoji.router)
@@ -110,7 +103,6 @@ app.include_router(peers.router)
app.include_router(discover.router)
app.include_router(annotations.router)
app.include_router(watches.router)
app.include_router(webhooks.router)
app.include_router(shares.router)
app.include_router(storage.router)
app.include_router(files.router)
backend/src/api/mcp/server.py
+24 -200
View File
@@ -1,4 +1,3 @@
import re
from collections.abc import Sequence
from datetime import datetime
from typing import Any
@@ -21,27 +20,12 @@ from api.mcp.download import (
from api.mcp.format import build_transcript, load_notes, load_photos, resolve_names
from dependencies.container import container
from utils.files import resolve_mime
from utils.read import (
annotations,
chats,
feed,
media,
peers,
people,
presence,
presence_stats,
social,
watches,
)
from utils.read import annotations, chats, media, peers, presence, social, watches
from utils.read.accounts import self_user_id, unknown_account_hint
from utils.read.feed import CURSOR_PATTERN, FeedChatKind, FeedFilter, FeedKind
from utils.read.models import DEFAULT_LIMIT, Page
from utils.read.period import DEFAULT_TZ, MAX_DAYS, known_timezone
from utils.read.read_receipts import read_info
from utils.search.models import ChatKind, SearchFilters
from utils.search.models import SearchFilters
from utils.search.repository import search_messages
from utils.storage import ContentAddressedStorage
from utils.watch_rules import WatchKind, WatchParamsError, clean_name, normalize_params
INSTRUCTIONS = """\
beavergram archives Telegram data (chats, messages, media, presence, stories,
@@ -72,25 +56,17 @@ Typical flows:
"📝 [your private note, NOT in Telegram]" are the user's own annotations
attached locally via the web UI; they never existed in Telegram. Treat them
as private notes from the user to you, not as chat content.
- Find something: `search_messages_tool` (words by prefix over message text,
STT transcripts and earlier versions of edited messages; filters by chat,
sender, chat kind, media, deleted, edited, dates, regex).
- People: `get_person` gives one person in full: profile changes, shared groups,
joins and leaves, deleted message count, the private chat.
- Forensics: `get_feed` lists what was deleted or edited in Telegram across
all chats, newest first, with the text before and after each edit; a bulk
deletion is one item with `message_count`. Its `profile` kind lists name,
username and photo changes of people and renames of chats. `get_deleted_messages` and
`get_message_versions` recover the content of one chat or one message.
- Find something: `search_messages_tool` (full-text over message text and STT
transcripts; supports chat/sender/date filters and regex).
- Forensics: `get_deleted_messages` and `get_message_versions` recover content
removed or edited in Telegram but kept in the archive.
- Files: `download_media` gets any message attachment (photo, video, voice,
document, ...). It pulls the file from Telegram if the archive lacks it and
returns a public link valid 24h that needs no auth (`curl -O <link>`), plus
the content itself for small images and text files. `get_media` returns
metadata only.
- Monitoring: `set_watch` creates a local rule (kinds `keyword`,
`new_message`, `message_deleted`, `message_edited`, `peer_online`; params in
its description), `list_watches` lists rules, and `list_alerts` reads what
those rules fired, with chat, message preview and a web client `path`.
- Monitoring: `set_watch` creates a local rule, `list_watches` lists rules, and
`list_alerts` reads what those rules fired.
Watched bots:
- a bot whose token the user added is just another archived account: it has its
@@ -100,8 +76,7 @@ Watched bots:
Writes: everything is read-only except `set_watch` and the share links
`download_media` creates (visible and revocable in the web UI).
Paging: list tools accept `limit` and `offset`; `search_messages_tool` returns
`next_cursor` to pass back as `cursor`.
Paging: list/search tools accept `limit` and `offset`.
"""
mcp: FastMCP = FastMCP("beavergram", instructions=INSTRUCTIONS)
@@ -140,49 +115,27 @@ async def search_messages_tool(
query: str | None = None,
chat_id: int | None = None,
sender_id: int | None = None,
kind: ChatKind | None = None,
has_media: bool | None = None,
deleted: bool | None = None,
edited: bool | None = None,
date_from: datetime | None = None,
date_to: datetime | None = None,
regex: str | None = None,
cursor: str | None = None,
limit: int = DEFAULT_LIMIT,
) -> dict[str, Any]:
"""Search messages by words, newest first, including deleted ones.
Words match by prefix (3+ letters) and by stem, Russian and English.
`"exact phrase"`, `-word` and `or` work as in web search. Besides the
current text it searches voice transcripts and earlier versions of edited
messages: a hit with `source="version"` matched only an old text, given in
`version_text`. Every hit carries `deleted_at` and `edited_at`.
Filters: `chat_id`, `sender_id` (peer id), `kind` (private, group,
channel), `has_media`, `deleted` / `edited` (true for only those, false to
exclude them), `date_from` / `date_to`, `regex` over the current text. With
no `query` it lists messages that pass the filters.
Paging: pass `next_cursor` from the previous result as `cursor`; it is null
on the last page.
"""
offset: int = 0,
) -> list[dict[str, Any]]:
"""Full-text search over message text and STT transcripts."""
filters = SearchFilters(
account_id=account_id,
query=query,
chat_id=chat_id,
sender_id=sender_id,
kind=kind,
has_media=has_media,
deleted=deleted,
edited=edited,
date_from=date_from,
date_to=date_to,
regex=regex,
cursor=cursor,
limit=limit,
offset=offset,
)
page = await search_messages(await _pool(), filters)
return page.model_dump(mode="json", exclude={"terms"})
return _dump(await search_messages(await _pool(), filters))
@mcp.tool
@@ -285,41 +238,6 @@ async def get_deleted_messages(
)
@mcp.tool
async def get_feed(
account_id: int,
kind: FeedKind = FeedKind.ALL,
since: datetime | None = None,
chat_id: int | None = None,
chat_kind: FeedChatKind | None = None,
limit: int = DEFAULT_LIMIT,
cursor: str | None = None,
) -> dict[str, Any]:
"""Deleted and edited messages and profile changes across chats, newest first.
`kind` picks deleted, edited, profile, messages (deleted and edited) or all;
`since` keeps items at or after that time; `chat_id` or `chat_kind` (dm,
group, channel) narrow it down. Edited items carry `before` and `after`
texts. A deleted item with `message_count` above 1 is a bulk deletion in
one chat; read its messages with `get_deleted_messages` for that chat.
Profile items cover people in private chats or contacts and chats in the
dialog list: `detail` maps `name`, `username`, `photo` (unique file ids) or
`title` to `[old, new]`, changes a day apart or closer are merged. Pass
`next_cursor` back as `cursor` for the next page.
"""
if cursor is not None and re.fullmatch(CURSOR_PATTERN, cursor) is None:
msg = "cursor must be a next_cursor returned by get_feed"
raise ToolError(msg)
page = await feed.list_feed(
await _pool(),
account_id,
FeedFilter(kind=kind, chat_id=chat_id, chat_kind=chat_kind, since=since),
limit=limit,
cursor=cursor,
)
return page.model_dump(mode="json")
@mcp.tool
async def get_message_versions(
account_id: int, chat_id: int, message_id: int
@@ -377,23 +295,6 @@ async def get_callbacks(
return _dump(items)
@mcp.tool
async def get_read_info(
account_id: int, chat_id: int, message_id: int
) -> dict[str, Any] | None:
"""When the user's own message was read, from the archive only.
`observed_at` is when the archive saw the chat read up to this message,
`exact_at` the read date Telegram reported for a private chat, `readers`
who read it in a group and when, `listened_at` when a voice or round
video was played. `outcome` is the last Telegram answer: ok, not_read_yet,
user_privacy (they hide read time), your_privacy (the user hides theirs),
chat_too_big, too_old, unavailable, flood or error.
"""
info = await read_info(await _pool(), account_id, chat_id, message_id)
return info.model_dump(mode="json") if info else None
@mcp.tool
async def presence_history(
account_id: int,
@@ -416,58 +317,12 @@ async def presence_history(
)
@mcp.tool(name="presence_stats")
async def presence_stats_tool(
account_id: int,
peer_id: int,
days: int = 30,
tz: str = DEFAULT_TZ,
limit: int = DEFAULT_LIMIT,
offset: int = 0,
) -> dict[str, Any]:
"""Time a peer spent online, from status pushes turned into online sessions.
`tz` is an IANA zone name; days and hours are counted in it. `week` holds
the average online minutes for each weekday (0 is Monday) and hour, null
where the archive has no data for that hour. `days` gives online seconds
per day, `gaps` the stretches when capture was down, `sessions` the latest
sessions (online stretches with breaks under 5 minutes), paged by
`limit`/`offset`. `hidden` means the peer hides their last seen time.
"""
pool = await _pool()
if not 1 <= days <= MAX_DAYS:
msg = f"days must be between 1 and {MAX_DAYS}"
raise ToolError(msg)
if not await known_timezone(pool, tz):
msg = f"Unknown time zone: {tz}"
raise ToolError(msg)
stats = await presence_stats.presence_stats(
pool, account_id, peer_id, Page(limit=limit, offset=offset), days=days, tz=tz
)
return stats.model_dump(mode="json")
@mcp.tool
async def get_peer_history(account_id: int, peer_id: int) -> list[dict[str, Any]]:
"""Get name/username/avatar change history of a contact."""
return _dump(await peers.get_peer_history(await _pool(), account_id, peer_id))
@mcp.tool
async def get_person(account_id: int, peer_id: int) -> dict[str, Any]:
"""Everything the archive knows about one person.
Current profile, name/username/phone changes with times (newest first),
avatar history, groups shared in the archive with their message counts and
last message, joins and leaves with who added or removed them, the number
of their deleted messages, and the private chat if there is one.
"""
pool = await _pool()
overview = await people.get_overview(pool, account_id, peer_id)
groups = await people.get_groups(pool, account_id, peer_id)
return {**overview.model_dump(mode="json"), **groups.model_dump(mode="json")}
@mcp.tool
async def get_stories(
account_id: int,
@@ -506,37 +361,13 @@ async def get_annotations(
@mcp.tool
async def set_watch(
account_id: int,
kind: WatchKind,
kind: str,
params: dict[str, Any] | None = None,
name: str | None = None,
enabled: bool = True,
) -> dict[str, Any]:
"""Create a local watch rule (the only local write MCP is allowed).
A watch fires an alert into the web client (live counter and list) and to
the user's webhooks. The user's own messages never fire. Kinds and params:
- `keyword`: `{"pattern": str, "regex": bool = false, "chat_id"?: int,
"sender_id"?: int}`. Case-insensitive substring, or a Python regex when
`regex` is true (at most 200 characters, no nested repeats like
`(a+)+`). Checked on new messages, on edits (the new text) and on voice
transcripts. Without `chat_id` it watches every chat.
- `new_message`: `{"chat_id"?: int, "sender_id"?: int}`, at least one: any
new message in that chat, or from that person anywhere, or both.
- `message_deleted`: `{"chat_id"?: int}`. A stored message deleted in that
chat; without `chat_id`, in any private chat.
- `message_edited`: `{"chat_id"?: int}`. Same scopes as `message_deleted`.
- `peer_online`: `{"peer_id": int}`. The person comes online after at
least 15 minutes away.
`name` is an optional human label shown in the list and in alerts.
"""
try:
clean = normalize_params(kind, params or {})
except WatchParamsError as exc:
raise ToolError(str(exc)) from exc
"""Create a local watch rule (the only local write MCP is allowed)."""
watch = await watches.create_watch(
await _pool(), account_id, kind, clean, name=clean_name(name), enabled=enabled
await _pool(), account_id, kind, params or {}, enabled=enabled
)
return watch.model_dump(mode="json")
@@ -551,19 +382,12 @@ async def list_watches(account_id: int) -> list[dict[str, Any]]:
async def list_alerts(
account_id: int,
seen: bool | None = None,
before: int | None = None,
limit: int = DEFAULT_LIMIT,
) -> dict[str, Any]:
"""List fired alerts from watch rules, newest first.
Each alert names its watch (id, kind, name), the event (`message`, `edit`,
`transcript`, `delete`, `online`), the chat, the message preview with its
sender and deleted/edited flags, the peer for `peer_online`, and `path`,
the web client link. `seen=false` keeps unseen ones. Page older with
`before` set to `next_before` from the previous call; `unseen` is the
account's unseen count.
"""
page = await watches.list_alerts(
await _pool(), account_id, seen=seen, before=before, limit=limit
offset: int = 0,
) -> list[dict[str, Any]]:
"""List fired alerts from watch rules."""
return _dump(
await watches.list_alerts(
await _pool(), account_id, Page(limit=limit, offset=offset), seen=seen
)
)
return page.model_dump(mode="json")
backend/src/api/realtime.py
+19 -54
View File
@@ -1,19 +1,14 @@
import asyncio
import json
import logging
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from typing import Any
import asyncpg
from utils.env import env
from utils.events import BG_EVENTS_CHANNEL
from utils.listen import Listener
from utils.read import chats as chats_read
from utils.read import presence as presence_read
from utils.read import watches as watches_read
if TYPE_CHECKING:
from collections.abc import Callable
logger = logging.getLogger(__name__)
@@ -31,15 +26,8 @@ class EventHub:
def __init__(self) -> None:
self._subscribers: set[Subscriber] = set()
self._pool: asyncpg.Pool | None = None
self._listener = Listener(
"realtime",
{
BG_EVENTS_CHANNEL: self._on_notify,
watches_read.ALERTS_CHANGED_CHANNEL: self._on_alert,
},
)
self._conn: asyncpg.Connection | None = None
self._tasks: set[asyncio.Task] = set()
self.alert_listeners: list[Callable[[], None]] = []
def subscribe(self, account_id: int, chat_id: int | None) -> Subscriber:
sub = Subscriber(account_id, chat_id)
@@ -51,51 +39,25 @@ class EventHub:
async def start(self, pool: asyncpg.Pool) -> None:
self._pool = pool
await self._listener.start()
conn = await asyncpg.connect(dsn=env.db.connection_url)
await conn.add_listener(BG_EVENTS_CHANNEL, self._on_notify)
self._conn = conn
logger.info("Realtime hub listening on %s", BG_EVENTS_CHANNEL)
async def stop(self) -> None:
for task in self._tasks:
task.cancel()
await self._listener.stop()
if self._conn is not None:
await self._conn.close()
self._conn = None
def _on_notify(self, payload: str) -> None:
def _on_notify(
self, _conn: asyncpg.Connection, _pid: int, _channel: str, payload: str
) -> None:
task = asyncio.create_task(self._dispatch(payload))
self._tasks.add(task)
task.add_done_callback(self._tasks.discard)
def _on_alert(self, payload: str) -> None:
for listener in self.alert_listeners:
listener()
task = asyncio.create_task(self._dispatch_alert(payload))
self._tasks.add(task)
task.add_done_callback(self._tasks.discard)
async def _dispatch_alert(self, payload: str) -> None:
try:
event = json.loads(payload)
except json.JSONDecodeError:
return
account_id = event.get("account_id")
targets = [
sub
for sub in self._subscribers
if sub.account_id == account_id and sub.chat_id is None
]
if not targets or self._pool is None:
return
alert_id = event.get("alert_id")
item = await watches_read.get_alert(self._pool, alert_id) if alert_id else None
frame = {
"type": "alert",
"alert": item.model_dump(mode="json") if item else None,
"unseen": await watches_read.unseen_count(self._pool, account_id),
}
for sub in targets:
try:
sub.queue.put_nowait(frame)
except asyncio.QueueFull:
logger.warning("Dropping alert for slow subscriber")
async def _dispatch(self, payload: str) -> None:
try:
event = json.loads(payload)
@@ -103,12 +65,16 @@ class EventHub:
return
account_id = event.get("account_id")
chat_id = event.get("chat_id")
account_wide = event.get("kind") == "presence"
scoped = event.get("kind") == "presence"
targets = [
sub
for sub in self._subscribers
if sub.account_id == account_id
and (account_wide or sub.chat_id is None or sub.chat_id == chat_id)
and (
sub.chat_id == chat_id
if scoped
else sub.chat_id is None or sub.chat_id == chat_id
)
]
if not targets:
return
@@ -155,7 +121,6 @@ class EventHub:
"type": "receipt",
"chat_id": event["chat_id"],
"read_up_to": event["message_id"],
"read_at": datetime.now(UTC).isoformat(),
}
return None
backend/src/api/routers/analytics.py
+2 -7
View File
@@ -4,10 +4,8 @@ import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, Query
from api.timezone import checked_tz
from utils.read import analytics
from utils.read.models import ResponseStats, VolumeBucket
from utils.read.period import DEFAULT_TZ, MAX_DAYS
router = APIRouter(prefix="/api/analytics", tags=["analytics"], route_class=DishkaRoute)
@@ -20,12 +18,9 @@ async def volume(
pool: FromDishka[asyncpg.Pool],
account_id: AccountId,
chat_id: ChatId,
days: Annotated[int, Query(ge=1, le=MAX_DAYS)] = 90,
tz: Annotated[str, Query()] = DEFAULT_TZ,
days: Annotated[int, Query()] = 90,
) -> list[VolumeBucket]:
return await analytics.message_volume(
pool, account_id, chat_id, days=days, tz=await checked_tz(pool, tz)
)
return await analytics.message_volume(pool, account_id, chat_id, days=days)
@router.get("/response-time")
backend/src/api/routers/feed.py
-70
View File
@@ -1,70 +0,0 @@
from datetime import datetime
from typing import Annotated
import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel
from utils.read import feed
from utils.read.feed import CURSOR_PATTERN, FeedChatKind, FeedFilter, FeedKind
from utils.read.models import DEFAULT_LIMIT, FeedMessagesPage, FeedPage, FeedUnseen
router = APIRouter(prefix="/api/feed", tags=["feed"], route_class=DishkaRoute)
AccountId = Annotated[int, Query()]
Limit = Annotated[int, Query(ge=1)]
Cursor = Annotated[str | None, Query(pattern=CURSOR_PATTERN)]
class SeenRequest(BaseModel):
account_id: int
at: datetime | None = None
@router.get("")
async def list_feed(
pool: FromDishka[asyncpg.Pool],
account_id: AccountId,
kind: Annotated[FeedKind, Query()] = FeedKind.ALL,
chat_id: Annotated[int | None, Query()] = None,
chat_kind: Annotated[FeedChatKind | None, Query()] = None,
since: Annotated[datetime | None, Query()] = None,
cursor: Cursor = None,
limit: Limit = DEFAULT_LIMIT,
) -> FeedPage:
return await feed.list_feed(
pool,
account_id,
FeedFilter(kind=kind, chat_id=chat_id, chat_kind=chat_kind, since=since),
limit=limit,
cursor=cursor,
)
@router.get("/unseen")
async def feed_unseen(
pool: FromDishka[asyncpg.Pool], account_id: AccountId
) -> FeedUnseen:
return await feed.unseen(pool, account_id)
@router.post("/seen")
async def feed_seen(pool: FromDishka[asyncpg.Pool], body: SeenRequest) -> FeedUnseen:
return await feed.mark_seen(pool, body.account_id, body.at)
@router.get("/{item_id}/messages")
async def feed_item_messages(
pool: FromDishka[asyncpg.Pool],
item_id: int,
account_id: AccountId,
cursor: Cursor = None,
limit: Limit = DEFAULT_LIMIT,
) -> FeedMessagesPage:
page = await feed.item_messages(
pool, account_id, item_id, limit=limit, cursor=cursor
)
if page is None:
raise HTTPException(status_code=404, detail="Feed item not found")
return page
backend/src/api/routers/peers.py
+2 -29
View File
@@ -4,14 +4,8 @@ import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, HTTPException, Query
from utils.read import peers, people
from utils.read.models import (
PeerGroups,
PeerHistoryView,
PeerOverview,
PeerView,
ProfileChange,
)
from utils.read import peers
from utils.read.models import PeerHistoryView, PeerView
router = APIRouter(prefix="/api", tags=["peers"], route_class=DishkaRoute)
@@ -41,24 +35,3 @@ async def peer_history(
pool: FromDishka[asyncpg.Pool], peer_id: int, account_id: AccountId
) -> list[PeerHistoryView]:
return await peers.get_peer_history(pool, account_id, peer_id)
@router.get("/peers/{peer_id}/overview")
async def peer_overview(
pool: FromDishka[asyncpg.Pool], peer_id: int, account_id: AccountId
) -> PeerOverview:
return await people.get_overview(pool, account_id, peer_id)
@router.get("/peers/{peer_id}/groups")
async def peer_groups(
pool: FromDishka[asyncpg.Pool], peer_id: int, account_id: AccountId
) -> PeerGroups:
return await people.get_groups(pool, account_id, peer_id)
@router.get("/chats/{chat_id}/changes")
async def chat_changes(
pool: FromDishka[asyncpg.Pool], chat_id: int, account_id: AccountId
) -> list[ProfileChange]:
return await people.get_chat_changes(pool, account_id, chat_id)
backend/src/api/routers/presence.py
+2 -53
View File
@@ -5,17 +5,8 @@ import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, Query
from api.timezone import checked_tz
from utils.read import presence, presence_stats
from utils.read.models import (
DEFAULT_LIMIT,
Page,
PresenceHourly,
PresenceSample,
PresenceSession,
PresenceStats,
)
from utils.read.period import DEFAULT_TZ, MAX_DAYS
from utils.read import presence
from utils.read.models import DEFAULT_LIMIT, Page, PresenceHourly, PresenceSample
router = APIRouter(prefix="/api/presence", tags=["presence"], route_class=DishkaRoute)
@@ -23,8 +14,6 @@ AccountId = Annotated[int, Query()]
PeerId = Annotated[int, Query()]
DateFrom = Annotated[datetime | None, Query()]
DateTo = Annotated[datetime | None, Query()]
Days = Annotated[int, Query(ge=1, le=MAX_DAYS)]
Tz = Annotated[str, Query()]
@router.get("")
@@ -65,43 +54,3 @@ async def presence_hourly(
return await presence.presence_hourly(
pool, account_id, peer_id, date_from=date_from, date_to=date_to
)
@router.get("/stats")
async def stats(
pool: FromDishka[asyncpg.Pool],
account_id: AccountId,
peer_id: PeerId,
days: Days = 30,
tz: Tz = DEFAULT_TZ,
limit: Annotated[int, Query()] = DEFAULT_LIMIT,
offset: Annotated[int, Query()] = 0,
) -> PresenceStats:
return await presence_stats.presence_stats(
pool,
account_id,
peer_id,
Page(limit=limit, offset=offset),
days=days,
tz=await checked_tz(pool, tz),
)
@router.get("/sessions")
async def sessions(
pool: FromDishka[asyncpg.Pool],
account_id: AccountId,
peer_id: PeerId,
days: Days = 30,
tz: Tz = DEFAULT_TZ,
limit: Annotated[int, Query()] = DEFAULT_LIMIT,
offset: Annotated[int, Query()] = 0,
) -> list[PresenceSession]:
return await presence_stats.presence_sessions(
pool,
account_id,
peer_id,
Page(limit=limit, offset=offset),
days=days,
tz=await checked_tz(pool, tz),
)
backend/src/api/routers/search.py
+2 -2
View File
@@ -4,7 +4,7 @@ import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, Query
from utils.search.models import SearchFilters, SearchPage
from utils.search.models import SearchFilters, SearchHit
from utils.search.repository import search_messages
router = APIRouter(prefix="/api/search", tags=["search"], route_class=DishkaRoute)
@@ -13,5 +13,5 @@ router = APIRouter(prefix="/api/search", tags=["search"], route_class=DishkaRout
@router.get("")
async def search(
pool: FromDishka[asyncpg.Pool], filters: Annotated[SearchFilters, Query()]
) -> SearchPage:
) -> list[SearchHit]:
return await search_messages(pool, filters)
backend/src/api/routers/social.py
+2 -35
View File
@@ -2,13 +2,10 @@ from typing import Annotated
import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel
from fastapi import APIRouter, Query
from utils.jobs import enqueue
from utils.read import social
from utils.read.models import CallbackView, LinkView, ReactionView, ReadInfoView
from utils.read.read_receipts import read_info
from utils.read.models import CallbackView, LinkView, ReactionView
router = APIRouter(
prefix="/api/messages/{chat_id}/{message_id}",
@@ -19,10 +16,6 @@ router = APIRouter(
AccountId = Annotated[int, Query()]
class RefreshRequest(BaseModel):
account_id: int
@router.get("/callbacks")
async def message_callbacks(
pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, account_id: AccountId
@@ -42,29 +35,3 @@ async def message_links(
pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, account_id: AccountId
) -> list[LinkView]:
return await social.get_links(pool, account_id, chat_id, message_id)
@router.get("/read")
async def message_read(
pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, account_id: AccountId
) -> ReadInfoView:
info = await read_info(pool, account_id, chat_id, message_id)
if info is None:
raise HTTPException(status_code=404, detail="message not found")
return info
@router.post("/read/refresh")
async def refresh_message_read(
pool: FromDishka[asyncpg.Pool], chat_id: int, message_id: int, body: RefreshRequest
) -> dict[str, int | None]:
info = await read_info(pool, body.account_id, chat_id, message_id)
if info is None or not info.can_fetch:
return {"job_id": None}
job_id = await enqueue(
pool,
body.account_id,
"fetch_read_info",
{"chat_id": chat_id, "message_id": message_id},
)
return {"job_id": job_id}
backend/src/api/routers/watches.py
+21 -69
View File
@@ -6,88 +6,55 @@ from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel
from utils.read import watches
from utils.read.models import AlertPage, WatchItem, WatchView
from utils.watch_rules import WatchParamsError, clean_name, normalize_params
from utils.read.models import DEFAULT_LIMIT, AlertView, Page, WatchView
router = APIRouter(prefix="/api", tags=["watches"], route_class=DishkaRoute)
class WatchUpdate(BaseModel):
kind: str | None = None
class WatchCreate(BaseModel):
account_id: int
kind: str
params: dict[str, Any] = {}
name: str | None = None
enabled: bool = True
class WatchCreate(WatchUpdate):
account_id: int
kind: str
class SeenAll(BaseModel):
account_id: int
up_to: int | None = None
def _params(kind: str, params: dict[str, Any]) -> dict[str, Any]:
try:
return normalize_params(kind, params)
except WatchParamsError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
async def _item(pool: asyncpg.Pool, watch: WatchView) -> WatchItem:
return (await watches.describe_watches(pool, watch.account_id, [watch]))[0]
class WatchUpdate(BaseModel):
params: dict[str, Any] = {}
enabled: bool = True
@router.get("/watches")
async def list_watches(
pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
) -> list[WatchItem]:
rows = await watches.list_watches(pool, account_id)
return await watches.describe_watches(pool, account_id, rows)
) -> list[WatchView]:
return await watches.list_watches(pool, account_id)
@router.post("/watches", status_code=201)
async def create_watch(pool: FromDishka[asyncpg.Pool], body: WatchCreate) -> WatchItem:
watch = await watches.create_watch(
pool,
body.account_id,
body.kind,
_params(body.kind, body.params),
name=clean_name(body.name),
enabled=body.enabled,
async def create_watch(pool: FromDishka[asyncpg.Pool], body: WatchCreate) -> WatchView:
return await watches.create_watch(
pool, body.account_id, body.kind, body.params, enabled=body.enabled
)
return await _item(pool, watch)
@router.get("/watches/{watch_id}")
async def get_watch(pool: FromDishka[asyncpg.Pool], watch_id: int) -> WatchItem:
async def get_watch(pool: FromDishka[asyncpg.Pool], watch_id: int) -> WatchView:
watch = await watches.get_watch(pool, watch_id)
if watch is None:
raise HTTPException(status_code=404, detail="watch not found")
return await _item(pool, watch)
return watch
@router.put("/watches/{watch_id}")
async def update_watch(
pool: FromDishka[asyncpg.Pool], watch_id: int, body: WatchUpdate
) -> WatchItem:
current = await watches.get_watch(pool, watch_id)
if current is None:
raise HTTPException(status_code=404, detail="watch not found")
kind = body.kind or current.kind
) -> WatchView:
watch = await watches.update_watch(
pool,
watch_id,
kind,
_params(kind, body.params),
name=clean_name(body.name),
enabled=body.enabled,
pool, watch_id, body.params, enabled=body.enabled
)
if watch is None:
raise HTTPException(status_code=404, detail="watch not found")
return await _item(pool, watch)
return watch
@router.delete("/watches/{watch_id}", status_code=204)
@@ -101,29 +68,14 @@ async def list_alerts(
pool: FromDishka[asyncpg.Pool],
account_id: Annotated[int, Query()],
seen: Annotated[bool | None, Query()] = None,
before: Annotated[int | None, Query()] = None,
limit: Annotated[int, Query()] = watches.ALERT_PAGE,
) -> AlertPage:
limit: Annotated[int, Query()] = DEFAULT_LIMIT,
offset: Annotated[int, Query()] = 0,
) -> list[AlertView]:
return await watches.list_alerts(
pool, account_id, seen=seen, before=before, limit=limit
pool, account_id, Page(limit=limit, offset=offset), seen=seen
)
@router.get("/alerts/unseen")
async def unseen_alerts(
pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
) -> dict[str, int]:
return {"unseen": await watches.unseen_count(pool, account_id)}
@router.post("/alerts/seen")
async def mark_all_seen(
pool: FromDishka[asyncpg.Pool], body: SeenAll
) -> dict[str, int]:
marked = await watches.mark_all_seen(pool, body.account_id, body.up_to)
return {"marked": marked}
@router.post("/alerts/{alert_id}/seen", status_code=204)
async def mark_alert_seen(pool: FromDishka[asyncpg.Pool], alert_id: int) -> None:
if not await watches.mark_alert_seen(pool, alert_id):
backend/src/api/routers/webhooks.py
-121
View File
@@ -1,121 +0,0 @@
from typing import Annotated
from urllib.parse import urlsplit
import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel
from api.webhooks import send_test
from utils.read import webhooks
from utils.read.models import DeliveryView, WebhookView
router = APIRouter(prefix="/api/webhooks", tags=["webhooks"], route_class=DishkaRoute)
MAX_URL = 2048
class WebhookUpdate(BaseModel):
url: str
watch_ids: list[int] | None = None
enabled: bool = True
class WebhookCreate(WebhookUpdate):
account_id: int
def _url(value: str) -> str:
url = value.strip()
parts = urlsplit(url)
if (
len(url) > MAX_URL
or parts.scheme not in {"http", "https"}
or not parts.hostname
):
raise HTTPException(
status_code=422, detail="url must be an http or https address"
)
return url
def _watch_ids(ids: list[int] | None) -> list[int] | None:
return sorted(set(ids)) if ids else None
async def _webhook(pool: asyncpg.Pool, webhook_id: int) -> WebhookView:
webhook = await webhooks.get_webhook(pool, webhook_id)
if webhook is None:
raise HTTPException(status_code=404, detail="webhook not found")
return webhook
@router.get("")
async def list_webhooks(
pool: FromDishka[asyncpg.Pool], account_id: Annotated[int, Query()]
) -> list[WebhookView]:
return await webhooks.list_webhooks(pool, account_id)
@router.post("", status_code=201)
async def create_webhook(
pool: FromDishka[asyncpg.Pool], body: WebhookCreate
) -> WebhookView:
webhook = await webhooks.create_webhook(
pool,
body.account_id,
_url(body.url),
_watch_ids(body.watch_ids),
enabled=body.enabled,
)
if webhook is None:
raise HTTPException(status_code=500, detail="webhook was not saved")
return webhook
@router.put("/{webhook_id}")
async def update_webhook(
pool: FromDishka[asyncpg.Pool], webhook_id: int, body: WebhookUpdate
) -> WebhookView:
webhook = await webhooks.update_webhook(
pool,
webhook_id,
_url(body.url),
_watch_ids(body.watch_ids),
enabled=body.enabled,
)
if webhook is None:
raise HTTPException(status_code=404, detail="webhook not found")
return webhook
@router.delete("/{webhook_id}", status_code=204)
async def delete_webhook(pool: FromDishka[asyncpg.Pool], webhook_id: int) -> None:
if not await webhooks.delete_webhook(pool, webhook_id):
raise HTTPException(status_code=404, detail="webhook not found")
@router.post("/{webhook_id}/secret")
async def rotate_secret(pool: FromDishka[asyncpg.Pool], webhook_id: int) -> WebhookView:
webhook = await webhooks.rotate_secret(pool, webhook_id)
if webhook is None:
raise HTTPException(status_code=404, detail="webhook not found")
return webhook
@router.post("/{webhook_id}/test")
async def test_webhook(pool: FromDishka[asyncpg.Pool], webhook_id: int) -> DeliveryView:
delivery = await send_test(pool, await _webhook(pool, webhook_id))
if delivery is None:
raise HTTPException(status_code=500, detail="test delivery was not recorded")
return delivery
@router.get("/{webhook_id}/deliveries")
async def list_deliveries(
pool: FromDishka[asyncpg.Pool],
webhook_id: int,
limit: Annotated[int, Query(ge=1, le=100)] = webhooks.DELIVERY_LOG,
) -> list[DeliveryView]:
await _webhook(pool, webhook_id)
return await webhooks.list_deliveries(pool, webhook_id, limit)
backend/src/api/timezone.py
-10
View File
@@ -1,10 +0,0 @@
import asyncpg
from fastapi import HTTPException
from utils.read.period import known_timezone
async def checked_tz(pool: asyncpg.Pool, tz: str) -> str:
if not await known_timezone(pool, tz):
raise HTTPException(status_code=422, detail=f"Unknown time zone: {tz}")
return tz
backend/src/api/webhooks.py
-191
View File
@@ -1,191 +0,0 @@
import asyncio
import contextlib
import hashlib
import hmac
import json
import logging
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
import asyncpg
import httpx2
from utils.env import env
from utils.read import watches
from utils.read import webhooks as webhooks_read
from utils.read.models import AlertItem, DeliveryView, WebhookView
logger = logging.getLogger(__name__)
TIMEOUT_SECONDS = 10
SCAN_SECONDS = 30
BATCH = 20
RETRY_DELAYS = (30, 90, 270, 810, 2430)
ERROR_CHARS = 300
USER_AGENT = "beavergram-webhooks"
def sign(secret: str, body: bytes) -> str:
digest = hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
return f"sha256={digest}"
def _encode(body: dict[str, Any]) -> str:
return json.dumps(body, ensure_ascii=False, separators=(",", ":"))
def _with_url(body: dict[str, Any], path: str) -> dict[str, Any]:
if env.api.public_url:
body["url"] = env.api.public_url.rstrip("/") + path
return body
def alert_body(item: AlertItem) -> dict[str, Any]:
data = item.model_dump(mode="json", exclude={"id", "seen"})
return _with_url({"type": "alert", "alert_id": item.id, **data}, item.path)
def test_body(webhook: WebhookView) -> dict[str, Any]:
body = {
"type": "test",
"alert_id": None,
"account_id": webhook.account_id,
"ts": datetime.now(UTC).isoformat(),
"event": "test",
"watch": None,
"chat": None,
"message": None,
"peer": None,
"path": "/app",
}
return _with_url(body, "/app")
@dataclass(frozen=True, slots=True)
class Target:
delivery_id: int
event: str
url: str
secret: str
async def post(
client: httpx2.AsyncClient, target: Target, body: str
) -> tuple[int | None, str | None]:
raw = body.encode()
headers = {
"Content-Type": "application/json",
"User-Agent": USER_AGENT,
"X-Beavergram-Event": target.event,
"X-Beavergram-Delivery": str(target.delivery_id),
"X-Beavergram-Signature": sign(target.secret, raw),
}
try:
response = await client.post(target.url, content=raw, headers=headers)
except httpx2.HTTPError as exc:
return None, (str(exc) or type(exc).__name__)[:ERROR_CHARS]
if response.is_success:
return response.status_code, None
reason = response.text.strip()[:ERROR_CHARS] or response.reason_phrase
return response.status_code, reason or f"HTTP {response.status_code}"
def new_client() -> httpx2.AsyncClient:
return httpx2.AsyncClient(timeout=TIMEOUT_SECONDS, follow_redirects=False)
async def _body(pool: asyncpg.Pool, row: asyncpg.Record) -> str | None:
if row["body"] is not None:
return row["body"]
if row["alert_id"] is None:
return None
item = await watches.get_alert(pool, row["alert_id"])
if item is None:
return None
body = _encode(alert_body(item))
await webhooks_read.save_body(pool, row["id"], body)
return body
async def _deliver(
pool: asyncpg.Pool, client: httpx2.AsyncClient, row: asyncpg.Record
) -> None:
body = await _body(pool, row)
if body is None:
await webhooks_read.finish(
pool, row["id"], webhooks_read.STATUS_FAILED, None, "alert is gone"
)
return
target = Target(row["id"], row["event"], row["url"], row["secret"])
code, error = await post(client, target, body)
if error is None:
await webhooks_read.finish(
pool, row["id"], webhooks_read.STATUS_DELIVERED, code, None
)
return
attempt = row["attempts"]
if attempt > len(RETRY_DELAYS):
await webhooks_read.finish(
pool, row["id"], webhooks_read.STATUS_FAILED, code, error
)
return
retry_at = datetime.now(UTC) + timedelta(seconds=RETRY_DELAYS[attempt - 1])
await webhooks_read.finish(
pool, row["id"], webhooks_read.STATUS_PENDING, code, error, retry_at
)
async def send_test(pool: asyncpg.Pool, webhook: WebhookView) -> DeliveryView | None:
body = _encode(test_body(webhook))
delivery_id = await webhooks_read.start_test(pool, webhook.id, body)
async with new_client() as client:
target = Target(delivery_id, "test", webhook.url, webhook.secret)
code, error = await post(client, target, body)
status = webhooks_read.STATUS_FAILED if error else webhooks_read.STATUS_DELIVERED
await webhooks_read.finish(pool, delivery_id, status, code, error)
return await webhooks_read.get_delivery(pool, delivery_id)
class Outbox:
def __init__(self) -> None:
self._wake = asyncio.Event()
def wake(self) -> None:
self._wake.set()
async def run(self, pool: asyncpg.Pool) -> None:
released = False
async with new_client() as client:
while True:
self._wake.clear()
try:
if not released:
await webhooks_read.release_stuck(pool)
released = True
await self._drain(pool, client)
delay = await self._delay(pool)
except Exception:
logger.exception("Webhook delivery failed")
delay = SCAN_SECONDS
with contextlib.suppress(TimeoutError):
await asyncio.wait_for(self._wake.wait(), timeout=delay)
async def _drain(self, pool: asyncpg.Pool, client: httpx2.AsyncClient) -> None:
while rows := await webhooks_read.claim_due(pool, BATCH):
results = await asyncio.gather(
*(_deliver(pool, client, row) for row in rows), return_exceptions=True
)
for result in results:
if isinstance(result, Exception):
logger.error("Webhook delivery crashed: %r", result)
async def _delay(self, pool: asyncpg.Pool) -> float:
due = await webhooks_read.next_due(pool)
if due is None:
return SCAN_SECONDS
wait = (due - datetime.now(UTC)).total_seconds()
return min(SCAN_SECONDS, max(wait, 1.0))
outbox = Outbox()
backend/src/userbot/handlers/deletes.py
-5
View File
@@ -20,16 +20,11 @@ async def on_deleted_messages(client: PyroClient, messages: list[Message]) -> No
box.append(message.id)
else:
channels.setdefault(chat_id, []).append(message.id)
ctx.pending_deletes.remember(None, box)
for chat_id, ids in channels.items():
ctx.pending_deletes.remember(chat_id, ids)
if box:
await repository.mark_deleted_box(ctx.pool, ctx.account_id, box)
await ctx.watches.on_deleted(None, box)
await notify_bg_event(ctx.pool, "delete", ctx.account_id, message_ids=box)
for chat_id, ids in channels.items():
await repository.mark_deleted_channel(ctx.pool, ctx.account_id, chat_id, ids)
await ctx.watches.on_deleted(chat_id, ids)
await notify_bg_event(
ctx.pool, "delete", ctx.account_id, chat_id=chat_id, message_ids=ids
)
backend/src/userbot/handlers/edits.py
+16 -6
View File
@@ -2,10 +2,10 @@ from pyrogram.types import Message
from userbot.modules.capture import repository
from userbot.modules.capture.chat_meta import meta_from_chat
from userbot.modules.capture.message import message_row
from userbot.modules.capture.message import sender_id
from userbot.modules.client import PyroClient
from userbot.modules.media import capture_media
from userbot.modules.watches.seen import seen_message
from userbot.modules.media import capture_media, media_unique_id, self_destruct_ttl
from userbot.modules.raw_json import raw_json
from utils.events import notify_bg_event
@@ -23,12 +23,22 @@ async def on_edited_message(client: PyroClient, message: Message) -> None:
chat = message.chat
chat_id = chat.id or 0
meta = meta_from_chat(chat, ctx.contacts.ids)
await ctx.watches.on_edit(seen_message(chat_id, message))
toggles = ctx.resolve(meta)
if not (toggles.messages and toggles.track_edits_deletes):
if not toggles.track_edits_deletes:
return
changed = await repository.add_version(
ctx.pool, ctx.account_id, message_row(message, chat_id, message.date)
ctx.pool,
ctx.account_id,
chat_id,
message.id,
message.date,
sender_id(message),
message.text or message.caption,
raw_json(message),
message.edit_date,
media_unique_id(message),
has_media=message.media is not None,
is_self_destruct=self_destruct_ttl(message) is not None,
)
if not changed:
return
backend/src/userbot/handlers/messages.py
+1 -2
View File
@@ -5,7 +5,6 @@ from userbot.modules.capture.chat_meta import meta_from_chat
from userbot.modules.client import PyroClient
from userbot.modules.stt import is_transcribable
from userbot.modules.stt.gate import safe_transcribe
from userbot.modules.watches.seen import seen_message
from utils.events import notify_bg_event
@@ -21,7 +20,7 @@ async def on_message(client: PyroClient, message: Message) -> None:
):
return
meta = meta_from_chat(message.chat, ctx.contacts.ids)
await ctx.watches.on_message(seen_message(meta.chat_id, message))
await ctx.watches.on_text(meta.chat_id, message.id, message.text or message.caption)
toggles = ctx.resolve(meta)
if not toggles.messages:
return
backend/src/userbot/handlers/presence.py
+12 -15
View File
@@ -13,21 +13,18 @@ async def on_user_status(client: PyroClient, user: User) -> None:
ctx = client.capture
if ctx is None or user.status is None:
return
at = datetime.now(UTC)
status = user.status.name.lower()
if ctx.resolve_peer(user.id, is_bot=bool(user.is_bot)).presence:
await repository.insert_status(
ctx.pool,
ctx.account_id,
user.id,
at,
status,
user.last_online_date,
user.next_offline_date,
raw_json(user.raw),
)
await notify_bg_event(ctx.pool, "presence", ctx.account_id, chat_id=user.id)
await ctx.watches.on_status(user.id, status, at, user.last_online_date)
await repository.insert_status(
ctx.pool,
ctx.account_id,
user.id,
datetime.now(UTC),
user.status.name.lower(),
user.last_online_date,
user.next_offline_date,
raw_json(user.raw),
)
await ctx.watches.on_status(user.id, is_online=user.status.name.lower() == "online")
await notify_bg_event(ctx.pool, "presence", ctx.account_id, chat_id=user.id)
handlers = on_user_status.handlers # ty: ignore[unresolved-attribute]
backend/src/userbot/handlers/raw/profiles.py
+2 -15
View File
@@ -25,14 +25,8 @@ async def _handle_user(
current = await get_peer(ctx.pool, ctx.account_id, update.user_id)
if current == fields:
return
is_bot = bool(getattr(raw_user, "bot", False))
await write_profile(
ctx.pool,
ctx.account_id,
update.user_id,
fields,
raw_json(raw_user),
history=ctx.resolve_peer(update.user_id, is_bot=is_bot).profile_history,
ctx.pool, ctx.account_id, update.user_id, fields, raw_json(raw_user)
)
changed_photo = current is None or current.photo_unique_id != photo_unique_id
if photo_file_id and photo_unique_id and changed_photo:
@@ -57,14 +51,7 @@ async def _merge_partial(
merged = replace(current or ProfileFields(), **overrides)
if merged == current:
return
await write_profile(
ctx.pool,
ctx.account_id,
peer_id,
merged,
raw_str,
history=ctx.resolve_peer(peer_id).profile_history,
)
await write_profile(ctx.pool, ctx.account_id, peer_id, merged, raw_str)
async def handle(
backend/src/userbot/handlers/raw/read_receipts.py
+5 -15
View File
@@ -7,35 +7,25 @@ from userbot.modules.raw_json import raw_json
from userbot.modules.read_receipts import repository
from utils.events import notify_bg_event
HANDLES = (raw.types.UpdateReadHistoryOutbox, raw.types.UpdateReadChannelOutbox)
def _watermark(update: raw.base.Update) -> tuple[int, int] | None:
if isinstance(update, raw.types.UpdateReadHistoryOutbox):
return utils.get_peer_id(update.peer), update.max_id
if isinstance(update, raw.types.UpdateReadChannelOutbox):
peer = raw.types.PeerChannel(channel_id=update.channel_id)
return utils.get_peer_id(peer), update.max_id
return None
HANDLES = (raw.types.UpdateReadHistoryOutbox,)
async def handle(
client: PyroClient, update: raw.base.Update, _users: dict, _chats: dict
) -> None:
ctx = client.capture
watermark = _watermark(update)
if ctx is None or watermark is None:
if ctx is None or not isinstance(update, raw.types.UpdateReadHistoryOutbox):
return
chat_id, max_id = watermark
chat_id = utils.get_peer_id(update.peer)
await repository.insert_read(
ctx.pool,
ctx.account_id,
chat_id,
chat_id,
datetime.now(UTC),
max_id,
update.max_id,
raw_json(update),
)
await notify_bg_event(
ctx.pool, "receipt", ctx.account_id, chat_id=chat_id, message_id=max_id
ctx.pool, "receipt", ctx.account_id, chat_id=chat_id, message_id=update.max_id
)
backend/src/userbot/handlers/raw/transcribed.py
+1 -1
View File
@@ -20,4 +20,4 @@ async def handle(
await repository.set_extracted_text(
ctx.pool, ctx.account_id, meta.chat_id, update.msg_id, update.text
)
await ctx.watches.on_transcript(meta.chat_id, update.msg_id, update.text)
await ctx.watches.on_text(meta.chat_id, update.msg_id, update.text)
backend/src/userbot/handlers/stories.py
+3 -8
View File
@@ -1,19 +1,14 @@
from pyrogram.types import Story
from userbot.modules.client import PyroClient
from userbot.modules.stories.service import save_story, story_peer_id
from userbot.modules.stories.service import save_story
@PyroClient.on_story()
async def on_story(client: PyroClient, story: Story) -> None:
ctx = client.capture
if ctx is None:
if client.capture is None:
return
peer_id = story_peer_id(story)
is_bot = bool(story.from_user and story.from_user.is_bot)
if peer_id > 0 and not ctx.resolve_peer(peer_id, is_bot=is_bot).stories:
return
await save_story(client, ctx, story)
await save_story(client, client.capture, story)
handlers = on_story.handlers # ty: ignore[unresolved-attribute]
backend/src/userbot/modules/capture/context.py
+1 -13
View File
@@ -2,11 +2,10 @@ import asyncpg
from pyrogram import Client
from userbot.modules.capture.identity import ChatMetaCache, PeerIdentityCache
from userbot.modules.capture.pending import PendingDeletes
from userbot.modules.contacts import ContactCache
from userbot.modules.folders import FolderCache
from userbot.modules.watches import WatchCache
from utils.policy.models import CaptureToggles, ChatKind, ChatMeta, PolicySet
from utils.policy.models import CaptureToggles, ChatMeta, PolicySet
from utils.policy.repository import load_policy_set
from utils.policy.resolver import resolve
from utils.storage import ContentAddressedStorage
@@ -29,7 +28,6 @@ class CaptureContext:
self.watches = WatchCache(pool, account_id)
self.peer_identity = PeerIdentityCache()
self.chat_meta = ChatMetaCache()
self.pending_deletes = PendingDeletes()
self.policies = PolicySet()
async def reload_policies(self) -> None:
@@ -38,16 +36,6 @@ class CaptureContext:
def resolve(self, chat: ChatMeta) -> CaptureToggles:
return resolve(chat, self.folders.folders, self.policies)
def resolve_peer(self, peer_id: int, *, is_bot: bool = False) -> CaptureToggles:
return self.resolve(
ChatMeta(
chat_id=peer_id,
kind=ChatKind.DM,
is_bot=is_bot,
is_contact=peer_id in self.contacts.ids,
)
)
async def build_capture_context(
client: Client,
backend/src/userbot/modules/capture/identity.py
+5 -10
View File
@@ -58,14 +58,7 @@ async def _capture_peer(message: Message, ctx: CaptureContext) -> None:
fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
if not await ctx.peer_identity.changed(ctx.pool, ctx.account_id, user.id, fields):
return
await write_profile(
ctx.pool,
ctx.account_id,
user.id,
fields,
raw_json(user),
history=ctx.resolve_peer(user.id, is_bot=bool(user.is_bot)).profile_history,
)
await write_profile(ctx.pool, ctx.account_id, user.id, fields, raw_json(user))
if photo_file_id and photo_unique_id:
from userbot.modules.avatars import note_avatar # noqa: PLC0415
@@ -89,15 +82,17 @@ async def _capture_chat(message: Message, ctx: CaptureContext) -> None:
meta = (chat.title, photo_unique_id)
if not await ctx.chat_meta.changed(ctx.pool, ctx.account_id, chat.id, meta):
return
from userbot.modules.groups.repository import record_chat_meta # noqa: PLC0415
from userbot.modules.groups.repository import insert_chat_history # noqa: PLC0415
await record_chat_meta(
await insert_chat_history(
ctx.pool,
ctx.account_id,
chat.id,
message.id,
"meta",
chat.title,
photo_unique_id,
None,
message.date,
raw_json(message),
)
backend/src/userbot/modules/capture/message.py
+12 -27
View File
@@ -1,14 +1,11 @@
from datetime import datetime
from pyrogram import Client
from pyrogram.types import Message
from userbot.modules.capture import repository
from userbot.modules.capture.context import CaptureContext
from userbot.modules.capture.repository import MessageRow
from userbot.modules.links import extract_links
from userbot.modules.links import repository as links_repository
from userbot.modules.media import capture_media, media_unique_id, self_destruct_ttl
from userbot.modules.media import capture_media, self_destruct_ttl
from userbot.modules.raw_json import raw_json
from utils.policy.models import CaptureToggles
@@ -37,39 +34,27 @@ def callbacks(message: Message) -> list[tuple[int, str | None, bytes | None]]:
return buttons
def message_row(message: Message, chat_id: int, date: datetime) -> MessageRow:
return MessageRow(
chat_id=chat_id,
message_id=message.id,
date=date,
sender_id=sender_id(message),
text=message.text or message.caption,
raw=raw_json(message),
has_media=message.media is not None,
is_self_destruct=self_destruct_ttl(message) is not None,
edit_date=message.edit_date,
media_unique_id=media_unique_id(message),
)
async def capture_message(
client: Client, message: Message, ctx: CaptureContext, toggles: CaptureToggles
) -> None:
if message.empty or message.chat is None or message.date is None:
return
chat_id = message.chat.id or 0
current = await repository.upsert_message(
ctx.pool, ctx.account_id, message_row(message, chat_id, message.date)
await repository.upsert_message(
ctx.pool,
ctx.account_id,
chat_id,
message.id,
message.date,
sender_id(message),
message.text or message.caption,
raw_json(message),
has_media=message.media is not None,
is_self_destruct=self_destruct_ttl(message) is not None,
)
if ctx.pending_deletes.take(chat_id, message.id):
await repository.mark_deleted_late(
ctx.pool, ctx.account_id, chat_id, message.id, message.date
)
from userbot.modules.capture.identity import capture_identity # noqa: PLC0415
await capture_identity(message, ctx)
if not current:
return
await capture_media(client, message, ctx, chat_id, message.id, toggles)
buttons = callbacks(message)
if buttons:
backend/src/userbot/modules/capture/pending.py
-36
View File
@@ -1,36 +0,0 @@
import time
from userbot.modules.capture.repository import CHANNEL_ID_THRESHOLD
PENDING_DELETE_SECONDS = 120.0
PENDING_DELETE_LIMIT = 10_000
DeleteKey = tuple[int | None, int]
def _key(chat_id: int | None, message_id: int) -> DeleteKey:
if chat_id is None or chat_id > CHANNEL_ID_THRESHOLD:
return None, message_id
return chat_id, message_id
class PendingDeletes:
def __init__(self) -> None:
self._expires: dict[DeleteKey, float] = {}
def remember(self, chat_id: int | None, message_ids: list[int]) -> None:
now = time.monotonic()
while self._expires:
oldest = next(iter(self._expires))
fresh = self._expires[oldest] > now
if fresh and len(self._expires) < PENDING_DELETE_LIMIT:
break
del self._expires[oldest]
for message_id in message_ids:
key = _key(chat_id, message_id)
self._expires.pop(key, None)
self._expires[key] = now + PENDING_DELETE_SECONDS
def take(self, chat_id: int, message_id: int) -> bool:
expires = self._expires.pop(_key(chat_id, message_id), None)
return expires is not None and expires > time.monotonic()
backend/src/userbot/modules/capture/repository.py
+65 -158
View File
@@ -1,4 +1,3 @@
from dataclasses import dataclass
from datetime import datetime
import asyncpg
@@ -22,19 +21,13 @@ _TOUCH_EDITED = """
INSERT INTO messages
(account_id, chat_id, message_id, date, sender_id, text, raw,
has_media, is_self_destruct, edited_at)
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8, $9, CASE WHEN $10 THEN now() END)
VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8, $9, now())
ON CONFLICT (account_id, chat_id, message_id, date) DO UPDATE SET
text = EXCLUDED.text,
raw = EXCLUDED.raw,
has_media = EXCLUDED.has_media,
is_self_destruct = EXCLUDED.is_self_destruct,
edited_at = coalesce(EXCLUDED.edited_at, messages.edited_at)
"""
_LOCK_MESSAGE = """
SELECT pg_advisory_xact_lock(
hashtextextended(concat_ws(':', 'message', $1::int, $2::bigint, $3::bigint), 0)
)
edited_at = now()
"""
_FIND_DELETED = """
@@ -47,12 +40,6 @@ UPDATE messages SET deleted_at = now()
WHERE account_id = $1 AND chat_id = $2 AND message_id = $3 AND date = $4
"""
_MARK_DELETED_LATE = """
UPDATE messages SET deleted_at = now()
WHERE account_id = $1 AND chat_id = $2 AND message_id = $3 AND date = $4
AND deleted_at IS NULL
"""
_INSERT_VERSION = """
INSERT INTO message_versions
(account_id, chat_id, message_id, observed_at, edit_date, text, raw)
@@ -65,7 +52,7 @@ INSERT INTO message_versions
(account_id, chat_id, message_id, observed_at, edit_date, text, raw)
SELECT account_id, chat_id, message_id, clock_timestamp(), NULL, text, raw
FROM messages m
WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3 AND m.date = $4
WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3
AND NOT EXISTS (
SELECT 1 FROM message_versions v
WHERE v.account_id = m.account_id AND v.chat_id = m.chat_id
@@ -74,37 +61,15 @@ WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3 AND m.date = $4
ON CONFLICT DO NOTHING
"""
_INSERT_PAST_VERSION = """
INSERT INTO message_versions
(account_id, chat_id, message_id, observed_at, edit_date, text, raw)
SELECT $1::int, $2::bigint, $3::bigint,
coalesce(
(SELECT min(v.observed_at) - interval '1 microsecond'
FROM message_versions v
WHERE v.account_id = $1 AND v.chat_id = $2 AND v.message_id = $3
AND v.edit_date > $4::timestamptz),
$4::timestamptz,
$5::timestamptz
),
$4::timestamptz, $6::text, $7::jsonb
WHERE NOT EXISTS (
SELECT 1 FROM message_versions v
WHERE v.account_id = $1 AND v.chat_id = $2 AND v.message_id = $3
AND v.edit_date IS NOT DISTINCT FROM $4::timestamptz
AND v.text IS NOT DISTINCT FROM $6::text
)
ON CONFLICT DO NOTHING
"""
_CURRENT_CONTENT = """
SELECT
m.text AS text,
m.raw->>'edit_date' AS edit_date,
(SELECT d.unique_id FROM media d
WHERE d.account_id = m.account_id AND d.chat_id = m.chat_id
AND d.message_id = m.message_id) AS media_unique_id
FROM messages m
WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3 AND m.date = $4
WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3
ORDER BY m.date DESC LIMIT 1
"""
_CURRENT_MEDIA = """
@@ -156,110 +121,33 @@ WHERE evicted_key = $2 OR (unique_id = $1 AND evicted_at IS NOT NULL)
"""
@dataclass(frozen=True, slots=True)
class MessageRow:
chat_id: int
message_id: int
date: datetime
sender_id: int | None
text: str | None
raw: str
has_media: bool
is_self_destruct: bool
edit_date: datetime | None
media_unique_id: str | None
@property
def columns(self) -> tuple:
return (
self.chat_id,
self.message_id,
self.date,
self.sender_id,
self.text,
self.raw,
self.has_media,
self.is_self_destruct,
)
async def _lock_current(
conn: asyncpg.Connection, account_id: int, row: MessageRow
) -> asyncpg.Record | None:
await conn.execute(_LOCK_MESSAGE, account_id, row.chat_id, row.message_id)
return await conn.fetchrow(
_CURRENT_CONTENT, account_id, row.chat_id, row.message_id, row.date
)
def _is_stale(current: asyncpg.Record, row: MessageRow) -> bool:
stored = current["edit_date"]
if stored is None:
return False
if row.edit_date is None:
return True
try:
return row.edit_date < datetime.fromisoformat(stored)
except (TypeError, ValueError):
return False
async def _record_past(
conn: asyncpg.Connection, account_id: int, row: MessageRow
async def upsert_message( # noqa: PLR0913
pool: asyncpg.Pool,
account_id: int,
chat_id: int,
message_id: int,
date: datetime,
sender_id: int | None,
text: str | None,
raw: str,
*,
has_media: bool,
is_self_destruct: bool,
) -> None:
await conn.execute(
_INSERT_PAST_VERSION,
await pool.execute(
_UPSERT_MESSAGE,
account_id,
row.chat_id,
row.message_id,
row.edit_date,
row.date,
row.text,
row.raw,
chat_id,
message_id,
date,
sender_id,
text,
raw,
has_media,
is_self_destruct,
)
async def _record_edit(
conn: asyncpg.Connection, account_id: int, row: MessageRow, *, mark_edited: bool
) -> None:
await conn.execute(
_SNAPSHOT_ORIGINAL, account_id, row.chat_id, row.message_id, row.date
)
await conn.execute(_TOUCH_EDITED, account_id, *row.columns, mark_edited)
await conn.execute(
_INSERT_VERSION,
account_id,
row.chat_id,
row.message_id,
row.edit_date,
row.text,
row.raw,
)
async def upsert_message(pool: asyncpg.Pool, account_id: int, row: MessageRow) -> bool:
async with pool.acquire() as conn, conn.transaction():
current = await _lock_current(conn, account_id, row)
if current is None:
await conn.execute(_UPSERT_MESSAGE, account_id, *row.columns)
return True
stored_media = current["media_unique_id"]
changed = current["text"] != row.text or (
stored_media is not None and stored_media != row.media_unique_id
)
if _is_stale(current, row):
if changed:
await _record_past(conn, account_id, row)
return False
if changed:
await _record_edit(
conn, account_id, row, mark_edited=row.edit_date is not None
)
else:
await conn.execute(_UPSERT_MESSAGE, account_id, *row.columns)
return True
async def max_message_id(
pool: asyncpg.Pool, account_id: int, chat_id: int
) -> int | None:
@@ -300,29 +188,48 @@ async def mark_deleted_channel(
await _mark_deleted(pool, account_id, rows)
async def add_version(pool: asyncpg.Pool, account_id: int, row: MessageRow) -> bool:
async def add_version( # noqa: PLR0913
pool: asyncpg.Pool,
account_id: int,
chat_id: int,
message_id: int,
date: datetime,
sender_id: int | None,
text: str | None,
raw: str,
edit_date: datetime | None,
media_unique_id: str | None,
*,
has_media: bool,
is_self_destruct: bool,
) -> bool:
async with pool.acquire() as conn, conn.transaction():
current = await _lock_current(conn, account_id, row)
if current is not None:
changed = (
current["text"] != row.text
or current["media_unique_id"] != row.media_unique_id
)
if not changed:
return False
if _is_stale(current, row):
await _record_past(conn, account_id, row)
return False
await _record_edit(conn, account_id, row, mark_edited=True)
current = await conn.fetchrow(_CURRENT_CONTENT, account_id, chat_id, message_id)
text_changed = current is None or current["text"] != text
media_changed = (
current is not None and current["media_unique_id"] != media_unique_id
)
if not (text_changed or media_changed):
return False
await conn.execute(_SNAPSHOT_ORIGINAL, account_id, chat_id, message_id)
await conn.execute(
_TOUCH_EDITED,
account_id,
chat_id,
message_id,
date,
sender_id,
text,
raw,
has_media,
is_self_destruct,
)
await conn.execute(
_INSERT_VERSION, account_id, chat_id, message_id, edit_date, text, raw
)
return True
async def mark_deleted_late(
pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int, date: datetime
) -> None:
await pool.execute(_MARK_DELETED_LATE, account_id, chat_id, message_id, date)
async def current_media(
pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
) -> asyncpg.Record | None:
backend/src/userbot/modules/catchup/__init__.py
-3
View File
@@ -1,3 +0,0 @@
from userbot.modules.catchup.service import SavedState, catch_up, read_saved_state
__all__ = ["SavedState", "catch_up", "read_saved_state"]
backend/src/userbot/modules/catchup/repository.py
-41
View File
@@ -1,41 +0,0 @@
from dataclasses import dataclass
import asyncpg
_LOAD = """
SELECT catchup_pts, catchup_qts, catchup_date FROM accounts WHERE account_id = $1
"""
_SAVE = """
UPDATE accounts SET catchup_pts = $2, catchup_qts = $3, catchup_date = $4
WHERE account_id = $1
"""
_CLEAR = """
UPDATE accounts SET catchup_pts = NULL, catchup_qts = NULL, catchup_date = NULL
WHERE account_id = $1
"""
@dataclass(frozen=True)
class UpdateCursor:
pts: int
qts: int
date: int
async def load_cursor(pool: asyncpg.Pool, account_id: int) -> UpdateCursor | None:
row = await pool.fetchrow(_LOAD, account_id)
if row is None or row["catchup_pts"] is None:
return None
return UpdateCursor(row["catchup_pts"], row["catchup_qts"], row["catchup_date"])
async def save_cursor(
pool: asyncpg.Pool, account_id: int, cursor: UpdateCursor
) -> None:
await pool.execute(_SAVE, account_id, cursor.pts, cursor.qts, cursor.date)
async def clear_cursor(pool: asyncpg.Pool, account_id: int) -> None:
await pool.execute(_CLEAR, account_id)
backend/src/userbot/modules/catchup/service.py
-255
View File
@@ -1,255 +0,0 @@
import asyncio
import sqlite3
import time
from contextlib import closing
from dataclasses import dataclass, field
from pathlib import Path
from pyrogram import raw, utils
from pyrogram.errors import FloodPremiumWait, FloodWait, RPCError
from userbot.modules.capture.chat_meta import meta_from_peer
from userbot.modules.capture.context import CaptureContext
from userbot.modules.catchup import repository
from userbot.modules.catchup.repository import UpdateCursor
from userbot.modules.client import PyroClient
from utils.logging import logger
from utils.policy.models import ChatKind
BOX_SLICES = 50
CHANNEL_SLICES = 10
CHANNEL_LIMIT = 100
MAX_CHANNELS = 50
QUEUE_HIGH_WATER = 200
QUEUE_WAIT_SECONDS = 0.5
NO_QTS = -1
MAX_AGE_SECONDS = 3 * 24 * 3600
@dataclass(frozen=True)
class SavedState:
box: UpdateCursor | None = None
channels: dict[int, int] = field(default_factory=dict)
@dataclass
class BoxWalk:
cursor: UpdateCursor
done: bool = False
updates: int = 0
channels: dict[int, raw.base.Chat] = field(default_factory=dict)
def read_saved_state(path: Path) -> SavedState:
try:
uri = f"{path.resolve().as_uri()}?mode=ro"
with closing(sqlite3.connect(uri, uri=True)) as conn:
rows = conn.execute(
"SELECT id, pts, qts, date FROM update_state"
).fetchall()
except Exception as exc:
logger.warning(f"[yellow]Session {path.stem}: no saved update state ({exc})[/]")
return SavedState()
box = None
channels: dict[int, int] = {}
for state_id, pts, qts, date in rows:
if pts is None:
continue
if state_id == 0:
box = UpdateCursor(
pts, NO_QTS if qts is None else qts, date or int(time.time())
)
else:
channels[state_id] = pts
return SavedState(box, channels)
async def _dispatch(
client: PyroClient,
new_messages: list,
other_updates: list,
users: list,
chats: list,
) -> int:
await client.fetch_peers(users)
await client.fetch_peers(chats)
user_map = {user.id: user for user in users}
chat_map = {chat.id: chat for chat in chats}
queue = client.dispatcher.updates_queue
for message in new_messages:
update = raw.types.UpdateNewMessage(message=message, pts=0, pts_count=0)
queue.put_nowait((update, user_map, chat_map))
for update in other_updates:
queue.put_nowait((update, user_map, chat_map))
while queue.qsize() > QUEUE_HIGH_WATER: # noqa: ASYNC110
await asyncio.sleep(QUEUE_WAIT_SECONDS)
return len(new_messages) + len(other_updates)
def _channel_hints(other_updates: list, chats: list) -> dict[int, raw.base.Chat]:
by_id = {chat.id: chat for chat in chats}
return {
utils.get_channel_id(update.channel_id): by_id[update.channel_id]
for update in other_updates
if isinstance(update, raw.types.UpdateChannelTooLong)
and update.channel_id in by_id
}
async def _walk_box(
client: PyroClient, ctx: CaptureContext, cursor: UpdateCursor
) -> BoxWalk:
walk = BoxWalk(cursor)
for _ in range(BOX_SLICES):
diff = await client.invoke(
raw.functions.updates.GetDifference(
pts=walk.cursor.pts, date=walk.cursor.date, qts=walk.cursor.qts
)
)
if isinstance(diff, raw.types.updates.DifferenceEmpty):
walk.done = True
return walk
if isinstance(diff, raw.types.updates.DifferenceTooLong):
logger.warning(
f"[yellow]Account {ctx.account_id}: too much missed to catch up "
f"(pts {walk.cursor.pts} to {diff.pts}), that gap stays[/]"
)
walk.done = True
return walk
walk.updates += await _dispatch(
client, diff.new_messages, diff.other_updates, diff.users, diff.chats
)
walk.channels.update(_channel_hints(diff.other_updates, diff.chats))
final = isinstance(diff, raw.types.updates.Difference)
state = diff.state if final else diff.intermediate_state
moved = UpdateCursor(state.pts, state.qts, state.date or walk.cursor.date)
if moved == walk.cursor and not final:
logger.warning(
f"[yellow]Account {ctx.account_id}: catch-up stopped advancing "
f"at pts={moved.pts}[/]"
)
return walk
walk.cursor = moved
await repository.save_cursor(ctx.pool, ctx.account_id, moved)
if final:
walk.done = True
return walk
return walk
async def _walk_channel(client: PyroClient, peer_id: int, pts: int) -> int:
peer = await client.resolve_peer(peer_id)
if not isinstance(peer, raw.types.InputPeerChannel):
return 0
channel = raw.types.InputChannel(
channel_id=peer.channel_id, access_hash=peer.access_hash
)
updates = 0
for _ in range(CHANNEL_SLICES):
diff = await client.invoke(
raw.functions.updates.GetChannelDifference(
channel=channel,
filter=raw.types.ChannelMessagesFilterEmpty(),
pts=pts,
limit=CHANNEL_LIMIT,
)
)
if isinstance(diff, raw.types.updates.ChannelDifferenceEmpty):
return updates
if isinstance(diff, raw.types.updates.ChannelDifferenceTooLong):
logger.info(f"Channel {peer_id}: too much missed to catch up")
return updates
updates += await _dispatch(
client, diff.new_messages, diff.other_updates, diff.users, diff.chats
)
if diff.final:
return updates
pts = diff.pts
return updates
def _captured_channels(
ctx: CaptureContext, hints: dict[int, raw.base.Chat], saved: dict[int, int]
) -> list[int]:
wanted: list[tuple[bool, int]] = []
for peer_id, chat in hints.items():
if peer_id not in saved:
continue
meta = meta_from_peer(
raw.types.PeerChannel(channel_id=chat.id),
{chat.id: chat},
{},
ctx.contacts.ids,
)
toggles = ctx.resolve(meta)
if toggles.messages or toggles.track_edits_deletes:
wanted.append((meta.kind is ChatKind.CHANNEL, peer_id))
return [peer_id for _, peer_id in sorted(wanted)[:MAX_CHANNELS]]
async def _walk_channels(
client: PyroClient, ctx: CaptureContext, walk: BoxWalk, saved: dict[int, int]
) -> int:
peer_ids = _captured_channels(ctx, walk.channels, saved)
updates = 0
for peer_id in peer_ids:
try:
updates += await _walk_channel(client, peer_id, saved[peer_id])
except (FloodWait, FloodPremiumWait) as exc:
logger.warning(
f"[yellow]Account {ctx.account_id}: channel catch-up stopped "
f"by a flood wait:[/] {exc}"
)
break
except RPCError as exc:
logger.warning(
f"[yellow]Account {ctx.account_id}: channel {peer_id} "
f"catch-up failed:[/] {exc}"
)
skipped = len(walk.channels) - len(peer_ids)
if skipped:
logger.info(f"Account {ctx.account_id}: {skipped} channel gap(s) left as is")
return updates
async def catch_up(client: PyroClient, saved: SavedState) -> None:
ctx = client.capture
if ctx is None:
return
try:
pending = await repository.load_cursor(ctx.pool, ctx.account_id)
cursor = pending or saved.box
if cursor is None:
logger.info(f"Account {ctx.account_id}: no saved update state to catch up")
return
if time.time() - cursor.date > MAX_AGE_SECONDS:
logger.warning(
f"[yellow]Account {ctx.account_id}: last update seen too long ago "
f"(pts={cursor.pts}), not catching up[/]"
)
await repository.clear_cursor(ctx.pool, ctx.account_id)
return
await repository.save_cursor(ctx.pool, ctx.account_id, cursor)
logger.info(
f"Account {ctx.account_id}: catching up from pts={cursor.pts}"
f"{' (resumed)' if pending else ''}"
)
walk = await _walk_box(client, ctx, cursor)
if not walk.done:
logger.warning(
f"[yellow]Account {ctx.account_id}: catch-up unfinished at "
f"pts={walk.cursor.pts},[/] {walk.updates} updates replayed, "
f"resumes on the next start"
)
return
from_channels = await _walk_channels(client, ctx, walk, saved.channels)
await repository.clear_cursor(ctx.pool, ctx.account_id)
logger.info(
f"[green]Account {ctx.account_id}: caught up,[/] {walk.updates} updates "
f"from the main box, {from_channels} from channels"
)
except Exception:
logger.exception(
f"[red]Account {ctx.account_id}: catch-up failed,[/] "
f"resumes on the next start"
)
backend/src/userbot/modules/groups/repository.py
+3 -43
View File
@@ -7,23 +7,7 @@ INSERT INTO chat_history
(account_id, chat_id, message_id, event, title, photo_unique_id, actor_id,
ts, raw)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb)
ON CONFLICT DO NOTHING
"""
_INSERT_CHAT_META = """
INSERT INTO chat_history
(account_id, chat_id, message_id, event, title, photo_unique_id, ts, raw)
SELECT $1, $2, $3, 'meta', $4::text, $5::text, $6, $7::jsonb
WHERE ($4::text, $5::text) IS DISTINCT FROM (
(SELECT title FROM chat_history
WHERE account_id = $1 AND chat_id = $2 AND event IN ('meta', 'title')
ORDER BY ts DESC LIMIT 1),
(SELECT photo_unique_id FROM chat_history
WHERE account_id = $1 AND chat_id = $2
AND event IN ('meta', 'photo', 'photo_deleted')
ORDER BY ts DESC LIMIT 1)
)
ON CONFLICT DO NOTHING
ON CONFLICT (account_id, chat_id, message_id) DO NOTHING
"""
_INSERT_MEMBERSHIP = """
@@ -35,14 +19,13 @@ ON CONFLICT (account_id, chat_id, message_id, user_id) DO NOTHING
_LATEST_TITLE = """
SELECT title FROM chat_history
WHERE account_id = $1 AND chat_id = $2 AND event IN ('meta', 'title')
WHERE account_id = $1 AND chat_id = $2 AND title IS NOT NULL
ORDER BY ts DESC LIMIT 1
"""
_LATEST_PHOTO = """
SELECT photo_unique_id FROM chat_history
WHERE account_id = $1 AND chat_id = $2
AND event IN ('meta', 'photo', 'photo_deleted')
WHERE account_id = $1 AND chat_id = $2 AND photo_unique_id IS NOT NULL
ORDER BY ts DESC LIMIT 1
"""
@@ -81,29 +64,6 @@ async def insert_chat_history( # noqa: PLR0913
)
async def record_chat_meta( # noqa: PLR0913
pool: asyncpg.Pool,
account_id: int,
chat_id: int,
message_id: int,
title: str | None,
photo_unique_id: str | None,
ts: datetime,
raw: str,
) -> bool:
status = await pool.execute(
_INSERT_CHAT_META,
account_id,
chat_id,
message_id,
title,
photo_unique_id,
ts,
raw,
)
return status == "INSERT 0 1"
async def insert_membership( # noqa: PLR0913
pool: asyncpg.Pool,
account_id: int,
backend/src/userbot/modules/jobs/consumer.py
+26 -40
View File
@@ -9,12 +9,11 @@ from userbot.modules.jobs import repository
from userbot.modules.jobs.context import JobContext
from userbot.modules.jobs.registry import JOB_HANDLERS, JobHandler
from utils.db.models import Job, JobStatus
from utils.listen import Listener
from utils.env import env
from utils.logging import logger
JOBS_CHANGED_CHANNEL = "jobs_changed"
POLL_INTERVAL_SECONDS = 60.0
RESTART_SECONDS = 10.0
class JobConsumer:
@@ -39,20 +38,25 @@ class JobConsumer:
self.pool, ctx.job_id, JobStatus.FAILED, f"no handler for {job.kind!r}"
)
return
try:
await handler(ctx)
except (FloodWait, FloodPremiumWait) as exc:
wait = exc.value if isinstance(exc.value, int) else 1
await repository.defer_flood_wait(self.pool, ctx.job_id, wait)
logger.warning(
f"[yellow]FloodWait {wait}s on job {ctx.job_id} ({job.kind}), "
f"deferred[/]"
)
except Exception as exc:
logger.exception(f"Job {ctx.job_id} ({job.kind}) failed")
await repository.finish(self.pool, ctx.job_id, JobStatus.FAILED, str(exc))
else:
await repository.finish(self.pool, ctx.job_id, JobStatus.DONE)
while True:
try:
await handler(ctx)
except (FloodWait, FloodPremiumWait) as exc:
await repository.bump_flood_wait(self.pool, ctx.job_id)
wait = exc.value if isinstance(exc.value, int) else 1
logger.warning(
f"[yellow]FloodWait {wait}s on job {ctx.job_id} ({job.kind})[/]"
)
await asyncio.sleep(wait)
except Exception as exc:
logger.exception(f"Job {ctx.job_id} ({job.kind}) failed")
await repository.finish(
self.pool, ctx.job_id, JobStatus.FAILED, str(exc)
)
return
else:
await repository.finish(self.pool, ctx.job_id, JobStatus.DONE)
return
async def drain(self) -> None:
while True:
@@ -62,41 +66,23 @@ class JobConsumer:
await self.execute(job)
async def run(self) -> None:
while True:
try:
await self._serve()
except Exception:
logger.exception(
f"[red]Job consumer for account {self.account_id} failed,[/] "
f"restarting in {RESTART_SECONDS:.0f}s"
)
await asyncio.sleep(RESTART_SECONDS)
async def _serve(self) -> None:
requeued = await repository.requeue_running(self.pool, self.account_id)
if requeued:
logger.info(f"[yellow]Requeued {requeued} stale running job(s).[/]")
pruned = await repository.prune_finished(self.pool, self.account_id)
if pruned:
logger.info(f"[yellow]Pruned {pruned} finished job(s).[/]")
listener = Listener(
f"jobs {self.account_id}",
{JOBS_CHANGED_CHANNEL: lambda _payload: self._wake.set()},
self._wake.set,
)
await listener.start()
conn = await asyncpg.connect(dsn=env.db.connection_url)
await conn.add_listener(JOBS_CHANGED_CHANNEL, lambda *_: self._wake.set())
logger.info(f"[green]Job consumer running for account {self.account_id}.[/]")
try:
while True:
self._wake.clear()
await self.drain()
self._wake.clear()
with contextlib.suppress(TimeoutError):
await asyncio.wait_for(
self._wake.wait(), timeout=await self._idle_seconds()
self._wake.wait(), timeout=POLL_INTERVAL_SECONDS
)
finally:
await listener.stop()
async def _idle_seconds(self) -> float:
due = await repository.seconds_to_next_due(self.pool, self.account_id)
return POLL_INTERVAL_SECONDS if due is None else min(due, POLL_INTERVAL_SECONDS)
with contextlib.suppress(Exception):
await conn.close()
backend/src/userbot/modules/jobs/handlers/__init__.py
-2
View File
@@ -5,7 +5,6 @@ from userbot.modules.jobs.handlers import (
fetch_avatar,
fetch_custom_emoji,
fetch_media,
fetch_read_info,
search_peers,
sync_contacts,
sync_dialogs,
@@ -19,7 +18,6 @@ __all__ = [
"fetch_avatar",
"fetch_custom_emoji",
"fetch_media",
"fetch_read_info",
"search_peers",
"sync_contacts",
"sync_dialogs",
backend/src/userbot/modules/jobs/handlers/enrich_chat.py
+39 -5
View File
@@ -1,11 +1,17 @@
from datetime import UTC, datetime
from pyrogram import Client
from pyrogram.errors import BadRequest, Forbidden
from pyrogram.types import User
from userbot.modules.avatars import note_avatar
from userbot.modules.capture.context import CaptureContext
from userbot.modules.groups.repository import insert_chat_history
from userbot.modules.jobs.context import JobContext
from userbot.modules.jobs.registry import register
from userbot.modules.profiles.snapshots import save_group, save_user
from userbot.modules.profiles.parse import snapshot_from_high_level
from userbot.modules.profiles.repository import write_profile
from userbot.modules.raw_json import raw_json
MEMBER_CAP = 200
@@ -19,17 +25,45 @@ LIMIT 100
"""
async def _save_user(ctx: CaptureContext, user: User) -> None:
fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
await write_profile(ctx.pool, ctx.account_id, user.id, fields, raw_json(user))
if photo_file_id and photo_unique_id:
await note_avatar(
ctx.pool, ctx.account_id, user.id, "peer", photo_unique_id, photo_file_id
)
async def _enrich_chat_meta(client: Client, ctx: CaptureContext, chat_id: int) -> None:
chat = await client.get_chat(chat_id)
if chat is not None:
await save_group(ctx, chat)
if chat is None:
return
photo = chat.photo
photo_unique_id = photo.big_photo_unique_id if photo else None
photo_file_id = photo.big_file_id if photo else None
await insert_chat_history(
ctx.pool,
ctx.account_id,
chat_id,
0,
"meta",
chat.title,
photo_unique_id,
None,
datetime.now(UTC),
raw_json(chat),
)
if photo_file_id and photo_unique_id:
await note_avatar(
ctx.pool, ctx.account_id, chat_id, "chat", photo_unique_id, photo_file_id
)
async def _enrich_members(client: Client, ctx: CaptureContext, chat_id: int) -> None:
try:
async for member in client.get_chat_members(chat_id, limit=MEMBER_CAP):
if isinstance(member.user, User):
await save_user(ctx, member.user)
await _save_user(ctx, member.user)
except (BadRequest, Forbidden):
return
@@ -55,7 +89,7 @@ async def _enrich_senders(client: Client, ctx: CaptureContext, chat_id: int) ->
if not ids:
return
for user in await _fetch_users(client, ids):
await save_user(ctx, user)
await _save_user(ctx, user)
@register("enrich_chat")
backend/src/userbot/modules/jobs/handlers/fetch_read_info.py
-124
View File
@@ -1,124 +0,0 @@
from datetime import UTC, datetime, timedelta
from pyrogram import Client, raw
from pyrogram.errors import (
ChannelInvalid,
ChannelPrivate,
ChatTooBig,
FloodPremiumWait,
FloodWait,
MessageIdInvalid,
MessageNotReadYet,
MessageTooOld,
MsgIdInvalid,
MsgTooOld,
PeerIdInvalid,
RPCError,
UserPrivacyRestricted,
YourPrivacyRestricted,
)
from userbot.modules.jobs.context import JobContext
from userbot.modules.jobs.registry import register
from userbot.modules.read_receipts.repository import save_fetch, save_reads
from utils.jobs import enqueue
from utils.logging import logger
from utils.policy.models import ChatKind
from utils.read.peers import get_peers
from utils.read.read_receipts import read_info
RETRY = timedelta(seconds=60)
LATER = timedelta(days=1)
Refusal = tuple[tuple[type[Exception], ...], str, timedelta]
REFUSALS: tuple[Refusal, ...] = (
((MessageNotReadYet,), "not_read_yet", RETRY),
((UserPrivacyRestricted,), "user_privacy", RETRY),
((YourPrivacyRestricted,), "your_privacy", LATER),
((ChatTooBig,), "chat_too_big", LATER),
((MsgTooOld, MessageTooOld), "too_old", LATER),
(
(
PeerIdInvalid,
ChannelInvalid,
ChannelPrivate,
MessageIdInvalid,
MsgIdInvalid,
KeyError,
ValueError,
),
"unavailable",
RETRY,
),
)
def _refusal(exc: Exception) -> tuple[str, timedelta]:
for kinds, outcome, retry in REFUSALS:
if isinstance(exc, kinds):
return outcome, retry
return "error", RETRY
def _at(timestamp: int) -> datetime:
return datetime.fromtimestamp(timestamp, UTC)
async def ask_telegram(
client: Client, chat_id: int, message_id: int, *, group: bool
) -> list[tuple[int, datetime]]:
peer = await client.resolve_peer(chat_id)
if peer is None:
msg = f"unknown peer {chat_id}"
raise ValueError(msg)
if group:
participants = await client.invoke(
raw.functions.messages.GetMessageReadParticipants(
peer=peer, msg_id=message_id
)
)
return [(item.user_id, _at(item.date)) for item in participants]
outbox = await client.invoke(
raw.functions.messages.GetOutboxReadDate(peer=peer, msg_id=message_id)
)
return [(chat_id, _at(outbox.date))]
async def _enrich_unknown(ctx: JobContext, chat_id: int, reader_ids: list[int]) -> None:
known = {
peer.peer_id for peer in await get_peers(ctx.pool, ctx.account_id, reader_ids)
}
if set(reader_ids) - known:
await enqueue(ctx.pool, ctx.account_id, "enrich_chat", {"chat_id": chat_id})
@register("fetch_read_info")
async def fetch_read_info(ctx: JobContext) -> None:
client = ctx.client
if client is None:
return
chat_id = ctx.job.params["chat_id"]
message_id = ctx.job.params["message_id"]
info = await read_info(ctx.pool, ctx.account_id, chat_id, message_id)
if info is None or not info.can_fetch:
return
group = info.chat_kind == ChatKind.GROUP
try:
reads = await ask_telegram(client, chat_id, message_id, group=group)
except (FloodWait, FloodPremiumWait) as exc:
wait = exc.value if isinstance(exc.value, int) else 0
retry = max(RETRY, timedelta(seconds=wait))
await save_fetch(ctx.pool, ctx.account_id, chat_id, message_id, "flood", retry)
return
except (RPCError, KeyError, ValueError) as exc:
outcome, retry = _refusal(exc)
if outcome == "error":
logger.warning(f"[yellow]Read info {chat_id}/{message_id}: {exc!r}[/]")
await save_fetch(ctx.pool, ctx.account_id, chat_id, message_id, outcome, retry)
return
source = "participants" if group else "outbox_date"
await save_reads(ctx.pool, ctx.account_id, chat_id, message_id, source, reads)
await save_fetch(ctx.pool, ctx.account_id, chat_id, message_id, "ok", RETRY)
if group and reads:
await _enrich_unknown(ctx, chat_id, [reader_id for reader_id, _ in reads])
backend/src/userbot/modules/jobs/handlers/sync_contacts.py
+15 -2
View File
@@ -1,8 +1,11 @@
from pyrogram.types import User
from userbot.modules.avatars import note_avatar
from userbot.modules.jobs.context import JobContext
from userbot.modules.jobs.registry import register
from userbot.modules.profiles.snapshots import save_user
from userbot.modules.profiles.parse import snapshot_from_high_level
from userbot.modules.profiles.repository import write_profile
from userbot.modules.raw_json import raw_json
@register("sync_contacts")
@@ -18,7 +21,17 @@ async def sync_contacts(ctx: JobContext) -> None:
for user in contacts:
if not isinstance(user, User):
continue
await save_user(capture, user)
fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
await write_profile(ctx.pool, ctx.account_id, user.id, fields, raw_json(user))
if photo_file_id and photo_unique_id:
await note_avatar(
ctx.pool,
ctx.account_id,
user.id,
"peer",
photo_unique_id,
photo_file_id,
)
processed += 1
await capture.contacts.refresh()
await ctx.report_progress({"processed": processed, "done": True})
backend/src/userbot/modules/jobs/handlers/sync_dialogs.py
+20 -3
View File
@@ -2,10 +2,14 @@ from pyrogram import Client
from pyrogram.errors import BadRequest, Forbidden
from pyrogram.types import User
from userbot.modules.avatars import note_avatar
from userbot.modules.capture.context import CaptureContext
from userbot.modules.jobs.context import JobContext
from userbot.modules.jobs.registry import register
from userbot.modules.profiles.snapshots import save_group, save_private, save_user
from userbot.modules.profiles.parse import snapshot_from_high_level
from userbot.modules.profiles.repository import write_profile
from userbot.modules.profiles.snapshots import save_group, save_private
from userbot.modules.raw_json import raw_json
from utils.logging import logger
SAVE_EVERY = 100
@@ -26,8 +30,21 @@ async def _enrich_users(client: Client, ctx: CaptureContext, ids: list[int]) ->
continue
users = result if isinstance(result, list) else [result]
for user in users:
if isinstance(user, User):
await save_user(ctx, user)
if not isinstance(user, User):
continue
fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
await write_profile(
ctx.pool, ctx.account_id, user.id, fields, raw_json(user)
)
if photo_file_id and photo_unique_id:
await note_avatar(
ctx.pool,
ctx.account_id,
user.id,
"peer",
photo_unique_id,
photo_file_id,
)
@register("sync_dialogs")
backend/src/userbot/modules/jobs/repository.py
+3 -14
View File
@@ -32,7 +32,6 @@ async def claim_next(pool: asyncpg.Pool, account_id: int) -> Job | None:
"updated_at = now(), attempts = attempts + 1 "
"WHERE id = ("
" SELECT id FROM jobs WHERE account_id = $1 AND status = 'pending' "
" AND (run_after IS NULL OR run_after <= now()) "
" ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1"
") RETURNING *",
account_id,
@@ -77,21 +76,11 @@ async def report_progress(
)
async def defer_flood_wait(pool: asyncpg.Pool, job_id: int, seconds: int) -> None:
async def bump_flood_wait(pool: asyncpg.Pool, job_id: int) -> None:
await pool.execute(
"UPDATE jobs SET status = 'pending', flood_waits = flood_waits + 1, "
"run_after = now() + make_interval(secs => $2), updated_at = now() "
"WHERE id = $1 AND status = 'running'",
"UPDATE jobs SET flood_waits = flood_waits + 1, updated_at = now() "
"WHERE id = $1",
job_id,
seconds,
)
async def seconds_to_next_due(pool: asyncpg.Pool, account_id: int) -> float | None:
return await pool.fetchval(
"SELECT extract(epoch FROM min(run_after) - now())::float FROM jobs "
"WHERE account_id = $1 AND status = 'pending' AND run_after > now()",
account_id,
)
backend/src/userbot/modules/links/repository.py
-8
View File
@@ -7,14 +7,6 @@ INSERT INTO links
(account_id, chat_id, message_id, position, url, kind, web_url, web_title,
web_description, web_site_name, raw)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::jsonb)
ON CONFLICT (account_id, chat_id, message_id, position) DO UPDATE SET
url = EXCLUDED.url,
kind = EXCLUDED.kind,
web_url = EXCLUDED.web_url,
web_title = EXCLUDED.web_title,
web_description = EXCLUDED.web_description,
web_site_name = EXCLUDED.web_site_name,
raw = EXCLUDED.raw
"""
backend/src/userbot/modules/profiles/parse.py
+3 -9
View File
@@ -1,4 +1,4 @@
from dataclasses import dataclass, field
from dataclasses import dataclass
from pyrogram import Client, raw
from pyrogram.types import Chat, User
@@ -12,11 +12,6 @@ class ProfileFields:
phone: str | None = None
photo_unique_id: str | None = None
is_deleted_account: bool = False
unknown: frozenset[str] = field(default=frozenset(), compare=False)
MIN_UNKNOWN = frozenset({"username", "phone"})
CHAT_UNKNOWN = frozenset({"phone", "is_deleted_account"})
def active_username(usernames: list[raw.types.Username]) -> str | None:
@@ -40,7 +35,6 @@ async def snapshot_from_user(
phone=user.phone_number if user else None,
photo_unique_id=photo_unique_id,
is_deleted_account=bool(getattr(raw_user, "deleted", False)),
unknown=MIN_UNKNOWN if getattr(raw_user, "min", False) else frozenset(),
)
return fields, photo_file_id, photo_unique_id
@@ -58,7 +52,6 @@ def snapshot_from_high_level(
phone=user.phone_number,
photo_unique_id=photo_unique_id,
is_deleted_account=bool(user.is_deleted),
unknown=MIN_UNKNOWN if user.is_min else frozenset(),
)
return fields, photo_file_id, photo_unique_id
@@ -71,7 +64,8 @@ def snapshot_from_chat(chat: Chat) -> tuple[ProfileFields, str | None, str | Non
first_name=chat.first_name,
last_name=chat.last_name,
username=chat.username,
phone=None,
photo_unique_id=photo_unique_id,
unknown=CHAT_UNKNOWN,
is_deleted_account=False,
)
return fields, photo_file_id, photo_unique_id
backend/src/userbot/modules/profiles/repository.py
+20 -52
View File
@@ -9,46 +9,26 @@ WHERE account_id = $1 AND peer_id = $2
"""
_UPSERT_PEER = """
INSERT INTO peers AS p
INSERT INTO peers
(account_id, peer_id, first_name, last_name, username, phone,
photo_unique_id, is_deleted_account, raw, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, now())
ON CONFLICT (account_id, peer_id) DO UPDATE SET
first_name = EXCLUDED.first_name,
last_name = EXCLUDED.last_name,
username = CASE WHEN 'username' = ANY($10::text[])
THEN COALESCE(EXCLUDED.username, p.username) ELSE EXCLUDED.username END,
phone = CASE WHEN 'phone' = ANY($10::text[])
THEN COALESCE(EXCLUDED.phone, p.phone) ELSE EXCLUDED.phone END,
username = EXCLUDED.username,
phone = EXCLUDED.phone,
photo_unique_id = EXCLUDED.photo_unique_id,
is_deleted_account = CASE WHEN 'is_deleted_account' = ANY($10::text[])
THEN p.is_deleted_account ELSE EXCLUDED.is_deleted_account END,
is_deleted_account = EXCLUDED.is_deleted_account,
raw = EXCLUDED.raw,
updated_at = now()
"""
_INSERT_CHANGE = """
_INSERT_HISTORY = """
INSERT INTO peer_history
(account_id, peer_id, observed_at, first_name, last_name, username, phone,
photo_unique_id, is_deleted_account, raw)
SELECT account_id, peer_id, clock_timestamp(), first_name, last_name, username,
phone, photo_unique_id, is_deleted_account, raw
FROM peers p
WHERE account_id = $1 AND peer_id = $2
AND NOT EXISTS (
SELECT 1 FROM (
SELECT first_name, last_name, username, phone, photo_unique_id,
is_deleted_account
FROM peer_history
WHERE account_id = $1 AND peer_id = $2
ORDER BY observed_at DESC LIMIT 1
) latest
WHERE (latest.first_name, latest.last_name, latest.username, latest.phone,
latest.photo_unique_id, latest.is_deleted_account)
IS NOT DISTINCT FROM
(p.first_name, p.last_name, p.username, p.phone,
p.photo_unique_id, p.is_deleted_account)
)
VALUES ($1, $2, now(), $3, $4, $5, $6, $7, $8, $9::jsonb)
ON CONFLICT DO NOTHING
"""
@@ -69,30 +49,18 @@ async def get_peer(
)
async def write_profile( # noqa: PLR0913
pool: asyncpg.Pool,
account_id: int,
peer_id: int,
fields: ProfileFields,
raw: str,
*,
history: bool,
) -> bool:
async def write_profile(
pool: asyncpg.Pool, account_id: int, peer_id: int, fields: ProfileFields, raw: str
) -> None:
args = (
fields.first_name,
fields.last_name,
fields.username,
fields.phone,
fields.photo_unique_id,
fields.is_deleted_account,
raw,
)
async with pool.acquire() as conn, conn.transaction():
await conn.execute(
_UPSERT_PEER,
account_id,
peer_id,
fields.first_name,
fields.last_name,
fields.username,
fields.phone,
fields.photo_unique_id,
fields.is_deleted_account,
raw,
sorted(fields.unknown),
)
if not history:
return False
status = await conn.execute(_INSERT_CHANGE, account_id, peer_id)
return status == "INSERT 0 1"
await conn.execute(_UPSERT_PEER, account_id, peer_id, *args)
await conn.execute(_INSERT_HISTORY, account_id, peer_id, *args)
backend/src/userbot/modules/profiles/snapshots.py
+7 -32
View File
@@ -1,46 +1,19 @@
from datetime import UTC, datetime
from pyrogram.enums import ChatType
from pyrogram.types import Chat, User
from pyrogram.types import Chat
from userbot.modules.avatars import note_avatar
from userbot.modules.capture.context import CaptureContext
from userbot.modules.groups.repository import record_chat_meta
from userbot.modules.profiles.parse import snapshot_from_chat, snapshot_from_high_level
from userbot.modules.groups.repository import insert_chat_history
from userbot.modules.profiles.parse import snapshot_from_chat
from userbot.modules.profiles.repository import write_profile
from userbot.modules.raw_json import raw_json
async def save_user(ctx: CaptureContext, user: User) -> None:
fields, photo_file_id, photo_unique_id = snapshot_from_high_level(user)
if not await ctx.peer_identity.changed(ctx.pool, ctx.account_id, user.id, fields):
return
await write_profile(
ctx.pool,
ctx.account_id,
user.id,
fields,
raw_json(user),
history=ctx.resolve_peer(user.id, is_bot=bool(user.is_bot)).profile_history,
)
if photo_file_id and photo_unique_id:
await note_avatar(
ctx.pool, ctx.account_id, user.id, "peer", photo_unique_id, photo_file_id
)
async def save_private(ctx: CaptureContext, chat: Chat) -> bool:
chat_id = chat.id or 0
fields, photo_file_id, photo_unique_id = snapshot_from_chat(chat)
is_bot = chat.type is ChatType.BOT
await write_profile(
ctx.pool,
ctx.account_id,
chat_id,
fields,
raw_json(chat),
history=ctx.resolve_peer(chat_id, is_bot=is_bot).profile_history,
)
await write_profile(ctx.pool, ctx.account_id, chat_id, fields, raw_json(chat))
if photo_file_id and photo_unique_id:
await note_avatar(
ctx.pool, ctx.account_id, chat_id, "peer", photo_unique_id, photo_file_id
@@ -53,13 +26,15 @@ async def save_group(ctx: CaptureContext, chat: Chat) -> None:
photo = chat.photo
photo_unique_id = photo.big_photo_unique_id if photo else None
photo_file_id = photo.big_file_id if photo else None
await record_chat_meta(
await insert_chat_history(
ctx.pool,
ctx.account_id,
chat_id,
0,
"meta",
chat.title,
photo_unique_id,
None,
datetime.now(UTC),
raw_json(chat),
)
backend/src/userbot/modules/read_receipts/repository.py
+1 -51
View File
@@ -1,4 +1,4 @@
from datetime import datetime, timedelta
from datetime import datetime
import asyncpg
@@ -9,26 +9,6 @@ VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
ON CONFLICT (account_id, chat_id, message_id, ts) DO NOTHING
"""
_UPSERT_READ = """
INSERT INTO message_reads
(account_id, chat_id, message_id, reader_id, read_at, source, fetched_at)
VALUES ($1, $2, $3, $4, $5, $6, now())
ON CONFLICT (account_id, chat_id, message_id, reader_id) DO UPDATE SET
read_at = EXCLUDED.read_at,
source = EXCLUDED.source,
fetched_at = EXCLUDED.fetched_at
"""
_UPSERT_FETCH = """
INSERT INTO read_fetches
(account_id, chat_id, message_id, outcome, fetched_at, retry_at)
VALUES ($1, $2, $3, $4, now(), now() + $5::interval)
ON CONFLICT (account_id, chat_id, message_id) DO UPDATE SET
outcome = EXCLUDED.outcome,
fetched_at = EXCLUDED.fetched_at,
retry_at = EXCLUDED.retry_at
"""
async def insert_read( # noqa: PLR0913
pool: asyncpg.Pool,
@@ -54,33 +34,3 @@ async def insert_listened( # noqa: PLR0913
await pool.execute(
_INSERT, account_id, chat_id, message_id, ts, reader_id, "listened", raw
)
async def save_reads( # noqa: PLR0913
pool: asyncpg.Pool,
account_id: int,
chat_id: int,
message_id: int,
source: str,
reads: list[tuple[int, datetime]],
) -> None:
await pool.executemany(
_UPSERT_READ,
[
(account_id, chat_id, message_id, reader_id, read_at, source)
for reader_id, read_at in reads
],
)
async def save_fetch( # noqa: PLR0913
pool: asyncpg.Pool,
account_id: int,
chat_id: int,
message_id: int,
outcome: str,
retry_after: timedelta,
) -> None:
await pool.execute(
_UPSERT_FETCH, account_id, chat_id, message_id, outcome, retry_after
)
backend/src/userbot/modules/uptime/__init__.py
View File
Whitespace-only changes.
backend/src/userbot/modules/uptime/repository.py
-16
View File
@@ -1,16 +0,0 @@
import asyncpg
_HEARTBEAT = """
WITH extended AS (
UPDATE capture_uptime SET seen_at = now()
WHERE account_id = $1 AND seen_at >= now() - interval '150 seconds'
RETURNING account_id
)
INSERT INTO capture_uptime (account_id, started_at, seen_at)
SELECT $1, now(), now()
WHERE NOT EXISTS (SELECT 1 FROM extended)
"""
async def heartbeat(pool: asyncpg.Pool, account_id: int) -> None:
await pool.execute(_HEARTBEAT, account_id)
backend/src/userbot/modules/watches/cache.py
+34 -156
View File
@@ -1,182 +1,60 @@
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING
from userbot.modules.watches import repository
from userbot.modules.watches.evaluator import (
Rules,
Seen,
build_rules,
match_chat_scope,
KIND_KEYWORD,
KIND_PEER_ONLINE,
match_keyword,
match_new_message,
match_peer_online,
)
from utils.logging import logger
from utils.read import watches as watches_read
from utils.read.accounts import self_user_id
if TYPE_CHECKING:
from collections.abc import Coroutine
import asyncpg
ONLINE_QUIET = timedelta(minutes=15)
PREVIEW_CHARS = 500
def _message_payload(event: str, seen: Seen) -> dict[str, Any]:
payload: dict[str, Any] = {
"event": event,
"chat_id": seen.chat_id,
"message_id": seen.message_id,
"sender_id": seen.sender_id,
"text": (seen.text or "")[:PREVIEW_CHARS] or None,
}
if seen.date is not None:
payload["date"] = seen.date.isoformat()
return payload
from utils.read.models import WatchView
class WatchCache:
def __init__(self, pool: asyncpg.Pool, account_id: int) -> None:
self._pool = pool
self._account_id = account_id
self.rules = Rules()
self._self_id: int | None = None
self._last_seen: dict[int, datetime] = {}
self.watches: list[WatchView] = []
self._online: set[int] = set()
async def refresh(self) -> None:
rows = await watches_read.list_watches(self._pool, self._account_id)
self.rules = build_rules(rows)
self._self_id = await self_user_id(self._pool, self._account_id)
logger.info(f"[green]Watches cached:[/] {len(self.rules)}")
self.watches = [watch for watch in rows if watch.enabled]
logger.info(f"[green]Watches cached:[/] {len(self.watches)}")
def _own(self, seen: Seen) -> bool:
return seen.outgoing or (
self._self_id is not None and seen.sender_id == self._self_id
)
async def _guard(self, work: Coroutine[Any, Any, None]) -> None:
try:
await work
except Exception:
logger.exception("[red]Watch evaluation failed[/]")
async def _fire(self, watch_id: int, payload: dict[str, Any], key: str) -> None:
await watches_read.insert_alert(
self._pool, self._account_id, watch_id, payload, dedup_key=key
)
async def _keyword(self, event: str, seen: Seen) -> None:
for rule in self.rules.keyword:
if match_keyword(rule, seen):
await self._fire(
rule.watch_id,
_message_payload(event, seen),
f"{seen.chat_id}:{seen.message_id}",
async def on_text(self, chat_id: int, message_id: int, text: str | None) -> None:
if not text:
return
for watch in self.watches:
if watch.kind == KIND_KEYWORD and match_keyword(
text, chat_id, watch.params
):
await watches_read.insert_alert(
self._pool,
self._account_id,
watch.id,
{"chat_id": chat_id, "message_id": message_id},
dedup_key=f"{chat_id}:{message_id}",
)
async def on_message(self, seen: Seen) -> None:
if self._own(seen):
async def on_status(self, peer_id: int, *, is_online: bool) -> None:
if not is_online:
self._online.discard(peer_id)
return
await self._guard(self._on_message(seen))
async def _on_message(self, seen: Seen) -> None:
await self._keyword("message", seen)
for watch_id, params in self.rules.new_message:
if match_new_message(params, seen):
await self._fire(
watch_id,
_message_payload("message", seen),
f"{seen.chat_id}:{seen.message_id}",
if peer_id in self._online:
return
self._online.add(peer_id)
for watch in self.watches:
if watch.kind == KIND_PEER_ONLINE and match_peer_online(
peer_id, watch.params
):
await watches_read.insert_alert(
self._pool, self._account_id, watch.id, {"peer_id": peer_id}
)
async def on_edit(self, seen: Seen) -> None:
if self._own(seen):
return
await self._guard(self._on_edit(seen))
async def _on_edit(self, seen: Seen) -> None:
await self._keyword("edit", seen)
if seen.edit_date is None:
return
stamp = int(seen.edit_date.timestamp())
for watch_id, params in self.rules.edited:
if match_chat_scope(params, seen):
await self._fire(
watch_id,
_message_payload("edit", seen),
f"{seen.chat_id}:{seen.message_id}:{stamp}",
)
async def on_transcript(self, chat_id: int, message_id: int, text: str) -> None:
if self.rules.keyword:
await self._guard(self._on_transcript(chat_id, message_id, text))
async def _on_transcript(self, chat_id: int, message_id: int, text: str) -> None:
sender_id = await repository.message_sender(
self._pool, self._account_id, chat_id, message_id
)
seen = Seen(chat_id, message_id, sender_id, text)
if not self._own(seen):
await self._keyword("transcript", seen)
async def on_deleted(self, chat_id: int | None, message_ids: list[int]) -> None:
if self.rules.deleted and message_ids:
await self._guard(self._on_deleted(chat_id, message_ids))
async def _on_deleted(self, chat_id: int | None, message_ids: list[int]) -> None:
rows = await repository.deleted_messages(
self._pool, self._account_id, chat_id, message_ids
)
for row in rows:
seen = Seen(
row["chat_id"],
row["message_id"],
row["sender_id"],
row["text"],
date=row["date"],
)
if self._own(seen):
continue
for watch_id, params in self.rules.deleted:
if match_chat_scope(params, seen):
await self._fire(
watch_id,
_message_payload("delete", seen),
f"{seen.chat_id}:{seen.message_id}",
)
async def on_status(
self, peer_id: int, status: str, at: datetime, last_online: datetime | None
) -> None:
watch_ids = [
watch_id
for watch_id, params in self.rules.online
if params.peer_id == peer_id
]
if not watch_ids:
return
if status != "online":
self._last_seen[peer_id] = (last_online or at).astimezone(UTC)
return
await self._guard(self._on_online(peer_id, at, watch_ids))
async def _on_online(
self, peer_id: int, at: datetime, watch_ids: list[int]
) -> None:
stored = await repository.last_seen_before(
self._pool, self._account_id, peer_id, at
)
known = [value for value in (stored, self._last_seen.get(peer_id)) if value]
previous = max(known, default=None)
self._last_seen[peer_id] = at
if previous is not None and at - previous < ONLINE_QUIET:
return
session = int((previous or at).timestamp())
payload: dict[str, Any] = {"event": "online", "peer_id": peer_id}
if previous is not None:
payload["since"] = previous.isoformat()
for watch_id in watch_ids:
await self._fire(watch_id, payload, f"{peer_id}:{session}")
backend/src/userbot/modules/watches/evaluator.py
+17 -97
View File
@@ -1,104 +1,24 @@
import re
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any
from utils.logging import logger
from utils.read.models import WatchView
from utils.watch_rules import (
MAX_TEXT,
DeletedParams,
EditedParams,
KeywordParams,
NewMessageParams,
PeerOnlineParams,
WatchParamsError,
compile_pattern,
parse_params,
)
KIND_KEYWORD = "keyword"
KIND_PEER_ONLINE = "peer_online"
@dataclass(frozen=True, slots=True)
class Seen:
chat_id: int
message_id: int
sender_id: int | None
text: str | None
outgoing: bool = False
date: datetime | None = None
edit_date: datetime | None = None
@dataclass(frozen=True, slots=True)
class KeywordRule:
watch_id: int
params: KeywordParams
regex: re.Pattern[str] | None
@dataclass(slots=True)
class Rules:
keyword: list[KeywordRule] = field(default_factory=list)
new_message: list[tuple[int, NewMessageParams]] = field(default_factory=list)
deleted: list[tuple[int, DeletedParams]] = field(default_factory=list)
edited: list[tuple[int, EditedParams]] = field(default_factory=list)
online: list[tuple[int, PeerOnlineParams]] = field(default_factory=list)
def __len__(self) -> int:
return (
len(self.keyword)
+ len(self.new_message)
+ len(self.deleted)
+ len(self.edited)
+ len(self.online)
)
def build_rules(watches: list[WatchView]) -> Rules:
rules = Rules()
for watch in watches:
if not watch.enabled:
continue
def match_keyword(text: str, chat_id: int, params: dict[str, Any]) -> bool:
target_chat = params.get("chat_id")
if target_chat is not None and target_chat != chat_id:
return False
pattern = params.get("pattern")
if not pattern:
return False
if params.get("regex"):
try:
params = parse_params(watch.kind, watch.params)
except WatchParamsError as exc:
logger.warning(f"[yellow]Watch {watch.id} skipped:[/] {exc}")
continue
match params:
case KeywordParams():
regex = compile_pattern(params.pattern) if params.regex else None
rules.keyword.append(KeywordRule(watch.id, params, regex))
case NewMessageParams():
rules.new_message.append((watch.id, params))
case DeletedParams():
rules.deleted.append((watch.id, params))
case EditedParams():
rules.edited.append((watch.id, params))
case PeerOnlineParams():
rules.online.append((watch.id, params))
return rules
return re.search(pattern, text) is not None
except re.error:
return False
return pattern.casefold() in text.casefold()
def _in_scope(chat_id: int | None, sender_id: int | None, seen: Seen) -> bool:
if chat_id is not None and chat_id != seen.chat_id:
return False
return sender_id is None or sender_id == seen.sender_id
def match_keyword(rule: KeywordRule, seen: Seen) -> bool:
params = rule.params
if not seen.text or not _in_scope(params.chat_id, params.sender_id, seen):
return False
sample = seen.text[:MAX_TEXT]
if rule.regex is not None:
return rule.regex.search(sample) is not None
return params.pattern.casefold() in sample.casefold()
def match_new_message(params: NewMessageParams, seen: Seen) -> bool:
return _in_scope(params.chat_id, params.sender_id, seen)
def match_chat_scope(params: DeletedParams | EditedParams, seen: Seen) -> bool:
if params.chat_id is None:
return seen.chat_id > 0
return params.chat_id == seen.chat_id
def match_peer_online(peer_id: int, params: dict[str, Any]) -> bool:
return params.get("peer_id") == peer_id
backend/src/userbot/modules/watches/repository.py
-48
View File
@@ -1,48 +0,0 @@
from datetime import datetime
import asyncpg
from userbot.modules.capture.repository import CHANNEL_ID_THRESHOLD
_DELETED = """
SELECT DISTINCT ON (chat_id, message_id) chat_id, message_id, sender_id, text, date
FROM messages
WHERE account_id = $1 AND message_id = ANY($2::bigint[])
AND deleted_at IS NOT NULL AND {scope}
ORDER BY chat_id, message_id, date DESC
"""
_SENDER = """
SELECT sender_id FROM messages
WHERE account_id = $1 AND chat_id = $2 AND message_id = $3
ORDER BY date DESC LIMIT 1
"""
_LAST_SEEN = """
SELECT CASE WHEN status = 'online' THEN ts ELSE coalesce(last_online_date, ts) END
FROM presence
WHERE account_id = $1 AND peer_id = $2 AND ts < $3
ORDER BY ts DESC LIMIT 1
"""
async def deleted_messages(
pool: asyncpg.Pool, account_id: int, chat_id: int | None, message_ids: list[int]
) -> list[asyncpg.Record]:
if chat_id is None:
query = _DELETED.format(scope="chat_id > $3")
return await pool.fetch(query, account_id, message_ids, CHANNEL_ID_THRESHOLD)
query = _DELETED.format(scope="chat_id = $3")
return await pool.fetch(query, account_id, message_ids, chat_id)
async def message_sender(
pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
) -> int | None:
return await pool.fetchval(_SENDER, account_id, chat_id, message_id)
async def last_seen_before(
pool: asyncpg.Pool, account_id: int, peer_id: int, at: datetime
) -> datetime | None:
return await pool.fetchval(_LAST_SEEN, account_id, peer_id, at)
backend/src/userbot/modules/watches/seen.py
-22
View File
@@ -1,22 +0,0 @@
from datetime import UTC, datetime
from pyrogram.types import Message
from userbot.modules.capture.message import sender_id
from userbot.modules.watches.evaluator import Seen
def utc(moment: datetime | None) -> datetime | None:
return moment.astimezone(UTC) if moment else None
def seen_message(chat_id: int, message: Message) -> Seen:
return Seen(
chat_id=chat_id,
message_id=message.id,
sender_id=sender_id(message),
text=message.text or message.caption,
outgoing=bool(message.outgoing),
date=utc(message.date),
edit_date=utc(message.edit_date),
)
backend/src/userbot/runner.py
+29 -66
View File
@@ -11,13 +11,10 @@ import uvloop
from dependencies.container import container
from userbot.modules.capture import CaptureContext, build_capture_context
from userbot.modules.catchup import SavedState, catch_up, read_saved_state
from userbot.modules.client import PyroClient
from userbot.modules.jobs import JobConsumer
from userbot.modules.uptime import repository as uptime
from utils.env import env
from utils.jobs import enqueue
from utils.listen import Listener
from utils.logging import logger, setup_logging
from utils.read.accounts import (
ACCOUNTS_CHANGED_CHANNEL,
@@ -31,14 +28,12 @@ from utils.storage import ContentAddressedStorage
setup_logging()
SESSION_FLUSH_INTERVAL = 60
SESSION_RETRY_INTERVAL = 60
UPTIME_INTERVAL = 60
@dataclass
class RunningAccount:
client: PyroClient
tasks: list[asyncio.Task]
consumer_task: asyncio.Task
device_model: str | None
@@ -59,7 +54,6 @@ class AccountRegistry:
self._pool = pool
self._storage = storage
self._running: dict[str, RunningAccount] = {}
self._saved: dict[str, SavedState] = {}
self._lock = asyncio.Lock()
@property
@@ -92,8 +86,6 @@ class AccountRegistry:
async def _start(self, path: Path, device_model: str | None) -> None:
session_name = path.stem
if session_name not in self._saved:
self._saved[session_name] = await asyncio.to_thread(read_saved_state, path)
client = PyroClient(
session_name, workdir=str(path.parent), device_model=device_model
)
@@ -108,20 +100,14 @@ class AccountRegistry:
client, self._pool, self._storage, account_id
)
except Exception:
logger.exception(
f"[red]Failed to start session:[/] {session_name}, "
f"retrying in {SESSION_RETRY_INTERVAL}s"
)
logger.exception(f"[red]Failed to start session:[/] {session_name}")
with contextlib.suppress(Exception):
await client.stop()
return
consumer = JobConsumer(client, self._pool, account_id)
saved = self._saved.pop(session_name)
tasks = [
asyncio.create_task(consumer.run()),
asyncio.create_task(catch_up(client, saved)),
]
self._running[session_name] = RunningAccount(client, tasks, device_model)
self._running[session_name] = RunningAccount(
client, asyncio.create_task(consumer.run()), device_model
)
logger.info(f"[green]Client started:[/] {me.full_name} ({me.id})")
await enqueue(self._pool, account_id, "sync_dialogs", {})
await enqueue(self._pool, account_id, "sync_contacts", {})
@@ -130,8 +116,7 @@ class AccountRegistry:
account = self._running.pop(session_name, None)
if account is None:
return
for task in account.tasks:
await _cancel(task)
await _cancel(account.consumer_task)
with contextlib.suppress(Exception):
await account.client.stop()
logger.info(f"[yellow]Client stopped:[/] {session_name}")
@@ -139,8 +124,7 @@ class AccountRegistry:
async def _log_out(self, path: Path) -> None:
account = self._running.pop(path.stem, None)
if account is not None:
for task in account.tasks:
await _cancel(task)
await _cancel(account.consumer_task)
client = account.client
else:
client = PyroClient(
@@ -164,25 +148,9 @@ async def _flush_sessions(registry: AccountRegistry) -> None:
await client.storage.save()
async def _retry_sessions(registry: AccountRegistry) -> None:
while True:
await asyncio.sleep(SESSION_RETRY_INTERVAL)
try:
await registry.sync()
except Exception:
logger.exception("[red]Session sync failed[/]")
async def _keep_uptime(registry: AccountRegistry, pool: asyncpg.Pool) -> None:
while True:
for client in registry.clients:
if client.capture is not None and client.is_connected:
with contextlib.suppress(Exception):
await uptime.heartbeat(pool, client.capture.account_id)
await asyncio.sleep(UPTIME_INTERVAL)
def _change_listener(registry: AccountRegistry, tasks: set[asyncio.Task]) -> Listener:
async def _listen_changes(
registry: AccountRegistry, tasks: set[asyncio.Task]
) -> asyncpg.Connection:
def spawn(coro: Coroutine[Any, Any, None]) -> None:
task = asyncio.create_task(coro)
tasks.add(task)
@@ -195,29 +163,26 @@ def _change_listener(registry: AccountRegistry, tasks: set[asyncio.Task]) -> Lis
if client.capture is not None:
spawn(make_coro(client.capture))
def on_policy(_payload: str) -> None:
def on_policy(
_conn: asyncpg.Connection, _pid: int, _channel: str, _payload: str
) -> None:
reload(lambda capture: capture.reload_policies())
def on_watch(_payload: str) -> None:
def on_watch(
_conn: asyncpg.Connection, _pid: int, _channel: str, _payload: str
) -> None:
reload(lambda capture: capture.watches.refresh())
def on_accounts(_payload: str) -> None:
def on_accounts(
_conn: asyncpg.Connection, _pid: int, _channel: str, _payload: str
) -> None:
spawn(registry.sync())
def on_reconnect() -> None:
on_policy("")
on_watch("")
on_accounts("")
return Listener(
"userbot",
{
"policy_changed": on_policy,
WATCHES_CHANGED_CHANNEL: on_watch,
ACCOUNTS_CHANGED_CHANNEL: on_accounts,
},
on_reconnect,
)
conn = await asyncpg.connect(dsn=env.db.connection_url)
await conn.add_listener("policy_changed", on_policy)
await conn.add_listener(WATCHES_CHANGED_CHANNEL, on_watch)
await conn.add_listener(ACCOUNTS_CHANGED_CHANNEL, on_accounts)
return conn
async def runner() -> None:
@@ -226,24 +191,22 @@ async def runner() -> None:
registry = AccountRegistry(pool, storage)
tasks: set[asyncio.Task] = set()
listener = _change_listener(registry, tasks)
listen_conn: asyncpg.Connection | None = None
stop = asyncio.Event()
asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, stop.set)
flush = asyncio.create_task(_flush_sessions(registry))
retry = asyncio.create_task(_retry_sessions(registry))
beat = asyncio.create_task(_keep_uptime(registry, pool))
try:
await registry.sync()
if not registry.clients:
logger.warning("[yellow]No sessions yet. Add an account in the web UI.[/]")
await listener.start()
listen_conn = await _listen_changes(registry, tasks)
logger.info("[green]Userbot running.[/]")
await stop.wait()
finally:
await _cancel(flush)
await _cancel(retry)
await _cancel(beat)
await listener.stop()
if listen_conn is not None:
with contextlib.suppress(Exception):
await listen_conn.close()
await registry.close()
await container.close()
backend/src/utils/db/models.py
+4 -80
View File
@@ -2,17 +2,8 @@ from datetime import datetime
from enum import StrEnum
from typing import Any
from sqlalchemy import (
BigInteger,
Column,
DateTime,
ForeignKey,
Integer,
LargeBinary,
Text,
func,
)
from sqlalchemy.dialects.postgresql import ARRAY, JSONB
from sqlalchemy import BigInteger, Column, DateTime, LargeBinary, Text, func
from sqlalchemy.dialects.postgresql import JSONB
from sqlmodel import Field, SQLModel
@@ -212,9 +203,6 @@ class Job(SQLModel, table=True):
finished_at: datetime | None = Field(
default=None, sa_column=Column(DateTime(timezone=True))
)
run_after: datetime | None = Field(
default=None, sa_column=Column(DateTime(timezone=True))
)
class CapturePolicy(SQLModel, table=True):
@@ -356,7 +344,7 @@ class ChatHistory(SQLModel, table=True):
title: str | None = None
photo_unique_id: str | None = None
actor_id: int | None = Field(default=None, sa_column=Column(BigInteger))
ts: datetime = Field(sa_column=Column(DateTime(timezone=True), primary_key=True))
ts: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=False))
raw: dict[str, Any] = Field(
default_factory=dict, sa_column=Column(JSONB, nullable=False)
)
@@ -454,7 +442,6 @@ class Watch(SQLModel, table=True):
id: int | None = Field(default=None, primary_key=True)
account_id: int
kind: str
name: str | None = Field(default=None, sa_column=Column(Text, nullable=True))
params: dict[str, Any] = Field(
default_factory=dict, sa_column=Column(JSONB, nullable=False)
)
@@ -479,11 +466,7 @@ class Alert(SQLModel, table=True):
id: int | None = Field(default=None, primary_key=True)
account_id: int
watch_id: int = Field(
sa_column=Column(
Integer, ForeignKey("watches.id", ondelete="CASCADE"), nullable=False
)
)
watch_id: int = Field(foreign_key="watches.id")
ts: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=False))
payload: dict[str, Any] = Field(
default_factory=dict, sa_column=Column(JSONB, nullable=False)
@@ -560,62 +543,3 @@ class FileShareHit(SQLModel, table=True):
ip: str | None = None
user_agent: str | None = None
counted: bool = False
class Webhook(SQLModel, table=True):
__tablename__ = "webhooks"
id: int | None = Field(default=None, primary_key=True)
account_id: int
url: str = Field(sa_column=Column(Text, nullable=False))
secret: str = Field(sa_column=Column(Text, nullable=False))
enabled: bool = True
watch_ids: list[int] | None = Field(
default=None, sa_column=Column(ARRAY(Integer), nullable=True)
)
created_at: datetime = Field(
sa_column=Column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
)
updated_at: datetime = Field(
sa_column=Column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
)
class WebhookDelivery(SQLModel, table=True):
__tablename__ = "webhook_deliveries"
id: int | None = Field(default=None, sa_column=Column(BigInteger, primary_key=True))
webhook_id: int = Field(
sa_column=Column(
Integer, ForeignKey("webhooks.id", ondelete="CASCADE"), nullable=False
)
)
alert_id: int | None = Field(
default=None,
sa_column=Column(
Integer, ForeignKey("alerts.id", ondelete="CASCADE"), nullable=True
),
)
event: str = Field(sa_column=Column(Text, nullable=False))
status: str = Field(default="pending", sa_column=Column(Text, nullable=False))
attempts: int = 0
next_attempt_at: datetime = Field(
sa_column=Column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
)
response_code: int | None = None
last_error: str | None = Field(default=None, sa_column=Column(Text))
body: str | None = Field(default=None, sa_column=Column(Text))
created_at: datetime = Field(
sa_column=Column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
)
finished_at: datetime | None = Field(
default=None, sa_column=Column(DateTime(timezone=True))
)
backend/src/utils/env.py
-1
View File
@@ -41,7 +41,6 @@ class ApiSettings(BaseSettings):
host: str = "0.0.0.0" # noqa: S104
port: int = 8080
static_dir: str = "static"
public_url: str | None = None
class AuthSettings(BaseSettings):
backend/src/utils/events.py
+2 -10
View File
@@ -4,7 +4,6 @@ from typing import Literal
import asyncpg
BG_EVENTS_CHANNEL = "bg_events"
MESSAGE_IDS_PER_EVENT = 500
EventKind = Literal["message", "edit", "delete", "reaction", "presence", "receipt"]
@@ -23,15 +22,8 @@ async def notify_bg_event( # noqa: PLR0913
payload["chat_id"] = chat_id
if message_id is not None:
payload["message_id"] = message_id
if message_ids is None:
await _notify(pool, payload)
return
for start in range(0, len(message_ids), MESSAGE_IDS_PER_EVENT):
chunk = message_ids[start : start + MESSAGE_IDS_PER_EVENT]
await _notify(pool, {**payload, "message_ids": chunk})
async def _notify(pool: asyncpg.Pool, payload: dict[str, object]) -> None:
if message_ids is not None:
payload["message_ids"] = message_ids
await pool.execute(
"SELECT pg_notify($1, $2)", BG_EVENTS_CHANNEL, json.dumps(payload)
)
backend/src/utils/jobs.py
+2 -10
View File
@@ -1,5 +1,4 @@
import json
from datetime import timedelta
from typing import Any
import asyncpg
@@ -8,22 +7,15 @@ JOBS_CHANGED_CHANNEL = "jobs_changed"
async def enqueue(
pool: asyncpg.Pool,
account_id: int,
kind: str,
params: dict[str, Any],
*,
cooldown: timedelta | None = None,
pool: asyncpg.Pool, account_id: int, kind: str, params: dict[str, Any]
) -> int:
params_json = json.dumps(params)
existing = await pool.fetchval(
"SELECT id FROM jobs WHERE account_id = $1 AND kind = $2 "
"AND params = $3::jsonb AND (status IN ('pending', 'running') "
"OR created_at > now() - $4::interval) LIMIT 1",
"AND params = $3::jsonb AND status IN ('pending', 'running') LIMIT 1",
account_id,
kind,
params_json,
cooldown,
)
if existing is not None:
return existing
backend/src/utils/listen.py
-87
View File
@@ -1,87 +0,0 @@
import asyncio
import contextlib
from collections.abc import Callable, Mapping
import asyncpg
from utils.env import env
from utils.logging import logger
RETRY_MIN_SECONDS = 1.0
RETRY_MAX_SECONDS = 60.0
PING_INTERVAL_SECONDS = 60.0
PING_TIMEOUT_SECONDS = 10.0
CLOSE_TIMEOUT_SECONDS = 5.0
OnNotify = Callable[[str], None]
class Listener:
def __init__(
self,
name: str,
channels: Mapping[str, OnNotify],
on_reconnect: Callable[[], None] | None = None,
) -> None:
self._name = name
self._channels = channels
self._on_reconnect = on_reconnect
self._task: asyncio.Task | None = None
self._ready = asyncio.Event()
self._listening = False
async def start(self) -> None:
self._task = asyncio.create_task(self._run())
await self._ready.wait()
async def stop(self) -> None:
if self._task is None:
return
self._task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._task
self._task = None
async def _run(self) -> None:
delay = RETRY_MIN_SECONDS
while True:
try:
await self._listen()
except Exception as exc:
if self._listening:
delay = RETRY_MIN_SECONDS
logger.warning(
f"{self._name}: no listen connection ({exc!r}), "
f"reconnecting in {delay:.0f}s"
)
self._listening = False
self._ready.set()
await asyncio.sleep(delay)
delay = min(delay * 2, RETRY_MAX_SECONDS)
async def _listen(self) -> None:
conn = await asyncpg.connect(dsn=env.db.connection_url)
try:
lost = asyncio.Event()
conn.add_termination_listener(lambda _conn: lost.set())
for channel, on_notify in self._channels.items():
await conn.add_listener(
channel,
lambda _conn, _pid, _channel, payload, f=on_notify: f(payload),
)
logger.info(f"{self._name}: listening on {', '.join(self._channels)}")
self._listening = True
if self._ready.is_set() and self._on_reconnect is not None:
self._on_reconnect()
self._ready.set()
while True:
try:
await asyncio.wait_for(lost.wait(), PING_INTERVAL_SECONDS)
except TimeoutError:
await conn.fetchval("SELECT 1", timeout=PING_TIMEOUT_SECONDS)
else:
msg = "closed by the server"
raise ConnectionError(msg)
finally:
with contextlib.suppress(Exception):
await conn.close(timeout=CLOSE_TIMEOUT_SECONDS)
backend/src/utils/policy/defaults.py
+2 -2
View File
@@ -1,8 +1,8 @@
from utils.policy.models import CaptureToggles, ChatKind
DEFAULTS: dict[ChatKind, CaptureToggles] = {
ChatKind.CHANNEL: CaptureToggles(track_edits_deletes=True),
ChatKind.GROUP: CaptureToggles(messages=True, track_edits_deletes=True),
ChatKind.CHANNEL: CaptureToggles(),
ChatKind.GROUP: CaptureToggles(messages=True),
ChatKind.DM: CaptureToggles(
messages=True,
media=True,
backend/src/utils/read/analytics.py
+12 -20
View File
@@ -4,31 +4,23 @@ import asyncpg
from utils.read.accounts import self_user_id
from utils.read.models import ResponseStats, VolumeBucket
from utils.read.period import period_bounds
_VOLUME = """
WITH counts AS (
SELECT (date AT TIME ZONE $5)::date AS day, count(*) AS total,
count(*) FILTER (WHERE sender_id = $3) AS outgoing
FROM messages
WHERE account_id = $1 AND chat_id = $2 AND date >= $4
GROUP BY 1
)
SELECT local_day::date AS bucket, coalesce(total, 0) AS total,
coalesce(outgoing, 0) AS outgoing
FROM generate_series(($4 AT TIME ZONE $5)::date, (now() AT TIME ZONE $5)::date,
interval '1 day') AS local_day
LEFT JOIN counts ON counts.day = local_day::date
ORDER BY 1
"""
async def message_volume(
pool: asyncpg.Pool, account_id: int, chat_id: int, *, days: int, tz: str
pool: asyncpg.Pool, account_id: int, chat_id: int, *, days: int = 90
) -> list[VolumeBucket]:
self_id = await self_user_id(pool, account_id)
since, _ = await period_bounds(pool, days, tz)
rows = await pool.fetch(_VOLUME, account_id, chat_id, self_id, since, tz)
rows = await pool.fetch(
"SELECT date_trunc('day', date) AS bucket, count(*) AS total, "
"count(*) FILTER (WHERE sender_id = $3) AS outgoing "
"FROM messages "
"WHERE account_id = $1 AND chat_id = $2 AND date >= $4 "
"GROUP BY bucket ORDER BY bucket",
account_id,
chat_id,
self_id,
datetime.now(UTC) - timedelta(days=days),
)
return [
VolumeBucket(
bucket=row["bucket"],
backend/src/utils/read/chats.py
+6 -81
View File
@@ -1,5 +1,3 @@
from datetime import datetime
import asyncpg
from utils.policy.models import FolderSpec
@@ -12,7 +10,7 @@ from utils.read.models import (
MessageView,
Page,
)
from utils.read.read_receipts import read_times, read_up_to
from utils.read.read_receipts import read_up_to
_MESSAGE_COLS = (
"chat_id, message_id, date, sender_id, text, has_media, is_self_destruct, "
@@ -287,18 +285,12 @@ async def _apply_read_status(
self_id = await self_user_id(pool, account_id)
if self_id is None:
return
own = [view for view in views if view.sender_id == self_id]
if not own:
return
marker = await read_up_to(pool, account_id, chat_id)
times = await read_times(
pool, account_id, chat_id, [(view.message_id, view.date) for view in own]
)
for view in own:
view.read_at = times.get(view.message_id)
view.read = view.read_at is not None or (
marker is not None and view.message_id <= marker
)
if marker is None:
return
for view in views:
if view.sender_id == self_id and view.message_id <= marker:
view.read = True
def _build_album(
@@ -333,12 +325,6 @@ async def get_deleted_messages(
f"ORDER BY deleted_at DESC LIMIT ${len(params) - 1} OFFSET ${len(params)}",
*params,
)
return await _single_views(pool, account_id, rows)
async def _single_views(
pool: asyncpg.Pool, account_id: int, rows: list[asyncpg.Record]
) -> list[MessageView]:
media_by_key = await _media_map(pool, account_id, rows)
views: list[MessageView] = []
for row in rows:
@@ -349,67 +335,6 @@ async def _single_views(
return views
async def _rows_at(
pool: asyncpg.Pool, account_id: int, keys: list[tuple[int, int, datetime]]
) -> list[asyncpg.Record]:
if not keys:
return []
params: list[object] = [account_id]
clauses: list[str] = []
for key in keys:
params.extend(key)
n = len(params)
clauses.append(
f"(chat_id = ${n - 2} AND message_id = ${n - 1} AND date = ${n})"
)
return await pool.fetch(
f"SELECT {_MESSAGE_COLS} FROM messages " # noqa: S608
f"WHERE account_id = $1 AND ({' OR '.join(clauses)})",
*params,
)
async def get_messages_at(
pool: asyncpg.Pool, account_id: int, keys: list[tuple[int, int, datetime | None]]
) -> dict[tuple[int, int], MessageView]:
dated = list(
dict.fromkeys((chat, message, date) for chat, message, date in keys if date)
)
rows = await _rows_at(pool, account_id, dated)
views = {
(view.chat_id, view.message_id): view
for view in await _single_views(pool, account_id, rows)
}
for chat_id, message_id, _ in keys:
if (chat_id, message_id) not in views:
view = await get_message(pool, account_id, chat_id, message_id)
if view is not None:
views[chat_id, message_id] = view
return views
async def get_deleted_between( # noqa: PLR0913
pool: asyncpg.Pool,
account_id: int,
chat_id: int,
since: datetime,
until: datetime,
limit: int,
before: tuple[datetime, int] | None = None,
) -> list[MessageView]:
params: list[object] = [account_id, chat_id, since, until, limit]
where = "account_id = $1 AND chat_id = $2 AND deleted_at BETWEEN $3 AND $4"
if before is not None:
params.extend(before)
where += " AND (date, message_id) < ($6, $7)"
rows = await pool.fetch(
f"SELECT {_MESSAGE_COLS} FROM messages WHERE {where} " # noqa: S608
"ORDER BY date DESC, message_id DESC LIMIT $5",
*params,
)
return await _single_views(pool, account_id, rows)
async def get_message_versions(
pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
) -> list[MessageVersionView]:
backend/src/utils/read/feed.py
-291
View File
@@ -1,291 +0,0 @@
import json
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from enum import StrEnum
import asyncpg
from utils.read import chats, peers
from utils.read.accounts import self_user_id
from utils.read.models import (
DEFAULT_LIMIT,
MAX_LIMIT,
ChatListItem,
FeedChat,
FeedItem,
FeedMessagesPage,
FeedPage,
FeedUnseen,
MessageView,
PeerView,
)
CURSOR_PATTERN = r"^\d+\.\d+$"
_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
_MICROSECOND = timedelta(microseconds=1)
_ITEM_COLS = (
"id, kind, chat_id, chat_kind, message_id, message_date, at, since, "
"message_count, detail"
)
_VERSIONS = """
SELECT v.chat_id, v.message_id, v.observed_at, v.edit_date, v.text,
p.text AS before, p.observed_at IS NOT NULL AS has_before
FROM unnest($2::bigint[], $3::bigint[], $4::timestamptz[])
AS k(chat_id, message_id, observed_at)
JOIN message_versions v
ON v.account_id = $1 AND v.chat_id = k.chat_id
AND v.message_id = k.message_id AND v.observed_at = k.observed_at
LEFT JOIN LATERAL (
SELECT text, observed_at FROM message_versions p
WHERE p.account_id = $1 AND p.chat_id = v.chat_id
AND p.message_id = v.message_id AND p.observed_at < v.observed_at
ORDER BY p.observed_at DESC LIMIT 1
) p ON true
"""
_UNSEEN = """
SELECT count(*) FILTER (WHERE kind = 'deleted') AS deleted,
count(*) FILTER (WHERE kind = 'edited') AS edited
FROM feed_items
WHERE account_id = $1 AND at > coalesce($2, '-infinity'::timestamptz)
"""
_MARK_SEEN = """
INSERT INTO feed_seen (account_id, seen_at) VALUES ($1, coalesce($2, now()))
ON CONFLICT (account_id) DO UPDATE
SET seen_at = greatest(feed_seen.seen_at, EXCLUDED.seen_at)
"""
class FeedKind(StrEnum):
ALL = "all"
MESSAGES = "messages"
DELETED = "deleted"
EDITED = "edited"
PROFILE = "profile"
class FeedChatKind(StrEnum):
DM = "dm"
GROUP = "group"
CHANNEL = "channel"
@dataclass(slots=True)
class FeedFilter:
kind: FeedKind = FeedKind.ALL
chat_id: int | None = None
chat_kind: FeedChatKind | None = None
since: datetime | None = None
def encode_cursor(at: datetime, key: int) -> str:
return f"{(at - _EPOCH) // _MICROSECOND}.{key}"
def decode_cursor(cursor: str) -> tuple[datetime, int]:
micros, _, key = cursor.partition(".")
return _EPOCH + int(micros) * _MICROSECOND, int(key)
def _where(
account_id: int, feed: FeedFilter, cursor: str | None
) -> tuple[str, list[object]]:
params: list[object] = [account_id]
clauses = ["account_id = $1"]
if feed.kind is FeedKind.MESSAGES:
clauses.append("kind <> 'profile'")
exact = feed.kind not in {FeedKind.ALL, FeedKind.MESSAGES}
conditions = (
("kind = {}", feed.kind.value if exact else None),
("chat_id = {}", feed.chat_id),
("chat_kind = {}", feed.chat_kind.value if feed.chat_kind else None),
("at >= {}", feed.since),
)
for template, value in conditions:
if value is not None:
params.append(value)
clauses.append(template.format(f"${len(params)}"))
if cursor is not None:
params.extend(decode_cursor(cursor))
clauses.append(f"(at, id) < (${len(params) - 1}, ${len(params)})")
return " AND ".join(clauses), params
def _peer_name(peer: PeerView) -> str | None:
name = " ".join(part for part in (peer.first_name, peer.last_name) if part)
return name or peer.username
async def _versions(
pool: asyncpg.Pool, account_id: int, rows: list[asyncpg.Record]
) -> dict[tuple[int, int, datetime], asyncpg.Record]:
edited = [row for row in rows if row["kind"] == FeedKind.EDITED]
if not edited:
return {}
found = await pool.fetch(
_VERSIONS,
account_id,
[row["chat_id"] for row in edited],
[row["message_id"] for row in edited],
[row["at"] for row in edited],
)
return {
(row["chat_id"], row["message_id"], row["observed_at"]): row for row in found
}
async def _sender_names(
pool: asyncpg.Pool,
account_id: int,
messages: dict[tuple[int, int], MessageView],
chat_views: dict[int, ChatListItem],
) -> dict[int, str | None]:
ids = {m.sender_id for m in messages.values() if m.sender_id is not None}
found = await peers.get_peers(pool, account_id, list(ids))
names: dict[int, str | None] = {
chat_id: chat.title for chat_id, chat in chat_views.items()
}
names.update({peer.peer_id: _peer_name(peer) for peer in found})
return names
async def _build_items(
pool: asyncpg.Pool, account_id: int, rows: list[asyncpg.Record]
) -> list[FeedItem]:
if not rows:
return []
messages = await chats.get_messages_at(
pool,
account_id,
[
(row["chat_id"], row["message_id"], row["message_date"])
for row in rows
if row["kind"] != FeedKind.PROFILE
],
)
versions = await _versions(pool, account_id, rows)
chat_views = {
chat.chat_id: chat
for chat in await chats.get_chats(
pool, account_id, list({row["chat_id"] for row in rows})
)
}
names = await _sender_names(pool, account_id, messages, chat_views)
self_id = await self_user_id(pool, account_id)
items: list[FeedItem] = []
for row in rows:
message = messages.get((row["chat_id"], row["message_id"]))
chat = chat_views.get(row["chat_id"])
version = versions.get((row["chat_id"], row["message_id"], row["at"]))
sender_id = message.sender_id if message else None
items.append(
FeedItem(
id=row["id"],
kind=row["kind"],
at=row["at"],
since=row["since"],
message_count=row["message_count"],
chat=FeedChat(
chat_id=row["chat_id"],
title=chat.title if chat else None,
kind=row["chat_kind"],
has_avatar=chat.has_avatar if chat else False,
),
message_id=row["message_id"],
message=message,
sender_id=sender_id,
sender_name=names.get(sender_id) if sender_id is not None else None,
outgoing=self_id is not None and sender_id == self_id,
before=version["before"] if version else None,
after=version["text"] if version else None,
has_before=bool(version and version["has_before"]),
edit_date=version["edit_date"] if version else None,
detail=json.loads(row["detail"]) if row["detail"] else None,
)
)
return items
async def list_feed(
pool: asyncpg.Pool,
account_id: int,
feed: FeedFilter,
*,
limit: int = DEFAULT_LIMIT,
cursor: str | None = None,
) -> FeedPage:
size = max(1, min(limit, MAX_LIMIT))
where, params = _where(account_id, feed, cursor)
params.append(size + 1)
rows = await pool.fetch(
f"SELECT {_ITEM_COLS} FROM feed_items WHERE {where} " # noqa: S608
f"ORDER BY at DESC, id DESC LIMIT ${len(params)}",
*params,
)
page = rows[:size]
more = len(rows) > size
return FeedPage(
items=await _build_items(pool, account_id, page),
next_cursor=encode_cursor(page[-1]["at"], page[-1]["id"]) if more else None,
)
async def unseen(pool: asyncpg.Pool, account_id: int) -> FeedUnseen:
seen_at = await pool.fetchval(
"SELECT seen_at FROM feed_seen WHERE account_id = $1", account_id
)
counts = await pool.fetchrow(_UNSEEN, account_id, seen_at)
latest = await list_feed(
pool, account_id, FeedFilter(kind=FeedKind.MESSAGES), limit=1
)
return FeedUnseen(
count=counts["deleted"] + counts["edited"],
deleted=counts["deleted"],
edited=counts["edited"],
seen_at=seen_at,
latest=latest.items[0] if latest.items else None,
)
async def mark_seen(
pool: asyncpg.Pool, account_id: int, at: datetime | None
) -> FeedUnseen:
await pool.execute(_MARK_SEEN, account_id, at)
return await unseen(pool, account_id)
async def item_messages(
pool: asyncpg.Pool,
account_id: int,
item_id: int,
*,
limit: int = DEFAULT_LIMIT,
cursor: str | None = None,
) -> FeedMessagesPage | None:
row = await pool.fetchrow(
"SELECT chat_id, since, at FROM feed_items "
"WHERE id = $1 AND account_id = $2 AND kind = 'deleted'",
item_id,
account_id,
)
if row is None:
return None
size = max(1, min(limit, MAX_LIMIT))
views = await chats.get_deleted_between(
pool,
account_id,
row["chat_id"],
row["since"],
row["at"],
size + 1,
decode_cursor(cursor) if cursor else None,
)
page = views[:size]
more = len(views) > size
return FeedMessagesPage(
messages=page,
next_cursor=encode_cursor(page[-1].date, page[-1].message_id) if more else None,
)
backend/src/utils/read/models.py
+2 -254
View File
@@ -1,4 +1,4 @@
from datetime import date, datetime
from datetime import datetime
from typing import Any
from pydantic import BaseModel
@@ -203,7 +203,6 @@ class MessageView(BaseModel):
is_sticker: bool = False
is_animated_emoji: bool = False
read: bool = False
read_at: datetime | None = None
class MessageVersionView(BaseModel):
@@ -212,50 +211,6 @@ class MessageVersionView(BaseModel):
text: str | None
class FeedChat(BaseModel):
chat_id: int
title: str | None
kind: str
has_avatar: bool
class FeedItem(BaseModel):
id: int
kind: str
at: datetime
since: datetime
message_count: int
chat: FeedChat
message_id: int
message: MessageView | None
sender_id: int | None
sender_name: str | None
outgoing: bool
before: str | None = None
after: str | None = None
has_before: bool = False
edit_date: datetime | None = None
detail: dict[str, list[str | None]] | None = None
class FeedPage(BaseModel):
items: list[FeedItem]
next_cursor: str | None
class FeedUnseen(BaseModel):
count: int
deleted: int
edited: int
seen_at: datetime | None
latest: FeedItem | None
class FeedMessagesPage(BaseModel):
messages: list[MessageView]
next_cursor: str | None
class MediaView(BaseModel):
id: int
account_id: int
@@ -406,61 +361,8 @@ class PresenceHourly(BaseModel):
last_seen: datetime | None
class PresenceSession(BaseModel):
started: datetime
ended: datetime
seconds: float
online_seconds: float
ongoing: bool
class PresenceHour(BaseModel):
weekday: int
hour: int
hours: int
online_minutes: float | None
covered_hours: float
class PresenceDay(BaseModel):
day: date
online_seconds: float
covered_seconds: float
class CoverageGap(BaseModel):
started: datetime
ended: datetime
class PresenceSummary(BaseModel):
online_seconds: float
daily_seconds: float | None
covered_seconds: float
session_count: int
median_session_seconds: float | None
first_online: int | None
last_offline: int | None
last_seen: datetime | None
class PresenceStats(BaseModel):
peer_id: int
tz: str
since: datetime
until: datetime
current: PresenceSample | None
online: bool
hidden: bool
summary: PresenceSummary
week: list[PresenceHour]
days: list[PresenceDay]
gaps: list[CoverageGap]
sessions: list[PresenceSession]
class VolumeBucket(BaseModel):
bucket: date
bucket: datetime
total: int
outgoing: int
incoming: int
@@ -485,29 +387,6 @@ class PeerView(BaseModel):
updated_at: datetime
class ReaderView(BaseModel):
peer_id: int
read_at: datetime
peer: PeerView | None
class ReadInfoView(BaseModel):
chat_id: int
message_id: int
chat_kind: str
own: bool
voice: bool
observed_at: datetime | None
listened_at: datetime | None
exact_at: datetime | None
readers: list[ReaderView]
read: bool
unread: bool
outcome: str | None
fetched_at: datetime | None
can_fetch: bool
class PeerHistoryView(BaseModel):
observed_at: datetime
first_name: str | None
@@ -518,54 +397,6 @@ class PeerHistoryView(BaseModel):
is_deleted_account: bool
class ProfileChange(BaseModel):
field: str
old: str | None
new: str | None
changed_at: datetime
actor_id: int | None = None
message_id: int | None = None
class MembershipView(BaseModel):
chat_id: int
chat_title: str | None
has_avatar: bool
message_id: int
event: str
actor_id: int | None
actor_name: str | None
ts: datetime
class DirectChatView(BaseModel):
chat_id: int
message_count: int
last_date: datetime | None
class PeerOverview(BaseModel):
peer: PeerView | None
changes: list[ProfileChange]
avatars: list[AvatarHistoryView]
memberships: list[MembershipView]
dm: DirectChatView | None
class SharedGroupView(BaseModel):
chat_id: int
title: str | None
has_avatar: bool
message_count: int
last_date: datetime
last_message_id: int
class PeerGroups(BaseModel):
groups: list[SharedGroupView]
deleted_count: int
class ChatLinkView(BaseModel):
message_id: int
date: datetime | None
@@ -615,33 +446,12 @@ class WatchView(BaseModel):
id: int
account_id: int
kind: str
name: str | None
params: dict[str, Any]
enabled: bool
created_at: datetime
updated_at: datetime
class ChatRef(BaseModel):
id: int
title: str | None
kind: str
has_avatar: bool
class PersonRef(BaseModel):
id: int
name: str | None
has_avatar: bool
class WatchItem(WatchView):
chat: ChatRef | None
person: PersonRef | None
alert_count: int
last_alert_at: datetime | None
class AlertView(BaseModel):
id: int
account_id: int
@@ -652,68 +462,6 @@ class AlertView(BaseModel):
created_at: datetime
class AlertWatch(BaseModel):
id: int
kind: str
name: str | None
params: dict[str, Any]
class AlertMessage(BaseModel):
id: int
text: str | None
sender: PersonRef | None
date: datetime | None
deleted: bool
edited: bool
class AlertItem(BaseModel):
id: int
account_id: int
ts: datetime
seen: bool
event: str
watch: AlertWatch
chat: ChatRef | None
message: AlertMessage | None
peer: PersonRef | None
away_since: datetime | None
path: str
class AlertPage(BaseModel):
items: list[AlertItem]
next_before: int | None
unseen: int
class DeliveryView(BaseModel):
id: int
webhook_id: int
alert_id: int | None
event: str
status: str
attempts: int
next_attempt_at: datetime
response_code: int | None
last_error: str | None
created_at: datetime
finished_at: datetime | None
class WebhookView(BaseModel):
id: int
account_id: int
url: str
secret: str
enabled: bool
watch_ids: list[int] | None
created_at: datetime
updated_at: datetime
last_delivery: DeliveryView | None
class FileShareView(BaseModel):
id: int
account_id: int
backend/src/utils/read/peers.py
+8 -20
View File
@@ -2,8 +2,6 @@ import asyncpg
from utils.read.models import Page, PeerHistoryView, PeerView, StoryView
HISTORY_LIMIT = 500
_PEER_COLS = (
"peer_id, first_name, last_name, username, phone, photo_unique_id, "
"is_deleted_account, updated_at, "
@@ -11,23 +9,6 @@ _PEER_COLS = (
"AND a.owner_id = peers.peer_id) AS has_avatar"
)
_HISTORY = """
SELECT observed_at, first_name, last_name, username, phone, photo_unique_id,
is_deleted_account
FROM (
SELECT observed_at, first_name, last_name, username, phone, photo_unique_id,
is_deleted_account,
lag((first_name, last_name, username, phone, photo_unique_id,
is_deleted_account)) OVER (ORDER BY observed_at) AS previous
FROM peer_history
WHERE account_id = $1 AND peer_id = $2
) states
WHERE previous IS DISTINCT FROM
(first_name, last_name, username, phone, photo_unique_id, is_deleted_account)
ORDER BY observed_at DESC
LIMIT $3
"""
async def get_peer(
pool: asyncpg.Pool, account_id: int, peer_id: int
@@ -58,7 +39,14 @@ async def get_peers(
async def get_peer_history(
pool: asyncpg.Pool, account_id: int, peer_id: int
) -> list[PeerHistoryView]:
rows = await pool.fetch(_HISTORY, account_id, peer_id, HISTORY_LIMIT)
rows = await pool.fetch(
"SELECT observed_at, first_name, last_name, username, phone, "
"photo_unique_id, is_deleted_account FROM peer_history "
"WHERE account_id = $1 AND peer_id = $2 "
"ORDER BY observed_at DESC LIMIT 500",
account_id,
peer_id,
)
return [PeerHistoryView(**dict(row)) for row in rows]
backend/src/utils/read/people.py
-194
View File
@@ -1,194 +0,0 @@
from collections.abc import Callable
from datetime import datetime
from itertools import pairwise
import asyncpg
from utils.read import chats
from utils.read.avatars import avatar_history
from utils.read.models import (
DirectChatView,
MembershipView,
PeerGroups,
PeerHistoryView,
PeerOverview,
ProfileChange,
SharedGroupView,
)
from utils.read.peers import HISTORY_LIMIT, get_peer, get_peer_history
_CHAT_TITLES = """
SELECT ts, title, previous, actor_id, message_id
FROM (
SELECT ts, title,
CASE WHEN event = 'title' THEN actor_id END AS actor_id,
CASE WHEN event = 'title' THEN message_id END AS message_id,
lag(title) OVER (ORDER BY ts, message_id) AS previous
FROM chat_history
WHERE account_id = $1 AND chat_id = $2 AND event IN ('meta', 'title')
AND title IS NOT NULL
) titles
WHERE previous IS DISTINCT FROM title AND previous IS NOT NULL
ORDER BY ts DESC
LIMIT $3
"""
_MEMBERSHIPS = """
SELECT m.chat_id, m.message_id, m.event, m.actor_id, m.ts,
COALESCE(NULLIF(trim(concat_ws(' ', a.first_name, a.last_name)), ''),
a.username) AS actor_name
FROM memberships m
LEFT JOIN peers a ON a.account_id = m.account_id AND a.peer_id = m.actor_id
WHERE m.account_id = $1 AND m.user_id = $2
ORDER BY m.ts DESC
LIMIT 200
"""
_DIRECT_CHAT = """
SELECT $2::bigint AS chat_id, COALESCE(cs.message_count, 0) AS message_count,
cs.last_date
FROM (SELECT 1) one
LEFT JOIN chat_stats cs ON cs.account_id = $1 AND cs.chat_id = $2
WHERE cs.chat_id IS NOT NULL
OR EXISTS (SELECT 1 FROM dialogs WHERE account_id = $1 AND chat_id = $2)
"""
_SENT = """
SELECT chat_id, count(*) AS message_count, max(date) AS last_date,
max(message_id) AS last_message_id,
count(*) FILTER (WHERE deleted_at IS NOT NULL) AS deleted_count
FROM messages
WHERE account_id = $1 AND sender_id = $2
GROUP BY chat_id
"""
type Point = tuple[datetime, str | None]
def _full_name(state: PeerHistoryView) -> str | None:
parts = [part for part in (state.first_name, state.last_name) if part]
return " ".join(parts) or None
def _timeline(
states: list[PeerHistoryView], value: Callable[[PeerHistoryView], str | None]
) -> list[Point]:
points: list[Point] = []
for state in states:
current = value(state)
if not points or points[-1][1] != current:
points.append((state.observed_at, current))
return points
def _without_gaps(points: list[Point]) -> list[Point]:
kept: list[Point] = []
for at, current in points:
gap = len(kept) > 1 and kept[-1][1] is None and kept[-2][1] == current
if gap:
kept.pop()
elif not kept or kept[-1][1] != current:
kept.append((at, current))
return kept
def _known(points: list[Point]) -> list[Point]:
return _without_gaps([point for point in points if point[1] is not None])
def _changes(field: str, points: list[Point]) -> list[ProfileChange]:
return [
ProfileChange(field=field, old=old, new=new, changed_at=at)
for (_, old), (at, new) in pairwise(points)
]
def profile_changes(states: list[PeerHistoryView]) -> list[ProfileChange]:
ordered = sorted(states, key=lambda state: state.observed_at)
alive = [state for state in ordered if not state.is_deleted_account]
deleted = _timeline(
ordered, lambda state: "deleted" if state.is_deleted_account else None
)
changes = [
*_changes("name", _without_gaps(_timeline(alive, _full_name))),
*_changes(
"username", _without_gaps(_timeline(alive, lambda state: state.username))
),
*_changes("phone", _known(_timeline(alive, lambda state: state.phone))),
*[change for change in _changes("deleted", deleted) if change.new is not None],
]
return sorted(changes, key=lambda change: change.changed_at, reverse=True)
async def get_chat_changes(
pool: asyncpg.Pool, account_id: int, chat_id: int
) -> list[ProfileChange]:
rows = await pool.fetch(_CHAT_TITLES, account_id, chat_id, HISTORY_LIMIT)
return [
ProfileChange(
field="title",
old=row["previous"],
new=row["title"],
changed_at=row["ts"],
actor_id=row["actor_id"],
message_id=row["message_id"],
)
for row in rows
]
async def _memberships(
pool: asyncpg.Pool, account_id: int, peer_id: int
) -> list[MembershipView]:
rows = await pool.fetch(_MEMBERSHIPS, account_id, peer_id)
ids = list({row["chat_id"] for row in rows})
known = {
chat.chat_id: chat for chat in await chats.get_chats(pool, account_id, ids)
}
views: list[MembershipView] = []
for row in rows:
chat = known.get(row["chat_id"])
views.append(
MembershipView(
**dict(row),
chat_title=chat.title if chat else None,
has_avatar=bool(chat and chat.has_avatar),
)
)
return views
async def get_overview(
pool: asyncpg.Pool, account_id: int, peer_id: int
) -> PeerOverview:
direct = await pool.fetchrow(_DIRECT_CHAT, account_id, peer_id)
return PeerOverview(
peer=await get_peer(pool, account_id, peer_id),
changes=profile_changes(await get_peer_history(pool, account_id, peer_id)),
avatars=await avatar_history(pool, account_id, peer_id),
memberships=await _memberships(pool, account_id, peer_id),
dm=DirectChatView(**dict(direct)) if direct else None,
)
async def get_groups(pool: asyncpg.Pool, account_id: int, peer_id: int) -> PeerGroups:
async with pool.acquire() as conn, conn.transaction():
await conn.execute("SET LOCAL jit = off")
rows = await conn.fetch(_SENT, account_id, peer_id)
sent = {row["chat_id"]: row for row in rows if row["chat_id"] < 0}
found = await chats.get_chats(pool, account_id, list(sent))
groups = [
SharedGroupView(
chat_id=chat.chat_id,
title=chat.title,
has_avatar=chat.has_avatar,
message_count=sent[chat.chat_id]["message_count"],
last_date=sent[chat.chat_id]["last_date"],
last_message_id=sent[chat.chat_id]["last_message_id"],
)
for chat in found
]
groups.sort(key=lambda group: group.last_date, reverse=True)
return PeerGroups(
groups=groups, deleted_count=sum(row["deleted_count"] for row in rows)
)
backend/src/utils/read/period.py
-27
View File
@@ -1,27 +0,0 @@
from datetime import datetime
import asyncpg
DEFAULT_TZ = "UTC"
MAX_DAYS = 365
_timezones: set[str] = set()
async def known_timezone(pool: asyncpg.Pool, tz: str) -> bool:
if not _timezones:
rows = await pool.fetch("SELECT name FROM pg_timezone_names")
_timezones.update(row["name"] for row in rows)
return tz in _timezones
async def period_bounds(
pool: asyncpg.Pool, days: int, tz: str
) -> tuple[datetime, datetime]:
row = await pool.fetchrow(
"SELECT (date_trunc('day', now() AT TIME ZONE $1) "
"- make_interval(days => $2 - 1)) AT TIME ZONE $1 AS since, now() AS until",
tz,
days,
)
return row["since"], row["until"]
backend/src/utils/read/presence_stats.py
-289
View File
@@ -1,289 +0,0 @@
import json
import asyncpg
from utils.read.models import (
CoverageGap,
Page,
PresenceDay,
PresenceHour,
PresenceSession,
PresenceStats,
PresenceSummary,
)
from utils.read.period import period_bounds
from utils.read.presence import current_presence
SECONDS_IN_HOUR = 3600
SECONDS_IN_DAY = 86_400
MIN_CELL_COVERAGE = 1800
MIN_CELL_SHARE = 0.5
STATS_WORK_MEM = "32MB"
FUZZY_STATUSES = {"recently", "last_week", "last_month", "long_time_ago"}
_SESSIONS = """
marks AS (
SELECT ts, status,
least(next_offline_date, ts + interval '5 minutes',
lead(coalesce(last_online_date, ts)) OVER w, $4) AS ended,
lag(CASE WHEN status = 'online'
THEN least(next_offline_date, ts + interval '5 minutes')
ELSE coalesce(last_online_date, ts) END) OVER w AS previous
FROM presence
WHERE account_id = $1 AND peer_id = $2
AND ts >= $3::timestamptz - interval '5 minutes' AND ts <= $4
WINDOW w AS (ORDER BY ts)
),
online AS (
SELECT greatest(ts, $3) AS started, ended,
count(*) FILTER (WHERE previous IS NULL
OR ts > previous + interval '5 minutes')
OVER (ORDER BY ts) AS visit
FROM marks
WHERE status = 'online' AND ended > greatest(ts, $3)
),
sessions AS (
SELECT min(started) AS started, max(ended) AS ended,
extract(epoch FROM sum(ended - started))::float8 AS online_seconds
FROM online
GROUP BY visit
)
"""
_SESSION_FIELDS = """
started, ended, extract(epoch FROM ended - started)::float8 AS seconds,
online_seconds, ended >= $4 AS ongoing
"""
_SESSION_PAGE = f"""
WITH {_SESSIONS}
SELECT {_SESSION_FIELDS}
FROM sessions
ORDER BY started DESC
LIMIT $5 OFFSET $6
""" # noqa: S608
_COVERED_HOURS = """
SELECT local_hour,
extract(epoch FROM sum(
least(ended, (local_hour + interval '1 hour') AT TIME ZONE $5)
- greatest(started, local_hour AT TIME ZONE $5)))::float8 AS seconds
FROM uptime,
generate_series(date_trunc('hour', started AT TIME ZONE $5),
date_trunc('hour', ended AT TIME ZONE $5),
interval '1 hour') AS local_hour
GROUP BY local_hour
"""
_STATS = f"""
WITH {_SESSIONS},
uptime AS (
SELECT greatest(started_at, $3) AS started,
least(seen_at + interval '90 seconds', $4) AS ended
FROM capture_uptime
WHERE account_id = $1 AND started_at < $4
AND seen_at + interval '90 seconds' > $3
),
quarters AS (
SELECT quarter, sum(ended - started - spill) AS inside, sum(spill) AS spill
FROM (SELECT started, ended, quarter,
greatest(ended - quarter - interval '15 minutes',
interval '0') AS spill
FROM (SELECT started, ended,
date_bin('15 minutes', started, timestamptz 'epoch') AS quarter
FROM online) binned) split
GROUP BY quarter
),
online_hours AS (
SELECT date_trunc('hour', quarter AT TIME ZONE $5) AS local_hour,
extract(epoch FROM sum(share))::float8 AS seconds
FROM (SELECT quarter, inside AS share FROM quarters
UNION ALL
SELECT quarter + interval '15 minutes', spill FROM quarters) shares
GROUP BY 1
),
covered_hours AS ({_COVERED_HOURS}),
hours AS (
SELECT local_hour,
extract(epoch FROM
least((local_hour + interval '1 hour') AT TIME ZONE $5, $4)
- greatest(local_hour AT TIME ZONE $5, $3))::float8 AS span,
coalesce(online.seconds, 0) AS online,
least(3600, greatest(coalesce(covered.seconds, 0),
coalesce(online.seconds, 0))) AS covered
FROM generate_series(date_trunc('hour', $3 AT TIME ZONE $5),
date_trunc('hour', $4 AT TIME ZONE $5),
interval '1 hour') AS local_hour
LEFT JOIN online_hours AS online USING (local_hour)
LEFT JOIN covered_hours AS covered USING (local_hour)
),
week AS (
SELECT extract(isodow FROM local_hour)::int - 1 AS weekday,
extract(hour FROM local_hour)::int AS hour,
count(*) AS hours, sum(span) AS span,
sum(online) AS online, sum(covered) AS covered
FROM hours
GROUP BY 1, 2
),
days AS (
SELECT local_hour::date AS day, sum(online) AS online, sum(covered) AS covered
FROM hours
GROUP BY 1
),
uptime_reach AS (
SELECT started,
coalesce(max(ended) OVER (ORDER BY started
ROWS BETWEEN UNBOUNDED PRECEDING
AND 1 PRECEDING), $3) AS reached
FROM uptime
),
gaps AS (
SELECT reached AS started, started AS ended FROM uptime_reach
UNION ALL
SELECT coalesce(max(ended), $3), $4 FROM uptime
),
quiet AS (
SELECT hour
FROM week
GROUP BY hour
ORDER BY sum(online) / nullif(sum(covered), 0) NULLS LAST, hour
LIMIT 1
),
activity_days AS (
SELECT (sessions.started AT TIME ZONE $5) - make_interval(hours => quiet.hour)
AS started,
(sessions.ended AT TIME ZONE $5) - make_interval(hours => quiet.hour)
AS ended,
(($4 AT TIME ZONE $5) - make_interval(hours => quiet.hour))::date AS today
FROM sessions, quiet
),
daily AS (
SELECT extract(epoch FROM min(started - started::date))::float8 AS first_online,
extract(epoch FROM max(ended - started::date))::float8 AS last_offline
FROM activity_days
WHERE started::date < today
GROUP BY started::date
)
SELECT
(SELECT count(*) FROM sessions) AS session_count,
(SELECT coalesce(sum(online_seconds), 0) FROM sessions) AS online_seconds,
(SELECT sum(covered) FROM hours) AS covered_seconds,
(SELECT percentile_cont(0.5) WITHIN GROUP (
ORDER BY extract(epoch FROM ended - started)) FROM sessions) AS median_session,
(SELECT max(ended) FROM sessions) AS last_seen,
(SELECT hour FROM quiet) AS day_start,
(SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY first_online)
FROM daily) AS first_online,
(SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY last_offline)
FROM daily) AS last_offline,
(SELECT json_agg(json_build_object(
'weekday', weekday, 'hour', hour, 'hours', hours, 'span', span,
'online', online, 'covered', covered))
FROM week) AS week,
(SELECT json_agg(json_build_object(
'day', day, 'online_seconds', online, 'covered_seconds', covered)
ORDER BY day)
FROM days) AS days,
(SELECT json_agg(json_build_object('started', started, 'ended', ended)
ORDER BY started)
FROM gaps WHERE ended - started > interval '5 minutes') AS gaps,
(SELECT json_agg(row_to_json(page) ORDER BY page.started DESC)
FROM (SELECT {_SESSION_FIELDS} FROM sessions
ORDER BY started DESC LIMIT $6 OFFSET $7) page) AS sessions
""" # noqa: S608
def _time_of_day(shifted: float | None, day_start: int | None) -> int | None:
if shifted is None or day_start is None:
return None
return round(shifted + day_start * SECONDS_IN_HOUR) % SECONDS_IN_DAY
def _week(rows: list[dict]) -> list[PresenceHour]:
cells = {(row["weekday"], row["hour"]): row for row in rows}
week: list[PresenceHour] = []
for weekday in range(7):
for hour in range(24):
row = cells.get(
(weekday, hour), {"hours": 0, "span": 0, "online": 0, "covered": 0}
)
covered = row["covered"]
minutes = (
min(60.0, row["online"] / covered * 60)
if covered > 0
and covered >= min(MIN_CELL_COVERAGE, row["span"] * MIN_CELL_SHARE)
else None
)
week.append(
PresenceHour(
weekday=weekday,
hour=hour,
hours=row["hours"],
online_minutes=minutes,
covered_hours=covered / SECONDS_IN_HOUR,
)
)
return week
async def presence_stats( # noqa: PLR0913
pool: asyncpg.Pool, account_id: int, peer_id: int, page: Page, *, days: int, tz: str
) -> PresenceStats:
since, until = await period_bounds(pool, days, tz)
async with pool.acquire() as conn, conn.transaction():
await conn.execute(f"SET LOCAL work_mem = '{STATS_WORK_MEM}'")
row = await conn.fetchrow(
_STATS,
account_id,
peer_id,
since,
until,
tz,
page.capped_limit,
page.offset,
)
current = await current_presence(pool, account_id, peer_id)
sessions = [PresenceSession(**item) for item in json.loads(row["sessions"] or "[]")]
covered = row["covered_seconds"] or 0
online = row["online_seconds"]
return PresenceStats(
peer_id=peer_id,
tz=tz,
since=since,
until=until,
current=current,
online=current is not None
and current.status == "online"
and current.next_offline_date is not None
and current.next_offline_date > until,
hidden=row["session_count"] == 0
and current is not None
and current.status in FUZZY_STATUSES,
summary=PresenceSummary(
online_seconds=online,
daily_seconds=online / covered * SECONDS_IN_DAY
if covered >= SECONDS_IN_HOUR
else None,
covered_seconds=covered,
session_count=row["session_count"],
median_session_seconds=row["median_session"],
first_online=_time_of_day(row["first_online"], row["day_start"]),
last_offline=_time_of_day(row["last_offline"], row["day_start"]),
last_seen=row["last_seen"]
or (current.last_online_date if current else None),
),
week=_week(json.loads(row["week"] or "[]")),
days=[PresenceDay(**item) for item in json.loads(row["days"] or "[]")],
gaps=[CoverageGap(**item) for item in json.loads(row["gaps"] or "[]")],
sessions=sessions,
)
async def presence_sessions( # noqa: PLR0913
pool: asyncpg.Pool, account_id: int, peer_id: int, page: Page, *, days: int, tz: str
) -> list[PresenceSession]:
since, until = await period_bounds(pool, days, tz)
rows = await pool.fetch(
_SESSION_PAGE, account_id, peer_id, since, until, page.capped_limit, page.offset
)
return [PresenceSession(**dict(row)) for row in rows]
backend/src/utils/read/read_receipts.py
-178
View File
@@ -1,89 +1,5 @@
from datetime import UTC, datetime, timedelta
import asyncpg
from utils.policy.models import ChatKind
from utils.read.discover import chat_kind
from utils.read.models import ReaderView, ReadInfoView
from utils.read.peers import get_peers
FETCH_WINDOW = timedelta(days=7)
CLOCK_SKEW = timedelta(minutes=1)
VOICE_KINDS = ["voice", "video_note"]
_READ_TIMES = """
SELECT m.id,
COALESCE(
(SELECT min(mr.read_at) FROM message_reads mr
WHERE mr.account_id = $1 AND mr.chat_id = $2 AND mr.message_id = m.id),
CASE WHEN m.date >= $5 THEN (
SELECT r.ts FROM read_receipts r
WHERE r.account_id = $1 AND r.chat_id = $2 AND r.kind = 'read'
AND r.message_id >= m.id AND r.ts >= m.date - $6::interval
ORDER BY r.ts LIMIT 1
) END
) AS read_at
FROM unnest($3::bigint[], $4::timestamptz[]) AS m(id, date)
"""
_MESSAGE = """
SELECT m.sender_id, m.date, a.tg_user_id, a.is_bot,
EXISTS (
SELECT 1 FROM media md
WHERE md.account_id = m.account_id AND md.chat_id = m.chat_id
AND md.message_id = m.message_id AND md.kind = ANY($4::text[])
) AS voice
FROM messages m
JOIN accounts a ON a.account_id = m.account_id
WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3
LIMIT 1
"""
_RECEIPTS = """
SELECT
(SELECT ts FROM read_receipts
WHERE account_id = $1 AND chat_id = $2 AND kind = 'read'
AND message_id >= $3 AND ts >= $4
ORDER BY ts LIMIT 1) AS observed_at,
(SELECT min(ts) FROM read_receipts
WHERE account_id = $1 AND chat_id = $2 AND kind = 'listened'
AND message_id = $3) AS listened_at,
(SELECT max(message_id) FROM read_receipts
WHERE account_id = $1 AND chat_id = $2 AND kind = 'read') AS watermark
"""
_READS = """
SELECT reader_id, read_at, source FROM message_reads
WHERE account_id = $1 AND chat_id = $2 AND message_id = $3
ORDER BY read_at
"""
_FETCH = """
SELECT outcome, fetched_at, retry_at FROM read_fetches
WHERE account_id = $1 AND chat_id = $2 AND message_id = $3
"""
_INHERITED = """
SELECT outcome FROM read_fetches
WHERE account_id = $1 AND retry_at > now()
AND ((chat_id = $2 AND outcome = 'chat_too_big') OR outcome = 'your_privacy')
LIMIT 1
"""
_capture_starts: dict[int, datetime] = {}
async def capture_start(pool: asyncpg.Pool, account_id: int) -> datetime | None:
cached = _capture_starts.get(account_id)
if cached is not None:
return cached
start = await pool.fetchval(
"SELECT min(ts) FROM read_receipts WHERE account_id = $1", account_id
)
if start is not None:
_capture_starts[account_id] = start
return start
async def read_up_to(pool: asyncpg.Pool, account_id: int, chat_id: int) -> int | None:
return await pool.fetchval(
@@ -92,97 +8,3 @@ async def read_up_to(pool: asyncpg.Pool, account_id: int, chat_id: int) -> int |
account_id,
chat_id,
)
async def read_times(
pool: asyncpg.Pool,
account_id: int,
chat_id: int,
messages: list[tuple[int, datetime]],
) -> dict[int, datetime]:
if not messages:
return {}
start = await capture_start(pool, account_id) or datetime.max.replace(tzinfo=UTC)
rows = await pool.fetch(
_READ_TIMES,
account_id,
chat_id,
[message_id for message_id, _ in messages],
[date for _, date in messages],
start,
CLOCK_SKEW,
)
return {row["id"]: row["read_at"] for row in rows if row["read_at"] is not None}
async def read_info(
pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
) -> ReadInfoView | None:
message = await pool.fetchrow(
_MESSAGE, account_id, chat_id, message_id, VOICE_KINDS
)
if message is None:
return None
kind = await chat_kind(pool, account_id, chat_id)
date: datetime = message["date"]
receipts = await pool.fetchrow(
_RECEIPTS, account_id, chat_id, message_id, date - CLOCK_SKEW
)
reads = await pool.fetch(_READS, account_id, chat_id, message_id)
fetch = await pool.fetchrow(_FETCH, account_id, chat_id, message_id)
inherited = await pool.fetchval(_INHERITED, account_id, chat_id)
start = await capture_start(pool, account_id)
observed_at = receipts["observed_at"] if start and date >= start else None
exact_at = next(
(row["read_at"] for row in reads if row["source"] == "outbox_date"), None
)
reader_rows = [row for row in reads if row["source"] == "participants"]
known = {
peer.peer_id: peer
for peer in await get_peers(
pool, account_id, [row["reader_id"] for row in reader_rows]
)
}
readers = [
ReaderView(
peer_id=row["reader_id"],
read_at=row["read_at"],
peer=known.get(row["reader_id"]),
)
for row in reader_rows
]
outcome = fetch["outcome"] if fetch else inherited
now = datetime.now(UTC)
own = (
message["sender_id"] is not None
and message["sender_id"] == (message["tg_user_id"])
)
applicable = own and kind != ChatKind.CHANNEL and chat_id != message["tg_user_id"]
watermark = receipts["watermark"]
covered = watermark is not None and message_id <= watermark
seen = covered or any((exact_at, readers, receipts["listened_at"]))
evidence = watermark is not None or outcome in {"ok", "not_read_yet"}
due = fetch is None or fetch["retry_at"] <= now
return ReadInfoView(
chat_id=chat_id,
message_id=message_id,
chat_kind=kind.value,
own=own,
voice=message["voice"],
observed_at=observed_at,
listened_at=receipts["listened_at"],
exact_at=exact_at,
readers=readers,
read=applicable and seen,
unread=applicable and not seen and evidence,
outcome=outcome,
fetched_at=fetch["fetched_at"] if fetch else None,
can_fetch=applicable
and not message["is_bot"]
and now - date < FETCH_WINDOW
and exact_at is None
and inherited is None
and due,
)
backend/src/utils/read/watches.py
+27 -299
View File
@@ -4,70 +4,13 @@ from typing import Any
import asyncpg
from utils.read import chats as chats_read
from utils.read import peers as peers_read
from utils.read.models import (
AlertItem,
AlertMessage,
AlertPage,
AlertView,
AlertWatch,
ChatListItem,
ChatRef,
PeerView,
PersonRef,
WatchItem,
WatchView,
)
from utils.read.models import AlertView, Page, WatchView
WATCHES_CHANGED_CHANNEL = "watches_changed"
ALERTS_CHANGED_CHANNEL = "alerts_changed"
ALERT_PAGE = 50
MAX_ALERT_PAGE = 200
_WATCH_COLS = "id, account_id, kind, name, params, enabled, created_at, updated_at"
_WATCH_COLS = "id, account_id, kind, params, enabled, created_at, updated_at"
_ALERT_COLS = "id, account_id, watch_id, ts, payload, seen, created_at"
_LEGACY_EVENTS = {"keyword": "message", "peer_online": "online"}
_INSERT_ALERT = f"""
WITH alert AS (
INSERT INTO alerts (account_id, watch_id, ts, payload, seen, dedup_key)
VALUES ($1, $2, $3, $4::jsonb, false, $5)
ON CONFLICT (watch_id, dedup_key) WHERE dedup_key IS NOT NULL DO NOTHING
RETURNING {_ALERT_COLS}
), hooks AS (
INSERT INTO webhook_deliveries (webhook_id, alert_id, event)
SELECT w.id, alert.id, 'alert' FROM alert
JOIN webhooks w ON w.account_id = alert.account_id AND w.enabled
AND (w.watch_ids IS NULL OR alert.watch_id = ANY(w.watch_ids))
)
SELECT * FROM alert
""" # noqa: S608
_WATCH_STATS = """
SELECT watch_id, count(*) AS count, max(ts) AS last FROM alerts
WHERE watch_id = ANY($1::int[]) GROUP BY watch_id
"""
_ALERT_ROWS = """
SELECT a.id, a.account_id, a.ts, a.seen, a.payload,
w.id AS watch_id, w.kind AS watch_kind, w.name AS watch_name,
w.params AS watch_params,
m.text AS stored_text,
m.sender_id AS stored_sender, m.date AS stored_date,
m.deleted_at IS NOT NULL AS stored_deleted,
m.edited_at IS NOT NULL AS stored_edited
FROM alerts a
JOIN watches w ON w.id = a.watch_id
LEFT JOIN LATERAL (
SELECT text, sender_id, date, deleted_at, edited_at
FROM messages
WHERE account_id = a.account_id
AND chat_id = (a.payload->>'chat_id')::bigint
AND message_id = (a.payload->>'message_id')::bigint
ORDER BY date DESC LIMIT 1
) m ON true
"""
def _to_watch(row: asyncpg.Record) -> WatchView:
@@ -82,10 +25,6 @@ def _to_alert(row: asyncpg.Record) -> AlertView:
return AlertView(**data)
async def _notify(pool: asyncpg.Pool, channel: str, payload: dict[str, Any]) -> None:
await pool.execute("SELECT pg_notify($1, $2)", channel, json.dumps(payload))
async def list_watches(pool: asyncpg.Pool, account_id: int) -> list[WatchView]:
rows = await pool.fetch(
f"SELECT {_WATCH_COLS} FROM watches WHERE account_id = $1 " # noqa: S608
@@ -103,21 +42,19 @@ async def get_watch(pool: asyncpg.Pool, watch_id: int) -> WatchView | None:
return _to_watch(row) if row else None
async def create_watch( # noqa: PLR0913
async def create_watch(
pool: asyncpg.Pool,
account_id: int,
kind: str,
params: dict[str, Any],
*,
name: str | None = None,
enabled: bool = True,
) -> WatchView:
row = await pool.fetchrow(
"INSERT INTO watches (account_id, kind, name, params, enabled) " # noqa: S608
f"VALUES ($1, $2, $3, $4::jsonb, $5) RETURNING {_WATCH_COLS}",
"INSERT INTO watches (account_id, kind, params, enabled) " # noqa: S608
f"VALUES ($1, $2, $3::jsonb, $4) RETURNING {_WATCH_COLS}",
account_id,
kind,
name,
json.dumps(params),
enabled,
)
@@ -125,22 +62,13 @@ async def create_watch( # noqa: PLR0913
return _to_watch(row)
async def update_watch( # noqa: PLR0913
pool: asyncpg.Pool,
watch_id: int,
kind: str,
params: dict[str, Any],
*,
name: str | None,
enabled: bool,
async def update_watch(
pool: asyncpg.Pool, watch_id: int, params: dict[str, Any], *, enabled: bool
) -> WatchView | None:
row = await pool.fetchrow(
"UPDATE watches SET kind = $2, name = $3, params = $4::jsonb, " # noqa: S608
"enabled = $5, updated_at = now() "
"UPDATE watches SET params = $2::jsonb, enabled = $3, updated_at = now() " # noqa: S608
f"WHERE id = $1 RETURNING {_WATCH_COLS}",
watch_id,
kind,
name,
json.dumps(params),
enabled,
)
@@ -151,210 +79,29 @@ async def update_watch( # noqa: PLR0913
async def delete_watch(pool: asyncpg.Pool, watch_id: int) -> bool:
account_id = await pool.fetchval(
"DELETE FROM watches WHERE id = $1 RETURNING account_id", watch_id
)
if account_id is None:
result = await pool.execute("DELETE FROM watches WHERE id = $1", watch_id)
if not result.endswith("1"):
return False
await pool.execute(f"NOTIFY {WATCHES_CHANGED_CHANNEL}")
await _notify(pool, ALERTS_CHANGED_CHANNEL, {"account_id": account_id})
return True
def _chat_ref(chat_id: int, chat: ChatListItem | None, name: str | None) -> ChatRef:
if chat is None:
kind = "private" if chat_id > 0 else "group"
return ChatRef(id=chat_id, title=name, kind=kind, has_avatar=False)
kind = "channel" if chat.is_broadcast else chat.kind
return ChatRef(
id=chat_id, title=chat.title or name, kind=kind, has_avatar=chat.has_avatar
)
def _peer_name(peer: PeerView) -> str | None:
full = " ".join(part for part in (peer.first_name, peer.last_name) if part)
return full or peer.username
class Refs:
def __init__(self, chats: list[ChatListItem], people: list[PeerView]) -> None:
self._chats = {chat.chat_id: chat for chat in chats}
self._people = {peer.peer_id: peer for peer in people}
def chat(self, chat_id: int | None) -> ChatRef | None:
if chat_id is None:
return None
peer = self._people.get(chat_id)
name = _peer_name(peer) if peer else None
return _chat_ref(chat_id, self._chats.get(chat_id), name)
def person(self, person_id: int | None) -> PersonRef | None:
if person_id is None:
return None
peer = self._people.get(person_id)
chat = self._chats.get(person_id)
name = (_peer_name(peer) if peer else None) or (chat.title if chat else None)
has_avatar = (peer.has_avatar if peer else False) or bool(
chat and chat.has_avatar
)
return PersonRef(id=person_id, name=name, has_avatar=has_avatar)
async def load_refs(
pool: asyncpg.Pool, account_id: int, chat_ids: set[int], person_ids: set[int]
) -> Refs:
chats = await chats_read.get_chats(pool, account_id, sorted(chat_ids | person_ids))
people = await peers_read.get_peers(
pool, account_id, sorted(person_ids | {i for i in chat_ids if i > 0})
)
return Refs(chats, people)
def _int(value: object) -> int | None:
return value if isinstance(value, int) and not isinstance(value, bool) else None
def _watch_person(params: dict[str, Any]) -> int | None:
return _int(params.get("sender_id")) or _int(params.get("peer_id"))
async def describe_watches(
pool: asyncpg.Pool, account_id: int, watches: list[WatchView]
) -> list[WatchItem]:
targets = [
(_int(watch.params.get("chat_id")), _watch_person(watch.params))
for watch in watches
]
refs = await load_refs(
pool,
account_id,
{chat for chat, _ in targets if chat is not None},
{person for _, person in targets if person is not None},
)
stats = {
row["watch_id"]: row
for row in await pool.fetch(_WATCH_STATS, [watch.id for watch in watches])
}
return [
WatchItem(
**watch.model_dump(),
chat=refs.chat(chat),
person=refs.person(person),
alert_count=stats[watch.id]["count"] if watch.id in stats else 0,
last_alert_at=stats[watch.id]["last"] if watch.id in stats else None,
)
for watch, (chat, person) in zip(watches, targets, strict=True)
]
def _path(chat_id: int | None, message_id: int | None, peer_id: int | None) -> str:
if chat_id is not None and message_id is not None:
return f"/app/{chat_id}?message={message_id}"
if peer_id is not None:
return f"/app/{peer_id}"
return "/app"
def _alert_item(row: asyncpg.Record, refs: Refs) -> AlertItem:
payload = json.loads(row["payload"])
event = payload.get("event") or _LEGACY_EVENTS.get(row["watch_kind"], "message")
chat_id = _int(payload.get("chat_id"))
message_id = _int(payload.get("message_id"))
peer_id = _int(payload.get("peer_id"))
message = None
if message_id is not None:
date = row["stored_date"] or (
datetime.fromisoformat(payload["date"]) if payload.get("date") else None
)
message = AlertMessage(
id=message_id,
text=payload.get("text") or row["stored_text"],
sender=refs.person(_int(payload.get("sender_id")) or row["stored_sender"]),
date=date,
deleted=bool(row["stored_deleted"]) or event == "delete",
edited=bool(row["stored_edited"]) or event == "edit",
)
return AlertItem(
id=row["id"],
account_id=row["account_id"],
ts=row["ts"],
seen=row["seen"],
event=event,
watch=AlertWatch(
id=row["watch_id"],
kind=row["watch_kind"],
name=row["watch_name"],
params=json.loads(row["watch_params"]),
),
chat=refs.chat(chat_id),
message=message,
peer=refs.person(peer_id),
away_since=payload.get("since"),
path=_path(chat_id, message_id, peer_id),
)
async def _alert_items(
pool: asyncpg.Pool, account_id: int, rows: list[asyncpg.Record]
) -> list[AlertItem]:
chat_ids: set[int] = set()
person_ids: set[int] = set()
for row in rows:
payload = json.loads(row["payload"])
for key, bucket in (
("chat_id", chat_ids),
("sender_id", person_ids),
("peer_id", person_ids),
):
value = _int(payload.get(key))
if value is not None:
bucket.add(value)
if row["stored_sender"] is not None:
person_ids.add(row["stored_sender"])
refs = await load_refs(pool, account_id, chat_ids, person_ids)
return [_alert_item(row, refs) for row in rows]
async def list_alerts(
pool: asyncpg.Pool,
account_id: int,
*,
seen: bool | None = None,
before: int | None = None,
limit: int = ALERT_PAGE,
) -> AlertPage:
size = max(1, min(limit, MAX_ALERT_PAGE))
pool: asyncpg.Pool, account_id: int, page: Page, *, seen: bool | None = None
) -> list[AlertView]:
params: list[object] = [account_id]
where = "a.account_id = $1"
where = "account_id = $1"
if seen is not None:
params.append(seen)
where += f" AND a.seen = ${len(params)}"
if before is not None:
params.append(before)
where += f" AND a.id < ${len(params)}"
params.append(size + 1)
where += f" AND seen = ${len(params)}"
params.append(page.capped_limit)
params.append(page.offset)
rows = await pool.fetch(
f"{_ALERT_ROWS} WHERE {where} ORDER BY a.id DESC LIMIT ${len(params)}", *params
)
items = await _alert_items(pool, account_id, rows[:size])
return AlertPage(
items=items,
next_before=items[-1].id if len(rows) > size else None,
unseen=await unseen_count(pool, account_id),
)
async def get_alert(pool: asyncpg.Pool, alert_id: int) -> AlertItem | None:
row = await pool.fetchrow(f"{_ALERT_ROWS} WHERE a.id = $1", alert_id)
if row is None:
return None
return (await _alert_items(pool, row["account_id"], [row]))[0]
async def unseen_count(pool: asyncpg.Pool, account_id: int) -> int:
return await pool.fetchval(
"SELECT count(*) FROM alerts WHERE account_id = $1 AND NOT seen", account_id
f"SELECT {_ALERT_COLS} FROM alerts WHERE {where} " # noqa: S608
f"ORDER BY ts DESC LIMIT ${len(params) - 1} OFFSET ${len(params)}",
*params,
)
return [_to_alert(row) for row in rows]
async def insert_alert(
@@ -366,7 +113,10 @@ async def insert_alert(
dedup_key: str | None = None,
) -> AlertView | None:
row = await pool.fetchrow(
_INSERT_ALERT,
"INSERT INTO alerts (account_id, watch_id, ts, payload, seen, dedup_key) " # noqa: S608
"VALUES ($1, $2, $3, $4::jsonb, false, $5) "
"ON CONFLICT (watch_id, dedup_key) WHERE dedup_key IS NOT NULL "
f"DO NOTHING RETURNING {_ALERT_COLS}",
account_id,
watch_id,
datetime.now(UTC),
@@ -375,32 +125,10 @@ async def insert_alert(
)
if row is None:
return None
await _notify(
pool, ALERTS_CHANGED_CHANNEL, {"account_id": account_id, "alert_id": row["id"]}
)
await pool.execute(f"NOTIFY {ALERTS_CHANGED_CHANNEL}")
return _to_alert(row)
async def mark_alert_seen(pool: asyncpg.Pool, alert_id: int) -> bool:
account_id = await pool.fetchval(
"UPDATE alerts SET seen = true WHERE id = $1 RETURNING account_id", alert_id
)
if account_id is None:
return False
await _notify(pool, ALERTS_CHANGED_CHANNEL, {"account_id": account_id})
return True
async def mark_all_seen(
pool: asyncpg.Pool, account_id: int, up_to: int | None = None
) -> int:
result = await pool.execute(
"UPDATE alerts SET seen = true WHERE account_id = $1 AND NOT seen "
"AND ($2::int IS NULL OR id <= $2)",
account_id,
up_to,
)
marked = int(result.split()[-1])
if marked:
await _notify(pool, ALERTS_CHANGED_CHANNEL, {"account_id": account_id})
return marked
result = await pool.execute("UPDATE alerts SET seen = true WHERE id = $1", alert_id)
return result.endswith("1")
backend/src/utils/read/webhooks.py
-198
View File
@@ -1,198 +0,0 @@
import json
import secrets
from datetime import datetime
import asyncpg
from utils.read.models import DeliveryView, WebhookView
STATUS_PENDING = "pending"
STATUS_SENDING = "sending"
STATUS_DELIVERED = "delivered"
STATUS_FAILED = "failed"
DELIVERY_LOG = 20
_DELIVERY_COLS = (
"id, webhook_id, alert_id, event, status, attempts, next_attempt_at, "
"response_code, last_error, created_at, finished_at"
)
_HOOKS = f"""
SELECT w.id, w.account_id, w.url, w.secret, w.enabled, w.watch_ids,
w.created_at, w.updated_at, to_jsonb(d) AS last_delivery
FROM webhooks w
LEFT JOIN LATERAL (
SELECT {_DELIVERY_COLS} FROM webhook_deliveries
WHERE webhook_id = w.id ORDER BY id DESC LIMIT 1
) d ON true
""" # noqa: S608
_CLAIM = """
UPDATE webhook_deliveries d SET status = 'sending', attempts = d.attempts + 1
FROM webhooks w
WHERE w.id = d.webhook_id AND d.id IN (
SELECT id FROM webhook_deliveries
WHERE status = 'pending' AND next_attempt_at <= now()
ORDER BY next_attempt_at LIMIT $1
FOR UPDATE SKIP LOCKED
)
RETURNING d.id, d.alert_id, d.event, d.attempts, d.body,
w.id AS webhook_id, w.account_id, w.url, w.secret
"""
_FINISH = """
UPDATE webhook_deliveries SET
status = $2, response_code = $3, last_error = $4,
next_attempt_at = coalesce($5, next_attempt_at),
finished_at = CASE WHEN $2 = 'pending' THEN NULL ELSE now() END
WHERE id = $1
"""
def new_secret() -> str:
return secrets.token_hex(32)
def _to_webhook(row: asyncpg.Record) -> WebhookView:
data = dict(row)
last = data.pop("last_delivery")
return WebhookView(
**data, last_delivery=DeliveryView(**json.loads(last)) if last else None
)
async def list_webhooks(pool: asyncpg.Pool, account_id: int) -> list[WebhookView]:
rows = await pool.fetch(
f"{_HOOKS} WHERE w.account_id = $1 ORDER BY w.id", account_id
)
return [_to_webhook(row) for row in rows]
async def get_webhook(pool: asyncpg.Pool, webhook_id: int) -> WebhookView | None:
row = await pool.fetchrow(f"{_HOOKS} WHERE w.id = $1", webhook_id)
return _to_webhook(row) if row else None
async def create_webhook(
pool: asyncpg.Pool,
account_id: int,
url: str,
watch_ids: list[int] | None,
*,
enabled: bool,
) -> WebhookView | None:
webhook_id = await pool.fetchval(
"INSERT INTO webhooks (account_id, url, secret, enabled, watch_ids) "
"VALUES ($1, $2, $3, $4, $5) RETURNING id",
account_id,
url,
new_secret(),
enabled,
watch_ids,
)
return await get_webhook(pool, webhook_id)
async def update_webhook(
pool: asyncpg.Pool,
webhook_id: int,
url: str,
watch_ids: list[int] | None,
*,
enabled: bool,
) -> WebhookView | None:
async with pool.acquire() as conn, conn.transaction():
found = await conn.fetchval(
"UPDATE webhooks SET url = $2, enabled = $3, watch_ids = $4, "
"updated_at = now() WHERE id = $1 RETURNING id",
webhook_id,
url,
enabled,
watch_ids,
)
if found is not None and not enabled:
await conn.execute(
"UPDATE webhook_deliveries SET status = 'failed', "
"last_error = 'endpoint disabled', finished_at = now() "
"WHERE webhook_id = $1 AND status = 'pending'",
webhook_id,
)
return await get_webhook(pool, webhook_id)
async def rotate_secret(pool: asyncpg.Pool, webhook_id: int) -> WebhookView | None:
await pool.execute(
"UPDATE webhooks SET secret = $2, updated_at = now() WHERE id = $1",
webhook_id,
new_secret(),
)
return await get_webhook(pool, webhook_id)
async def delete_webhook(pool: asyncpg.Pool, webhook_id: int) -> bool:
result = await pool.execute("DELETE FROM webhooks WHERE id = $1", webhook_id)
return result.endswith("1")
async def list_deliveries(
pool: asyncpg.Pool, webhook_id: int, limit: int = DELIVERY_LOG
) -> list[DeliveryView]:
rows = await pool.fetch(
f"SELECT {_DELIVERY_COLS} FROM webhook_deliveries " # noqa: S608
"WHERE webhook_id = $1 ORDER BY id DESC LIMIT $2",
webhook_id,
limit,
)
return [DeliveryView(**dict(row)) for row in rows]
async def get_delivery(pool: asyncpg.Pool, delivery_id: int) -> DeliveryView | None:
row = await pool.fetchrow(
f"SELECT {_DELIVERY_COLS} FROM webhook_deliveries WHERE id = $1", # noqa: S608
delivery_id,
)
return DeliveryView(**dict(row)) if row else None
async def start_test(pool: asyncpg.Pool, webhook_id: int, body: str) -> int:
return await pool.fetchval(
"INSERT INTO webhook_deliveries (webhook_id, event, status, attempts, body) "
"VALUES ($1, 'test', 'sending', 1, $2) RETURNING id",
webhook_id,
body,
)
async def release_stuck(pool: asyncpg.Pool) -> None:
await pool.execute(
"UPDATE webhook_deliveries SET status = 'pending' WHERE status = 'sending'"
)
async def claim_due(pool: asyncpg.Pool, limit: int) -> list[asyncpg.Record]:
return await pool.fetch(_CLAIM, limit)
async def save_body(pool: asyncpg.Pool, delivery_id: int, body: str) -> None:
await pool.execute(
"UPDATE webhook_deliveries SET body = $2 WHERE id = $1", delivery_id, body
)
async def finish( # noqa: PLR0913
pool: asyncpg.Pool,
delivery_id: int,
status: str,
response_code: int | None,
error: str | None,
next_attempt_at: datetime | None = None,
) -> None:
await pool.execute(
_FINISH, delivery_id, status, response_code, error, next_attempt_at
)
async def next_due(pool: asyncpg.Pool) -> datetime | None:
return await pool.fetchval(
"SELECT min(next_attempt_at) FROM webhook_deliveries WHERE status = 'pending'"
)
backend/src/utils/search/models.py
+4 -41
View File
@@ -1,48 +1,23 @@
from datetime import UTC, datetime, timedelta
from datetime import datetime
from typing import Literal
from pydantic import BaseModel, field_validator
from pydantic import BaseModel
DEFAULT_LIMIT = 50
MAX_LIMIT = 500
_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
_MICROSECOND = timedelta(microseconds=1)
ChatKind = Literal["private", "group", "channel"]
HitSource = Literal["text", "stt", "version"]
def encode_cursor(date: datetime, chat_id: int, message_id: int) -> str:
return f"{(date - _EPOCH) // _MICROSECOND}_{chat_id}_{message_id}"
def decode_cursor(cursor: str) -> tuple[datetime, int, int]:
micros, chat_id, message_id = (int(part) for part in cursor.split("_"))
return _EPOCH + micros * _MICROSECOND, chat_id, message_id
class SearchFilters(BaseModel):
account_id: int
query: str | None = None
chat_id: int | None = None
sender_id: int | None = None
kind: ChatKind | None = None
has_media: bool | None = None
deleted: bool | None = None
edited: bool | None = None
date_from: datetime | None = None
date_to: datetime | None = None
regex: str | None = None
cursor: str | None = None
limit: int = DEFAULT_LIMIT
@field_validator("cursor")
@classmethod
def _valid_cursor(cls, value: str | None) -> str | None:
if value is not None:
decode_cursor(value)
return value
offset: int = 0
class SearchHit(BaseModel):
@@ -51,17 +26,5 @@ class SearchHit(BaseModel):
date: datetime
sender_id: int | None
text: str | None
has_media: bool
media_kind: str | None
edited_at: datetime | None
deleted_at: datetime | None
source: HitSource
source: Literal["text", "stt"]
extracted_text: str | None
version_text: str | None
version_at: datetime | None
class SearchPage(BaseModel):
hits: list[SearchHit]
next_cursor: str | None
terms: list[str]
backend/src/utils/search/repository.py
+72 -171
View File
@@ -1,187 +1,88 @@
import asyncpg
from utils.search.models import (
MAX_LIMIT,
ChatKind,
SearchFilters,
SearchHit,
SearchPage,
decode_cursor,
encode_cursor,
from utils.search.models import MAX_LIMIT, SearchFilters, SearchHit
_TEXT_TSV = "to_tsvector('russian', coalesce(m.text, ''))"
_STT_TSV = "to_tsvector('russian', coalesce(md.extracted_text, ''))"
_TSQUERY = "websearch_to_tsquery('russian', ${})"
_HITS = "{cols} FROM messages m WHERE {where}" # text branch
_STT_HITS = (
"{cols} FROM messages m "
"JOIN media md ON md.account_id = m.account_id "
"AND md.chat_id = m.chat_id AND md.message_id = m.message_id "
"WHERE {where}"
)
_TEXT_COLS = (
"SELECT m.chat_id, m.message_id, m.date, m.sender_id, m.text, "
"'text' AS source, NULL::text AS extracted_text"
)
_STT_COLS = (
"SELECT m.chat_id, m.message_id, m.date, m.sender_id, m.text, "
"'stt' AS source, md.extracted_text"
)
_PARSE = """
SELECT (regexp_replace(websearch_to_tsquery('russian', q)::text,
'''((?:[^'']|''''){3,})''', '''\\1'':*', 'g')::tsquery
|| regexp_replace(websearch_to_tsquery('simple', q)::text,
'''((?:[^'']|''''){3,})''', '''\\1'':*', 'g')::tsquery
)::text AS tsquery,
tsvector_to_array(search_vector(q)) AS terms
FROM translate($1, 'Ёё', 'Ее') AS q
"""
_CHANNELS = """
SELECT coalesce(array_agg(chat_id), '{}'::bigint[]) FROM (
SELECT DISTINCT ON (chat_id) chat_id,
coalesce(raw->'chat'->>'type', raw->>'type') AS type
FROM chat_history
WHERE account_id = $1
AND coalesce(raw->'chat'->>'type', raw->>'type') IS NOT NULL
ORDER BY chat_id, ts DESC
) typed WHERE type = 'ChatType.CHANNEL'
"""
_KEY_ORDER = "s.date DESC, s.chat_id DESC, s.message_id DESC"
_COLS = (
"s.chat_id, s.message_id, s.date, s.sender_id, s.text, s.has_media, "
"s.edited_at, s.deleted_at"
)
_TEXT = (
f"SELECT {_COLS}, 0 AS rank, NULL::text AS extracted_text, " # noqa: S608
"NULL::text AS version_text, NULL::timestamptz AS version_at "
"FROM message_search s WHERE {where}"
)
_PROBE = (
"CROSS JOIN LATERAL (SELECT * FROM message_search s "
"WHERE s.chat_id = {source}.chat_id AND s.message_id = {source}.message_id "
"AND {where} LIMIT 1) s "
)
_STT = (
f"SELECT {_COLS}, 1 AS rank, md.extracted_text, NULL::text, " # noqa: S608
"NULL::timestamptz FROM media md "
f"{_PROBE.replace('{source}', 'md')}"
"WHERE md.account_id = {account} "
"AND search_vector(md.extracted_text) @@ {tsquery}"
)
_VERSION = (
"SELECT DISTINCT ON (s.date, s.chat_id, s.message_id) " # noqa: S608
f"{_COLS}, 2 AS rank, NULL::text, v.text, v.observed_at "
"FROM message_versions v "
f"{_PROBE.replace('{source}', 'v')}"
"WHERE v.account_id = {account} "
"AND search_vector(v.text) @@ {tsquery}"
)
_WITH_MEDIA = (
"SELECT h.*, md.kind AS media_kind FROM ({hits}) h "
"LEFT JOIN media md ON md.account_id = {account} "
"AND md.chat_id = h.chat_id AND md.message_id = h.message_id "
"ORDER BY h.date DESC, h.chat_id DESC, h.message_id DESC"
)
_SOURCES = ("text", "stt", "version")
class _Binder:
def __init__(self) -> None:
self.params: list[object] = []
def __call__(self, value: object) -> str:
self.params.append(value)
return f"${len(self.params)}"
def _kind(kind: ChatKind, channels: list[int], bind: _Binder) -> str:
if kind == "private":
return "s.chat_id > 0"
member = f"s.chat_id = ANY({bind(channels)}::bigint[])"
return member if kind == "channel" else f"s.chat_id < 0 AND NOT {member}"
def _conditions(
filters: SearchFilters, channels: list[int], bind: _Binder
) -> list[str]:
conds = [f"s.account_id = {bind(filters.account_id)}"]
compared = (
("s.chat_id =", filters.chat_id),
("s.sender_id =", filters.sender_id),
("s.has_media =", filters.has_media),
("s.date >=", filters.date_from),
("s.date <=", filters.date_to),
("s.text ~", filters.regex),
)
conds += [f"{left} {bind(value)}" for left, value in compared if value is not None]
flags = (("s.deleted_at", filters.deleted), ("s.edited_at", filters.edited))
conds += [
f"{column} IS {'NOT ' if flag else ''}NULL"
for column, flag in flags
if flag is not None
]
if filters.kind is not None:
conds.append(_kind(filters.kind, channels, bind))
if filters.cursor is not None:
date, chat_id, message_id = decode_cursor(filters.cursor)
conds.append(
"(s.date, s.chat_id, s.message_id) < "
f"({bind(date)}, {bind(chat_id)}, {bind(message_id)})"
)
def _common_conditions(filters: SearchFilters, params: list[object]) -> list[str]:
params.append(filters.account_id)
conds = [f"m.account_id = ${len(params)}"]
if filters.chat_id is not None:
params.append(filters.chat_id)
conds.append(f"m.chat_id = ${len(params)}")
if filters.sender_id is not None:
params.append(filters.sender_id)
conds.append(f"m.sender_id = ${len(params)}")
if filters.has_media is not None:
params.append(filters.has_media)
conds.append(f"m.has_media = ${len(params)}")
if filters.date_from is not None:
params.append(filters.date_from)
conds.append(f"m.date >= ${len(params)}")
if filters.date_to is not None:
params.append(filters.date_to)
conds.append(f"m.date <= ${len(params)}")
return conds
def _branch(sql: str, limit: str, order: str = _KEY_ORDER, **parts: str) -> str:
return f"({sql.format(**parts)} ORDER BY {order} LIMIT {limit})"
async def search_messages(
pool: asyncpg.Pool, filters: SearchFilters
) -> list[SearchHit]:
query = filters.query.strip() if filters.query else None
params: list[object] = []
common = _common_conditions(filters, params)
if filters.regex is not None:
params.append(filters.regex)
common.append(f"m.text ~ ${len(params)}")
q_idx: int | None = None
if query is not None:
params.append(query)
q_idx = len(params)
async def search_messages(pool: asyncpg.Pool, filters: SearchFilters) -> SearchPage:
async with pool.acquire() as conn, conn.transaction():
await conn.execute("SET LOCAL jit = off")
return await _search(conn, filters)
text_conds = [*common]
if q_idx is not None:
text_conds.append(f"{_TEXT_TSV} @@ {_TSQUERY.format(q_idx)}")
branches = [_HITS.format(cols=_TEXT_COLS, where=" AND ".join(text_conds))]
if q_idx is not None:
stt_conds = [*common, f"{_STT_TSV} @@ {_TSQUERY.format(q_idx)}"]
branches.append(_STT_HITS.format(cols=_STT_COLS, where=" AND ".join(stt_conds)))
async def _search(conn: asyncpg.Connection, filters: SearchFilters) -> SearchPage:
query = (filters.query or "").strip()
terms: list[str] = []
tsquery = None
if query:
parsed = await conn.fetchrow(_PARSE, query)
terms = list(parsed["terms"])
if not parsed["tsquery"]:
return SearchPage(hits=[], next_cursor=None, terms=terms)
tsquery = parsed["tsquery"]
params.append(min(filters.limit, MAX_LIMIT))
limit_idx = len(params)
params.append(filters.offset)
offset_idx = len(params)
page = f"ORDER BY date DESC LIMIT ${limit_idx} OFFSET ${offset_idx}"
channels: list[int] = []
if filters.kind in {"group", "channel"}:
channels = await conn.fetchval(_CHANNELS, filters.account_id)
bind = _Binder()
where = " AND ".join(_conditions(filters, channels, bind))
size = min(filters.limit, MAX_LIMIT)
limit = bind(size + 1)
account = bind(filters.account_id)
if tsquery is None:
sql = _branch(_TEXT, limit, where=where)
if len(branches) == 1:
sql = f"SELECT * FROM ({branches[0]}) hits {page}" # noqa: S608
else:
match = f"{bind(tsquery)}::tsquery"
branches = (
_branch(_TEXT, limit, where=f"{where} AND s.tsv @@ {match}"),
_branch(_STT, limit, where=where, account=account, tsquery=match),
_branch(
_VERSION,
limit,
f"{_KEY_ORDER}, v.observed_at DESC",
where=where,
account=account,
tsquery=match,
),
)
sql = (
"SELECT DISTINCT ON (date, chat_id, message_id) * FROM (" # noqa: S608
f"{' UNION ALL '.join(branches)}) s "
f"ORDER BY {_KEY_ORDER}, rank LIMIT {limit}"
union = " UNION ALL ".join(f"({b})" for b in branches)
dedup = (
"SELECT DISTINCT ON (chat_id, message_id) * " # noqa: S608
f"FROM ({union}) hits ORDER BY chat_id, message_id, source DESC"
)
sql = f"SELECT * FROM ({dedup}) d {page}" # noqa: S608
rows = await conn.fetch(_WITH_MEDIA.format(hits=sql, account=account), *bind.params)
hits = [
SearchHit(**{**dict(row), "source": _SOURCES[row["rank"]]})
for row in rows[:size]
]
next_cursor = None
if len(rows) > size:
last = hits[-1]
next_cursor = encode_cursor(last.date, last.chat_id, last.message_id)
return SearchPage(hits=hits, next_cursor=next_cursor, terms=terms)
rows = await pool.fetch(sql, *params)
return [SearchHit(**dict(row)) for row in rows]
backend/src/utils/watch_rules.py
-145
View File
@@ -1,145 +0,0 @@
import re
from collections.abc import Iterable, Iterator
from importlib import import_module
from typing import Any, Literal, Self, get_args
from pydantic import BaseModel, Field, ValidationError, model_validator
MAX_PATTERN = 200
MAX_TEXT = 4096
MAX_NAME = 120
WatchKind = Literal[
"keyword", "new_message", "message_deleted", "message_edited", "peer_online"
]
WATCH_KINDS: tuple[str, ...] = get_args(WatchKind)
sre_parser = import_module("re._parser")
_REPEATS = {sre_parser.MAX_REPEAT, sre_parser.MIN_REPEAT, sre_parser.POSSESSIVE_REPEAT}
class WatchParamsError(ValueError):
pass
def _subpatterns(value: object) -> Iterator[Any]:
if isinstance(value, sre_parser.SubPattern):
yield value
elif isinstance(value, tuple | list):
for item in value:
yield from _subpatterns(item)
def _repeat_depth(items: Iterable[tuple[Any, Any]]) -> int:
depth = 0
for op, value in items:
if op in _REPEATS:
low, high, body = value
depth = max(depth, _repeat_depth(body) + (high > max(low, 1)))
else:
inner = (_repeat_depth(sub) for sub in _subpatterns(value))
depth = max(depth, *inner, 0)
return depth
def compile_pattern(pattern: str) -> re.Pattern[str]:
try:
compiled = re.compile(pattern, re.IGNORECASE)
except re.error as exc:
msg = f"invalid regex: {exc}"
raise ValueError(msg) from exc
if _repeat_depth(sre_parser.parse(pattern)) > 1:
msg = "regex repeats inside a repeat, like (a+)+, which can hang"
raise ValueError(msg)
return compiled
class KeywordParams(BaseModel):
pattern: str = Field(min_length=1, max_length=MAX_PATTERN)
regex: bool = False
chat_id: int | None = None
sender_id: int | None = None
@model_validator(mode="before")
@classmethod
def _legacy_keyword(cls, data: object) -> object:
if isinstance(data, dict) and "pattern" not in data and "keyword" in data:
return {**data, "pattern": data["keyword"]}
return data
@model_validator(mode="after")
def _valid_pattern(self) -> Self:
if not self.pattern.strip():
msg = "pattern is blank"
raise ValueError(msg)
if self.regex:
compile_pattern(self.pattern)
return self
class NewMessageParams(BaseModel):
chat_id: int | None = None
sender_id: int | None = None
@model_validator(mode="after")
def _scoped(self) -> Self:
if self.chat_id is None and self.sender_id is None:
msg = "set chat_id, sender_id or both"
raise ValueError(msg)
return self
class DeletedParams(BaseModel):
chat_id: int | None = None
class EditedParams(BaseModel):
chat_id: int | None = None
class PeerOnlineParams(BaseModel):
peer_id: int
WatchParams = (
KeywordParams | NewMessageParams | DeletedParams | EditedParams | PeerOnlineParams
)
PARAMS_BY_KIND: dict[str, type[WatchParams]] = {
"keyword": KeywordParams,
"new_message": NewMessageParams,
"message_deleted": DeletedParams,
"message_edited": EditedParams,
"peer_online": PeerOnlineParams,
}
def _describe(exc: ValidationError) -> str:
parts = []
for error in exc.errors():
where = ".".join(str(part) for part in error["loc"])
message = error["msg"].removeprefix("Value error, ")
parts.append(f"{where}: {message}" if where else message)
return "; ".join(parts)
def parse_params(kind: str, params: dict[str, Any]) -> WatchParams:
model = PARAMS_BY_KIND.get(kind)
if model is None:
msg = f"unknown kind {kind!r}, expected one of {', '.join(WATCH_KINDS)}"
raise WatchParamsError(msg)
try:
return model.model_validate(params)
except ValidationError as exc:
raise WatchParamsError(_describe(exc)) from exc
def normalize_params(kind: str, params: dict[str, Any]) -> dict[str, Any]:
return parse_params(kind, params).model_dump(exclude_none=True)
def clean_name(name: str | None) -> str | None:
if name is None:
return None
name = name.strip()[:MAX_NAME]
return name or None
backend/src/watcher/registry.py
+16 -53
View File
@@ -1,7 +1,6 @@
import asyncio
import contextlib
from dataclasses import dataclass
from datetime import timedelta
import asyncpg
from pyrogram import Client, enums
@@ -16,7 +15,6 @@ from userbot.modules.capture import capture_message
from userbot.modules.capture.chat_meta import meta_from_chat
from userbot.modules.client import PyroClient
from userbot.modules.jobs import JobConsumer
from userbot.modules.watches.seen import seen_message
from utils.events import notify_bg_event
from utils.jobs import enqueue
from utils.logging import logger
@@ -31,7 +29,6 @@ TOKEN_MISSING = "Токен не найден на диске" # noqa: S105
INVALID_TOKEN_ERRORS = ("ACCESS_TOKEN_INVALID", "ACCESS_TOKEN_EXPIRED")
BACKLOG_ATTEMPTS = 3
BACKLOG_RETRY_PAUSE = 60
ENRICH_COOLDOWN = timedelta(hours=6)
@dataclass
@@ -54,11 +51,9 @@ def _handlers(pool: asyncpg.Pool) -> tuple:
if ctx is None or message.empty or message.chat is None or message.date is None:
return
meta = meta_from_chat(message.chat, ctx.contacts.ids)
seen = seen_message(meta.chat_id, message)
if message.edit_date is None:
await ctx.watches.on_message(seen)
else:
await ctx.watches.on_edit(seen)
await ctx.watches.on_text(
meta.chat_id, message.id, message.text or message.caption
)
toggles = ctx.resolve(meta)
if not toggles.messages:
return
@@ -72,24 +67,14 @@ def _handlers(pool: asyncpg.Pool) -> tuple:
)
if message.chat.type is not enums.ChatType.PRIVATE:
await enqueue(
pool,
ctx.account_id,
"enrich_chat",
{"chat_id": meta.chat_id},
cooldown=ENRICH_COOLDOWN,
pool, ctx.account_id, "enrich_chat", {"chat_id": meta.chat_id}
)
async def on_member(client: Client, update) -> None: # noqa: ANN001
ctx = getattr(client, "capture", None)
if ctx is None or update.chat is None:
return
await enqueue(
pool,
ctx.account_id,
"enrich_chat",
{"chat_id": update.chat.id},
cooldown=ENRICH_COOLDOWN,
)
await enqueue(pool, ctx.account_id, "enrich_chat", {"chat_id": update.chat.id})
return (
MessageHandler(on_message),
@@ -130,12 +115,6 @@ class BotRegistry:
for bot_id in list(self._running):
await self._stop(bot_id)
async def refresh_watches(self) -> None:
for running in list(self._running.values()):
capture = getattr(running.client, "capture", None)
if capture is not None:
await capture.watches.refresh()
async def _adopt_dropped_in(self) -> None:
unknown = tokens.stored_bot_ids() - await bots.known_bot_ids(self._pool)
for bot_id in unknown:
@@ -188,33 +167,17 @@ class BotRegistry:
await client.stop()
return
try:
account_id = await accounts.sync_account(
self._pool, me, str(bot_id), is_bot=True
)
client.capture = await build_bot_capture_context(
client, self._pool, self._storage, account_id
)
await bots.link_account(self._pool, bot_id, account_id)
await bots.set_status(
self._pool,
bot_id,
bots.STATUS_OK,
username=me.username,
title=me.full_name,
)
await accounts.notify_accounts_changed(self._pool)
except Exception as exc:
logger.exception(
f"[red]Bot {bot_id} failed to start,[/] retrying on the next sync"
)
with contextlib.suppress(Exception):
await bots.set_status(
self._pool, bot_id, bots.STATUS_ERROR, detail=str(exc)
)
with contextlib.suppress(Exception):
await client.stop()
return
account_id = await accounts.sync_account(
self._pool, me, str(bot_id), is_bot=True
)
client.capture = await build_bot_capture_context(
client, self._pool, self._storage, account_id
)
await bots.link_account(self._pool, bot_id, account_id)
await bots.set_status(
self._pool, bot_id, bots.STATUS_OK, username=me.username, title=me.full_name
)
await accounts.notify_accounts_changed(self._pool)
consumer = JobConsumer(client, self._pool, account_id)
self._running[bot_id] = RunningBot(
backend/src/watcher/runner.py
+16 -24
View File
@@ -6,10 +6,9 @@ import asyncpg
import uvloop
from dependencies.container import container
from utils.listen import Listener
from utils.env import env
from utils.logging import logger, setup_logging
from utils.read.bots import BOTS_CHANGED_CHANNEL
from utils.read.watches import WATCHES_CHANGED_CHANNEL
from utils.storage import ContentAddressedStorage
from watcher.registry import BotRegistry
@@ -21,32 +20,23 @@ RETRY_INTERVAL = 300
async def _retry_loop(registry: BotRegistry) -> None:
while True:
await asyncio.sleep(RETRY_INTERVAL)
try:
with contextlib.suppress(Exception):
await registry.sync()
except Exception:
logger.exception("[red]Bot sync failed[/]")
def _change_listener(registry: BotRegistry, tasks: set[asyncio.Task]) -> Listener:
def on_change(_payload: str) -> None:
async def _listen_changes(
registry: BotRegistry, tasks: set[asyncio.Task]
) -> asyncpg.Connection:
def on_change(
_conn: asyncpg.Connection, _pid: int, _channel: str, _payload: str
) -> None:
task = asyncio.create_task(registry.sync())
tasks.add(task)
task.add_done_callback(tasks.discard)
def on_watch(_payload: str) -> None:
task = asyncio.create_task(registry.refresh_watches())
tasks.add(task)
task.add_done_callback(tasks.discard)
def on_reconnect() -> None:
on_change("")
on_watch("")
return Listener(
"watcher",
{BOTS_CHANGED_CHANNEL: on_change, WATCHES_CHANGED_CHANNEL: on_watch},
on_reconnect,
)
conn = await asyncpg.connect(dsn=env.db.connection_url)
await conn.add_listener(BOTS_CHANGED_CHANNEL, on_change)
return conn
async def runner() -> None:
@@ -54,7 +44,7 @@ async def runner() -> None:
storage = await container.get(ContentAddressedStorage)
registry = BotRegistry(pool, storage)
tasks: set[asyncio.Task] = set()
listener = _change_listener(registry, tasks)
listen_conn: asyncpg.Connection | None = None
stop = asyncio.Event()
asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, stop.set)
retry = asyncio.create_task(_retry_loop(registry))
@@ -62,14 +52,16 @@ async def runner() -> None:
await registry.sync()
if not registry.running:
logger.warning("[yellow]No bots yet. Add a bot token in the web UI.[/]")
await listener.start()
listen_conn = await _listen_changes(registry, tasks)
logger.info("[green]Bot watcher running.[/]")
await stop.wait()
finally:
retry.cancel()
with contextlib.suppress(asyncio.CancelledError):
await retry
await listener.stop()
if listen_conn is not None:
with contextlib.suppress(Exception):
await listen_conn.close()
await registry.close()
await container.close()
backend/uv.lock
Generated
+7 -8
View File
@@ -149,7 +149,6 @@ dependencies = [
{ name = "fastapi" },
{ name = "fastmcp" },
{ name = "greenlet" },
{ name = "httpx2" },
{ name = "kurigram" },
{ name = "pydantic-settings" },
{ name = "rich" },
@@ -167,7 +166,6 @@ requires-dist = [
{ name = "fastapi", specifier = ">=0.115.0" },
{ name = "fastmcp", specifier = ">=3.3" },
{ name = "greenlet", specifier = ">=3.1.0" },
{ name = "httpx2", specifier = ">=2.13.1" },
{ name = "kurigram", specifier = ">=2.2.23" },
{ name = "pydantic-settings", specifier = ">=2.7.0" },
{ name = "rich", specifier = ">=13.9.0" },
@@ -413,7 +411,7 @@ wheels = [
[[package]]
name = "fastapi"
version = "0.143.0"
version = "0.143.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "annotated-doc" },
@@ -423,9 +421,9 @@ dependencies = [
{ name = "typing-extensions" },
{ name = "typing-inspection" },
]
sdist = { url = "https://files.pythonhosted.org/packages/0b/d7/6a8753ab6c1d432dc53703c3e1b92974a94531b7d047c32bbaae461ea844/fastapi-0.143.0.tar.gz", hash = "sha256:1acffe48206a80917cf7dac21992b5c44b25384e8902bf745c1fd9dabcf6c51f", size = 468391, upload-time = "2026-10-08T12:29:46.54Z" }
sdist = { url = "https://files.pythonhosted.org/packages/38/d5/b0aeb0b303650ac87b0411de4aaf3f901b06042dcc3fb00f30b97b7ff585/fastapi-0.143.3.tar.gz", hash = "sha256:b9ab567ee57afb4454004ce04c889f8f0160ac47946b99795887cd6e88484848", size = 469834, upload-time = "2026-10-16T14:14:13.31Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/bd/f4/27e386913417ad32aae42bba48b0c0cce40e9ff2fba1a871ca2702c37324/fastapi-0.143.0-py3-none-any.whl", hash = "sha256:3e9395fd35276425b61b516a31fdd7c77fe2af83e41b4da22e30696fb1304c5d", size = 144665, upload-time = "2026-10-08T12:29:44.853Z" },
{ url = "https://files.pythonhosted.org/packages/2c/1a/03d246befcf307219bb2161c97d8c5d314f84125bf7dd245730a28589828/fastapi-0.143.3-py3-none-any.whl", hash = "sha256:75e8bd68e2f52f77d29b2879749c257d64de2d85282d25270caf2bfccce5efe5", size = 144849, upload-time = "2026-10-16T14:14:11.763Z" },
]
[[package]]
@@ -1082,16 +1080,17 @@ wheels = [
[[package]]
name = "pydantic-settings"
version = "2.15.0"
version = "2.16.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pydantic" },
{ name = "python-dotenv" },
{ name = "typing-extensions" },
{ name = "typing-inspection" },
]
sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" }
sdist = { url = "https://files.pythonhosted.org/packages/2e/3b/a5d2294799b53b448319978cfb5bd139d5a9d45e862af91661614f14c922/pydantic_settings-2.16.0.tar.gz", hash = "sha256:5b6c578049ede4db0e2ef3b4eaa4ad4069cfa9211f83fb38df899dfade50a614", size = 301031, upload-time = "2026-10-14T12:44:09.998Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" },
{ url = "https://files.pythonhosted.org/packages/53/f4/b987bf8c51e5b19a95fa66d1ee596074141e085d9c2ddf97920803c7029b/pydantic_settings-2.16.0-py3-none-any.whl", hash = "sha256:7e73acf7f61936a15e5a3b6eedaea29f133357faf7272f2607ba479b049dd7f2", size = 80364, upload-time = "2026-10-14T12:44:08.233Z" },
]
[[package]]
frontend/src/app.html
+1 -10
View File
@@ -1,18 +1,9 @@
<!doctype html>
<html lang="ru">
<html lang="en">
<head>
<meta charset="utf-8" />
<meta content="width=device-width, initial-scale=1" name="viewport" />
<meta content="scale" name="text-scale" />
<meta content="#ffffff" media="(prefers-color-scheme: light)" name="theme-color" />
<meta content="#212121" media="(prefers-color-scheme: dark)" name="theme-color" />
<meta content="yes" name="mobile-web-app-capable" />
<meta content="black-translucent" name="apple-mobile-web-app-status-bar-style" />
<meta content="Beavergram" name="apple-mobile-web-app-title" />
<link href="/icon.svg" rel="icon" type="image/svg+xml" />
<link href="/apple-touch-icon.png" rel="apple-touch-icon" />
<link href="/manifest.webmanifest" rel="manifest" />
<title>Beavergram</title>
<script>
(() => {
const stored = localStorage.getItem("bg.theme");
frontend/src/lib/api/avatars.ts
+7 -21
View File
@@ -4,27 +4,13 @@ import { PUBLIC_API_BASE_URL } from "$app/env/public";
const BASE = PUBLIC_API_BASE_URL;
const RETRY_DELAY = 2500;
const MISSING_TTL = 120_000;
export type AvatarKind = "peer" | "chat";
const MAX_CACHED = 240;
const ready = new Map<string, string>();
const missing = new Map<string, number>();
function isMissing(key: string): boolean {
const since = missing.get(key);
if (since === undefined) {
return false;
}
if (Date.now() - since < MISSING_TTL) {
return true;
}
missing.delete(key);
return false;
}
const missing = new Set<string>();
const inflight = new Map<string, Promise<string | null>>();
function cacheKey(account: number, kind: AvatarKind, id: number): string {
@@ -74,15 +60,15 @@ async function fetchAvatar(
await delay(RETRY_DELAY);
return fetchAvatar(account, kind, id, key, false);
}
missing.set(key, Date.now());
missing.add(key);
return null;
}
export function loadAvatar(
kind: AvatarKind,
id: number,
account: number | null = accounts.selectedId
id: number
): Promise<string | null> {
const account = accounts.selectedId;
if (account === null) {
return Promise.resolve(null);
}
@@ -91,7 +77,7 @@ export function loadAvatar(
if (cached) {
return Promise.resolve(cached);
}
if (isMissing(key)) {
if (missing.has(key)) {
return Promise.resolve(null);
}
const existing = inflight.get(key);
@@ -124,7 +110,7 @@ async function fetchVariant(
await delay(RETRY_DELAY);
return fetchVariant(account, kind, id, uniqueId, key, false);
}
missing.set(key, Date.now());
missing.add(key);
return null;
}
@@ -142,7 +128,7 @@ export function loadAvatarVariant(
if (cached) {
return Promise.resolve(cached);
}
if (isMissing(key)) {
if (missing.has(key)) {
return Promise.resolve(null);
}
const existing = inflight.get(key);
frontend/src/lib/api/endpoints.ts
+64 -94
View File
@@ -1,6 +1,7 @@
import { request } from "#lib/api/client.js";
import type {
Account,
Alert,
Annotation,
AvatarHistoryView,
CallbackView,
@@ -9,11 +10,6 @@ import type {
ChatLinkView,
DayCount,
DiscoverItem,
FeedChatKind,
FeedKind,
FeedMessagesPage,
FeedPage,
FeedUnseen,
Folder,
JobStatus,
JobView,
@@ -24,8 +20,7 @@ import type {
MessageAt,
MessageVersion,
MessageView,
PeerGroups,
PeerOverview,
PeerHistoryView,
PeerView,
PinnedView,
PolicyChatKind,
@@ -33,16 +28,12 @@ import type {
PolicyRecord,
PolicyRule,
PolicyScope,
PresenceHourly,
PresenceSample,
PresenceSession,
PresenceStats,
ProfileChange,
ReactionView,
ReadInfo,
ResponseStats,
ScopeTarget,
SearchPage,
SearchQuery,
SearchHit,
StorageChat,
StorageFile,
StorageGroupName,
@@ -52,6 +43,7 @@ import type {
TrashItem,
TrashOutcome,
VolumeBucket,
Watch,
WatchedBot,
} from "#lib/api/types.js";
import { accounts } from "#lib/stores/accounts.svelte.js";
@@ -213,38 +205,30 @@ export function getCurrentPresence(
});
}
export function getPresenceStats(
export function getPresenceHistory(
peerId: number,
days: number,
tz: string,
page: Page = {}
): Promise<PresenceStats> {
return request<PresenceStats>("/presence/stats", {
): Promise<PresenceSample[]> {
return request<PresenceSample[]>("/presence", {
account: true,
query: { days, peer_id: peerId, tz, ...page },
query: { peer_id: peerId, ...page },
});
}
export function getPresenceSessions(
peerId: number,
days: number,
tz: string,
page: Page = {}
): Promise<PresenceSession[]> {
return request<PresenceSession[]>("/presence/sessions", {
export function getPresenceHourly(peerId: number): Promise<PresenceHourly[]> {
return request<PresenceHourly[]>("/presence/hourly", {
account: true,
query: { days, peer_id: peerId, tz, ...page },
query: { peer_id: peerId },
});
}
export function getMessageVolume(
chatId: number,
days: number,
tz: string
days = 90
): Promise<VolumeBucket[]> {
return request<VolumeBucket[]>("/analytics/volume", {
account: true,
query: { chat_id: chatId, days, tz },
query: { chat_id: chatId, days },
});
}
@@ -280,53 +264,12 @@ export function listDeleted(
});
}
export interface FeedQuery {
chat_id?: number;
chat_kind?: FeedChatKind;
cursor?: string;
kind?: FeedKind;
limit?: number;
}
export function listFeed(query: FeedQuery = {}): Promise<FeedPage> {
return request<FeedPage>("/feed", { account: true, query: { ...query } });
}
export function getFeedUnseen(): Promise<FeedUnseen> {
return request<FeedUnseen>("/feed/unseen", { account: true });
}
export function markFeedSeen(at?: string): Promise<FeedUnseen> {
return request<FeedUnseen>("/feed/seen", {
body: { account_id: accounts.selectedId, at },
method: "POST",
});
}
export function listFeedMessages(
itemId: number,
query: { cursor?: string; limit?: number } = {}
): Promise<FeedMessagesPage> {
return request<FeedMessagesPage>(`/feed/${itemId}/messages`, {
account: true,
query: { ...query },
});
}
export function getPeer(peerId: number): Promise<PeerView> {
return request<PeerView>(`/peers/${peerId}`, { account: true });
}
export function getPeerOverview(peerId: number): Promise<PeerOverview> {
return request<PeerOverview>(`/peers/${peerId}/overview`, { account: true });
}
export function getPeerGroups(peerId: number): Promise<PeerGroups> {
return request<PeerGroups>(`/peers/${peerId}/groups`, { account: true });
}
export function getChatChanges(chatId: number): Promise<ProfileChange[]> {
return request<ProfileChange[]>(`/chats/${chatId}/changes`, {
export function getPeerHistory(peerId: number): Promise<PeerHistoryView[]> {
return request<PeerHistoryView[]>(`/peers/${peerId}/history`, {
account: true,
});
}
@@ -361,25 +304,6 @@ export function getChatLinks(
});
}
export function getReadInfo(
chatId: number,
messageId: number
): Promise<ReadInfo> {
return request<ReadInfo>(`/messages/${chatId}/${messageId}/read`, {
account: true,
});
}
export function refreshReadInfo(
chatId: number,
messageId: number
): Promise<{ job_id: number | null }> {
return request<{ job_id: number | null }>(
`/messages/${chatId}/${messageId}/read/refresh`,
{ body: { account_id: accounts.selectedId }, method: "POST" }
);
}
export function getMessageReactions(
chatId: number,
messageId: number
@@ -556,8 +480,14 @@ export function getMessageMedia(
});
}
export function searchMessages(query: SearchQuery): Promise<SearchPage> {
return request<SearchPage>("/search", { account: true, query: { ...query } });
export function searchMessages(
query: string,
options: Page & { chat_id?: number } = {}
): Promise<SearchHit[]> {
return request<SearchHit[]>("/search", {
account: true,
query: { query, ...options },
});
}
export function fetchMedia(
@@ -588,6 +518,46 @@ export function transcribeMedia(
});
}
export function listWatches(): Promise<Watch[]> {
return request<Watch[]>("/watches", { account: true });
}
export function createWatch(
kind: string,
params: Record<string, unknown>,
enabled: boolean
): Promise<Watch> {
return request<Watch>("/watches", {
body: { account_id: accounts.selectedId, enabled, kind, params },
method: "POST",
});
}
export function updateWatch(
id: number,
params: Record<string, unknown>,
enabled: boolean
): Promise<Watch> {
return request<Watch>(`/watches/${id}`, {
body: { enabled, params },
method: "PUT",
});
}
export function deleteWatch(id: number): Promise<void> {
return request<void>(`/watches/${id}`, { method: "DELETE" });
}
export function listAlerts(
options: Page & { seen?: boolean } = {}
): Promise<Alert[]> {
return request<Alert[]>("/alerts", { account: true, query: { ...options } });
}
export function markAlertSeen(id: number): Promise<void> {
return request<void>(`/alerts/${id}/seen`, { method: "POST" });
}
export function listAnnotations(
options: { chatId?: number; messageId?: number } = {}
): Promise<Annotation[]> {
frontend/src/lib/api/types.ts
+34 -213
View File
@@ -1,5 +1,3 @@
import type { LiveAlertEvent } from "#lib/api/watches.js";
export type ChatKind = "private" | "group";
export type PolicyScopeType =
| "default_dm"
@@ -7,9 +5,7 @@ export type PolicyScopeType =
| "default_channel"
| "folder"
| "chat";
export type SearchSource = "text" | "stt" | "version";
export type SearchChatKind = "private" | "group" | "channel";
export type SearchSource = "text" | "stt";
export type JobStatus =
| "pending"
| "running"
@@ -208,7 +204,6 @@ export interface MessageView {
quote: string | null;
reactions: ReactionCount[];
read: boolean;
read_at: string | null;
reply: ReplyView | null;
sender_id: number | null;
service: ServiceView | null;
@@ -224,53 +219,6 @@ export interface MessageVersion {
text: string | null;
}
export type FeedKind = "all" | "deleted" | "edited" | "profile";
export type FeedChatKind = "dm" | "group" | "channel";
export interface FeedChat {
chat_id: number;
has_avatar: boolean;
kind: FeedChatKind;
title: string | null;
}
export interface FeedItem {
after: string | null;
at: string;
before: string | null;
chat: FeedChat;
detail: Record<string, [string | null, string | null]> | null;
edit_date: string | null;
has_before: boolean;
id: number;
kind: Exclude<FeedKind, "all">;
message: MessageView | null;
message_count: number;
message_id: number;
outgoing: boolean;
sender_id: number | null;
sender_name: string | null;
since: string;
}
export interface FeedPage {
items: FeedItem[];
next_cursor: string | null;
}
export interface FeedUnseen {
count: number;
deleted: number;
edited: number;
latest: FeedItem | null;
seen_at: string | null;
}
export interface FeedMessagesPage {
messages: MessageView[];
next_cursor: string | null;
}
export interface MediaVersion {
file_size: number | null;
id: number;
@@ -283,37 +231,11 @@ export interface MediaVersion {
export interface SearchHit {
chat_id: number;
date: string;
deleted_at: string | null;
edited_at: string | null;
extracted_text: string | null;
has_media: boolean;
media_kind: string | null;
message_id: number;
sender_id: number | null;
source: SearchSource;
text: string | null;
version_at: string | null;
version_text: string | null;
}
export interface SearchPage {
hits: SearchHit[];
next_cursor: string | null;
terms: string[];
}
export interface SearchQuery {
chat_id?: number;
cursor?: string;
date_from?: string;
date_to?: string;
deleted?: boolean;
edited?: boolean;
has_media?: boolean;
kind?: SearchChatKind;
limit?: number;
query?: string;
sender_id?: number;
}
export interface MediaView {
@@ -413,57 +335,12 @@ export interface PresenceSample {
ts: string;
}
export interface PresenceSession {
ended: string;
ongoing: boolean;
online_seconds: number;
seconds: number;
started: string;
}
export interface PresenceHour {
covered_hours: number;
hour: number;
hours: number;
online_minutes: number | null;
weekday: number;
}
export interface PresenceDay {
covered_seconds: number;
day: string;
online_seconds: number;
}
export interface CoverageGap {
ended: string;
started: string;
}
export interface PresenceSummary {
covered_seconds: number;
daily_seconds: number | null;
first_online: number | null;
last_offline: number | null;
export interface PresenceHourly {
bucket: string;
last_seen: string | null;
median_session_seconds: number | null;
online_seconds: number;
session_count: number;
}
export interface PresenceStats {
current: PresenceSample | null;
days: PresenceDay[];
gaps: CoverageGap[];
hidden: boolean;
online: boolean;
online_samples: number;
peer_id: number;
sessions: PresenceSession[];
since: string;
summary: PresenceSummary;
tz: string;
until: string;
week: PresenceHour[];
samples: number;
}
export interface VolumeBucket {
@@ -492,88 +369,14 @@ export interface PeerView {
username: string | null;
}
export interface ReaderView {
peer: PeerView | null;
peer_id: number;
read_at: string;
}
export type ReadOutcome =
| "ok"
| "not_read_yet"
| "user_privacy"
| "your_privacy"
| "chat_too_big"
| "too_old"
| "unavailable"
| "flood"
| "error";
export interface ReadInfo {
can_fetch: boolean;
chat_id: number;
chat_kind: "dm" | "group" | "channel";
exact_at: string | null;
fetched_at: string | null;
listened_at: string | null;
message_id: number;
observed_at: string | null;
outcome: ReadOutcome | null;
own: boolean;
read: boolean;
readers: ReaderView[];
unread: boolean;
voice: boolean;
}
export type ProfileField = "name" | "username" | "phone" | "deleted" | "title";
export interface ProfileChange {
actor_id: number | null;
changed_at: string;
field: ProfileField;
message_id: number | null;
new: string | null;
old: string | null;
}
export interface MembershipView {
actor_id: number | null;
actor_name: string | null;
chat_id: number;
chat_title: string | null;
event: "join" | "leave";
has_avatar: boolean;
message_id: number;
ts: string;
}
export interface DirectChatView {
chat_id: number;
last_date: string | null;
message_count: number;
}
export interface PeerOverview {
avatars: AvatarHistoryView[];
changes: ProfileChange[];
dm: DirectChatView | null;
memberships: MembershipView[];
peer: PeerView | null;
}
export interface SharedGroupView {
chat_id: number;
has_avatar: boolean;
last_date: string;
last_message_id: number;
message_count: number;
title: string | null;
}
export interface PeerGroups {
deleted_count: number;
groups: SharedGroupView[];
export interface PeerHistoryView {
first_name: string | null;
is_deleted_account: boolean;
last_name: string | null;
observed_at: string;
phone: string | null;
photo_unique_id: string | null;
username: string | null;
}
export interface StoryView {
@@ -660,6 +463,26 @@ export interface Folder {
title: string;
}
export interface Watch {
account_id: number;
created_at: string;
enabled: boolean;
id: number;
kind: string;
params: Record<string, unknown>;
updated_at: string;
}
export interface Alert {
account_id: number;
created_at: string;
id: number;
payload: Record<string, unknown>;
seen: boolean;
ts: string;
watch_id: number;
}
export interface JobView {
account_id: number;
attempts: number;
@@ -695,7 +518,6 @@ export interface LivePresenceEvent {
export interface LiveReceiptEvent {
chat_id: number;
read_at: string;
read_up_to: number;
type: "receipt";
}
@@ -704,8 +526,7 @@ export type LiveEvent =
| LiveMessageEvent
| LiveDeleteEvent
| LivePresenceEvent
| LiveReceiptEvent
| LiveAlertEvent;
| LiveReceiptEvent;
export interface FileShare {
account_id: number;
frontend/src/lib/api/watches.ts
-210
View File
@@ -1,210 +0,0 @@
import { request } from "#lib/api/client.js";
import { accounts } from "#lib/stores/accounts.svelte.js";
export type WatchKind =
| "keyword"
| "new_message"
| "message_deleted"
| "message_edited"
| "peer_online";
export interface WatchParams {
chat_id?: number;
pattern?: string;
peer_id?: number;
regex?: boolean;
sender_id?: number;
}
export type StoredParams = WatchParams & { keyword?: string };
export interface ChatRef {
has_avatar: boolean;
id: number;
kind: "private" | "group" | "channel";
title: string | null;
}
export interface PersonRef {
has_avatar: boolean;
id: number;
name: string | null;
}
export interface Watch {
account_id: number;
alert_count: number;
chat: ChatRef | null;
created_at: string;
enabled: boolean;
id: number;
kind: WatchKind;
last_alert_at: string | null;
name: string | null;
params: StoredParams;
person: PersonRef | null;
updated_at: string;
}
export interface WatchDraft {
enabled: boolean;
kind: WatchKind;
name: string | null;
params: WatchParams;
}
export type AlertEvent =
| "message"
| "edit"
| "transcript"
| "delete"
| "online";
export interface AlertMessage {
date: string | null;
deleted: boolean;
edited: boolean;
id: number;
sender: PersonRef | null;
text: string | null;
}
export interface AlertItem {
account_id: number;
away_since: string | null;
chat: ChatRef | null;
event: AlertEvent;
id: number;
message: AlertMessage | null;
path: string;
peer: PersonRef | null;
seen: boolean;
ts: string;
watch: {
id: number;
kind: WatchKind;
name: string | null;
params: StoredParams;
};
}
export interface AlertPage {
items: AlertItem[];
next_before: number | null;
unseen: number;
}
export interface LiveAlertEvent {
alert: AlertItem | null;
type: "alert";
unseen: number;
}
export interface Webhook {
account_id: number;
created_at: string;
enabled: boolean;
id: number;
last_delivery: Delivery | null;
secret: string;
updated_at: string;
url: string;
watch_ids: number[] | null;
}
export interface WebhookDraft {
enabled: boolean;
url: string;
watch_ids: number[] | null;
}
export interface Delivery {
alert_id: number | null;
attempts: number;
created_at: string;
event: "alert" | "test";
finished_at: string | null;
id: number;
last_error: string | null;
next_attempt_at: string;
response_code: number | null;
status: "pending" | "sending" | "delivered" | "failed";
webhook_id: number;
}
export function listWatches(): Promise<Watch[]> {
return request<Watch[]>("/watches", { account: true });
}
export function createWatch(draft: WatchDraft): Promise<Watch> {
return request<Watch>("/watches", {
body: { account_id: accounts.selectedId, ...draft },
method: "POST",
});
}
export function updateWatch(id: number, draft: WatchDraft): Promise<Watch> {
return request<Watch>(`/watches/${id}`, { body: draft, method: "PUT" });
}
export function deleteWatch(id: number): Promise<void> {
return request<void>(`/watches/${id}`, { method: "DELETE" });
}
export function listAlerts(
options: { before?: number; limit?: number; seen?: boolean } = {}
): Promise<AlertPage> {
return request<AlertPage>("/alerts", { account: true, query: options });
}
export async function unseenAlerts(): Promise<number> {
const { unseen } = await request<{ unseen: number }>("/alerts/unseen", {
account: true,
});
return unseen;
}
export function markAlertSeen(id: number): Promise<void> {
return request<void>(`/alerts/${id}/seen`, { method: "POST" });
}
export function markAllAlertsSeen(upTo?: number): Promise<{ marked: number }> {
return request<{ marked: number }>("/alerts/seen", {
body: { account_id: accounts.selectedId, up_to: upTo ?? null },
method: "POST",
});
}
export function listWebhooks(): Promise<Webhook[]> {
return request<Webhook[]>("/webhooks", { account: true });
}
export function createWebhook(draft: WebhookDraft): Promise<Webhook> {
return request<Webhook>("/webhooks", {
body: { account_id: accounts.selectedId, ...draft },
method: "POST",
});
}
export function updateWebhook(
id: number,
draft: WebhookDraft
): Promise<Webhook> {
return request<Webhook>(`/webhooks/${id}`, { body: draft, method: "PUT" });
}
export function deleteWebhook(id: number): Promise<void> {
return request<void>(`/webhooks/${id}`, { method: "DELETE" });
}
export function rotateWebhookSecret(id: number): Promise<Webhook> {
return request<Webhook>(`/webhooks/${id}/secret`, { method: "POST" });
}
export function testWebhook(id: number): Promise<Delivery> {
return request<Delivery>(`/webhooks/${id}/test`, { method: "POST" });
}
export function listDeliveries(id: number): Promise<Delivery[]> {
return request<Delivery[]>(`/webhooks/${id}/deliveries`);
}
frontend/src/lib/assets/favicon.svg
+16
View File
@@ -0,0 +1,16 @@
<svg
height="128"
viewBox="0 0 107 128"
width="107"
xmlns="http://www.w3.org/2000/svg"
>
<title>svelte-logo</title>
<path
d="M94.157 22.819c-10.4-14.885-30.94-19.297-45.792-9.835L22.282 29.608A29.92 29.92 0 0 0 8.764 49.65a31.5 31.5 0 0 0 3.108 20.231 30 30 0 0 0-4.477 11.183 31.9 31.9 0 0 0 5.448 24.116c10.402 14.887 30.942 19.297 45.791 9.835l26.083-16.624A29.92 29.92 0 0 0 98.235 78.35a31.53 31.53 0 0 0-3.105-20.232 30 30 0 0 0 4.474-11.182 31.88 31.88 0 0 0-5.447-24.116"
style="fill:#ff3e00"
/>
<path
d="M45.817 106.582a20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.503 18 18 0 0 1 .624-2.435l.49-1.498 1.337.981a33.6 33.6 0 0 0 10.203 5.098l.97.294-.09.968a5.85 5.85 0 0 0 1.052 3.878 6.24 6.24 0 0 0 6.695 2.485 5.8 5.8 0 0 0 1.603-.704L69.27 76.28a5.43 5.43 0 0 0 2.45-3.631 5.8 5.8 0 0 0-.987-4.371 6.24 6.24 0 0 0-6.698-2.487 5.7 5.7 0 0 0-1.6.704l-9.953 6.345a19 19 0 0 1-5.296 2.326 20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.502 17.99 17.99 0 0 1 8.13-12.052l26.081-16.623a19 19 0 0 1 5.3-2.329 20.72 20.72 0 0 1 22.237 8.243 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-.624 2.435l-.49 1.498-1.337-.98a33.6 33.6 0 0 0-10.203-5.1l-.97-.294.09-.968a5.86 5.86 0 0 0-1.052-3.878 6.24 6.24 0 0 0-6.696-2.485 5.8 5.8 0 0 0-1.602.704L37.73 51.72a5.42 5.42 0 0 0-2.449 3.63 5.79 5.79 0 0 0 .986 4.372 6.24 6.24 0 0 0 6.698 2.486 5.8 5.8 0 0 0 1.602-.704l9.952-6.342a19 19 0 0 1 5.295-2.328 20.72 20.72 0 0 1 22.237 8.242 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-8.13 12.053l-26.081 16.622a19 19 0 0 1-5.3 2.328"
style="fill:#fff"
/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.6 KiB

frontend/src/lib/components/ChatHeader.svelte
+54 -56
View File
@@ -1,6 +1,10 @@
<script lang="ts">
import { enqueueBackfill, getPeer } from "#lib/api/endpoints.js";
import type { PeerView } from "#lib/api/types.js";
import {
enqueueBackfill,
getCurrentPresence,
getPeer,
} from "#lib/api/endpoints.js";
import type { PeerView, PresenceSample } from "#lib/api/types.js";
import Avatar from "#lib/components/ui/Avatar.svelte";
import Button from "#lib/components/ui/Button.svelte";
import ContextMenu from "#lib/components/ui/ContextMenu.svelte";
@@ -8,17 +12,13 @@
import Icon from "#lib/components/ui/Icon.svelte";
import { peerName } from "#lib/format/peer.js";
import { formatPresence, isOnline } from "#lib/format/presence.js";
import { t } from "#lib/i18n/locale.svelte.js";
import { accounts } from "#lib/stores/accounts.svelte.js";
import { chats } from "#lib/stores/chats.svelte.js";
import { discover } from "#lib/stores/discover.svelte.js";
import { withoutMessage } from "#lib/stores/feed.svelte.js";
import { policyUi } from "#lib/stores/policy.svelte.js";
import { presence } from "#lib/stores/presence.svelte.js";
import { toasts } from "#lib/stores/toasts.svelte.js";
import { ui } from "#lib/stores/ui.svelte.js";
import { goto } from "$app/navigation";
import { page } from "$app/state";
interface Props {
chatId: number;
@@ -26,10 +26,13 @@
let { chatId }: Props = $props();
const PRESENCE_INTERVAL = 30_000;
const isDm = $derived(chatId > 0);
const chat = $derived(chats.byId(chatId));
const discovered = $derived(discover.get(chatId));
let peer = $state<PeerView | null>(null);
let presence = $state<PresenceSample | null>(null);
let backfilling = $state(false);
async function backfill() {
@@ -39,20 +42,14 @@
backfilling = true;
try {
await enqueueBackfill(chatId, true);
toasts.success(t("chat.backfill.started"));
toasts.success("Догружаем новые сообщения");
} catch {
toasts.error(t("chat.backfill.failed"));
toasts.error("Не удалось запустить бэкфилл");
} finally {
backfilling = false;
}
}
function back() {
const target = new URL(page.url.href);
target.pathname = "/app";
goto(withoutMessage(target));
}
$effect(() => {
if (accounts.selectedId === null) {
return;
@@ -84,22 +81,46 @@
});
$effect(() => {
presence.follow(accounts.selectedId, isDm ? chatId : null);
return () => presence.follow(null, null);
if (accounts.selectedId === null || !isDm) {
presence = null;
return;
}
let active = true;
presence = null;
const refresh = () => {
if (document.visibilityState !== "visible") {
return;
}
getCurrentPresence(chatId)
.then((result) => {
if (active) {
presence = result;
}
})
.catch(() => {
if (active) {
presence = null;
}
});
};
refresh();
const timer = setInterval(refresh, PRESENCE_INTERVAL);
return () => {
active = false;
clearInterval(timer);
};
});
const fallbackTitle = $derived(
chat?.title ??
discovered?.title ??
(isDm
? t("chat.deletedAccount")
: t("chat.fallbackTitle", { id: chatId }))
(isDm ? "Удалённый аккаунт" : `Chat ${chatId}`)
);
const title = $derived(isDm && peer ? peerName(peer) : fallbackTitle);
const subtitle = $derived.by(() => {
if (isDm) {
if (presence.sample) {
return formatPresence(presence.sample, presence.now);
if (presence) {
return formatPresence(presence);
}
if (peer?.username) {
return `@${peer.username}`;
@@ -108,11 +129,9 @@
}
const count = chat?.message_count ?? discovered?.message_count ?? 0;
if (count > 0) {
return t("chat.messages", { count });
return `${count} messages`;
}
return discovered?.kind === "channel"
? t("chat.kind.channel")
: t("chat.kind.group");
return discovered?.kind === "channel" ? "channel" : "group";
});
const avatarKind = $derived(isDm ? "peer" : "chat");
const hasAvatar = $derived(
@@ -120,15 +139,11 @@
);
</script>
<svelte:head>
<title>{title} · Beavergram</title>
</svelte:head>
<header class="chat-header">
<div class="back">
<Button
aria-label={t("common.back")}
onclick={back}
aria-label="Назад"
onclick={() => goto("/app")}
round
smaller
variant="translucent"
@@ -142,7 +157,7 @@
})}
<button
{...props}
aria-label={t("chat.openProfile")}
aria-label="Открыть профиль"
class="peek"
onclick={() => ui.openPanel("profile")}
type="button"
@@ -157,12 +172,7 @@
/>
<div class="info">
<h2 class="title">{title}</h2>
<span
class={[
"subtitle",
{ online: isDm && isOnline(presence.sample, presence.now) },
]}
>
<span class="subtitle" class:online={isDm && isOnline(presence)}>
{subtitle}
</span>
</div>
@@ -170,26 +180,23 @@
{/snippet}
{#snippet menu()}
<ContextMenuItem icon="info" onselect={() => ui.openPanel("profile")}
>{t("chat.openProfile")}</ContextMenuItem
>
<ContextMenuItem icon="recent" onselect={() => ui.openPanel("changes")}
>{t("feed.changes")}</ContextMenuItem
>Открыть профиль</ContextMenuItem
>
<ContextMenuItem
icon="play-story"
onselect={() => ui.openPanel("stories")}
>{t("settings.stories")}</ContextMenuItem
>Сторис</ContextMenuItem
>
<ContextMenuItem
icon="settings"
onselect={() => policyUi.edit({ chatId }, title)}
>{t("settings.policy")}</ContextMenuItem
>Политика захвата</ContextMenuItem
>
{/snippet}
</ContextMenu>
<div class="actions">
<Button
aria-label={t("chat.search")}
aria-label="Поиск в чате"
onclick={() => ui.openPanel("search")}
round
smaller
@@ -198,16 +205,7 @@
<Icon name="search" />
</Button>
<Button
aria-label={t("feed.changes")}
onclick={() => ui.openPanel("changes")}
round
smaller
variant="translucent"
>
<Icon name="recent" />
</Button>
<Button
aria-label={t("chat.analytics")}
aria-label="Аналитика"
onclick={() => ui.openPanel("presence")}
round
smaller
@@ -216,7 +214,7 @@
<Icon name="stats" />
</Button>
<Button
aria-label={t("chat.backfill")}
aria-label="Догрузить новые сообщения"
loading={backfilling}
onclick={backfill}
round
frontend/src/lib/components/ChatList.svelte
+4 -11
View File
@@ -1,10 +1,8 @@
<script lang="ts">
import { untrack } from "svelte";
import ChatListItem from "#lib/components/ChatListItem.svelte";
import FeedRow from "#lib/components/feed/FeedRow.svelte";
import EmptyState from "#lib/components/ui/EmptyState.svelte";
import Skeleton from "#lib/components/ui/Skeleton.svelte";
import { t } from "#lib/i18n/locale.svelte.js";
import { accounts } from "#lib/stores/accounts.svelte.js";
import { chats } from "#lib/stores/chats.svelte.js";
import { folders } from "#lib/stores/folders.svelte.js";
@@ -48,7 +46,7 @@
return;
}
untrack(() => folders.load()).catch(() =>
toasts.error(t("chats.foldersFailed"))
toasts.error("Failed to load folders")
);
});
@@ -62,7 +60,7 @@
scrollTop = 0;
}
untrack(() => chats.load(folderId)).catch(() =>
toasts.error(t("chats.loadFailed"))
toasts.error("Failed to load chats")
);
});
@@ -101,9 +99,6 @@
bind:this={viewport}
bind:clientHeight={viewportHeight}
>
{#if folders.selectedId === null}
<FeedRow />
{/if}
{#if chats.loading && list.length === 0}
{#each skeletonRows as index (index)}
<div class="row-skeleton">
@@ -118,10 +113,8 @@
<EmptyState
description={folders.selectedId === null
? undefined
: t("chats.emptyFolderHint")}
title={folders.selectedId === null
? t("chats.empty")
: t("chats.emptyFolder")}
: "No chats match this folder yet"}
title={folders.selectedId === null ? "No chats yet" : "Empty folder"}
/>
{:else}
<div style:padding-bottom="{padBottom}px" style:padding-top="{padTop}px">
frontend/src/lib/components/ChatListItem.svelte
+9 -15
View File
@@ -5,7 +5,6 @@
import ContextMenu from "#lib/components/ui/ContextMenu.svelte";
import ContextMenuItem from "#lib/components/ui/ContextMenuItem.svelte";
import { formatListDate } from "#lib/format/datetime.js";
import { t } from "#lib/i18n/locale.svelte.js";
import { accounts } from "#lib/stores/accounts.svelte.js";
import { peers } from "#lib/stores/peers.svelte.js";
import { policyUi } from "#lib/stores/policy.svelte.js";
@@ -27,9 +26,7 @@
const title = $derived(
chat.title ??
(chat.chat_id > 0
? t("chat.deletedAccount")
: t("chat.fallbackTitle", { id: chat.chat_id }))
(chat.chat_id > 0 ? "Удалённый аккаунт" : `Chat ${chat.chat_id}`)
);
const avatarKind = $derived(chat.chat_id > 0 ? "peer" : "chat");
const ownId = $derived(accounts.selected?.tg_user_id ?? null);
@@ -48,7 +45,7 @@
return "";
}
if (chat.last_sender_id === ownId) {
return `${t("chat.you")}: `;
return "You: ";
}
const peer = peers.get(chat.last_sender_id as number);
if (!peer) {
@@ -58,7 +55,7 @@
});
const preview = $derived(
chat.last_text ?? (chat.message_count > 0 ? t("chat.media") : "")
chat.last_text ?? (chat.message_count > 0 ? "Media" : "")
);
</script>
@@ -98,27 +95,24 @@
{/snippet}
{#snippet menu()}
<ContextMenuItem icon="open-in-new-tab" onselect={onclick}
>{t("chat.open")}</ContextMenuItem
>Открыть</ContextMenuItem
>
<ContextMenuItem icon="info" onselect={() => openWith("profile")}
>{t("chat.profile")}</ContextMenuItem
>Профиль</ContextMenuItem
>
<ContextMenuItem icon="search" onselect={() => openWith("search")}
>{t("chat.search")}</ContextMenuItem
>
<ContextMenuItem icon="recent" onselect={() => openWith("changes")}
>{t("feed.changes")}</ContextMenuItem
>Поиск в чате</ContextMenuItem
>
<ContextMenuItem icon="stats" onselect={() => openWith("presence")}
>{t("chat.analytics")}</ContextMenuItem
>Аналитика</ContextMenuItem
>
<ContextMenuItem icon="play-story" onselect={() => openWith("stories")}
>{t("settings.stories")}</ContextMenuItem
>Сторис</ContextMenuItem
>
<ContextMenuItem
icon="settings"
onselect={() => policyUi.edit({ chatId: chat.chat_id }, title)}
>{t("settings.policy")}</ContextMenuItem
>Политика захвата</ContextMenuItem
>
{/snippet}
</ContextMenu>
frontend/src/lib/components/MainMenu.svelte
-172
View File
@@ -1,172 +0,0 @@
<script lang="ts">
import { DropdownMenu } from "bits-ui";
import AlertBadge from "#lib/components/alerts/AlertBadge.svelte";
import AccountAvatar from "#lib/components/settings/AccountAvatar.svelte";
import AddAccountDialog from "#lib/components/settings/AddAccountDialog.svelte";
import Button from "#lib/components/ui/Button.svelte";
import Icon from "#lib/components/ui/Icon.svelte";
import { accountName } from "#lib/format/peer.js";
import { t } from "#lib/i18n/locale.svelte.js";
import type { MessageKey } from "#lib/i18n/messages.js";
import { accounts } from "#lib/stores/accounts.svelte.js";
import { theme } from "#lib/stores/theme.svelte.js";
import { type RightPanel, ui } from "#lib/stores/ui.svelte.js";
interface Entry {
icon: string;
label: MessageKey;
panel: RightPanel;
}
const DIGIT = /^Digit([1-9])$/;
const entries: Entry[] = [
{ icon: "eye", label: "settings.watches", panel: "watches" },
{ icon: "unmute", label: "settings.alerts", panel: "alerts" },
{ icon: "data", label: "settings.storage", panel: "storage" },
{ icon: "play-story", label: "settings.stories", panel: "stories-all" },
{ icon: "cloud-download", label: "settings.jobs", panel: "jobs" },
{ icon: "permissions", label: "settings.policy", panel: "policy" },
];
let adding = $state(false);
function onkeydown(event: KeyboardEvent) {
const match = DIGIT.exec(event.code);
if (!(event.altKey && match)) {
return;
}
const account = accounts.list[Number(match[1]) - 1];
if (account) {
event.preventDefault();
accounts.switchTo(account.account_id);
}
}
</script>
<svelte:window {onkeydown} />
<DropdownMenu.Root>
<DropdownMenu.Trigger>
{#snippet child({
props,
})}
<span class="menu-trigger">
<Button
{...props}
aria-label={t("menu.open")}
round
smaller
variant="translucent"
>
<Icon name="menu" />
</Button>
<AlertBadge />
</span>
{/snippet}
</DropdownMenu.Trigger>
<DropdownMenu.Portal>
<DropdownMenu.Content
align="start"
class="bg-menu-content main-menu"
sideOffset={6}
>
{#each accounts.list as account (account.account_id)}
<DropdownMenu.Item
class="bg-menu-item account-item"
onSelect={() => accounts.switchTo(account.account_id)}
>
<AccountAvatar {account} size={1.75} />
<span class="account-name">{accountName(account)}</span>
{#if !account.is_active}
<span class="account-tag">{t("menu.archived")}</span>
{/if}
{#if account.account_id === accounts.selectedId}
<Icon class="account-check" name="check" size="1.125rem" />
{/if}
</DropdownMenu.Item>
{/each}
<DropdownMenu.Item
class="bg-menu-item"
onSelect={() => {
adding = true;
}}
>
<Icon name="add" size="1.125rem" />
{t("menu.addAccount")}
</DropdownMenu.Item>
<DropdownMenu.Separator class="bg-menu-separator" />
<DropdownMenu.Item
class="bg-menu-item"
onSelect={() => ui.openSettings()}
>
<Icon name="settings" size="1.125rem" />
{t("settings.title")}
</DropdownMenu.Item>
{#each entries as entry (entry.panel)}
<DropdownMenu.Item
class="bg-menu-item"
onSelect={() => ui.openPanel(entry.panel)}
>
<Icon name={entry.icon} size="1.125rem" />
{#if entry.panel === "alerts"}
<span class="account-name">{t(entry.label)}</span>
<AlertBadge inline />
{:else}
{t(entry.label)}
{/if}
</DropdownMenu.Item>
{/each}
<DropdownMenu.Separator class="bg-menu-separator" />
<DropdownMenu.Item
class="bg-menu-item"
closeOnSelect={false}
onSelect={() => theme.toggle()}
>
<Icon name="darkmode" size="1.125rem" />
<span class="account-name">{t("menu.nightMode")}</span>
{#if theme.resolved === "dark"}
<Icon class="account-check" name="check" size="1.125rem" />
{/if}
</DropdownMenu.Item>
</DropdownMenu.Content>
</DropdownMenu.Portal>
</DropdownMenu.Root>
<AddAccountDialog bind:open={adding} />
<style lang="scss">
.menu-trigger {
position: relative;
display: flex;
flex-shrink: 0;
}
:global(.main-menu) {
width: 17rem;
max-height: calc(100dvh - 5rem);
overflow-y: auto;
}
:global(.main-menu .account-item) {
padding-block: 0.375rem;
}
.account-name {
overflow: hidden;
flex: 1;
text-overflow: ellipsis;
white-space: nowrap;
}
.account-tag {
flex-shrink: 0;
font-size: 0.75rem;
color: var(--color-text-secondary);
}
:global(.main-menu .account-check) {
flex-shrink: 0;
color: var(--color-primary);
}
</style>
frontend/src/lib/components/MessageBubble.svelte
+33 -102
View File
@@ -11,16 +11,13 @@
import Poll from "#lib/components/Poll.svelte";
import Reactions from "#lib/components/Reactions.svelte";
import ReplyHeader from "#lib/components/ReplyHeader.svelte";
import MessageReadItem from "#lib/components/read/MessageReadItem.svelte";
import Avatar from "#lib/components/ui/Avatar.svelte";
import ContextMenu from "#lib/components/ui/ContextMenu.svelte";
import ContextMenuItem from "#lib/components/ui/ContextMenuItem.svelte";
import Icon from "#lib/components/ui/Icon.svelte";
import WebPage from "#lib/components/WebPage.svelte";
import { accountName, peerColorIndex, peerName } from "#lib/format/peer.js";
import { t } from "#lib/i18n/locale.svelte.js";
import { accounts } from "#lib/stores/accounts.svelte.js";
import { chats } from "#lib/stores/chats.svelte.js";
import { peers } from "#lib/stores/peers.svelte.js";
import { toasts } from "#lib/stores/toasts.svelte.js";
import { ui } from "#lib/stores/ui.svelte.js";
@@ -62,47 +59,27 @@
);
const ownId = $derived(accounts.selected?.tg_user_id ?? null);
const senderChat = $derived(
message.sender_id !== null && message.sender_id < 0
? message.sender_id
: null
);
const sender = $derived(
message.sender_id === null || senderChat !== null
? undefined
: peers.get(message.sender_id)
message.sender_id === null ? undefined : peers.get(message.sender_id)
);
const senderName = $derived.by(() => {
if (own) {
return accounts.selected ? accountName(accounts.selected) : t("chat.you");
}
if (senderChat !== null) {
return chats.byId(senderChat)?.title ?? "";
return accounts.selected ? accountName(accounts.selected) : "You";
}
if (sender) {
return peerName(sender);
}
return message.sender_id === null ? "" : String(message.sender_id);
});
$effect(() => {
if (senderChat !== null) {
chats.ensure(senderChat);
}
});
const colorIndex = $derived(peerColorIndex(message.sender_id ?? 0));
const showName = $derived(isGroupChat && !own && firstInGroup);
const avatarId = $derived(own ? ownId : message.sender_id);
const avatarHas = $derived.by(() => {
if (own) {
return ownId !== null && Boolean(peers.get(ownId)?.has_avatar);
}
if (senderChat !== null) {
return chats.byId(senderChat)?.has_avatar ?? false;
}
return sender?.has_avatar ?? false;
});
const avatarHas = $derived(
own
? ownId !== null && peers.get(ownId)?.has_avatar
: (sender?.has_avatar ?? false)
);
const hasCallbacks = $derived(
message.inline_buttons.some((row) =>
@@ -139,19 +116,16 @@
}
async function copyText() {
await navigator.clipboard.writeText(message.text ?? "");
toasts.success(t("thread.copiedText"));
toasts.success("Текст скопирован");
}
async function copyId() {
await navigator.clipboard.writeText(String(message.message_id));
toasts.success(t("thread.copiedId"));
toasts.success("ID скопирован");
}
function openSenderProfile() {
if (senderChat === message.chat_id) {
if (message.sender_id !== null) {
goto(`/app/${message.sender_id}`);
ui.openPanel("profile");
} else if (senderChat !== null) {
goto(`/app/${senderChat}`);
} else if (message.sender_id !== null) {
ui.openPerson(message.chat_id, message.sender_id);
}
}
</script>
@@ -175,36 +149,20 @@
{#if isGroupChat}
<div class="avatar-slot">
{#if lastInGroup && avatarId !== null}
<button
aria-label={senderName}
class="sender-avatar"
onclick={openSenderProfile}
type="button"
>
<Avatar
avatar={{
id: avatarId,
kind: senderChat === null ? "peer" : "chat",
}}
colorKey={avatarId}
hasAvatar={avatarHas}
name={senderName}
size={2.125}
/>
</button>
<Avatar
avatar={{ id: avatarId, kind: "peer" }}
colorKey={avatarId}
hasAvatar={avatarHas}
name={senderName}
size={2.125}
/>
{/if}
</div>
{/if}
<div class="message-content" class:has-appendix={lastInGroup}>
{#if showName}
<button
class="sender-name peer-color-{colorIndex}"
onclick={openSenderProfile}
type="button"
>
{senderName}
</button>
<div class="sender-name peer-color-{colorIndex}">{senderName}</div>
{/if}
{#if message.forward}
<ForwardHeader forward={message.forward} />
@@ -215,7 +173,7 @@
{#if deleted}
<span class="deleted-tag">
<Icon name="delete" size="0.875rem" />
{t("thread.deleted")}
deleted
</span>
{/if}
{#if message.poll}
@@ -248,7 +206,7 @@
/>
</div>
{:else if !(message.has_media || deleted || special)}
<div class="text empty">{t("feed.noText")}</div>
<div class="text empty">(no text)</div>
{/if}
{#if message.web_page}
<WebPage {message} {onmedia} {own} />
@@ -263,7 +221,8 @@
{#if linkCount > 0}
<button class="detail-chip" onclick={openLinks} type="button">
<Icon name="link" size="0.875rem" />
{t("thread.links", { count: linkCount })}
{linkCount}
{linkCount === 1 ? "ссылка" : "ссылки"}
</button>
{/if}
<MessageMeta {message} {onversions} {own} />
@@ -273,7 +232,7 @@
{#if hasCallbacks}
<button class="detail-chip" onclick={openCallbacks} type="button">
<Icon name="bot-command" size="0.875rem" />
{t("panel.callbacks")}
Callback-данные
</button>
{/if}
{#if lastInGroup}
@@ -288,48 +247,43 @@
</div>
{/snippet}
{#snippet menu()}
{#if own}
<MessageReadItem {message} />
{/if}
<ContextMenuItem icon="note" onselect={openAnnotations}
>{t("panel.annotations")}</ContextMenuItem
>Заметки</ContextMenuItem
>
{#if message.reactions.length}
<ContextMenuItem icon="heart" onselect={openReactions}
>{t("panel.reactions")}</ContextMenuItem
>Реакции</ContextMenuItem
>
{/if}
{#if linkCount > 0}
<ContextMenuItem icon="link" onselect={openLinks}
>{t("panel.links")}</ContextMenuItem
>
<ContextMenuItem icon="link" onselect={openLinks}>Ссылки</ContextMenuItem>
{/if}
{#if hasCallbacks}
<ContextMenuItem icon="bot-command" onselect={openCallbacks}
>{t("panel.callbacks")}</ContextMenuItem
>Callback-данные</ContextMenuItem
>
{/if}
{#if message.edited_at}
<ContextMenuItem icon="edit" onselect={onversions}
>{t("panel.versions")}</ContextMenuItem
>Версии</ContextMenuItem
>
{/if}
{#if (message.reply?.message_id ?? null) !== null}
<ContextMenuItem icon="reply" onselect={jumpToReply}
>{t("thread.toReply")}</ContextMenuItem
>Перейти к ответу</ContextMenuItem
>
{/if}
{#if hasText}
<ContextMenuItem icon="copy" onselect={copyText}
>{t("thread.copyText")}</ContextMenuItem
>Копировать текст</ContextMenuItem
>
{/if}
<ContextMenuItem icon="hashtag" onselect={copyId}
>{t("thread.copyId")}</ContextMenuItem
>Копировать ID</ContextMenuItem
>
{#if isGroupChat && !own && message.sender_id !== null}
<ContextMenuItem icon="info" onselect={openSenderProfile}
>{t("thread.senderProfile")}</ContextMenuItem
>Профиль автора</ContextMenuItem
>
{/if}
{/snippet}
@@ -406,35 +360,12 @@
border-bottom-left-radius: 0;
}
.sender-avatar,
.sender-name {
cursor: pointer;
padding: 0;
border: 0;
font: inherit;
background: none;
&:focus-visible {
outline: 2px solid var(--color-primary);
outline-offset: 2px;
}
}
.sender-avatar {
display: block;
border-radius: 50%;
}
.sender-name {
overflow: hidden;
display: block;
max-width: 100%;
margin-bottom: 0.0625rem;
font-size: 0.875rem;
font-weight: var(--font-weight-medium);
text-align: start;
text-overflow: ellipsis;
white-space: nowrap;
}
frontend/src/lib/components/MessageList.svelte
+40 -141
View File
@@ -12,10 +12,7 @@
import Icon from "#lib/components/ui/Icon.svelte";
import Spinner from "#lib/components/ui/Spinner.svelte";
import { formatDay } from "#lib/format/datetime.js";
import { t } from "#lib/i18n/locale.svelte.js";
import { accounts } from "#lib/stores/accounts.svelte.js";
import { chats } from "#lib/stores/chats.svelte.js";
import { discover } from "#lib/stores/discover.svelte.js";
import { events } from "#lib/stores/events.svelte.js";
import { peers } from "#lib/stores/peers.svelte.js";
import { toasts } from "#lib/stores/toasts.svelte.js";
@@ -56,11 +53,7 @@
let highlightTimer: ReturnType<typeof setTimeout> | null = null;
const ownId = $derived(accounts.selected?.tg_user_id ?? null);
const isChannel = $derived(
Boolean(chats.byId(chatId)?.is_broadcast) ||
discover.get(chatId)?.kind === "channel"
);
const isGroupChat = $derived(chatId < 0 && !isChannel);
const isGroupChat = $derived(chatId < 0);
interface Row {
dayKey: string;
@@ -124,81 +117,16 @@
peers.ensure(ids);
}
function memberIds(message: MessageView): number[] {
return [message.message_id, ...message.media.map((ref) => ref.message_id)];
}
function holds(message: MessageView, messageId: number): boolean {
return memberIds(message).includes(messageId);
}
function sameAlbum(a: MessageView, b: MessageView): boolean {
return (
a.media_group_id !== null &&
a.media_group_id === b.media_group_id &&
a.chat_id === b.chat_id
);
}
function albumPrimary(base: MessageView, incoming: MessageView) {
if (
base.message_id === incoming.message_id ||
(incoming.text && !base.text)
) {
return incoming;
}
if (base.text) {
return base;
}
return incoming.message_id < base.message_id ? incoming : base;
}
function joinAlbum(base: MessageView, incoming: MessageView): MessageView {
const media = new Map(base.media.map((ref) => [ref.message_id, ref]));
for (const ref of incoming.media) {
media.set(ref.message_id, ref);
}
return {
...albumPrimary(base, incoming),
media: [...media.values()].sort((a, b) => a.message_id - b.message_id),
};
}
function mergeAlbums(list: MessageView[]): MessageView[] {
const out: MessageView[] = [];
for (const message of list) {
const prev = out.at(-1);
if (prev && sameAlbum(prev, message)) {
out[out.length - 1] = joinAlbum(prev, message);
} else {
out.push(message);
}
}
return out;
}
function knownIds(): Set<number> {
return new Set(messages.flatMap(memberIds));
}
function ownerOf(messageId: number): number | null {
return messages.find((m) => holds(m, messageId))?.message_id ?? null;
}
function jumpToMessage(messageId: number, smooth = true) {
const owner = ownerOf(messageId) ?? messageId;
function jumpToMessage(messageId: number) {
const target = container?.querySelector<HTMLElement>(
`[data-message-id="${owner}"]`
`[data-message-id="${messageId}"]`
);
if (!target) {
toasts.error(t("thread.notFound"));
toasts.error("Message not loaded");
return;
}
target.scrollIntoView({
behavior: smooth ? "smooth" : "auto",
block: "center",
});
highlightId = owner;
target.scrollIntoView({ behavior: "smooth", block: "center" });
highlightId = messageId;
if (highlightTimer) {
clearTimeout(highlightTimer);
}
@@ -232,28 +160,20 @@
loading = true;
hasMore = true;
hasNewer = false;
const pending = ui.jumpTarget?.chatId === chatId ? ui.jumpTarget : null;
try {
if (pending) {
ui.clearJump();
await loadAround(pending.messageId);
await tick();
jumpToMessage(pending.messageId, false);
return;
}
const page = await listMessages(chatId, {
include_deleted: true,
limit: PAGE,
offset: 0,
});
messages = mergeAlbums([...page].reverse());
messages = [...page].reverse();
hasMore = page.length > 0;
ensurePeers(messages);
await tick();
scrollToBottom();
updateStuck();
} catch {
toasts.error(t("thread.loadFailed"));
toasts.error("Failed to load messages");
} finally {
loading = false;
}
@@ -295,18 +215,16 @@
const prevTop = el.scrollTop;
try {
const page = await listMessages(chatId, {
before_id: Math.min(...memberIds(messages[0])),
before_id: messages[0].message_id,
include_deleted: true,
limit: PAGE,
});
const known = knownIds();
const fresh = page.filter((m) =>
memberIds(m).some((id) => !known.has(id))
);
const known = new Set(messages.map((m) => m.message_id));
const fresh = page.filter((m) => !known.has(m.message_id));
hasMore = fresh.length > 0;
if (fresh.length > 0) {
suppressAppear = true;
messages = mergeAlbums([...[...fresh].reverse(), ...messages]);
messages = [...[...fresh].reverse(), ...messages];
ensurePeers(fresh);
await tick();
el.scrollTop = prevTop + (el.scrollHeight - prevHeight);
@@ -324,20 +242,17 @@
}
loadingNewer = true;
try {
const last = messages.at(-1);
const page = await listMessages(chatId, {
after_id: last ? Math.max(...memberIds(last)) : undefined,
after_id: messages.at(-1)?.message_id,
include_deleted: true,
limit: PAGE,
});
const known = knownIds();
const fresh = page.filter((m) =>
memberIds(m).some((id) => !known.has(id))
);
const known = new Set(messages.map((m) => m.message_id));
const fresh = page.filter((m) => !known.has(m.message_id));
hasNewer = fresh.length > 0;
if (fresh.length > 0) {
suppressAppear = true;
messages = mergeAlbums([...messages, ...fresh]);
messages = [...messages, ...fresh];
ensurePeers(fresh);
await tick();
await trimHead();
@@ -363,7 +278,7 @@
limit: PAGE,
}),
]);
messages = mergeAlbums([...[...older].reverse(), ...newer]);
messages = [...[...older].reverse(), ...newer];
hasMore = older.length > 0;
hasNewer = newer.length > 0;
ensurePeers(messages);
@@ -416,20 +331,13 @@
}
function replaceMessage(message: MessageView) {
messages = messages.map((m) => {
if (sameAlbum(m, message)) {
return joinAlbum(m, message);
}
return m.message_id === message.message_id ? message : m;
});
messages = messages.map((m) =>
m.message_id === message.message_id ? message : m
);
}
async function appendMessage(message: MessageView) {
const last = messages.at(-1);
if (
messages.some((m) => holds(m, message.message_id)) ||
(last && sameAlbum(last, message))
) {
if (messages.some((m) => m.message_id === message.message_id)) {
replaceMessage(message);
return;
}
@@ -449,19 +357,19 @@
function applyDelete(ids: number[]) {
const now = new Date().toISOString();
messages = messages.map((m) =>
m.deleted_at === null && ids.some((id) => holds(m, id))
ids.includes(m.message_id) && m.deleted_at === null
? { ...m, deleted_at: now }
: m
);
}
function applyReceipt(readUpTo: number, readAt: string) {
function applyReceipt(readUpTo: number) {
if (ownId === null) {
return;
}
messages = messages.map((m) =>
m.sender_id === ownId && m.message_id <= readUpTo && !m.read
? { ...m, read: true, read_at: readAt }
? { ...m, read: true }
: m
);
}
@@ -480,7 +388,7 @@
) {
applyDelete(event.message_ids);
} else if (event.type === "receipt" && event.chat_id === chatId) {
applyReceipt(event.read_up_to, event.read_at);
applyReceipt(event.read_up_to);
}
}
@@ -495,24 +403,20 @@
});
const fresh = [...page].reverse();
const known = new Map(messages.map((m) => [m.message_id, m]));
const maxId = Math.max(0, ...messages.flatMap(memberIds));
const maxId = messages.reduce((acc, m) => Math.max(acc, m.message_id), 0);
const stick = isNearBottom();
const appended: MessageView[] = [];
for (const message of fresh) {
const current = known.get(message.message_id);
if (current) {
known.set(
message.message_id,
sameAlbum(current, message) ? joinAlbum(current, message) : message
);
if (known.has(message.message_id)) {
known.set(message.message_id, message);
} else if (message.message_id > maxId) {
appended.push(message);
}
}
messages = mergeAlbums([
messages = [
...messages.map((m) => known.get(m.message_id) ?? m),
...appended,
]);
];
ensurePeers(fresh);
await trimHead();
if (appended.length > 0 && stick) {
@@ -521,17 +425,12 @@
}
}
async function goTo(messageId: number) {
try {
const loaded = ownerOf(messageId) !== null;
if (!loaded) {
await loadAround(messageId);
}
await tick();
jumpToMessage(messageId, loaded);
} catch {
toasts.error(t("thread.loadFailed"));
async function jumpToTarget(messageId: number) {
if (!messages.some((m) => m.message_id === messageId)) {
await loadAround(messageId);
}
await tick();
jumpToMessage(messageId);
}
$effect(() => {
@@ -540,7 +439,7 @@
return;
}
ui.clearJump();
goTo(target.messageId);
jumpToTarget(target.messageId);
});
$effect(() => {
@@ -561,7 +460,7 @@
</script>
<div class="message-pane">
<PinnedBar {chatId} onjump={goTo} />
<PinnedBar {chatId} onjump={jumpToMessage} />
<div
class="message-list custom-scroll"
onscroll={onScroll}
@@ -591,7 +490,7 @@
</div>
{/if}
{#if row.message.service}
<ActionMessage message={row.message} onjump={goTo} />
<ActionMessage message={row.message} onjump={jumpToMessage} />
{:else}
<MessageBubble
animate={!suppressAppear}
@@ -600,7 +499,7 @@
{isGroupChat}
lastInGroup={row.lastInGroup}
message={row.message}
onjump={goTo}
onjump={jumpToMessage}
onmedia={(index) => openMedia(row.message, index)}
onversions={() => openVersions(row.message.message_id)}
own={row.own}
@@ -615,7 +514,7 @@
</div>
{#if hasNewer}
<button
aria-label={t("thread.toLatest")}
aria-label="К последним сообщениям"
class="to-latest"
onclick={goToLatest}
type="button"
frontend/src/lib/components/MessageMedia.svelte
-4
View File
@@ -14,7 +14,6 @@
import TgsSticker from "#lib/components/media/TgsSticker.svelte";
import VideoNote from "#lib/components/media/VideoNote.svelte";
import VoiceMessage from "#lib/components/media/VoiceMessage.svelte";
import MessageReadItem from "#lib/components/read/MessageReadItem.svelte";
import ContextMenu from "#lib/components/ui/ContextMenu.svelte";
import ContextMenuItem from "#lib/components/ui/ContextMenuItem.svelte";
import Icon from "#lib/components/ui/Icon.svelte";
@@ -223,9 +222,6 @@
</div>
{/snippet}
{#snippet menu()}
{#if own}
<MessageReadItem {message} />
{/if}
<ContextMenuItem icon="open-in-new-tab" onselect={onopen}
>Открыть на весь экран</ContextMenuItem
>
Loaded 100 of 189 files, more files were not shown because too many files have changed in this diff. Show more