feat(userbot,api,frontend): file links, media downloads, story hold-pause, drop scheduled dupes

This commit is contained in:
hh
2026-08-13 03:17:36 +02:00
parent 683b9a31a3
commit 3e08698b62
38 changed files with 2889 additions and 134 deletions
@@ -0,0 +1,70 @@
"""drop scheduled message duplicates
Revision ID: e1c7a4b62d90
Revises: d4a7e2b91f38
Create Date: 2026-08-13 10:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
revision: str = "e1c7a4b62d90"
down_revision: str | None = "d4a7e2b91f38"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_STAGE = """
CREATE TEMP TABLE scheduled_keys ON COMMIT DROP AS
SELECT DISTINCT s.account_id, s.chat_id, s.message_id
FROM messages s
WHERE s.raw->>'scheduled' = 'true'
AND NOT EXISTS (
SELECT 1 FROM messages r
WHERE r.account_id = s.account_id AND r.chat_id = s.chat_id
AND r.message_id = s.message_id
AND r.raw->>'scheduled' IS DISTINCT FROM 'true'
)
"""
_DELETE_CHILD = """
DELETE FROM {table} t
USING scheduled_keys k
WHERE t.account_id = k.account_id AND t.chat_id = k.chat_id
AND t.message_id = k.message_id
"""
_DELETE_MESSAGES = """
DELETE FROM messages m
USING scheduled_keys k
WHERE m.account_id = k.account_id AND m.chat_id = k.chat_id
AND m.message_id = k.message_id AND m.raw->>'scheduled' = 'true'
"""
_RECOUNT = """
UPDATE chat_stats cs
SET message_count = fresh.message_count
FROM (
SELECT k.account_id, k.chat_id,
(SELECT count(*) FROM messages m
WHERE m.account_id = k.account_id AND m.chat_id = k.chat_id)
AS message_count
FROM (SELECT DISTINCT account_id, chat_id FROM scheduled_keys) k
) fresh
WHERE cs.account_id = fresh.account_id AND cs.chat_id = fresh.chat_id
"""
_CHILD_TABLES = ("media", "media_versions", "message_versions", "links", "callbacks")
def upgrade() -> None:
op.execute(_STAGE)
for table in _CHILD_TABLES:
op.execute(_DELETE_CHILD.format(table=table))
op.execute(_DELETE_MESSAGES)
op.execute(_RECOUNT)
def downgrade() -> None:
pass
@@ -0,0 +1,73 @@
"""file shares
Revision ID: f2b8d3c9a51e
Revises: e1c7a4b62d90
Create Date: 2026-08-13 10:30:00.000000
"""
from collections.abc import Sequence
from alembic import op
revision: str = "f2b8d3c9a51e"
down_revision: str | None = "e1c7a4b62d90"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
_SHARES = """
CREATE TABLE file_shares (
id serial PRIMARY KEY,
account_id integer NOT NULL,
token text NOT NULL UNIQUE,
kind text NOT NULL,
storage_key text NOT NULL,
file_name text NOT NULL,
mime text,
file_size bigint,
title text,
chat_id bigint,
message_id bigint,
peer_id bigint,
story_id bigint,
expires_at timestamptz,
max_downloads integer,
download_count integer NOT NULL DEFAULT 0,
last_download_at timestamptz,
revoked_at timestamptz,
created_at timestamptz NOT NULL DEFAULT now()
)
"""
_HITS = """
CREATE TABLE file_share_hits (
id bigserial PRIMARY KEY,
share_id integer NOT NULL REFERENCES file_shares (id) ON DELETE CASCADE,
ts timestamptz NOT NULL DEFAULT now(),
method text NOT NULL,
ip text,
user_agent text,
counted boolean NOT NULL DEFAULT false
)
"""
def upgrade() -> None:
op.execute(_SHARES)
op.execute(
"CREATE INDEX ix_file_shares_account ON file_shares "
"(account_id, created_at DESC)"
)
op.execute(
"CREATE INDEX ix_file_shares_subject ON file_shares "
"(account_id, storage_key) WHERE revoked_at IS NULL"
)
op.execute(_HITS)
op.execute(
"CREATE INDEX ix_file_share_hits_share ON file_share_hits (share_id, ts DESC)"
)
def downgrade() -> None:
op.execute("DROP TABLE IF EXISTS file_share_hits")
op.execute("DROP TABLE IF EXISTS file_shares")
+4
View File
@@ -22,6 +22,7 @@ from api.routers import (
custom_emoji,
discover,
events,
files,
folders,
media,
peers,
@@ -29,6 +30,7 @@ from api.routers import (
presence,
profile,
search,
shares,
social,
stories,
watches,
@@ -88,6 +90,8 @@ app.include_router(peers.router)
app.include_router(discover.router)
app.include_router(annotations.router)
app.include_router(watches.router)
app.include_router(shares.router)
app.include_router(files.router)
app.mount("/mcp", mcp_app)
+78
View File
@@ -0,0 +1,78 @@
from datetime import UTC, datetime
from typing import Annotated
import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, Query, Request
from fastapi.responses import FileResponse, PlainTextResponse
from utils.files import content_disposition, counts_as_download, is_inline_mime
from utils.read import shares
from utils.storage import ContentAddressedStorage
router = APIRouter(tags=["files"], route_class=DishkaRoute)
_GONE = "This link is no longer available.\n"
_MISSING = "Not found.\n"
_NO_STORE = {"Cache-Control": "private, no-store", "X-Robots-Tag": "noindex, nofollow"}
def _client_ip(request: Request) -> str | None:
forwarded = request.headers.get("x-forwarded-for")
if forwarded:
return forwarded.split(",")[0].strip()
return request.client.host if request.client else None
def _expired(row: asyncpg.Record) -> bool:
expires_at = row["expires_at"]
return expires_at is not None and expires_at <= datetime.now(UTC)
def _exhausted(row: asyncpg.Record) -> bool:
limit = row["max_downloads"]
return limit is not None and row["download_count"] >= limit
@router.api_route("/f/{token}", methods=["GET", "HEAD"], response_model=None)
async def serve_shared_file(
request: Request,
pool: FromDishka[asyncpg.Pool],
storage: FromDishka[ContentAddressedStorage],
token: str,
dl: Annotated[bool, Query()] = False,
) -> FileResponse | PlainTextResponse:
row = await shares.share_by_token(pool, token)
if row is None:
return PlainTextResponse(_MISSING, status_code=404, headers=_NO_STORE)
method = request.method
agent = request.headers.get("user-agent")
ip = _client_ip(request)
counts = counts_as_download(method, agent, request.headers.get("range"))
if row["revoked_at"] is not None or _expired(row) or _exhausted(row):
await shares.record_hit(pool, row["id"], method, ip, agent, counted=False)
return PlainTextResponse(_GONE, status_code=410, headers=_NO_STORE)
if not storage.exists(row["storage_key"]):
return PlainTextResponse(_MISSING, status_code=404, headers=_NO_STORE)
if counts and not await shares.consume_download(pool, row["id"]):
await shares.record_hit(pool, row["id"], method, ip, agent, counted=False)
return PlainTextResponse(_GONE, status_code=410, headers=_NO_STORE)
await shares.record_hit(pool, row["id"], method, ip, agent, counted=counts)
mime = row["mime"] or "application/octet-stream"
attachment = dl or not is_inline_mime(row["mime"])
return FileResponse(
storage.url(row["storage_key"]),
media_type=mime,
headers={
**_NO_STORE,
"Content-Disposition": content_disposition(
row["file_name"], attachment=attachment
),
},
)
+22 -2
View File
@@ -6,6 +6,7 @@ from fastapi import APIRouter, HTTPException, Query
from fastapi.responses import FileResponse
from utils.cache import DAY_HEADERS, IMMUTABLE_HEADERS
from utils.files import content_disposition, media_file_name
from utils.read.media import (
get_media,
get_media_version,
@@ -17,6 +18,15 @@ from utils.storage import ContentAddressedStorage
router = APIRouter(prefix="/api/media", tags=["media"], route_class=DishkaRoute)
Download = Annotated[bool, Query()]
def _attachment(headers: dict[str, str], file_name: str) -> dict[str, str]:
return {
**headers,
"Content-Disposition": content_disposition(file_name, attachment=True),
}
@router.get("/{media_id}/meta")
async def media_meta(pool: FromDishka[asyncpg.Pool], media_id: int) -> MediaView:
@@ -41,14 +51,20 @@ async def serve_media_version(
pool: FromDishka[asyncpg.Pool],
storage: FromDishka[ContentAddressedStorage],
version_id: int,
download: Download = False,
) -> FileResponse:
version = await get_media_version(pool, version_id)
if version is None:
raise HTTPException(status_code=404, detail="media version not found")
headers = IMMUTABLE_HEADERS
if download:
headers = _attachment(
headers, media_file_name(version.kind, version.mime, version_id)
)
return FileResponse(
storage.url(version.storage_key),
media_type=version.mime or "application/octet-stream",
headers=IMMUTABLE_HEADERS,
headers=headers,
)
@@ -70,6 +86,7 @@ async def serve_media(
pool: FromDishka[asyncpg.Pool],
storage: FromDishka[ContentAddressedStorage],
media_id: int,
download: Download = False,
) -> FileResponse:
media = await get_media(pool, media_id)
if media is None:
@@ -79,8 +96,11 @@ async def serve_media(
status_code=409,
detail="media not downloaded; enqueue fetch via POST /api/media/fetch",
)
headers = DAY_HEADERS
if download:
headers = _attachment(headers, media.file_name or f"media_{media_id}")
return FileResponse(
storage.url(media.storage_key),
media_type=media.mime or "application/octet-stream",
headers=DAY_HEADERS,
headers=headers,
)
+226
View File
@@ -0,0 +1,226 @@
from typing import Annotated
import asyncpg
from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel
from utils.files import expiry_from_seconds, media_file_name, story_file_name
from utils.read import chats, peers, shares
from utils.read.media import get_media, get_media_version
from utils.read.models import DEFAULT_LIMIT, FileShareHitView, FileShareView, Page
router = APIRouter(prefix="/api/shares", tags=["shares"], route_class=DishkaRoute)
class ShareCreate(BaseModel):
account_id: int
kind: str
media_id: int | None = None
version_id: int | None = None
peer_id: int | None = None
story_id: int | None = None
expires_in_seconds: int | None = None
max_downloads: int | None = None
class ShareUpdate(BaseModel):
expires_in_seconds: int | None = None
max_downloads: int | None = None
keep_expiry: bool = False
class Subject(BaseModel):
storage_key: str
file_name: str
mime: str | None
file_size: int | None
title: str | None
chat_id: int | None = None
message_id: int | None = None
peer_id: int | None = None
story_id: int | None = None
_NOT_STORED = "file is not downloaded yet"
async def _chat_title(pool: asyncpg.Pool, account_id: int, chat_id: int) -> str | None:
chat = await chats.get_chat(pool, account_id, chat_id)
return chat.title if chat else None
async def _media_subject(pool: asyncpg.Pool, media_id: int) -> Subject:
media = await get_media(pool, media_id)
if media is None:
raise HTTPException(status_code=404, detail="media not found")
if not media.downloaded or media.storage_key is None:
raise HTTPException(status_code=409, detail=_NOT_STORED)
return Subject(
storage_key=media.storage_key,
file_name=media.file_name or f"media_{media.id}",
mime=media.mime,
file_size=media.file_size,
title=await _chat_title(pool, media.account_id, media.chat_id),
chat_id=media.chat_id,
message_id=media.message_id,
)
async def _version_subject(pool: asyncpg.Pool, version_id: int) -> Subject:
version = await get_media_version(pool, version_id)
if version is None:
raise HTTPException(status_code=404, detail="media version not found")
return Subject(
storage_key=version.storage_key,
file_name=media_file_name(version.kind, version.mime, version_id),
mime=version.mime,
file_size=version.file_size,
title=None,
chat_id=None,
message_id=None,
)
async def _story_subject(
pool: asyncpg.Pool, account_id: int, peer_id: int, story_id: int
) -> Subject:
story = await peers.get_story(pool, account_id, peer_id, story_id)
if story is None:
raise HTTPException(status_code=404, detail="story not found")
if not story.downloaded or story.storage_key is None:
raise HTTPException(status_code=409, detail=_NOT_STORED)
return Subject(
storage_key=story.storage_key,
file_name=story_file_name(peer_id, story_id, story.media_kind),
mime="video/mp4" if story.media_kind == "video" else "image/jpeg",
file_size=None,
title=await _chat_title(pool, account_id, peer_id),
peer_id=peer_id,
story_id=story_id,
)
async def _resolve(pool: asyncpg.Pool, body: ShareCreate) -> Subject:
if body.kind == "media" and body.media_id is not None:
return await _media_subject(pool, body.media_id)
if body.kind == "media_version" and body.version_id is not None:
return await _version_subject(pool, body.version_id)
if body.kind == "story" and body.peer_id is not None and body.story_id is not None:
return await _story_subject(pool, body.account_id, body.peer_id, body.story_id)
raise HTTPException(status_code=422, detail="unsupported share subject")
@router.get("")
async def list_shares(
pool: FromDishka[asyncpg.Pool],
account_id: Annotated[int, Query()],
active_only: Annotated[bool, Query()] = False,
limit: Annotated[int, Query()] = DEFAULT_LIMIT,
offset: Annotated[int, Query()] = 0,
) -> list[FileShareView]:
return await shares.list_shares(
pool, account_id, Page(limit=limit, offset=offset), active_only=active_only
)
@router.post("", status_code=201)
async def create_share(
pool: FromDishka[asyncpg.Pool], body: ShareCreate
) -> FileShareView:
subject = await _resolve(pool, body)
existing = await shares.find_active_share(
pool, body.account_id, subject.storage_key
)
if existing is not None:
return existing
return await shares.create_share(
pool,
body.account_id,
body.kind,
subject.storage_key,
subject.file_name,
mime=subject.mime,
file_size=subject.file_size,
title=subject.title,
chat_id=subject.chat_id,
message_id=subject.message_id,
peer_id=subject.peer_id,
story_id=subject.story_id,
expires_at=expiry_from_seconds(body.expires_in_seconds),
max_downloads=body.max_downloads,
)
@router.get("/lookup")
async def lookup_share(
pool: FromDishka[asyncpg.Pool],
account_id: Annotated[int, Query()],
kind: Annotated[str, Query()],
media_id: Annotated[int | None, Query()] = None,
version_id: Annotated[int | None, Query()] = None,
peer_id: Annotated[int | None, Query()] = None,
story_id: Annotated[int | None, Query()] = None,
) -> FileShareView | None:
body = ShareCreate(
account_id=account_id,
kind=kind,
media_id=media_id,
version_id=version_id,
peer_id=peer_id,
story_id=story_id,
)
subject = await _resolve(pool, body)
return await shares.find_active_share(pool, account_id, subject.storage_key)
@router.get("/{share_id}/hits")
async def share_hits(
pool: FromDishka[asyncpg.Pool],
share_id: int,
limit: Annotated[int, Query()] = DEFAULT_LIMIT,
offset: Annotated[int, Query()] = 0,
) -> list[FileShareHitView]:
return await shares.list_hits(pool, share_id, Page(limit=limit, offset=offset))
@router.patch("/{share_id}")
async def update_share(
pool: FromDishka[asyncpg.Pool], share_id: int, body: ShareUpdate
) -> FileShareView:
current = await shares.get_share(pool, share_id)
if current is None:
raise HTTPException(status_code=404, detail="share not found")
expires_at = (
current.expires_at
if body.keep_expiry
else expiry_from_seconds(body.expires_in_seconds)
)
updated = await shares.update_share(
pool, share_id, expires_at=expires_at, max_downloads=body.max_downloads
)
if updated is None:
raise HTTPException(status_code=404, detail="share not found")
return updated
@router.post("/{share_id}/revoke")
async def revoke_share(pool: FromDishka[asyncpg.Pool], share_id: int) -> FileShareView:
share = await shares.revoke_share(pool, share_id)
if share is None:
raise HTTPException(status_code=404, detail="share not found")
return share
@router.post("/{share_id}/reissue")
async def reissue_share(pool: FromDishka[asyncpg.Pool], share_id: int) -> FileShareView:
share = await shares.rotate_token(pool, share_id)
if share is None:
raise HTTPException(status_code=404, detail="share not found")
return share
@router.delete("/{share_id}", status_code=204)
async def delete_share(pool: FromDishka[asyncpg.Pool], share_id: int) -> None:
if not await shares.delete_share(pool, share_id):
raise HTTPException(status_code=404, detail="share not found")
+8
View File
@@ -5,6 +5,7 @@ from dishka.integrations.fastapi import DishkaRoute, FromDishka
from fastapi import APIRouter, HTTPException, Query
from fastapi.responses import FileResponse
from utils.files import content_disposition, story_file_name
from utils.read import peers
from utils.read.models import DEFAULT_LIMIT, Page, StoryView
from utils.storage import ContentAddressedStorage
@@ -36,13 +37,20 @@ async def serve_story_media(
peer_id: int,
story_id: int,
account_id: AccountId,
download: Annotated[bool, Query()] = False,
) -> FileResponse:
story = await peers.get_story(pool, account_id, peer_id, story_id)
if story is None:
raise HTTPException(status_code=404, detail="story not found")
if not story.downloaded or story.storage_key is None:
raise HTTPException(status_code=409, detail="story media not downloaded")
headers = {}
if download:
headers["Content-Disposition"] = content_disposition(
story_file_name(peer_id, story_id, story.media_kind), attachment=True
)
return FileResponse(
storage.url(story.storage_key),
media_type=_STORY_MIME.get(story.media_kind or "", "application/octet-stream"),
headers=headers,
)
+7 -1
View File
@@ -11,7 +11,13 @@ from utils.events import notify_bg_event
@PyroClient.on_edited_message()
async def on_edited_message(client: PyroClient, message: Message) -> None:
ctx = client.capture
if ctx is None or message.empty or message.chat is None or message.date is None:
if (
ctx is None
or message.empty
or message.scheduled
or message.chat is None
or message.date is None
):
return
chat = message.chat
chat_id = chat.id or 0
+7 -1
View File
@@ -11,7 +11,13 @@ from utils.events import notify_bg_event
@PyroClient.on_message()
async def on_message(client: PyroClient, message: Message) -> None:
ctx = client.capture
if ctx is None or message.empty or message.chat is None or message.date is None:
if (
ctx is None
or message.empty
or message.scheduled
or message.chat is None
or message.date is None
):
return
meta = meta_from_chat(message.chat, ctx.contacts.ids)
await ctx.watches.on_text(meta.chat_id, message.id, message.text or message.caption)
+50
View File
@@ -493,3 +493,53 @@ class Dialog(SQLModel, table=True):
onupdate=func.now(),
)
)
class FileShare(SQLModel, table=True):
__tablename__ = "file_shares"
id: int | None = Field(default=None, primary_key=True)
account_id: int
token: str = Field(unique=True)
kind: str
storage_key: str
file_name: str
mime: str | None = None
file_size: int | None = Field(default=None, sa_column=Column(BigInteger))
title: str | None = None
chat_id: int | None = Field(default=None, sa_column=Column(BigInteger))
message_id: int | None = Field(default=None, sa_column=Column(BigInteger))
peer_id: int | None = Field(default=None, sa_column=Column(BigInteger))
story_id: int | None = Field(default=None, sa_column=Column(BigInteger))
expires_at: datetime | None = Field(
default=None, sa_column=Column(DateTime(timezone=True))
)
max_downloads: int | None = None
download_count: int = 0
last_download_at: datetime | None = Field(
default=None, sa_column=Column(DateTime(timezone=True))
)
revoked_at: datetime | None = Field(
default=None, sa_column=Column(DateTime(timezone=True))
)
created_at: datetime = Field(
sa_column=Column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
)
class FileShareHit(SQLModel, table=True):
__tablename__ = "file_share_hits"
id: int | None = Field(default=None, sa_column=Column(BigInteger, primary_key=True))
share_id: int = Field(foreign_key="file_shares.id")
ts: datetime = Field(
sa_column=Column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
)
method: str
ip: str | None = None
user_agent: str | None = None
counted: bool = False
+134
View File
@@ -0,0 +1,134 @@
import re
from datetime import UTC, datetime, timedelta
from urllib.parse import quote
_MIME_EXTENSIONS = {
"application/pdf": ".pdf",
"application/x-tgsticker": ".tgs",
"application/zip": ".zip",
"audio/mpeg": ".mp3",
"audio/ogg": ".ogg",
"image/gif": ".gif",
"image/jpeg": ".jpg",
"image/png": ".png",
"image/webp": ".webp",
"text/plain": ".txt",
"video/mp4": ".mp4",
"video/quicktime": ".mov",
"video/webm": ".webm",
}
_KIND_EXTENSIONS = {
"animation": ".mp4",
"audio": ".mp3",
"gif": ".mp4",
"photo": ".jpg",
"sticker": ".webp",
"video": ".mp4",
"video_note": ".mp4",
"voice": ".ogg",
}
_INLINE_MIME_PREFIXES = ("image/", "video/", "audio/", "text/")
_INLINE_MIMES = {"application/pdf", "application/json"}
_PREVIEW_AGENTS = (
"telegrambot",
"twitterbot",
"facebookexternalhit",
"whatsapp",
"discordbot",
"slackbot",
"skypeuripreview",
"vkshare",
"redditbot",
"linkedinbot",
"embedly",
"quora link preview",
"googlebot",
"bingbot",
"applebot",
"yandexbot",
"duckduckbot",
"petalbot",
"ahrefsbot",
"semrushbot",
"headlesschrome",
"python-requests",
)
_UNSAFE_CHARS = re.compile(r'[\\/:*?"<>|\x00-\x1f]+')
_SPACES = re.compile(r"\s+")
_NAME_LIMIT = 120
def extension_for(kind: str | None, mime: str | None) -> str:
if mime and mime in _MIME_EXTENSIONS:
return _MIME_EXTENSIONS[mime]
if kind and kind in _KIND_EXTENSIONS:
return _KIND_EXTENSIONS[kind]
if mime and "/" in mime:
tail = mime.rsplit("/", 1)[1].split(";")[0].strip()
if tail.isalnum():
return f".{tail}"
return ".bin"
def sanitize_name(name: str) -> str:
cleaned = _SPACES.sub(" ", _UNSAFE_CHARS.sub("_", name)).strip(" .")
if len(cleaned) > _NAME_LIMIT:
head, dot, tail = cleaned.rpartition(".")
cleaned = (
f"{head[: _NAME_LIMIT - len(tail) - 1]}{dot}{tail}"
if dot and len(tail) < 12 # noqa: PLR2004
else cleaned[:_NAME_LIMIT]
)
return cleaned or "file"
def media_file_name(
kind: str | None, mime: str | None, message_id: int, original: str | None = None
) -> str:
if original:
name = sanitize_name(original)
return name if "." in name else f"{name}{extension_for(kind, mime)}"
return f"{kind or 'media'}_{message_id}{extension_for(kind, mime)}"
def story_file_name(peer_id: int, story_id: int, media_kind: str | None) -> str:
return f"story_{peer_id}_{story_id}{extension_for(media_kind or 'photo', None)}"
def is_inline_mime(mime: str | None) -> bool:
if not mime:
return False
return mime in _INLINE_MIMES or mime.startswith(_INLINE_MIME_PREFIXES)
def content_disposition(file_name: str, *, attachment: bool) -> str:
kind = "attachment" if attachment else "inline"
ascii_name = file_name.encode("ascii", "replace").decode("ascii").replace('"', "_")
return f"{kind}; filename=\"{ascii_name}\"; filename*=UTF-8''{quote(file_name)}"
def is_preview_agent(user_agent: str | None) -> bool:
if not user_agent:
return False
lowered = user_agent.lower()
return any(marker in lowered for marker in _PREVIEW_AGENTS)
def counts_as_download(
method: str, user_agent: str | None, range_header: str | None
) -> bool:
if method.upper() != "GET" or is_preview_agent(user_agent):
return False
if range_header is None:
return True
return range_header.replace(" ", "").startswith("bytes=0-")
def expiry_from_seconds(seconds: int | None) -> datetime | None:
if seconds is None or seconds <= 0:
return None
return datetime.now(UTC) + timedelta(seconds=seconds)
+34 -8
View File
@@ -1,18 +1,41 @@
import asyncpg
from utils.files import media_file_name
from utils.read.message_view import load_raw
from utils.read.models import MediaVersionView, MediaView
_MEDIA_COLS = (
"id, account_id, chat_id, message_id, kind, storage_key, file_size, "
"mime, ttl_seconds, downloaded, extracted_text, created_at"
MEDIA_COLS = (
"m.id, m.account_id, m.chat_id, m.message_id, m.kind, m.storage_key, "
"m.file_size, m.mime, m.ttl_seconds, m.downloaded, m.extracted_text, "
"m.created_at, src.original_name"
)
ORIGINAL_NAME_JOIN = """
LEFT JOIN LATERAL (
SELECT msg.raw->m.kind->>'file_name' AS original_name
FROM messages msg
WHERE msg.account_id = m.account_id AND msg.chat_id = m.chat_id
AND msg.message_id = m.message_id
ORDER BY msg.date DESC LIMIT 1
) src ON true
"""
_VERSION_COLS = "id, kind, storage_key, file_size, mime, observed_at"
_WEB_PAGE_MEDIA_KINDS = ("photo", "video", "animation", "document", "audio")
def media_view(row: asyncpg.Record) -> MediaView:
fields = dict(row)
original = fields.pop("original_name", None)
return MediaView(
**fields,
file_name=media_file_name(
fields["kind"], fields["mime"], fields["message_id"], original
),
)
async def _web_page_media_stub(
pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
) -> MediaView | None:
@@ -47,29 +70,32 @@ async def _web_page_media_stub(
downloaded=False,
extracted_text=None,
created_at=row["date"],
file_name=media_file_name(
kind, obj.get("mime_type"), message_id, obj.get("file_name")
),
)
async def get_media(pool: asyncpg.Pool, media_id: int) -> MediaView | None:
row = await pool.fetchrow(
f"SELECT {_MEDIA_COLS} FROM media WHERE id = $1", # noqa: S608
f"SELECT {MEDIA_COLS} FROM media m {ORIGINAL_NAME_JOIN} WHERE m.id = $1", # noqa: S608
media_id,
)
return MediaView(**dict(row)) if row else None
return media_view(row) if row else None
async def get_message_media(
pool: asyncpg.Pool, account_id: int, chat_id: int, message_id: int
) -> MediaView | None:
row = await pool.fetchrow(
f"SELECT {_MEDIA_COLS} FROM media " # noqa: S608
"WHERE account_id = $1 AND chat_id = $2 AND message_id = $3",
f"SELECT {MEDIA_COLS} FROM media m {ORIGINAL_NAME_JOIN} " # noqa: S608
"WHERE m.account_id = $1 AND m.chat_id = $2 AND m.message_id = $3",
account_id,
chat_id,
message_id,
)
if row is not None:
return MediaView(**dict(row))
return media_view(row)
return await _web_page_media_stub(pool, account_id, chat_id, message_id)
+4 -1
View File
@@ -5,6 +5,7 @@ from typing import Any
import asyncpg
from pydantic import ValidationError
from utils.files import media_file_name
from utils.read.models import (
ContactView,
EntityView,
@@ -344,6 +345,7 @@ def media_ref_from(
obj = obj if isinstance(obj, dict) else {}
width = obj.get("width") or obj.get("length")
height = obj.get("height") or obj.get("length")
mime = (media_row["mime"] if media_row else None) or obj.get("mime_type")
return MediaRef(
message_id=message_id,
id=media_row["id"] if media_row else None,
@@ -352,11 +354,12 @@ def media_ref_from(
width=width,
height=height,
duration=obj.get("duration"),
mime=(media_row["mime"] if media_row else None) or obj.get("mime_type"),
mime=mime,
file_size=(media_row["file_size"] if media_row else None)
or obj.get("file_size"),
ttl_seconds=media_row["ttl_seconds"] if media_row else None,
extracted_text=media_row["extracted_text"] if media_row else None,
file_name=media_file_name(kind, mime, message_id, obj.get("file_name")),
)
+31
View File
@@ -92,6 +92,7 @@ class MediaRef(BaseModel):
file_size: int | None = None
ttl_seconds: int | None = None
extracted_text: str | None = None
file_name: str | None = None
class ReactionCount(BaseModel):
@@ -222,6 +223,7 @@ class MediaView(BaseModel):
downloaded: bool
extracted_text: str | None
created_at: datetime
file_name: str | None = None
class MediaVersionView(BaseModel):
@@ -391,3 +393,32 @@ class AlertView(BaseModel):
payload: dict[str, Any]
seen: bool
created_at: datetime
class FileShareView(BaseModel):
id: int
account_id: int
token: str
kind: str
file_name: str
mime: str | None
file_size: int | None
title: str | None
chat_id: int | None
message_id: int | None
peer_id: int | None
story_id: int | None
expires_at: datetime | None
max_downloads: int | None
download_count: int
last_download_at: datetime | None
revoked_at: datetime | None
created_at: datetime
class FileShareHitView(BaseModel):
ts: datetime
method: str
ip: str | None
user_agent: str | None
counted: bool
+5 -9
View File
@@ -3,28 +3,24 @@ from datetime import datetime, timedelta
import asyncpg
from utils.read.accounts import self_user_id
from utils.read.media import MEDIA_COLS, ORIGINAL_NAME_JOIN, media_view
from utils.read.models import ChatLinkView, DayCount, MediaView, MessageAt, Page
_MEDIA_COLS = (
"id, account_id, chat_id, message_id, kind, storage_key, file_size, "
"mime, ttl_seconds, downloaded, extracted_text, created_at"
)
async def chat_media(
pool: asyncpg.Pool, account_id: int, chat_id: int, kinds: list[str], page: Page
) -> list[MediaView]:
rows = await pool.fetch(
f"SELECT {_MEDIA_COLS} FROM media " # noqa: S608
"WHERE account_id = $1 AND chat_id = $2 AND kind = ANY($3) "
"ORDER BY message_id DESC LIMIT $4 OFFSET $5",
f"SELECT {MEDIA_COLS} FROM media m {ORIGINAL_NAME_JOIN} " # noqa: S608
"WHERE m.account_id = $1 AND m.chat_id = $2 AND m.kind = ANY($3) "
"ORDER BY m.message_id DESC LIMIT $4 OFFSET $5",
account_id,
chat_id,
kinds,
page.capped_limit,
page.offset,
)
return [MediaView(**dict(row)) for row in rows]
return [media_view(row) for row in rows]
async def chat_links(
+204
View File
@@ -0,0 +1,204 @@
import secrets
from datetime import datetime
import asyncpg
from utils.read.models import FileShareHitView, FileShareView, Page
_COLS = (
"id, account_id, token, kind, file_name, mime, file_size, title, "
"chat_id, message_id, peer_id, story_id, expires_at, max_downloads, "
"download_count, last_download_at, revoked_at, created_at"
)
_TOKEN_BYTES = 12
_SERVE = """
SELECT id, storage_key, file_name, mime, expires_at, max_downloads,
download_count, revoked_at
FROM file_shares WHERE token = $1
"""
_INSERT = (
"INSERT INTO file_shares " # noqa: S608
"(account_id, token, kind, storage_key, file_name, mime, file_size, title, "
"chat_id, message_id, peer_id, story_id, expires_at, max_downloads) "
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) "
f"RETURNING {_COLS}"
)
_ACTIVE = "revoked_at IS NULL AND (expires_at IS NULL OR expires_at > now())"
_LOOKUP = (
f"SELECT {_COLS} FROM file_shares " # noqa: S608
f"WHERE account_id = $1 AND storage_key = $2 AND {_ACTIVE} "
"AND (max_downloads IS NULL OR download_count < max_downloads) "
"ORDER BY created_at DESC LIMIT 1"
)
_CONSUME = """
UPDATE file_shares
SET download_count = download_count + 1, last_download_at = now()
WHERE id = $1 AND (max_downloads IS NULL OR download_count < max_downloads)
RETURNING download_count
"""
def new_token() -> str:
return secrets.token_urlsafe(_TOKEN_BYTES)
def _view(row: asyncpg.Record) -> FileShareView:
return FileShareView(**dict(row))
async def list_shares(
pool: asyncpg.Pool, account_id: int, page: Page, *, active_only: bool = False
) -> list[FileShareView]:
where = "account_id = $1"
if active_only:
where += f" AND {_ACTIVE}"
rows = await pool.fetch(
f"SELECT {_COLS} FROM file_shares WHERE {where} " # noqa: S608
"ORDER BY created_at DESC LIMIT $2 OFFSET $3",
account_id,
page.capped_limit,
page.offset,
)
return [_view(row) for row in rows]
async def get_share(pool: asyncpg.Pool, share_id: int) -> FileShareView | None:
row = await pool.fetchrow(
f"SELECT {_COLS} FROM file_shares WHERE id = $1", # noqa: S608
share_id,
)
return _view(row) if row else None
async def find_active_share(
pool: asyncpg.Pool, account_id: int, storage_key: str
) -> FileShareView | None:
row = await pool.fetchrow(_LOOKUP, account_id, storage_key)
return _view(row) if row else None
async def create_share( # noqa: PLR0913
pool: asyncpg.Pool,
account_id: int,
kind: str,
storage_key: str,
file_name: str,
*,
mime: str | None = None,
file_size: int | None = None,
title: str | None = None,
chat_id: int | None = None,
message_id: int | None = None,
peer_id: int | None = None,
story_id: int | None = None,
expires_at: datetime | None = None,
max_downloads: int | None = None,
) -> FileShareView:
row = await pool.fetchrow(
_INSERT,
account_id,
new_token(),
kind,
storage_key,
file_name,
mime,
file_size,
title,
chat_id,
message_id,
peer_id,
story_id,
expires_at,
max_downloads,
)
return _view(row)
async def update_share(
pool: asyncpg.Pool,
share_id: int,
*,
expires_at: datetime | None,
max_downloads: int | None,
) -> FileShareView | None:
row = await pool.fetchrow(
"UPDATE file_shares SET expires_at = $2, max_downloads = $3 " # noqa: S608
f"WHERE id = $1 RETURNING {_COLS}",
share_id,
expires_at,
max_downloads,
)
return _view(row) if row else None
async def revoke_share(pool: asyncpg.Pool, share_id: int) -> FileShareView | None:
row = await pool.fetchrow(
"UPDATE file_shares SET revoked_at = now() " # noqa: S608
f"WHERE id = $1 AND revoked_at IS NULL RETURNING {_COLS}",
share_id,
)
if row is not None:
return _view(row)
return await get_share(pool, share_id)
async def delete_share(pool: asyncpg.Pool, share_id: int) -> bool:
result = await pool.execute("DELETE FROM file_shares WHERE id = $1", share_id)
return result.endswith("1")
async def rotate_token(pool: asyncpg.Pool, share_id: int) -> FileShareView | None:
row = await pool.fetchrow(
"UPDATE file_shares SET token = $2, revoked_at = NULL " # noqa: S608
f"WHERE id = $1 RETURNING {_COLS}",
share_id,
new_token(),
)
return _view(row) if row else None
async def share_by_token(pool: asyncpg.Pool, token: str) -> asyncpg.Record | None:
return await pool.fetchrow(_SERVE, token)
async def consume_download(pool: asyncpg.Pool, share_id: int) -> bool:
return await pool.fetchval(_CONSUME, share_id) is not None
async def record_hit( # noqa: PLR0913
pool: asyncpg.Pool,
share_id: int,
method: str,
ip: str | None,
user_agent: str | None,
*,
counted: bool,
) -> None:
await pool.execute(
"INSERT INTO file_share_hits (share_id, method, ip, user_agent, counted) "
"VALUES ($1, $2, $3, $4, $5)",
share_id,
method,
ip,
user_agent,
counted,
)
async def list_hits(
pool: asyncpg.Pool, share_id: int, page: Page
) -> list[FileShareHitView]:
rows = await pool.fetch(
"SELECT ts, method, ip, user_agent, counted FROM file_share_hits "
"WHERE share_id = $1 ORDER BY ts DESC LIMIT $2 OFFSET $3",
share_id,
page.capped_limit,
page.offset,
)
return [FileShareHitView(**dict(row)) for row in rows]