feat(backend): storage trash with seven day retention, trashed files are not fetched again automatically

backend/src/userbot/modules/capture/repository.py

@@ -88,8 +88,8 @@ ON CONFLICT (account_id, chat_id, message_id, storage_key) DO NOTHING
 _INSERT_MEDIA = """
 INSERT INTO media
     (account_id, chat_id, message_id, kind, storage_key, file_size, mime,
-     ttl_seconds, downloaded, unique_id)
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+     ttl_seconds, downloaded, unique_id, evicted_at)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, CASE WHEN $11 THEN now() END)
 ON CONFLICT (account_id, chat_id, message_id) DO UPDATE SET
     kind = EXCLUDED.kind,
     storage_key = EXCLUDED.storage_key,
@@ -97,7 +97,27 @@ ON CONFLICT (account_id, chat_id, message_id) DO UPDATE SET
     mime = EXCLUDED.mime,
     ttl_seconds = EXCLUDED.ttl_seconds,
     downloaded = EXCLUDED.downloaded,
-    unique_id = EXCLUDED.unique_id
+    unique_id = EXCLUDED.unique_id,
+    evicted_at = CASE
+        WHEN EXCLUDED.downloaded THEN NULL
+        WHEN media.unique_id IS DISTINCT FROM EXCLUDED.unique_id
+            THEN EXCLUDED.evicted_at
+        ELSE coalesce(media.evicted_at, EXCLUDED.evicted_at) END,
+    evicted_key = CASE WHEN EXCLUDED.downloaded
+        OR media.unique_id IS DISTINCT FROM EXCLUDED.unique_id
+        THEN NULL ELSE media.evicted_key END
+"""
+
+_IS_EVICTED = """
+SELECT EXISTS (
+    SELECT 1 FROM media WHERE unique_id = $1 AND evicted_at IS NOT NULL
+)
+"""
+
+_UNEVICT = """
+UPDATE media SET storage_key = $2, file_size = $3, downloaded = true,
+    evicted_at = NULL, evicted_key = NULL
+WHERE evicted_key = $2 OR (unique_id = $1 AND evicted_at IS NOT NULL)
 """
 
 
@@ -229,6 +249,7 @@ async def insert_media(  # noqa: PLR0913
     unique_id: str | None,
     *,
     downloaded: bool,
+    evicted: bool = False,
 ) -> None:
     async with pool.acquire() as conn, conn.transaction():
         await conn.execute(
@@ -243,6 +264,7 @@ async def insert_media(  # noqa: PLR0913
             ttl_seconds,
             downloaded,
             unique_id,
+            evicted,
         )
         if storage_key is not None:
             await conn.execute(
@@ -259,6 +281,22 @@ async def insert_media(  # noqa: PLR0913
             )
 
 
+async def is_evicted(pool: asyncpg.Pool, unique_id: str | None) -> bool:
+    return unique_id is not None and await pool.fetchval(_IS_EVICTED, unique_id)
+
+
+async def unevict(
+    pool: asyncpg.Pool, unique_id: str | None, storage_key: str, file_size: int | None
+) -> bool:
+    async with pool.acquire() as conn, conn.transaction():
+        await conn.execute(_UNEVICT, unique_id, storage_key, file_size)
+        dropped = await conn.fetchval(
+            "DELETE FROM media_trash WHERE storage_key = $1 RETURNING storage_key",
+            storage_key,
+        )
+    return dropped is not None
+
+
 async def insert_callbacks(
     pool: asyncpg.Pool,
     account_id: int,

backend/src/userbot/modules/jobs/handlers/fetch_media.py

@@ -21,7 +21,9 @@ async def fetch_media(ctx: JobContext) -> None:
         msg = f"message {chat_id}/{message_id} is unavailable"
         raise RuntimeError(msg)
     toggles = CaptureToggles(media=True, self_destruct_media=True)
-    ok = await capture_media(client, message, capture, chat_id, message_id, toggles)
+    ok = await capture_media(
+        client, message, capture, chat_id, message_id, toggles, force=True
+    )
     if not ok:
         msg = f"media download failed for {chat_id}/{message_id}"
         raise RuntimeError(msg)

backend/src/userbot/modules/media/downloader.py

@@ -106,6 +106,8 @@ async def capture_media(  # noqa: PLR0913
     chat_id: int,
     message_id: int,
     toggles: CaptureToggles,
+    *,
+    force: bool = False,
 ) -> bool:
     kind, obj = media_object(message)
     if obj is None:
@@ -117,6 +119,7 @@ async def capture_media(  # noqa: PLR0913
     mime = getattr(obj, "mime_type", None)
     storage_key: str | None = None
     downloaded = False
+    evicted = False
     flood: FloodWait | FloodPremiumWait | None = None
     if want:
         existing = await repository.current_media(
@@ -131,7 +134,7 @@ async def capture_media(  # noqa: PLR0913
             storage_key = existing["storage_key"]
             file_size = existing["file_size"]
             downloaded = True
-        else:
+        elif force or not await repository.is_evicted(ctx.pool, unique_id):
             target = _download_target(message)
             try:
                 data = await _download(client, target, chat_id, message_id)
@@ -142,6 +145,8 @@ async def capture_media(  # noqa: PLR0913
                 storage_key = ctx.storage.put(data)
                 file_size = len(data)
                 downloaded = True
+        else:
+            evicted = True
     await repository.insert_media(
         ctx.pool,
         ctx.account_id,
@@ -154,7 +159,14 @@ async def capture_media(  # noqa: PLR0913
         ttl,
         unique_id,
         downloaded=downloaded,
+        evicted=evicted,
     )
+    if (
+        force
+        and storage_key is not None
+        and await repository.unevict(ctx.pool, unique_id, storage_key, file_size)
+    ):
+        ctx.storage.purge(storage_key)
     if flood is not None:
         raise flood
     return downloaded or not want