feat(backend): policy rules show what they inherit, files of a chat or folder go to trash in one batch, trash list is paged

backend/src/utils/trash.py

@@ -4,6 +4,7 @@ from datetime import timedelta
 
 import asyncpg
 
+from utils.files import media_file_name
 from utils.read.media import get_media
 from utils.read.models import TrashOutcome
 from utils.storage import ContentAddressedStorage
@@ -12,6 +13,7 @@ logger = logging.getLogger(__name__)
 
 RETENTION = timedelta(days=7)
 PURGE_INTERVAL_SECONDS = 3600
+BATCH = 200
 
 _SHARED_ELSEWHERE = """
 SELECT EXISTS (SELECT 1 FROM stories WHERE storage_key = $1)
@@ -47,6 +49,12 @@ _FORGET_KEYS = """
 UPDATE media SET evicted_key = NULL WHERE evicted_key = ANY($1::text[])
 """
 
+_EVICT_MANY = """
+UPDATE media SET evicted_key = storage_key, storage_key = NULL,
+    downloaded = false, evicted_at = now()
+WHERE storage_key = ANY($1::text[])
+"""
+
 
 async def move_to_trash(
     pool: asyncpg.Pool,
@@ -86,12 +94,65 @@ async def move_to_trash(
     return outcome
 
 
+def _move_batch(storage: ContentAddressedStorage, keys: list[str]) -> None:
+    moved: list[str] = []
+    try:
+        for key in keys:
+            if storage.exists(key):
+                storage.trash(key)
+                moved.append(key)
+    except OSError:
+        for key in moved:
+            storage.restore(key)
+        raise
+
+
+async def trash_blobs(
+    pool: asyncpg.Pool,
+    storage: ContentAddressedStorage,
+    account_id: int,
+    blobs: list[asyncpg.Record],
+) -> TrashOutcome:
+    outcome = TrashOutcome()
+    for start in range(0, len(blobs), BATCH):
+        batch = blobs[start : start + BATCH]
+        keys = [blob["storage_key"] for blob in batch]
+        async with pool.acquire() as conn, conn.transaction():
+            await conn.execute(_EVICT_MANY, keys)
+            await conn.executemany(
+                _INSERT,
+                [
+                    (
+                        blob["storage_key"],
+                        account_id,
+                        blob["chat_id"],
+                        blob["message_id"],
+                        blob["kind"],
+                        blob["mime"],
+                        media_file_name(blob["kind"], blob["mime"], blob["message_id"]),
+                        blob["file_size"],
+                    )
+                    for blob in batch
+                ],
+            )
+            await asyncio.to_thread(_move_batch, storage, keys)
+        outcome.keys.extend(keys)
+        outcome.files += len(batch)
+        outcome.bytes += sum(blob["file_size"] or 0 for blob in batch)
+    return outcome
+
+
 async def restore(
     pool: asyncpg.Pool,
     storage: ContentAddressedStorage,
     account_id: int,
-    keys: list[str],
+    keys: list[str] | None,
 ) -> TrashOutcome:
+    if keys is None:
+        rows = await pool.fetch(
+            "SELECT storage_key FROM media_trash WHERE account_id = $1", account_id
+        )
+        keys = [row["storage_key"] for row in rows]
     outcome = TrashOutcome()
     for key in keys:
         async with pool.acquire() as conn, conn.transaction():