fix(backend): a session or bot that fails to start is retried and the failure is logged

backend/src/userbot/runner.py

@@ -29,6 +29,7 @@ from utils.storage import ContentAddressedStorage
 setup_logging()
 
 SESSION_FLUSH_INTERVAL = 60
+SESSION_RETRY_INTERVAL = 60
 
 
 @dataclass
@@ -101,7 +102,10 @@ class AccountRegistry:
                 client, self._pool, self._storage, account_id
             )
         except Exception:
-            logger.exception(f"[red]Failed to start session:[/] {session_name}")
+            logger.exception(
+                f"[red]Failed to start session:[/] {session_name}, "
+                f"retrying in {SESSION_RETRY_INTERVAL}s"
+            )
             with contextlib.suppress(Exception):
                 await client.stop()
             return
@@ -149,6 +153,15 @@ async def _flush_sessions(registry: AccountRegistry) -> None:
                 await client.storage.save()
 
 
+async def _retry_sessions(registry: AccountRegistry) -> None:
+    while True:
+        await asyncio.sleep(SESSION_RETRY_INTERVAL)
+        try:
+            await registry.sync()
+        except Exception:
+            logger.exception("[red]Session sync failed[/]")
+
+
 def _change_listener(registry: AccountRegistry, tasks: set[asyncio.Task]) -> Listener:
     def spawn(coro: Coroutine[Any, Any, None]) -> None:
         task = asyncio.create_task(coro)
@@ -197,6 +210,7 @@ async def runner() -> None:
     stop = asyncio.Event()
     asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, stop.set)
     flush = asyncio.create_task(_flush_sessions(registry))
+    retry = asyncio.create_task(_retry_sessions(registry))
     try:
         await registry.sync()
         if not registry.clients:
@@ -206,6 +220,7 @@ async def runner() -> None:
         await stop.wait()
     finally:
         await _cancel(flush)
+        await _cancel(retry)
         await listener.stop()
         await registry.close()
         await container.close()

backend/src/watcher/registry.py

@@ -167,6 +167,7 @@ class BotRegistry:
                 await client.stop()
             return
 
+        try:
             account_id = await accounts.sync_account(
                 self._pool, me, str(bot_id), is_bot=True
             )
@@ -175,9 +176,24 @@ class BotRegistry:
             )
             await bots.link_account(self._pool, bot_id, account_id)
             await bots.set_status(
-            self._pool, bot_id, bots.STATUS_OK, username=me.username, title=me.full_name
+                self._pool,
+                bot_id,
+                bots.STATUS_OK,
+                username=me.username,
+                title=me.full_name,
             )
             await accounts.notify_accounts_changed(self._pool)
+        except Exception as exc:
+            logger.exception(
+                f"[red]Bot {bot_id} failed to start,[/] retrying on the next sync"
+            )
+            with contextlib.suppress(Exception):
+                await bots.set_status(
+                    self._pool, bot_id, bots.STATUS_ERROR, detail=str(exc)
+                )
+            with contextlib.suppress(Exception):
+                await client.stop()
+            return
 
         consumer = JobConsumer(client, self._pool, account_id)
         self._running[bot_id] = RunningBot(

backend/src/watcher/runner.py

@@ -20,8 +20,10 @@ RETRY_INTERVAL = 300
 async def _retry_loop(registry: BotRegistry) -> None:
     while True:
         await asyncio.sleep(RETRY_INTERVAL)
-        with contextlib.suppress(Exception):
+        try:
             await registry.sync()
+        except Exception:
+            logger.exception("[red]Bot sync failed[/]")
 
 
 def _change_listener(registry: BotRegistry, tasks: set[asyncio.Task]) -> Listener: