fix(backend): a session or bot that fails to start is retried and the failure is logged

backend/src/userbot/runner.py

@@ -29,6 +29,7 @@ from utils.storage import ContentAddressedStorage
 setup_logging()
 
 SESSION_FLUSH_INTERVAL = 60
+SESSION_RETRY_INTERVAL = 60
 
 
 @dataclass
@@ -101,7 +102,10 @@ class AccountRegistry:
                 client, self._pool, self._storage, account_id
             )
         except Exception:
-            logger.exception(f"[red]Failed to start session:[/] {session_name}")
+            logger.exception(
+                f"[red]Failed to start session:[/] {session_name}, "
+                f"retrying in {SESSION_RETRY_INTERVAL}s"
+            )
             with contextlib.suppress(Exception):
                 await client.stop()
             return
@@ -149,6 +153,15 @@ async def _flush_sessions(registry: AccountRegistry) -> None:
                 await client.storage.save()
 
 
+async def _retry_sessions(registry: AccountRegistry) -> None:
+    while True:
+        await asyncio.sleep(SESSION_RETRY_INTERVAL)
+        try:
+            await registry.sync()
+        except Exception:
+            logger.exception("[red]Session sync failed[/]")
+
+
 def _change_listener(registry: AccountRegistry, tasks: set[asyncio.Task]) -> Listener:
     def spawn(coro: Coroutine[Any, Any, None]) -> None:
         task = asyncio.create_task(coro)
@@ -197,6 +210,7 @@ async def runner() -> None:
     stop = asyncio.Event()
     asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, stop.set)
     flush = asyncio.create_task(_flush_sessions(registry))
+    retry = asyncio.create_task(_retry_sessions(registry))
     try:
         await registry.sync()
         if not registry.clients:
@@ -206,6 +220,7 @@ async def runner() -> None:
         await stop.wait()
     finally:
         await _cancel(flush)
+        await _cancel(retry)
         await listener.stop()
         await registry.close()
         await container.close()