feat(*): komodo actions menu with pins and commands, fleet by server, uptime history

pyproject.toml

@@ -11,7 +11,9 @@ dependencies = [
     "aiohttp>=3.10.0",
     "aiosqlite>=0.20.0",
     "pydantic-settings>=2.14.2",
+    "pyyaml>=6.0.3",
     "rich>=15.0.0",
+    "tzdata>=2026.5",
 ]
 
 [build-system]

src/healthbot/__init__.py

@@ -12,6 +12,7 @@ async def runner() -> None:
 
     from . import handlers  # noqa: PLC0415
     from .common import bot, dp  # noqa: PLC0415
+    from .digest import run_digest  # noqa: PLC0415
     from .komodo import komodo  # noqa: PLC0415
     from .monitor import run_monitor  # noqa: PLC0415
     from .web import run_web  # noqa: PLC0415
@@ -20,14 +21,16 @@ async def runner() -> None:
     dp.include_routers(handlers.router)
     await bot.delete_webhook(drop_pending_updates=True)
 
-    monitor = asyncio.create_task(run_monitor())
+    background = [asyncio.create_task(run_monitor()), asyncio.create_task(run_digest())]
     web = await run_web() if env.web.enabled else None
     try:
         await dp.start_polling(bot)
     finally:
-        monitor.cancel()
+        for task in background:
+            task.cancel()
+        for task in background:
             with contextlib.suppress(asyncio.CancelledError):
-            await monitor
+                await task
         if web is not None:
             await web.cleanup()
         await komodo.close()

src/healthbot/actionargs.py

@@ -0,0 +1,186 @@
+import json
+import re
+import shlex
+import tomllib
+from typing import Any
+
+import yaml
+
+QUOTES = str.maketrans({"“": '"', "”": '"', "„": '"', "«": '"', "»": '"'})
+KEY_RE = re.compile(r"[\w.-]+")
+TRUE = {"true", "yes", "on", "1", "да", "вкл"}
+FALSE = {"false", "no", "off", "0", "нет", "выкл"}
+VARIABLE_MARK = "[["
+DASHES = "-\u2014\u2013"
+
+
+class ArgError(ValueError):
+    pass
+
+
+def _key_value(text: str) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for raw in text.splitlines():
+        line = raw.strip().removeprefix("- ").strip()
+        if not line or line.startswith("#"):
+            continue
+        match = re.match(r"([^=:\s]+)\s*[=:]\s*(.*)", line)
+        if not match:
+            continue
+        value = match.group(2).strip()
+        if len(value) > 1 and value[0] == value[-1] and value[0] in "\"'":
+            value = value[1:-1]
+        result[match.group(1)] = value
+    return result
+
+
+def parse_defaults(fmt: str, text: str) -> dict[str, Any]:
+    if not text.strip():
+        return {}
+    try:
+        if fmt == "json":
+            data = json.loads(text)
+        elif fmt == "toml":
+            data = tomllib.loads(text)
+        elif fmt == "yaml":
+            data = yaml.safe_load(text)
+        else:
+            data = _key_value(text)
+    except (ValueError, yaml.YAMLError):
+        return {}
+    if not isinstance(data, dict):
+        return {}
+    return json.loads(json.dumps(data, default=str))
+
+
+def type_name(value: object) -> str:
+    if isinstance(value, bool):
+        return "да или нет"
+    if isinstance(value, (int, float)):
+        return "число"
+    if isinstance(value, str):
+        return "текст"
+    return "JSON"
+
+
+def render(value: object, limit: int = 0) -> str:
+    text = json.dumps(value, ensure_ascii=False)
+    return text if not limit or len(text) <= limit else text[: limit - 1] + "…"
+
+
+def _guess(text: str) -> Any:  # noqa: ANN401
+    try:
+        return json.loads(text)
+    except ValueError:
+        return text
+
+
+def _to_bool(text: str) -> bool:
+    lowered = text.strip().lower()
+    if lowered in TRUE:
+        return True
+    if lowered in FALSE:
+        return False
+    msg = "нужно true или false"
+    raise ArgError(msg)
+
+
+def _to_number(text: str, *, keep_float: bool) -> int | float:
+    try:
+        number = float(text.strip().replace(",", "."))
+    except ValueError as exc:
+        msg = "нужно число"
+        raise ArgError(msg) from exc
+    return number if keep_float or not number.is_integer() else int(number)
+
+
+def _to_json(default: object, text: str) -> Any:  # noqa: ANN401
+    try:
+        value = json.loads(text)
+    except ValueError as exc:
+        msg = "нужен JSON"
+        raise ArgError(msg) from exc
+    if type(value) is not type(default):
+        msg = "нужен JSON-объект" if isinstance(default, dict) else "нужен JSON-массив"
+        raise ArgError(msg)
+    return value
+
+
+def coerce(default: object, text: str) -> Any:  # noqa: ANN401
+    if isinstance(default, bool):
+        return _to_bool(text)
+    if isinstance(default, (int, float)):
+        return _to_number(text, keep_float=isinstance(default, float))
+    if isinstance(default, str):
+        return text
+    if default is None:
+        return _guess(text)
+    return _to_json(default, text)
+
+
+def _tokens(text: str) -> list[str]:
+    try:
+        return shlex.split(text.translate(QUOTES))
+    except ValueError as exc:
+        msg = "кавычка не закрыта"
+        raise ArgError(msg) from exc
+
+
+def _pairs(tokens: list[str], defaults: dict[str, Any]) -> list[tuple[str, str]]:
+    pairs: list[tuple[str, str]] = []
+    index = 0
+    while index < len(tokens):
+        token = tokens[index].lstrip(DASHES)
+        key, sep, value = token.partition("=")
+        inline = bool(sep) and bool(KEY_RE.fullmatch(key))
+        name = key if inline else token
+        index += 1
+        if defaults and name not in defaults:
+            msg = f"нет поля {name}, есть: {', '.join(defaults)}"
+            raise ArgError(msg)
+        if inline:
+            pairs.append((key, value))
+        elif isinstance(defaults.get(token), bool) and not (
+            index < len(tokens) and tokens[index].lower() in TRUE | FALSE
+        ):
+            pairs.append((token, "true"))
+        elif index < len(tokens):
+            pairs.append((token, tokens[index]))
+            index += 1
+        else:
+            msg = f"у поля {token} нет значения"
+            raise ArgError(msg)
+    return pairs
+
+
+def parse_cli(text: str, defaults: dict[str, Any]) -> dict[str, Any]:
+    text = text.strip()
+    if not text:
+        return {}
+    tokens = _tokens(text)
+    first = tokens[0].lstrip(DASHES).partition("=")[0]
+    if len(defaults) == 1 and first not in defaults:
+        key = next(iter(defaults))
+        return {key: coerce(defaults[key], " ".join(tokens))}
+    result: dict[str, Any] = {}
+    for key, value in _pairs(tokens, defaults):
+        try:
+            result[key] = coerce(defaults[key], value) if defaults else _guess(value)
+        except ArgError as exc:
+            msg = f"{key}: {exc}"
+            raise ArgError(msg) from exc
+    return result
+
+
+def changed(defaults: dict[str, Any], values: dict[str, Any]) -> dict[str, Any]:
+    return {k: v for k, v in values.items() if k not in defaults or defaults[k] != v}
+
+
+def ensure_safe(args: dict[str, Any]) -> None:
+    if VARIABLE_MARK in json.dumps(args, ensure_ascii=False):
+        msg = "в значениях нельзя «[[»: Komodo читает это как переменную"
+        raise ArgError(msg)
+
+
+def command_name(name: str) -> str:
+    return re.sub(r"[^a-z0-9_]", "_", name.lower()).strip("_")[:32]

src/healthbot/alerts.py

@@ -1,8 +1,11 @@
-"""Превращение алерта Komodo в сообщение для телеграма."""
-
-import html
 from typing import Any
 
+from aiogram.types import InlineKeyboardMarkup
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+from healthbot.ui import esc
+from utils.env import env
+
 LEVEL_ICON = {"OK": "✅", "WARNING": "🟠", "CRITICAL": "🔴"}
 
 INFORMATIONAL = frozenset(
@@ -21,13 +24,33 @@ FAILURE = frozenset(
     {"ProcedureFailed", "ActionFailed", "BuildFailed", "RepoBuildFailed"}
 )
 
-# Komodo создаёт их сразу с resolved=true (resolved_ts == ts): это факт
-# «состояние сменилось», а не тревога, которую потом снимают. Без этого
-# и running → unhealthy, и unhealthy → running приходили как «Отбой».
 STATE_CHANGE = frozenset({"StackStateChange", "ContainerStateChange"})
 HEALTHY_STATES = frozenset({"running", "healthy"})
 
-ONE_SHOT = INFORMATIONAL | FAILURE | STATE_CHANGE
+CREATED_ALREADY_RESOLVED = INFORMATIONAL | FAILURE | STATE_CHANGE
+
+KNOWN_TYPES = (
+    "ServerUnreachable",
+    "ServerCpu",
+    "ServerMem",
+    "ServerDisk",
+    "ServerVersionMismatch",
+    "StackStateChange",
+    "ContainerStateChange",
+    "StackImageUpdateAvailable",
+    "DeploymentImageUpdateAvailable",
+    "StackAutoUpdated",
+    "DeploymentAutoUpdated",
+    "ResourceSyncPendingUpdates",
+    "ProcedureFailed",
+    "ActionFailed",
+    "BuildFailed",
+    "RepoBuildFailed",
+    "ScheduleRun",
+    "Custom",
+)
+
+CALLBACK_LIMIT = 64
 
 
 def _gb(value: Any) -> str:  # noqa: ANN401
@@ -44,79 +67,88 @@ def _pct(value: Any) -> str:  # noqa: ANN401
         return str(value)
 
 
+def _error(value: Any) -> str:  # noqa: ANN401
+    if isinstance(value, dict):
+        return str(value.get("error") or value)
+    return str(value or "нет деталей")
+
+
 def _describe(kind: str, d: dict) -> tuple[str, list[str]]:  # noqa: C901, PLR0911, PLR0912
-    """Заголовок и строки подробностей для конкретного типа алерта."""
     name = d.get("name") or d.get("id") or "?"
     server = d.get("server_name")
 
     if kind == "ServerUnreachable":
-        return f"Сервер <b>{html.escape(name)}</b> не отвечает", [
+        return f"Сервер <b>{esc(name)}</b> не отвечает", [
-            f"Ошибка: {html.escape(str(d.get('err') or 'нет деталей'))}"
+            f"Ошибка: {esc(_error(d.get('err')))}"
         ]
     if kind == "ServerCpu":
-        return f"CPU на <b>{html.escape(name)}</b>", [
+        return f"CPU на <b>{esc(name)}</b>", [f"Загрузка: {_pct(d.get('percentage'))}"]
-            f"Загрузка: {_pct(d.get('percentage'))}"
-        ]
     if kind == "ServerMem":
-        return f"Память на <b>{html.escape(name)}</b>", [
+        return f"Память на <b>{esc(name)}</b>", [
             f"Занято: {_gb(d.get('used_gb'))} из {_gb(d.get('total_gb'))}"
         ]
     if kind == "ServerDisk":
-        return f"Диск на <b>{html.escape(name)}</b>", [
+        return f"Диск на <b>{esc(name)}</b>", [
-            f"Точка: <code>{html.escape(str(d.get('path', '/')))}</code>",
+            f"Точка: <code>{esc(d.get('path', '/'))}</code>",
             f"Занято: {_gb(d.get('used_gb'))} из {_gb(d.get('total_gb'))}",
         ]
     if kind == "ServerVersionMismatch":
-        return f"Разъехались версии на <b>{html.escape(name)}</b>", [
+        return f"Разъехались версии на <b>{esc(name)}</b>", [
-            f"Periphery: {d.get('version')}, Core: {d.get('core_version')}",
+            f"Periphery: {esc(d.get('version'))}, Core: {esc(d.get('core_version'))}",
             "Чинится только по ssh - дашборд к такому серверу не достучится.",
         ]
-    if kind in {"StackStateChange", "ContainerStateChange"}:
+    if kind in STATE_CHANGE:
-        return f"<b>{html.escape(name)}</b>: {d.get('from')} → {d.get('to')}", (
+        return f"<b>{esc(name)}</b>: {esc(d.get('from'))} → {esc(d.get('to'))}", (
-            [f"Сервер: {html.escape(server)}"] if server else []
+            [f"Сервер: {esc(server)}"] if server else []
         )
     if kind in {"StackImageUpdateAvailable", "DeploymentImageUpdateAvailable"}:
-        return f"Обновление образа для <b>{html.escape(name)}</b>", [
+        return f"Обновление образа для <b>{esc(name)}</b>", [
-            f"<code>{html.escape(str(d.get('image', '?')))}</code>"
+            f"<code>{esc(d.get('image', '?'))}</code>"
         ]
     if kind in {"StackAutoUpdated", "DeploymentAutoUpdated"}:
-        return f"<b>{html.escape(name)}</b> обновлён автоматически", [
+        images = d.get("images") or d.get("image") or ""
-            f"<code>{html.escape(str(d.get('images') or d.get('image') or ''))}</code>"
+        if isinstance(images, list):
+            images = ", ".join(str(image) for image in images)
+        return f"<b>{esc(name)}</b> обновлён автоматически", [
+            f"<code>{esc(images)}</code>"
         ]
     if kind == "ResourceSyncPendingUpdates":
-        return f"Синк <b>{html.escape(name)}</b>: git разошёлся с Komodo", [
+        return f"Синк <b>{esc(name)}</b>: git разошёлся с Komodo", [
             "Кто-то правил ресурсы мимо git, либо пуш не доехал."
         ]
-    if kind in {"ProcedureFailed", "ActionFailed", "BuildFailed", "RepoBuildFailed"}:
+    if kind in FAILURE:
         what = {"ProcedureFailed": "Процедура", "ActionFailed": "Действие"}.get(
             kind, "Сборка"
         )
-        return f"{what} <b>{html.escape(name)}</b> упала", []
+        fell = "упало" if kind == "ActionFailed" else "упала"
+        return f"{what} <b>{esc(name)}</b> {fell}", []
     if kind == "ScheduleRun":
-        return f"По расписанию запущено: <b>{html.escape(name)}</b>", []
+        return f"По расписанию запущено: <b>{esc(name)}</b>", []
     if kind == "Test":
         return "Проверка связи с Komodo", ["Если ты это видишь - канал работает."]
     if kind == "Custom":
-        return html.escape(str(d.get("title") or "Сообщение из Komodo")), [
+        return esc(d.get("title") or "Сообщение из Komodo"), [esc(d.get("body") or "")]
-            html.escape(str(d.get("body") or ""))
-        ]
 
     rest = [
-        f"{html.escape(str(k))}: {html.escape(str(v))}"
+        f"{esc(k)}: {esc(v)}"
         for k, v in d.items()
         if k not in {"id", "server_id", "swarm_id"}
     ]
-    return f"{html.escape(kind)}: <b>{html.escape(name)}</b>", rest[:8]
+    return f"{esc(kind)}: <b>{esc(name)}</b>", rest[:8]
+
+
+def alert_kind(payload: dict) -> str:
+    return str((payload.get("data") or {}).get("type") or "Unknown")
 
 
 def format_alert(payload: dict) -> str:
     level = str(payload.get("level", "")).upper()
     data = payload.get("data") or {}
-    kind = str(data.get("type") or "Unknown")
+    kind = alert_kind(payload)
     inner = data.get("data") or {}
 
     informational = kind in INFORMATIONAL
-    resolved = bool(payload.get("resolved")) and kind not in ONE_SHOT
+    resolved = bool(payload.get("resolved")) and kind not in CREATED_ALREADY_RESOLVED
 
     if informational:
         icon = "ℹ️"
@@ -131,11 +163,34 @@ def format_alert(payload: dict) -> str:
     title, details = _describe(kind, inner if isinstance(inner, dict) else {})
 
     if resolved:
-        return "\n".join(
+        return "\n".join([f"{icon} Отбой: {title}", f"<i>Komodo · {esc(kind)}</i>"])
-            [f"{icon} Отбой: {title}", f"<i>Komodo · {html.escape(kind)}</i>"]
-        )
 
     lines = [f"{icon} {title}"]
     lines.extend(x for x in details if x)
-    lines.append(f"<i>Komodo · {html.escape(kind)}</i>")
+    lines.append(f"<i>Komodo · {esc(kind)}</i>")
     return "\n".join(lines)
+
+
+def alert_keyboard(payload: dict) -> InlineKeyboardMarkup | None:
+    kind = alert_kind(payload)
+    target = payload.get("target") or {}
+    target_type, target_id = target.get("type"), target.get("id")
+    kb = InlineKeyboardBuilder()
+    if env.komodo.enabled and target_id:
+        if target_type == "Stack":
+            kb.button(text="Стек", callback_data=f"go:stack:{target_id}")
+            kb.button(text="Логи", callback_data=f"go:log:{target_id}")
+        elif target_type == "Server":
+            kb.button(text="Сервер", callback_data=f"go:srv:{target_id}")
+        elif target_type == "Action":
+            kb.button(text="Действие", callback_data=f"go:act:a:{target_id}")
+        elif target_type == "Procedure":
+            kb.button(text="Процедура", callback_data=f"go:act:p:{target_id}")
+    mute = f"al:mute:{kind}"
+    if kind != "Test" and len(mute) <= CALLBACK_LIMIT:
+        kb.button(text="Не присылать такие", callback_data=mute)
+    buttons = len(list(kb.buttons))
+    if not buttons:
+        return None
+    kb.adjust(buttons - 1 or 1, 1)
+    return kb.as_markup()

src/healthbot/checker.py

@@ -1,21 +1,27 @@
-"""HTTP-пробинг сервисов и логика сравнения ответа с эталоном."""
+import asyncio
-
 import difflib
-import html
 import json
+import ssl
 import time
 from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from urllib.parse import urlparse
 
 import aiohttp
 
+from healthbot.ui import esc
 from utils.db.models import MatchType, Service
 from utils.env import env
 from utils.format import short
 
-_MISSING = object()
+MISSING = object()
 
 DIFF_LINES = 12
 DIFF_HEADER_LINES = 2
+BODY_LIMIT = 1024 * 1024
+CHUNK = 64 * 1024
+TLS_TIMEOUT = 10
+HEADERS = {"User-Agent": "healthbot"}
 
 
 @dataclass(slots=True)
@@ -29,8 +35,6 @@ class Probe:
 
 @dataclass(slots=True)
 class Verdict:
-    """Результат сравнения."""
-
     healthy: bool
     reason: str = ""
     expected: str | None = None
@@ -38,11 +42,10 @@ class Verdict:
     details: list[str] = field(default_factory=list)
 
     def report(self) -> str:
-        """Человекочитаемое объяснение для сообщения в телеграм."""
         lines: list[str] = []
         if self.expected is not None or self.actual is not None:
-            lines.append(f"Ожидали: <code>{html.escape(self.expected or '-')}</code>")
+            lines.append(f"Ожидали: <code>{esc(self.expected or '-')}</code>")
-            lines.append(f"Сейчас: <code>{html.escape(self.actual or '-')}</code>")
+            lines.append(f"Сейчас: <code>{esc(self.actual or '-')}</code>")
         lines.extend(self.details)
         return "\n".join(lines)
 
@@ -51,9 +54,19 @@ async def probe(session: aiohttp.ClientSession, url: str, timeout: int) -> Probe
     started = time.monotonic()
     try:
         async with session.get(
-            url, timeout=aiohttp.ClientTimeout(total=timeout), allow_redirects=True
+            url,
+            timeout=aiohttp.ClientTimeout(total=timeout),
+            allow_redirects=True,
+            headers=HEADERS,
         ) as resp:
-            body = await resp.text()
+            raw = bytearray()
+            async for chunk in resp.content.iter_chunked(CHUNK):
+                raw += chunk
+                if len(raw) >= BODY_LIMIT:
+                    break
+            body = bytes(raw[:BODY_LIMIT]).decode(
+                resp.charset or "utf-8", errors="replace"
+            )
             return Probe(
                 reachable=True,
                 status=resp.status,
@@ -65,40 +78,65 @@ async def probe(session: aiohttp.ClientSession, url: str, timeout: int) -> Probe
         return Probe(
             reachable=False, status=None, body="", error=f"таймаут ({timeout}с)"
         )
-    except aiohttp.ClientError as exc:
+    except aiohttp.ClientConnectorError as exc:
+        reason = exc.os_error.strerror or str(exc.os_error)
+        return Probe(
+            reachable=False, status=None, body="", error=f"нет связи: {reason}"
+        )
+    except (aiohttp.ClientError, ValueError, LookupError) as exc:
         return Probe(reachable=False, status=None, body="", error=f"нет связи: {exc}")
 
 
+async def tls_expiry(url: str) -> datetime | None:
+    parsed = urlparse(url)
+    if parsed.scheme != "https" or not parsed.hostname:
+        return None
+    try:
+        _, writer = await asyncio.wait_for(
+            asyncio.open_connection(
+                parsed.hostname, parsed.port or 443, ssl=ssl.create_default_context()
+            ),
+            TLS_TIMEOUT,
+        )
+    except (OSError, TimeoutError):
+        return None
+    try:
+        cert = writer.get_extra_info("peercert") or {}
+    finally:
+        writer.close()
+    not_after = cert.get("notAfter")
+    if not not_after:
+        return None
+    return datetime.fromtimestamp(ssl.cert_time_to_seconds(not_after), UTC)
+
+
 def dig(data: object, path: str) -> object:
-    """Достаёт значение по dot-пути (индексы списков - числами)."""
     node = data
     for segment in path.split("."):
         if isinstance(node, dict):
             if segment not in node:
-                return _MISSING
+                return MISSING
             node = node[segment]
         elif isinstance(node, list):
             if not segment.isdigit() or int(segment) >= len(node):
-                return _MISSING
+                return MISSING
             node = node[int(segment)]
         else:
-            return _MISSING
+            return MISSING
     return node
 
 
 def snippet(text: str, limit: int | None = None) -> list[str]:
-    """Кусок ответа в <pre>, готовый к вставке в сообщение."""
     limit = limit or env.monitor.body_preview
     body = text.strip()
     if not body:
-        return ["<i>(пустой ответ)</i>"]
+        return ["<i>пустой ответ</i>"]
     clipped = body[:limit]
     tail = "\n…" if len(body) > limit else ""
-    return [f"<pre>{html.escape(clipped)}{tail}</pre>"]
+    return [f"<pre>{esc(clipped)}{tail}</pre>"]
 
 
 def body_diff(expected: str, actual: str) -> list[str]:
-    """Чем именно тело отличается от эталона."""
     diff = list(
         difflib.unified_diff(
             expected.strip().splitlines(), actual.strip().splitlines(), lineterm="", n=0
@@ -108,57 +146,34 @@ def body_diff(expected: str, actual: str) -> list[str]:
         return ["Отличие в невидимых символах.", *snippet(actual)]
     shown = diff[:DIFF_LINES]
     text = "\n".join(shown)
-    more = f"\n… ещё {len(diff) - len(shown)} строк" if len(diff) > len(shown) else ""
+    more = f"\nещё строк: {len(diff) - len(shown)}" if len(diff) > len(shown) else ""
-    return [f"<pre>{html.escape(text)}{more}</pre>"]
+    return [f"<pre>{esc(text)}{more}</pre>"]
 
 
-def evaluate(service: Service, result: Probe) -> Verdict:  # noqa: PLR0911
+def _json_verdict(service: Service, result: Probe) -> Verdict:
-    """Жив ли сервис - и если нет, то что именно он сейчас отдаёт."""
+    expected = f"{service.json_path} = {service.json_value}"
-    if not result.reachable:
-        return Verdict(healthy=False, reason=result.error or "недоступен")
-
-    if service.match_type is MatchType.STATUS:
-        if result.status != service.ok_status:
-            return Verdict(
-                healthy=False,
-                reason=f"HTTP {result.status}",
-                expected=f"HTTP {service.ok_status}",
-                actual=f"HTTP {result.status}",
-                details=snippet(result.body),
-            )
-        return Verdict(healthy=True)
-
-    if service.match_type is MatchType.BODY:
-        if result.body.strip() != service.ok_body.strip():
-            return Verdict(
-                healthy=False,
-                reason="тело ответа изменилось",
-                details=body_diff(service.ok_body, result.body),
-            )
-        return Verdict(healthy=True)
-
     try:
         data = json.loads(result.body)
     except ValueError:
         return Verdict(
             healthy=False,
             reason="ответ перестал быть JSON",
-            expected=f"{service.json_path} = {service.json_value}",
+            expected=expected,
             actual=f"не JSON, HTTP {result.status}",
             details=snippet(result.body),
         )
 
     value = dig(data, service.json_path or "")
-    if value is _MISSING:
+    if value is MISSING:
         return Verdict(
             healthy=False,
             reason=f"нет поля {service.json_path}",
-            expected=f"{service.json_path} = {service.json_value}",
+            expected=expected,
             actual="поля нет в ответе",
             details=snippet(result.body),
         )
 
-    current = _scalar_str(value)
+    current = scalar_str(value)
     if current != service.json_value:
         return Verdict(
             healthy=False,
@@ -170,8 +185,56 @@ def evaluate(service: Service, result: Probe) -> Verdict:  # noqa: PLR0911
     return Verdict(healthy=True)
 
 
-def _scalar_str(value: object) -> str:
+def _status_verdict(service: Service, result: Probe) -> Verdict:
-    """Строковое представление скаляра JSON (bool/null как в JSON)."""
+    if result.status == service.ok_status:
+        return Verdict(healthy=True)
+    return Verdict(
+        healthy=False,
+        reason=f"HTTP {result.status}",
+        expected=f"HTTP {service.ok_status}",
+        actual=f"HTTP {result.status}",
+        details=snippet(result.body),
+    )
+
+
+def _body_verdict(service: Service, result: Probe) -> Verdict:
+    if result.body.strip() == service.ok_body.strip():
+        return Verdict(healthy=True)
+    return Verdict(
+        healthy=False,
+        reason="тело ответа изменилось",
+        details=body_diff(service.ok_body, result.body),
+    )
+
+
+def _contains_verdict(service: Service, result: Probe) -> Verdict:
+    needle = service.needle or ""
+    if needle in result.body:
+        return Verdict(healthy=True)
+    return Verdict(
+        healthy=False,
+        reason=f"в ответе нет «{short(needle, 40)}»",
+        expected=f"есть {needle}",
+        actual=f"нет, HTTP {result.status}",
+        details=snippet(result.body),
+    )
+
+
+VERDICTS = {
+    MatchType.STATUS: _status_verdict,
+    MatchType.BODY: _body_verdict,
+    MatchType.CONTAINS: _contains_verdict,
+    MatchType.JSON: _json_verdict,
+}
+
+
+def evaluate(service: Service, result: Probe) -> Verdict:
+    if not result.reachable:
+        return Verdict(healthy=False, reason=result.error or "недоступен")
+    return VERDICTS[service.match_type](service, result)
+
+
+def scalar_str(value: object) -> str:
     if isinstance(value, bool):
         return "true" if value else "false"
     if value is None:

src/healthbot/common.py

@@ -6,7 +6,7 @@ from utils.env import env
 
 bot = Bot(
     token=env.bot.token.get_secret_value(),
-    default=DefaultBotProperties(parse_mode="HTML"),
+    default=DefaultBotProperties(parse_mode="HTML", link_preview_is_disabled=True),
 )
 dp = Dispatcher(storage=MemoryStorage())
 

src/healthbot/digest.py

@@ -0,0 +1,118 @@
+import asyncio
+from datetime import timedelta
+
+from healthbot.keyboards import back_to_menu
+from healthbot.komodo import KomodoError, Update, komodo
+from healthbot.notify import notify
+from healthbot.stats import DAY, downtime
+from healthbot.ui import esc, pct, plural
+from utils.db import db
+from utils.env import env
+from utils.format import from_ms, human_duration, now, parse, stamp, zone
+from utils.logging import logger
+
+TICK = 60
+MAX_PAGES = 5
+PAGE_SIZE = 100
+SHOWN = 6
+
+
+async def _services_block() -> list[str]:
+    services = await db.services.all()
+    if not services:
+        return []
+    end = now()
+    start = end - timedelta(seconds=DAY)
+    recent = await db.incidents.since(start.isoformat())
+    window = lost = 0.0
+    rows = []
+    for service in services:
+        mine = [i for i in recent if i.service_id == service.id]
+        began = max(start, parse(service.created_at))
+        span = max(0.0, (end - began).total_seconds())
+        down = downtime(mine, began, end)
+        window += span
+        lost += down
+        if mine:
+            rows.append((down, service, len(mine)))
+    uptime = (window - lost) / window * 100 if window else None
+    falls = sum(row[2] for row in rows)
+    if not falls:
+        return [f"<b>Сервисы</b>: аптайм {pct(uptime)}, падений не было"]
+    word = plural(falls, "падение", "падения", "падений")
+    lines = [
+        f"<b>Сервисы</b>: аптайм {pct(uptime)}, {falls} {word}, "
+        f"даунтайм {human_duration(lost)}"
+    ]
+    for down, service, count in sorted(rows, key=lambda row: -row[0])[:SHOWN]:
+        lines.append(
+            f"{service.mark} {esc(service.name)}: {count} "
+            f"{plural(count, 'падение', 'падения', 'падений')}, {human_duration(down)}"
+        )
+    return lines
+
+
+async def _recent_updates(since_ms: int) -> list[Update]:
+    result: list[Update] = []
+    for page in range(MAX_PAGES):
+        chunk = await komodo.updates({"start_ts": {"$gte": since_ms}}, page)
+        result.extend(chunk)
+        if len(chunk) < PAGE_SIZE:
+            break
+    return result
+
+
+async def _fleet_block() -> list[str]:
+    since = now() - timedelta(seconds=DAY)
+    try:
+        updates, alerts, names = await asyncio.gather(
+            _recent_updates(int(since.timestamp() * 1000)),
+            komodo.open_alerts(),
+            komodo.names(),
+        )
+    except KomodoError as exc:
+        return [f"<b>Флот</b>: {esc(exc)}"]
+    tz = db.settings.get("tz")
+    failed = [u for u in updates if u.done and not u.success]
+    lines = [f"<b>Флот</b>: операций {len(updates)}, неудачных {len(failed)}"]
+    grouped: dict[tuple[str, str], list[Update]] = {}
+    for update in failed:
+        grouped.setdefault((update.operation, update.target_id), []).append(update)
+    for (operation, target_id), group in list(grouped.items())[:SHOWN]:
+        times = f" ×{len(group)}" if len(group) > 1 else ""
+        lines.append(
+            f"🔴 {esc(operation)} {esc(names.get(target_id, group[0].target_type))}"
+            f"{times}, последний раз {stamp(from_ms(group[0].start_ts), tz)}"
+        )
+    if alerts:
+        lines.append(f"Открытых алертов: {len(alerts)}")
+    return lines
+
+
+async def digest_text() -> str:
+    blocks = [await _services_block()]
+    if env.komodo.enabled:
+        blocks.append(await _fleet_block())
+    lines = ["<b>Сводка за сутки</b>"]
+    for block in blocks:
+        if block:
+            lines += ["", *block]
+    return "\n".join(lines)
+
+
+async def run_digest() -> None:
+    while True:
+        try:
+            hour = db.settings.get("digest")
+            local = now().astimezone(zone(db.settings.get("tz")))
+            today = local.strftime("%Y-%m-%d")
+            if (
+                hour is not None
+                and local.hour == hour
+                and db.settings.get("digest_sent") != today
+            ):
+                await db.settings.set("digest_sent", today)
+                await notify(await digest_text(), back_to_menu())
+        except Exception:
+            logger.exception("Digest failed")
+        await asyncio.sleep(TICK)

src/healthbot/handlers/__init__.py

@@ -1,8 +1,33 @@
-from aiogram import Router
+from aiogram import Router, types
 
-from . import add, commands, fleet
+from utils.logging import logger
+
+from . import actions, add, fleet, menu, runs, services, settings
 
 router = Router()
-router.include_routers(commands.router, add.router, fleet.router)
+router.include_routers(
+    menu.router,
+    services.router,
+    add.router,
+    fleet.router,
+    actions.router,
+    settings.router,
+    add.free,
+    actions.commands,
+    runs.router,
+)
+
+
+@router.errors()
+async def on_error(event: types.ErrorEvent) -> bool:
+    logger.error("Handler failed", exc_info=event.exception)
+    callback = event.update.callback_query
+    if callback is not None:
+        try:
+            await callback.answer("Не получилось, подробности в логах", show_alert=True)
+        except Exception:
+            logger.debug("Callback answer failed")
+    return True
+
 
 __all__ = ["router"]

src/healthbot/handlers/actions.py

@@ -0,0 +1,722 @@
+from typing import Any
+
+from aiogram import Bot, F, Router, types
+from aiogram.filters import Command
+from aiogram.fsm.context import FSMContext
+from aiogram.fsm.state import State, StatesGroup
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+from healthbot.actionargs import (
+    ArgError,
+    changed,
+    coerce,
+    command_name,
+    ensure_safe,
+    parse_cli,
+    parse_defaults,
+    render,
+    type_name,
+)
+from healthbot.filters import Admin
+from healthbot.handlers.menu import sync_commands
+from healthbot.keyboards import back_to, back_to_menu
+from healthbot.komodo import Detail, KomodoError, Resource, Update, komodo
+from healthbot.runs import track
+from healthbot.ui import (
+    ANSWER,
+    DANGER,
+    SUCCESS,
+    esc,
+    int_arg,
+    label,
+    pager,
+    pages,
+    show,
+)
+from utils.db import db
+from utils.db.models import Pin, PinMode
+from utils.format import ago, from_ms, until
+from utils.logging import logger
+
+router = Router()
+router.message.filter(Admin())
+router.callback_query.filter(Admin())
+
+PAGE = 20
+WIDE_FORM = 6
+MAX_LABEL = 40
+KIND_TITLE = {"a": "Действия", "p": "Процедуры"}
+KIND_ONE = {"a": "Действие", "p": "Процедура"}
+KIND_BACK = {"a": "К действию", "p": "К процедуре"}
+USAGE_FIELDS = 3
+RUN_STATE = {"Ok": "успешно", "Failed": "с ошибкой", "Running": "идёт сейчас"}
+MODE_TEXT = {PinMode.INSTANT: "запускается сразу", PinMode.FORM: "открывает форму"}
+
+
+class ActionFlow(StatesGroup):
+    waiting_value = State()
+    waiting_label = State()
+
+
+def enabled() -> bool:
+    return bool(komodo.enabled and db.settings.get("actions"))
+
+
+async def _guard(callback: types.CallbackQuery) -> bool:
+    if enabled():
+        return True
+    await callback.answer("Действия выключены в настройках", show_alert=True)
+    return False
+
+
+async def _fail(event: types.Message | types.CallbackQuery, exc: Exception) -> None:
+    if isinstance(event, types.CallbackQuery):
+        await event.answer(str(exc)[:190], show_alert=True)
+    else:
+        await show(event, f"🔴 {esc(exc)}", back_to_menu())
+
+
+async def list_view(kind: str, page: int) -> tuple[str, types.InlineKeyboardMarkup]:
+    items = await komodo.resources(kind)
+    other = "p" if kind == "a" else "a"
+    try:
+        other_count = len(await komodo.resources(other))
+    except KomodoError:
+        other_count = 0
+    total = pages(len(items), PAGE)
+    page = min(page, total - 1)
+    chunk = items[page * PAGE : (page + 1) * PAGE]
+
+    kb = InlineKeyboardBuilder()
+    for item in chunk:
+        kb.button(
+            text=f"{item.mark} {label(item.name, 26)}",
+            callback_data=f"act:open:{kind}:{item.id}",
+        )
+    rows = [2] * (len(chunk) // 2) + [1] * (len(chunk) % 2)
+    nav = pager(kb, f"act:list:{kind}", page, total)
+    kb.button(
+        text=f"{KIND_TITLE[other]}: {other_count}", callback_data=f"act:list:{other}:0"
+    )
+    kb.button(text="Закреплённые", callback_data="set:pins")
+    kb.button(text="Меню", callback_data="menu:main")
+    kb.adjust(*rows, *([nav] if nav else []), 2, 1)
+    text = f"<b>{KIND_TITLE[kind]}</b>: {len(items)}"
+    if not items:
+        text += "\nКлюч бота не видит ни одного. Нужен доступ Execute в Komodo."
+    return text, kb.as_markup()
+
+
+def _values_line(values: dict[str, Any]) -> str:
+    return ", ".join(f"{esc(k)} = {esc(render(v, 60))}" for k, v in values.items())
+
+
+def _args_code(values: dict[str, Any]) -> str:
+    if not values:
+        return ""
+    return (
+        "<code>"
+        + esc(" ".join(f"{k}={render(v)}" for k, v in values.items()))
+        + "</code>"
+    )
+
+
+def _run_line(resource: Resource | None) -> str:
+    if resource is None or not resource.last_run_at:
+        return "Ещё не запускалось"
+    state = RUN_STATE.get(resource.state, "")
+    return f"Последний запуск: {ago(from_ms(resource.last_run_at))}" + (
+        f", {state}" if state else ""
+    )
+
+
+def _schedule_line(detail: Detail, resource: Resource | None) -> str:
+    if not detail.schedule or not detail.schedule_enabled:
+        return ""
+    line = f"Расписание: {esc(detail.schedule)}"
+    if resource and resource.schedule_error:
+        return f"{line}, ошибка: {esc(resource.schedule_error)}"
+    if resource and resource.next_run_at:
+        return f"{line}, следующий запуск {until(from_ms(resource.next_run_at))}"
+    return line
+
+
+async def _load(kind: str, target_id: str) -> tuple[Detail, Resource | None, dict]:
+    detail = await komodo.detail(kind, target_id)
+    resource = next(
+        (r for r in await komodo.resources(kind) if r.id == detail.id), None
+    )
+    defaults = (
+        parse_defaults(detail.args_format, detail.args_text) if kind == "a" else {}
+    )
+    return detail, resource, defaults
+
+
+async def _form(state: FSMContext) -> dict[str, Any]:
+    return (await state.get_data()).get("form") or {}
+
+
+async def card_view(
+    state: FSMContext, kind: str, target_id: str, preset: dict | None = None
+) -> tuple[str, types.InlineKeyboardMarkup]:
+    detail, resource, defaults = await _load(kind, target_id)
+    form: dict[str, Any] = await _form(state)
+    if form.get("id") != detail.id or preset is not None:
+        form = {"kind": kind, "id": detail.id, "values": dict(preset or {})}
+    form["name"] = detail.name
+    form["keys"] = list(defaults)
+    values = {k: v for k, v in form["values"].items() if k in defaults}
+    form["values"] = changed(defaults, values)
+    await state.set_state(None)
+    await state.update_data(form=form)
+
+    lines = [f"<b>{esc(detail.name)}</b>"]
+    if detail.description:
+        lines.append(esc(detail.description))
+    if detail.stages:
+        lines += ["", "Этапы: " + " → ".join(esc(s) for s in detail.stages)]
+    if defaults:
+        lines.append("")
+        for key, default in defaults.items():
+            if key in form["values"]:
+                lines.append(
+                    f"{esc(key)} = <b>{esc(render(form['values'][key], 200))}</b>"
+                    f" (в Komodo {esc(render(default, 60))})"
+                )
+            else:
+                lines.append(f"{esc(key)} = {esc(render(default, 200))}")
+    lines += ["", _run_line(resource)]
+    schedule = _schedule_line(detail, resource)
+    if schedule:
+        lines.append(schedule)
+
+    ref = _ref(form)
+    kb = InlineKeyboardBuilder()
+    for index, key in enumerate(defaults):
+        value = form["values"].get(key, defaults[key])
+        kb.button(
+            text=label(f"{key}: {render(value)}", 40),
+            callback_data=f"act:f:{index}:{ref}",
+        )
+    width = 2 if len(defaults) > WIDE_FORM else 1
+    rows = [width] * (len(defaults) // width) + [1] * (len(defaults) % width)
+    kb.button(text="Запустить", callback_data=f"act:run:{ref}", style=SUCCESS)
+    kb.button(text="Закрепить", callback_data=f"act:pin:{ref}")
+    tail = 2
+    if form["values"]:
+        kb.button(text="Сбросить", callback_data=f"act:reset:{ref}")
+        tail = 3
+    kb.button(text="К списку", callback_data=f"act:list:{kind}:0")
+    kb.adjust(*rows, tail, 1)
+    return "\n".join(lines), kb.as_markup()
+
+
+async def _show_card(  # noqa: PLR0913
+    event: types.Message | types.CallbackQuery,
+    state: FSMContext,
+    kind: str,
+    target_id: str,
+    *,
+    preset: dict | None = None,
+    new: bool = False,
+) -> None:
+    try:
+        text, markup = await card_view(state, kind, target_id, preset)
+    except KomodoError as exc:
+        await _fail(event, exc)
+        return
+    await show(event, text, markup, new=new)
+    if isinstance(event, types.CallbackQuery):
+        await event.answer()
+
+
+@router.message(Command("actions"))
+async def cmd_actions(message: types.Message, state: FSMContext) -> None:
+    await state.clear()
+    if not enabled():
+        await show(
+            message, "Действия выключены или Komodo не настроен.", back_to_menu()
+        )
+        return
+    try:
+        text, markup = await list_view("a", 0)
+    except KomodoError as exc:
+        await _fail(message, exc)
+        return
+    await show(message, text, markup)
+
+
+@router.callback_query(F.data.startswith("act:list:"))
+async def act_list(callback: types.CallbackQuery, state: FSMContext) -> None:
+    if not await _guard(callback):
+        return
+    await state.set_state(None)
+    kind = (callback.data or "").split(":")[2]
+    try:
+        text, markup = await list_view(kind, int_arg(callback.data))
+    except KomodoError as exc:
+        await _fail(callback, exc)
+        return
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("act:open:"))
+async def act_open(callback: types.CallbackQuery, state: FSMContext) -> None:
+    if not await _guard(callback):
+        return
+    _, _, kind, target_id = (callback.data or "").split(":", 3)
+    await _show_card(callback, state, kind, target_id)
+
+
+@router.callback_query(F.data.startswith("go:act:"))
+async def go_action(callback: types.CallbackQuery, state: FSMContext) -> None:
+    if not await _guard(callback):
+        return
+    _, _, kind, target_id = (callback.data or "").split(":", 3)
+    await _show_card(callback, state, kind, target_id, new=True)
+
+
+def _ref(form: dict[str, Any]) -> str:
+    return f"{form['kind']}:{form['id']}"
+
+
+async def _current(callback: types.CallbackQuery, state: FSMContext) -> dict | None:
+    kind, target_id = (callback.data or "").rsplit(":", 2)[-2:]
+    form = await _form(state)
+    if form.get("id") == target_id:
+        return form
+    try:
+        text, markup = await card_view(state, kind, target_id, {})
+    except KomodoError as exc:
+        await _fail(callback, exc)
+        return None
+    await show(callback, text, markup)
+    await callback.answer("Форма устарела, открыл её заново", show_alert=True)
+    return None
+
+
+@router.callback_query(F.data.startswith("act:back:"))
+async def act_back(callback: types.CallbackQuery, state: FSMContext) -> None:
+    form = await _current(callback, state)
+    if form is None:
+        return
+    await _show_card(callback, state, form["kind"], form["id"])
+
+
+@router.callback_query(F.data.startswith("act:reset:"))
+async def act_reset(callback: types.CallbackQuery, state: FSMContext) -> None:
+    form = await _current(callback, state)
+    if form is None:
+        return
+    await _show_card(callback, state, form["kind"], form["id"], preset={})
+
+
+@router.callback_query(F.data.startswith("act:f:"))
+async def act_field(callback: types.CallbackQuery, state: FSMContext) -> None:
+    form = await _current(callback, state)
+    if form is None:
+        return
+    index = int((callback.data or "").split(":")[2])
+    if index >= len(form["keys"]):
+        await callback.answer()
+        return
+    key = form["keys"][index]
+    try:
+        _, _, defaults = await _load(form["kind"], form["id"])
+    except KomodoError as exc:
+        await _fail(callback, exc)
+        return
+    default = defaults.get(key)
+    value = form["values"].get(key, default)
+    if isinstance(default, bool):
+        form["values"] = {**form["values"], key: not value}
+        await state.update_data(form=form)
+        await _show_card(callback, state, form["kind"], form["id"])
+        return
+
+    form["field"] = key
+    await state.update_data(form=form)
+    await state.set_state(ActionFlow.waiting_value)
+    kb = InlineKeyboardBuilder()
+    if key in form["values"]:
+        kb.button(
+            text=label(f"Вернуть {render(default)}", 40),
+            callback_data=f"act:fd:{_ref(form)}",
+        )
+    kb.button(text="Отмена", callback_data=f"act:back:{_ref(form)}")
+    kb.adjust(1)
+    await show(
+        callback,
+        f"<b>{esc(key)}</b>: пришли новое значение ({type_name(default)}).\n"
+        f"Сейчас: <code>{esc(render(value, 300))}</code>",
+        kb.as_markup(),
+    )
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("act:fd:"))
+async def act_field_default(callback: types.CallbackQuery, state: FSMContext) -> None:
+    form = await _current(callback, state)
+    if form is None:
+        return
+    form["values"] = {k: v for k, v in form["values"].items() if k != form.get("field")}
+    await state.update_data(form=form)
+    await _show_card(callback, state, form["kind"], form["id"])
+
+
+@router.message(ActionFlow.waiting_value, ANSWER)
+async def on_value(message: types.Message, state: FSMContext) -> None:
+    form = await _form(state)
+    key = form.get("field")
+    if not form or key is None:
+        await state.set_state(None)
+        return
+    try:
+        _, _, defaults = await _load(form["kind"], form["id"])
+        values = {**form["values"], key: coerce(defaults.get(key), message.text or "")}
+        ensure_safe(values)
+    except ArgError as exc:
+        await show(
+            message,
+            f"<b>{esc(key)}</b>: {esc(exc)}. Пришли ещё раз.",
+            back_to("Отмена", f"act:back:{_ref(form)}"),
+        )
+        return
+    except KomodoError as exc:
+        await _fail(message, exc)
+        return
+    form["values"] = values
+    await state.update_data(form=form)
+    await _show_card(message, state, form["kind"], form["id"])
+
+
+async def _execute(  # noqa: PLR0913
+    origin: types.Message | types.CallbackQuery,
+    kind: str,
+    target_id: str,
+    name: str,
+    args: dict,
+    *,
+    title: str | None = None,
+    new: bool = False,
+) -> None:
+    async def starter() -> Update:
+        try:
+            ensure_safe(args)
+        except ArgError as exc:
+            raise KomodoError(str(exc)) from exc
+        logger.info(f"Run{'Action' if kind == 'a' else 'Procedure'} {name}")
+        return await komodo.run(kind, target_id, args)
+
+    await track(
+        origin,
+        title or name,
+        starter,
+        back=[(KIND_BACK[kind], f"act:open:{kind}:{target_id}"), ("Меню", "menu:main")],
+        verbose=kind == "a",
+        extra=_args_code(args),
+        new=new,
+    )
+
+
+@router.callback_query(F.data.startswith("act:run:"))
+async def act_run(callback: types.CallbackQuery, state: FSMContext) -> None:
+    form = await _current(callback, state)
+    if form is None or not await _guard(callback):
+        return
+    if not db.settings.get("confirm"):
+        await callback.answer()
+        await _execute(callback, form["kind"], form["id"], form["name"], form["values"])
+        return
+    kb = InlineKeyboardBuilder()
+    kb.button(text="Запустить", callback_data=f"act:go:{_ref(form)}", style=SUCCESS)
+    kb.button(text="Отмена", callback_data=f"act:back:{_ref(form)}")
+    kb.adjust(2)
+    changes = f"\nИзменено: {_values_line(form['values'])}" if form["values"] else ""
+    await show(
+        callback, f"Запустить <b>{esc(form['name'])}</b>?{changes}", kb.as_markup()
+    )
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("act:go:"))
+async def act_go(callback: types.CallbackQuery, state: FSMContext) -> None:
+    form = await _current(callback, state)
+    if form is None or not await _guard(callback):
+        return
+    await callback.answer()
+    await _execute(callback, form["kind"], form["id"], form["name"], form["values"])
+
+
+@router.callback_query(F.data.startswith("act:pin:"))
+async def act_pin(callback: types.CallbackQuery, state: FSMContext) -> None:
+    form = await _current(callback, state)
+    if form is None:
+        return
+    remembered = (
+        f"Кнопка запомнит: {_values_line(form['values'])}."
+        if form["values"]
+        else "Значения кнопка возьмёт из Komodo в момент запуска."
+    )
+    kb = InlineKeyboardBuilder()
+    ref = _ref(form)
+    kb.button(text="Запускать сразу", callback_data=f"act:pinmode:instant:{ref}")
+    kb.button(text="Открывать форму", callback_data=f"act:pinmode:form:{ref}")
+    kb.button(text="Отмена", callback_data=f"act:back:{ref}")
+    kb.adjust(2, 1)
+    await show(
+        callback,
+        f"Закрепить <b>{esc(form['name'])}</b> на главной.\n{remembered}",
+        kb.as_markup(),
+    )
+    await callback.answer()
+
+
+def pin_view(pin: Pin) -> tuple[str, types.InlineKeyboardMarkup]:
+    values = (
+        f"Значения: {_values_line(pin.args)}"
+        if pin.args
+        else "Значения: по умолчанию из Komodo"
+    )
+    lines = [
+        f"Кнопка <b>{esc(pin.label)}</b>",
+        f"{KIND_ONE[pin.kind]}: {esc(pin.name)}",
+        f"Режим: {MODE_TEXT[pin.mode]}",
+        values,
+    ]
+    kb = InlineKeyboardBuilder()
+    kb.button(text="Переименовать", callback_data=f"pin:ren:{pin.id}")
+    other = PinMode.FORM if pin.mode is PinMode.INSTANT else PinMode.INSTANT
+    kb.button(
+        text="Открывать форму" if other is PinMode.FORM else "Запускать сразу",
+        callback_data=f"pin:mode:{pin.id}",
+    )
+    kb.button(text="Выше", callback_data=f"pin:up:{pin.id}")
+    kb.button(text="Ниже", callback_data=f"pin:down:{pin.id}")
+    kb.button(text="Открепить", callback_data=f"pin:del:{pin.id}", style=DANGER)
+    kb.button(text="Закреплённые", callback_data="set:pins")
+    kb.button(text="Меню", callback_data="menu:main")
+    kb.adjust(2, 2, 1, 2)
+    return "\n".join(lines), kb.as_markup()
+
+
+@router.callback_query(F.data.startswith("act:pinmode:"))
+async def act_pin_mode(
+    callback: types.CallbackQuery, state: FSMContext, bot: Bot
+) -> None:
+    form = await _current(callback, state)
+    if form is None:
+        return
+    mode = PinMode((callback.data or "").split(":")[2])
+    pin = await db.pins.add(
+        kind=form["kind"],
+        target_id=form["id"],
+        name=form["name"],
+        label=form["name"][:MAX_LABEL],
+        mode=mode,
+        args=form["values"],
+    )
+    await sync_commands(bot)
+    text, markup = pin_view(pin)
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+async def pins_view() -> tuple[str, types.InlineKeyboardMarkup]:
+    pins = await db.pins.all()
+    kb = InlineKeyboardBuilder()
+    for pin in pins:
+        kb.button(text=label(pin.label, 40), callback_data=f"pin:open:{pin.id}")
+    if enabled():
+        kb.button(text="Действия", callback_data="act:list:a:0")
+    kb.button(text="Меню", callback_data="menu:main")
+    kb.adjust(*[1] * len(pins), 2)
+    text = "<b>Закреплённые</b>"
+    if not pins:
+        text += "\nПока пусто. Открой действие и нажми «Закрепить»."
+    return text, kb.as_markup()
+
+
+@router.callback_query(F.data == "set:pins")
+async def pins_list(callback: types.CallbackQuery, state: FSMContext) -> None:
+    await state.set_state(None)
+    text, markup = await pins_view()
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+async def _pin(callback: types.CallbackQuery) -> Pin | None:
+    pin = await db.pins.get(int_arg(callback.data))
+    if pin is None:
+        await callback.answer("Эта кнопка уже откреплена", show_alert=True)
+    return pin
+
+
+@router.callback_query(F.data.startswith("pin:open:"))
+async def pin_open(callback: types.CallbackQuery, state: FSMContext) -> None:
+    pin = await _pin(callback)
+    if pin is None:
+        return
+    await state.set_state(None)
+    text, markup = pin_view(pin)
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("pin:run:"))
+async def pin_run(callback: types.CallbackQuery, state: FSMContext) -> None:
+    pin = await _pin(callback)
+    if pin is None or not await _guard(callback):
+        return
+    if pin.mode is PinMode.FORM:
+        await _show_card(callback, state, pin.kind, pin.target_id, preset=pin.args)
+        return
+    await callback.answer()
+    await _execute(
+        callback, pin.kind, pin.target_id, pin.name, pin.args, title=pin.label, new=True
+    )
+
+
+@router.callback_query(F.data.startswith("pin:mode:"))
+async def pin_mode(callback: types.CallbackQuery) -> None:
+    pin = await _pin(callback)
+    if pin is None:
+        return
+    other = PinMode.FORM if pin.mode is PinMode.INSTANT else PinMode.INSTANT
+    await db.pins.set_mode(pin.id, other)
+    pin.mode = other
+    text, markup = pin_view(pin)
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+async def _move(callback: types.CallbackQuery, step: int) -> None:
+    pin = await _pin(callback)
+    if pin is None:
+        return
+    await db.pins.move(pin.id, step)
+    text, markup = await pins_view()
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("pin:up:"))
+async def pin_up(callback: types.CallbackQuery) -> None:
+    await _move(callback, -1)
+
+
+@router.callback_query(F.data.startswith("pin:down:"))
+async def pin_down(callback: types.CallbackQuery) -> None:
+    await _move(callback, 1)
+
+
+@router.callback_query(F.data.startswith("pin:del:"))
+async def pin_delete(callback: types.CallbackQuery, bot: Bot) -> None:
+    await db.pins.delete(int_arg(callback.data))
+    await sync_commands(bot)
+    text, markup = await pins_view()
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("pin:ren:"))
+async def pin_rename(callback: types.CallbackQuery, state: FSMContext) -> None:
+    pin = await _pin(callback)
+    if pin is None:
+        return
+    await state.set_state(ActionFlow.waiting_label)
+    await state.update_data(pin_id=pin.id)
+    await show(
+        callback,
+        f"Пришли новое название для кнопки <b>{esc(pin.label)}</b>.",
+        back_to("Отмена", f"pin:open:{pin.id}"),
+    )
+    await callback.answer()
+
+
+@router.message(ActionFlow.waiting_label, ANSWER)
+async def on_label(message: types.Message, state: FSMContext, bot: Bot) -> None:
+    text = " ".join((message.text or "").split())[:MAX_LABEL]
+    data = await state.get_data()
+    await state.set_state(None)
+    pin = await db.pins.get(data.get("pin_id", 0))
+    if pin is None or not text:
+        await show(message, "Эта кнопка уже откреплена.", back_to_menu())
+        return
+    await db.pins.rename(pin.id, text)
+    await sync_commands(bot)
+    pin.label = text
+    view, markup = pin_view(pin)
+    await show(message, view, markup)
+
+
+def _usage(detail: Detail, defaults: dict) -> str:
+    parts = [f"/{command_name(detail.name)}"]
+    for key, value in list(defaults.items())[:USAGE_FIELDS]:
+        parts += [key, '"текст"' if isinstance(value, str) else render(value)]
+    return f"Пример: <code>{esc(' '.join(parts))}</code>"
+
+
+async def _find(name: str) -> Resource | None:
+    wanted = command_name(name)
+    for kind in ("a", "p"):
+        try:
+            items = await komodo.resources(kind)
+        except KomodoError:
+            continue
+        for item in items:
+            if command_name(item.name) == wanted:
+                return item
+    return None
+
+
+commands = Router()
+commands.message.filter(Admin())
+
+
+@commands.message(F.text.startswith("/"))
+async def on_command(message: types.Message, state: FSMContext) -> None:
+    head, *tail = (message.text or "").split(maxsplit=1)
+    rest = tail[0] if tail else ""
+    name = head[1:].split("@")[0]
+    resource = await _find(name) if enabled() else None
+    if resource is not None and not db.settings.get("commands"):
+        await show(
+            message,
+            "Запуск действий командами выключен.",
+            back_to("Настройки", "set:main"),
+        )
+        return
+    if resource is None and not (enabled() and db.settings.get("commands")):
+        await show(message, "Не знаю такую команду.", back_to_menu())
+        return
+    if resource is None:
+        await show(
+            message, f"Действия <b>{esc(name)}</b> в Komodo нет.", back_to_menu()
+        )
+        return
+    await state.set_state(None)
+    try:
+        detail, _, defaults = await _load(resource.kind, resource.id)
+        args = changed(defaults, parse_cli(rest, defaults))
+        ensure_safe(args)
+    except ArgError as exc:
+        detail = await komodo.detail(resource.kind, resource.id)
+        defaults = parse_defaults(detail.args_format, detail.args_text)
+        await show(
+            message,
+            f"<b>{esc(resource.name)}</b>: {esc(exc)}\n{_usage(detail, defaults)}",
+        )
+        return
+    except KomodoError as exc:
+        await _fail(message, exc)
+        return
+    await _execute(message, resource.kind, resource.id, detail.name, args)
+
+
+@commands.message(F.text)
+async def on_unknown(message: types.Message) -> None:
+    await show(
+        message, "Не понял. Пришли ссылку на сервис или открой меню.", back_to_menu()
+    )

src/healthbot/handlers/add.py

@@ -1,6 +1,6 @@
-"""Добавление сервиса: прислал ссылку → выбрал в меню за чем следить."""
+import asyncio
-
+import json
-import html
+import re
 from urllib.parse import urlparse
 
 import aiohttp
@@ -9,9 +9,12 @@ from aiogram.fsm.context import FSMContext
 from aiogram.fsm.state import State, StatesGroup
 from aiogram.utils.keyboard import InlineKeyboardBuilder
 
-from healthbot.checker import _scalar_str, probe
+from healthbot.checker import Probe, probe, scalar_str
 from healthbot.filters import Admin
-from healthbot.keyboards import back_to_menu
+from healthbot.handlers.menu import dashboard
+from healthbot.handlers.services import show_card
+from healthbot.keyboards import back_to, back_to_menu
+from healthbot.ui import ANSWER, SUCCESS, esc, int_arg, label, show
 from utils.db import db
 from utils.db.models import MatchType
 from utils.env import env
@@ -20,14 +23,51 @@ router = Router()
 router.message.filter(Admin())
 router.callback_query.filter(Admin())
 
-URL_RE = r"^https?://\S+$"
+FULL_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE)
-MAX_CHILDREN = 40
+BARE_URL_RE = re.compile(
+    r"(localhost|(\d{1,3}\.){3}\d{1,3}|[\w-]+(\.[\w-]+)*\.[a-z]{2,})(:\d+)?(/\S*)?",
+    re.IGNORECASE | re.ASCII,
+)
+MAX_CHILDREN = 30
+MAX_BULK = 20
 PREVIEW_LEN = 30
+BODY_PREVIEW = 350
+HEALTH_KEYS = (
+    "status",
+    "ok",
+    "healthy",
+    "health",
+    "state",
+    "alive",
+    "up",
+    "ready",
+    "success",
+)
 
 
 class AddFlow(StatesGroup):
     waiting_url = State()
     choosing = State()
+    waiting_needle = State()
+    waiting_new_url = State()
+
+
+def normalize(token: str) -> str | None:
+    if FULL_URL_RE.fullmatch(token):
+        return token
+    if BARE_URL_RE.fullmatch(token):
+        return f"https://{token}"
+    return None
+
+
+def extract_urls(text: str | None) -> list[str] | None:
+    tokens = (text or "").split()
+    if not tokens or len(tokens) > MAX_BULK:
+        return None
+    urls = [normalize(token) for token in tokens]
+    if any(url is None for url in urls):
+        return None
+    return list(dict.fromkeys(url for url in urls if url))
 
 
 def _name_from_url(url: str) -> str:
@@ -45,7 +85,8 @@ def _node_at(root: object, path: list[str]) -> object:
 
 def _children(node: object) -> list[tuple[str, object]]:
     if isinstance(node, dict):
-        return list(node.items())
+        kids: list[tuple[str, object]] = [(str(k), v) for k, v in node.items()]
+        return sorted(kids, key=lambda kid: kid[0].lower() not in HEALTH_KEYS)
     if isinstance(node, list):
         return [(str(i), v) for i, v in enumerate(node)]
     return []
@@ -56,140 +97,298 @@ def _preview(value: object) -> str:
         return "{…}"
     if isinstance(value, list):
         return "[…]"
-    text = _scalar_str(value)
+    text = scalar_str(value)
     return text if len(text) <= PREVIEW_LEN else text[: PREVIEW_LEN - 1] + "…"
 
 
 def _choose_view(data: dict) -> tuple[str, types.InlineKeyboardMarkup]:
     parsed = data["parsed"]
     path: list[str] = data["path"]
-    body_preview = html.escape(data["body"][:350]) or "(пусто)"
 
     lines = [
-        f"🔗 <b>{html.escape(data['name'])}</b>",
+        f"<b>{esc(data['name'])}</b>",
-        f"<code>{html.escape(data['url'])}</code>",
+        f"<code>{esc(data['url'])}</code>",
-        f"Ответ: HTTP {data['status']}",
+        f"HTTP {data['status']}, отклик {data['latency']} мс",
-        f"<pre>{body_preview}</pre>",
+        f"<pre>{esc(data['body'][:BODY_PREVIEW]) or 'пустой ответ'}</pre>",
     ]
 
     kb = InlineKeyboardBuilder()
+    rows: list[int] = []
     if parsed is not None:
-        crumb = " › ".join(["корень", *path])
+        crumb = " › ".join(["ответ", *path])
-        lines.append(f"За каким полем следить? <i>{html.escape(crumb)}</i>")
+        lines.append(f"За каким полем следить? <i>{esc(crumb)}</i>")
-        node = _node_at(parsed, path)
+        kids = _children(_node_at(parsed, path))
-        kids = _children(node)
         for i, (key, value) in enumerate(kids[:MAX_CHILDREN]):
-            kb.button(text=f"{key}: {_preview(value)}", callback_data=f"add:nav:{i}")
+            healthy = not path and key.lower() in HEALTH_KEYS
+            kb.button(
+                text=label(f"{key}: {_preview(value)}"),
+                callback_data=f"add:nav:{i}",
+                style=SUCCESS
+                if healthy and not isinstance(value, (dict, list))
+                else None,
+            )
+        rows += [1] * min(len(kids), MAX_CHILDREN)
         if len(kids) > MAX_CHILDREN:
-            lines.append(f"…показаны первые {MAX_CHILDREN} полей")
+            lines.append(f"Показаны первые {MAX_CHILDREN} полей.")
     else:
-        lines.append("Ответ не JSON - можно следить за статусом или телом целиком.")
+        lines.append("За чем следить?")
 
     if path:
-        kb.button(text="⬆️ Назад", callback_data="add:up")
+        kb.button(text="Назад", callback_data="add:up")
-    kb.button(text="🔢 Только HTTP-статус", callback_data="add:status")
+        rows.append(1)
-    kb.button(text="📄 Всё тело целиком", callback_data="add:body")
+    kb.button(text=f"HTTP-статус {data['status']}", callback_data="add:status")
-    kb.button(text="❌ Отмена", callback_data="add:cancel")
+    kb.button(text="Текст в ответе", callback_data="add:text")
-    kb.adjust(1)
+    kb.button(text="Тело целиком", callback_data="add:body")
+    kb.button(text="Отмена", callback_data="add:cancel")
+    kb.adjust(*rows, 2, 2)
     return "\n".join(lines), kb.as_markup()
 
 
+async def _probe(url: str) -> Probe:
+    async with aiohttp.ClientSession() as session:
+        return await probe(session, url, env.monitor.request_timeout)
+
+
+def _unreachable_kb(*, editing: bool) -> types.InlineKeyboardMarkup:
+    kb = InlineKeyboardBuilder()
+    kb.button(text="Повторить", callback_data="add:retry")
+    if not editing:
+        kb.button(text="Добавить всё равно", callback_data="add:force")
+    kb.button(text="Отмена", callback_data="add:cancel")
+    kb.adjust(2, 1)
+    return kb.as_markup()
+
+
+async def _begin(
+    event: types.Message | types.CallbackQuery,
+    state: FSMContext,
+    url: str,
+    *,
+    edit_id: int | None = None,
+    name: str | None = None,
+) -> None:
+    result = await _probe(url)
+    name = name or _name_from_url(url)
+    await state.set_state(AddFlow.choosing)
+    await state.set_data({"url": url, "name": name, "edit_id": edit_id})
+
+    if not result.reachable:
+        hint = "" if edit_id else "\nМожно добавить и так: буду ждать HTTP 200."
+        await show(
+            event,
+            f"<b>{esc(name)}</b> не отвечает: {esc(result.error or '')}.{hint}",
+            _unreachable_kb(editing=edit_id is not None),
+        )
+        return
+
+    try:
+        parsed = json.loads(result.body)
+    except ValueError:
+        parsed = None
+
+    await state.update_data(
+        status=result.status,
+        latency=result.latency_ms,
+        body=result.body,
+        parsed=parsed if isinstance(parsed, (dict, list)) else None,
+        path=[],
+    )
+    text, markup = _choose_view(await state.get_data())
+    await show(event, text, markup)
+
+
 @router.callback_query(F.data == "menu:add")
 async def on_add_menu(callback: types.CallbackQuery, state: FSMContext) -> None:
     await state.set_state(AddFlow.waiting_url)
-    if isinstance(callback.message, types.Message):
+    await show(
-        await callback.message.edit_text(
+        callback,
-            "Пришли ссылку на сервис "
+        "Пришли ссылку на сервис, например <code>example.com/health</code>.\n"
-            "(например <code>https://example.com/health</code>) - "
+        "Несколько ссылок через пробел или с новой строки добавлю разом.",
-            "я запрошу её и предложу выбрать, за чем следить.",
+        back_to_menu(),
-            reply_markup=back_to_menu(),
     )
     await callback.answer()
 
 
-@router.message(F.text.regexp(URL_RE))
+async def _add_many(message: types.Message, urls: list[str]) -> None:
-async def on_url(message: types.Message, state: FSMContext) -> None:
+    fresh = [url for url in urls if await db.services.by_url(url) is None]
-    url = (message.text or "").strip()
+    results = await asyncio.gather(*(_probe(url) for url in fresh))
+    added, skipped = [], [f"{esc(u)}: уже есть" for u in urls if u not in fresh]
+    for url, result in zip(fresh, results, strict=True):
+        if not result.reachable or result.status is None:
+            skipped.append(f"{esc(url)}: {esc(result.error or 'нет ответа')}")
+            continue
+        service = await db.services.add(
+            name=_name_from_url(url),
+            url=url,
+            match_type=MatchType.STATUS,
+            ok_status=result.status,
+        )
+        added.append(f"🟢 {esc(service.name)} · HTTP {result.status}")
+    lines = [f"Добавил {len(added)} из {len(urls)}, слежу за HTTP-статусом.", *added]
+    if skipped:
+        lines += ["", "Пропустил:", *skipped]
+    await show(message, "\n".join(lines), back_to("Сервисы", "menu:list:0"))
 
-    if await db.services.by_url(url) is not None:
-        await message.answer("Этот сервис уже под наблюдением. /list - список.")
-        return
 
-    note = await message.answer("Запрашиваю…")
+async def _on_urls(message: types.Message, state: FSMContext, urls: list[str]) -> None:
-    async with aiohttp.ClientSession() as session:
+    if len(urls) > 1:
-        result = await probe(session, url, env.monitor.request_timeout)
-
-    if not result.reachable:
         await state.clear()
-        await note.edit_text(
+        await _add_many(message, urls)
-            f"Не смог достучаться: <b>{html.escape(result.error or '')}</b>.\n"
+        return
-            "Эталон снимаю только с живого сервиса - попробуй, когда он поднимется.",
+    existing = await db.services.by_url(urls[0])
-            reply_markup=back_to_menu(),
+    if existing is not None:
+        await state.clear()
+        await show_card(message, existing)
+        return
+    await _begin(message, state, urls[0])
+
+
+@router.message(AddFlow.waiting_url, ANSWER)
+async def on_url_expected(message: types.Message, state: FSMContext) -> None:
+    urls = extract_urls(message.text)
+    if urls is None:
+        await show(
+            message,
+            "Это не похоже на ссылку. Пример: <code>example.com/health</code>",
+            back_to_menu(),
         )
         return
+    await _on_urls(message, state, urls)
 
-    try:
-        import json  # noqa: PLC0415
 
-        parsed = json.loads(result.body)
+@router.message(AddFlow.waiting_new_url, ANSWER)
-    except ValueError:
+async def on_new_url(message: types.Message, state: FSMContext) -> None:
-        parsed = None
+    data = await state.get_data()
+    service = await db.services.get(data["edit_id"])
+    if service is None:
+        await state.clear()
+        await show(message, "Сервис уже удалён.", back_to_menu())
+        return
+    urls = extract_urls(message.text)
+    back = back_to("Отмена", f"svc:open:{service.id}")
+    if urls is None or len(urls) != 1:
+        await show(message, "Нужна одна ссылка.", back)
+        return
+    other = await db.services.by_url(urls[0])
+    if other is not None and other.id != service.id:
+        await show(message, f"Этот адрес уже у <b>{esc(other.name)}</b>.", back)
+        return
+    await state.clear()
+    await db.services.set_url(service.id, urls[0])
+    updated = await db.services.get(service.id)
+    if updated is not None:
+        await show_card(message, updated)
 
-    await state.set_state(AddFlow.choosing)
+
-    await state.set_data(
+@router.callback_query(F.data.startswith("svc:url:"))
-        {
+async def svc_url(callback: types.CallbackQuery, state: FSMContext) -> None:
-            "url": url,
+    service = await db.services.get(int_arg(callback.data))
-            "name": _name_from_url(url),
+    if service is None:
-            "status": result.status,
+        await callback.answer("Сервис не найден", show_alert=True)
-            "body": result.body,
+        return
-            "parsed": parsed,
+    await state.set_state(AddFlow.waiting_new_url)
-            "path": [],
+    await state.set_data({"edit_id": service.id})
-        }
+    await show(
+        callback,
+        f"Пришли новый адрес для <b>{esc(service.name)}</b>.\n"
+        f"Сейчас: <code>{esc(service.url)}</code>",
+        back_to("Отмена", f"svc:open:{service.id}"),
     )
-    text, markup = _choose_view(await state.get_data())
+    await callback.answer()
-    await note.edit_text(text, reply_markup=markup)
+
+
+@router.callback_query(F.data.startswith("svc:rule:"))
+async def svc_rule(callback: types.CallbackQuery, state: FSMContext) -> None:
+    service = await db.services.get(int_arg(callback.data))
+    if service is None:
+        await callback.answer("Сервис не найден", show_alert=True)
+        return
+    await callback.answer()
+    await _begin(callback, state, service.url, edit_id=service.id, name=service.name)
 
 
 async def _save(  # noqa: PLR0913
-    callback: types.CallbackQuery,
+    event: types.Message | types.CallbackQuery,
     state: FSMContext,
     *,
     match_type: MatchType,
     ok_body: str = "",
     json_path: str | None = None,
     json_value: str | None = None,
+    needle: str | None = None,
 ) -> None:
     data = await state.get_data()
-    service = await db.services.add(
-        name=data["name"],
-        url=data["url"],
-        match_type=match_type,
-        ok_status=data["status"],
-        ok_body=ok_body,
-        json_path=json_path,
-        json_value=json_value,
-    )
     await state.clear()
-    if isinstance(callback.message, types.Message):
+    rule = {
-        await callback.message.edit_text(
+        "match_type": match_type,
-            f"✅ Слежу за <b>{html.escape(service.name)}</b>\n"
+        "ok_status": data.get("status") or 200,
-            f"<code>{html.escape(service.url)}</code>\n"
+        "ok_body": ok_body,
-            f"Правило: {service.rule}\n"
+        "json_path": json_path,
-            f"Проверяю раз в {env.monitor.check_interval}с.",
+        "json_value": json_value,
-            reply_markup=back_to_menu(),
+        "needle": needle,
-        )
+    }
-    await callback.answer("Готово")
+    if data.get("edit_id"):
+        await db.services.set_rule(data["edit_id"], **rule)
+        service = await db.services.get(data["edit_id"])
+    else:
+        service = await db.services.add(name=data["name"], url=data["url"], **rule)
+    if service is None:
+        await show(event, "Сервис уже удалён.", back_to_menu())
+        return
+    await show_card(event, service)
 
 
 @router.callback_query(AddFlow.choosing, F.data == "add:status")
 async def on_pick_status(callback: types.CallbackQuery, state: FSMContext) -> None:
     await _save(callback, state, match_type=MatchType.STATUS)
+    await callback.answer()
+
+
+@router.callback_query(AddFlow.choosing, F.data == "add:force")
+async def on_force(callback: types.CallbackQuery, state: FSMContext) -> None:
+    await _save(callback, state, match_type=MatchType.STATUS)
+    await callback.answer()
+
+
+@router.callback_query(AddFlow.choosing, F.data == "add:retry")
+async def on_retry(callback: types.CallbackQuery, state: FSMContext) -> None:
+    data = await state.get_data()
+    await callback.answer()
+    await _begin(
+        callback, state, data["url"], edit_id=data.get("edit_id"), name=data["name"]
+    )
 
 
 @router.callback_query(AddFlow.choosing, F.data == "add:body")
 async def on_pick_body(callback: types.CallbackQuery, state: FSMContext) -> None:
     data = await state.get_data()
     await _save(callback, state, match_type=MatchType.BODY, ok_body=data["body"])
+    await callback.answer()
+
+
+@router.callback_query(AddFlow.choosing, F.data == "add:text")
+async def on_pick_text(callback: types.CallbackQuery, state: FSMContext) -> None:
+    await state.set_state(AddFlow.waiting_needle)
+    await show(
+        callback,
+        "Пришли текст, который должен быть в ответе.",
+        back_to("Отмена", "add:cancel"),
+    )
+    await callback.answer()
+
+
+@router.message(AddFlow.waiting_needle, ANSWER)
+async def on_needle(message: types.Message, state: FSMContext) -> None:
+    needle = message.text or ""
+    data = await state.get_data()
+    if needle not in data["body"]:
+        await show(
+            message,
+            "Сейчас такого текста в ответе нет. Пришли другой.",
+            back_to("Отмена", "add:cancel"),
+        )
+        return
+    await _save(message, state, match_type=MatchType.CONTAINS, needle=needle)
 
 
 @router.callback_query(AddFlow.choosing, F.data == "add:up")
@@ -204,9 +403,8 @@ async def on_up(callback: types.CallbackQuery, state: FSMContext) -> None:
 @router.callback_query(AddFlow.choosing, F.data.startswith("add:nav:"))
 async def on_nav(callback: types.CallbackQuery, state: FSMContext) -> None:
     data = await state.get_data()
-    index = int((callback.data or "").rsplit(":", 1)[-1])
+    kids = _children(_node_at(data["parsed"], data["path"]))
-    node = _node_at(data["parsed"], data["path"])
+    index = int_arg(callback.data)
-    kids = _children(node)
     if index >= len(kids):
         await callback.answer()
         return
@@ -221,20 +419,39 @@ async def on_nav(callback: types.CallbackQuery, state: FSMContext) -> None:
         state,
         match_type=MatchType.JSON,
         json_path=".".join([*data["path"], key]),
-        json_value=_scalar_str(value),
+        json_value=scalar_str(value),
     )
-
-
-@router.callback_query(AddFlow.choosing, F.data == "add:cancel")
-async def on_cancel(callback: types.CallbackQuery, state: FSMContext) -> None:
-    await state.clear()
-    if isinstance(callback.message, types.Message):
-        await callback.message.edit_text("Отменено.", reply_markup=back_to_menu())
     await callback.answer()
 
 
+@router.callback_query(F.data == "add:cancel")
+async def on_cancel(callback: types.CallbackQuery, state: FSMContext) -> None:
+    data = await state.get_data()
+    await state.clear()
+    service = await db.services.get(data["edit_id"]) if data.get("edit_id") else None
+    if service is not None:
+        await show_card(callback, service)
+    else:
+        text, markup = await dashboard()
+        await show(callback, text, markup)
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("add:"))
+async def on_stale(callback: types.CallbackQuery) -> None:
+    await callback.answer("Эта форма устарела, пришли ссылку заново", show_alert=True)
+
+
 async def _rerender(callback: types.CallbackQuery, state: FSMContext) -> None:
     text, markup = _choose_view(await state.get_data())
-    if isinstance(callback.message, types.Message):
+    await show(callback, text, markup)
-        await callback.message.edit_text(text, reply_markup=markup)
     await callback.answer()
+
+
+free = Router()
+free.message.filter(Admin())
+
+
+@free.message(F.text.func(lambda text: extract_urls(text) is not None))
+async def on_url(message: types.Message, state: FSMContext) -> None:
+    await _on_urls(message, state, extract_urls(message.text) or [])

src/healthbot/handlers/commands.py

@@ -1,358 +0,0 @@
-"""Меню, список сервисов, карточка, статистика, ручная проверка, удаление."""
-
-import html
-import json as jsonlib
-
-import aiohttp
-from aiogram import Bot, F, Router, types
-from aiogram.filters import Command, CommandStart
-from aiogram.fsm.context import FSMContext
-from aiogram.fsm.state import State, StatesGroup
-
-from healthbot.checker import _scalar_str, dig, evaluate, probe, snippet
-from healthbot.filters import Admin
-from healthbot.keyboards import (
-    after_check,
-    back_to_menu,
-    confirm_delete,
-    main_menu,
-    service_card,
-    services_list,
-)
-from utils.db import db
-from utils.db.models import MatchType, Service
-from utils.env import env
-from utils.format import human_duration, now, parse
-from utils.logging import logger
-
-router = Router()
-router.message.filter(Admin())
-router.callback_query.filter(Admin())
-
-MAX_NAME = 64
-
-GREETING = (
-    "👋 <b>healthbot</b> - слежу за твоими сервисами и пишу, если что-то легло.\n\n"
-    "Пришли ссылку на health-эндпоинт или жми кнопку 👇"
-)
-
-
-class RenameFlow(StatesGroup):
-    waiting_name = State()
-
-
-@router.startup()
-async def on_startup(bot: Bot) -> None:
-    commands = [
-        types.BotCommand(command="start", description="Меню"),
-        types.BotCommand(command="list", description="Сервисы"),
-        types.BotCommand(command="stats", description="Статистика"),
-    ]
-    if env.komodo.enabled:
-        commands.append(types.BotCommand(command="fleet", description="Флот"))
-    await bot.set_my_commands(commands)
-    logger.info(f"[green]Started as[/] @{(await bot.me()).username}")
-
-
-@router.shutdown()
-async def on_shutdown() -> None:
-    logger.info("Shutting down bot...")
-
-
-@router.message(CommandStart())
-async def cmd_start(message: types.Message) -> None:
-    await message.answer(GREETING, reply_markup=main_menu())
-
-
-@router.callback_query(F.data == "menu:main")
-async def menu_main(callback: types.CallbackQuery) -> None:
-    if isinstance(callback.message, types.Message):
-        await callback.message.edit_text(GREETING, reply_markup=main_menu())
-    await callback.answer()
-
-
-async def _list_view() -> tuple[str, types.InlineKeyboardMarkup]:
-    services = await db.services.all()
-    if not services:
-        return "Пока ни одного сервиса. Пришли ссылку, чтобы добавить.", back_to_menu()
-    watched = [s for s in services if not s.is_paused]
-    up = sum(s.is_up for s in watched)
-    paused = len(services) - len(watched)
-    tail = f", ⏸ {paused}" if paused else ""
-    text = f"📋 Сервисы - 🟢 {up} / 🔴 {len(watched) - up}{tail}\nТыкни, чтобы открыть."
-    return text, services_list(services)
-
-
-@router.message(Command("list"))
-async def cmd_list(message: types.Message) -> None:
-    text, markup = await _list_view()
-    await message.answer(text, reply_markup=markup)
-
-
-@router.callback_query(F.data == "menu:list")
-async def menu_list(callback: types.CallbackQuery) -> None:
-    text, markup = await _list_view()
-    if isinstance(callback.message, types.Message):
-        await callback.message.edit_text(text, reply_markup=markup)
-    await callback.answer()
-
-
-def _downtime(incidents: list) -> float:
-    total = 0.0
-    for inc in incidents:
-        end = parse(inc.ended_at) if inc.ended_at else now()
-        total += (end - parse(inc.started_at)).total_seconds()
-    return total
-
-
-async def _card_view(service: Service) -> str:
-    incidents = await db.incidents.for_service(service.id)
-    down = _downtime(incidents)
-    window = (now() - parse(service.created_at)).total_seconds()
-    uptime = 100.0 if window <= 0 else max(0.0, (window - down) / window * 100)
-
-    if service.is_paused:
-        status = "⏸ на паузе - не проверяется"
-    elif service.is_up:
-        status = "🟢 работает"
-        if service.fail_streak:
-            status += (
-                f"\n   ⚠️ неудач подряд: {service.fail_streak}"
-                f"/{env.monitor.failure_threshold}"
-            )
-    else:
-        current = await db.incidents.current(service.id)
-        since = (
-            human_duration((now() - parse(current.started_at)).total_seconds())
-            if current
-            else "?"
-        )
-        reason = html.escape(current.reason) if current else ""
-        status = f"🔴 лежит уже {since}\n   причина: {reason}"
-
-    latency = (
-        f"{service.last_latency_ms} мс" if service.last_latency_ms is not None else "-"
-    )
-
-    return (
-        f"<b>{html.escape(service.name)}</b>\n"
-        f"<code>{html.escape(service.url)}</code>\n\n"
-        f"Правило: {service.rule}\n"
-        f"Статус: {status}\n"
-        f"Отклик: {latency}\n\n"
-        f"📊 Аптайм: <b>{uptime:.2f}%</b>\n"
-        f"Инцидентов: {len(incidents)}\n"
-        f"Суммарный даунтайм: {human_duration(down) if down else '0с'}"
-    )
-
-
-async def _show_card(callback: types.CallbackQuery, service: Service) -> None:
-    if isinstance(callback.message, types.Message):
-        await callback.message.edit_text(
-            await _card_view(service), reply_markup=service_card(service)
-        )
-
-
-async def _load(callback: types.CallbackQuery) -> Service | None:
-    service_id = int((callback.data or "").rsplit(":", 1)[-1])
-    service = await db.services.get(service_id)
-    if service is None:
-        await callback.answer("Сервис не найден", show_alert=True)
-    return service
-
-
-@router.callback_query(F.data.startswith("svc:open:"))
-async def svc_open(callback: types.CallbackQuery) -> None:
-    service = await _load(callback)
-    if service is None:
-        return
-    await _show_card(callback, service)
-    await callback.answer()
-
-
-@router.callback_query(F.data.startswith("svc:check:"))
-async def svc_check(callback: types.CallbackQuery) -> None:
-    """Проверить прямо сейчас и показать, что сервис отдаёт."""
-    service = await _load(callback)
-    if service is None:
-        return
-    await callback.answer("Запрашиваю…")
-
-    async with aiohttp.ClientSession() as session:
-        result = await probe(session, service.url, env.monitor.request_timeout)
-    verdict = evaluate(service, result)
-
-    head = "✅ Правило выполняется" if verdict.healthy else "🔴 Правило нарушено"
-    lines = [
-        f"<b>{html.escape(service.name)}</b>",
-        f"<code>{html.escape(service.url)}</code>",
-        "",
-        head,
-        f"HTTP: {result.status if result.reachable else '-'}"
-        f"   отклик: {result.latency_ms if result.latency_ms is not None else '-'} мс",
-    ]
-    if not verdict.healthy:
-        lines.append(f"Причина: {html.escape(verdict.reason)}")
-        report = verdict.report()
-        if report:
-            lines.append(report)
-    elif result.reachable:
-        lines.extend(snippet(result.body))
-
-    if isinstance(callback.message, types.Message):
-        await callback.message.edit_text(
-            "\n".join(lines),
-            reply_markup=after_check(
-                service.id, offer_baseline=not verdict.healthy and result.reachable
-            ),
-        )
-
-
-@router.callback_query(F.data.startswith("svc:base:"))
-async def svc_baseline(callback: types.CallbackQuery) -> None:
-    """Принять текущий ответ за новый эталон."""
-    service = await _load(callback)
-    if service is None:
-        return
-    async with aiohttp.ClientSession() as session:
-        result = await probe(session, service.url, env.monitor.request_timeout)
-    if not result.reachable:
-        await callback.answer("Сервис недоступен - эталон снимать не с чего", True)  # noqa: FBT003
-        return
-
-    json_value = service.json_value
-    if service.match_type is MatchType.JSON:
-        try:
-            value = dig(jsonlib.loads(result.body), service.json_path or "")
-        except ValueError:
-            await callback.answer("Ответ не JSON - эталон не обновлён", True)  # noqa: FBT003
-            return
-        json_value = _scalar_str(value)
-
-    await db.services.set_baseline(
-        service.id,
-        ok_status=result.status or service.ok_status,
-        ok_body=(
-            result.body if service.match_type is MatchType.BODY else service.ok_body
-        ),
-        json_value=json_value,
-    )
-    await db.services.set_streak(service.id, 0)
-    updated = await db.services.get(service.id)
-    if updated is not None:
-        await _show_card(callback, updated)
-    await callback.answer("Эталон обновлён")
-
-
-@router.callback_query(F.data.startswith("svc:pause:"))
-async def svc_pause(callback: types.CallbackQuery) -> None:
-    service = await _load(callback)
-    if service is None:
-        return
-    await db.services.set_paused(service.id, is_paused=True)
-    updated = await db.services.get(service.id)
-    if updated is not None:
-        await _show_card(callback, updated)
-    await callback.answer("На паузе")
-
-
-@router.callback_query(F.data.startswith("svc:resume:"))
-async def svc_resume(callback: types.CallbackQuery) -> None:
-    service = await _load(callback)
-    if service is None:
-        return
-    await db.services.set_paused(service.id, is_paused=False)
-    updated = await db.services.get(service.id)
-    if updated is not None:
-        await _show_card(callback, updated)
-    await callback.answer("Снова слежу")
-
-
-@router.callback_query(F.data.startswith("svc:rename:"))
-async def svc_rename(callback: types.CallbackQuery, state: FSMContext) -> None:
-    service = await _load(callback)
-    if service is None:
-        return
-    await state.set_state(RenameFlow.waiting_name)
-    await state.set_data({"service_id": service.id})
-    if isinstance(callback.message, types.Message):
-        await callback.message.edit_text(
-            f"Как назвать вместо <b>{html.escape(service.name)}</b>?\n"
-            "Пришли новое имя сообщением.",
-            reply_markup=back_to_menu(),
-        )
-    await callback.answer()
-
-
-@router.message(RenameFlow.waiting_name)
-async def on_new_name(message: types.Message, state: FSMContext) -> None:
-    name = (message.text or "").strip()[:MAX_NAME]
-    if not name:
-        await message.answer("Пустое имя не годится.")
-        return
-    data = await state.get_data()
-    await state.clear()
-    await db.services.rename(data["service_id"], name)
-    service = await db.services.get(data["service_id"])
-    if service is None:
-        await message.answer("Сервис успел удалиться.")
-        return
-    await message.answer(await _card_view(service), reply_markup=service_card(service))
-
-
-@router.callback_query(F.data.startswith("svc:del:"))
-async def svc_del(callback: types.CallbackQuery) -> None:
-    service_id = int((callback.data or "").rsplit(":", 1)[-1])
-    if isinstance(callback.message, types.Message):
-        await callback.message.edit_text(
-            "Точно удалить сервис вместе со всей статистикой?",
-            reply_markup=confirm_delete(service_id),
-        )
-    await callback.answer()
-
-
-@router.callback_query(F.data.startswith("svc:delyes:"))
-async def svc_del_yes(callback: types.CallbackQuery) -> None:
-    service_id = int((callback.data or "").rsplit(":", 1)[-1])
-    await db.services.delete(service_id)
-    text, markup = await _list_view()
-    if isinstance(callback.message, types.Message):
-        await callback.message.edit_text(text, reply_markup=markup)
-    await callback.answer("Удалено")
-
-
-async def _stats_view() -> tuple[str, types.InlineKeyboardMarkup]:
-    services = await db.services.all()
-    if not services:
-        return "Пока нет данных.", back_to_menu()
-    lines = ["📊 <b>Статистика</b>\n"]
-    for service in services:
-        incidents = await db.incidents.for_service(service.id)
-        down = _downtime(incidents)
-        window = (now() - parse(service.created_at)).total_seconds()
-        uptime = 100.0 if window <= 0 else max(0.0, (window - down) / window * 100)
-        latency = (
-            f", отклик {service.last_latency_ms} мс"
-            if service.last_latency_ms is not None
-            else ""
-        )
-        lines.append(
-            f"{service.mark} <b>{html.escape(service.name)}</b> - "
-            f"аптайм {uptime:.2f}%, инцидентов {len(incidents)}, "
-            f"даунтайм {human_duration(down) if down else '0с'}{latency}"
-        )
-    return "\n".join(lines), back_to_menu()
-
-
-@router.message(Command("stats"))
-async def cmd_stats(message: types.Message) -> None:
-    text, markup = await _stats_view()
-    await message.answer(text, reply_markup=markup)
-
-
-@router.callback_query(F.data == "menu:stats")
-async def menu_stats(callback: types.CallbackQuery) -> None:
-    text, markup = await _stats_view()
-    if isinstance(callback.message, types.Message):
-        await callback.message.edit_text(text, reply_markup=markup)
-    await callback.answer()

src/healthbot/handlers/fleet.py

@@ -1,27 +1,43 @@
-"""Флот из телеграма: посмотреть сервера и стеки, нажать Deploy с телефона."""
+import asyncio
-
+import re
-import html
+from collections.abc import Awaitable
 
 from aiogram import F, Router, types
 from aiogram.filters import Command
+from aiogram.fsm.context import FSMContext
+from aiogram.types import BufferedInputFile
 from aiogram.utils.keyboard import InlineKeyboardBuilder
 
+from healthbot.alerts import format_alert
 from healthbot.filters import Admin
-from healthbot.keyboards import back_to_menu
+from healthbot.keyboards import back_to, back_to_menu
-from healthbot.komodo import KomodoError, Stack, komodo
+from healthbot.komodo import KomodoError, Server, Stack, Update, komodo
+from healthbot.runs import track
+from healthbot.ui import DANGER, PRIMARY, esc, label, pager, pages, pre, show
+from utils.db import db
+from utils.format import from_ms, stamp
 from utils.logging import logger
 
 router = Router()
 router.message.filter(Admin())
 router.callback_query.filter(Admin())
 
-LOG_TAIL = 30
+LOG_TAIL = 40
-LOG_CHARS = 2500
+LOG_FILE_TAIL = 1000
+LOG_CHARS = 3200
+STACKS_PAGE = 20
+HISTORY = 12
+JOURNAL = 15
+FAILED_BUTTONS = 5
+ALL_SERVICES = -1
+SMALL_DISK_GB = 2
+LOG_PREFIX_RE = re.compile(r"^\S+\s+\| ", re.MULTILINE)
 
-ACTIONS: dict[str, tuple[str, str, str]] = {
+OPS: dict[str, tuple[str, str, str | None]] = {
-    "dep": ("🚀 Deploy", "DeployStack", "Деплой запущен"),
+    "dep": ("Deploy", "DeployStack", PRIMARY),
-    "res": ("🔄 Restart", "RestartStack", "Перезапуск запущен"),
+    "res": ("Restart", "RestartStack", None),
-    "sto": ("⏹ Stop", "StopStack", "Остановка запущена"),
+    "sto": ("Stop", "StopStack", DANGER),
+    "sta": ("Start", "StartStack", None),
 }
 
 
@@ -32,122 +48,316 @@ async def _guard(callback: types.CallbackQuery) -> bool:
     return False
 
 
-def _fleet_kb(
+def _usage(server: Server) -> str:
-    stacks: list[Stack], servers: dict[str, str]
+    parts = []
-) -> types.InlineKeyboardMarkup:
+    if server.cpu is not None:
-    kb = InlineKeyboardBuilder()
+        parts.append(f"cpu {server.cpu:.0f}%")
-    for stack in sorted(stacks, key=lambda s: (servers.get(s.server_id, ""), s.name)):
+    if server.mem is not None:
-        kb.button(
+        parts.append(f"память {server.mem:.0f}%")
-            text=f"{stack.mark} {stack.name}", callback_data=f"fleet:stack:{stack.id}"
+    if server.disk is not None:
+        parts.append(f"диск {server.disk:.0f}%")
+    return ", ".join(parts)
+
+
+def _stack_count(stacks: list[Stack]) -> str:
+    good = sum(s.healthy for s in stacks)
+    if good == len(stacks):
+        return f"стеков {len(stacks)}, все работают"
+    return f"стеков {len(stacks)}, работают {good}"
+
+
+async def fleet_view() -> tuple[str, types.InlineKeyboardMarkup]:
+    servers, stacks, alerts = await asyncio.gather(
+        komodo.servers(), komodo.stacks(), komodo.open_alerts()
     )
-    kb.button(text="🔄 Обновить", callback_data="fleet:main")
+    servers.sort(key=lambda s: s.name)
-    kb.button(text="⬅️ Меню", callback_data="menu:main")
+    lines = [f"<b>Флот</b>: серверов {len(servers)}, стеков {len(stacks)}", ""]
-    kb.adjust(1)
+    kb = InlineKeyboardBuilder()
-    return kb.as_markup()
+    for server in servers:
-
-
-async def _fleet_view() -> tuple[str, types.InlineKeyboardMarkup]:
-    servers = await komodo.servers()
-    stacks = await komodo.stacks()
-    by_id = {s.id: s.name for s in servers}
-
-    lines = ["🛰 <b>Флот</b>", ""]
-    for server in sorted(servers, key=lambda s: s.name):
         mine = [s for s in stacks if s.server_id == server.id]
-        good = sum(s.state == "running" for s in mine)
+        if server.state == "NotOk":
+            lines.append(f"{server.mark} <b>{esc(server.name)}</b> не отвечает")
+        elif server.state == "Disabled":
+            lines.append(f"{server.mark} <b>{esc(server.name)}</b> выключен в Komodo")
+        else:
+            usage = _usage(server)
             lines.append(
-            f"{server.mark} <b>{html.escape(server.name)}</b> - "
+                f"{server.mark} <b>{esc(server.name)}</b>"
-            f"{good}/{len(mine)} стеков в строю"
+                + (f" · {usage}" if usage else "")
             )
+        lines.append(_stack_count(mine) if mine else "стеков нет")
+        kb.button(
+            text=f"{server.mark} {label(server.name, 28)}",
+            callback_data=f"fleet:srv:{server.id}:0",
+        )
+    rows = [2] * (len(servers) // 2) + [1] * (len(servers) % 2)
 
-    broken = [s for s in stacks if s.state not in {"running", "unknown"}]
+    broken = [s for s in stacks if not s.healthy]
+    extra = 0
     if broken:
-        lines.append("")
+        kb.button(text=f"Не работают: {len(broken)}", callback_data="fleet:bad")
-        lines.append("Не в строю:")
+        extra += 1
-        lines.extend(
+    if alerts:
-            f"{s.mark} {html.escape(s.name)} - {html.escape(s.state)}" for s in broken
+        kb.button(text=f"Алерты: {len(alerts)}", callback_data="fleet:alerts")
+        extra += 1
+    kb.button(text="Журнал", callback_data="fleet:journal")
+    kb.button(text="Обновить", callback_data="fleet:main")
+    kb.button(text="Меню", callback_data="menu:main")
+    kb.adjust(*rows, *([extra] if extra else []), 3)
+    return "\n".join(lines), kb.as_markup()
+
+
+def _stack_buttons(kb: InlineKeyboardBuilder, stacks: list[Stack]) -> list[int]:
+    for stack in stacks:
+        kb.button(
+            text=f"{stack.mark} {label(stack.name, 28)}",
+            callback_data=f"fleet:stack:{stack.id}",
+        )
+    return [2] * (len(stacks) // 2) + [1] * (len(stacks) % 2)
+
+
+async def server_view(
+    server_id: str, page: int
+) -> tuple[str, types.InlineKeyboardMarkup]:
+    servers, stacks = await asyncio.gather(komodo.servers(), komodo.stacks())
+    server = next((s for s in servers if s.id == server_id), None)
+    if server is None:
+        return "Сервер не найден.", back_to("К флоту", "fleet:main")
+
+    lines = [f"{server.mark} <b>{esc(server.name)}</b>"]
+    if server.state == "NotOk":
+        lines.append("не отвечает")
+    facts = []
+    if server.version:
+        facts.append(f"Komodo {esc(server.version)}")
+    if server.cpu is not None:
+        facts.append(f"cpu {server.cpu:.0f}%")
+    if server.load is not None:
+        facts.append(f"load {server.load:.2f}")
+    if facts:
+        lines.append(" · ".join(facts))
+    if server.mem is not None:
+        lines.append(
+            f"Память: {server.mem_used:.1f} из {server.mem_total:.1f} ГБ"
+            f" ({server.mem:.0f}%)"
+        )
+    try:
+        disks = await komodo.disks(server_id)
+    except KomodoError:
+        disks = []
+    for mount, used, total in [d for d in disks if d[2] >= SMALL_DISK_GB][:4]:
+        lines.append(
+            f"Диск <code>{esc(mount)}</code>: {used:.1f} из {total:.1f} ГБ"
+            f" ({used / total * 100:.0f}%)"
         )
 
-    return "\n".join(lines), _fleet_kb(stacks, by_id)
+    mine = sorted(
+        (s for s in stacks if s.server_id == server_id),
+        key=lambda s: (s.healthy, s.name),
+    )
+    lines += ["", _stack_count(mine) if mine else "стеков нет"]
+    total_pages = pages(len(mine), STACKS_PAGE)
+    page = min(page, total_pages - 1)
+    kb = InlineKeyboardBuilder()
+    rows = _stack_buttons(kb, mine[page * STACKS_PAGE : (page + 1) * STACKS_PAGE])
+    nav = pager(kb, f"fleet:srv:{server_id}", page, total_pages)
+    kb.button(text="Обновить", callback_data=f"fleet:srv:{server_id}:{page}")
+    kb.button(text="К флоту", callback_data="fleet:main")
+    kb.adjust(*rows, *([nav] if nav else []), 2)
+    return "\n".join(lines), kb.as_markup()
 
 
-async def _stack_view(stack_id: str) -> tuple[str, types.InlineKeyboardMarkup]:
+async def stack_view(stack_id: str) -> tuple[str, types.InlineKeyboardMarkup]:
-    stacks = await komodo.stacks()
+    stack = await komodo.stack(stack_id)
-    stack = next((s for s in stacks if s.id == stack_id), None)
     if stack is None:
-        return "Стек не найден - возможно, его удалили.", back_to_menu()
+        return "Стек не найден, возможно его удалили.", back_to("К флоту", "fleet:main")
 
-    servers = {s.id: s.name for s in await komodo.servers()}
+    lines = [
-    text = (
+        f"{stack.mark} <b>{esc(stack.name)}</b>",
-        f"{stack.mark} <b>{html.escape(stack.name)}</b>\n"
+        f"Сервер: {esc(stack.server_name or 'не назначен')}",
-        f"Состояние: <code>{html.escape(stack.state or '?')}</code>\n"
+        f"Состояние: {esc(stack.status or stack.state or 'неизвестно')}",
-        f"Сервер: {html.escape(servers.get(stack.server_id, '?'))}"
+    ]
+    if stack.repo:
+        lines.append(f"Репозиторий: {esc(stack.repo)}, ветка {esc(stack.branch)}")
+    if stack.deployed_hash:
+        newer = f", в репозитории уже {esc(stack.latest_hash)}" if stack.behind else ""
+        lines.append(f"Выкачен <code>{esc(stack.deployed_hash)}</code>{newer}")
+    if stack.image_updates:
+        lines.append(f"Новых образов: {stack.image_updates}")
+
+    try:
+        containers = await komodo.containers(stack_id)
+    except KomodoError:
+        containers = []
+    if containers:
+        lines.append("")
+        for item in containers:
+            facts = [item.status, f"cpu {item.cpu}" if item.cpu else "", item.mem]
+            lines.append(
+                f"{item.mark} {esc(item.service)}"
+                + "".join(f" · {esc(fact)}" for fact in facts if fact)
             )
 
     kb = InlineKeyboardBuilder()
-    for key, (label, _, _) in ACTIONS.items():
+    shown = ("dep", "res", "sto") if stack.state != "down" else ("dep", "sta")
-        kb.button(text=label, callback_data=f"fleet:ask:{key}:{stack.id}")
+    for key in shown:
-    kb.button(text="📜 Логи", callback_data=f"fleet:log:{stack.id}")
+        text, _, style = OPS[key]
-    kb.button(text="⬅️ К флоту", callback_data="fleet:main")
+        kb.button(text=text, callback_data=f"fleet:ask:{key}:{stack.id}", style=style)
-    kb.adjust(3, 2)
+    kb.button(text="Логи", callback_data=f"fleet:logs:{stack.id}")
-    return text, kb.as_markup()
+    kb.button(text="История", callback_data=f"fleet:hist:{stack.id}")
+    kb.button(text="Обновить", callback_data=f"fleet:stack:{stack.id}")
+    kb.button(
+        text="К серверу" if stack.server_id else "К флоту",
+        callback_data=f"fleet:srv:{stack.server_id}:0"
+        if stack.server_id
+        else "fleet:main",
+    )
+    kb.adjust(len(shown), 2, 2)
+    return "\n".join(lines), kb.as_markup()
+
+
+async def _render(
+    event: types.Message | types.CallbackQuery,
+    view: Awaitable[tuple[str, types.InlineKeyboardMarkup]],
+    *,
+    new: bool = False,
+) -> None:
+    try:
+        text, markup = await view
+    except KomodoError as exc:
+        if isinstance(event, types.CallbackQuery):
+            await event.answer(str(exc)[:190], show_alert=True)
+        else:
+            await show(event, f"🔴 {esc(exc)}", back_to_menu())
+        return
+    await show(event, text, markup, new=new)
+    if isinstance(event, types.CallbackQuery):
+        await event.answer()
 
 
 @router.message(Command("fleet"))
-async def cmd_fleet(message: types.Message) -> None:
+async def cmd_fleet(message: types.Message, state: FSMContext) -> None:
+    await state.clear()
     if not komodo.enabled:
-        await message.answer("Komodo не настроен - см. README, раздел про KOMODO__*.")
+        await show(message, "Komodo не настроен: нужны переменные KOMODO__*.")
         return
-    try:
+    await _render(message, fleet_view())
-        text, markup = await _fleet_view()
-    except KomodoError as exc:
-        await message.answer(f"⚠️ {html.escape(str(exc))}")
-        return
-    await message.answer(text, reply_markup=markup)
 
 
 @router.callback_query(F.data == "fleet:main")
-async def fleet_main(callback: types.CallbackQuery) -> None:
+async def fleet_main(callback: types.CallbackQuery, state: FSMContext) -> None:
     if not await _guard(callback):
         return
-    try:
+    await state.clear()
-        text, markup = await _fleet_view()
+    komodo.forget()
-    except KomodoError as exc:
+    await _render(callback, fleet_view())
-        await callback.answer(str(exc), show_alert=True)
+
+
+def _id_and_page(data: str | None) -> tuple[str, int]:
+    parts = (data or "").split(":")
+    page = int(parts[3]) if len(parts) > 3 and parts[3].lstrip("-").isdigit() else 0  # noqa: PLR2004
+    return parts[2], page
+
+
+@router.callback_query(F.data.startswith("fleet:srv:"))
+async def fleet_server(callback: types.CallbackQuery) -> None:
+    if not await _guard(callback):
         return
-    if isinstance(callback.message, types.Message):
+    server_id, page = _id_and_page(callback.data)
-        await callback.message.edit_text(text, reply_markup=markup)
+    await _render(callback, server_view(server_id, page))
-    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("go:srv:"))
+async def go_server(callback: types.CallbackQuery) -> None:
+    if not await _guard(callback):
+        return
+    server_id, _ = _id_and_page(callback.data)
+    await _render(callback, server_view(server_id, 0), new=True)
 
 
 @router.callback_query(F.data.startswith("fleet:stack:"))
 async def fleet_stack(callback: types.CallbackQuery) -> None:
     if not await _guard(callback):
         return
-    stack_id = (callback.data or "").rsplit(":", 1)[-1]
+    stack_id, _ = _id_and_page(callback.data)
-    try:
+    await _render(callback, stack_view(stack_id))
-        text, markup = await _stack_view(stack_id)
+
-    except KomodoError as exc:
+
-        await callback.answer(str(exc), show_alert=True)
+@router.callback_query(F.data.startswith("go:stack:"))
+async def go_stack(callback: types.CallbackQuery) -> None:
+    if not await _guard(callback):
         return
-    if isinstance(callback.message, types.Message):
+    stack_id, _ = _id_and_page(callback.data)
-        await callback.message.edit_text(text, reply_markup=markup)
+    await _render(callback, stack_view(stack_id), new=True)
-    await callback.answer()
+
+
+async def _bad_view() -> tuple[str, types.InlineKeyboardMarkup]:
+    broken = sorted(
+        (s for s in await komodo.stacks() if not s.healthy),
+        key=lambda s: (s.server_name, s.name),
+    )
+    lines = ["<b>Не работают</b>", ""]
+    lines.extend(
+        f"{s.mark} {esc(s.name)} · {esc(s.server_name)} · {esc(s.state)}"
+        for s in broken
+    )
+    kb = InlineKeyboardBuilder()
+    rows = _stack_buttons(kb, broken[:40])
+    kb.button(text="К флоту", callback_data="fleet:main")
+    kb.adjust(*rows, 1)
+    return "\n".join(lines), kb.as_markup()
+
+
+@router.callback_query(F.data == "fleet:bad")
+async def fleet_bad(callback: types.CallbackQuery) -> None:
+    if not await _guard(callback):
+        return
+    await _render(callback, _bad_view())
+
+
+async def _alerts_view() -> tuple[str, types.InlineKeyboardMarkup]:
+    tz = db.settings.get("tz")
+    alerts = await komodo.open_alerts()
+    lines = ["<b>Открытые алерты</b>"]
+    if not alerts:
+        lines.append("Сейчас ни одного.")
+    for alert in alerts[:10]:
+        since = stamp(from_ms(alert.get("ts") or 0), tz)
+        lines += ["", format_alert(alert), f"с {since}"]
+    return "\n".join(lines), back_to("К флоту", "fleet:main")
+
+
+@router.callback_query(F.data == "fleet:alerts")
+async def fleet_alerts(callback: types.CallbackQuery) -> None:
+    if not await _guard(callback):
+        return
+    await _render(callback, _alerts_view())
 
 
 @router.callback_query(F.data.startswith("fleet:ask:"))
 async def fleet_ask(callback: types.CallbackQuery) -> None:
-    """Подтверждение. Deploy с телефона слишком легко нажать штаниной."""
+    if not await _guard(callback):
+        return
     _, _, key, stack_id = (callback.data or "").split(":", 3)
-    label = ACTIONS[key][0]
+    text, _, style = OPS[key]
+    try:
+        stack = await komodo.stack(stack_id)
+    except KomodoError as exc:
+        await callback.answer(str(exc)[:190], show_alert=True)
+        return
+    if stack is None:
+        await callback.answer("Стек не найден", show_alert=True)
+        return
     kb = InlineKeyboardBuilder()
-    kb.button(text=f"✅ Да, {label}", callback_data=f"fleet:do:{key}:{stack_id}")
+    kb.button(
-    kb.button(text="⬅️ Отмена", callback_data=f"fleet:stack:{stack_id}")
+        text=text, callback_data=f"fleet:do:{key}:{stack_id}", style=style or PRIMARY
-    kb.adjust(1)
+    )
-    if isinstance(callback.message, types.Message):
+    kb.button(text="Отмена", callback_data=f"fleet:stack:{stack_id}")
-        await callback.message.edit_text(
+    kb.adjust(2)
-            f"{label} - подтверди, это подействует на прод.",
+    await show(
-            reply_markup=kb.as_markup(),
+        callback,
+        f"{text} <b>{esc(stack.name)}</b> на {esc(stack.server_name)}?",
+        kb.as_markup(),
     )
     await callback.answer()
 
@@ -157,49 +367,226 @@ async def fleet_do(callback: types.CallbackQuery) -> None:
     if not await _guard(callback):
         return
     _, _, key, stack_id = (callback.data or "").split(":", 3)
-    _, method, done = ACTIONS[key]
+    text, method, _ = OPS[key]
-
+    await callback.answer()
-    await callback.answer(done)
-    if isinstance(callback.message, types.Message):
-        await callback.message.edit_text(f"⏳ {done}…")
-
     try:
-        await komodo.execute(method, {"stack": stack_id, "services": []})
+        stack = await komodo.stack(stack_id)
-        logger.info(f"{method} {stack_id} - запущено из телеграма")
-    except KomodoError as exc:
-        if isinstance(callback.message, types.Message):
-            await callback.message.edit_text(
-                f"⚠️ Не вышло: {html.escape(str(exc))}", reply_markup=back_to_menu()
-            )
-        return
-
-    try:
-        text, markup = await _stack_view(stack_id)
     except KomodoError:
-        text, markup = "Готово.", back_to_menu()
+        stack = None
-    if isinstance(callback.message, types.Message):
+    name = stack.name if stack else stack_id
-        await callback.message.edit_text(f"✅ {done}.\n\n{text}", reply_markup=markup)
+
+    async def starter() -> Update:
+        logger.info(f"{method} {name} started from telegram")
+        return await komodo.execute(method, {"stack": stack_id, "services": []})
+
+    await track(
+        callback,
+        f"{text} {name}",
+        starter,
+        back=[
+            ("К стеку", f"fleet:stack:{stack_id}"),
+            ("Логи", f"fleet:logs:{stack_id}"),
+        ],
+        verbose=False,
+    )
+
+
+def _picked(stack: Stack, index: int) -> list[str]:
+    if index == ALL_SERVICES or index >= len(stack.services):
+        return []
+    return [stack.services[index]]
+
+
+async def _log_view(
+    stack_id: str, index: int
+) -> tuple[str, types.InlineKeyboardMarkup]:
+    stack = await komodo.stack(stack_id)
+    if stack is None:
+        return "Стек не найден.", back_to("К флоту", "fleet:main")
+    services = _picked(stack, index)
+    log = await komodo.stack_log(stack_id, services, LOG_TAIL)
+    if services or len(stack.services) == 1:
+        log = LOG_PREFIX_RE.sub("", log)
+    scope = f" · {esc(services[0])}" if services else ""
+    body = pre(log, LOG_CHARS) if log else "<i>лог пуст</i>"
+    kb = InlineKeyboardBuilder()
+    kb.button(text="Обновить", callback_data=f"fleet:log:{stack_id}:{index}")
+    kb.button(text="Файлом", callback_data=f"fleet:logf:{stack_id}:{index}")
+    rows = [2]
+    if len(stack.services) > 1:
+        kb.button(text="Другой сервис", callback_data=f"fleet:logs:{stack_id}")
+        kb.button(text="К стеку", callback_data=f"fleet:stack:{stack_id}")
+        rows.append(2)
+    else:
+        kb.button(text="К стеку", callback_data=f"fleet:stack:{stack_id}")
+        rows.append(1)
+    kb.adjust(*rows)
+    return f"<b>{esc(stack.name)}</b>{scope}\n{body}", kb.as_markup()
+
+
+async def _log_picker(stack_id: str) -> tuple[str, types.InlineKeyboardMarkup]:
+    stack = await komodo.stack(stack_id)
+    if stack is None:
+        return "Стек не найден.", back_to("К флоту", "fleet:main")
+    if len(stack.services) <= 1:
+        return await _log_view(stack_id, ALL_SERVICES)
+    kb = InlineKeyboardBuilder()
+    for index, service in enumerate(stack.services):
+        kb.button(
+            text=label(service, 28), callback_data=f"fleet:log:{stack_id}:{index}"
+        )
+    kb.button(text="Все сервисы", callback_data=f"fleet:log:{stack_id}:{ALL_SERVICES}")
+    kb.button(text="К стеку", callback_data=f"fleet:stack:{stack_id}")
+    count = len(stack.services)
+    kb.adjust(*[2] * (count // 2), *[1] * (count % 2), 2)
+    return f"<b>{esc(stack.name)}</b>: чьи логи?", kb.as_markup()
+
+
+@router.callback_query(F.data.startswith("fleet:logs:"))
+async def fleet_logs(callback: types.CallbackQuery) -> None:
+    if not await _guard(callback):
+        return
+    stack_id, _ = _id_and_page(callback.data)
+    await _render(callback, _log_picker(stack_id))
+
+
+@router.callback_query(F.data.startswith("go:log:"))
+async def go_log(callback: types.CallbackQuery) -> None:
+    if not await _guard(callback):
+        return
+    stack_id, _ = _id_and_page(callback.data)
+    await _render(callback, _log_view(stack_id, ALL_SERVICES), new=True)
 
 
 @router.callback_query(F.data.startswith("fleet:log:"))
 async def fleet_log(callback: types.CallbackQuery) -> None:
     if not await _guard(callback):
         return
-    stack_id = (callback.data or "").rsplit(":", 1)[-1]
+    stack_id, index = _id_and_page(callback.data)
-    await callback.answer("Забираю логи…")
+    await _render(callback, _log_view(stack_id, index))
-    try:
-        log = await komodo.stack_log(stack_id, tail=LOG_TAIL)
-    except KomodoError as exc:
-        await callback.answer(str(exc), show_alert=True)
-        return
 
-    kb = InlineKeyboardBuilder()
+
-    kb.button(text="🔄 Ещё раз", callback_data=f"fleet:log:{stack_id}")
+@router.callback_query(F.data.startswith("fleet:logf:"))
-    kb.button(text="⬅️ К стеку", callback_data=f"fleet:stack:{stack_id}")
+async def fleet_log_file(callback: types.CallbackQuery) -> None:
-    kb.adjust(2)
+    if not await _guard(callback):
-    tail = log[-LOG_CHARS:]
+        return
+    stack_id, index = _id_and_page(callback.data)
+    try:
+        stack = await komodo.stack(stack_id)
+        if stack is None:
+            await callback.answer("Стек не найден", show_alert=True)
+            return
+        services = _picked(stack, index)
+        log = await komodo.stack_log(stack_id, services, LOG_FILE_TAIL)
+    except KomodoError as exc:
+        await callback.answer(str(exc)[:190], show_alert=True)
+        return
+    await callback.answer()
     if isinstance(callback.message, types.Message):
-        await callback.message.edit_text(
+        name = "-".join([stack.name, *services])
-            f"📜 Последние {LOG_TAIL} строк:\n<pre>{html.escape(tail)}</pre>",
+        await callback.message.answer_document(
-            reply_markup=kb.as_markup(),
+            BufferedInputFile((log or "empty").encode(), filename=f"{name}.log")
         )
+
+
+def _same(one: Update, other: Update) -> bool:
+    return (one.operation, one.target_id, one.success, one.done) == (
+        other.operation,
+        other.target_id,
+        other.success,
+        other.done,
+    )
+
+
+def _collapse(updates: list[Update], *, everywhere: bool) -> list[tuple[Update, int]]:
+    result: list[tuple[Update, int]] = []
+    for update in updates:
+        scope = (
+            range(len(result)) if everywhere else range(len(result) - 1, len(result))
+        )
+        match = next((i for i in scope if i >= 0 and _same(result[i][0], update)), None)
+        if match is None:
+            result.append((update, 1))
+        else:
+            result[match] = (result[match][0], result[match][1] + 1)
+    return result
+
+
+def _update_mark(update: Update) -> str:
+    if not update.done:
+        return "🔄"
+    return "✅" if update.success else "🔴"
+
+
+async def _update_lines(
+    updates: list[Update], limit: int, *, with_target: bool
+) -> tuple[list[str], list[Update]]:
+    tz = db.settings.get("tz")
+    names = await komodo.names() if with_target else {}
+    lines, failed = [], []
+    for update, count in _collapse(updates, everywhere=with_target)[:limit]:
+        who = await komodo.username(update.operator)
+        target = (
+            f" {esc(names.get(update.target_id, update.target_type))}"
+            if with_target
+            else ""
+        )
+        times = f" ×{count}" if count > 1 else ""
+        lines.append(
+            f"{_update_mark(update)} {stamp(from_ms(update.start_ts), tz)} "
+            f"{esc(update.operation)}{target}{times} · {esc(who)}"
+        )
+        if update.done and not update.success:
+            failed.append(update)
+    return lines, failed
+
+
+def _failed_buttons(kb: InlineKeyboardBuilder, failed: list[Update]) -> int:
+    tz = db.settings.get("tz")
+    shown = failed[:FAILED_BUTTONS]
+    for update in shown:
+        kb.button(
+            text=f"🔴 {update.operation} {stamp(from_ms(update.start_ts), tz)}",
+            callback_data=f"run:st:{update.id}",
+        )
+    return len(shown)
+
+
+async def _history_view(stack_id: str) -> tuple[str, types.InlineKeyboardMarkup]:
+    stack = await komodo.stack(stack_id)
+    updates = await komodo.updates({"target.id": stack_id})
+    lines, failed = await _update_lines(updates, HISTORY, with_target=False)
+    title = esc(stack.name) if stack else "Стек"
+    kb = InlineKeyboardBuilder()
+    count = _failed_buttons(kb, failed)
+    kb.button(text="К стеку", callback_data=f"fleet:stack:{stack_id}")
+    kb.adjust(*[1] * count, 1)
+    body = "\n".join(lines) if lines else "Операций ещё не было."
+    return f"<b>{title}</b>: операции\n\n{body}", kb.as_markup()
+
+
+@router.callback_query(F.data.startswith("fleet:hist:"))
+async def fleet_history(callback: types.CallbackQuery) -> None:
+    if not await _guard(callback):
+        return
+    stack_id, _ = _id_and_page(callback.data)
+    await _render(callback, _history_view(stack_id))
+
+
+async def _journal_view() -> tuple[str, types.InlineKeyboardMarkup]:
+    updates = await komodo.updates()
+    lines, failed = await _update_lines(updates, JOURNAL, with_target=True)
+    kb = InlineKeyboardBuilder()
+    count = _failed_buttons(kb, failed)
+    kb.button(text="Обновить", callback_data="fleet:journal")
+    kb.button(text="К флоту", callback_data="fleet:main")
+    kb.adjust(*[1] * count, 2)
+    body = "\n".join(lines) if lines else "Операций ещё не было."
+    return f"<b>Журнал</b>\n\n{body}", kb.as_markup()
+
+
+@router.callback_query(F.data == "fleet:journal")
+async def fleet_journal(callback: types.CallbackQuery) -> None:
+    if not await _guard(callback):
+        return
+    await _render(callback, _journal_view())

src/healthbot/handlers/menu.py

@@ -0,0 +1,145 @@
+import asyncio
+
+from aiogram import Bot, F, Router, types
+from aiogram.filters import CommandStart
+from aiogram.fsm.context import FSMContext
+
+from healthbot.actionargs import command_name
+from healthbot.filters import Admin
+from healthbot.keyboards import main_menu
+from healthbot.komodo import KomodoError, komodo
+from healthbot.ui import MENU_COMMANDS, esc, plural, show
+from utils.db import db
+from utils.db.models import Service
+from utils.env import env
+from utils.format import human_duration, now, parse, short
+from utils.logging import logger
+
+router = Router()
+router.message.filter(Admin())
+router.callback_query.filter(Admin())
+
+SHOWN_PROBLEMS = 6
+FLEET_TIMEOUT = 6
+
+
+async def sync_commands(bot: Bot) -> None:
+    commands = [
+        types.BotCommand(command="start", description="Меню"),
+        types.BotCommand(command="list", description="Сервисы"),
+        types.BotCommand(command="stats", description="Статистика"),
+    ]
+    actions = env.komodo.enabled and db.settings.get("actions")
+    if env.komodo.enabled:
+        commands.append(types.BotCommand(command="fleet", description="Флот"))
+    if actions:
+        commands.append(types.BotCommand(command="actions", description="Действия"))
+    commands.append(types.BotCommand(command="settings", description="Настройки"))
+    if actions and db.settings.get("commands"):
+        taken = set(MENU_COMMANDS)
+        for pin in await db.pins.all():
+            name = command_name(pin.name)
+            if not name or name in taken:
+                continue
+            taken.add(name)
+            commands.append(
+                types.BotCommand(command=name, description=short(pin.label, 60))
+            )
+    await bot.set_my_commands(commands)
+
+
+@router.startup()
+async def on_startup(bot: Bot) -> None:
+    await sync_commands(bot)
+    logger.info(f"[green]Started as[/] @{(await bot.me()).username}")
+
+
+@router.shutdown()
+async def on_shutdown() -> None:
+    logger.info("Shutting down bot")
+
+
+async def _down_line(service: Service) -> str:
+    current = await db.incidents.current(service.id)
+    if current is None:
+        return f"🔴 {esc(service.name)}"
+    since = human_duration((now() - parse(current.started_at)).total_seconds())
+    return f"🔴 {esc(service.name)}: {since}, {esc(short(current.reason, 50))}"
+
+
+async def _services_block() -> list[str]:
+    services = await db.services.all()
+    if not services:
+        return ["Сервисов пока нет. Пришли ссылку, и я начну за ней следить."]
+    watched = [s for s in services if not s.is_paused]
+    down = [s for s in watched if not s.is_up]
+    paused = len(services) - len(watched)
+    working = len(watched) - len(down)
+    works = plural(working, "работает", "работают", "работают")
+    if not down and not paused:
+        every = "все " if working > 1 else ""
+        return [f"<b>Сервисы</b>: {every}{working} {works}"]
+    parts = [f"{working} {works}"]
+    if down:
+        parts.append(f"{len(down)} {plural(len(down), 'лежит', 'лежат', 'лежат')}")
+    if paused:
+        parts.append(f"{paused} на паузе")
+    lines = [f"<b>Сервисы</b>: {', '.join(parts)}"]
+    lines.extend([await _down_line(s) for s in down[:SHOWN_PROBLEMS]])
+    return lines
+
+
+async def _fleet_block() -> list[str]:
+    try:
+        servers, stacks, alerts = await asyncio.gather(
+            komodo.servers(), komodo.stacks(), komodo.open_alerts()
+        )
+    except KomodoError as exc:
+        return [f"<b>Флот</b>: {esc(exc)}"]
+    reachable = {s.id for s in servers if s.state == "Ok"}
+    good = sum(s.healthy for s in stacks)
+    summary = "все работают" if good == len(stacks) else f"работают {good}"
+    lines = [f"<b>Флот</b>: серверов {len(servers)}, стеков {len(stacks)}, {summary}"]
+    for server in servers:
+        if server.state == "NotOk":
+            lines.append(f"🔴 {esc(server.name)} не отвечает")
+        elif server.strained:
+            lines.append(f"🟠 {esc(server.name)}: {', '.join(server.strains)}")
+    troubled = [s for s in stacks if s.troubled and s.server_id in reachable]
+    lines.extend(
+        f"{s.mark} {esc(s.name)}: {esc(s.state)}" for s in troubled[:SHOWN_PROBLEMS]
+    )
+    if len(troubled) > SHOWN_PROBLEMS:
+        lines.append(f"и ещё {len(troubled) - SHOWN_PROBLEMS}")
+    if alerts:
+        lines.append(f"Открытых алертов: {len(alerts)}")
+    return lines
+
+
+async def dashboard() -> tuple[str, types.InlineKeyboardMarkup]:
+    lines = ["<b>healthbot</b>", "", *await _services_block()]
+    actions = env.komodo.enabled and db.settings.get("actions")
+    if env.komodo.enabled:
+        try:
+            fleet = await asyncio.wait_for(_fleet_block(), FLEET_TIMEOUT)
+        except TimeoutError:
+            fleet = ["<b>Флот</b>: Komodo не отвечает"]
+        lines += ["", *fleet]
+    pins = await db.pins.all() if actions else []
+    return "\n".join(lines), main_menu(pins, actions=bool(actions))
+
+
+@router.message(CommandStart())
+async def cmd_start(message: types.Message, state: FSMContext) -> None:
+    await state.clear()
+    text, markup = await dashboard()
+    await show(message, text, markup)
+
+
+@router.callback_query(F.data == "menu:main")
+async def menu_main(callback: types.CallbackQuery, state: FSMContext) -> None:
+    await state.clear()
+    komodo.forget()
+    text, markup = await dashboard()
+    await show(callback, text, markup)
+    await callback.answer()

src/healthbot/handlers/runs.py

@@ -0,0 +1,69 @@
+from aiogram import F, Router, types
+from aiogram.types import BufferedInputFile
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+from healthbot.filters import Admin
+from healthbot.komodo import KomodoError, Update, komodo
+from healthbot.runs import output_of, result_kb, result_text
+from healthbot.ui import esc, show, tail_arg
+from utils.format import clock
+
+router = Router()
+router.callback_query.filter(Admin())
+
+
+async def _title(update: Update) -> str:
+    try:
+        name = (await komodo.names()).get(update.target_id, "")
+    except KomodoError:
+        name = ""
+    return f"{update.operation} {name}".strip()
+
+
+@router.callback_query(F.data.startswith("run:st:"))
+async def run_status(callback: types.CallbackQuery) -> None:
+    update_id = tail_arg(callback.data)
+    try:
+        update = await komodo.update(update_id)
+    except KomodoError as exc:
+        await callback.answer(str(exc)[:190], show_alert=True)
+        return
+    title = await _title(update)
+    await callback.answer()
+    if not update.done:
+        kb = InlineKeyboardBuilder()
+        kb.button(text="Проверить", callback_data=f"run:st:{update_id}")
+        await show(
+            callback,
+            f"⏳ <b>{esc(title)}</b> идёт уже {clock(update.seconds)}",
+            kb.as_markup(),
+            new=True,
+        )
+        return
+    await show(
+        callback,
+        result_text(title, update, verbose=update.success),
+        result_kb(update, []),
+        new=True,
+    )
+
+
+@router.callback_query(F.data.startswith("run:file:"))
+async def run_file(callback: types.CallbackQuery) -> None:
+    update_id = tail_arg(callback.data)
+    try:
+        update = await komodo.update(update_id)
+    except KomodoError as exc:
+        await callback.answer(str(exc)[:190], show_alert=True)
+        return
+    await callback.answer()
+    if isinstance(callback.message, types.Message):
+        text = output_of(update, only_failed=False) or "empty"
+        await callback.message.answer_document(
+            BufferedInputFile(text.encode(), filename=f"{update.operation}.log")
+        )
+
+
+@router.callback_query()
+async def stale_button(callback: types.CallbackQuery) -> None:
+    await callback.answer("Эта кнопка устарела, открой /start", show_alert=True)

src/healthbot/handlers/services.py

@@ -0,0 +1,410 @@
+import json as jsonlib
+
+import aiohttp
+from aiogram import F, Router, types
+from aiogram.filters import Command
+from aiogram.fsm.context import FSMContext
+from aiogram.fsm.state import State, StatesGroup
+
+from healthbot.checker import MISSING, dig, evaluate, probe, scalar_str, snippet
+from healthbot.filters import Admin
+from healthbot.keyboards import (
+    after_check,
+    back_to,
+    back_to_menu,
+    confirm_delete,
+    service_card,
+    services_list,
+)
+from healthbot.stats import (
+    PERIODS,
+    current_hour,
+    day_squares,
+    downtime,
+    latency_chart,
+    uptime,
+    uptime_line,
+)
+from healthbot.ui import ANSWER, esc, int_arg, plural, show
+from utils.db import db
+from utils.db.models import Incident, MatchType, Service
+from utils.env import env
+from utils.format import ago, human_duration, now, parse, short, stamp
+
+router = Router()
+router.message.filter(Admin())
+router.callback_query.filter(Admin())
+
+MAX_NAME = 64
+SHOWN_INCIDENTS = 10
+CHART_HOURS = 24
+
+
+class RenameFlow(StatesGroup):
+    waiting_name = State()
+
+
+async def list_view(page: int = 0) -> tuple[str, types.InlineKeyboardMarkup]:
+    services = await db.services.all()
+    if not services:
+        return "Сервисов пока нет. Пришли ссылку, чтобы добавить.", back_to_menu()
+    watched = [s for s in services if not s.is_paused]
+    up = sum(s.is_up for s in watched)
+    paused = len(services) - len(watched)
+    parts = [f"🟢 {up}", f"🔴 {len(watched) - up}"]
+    if paused:
+        parts.append(f"⏸ {paused}")
+    return f"<b>Сервисы</b>  {'  '.join(parts)}", services_list(services, page)
+
+
+@router.message(Command("list"))
+async def cmd_list(message: types.Message, state: FSMContext) -> None:
+    await state.clear()
+    text, markup = await list_view()
+    await show(message, text, markup)
+
+
+@router.callback_query(F.data.startswith("menu:list"))
+async def menu_list(callback: types.CallbackQuery, state: FSMContext) -> None:
+    await state.clear()
+    text, markup = await list_view(int_arg(callback.data))
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+async def _status_line(service: Service) -> str:
+    if service.is_paused:
+        return "на паузе, не проверяется"
+    if not service.is_up:
+        current = await db.incidents.current(service.id)
+        if current is None:
+            return "лежит"
+        since = human_duration((now() - parse(current.started_at)).total_seconds())
+        muted = ", напоминания выключены" if current.muted else ""
+        return f"лежит {since}: {esc(current.reason)}{muted}"
+    parts = ["работает"]
+    if service.last_latency_ms is not None:
+        parts.append(f"отклик {service.last_latency_ms} мс")
+    if service.last_checked_at:
+        parts.append(f"проверен {ago(parse(service.last_checked_at))}")
+    if service.fail_streak:
+        parts.append(
+            f"неудач подряд {service.fail_streak} из {env.monitor.failure_threshold}"
+        )
+    return ", ".join(parts)
+
+
+def _incident_line(incident: Incident, tz: str) -> str:
+    began = parse(incident.started_at)
+    if incident.ended_at:
+        length = human_duration((parse(incident.ended_at) - began).total_seconds())
+    else:
+        length = "идёт сейчас"
+    return f"{stamp(began, tz)} · {length} · {esc(short(incident.reason, 60))}"
+
+
+async def card_view(service: Service) -> str:
+    tz = db.settings.get("tz")
+    incidents = await db.incidents.for_service(service.id)
+    hours = await db.stats.since(service.id, current_hour() - CHART_HOURS)
+    chart, average, peak = latency_chart(hours)
+
+    lines = [
+        f"{service.mark} <b>{esc(service.name)}</b>",
+        f"<code>{esc(service.url)}</code>",
+        "",
+        f"Правило: {service.rule}",
+        f"Статус: {await _status_line(service)}",
+        "",
+        f"Аптайм: {uptime_line(service, incidents)}",
+        f"14 дней: {day_squares(service, incidents, tz)}",
+    ]
+    if chart:
+        lines.append(f"Отклик за сутки: <code>{chart}</code>")
+        lines.append(f"в среднем {average} мс, пик {peak} мс")
+    if incidents:
+        last = _incident_line(incidents[-1], tz)
+        lines.append(f"Инцидентов: {len(incidents)}, последний {last}")
+    if service.tls_expires_at:
+        left = (parse(service.tls_expires_at) - now()).total_seconds()
+        state = f"ещё {human_duration(left)}" if left > 0 else "истёк"
+        lines.append(f"Сертификат: {state}")
+    return "\n".join(lines)
+
+
+async def show_card(
+    event: types.Message | types.CallbackQuery, service: Service, *, new: bool = False
+) -> None:
+    await show(event, await card_view(service), service_card(service), new=new)
+
+
+async def _load(callback: types.CallbackQuery) -> Service | None:
+    service = await db.services.get(int_arg(callback.data))
+    if service is None:
+        await callback.answer("Сервис не найден", show_alert=True)
+    return service
+
+
+@router.callback_query(F.data.startswith("svc:open:"))
+async def svc_open(callback: types.CallbackQuery, state: FSMContext) -> None:
+    service = await _load(callback)
+    if service is None:
+        return
+    await state.clear()
+    await show_card(callback, service)
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("go:svc:"))
+async def go_service(callback: types.CallbackQuery) -> None:
+    service = await _load(callback)
+    if service is None:
+        return
+    await show_card(callback, service, new=True)
+    await callback.answer()
+
+
+async def _check_view(service: Service) -> tuple[str, types.InlineKeyboardMarkup]:
+    async with aiohttp.ClientSession() as session:
+        result = await probe(session, service.url, env.monitor.request_timeout)
+    verdict = evaluate(service, result)
+
+    head = "✅ Правило выполняется" if verdict.healthy else "🔴 Правило нарушено"
+    lines = [
+        f"<b>{esc(service.name)}</b>",
+        f"<code>{esc(service.url)}</code>",
+        "",
+        head,
+    ]
+    if result.reachable:
+        lines.append(f"HTTP {result.status}, отклик {result.latency_ms} мс")
+    if not verdict.healthy:
+        lines.append(f"Причина: {esc(verdict.reason)}")
+        report = verdict.report()
+        if report:
+            lines.append(report)
+    elif result.reachable:
+        lines.extend(snippet(result.body))
+    markup = after_check(
+        service.id, offer_baseline=not verdict.healthy and result.reachable
+    )
+    return "\n".join(lines), markup
+
+
+@router.callback_query(F.data.startswith("svc:check:"))
+async def svc_check(callback: types.CallbackQuery) -> None:
+    service = await _load(callback)
+    if service is None:
+        return
+    await callback.answer()
+    text, markup = await _check_view(service)
+    await show(callback, text, markup)
+
+
+@router.callback_query(F.data.startswith("go:check:"))
+async def go_check(callback: types.CallbackQuery) -> None:
+    service = await _load(callback)
+    if service is None:
+        return
+    await callback.answer()
+    text, markup = await _check_view(service)
+    await show(callback, text, markup, new=True)
+
+
+@router.callback_query(F.data.startswith("svc:base:"))
+async def svc_baseline(callback: types.CallbackQuery) -> None:
+    service = await _load(callback)
+    if service is None:
+        return
+    async with aiohttp.ClientSession() as session:
+        result = await probe(session, service.url, env.monitor.request_timeout)
+    if not result.reachable:
+        await callback.answer("Сервис недоступен, эталон снять не с чего", True)  # noqa: FBT003
+        return
+
+    json_value = service.json_value
+    if service.match_type is MatchType.JSON:
+        try:
+            value = dig(jsonlib.loads(result.body), service.json_path or "")
+        except ValueError:
+            await callback.answer("Ответ не JSON, эталон не обновлён", True)  # noqa: FBT003
+            return
+        if value is MISSING or isinstance(value, (dict, list)):
+            await callback.answer(
+                f"Поля {service.json_path} в ответе нет, эталон не обновлён",
+                True,  # noqa: FBT003
+            )
+            return
+        json_value = scalar_str(value)
+
+    await db.services.set_baseline(
+        service.id,
+        ok_status=result.status or service.ok_status,
+        ok_body=(
+            result.body if service.match_type is MatchType.BODY else service.ok_body
+        ),
+        json_value=json_value,
+    )
+    await db.services.set_streak(service.id, 0)
+    updated = await db.services.get(service.id)
+    if updated is not None:
+        await show_card(callback, updated)
+    await callback.answer()
+
+
+async def _set_paused(callback: types.CallbackQuery, *, is_paused: bool) -> None:
+    service = await _load(callback)
+    if service is None:
+        return
+    await db.services.set_paused(service.id, is_paused=is_paused)
+    updated = await db.services.get(service.id)
+    if updated is not None:
+        await show_card(callback, updated)
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("svc:pause:"))
+async def svc_pause(callback: types.CallbackQuery) -> None:
+    await _set_paused(callback, is_paused=True)
+
+
+@router.callback_query(F.data.startswith("svc:resume:"))
+async def svc_resume(callback: types.CallbackQuery) -> None:
+    await _set_paused(callback, is_paused=False)
+
+
+@router.callback_query(F.data.startswith("svc:mute:"))
+async def svc_mute(callback: types.CallbackQuery) -> None:
+    incident = await db.incidents.get(int_arg(callback.data))
+    if incident is None or incident.ended_at:
+        await callback.answer("Инцидент уже закрыт", show_alert=True)
+        return
+    await db.incidents.mute(incident.id)
+    if isinstance(callback.message, types.Message):
+        kb = types.InlineKeyboardMarkup(
+            inline_keyboard=[
+                row
+                for row in (
+                    callback.message.reply_markup or back_to_menu()
+                ).inline_keyboard
+                if not any((b.callback_data or "").startswith("svc:mute:") for b in row)
+            ]
+        )
+        await callback.message.edit_reply_markup(reply_markup=kb)
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("svc:inc:"))
+async def svc_incidents(callback: types.CallbackQuery) -> None:
+    service = await _load(callback)
+    if service is None:
+        return
+    tz = db.settings.get("tz")
+    incidents = await db.incidents.for_service(service.id)
+    lines = [f"<b>{esc(service.name)}</b>: инциденты", ""]
+    if not incidents:
+        lines.append("Ни одного падения.")
+    else:
+        lines.extend(
+            _incident_line(i, tz) for i in reversed(incidents[-SHOWN_INCIDENTS:])
+        )
+        total = downtime(incidents, parse(service.created_at), now())
+        lines += ["", f"Всего {len(incidents)}, даунтайм {human_duration(total)}"]
+    await show(
+        callback, "\n".join(lines), back_to("К сервису", f"svc:open:{service.id}")
+    )
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("svc:rename:"))
+async def svc_rename(callback: types.CallbackQuery, state: FSMContext) -> None:
+    service = await _load(callback)
+    if service is None:
+        return
+    await state.set_state(RenameFlow.waiting_name)
+    await state.set_data({"service_id": service.id})
+    await show(
+        callback,
+        f"Пришли новое имя для <b>{esc(service.name)}</b>.",
+        back_to("Отмена", f"svc:open:{service.id}"),
+    )
+    await callback.answer()
+
+
+@router.message(RenameFlow.waiting_name, ANSWER)
+async def on_new_name(message: types.Message, state: FSMContext) -> None:
+    name = (message.text or "").strip()[:MAX_NAME]
+    if not name:
+        return
+    data = await state.get_data()
+    await state.clear()
+    await db.services.rename(data["service_id"], name)
+    service = await db.services.get(data["service_id"])
+    if service is None:
+        await show(message, "Сервис уже удалён.", back_to_menu())
+        return
+    await show_card(message, service)
+
+
+@router.callback_query(F.data.startswith("svc:del:"))
+async def svc_del(callback: types.CallbackQuery) -> None:
+    service = await _load(callback)
+    if service is None:
+        return
+    await show(
+        callback,
+        f"Удалить <b>{esc(service.name)}</b> вместе со статистикой?",
+        confirm_delete(service.id),
+    )
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("svc:delyes:"))
+async def svc_del_yes(callback: types.CallbackQuery) -> None:
+    await db.services.delete(int_arg(callback.data))
+    text, markup = await list_view()
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+async def stats_view() -> tuple[str, types.InlineKeyboardMarkup]:
+    services = await db.services.all()
+    if not services:
+        return "Сервисов пока нет.", back_to_menu()
+    month = PERIODS[-1][1]
+    rows = []
+    for service in services:
+        incidents = await db.incidents.for_service(service.id)
+        rows.append((uptime(service, incidents, month) or 0.0, service, incidents))
+    rows.sort(key=lambda row: row[0])
+
+    lines = ["<b>Статистика</b>", ""]
+    for _, service, incidents in rows:
+        speed = (
+            f" · {service.last_latency_ms} мс"
+            if service.last_latency_ms is not None
+            else ""
+        )
+        count = len(incidents)
+        falls = (
+            f"{count} {plural(count, 'падение', 'падения', 'падений')}"
+            if count
+            else "без падений"
+        )
+        lines.append(f"{service.mark} <b>{esc(service.name)}</b>{speed}")
+        lines.append(f"{uptime_line(service, incidents)} · {falls}")
+    return "\n".join(lines), back_to_menu()
+
+
+@router.message(Command("stats"))
+async def cmd_stats(message: types.Message, state: FSMContext) -> None:
+    await state.clear()
+    text, markup = await stats_view()
+    await show(message, text, markup)
+
+
+@router.callback_query(F.data == "menu:stats")
+async def menu_stats(callback: types.CallbackQuery) -> None:
+    text, markup = await stats_view()
+    await show(callback, text, markup)
+    await callback.answer()

src/healthbot/handlers/settings.py

@@ -0,0 +1,229 @@
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+from aiogram import Bot, F, Router, types
+from aiogram.filters import Command
+from aiogram.fsm.context import FSMContext
+from aiogram.fsm.state import State, StatesGroup
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+from healthbot.alerts import KNOWN_TYPES
+from healthbot.filters import Admin
+from healthbot.handlers.menu import sync_commands
+from healthbot.keyboards import back_to
+from healthbot.komodo import KomodoError, komodo
+from healthbot.ui import ANSWER, esc, int_arg, show, tail_arg
+from utils.db import db
+from utils.env import env
+from utils.format import now, zone
+
+router = Router()
+router.message.filter(Admin())
+router.callback_query.filter(Admin())
+
+TOGGLES = {
+    "actions": ("Действия Komodo", "включены", "выключены"),
+    "commands": ("Действия как команды", "включены", "выключены"),
+    "confirm": ("Подтверждение запуска", "включено", "выключено"),
+}
+DIGEST_HOURS = (7, 8, 9, 10, 12, 18, 21)
+
+
+class SettingsFlow(StatesGroup):
+    waiting_tz = State()
+
+
+async def _komodo_line() -> str:
+    if not env.komodo.enabled:
+        return "Komodo не подключён: нужны переменные KOMODO__*."
+    try:
+        return f"Komodo {esc(await komodo.version())} подключён."
+    except KomodoError as exc:
+        return f"Komodo: {esc(exc)}"
+
+
+async def settings_view() -> tuple[str, types.InlineKeyboardMarkup]:
+    tz = db.settings.get("tz")
+    lines = ["<b>Настройки</b>", await _komodo_line()]
+    kb = InlineKeyboardBuilder()
+    if env.komodo.enabled:
+        for key, (title, on, off) in TOGGLES.items():
+            if key != "actions" and not db.settings.get("actions"):
+                continue
+            state = on if db.settings.get(key) else off
+            kb.button(text=f"{title}: {state}", callback_data=f"set:t:{key}")
+        if db.settings.get("actions") and db.settings.get("commands"):
+            lines.append(
+                "Команда запускает действие по имени: "
+                "<code>/имя_действия поле значение</code>. "
+                "Закреплённые видны в меню команд."
+            )
+    if env.web.enabled:
+        muted = len(db.settings.get("muted"))
+        kb.button(text=f"Алерты Komodo, скрыто: {muted}", callback_data="set:alerts")
+    kb.button(text=f"Часовой пояс: {tz}", callback_data="set:tz")
+    hour = db.settings.get("digest")
+    digest = "выключена" if hour is None else f"в {hour:02d}:00"
+    kb.button(text=f"Сводка за сутки: {digest}", callback_data="set:digest")
+    if env.komodo.enabled and db.settings.get("actions"):
+        kb.button(text="Закреплённые", callback_data="set:pins")
+    kb.button(text="Меню", callback_data="menu:main")
+    kb.adjust(1)
+    return "\n".join(lines), kb.as_markup()
+
+
+@router.message(Command("settings"))
+async def cmd_settings(message: types.Message, state: FSMContext) -> None:
+    await state.clear()
+    text, markup = await settings_view()
+    await show(message, text, markup)
+
+
+@router.callback_query(F.data == "set:main")
+async def settings_main(callback: types.CallbackQuery, state: FSMContext) -> None:
+    await state.clear()
+    text, markup = await settings_view()
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("set:t:"))
+async def settings_toggle(callback: types.CallbackQuery, bot: Bot) -> None:
+    key = tail_arg(callback.data)
+    if key not in TOGGLES:
+        await callback.answer()
+        return
+    await db.settings.set(key, not db.settings.get(key))
+    await sync_commands(bot)
+    text, markup = await settings_view()
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+def _alerts_view() -> tuple[str, types.InlineKeyboardMarkup]:
+    muted: list[str] = db.settings.get("muted")
+    kinds = [*KNOWN_TYPES, *(kind for kind in muted if kind not in KNOWN_TYPES)]
+    kb = InlineKeyboardBuilder()
+    for kind in kinds:
+        mark = "🔕" if kind in muted else "🔔"
+        kb.button(text=f"{mark} {kind}", callback_data=f"set:mute:{kind}")
+    kb.button(text="К настройкам", callback_data="set:main")
+    kb.adjust(1)
+    return "<b>Алерты Komodo</b>", kb.as_markup()
+
+
+@router.callback_query(F.data == "set:alerts")
+async def settings_alerts(callback: types.CallbackQuery) -> None:
+    text, markup = _alerts_view()
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+async def _flip_mute(kind: str) -> None:
+    muted: list[str] = list(db.settings.get("muted"))
+    if kind in muted:
+        muted.remove(kind)
+    else:
+        muted.append(kind)
+    await db.settings.set("muted", muted)
+
+
+@router.callback_query(F.data.startswith("set:mute:"))
+async def settings_mute(callback: types.CallbackQuery) -> None:
+    await _flip_mute(tail_arg(callback.data))
+    text, markup = _alerts_view()
+    await show(callback, text, markup)
+    await callback.answer()
+
+
+async def _swap_alert_button(
+    callback: types.CallbackQuery, text: str, data: str
+) -> None:
+    message = callback.message
+    if isinstance(message, types.Message):
+        old = message.reply_markup.inline_keyboard if message.reply_markup else []
+        rows = [
+            [b for b in row if not (b.callback_data or "").startswith("al:")]
+            for row in old
+        ]
+        rows = [row for row in rows if row]
+        rows.append([types.InlineKeyboardButton(text=text, callback_data=data)])
+        await message.edit_reply_markup(
+            reply_markup=types.InlineKeyboardMarkup(inline_keyboard=rows)
+        )
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("al:mute:"))
+async def alert_mute(callback: types.CallbackQuery) -> None:
+    kind = tail_arg(callback.data)
+    if kind not in db.settings.get("muted"):
+        await _flip_mute(kind)
+    await _swap_alert_button(callback, f"{kind} скрыт, вернуть", f"al:show:{kind}")
+
+
+@router.callback_query(F.data.startswith("al:show:"))
+async def alert_unmute(callback: types.CallbackQuery) -> None:
+    kind = tail_arg(callback.data)
+    if kind in db.settings.get("muted"):
+        await _flip_mute(kind)
+    await _swap_alert_button(callback, "Не присылать такие", f"al:mute:{kind}")
+
+
+@router.callback_query(F.data == "set:tz")
+async def settings_tz(callback: types.CallbackQuery, state: FSMContext) -> None:
+    tz = db.settings.get("tz")
+    local = now().astimezone(zone(tz)).strftime("%H:%M")
+    await state.set_state(SettingsFlow.waiting_tz)
+    await show(
+        callback,
+        "Пришли часовой пояс, например <code>Europe/Warsaw</code>.\n"
+        f"Сейчас: {esc(tz)}, на часах {local}",
+        back_to("Отмена", "set:main"),
+    )
+    await callback.answer()
+
+
+@router.message(SettingsFlow.waiting_tz, ANSWER)
+async def on_tz(message: types.Message, state: FSMContext) -> None:
+    name = (message.text or "").strip()
+    try:
+        ZoneInfo(name)
+    except (ZoneInfoNotFoundError, ValueError, OSError):
+        await show(
+            message,
+            "Такого пояса нет. Нужно имя из базы tz, например "
+            "<code>Europe/Warsaw</code> или <code>UTC</code>.",
+            back_to("Отмена", "set:main"),
+        )
+        return
+    await state.clear()
+    await db.settings.set("tz", name)
+    text, markup = await settings_view()
+    await show(message, text, markup)
+
+
+@router.callback_query(F.data == "set:digest")
+async def settings_digest(callback: types.CallbackQuery) -> None:
+    kb = InlineKeyboardBuilder()
+    for hour in DIGEST_HOURS:
+        kb.button(text=f"{hour:02d}:00", callback_data=f"set:dg:{hour}")
+    kb.button(text="Выключить", callback_data="set:dg:off")
+    kb.button(text="К настройкам", callback_data="set:main")
+    kb.adjust(4, 3, 2)
+    await show(
+        callback,
+        "<b>Сводка за сутки</b>\n"
+        "Раз в день: аптайм и падения сервисов, операции и алерты флота.\n"
+        f"Время по поясу {esc(db.settings.get('tz'))}.",
+        kb.as_markup(),
+    )
+    await callback.answer()
+
+
+@router.callback_query(F.data.startswith("set:dg:"))
+async def settings_digest_set(callback: types.CallbackQuery) -> None:
+    value = tail_arg(callback.data)
+    await db.settings.set("digest", None if value == "off" else int_arg(callback.data))
+    text, markup = await settings_view()
+    await show(callback, text, markup)
+    await callback.answer()

src/healthbot/keyboards/__init__.py

@@ -1,69 +1,111 @@
 from aiogram.types import InlineKeyboardButton, InlineKeyboardMarkup
 from aiogram.utils.keyboard import InlineKeyboardBuilder
 
-from utils.db.models import Service
+from healthbot.ui import DANGER, PRIMARY, label, pager, pages
+from utils.db.models import Pin, Service
 from utils.env import env
 
+SERVICES_PAGE = 12
 
-def main_menu() -> InlineKeyboardMarkup:
+
+def main_menu(pins: list[Pin], *, actions: bool) -> InlineKeyboardMarkup:
     kb = InlineKeyboardBuilder()
-    kb.button(text="➕ Добавить сервис", callback_data="menu:add")
+    rows: list[int] = []
-    kb.button(text="📋 Сервисы", callback_data="menu:list")
+    for pin in pins:
-    kb.button(text="📊 Статистика", callback_data="menu:stats")
+        kb.button(
+            text=label(pin.label, 30), callback_data=f"pin:run:{pin.id}", style=PRIMARY
+        )
+    rows += [2] * (len(pins) // 2) + [1] * (len(pins) % 2)
+
+    kb.button(text="Сервисы", callback_data="menu:list:0")
     if env.komodo.enabled:
-        kb.button(text="🛰 Флот", callback_data="fleet:main")
+        kb.button(text="Флот", callback_data="fleet:main")
-        kb.adjust(1, 2, 1)
+        if actions:
+            kb.button(text="Действия", callback_data="act:list:a:0")
+            kb.button(text="Статистика", callback_data="menu:stats")
+            rows += [2, 2]
         else:
-        kb.adjust(1, 2)
+            kb.button(text="Статистика", callback_data="menu:stats")
+            rows += [2, 1]
+    else:
+        kb.button(text="Статистика", callback_data="menu:stats")
+        rows += [2]
+    kb.button(text="Добавить сервис", callback_data="menu:add")
+    kb.button(text="Настройки", callback_data="set:main")
+    kb.button(text="Обновить", callback_data="menu:main")
+    kb.adjust(*rows, 2, 1)
     return kb.as_markup()
 
 
-def services_list(services: list[Service]) -> InlineKeyboardMarkup:
+def _service_order(service: Service) -> int:
+    if service.is_paused:
+        return 2
+    return 1 if service.is_up else 0
+
+
+def services_list(services: list[Service], page: int) -> InlineKeyboardMarkup:
+    ordered = sorted(services, key=_service_order)
+    total = pages(len(ordered), SERVICES_PAGE)
+    page = min(page, total - 1)
+    chunk = ordered[page * SERVICES_PAGE : (page + 1) * SERVICES_PAGE]
     kb = InlineKeyboardBuilder()
-    for svc in services:
+    for svc in chunk:
-        kb.button(text=f"{svc.mark} {svc.name}", callback_data=f"svc:open:{svc.id}")
+        speed = (
-    kb.button(text="➕ Добавить", callback_data="menu:add")
+            f" · {svc.last_latency_ms} мс"
-    kb.button(text="⬅️ Меню", callback_data="menu:main")
+            if svc.is_up and not svc.is_paused and svc.last_latency_ms is not None
-    kb.adjust(1)
+            else ""
+        )
+        kb.button(
+            text=f"{svc.mark} {label(svc.name, 36)}{speed}",
+            callback_data=f"svc:open:{svc.id}",
+        )
+    nav = pager(kb, "menu:list", page, total)
+    kb.button(text="Добавить", callback_data="menu:add")
+    kb.button(text="Меню", callback_data="menu:main")
+    kb.adjust(*[1] * len(chunk), *([nav] if nav else []), 2)
     return kb.as_markup()
 
 
 def service_card(service: Service) -> InlineKeyboardMarkup:
     kb = InlineKeyboardBuilder()
-    kb.button(text="🔍 Проверить сейчас", callback_data=f"svc:check:{service.id}")
+    kb.button(text="Проверить", callback_data=f"svc:check:{service.id}", style=PRIMARY)
+    kb.button(text="Инциденты", callback_data=f"svc:inc:{service.id}")
     if service.is_paused:
-        kb.button(text="▶️ Возобновить", callback_data=f"svc:resume:{service.id}")
+        kb.button(text="Возобновить", callback_data=f"svc:resume:{service.id}")
     else:
-        kb.button(text="⏸ Пауза", callback_data=f"svc:pause:{service.id}")
+        kb.button(text="Пауза", callback_data=f"svc:pause:{service.id}")
-    kb.button(text="✏️ Переименовать", callback_data=f"svc:rename:{service.id}")
+    kb.button(text="Правило", callback_data=f"svc:rule:{service.id}")
-    kb.button(text="🗑 Удалить", callback_data=f"svc:del:{service.id}")
+    kb.button(text="Имя", callback_data=f"svc:rename:{service.id}")
-    kb.button(text="⬅️ К списку", callback_data="menu:list")
+    kb.button(text="Адрес", callback_data=f"svc:url:{service.id}")
-    kb.adjust(1, 2, 2)
+    kb.button(text="Удалить", callback_data=f"svc:del:{service.id}")
+    kb.button(text="К списку", callback_data="menu:list:0")
+    kb.adjust(2)
     return kb.as_markup()
 
 
 def confirm_delete(service_id: int) -> InlineKeyboardMarkup:
     kb = InlineKeyboardBuilder()
-    kb.button(text="✅ Да, удалить", callback_data=f"svc:delyes:{service_id}")
+    kb.button(text="Удалить", callback_data=f"svc:delyes:{service_id}", style=DANGER)
-    kb.button(text="⬅️ Отмена", callback_data=f"svc:open:{service_id}")
+    kb.button(text="Отмена", callback_data=f"svc:open:{service_id}")
     kb.adjust(2)
     return kb.as_markup()
 
 
 def after_check(service_id: int, *, offer_baseline: bool) -> InlineKeyboardMarkup:
-    """Клавиатура под результатом ручной проверки."""
     kb = InlineKeyboardBuilder()
     if offer_baseline:
-        kb.button(text="📌 Принять как эталон", callback_data=f"svc:base:{service_id}")
+        kb.button(text="Принять как эталон", callback_data=f"svc:base:{service_id}")
-    kb.button(text="🔄 Ещё раз", callback_data=f"svc:check:{service_id}")
+    kb.button(text="Ещё раз", callback_data=f"svc:check:{service_id}")
-    kb.button(text="⬅️ К сервису", callback_data=f"svc:open:{service_id}")
+    kb.button(text="К сервису", callback_data=f"svc:open:{service_id}")
-    kb.adjust(1, 2)
+    kb.adjust(*([1] if offer_baseline else []), 2)
     return kb.as_markup()
 
 
-def back_to_menu() -> InlineKeyboardMarkup:
+def back_to(text: str, data: str) -> InlineKeyboardMarkup:
     return InlineKeyboardMarkup(
-        inline_keyboard=[
+        inline_keyboard=[[InlineKeyboardButton(text=text, callback_data=data)]]
-            [InlineKeyboardButton(text="⬅️ Меню", callback_data="menu:main")]
-        ]
     )
+
+
+def back_to_menu() -> InlineKeyboardMarkup:
+    return back_to("Меню", "menu:main")

src/healthbot/komodo.py

@@ -1,7 +1,8 @@
-"""Тонкий клиент Komodo: посмотреть флот и нажать кнопку."""
+import asyncio
-
+import json
 import re
-from dataclasses import dataclass
+import time
+from dataclasses import dataclass, field
 from typing import Any
 
 import aiohttp
@@ -10,27 +11,123 @@ from utils.env import env
 from utils.logging import logger
 
 ANSI_RE = re.compile(r"\x1b\[[0-9;]*[a-zA-Z]")
+TAG_RE = re.compile(r"</?span[^>]*>")
+SECRET_RE = re.compile(
+    r"\b(key|secret|token|password)(['\"]?\s*[:=]\s*)'[^']*'", re.IGNORECASE
+)
+SYSTEM_USER_RE = re.compile(r"[0-9a-f]{24}")
 
 OK = 200
+CACHE_TTL = 8
+NAMES_TTL = 60
+BUSY_PAUSES = (1, 2, 4)
+POLL_FAST = 1
+POLL_SLOW = 3
+POLL_FAST_FOR = 15
+WAIT_FAILURES = 5
+
+STACK_MARKS = {
+    "running": "🟢",
+    "deploying": "🔄",
+    "restarting": "🔄",
+    "unhealthy": "🟠",
+    "paused": "⏸",
+    "created": "⚪️",
+    "down": "⚫️",
+    "stopped": "🔴",
+    "dead": "🔴",
+    "removing": "🔴",
+    "unknown": "⚪️",
+}
+IDLE_STATES = frozenset({"running", "down", "unknown"})
+SERVER_MARKS = {"Ok": "🟢", "NotOk": "🔴", "Disabled": "⏸"}
+RUN_MARKS = {"Ok": "🟢", "Failed": "🔴", "Running": "🔄"}
+
+KIND_API = {"a": ("Action", "action"), "p": ("Procedure", "procedure")}
 
 
 def strip_ansi(text: str) -> str:
     return ANSI_RE.sub("", text)
 
 
+def clean_output(text: str) -> str:
+    text = TAG_RE.sub("", strip_ansi(text))
+    return SECRET_RE.sub(r"\1\2'***'", text).strip()
+
+
+def oid(raw: dict) -> str:
+    value = raw.get("id") or raw.get("_id") or ""
+    if isinstance(value, dict):
+        return str(value.get("$oid", ""))
+    return str(value)
+
+
 class KomodoError(RuntimeError):
     pass
 
 
+def _error_text(status: int, body: str) -> str:
+    try:
+        data = json.loads(body)
+    except ValueError:
+        return f"HTTP {status}: {body[:300]}".strip()
+    if not isinstance(data, dict):
+        return f"HTTP {status}: {body[:300]}"
+    message = str(data.get("error") or body[:300])
+    trace = [str(item) for item in data.get("trace") or [] if item]
+    return message + (f" ({'; '.join(trace)})" if trace else "")
+
+
+def _percent(used: float | None, total: float | None) -> float | None:
+    if not used or not total:
+        return None
+    return used / total * 100
+
+
 @dataclass(slots=True)
 class Server:
     id: str
     name: str
     state: str
+    version: str = ""
+    cpu: float | None = None
+    load: float | None = None
+    mem_used: float | None = None
+    mem_total: float | None = None
+    disk_used: float | None = None
+    disk_total: float | None = None
+    thresholds: dict = field(default_factory=dict)
 
     @property
     def mark(self) -> str:
-        return {"Ok": "🟢", "NotOk": "🔴", "Disabled": "⏸"}.get(self.state, "⚪️")
+        if self.state == "Ok" and self.strained:
+            return "🟠"
+        return SERVER_MARKS.get(self.state, "⚪️")
+
+    @property
+    def mem(self) -> float | None:
+        return _percent(self.mem_used, self.mem_total)
+
+    @property
+    def disk(self) -> float | None:
+        return _percent(self.disk_used, self.disk_total)
+
+    @property
+    def strains(self) -> list[str]:
+        found = []
+        for label, value, key in (
+            ("cpu", self.cpu, "cpu_warning"),
+            ("память", self.mem, "mem_warning"),
+            ("диск", self.disk, "disk_warning"),
+        ):
+            limit = self.thresholds.get(key)
+            if value is not None and limit and value >= limit:
+                found.append(f"{label} {value:.0f}%")
+        return found
+
+    @property
+    def strained(self) -> bool:
+        return bool(self.strains)
 
 
 @dataclass(slots=True)
@@ -38,52 +135,208 @@ class Stack:
     id: str
     name: str
     state: str
+    status: str
     server_id: str
+    server_name: str
+    repo: str = ""
+    branch: str = ""
+    deployed_hash: str = ""
+    latest_hash: str = ""
+    image_updates: int = 0
+    services: list[str] = field(default_factory=list)
 
     @property
     def mark(self) -> str:
-        return {
+        return STACK_MARKS.get(self.state, "⚪️")
-            "running": "🟢",
+
-            "unhealthy": "🟠",
+    @property
-            "paused": "⏸",
+    def healthy(self) -> bool:
-            "down": "⚫️",
+        return self.state == "running"
-            "stopped": "🔴",
+
-            "restarting": "🔄",
+    @property
-            "unknown": "⚪️",
+    def troubled(self) -> bool:
-        }.get(self.state, "⚪️")
+        return self.state not in IDLE_STATES
+
+    @property
+    def behind(self) -> bool:
+        return bool(
+            self.deployed_hash
+            and self.latest_hash
+            and self.deployed_hash != self.latest_hash
+        )
+
+
+@dataclass(slots=True)
+class Container:
+    service: str
+    state: str
+    status: str
+    image: str
+    cpu: str = ""
+    mem: str = ""
+
+    @property
+    def mark(self) -> str:
+        return STACK_MARKS.get(self.state.lower(), "⚪️")
+
+
+@dataclass(slots=True)
+class Resource:
+    kind: str
+    id: str
+    name: str
+    state: str
+    last_run_at: int | None = None
+    next_run_at: int | None = None
+    schedule_error: str | None = None
+
+    @property
+    def mark(self) -> str:
+        return RUN_MARKS.get(self.state, "⚪️")
+
+
+@dataclass(slots=True)
+class Detail:
+    kind: str
+    id: str
+    name: str
+    description: str = ""
+    args_format: str = ""
+    args_text: str = ""
+    schedule: str = ""
+    schedule_enabled: bool = False
+    stages: list[str] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class Stage:
+    name: str
+    success: bool
+    output: str
+
+
+@dataclass(slots=True)
+class Update:
+    id: str
+    operation: str
+    status: str
+    success: bool
+    start_ts: int
+    end_ts: int | None
+    target_type: str
+    target_id: str
+    operator: str
+    stages: list[Stage] = field(default_factory=list)
+
+    @property
+    def done(self) -> bool:
+        return self.status == "Complete"
+
+    @property
+    def seconds(self) -> float:
+        end = self.end_ts or int(time.time() * 1000)
+        return max(0.0, (end - self.start_ts) / 1000)
+
+
+def _server(raw: dict) -> Server:
+    info = raw.get("info") or {}
+    stats = info.get("stats") or {}
+    load = (stats.get("load_average") or {}).get("one")
+    return Server(
+        id=raw["id"],
+        name=raw["name"],
+        state=info.get("state", ""),
+        version=info.get("version") or "",
+        cpu=stats.get("cpu_perc"),
+        load=load,
+        mem_used=stats.get("mem_used_gb"),
+        mem_total=stats.get("mem_total_gb"),
+        disk_used=stats.get("disk_used_gb"),
+        disk_total=stats.get("disk_total_gb"),
+        thresholds=info.get("alerting_thresholds") or {},
+    )
+
+
+def _stack(raw: dict) -> Stack:
+    info = raw.get("info") or {}
+    services = info.get("services") or []
+    return Stack(
+        id=raw["id"],
+        name=raw["name"],
+        state=info.get("state", ""),
+        status=info.get("status") or "",
+        server_id=info.get("server_id", ""),
+        server_name=info.get("server_name") or "",
+        repo=info.get("repo") or "",
+        branch=info.get("branch") or "",
+        deployed_hash=info.get("deployed_hash") or "",
+        latest_hash=info.get("latest_hash") or "",
+        image_updates=sum(bool(s.get("update_available")) for s in services),
+        services=[s.get("service", "") for s in services],
+    )
+
+
+def _resource(kind: str, raw: dict) -> Resource:
+    info = raw.get("info") or {}
+    return Resource(
+        kind=kind,
+        id=raw["id"],
+        name=raw["name"],
+        state=info.get("state", ""),
+        last_run_at=info.get("last_run_at"),
+        next_run_at=info.get("next_scheduled_run"),
+        schedule_error=info.get("schedule_error"),
+    )
+
+
+def _update(raw: dict) -> Update:
+    target = raw.get("target") or {}
+    return Update(
+        id=oid(raw),
+        operation=raw.get("operation", ""),
+        status=raw.get("status", ""),
+        success=bool(raw.get("success")),
+        start_ts=raw.get("start_ts") or 0,
+        end_ts=raw.get("end_ts"),
+        target_type=target.get("type", ""),
+        target_id=target.get("id", ""),
+        operator=raw.get("username") or raw.get("operator") or "",
+        stages=[
+            Stage(
+                name=log.get("stage", ""),
+                success=bool(log.get("success")),
+                output=(log.get("stdout") or "") + "\n" + (log.get("stderr") or ""),
+            )
+            for log in raw.get("logs") or []
+        ],
+    )
 
 
 class Komodo:
     def __init__(self) -> None:
         self._session: aiohttp.ClientSession | None = None
+        self._cache: dict[str, tuple[float, Any]] = {}
+        self._usernames: dict[str, str] = {}
 
     @property
     def enabled(self) -> bool:
         return env.komodo.enabled
 
-    async def _call(self, route: str, kind: str, params: dict | None = None) -> Any:  # noqa: ANN401
+    async def _post(self, route: str, kind: str, params: dict) -> tuple[int, str]:
-        if not self.enabled:
-            msg = "Komodo не настроен (KOMODO__URL / KOMODO__KEY / KOMODO__SECRET)"
-            raise KomodoError(msg)
         if self._session is None or self._session.closed:
             self._session = aiohttp.ClientSession()
         url = env.komodo.url.rstrip("/") + route
         try:
             async with self._session.post(
                 url,
-                json={"type": kind, "params": params or {}},
+                json={"type": kind, "params": params},
                 headers={
                     "X-Api-Key": env.komodo.key.get_secret_value(),
                     "X-Api-Secret": env.komodo.secret.get_secret_value(),
                 },
                 timeout=aiohttp.ClientTimeout(total=env.komodo.timeout),
             ) as resp:
-                text = await resp.text()
+                return resp.status, await resp.text()
-                if resp.status != OK:
-                    logger.warning(f"Komodo {kind} -> HTTP {resp.status}: {text[:300]}")
-                    msg = f"Komodo ответил HTTP {resp.status}"
-                    raise KomodoError(msg)
-                return await resp.json()
         except TimeoutError as exc:
             msg = f"Komodo не ответил за {env.komodo.timeout}с"
             raise KomodoError(msg) from exc
@@ -91,38 +344,194 @@ class Komodo:
             msg = f"Нет связи с Komodo: {exc}"
             raise KomodoError(msg) from exc
 
-    async def read(self, kind: str, params: dict | None = None) -> Any:  # noqa: ANN401
+    async def _call(self, route: str, kind: str, params: dict | None = None) -> Any:  # noqa: ANN401
-        return await self._call("/read", kind, params)
+        if not self.enabled:
+            msg = "Komodo не настроен: нужны KOMODO__URL, KOMODO__KEY, KOMODO__SECRET"
+            raise KomodoError(msg)
+        status, body = 0, ""
+        pauses = BUSY_PAUSES if route == "/read" else ()
+        for pause in (*pauses, None):
+            status, body = await self._post(route, kind, params or {})
+            if status == OK:
+                return json.loads(body)
+            if pause is None or "busy" not in body.lower():
+                break
+            await asyncio.sleep(pause)
+        logger.warning(f"Komodo {kind} -> HTTP {status}: {body[:300]}")
+        raise KomodoError(_error_text(status, body))
 
-    async def execute(self, kind: str, params: dict | None = None) -> Any:  # noqa: ANN401
+    async def read(
-        return await self._call("/execute", kind, params)
+        self, kind: str, params: dict | None = None, *, ttl: float = 0
+    ) -> Any:  # noqa: ANN401
+        key = kind + json.dumps(params or {}, sort_keys=True)
+        cached = self._cache.get(key)
+        if ttl and cached and time.monotonic() - cached[0] < ttl:
+            return cached[1]
+        data = await self._call("/read", kind, params)
+        if ttl:
+            self._cache[key] = (time.monotonic(), data)
+        return data
+
+    async def execute(self, kind: str, params: dict | None = None) -> Update:
+        raw = await self._call("/execute", kind, params)
+        self._cache.clear()
+        return _update(raw if isinstance(raw, dict) else {})
+
+    def forget(self) -> None:
+        self._cache.clear()
+
+    async def _list(self, kind: str, ttl: float = CACHE_TTL) -> list[dict]:
+        return await self.read(kind, {"limit": 0}, ttl=ttl)
+
+    async def version(self) -> str:
+        return (await self.read("GetVersion", ttl=NAMES_TTL)).get("version", "")
 
     async def servers(self) -> list[Server]:
-        raw = await self.read("ListServers")
+        return [_server(s) for s in await self._list("ListServers")]
-        return [
-            Server(id=s["id"], name=s["name"], state=s.get("info", {}).get("state", ""))
-            for s in raw
-        ]
 
     async def stacks(self) -> list[Stack]:
-        raw = await self.read("ListStacks")
+        return [_stack(s) for s in await self._list("ListStacks")]
-        return [
-            Stack(
-                id=s["id"],
-                name=s["name"],
-                state=s.get("info", {}).get("state", ""),
-                server_id=s.get("info", {}).get("server_id", ""),
-            )
-            for s in raw
-        ]
 
-    async def stack_log(self, stack: str, tail: int = 40) -> str:
+    async def stack(self, stack_id: str) -> Stack | None:
-        """Хвост лога стека одной строкой."""
+        return next((s for s in await self.stacks() if s.id == stack_id), None)
+
+    async def containers(self, stack_id: str) -> list[Container]:
+        raw = await self.read("ListStackServices", {"stack": stack_id})
+        result = []
+        for item in raw:
+            container = item.get("container") or {}
+            stats = container.get("stats") or {}
+            result.append(
+                Container(
+                    service=item.get("service", ""),
+                    state=container.get("state") or "down",
+                    status=container.get("status") or "",
+                    image=item.get("image") or container.get("image") or "",
+                    cpu=stats.get("cpu_perc") or "",
+                    mem=(stats.get("mem_usage") or "").split(" / ")[0],
+                )
+            )
+        return result
+
+    async def disks(self, server_id: str) -> list[tuple[str, float, float]]:
+        raw = await self.read("GetSystemStats", {"server": server_id}, ttl=CACHE_TTL)
+        seen: set[tuple[float, float]] = set()
+        result = []
+        for disk in raw.get("disks") or []:
+            used, total = disk.get("used_gb") or 0.0, disk.get("total_gb") or 0.0
+            if not total or (used, total) in seen:
+                continue
+            seen.add((used, total))
+            result.append((disk.get("mount", "?"), used, total))
+        return result
+
+    async def stack_log(self, stack: str, services: list[str], tail: int) -> str:
         raw = await self.read(
-            "GetStackLog", {"stack": stack, "services": [], "tail": tail}
+            "GetStackLog", {"stack": stack, "services": services, "tail": tail}
         )
         out = (raw.get("stdout") or "") + (raw.get("stderr") or "")
-        return strip_ansi(out).strip() or "(пусто)"
+        return strip_ansi(out).strip()
+
+    async def resources(self, kind: str) -> list[Resource]:
+        api, _ = KIND_API[kind]
+        items = [_resource(kind, r) for r in await self._list(f"List{api}s")]
+        return sorted(items, key=lambda r: r.name.lower())
+
+    async def detail(self, kind: str, target_id: str) -> Detail:
+        api, param = KIND_API[kind]
+        raw = await self.read(f"Get{api}", {param: target_id}, ttl=CACHE_TTL)
+        config = raw.get("config") or {}
+        return Detail(
+            kind=kind,
+            id=oid(raw),
+            name=raw.get("name", ""),
+            description=raw.get("description") or "",
+            args_format=config.get("arguments_format") or "",
+            args_text=config.get("arguments") or "",
+            schedule=config.get("schedule") or "",
+            schedule_enabled=bool(config.get("schedule_enabled")),
+            stages=[
+                stage.get("name", "")
+                for stage in config.get("stages") or []
+                if stage.get("enabled", True)
+            ],
+        )
+
+    async def run(self, kind: str, target_id: str, args: dict | None) -> Update:
+        api, param = KIND_API[kind]
+        params: dict[str, Any] = {param: target_id}
+        if kind == "a" and args:
+            params["args"] = args
+        return await self.execute(f"Run{api}", params)
+
+    async def update(self, update_id: str) -> Update:
+        return _update(await self.read("GetUpdate", {"id": update_id}))
+
+    async def updates(self, query: dict | None = None, page: int = 0) -> list[Update]:
+        raw = await self.read("ListUpdates", {"query": query or {}, "page": page})
+        return [_update(u) for u in raw.get("updates") or []]
+
+    async def open_alerts(self) -> list[dict]:
+        try:
+            raw = await self.read(
+                "ListAlerts", {"query": {"resolved": False}}, ttl=CACHE_TTL
+            )
+        except KomodoError:
+            return []
+        return raw.get("alerts") or []
+
+    async def wait(self, update_id: str, timeout: float) -> Update | None:  # noqa: ASYNC109
+        started = time.monotonic()
+        failures = 0
+        while True:
+            elapsed = time.monotonic() - started
+            try:
+                update = await self.update(update_id)
+                failures = 0
+            except KomodoError:
+                failures += 1
+                if failures >= WAIT_FAILURES:
+                    raise
+                update = None
+            if update is not None and update.done:
+                self._cache.clear()
+                return update
+            if elapsed >= timeout:
+                return None
+            await asyncio.sleep(POLL_FAST if elapsed < POLL_FAST_FOR else POLL_SLOW)
+
+    async def names(self) -> dict[str, str]:
+        kinds = (
+            "ListServers",
+            "ListStacks",
+            "ListActions",
+            "ListProcedures",
+            "ListResourceSyncs",
+            "ListRepos",
+            "ListDeployments",
+            "ListBuilds",
+            "ListAlerters",
+        )
+        lists = await asyncio.gather(
+            *(self._list(kind, ttl=NAMES_TTL) for kind in kinds), return_exceptions=True
+        )
+        result: dict[str, str] = {}
+        for items in lists:
+            if isinstance(items, BaseException):
+                continue
+            result.update({item["id"]: item["name"] for item in items})
+        return result
+
+    async def username(self, user: str) -> str:
+        if not SYSTEM_USER_RE.fullmatch(user):
+            return user
+        if user not in self._usernames:
+            try:
+                raw = await self.read("GetUsername", {"user_id": user})
+                self._usernames[user] = raw.get("username") or user
+            except KomodoError:
+                self._usernames[user] = "Komodo"
+        return self._usernames[user]
 
     async def close(self) -> None:
         if self._session is not None and not self._session.closed:

src/healthbot/monitor.py

@@ -1,24 +1,47 @@
-"""Фоновый цикл: пингует сервисы и уведомляет о падениях/восстановлениях."""
-
 import asyncio
-import html
+import time
+from collections.abc import Awaitable
+from datetime import datetime
 
 import aiohttp
+from aiogram.types import InlineKeyboardMarkup
+from aiogram.utils.keyboard import InlineKeyboardBuilder
 
-from healthbot.checker import Verdict, evaluate, probe
+from healthbot.checker import Verdict, evaluate, probe, tls_expiry
 from healthbot.notify import notify
+from healthbot.stats import current_hour
+from healthbot.ui import esc
 from utils.db import db
 from utils.db.models import Service
 from utils.env import env
 from utils.format import human_duration, now, parse
 from utils.logging import logger
 
+CONCURRENCY = 10
+KEEP_HOURS = 24 * 40
+TLS_EVERY = 6 * 3600
+TLS_WARN_DAYS = (14, 7, 3, 1)
+DAY = 86400
+
 
 def _head(service: Service, icon: str, tail: str) -> str:
-    return (
+    return f"{icon} <b>{esc(service.name)}</b> {tail}\n<code>{esc(service.url)}</code>"
-        f"{icon} <b>{html.escape(service.name)}</b> {tail}\n"
+
-        f"<code>{html.escape(service.url)}</code>"
+
-    )
+def _down_kb(service: Service, incident_id: int | None) -> InlineKeyboardMarkup:
+    kb = InlineKeyboardBuilder()
+    kb.button(text="Проверить", callback_data=f"go:check:{service.id}")
+    kb.button(text="Открыть", callback_data=f"go:svc:{service.id}")
+    if incident_id is not None:
+        kb.button(text="Не напоминать", callback_data=f"svc:mute:{incident_id}")
+    kb.adjust(2, 1)
+    return kb.as_markup()
+
+
+def _open_kb(service: Service) -> InlineKeyboardMarkup:
+    kb = InlineKeyboardBuilder()
+    kb.button(text="Открыть", callback_data=f"go:svc:{service.id}")
+    return kb.as_markup()
 
 
 async def _on_recovered(service: Service) -> None:
@@ -28,24 +51,27 @@ async def _on_recovered(service: Service) -> None:
         await db.incidents.close(current.id)
         downtime = human_duration((now() - parse(current.started_at)).total_seconds())
     await db.services.set_up(service.id, is_up=True)
-    await notify(f"{_head(service, '✅', 'снова в строю')}\nДаунтайм: {downtime}")
+    await notify(
+        f"{_head(service, '✅', 'снова в строю')}\nДаунтайм: {downtime}",
+        _open_kb(service),
+    )
     logger.info(f"[green]UP[/] {service.name}")
 
 
 async def _on_failed(service: Service, verdict: Verdict) -> None:
-    await db.incidents.open(service.id, verdict.reason)
+    incident = await db.incidents.open(service.id, verdict.reason)
     await db.services.set_up(service.id, is_up=False)
-    lines = [_head(service, "🔴", "лёг"), f"Причина: {html.escape(verdict.reason)}"]
+    lines = [_head(service, "🔴", "лёг"), f"Причина: {esc(verdict.reason)}"]
     report = verdict.report()
     if report:
         lines.append(report)
-    await notify("\n".join(lines))
+    await notify("\n".join(lines), _down_kb(service, incident.id))
     logger.warning(f"[red]DOWN[/] {service.name}: {verdict.reason}")
 
 
 async def _on_still_down(service: Service, verdict: Verdict) -> None:
     current = await db.incidents.current(service.id)
-    if current is None:
+    if current is None or current.muted:
         return
     idle = (now() - parse(current.last_reminder_at)).total_seconds()
     if idle < env.monitor.reminder_interval:
@@ -54,19 +80,24 @@ async def _on_still_down(service: Service, verdict: Verdict) -> None:
     since = human_duration((now() - parse(current.started_at)).total_seconds())
     lines = [
         _head(service, "⏰", f"всё ещё лежит (уже {since})"),
-        f"Причина: {html.escape(verdict.reason)}",
+        f"Причина: {esc(verdict.reason)}",
     ]
     report = verdict.report()
     if report:
         lines.append(report)
-    await notify("\n".join(lines))
+    await notify("\n".join(lines), _down_kb(service, current.id))
 
 
 async def check_one(session: aiohttp.ClientSession, service: Service) -> Verdict:
-    """Одна проверка со всеми побочными эффектами. Возвращает вердикт."""
     result = await probe(session, service.url, env.monitor.request_timeout)
     verdict = evaluate(service, result)
     await db.services.touch_probe(service.id, result.latency_ms)
+    await db.stats.record(
+        service.id,
+        current_hour(),
+        healthy=verdict.healthy,
+        latency_ms=result.latency_ms,
+    )
 
     if verdict.healthy:
         if service.fail_streak:
@@ -91,21 +122,65 @@ async def check_one(session: aiohttp.ClientSession, service: Service) -> Verdict
     return verdict
 
 
+def tls_step(expires: datetime, warned: int) -> int:
+    days = (expires - now()).total_seconds() / DAY
+    due = [limit for limit in TLS_WARN_DAYS if days <= limit]
+    if not due:
+        return 0
+    step = min(due)
+    return step if not warned or step < warned else warned
+
+
+async def check_tls(service: Service) -> None:
+    expires = await tls_expiry(service.url)
+    if expires is None:
+        return
+    step = tls_step(expires, service.tls_warned)
+    await db.services.set_tls(service.id, expires.isoformat(), step)
+    if step and step != service.tls_warned:
+        left = max(0.0, (expires - now()).total_seconds())
+        await notify(
+            f"{_head(service, '🟠', 'сертификат скоро истечёт')}\n"
+            f"Осталось: {human_duration(left)}",
+            _open_kb(service),
+        )
+
+
+async def _guarded(sem: asyncio.Semaphore, job: Awaitable[object], what: str) -> None:
+    async with sem:
+        try:
+            await job
+        except Exception:
+            logger.exception(f"Check failed: {what}")
+
+
+async def _sweep(session: aiohttp.ClientSession, *, with_tls: bool) -> None:
+    services = [s for s in await db.services.all() if not s.is_paused]
+    sem = asyncio.Semaphore(CONCURRENCY)
+    jobs = [_guarded(sem, check_one(session, s), s.url) for s in services]
+    if with_tls:
+        jobs += [_guarded(sem, check_tls(s), f"tls {s.url}") for s in services]
+    await asyncio.gather(*jobs)
+
+
 async def run_monitor() -> None:
     logger.info(
-        f"Monitor: каждые {env.monitor.check_interval}с, "
+        f"Monitor: every {env.monitor.check_interval}s, "
-        f"порог падения {env.monitor.failure_threshold}"
+        f"failure threshold {env.monitor.failure_threshold}"
     )
+    next_tls = 0.0
+    purged_hour = 0
     async with aiohttp.ClientSession() as session:
         while True:
             try:
-                for service in await db.services.all():
+                with_tls = time.monotonic() >= next_tls
-                    if service.is_paused:
+                await _sweep(session, with_tls=with_tls)
-                        continue
+                if with_tls:
-                    try:
+                    next_tls = time.monotonic() + TLS_EVERY
-                        await check_one(session, service)
+                hour = current_hour()
+                if hour != purged_hour:
+                    await db.stats.purge(hour - KEEP_HOURS)
+                    purged_hour = hour
             except Exception:
-                        logger.exception(f"Ошибка проверки {service.url}")
+                logger.exception("Monitor loop failed")
-            except Exception:
-                logger.exception("Ошибка цикла мониторинга")
             await asyncio.sleep(env.monitor.check_interval)

src/healthbot/notify.py

@@ -1,28 +1,15 @@
-"""Отправка уведомлений админам."""
+from aiogram.types import InlineKeyboardMarkup
 
 from healthbot.common import bot
+from healthbot.ui import fit
 from utils.env import env
 from utils.logging import logger
 
-TELEGRAM_LIMIT = 4096
-SAFE_LIMIT = 3900
 
-
+async def notify(text: str, markup: InlineKeyboardMarkup | None = None) -> None:
-def clip(text: str) -> str:
+    payload = fit(text)
-    if len(text) <= TELEGRAM_LIMIT:
-        return text
-    cut = text[:SAFE_LIMIT]
-    cut = cut[: cut.rfind("\n")] if "\n" in cut else cut
-    if cut.count("<pre>") > cut.count("</pre>"):
-        cut += "</pre>"
-    return cut + "\n<i>…обрезано</i>"
-
-
-async def notify(text: str) -> None:
-    """Разослать всем админам. Падение одного не мешает остальным."""
-    payload = clip(text)
     for admin in env.admins:
         try:
-            await bot.send_message(admin, payload)
+            await bot.send_message(admin, payload, reply_markup=markup)
         except Exception:
-            logger.exception(f"Не смог отправить уведомление {admin}")
+            logger.exception(f"Notification to {admin} failed")

src/healthbot/runs.py

@@ -0,0 +1,147 @@
+import contextlib
+from collections.abc import Awaitable, Callable
+
+from aiogram.exceptions import TelegramBadRequest
+from aiogram.types import CallbackQuery, InlineKeyboardMarkup, Message
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+from healthbot.komodo import KomodoError, Update, clean_output, komodo
+from healthbot.ui import edit, esc, pre, show
+from utils.format import clock
+
+RUN_TIMEOUT = 30 * 60
+OUTPUT_LIMIT = 3000
+QUICK_SECONDS = 10
+NOISE = ("Action completed successfully",)
+ACTIVE: set[str] = set()
+
+Back = list[tuple[str, str]]
+
+
+def _stage_text(output: str) -> str:
+    lines = [
+        line
+        for line in clean_output(output).splitlines()
+        if not any(mark in line for mark in NOISE)
+    ]
+    return "\n".join(lines).strip()
+
+
+def output_of(update: Update, *, only_failed: bool) -> str:
+    parts = []
+    for stage in update.stages:
+        if "config" in stage.name.lower():
+            if not stage.success:
+                parts.append(f"{stage.name}: вывод скрыт, в нём бывают секреты")
+            continue
+        if only_failed and stage.success:
+            continue
+        text = _stage_text(stage.output)
+        if not text:
+            continue
+        parts.append(f"{stage.name}:\n{text}" if len(update.stages) > 1 else text)
+    return "\n\n".join(parts)
+
+
+def result_text(title: str, update: Update, *, verbose: bool, extra: str = "") -> str:
+    icon = "✅" if update.success else "🔴"
+    state = "" if update.success else " не удалось"
+    lines = [f"{icon} <b>{esc(title)}</b>{state} · {clock(update.seconds)}"]
+    if extra:
+        lines.append(extra)
+    if verbose or not update.success:
+        output = output_of(update, only_failed=not verbose)
+        if output:
+            lines.append(pre(output, OUTPUT_LIMIT))
+    return "\n".join(lines)
+
+
+def result_kb(update: Update | None, back: Back) -> InlineKeyboardMarkup:
+    kb = InlineKeyboardBuilder()
+    for text, data in back:
+        kb.button(text=text, callback_data=data)
+    rows = [len(back)] if back else []
+    if update is not None and len(output_of(update, only_failed=False)) > OUTPUT_LIMIT:
+        kb.button(text="Весь вывод файлом", callback_data=f"run:file:{update.id}")
+        rows.append(1)
+    kb.adjust(*rows or [1])
+    return kb.as_markup()
+
+
+async def track(  # noqa: PLR0913
+    origin: Message | CallbackQuery,
+    title: str,
+    starter: Callable[[], Awaitable[Update]],
+    *,
+    back: Back,
+    verbose: bool,
+    extra: str = "",
+    new: bool = False,
+) -> None:
+    source = origin.message if isinstance(origin, CallbackQuery) else origin
+    key = f"{source.message_id if source else 0}:{title}"
+    if key in ACTIVE:
+        return
+    ACTIVE.add(key)
+    try:
+        await _track(
+            origin, title, starter, back, verbose=verbose, extra=extra, new=new
+        )
+    finally:
+        ACTIVE.discard(key)
+
+
+async def _track(  # noqa: PLR0913
+    origin: Message | CallbackQuery,
+    title: str,
+    starter: Callable[[], Awaitable[Update]],
+    back: Back,
+    *,
+    verbose: bool,
+    extra: str,
+    new: bool,
+) -> None:
+    progress = await show(origin, f"⏳ <b>{esc(title)}</b> выполняется", new=new)
+    if progress is None:
+        return
+    try:
+        started = await starter()
+    except KomodoError as exc:
+        await edit(
+            progress,
+            f"🔴 <b>{esc(title)}</b> не запустилось\n{esc(exc)}",
+            result_kb(None, back),
+        )
+        return
+
+    if not started.id:
+        await edit(progress, f"✅ <b>{esc(title)}</b> запущено", result_kb(None, back))
+        return
+
+    check = InlineKeyboardBuilder()
+    check.button(text="Проверить", callback_data=f"run:st:{started.id}")
+    try:
+        update = await komodo.wait(started.id, RUN_TIMEOUT)
+    except KomodoError as exc:
+        await edit(
+            progress,
+            f"⏳ <b>{esc(title)}</b> запущено, результат узнать не удалось\n{esc(exc)}",
+            check.as_markup(),
+        )
+        return
+    if update is None:
+        await edit(
+            progress,
+            f"⏳ <b>{esc(title)}</b> идёт дольше {RUN_TIMEOUT // 60} минут",
+            check.as_markup(),
+        )
+        return
+
+    text = result_text(title, update, verbose=verbose, extra=extra)
+    markup = result_kb(update, back)
+    if update.seconds < QUICK_SECONDS:
+        await edit(progress, text, markup)
+        return
+    await show(progress, text, markup)
+    with contextlib.suppress(TelegramBadRequest):
+        await progress.delete()

src/healthbot/stats.py

@@ -0,0 +1,67 @@
+from datetime import datetime, timedelta
+
+from healthbot.ui import pct, spark
+from utils.db.models import Hour, Incident, Service
+from utils.format import now, parse, zone
+
+DAY = 86400
+PERIODS = (("24ч", DAY), ("7д", 7 * DAY), ("30д", 30 * DAY))
+CHART_DAYS = 14
+CHART_HOURS = 24
+BLIP = 300
+
+
+def current_hour() -> int:
+    return int(now().timestamp()) // 3600
+
+
+def downtime(incidents: list[Incident], start: datetime, end: datetime) -> float:
+    total = 0.0
+    for incident in incidents:
+        began = max(parse(incident.started_at), start)
+        ended = min(parse(incident.ended_at) if incident.ended_at else end, end)
+        if ended > began:
+            total += (ended - began).total_seconds()
+    return total
+
+
+def uptime(service: Service, incidents: list[Incident], period: float) -> float | None:
+    end = now()
+    start = max(end - timedelta(seconds=period), parse(service.created_at))
+    window = (end - start).total_seconds()
+    if window <= 0:
+        return None
+    return max(0.0, (window - downtime(incidents, start, end)) / window * 100)
+
+
+def uptime_line(service: Service, incidents: list[Incident]) -> str:
+    return ", ".join(
+        f"{name} {pct(uptime(service, incidents, period))}" for name, period in PERIODS
+    )
+
+
+def day_squares(service: Service, incidents: list[Incident], tz_name: str) -> str:
+    tz = zone(tz_name)
+    today = now().astimezone(tz).replace(hour=0, minute=0, second=0, microsecond=0)
+    created = parse(service.created_at)
+    squares = []
+    for offset in range(CHART_DAYS - 1, -1, -1):
+        start = today - timedelta(days=offset)
+        end = min(start + timedelta(days=1), now())
+        if end <= created:
+            squares.append("⬜")
+            continue
+        lost = downtime(incidents, max(start, created), end)
+        squares.append("🟩" if not lost else "🟨" if lost < BLIP else "🟥")
+    return "".join(squares)
+
+
+def latency_chart(hours: dict[int, Hour]) -> tuple[str, int | None, int | None]:
+    last = current_hour()
+    window = [hours.get(h) for h in range(last - CHART_HOURS + 1, last + 1)]
+    values = [h.latency if h else None for h in window]
+    present = [v for v in values if v is not None]
+    if not present:
+        return "", None, None
+    peak = max(h.latency_max for h in window if h and h.latency_n)
+    return spark(values), sum(present) // len(present), peak

src/healthbot/ui.py

@@ -0,0 +1,153 @@
+import html
+from collections.abc import Sequence
+
+from aiogram import F
+from aiogram.enums import ButtonStyle
+from aiogram.exceptions import TelegramBadRequest
+from aiogram.types import CallbackQuery, InlineKeyboardMarkup, Message
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+TELEGRAM_LIMIT = 4096
+SAFE_LIMIT = 3900
+BUTTON_LIMIT = 48
+SPARK = "▁▂▃▄▅▆▇█"
+FULL = 100
+
+MENU_COMMANDS = ("start", "list", "stats", "fleet", "actions", "settings")
+ANSWER = F.text & ~F.text.regexp(rf"^/({'|'.join(MENU_COMMANDS)})(@\w+)?$")
+
+PRIMARY = ButtonStyle.PRIMARY
+SUCCESS = ButtonStyle.SUCCESS
+DANGER = ButtonStyle.DANGER
+
+
+def esc(value: object) -> str:
+    return html.escape(str(value), quote=False)
+
+
+def fit(text: str) -> str:
+    if len(text) <= TELEGRAM_LIMIT:
+        return text
+    cut = text[:SAFE_LIMIT]
+    cut = cut[: cut.rfind("\n")] if "\n" in cut else cut
+    cut = cut[: cut.rfind("<")] if cut.rfind("<") > cut.rfind(">") else cut
+    for tag in ("code", "pre", "b", "i"):
+        if cut.count(f"<{tag}>") > cut.count(f"</{tag}>"):
+            cut += f"</{tag}>"
+    return cut + "\n<i>обрезано</i>"
+
+
+def tail(text: str, limit: int) -> str:
+    text = text.strip()
+    if len(text) <= limit:
+        return text
+    cut = text[-limit:]
+    return cut[cut.find("\n") + 1 :] if "\n" in cut else cut
+
+
+def pre(text: str, limit: int = 3000) -> str:
+    return f"<pre>{esc(tail(text, limit))}</pre>"
+
+
+def label(text: str, limit: int = BUTTON_LIMIT) -> str:
+    text = " ".join(text.split())
+    return text if len(text) <= limit else text[: limit - 1] + "…"
+
+
+def pct(value: float | None) -> str:
+    if value is None:
+        return "нет данных"
+    if value >= FULL:
+        return "100%"
+    return f"{min(value, 99.99):.2f}%"
+
+
+def spark(values: Sequence[float | None]) -> str:
+    present = [v for v in values if v is not None]
+    if not present:
+        return ""
+    low, high = min(present), max(present)
+    span = high - low
+    chars = []
+    for value in values:
+        if value is None:
+            chars.append("·")
+        elif not span:
+            chars.append(SPARK[2])
+        else:
+            chars.append(SPARK[round((value - low) / span * (len(SPARK) - 1))])
+    return "".join(chars)
+
+
+def plural(count: int, one: str, few: str, many: str) -> str:
+    tens, units = count % 100, count % 10
+    if units == 1 and tens != 11:  # noqa: PLR2004
+        return one
+    if units in {2, 3, 4} and tens not in {12, 13, 14}:
+        return few
+    return many
+
+
+def pages(total: int, size: int) -> int:
+    return max(1, (total + size - 1) // size)
+
+
+def pager(kb: InlineKeyboardBuilder, prefix: str, page: int, total_pages: int) -> int:
+    if total_pages <= 1:
+        return 0
+    count = 0
+    if page > 0:
+        kb.button(text="‹ Назад", callback_data=f"{prefix}:{page - 1}")
+        count += 1
+    if page < total_pages - 1:
+        kb.button(text="Дальше ›", callback_data=f"{prefix}:{page + 1}")
+        count += 1
+    return count
+
+
+async def show(
+    event: Message | CallbackQuery,
+    text: str,
+    markup: InlineKeyboardMarkup | None = None,
+    *,
+    new: bool = False,
+) -> Message | None:
+    text = fit(text)
+    if isinstance(event, Message):
+        return await event.answer(text, reply_markup=markup)
+    message = event.message
+    if isinstance(message, Message) and not new:
+        try:
+            edited = await message.edit_text(text, reply_markup=markup)
+        except TelegramBadRequest as exc:
+            if "not modified" in str(exc):
+                return message
+        else:
+            return edited if isinstance(edited, Message) else message
+    if isinstance(message, Message):
+        return await message.answer(text, reply_markup=markup)
+    if event.bot is not None:
+        return await event.bot.send_message(
+            event.from_user.id, text, reply_markup=markup
+        )
+    return None
+
+
+async def edit(
+    message: Message, text: str, markup: InlineKeyboardMarkup | None = None
+) -> None:
+    text = fit(text)
+    try:
+        await message.edit_text(text, reply_markup=markup)
+    except TelegramBadRequest as exc:
+        if "not modified" not in str(exc):
+            await message.answer(text, reply_markup=markup)
+
+
+def tail_arg(data: str | None) -> str:
+    return (data or "").rsplit(":", 1)[-1]
+
+
+def int_arg(data: str | None, default: int = 0) -> int:
+    value = tail_arg(data)
+    return int(value) if value.lstrip("-").isdigit() else default

src/healthbot/web.py

@@ -1,17 +1,15 @@
-"""HTTP-приёмник входящих алертов."""
-
 import hmac
 import json
 
 from aiohttp import web
 
-from healthbot.alerts import format_alert
+from healthbot.alerts import alert_keyboard, alert_kind, format_alert
 from healthbot.notify import notify
+from healthbot.ui import esc
+from utils.db import db
 from utils.env import env
 from utils.logging import logger
 
-MUTED = frozenset({"ScheduleRun"})
-
 
 def _authorized(request: web.Request) -> bool:
     expected = env.web.token.get_secret_value()
@@ -23,32 +21,34 @@ def _authorized(request: web.Request) -> bool:
 
 async def handle_alert(request: web.Request) -> web.Response:
     if not _authorized(request):
-        logger.warning(f"Отклонён алерт без токена от {request.remote}")
+        logger.warning(f"Alert without a valid token from {request.remote}")
         return web.json_response({"error": "unauthorized"}, status=401)
 
     raw = await request.text()
     try:
         payload = json.loads(raw)
     except ValueError:
-        logger.warning(f"Алерт не разобрался как JSON: {raw[:200]}")
+        logger.warning(f"Alert is not JSON: {raw[:200]}")
         return web.json_response({"error": "bad json"}, status=400)
 
     if not isinstance(payload, dict):
         return web.json_response({"error": "expected object"}, status=400)
 
-    try:
+    kind = alert_kind(payload)
-        text = format_alert(payload)
+    logger.info(f"Alert received: {kind} (level={payload.get('level')})")
-    except Exception:
-        logger.exception("Не смог отформатировать алерт")
-        text = f"🔔 Алерт из Komodo (не разобрал формат)\n<pre>{raw[:1000]}</pre>"
 
-    kind = (payload.get("data") or {}).get("type", "?")
+    if kind in db.settings.get("muted"):
-    logger.info(f"Алерт принят: {kind} (level={payload.get('level')})")
-
-    if kind in MUTED:
         return web.json_response({"ok": True, "muted": True})
 
-    await notify(text)
+    try:
+        text = format_alert(payload)
+        markup = alert_keyboard(payload)
+    except Exception:
+        logger.exception("Alert formatting failed")
+        text = f"🔔 Алерт из Komodo (не разобрал формат)\n<pre>{esc(raw[:1000])}</pre>"
+        markup = None
+
+    await notify(text, markup)
     return web.json_response({"ok": True})
 
 
@@ -65,5 +65,5 @@ async def run_web() -> web.AppRunner:
     await runner.setup()
     site = web.TCPSite(runner, env.web.host, env.web.port)
     await site.start()
-    logger.info(f"Web: слушаю {env.web.host}:{env.web.port}")
+    logger.info(f"Web: listening on {env.web.host}:{env.web.port}")
     return runner

src/utils/db/__init__.py

@@ -1,10 +1,20 @@
+import shutil
 from pathlib import Path
 
 import aiosqlite
 
 from utils.env import env
+from utils.logging import logger
 
-from .repositories import IncidentRepository, ServiceRepository
+from .repositories import (
+    IncidentRepository,
+    PinRepository,
+    ServiceRepository,
+    SettingRepository,
+    StatRepository,
+)
+
+SCHEMA_VERSION = 2
 
 SCHEMA = """
 CREATE TABLE IF NOT EXISTS services (
@@ -30,6 +40,33 @@ CREATE TABLE IF NOT EXISTS incidents (
 );
 
 CREATE INDEX IF NOT EXISTS idx_incidents_service ON incidents(service_id);
+
+CREATE TABLE IF NOT EXISTS settings (
+    key   TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS pins (
+    id        INTEGER PRIMARY KEY AUTOINCREMENT,
+    kind      TEXT    NOT NULL,
+    target_id TEXT    NOT NULL,
+    name      TEXT    NOT NULL,
+    label     TEXT    NOT NULL,
+    mode      TEXT    NOT NULL,
+    args      TEXT    NOT NULL DEFAULT '{}',
+    position  INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS hourly (
+    service_id  INTEGER NOT NULL REFERENCES services(id) ON DELETE CASCADE,
+    hour        INTEGER NOT NULL,
+    checks      INTEGER NOT NULL DEFAULT 0,
+    fails       INTEGER NOT NULL DEFAULT 0,
+    latency_sum INTEGER NOT NULL DEFAULT 0,
+    latency_n   INTEGER NOT NULL DEFAULT 0,
+    latency_max INTEGER NOT NULL DEFAULT 0,
+    PRIMARY KEY (service_id, hour)
+);
 """
 
 MIGRATIONS: tuple[tuple[str, str, str], ...] = (
@@ -37,26 +74,66 @@ MIGRATIONS: tuple[tuple[str, str, str], ...] = (
     ("services", "is_paused", "INTEGER NOT NULL DEFAULT 0"),
     ("services", "last_latency_ms", "INTEGER"),
     ("services", "last_checked_at", "TEXT"),
+    ("services", "needle", "TEXT"),
+    ("services", "tls_expires_at", "TEXT"),
+    ("services", "tls_warned", "INTEGER NOT NULL DEFAULT 0"),
+    ("incidents", "muted", "INTEGER NOT NULL DEFAULT 0"),
 )
 
 
+def _prepare(path: str) -> None:
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+
+
+def _has_data(path: str) -> bool:
+    source = Path(path)
+    return source.is_file() and source.stat().st_size > 0
+
+
+def _copy(path: str, version: int) -> Path:
+    source = Path(path)
+    target = source.with_name(f"{source.name}.v{version}.bak")
+    shutil.copy2(source, target)
+    return target
+
+
 class Database:
     def __init__(self, path: str) -> None:
         self._path = path
         self._conn: aiosqlite.Connection | None = None
         self.services: ServiceRepository = None  # type: ignore[assignment]
         self.incidents: IncidentRepository = None  # type: ignore[assignment]
+        self.pins: PinRepository = None  # type: ignore[assignment]
+        self.settings: SettingRepository = None  # type: ignore[assignment]
+        self.stats: StatRepository = None  # type: ignore[assignment]
 
     async def connect(self) -> None:
-        Path(self._path).parent.mkdir(parents=True, exist_ok=True)
+        _prepare(self._path)
+        await self._backup_before_migration()
         self._conn = await aiosqlite.connect(self._path)
         self._conn.row_factory = aiosqlite.Row
         await self._conn.execute("PRAGMA foreign_keys = ON")
         await self._conn.executescript(SCHEMA)
         await self._migrate()
+        await self._conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
         await self._conn.commit()
         self.services = ServiceRepository(self._conn)
         self.incidents = IncidentRepository(self._conn)
+        self.pins = PinRepository(self._conn)
+        self.settings = SettingRepository(self._conn)
+        self.stats = StatRepository(self._conn)
+        await self.settings.load()
+
+    async def _backup_before_migration(self) -> None:
+        if not _has_data(self._path):
+            return
+        async with aiosqlite.connect(self._path) as conn:
+            cursor = await conn.execute("PRAGMA user_version")
+            row = await cursor.fetchone()
+        version = row[0] if row else 0
+        if version < SCHEMA_VERSION:
+            target = _copy(self._path, version)
+            logger.info(f"Database copied to {target} before schema {SCHEMA_VERSION}")
 
     async def _migrate(self) -> None:
         assert self._conn is not None

src/utils/db/models.py

@@ -1,11 +1,18 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import StrEnum
+from html import escape
 
 
 class MatchType(StrEnum):
     STATUS = "status"
     BODY = "body"
     JSON = "json"
+    CONTAINS = "contains"
+
+
+class PinMode(StrEnum):
+    INSTANT = "instant"
+    FORM = "form"
 
 
 @dataclass(slots=True)
@@ -24,14 +31,23 @@ class Service:
     is_paused: bool = False
     last_latency_ms: int | None = None
     last_checked_at: str | None = None
+    needle: str | None = None
+    tls_expires_at: str | None = None
+    tls_warned: int = 0
 
     @property
     def rule(self) -> str:
         if self.match_type is MatchType.STATUS:
-            return f"HTTP-статус = {self.ok_status}"
+            return f"HTTP-статус {self.ok_status}"
         if self.match_type is MatchType.JSON:
-            return f"<code>{self.json_path}</code> = <code>{self.json_value}</code>"
+            path = escape(self.json_path or "", quote=False)
-        return "тело ответа неизменно"
+            value = escape(self.json_value or "", quote=False)
+            return f"<code>{path}</code> = <code>{value}</code>"
+        if self.match_type is MatchType.CONTAINS:
+            return (
+                f"в ответе есть <code>{escape(self.needle or '', quote=False)}</code>"
+            )
+        return "тело ответа не меняется"
 
     @property
     def mark(self) -> str:
@@ -48,3 +64,30 @@ class Incident:
     ended_at: str | None
     reason: str
     last_reminder_at: str
+    muted: bool = False
+
+
+@dataclass(slots=True)
+class Pin:
+    id: int
+    kind: str
+    target_id: str
+    name: str
+    label: str
+    mode: PinMode
+    args: dict = field(default_factory=dict)
+    position: int = 0
+
+
+@dataclass(slots=True)
+class Hour:
+    hour: int
+    checks: int
+    fails: int
+    latency_sum: int
+    latency_n: int
+    latency_max: int
+
+    @property
+    def latency(self) -> int | None:
+        return self.latency_sum // self.latency_n if self.latency_n else None

src/utils/db/repositories/__init__.py

@@ -1,4 +1,13 @@
 from .incident import IncidentRepository
+from .pin import PinRepository
 from .service import ServiceRepository
+from .setting import SettingRepository
+from .stat import StatRepository
 
-__all__ = ["IncidentRepository", "ServiceRepository"]
+__all__ = [
+    "IncidentRepository",
+    "PinRepository",
+    "ServiceRepository",
+    "SettingRepository",
+    "StatRepository",
+]

src/utils/db/repositories/incident.py

@@ -3,7 +3,7 @@ import aiosqlite
 from utils.db.models import Incident
 from utils.format import now
 
-_COLUMNS = "id, service_id, started_at, ended_at, reason, last_reminder_at"
+_COLUMNS = "id, service_id, started_at, ended_at, reason, last_reminder_at, muted"
 
 
 def _row_to_incident(row: aiosqlite.Row) -> Incident:
@@ -14,6 +14,7 @@ def _row_to_incident(row: aiosqlite.Row) -> Incident:
         ended_at=row["ended_at"],
         reason=row["reason"],
         last_reminder_at=row["last_reminder_at"],
+        muted=bool(row["muted"]),
     )
 
 
@@ -29,7 +30,7 @@ class IncidentRepository:
             (service_id, stamp, reason, stamp),
         )
         await self._conn.commit()
-        incident = await self.get(cursor.lastrowid)  # type: ignore[arg-type]
+        incident = await self.get(cursor.lastrowid or 0)
         assert incident is not None
         return incident
 
@@ -65,6 +66,12 @@ class IncidentRepository:
         )
         await self._conn.commit()
 
+    async def mute(self, incident_id: int) -> None:
+        await self._conn.execute(
+            "UPDATE incidents SET muted = 1 WHERE id = ?", (incident_id,)
+        )
+        await self._conn.commit()
+
     async def for_service(self, service_id: int) -> list[Incident]:
         cursor = await self._conn.execute(
             f"SELECT {_COLUMNS} FROM incidents "  # noqa: S608
@@ -72,3 +79,11 @@ class IncidentRepository:
             (service_id,),
         )
         return [_row_to_incident(row) for row in await cursor.fetchall()]
+
+    async def since(self, stamp: str) -> list[Incident]:
+        cursor = await self._conn.execute(
+            f"SELECT {_COLUMNS} FROM incidents "  # noqa: S608
+            "WHERE ended_at IS NULL OR ended_at >= ? ORDER BY id",
+            (stamp,),
+        )
+        return [_row_to_incident(row) for row in await cursor.fetchall()]

src/utils/db/repositories/pin.py

@@ -0,0 +1,98 @@
+import json
+
+import aiosqlite
+
+from utils.db.models import Pin, PinMode
+
+_COLUMNS = "id, kind, target_id, name, label, mode, args, position"
+
+
+def _row_to_pin(row: aiosqlite.Row) -> Pin:
+    return Pin(
+        id=row["id"],
+        kind=row["kind"],
+        target_id=row["target_id"],
+        name=row["name"],
+        label=row["label"],
+        mode=PinMode(row["mode"]),
+        args=json.loads(row["args"] or "{}"),
+        position=row["position"],
+    )
+
+
+class PinRepository:
+    def __init__(self, conn: aiosqlite.Connection) -> None:
+        self._conn = conn
+
+    async def all(self) -> list[Pin]:
+        cursor = await self._conn.execute(
+            f"SELECT {_COLUMNS} FROM pins ORDER BY position, id"  # noqa: S608
+        )
+        return [_row_to_pin(row) for row in await cursor.fetchall()]
+
+    async def get(self, pin_id: int) -> Pin | None:
+        cursor = await self._conn.execute(
+            f"SELECT {_COLUMNS} FROM pins WHERE id = ?",  # noqa: S608
+            (pin_id,),
+        )
+        row = await cursor.fetchone()
+        return _row_to_pin(row) if row else None
+
+    async def add(  # noqa: PLR0913
+        self,
+        *,
+        kind: str,
+        target_id: str,
+        name: str,
+        label: str,
+        mode: PinMode,
+        args: dict,
+    ) -> Pin:
+        cursor = await self._conn.execute(
+            "INSERT INTO pins (kind, target_id, name, label, mode, args, position) "
+            "VALUES (?, ?, ?, ?, ?, ?, "
+            "(SELECT COALESCE(MAX(position), 0) + 1 FROM pins))",
+            (
+                kind,
+                target_id,
+                name,
+                label,
+                mode.value,
+                json.dumps(args, ensure_ascii=False),
+            ),
+        )
+        await self._conn.commit()
+        pin = await self.get(cursor.lastrowid or 0)
+        assert pin is not None
+        return pin
+
+    async def rename(self, pin_id: int, label: str) -> None:
+        await self._conn.execute(
+            "UPDATE pins SET label = ? WHERE id = ?", (label, pin_id)
+        )
+        await self._conn.commit()
+
+    async def set_mode(self, pin_id: int, mode: PinMode) -> None:
+        await self._conn.execute(
+            "UPDATE pins SET mode = ? WHERE id = ?", (mode.value, pin_id)
+        )
+        await self._conn.commit()
+
+    async def move(self, pin_id: int, step: int) -> None:
+        pins = await self.all()
+        index = next((i for i, p in enumerate(pins) if p.id == pin_id), None)
+        if index is None:
+            return
+        target = index + step
+        if not 0 <= target < len(pins):
+            return
+        pins[index], pins[target] = pins[target], pins[index]
+        for position, pin in enumerate(pins, start=1):
+            await self._conn.execute(
+                "UPDATE pins SET position = ? WHERE id = ?", (position, pin.id)
+            )
+        await self._conn.commit()
+
+    async def delete(self, pin_id: int) -> None:
+        await self._conn.execute("DELETE FROM pins WHERE id = ?", (pin_id,))
+        await self._conn.commit()

src/utils/db/repositories/service.py

@@ -6,7 +6,8 @@ from utils.format import now
 _COLUMNS = (
     "id, name, url, match_type, ok_status, ok_body, "
     "json_path, json_value, is_up, created_at, "
-    "fail_streak, is_paused, last_latency_ms, last_checked_at"
+    "fail_streak, is_paused, last_latency_ms, last_checked_at, "
+    "needle, tls_expires_at, tls_warned"
 )
 
 
@@ -26,6 +27,9 @@ def _row_to_service(row: aiosqlite.Row) -> Service:
         is_paused=bool(row["is_paused"]),
         last_latency_ms=row["last_latency_ms"],
         last_checked_at=row["last_checked_at"],
+        needle=row["needle"],
+        tls_expires_at=row["tls_expires_at"],
+        tls_warned=row["tls_warned"],
     )
 
 
@@ -65,11 +69,12 @@ class ServiceRepository:
         ok_body: str = "",
         json_path: str | None = None,
         json_value: str | None = None,
+        needle: str | None = None,
     ) -> Service:
         cursor = await self._conn.execute(
             "INSERT INTO services "
             "(name, url, match_type, ok_status, ok_body, json_path, json_value, "
-            "is_up, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?)",
+            "needle, is_up, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?)",
             (
                 name,
                 url,
@@ -78,14 +83,49 @@ class ServiceRepository:
                 ok_body,
                 json_path,
                 json_value,
+                needle,
                 now().isoformat(),
             ),
         )
         await self._conn.commit()
-        service = await self.get(cursor.lastrowid)  # type: ignore[arg-type]
+        service = await self.get(cursor.lastrowid or 0)
         assert service is not None
         return service
 
+    async def set_rule(  # noqa: PLR0913
+        self,
+        service_id: int,
+        *,
+        match_type: MatchType,
+        ok_status: int,
+        ok_body: str = "",
+        json_path: str | None = None,
+        json_value: str | None = None,
+        needle: str | None = None,
+    ) -> None:
+        await self._conn.execute(
+            "UPDATE services SET match_type = ?, ok_status = ?, ok_body = ?, "
+            "json_path = ?, json_value = ?, needle = ?, fail_streak = 0 WHERE id = ?",
+            (
+                match_type.value,
+                ok_status,
+                ok_body,
+                json_path,
+                json_value,
+                needle,
+                service_id,
+            ),
+        )
+        await self._conn.commit()
+
+    async def set_url(self, service_id: int, url: str) -> None:
+        await self._conn.execute(
+            "UPDATE services SET url = ?, fail_streak = 0, tls_expires_at = NULL, "
+            "tls_warned = 0 WHERE id = ?",
+            (url, service_id),
+        )
+        await self._conn.commit()
+
     async def set_up(self, service_id: int, *, is_up: bool) -> None:
         await self._conn.execute(
             "UPDATE services SET is_up = ? WHERE id = ?", (int(is_up), service_id)
@@ -112,6 +152,13 @@ class ServiceRepository:
         )
         await self._conn.commit()
 
+    async def set_tls(self, service_id: int, expires_at: str, warned: int) -> None:
+        await self._conn.execute(
+            "UPDATE services SET tls_expires_at = ?, tls_warned = ? WHERE id = ?",
+            (expires_at, warned, service_id),
+        )
+        await self._conn.commit()
+
     async def rename(self, service_id: int, name: str) -> None:
         await self._conn.execute(
             "UPDATE services SET name = ? WHERE id = ?", (name, service_id)
@@ -121,7 +168,6 @@ class ServiceRepository:
     async def set_baseline(
         self, service_id: int, *, ok_status: int, ok_body: str, json_value: str | None
     ) -> None:
-        """Переснять эталон с текущего ответа."""
         await self._conn.execute(
             "UPDATE services SET ok_status = ?, ok_body = ?, json_value = ? "
             "WHERE id = ?",

src/utils/db/repositories/setting.py

@@ -0,0 +1,38 @@
+import json
+from typing import Any
+
+import aiosqlite
+
+DEFAULTS: dict[str, Any] = {
+    "actions": True,
+    "commands": False,
+    "confirm": True,
+    "muted": ["ScheduleRun"],
+    "tz": "UTC",
+    "digest": None,
+    "digest_sent": "",
+}
+
+
+class SettingRepository:
+    def __init__(self, conn: aiosqlite.Connection) -> None:
+        self._conn = conn
+        self._cache: dict[str, Any] = {}
+
+    async def load(self) -> None:
+        cursor = await self._conn.execute("SELECT key, value FROM settings")
+        self._cache = {
+            row["key"]: json.loads(row["value"]) for row in await cursor.fetchall()
+        }
+
+    def get(self, key: str) -> Any:  # noqa: ANN401
+        return self._cache.get(key, DEFAULTS[key])
+
+    async def set(self, key: str, value: Any) -> None:  # noqa: ANN401
+        self._cache[key] = value
+        await self._conn.execute(
+            "INSERT INTO settings (key, value) VALUES (?, ?) "
+            "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+            (key, json.dumps(value, ensure_ascii=False)),
+        )
+        await self._conn.commit()

src/utils/db/repositories/stat.py

@@ -0,0 +1,50 @@
+import aiosqlite
+
+from utils.db.models import Hour
+
+SECONDS_IN_HOUR = 3600
+
+
+class StatRepository:
+    def __init__(self, conn: aiosqlite.Connection) -> None:
+        self._conn = conn
+
+    async def record(
+        self, service_id: int, hour: int, *, healthy: bool, latency_ms: int | None
+    ) -> None:
+        measured = latency_ms is not None
+        latency = latency_ms or 0
+        await self._conn.execute(
+            "INSERT INTO hourly "
+            "(service_id, hour, checks, fails, latency_sum, latency_n, latency_max) "
+            "VALUES (?, ?, 1, ?, ?, ?, ?) "
+            "ON CONFLICT(service_id, hour) DO UPDATE SET "
+            "checks = checks + 1, fails = fails + excluded.fails, "
+            "latency_sum = latency_sum + excluded.latency_sum, "
+            "latency_n = latency_n + excluded.latency_n, "
+            "latency_max = MAX(latency_max, excluded.latency_max)",
+            (service_id, hour, int(not healthy), latency, int(measured), latency),
+        )
+        await self._conn.commit()
+
+    async def since(self, service_id: int, hour: int) -> dict[int, Hour]:
+        cursor = await self._conn.execute(
+            "SELECT hour, checks, fails, latency_sum, latency_n, latency_max "
+            "FROM hourly WHERE service_id = ? AND hour >= ? ORDER BY hour",
+            (service_id, hour),
+        )
+        return {
+            row["hour"]: Hour(
+                hour=row["hour"],
+                checks=row["checks"],
+                fails=row["fails"],
+                latency_sum=row["latency_sum"],
+                latency_n=row["latency_n"],
+                latency_max=row["latency_max"],
+            )
+            for row in await cursor.fetchall()
+        }
+
+    async def purge(self, before_hour: int) -> None:
+        await self._conn.execute("DELETE FROM hourly WHERE hour < ?", (before_hour,))
+        await self._conn.commit()

src/utils/format.py

@@ -1,4 +1,5 @@
-from datetime import UTC, datetime
+from datetime import UTC, datetime, tzinfo
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 
 def now() -> datetime:
@@ -9,7 +10,23 @@ def parse(ts: str) -> datetime:
     return datetime.fromisoformat(ts)
 
 
+def from_ms(ms: float) -> datetime:
+    return datetime.fromtimestamp(ms / 1000, UTC)
+
+
+def zone(name: str) -> tzinfo:
+    try:
+        return ZoneInfo(name)
+    except (ZoneInfoNotFoundError, ValueError):
+        return UTC
+
+
+def stamp(moment: datetime, tz_name: str) -> str:
+    return moment.astimezone(zone(tz_name)).strftime("%d.%m %H:%M")
+
+
 MINUTE = 60
+HOUR = 3600
 MAX_PARTS = 2
 
 
@@ -27,6 +44,23 @@ def human_duration(seconds: float) -> str:
     return " ".join(parts[:MAX_PARTS])
 
 
+def ago(moment: datetime) -> str:
+    return f"{human_duration(max(0.0, (now() - moment).total_seconds()))} назад"
+
+
+def until(moment: datetime) -> str:
+    return f"через {human_duration(max(0.0, (moment - now()).total_seconds()))}"
+
+
+def clock(seconds: float) -> str:
+    if seconds < MINUTE:
+        return f"{seconds:.1f}с" if seconds < 10 else f"{int(seconds)}с"  # noqa: PLR2004
+    total = int(seconds)
+    if total < HOUR:
+        return f"{total // MINUTE}:{total % MINUTE:02d}"
+    return f"{total // HOUR}:{total % HOUR // MINUTE:02d}:{total % MINUTE:02d}"
+
+
 def short(value: object, limit: int = 60) -> str:
     text = str(value).replace("\n", " ").strip()
     return text if len(text) <= limit else text[: limit - 1] + "…"

src/utils/logging.py

@@ -22,14 +22,14 @@ def setup_logging() -> None:
                 markup=True,
                 rich_tracebacks=True,
                 enable_link_path=False,
-                tracebacks_show_locals=True,
+                tracebacks_show_locals=False,
                 omit_repeated_times=False,
                 show_time=env.log.show_time,
                 tracebacks_suppress=[router],
             )
         ],
     )
-    install(console=console, show_locals=True)
+    install(console=console, show_locals=False)
 
 
 logger = logging.getLogger("healthbot")

uv.lock

@@ -248,7 +248,9 @@ dependencies = [
     { name = "aiohttp" },
     { name = "aiosqlite" },
     { name = "pydantic-settings" },
+    { name = "pyyaml" },
     { name = "rich" },
+    { name = "tzdata" },
 ]
 
 [package.dev-dependencies]
@@ -263,7 +265,9 @@ requires-dist = [
     { name = "aiohttp", specifier = ">=3.10.0" },
     { name = "aiosqlite", specifier = ">=0.20.0" },
     { name = "pydantic-settings", specifier = ">=2.14.2" },
+    { name = "pyyaml", specifier = ">=6.0.3" },
     { name = "rich", specifier = ">=15.0.0" },
+    { name = "tzdata", specifier = ">=2026.5" },
 ]
 
 [package.metadata.requires-dev]
@@ -572,6 +576,42 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" },
 ]
 
+[[package]]
+name = "pyyaml"
+version = "6.0.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" },
+    { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" },
+    { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" },
+    { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" },
+    { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" },
+    { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" },
+    { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" },
+    { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" },
+    { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" },
+    { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" },
+    { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" },
+    { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
+]
+
 [[package]]
 name = "rich"
 version = "15.0.0"
@@ -656,6 +696,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" },
 ]
 
+[[package]]
+name = "tzdata"
+version = "2026.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" },
+]
+
 [[package]]
 name = "yarl"
 version = "1.24.2"