diff --git a/pyproject.toml b/pyproject.toml
index c304067..a08f2ed 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,7 +11,9 @@ dependencies = [
"aiohttp>=3.10.0",
"aiosqlite>=0.20.0",
"pydantic-settings>=2.14.2",
+ "pyyaml>=6.0.3",
"rich>=15.0.0",
+ "tzdata>=2026.5",
]
[build-system]
diff --git a/src/healthbot/__init__.py b/src/healthbot/__init__.py
index 0e8e038..5a2125b 100644
--- a/src/healthbot/__init__.py
+++ b/src/healthbot/__init__.py
@@ -12,6 +12,7 @@ async def runner() -> None:
from . import handlers # noqa: PLC0415
from .common import bot, dp # noqa: PLC0415
+ from .digest import run_digest # noqa: PLC0415
from .komodo import komodo # noqa: PLC0415
from .monitor import run_monitor # noqa: PLC0415
from .web import run_web # noqa: PLC0415
@@ -20,14 +21,16 @@ async def runner() -> None:
dp.include_routers(handlers.router)
await bot.delete_webhook(drop_pending_updates=True)
- monitor = asyncio.create_task(run_monitor())
+ background = [asyncio.create_task(run_monitor()), asyncio.create_task(run_digest())]
web = await run_web() if env.web.enabled else None
try:
await dp.start_polling(bot)
finally:
- monitor.cancel()
- with contextlib.suppress(asyncio.CancelledError):
- await monitor
+ for task in background:
+ task.cancel()
+ for task in background:
+ with contextlib.suppress(asyncio.CancelledError):
+ await task
if web is not None:
await web.cleanup()
await komodo.close()
diff --git a/src/healthbot/actionargs.py b/src/healthbot/actionargs.py
new file mode 100644
index 0000000..3e1afaf
--- /dev/null
+++ b/src/healthbot/actionargs.py
@@ -0,0 +1,186 @@
+import json
+import re
+import shlex
+import tomllib
+from typing import Any
+
+import yaml
+
+QUOTES = str.maketrans({"“": '"', "”": '"', "„": '"', "«": '"', "»": '"'})
+KEY_RE = re.compile(r"[\w.-]+")
+TRUE = {"true", "yes", "on", "1", "да", "вкл"}
+FALSE = {"false", "no", "off", "0", "нет", "выкл"}
+VARIABLE_MARK = "[["
+DASHES = "-\u2014\u2013"
+
+
+class ArgError(ValueError):
+ pass
+
+
+def _key_value(text: str) -> dict[str, Any]:
+ result: dict[str, Any] = {}
+ for raw in text.splitlines():
+ line = raw.strip().removeprefix("- ").strip()
+ if not line or line.startswith("#"):
+ continue
+ match = re.match(r"([^=:\s]+)\s*[=:]\s*(.*)", line)
+ if not match:
+ continue
+ value = match.group(2).strip()
+ if len(value) > 1 and value[0] == value[-1] and value[0] in "\"'":
+ value = value[1:-1]
+ result[match.group(1)] = value
+ return result
+
+
+def parse_defaults(fmt: str, text: str) -> dict[str, Any]:
+ if not text.strip():
+ return {}
+ try:
+ if fmt == "json":
+ data = json.loads(text)
+ elif fmt == "toml":
+ data = tomllib.loads(text)
+ elif fmt == "yaml":
+ data = yaml.safe_load(text)
+ else:
+ data = _key_value(text)
+ except (ValueError, yaml.YAMLError):
+ return {}
+ if not isinstance(data, dict):
+ return {}
+ return json.loads(json.dumps(data, default=str))
+
+
+def type_name(value: object) -> str:
+ if isinstance(value, bool):
+ return "да или нет"
+ if isinstance(value, (int, float)):
+ return "число"
+ if isinstance(value, str):
+ return "текст"
+ return "JSON"
+
+
+def render(value: object, limit: int = 0) -> str:
+ text = json.dumps(value, ensure_ascii=False)
+ return text if not limit or len(text) <= limit else text[: limit - 1] + "…"
+
+
+def _guess(text: str) -> Any: # noqa: ANN401
+ try:
+ return json.loads(text)
+ except ValueError:
+ return text
+
+
+def _to_bool(text: str) -> bool:
+ lowered = text.strip().lower()
+ if lowered in TRUE:
+ return True
+ if lowered in FALSE:
+ return False
+ msg = "нужно true или false"
+ raise ArgError(msg)
+
+
+def _to_number(text: str, *, keep_float: bool) -> int | float:
+ try:
+ number = float(text.strip().replace(",", "."))
+ except ValueError as exc:
+ msg = "нужно число"
+ raise ArgError(msg) from exc
+ return number if keep_float or not number.is_integer() else int(number)
+
+
+def _to_json(default: object, text: str) -> Any: # noqa: ANN401
+ try:
+ value = json.loads(text)
+ except ValueError as exc:
+ msg = "нужен JSON"
+ raise ArgError(msg) from exc
+ if type(value) is not type(default):
+ msg = "нужен JSON-объект" if isinstance(default, dict) else "нужен JSON-массив"
+ raise ArgError(msg)
+ return value
+
+
+def coerce(default: object, text: str) -> Any: # noqa: ANN401
+ if isinstance(default, bool):
+ return _to_bool(text)
+ if isinstance(default, (int, float)):
+ return _to_number(text, keep_float=isinstance(default, float))
+ if isinstance(default, str):
+ return text
+ if default is None:
+ return _guess(text)
+ return _to_json(default, text)
+
+
+def _tokens(text: str) -> list[str]:
+ try:
+ return shlex.split(text.translate(QUOTES))
+ except ValueError as exc:
+ msg = "кавычка не закрыта"
+ raise ArgError(msg) from exc
+
+
+def _pairs(tokens: list[str], defaults: dict[str, Any]) -> list[tuple[str, str]]:
+ pairs: list[tuple[str, str]] = []
+ index = 0
+ while index < len(tokens):
+ token = tokens[index].lstrip(DASHES)
+ key, sep, value = token.partition("=")
+ inline = bool(sep) and bool(KEY_RE.fullmatch(key))
+ name = key if inline else token
+ index += 1
+ if defaults and name not in defaults:
+ msg = f"нет поля {name}, есть: {', '.join(defaults)}"
+ raise ArgError(msg)
+ if inline:
+ pairs.append((key, value))
+ elif isinstance(defaults.get(token), bool) and not (
+ index < len(tokens) and tokens[index].lower() in TRUE | FALSE
+ ):
+ pairs.append((token, "true"))
+ elif index < len(tokens):
+ pairs.append((token, tokens[index]))
+ index += 1
+ else:
+ msg = f"у поля {token} нет значения"
+ raise ArgError(msg)
+ return pairs
+
+
+def parse_cli(text: str, defaults: dict[str, Any]) -> dict[str, Any]:
+ text = text.strip()
+ if not text:
+ return {}
+ tokens = _tokens(text)
+ first = tokens[0].lstrip(DASHES).partition("=")[0]
+ if len(defaults) == 1 and first not in defaults:
+ key = next(iter(defaults))
+ return {key: coerce(defaults[key], " ".join(tokens))}
+ result: dict[str, Any] = {}
+ for key, value in _pairs(tokens, defaults):
+ try:
+ result[key] = coerce(defaults[key], value) if defaults else _guess(value)
+ except ArgError as exc:
+ msg = f"{key}: {exc}"
+ raise ArgError(msg) from exc
+ return result
+
+
+def changed(defaults: dict[str, Any], values: dict[str, Any]) -> dict[str, Any]:
+ return {k: v for k, v in values.items() if k not in defaults or defaults[k] != v}
+
+
+def ensure_safe(args: dict[str, Any]) -> None:
+ if VARIABLE_MARK in json.dumps(args, ensure_ascii=False):
+ msg = "в значениях нельзя «[[»: Komodo читает это как переменную"
+ raise ArgError(msg)
+
+
+def command_name(name: str) -> str:
+ return re.sub(r"[^a-z0-9_]", "_", name.lower()).strip("_")[:32]
diff --git a/src/healthbot/alerts.py b/src/healthbot/alerts.py
index 595dab5..9e5f561 100644
--- a/src/healthbot/alerts.py
+++ b/src/healthbot/alerts.py
@@ -1,8 +1,11 @@
-"""Превращение алерта Komodo в сообщение для телеграма."""
-
-import html
from typing import Any
+from aiogram.types import InlineKeyboardMarkup
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+from healthbot.ui import esc
+from utils.env import env
+
LEVEL_ICON = {"OK": "✅", "WARNING": "🟠", "CRITICAL": "🔴"}
INFORMATIONAL = frozenset(
@@ -21,13 +24,33 @@ FAILURE = frozenset(
{"ProcedureFailed", "ActionFailed", "BuildFailed", "RepoBuildFailed"}
)
-# Komodo создаёт их сразу с resolved=true (resolved_ts == ts): это факт
-# «состояние сменилось», а не тревога, которую потом снимают. Без этого
-# и running → unhealthy, и unhealthy → running приходили как «Отбой».
STATE_CHANGE = frozenset({"StackStateChange", "ContainerStateChange"})
HEALTHY_STATES = frozenset({"running", "healthy"})
-ONE_SHOT = INFORMATIONAL | FAILURE | STATE_CHANGE
+CREATED_ALREADY_RESOLVED = INFORMATIONAL | FAILURE | STATE_CHANGE
+
+KNOWN_TYPES = (
+ "ServerUnreachable",
+ "ServerCpu",
+ "ServerMem",
+ "ServerDisk",
+ "ServerVersionMismatch",
+ "StackStateChange",
+ "ContainerStateChange",
+ "StackImageUpdateAvailable",
+ "DeploymentImageUpdateAvailable",
+ "StackAutoUpdated",
+ "DeploymentAutoUpdated",
+ "ResourceSyncPendingUpdates",
+ "ProcedureFailed",
+ "ActionFailed",
+ "BuildFailed",
+ "RepoBuildFailed",
+ "ScheduleRun",
+ "Custom",
+)
+
+CALLBACK_LIMIT = 64
def _gb(value: Any) -> str: # noqa: ANN401
@@ -44,79 +67,88 @@ def _pct(value: Any) -> str: # noqa: ANN401
return str(value)
+def _error(value: Any) -> str: # noqa: ANN401
+ if isinstance(value, dict):
+ return str(value.get("error") or value)
+ return str(value or "нет деталей")
+
+
def _describe(kind: str, d: dict) -> tuple[str, list[str]]: # noqa: C901, PLR0911, PLR0912
- """Заголовок и строки подробностей для конкретного типа алерта."""
name = d.get("name") or d.get("id") or "?"
server = d.get("server_name")
if kind == "ServerUnreachable":
- return f"Сервер {html.escape(name)} не отвечает", [
- f"Ошибка: {html.escape(str(d.get('err') or 'нет деталей'))}"
+ return f"Сервер {esc(name)} не отвечает", [
+ f"Ошибка: {esc(_error(d.get('err')))}"
]
if kind == "ServerCpu":
- return f"CPU на {html.escape(name)}", [
- f"Загрузка: {_pct(d.get('percentage'))}"
- ]
+ return f"CPU на {esc(name)}", [f"Загрузка: {_pct(d.get('percentage'))}"]
if kind == "ServerMem":
- return f"Память на {html.escape(name)}", [
+ return f"Память на {esc(name)}", [
f"Занято: {_gb(d.get('used_gb'))} из {_gb(d.get('total_gb'))}"
]
if kind == "ServerDisk":
- return f"Диск на {html.escape(name)}", [
- f"Точка: {html.escape(str(d.get('path', '/')))}",
+ return f"Диск на {esc(name)}", [
+ f"Точка: {esc(d.get('path', '/'))}",
f"Занято: {_gb(d.get('used_gb'))} из {_gb(d.get('total_gb'))}",
]
if kind == "ServerVersionMismatch":
- return f"Разъехались версии на {html.escape(name)}", [
- f"Periphery: {d.get('version')}, Core: {d.get('core_version')}",
+ return f"Разъехались версии на {esc(name)}", [
+ f"Periphery: {esc(d.get('version'))}, Core: {esc(d.get('core_version'))}",
"Чинится только по ssh - дашборд к такому серверу не достучится.",
]
- if kind in {"StackStateChange", "ContainerStateChange"}:
- return f"{html.escape(name)}: {d.get('from')} → {d.get('to')}", (
- [f"Сервер: {html.escape(server)}"] if server else []
+ if kind in STATE_CHANGE:
+ return f"{esc(name)}: {esc(d.get('from'))} → {esc(d.get('to'))}", (
+ [f"Сервер: {esc(server)}"] if server else []
)
if kind in {"StackImageUpdateAvailable", "DeploymentImageUpdateAvailable"}:
- return f"Обновление образа для {html.escape(name)}", [
- f"{html.escape(str(d.get('image', '?')))}"
+ return f"Обновление образа для {esc(name)}", [
+ f"{esc(d.get('image', '?'))}"
]
if kind in {"StackAutoUpdated", "DeploymentAutoUpdated"}:
- return f"{html.escape(name)} обновлён автоматически", [
- f"{html.escape(str(d.get('images') or d.get('image') or ''))}"
+ images = d.get("images") or d.get("image") or ""
+ if isinstance(images, list):
+ images = ", ".join(str(image) for image in images)
+ return f"{esc(name)} обновлён автоматически", [
+ f"{esc(images)}"
]
if kind == "ResourceSyncPendingUpdates":
- return f"Синк {html.escape(name)}: git разошёлся с Komodo", [
+ return f"Синк {esc(name)}: git разошёлся с Komodo", [
"Кто-то правил ресурсы мимо git, либо пуш не доехал."
]
- if kind in {"ProcedureFailed", "ActionFailed", "BuildFailed", "RepoBuildFailed"}:
+ if kind in FAILURE:
what = {"ProcedureFailed": "Процедура", "ActionFailed": "Действие"}.get(
kind, "Сборка"
)
- return f"{what} {html.escape(name)} упала", []
+ fell = "упало" if kind == "ActionFailed" else "упала"
+ return f"{what} {esc(name)} {fell}", []
if kind == "ScheduleRun":
- return f"По расписанию запущено: {html.escape(name)}", []
+ return f"По расписанию запущено: {esc(name)}", []
if kind == "Test":
return "Проверка связи с Komodo", ["Если ты это видишь - канал работает."]
if kind == "Custom":
- return html.escape(str(d.get("title") or "Сообщение из Komodo")), [
- html.escape(str(d.get("body") or ""))
- ]
+ return esc(d.get("title") or "Сообщение из Komodo"), [esc(d.get("body") or "")]
rest = [
- f"{html.escape(str(k))}: {html.escape(str(v))}"
+ f"{esc(k)}: {esc(v)}"
for k, v in d.items()
if k not in {"id", "server_id", "swarm_id"}
]
- return f"{html.escape(kind)}: {html.escape(name)}", rest[:8]
+ return f"{esc(kind)}: {esc(name)}", rest[:8]
+
+
+def alert_kind(payload: dict) -> str:
+ return str((payload.get("data") or {}).get("type") or "Unknown")
def format_alert(payload: dict) -> str:
level = str(payload.get("level", "")).upper()
data = payload.get("data") or {}
- kind = str(data.get("type") or "Unknown")
+ kind = alert_kind(payload)
inner = data.get("data") or {}
informational = kind in INFORMATIONAL
- resolved = bool(payload.get("resolved")) and kind not in ONE_SHOT
+ resolved = bool(payload.get("resolved")) and kind not in CREATED_ALREADY_RESOLVED
if informational:
icon = "ℹ️"
@@ -131,11 +163,34 @@ def format_alert(payload: dict) -> str:
title, details = _describe(kind, inner if isinstance(inner, dict) else {})
if resolved:
- return "\n".join(
- [f"{icon} Отбой: {title}", f"Komodo · {html.escape(kind)}"]
- )
+ return "\n".join([f"{icon} Отбой: {title}", f"Komodo · {esc(kind)}"])
lines = [f"{icon} {title}"]
lines.extend(x for x in details if x)
- lines.append(f"Komodo · {html.escape(kind)}")
+ lines.append(f"Komodo · {esc(kind)}")
return "\n".join(lines)
+
+
+def alert_keyboard(payload: dict) -> InlineKeyboardMarkup | None:
+ kind = alert_kind(payload)
+ target = payload.get("target") or {}
+ target_type, target_id = target.get("type"), target.get("id")
+ kb = InlineKeyboardBuilder()
+ if env.komodo.enabled and target_id:
+ if target_type == "Stack":
+ kb.button(text="Стек", callback_data=f"go:stack:{target_id}")
+ kb.button(text="Логи", callback_data=f"go:log:{target_id}")
+ elif target_type == "Server":
+ kb.button(text="Сервер", callback_data=f"go:srv:{target_id}")
+ elif target_type == "Action":
+ kb.button(text="Действие", callback_data=f"go:act:a:{target_id}")
+ elif target_type == "Procedure":
+ kb.button(text="Процедура", callback_data=f"go:act:p:{target_id}")
+ mute = f"al:mute:{kind}"
+ if kind != "Test" and len(mute) <= CALLBACK_LIMIT:
+ kb.button(text="Не присылать такие", callback_data=mute)
+ buttons = len(list(kb.buttons))
+ if not buttons:
+ return None
+ kb.adjust(buttons - 1 or 1, 1)
+ return kb.as_markup()
diff --git a/src/healthbot/checker.py b/src/healthbot/checker.py
index 9882385..0b03646 100644
--- a/src/healthbot/checker.py
+++ b/src/healthbot/checker.py
@@ -1,21 +1,27 @@
-"""HTTP-пробинг сервисов и логика сравнения ответа с эталоном."""
-
+import asyncio
import difflib
-import html
import json
+import ssl
import time
from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from urllib.parse import urlparse
import aiohttp
+from healthbot.ui import esc
from utils.db.models import MatchType, Service
from utils.env import env
from utils.format import short
-_MISSING = object()
+MISSING = object()
DIFF_LINES = 12
DIFF_HEADER_LINES = 2
+BODY_LIMIT = 1024 * 1024
+CHUNK = 64 * 1024
+TLS_TIMEOUT = 10
+HEADERS = {"User-Agent": "healthbot"}
@dataclass(slots=True)
@@ -29,8 +35,6 @@ class Probe:
@dataclass(slots=True)
class Verdict:
- """Результат сравнения."""
-
healthy: bool
reason: str = ""
expected: str | None = None
@@ -38,11 +42,10 @@ class Verdict:
details: list[str] = field(default_factory=list)
def report(self) -> str:
- """Человекочитаемое объяснение для сообщения в телеграм."""
lines: list[str] = []
if self.expected is not None or self.actual is not None:
- lines.append(f"Ожидали: {html.escape(self.expected or '-')}")
- lines.append(f"Сейчас: {html.escape(self.actual or '-')}")
+ lines.append(f"Ожидали: {esc(self.expected or '-')}")
+ lines.append(f"Сейчас: {esc(self.actual or '-')}")
lines.extend(self.details)
return "\n".join(lines)
@@ -51,9 +54,19 @@ async def probe(session: aiohttp.ClientSession, url: str, timeout: int) -> Probe
started = time.monotonic()
try:
async with session.get(
- url, timeout=aiohttp.ClientTimeout(total=timeout), allow_redirects=True
+ url,
+ timeout=aiohttp.ClientTimeout(total=timeout),
+ allow_redirects=True,
+ headers=HEADERS,
) as resp:
- body = await resp.text()
+ raw = bytearray()
+ async for chunk in resp.content.iter_chunked(CHUNK):
+ raw += chunk
+ if len(raw) >= BODY_LIMIT:
+ break
+ body = bytes(raw[:BODY_LIMIT]).decode(
+ resp.charset or "utf-8", errors="replace"
+ )
return Probe(
reachable=True,
status=resp.status,
@@ -65,40 +78,65 @@ async def probe(session: aiohttp.ClientSession, url: str, timeout: int) -> Probe
return Probe(
reachable=False, status=None, body="", error=f"таймаут ({timeout}с)"
)
- except aiohttp.ClientError as exc:
+ except aiohttp.ClientConnectorError as exc:
+ reason = exc.os_error.strerror or str(exc.os_error)
+ return Probe(
+ reachable=False, status=None, body="", error=f"нет связи: {reason}"
+ )
+ except (aiohttp.ClientError, ValueError, LookupError) as exc:
return Probe(reachable=False, status=None, body="", error=f"нет связи: {exc}")
+async def tls_expiry(url: str) -> datetime | None:
+ parsed = urlparse(url)
+ if parsed.scheme != "https" or not parsed.hostname:
+ return None
+ try:
+ _, writer = await asyncio.wait_for(
+ asyncio.open_connection(
+ parsed.hostname, parsed.port or 443, ssl=ssl.create_default_context()
+ ),
+ TLS_TIMEOUT,
+ )
+ except (OSError, TimeoutError):
+ return None
+ try:
+ cert = writer.get_extra_info("peercert") or {}
+ finally:
+ writer.close()
+ not_after = cert.get("notAfter")
+ if not not_after:
+ return None
+ return datetime.fromtimestamp(ssl.cert_time_to_seconds(not_after), UTC)
+
+
def dig(data: object, path: str) -> object:
- """Достаёт значение по dot-пути (индексы списков - числами)."""
node = data
for segment in path.split("."):
if isinstance(node, dict):
if segment not in node:
- return _MISSING
+ return MISSING
node = node[segment]
elif isinstance(node, list):
if not segment.isdigit() or int(segment) >= len(node):
- return _MISSING
+ return MISSING
node = node[int(segment)]
else:
- return _MISSING
+ return MISSING
return node
def snippet(text: str, limit: int | None = None) -> list[str]:
- """Кусок ответа в , готовый к вставке в сообщение."""
limit = limit or env.monitor.body_preview
body = text.strip()
if not body:
- return ["(пустой ответ)"]
+ return ["пустой ответ"]
clipped = body[:limit]
tail = "\n…" if len(body) > limit else ""
- return [f"{html.escape(clipped)}{tail}"]
+ return [f"{esc(clipped)}{tail}"]
def body_diff(expected: str, actual: str) -> list[str]:
- """Чем именно тело отличается от эталона."""
diff = list(
difflib.unified_diff(
expected.strip().splitlines(), actual.strip().splitlines(), lineterm="", n=0
@@ -108,57 +146,34 @@ def body_diff(expected: str, actual: str) -> list[str]:
return ["Отличие в невидимых символах.", *snippet(actual)]
shown = diff[:DIFF_LINES]
text = "\n".join(shown)
- more = f"\n… ещё {len(diff) - len(shown)} строк" if len(diff) > len(shown) else ""
- return [f"{html.escape(text)}{more}"]
+ more = f"\nещё строк: {len(diff) - len(shown)}" if len(diff) > len(shown) else ""
+ return [f"{esc(text)}{more}"]
-def evaluate(service: Service, result: Probe) -> Verdict: # noqa: PLR0911
- """Жив ли сервис - и если нет, то что именно он сейчас отдаёт."""
- if not result.reachable:
- return Verdict(healthy=False, reason=result.error or "недоступен")
-
- if service.match_type is MatchType.STATUS:
- if result.status != service.ok_status:
- return Verdict(
- healthy=False,
- reason=f"HTTP {result.status}",
- expected=f"HTTP {service.ok_status}",
- actual=f"HTTP {result.status}",
- details=snippet(result.body),
- )
- return Verdict(healthy=True)
-
- if service.match_type is MatchType.BODY:
- if result.body.strip() != service.ok_body.strip():
- return Verdict(
- healthy=False,
- reason="тело ответа изменилось",
- details=body_diff(service.ok_body, result.body),
- )
- return Verdict(healthy=True)
-
+def _json_verdict(service: Service, result: Probe) -> Verdict:
+ expected = f"{service.json_path} = {service.json_value}"
try:
data = json.loads(result.body)
except ValueError:
return Verdict(
healthy=False,
reason="ответ перестал быть JSON",
- expected=f"{service.json_path} = {service.json_value}",
+ expected=expected,
actual=f"не JSON, HTTP {result.status}",
details=snippet(result.body),
)
value = dig(data, service.json_path or "")
- if value is _MISSING:
+ if value is MISSING:
return Verdict(
healthy=False,
reason=f"нет поля {service.json_path}",
- expected=f"{service.json_path} = {service.json_value}",
+ expected=expected,
actual="поля нет в ответе",
details=snippet(result.body),
)
- current = _scalar_str(value)
+ current = scalar_str(value)
if current != service.json_value:
return Verdict(
healthy=False,
@@ -170,8 +185,56 @@ def evaluate(service: Service, result: Probe) -> Verdict: # noqa: PLR0911
return Verdict(healthy=True)
-def _scalar_str(value: object) -> str:
- """Строковое представление скаляра JSON (bool/null как в JSON)."""
+def _status_verdict(service: Service, result: Probe) -> Verdict:
+ if result.status == service.ok_status:
+ return Verdict(healthy=True)
+ return Verdict(
+ healthy=False,
+ reason=f"HTTP {result.status}",
+ expected=f"HTTP {service.ok_status}",
+ actual=f"HTTP {result.status}",
+ details=snippet(result.body),
+ )
+
+
+def _body_verdict(service: Service, result: Probe) -> Verdict:
+ if result.body.strip() == service.ok_body.strip():
+ return Verdict(healthy=True)
+ return Verdict(
+ healthy=False,
+ reason="тело ответа изменилось",
+ details=body_diff(service.ok_body, result.body),
+ )
+
+
+def _contains_verdict(service: Service, result: Probe) -> Verdict:
+ needle = service.needle or ""
+ if needle in result.body:
+ return Verdict(healthy=True)
+ return Verdict(
+ healthy=False,
+ reason=f"в ответе нет «{short(needle, 40)}»",
+ expected=f"есть {needle}",
+ actual=f"нет, HTTP {result.status}",
+ details=snippet(result.body),
+ )
+
+
+VERDICTS = {
+ MatchType.STATUS: _status_verdict,
+ MatchType.BODY: _body_verdict,
+ MatchType.CONTAINS: _contains_verdict,
+ MatchType.JSON: _json_verdict,
+}
+
+
+def evaluate(service: Service, result: Probe) -> Verdict:
+ if not result.reachable:
+ return Verdict(healthy=False, reason=result.error or "недоступен")
+ return VERDICTS[service.match_type](service, result)
+
+
+def scalar_str(value: object) -> str:
if isinstance(value, bool):
return "true" if value else "false"
if value is None:
diff --git a/src/healthbot/common.py b/src/healthbot/common.py
index d60cc9e..431cad5 100644
--- a/src/healthbot/common.py
+++ b/src/healthbot/common.py
@@ -6,7 +6,7 @@ from utils.env import env
bot = Bot(
token=env.bot.token.get_secret_value(),
- default=DefaultBotProperties(parse_mode="HTML"),
+ default=DefaultBotProperties(parse_mode="HTML", link_preview_is_disabled=True),
)
dp = Dispatcher(storage=MemoryStorage())
diff --git a/src/healthbot/digest.py b/src/healthbot/digest.py
new file mode 100644
index 0000000..c9dc289
--- /dev/null
+++ b/src/healthbot/digest.py
@@ -0,0 +1,118 @@
+import asyncio
+from datetime import timedelta
+
+from healthbot.keyboards import back_to_menu
+from healthbot.komodo import KomodoError, Update, komodo
+from healthbot.notify import notify
+from healthbot.stats import DAY, downtime
+from healthbot.ui import esc, pct, plural
+from utils.db import db
+from utils.env import env
+from utils.format import from_ms, human_duration, now, parse, stamp, zone
+from utils.logging import logger
+
+TICK = 60
+MAX_PAGES = 5
+PAGE_SIZE = 100
+SHOWN = 6
+
+
+async def _services_block() -> list[str]:
+ services = await db.services.all()
+ if not services:
+ return []
+ end = now()
+ start = end - timedelta(seconds=DAY)
+ recent = await db.incidents.since(start.isoformat())
+ window = lost = 0.0
+ rows = []
+ for service in services:
+ mine = [i for i in recent if i.service_id == service.id]
+ began = max(start, parse(service.created_at))
+ span = max(0.0, (end - began).total_seconds())
+ down = downtime(mine, began, end)
+ window += span
+ lost += down
+ if mine:
+ rows.append((down, service, len(mine)))
+ uptime = (window - lost) / window * 100 if window else None
+ falls = sum(row[2] for row in rows)
+ if not falls:
+ return [f"Сервисы: аптайм {pct(uptime)}, падений не было"]
+ word = plural(falls, "падение", "падения", "падений")
+ lines = [
+ f"Сервисы: аптайм {pct(uptime)}, {falls} {word}, "
+ f"даунтайм {human_duration(lost)}"
+ ]
+ for down, service, count in sorted(rows, key=lambda row: -row[0])[:SHOWN]:
+ lines.append(
+ f"{service.mark} {esc(service.name)}: {count} "
+ f"{plural(count, 'падение', 'падения', 'падений')}, {human_duration(down)}"
+ )
+ return lines
+
+
+async def _recent_updates(since_ms: int) -> list[Update]:
+ result: list[Update] = []
+ for page in range(MAX_PAGES):
+ chunk = await komodo.updates({"start_ts": {"$gte": since_ms}}, page)
+ result.extend(chunk)
+ if len(chunk) < PAGE_SIZE:
+ break
+ return result
+
+
+async def _fleet_block() -> list[str]:
+ since = now() - timedelta(seconds=DAY)
+ try:
+ updates, alerts, names = await asyncio.gather(
+ _recent_updates(int(since.timestamp() * 1000)),
+ komodo.open_alerts(),
+ komodo.names(),
+ )
+ except KomodoError as exc:
+ return [f"Флот: {esc(exc)}"]
+ tz = db.settings.get("tz")
+ failed = [u for u in updates if u.done and not u.success]
+ lines = [f"Флот: операций {len(updates)}, неудачных {len(failed)}"]
+ grouped: dict[tuple[str, str], list[Update]] = {}
+ for update in failed:
+ grouped.setdefault((update.operation, update.target_id), []).append(update)
+ for (operation, target_id), group in list(grouped.items())[:SHOWN]:
+ times = f" ×{len(group)}" if len(group) > 1 else ""
+ lines.append(
+ f"🔴 {esc(operation)} {esc(names.get(target_id, group[0].target_type))}"
+ f"{times}, последний раз {stamp(from_ms(group[0].start_ts), tz)}"
+ )
+ if alerts:
+ lines.append(f"Открытых алертов: {len(alerts)}")
+ return lines
+
+
+async def digest_text() -> str:
+ blocks = [await _services_block()]
+ if env.komodo.enabled:
+ blocks.append(await _fleet_block())
+ lines = ["Сводка за сутки"]
+ for block in blocks:
+ if block:
+ lines += ["", *block]
+ return "\n".join(lines)
+
+
+async def run_digest() -> None:
+ while True:
+ try:
+ hour = db.settings.get("digest")
+ local = now().astimezone(zone(db.settings.get("tz")))
+ today = local.strftime("%Y-%m-%d")
+ if (
+ hour is not None
+ and local.hour == hour
+ and db.settings.get("digest_sent") != today
+ ):
+ await db.settings.set("digest_sent", today)
+ await notify(await digest_text(), back_to_menu())
+ except Exception:
+ logger.exception("Digest failed")
+ await asyncio.sleep(TICK)
diff --git a/src/healthbot/handlers/__init__.py b/src/healthbot/handlers/__init__.py
index b4d3c2d..33d036d 100644
--- a/src/healthbot/handlers/__init__.py
+++ b/src/healthbot/handlers/__init__.py
@@ -1,8 +1,33 @@
-from aiogram import Router
+from aiogram import Router, types
-from . import add, commands, fleet
+from utils.logging import logger
+
+from . import actions, add, fleet, menu, runs, services, settings
router = Router()
-router.include_routers(commands.router, add.router, fleet.router)
+router.include_routers(
+ menu.router,
+ services.router,
+ add.router,
+ fleet.router,
+ actions.router,
+ settings.router,
+ add.free,
+ actions.commands,
+ runs.router,
+)
+
+
+@router.errors()
+async def on_error(event: types.ErrorEvent) -> bool:
+ logger.error("Handler failed", exc_info=event.exception)
+ callback = event.update.callback_query
+ if callback is not None:
+ try:
+ await callback.answer("Не получилось, подробности в логах", show_alert=True)
+ except Exception:
+ logger.debug("Callback answer failed")
+ return True
+
__all__ = ["router"]
diff --git a/src/healthbot/handlers/actions.py b/src/healthbot/handlers/actions.py
new file mode 100644
index 0000000..10eb4b6
--- /dev/null
+++ b/src/healthbot/handlers/actions.py
@@ -0,0 +1,722 @@
+from typing import Any
+
+from aiogram import Bot, F, Router, types
+from aiogram.filters import Command
+from aiogram.fsm.context import FSMContext
+from aiogram.fsm.state import State, StatesGroup
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+from healthbot.actionargs import (
+ ArgError,
+ changed,
+ coerce,
+ command_name,
+ ensure_safe,
+ parse_cli,
+ parse_defaults,
+ render,
+ type_name,
+)
+from healthbot.filters import Admin
+from healthbot.handlers.menu import sync_commands
+from healthbot.keyboards import back_to, back_to_menu
+from healthbot.komodo import Detail, KomodoError, Resource, Update, komodo
+from healthbot.runs import track
+from healthbot.ui import (
+ ANSWER,
+ DANGER,
+ SUCCESS,
+ esc,
+ int_arg,
+ label,
+ pager,
+ pages,
+ show,
+)
+from utils.db import db
+from utils.db.models import Pin, PinMode
+from utils.format import ago, from_ms, until
+from utils.logging import logger
+
+router = Router()
+router.message.filter(Admin())
+router.callback_query.filter(Admin())
+
+PAGE = 20
+WIDE_FORM = 6
+MAX_LABEL = 40
+KIND_TITLE = {"a": "Действия", "p": "Процедуры"}
+KIND_ONE = {"a": "Действие", "p": "Процедура"}
+KIND_BACK = {"a": "К действию", "p": "К процедуре"}
+USAGE_FIELDS = 3
+RUN_STATE = {"Ok": "успешно", "Failed": "с ошибкой", "Running": "идёт сейчас"}
+MODE_TEXT = {PinMode.INSTANT: "запускается сразу", PinMode.FORM: "открывает форму"}
+
+
+class ActionFlow(StatesGroup):
+ waiting_value = State()
+ waiting_label = State()
+
+
+def enabled() -> bool:
+ return bool(komodo.enabled and db.settings.get("actions"))
+
+
+async def _guard(callback: types.CallbackQuery) -> bool:
+ if enabled():
+ return True
+ await callback.answer("Действия выключены в настройках", show_alert=True)
+ return False
+
+
+async def _fail(event: types.Message | types.CallbackQuery, exc: Exception) -> None:
+ if isinstance(event, types.CallbackQuery):
+ await event.answer(str(exc)[:190], show_alert=True)
+ else:
+ await show(event, f"🔴 {esc(exc)}", back_to_menu())
+
+
+async def list_view(kind: str, page: int) -> tuple[str, types.InlineKeyboardMarkup]:
+ items = await komodo.resources(kind)
+ other = "p" if kind == "a" else "a"
+ try:
+ other_count = len(await komodo.resources(other))
+ except KomodoError:
+ other_count = 0
+ total = pages(len(items), PAGE)
+ page = min(page, total - 1)
+ chunk = items[page * PAGE : (page + 1) * PAGE]
+
+ kb = InlineKeyboardBuilder()
+ for item in chunk:
+ kb.button(
+ text=f"{item.mark} {label(item.name, 26)}",
+ callback_data=f"act:open:{kind}:{item.id}",
+ )
+ rows = [2] * (len(chunk) // 2) + [1] * (len(chunk) % 2)
+ nav = pager(kb, f"act:list:{kind}", page, total)
+ kb.button(
+ text=f"{KIND_TITLE[other]}: {other_count}", callback_data=f"act:list:{other}:0"
+ )
+ kb.button(text="Закреплённые", callback_data="set:pins")
+ kb.button(text="Меню", callback_data="menu:main")
+ kb.adjust(*rows, *([nav] if nav else []), 2, 1)
+ text = f"{KIND_TITLE[kind]}: {len(items)}"
+ if not items:
+ text += "\nКлюч бота не видит ни одного. Нужен доступ Execute в Komodo."
+ return text, kb.as_markup()
+
+
+def _values_line(values: dict[str, Any]) -> str:
+ return ", ".join(f"{esc(k)} = {esc(render(v, 60))}" for k, v in values.items())
+
+
+def _args_code(values: dict[str, Any]) -> str:
+ if not values:
+ return ""
+ return (
+ ""
+ + esc(" ".join(f"{k}={render(v)}" for k, v in values.items()))
+ + ""
+ )
+
+
+def _run_line(resource: Resource | None) -> str:
+ if resource is None or not resource.last_run_at:
+ return "Ещё не запускалось"
+ state = RUN_STATE.get(resource.state, "")
+ return f"Последний запуск: {ago(from_ms(resource.last_run_at))}" + (
+ f", {state}" if state else ""
+ )
+
+
+def _schedule_line(detail: Detail, resource: Resource | None) -> str:
+ if not detail.schedule or not detail.schedule_enabled:
+ return ""
+ line = f"Расписание: {esc(detail.schedule)}"
+ if resource and resource.schedule_error:
+ return f"{line}, ошибка: {esc(resource.schedule_error)}"
+ if resource and resource.next_run_at:
+ return f"{line}, следующий запуск {until(from_ms(resource.next_run_at))}"
+ return line
+
+
+async def _load(kind: str, target_id: str) -> tuple[Detail, Resource | None, dict]:
+ detail = await komodo.detail(kind, target_id)
+ resource = next(
+ (r for r in await komodo.resources(kind) if r.id == detail.id), None
+ )
+ defaults = (
+ parse_defaults(detail.args_format, detail.args_text) if kind == "a" else {}
+ )
+ return detail, resource, defaults
+
+
+async def _form(state: FSMContext) -> dict[str, Any]:
+ return (await state.get_data()).get("form") or {}
+
+
+async def card_view(
+ state: FSMContext, kind: str, target_id: str, preset: dict | None = None
+) -> tuple[str, types.InlineKeyboardMarkup]:
+ detail, resource, defaults = await _load(kind, target_id)
+ form: dict[str, Any] = await _form(state)
+ if form.get("id") != detail.id or preset is not None:
+ form = {"kind": kind, "id": detail.id, "values": dict(preset or {})}
+ form["name"] = detail.name
+ form["keys"] = list(defaults)
+ values = {k: v for k, v in form["values"].items() if k in defaults}
+ form["values"] = changed(defaults, values)
+ await state.set_state(None)
+ await state.update_data(form=form)
+
+ lines = [f"{esc(detail.name)}"]
+ if detail.description:
+ lines.append(esc(detail.description))
+ if detail.stages:
+ lines += ["", "Этапы: " + " → ".join(esc(s) for s in detail.stages)]
+ if defaults:
+ lines.append("")
+ for key, default in defaults.items():
+ if key in form["values"]:
+ lines.append(
+ f"{esc(key)} = {esc(render(form['values'][key], 200))}"
+ f" (в Komodo {esc(render(default, 60))})"
+ )
+ else:
+ lines.append(f"{esc(key)} = {esc(render(default, 200))}")
+ lines += ["", _run_line(resource)]
+ schedule = _schedule_line(detail, resource)
+ if schedule:
+ lines.append(schedule)
+
+ ref = _ref(form)
+ kb = InlineKeyboardBuilder()
+ for index, key in enumerate(defaults):
+ value = form["values"].get(key, defaults[key])
+ kb.button(
+ text=label(f"{key}: {render(value)}", 40),
+ callback_data=f"act:f:{index}:{ref}",
+ )
+ width = 2 if len(defaults) > WIDE_FORM else 1
+ rows = [width] * (len(defaults) // width) + [1] * (len(defaults) % width)
+ kb.button(text="Запустить", callback_data=f"act:run:{ref}", style=SUCCESS)
+ kb.button(text="Закрепить", callback_data=f"act:pin:{ref}")
+ tail = 2
+ if form["values"]:
+ kb.button(text="Сбросить", callback_data=f"act:reset:{ref}")
+ tail = 3
+ kb.button(text="К списку", callback_data=f"act:list:{kind}:0")
+ kb.adjust(*rows, tail, 1)
+ return "\n".join(lines), kb.as_markup()
+
+
+async def _show_card( # noqa: PLR0913
+ event: types.Message | types.CallbackQuery,
+ state: FSMContext,
+ kind: str,
+ target_id: str,
+ *,
+ preset: dict | None = None,
+ new: bool = False,
+) -> None:
+ try:
+ text, markup = await card_view(state, kind, target_id, preset)
+ except KomodoError as exc:
+ await _fail(event, exc)
+ return
+ await show(event, text, markup, new=new)
+ if isinstance(event, types.CallbackQuery):
+ await event.answer()
+
+
+@router.message(Command("actions"))
+async def cmd_actions(message: types.Message, state: FSMContext) -> None:
+ await state.clear()
+ if not enabled():
+ await show(
+ message, "Действия выключены или Komodo не настроен.", back_to_menu()
+ )
+ return
+ try:
+ text, markup = await list_view("a", 0)
+ except KomodoError as exc:
+ await _fail(message, exc)
+ return
+ await show(message, text, markup)
+
+
+@router.callback_query(F.data.startswith("act:list:"))
+async def act_list(callback: types.CallbackQuery, state: FSMContext) -> None:
+ if not await _guard(callback):
+ return
+ await state.set_state(None)
+ kind = (callback.data or "").split(":")[2]
+ try:
+ text, markup = await list_view(kind, int_arg(callback.data))
+ except KomodoError as exc:
+ await _fail(callback, exc)
+ return
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("act:open:"))
+async def act_open(callback: types.CallbackQuery, state: FSMContext) -> None:
+ if not await _guard(callback):
+ return
+ _, _, kind, target_id = (callback.data or "").split(":", 3)
+ await _show_card(callback, state, kind, target_id)
+
+
+@router.callback_query(F.data.startswith("go:act:"))
+async def go_action(callback: types.CallbackQuery, state: FSMContext) -> None:
+ if not await _guard(callback):
+ return
+ _, _, kind, target_id = (callback.data or "").split(":", 3)
+ await _show_card(callback, state, kind, target_id, new=True)
+
+
+def _ref(form: dict[str, Any]) -> str:
+ return f"{form['kind']}:{form['id']}"
+
+
+async def _current(callback: types.CallbackQuery, state: FSMContext) -> dict | None:
+ kind, target_id = (callback.data or "").rsplit(":", 2)[-2:]
+ form = await _form(state)
+ if form.get("id") == target_id:
+ return form
+ try:
+ text, markup = await card_view(state, kind, target_id, {})
+ except KomodoError as exc:
+ await _fail(callback, exc)
+ return None
+ await show(callback, text, markup)
+ await callback.answer("Форма устарела, открыл её заново", show_alert=True)
+ return None
+
+
+@router.callback_query(F.data.startswith("act:back:"))
+async def act_back(callback: types.CallbackQuery, state: FSMContext) -> None:
+ form = await _current(callback, state)
+ if form is None:
+ return
+ await _show_card(callback, state, form["kind"], form["id"])
+
+
+@router.callback_query(F.data.startswith("act:reset:"))
+async def act_reset(callback: types.CallbackQuery, state: FSMContext) -> None:
+ form = await _current(callback, state)
+ if form is None:
+ return
+ await _show_card(callback, state, form["kind"], form["id"], preset={})
+
+
+@router.callback_query(F.data.startswith("act:f:"))
+async def act_field(callback: types.CallbackQuery, state: FSMContext) -> None:
+ form = await _current(callback, state)
+ if form is None:
+ return
+ index = int((callback.data or "").split(":")[2])
+ if index >= len(form["keys"]):
+ await callback.answer()
+ return
+ key = form["keys"][index]
+ try:
+ _, _, defaults = await _load(form["kind"], form["id"])
+ except KomodoError as exc:
+ await _fail(callback, exc)
+ return
+ default = defaults.get(key)
+ value = form["values"].get(key, default)
+ if isinstance(default, bool):
+ form["values"] = {**form["values"], key: not value}
+ await state.update_data(form=form)
+ await _show_card(callback, state, form["kind"], form["id"])
+ return
+
+ form["field"] = key
+ await state.update_data(form=form)
+ await state.set_state(ActionFlow.waiting_value)
+ kb = InlineKeyboardBuilder()
+ if key in form["values"]:
+ kb.button(
+ text=label(f"Вернуть {render(default)}", 40),
+ callback_data=f"act:fd:{_ref(form)}",
+ )
+ kb.button(text="Отмена", callback_data=f"act:back:{_ref(form)}")
+ kb.adjust(1)
+ await show(
+ callback,
+ f"{esc(key)}: пришли новое значение ({type_name(default)}).\n"
+ f"Сейчас: {esc(render(value, 300))}",
+ kb.as_markup(),
+ )
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("act:fd:"))
+async def act_field_default(callback: types.CallbackQuery, state: FSMContext) -> None:
+ form = await _current(callback, state)
+ if form is None:
+ return
+ form["values"] = {k: v for k, v in form["values"].items() if k != form.get("field")}
+ await state.update_data(form=form)
+ await _show_card(callback, state, form["kind"], form["id"])
+
+
+@router.message(ActionFlow.waiting_value, ANSWER)
+async def on_value(message: types.Message, state: FSMContext) -> None:
+ form = await _form(state)
+ key = form.get("field")
+ if not form or key is None:
+ await state.set_state(None)
+ return
+ try:
+ _, _, defaults = await _load(form["kind"], form["id"])
+ values = {**form["values"], key: coerce(defaults.get(key), message.text or "")}
+ ensure_safe(values)
+ except ArgError as exc:
+ await show(
+ message,
+ f"{esc(key)}: {esc(exc)}. Пришли ещё раз.",
+ back_to("Отмена", f"act:back:{_ref(form)}"),
+ )
+ return
+ except KomodoError as exc:
+ await _fail(message, exc)
+ return
+ form["values"] = values
+ await state.update_data(form=form)
+ await _show_card(message, state, form["kind"], form["id"])
+
+
+async def _execute( # noqa: PLR0913
+ origin: types.Message | types.CallbackQuery,
+ kind: str,
+ target_id: str,
+ name: str,
+ args: dict,
+ *,
+ title: str | None = None,
+ new: bool = False,
+) -> None:
+ async def starter() -> Update:
+ try:
+ ensure_safe(args)
+ except ArgError as exc:
+ raise KomodoError(str(exc)) from exc
+ logger.info(f"Run{'Action' if kind == 'a' else 'Procedure'} {name}")
+ return await komodo.run(kind, target_id, args)
+
+ await track(
+ origin,
+ title or name,
+ starter,
+ back=[(KIND_BACK[kind], f"act:open:{kind}:{target_id}"), ("Меню", "menu:main")],
+ verbose=kind == "a",
+ extra=_args_code(args),
+ new=new,
+ )
+
+
+@router.callback_query(F.data.startswith("act:run:"))
+async def act_run(callback: types.CallbackQuery, state: FSMContext) -> None:
+ form = await _current(callback, state)
+ if form is None or not await _guard(callback):
+ return
+ if not db.settings.get("confirm"):
+ await callback.answer()
+ await _execute(callback, form["kind"], form["id"], form["name"], form["values"])
+ return
+ kb = InlineKeyboardBuilder()
+ kb.button(text="Запустить", callback_data=f"act:go:{_ref(form)}", style=SUCCESS)
+ kb.button(text="Отмена", callback_data=f"act:back:{_ref(form)}")
+ kb.adjust(2)
+ changes = f"\nИзменено: {_values_line(form['values'])}" if form["values"] else ""
+ await show(
+ callback, f"Запустить {esc(form['name'])}?{changes}", kb.as_markup()
+ )
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("act:go:"))
+async def act_go(callback: types.CallbackQuery, state: FSMContext) -> None:
+ form = await _current(callback, state)
+ if form is None or not await _guard(callback):
+ return
+ await callback.answer()
+ await _execute(callback, form["kind"], form["id"], form["name"], form["values"])
+
+
+@router.callback_query(F.data.startswith("act:pin:"))
+async def act_pin(callback: types.CallbackQuery, state: FSMContext) -> None:
+ form = await _current(callback, state)
+ if form is None:
+ return
+ remembered = (
+ f"Кнопка запомнит: {_values_line(form['values'])}."
+ if form["values"]
+ else "Значения кнопка возьмёт из Komodo в момент запуска."
+ )
+ kb = InlineKeyboardBuilder()
+ ref = _ref(form)
+ kb.button(text="Запускать сразу", callback_data=f"act:pinmode:instant:{ref}")
+ kb.button(text="Открывать форму", callback_data=f"act:pinmode:form:{ref}")
+ kb.button(text="Отмена", callback_data=f"act:back:{ref}")
+ kb.adjust(2, 1)
+ await show(
+ callback,
+ f"Закрепить {esc(form['name'])} на главной.\n{remembered}",
+ kb.as_markup(),
+ )
+ await callback.answer()
+
+
+def pin_view(pin: Pin) -> tuple[str, types.InlineKeyboardMarkup]:
+ values = (
+ f"Значения: {_values_line(pin.args)}"
+ if pin.args
+ else "Значения: по умолчанию из Komodo"
+ )
+ lines = [
+ f"Кнопка {esc(pin.label)}",
+ f"{KIND_ONE[pin.kind]}: {esc(pin.name)}",
+ f"Режим: {MODE_TEXT[pin.mode]}",
+ values,
+ ]
+ kb = InlineKeyboardBuilder()
+ kb.button(text="Переименовать", callback_data=f"pin:ren:{pin.id}")
+ other = PinMode.FORM if pin.mode is PinMode.INSTANT else PinMode.INSTANT
+ kb.button(
+ text="Открывать форму" if other is PinMode.FORM else "Запускать сразу",
+ callback_data=f"pin:mode:{pin.id}",
+ )
+ kb.button(text="Выше", callback_data=f"pin:up:{pin.id}")
+ kb.button(text="Ниже", callback_data=f"pin:down:{pin.id}")
+ kb.button(text="Открепить", callback_data=f"pin:del:{pin.id}", style=DANGER)
+ kb.button(text="Закреплённые", callback_data="set:pins")
+ kb.button(text="Меню", callback_data="menu:main")
+ kb.adjust(2, 2, 1, 2)
+ return "\n".join(lines), kb.as_markup()
+
+
+@router.callback_query(F.data.startswith("act:pinmode:"))
+async def act_pin_mode(
+ callback: types.CallbackQuery, state: FSMContext, bot: Bot
+) -> None:
+ form = await _current(callback, state)
+ if form is None:
+ return
+ mode = PinMode((callback.data or "").split(":")[2])
+ pin = await db.pins.add(
+ kind=form["kind"],
+ target_id=form["id"],
+ name=form["name"],
+ label=form["name"][:MAX_LABEL],
+ mode=mode,
+ args=form["values"],
+ )
+ await sync_commands(bot)
+ text, markup = pin_view(pin)
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+async def pins_view() -> tuple[str, types.InlineKeyboardMarkup]:
+ pins = await db.pins.all()
+ kb = InlineKeyboardBuilder()
+ for pin in pins:
+ kb.button(text=label(pin.label, 40), callback_data=f"pin:open:{pin.id}")
+ if enabled():
+ kb.button(text="Действия", callback_data="act:list:a:0")
+ kb.button(text="Меню", callback_data="menu:main")
+ kb.adjust(*[1] * len(pins), 2)
+ text = "Закреплённые"
+ if not pins:
+ text += "\nПока пусто. Открой действие и нажми «Закрепить»."
+ return text, kb.as_markup()
+
+
+@router.callback_query(F.data == "set:pins")
+async def pins_list(callback: types.CallbackQuery, state: FSMContext) -> None:
+ await state.set_state(None)
+ text, markup = await pins_view()
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+async def _pin(callback: types.CallbackQuery) -> Pin | None:
+ pin = await db.pins.get(int_arg(callback.data))
+ if pin is None:
+ await callback.answer("Эта кнопка уже откреплена", show_alert=True)
+ return pin
+
+
+@router.callback_query(F.data.startswith("pin:open:"))
+async def pin_open(callback: types.CallbackQuery, state: FSMContext) -> None:
+ pin = await _pin(callback)
+ if pin is None:
+ return
+ await state.set_state(None)
+ text, markup = pin_view(pin)
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("pin:run:"))
+async def pin_run(callback: types.CallbackQuery, state: FSMContext) -> None:
+ pin = await _pin(callback)
+ if pin is None or not await _guard(callback):
+ return
+ if pin.mode is PinMode.FORM:
+ await _show_card(callback, state, pin.kind, pin.target_id, preset=pin.args)
+ return
+ await callback.answer()
+ await _execute(
+ callback, pin.kind, pin.target_id, pin.name, pin.args, title=pin.label, new=True
+ )
+
+
+@router.callback_query(F.data.startswith("pin:mode:"))
+async def pin_mode(callback: types.CallbackQuery) -> None:
+ pin = await _pin(callback)
+ if pin is None:
+ return
+ other = PinMode.FORM if pin.mode is PinMode.INSTANT else PinMode.INSTANT
+ await db.pins.set_mode(pin.id, other)
+ pin.mode = other
+ text, markup = pin_view(pin)
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+async def _move(callback: types.CallbackQuery, step: int) -> None:
+ pin = await _pin(callback)
+ if pin is None:
+ return
+ await db.pins.move(pin.id, step)
+ text, markup = await pins_view()
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("pin:up:"))
+async def pin_up(callback: types.CallbackQuery) -> None:
+ await _move(callback, -1)
+
+
+@router.callback_query(F.data.startswith("pin:down:"))
+async def pin_down(callback: types.CallbackQuery) -> None:
+ await _move(callback, 1)
+
+
+@router.callback_query(F.data.startswith("pin:del:"))
+async def pin_delete(callback: types.CallbackQuery, bot: Bot) -> None:
+ await db.pins.delete(int_arg(callback.data))
+ await sync_commands(bot)
+ text, markup = await pins_view()
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("pin:ren:"))
+async def pin_rename(callback: types.CallbackQuery, state: FSMContext) -> None:
+ pin = await _pin(callback)
+ if pin is None:
+ return
+ await state.set_state(ActionFlow.waiting_label)
+ await state.update_data(pin_id=pin.id)
+ await show(
+ callback,
+ f"Пришли новое название для кнопки {esc(pin.label)}.",
+ back_to("Отмена", f"pin:open:{pin.id}"),
+ )
+ await callback.answer()
+
+
+@router.message(ActionFlow.waiting_label, ANSWER)
+async def on_label(message: types.Message, state: FSMContext, bot: Bot) -> None:
+ text = " ".join((message.text or "").split())[:MAX_LABEL]
+ data = await state.get_data()
+ await state.set_state(None)
+ pin = await db.pins.get(data.get("pin_id", 0))
+ if pin is None or not text:
+ await show(message, "Эта кнопка уже откреплена.", back_to_menu())
+ return
+ await db.pins.rename(pin.id, text)
+ await sync_commands(bot)
+ pin.label = text
+ view, markup = pin_view(pin)
+ await show(message, view, markup)
+
+
+def _usage(detail: Detail, defaults: dict) -> str:
+ parts = [f"/{command_name(detail.name)}"]
+ for key, value in list(defaults.items())[:USAGE_FIELDS]:
+ parts += [key, '"текст"' if isinstance(value, str) else render(value)]
+ return f"Пример: {esc(' '.join(parts))}"
+
+
+async def _find(name: str) -> Resource | None:
+ wanted = command_name(name)
+ for kind in ("a", "p"):
+ try:
+ items = await komodo.resources(kind)
+ except KomodoError:
+ continue
+ for item in items:
+ if command_name(item.name) == wanted:
+ return item
+ return None
+
+
+commands = Router()
+commands.message.filter(Admin())
+
+
+@commands.message(F.text.startswith("/"))
+async def on_command(message: types.Message, state: FSMContext) -> None:
+ head, *tail = (message.text or "").split(maxsplit=1)
+ rest = tail[0] if tail else ""
+ name = head[1:].split("@")[0]
+ resource = await _find(name) if enabled() else None
+ if resource is not None and not db.settings.get("commands"):
+ await show(
+ message,
+ "Запуск действий командами выключен.",
+ back_to("Настройки", "set:main"),
+ )
+ return
+ if resource is None and not (enabled() and db.settings.get("commands")):
+ await show(message, "Не знаю такую команду.", back_to_menu())
+ return
+ if resource is None:
+ await show(
+ message, f"Действия {esc(name)} в Komodo нет.", back_to_menu()
+ )
+ return
+ await state.set_state(None)
+ try:
+ detail, _, defaults = await _load(resource.kind, resource.id)
+ args = changed(defaults, parse_cli(rest, defaults))
+ ensure_safe(args)
+ except ArgError as exc:
+ detail = await komodo.detail(resource.kind, resource.id)
+ defaults = parse_defaults(detail.args_format, detail.args_text)
+ await show(
+ message,
+ f"{esc(resource.name)}: {esc(exc)}\n{_usage(detail, defaults)}",
+ )
+ return
+ except KomodoError as exc:
+ await _fail(message, exc)
+ return
+ await _execute(message, resource.kind, resource.id, detail.name, args)
+
+
+@commands.message(F.text)
+async def on_unknown(message: types.Message) -> None:
+ await show(
+ message, "Не понял. Пришли ссылку на сервис или открой меню.", back_to_menu()
+ )
diff --git a/src/healthbot/handlers/add.py b/src/healthbot/handlers/add.py
index 624a35e..2ff1a2a 100644
--- a/src/healthbot/handlers/add.py
+++ b/src/healthbot/handlers/add.py
@@ -1,6 +1,6 @@
-"""Добавление сервиса: прислал ссылку → выбрал в меню за чем следить."""
-
-import html
+import asyncio
+import json
+import re
from urllib.parse import urlparse
import aiohttp
@@ -9,9 +9,12 @@ from aiogram.fsm.context import FSMContext
from aiogram.fsm.state import State, StatesGroup
from aiogram.utils.keyboard import InlineKeyboardBuilder
-from healthbot.checker import _scalar_str, probe
+from healthbot.checker import Probe, probe, scalar_str
from healthbot.filters import Admin
-from healthbot.keyboards import back_to_menu
+from healthbot.handlers.menu import dashboard
+from healthbot.handlers.services import show_card
+from healthbot.keyboards import back_to, back_to_menu
+from healthbot.ui import ANSWER, SUCCESS, esc, int_arg, label, show
from utils.db import db
from utils.db.models import MatchType
from utils.env import env
@@ -20,14 +23,51 @@ router = Router()
router.message.filter(Admin())
router.callback_query.filter(Admin())
-URL_RE = r"^https?://\S+$"
-MAX_CHILDREN = 40
+FULL_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE)
+BARE_URL_RE = re.compile(
+ r"(localhost|(\d{1,3}\.){3}\d{1,3}|[\w-]+(\.[\w-]+)*\.[a-z]{2,})(:\d+)?(/\S*)?",
+ re.IGNORECASE | re.ASCII,
+)
+MAX_CHILDREN = 30
+MAX_BULK = 20
PREVIEW_LEN = 30
+BODY_PREVIEW = 350
+HEALTH_KEYS = (
+ "status",
+ "ok",
+ "healthy",
+ "health",
+ "state",
+ "alive",
+ "up",
+ "ready",
+ "success",
+)
class AddFlow(StatesGroup):
waiting_url = State()
choosing = State()
+ waiting_needle = State()
+ waiting_new_url = State()
+
+
+def normalize(token: str) -> str | None:
+ if FULL_URL_RE.fullmatch(token):
+ return token
+ if BARE_URL_RE.fullmatch(token):
+ return f"https://{token}"
+ return None
+
+
+def extract_urls(text: str | None) -> list[str] | None:
+ tokens = (text or "").split()
+ if not tokens or len(tokens) > MAX_BULK:
+ return None
+ urls = [normalize(token) for token in tokens]
+ if any(url is None for url in urls):
+ return None
+ return list(dict.fromkeys(url for url in urls if url))
def _name_from_url(url: str) -> str:
@@ -45,7 +85,8 @@ def _node_at(root: object, path: list[str]) -> object:
def _children(node: object) -> list[tuple[str, object]]:
if isinstance(node, dict):
- return list(node.items())
+ kids: list[tuple[str, object]] = [(str(k), v) for k, v in node.items()]
+ return sorted(kids, key=lambda kid: kid[0].lower() not in HEALTH_KEYS)
if isinstance(node, list):
return [(str(i), v) for i, v in enumerate(node)]
return []
@@ -56,140 +97,298 @@ def _preview(value: object) -> str:
return "{…}"
if isinstance(value, list):
return "[…]"
- text = _scalar_str(value)
+ text = scalar_str(value)
return text if len(text) <= PREVIEW_LEN else text[: PREVIEW_LEN - 1] + "…"
def _choose_view(data: dict) -> tuple[str, types.InlineKeyboardMarkup]:
parsed = data["parsed"]
path: list[str] = data["path"]
- body_preview = html.escape(data["body"][:350]) or "(пусто)"
lines = [
- f"🔗 {html.escape(data['name'])}",
- f"{html.escape(data['url'])}",
- f"Ответ: HTTP {data['status']}",
- f"{body_preview}",
+ f"{esc(data['name'])}",
+ f"{esc(data['url'])}",
+ f"HTTP {data['status']}, отклик {data['latency']} мс",
+ f"{esc(data['body'][:BODY_PREVIEW]) or 'пустой ответ'}",
]
kb = InlineKeyboardBuilder()
+ rows: list[int] = []
if parsed is not None:
- crumb = " › ".join(["корень", *path])
- lines.append(f"За каким полем следить? {html.escape(crumb)}")
- node = _node_at(parsed, path)
- kids = _children(node)
+ crumb = " › ".join(["ответ", *path])
+ lines.append(f"За каким полем следить? {esc(crumb)}")
+ kids = _children(_node_at(parsed, path))
for i, (key, value) in enumerate(kids[:MAX_CHILDREN]):
- kb.button(text=f"{key}: {_preview(value)}", callback_data=f"add:nav:{i}")
+ healthy = not path and key.lower() in HEALTH_KEYS
+ kb.button(
+ text=label(f"{key}: {_preview(value)}"),
+ callback_data=f"add:nav:{i}",
+ style=SUCCESS
+ if healthy and not isinstance(value, (dict, list))
+ else None,
+ )
+ rows += [1] * min(len(kids), MAX_CHILDREN)
if len(kids) > MAX_CHILDREN:
- lines.append(f"…показаны первые {MAX_CHILDREN} полей")
+ lines.append(f"Показаны первые {MAX_CHILDREN} полей.")
else:
- lines.append("Ответ не JSON - можно следить за статусом или телом целиком.")
+ lines.append("За чем следить?")
if path:
- kb.button(text="⬆️ Назад", callback_data="add:up")
- kb.button(text="🔢 Только HTTP-статус", callback_data="add:status")
- kb.button(text="📄 Всё тело целиком", callback_data="add:body")
- kb.button(text="❌ Отмена", callback_data="add:cancel")
- kb.adjust(1)
+ kb.button(text="Назад", callback_data="add:up")
+ rows.append(1)
+ kb.button(text=f"HTTP-статус {data['status']}", callback_data="add:status")
+ kb.button(text="Текст в ответе", callback_data="add:text")
+ kb.button(text="Тело целиком", callback_data="add:body")
+ kb.button(text="Отмена", callback_data="add:cancel")
+ kb.adjust(*rows, 2, 2)
return "\n".join(lines), kb.as_markup()
+async def _probe(url: str) -> Probe:
+ async with aiohttp.ClientSession() as session:
+ return await probe(session, url, env.monitor.request_timeout)
+
+
+def _unreachable_kb(*, editing: bool) -> types.InlineKeyboardMarkup:
+ kb = InlineKeyboardBuilder()
+ kb.button(text="Повторить", callback_data="add:retry")
+ if not editing:
+ kb.button(text="Добавить всё равно", callback_data="add:force")
+ kb.button(text="Отмена", callback_data="add:cancel")
+ kb.adjust(2, 1)
+ return kb.as_markup()
+
+
+async def _begin(
+ event: types.Message | types.CallbackQuery,
+ state: FSMContext,
+ url: str,
+ *,
+ edit_id: int | None = None,
+ name: str | None = None,
+) -> None:
+ result = await _probe(url)
+ name = name or _name_from_url(url)
+ await state.set_state(AddFlow.choosing)
+ await state.set_data({"url": url, "name": name, "edit_id": edit_id})
+
+ if not result.reachable:
+ hint = "" if edit_id else "\nМожно добавить и так: буду ждать HTTP 200."
+ await show(
+ event,
+ f"{esc(name)} не отвечает: {esc(result.error or '')}.{hint}",
+ _unreachable_kb(editing=edit_id is not None),
+ )
+ return
+
+ try:
+ parsed = json.loads(result.body)
+ except ValueError:
+ parsed = None
+
+ await state.update_data(
+ status=result.status,
+ latency=result.latency_ms,
+ body=result.body,
+ parsed=parsed if isinstance(parsed, (dict, list)) else None,
+ path=[],
+ )
+ text, markup = _choose_view(await state.get_data())
+ await show(event, text, markup)
+
+
@router.callback_query(F.data == "menu:add")
async def on_add_menu(callback: types.CallbackQuery, state: FSMContext) -> None:
await state.set_state(AddFlow.waiting_url)
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(
- "Пришли ссылку на сервис "
- "(например https://example.com/health) - "
- "я запрошу её и предложу выбрать, за чем следить.",
- reply_markup=back_to_menu(),
- )
+ await show(
+ callback,
+ "Пришли ссылку на сервис, например example.com/health.\n"
+ "Несколько ссылок через пробел или с новой строки добавлю разом.",
+ back_to_menu(),
+ )
await callback.answer()
-@router.message(F.text.regexp(URL_RE))
-async def on_url(message: types.Message, state: FSMContext) -> None:
- url = (message.text or "").strip()
+async def _add_many(message: types.Message, urls: list[str]) -> None:
+ fresh = [url for url in urls if await db.services.by_url(url) is None]
+ results = await asyncio.gather(*(_probe(url) for url in fresh))
+ added, skipped = [], [f"{esc(u)}: уже есть" for u in urls if u not in fresh]
+ for url, result in zip(fresh, results, strict=True):
+ if not result.reachable or result.status is None:
+ skipped.append(f"{esc(url)}: {esc(result.error or 'нет ответа')}")
+ continue
+ service = await db.services.add(
+ name=_name_from_url(url),
+ url=url,
+ match_type=MatchType.STATUS,
+ ok_status=result.status,
+ )
+ added.append(f"🟢 {esc(service.name)} · HTTP {result.status}")
+ lines = [f"Добавил {len(added)} из {len(urls)}, слежу за HTTP-статусом.", *added]
+ if skipped:
+ lines += ["", "Пропустил:", *skipped]
+ await show(message, "\n".join(lines), back_to("Сервисы", "menu:list:0"))
- if await db.services.by_url(url) is not None:
- await message.answer("Этот сервис уже под наблюдением. /list - список.")
- return
- note = await message.answer("Запрашиваю…")
- async with aiohttp.ClientSession() as session:
- result = await probe(session, url, env.monitor.request_timeout)
-
- if not result.reachable:
+async def _on_urls(message: types.Message, state: FSMContext, urls: list[str]) -> None:
+ if len(urls) > 1:
await state.clear()
- await note.edit_text(
- f"Не смог достучаться: {html.escape(result.error or '')}.\n"
- "Эталон снимаю только с живого сервиса - попробуй, когда он поднимется.",
- reply_markup=back_to_menu(),
+ await _add_many(message, urls)
+ return
+ existing = await db.services.by_url(urls[0])
+ if existing is not None:
+ await state.clear()
+ await show_card(message, existing)
+ return
+ await _begin(message, state, urls[0])
+
+
+@router.message(AddFlow.waiting_url, ANSWER)
+async def on_url_expected(message: types.Message, state: FSMContext) -> None:
+ urls = extract_urls(message.text)
+ if urls is None:
+ await show(
+ message,
+ "Это не похоже на ссылку. Пример: example.com/health",
+ back_to_menu(),
)
return
+ await _on_urls(message, state, urls)
- try:
- import json # noqa: PLC0415
- parsed = json.loads(result.body)
- except ValueError:
- parsed = None
+@router.message(AddFlow.waiting_new_url, ANSWER)
+async def on_new_url(message: types.Message, state: FSMContext) -> None:
+ data = await state.get_data()
+ service = await db.services.get(data["edit_id"])
+ if service is None:
+ await state.clear()
+ await show(message, "Сервис уже удалён.", back_to_menu())
+ return
+ urls = extract_urls(message.text)
+ back = back_to("Отмена", f"svc:open:{service.id}")
+ if urls is None or len(urls) != 1:
+ await show(message, "Нужна одна ссылка.", back)
+ return
+ other = await db.services.by_url(urls[0])
+ if other is not None and other.id != service.id:
+ await show(message, f"Этот адрес уже у {esc(other.name)}.", back)
+ return
+ await state.clear()
+ await db.services.set_url(service.id, urls[0])
+ updated = await db.services.get(service.id)
+ if updated is not None:
+ await show_card(message, updated)
- await state.set_state(AddFlow.choosing)
- await state.set_data(
- {
- "url": url,
- "name": _name_from_url(url),
- "status": result.status,
- "body": result.body,
- "parsed": parsed,
- "path": [],
- }
+
+@router.callback_query(F.data.startswith("svc:url:"))
+async def svc_url(callback: types.CallbackQuery, state: FSMContext) -> None:
+ service = await db.services.get(int_arg(callback.data))
+ if service is None:
+ await callback.answer("Сервис не найден", show_alert=True)
+ return
+ await state.set_state(AddFlow.waiting_new_url)
+ await state.set_data({"edit_id": service.id})
+ await show(
+ callback,
+ f"Пришли новый адрес для {esc(service.name)}.\n"
+ f"Сейчас: {esc(service.url)}",
+ back_to("Отмена", f"svc:open:{service.id}"),
)
- text, markup = _choose_view(await state.get_data())
- await note.edit_text(text, reply_markup=markup)
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("svc:rule:"))
+async def svc_rule(callback: types.CallbackQuery, state: FSMContext) -> None:
+ service = await db.services.get(int_arg(callback.data))
+ if service is None:
+ await callback.answer("Сервис не найден", show_alert=True)
+ return
+ await callback.answer()
+ await _begin(callback, state, service.url, edit_id=service.id, name=service.name)
async def _save( # noqa: PLR0913
- callback: types.CallbackQuery,
+ event: types.Message | types.CallbackQuery,
state: FSMContext,
*,
match_type: MatchType,
ok_body: str = "",
json_path: str | None = None,
json_value: str | None = None,
+ needle: str | None = None,
) -> None:
data = await state.get_data()
- service = await db.services.add(
- name=data["name"],
- url=data["url"],
- match_type=match_type,
- ok_status=data["status"],
- ok_body=ok_body,
- json_path=json_path,
- json_value=json_value,
- )
await state.clear()
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(
- f"✅ Слежу за {html.escape(service.name)}\n"
- f"{html.escape(service.url)}\n"
- f"Правило: {service.rule}\n"
- f"Проверяю раз в {env.monitor.check_interval}с.",
- reply_markup=back_to_menu(),
- )
- await callback.answer("Готово")
+ rule = {
+ "match_type": match_type,
+ "ok_status": data.get("status") or 200,
+ "ok_body": ok_body,
+ "json_path": json_path,
+ "json_value": json_value,
+ "needle": needle,
+ }
+ if data.get("edit_id"):
+ await db.services.set_rule(data["edit_id"], **rule)
+ service = await db.services.get(data["edit_id"])
+ else:
+ service = await db.services.add(name=data["name"], url=data["url"], **rule)
+ if service is None:
+ await show(event, "Сервис уже удалён.", back_to_menu())
+ return
+ await show_card(event, service)
@router.callback_query(AddFlow.choosing, F.data == "add:status")
async def on_pick_status(callback: types.CallbackQuery, state: FSMContext) -> None:
await _save(callback, state, match_type=MatchType.STATUS)
+ await callback.answer()
+
+
+@router.callback_query(AddFlow.choosing, F.data == "add:force")
+async def on_force(callback: types.CallbackQuery, state: FSMContext) -> None:
+ await _save(callback, state, match_type=MatchType.STATUS)
+ await callback.answer()
+
+
+@router.callback_query(AddFlow.choosing, F.data == "add:retry")
+async def on_retry(callback: types.CallbackQuery, state: FSMContext) -> None:
+ data = await state.get_data()
+ await callback.answer()
+ await _begin(
+ callback, state, data["url"], edit_id=data.get("edit_id"), name=data["name"]
+ )
@router.callback_query(AddFlow.choosing, F.data == "add:body")
async def on_pick_body(callback: types.CallbackQuery, state: FSMContext) -> None:
data = await state.get_data()
await _save(callback, state, match_type=MatchType.BODY, ok_body=data["body"])
+ await callback.answer()
+
+
+@router.callback_query(AddFlow.choosing, F.data == "add:text")
+async def on_pick_text(callback: types.CallbackQuery, state: FSMContext) -> None:
+ await state.set_state(AddFlow.waiting_needle)
+ await show(
+ callback,
+ "Пришли текст, который должен быть в ответе.",
+ back_to("Отмена", "add:cancel"),
+ )
+ await callback.answer()
+
+
+@router.message(AddFlow.waiting_needle, ANSWER)
+async def on_needle(message: types.Message, state: FSMContext) -> None:
+ needle = message.text or ""
+ data = await state.get_data()
+ if needle not in data["body"]:
+ await show(
+ message,
+ "Сейчас такого текста в ответе нет. Пришли другой.",
+ back_to("Отмена", "add:cancel"),
+ )
+ return
+ await _save(message, state, match_type=MatchType.CONTAINS, needle=needle)
@router.callback_query(AddFlow.choosing, F.data == "add:up")
@@ -204,9 +403,8 @@ async def on_up(callback: types.CallbackQuery, state: FSMContext) -> None:
@router.callback_query(AddFlow.choosing, F.data.startswith("add:nav:"))
async def on_nav(callback: types.CallbackQuery, state: FSMContext) -> None:
data = await state.get_data()
- index = int((callback.data or "").rsplit(":", 1)[-1])
- node = _node_at(data["parsed"], data["path"])
- kids = _children(node)
+ kids = _children(_node_at(data["parsed"], data["path"]))
+ index = int_arg(callback.data)
if index >= len(kids):
await callback.answer()
return
@@ -221,20 +419,39 @@ async def on_nav(callback: types.CallbackQuery, state: FSMContext) -> None:
state,
match_type=MatchType.JSON,
json_path=".".join([*data["path"], key]),
- json_value=_scalar_str(value),
+ json_value=scalar_str(value),
)
-
-
-@router.callback_query(AddFlow.choosing, F.data == "add:cancel")
-async def on_cancel(callback: types.CallbackQuery, state: FSMContext) -> None:
- await state.clear()
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text("Отменено.", reply_markup=back_to_menu())
await callback.answer()
+@router.callback_query(F.data == "add:cancel")
+async def on_cancel(callback: types.CallbackQuery, state: FSMContext) -> None:
+ data = await state.get_data()
+ await state.clear()
+ service = await db.services.get(data["edit_id"]) if data.get("edit_id") else None
+ if service is not None:
+ await show_card(callback, service)
+ else:
+ text, markup = await dashboard()
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("add:"))
+async def on_stale(callback: types.CallbackQuery) -> None:
+ await callback.answer("Эта форма устарела, пришли ссылку заново", show_alert=True)
+
+
async def _rerender(callback: types.CallbackQuery, state: FSMContext) -> None:
text, markup = _choose_view(await state.get_data())
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(text, reply_markup=markup)
+ await show(callback, text, markup)
await callback.answer()
+
+
+free = Router()
+free.message.filter(Admin())
+
+
+@free.message(F.text.func(lambda text: extract_urls(text) is not None))
+async def on_url(message: types.Message, state: FSMContext) -> None:
+ await _on_urls(message, state, extract_urls(message.text) or [])
diff --git a/src/healthbot/handlers/commands.py b/src/healthbot/handlers/commands.py
deleted file mode 100644
index 5b2eac3..0000000
--- a/src/healthbot/handlers/commands.py
+++ /dev/null
@@ -1,358 +0,0 @@
-"""Меню, список сервисов, карточка, статистика, ручная проверка, удаление."""
-
-import html
-import json as jsonlib
-
-import aiohttp
-from aiogram import Bot, F, Router, types
-from aiogram.filters import Command, CommandStart
-from aiogram.fsm.context import FSMContext
-from aiogram.fsm.state import State, StatesGroup
-
-from healthbot.checker import _scalar_str, dig, evaluate, probe, snippet
-from healthbot.filters import Admin
-from healthbot.keyboards import (
- after_check,
- back_to_menu,
- confirm_delete,
- main_menu,
- service_card,
- services_list,
-)
-from utils.db import db
-from utils.db.models import MatchType, Service
-from utils.env import env
-from utils.format import human_duration, now, parse
-from utils.logging import logger
-
-router = Router()
-router.message.filter(Admin())
-router.callback_query.filter(Admin())
-
-MAX_NAME = 64
-
-GREETING = (
- "👋 healthbot - слежу за твоими сервисами и пишу, если что-то легло.\n\n"
- "Пришли ссылку на health-эндпоинт или жми кнопку 👇"
-)
-
-
-class RenameFlow(StatesGroup):
- waiting_name = State()
-
-
-@router.startup()
-async def on_startup(bot: Bot) -> None:
- commands = [
- types.BotCommand(command="start", description="Меню"),
- types.BotCommand(command="list", description="Сервисы"),
- types.BotCommand(command="stats", description="Статистика"),
- ]
- if env.komodo.enabled:
- commands.append(types.BotCommand(command="fleet", description="Флот"))
- await bot.set_my_commands(commands)
- logger.info(f"[green]Started as[/] @{(await bot.me()).username}")
-
-
-@router.shutdown()
-async def on_shutdown() -> None:
- logger.info("Shutting down bot...")
-
-
-@router.message(CommandStart())
-async def cmd_start(message: types.Message) -> None:
- await message.answer(GREETING, reply_markup=main_menu())
-
-
-@router.callback_query(F.data == "menu:main")
-async def menu_main(callback: types.CallbackQuery) -> None:
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(GREETING, reply_markup=main_menu())
- await callback.answer()
-
-
-async def _list_view() -> tuple[str, types.InlineKeyboardMarkup]:
- services = await db.services.all()
- if not services:
- return "Пока ни одного сервиса. Пришли ссылку, чтобы добавить.", back_to_menu()
- watched = [s for s in services if not s.is_paused]
- up = sum(s.is_up for s in watched)
- paused = len(services) - len(watched)
- tail = f", ⏸ {paused}" if paused else ""
- text = f"📋 Сервисы - 🟢 {up} / 🔴 {len(watched) - up}{tail}\nТыкни, чтобы открыть."
- return text, services_list(services)
-
-
-@router.message(Command("list"))
-async def cmd_list(message: types.Message) -> None:
- text, markup = await _list_view()
- await message.answer(text, reply_markup=markup)
-
-
-@router.callback_query(F.data == "menu:list")
-async def menu_list(callback: types.CallbackQuery) -> None:
- text, markup = await _list_view()
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(text, reply_markup=markup)
- await callback.answer()
-
-
-def _downtime(incidents: list) -> float:
- total = 0.0
- for inc in incidents:
- end = parse(inc.ended_at) if inc.ended_at else now()
- total += (end - parse(inc.started_at)).total_seconds()
- return total
-
-
-async def _card_view(service: Service) -> str:
- incidents = await db.incidents.for_service(service.id)
- down = _downtime(incidents)
- window = (now() - parse(service.created_at)).total_seconds()
- uptime = 100.0 if window <= 0 else max(0.0, (window - down) / window * 100)
-
- if service.is_paused:
- status = "⏸ на паузе - не проверяется"
- elif service.is_up:
- status = "🟢 работает"
- if service.fail_streak:
- status += (
- f"\n ⚠️ неудач подряд: {service.fail_streak}"
- f"/{env.monitor.failure_threshold}"
- )
- else:
- current = await db.incidents.current(service.id)
- since = (
- human_duration((now() - parse(current.started_at)).total_seconds())
- if current
- else "?"
- )
- reason = html.escape(current.reason) if current else ""
- status = f"🔴 лежит уже {since}\n причина: {reason}"
-
- latency = (
- f"{service.last_latency_ms} мс" if service.last_latency_ms is not None else "-"
- )
-
- return (
- f"{html.escape(service.name)}\n"
- f"{html.escape(service.url)}\n\n"
- f"Правило: {service.rule}\n"
- f"Статус: {status}\n"
- f"Отклик: {latency}\n\n"
- f"📊 Аптайм: {uptime:.2f}%\n"
- f"Инцидентов: {len(incidents)}\n"
- f"Суммарный даунтайм: {human_duration(down) if down else '0с'}"
- )
-
-
-async def _show_card(callback: types.CallbackQuery, service: Service) -> None:
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(
- await _card_view(service), reply_markup=service_card(service)
- )
-
-
-async def _load(callback: types.CallbackQuery) -> Service | None:
- service_id = int((callback.data or "").rsplit(":", 1)[-1])
- service = await db.services.get(service_id)
- if service is None:
- await callback.answer("Сервис не найден", show_alert=True)
- return service
-
-
-@router.callback_query(F.data.startswith("svc:open:"))
-async def svc_open(callback: types.CallbackQuery) -> None:
- service = await _load(callback)
- if service is None:
- return
- await _show_card(callback, service)
- await callback.answer()
-
-
-@router.callback_query(F.data.startswith("svc:check:"))
-async def svc_check(callback: types.CallbackQuery) -> None:
- """Проверить прямо сейчас и показать, что сервис отдаёт."""
- service = await _load(callback)
- if service is None:
- return
- await callback.answer("Запрашиваю…")
-
- async with aiohttp.ClientSession() as session:
- result = await probe(session, service.url, env.monitor.request_timeout)
- verdict = evaluate(service, result)
-
- head = "✅ Правило выполняется" if verdict.healthy else "🔴 Правило нарушено"
- lines = [
- f"{html.escape(service.name)}",
- f"{html.escape(service.url)}",
- "",
- head,
- f"HTTP: {result.status if result.reachable else '-'}"
- f" отклик: {result.latency_ms if result.latency_ms is not None else '-'} мс",
- ]
- if not verdict.healthy:
- lines.append(f"Причина: {html.escape(verdict.reason)}")
- report = verdict.report()
- if report:
- lines.append(report)
- elif result.reachable:
- lines.extend(snippet(result.body))
-
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(
- "\n".join(lines),
- reply_markup=after_check(
- service.id, offer_baseline=not verdict.healthy and result.reachable
- ),
- )
-
-
-@router.callback_query(F.data.startswith("svc:base:"))
-async def svc_baseline(callback: types.CallbackQuery) -> None:
- """Принять текущий ответ за новый эталон."""
- service = await _load(callback)
- if service is None:
- return
- async with aiohttp.ClientSession() as session:
- result = await probe(session, service.url, env.monitor.request_timeout)
- if not result.reachable:
- await callback.answer("Сервис недоступен - эталон снимать не с чего", True) # noqa: FBT003
- return
-
- json_value = service.json_value
- if service.match_type is MatchType.JSON:
- try:
- value = dig(jsonlib.loads(result.body), service.json_path or "")
- except ValueError:
- await callback.answer("Ответ не JSON - эталон не обновлён", True) # noqa: FBT003
- return
- json_value = _scalar_str(value)
-
- await db.services.set_baseline(
- service.id,
- ok_status=result.status or service.ok_status,
- ok_body=(
- result.body if service.match_type is MatchType.BODY else service.ok_body
- ),
- json_value=json_value,
- )
- await db.services.set_streak(service.id, 0)
- updated = await db.services.get(service.id)
- if updated is not None:
- await _show_card(callback, updated)
- await callback.answer("Эталон обновлён")
-
-
-@router.callback_query(F.data.startswith("svc:pause:"))
-async def svc_pause(callback: types.CallbackQuery) -> None:
- service = await _load(callback)
- if service is None:
- return
- await db.services.set_paused(service.id, is_paused=True)
- updated = await db.services.get(service.id)
- if updated is not None:
- await _show_card(callback, updated)
- await callback.answer("На паузе")
-
-
-@router.callback_query(F.data.startswith("svc:resume:"))
-async def svc_resume(callback: types.CallbackQuery) -> None:
- service = await _load(callback)
- if service is None:
- return
- await db.services.set_paused(service.id, is_paused=False)
- updated = await db.services.get(service.id)
- if updated is not None:
- await _show_card(callback, updated)
- await callback.answer("Снова слежу")
-
-
-@router.callback_query(F.data.startswith("svc:rename:"))
-async def svc_rename(callback: types.CallbackQuery, state: FSMContext) -> None:
- service = await _load(callback)
- if service is None:
- return
- await state.set_state(RenameFlow.waiting_name)
- await state.set_data({"service_id": service.id})
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(
- f"Как назвать вместо {html.escape(service.name)}?\n"
- "Пришли новое имя сообщением.",
- reply_markup=back_to_menu(),
- )
- await callback.answer()
-
-
-@router.message(RenameFlow.waiting_name)
-async def on_new_name(message: types.Message, state: FSMContext) -> None:
- name = (message.text or "").strip()[:MAX_NAME]
- if not name:
- await message.answer("Пустое имя не годится.")
- return
- data = await state.get_data()
- await state.clear()
- await db.services.rename(data["service_id"], name)
- service = await db.services.get(data["service_id"])
- if service is None:
- await message.answer("Сервис успел удалиться.")
- return
- await message.answer(await _card_view(service), reply_markup=service_card(service))
-
-
-@router.callback_query(F.data.startswith("svc:del:"))
-async def svc_del(callback: types.CallbackQuery) -> None:
- service_id = int((callback.data or "").rsplit(":", 1)[-1])
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(
- "Точно удалить сервис вместе со всей статистикой?",
- reply_markup=confirm_delete(service_id),
- )
- await callback.answer()
-
-
-@router.callback_query(F.data.startswith("svc:delyes:"))
-async def svc_del_yes(callback: types.CallbackQuery) -> None:
- service_id = int((callback.data or "").rsplit(":", 1)[-1])
- await db.services.delete(service_id)
- text, markup = await _list_view()
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(text, reply_markup=markup)
- await callback.answer("Удалено")
-
-
-async def _stats_view() -> tuple[str, types.InlineKeyboardMarkup]:
- services = await db.services.all()
- if not services:
- return "Пока нет данных.", back_to_menu()
- lines = ["📊 Статистика\n"]
- for service in services:
- incidents = await db.incidents.for_service(service.id)
- down = _downtime(incidents)
- window = (now() - parse(service.created_at)).total_seconds()
- uptime = 100.0 if window <= 0 else max(0.0, (window - down) / window * 100)
- latency = (
- f", отклик {service.last_latency_ms} мс"
- if service.last_latency_ms is not None
- else ""
- )
- lines.append(
- f"{service.mark} {html.escape(service.name)} - "
- f"аптайм {uptime:.2f}%, инцидентов {len(incidents)}, "
- f"даунтайм {human_duration(down) if down else '0с'}{latency}"
- )
- return "\n".join(lines), back_to_menu()
-
-
-@router.message(Command("stats"))
-async def cmd_stats(message: types.Message) -> None:
- text, markup = await _stats_view()
- await message.answer(text, reply_markup=markup)
-
-
-@router.callback_query(F.data == "menu:stats")
-async def menu_stats(callback: types.CallbackQuery) -> None:
- text, markup = await _stats_view()
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(text, reply_markup=markup)
- await callback.answer()
diff --git a/src/healthbot/handlers/fleet.py b/src/healthbot/handlers/fleet.py
index b3b9ac8..b57dba9 100644
--- a/src/healthbot/handlers/fleet.py
+++ b/src/healthbot/handlers/fleet.py
@@ -1,27 +1,43 @@
-"""Флот из телеграма: посмотреть сервера и стеки, нажать Deploy с телефона."""
-
-import html
+import asyncio
+import re
+from collections.abc import Awaitable
from aiogram import F, Router, types
from aiogram.filters import Command
+from aiogram.fsm.context import FSMContext
+from aiogram.types import BufferedInputFile
from aiogram.utils.keyboard import InlineKeyboardBuilder
+from healthbot.alerts import format_alert
from healthbot.filters import Admin
-from healthbot.keyboards import back_to_menu
-from healthbot.komodo import KomodoError, Stack, komodo
+from healthbot.keyboards import back_to, back_to_menu
+from healthbot.komodo import KomodoError, Server, Stack, Update, komodo
+from healthbot.runs import track
+from healthbot.ui import DANGER, PRIMARY, esc, label, pager, pages, pre, show
+from utils.db import db
+from utils.format import from_ms, stamp
from utils.logging import logger
router = Router()
router.message.filter(Admin())
router.callback_query.filter(Admin())
-LOG_TAIL = 30
-LOG_CHARS = 2500
+LOG_TAIL = 40
+LOG_FILE_TAIL = 1000
+LOG_CHARS = 3200
+STACKS_PAGE = 20
+HISTORY = 12
+JOURNAL = 15
+FAILED_BUTTONS = 5
+ALL_SERVICES = -1
+SMALL_DISK_GB = 2
+LOG_PREFIX_RE = re.compile(r"^\S+\s+\| ", re.MULTILINE)
-ACTIONS: dict[str, tuple[str, str, str]] = {
- "dep": ("🚀 Deploy", "DeployStack", "Деплой запущен"),
- "res": ("🔄 Restart", "RestartStack", "Перезапуск запущен"),
- "sto": ("⏹ Stop", "StopStack", "Остановка запущена"),
+OPS: dict[str, tuple[str, str, str | None]] = {
+ "dep": ("Deploy", "DeployStack", PRIMARY),
+ "res": ("Restart", "RestartStack", None),
+ "sto": ("Stop", "StopStack", DANGER),
+ "sta": ("Start", "StartStack", None),
}
@@ -32,123 +48,317 @@ async def _guard(callback: types.CallbackQuery) -> bool:
return False
-def _fleet_kb(
- stacks: list[Stack], servers: dict[str, str]
-) -> types.InlineKeyboardMarkup:
- kb = InlineKeyboardBuilder()
- for stack in sorted(stacks, key=lambda s: (servers.get(s.server_id, ""), s.name)):
- kb.button(
- text=f"{stack.mark} {stack.name}", callback_data=f"fleet:stack:{stack.id}"
- )
- kb.button(text="🔄 Обновить", callback_data="fleet:main")
- kb.button(text="⬅️ Меню", callback_data="menu:main")
- kb.adjust(1)
- return kb.as_markup()
+def _usage(server: Server) -> str:
+ parts = []
+ if server.cpu is not None:
+ parts.append(f"cpu {server.cpu:.0f}%")
+ if server.mem is not None:
+ parts.append(f"память {server.mem:.0f}%")
+ if server.disk is not None:
+ parts.append(f"диск {server.disk:.0f}%")
+ return ", ".join(parts)
-async def _fleet_view() -> tuple[str, types.InlineKeyboardMarkup]:
- servers = await komodo.servers()
- stacks = await komodo.stacks()
- by_id = {s.id: s.name for s in servers}
-
- lines = ["🛰 Флот", ""]
- for server in sorted(servers, key=lambda s: s.name):
- mine = [s for s in stacks if s.server_id == server.id]
- good = sum(s.state == "running" for s in mine)
- lines.append(
- f"{server.mark} {html.escape(server.name)} - "
- f"{good}/{len(mine)} стеков в строю"
- )
-
- broken = [s for s in stacks if s.state not in {"running", "unknown"}]
- if broken:
- lines.append("")
- lines.append("Не в строю:")
- lines.extend(
- f"{s.mark} {html.escape(s.name)} - {html.escape(s.state)}" for s in broken
- )
-
- return "\n".join(lines), _fleet_kb(stacks, by_id)
+def _stack_count(stacks: list[Stack]) -> str:
+ good = sum(s.healthy for s in stacks)
+ if good == len(stacks):
+ return f"стеков {len(stacks)}, все работают"
+ return f"стеков {len(stacks)}, работают {good}"
-async def _stack_view(stack_id: str) -> tuple[str, types.InlineKeyboardMarkup]:
- stacks = await komodo.stacks()
- stack = next((s for s in stacks if s.id == stack_id), None)
- if stack is None:
- return "Стек не найден - возможно, его удалили.", back_to_menu()
-
- servers = {s.id: s.name for s in await komodo.servers()}
- text = (
- f"{stack.mark} {html.escape(stack.name)}\n"
- f"Состояние: {html.escape(stack.state or '?')}\n"
- f"Сервер: {html.escape(servers.get(stack.server_id, '?'))}"
+async def fleet_view() -> tuple[str, types.InlineKeyboardMarkup]:
+ servers, stacks, alerts = await asyncio.gather(
+ komodo.servers(), komodo.stacks(), komodo.open_alerts()
)
+ servers.sort(key=lambda s: s.name)
+ lines = [f"Флот: серверов {len(servers)}, стеков {len(stacks)}", ""]
+ kb = InlineKeyboardBuilder()
+ for server in servers:
+ mine = [s for s in stacks if s.server_id == server.id]
+ if server.state == "NotOk":
+ lines.append(f"{server.mark} {esc(server.name)} не отвечает")
+ elif server.state == "Disabled":
+ lines.append(f"{server.mark} {esc(server.name)} выключен в Komodo")
+ else:
+ usage = _usage(server)
+ lines.append(
+ f"{server.mark} {esc(server.name)}"
+ + (f" · {usage}" if usage else "")
+ )
+ lines.append(_stack_count(mine) if mine else "стеков нет")
+ kb.button(
+ text=f"{server.mark} {label(server.name, 28)}",
+ callback_data=f"fleet:srv:{server.id}:0",
+ )
+ rows = [2] * (len(servers) // 2) + [1] * (len(servers) % 2)
+
+ broken = [s for s in stacks if not s.healthy]
+ extra = 0
+ if broken:
+ kb.button(text=f"Не работают: {len(broken)}", callback_data="fleet:bad")
+ extra += 1
+ if alerts:
+ kb.button(text=f"Алерты: {len(alerts)}", callback_data="fleet:alerts")
+ extra += 1
+ kb.button(text="Журнал", callback_data="fleet:journal")
+ kb.button(text="Обновить", callback_data="fleet:main")
+ kb.button(text="Меню", callback_data="menu:main")
+ kb.adjust(*rows, *([extra] if extra else []), 3)
+ return "\n".join(lines), kb.as_markup()
+
+
+def _stack_buttons(kb: InlineKeyboardBuilder, stacks: list[Stack]) -> list[int]:
+ for stack in stacks:
+ kb.button(
+ text=f"{stack.mark} {label(stack.name, 28)}",
+ callback_data=f"fleet:stack:{stack.id}",
+ )
+ return [2] * (len(stacks) // 2) + [1] * (len(stacks) % 2)
+
+
+async def server_view(
+ server_id: str, page: int
+) -> tuple[str, types.InlineKeyboardMarkup]:
+ servers, stacks = await asyncio.gather(komodo.servers(), komodo.stacks())
+ server = next((s for s in servers if s.id == server_id), None)
+ if server is None:
+ return "Сервер не найден.", back_to("К флоту", "fleet:main")
+
+ lines = [f"{server.mark} {esc(server.name)}"]
+ if server.state == "NotOk":
+ lines.append("не отвечает")
+ facts = []
+ if server.version:
+ facts.append(f"Komodo {esc(server.version)}")
+ if server.cpu is not None:
+ facts.append(f"cpu {server.cpu:.0f}%")
+ if server.load is not None:
+ facts.append(f"load {server.load:.2f}")
+ if facts:
+ lines.append(" · ".join(facts))
+ if server.mem is not None:
+ lines.append(
+ f"Память: {server.mem_used:.1f} из {server.mem_total:.1f} ГБ"
+ f" ({server.mem:.0f}%)"
+ )
+ try:
+ disks = await komodo.disks(server_id)
+ except KomodoError:
+ disks = []
+ for mount, used, total in [d for d in disks if d[2] >= SMALL_DISK_GB][:4]:
+ lines.append(
+ f"Диск {esc(mount)}: {used:.1f} из {total:.1f} ГБ"
+ f" ({used / total * 100:.0f}%)"
+ )
+
+ mine = sorted(
+ (s for s in stacks if s.server_id == server_id),
+ key=lambda s: (s.healthy, s.name),
+ )
+ lines += ["", _stack_count(mine) if mine else "стеков нет"]
+ total_pages = pages(len(mine), STACKS_PAGE)
+ page = min(page, total_pages - 1)
+ kb = InlineKeyboardBuilder()
+ rows = _stack_buttons(kb, mine[page * STACKS_PAGE : (page + 1) * STACKS_PAGE])
+ nav = pager(kb, f"fleet:srv:{server_id}", page, total_pages)
+ kb.button(text="Обновить", callback_data=f"fleet:srv:{server_id}:{page}")
+ kb.button(text="К флоту", callback_data="fleet:main")
+ kb.adjust(*rows, *([nav] if nav else []), 2)
+ return "\n".join(lines), kb.as_markup()
+
+
+async def stack_view(stack_id: str) -> tuple[str, types.InlineKeyboardMarkup]:
+ stack = await komodo.stack(stack_id)
+ if stack is None:
+ return "Стек не найден, возможно его удалили.", back_to("К флоту", "fleet:main")
+
+ lines = [
+ f"{stack.mark} {esc(stack.name)}",
+ f"Сервер: {esc(stack.server_name or 'не назначен')}",
+ f"Состояние: {esc(stack.status or stack.state or 'неизвестно')}",
+ ]
+ if stack.repo:
+ lines.append(f"Репозиторий: {esc(stack.repo)}, ветка {esc(stack.branch)}")
+ if stack.deployed_hash:
+ newer = f", в репозитории уже {esc(stack.latest_hash)}" if stack.behind else ""
+ lines.append(f"Выкачен {esc(stack.deployed_hash)}{newer}")
+ if stack.image_updates:
+ lines.append(f"Новых образов: {stack.image_updates}")
+
+ try:
+ containers = await komodo.containers(stack_id)
+ except KomodoError:
+ containers = []
+ if containers:
+ lines.append("")
+ for item in containers:
+ facts = [item.status, f"cpu {item.cpu}" if item.cpu else "", item.mem]
+ lines.append(
+ f"{item.mark} {esc(item.service)}"
+ + "".join(f" · {esc(fact)}" for fact in facts if fact)
+ )
kb = InlineKeyboardBuilder()
- for key, (label, _, _) in ACTIONS.items():
- kb.button(text=label, callback_data=f"fleet:ask:{key}:{stack.id}")
- kb.button(text="📜 Логи", callback_data=f"fleet:log:{stack.id}")
- kb.button(text="⬅️ К флоту", callback_data="fleet:main")
- kb.adjust(3, 2)
- return text, kb.as_markup()
+ shown = ("dep", "res", "sto") if stack.state != "down" else ("dep", "sta")
+ for key in shown:
+ text, _, style = OPS[key]
+ kb.button(text=text, callback_data=f"fleet:ask:{key}:{stack.id}", style=style)
+ kb.button(text="Логи", callback_data=f"fleet:logs:{stack.id}")
+ kb.button(text="История", callback_data=f"fleet:hist:{stack.id}")
+ kb.button(text="Обновить", callback_data=f"fleet:stack:{stack.id}")
+ kb.button(
+ text="К серверу" if stack.server_id else "К флоту",
+ callback_data=f"fleet:srv:{stack.server_id}:0"
+ if stack.server_id
+ else "fleet:main",
+ )
+ kb.adjust(len(shown), 2, 2)
+ return "\n".join(lines), kb.as_markup()
+
+
+async def _render(
+ event: types.Message | types.CallbackQuery,
+ view: Awaitable[tuple[str, types.InlineKeyboardMarkup]],
+ *,
+ new: bool = False,
+) -> None:
+ try:
+ text, markup = await view
+ except KomodoError as exc:
+ if isinstance(event, types.CallbackQuery):
+ await event.answer(str(exc)[:190], show_alert=True)
+ else:
+ await show(event, f"🔴 {esc(exc)}", back_to_menu())
+ return
+ await show(event, text, markup, new=new)
+ if isinstance(event, types.CallbackQuery):
+ await event.answer()
@router.message(Command("fleet"))
-async def cmd_fleet(message: types.Message) -> None:
+async def cmd_fleet(message: types.Message, state: FSMContext) -> None:
+ await state.clear()
if not komodo.enabled:
- await message.answer("Komodo не настроен - см. README, раздел про KOMODO__*.")
+ await show(message, "Komodo не настроен: нужны переменные KOMODO__*.")
return
- try:
- text, markup = await _fleet_view()
- except KomodoError as exc:
- await message.answer(f"⚠️ {html.escape(str(exc))}")
- return
- await message.answer(text, reply_markup=markup)
+ await _render(message, fleet_view())
@router.callback_query(F.data == "fleet:main")
-async def fleet_main(callback: types.CallbackQuery) -> None:
+async def fleet_main(callback: types.CallbackQuery, state: FSMContext) -> None:
if not await _guard(callback):
return
- try:
- text, markup = await _fleet_view()
- except KomodoError as exc:
- await callback.answer(str(exc), show_alert=True)
+ await state.clear()
+ komodo.forget()
+ await _render(callback, fleet_view())
+
+
+def _id_and_page(data: str | None) -> tuple[str, int]:
+ parts = (data or "").split(":")
+ page = int(parts[3]) if len(parts) > 3 and parts[3].lstrip("-").isdigit() else 0 # noqa: PLR2004
+ return parts[2], page
+
+
+@router.callback_query(F.data.startswith("fleet:srv:"))
+async def fleet_server(callback: types.CallbackQuery) -> None:
+ if not await _guard(callback):
return
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(text, reply_markup=markup)
- await callback.answer()
+ server_id, page = _id_and_page(callback.data)
+ await _render(callback, server_view(server_id, page))
+
+
+@router.callback_query(F.data.startswith("go:srv:"))
+async def go_server(callback: types.CallbackQuery) -> None:
+ if not await _guard(callback):
+ return
+ server_id, _ = _id_and_page(callback.data)
+ await _render(callback, server_view(server_id, 0), new=True)
@router.callback_query(F.data.startswith("fleet:stack:"))
async def fleet_stack(callback: types.CallbackQuery) -> None:
if not await _guard(callback):
return
- stack_id = (callback.data or "").rsplit(":", 1)[-1]
- try:
- text, markup = await _stack_view(stack_id)
- except KomodoError as exc:
- await callback.answer(str(exc), show_alert=True)
+ stack_id, _ = _id_and_page(callback.data)
+ await _render(callback, stack_view(stack_id))
+
+
+@router.callback_query(F.data.startswith("go:stack:"))
+async def go_stack(callback: types.CallbackQuery) -> None:
+ if not await _guard(callback):
return
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(text, reply_markup=markup)
- await callback.answer()
+ stack_id, _ = _id_and_page(callback.data)
+ await _render(callback, stack_view(stack_id), new=True)
+
+
+async def _bad_view() -> tuple[str, types.InlineKeyboardMarkup]:
+ broken = sorted(
+ (s for s in await komodo.stacks() if not s.healthy),
+ key=lambda s: (s.server_name, s.name),
+ )
+ lines = ["Не работают", ""]
+ lines.extend(
+ f"{s.mark} {esc(s.name)} · {esc(s.server_name)} · {esc(s.state)}"
+ for s in broken
+ )
+ kb = InlineKeyboardBuilder()
+ rows = _stack_buttons(kb, broken[:40])
+ kb.button(text="К флоту", callback_data="fleet:main")
+ kb.adjust(*rows, 1)
+ return "\n".join(lines), kb.as_markup()
+
+
+@router.callback_query(F.data == "fleet:bad")
+async def fleet_bad(callback: types.CallbackQuery) -> None:
+ if not await _guard(callback):
+ return
+ await _render(callback, _bad_view())
+
+
+async def _alerts_view() -> tuple[str, types.InlineKeyboardMarkup]:
+ tz = db.settings.get("tz")
+ alerts = await komodo.open_alerts()
+ lines = ["Открытые алерты"]
+ if not alerts:
+ lines.append("Сейчас ни одного.")
+ for alert in alerts[:10]:
+ since = stamp(from_ms(alert.get("ts") or 0), tz)
+ lines += ["", format_alert(alert), f"с {since}"]
+ return "\n".join(lines), back_to("К флоту", "fleet:main")
+
+
+@router.callback_query(F.data == "fleet:alerts")
+async def fleet_alerts(callback: types.CallbackQuery) -> None:
+ if not await _guard(callback):
+ return
+ await _render(callback, _alerts_view())
@router.callback_query(F.data.startswith("fleet:ask:"))
async def fleet_ask(callback: types.CallbackQuery) -> None:
- """Подтверждение. Deploy с телефона слишком легко нажать штаниной."""
+ if not await _guard(callback):
+ return
_, _, key, stack_id = (callback.data or "").split(":", 3)
- label = ACTIONS[key][0]
+ text, _, style = OPS[key]
+ try:
+ stack = await komodo.stack(stack_id)
+ except KomodoError as exc:
+ await callback.answer(str(exc)[:190], show_alert=True)
+ return
+ if stack is None:
+ await callback.answer("Стек не найден", show_alert=True)
+ return
kb = InlineKeyboardBuilder()
- kb.button(text=f"✅ Да, {label}", callback_data=f"fleet:do:{key}:{stack_id}")
- kb.button(text="⬅️ Отмена", callback_data=f"fleet:stack:{stack_id}")
- kb.adjust(1)
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(
- f"{label} - подтверди, это подействует на прод.",
- reply_markup=kb.as_markup(),
- )
+ kb.button(
+ text=text, callback_data=f"fleet:do:{key}:{stack_id}", style=style or PRIMARY
+ )
+ kb.button(text="Отмена", callback_data=f"fleet:stack:{stack_id}")
+ kb.adjust(2)
+ await show(
+ callback,
+ f"{text} {esc(stack.name)} на {esc(stack.server_name)}?",
+ kb.as_markup(),
+ )
await callback.answer()
@@ -157,49 +367,226 @@ async def fleet_do(callback: types.CallbackQuery) -> None:
if not await _guard(callback):
return
_, _, key, stack_id = (callback.data or "").split(":", 3)
- _, method, done = ACTIONS[key]
-
- await callback.answer(done)
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(f"⏳ {done}…")
-
+ text, method, _ = OPS[key]
+ await callback.answer()
try:
- await komodo.execute(method, {"stack": stack_id, "services": []})
- logger.info(f"{method} {stack_id} - запущено из телеграма")
- except KomodoError as exc:
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(
- f"⚠️ Не вышло: {html.escape(str(exc))}", reply_markup=back_to_menu()
- )
- return
-
- try:
- text, markup = await _stack_view(stack_id)
+ stack = await komodo.stack(stack_id)
except KomodoError:
- text, markup = "Готово.", back_to_menu()
- if isinstance(callback.message, types.Message):
- await callback.message.edit_text(f"✅ {done}.\n\n{text}", reply_markup=markup)
+ stack = None
+ name = stack.name if stack else stack_id
+
+ async def starter() -> Update:
+ logger.info(f"{method} {name} started from telegram")
+ return await komodo.execute(method, {"stack": stack_id, "services": []})
+
+ await track(
+ callback,
+ f"{text} {name}",
+ starter,
+ back=[
+ ("К стеку", f"fleet:stack:{stack_id}"),
+ ("Логи", f"fleet:logs:{stack_id}"),
+ ],
+ verbose=False,
+ )
+
+
+def _picked(stack: Stack, index: int) -> list[str]:
+ if index == ALL_SERVICES or index >= len(stack.services):
+ return []
+ return [stack.services[index]]
+
+
+async def _log_view(
+ stack_id: str, index: int
+) -> tuple[str, types.InlineKeyboardMarkup]:
+ stack = await komodo.stack(stack_id)
+ if stack is None:
+ return "Стек не найден.", back_to("К флоту", "fleet:main")
+ services = _picked(stack, index)
+ log = await komodo.stack_log(stack_id, services, LOG_TAIL)
+ if services or len(stack.services) == 1:
+ log = LOG_PREFIX_RE.sub("", log)
+ scope = f" · {esc(services[0])}" if services else ""
+ body = pre(log, LOG_CHARS) if log else "лог пуст"
+ kb = InlineKeyboardBuilder()
+ kb.button(text="Обновить", callback_data=f"fleet:log:{stack_id}:{index}")
+ kb.button(text="Файлом", callback_data=f"fleet:logf:{stack_id}:{index}")
+ rows = [2]
+ if len(stack.services) > 1:
+ kb.button(text="Другой сервис", callback_data=f"fleet:logs:{stack_id}")
+ kb.button(text="К стеку", callback_data=f"fleet:stack:{stack_id}")
+ rows.append(2)
+ else:
+ kb.button(text="К стеку", callback_data=f"fleet:stack:{stack_id}")
+ rows.append(1)
+ kb.adjust(*rows)
+ return f"{esc(stack.name)}{scope}\n{body}", kb.as_markup()
+
+
+async def _log_picker(stack_id: str) -> tuple[str, types.InlineKeyboardMarkup]:
+ stack = await komodo.stack(stack_id)
+ if stack is None:
+ return "Стек не найден.", back_to("К флоту", "fleet:main")
+ if len(stack.services) <= 1:
+ return await _log_view(stack_id, ALL_SERVICES)
+ kb = InlineKeyboardBuilder()
+ for index, service in enumerate(stack.services):
+ kb.button(
+ text=label(service, 28), callback_data=f"fleet:log:{stack_id}:{index}"
+ )
+ kb.button(text="Все сервисы", callback_data=f"fleet:log:{stack_id}:{ALL_SERVICES}")
+ kb.button(text="К стеку", callback_data=f"fleet:stack:{stack_id}")
+ count = len(stack.services)
+ kb.adjust(*[2] * (count // 2), *[1] * (count % 2), 2)
+ return f"{esc(stack.name)}: чьи логи?", kb.as_markup()
+
+
+@router.callback_query(F.data.startswith("fleet:logs:"))
+async def fleet_logs(callback: types.CallbackQuery) -> None:
+ if not await _guard(callback):
+ return
+ stack_id, _ = _id_and_page(callback.data)
+ await _render(callback, _log_picker(stack_id))
+
+
+@router.callback_query(F.data.startswith("go:log:"))
+async def go_log(callback: types.CallbackQuery) -> None:
+ if not await _guard(callback):
+ return
+ stack_id, _ = _id_and_page(callback.data)
+ await _render(callback, _log_view(stack_id, ALL_SERVICES), new=True)
@router.callback_query(F.data.startswith("fleet:log:"))
async def fleet_log(callback: types.CallbackQuery) -> None:
if not await _guard(callback):
return
- stack_id = (callback.data or "").rsplit(":", 1)[-1]
- await callback.answer("Забираю логи…")
- try:
- log = await komodo.stack_log(stack_id, tail=LOG_TAIL)
- except KomodoError as exc:
- await callback.answer(str(exc), show_alert=True)
- return
+ stack_id, index = _id_and_page(callback.data)
+ await _render(callback, _log_view(stack_id, index))
- kb = InlineKeyboardBuilder()
- kb.button(text="🔄 Ещё раз", callback_data=f"fleet:log:{stack_id}")
- kb.button(text="⬅️ К стеку", callback_data=f"fleet:stack:{stack_id}")
- kb.adjust(2)
- tail = log[-LOG_CHARS:]
+
+@router.callback_query(F.data.startswith("fleet:logf:"))
+async def fleet_log_file(callback: types.CallbackQuery) -> None:
+ if not await _guard(callback):
+ return
+ stack_id, index = _id_and_page(callback.data)
+ try:
+ stack = await komodo.stack(stack_id)
+ if stack is None:
+ await callback.answer("Стек не найден", show_alert=True)
+ return
+ services = _picked(stack, index)
+ log = await komodo.stack_log(stack_id, services, LOG_FILE_TAIL)
+ except KomodoError as exc:
+ await callback.answer(str(exc)[:190], show_alert=True)
+ return
+ await callback.answer()
if isinstance(callback.message, types.Message):
- await callback.message.edit_text(
- f"📜 Последние {LOG_TAIL} строк:\n{html.escape(tail)}",
- reply_markup=kb.as_markup(),
+ name = "-".join([stack.name, *services])
+ await callback.message.answer_document(
+ BufferedInputFile((log or "empty").encode(), filename=f"{name}.log")
)
+
+
+def _same(one: Update, other: Update) -> bool:
+ return (one.operation, one.target_id, one.success, one.done) == (
+ other.operation,
+ other.target_id,
+ other.success,
+ other.done,
+ )
+
+
+def _collapse(updates: list[Update], *, everywhere: bool) -> list[tuple[Update, int]]:
+ result: list[tuple[Update, int]] = []
+ for update in updates:
+ scope = (
+ range(len(result)) if everywhere else range(len(result) - 1, len(result))
+ )
+ match = next((i for i in scope if i >= 0 and _same(result[i][0], update)), None)
+ if match is None:
+ result.append((update, 1))
+ else:
+ result[match] = (result[match][0], result[match][1] + 1)
+ return result
+
+
+def _update_mark(update: Update) -> str:
+ if not update.done:
+ return "🔄"
+ return "✅" if update.success else "🔴"
+
+
+async def _update_lines(
+ updates: list[Update], limit: int, *, with_target: bool
+) -> tuple[list[str], list[Update]]:
+ tz = db.settings.get("tz")
+ names = await komodo.names() if with_target else {}
+ lines, failed = [], []
+ for update, count in _collapse(updates, everywhere=with_target)[:limit]:
+ who = await komodo.username(update.operator)
+ target = (
+ f" {esc(names.get(update.target_id, update.target_type))}"
+ if with_target
+ else ""
+ )
+ times = f" ×{count}" if count > 1 else ""
+ lines.append(
+ f"{_update_mark(update)} {stamp(from_ms(update.start_ts), tz)} "
+ f"{esc(update.operation)}{target}{times} · {esc(who)}"
+ )
+ if update.done and not update.success:
+ failed.append(update)
+ return lines, failed
+
+
+def _failed_buttons(kb: InlineKeyboardBuilder, failed: list[Update]) -> int:
+ tz = db.settings.get("tz")
+ shown = failed[:FAILED_BUTTONS]
+ for update in shown:
+ kb.button(
+ text=f"🔴 {update.operation} {stamp(from_ms(update.start_ts), tz)}",
+ callback_data=f"run:st:{update.id}",
+ )
+ return len(shown)
+
+
+async def _history_view(stack_id: str) -> tuple[str, types.InlineKeyboardMarkup]:
+ stack = await komodo.stack(stack_id)
+ updates = await komodo.updates({"target.id": stack_id})
+ lines, failed = await _update_lines(updates, HISTORY, with_target=False)
+ title = esc(stack.name) if stack else "Стек"
+ kb = InlineKeyboardBuilder()
+ count = _failed_buttons(kb, failed)
+ kb.button(text="К стеку", callback_data=f"fleet:stack:{stack_id}")
+ kb.adjust(*[1] * count, 1)
+ body = "\n".join(lines) if lines else "Операций ещё не было."
+ return f"{title}: операции\n\n{body}", kb.as_markup()
+
+
+@router.callback_query(F.data.startswith("fleet:hist:"))
+async def fleet_history(callback: types.CallbackQuery) -> None:
+ if not await _guard(callback):
+ return
+ stack_id, _ = _id_and_page(callback.data)
+ await _render(callback, _history_view(stack_id))
+
+
+async def _journal_view() -> tuple[str, types.InlineKeyboardMarkup]:
+ updates = await komodo.updates()
+ lines, failed = await _update_lines(updates, JOURNAL, with_target=True)
+ kb = InlineKeyboardBuilder()
+ count = _failed_buttons(kb, failed)
+ kb.button(text="Обновить", callback_data="fleet:journal")
+ kb.button(text="К флоту", callback_data="fleet:main")
+ kb.adjust(*[1] * count, 2)
+ body = "\n".join(lines) if lines else "Операций ещё не было."
+ return f"Журнал\n\n{body}", kb.as_markup()
+
+
+@router.callback_query(F.data == "fleet:journal")
+async def fleet_journal(callback: types.CallbackQuery) -> None:
+ if not await _guard(callback):
+ return
+ await _render(callback, _journal_view())
diff --git a/src/healthbot/handlers/menu.py b/src/healthbot/handlers/menu.py
new file mode 100644
index 0000000..d265213
--- /dev/null
+++ b/src/healthbot/handlers/menu.py
@@ -0,0 +1,145 @@
+import asyncio
+
+from aiogram import Bot, F, Router, types
+from aiogram.filters import CommandStart
+from aiogram.fsm.context import FSMContext
+
+from healthbot.actionargs import command_name
+from healthbot.filters import Admin
+from healthbot.keyboards import main_menu
+from healthbot.komodo import KomodoError, komodo
+from healthbot.ui import MENU_COMMANDS, esc, plural, show
+from utils.db import db
+from utils.db.models import Service
+from utils.env import env
+from utils.format import human_duration, now, parse, short
+from utils.logging import logger
+
+router = Router()
+router.message.filter(Admin())
+router.callback_query.filter(Admin())
+
+SHOWN_PROBLEMS = 6
+FLEET_TIMEOUT = 6
+
+
+async def sync_commands(bot: Bot) -> None:
+ commands = [
+ types.BotCommand(command="start", description="Меню"),
+ types.BotCommand(command="list", description="Сервисы"),
+ types.BotCommand(command="stats", description="Статистика"),
+ ]
+ actions = env.komodo.enabled and db.settings.get("actions")
+ if env.komodo.enabled:
+ commands.append(types.BotCommand(command="fleet", description="Флот"))
+ if actions:
+ commands.append(types.BotCommand(command="actions", description="Действия"))
+ commands.append(types.BotCommand(command="settings", description="Настройки"))
+ if actions and db.settings.get("commands"):
+ taken = set(MENU_COMMANDS)
+ for pin in await db.pins.all():
+ name = command_name(pin.name)
+ if not name or name in taken:
+ continue
+ taken.add(name)
+ commands.append(
+ types.BotCommand(command=name, description=short(pin.label, 60))
+ )
+ await bot.set_my_commands(commands)
+
+
+@router.startup()
+async def on_startup(bot: Bot) -> None:
+ await sync_commands(bot)
+ logger.info(f"[green]Started as[/] @{(await bot.me()).username}")
+
+
+@router.shutdown()
+async def on_shutdown() -> None:
+ logger.info("Shutting down bot")
+
+
+async def _down_line(service: Service) -> str:
+ current = await db.incidents.current(service.id)
+ if current is None:
+ return f"🔴 {esc(service.name)}"
+ since = human_duration((now() - parse(current.started_at)).total_seconds())
+ return f"🔴 {esc(service.name)}: {since}, {esc(short(current.reason, 50))}"
+
+
+async def _services_block() -> list[str]:
+ services = await db.services.all()
+ if not services:
+ return ["Сервисов пока нет. Пришли ссылку, и я начну за ней следить."]
+ watched = [s for s in services if not s.is_paused]
+ down = [s for s in watched if not s.is_up]
+ paused = len(services) - len(watched)
+ working = len(watched) - len(down)
+ works = plural(working, "работает", "работают", "работают")
+ if not down and not paused:
+ every = "все " if working > 1 else ""
+ return [f"Сервисы: {every}{working} {works}"]
+ parts = [f"{working} {works}"]
+ if down:
+ parts.append(f"{len(down)} {plural(len(down), 'лежит', 'лежат', 'лежат')}")
+ if paused:
+ parts.append(f"{paused} на паузе")
+ lines = [f"Сервисы: {', '.join(parts)}"]
+ lines.extend([await _down_line(s) for s in down[:SHOWN_PROBLEMS]])
+ return lines
+
+
+async def _fleet_block() -> list[str]:
+ try:
+ servers, stacks, alerts = await asyncio.gather(
+ komodo.servers(), komodo.stacks(), komodo.open_alerts()
+ )
+ except KomodoError as exc:
+ return [f"Флот: {esc(exc)}"]
+ reachable = {s.id for s in servers if s.state == "Ok"}
+ good = sum(s.healthy for s in stacks)
+ summary = "все работают" if good == len(stacks) else f"работают {good}"
+ lines = [f"Флот: серверов {len(servers)}, стеков {len(stacks)}, {summary}"]
+ for server in servers:
+ if server.state == "NotOk":
+ lines.append(f"🔴 {esc(server.name)} не отвечает")
+ elif server.strained:
+ lines.append(f"🟠 {esc(server.name)}: {', '.join(server.strains)}")
+ troubled = [s for s in stacks if s.troubled and s.server_id in reachable]
+ lines.extend(
+ f"{s.mark} {esc(s.name)}: {esc(s.state)}" for s in troubled[:SHOWN_PROBLEMS]
+ )
+ if len(troubled) > SHOWN_PROBLEMS:
+ lines.append(f"и ещё {len(troubled) - SHOWN_PROBLEMS}")
+ if alerts:
+ lines.append(f"Открытых алертов: {len(alerts)}")
+ return lines
+
+
+async def dashboard() -> tuple[str, types.InlineKeyboardMarkup]:
+ lines = ["healthbot", "", *await _services_block()]
+ actions = env.komodo.enabled and db.settings.get("actions")
+ if env.komodo.enabled:
+ try:
+ fleet = await asyncio.wait_for(_fleet_block(), FLEET_TIMEOUT)
+ except TimeoutError:
+ fleet = ["Флот: Komodo не отвечает"]
+ lines += ["", *fleet]
+ pins = await db.pins.all() if actions else []
+ return "\n".join(lines), main_menu(pins, actions=bool(actions))
+
+
+@router.message(CommandStart())
+async def cmd_start(message: types.Message, state: FSMContext) -> None:
+ await state.clear()
+ text, markup = await dashboard()
+ await show(message, text, markup)
+
+
+@router.callback_query(F.data == "menu:main")
+async def menu_main(callback: types.CallbackQuery, state: FSMContext) -> None:
+ await state.clear()
+ komodo.forget()
+ text, markup = await dashboard()
+ await show(callback, text, markup)
+ await callback.answer()
diff --git a/src/healthbot/handlers/runs.py b/src/healthbot/handlers/runs.py
new file mode 100644
index 0000000..24cf60d
--- /dev/null
+++ b/src/healthbot/handlers/runs.py
@@ -0,0 +1,69 @@
+from aiogram import F, Router, types
+from aiogram.types import BufferedInputFile
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+from healthbot.filters import Admin
+from healthbot.komodo import KomodoError, Update, komodo
+from healthbot.runs import output_of, result_kb, result_text
+from healthbot.ui import esc, show, tail_arg
+from utils.format import clock
+
+router = Router()
+router.callback_query.filter(Admin())
+
+
+async def _title(update: Update) -> str:
+ try:
+ name = (await komodo.names()).get(update.target_id, "")
+ except KomodoError:
+ name = ""
+ return f"{update.operation} {name}".strip()
+
+
+@router.callback_query(F.data.startswith("run:st:"))
+async def run_status(callback: types.CallbackQuery) -> None:
+ update_id = tail_arg(callback.data)
+ try:
+ update = await komodo.update(update_id)
+ except KomodoError as exc:
+ await callback.answer(str(exc)[:190], show_alert=True)
+ return
+ title = await _title(update)
+ await callback.answer()
+ if not update.done:
+ kb = InlineKeyboardBuilder()
+ kb.button(text="Проверить", callback_data=f"run:st:{update_id}")
+ await show(
+ callback,
+ f"⏳ {esc(title)} идёт уже {clock(update.seconds)}",
+ kb.as_markup(),
+ new=True,
+ )
+ return
+ await show(
+ callback,
+ result_text(title, update, verbose=update.success),
+ result_kb(update, []),
+ new=True,
+ )
+
+
+@router.callback_query(F.data.startswith("run:file:"))
+async def run_file(callback: types.CallbackQuery) -> None:
+ update_id = tail_arg(callback.data)
+ try:
+ update = await komodo.update(update_id)
+ except KomodoError as exc:
+ await callback.answer(str(exc)[:190], show_alert=True)
+ return
+ await callback.answer()
+ if isinstance(callback.message, types.Message):
+ text = output_of(update, only_failed=False) or "empty"
+ await callback.message.answer_document(
+ BufferedInputFile(text.encode(), filename=f"{update.operation}.log")
+ )
+
+
+@router.callback_query()
+async def stale_button(callback: types.CallbackQuery) -> None:
+ await callback.answer("Эта кнопка устарела, открой /start", show_alert=True)
diff --git a/src/healthbot/handlers/services.py b/src/healthbot/handlers/services.py
new file mode 100644
index 0000000..ba611d0
--- /dev/null
+++ b/src/healthbot/handlers/services.py
@@ -0,0 +1,410 @@
+import json as jsonlib
+
+import aiohttp
+from aiogram import F, Router, types
+from aiogram.filters import Command
+from aiogram.fsm.context import FSMContext
+from aiogram.fsm.state import State, StatesGroup
+
+from healthbot.checker import MISSING, dig, evaluate, probe, scalar_str, snippet
+from healthbot.filters import Admin
+from healthbot.keyboards import (
+ after_check,
+ back_to,
+ back_to_menu,
+ confirm_delete,
+ service_card,
+ services_list,
+)
+from healthbot.stats import (
+ PERIODS,
+ current_hour,
+ day_squares,
+ downtime,
+ latency_chart,
+ uptime,
+ uptime_line,
+)
+from healthbot.ui import ANSWER, esc, int_arg, plural, show
+from utils.db import db
+from utils.db.models import Incident, MatchType, Service
+from utils.env import env
+from utils.format import ago, human_duration, now, parse, short, stamp
+
+router = Router()
+router.message.filter(Admin())
+router.callback_query.filter(Admin())
+
+MAX_NAME = 64
+SHOWN_INCIDENTS = 10
+CHART_HOURS = 24
+
+
+class RenameFlow(StatesGroup):
+ waiting_name = State()
+
+
+async def list_view(page: int = 0) -> tuple[str, types.InlineKeyboardMarkup]:
+ services = await db.services.all()
+ if not services:
+ return "Сервисов пока нет. Пришли ссылку, чтобы добавить.", back_to_menu()
+ watched = [s for s in services if not s.is_paused]
+ up = sum(s.is_up for s in watched)
+ paused = len(services) - len(watched)
+ parts = [f"🟢 {up}", f"🔴 {len(watched) - up}"]
+ if paused:
+ parts.append(f"⏸ {paused}")
+ return f"Сервисы {' '.join(parts)}", services_list(services, page)
+
+
+@router.message(Command("list"))
+async def cmd_list(message: types.Message, state: FSMContext) -> None:
+ await state.clear()
+ text, markup = await list_view()
+ await show(message, text, markup)
+
+
+@router.callback_query(F.data.startswith("menu:list"))
+async def menu_list(callback: types.CallbackQuery, state: FSMContext) -> None:
+ await state.clear()
+ text, markup = await list_view(int_arg(callback.data))
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+async def _status_line(service: Service) -> str:
+ if service.is_paused:
+ return "на паузе, не проверяется"
+ if not service.is_up:
+ current = await db.incidents.current(service.id)
+ if current is None:
+ return "лежит"
+ since = human_duration((now() - parse(current.started_at)).total_seconds())
+ muted = ", напоминания выключены" if current.muted else ""
+ return f"лежит {since}: {esc(current.reason)}{muted}"
+ parts = ["работает"]
+ if service.last_latency_ms is not None:
+ parts.append(f"отклик {service.last_latency_ms} мс")
+ if service.last_checked_at:
+ parts.append(f"проверен {ago(parse(service.last_checked_at))}")
+ if service.fail_streak:
+ parts.append(
+ f"неудач подряд {service.fail_streak} из {env.monitor.failure_threshold}"
+ )
+ return ", ".join(parts)
+
+
+def _incident_line(incident: Incident, tz: str) -> str:
+ began = parse(incident.started_at)
+ if incident.ended_at:
+ length = human_duration((parse(incident.ended_at) - began).total_seconds())
+ else:
+ length = "идёт сейчас"
+ return f"{stamp(began, tz)} · {length} · {esc(short(incident.reason, 60))}"
+
+
+async def card_view(service: Service) -> str:
+ tz = db.settings.get("tz")
+ incidents = await db.incidents.for_service(service.id)
+ hours = await db.stats.since(service.id, current_hour() - CHART_HOURS)
+ chart, average, peak = latency_chart(hours)
+
+ lines = [
+ f"{service.mark} {esc(service.name)}",
+ f"{esc(service.url)}",
+ "",
+ f"Правило: {service.rule}",
+ f"Статус: {await _status_line(service)}",
+ "",
+ f"Аптайм: {uptime_line(service, incidents)}",
+ f"14 дней: {day_squares(service, incidents, tz)}",
+ ]
+ if chart:
+ lines.append(f"Отклик за сутки: {chart}")
+ lines.append(f"в среднем {average} мс, пик {peak} мс")
+ if incidents:
+ last = _incident_line(incidents[-1], tz)
+ lines.append(f"Инцидентов: {len(incidents)}, последний {last}")
+ if service.tls_expires_at:
+ left = (parse(service.tls_expires_at) - now()).total_seconds()
+ state = f"ещё {human_duration(left)}" if left > 0 else "истёк"
+ lines.append(f"Сертификат: {state}")
+ return "\n".join(lines)
+
+
+async def show_card(
+ event: types.Message | types.CallbackQuery, service: Service, *, new: bool = False
+) -> None:
+ await show(event, await card_view(service), service_card(service), new=new)
+
+
+async def _load(callback: types.CallbackQuery) -> Service | None:
+ service = await db.services.get(int_arg(callback.data))
+ if service is None:
+ await callback.answer("Сервис не найден", show_alert=True)
+ return service
+
+
+@router.callback_query(F.data.startswith("svc:open:"))
+async def svc_open(callback: types.CallbackQuery, state: FSMContext) -> None:
+ service = await _load(callback)
+ if service is None:
+ return
+ await state.clear()
+ await show_card(callback, service)
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("go:svc:"))
+async def go_service(callback: types.CallbackQuery) -> None:
+ service = await _load(callback)
+ if service is None:
+ return
+ await show_card(callback, service, new=True)
+ await callback.answer()
+
+
+async def _check_view(service: Service) -> tuple[str, types.InlineKeyboardMarkup]:
+ async with aiohttp.ClientSession() as session:
+ result = await probe(session, service.url, env.monitor.request_timeout)
+ verdict = evaluate(service, result)
+
+ head = "✅ Правило выполняется" if verdict.healthy else "🔴 Правило нарушено"
+ lines = [
+ f"{esc(service.name)}",
+ f"{esc(service.url)}",
+ "",
+ head,
+ ]
+ if result.reachable:
+ lines.append(f"HTTP {result.status}, отклик {result.latency_ms} мс")
+ if not verdict.healthy:
+ lines.append(f"Причина: {esc(verdict.reason)}")
+ report = verdict.report()
+ if report:
+ lines.append(report)
+ elif result.reachable:
+ lines.extend(snippet(result.body))
+ markup = after_check(
+ service.id, offer_baseline=not verdict.healthy and result.reachable
+ )
+ return "\n".join(lines), markup
+
+
+@router.callback_query(F.data.startswith("svc:check:"))
+async def svc_check(callback: types.CallbackQuery) -> None:
+ service = await _load(callback)
+ if service is None:
+ return
+ await callback.answer()
+ text, markup = await _check_view(service)
+ await show(callback, text, markup)
+
+
+@router.callback_query(F.data.startswith("go:check:"))
+async def go_check(callback: types.CallbackQuery) -> None:
+ service = await _load(callback)
+ if service is None:
+ return
+ await callback.answer()
+ text, markup = await _check_view(service)
+ await show(callback, text, markup, new=True)
+
+
+@router.callback_query(F.data.startswith("svc:base:"))
+async def svc_baseline(callback: types.CallbackQuery) -> None:
+ service = await _load(callback)
+ if service is None:
+ return
+ async with aiohttp.ClientSession() as session:
+ result = await probe(session, service.url, env.monitor.request_timeout)
+ if not result.reachable:
+ await callback.answer("Сервис недоступен, эталон снять не с чего", True) # noqa: FBT003
+ return
+
+ json_value = service.json_value
+ if service.match_type is MatchType.JSON:
+ try:
+ value = dig(jsonlib.loads(result.body), service.json_path or "")
+ except ValueError:
+ await callback.answer("Ответ не JSON, эталон не обновлён", True) # noqa: FBT003
+ return
+ if value is MISSING or isinstance(value, (dict, list)):
+ await callback.answer(
+ f"Поля {service.json_path} в ответе нет, эталон не обновлён",
+ True, # noqa: FBT003
+ )
+ return
+ json_value = scalar_str(value)
+
+ await db.services.set_baseline(
+ service.id,
+ ok_status=result.status or service.ok_status,
+ ok_body=(
+ result.body if service.match_type is MatchType.BODY else service.ok_body
+ ),
+ json_value=json_value,
+ )
+ await db.services.set_streak(service.id, 0)
+ updated = await db.services.get(service.id)
+ if updated is not None:
+ await show_card(callback, updated)
+ await callback.answer()
+
+
+async def _set_paused(callback: types.CallbackQuery, *, is_paused: bool) -> None:
+ service = await _load(callback)
+ if service is None:
+ return
+ await db.services.set_paused(service.id, is_paused=is_paused)
+ updated = await db.services.get(service.id)
+ if updated is not None:
+ await show_card(callback, updated)
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("svc:pause:"))
+async def svc_pause(callback: types.CallbackQuery) -> None:
+ await _set_paused(callback, is_paused=True)
+
+
+@router.callback_query(F.data.startswith("svc:resume:"))
+async def svc_resume(callback: types.CallbackQuery) -> None:
+ await _set_paused(callback, is_paused=False)
+
+
+@router.callback_query(F.data.startswith("svc:mute:"))
+async def svc_mute(callback: types.CallbackQuery) -> None:
+ incident = await db.incidents.get(int_arg(callback.data))
+ if incident is None or incident.ended_at:
+ await callback.answer("Инцидент уже закрыт", show_alert=True)
+ return
+ await db.incidents.mute(incident.id)
+ if isinstance(callback.message, types.Message):
+ kb = types.InlineKeyboardMarkup(
+ inline_keyboard=[
+ row
+ for row in (
+ callback.message.reply_markup or back_to_menu()
+ ).inline_keyboard
+ if not any((b.callback_data or "").startswith("svc:mute:") for b in row)
+ ]
+ )
+ await callback.message.edit_reply_markup(reply_markup=kb)
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("svc:inc:"))
+async def svc_incidents(callback: types.CallbackQuery) -> None:
+ service = await _load(callback)
+ if service is None:
+ return
+ tz = db.settings.get("tz")
+ incidents = await db.incidents.for_service(service.id)
+ lines = [f"{esc(service.name)}: инциденты", ""]
+ if not incidents:
+ lines.append("Ни одного падения.")
+ else:
+ lines.extend(
+ _incident_line(i, tz) for i in reversed(incidents[-SHOWN_INCIDENTS:])
+ )
+ total = downtime(incidents, parse(service.created_at), now())
+ lines += ["", f"Всего {len(incidents)}, даунтайм {human_duration(total)}"]
+ await show(
+ callback, "\n".join(lines), back_to("К сервису", f"svc:open:{service.id}")
+ )
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("svc:rename:"))
+async def svc_rename(callback: types.CallbackQuery, state: FSMContext) -> None:
+ service = await _load(callback)
+ if service is None:
+ return
+ await state.set_state(RenameFlow.waiting_name)
+ await state.set_data({"service_id": service.id})
+ await show(
+ callback,
+ f"Пришли новое имя для {esc(service.name)}.",
+ back_to("Отмена", f"svc:open:{service.id}"),
+ )
+ await callback.answer()
+
+
+@router.message(RenameFlow.waiting_name, ANSWER)
+async def on_new_name(message: types.Message, state: FSMContext) -> None:
+ name = (message.text or "").strip()[:MAX_NAME]
+ if not name:
+ return
+ data = await state.get_data()
+ await state.clear()
+ await db.services.rename(data["service_id"], name)
+ service = await db.services.get(data["service_id"])
+ if service is None:
+ await show(message, "Сервис уже удалён.", back_to_menu())
+ return
+ await show_card(message, service)
+
+
+@router.callback_query(F.data.startswith("svc:del:"))
+async def svc_del(callback: types.CallbackQuery) -> None:
+ service = await _load(callback)
+ if service is None:
+ return
+ await show(
+ callback,
+ f"Удалить {esc(service.name)} вместе со статистикой?",
+ confirm_delete(service.id),
+ )
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("svc:delyes:"))
+async def svc_del_yes(callback: types.CallbackQuery) -> None:
+ await db.services.delete(int_arg(callback.data))
+ text, markup = await list_view()
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+async def stats_view() -> tuple[str, types.InlineKeyboardMarkup]:
+ services = await db.services.all()
+ if not services:
+ return "Сервисов пока нет.", back_to_menu()
+ month = PERIODS[-1][1]
+ rows = []
+ for service in services:
+ incidents = await db.incidents.for_service(service.id)
+ rows.append((uptime(service, incidents, month) or 0.0, service, incidents))
+ rows.sort(key=lambda row: row[0])
+
+ lines = ["Статистика", ""]
+ for _, service, incidents in rows:
+ speed = (
+ f" · {service.last_latency_ms} мс"
+ if service.last_latency_ms is not None
+ else ""
+ )
+ count = len(incidents)
+ falls = (
+ f"{count} {plural(count, 'падение', 'падения', 'падений')}"
+ if count
+ else "без падений"
+ )
+ lines.append(f"{service.mark} {esc(service.name)}{speed}")
+ lines.append(f"{uptime_line(service, incidents)} · {falls}")
+ return "\n".join(lines), back_to_menu()
+
+
+@router.message(Command("stats"))
+async def cmd_stats(message: types.Message, state: FSMContext) -> None:
+ await state.clear()
+ text, markup = await stats_view()
+ await show(message, text, markup)
+
+
+@router.callback_query(F.data == "menu:stats")
+async def menu_stats(callback: types.CallbackQuery) -> None:
+ text, markup = await stats_view()
+ await show(callback, text, markup)
+ await callback.answer()
diff --git a/src/healthbot/handlers/settings.py b/src/healthbot/handlers/settings.py
new file mode 100644
index 0000000..2f5c691
--- /dev/null
+++ b/src/healthbot/handlers/settings.py
@@ -0,0 +1,229 @@
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+from aiogram import Bot, F, Router, types
+from aiogram.filters import Command
+from aiogram.fsm.context import FSMContext
+from aiogram.fsm.state import State, StatesGroup
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+from healthbot.alerts import KNOWN_TYPES
+from healthbot.filters import Admin
+from healthbot.handlers.menu import sync_commands
+from healthbot.keyboards import back_to
+from healthbot.komodo import KomodoError, komodo
+from healthbot.ui import ANSWER, esc, int_arg, show, tail_arg
+from utils.db import db
+from utils.env import env
+from utils.format import now, zone
+
+router = Router()
+router.message.filter(Admin())
+router.callback_query.filter(Admin())
+
+TOGGLES = {
+ "actions": ("Действия Komodo", "включены", "выключены"),
+ "commands": ("Действия как команды", "включены", "выключены"),
+ "confirm": ("Подтверждение запуска", "включено", "выключено"),
+}
+DIGEST_HOURS = (7, 8, 9, 10, 12, 18, 21)
+
+
+class SettingsFlow(StatesGroup):
+ waiting_tz = State()
+
+
+async def _komodo_line() -> str:
+ if not env.komodo.enabled:
+ return "Komodo не подключён: нужны переменные KOMODO__*."
+ try:
+ return f"Komodo {esc(await komodo.version())} подключён."
+ except KomodoError as exc:
+ return f"Komodo: {esc(exc)}"
+
+
+async def settings_view() -> tuple[str, types.InlineKeyboardMarkup]:
+ tz = db.settings.get("tz")
+ lines = ["Настройки", await _komodo_line()]
+ kb = InlineKeyboardBuilder()
+ if env.komodo.enabled:
+ for key, (title, on, off) in TOGGLES.items():
+ if key != "actions" and not db.settings.get("actions"):
+ continue
+ state = on if db.settings.get(key) else off
+ kb.button(text=f"{title}: {state}", callback_data=f"set:t:{key}")
+ if db.settings.get("actions") and db.settings.get("commands"):
+ lines.append(
+ "Команда запускает действие по имени: "
+ "/имя_действия поле значение. "
+ "Закреплённые видны в меню команд."
+ )
+ if env.web.enabled:
+ muted = len(db.settings.get("muted"))
+ kb.button(text=f"Алерты Komodo, скрыто: {muted}", callback_data="set:alerts")
+ kb.button(text=f"Часовой пояс: {tz}", callback_data="set:tz")
+ hour = db.settings.get("digest")
+ digest = "выключена" if hour is None else f"в {hour:02d}:00"
+ kb.button(text=f"Сводка за сутки: {digest}", callback_data="set:digest")
+ if env.komodo.enabled and db.settings.get("actions"):
+ kb.button(text="Закреплённые", callback_data="set:pins")
+ kb.button(text="Меню", callback_data="menu:main")
+ kb.adjust(1)
+ return "\n".join(lines), kb.as_markup()
+
+
+@router.message(Command("settings"))
+async def cmd_settings(message: types.Message, state: FSMContext) -> None:
+ await state.clear()
+ text, markup = await settings_view()
+ await show(message, text, markup)
+
+
+@router.callback_query(F.data == "set:main")
+async def settings_main(callback: types.CallbackQuery, state: FSMContext) -> None:
+ await state.clear()
+ text, markup = await settings_view()
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("set:t:"))
+async def settings_toggle(callback: types.CallbackQuery, bot: Bot) -> None:
+ key = tail_arg(callback.data)
+ if key not in TOGGLES:
+ await callback.answer()
+ return
+ await db.settings.set(key, not db.settings.get(key))
+ await sync_commands(bot)
+ text, markup = await settings_view()
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+def _alerts_view() -> tuple[str, types.InlineKeyboardMarkup]:
+ muted: list[str] = db.settings.get("muted")
+ kinds = [*KNOWN_TYPES, *(kind for kind in muted if kind not in KNOWN_TYPES)]
+ kb = InlineKeyboardBuilder()
+ for kind in kinds:
+ mark = "🔕" if kind in muted else "🔔"
+ kb.button(text=f"{mark} {kind}", callback_data=f"set:mute:{kind}")
+ kb.button(text="К настройкам", callback_data="set:main")
+ kb.adjust(1)
+ return "Алерты Komodo", kb.as_markup()
+
+
+@router.callback_query(F.data == "set:alerts")
+async def settings_alerts(callback: types.CallbackQuery) -> None:
+ text, markup = _alerts_view()
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+async def _flip_mute(kind: str) -> None:
+ muted: list[str] = list(db.settings.get("muted"))
+ if kind in muted:
+ muted.remove(kind)
+ else:
+ muted.append(kind)
+ await db.settings.set("muted", muted)
+
+
+@router.callback_query(F.data.startswith("set:mute:"))
+async def settings_mute(callback: types.CallbackQuery) -> None:
+ await _flip_mute(tail_arg(callback.data))
+ text, markup = _alerts_view()
+ await show(callback, text, markup)
+ await callback.answer()
+
+
+async def _swap_alert_button(
+ callback: types.CallbackQuery, text: str, data: str
+) -> None:
+ message = callback.message
+ if isinstance(message, types.Message):
+ old = message.reply_markup.inline_keyboard if message.reply_markup else []
+ rows = [
+ [b for b in row if not (b.callback_data or "").startswith("al:")]
+ for row in old
+ ]
+ rows = [row for row in rows if row]
+ rows.append([types.InlineKeyboardButton(text=text, callback_data=data)])
+ await message.edit_reply_markup(
+ reply_markup=types.InlineKeyboardMarkup(inline_keyboard=rows)
+ )
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("al:mute:"))
+async def alert_mute(callback: types.CallbackQuery) -> None:
+ kind = tail_arg(callback.data)
+ if kind not in db.settings.get("muted"):
+ await _flip_mute(kind)
+ await _swap_alert_button(callback, f"{kind} скрыт, вернуть", f"al:show:{kind}")
+
+
+@router.callback_query(F.data.startswith("al:show:"))
+async def alert_unmute(callback: types.CallbackQuery) -> None:
+ kind = tail_arg(callback.data)
+ if kind in db.settings.get("muted"):
+ await _flip_mute(kind)
+ await _swap_alert_button(callback, "Не присылать такие", f"al:mute:{kind}")
+
+
+@router.callback_query(F.data == "set:tz")
+async def settings_tz(callback: types.CallbackQuery, state: FSMContext) -> None:
+ tz = db.settings.get("tz")
+ local = now().astimezone(zone(tz)).strftime("%H:%M")
+ await state.set_state(SettingsFlow.waiting_tz)
+ await show(
+ callback,
+ "Пришли часовой пояс, например Europe/Warsaw.\n"
+ f"Сейчас: {esc(tz)}, на часах {local}",
+ back_to("Отмена", "set:main"),
+ )
+ await callback.answer()
+
+
+@router.message(SettingsFlow.waiting_tz, ANSWER)
+async def on_tz(message: types.Message, state: FSMContext) -> None:
+ name = (message.text or "").strip()
+ try:
+ ZoneInfo(name)
+ except (ZoneInfoNotFoundError, ValueError, OSError):
+ await show(
+ message,
+ "Такого пояса нет. Нужно имя из базы tz, например "
+ "Europe/Warsaw или UTC.",
+ back_to("Отмена", "set:main"),
+ )
+ return
+ await state.clear()
+ await db.settings.set("tz", name)
+ text, markup = await settings_view()
+ await show(message, text, markup)
+
+
+@router.callback_query(F.data == "set:digest")
+async def settings_digest(callback: types.CallbackQuery) -> None:
+ kb = InlineKeyboardBuilder()
+ for hour in DIGEST_HOURS:
+ kb.button(text=f"{hour:02d}:00", callback_data=f"set:dg:{hour}")
+ kb.button(text="Выключить", callback_data="set:dg:off")
+ kb.button(text="К настройкам", callback_data="set:main")
+ kb.adjust(4, 3, 2)
+ await show(
+ callback,
+ "Сводка за сутки\n"
+ "Раз в день: аптайм и падения сервисов, операции и алерты флота.\n"
+ f"Время по поясу {esc(db.settings.get('tz'))}.",
+ kb.as_markup(),
+ )
+ await callback.answer()
+
+
+@router.callback_query(F.data.startswith("set:dg:"))
+async def settings_digest_set(callback: types.CallbackQuery) -> None:
+ value = tail_arg(callback.data)
+ await db.settings.set("digest", None if value == "off" else int_arg(callback.data))
+ text, markup = await settings_view()
+ await show(callback, text, markup)
+ await callback.answer()
diff --git a/src/healthbot/keyboards/__init__.py b/src/healthbot/keyboards/__init__.py
index 867252b..3816729 100644
--- a/src/healthbot/keyboards/__init__.py
+++ b/src/healthbot/keyboards/__init__.py
@@ -1,69 +1,111 @@
from aiogram.types import InlineKeyboardButton, InlineKeyboardMarkup
from aiogram.utils.keyboard import InlineKeyboardBuilder
-from utils.db.models import Service
+from healthbot.ui import DANGER, PRIMARY, label, pager, pages
+from utils.db.models import Pin, Service
from utils.env import env
+SERVICES_PAGE = 12
-def main_menu() -> InlineKeyboardMarkup:
+
+def main_menu(pins: list[Pin], *, actions: bool) -> InlineKeyboardMarkup:
kb = InlineKeyboardBuilder()
- kb.button(text="➕ Добавить сервис", callback_data="menu:add")
- kb.button(text="📋 Сервисы", callback_data="menu:list")
- kb.button(text="📊 Статистика", callback_data="menu:stats")
+ rows: list[int] = []
+ for pin in pins:
+ kb.button(
+ text=label(pin.label, 30), callback_data=f"pin:run:{pin.id}", style=PRIMARY
+ )
+ rows += [2] * (len(pins) // 2) + [1] * (len(pins) % 2)
+
+ kb.button(text="Сервисы", callback_data="menu:list:0")
if env.komodo.enabled:
- kb.button(text="🛰 Флот", callback_data="fleet:main")
- kb.adjust(1, 2, 1)
+ kb.button(text="Флот", callback_data="fleet:main")
+ if actions:
+ kb.button(text="Действия", callback_data="act:list:a:0")
+ kb.button(text="Статистика", callback_data="menu:stats")
+ rows += [2, 2]
+ else:
+ kb.button(text="Статистика", callback_data="menu:stats")
+ rows += [2, 1]
else:
- kb.adjust(1, 2)
+ kb.button(text="Статистика", callback_data="menu:stats")
+ rows += [2]
+ kb.button(text="Добавить сервис", callback_data="menu:add")
+ kb.button(text="Настройки", callback_data="set:main")
+ kb.button(text="Обновить", callback_data="menu:main")
+ kb.adjust(*rows, 2, 1)
return kb.as_markup()
-def services_list(services: list[Service]) -> InlineKeyboardMarkup:
+def _service_order(service: Service) -> int:
+ if service.is_paused:
+ return 2
+ return 1 if service.is_up else 0
+
+
+def services_list(services: list[Service], page: int) -> InlineKeyboardMarkup:
+ ordered = sorted(services, key=_service_order)
+ total = pages(len(ordered), SERVICES_PAGE)
+ page = min(page, total - 1)
+ chunk = ordered[page * SERVICES_PAGE : (page + 1) * SERVICES_PAGE]
kb = InlineKeyboardBuilder()
- for svc in services:
- kb.button(text=f"{svc.mark} {svc.name}", callback_data=f"svc:open:{svc.id}")
- kb.button(text="➕ Добавить", callback_data="menu:add")
- kb.button(text="⬅️ Меню", callback_data="menu:main")
- kb.adjust(1)
+ for svc in chunk:
+ speed = (
+ f" · {svc.last_latency_ms} мс"
+ if svc.is_up and not svc.is_paused and svc.last_latency_ms is not None
+ else ""
+ )
+ kb.button(
+ text=f"{svc.mark} {label(svc.name, 36)}{speed}",
+ callback_data=f"svc:open:{svc.id}",
+ )
+ nav = pager(kb, "menu:list", page, total)
+ kb.button(text="Добавить", callback_data="menu:add")
+ kb.button(text="Меню", callback_data="menu:main")
+ kb.adjust(*[1] * len(chunk), *([nav] if nav else []), 2)
return kb.as_markup()
def service_card(service: Service) -> InlineKeyboardMarkup:
kb = InlineKeyboardBuilder()
- kb.button(text="🔍 Проверить сейчас", callback_data=f"svc:check:{service.id}")
+ kb.button(text="Проверить", callback_data=f"svc:check:{service.id}", style=PRIMARY)
+ kb.button(text="Инциденты", callback_data=f"svc:inc:{service.id}")
if service.is_paused:
- kb.button(text="▶️ Возобновить", callback_data=f"svc:resume:{service.id}")
+ kb.button(text="Возобновить", callback_data=f"svc:resume:{service.id}")
else:
- kb.button(text="⏸ Пауза", callback_data=f"svc:pause:{service.id}")
- kb.button(text="✏️ Переименовать", callback_data=f"svc:rename:{service.id}")
- kb.button(text="🗑 Удалить", callback_data=f"svc:del:{service.id}")
- kb.button(text="⬅️ К списку", callback_data="menu:list")
- kb.adjust(1, 2, 2)
+ kb.button(text="Пауза", callback_data=f"svc:pause:{service.id}")
+ kb.button(text="Правило", callback_data=f"svc:rule:{service.id}")
+ kb.button(text="Имя", callback_data=f"svc:rename:{service.id}")
+ kb.button(text="Адрес", callback_data=f"svc:url:{service.id}")
+ kb.button(text="Удалить", callback_data=f"svc:del:{service.id}")
+ kb.button(text="К списку", callback_data="menu:list:0")
+ kb.adjust(2)
return kb.as_markup()
def confirm_delete(service_id: int) -> InlineKeyboardMarkup:
kb = InlineKeyboardBuilder()
- kb.button(text="✅ Да, удалить", callback_data=f"svc:delyes:{service_id}")
- kb.button(text="⬅️ Отмена", callback_data=f"svc:open:{service_id}")
+ kb.button(text="Удалить", callback_data=f"svc:delyes:{service_id}", style=DANGER)
+ kb.button(text="Отмена", callback_data=f"svc:open:{service_id}")
kb.adjust(2)
return kb.as_markup()
def after_check(service_id: int, *, offer_baseline: bool) -> InlineKeyboardMarkup:
- """Клавиатура под результатом ручной проверки."""
kb = InlineKeyboardBuilder()
if offer_baseline:
- kb.button(text="📌 Принять как эталон", callback_data=f"svc:base:{service_id}")
- kb.button(text="🔄 Ещё раз", callback_data=f"svc:check:{service_id}")
- kb.button(text="⬅️ К сервису", callback_data=f"svc:open:{service_id}")
- kb.adjust(1, 2)
+ kb.button(text="Принять как эталон", callback_data=f"svc:base:{service_id}")
+ kb.button(text="Ещё раз", callback_data=f"svc:check:{service_id}")
+ kb.button(text="К сервису", callback_data=f"svc:open:{service_id}")
+ kb.adjust(*([1] if offer_baseline else []), 2)
return kb.as_markup()
-def back_to_menu() -> InlineKeyboardMarkup:
+def back_to(text: str, data: str) -> InlineKeyboardMarkup:
return InlineKeyboardMarkup(
- inline_keyboard=[
- [InlineKeyboardButton(text="⬅️ Меню", callback_data="menu:main")]
- ]
+ inline_keyboard=[[InlineKeyboardButton(text=text, callback_data=data)]]
)
+
+
+def back_to_menu() -> InlineKeyboardMarkup:
+ return back_to("Меню", "menu:main")
diff --git a/src/healthbot/komodo.py b/src/healthbot/komodo.py
index e8a970f..271d88f 100644
--- a/src/healthbot/komodo.py
+++ b/src/healthbot/komodo.py
@@ -1,7 +1,8 @@
-"""Тонкий клиент Komodo: посмотреть флот и нажать кнопку."""
-
+import asyncio
+import json
import re
-from dataclasses import dataclass
+import time
+from dataclasses import dataclass, field
from typing import Any
import aiohttp
@@ -10,27 +11,123 @@ from utils.env import env
from utils.logging import logger
ANSI_RE = re.compile(r"\x1b\[[0-9;]*[a-zA-Z]")
+TAG_RE = re.compile(r"]*>")
+SECRET_RE = re.compile(
+ r"\b(key|secret|token|password)(['\"]?\s*[:=]\s*)'[^']*'", re.IGNORECASE
+)
+SYSTEM_USER_RE = re.compile(r"[0-9a-f]{24}")
OK = 200
+CACHE_TTL = 8
+NAMES_TTL = 60
+BUSY_PAUSES = (1, 2, 4)
+POLL_FAST = 1
+POLL_SLOW = 3
+POLL_FAST_FOR = 15
+WAIT_FAILURES = 5
+
+STACK_MARKS = {
+ "running": "🟢",
+ "deploying": "🔄",
+ "restarting": "🔄",
+ "unhealthy": "🟠",
+ "paused": "⏸",
+ "created": "⚪️",
+ "down": "⚫️",
+ "stopped": "🔴",
+ "dead": "🔴",
+ "removing": "🔴",
+ "unknown": "⚪️",
+}
+IDLE_STATES = frozenset({"running", "down", "unknown"})
+SERVER_MARKS = {"Ok": "🟢", "NotOk": "🔴", "Disabled": "⏸"}
+RUN_MARKS = {"Ok": "🟢", "Failed": "🔴", "Running": "🔄"}
+
+KIND_API = {"a": ("Action", "action"), "p": ("Procedure", "procedure")}
def strip_ansi(text: str) -> str:
return ANSI_RE.sub("", text)
+def clean_output(text: str) -> str:
+ text = TAG_RE.sub("", strip_ansi(text))
+ return SECRET_RE.sub(r"\1\2'***'", text).strip()
+
+
+def oid(raw: dict) -> str:
+ value = raw.get("id") or raw.get("_id") or ""
+ if isinstance(value, dict):
+ return str(value.get("$oid", ""))
+ return str(value)
+
+
class KomodoError(RuntimeError):
pass
+def _error_text(status: int, body: str) -> str:
+ try:
+ data = json.loads(body)
+ except ValueError:
+ return f"HTTP {status}: {body[:300]}".strip()
+ if not isinstance(data, dict):
+ return f"HTTP {status}: {body[:300]}"
+ message = str(data.get("error") or body[:300])
+ trace = [str(item) for item in data.get("trace") or [] if item]
+ return message + (f" ({'; '.join(trace)})" if trace else "")
+
+
+def _percent(used: float | None, total: float | None) -> float | None:
+ if not used or not total:
+ return None
+ return used / total * 100
+
+
@dataclass(slots=True)
class Server:
id: str
name: str
state: str
+ version: str = ""
+ cpu: float | None = None
+ load: float | None = None
+ mem_used: float | None = None
+ mem_total: float | None = None
+ disk_used: float | None = None
+ disk_total: float | None = None
+ thresholds: dict = field(default_factory=dict)
@property
def mark(self) -> str:
- return {"Ok": "🟢", "NotOk": "🔴", "Disabled": "⏸"}.get(self.state, "⚪️")
+ if self.state == "Ok" and self.strained:
+ return "🟠"
+ return SERVER_MARKS.get(self.state, "⚪️")
+
+ @property
+ def mem(self) -> float | None:
+ return _percent(self.mem_used, self.mem_total)
+
+ @property
+ def disk(self) -> float | None:
+ return _percent(self.disk_used, self.disk_total)
+
+ @property
+ def strains(self) -> list[str]:
+ found = []
+ for label, value, key in (
+ ("cpu", self.cpu, "cpu_warning"),
+ ("память", self.mem, "mem_warning"),
+ ("диск", self.disk, "disk_warning"),
+ ):
+ limit = self.thresholds.get(key)
+ if value is not None and limit and value >= limit:
+ found.append(f"{label} {value:.0f}%")
+ return found
+
+ @property
+ def strained(self) -> bool:
+ return bool(self.strains)
@dataclass(slots=True)
@@ -38,52 +135,208 @@ class Stack:
id: str
name: str
state: str
+ status: str
server_id: str
+ server_name: str
+ repo: str = ""
+ branch: str = ""
+ deployed_hash: str = ""
+ latest_hash: str = ""
+ image_updates: int = 0
+ services: list[str] = field(default_factory=list)
@property
def mark(self) -> str:
- return {
- "running": "🟢",
- "unhealthy": "🟠",
- "paused": "⏸",
- "down": "⚫️",
- "stopped": "🔴",
- "restarting": "🔄",
- "unknown": "⚪️",
- }.get(self.state, "⚪️")
+ return STACK_MARKS.get(self.state, "⚪️")
+
+ @property
+ def healthy(self) -> bool:
+ return self.state == "running"
+
+ @property
+ def troubled(self) -> bool:
+ return self.state not in IDLE_STATES
+
+ @property
+ def behind(self) -> bool:
+ return bool(
+ self.deployed_hash
+ and self.latest_hash
+ and self.deployed_hash != self.latest_hash
+ )
+
+
+@dataclass(slots=True)
+class Container:
+ service: str
+ state: str
+ status: str
+ image: str
+ cpu: str = ""
+ mem: str = ""
+
+ @property
+ def mark(self) -> str:
+ return STACK_MARKS.get(self.state.lower(), "⚪️")
+
+
+@dataclass(slots=True)
+class Resource:
+ kind: str
+ id: str
+ name: str
+ state: str
+ last_run_at: int | None = None
+ next_run_at: int | None = None
+ schedule_error: str | None = None
+
+ @property
+ def mark(self) -> str:
+ return RUN_MARKS.get(self.state, "⚪️")
+
+
+@dataclass(slots=True)
+class Detail:
+ kind: str
+ id: str
+ name: str
+ description: str = ""
+ args_format: str = ""
+ args_text: str = ""
+ schedule: str = ""
+ schedule_enabled: bool = False
+ stages: list[str] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class Stage:
+ name: str
+ success: bool
+ output: str
+
+
+@dataclass(slots=True)
+class Update:
+ id: str
+ operation: str
+ status: str
+ success: bool
+ start_ts: int
+ end_ts: int | None
+ target_type: str
+ target_id: str
+ operator: str
+ stages: list[Stage] = field(default_factory=list)
+
+ @property
+ def done(self) -> bool:
+ return self.status == "Complete"
+
+ @property
+ def seconds(self) -> float:
+ end = self.end_ts or int(time.time() * 1000)
+ return max(0.0, (end - self.start_ts) / 1000)
+
+
+def _server(raw: dict) -> Server:
+ info = raw.get("info") or {}
+ stats = info.get("stats") or {}
+ load = (stats.get("load_average") or {}).get("one")
+ return Server(
+ id=raw["id"],
+ name=raw["name"],
+ state=info.get("state", ""),
+ version=info.get("version") or "",
+ cpu=stats.get("cpu_perc"),
+ load=load,
+ mem_used=stats.get("mem_used_gb"),
+ mem_total=stats.get("mem_total_gb"),
+ disk_used=stats.get("disk_used_gb"),
+ disk_total=stats.get("disk_total_gb"),
+ thresholds=info.get("alerting_thresholds") or {},
+ )
+
+
+def _stack(raw: dict) -> Stack:
+ info = raw.get("info") or {}
+ services = info.get("services") or []
+ return Stack(
+ id=raw["id"],
+ name=raw["name"],
+ state=info.get("state", ""),
+ status=info.get("status") or "",
+ server_id=info.get("server_id", ""),
+ server_name=info.get("server_name") or "",
+ repo=info.get("repo") or "",
+ branch=info.get("branch") or "",
+ deployed_hash=info.get("deployed_hash") or "",
+ latest_hash=info.get("latest_hash") or "",
+ image_updates=sum(bool(s.get("update_available")) for s in services),
+ services=[s.get("service", "") for s in services],
+ )
+
+
+def _resource(kind: str, raw: dict) -> Resource:
+ info = raw.get("info") or {}
+ return Resource(
+ kind=kind,
+ id=raw["id"],
+ name=raw["name"],
+ state=info.get("state", ""),
+ last_run_at=info.get("last_run_at"),
+ next_run_at=info.get("next_scheduled_run"),
+ schedule_error=info.get("schedule_error"),
+ )
+
+
+def _update(raw: dict) -> Update:
+ target = raw.get("target") or {}
+ return Update(
+ id=oid(raw),
+ operation=raw.get("operation", ""),
+ status=raw.get("status", ""),
+ success=bool(raw.get("success")),
+ start_ts=raw.get("start_ts") or 0,
+ end_ts=raw.get("end_ts"),
+ target_type=target.get("type", ""),
+ target_id=target.get("id", ""),
+ operator=raw.get("username") or raw.get("operator") or "",
+ stages=[
+ Stage(
+ name=log.get("stage", ""),
+ success=bool(log.get("success")),
+ output=(log.get("stdout") or "") + "\n" + (log.get("stderr") or ""),
+ )
+ for log in raw.get("logs") or []
+ ],
+ )
class Komodo:
def __init__(self) -> None:
self._session: aiohttp.ClientSession | None = None
+ self._cache: dict[str, tuple[float, Any]] = {}
+ self._usernames: dict[str, str] = {}
@property
def enabled(self) -> bool:
return env.komodo.enabled
- async def _call(self, route: str, kind: str, params: dict | None = None) -> Any: # noqa: ANN401
- if not self.enabled:
- msg = "Komodo не настроен (KOMODO__URL / KOMODO__KEY / KOMODO__SECRET)"
- raise KomodoError(msg)
+ async def _post(self, route: str, kind: str, params: dict) -> tuple[int, str]:
if self._session is None or self._session.closed:
self._session = aiohttp.ClientSession()
url = env.komodo.url.rstrip("/") + route
try:
async with self._session.post(
url,
- json={"type": kind, "params": params or {}},
+ json={"type": kind, "params": params},
headers={
"X-Api-Key": env.komodo.key.get_secret_value(),
"X-Api-Secret": env.komodo.secret.get_secret_value(),
},
timeout=aiohttp.ClientTimeout(total=env.komodo.timeout),
) as resp:
- text = await resp.text()
- if resp.status != OK:
- logger.warning(f"Komodo {kind} -> HTTP {resp.status}: {text[:300]}")
- msg = f"Komodo ответил HTTP {resp.status}"
- raise KomodoError(msg)
- return await resp.json()
+ return resp.status, await resp.text()
except TimeoutError as exc:
msg = f"Komodo не ответил за {env.komodo.timeout}с"
raise KomodoError(msg) from exc
@@ -91,38 +344,194 @@ class Komodo:
msg = f"Нет связи с Komodo: {exc}"
raise KomodoError(msg) from exc
- async def read(self, kind: str, params: dict | None = None) -> Any: # noqa: ANN401
- return await self._call("/read", kind, params)
+ async def _call(self, route: str, kind: str, params: dict | None = None) -> Any: # noqa: ANN401
+ if not self.enabled:
+ msg = "Komodo не настроен: нужны KOMODO__URL, KOMODO__KEY, KOMODO__SECRET"
+ raise KomodoError(msg)
+ status, body = 0, ""
+ pauses = BUSY_PAUSES if route == "/read" else ()
+ for pause in (*pauses, None):
+ status, body = await self._post(route, kind, params or {})
+ if status == OK:
+ return json.loads(body)
+ if pause is None or "busy" not in body.lower():
+ break
+ await asyncio.sleep(pause)
+ logger.warning(f"Komodo {kind} -> HTTP {status}: {body[:300]}")
+ raise KomodoError(_error_text(status, body))
- async def execute(self, kind: str, params: dict | None = None) -> Any: # noqa: ANN401
- return await self._call("/execute", kind, params)
+ async def read(
+ self, kind: str, params: dict | None = None, *, ttl: float = 0
+ ) -> Any: # noqa: ANN401
+ key = kind + json.dumps(params or {}, sort_keys=True)
+ cached = self._cache.get(key)
+ if ttl and cached and time.monotonic() - cached[0] < ttl:
+ return cached[1]
+ data = await self._call("/read", kind, params)
+ if ttl:
+ self._cache[key] = (time.monotonic(), data)
+ return data
+
+ async def execute(self, kind: str, params: dict | None = None) -> Update:
+ raw = await self._call("/execute", kind, params)
+ self._cache.clear()
+ return _update(raw if isinstance(raw, dict) else {})
+
+ def forget(self) -> None:
+ self._cache.clear()
+
+ async def _list(self, kind: str, ttl: float = CACHE_TTL) -> list[dict]:
+ return await self.read(kind, {"limit": 0}, ttl=ttl)
+
+ async def version(self) -> str:
+ return (await self.read("GetVersion", ttl=NAMES_TTL)).get("version", "")
async def servers(self) -> list[Server]:
- raw = await self.read("ListServers")
- return [
- Server(id=s["id"], name=s["name"], state=s.get("info", {}).get("state", ""))
- for s in raw
- ]
+ return [_server(s) for s in await self._list("ListServers")]
async def stacks(self) -> list[Stack]:
- raw = await self.read("ListStacks")
- return [
- Stack(
- id=s["id"],
- name=s["name"],
- state=s.get("info", {}).get("state", ""),
- server_id=s.get("info", {}).get("server_id", ""),
- )
- for s in raw
- ]
+ return [_stack(s) for s in await self._list("ListStacks")]
- async def stack_log(self, stack: str, tail: int = 40) -> str:
- """Хвост лога стека одной строкой."""
+ async def stack(self, stack_id: str) -> Stack | None:
+ return next((s for s in await self.stacks() if s.id == stack_id), None)
+
+ async def containers(self, stack_id: str) -> list[Container]:
+ raw = await self.read("ListStackServices", {"stack": stack_id})
+ result = []
+ for item in raw:
+ container = item.get("container") or {}
+ stats = container.get("stats") or {}
+ result.append(
+ Container(
+ service=item.get("service", ""),
+ state=container.get("state") or "down",
+ status=container.get("status") or "",
+ image=item.get("image") or container.get("image") or "",
+ cpu=stats.get("cpu_perc") or "",
+ mem=(stats.get("mem_usage") or "").split(" / ")[0],
+ )
+ )
+ return result
+
+ async def disks(self, server_id: str) -> list[tuple[str, float, float]]:
+ raw = await self.read("GetSystemStats", {"server": server_id}, ttl=CACHE_TTL)
+ seen: set[tuple[float, float]] = set()
+ result = []
+ for disk in raw.get("disks") or []:
+ used, total = disk.get("used_gb") or 0.0, disk.get("total_gb") or 0.0
+ if not total or (used, total) in seen:
+ continue
+ seen.add((used, total))
+ result.append((disk.get("mount", "?"), used, total))
+ return result
+
+ async def stack_log(self, stack: str, services: list[str], tail: int) -> str:
raw = await self.read(
- "GetStackLog", {"stack": stack, "services": [], "tail": tail}
+ "GetStackLog", {"stack": stack, "services": services, "tail": tail}
)
out = (raw.get("stdout") or "") + (raw.get("stderr") or "")
- return strip_ansi(out).strip() or "(пусто)"
+ return strip_ansi(out).strip()
+
+ async def resources(self, kind: str) -> list[Resource]:
+ api, _ = KIND_API[kind]
+ items = [_resource(kind, r) for r in await self._list(f"List{api}s")]
+ return sorted(items, key=lambda r: r.name.lower())
+
+ async def detail(self, kind: str, target_id: str) -> Detail:
+ api, param = KIND_API[kind]
+ raw = await self.read(f"Get{api}", {param: target_id}, ttl=CACHE_TTL)
+ config = raw.get("config") or {}
+ return Detail(
+ kind=kind,
+ id=oid(raw),
+ name=raw.get("name", ""),
+ description=raw.get("description") or "",
+ args_format=config.get("arguments_format") or "",
+ args_text=config.get("arguments") or "",
+ schedule=config.get("schedule") or "",
+ schedule_enabled=bool(config.get("schedule_enabled")),
+ stages=[
+ stage.get("name", "")
+ for stage in config.get("stages") or []
+ if stage.get("enabled", True)
+ ],
+ )
+
+ async def run(self, kind: str, target_id: str, args: dict | None) -> Update:
+ api, param = KIND_API[kind]
+ params: dict[str, Any] = {param: target_id}
+ if kind == "a" and args:
+ params["args"] = args
+ return await self.execute(f"Run{api}", params)
+
+ async def update(self, update_id: str) -> Update:
+ return _update(await self.read("GetUpdate", {"id": update_id}))
+
+ async def updates(self, query: dict | None = None, page: int = 0) -> list[Update]:
+ raw = await self.read("ListUpdates", {"query": query or {}, "page": page})
+ return [_update(u) for u in raw.get("updates") or []]
+
+ async def open_alerts(self) -> list[dict]:
+ try:
+ raw = await self.read(
+ "ListAlerts", {"query": {"resolved": False}}, ttl=CACHE_TTL
+ )
+ except KomodoError:
+ return []
+ return raw.get("alerts") or []
+
+ async def wait(self, update_id: str, timeout: float) -> Update | None: # noqa: ASYNC109
+ started = time.monotonic()
+ failures = 0
+ while True:
+ elapsed = time.monotonic() - started
+ try:
+ update = await self.update(update_id)
+ failures = 0
+ except KomodoError:
+ failures += 1
+ if failures >= WAIT_FAILURES:
+ raise
+ update = None
+ if update is not None and update.done:
+ self._cache.clear()
+ return update
+ if elapsed >= timeout:
+ return None
+ await asyncio.sleep(POLL_FAST if elapsed < POLL_FAST_FOR else POLL_SLOW)
+
+ async def names(self) -> dict[str, str]:
+ kinds = (
+ "ListServers",
+ "ListStacks",
+ "ListActions",
+ "ListProcedures",
+ "ListResourceSyncs",
+ "ListRepos",
+ "ListDeployments",
+ "ListBuilds",
+ "ListAlerters",
+ )
+ lists = await asyncio.gather(
+ *(self._list(kind, ttl=NAMES_TTL) for kind in kinds), return_exceptions=True
+ )
+ result: dict[str, str] = {}
+ for items in lists:
+ if isinstance(items, BaseException):
+ continue
+ result.update({item["id"]: item["name"] for item in items})
+ return result
+
+ async def username(self, user: str) -> str:
+ if not SYSTEM_USER_RE.fullmatch(user):
+ return user
+ if user not in self._usernames:
+ try:
+ raw = await self.read("GetUsername", {"user_id": user})
+ self._usernames[user] = raw.get("username") or user
+ except KomodoError:
+ self._usernames[user] = "Komodo"
+ return self._usernames[user]
async def close(self) -> None:
if self._session is not None and not self._session.closed:
diff --git a/src/healthbot/monitor.py b/src/healthbot/monitor.py
index 91ab5bd..3cee02a 100644
--- a/src/healthbot/monitor.py
+++ b/src/healthbot/monitor.py
@@ -1,24 +1,47 @@
-"""Фоновый цикл: пингует сервисы и уведомляет о падениях/восстановлениях."""
-
import asyncio
-import html
+import time
+from collections.abc import Awaitable
+from datetime import datetime
import aiohttp
+from aiogram.types import InlineKeyboardMarkup
+from aiogram.utils.keyboard import InlineKeyboardBuilder
-from healthbot.checker import Verdict, evaluate, probe
+from healthbot.checker import Verdict, evaluate, probe, tls_expiry
from healthbot.notify import notify
+from healthbot.stats import current_hour
+from healthbot.ui import esc
from utils.db import db
from utils.db.models import Service
from utils.env import env
from utils.format import human_duration, now, parse
from utils.logging import logger
+CONCURRENCY = 10
+KEEP_HOURS = 24 * 40
+TLS_EVERY = 6 * 3600
+TLS_WARN_DAYS = (14, 7, 3, 1)
+DAY = 86400
+
def _head(service: Service, icon: str, tail: str) -> str:
- return (
- f"{icon} {html.escape(service.name)} {tail}\n"
- f"{html.escape(service.url)}"
- )
+ return f"{icon} {esc(service.name)} {tail}\n{esc(service.url)}"
+
+
+def _down_kb(service: Service, incident_id: int | None) -> InlineKeyboardMarkup:
+ kb = InlineKeyboardBuilder()
+ kb.button(text="Проверить", callback_data=f"go:check:{service.id}")
+ kb.button(text="Открыть", callback_data=f"go:svc:{service.id}")
+ if incident_id is not None:
+ kb.button(text="Не напоминать", callback_data=f"svc:mute:{incident_id}")
+ kb.adjust(2, 1)
+ return kb.as_markup()
+
+
+def _open_kb(service: Service) -> InlineKeyboardMarkup:
+ kb = InlineKeyboardBuilder()
+ kb.button(text="Открыть", callback_data=f"go:svc:{service.id}")
+ return kb.as_markup()
async def _on_recovered(service: Service) -> None:
@@ -28,24 +51,27 @@ async def _on_recovered(service: Service) -> None:
await db.incidents.close(current.id)
downtime = human_duration((now() - parse(current.started_at)).total_seconds())
await db.services.set_up(service.id, is_up=True)
- await notify(f"{_head(service, '✅', 'снова в строю')}\nДаунтайм: {downtime}")
+ await notify(
+ f"{_head(service, '✅', 'снова в строю')}\nДаунтайм: {downtime}",
+ _open_kb(service),
+ )
logger.info(f"[green]UP[/] {service.name}")
async def _on_failed(service: Service, verdict: Verdict) -> None:
- await db.incidents.open(service.id, verdict.reason)
+ incident = await db.incidents.open(service.id, verdict.reason)
await db.services.set_up(service.id, is_up=False)
- lines = [_head(service, "🔴", "лёг"), f"Причина: {html.escape(verdict.reason)}"]
+ lines = [_head(service, "🔴", "лёг"), f"Причина: {esc(verdict.reason)}"]
report = verdict.report()
if report:
lines.append(report)
- await notify("\n".join(lines))
+ await notify("\n".join(lines), _down_kb(service, incident.id))
logger.warning(f"[red]DOWN[/] {service.name}: {verdict.reason}")
async def _on_still_down(service: Service, verdict: Verdict) -> None:
current = await db.incidents.current(service.id)
- if current is None:
+ if current is None or current.muted:
return
idle = (now() - parse(current.last_reminder_at)).total_seconds()
if idle < env.monitor.reminder_interval:
@@ -54,19 +80,24 @@ async def _on_still_down(service: Service, verdict: Verdict) -> None:
since = human_duration((now() - parse(current.started_at)).total_seconds())
lines = [
_head(service, "⏰", f"всё ещё лежит (уже {since})"),
- f"Причина: {html.escape(verdict.reason)}",
+ f"Причина: {esc(verdict.reason)}",
]
report = verdict.report()
if report:
lines.append(report)
- await notify("\n".join(lines))
+ await notify("\n".join(lines), _down_kb(service, current.id))
async def check_one(session: aiohttp.ClientSession, service: Service) -> Verdict:
- """Одна проверка со всеми побочными эффектами. Возвращает вердикт."""
result = await probe(session, service.url, env.monitor.request_timeout)
verdict = evaluate(service, result)
await db.services.touch_probe(service.id, result.latency_ms)
+ await db.stats.record(
+ service.id,
+ current_hour(),
+ healthy=verdict.healthy,
+ latency_ms=result.latency_ms,
+ )
if verdict.healthy:
if service.fail_streak:
@@ -91,21 +122,65 @@ async def check_one(session: aiohttp.ClientSession, service: Service) -> Verdict
return verdict
+def tls_step(expires: datetime, warned: int) -> int:
+ days = (expires - now()).total_seconds() / DAY
+ due = [limit for limit in TLS_WARN_DAYS if days <= limit]
+ if not due:
+ return 0
+ step = min(due)
+ return step if not warned or step < warned else warned
+
+
+async def check_tls(service: Service) -> None:
+ expires = await tls_expiry(service.url)
+ if expires is None:
+ return
+ step = tls_step(expires, service.tls_warned)
+ await db.services.set_tls(service.id, expires.isoformat(), step)
+ if step and step != service.tls_warned:
+ left = max(0.0, (expires - now()).total_seconds())
+ await notify(
+ f"{_head(service, '🟠', 'сертификат скоро истечёт')}\n"
+ f"Осталось: {human_duration(left)}",
+ _open_kb(service),
+ )
+
+
+async def _guarded(sem: asyncio.Semaphore, job: Awaitable[object], what: str) -> None:
+ async with sem:
+ try:
+ await job
+ except Exception:
+ logger.exception(f"Check failed: {what}")
+
+
+async def _sweep(session: aiohttp.ClientSession, *, with_tls: bool) -> None:
+ services = [s for s in await db.services.all() if not s.is_paused]
+ sem = asyncio.Semaphore(CONCURRENCY)
+ jobs = [_guarded(sem, check_one(session, s), s.url) for s in services]
+ if with_tls:
+ jobs += [_guarded(sem, check_tls(s), f"tls {s.url}") for s in services]
+ await asyncio.gather(*jobs)
+
+
async def run_monitor() -> None:
logger.info(
- f"Monitor: каждые {env.monitor.check_interval}с, "
- f"порог падения {env.monitor.failure_threshold}"
+ f"Monitor: every {env.monitor.check_interval}s, "
+ f"failure threshold {env.monitor.failure_threshold}"
)
+ next_tls = 0.0
+ purged_hour = 0
async with aiohttp.ClientSession() as session:
while True:
try:
- for service in await db.services.all():
- if service.is_paused:
- continue
- try:
- await check_one(session, service)
- except Exception:
- logger.exception(f"Ошибка проверки {service.url}")
+ with_tls = time.monotonic() >= next_tls
+ await _sweep(session, with_tls=with_tls)
+ if with_tls:
+ next_tls = time.monotonic() + TLS_EVERY
+ hour = current_hour()
+ if hour != purged_hour:
+ await db.stats.purge(hour - KEEP_HOURS)
+ purged_hour = hour
except Exception:
- logger.exception("Ошибка цикла мониторинга")
+ logger.exception("Monitor loop failed")
await asyncio.sleep(env.monitor.check_interval)
diff --git a/src/healthbot/notify.py b/src/healthbot/notify.py
index a57b046..84417dc 100644
--- a/src/healthbot/notify.py
+++ b/src/healthbot/notify.py
@@ -1,28 +1,15 @@
-"""Отправка уведомлений админам."""
+from aiogram.types import InlineKeyboardMarkup
from healthbot.common import bot
+from healthbot.ui import fit
from utils.env import env
from utils.logging import logger
-TELEGRAM_LIMIT = 4096
-SAFE_LIMIT = 3900
-
-def clip(text: str) -> str:
- if len(text) <= TELEGRAM_LIMIT:
- return text
- cut = text[:SAFE_LIMIT]
- cut = cut[: cut.rfind("\n")] if "\n" in cut else cut
- if cut.count("") > cut.count(""):
- cut += ""
- return cut + "\n…обрезано"
-
-
-async def notify(text: str) -> None:
- """Разослать всем админам. Падение одного не мешает остальным."""
- payload = clip(text)
+async def notify(text: str, markup: InlineKeyboardMarkup | None = None) -> None:
+ payload = fit(text)
for admin in env.admins:
try:
- await bot.send_message(admin, payload)
+ await bot.send_message(admin, payload, reply_markup=markup)
except Exception:
- logger.exception(f"Не смог отправить уведомление {admin}")
+ logger.exception(f"Notification to {admin} failed")
diff --git a/src/healthbot/runs.py b/src/healthbot/runs.py
new file mode 100644
index 0000000..4bbb5e4
--- /dev/null
+++ b/src/healthbot/runs.py
@@ -0,0 +1,147 @@
+import contextlib
+from collections.abc import Awaitable, Callable
+
+from aiogram.exceptions import TelegramBadRequest
+from aiogram.types import CallbackQuery, InlineKeyboardMarkup, Message
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+from healthbot.komodo import KomodoError, Update, clean_output, komodo
+from healthbot.ui import edit, esc, pre, show
+from utils.format import clock
+
+RUN_TIMEOUT = 30 * 60
+OUTPUT_LIMIT = 3000
+QUICK_SECONDS = 10
+NOISE = ("Action completed successfully",)
+ACTIVE: set[str] = set()
+
+Back = list[tuple[str, str]]
+
+
+def _stage_text(output: str) -> str:
+ lines = [
+ line
+ for line in clean_output(output).splitlines()
+ if not any(mark in line for mark in NOISE)
+ ]
+ return "\n".join(lines).strip()
+
+
+def output_of(update: Update, *, only_failed: bool) -> str:
+ parts = []
+ for stage in update.stages:
+ if "config" in stage.name.lower():
+ if not stage.success:
+ parts.append(f"{stage.name}: вывод скрыт, в нём бывают секреты")
+ continue
+ if only_failed and stage.success:
+ continue
+ text = _stage_text(stage.output)
+ if not text:
+ continue
+ parts.append(f"{stage.name}:\n{text}" if len(update.stages) > 1 else text)
+ return "\n\n".join(parts)
+
+
+def result_text(title: str, update: Update, *, verbose: bool, extra: str = "") -> str:
+ icon = "✅" if update.success else "🔴"
+ state = "" if update.success else " не удалось"
+ lines = [f"{icon} {esc(title)}{state} · {clock(update.seconds)}"]
+ if extra:
+ lines.append(extra)
+ if verbose or not update.success:
+ output = output_of(update, only_failed=not verbose)
+ if output:
+ lines.append(pre(output, OUTPUT_LIMIT))
+ return "\n".join(lines)
+
+
+def result_kb(update: Update | None, back: Back) -> InlineKeyboardMarkup:
+ kb = InlineKeyboardBuilder()
+ for text, data in back:
+ kb.button(text=text, callback_data=data)
+ rows = [len(back)] if back else []
+ if update is not None and len(output_of(update, only_failed=False)) > OUTPUT_LIMIT:
+ kb.button(text="Весь вывод файлом", callback_data=f"run:file:{update.id}")
+ rows.append(1)
+ kb.adjust(*rows or [1])
+ return kb.as_markup()
+
+
+async def track( # noqa: PLR0913
+ origin: Message | CallbackQuery,
+ title: str,
+ starter: Callable[[], Awaitable[Update]],
+ *,
+ back: Back,
+ verbose: bool,
+ extra: str = "",
+ new: bool = False,
+) -> None:
+ source = origin.message if isinstance(origin, CallbackQuery) else origin
+ key = f"{source.message_id if source else 0}:{title}"
+ if key in ACTIVE:
+ return
+ ACTIVE.add(key)
+ try:
+ await _track(
+ origin, title, starter, back, verbose=verbose, extra=extra, new=new
+ )
+ finally:
+ ACTIVE.discard(key)
+
+
+async def _track( # noqa: PLR0913
+ origin: Message | CallbackQuery,
+ title: str,
+ starter: Callable[[], Awaitable[Update]],
+ back: Back,
+ *,
+ verbose: bool,
+ extra: str,
+ new: bool,
+) -> None:
+ progress = await show(origin, f"⏳ {esc(title)} выполняется", new=new)
+ if progress is None:
+ return
+ try:
+ started = await starter()
+ except KomodoError as exc:
+ await edit(
+ progress,
+ f"🔴 {esc(title)} не запустилось\n{esc(exc)}",
+ result_kb(None, back),
+ )
+ return
+
+ if not started.id:
+ await edit(progress, f"✅ {esc(title)} запущено", result_kb(None, back))
+ return
+
+ check = InlineKeyboardBuilder()
+ check.button(text="Проверить", callback_data=f"run:st:{started.id}")
+ try:
+ update = await komodo.wait(started.id, RUN_TIMEOUT)
+ except KomodoError as exc:
+ await edit(
+ progress,
+ f"⏳ {esc(title)} запущено, результат узнать не удалось\n{esc(exc)}",
+ check.as_markup(),
+ )
+ return
+ if update is None:
+ await edit(
+ progress,
+ f"⏳ {esc(title)} идёт дольше {RUN_TIMEOUT // 60} минут",
+ check.as_markup(),
+ )
+ return
+
+ text = result_text(title, update, verbose=verbose, extra=extra)
+ markup = result_kb(update, back)
+ if update.seconds < QUICK_SECONDS:
+ await edit(progress, text, markup)
+ return
+ await show(progress, text, markup)
+ with contextlib.suppress(TelegramBadRequest):
+ await progress.delete()
diff --git a/src/healthbot/stats.py b/src/healthbot/stats.py
new file mode 100644
index 0000000..f750343
--- /dev/null
+++ b/src/healthbot/stats.py
@@ -0,0 +1,67 @@
+from datetime import datetime, timedelta
+
+from healthbot.ui import pct, spark
+from utils.db.models import Hour, Incident, Service
+from utils.format import now, parse, zone
+
+DAY = 86400
+PERIODS = (("24ч", DAY), ("7д", 7 * DAY), ("30д", 30 * DAY))
+CHART_DAYS = 14
+CHART_HOURS = 24
+BLIP = 300
+
+
+def current_hour() -> int:
+ return int(now().timestamp()) // 3600
+
+
+def downtime(incidents: list[Incident], start: datetime, end: datetime) -> float:
+ total = 0.0
+ for incident in incidents:
+ began = max(parse(incident.started_at), start)
+ ended = min(parse(incident.ended_at) if incident.ended_at else end, end)
+ if ended > began:
+ total += (ended - began).total_seconds()
+ return total
+
+
+def uptime(service: Service, incidents: list[Incident], period: float) -> float | None:
+ end = now()
+ start = max(end - timedelta(seconds=period), parse(service.created_at))
+ window = (end - start).total_seconds()
+ if window <= 0:
+ return None
+ return max(0.0, (window - downtime(incidents, start, end)) / window * 100)
+
+
+def uptime_line(service: Service, incidents: list[Incident]) -> str:
+ return ", ".join(
+ f"{name} {pct(uptime(service, incidents, period))}" for name, period in PERIODS
+ )
+
+
+def day_squares(service: Service, incidents: list[Incident], tz_name: str) -> str:
+ tz = zone(tz_name)
+ today = now().astimezone(tz).replace(hour=0, minute=0, second=0, microsecond=0)
+ created = parse(service.created_at)
+ squares = []
+ for offset in range(CHART_DAYS - 1, -1, -1):
+ start = today - timedelta(days=offset)
+ end = min(start + timedelta(days=1), now())
+ if end <= created:
+ squares.append("⬜")
+ continue
+ lost = downtime(incidents, max(start, created), end)
+ squares.append("🟩" if not lost else "🟨" if lost < BLIP else "🟥")
+ return "".join(squares)
+
+
+def latency_chart(hours: dict[int, Hour]) -> tuple[str, int | None, int | None]:
+ last = current_hour()
+ window = [hours.get(h) for h in range(last - CHART_HOURS + 1, last + 1)]
+ values = [h.latency if h else None for h in window]
+ present = [v for v in values if v is not None]
+ if not present:
+ return "", None, None
+ peak = max(h.latency_max for h in window if h and h.latency_n)
+ return spark(values), sum(present) // len(present), peak
diff --git a/src/healthbot/ui.py b/src/healthbot/ui.py
new file mode 100644
index 0000000..7ae7bf9
--- /dev/null
+++ b/src/healthbot/ui.py
@@ -0,0 +1,153 @@
+import html
+from collections.abc import Sequence
+
+from aiogram import F
+from aiogram.enums import ButtonStyle
+from aiogram.exceptions import TelegramBadRequest
+from aiogram.types import CallbackQuery, InlineKeyboardMarkup, Message
+from aiogram.utils.keyboard import InlineKeyboardBuilder
+
+TELEGRAM_LIMIT = 4096
+SAFE_LIMIT = 3900
+BUTTON_LIMIT = 48
+SPARK = "▁▂▃▄▅▆▇█"
+FULL = 100
+
+MENU_COMMANDS = ("start", "list", "stats", "fleet", "actions", "settings")
+ANSWER = F.text & ~F.text.regexp(rf"^/({'|'.join(MENU_COMMANDS)})(@\w+)?$")
+
+PRIMARY = ButtonStyle.PRIMARY
+SUCCESS = ButtonStyle.SUCCESS
+DANGER = ButtonStyle.DANGER
+
+
+def esc(value: object) -> str:
+ return html.escape(str(value), quote=False)
+
+
+def fit(text: str) -> str:
+ if len(text) <= TELEGRAM_LIMIT:
+ return text
+ cut = text[:SAFE_LIMIT]
+ cut = cut[: cut.rfind("\n")] if "\n" in cut else cut
+ cut = cut[: cut.rfind("<")] if cut.rfind("<") > cut.rfind(">") else cut
+ for tag in ("code", "pre", "b", "i"):
+ if cut.count(f"<{tag}>") > cut.count(f""):
+ cut += f""
+ return cut + "\nобрезано"
+
+
+def tail(text: str, limit: int) -> str:
+ text = text.strip()
+ if len(text) <= limit:
+ return text
+ cut = text[-limit:]
+ return cut[cut.find("\n") + 1 :] if "\n" in cut else cut
+
+
+def pre(text: str, limit: int = 3000) -> str:
+ return f"{esc(tail(text, limit))}"
+
+
+def label(text: str, limit: int = BUTTON_LIMIT) -> str:
+ text = " ".join(text.split())
+ return text if len(text) <= limit else text[: limit - 1] + "…"
+
+
+def pct(value: float | None) -> str:
+ if value is None:
+ return "нет данных"
+ if value >= FULL:
+ return "100%"
+ return f"{min(value, 99.99):.2f}%"
+
+
+def spark(values: Sequence[float | None]) -> str:
+ present = [v for v in values if v is not None]
+ if not present:
+ return ""
+ low, high = min(present), max(present)
+ span = high - low
+ chars = []
+ for value in values:
+ if value is None:
+ chars.append("·")
+ elif not span:
+ chars.append(SPARK[2])
+ else:
+ chars.append(SPARK[round((value - low) / span * (len(SPARK) - 1))])
+ return "".join(chars)
+
+
+def plural(count: int, one: str, few: str, many: str) -> str:
+ tens, units = count % 100, count % 10
+ if units == 1 and tens != 11: # noqa: PLR2004
+ return one
+ if units in {2, 3, 4} and tens not in {12, 13, 14}:
+ return few
+ return many
+
+
+def pages(total: int, size: int) -> int:
+ return max(1, (total + size - 1) // size)
+
+
+def pager(kb: InlineKeyboardBuilder, prefix: str, page: int, total_pages: int) -> int:
+ if total_pages <= 1:
+ return 0
+ count = 0
+ if page > 0:
+ kb.button(text="‹ Назад", callback_data=f"{prefix}:{page - 1}")
+ count += 1
+ if page < total_pages - 1:
+ kb.button(text="Дальше ›", callback_data=f"{prefix}:{page + 1}")
+ count += 1
+ return count
+
+
+async def show(
+ event: Message | CallbackQuery,
+ text: str,
+ markup: InlineKeyboardMarkup | None = None,
+ *,
+ new: bool = False,
+) -> Message | None:
+ text = fit(text)
+ if isinstance(event, Message):
+ return await event.answer(text, reply_markup=markup)
+ message = event.message
+ if isinstance(message, Message) and not new:
+ try:
+ edited = await message.edit_text(text, reply_markup=markup)
+ except TelegramBadRequest as exc:
+ if "not modified" in str(exc):
+ return message
+ else:
+ return edited if isinstance(edited, Message) else message
+ if isinstance(message, Message):
+ return await message.answer(text, reply_markup=markup)
+ if event.bot is not None:
+ return await event.bot.send_message(
+ event.from_user.id, text, reply_markup=markup
+ )
+ return None
+
+
+async def edit(
+ message: Message, text: str, markup: InlineKeyboardMarkup | None = None
+) -> None:
+ text = fit(text)
+ try:
+ await message.edit_text(text, reply_markup=markup)
+ except TelegramBadRequest as exc:
+ if "not modified" not in str(exc):
+ await message.answer(text, reply_markup=markup)
+
+
+def tail_arg(data: str | None) -> str:
+ return (data or "").rsplit(":", 1)[-1]
+
+
+def int_arg(data: str | None, default: int = 0) -> int:
+ value = tail_arg(data)
+ return int(value) if value.lstrip("-").isdigit() else default
diff --git a/src/healthbot/web.py b/src/healthbot/web.py
index dcbbde2..9066327 100644
--- a/src/healthbot/web.py
+++ b/src/healthbot/web.py
@@ -1,17 +1,15 @@
-"""HTTP-приёмник входящих алертов."""
-
import hmac
import json
from aiohttp import web
-from healthbot.alerts import format_alert
+from healthbot.alerts import alert_keyboard, alert_kind, format_alert
from healthbot.notify import notify
+from healthbot.ui import esc
+from utils.db import db
from utils.env import env
from utils.logging import logger
-MUTED = frozenset({"ScheduleRun"})
-
def _authorized(request: web.Request) -> bool:
expected = env.web.token.get_secret_value()
@@ -23,32 +21,34 @@ def _authorized(request: web.Request) -> bool:
async def handle_alert(request: web.Request) -> web.Response:
if not _authorized(request):
- logger.warning(f"Отклонён алерт без токена от {request.remote}")
+ logger.warning(f"Alert without a valid token from {request.remote}")
return web.json_response({"error": "unauthorized"}, status=401)
raw = await request.text()
try:
payload = json.loads(raw)
except ValueError:
- logger.warning(f"Алерт не разобрался как JSON: {raw[:200]}")
+ logger.warning(f"Alert is not JSON: {raw[:200]}")
return web.json_response({"error": "bad json"}, status=400)
if not isinstance(payload, dict):
return web.json_response({"error": "expected object"}, status=400)
- try:
- text = format_alert(payload)
- except Exception:
- logger.exception("Не смог отформатировать алерт")
- text = f"🔔 Алерт из Komodo (не разобрал формат)\n{raw[:1000]}"
+ kind = alert_kind(payload)
+ logger.info(f"Alert received: {kind} (level={payload.get('level')})")
- kind = (payload.get("data") or {}).get("type", "?")
- logger.info(f"Алерт принят: {kind} (level={payload.get('level')})")
-
- if kind in MUTED:
+ if kind in db.settings.get("muted"):
return web.json_response({"ok": True, "muted": True})
- await notify(text)
+ try:
+ text = format_alert(payload)
+ markup = alert_keyboard(payload)
+ except Exception:
+ logger.exception("Alert formatting failed")
+ text = f"🔔 Алерт из Komodo (не разобрал формат)\n{esc(raw[:1000])}"
+ markup = None
+
+ await notify(text, markup)
return web.json_response({"ok": True})
@@ -65,5 +65,5 @@ async def run_web() -> web.AppRunner:
await runner.setup()
site = web.TCPSite(runner, env.web.host, env.web.port)
await site.start()
- logger.info(f"Web: слушаю {env.web.host}:{env.web.port}")
+ logger.info(f"Web: listening on {env.web.host}:{env.web.port}")
return runner
diff --git a/src/utils/db/__init__.py b/src/utils/db/__init__.py
index 9342ffe..51430ee 100644
--- a/src/utils/db/__init__.py
+++ b/src/utils/db/__init__.py
@@ -1,10 +1,20 @@
+import shutil
from pathlib import Path
import aiosqlite
from utils.env import env
+from utils.logging import logger
-from .repositories import IncidentRepository, ServiceRepository
+from .repositories import (
+ IncidentRepository,
+ PinRepository,
+ ServiceRepository,
+ SettingRepository,
+ StatRepository,
+)
+
+SCHEMA_VERSION = 2
SCHEMA = """
CREATE TABLE IF NOT EXISTS services (
@@ -30,6 +40,33 @@ CREATE TABLE IF NOT EXISTS incidents (
);
CREATE INDEX IF NOT EXISTS idx_incidents_service ON incidents(service_id);
+
+CREATE TABLE IF NOT EXISTS settings (
+ key TEXT PRIMARY KEY,
+ value TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS pins (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ kind TEXT NOT NULL,
+ target_id TEXT NOT NULL,
+ name TEXT NOT NULL,
+ label TEXT NOT NULL,
+ mode TEXT NOT NULL,
+ args TEXT NOT NULL DEFAULT '{}',
+ position INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS hourly (
+ service_id INTEGER NOT NULL REFERENCES services(id) ON DELETE CASCADE,
+ hour INTEGER NOT NULL,
+ checks INTEGER NOT NULL DEFAULT 0,
+ fails INTEGER NOT NULL DEFAULT 0,
+ latency_sum INTEGER NOT NULL DEFAULT 0,
+ latency_n INTEGER NOT NULL DEFAULT 0,
+ latency_max INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY (service_id, hour)
+);
"""
MIGRATIONS: tuple[tuple[str, str, str], ...] = (
@@ -37,26 +74,66 @@ MIGRATIONS: tuple[tuple[str, str, str], ...] = (
("services", "is_paused", "INTEGER NOT NULL DEFAULT 0"),
("services", "last_latency_ms", "INTEGER"),
("services", "last_checked_at", "TEXT"),
+ ("services", "needle", "TEXT"),
+ ("services", "tls_expires_at", "TEXT"),
+ ("services", "tls_warned", "INTEGER NOT NULL DEFAULT 0"),
+ ("incidents", "muted", "INTEGER NOT NULL DEFAULT 0"),
)
+def _prepare(path: str) -> None:
+ Path(path).parent.mkdir(parents=True, exist_ok=True)
+
+
+def _has_data(path: str) -> bool:
+ source = Path(path)
+ return source.is_file() and source.stat().st_size > 0
+
+
+def _copy(path: str, version: int) -> Path:
+ source = Path(path)
+ target = source.with_name(f"{source.name}.v{version}.bak")
+ shutil.copy2(source, target)
+ return target
+
+
class Database:
def __init__(self, path: str) -> None:
self._path = path
self._conn: aiosqlite.Connection | None = None
self.services: ServiceRepository = None # type: ignore[assignment]
self.incidents: IncidentRepository = None # type: ignore[assignment]
+ self.pins: PinRepository = None # type: ignore[assignment]
+ self.settings: SettingRepository = None # type: ignore[assignment]
+ self.stats: StatRepository = None # type: ignore[assignment]
async def connect(self) -> None:
- Path(self._path).parent.mkdir(parents=True, exist_ok=True)
+ _prepare(self._path)
+ await self._backup_before_migration()
self._conn = await aiosqlite.connect(self._path)
self._conn.row_factory = aiosqlite.Row
await self._conn.execute("PRAGMA foreign_keys = ON")
await self._conn.executescript(SCHEMA)
await self._migrate()
+ await self._conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
await self._conn.commit()
self.services = ServiceRepository(self._conn)
self.incidents = IncidentRepository(self._conn)
+ self.pins = PinRepository(self._conn)
+ self.settings = SettingRepository(self._conn)
+ self.stats = StatRepository(self._conn)
+ await self.settings.load()
+
+ async def _backup_before_migration(self) -> None:
+ if not _has_data(self._path):
+ return
+ async with aiosqlite.connect(self._path) as conn:
+ cursor = await conn.execute("PRAGMA user_version")
+ row = await cursor.fetchone()
+ version = row[0] if row else 0
+ if version < SCHEMA_VERSION:
+ target = _copy(self._path, version)
+ logger.info(f"Database copied to {target} before schema {SCHEMA_VERSION}")
async def _migrate(self) -> None:
assert self._conn is not None
diff --git a/src/utils/db/models.py b/src/utils/db/models.py
index 782f630..5ab7a5c 100644
--- a/src/utils/db/models.py
+++ b/src/utils/db/models.py
@@ -1,11 +1,18 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from enum import StrEnum
+from html import escape
class MatchType(StrEnum):
STATUS = "status"
BODY = "body"
JSON = "json"
+ CONTAINS = "contains"
+
+
+class PinMode(StrEnum):
+ INSTANT = "instant"
+ FORM = "form"
@dataclass(slots=True)
@@ -24,14 +31,23 @@ class Service:
is_paused: bool = False
last_latency_ms: int | None = None
last_checked_at: str | None = None
+ needle: str | None = None
+ tls_expires_at: str | None = None
+ tls_warned: int = 0
@property
def rule(self) -> str:
if self.match_type is MatchType.STATUS:
- return f"HTTP-статус = {self.ok_status}"
+ return f"HTTP-статус {self.ok_status}"
if self.match_type is MatchType.JSON:
- return f"{self.json_path} = {self.json_value}"
- return "тело ответа неизменно"
+ path = escape(self.json_path or "", quote=False)
+ value = escape(self.json_value or "", quote=False)
+ return f"{path} = {value}"
+ if self.match_type is MatchType.CONTAINS:
+ return (
+ f"в ответе есть {escape(self.needle or '', quote=False)}"
+ )
+ return "тело ответа не меняется"
@property
def mark(self) -> str:
@@ -48,3 +64,30 @@ class Incident:
ended_at: str | None
reason: str
last_reminder_at: str
+ muted: bool = False
+
+
+@dataclass(slots=True)
+class Pin:
+ id: int
+ kind: str
+ target_id: str
+ name: str
+ label: str
+ mode: PinMode
+ args: dict = field(default_factory=dict)
+ position: int = 0
+
+
+@dataclass(slots=True)
+class Hour:
+ hour: int
+ checks: int
+ fails: int
+ latency_sum: int
+ latency_n: int
+ latency_max: int
+
+ @property
+ def latency(self) -> int | None:
+ return self.latency_sum // self.latency_n if self.latency_n else None
diff --git a/src/utils/db/repositories/__init__.py b/src/utils/db/repositories/__init__.py
index 69e1312..2458f3d 100644
--- a/src/utils/db/repositories/__init__.py
+++ b/src/utils/db/repositories/__init__.py
@@ -1,4 +1,13 @@
from .incident import IncidentRepository
+from .pin import PinRepository
from .service import ServiceRepository
+from .setting import SettingRepository
+from .stat import StatRepository
-__all__ = ["IncidentRepository", "ServiceRepository"]
+__all__ = [
+ "IncidentRepository",
+ "PinRepository",
+ "ServiceRepository",
+ "SettingRepository",
+ "StatRepository",
+]
diff --git a/src/utils/db/repositories/incident.py b/src/utils/db/repositories/incident.py
index a394db3..f5de842 100644
--- a/src/utils/db/repositories/incident.py
+++ b/src/utils/db/repositories/incident.py
@@ -3,7 +3,7 @@ import aiosqlite
from utils.db.models import Incident
from utils.format import now
-_COLUMNS = "id, service_id, started_at, ended_at, reason, last_reminder_at"
+_COLUMNS = "id, service_id, started_at, ended_at, reason, last_reminder_at, muted"
def _row_to_incident(row: aiosqlite.Row) -> Incident:
@@ -14,6 +14,7 @@ def _row_to_incident(row: aiosqlite.Row) -> Incident:
ended_at=row["ended_at"],
reason=row["reason"],
last_reminder_at=row["last_reminder_at"],
+ muted=bool(row["muted"]),
)
@@ -29,7 +30,7 @@ class IncidentRepository:
(service_id, stamp, reason, stamp),
)
await self._conn.commit()
- incident = await self.get(cursor.lastrowid) # type: ignore[arg-type]
+ incident = await self.get(cursor.lastrowid or 0)
assert incident is not None
return incident
@@ -65,6 +66,12 @@ class IncidentRepository:
)
await self._conn.commit()
+ async def mute(self, incident_id: int) -> None:
+ await self._conn.execute(
+ "UPDATE incidents SET muted = 1 WHERE id = ?", (incident_id,)
+ )
+ await self._conn.commit()
+
async def for_service(self, service_id: int) -> list[Incident]:
cursor = await self._conn.execute(
f"SELECT {_COLUMNS} FROM incidents " # noqa: S608
@@ -72,3 +79,11 @@ class IncidentRepository:
(service_id,),
)
return [_row_to_incident(row) for row in await cursor.fetchall()]
+
+ async def since(self, stamp: str) -> list[Incident]:
+ cursor = await self._conn.execute(
+ f"SELECT {_COLUMNS} FROM incidents " # noqa: S608
+ "WHERE ended_at IS NULL OR ended_at >= ? ORDER BY id",
+ (stamp,),
+ )
+ return [_row_to_incident(row) for row in await cursor.fetchall()]
diff --git a/src/utils/db/repositories/pin.py b/src/utils/db/repositories/pin.py
new file mode 100644
index 0000000..08da3bf
--- /dev/null
+++ b/src/utils/db/repositories/pin.py
@@ -0,0 +1,98 @@
+import json
+
+import aiosqlite
+
+from utils.db.models import Pin, PinMode
+
+_COLUMNS = "id, kind, target_id, name, label, mode, args, position"
+
+
+def _row_to_pin(row: aiosqlite.Row) -> Pin:
+ return Pin(
+ id=row["id"],
+ kind=row["kind"],
+ target_id=row["target_id"],
+ name=row["name"],
+ label=row["label"],
+ mode=PinMode(row["mode"]),
+ args=json.loads(row["args"] or "{}"),
+ position=row["position"],
+ )
+
+
+class PinRepository:
+ def __init__(self, conn: aiosqlite.Connection) -> None:
+ self._conn = conn
+
+ async def all(self) -> list[Pin]:
+ cursor = await self._conn.execute(
+ f"SELECT {_COLUMNS} FROM pins ORDER BY position, id" # noqa: S608
+ )
+ return [_row_to_pin(row) for row in await cursor.fetchall()]
+
+ async def get(self, pin_id: int) -> Pin | None:
+ cursor = await self._conn.execute(
+ f"SELECT {_COLUMNS} FROM pins WHERE id = ?", # noqa: S608
+ (pin_id,),
+ )
+ row = await cursor.fetchone()
+ return _row_to_pin(row) if row else None
+
+ async def add( # noqa: PLR0913
+ self,
+ *,
+ kind: str,
+ target_id: str,
+ name: str,
+ label: str,
+ mode: PinMode,
+ args: dict,
+ ) -> Pin:
+ cursor = await self._conn.execute(
+ "INSERT INTO pins (kind, target_id, name, label, mode, args, position) "
+ "VALUES (?, ?, ?, ?, ?, ?, "
+ "(SELECT COALESCE(MAX(position), 0) + 1 FROM pins))",
+ (
+ kind,
+ target_id,
+ name,
+ label,
+ mode.value,
+ json.dumps(args, ensure_ascii=False),
+ ),
+ )
+ await self._conn.commit()
+ pin = await self.get(cursor.lastrowid or 0)
+ assert pin is not None
+ return pin
+
+ async def rename(self, pin_id: int, label: str) -> None:
+ await self._conn.execute(
+ "UPDATE pins SET label = ? WHERE id = ?", (label, pin_id)
+ )
+ await self._conn.commit()
+
+ async def set_mode(self, pin_id: int, mode: PinMode) -> None:
+ await self._conn.execute(
+ "UPDATE pins SET mode = ? WHERE id = ?", (mode.value, pin_id)
+ )
+ await self._conn.commit()
+
+ async def move(self, pin_id: int, step: int) -> None:
+ pins = await self.all()
+ index = next((i for i, p in enumerate(pins) if p.id == pin_id), None)
+ if index is None:
+ return
+ target = index + step
+ if not 0 <= target < len(pins):
+ return
+ pins[index], pins[target] = pins[target], pins[index]
+ for position, pin in enumerate(pins, start=1):
+ await self._conn.execute(
+ "UPDATE pins SET position = ? WHERE id = ?", (position, pin.id)
+ )
+ await self._conn.commit()
+
+ async def delete(self, pin_id: int) -> None:
+ await self._conn.execute("DELETE FROM pins WHERE id = ?", (pin_id,))
+ await self._conn.commit()
diff --git a/src/utils/db/repositories/service.py b/src/utils/db/repositories/service.py
index 80db74a..b6f341c 100644
--- a/src/utils/db/repositories/service.py
+++ b/src/utils/db/repositories/service.py
@@ -6,7 +6,8 @@ from utils.format import now
_COLUMNS = (
"id, name, url, match_type, ok_status, ok_body, "
"json_path, json_value, is_up, created_at, "
- "fail_streak, is_paused, last_latency_ms, last_checked_at"
+ "fail_streak, is_paused, last_latency_ms, last_checked_at, "
+ "needle, tls_expires_at, tls_warned"
)
@@ -26,6 +27,9 @@ def _row_to_service(row: aiosqlite.Row) -> Service:
is_paused=bool(row["is_paused"]),
last_latency_ms=row["last_latency_ms"],
last_checked_at=row["last_checked_at"],
+ needle=row["needle"],
+ tls_expires_at=row["tls_expires_at"],
+ tls_warned=row["tls_warned"],
)
@@ -65,11 +69,12 @@ class ServiceRepository:
ok_body: str = "",
json_path: str | None = None,
json_value: str | None = None,
+ needle: str | None = None,
) -> Service:
cursor = await self._conn.execute(
"INSERT INTO services "
"(name, url, match_type, ok_status, ok_body, json_path, json_value, "
- "is_up, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?)",
+ "needle, is_up, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?)",
(
name,
url,
@@ -78,14 +83,49 @@ class ServiceRepository:
ok_body,
json_path,
json_value,
+ needle,
now().isoformat(),
),
)
await self._conn.commit()
- service = await self.get(cursor.lastrowid) # type: ignore[arg-type]
+ service = await self.get(cursor.lastrowid or 0)
assert service is not None
return service
+ async def set_rule( # noqa: PLR0913
+ self,
+ service_id: int,
+ *,
+ match_type: MatchType,
+ ok_status: int,
+ ok_body: str = "",
+ json_path: str | None = None,
+ json_value: str | None = None,
+ needle: str | None = None,
+ ) -> None:
+ await self._conn.execute(
+ "UPDATE services SET match_type = ?, ok_status = ?, ok_body = ?, "
+ "json_path = ?, json_value = ?, needle = ?, fail_streak = 0 WHERE id = ?",
+ (
+ match_type.value,
+ ok_status,
+ ok_body,
+ json_path,
+ json_value,
+ needle,
+ service_id,
+ ),
+ )
+ await self._conn.commit()
+
+ async def set_url(self, service_id: int, url: str) -> None:
+ await self._conn.execute(
+ "UPDATE services SET url = ?, fail_streak = 0, tls_expires_at = NULL, "
+ "tls_warned = 0 WHERE id = ?",
+ (url, service_id),
+ )
+ await self._conn.commit()
+
async def set_up(self, service_id: int, *, is_up: bool) -> None:
await self._conn.execute(
"UPDATE services SET is_up = ? WHERE id = ?", (int(is_up), service_id)
@@ -112,6 +152,13 @@ class ServiceRepository:
)
await self._conn.commit()
+ async def set_tls(self, service_id: int, expires_at: str, warned: int) -> None:
+ await self._conn.execute(
+ "UPDATE services SET tls_expires_at = ?, tls_warned = ? WHERE id = ?",
+ (expires_at, warned, service_id),
+ )
+ await self._conn.commit()
+
async def rename(self, service_id: int, name: str) -> None:
await self._conn.execute(
"UPDATE services SET name = ? WHERE id = ?", (name, service_id)
@@ -121,7 +168,6 @@ class ServiceRepository:
async def set_baseline(
self, service_id: int, *, ok_status: int, ok_body: str, json_value: str | None
) -> None:
- """Переснять эталон с текущего ответа."""
await self._conn.execute(
"UPDATE services SET ok_status = ?, ok_body = ?, json_value = ? "
"WHERE id = ?",
diff --git a/src/utils/db/repositories/setting.py b/src/utils/db/repositories/setting.py
new file mode 100644
index 0000000..a68fafa
--- /dev/null
+++ b/src/utils/db/repositories/setting.py
@@ -0,0 +1,38 @@
+import json
+from typing import Any
+
+import aiosqlite
+
+DEFAULTS: dict[str, Any] = {
+ "actions": True,
+ "commands": False,
+ "confirm": True,
+ "muted": ["ScheduleRun"],
+ "tz": "UTC",
+ "digest": None,
+ "digest_sent": "",
+}
+
+
+class SettingRepository:
+ def __init__(self, conn: aiosqlite.Connection) -> None:
+ self._conn = conn
+ self._cache: dict[str, Any] = {}
+
+ async def load(self) -> None:
+ cursor = await self._conn.execute("SELECT key, value FROM settings")
+ self._cache = {
+ row["key"]: json.loads(row["value"]) for row in await cursor.fetchall()
+ }
+
+ def get(self, key: str) -> Any: # noqa: ANN401
+ return self._cache.get(key, DEFAULTS[key])
+
+ async def set(self, key: str, value: Any) -> None: # noqa: ANN401
+ self._cache[key] = value
+ await self._conn.execute(
+ "INSERT INTO settings (key, value) VALUES (?, ?) "
+ "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+ (key, json.dumps(value, ensure_ascii=False)),
+ )
+ await self._conn.commit()
diff --git a/src/utils/db/repositories/stat.py b/src/utils/db/repositories/stat.py
new file mode 100644
index 0000000..a80fe29
--- /dev/null
+++ b/src/utils/db/repositories/stat.py
@@ -0,0 +1,50 @@
+import aiosqlite
+
+from utils.db.models import Hour
+
+SECONDS_IN_HOUR = 3600
+
+
+class StatRepository:
+ def __init__(self, conn: aiosqlite.Connection) -> None:
+ self._conn = conn
+
+ async def record(
+ self, service_id: int, hour: int, *, healthy: bool, latency_ms: int | None
+ ) -> None:
+ measured = latency_ms is not None
+ latency = latency_ms or 0
+ await self._conn.execute(
+ "INSERT INTO hourly "
+ "(service_id, hour, checks, fails, latency_sum, latency_n, latency_max) "
+ "VALUES (?, ?, 1, ?, ?, ?, ?) "
+ "ON CONFLICT(service_id, hour) DO UPDATE SET "
+ "checks = checks + 1, fails = fails + excluded.fails, "
+ "latency_sum = latency_sum + excluded.latency_sum, "
+ "latency_n = latency_n + excluded.latency_n, "
+ "latency_max = MAX(latency_max, excluded.latency_max)",
+ (service_id, hour, int(not healthy), latency, int(measured), latency),
+ )
+ await self._conn.commit()
+
+ async def since(self, service_id: int, hour: int) -> dict[int, Hour]:
+ cursor = await self._conn.execute(
+ "SELECT hour, checks, fails, latency_sum, latency_n, latency_max "
+ "FROM hourly WHERE service_id = ? AND hour >= ? ORDER BY hour",
+ (service_id, hour),
+ )
+ return {
+ row["hour"]: Hour(
+ hour=row["hour"],
+ checks=row["checks"],
+ fails=row["fails"],
+ latency_sum=row["latency_sum"],
+ latency_n=row["latency_n"],
+ latency_max=row["latency_max"],
+ )
+ for row in await cursor.fetchall()
+ }
+
+ async def purge(self, before_hour: int) -> None:
+ await self._conn.execute("DELETE FROM hourly WHERE hour < ?", (before_hour,))
+ await self._conn.commit()
diff --git a/src/utils/format.py b/src/utils/format.py
index facdf31..d134c16 100644
--- a/src/utils/format.py
+++ b/src/utils/format.py
@@ -1,4 +1,5 @@
-from datetime import UTC, datetime
+from datetime import UTC, datetime, tzinfo
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
def now() -> datetime:
@@ -9,7 +10,23 @@ def parse(ts: str) -> datetime:
return datetime.fromisoformat(ts)
+def from_ms(ms: float) -> datetime:
+ return datetime.fromtimestamp(ms / 1000, UTC)
+
+
+def zone(name: str) -> tzinfo:
+ try:
+ return ZoneInfo(name)
+ except (ZoneInfoNotFoundError, ValueError):
+ return UTC
+
+
+def stamp(moment: datetime, tz_name: str) -> str:
+ return moment.astimezone(zone(tz_name)).strftime("%d.%m %H:%M")
+
+
MINUTE = 60
+HOUR = 3600
MAX_PARTS = 2
@@ -27,6 +44,23 @@ def human_duration(seconds: float) -> str:
return " ".join(parts[:MAX_PARTS])
+def ago(moment: datetime) -> str:
+ return f"{human_duration(max(0.0, (now() - moment).total_seconds()))} назад"
+
+
+def until(moment: datetime) -> str:
+ return f"через {human_duration(max(0.0, (moment - now()).total_seconds()))}"
+
+
+def clock(seconds: float) -> str:
+ if seconds < MINUTE:
+ return f"{seconds:.1f}с" if seconds < 10 else f"{int(seconds)}с" # noqa: PLR2004
+ total = int(seconds)
+ if total < HOUR:
+ return f"{total // MINUTE}:{total % MINUTE:02d}"
+ return f"{total // HOUR}:{total % HOUR // MINUTE:02d}:{total % MINUTE:02d}"
+
+
def short(value: object, limit: int = 60) -> str:
text = str(value).replace("\n", " ").strip()
return text if len(text) <= limit else text[: limit - 1] + "…"
diff --git a/src/utils/logging.py b/src/utils/logging.py
index 79271b6..6d0b282 100644
--- a/src/utils/logging.py
+++ b/src/utils/logging.py
@@ -22,14 +22,14 @@ def setup_logging() -> None:
markup=True,
rich_tracebacks=True,
enable_link_path=False,
- tracebacks_show_locals=True,
+ tracebacks_show_locals=False,
omit_repeated_times=False,
show_time=env.log.show_time,
tracebacks_suppress=[router],
)
],
)
- install(console=console, show_locals=True)
+ install(console=console, show_locals=False)
logger = logging.getLogger("healthbot")
diff --git a/uv.lock b/uv.lock
index 46ba644..efe4114 100644
--- a/uv.lock
+++ b/uv.lock
@@ -248,7 +248,9 @@ dependencies = [
{ name = "aiohttp" },
{ name = "aiosqlite" },
{ name = "pydantic-settings" },
+ { name = "pyyaml" },
{ name = "rich" },
+ { name = "tzdata" },
]
[package.dev-dependencies]
@@ -263,7 +265,9 @@ requires-dist = [
{ name = "aiohttp", specifier = ">=3.10.0" },
{ name = "aiosqlite", specifier = ">=0.20.0" },
{ name = "pydantic-settings", specifier = ">=2.14.2" },
+ { name = "pyyaml", specifier = ">=6.0.3" },
{ name = "rich", specifier = ">=15.0.0" },
+ { name = "tzdata", specifier = ">=2026.5" },
]
[package.metadata.requires-dev]
@@ -572,6 +576,42 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" },
]
+[[package]]
+name = "pyyaml"
+version = "6.0.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" },
+ { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" },
+ { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" },
+ { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" },
+ { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" },
+ { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" },
+ { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" },
+ { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" },
+ { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" },
+ { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" },
+ { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" },
+ { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" },
+ { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" },
+ { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" },
+ { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" },
+ { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" },
+ { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" },
+ { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" },
+ { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" },
+ { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" },
+ { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" },
+ { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" },
+ { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" },
+ { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" },
+ { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
+]
+
[[package]]
name = "rich"
version = "15.0.0"
@@ -656,6 +696,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" },
]
+[[package]]
+name = "tzdata"
+version = "2026.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" },
+]
+
[[package]]
name = "yarl"
version = "1.24.2"