73 lines
2.4 KiB
Python
73 lines
2.4 KiB
Python
"""`POST /hooks/komodo`: алерты флота - флапы выдерживаются, остальное едет в мастер."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import TYPE_CHECKING
|
|
|
|
from beaver_agent.hands import KOMODO
|
|
from beaver_agent.hands.komodo_alerts import (
|
|
HOLD,
|
|
AlertMemory,
|
|
describe,
|
|
parse,
|
|
still_bad,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from beaver_gateway.jobs.scheduler import JobRun
|
|
|
|
_log = logging.getLogger(__name__)
|
|
|
|
ALERTS = AlertMemory()
|
|
HINT = (
|
|
"Это флот Бобра; ему такие алерты приходят и так (healthbot), тебя будят, "
|
|
"когда через 5 минут само не встало. Посмотри komodo(status/logs), почини, "
|
|
"если очевидно (restart стека), иначе - коротко скажи Бобру через say, что "
|
|
"видишь. Не деплой и не чини «заодно»."
|
|
)
|
|
|
|
|
|
async def komodo_alert(run: JobRun) -> None:
|
|
payload = run.payload
|
|
if payload.get("held"):
|
|
alert = parse(payload["alert"])
|
|
bad = await still_bad(KOMODO.read, alert) if KOMODO else True
|
|
verdict = ALERTS.after_hold(alert, still_bad=bad)
|
|
if verdict.action == "drop":
|
|
_log.info("komodo alert %s recovered during hold", alert.key)
|
|
return
|
|
ALERTS.mark_told(alert)
|
|
await run.inject_master(
|
|
f"Komodo: {describe(alert, held=HOLD)}\n{HINT}",
|
|
urgency=verdict.urgency,
|
|
origin="komodo",
|
|
)
|
|
return
|
|
alert = parse(payload)
|
|
verdict = ALERTS.verdict(alert)
|
|
match verdict.action:
|
|
case "drop":
|
|
_log.info("komodo alert %s dropped (%s)", alert.key, alert.kind)
|
|
case "hold":
|
|
await run.scheduler.trigger(
|
|
run.job,
|
|
{
|
|
"held": True,
|
|
"alert": {k: v for k, v in payload.items() if k != "trigger"},
|
|
},
|
|
delay=HOLD,
|
|
trigger="hold",
|
|
)
|
|
case "resolved":
|
|
await run.inject_master(
|
|
f"Komodo: отбой - {describe(alert)}", urgency="normal", origin="komodo"
|
|
)
|
|
case "inject":
|
|
ALERTS.mark_told(alert)
|
|
await run.inject_master(
|
|
f"Komodo: {describe(alert)}\n{HINT}",
|
|
urgency=verdict.urgency,
|
|
origin="komodo",
|
|
)
|