81 lines
2.9 KiB
Python
81 lines
2.9 KiB
Python
from datetime import timedelta
|
|
|
|
from mcps.komodo_alerts import HOLD, AlertMemory, describe, parse, still_bad
|
|
|
|
|
|
def alert(kind, *, resolved=False, level="CRITICAL", **data):
|
|
data.setdefault("name", "ms-agents-proxy-aeza")
|
|
return {"level": level, "resolved": resolved, "data": {"type": kind, "data": data}}
|
|
|
|
|
|
def test_state_change_is_one_shot_and_read_by_target_state():
|
|
up = parse(
|
|
alert(
|
|
"StackStateChange", resolved=True, **{"from": "running", "to": "unhealthy"}
|
|
)
|
|
)
|
|
down = parse(
|
|
alert(
|
|
"StackStateChange", resolved=True, **{"from": "unhealthy", "to": "running"}
|
|
)
|
|
)
|
|
assert up.bad and not down.bad
|
|
assert up.key == down.key
|
|
|
|
|
|
def test_flappy_alert_is_held_then_dropped_or_injected():
|
|
mem = AlertMemory()
|
|
first = parse(
|
|
alert("ServerUnreachable", err={"error": "Timed out waiting for Ping"})
|
|
)
|
|
assert mem.verdict(first).action == "hold"
|
|
# повтор той же тревоги, пока держим - тишина
|
|
assert mem.verdict(first).action == "drop"
|
|
# отбой во время выдержки - тоже тишина, и память чистая
|
|
assert (
|
|
mem.verdict(parse(alert("ServerUnreachable", resolved=True))).action == "drop"
|
|
)
|
|
assert not mem.holding and not mem.told
|
|
# выдержка истекла, всё ещё плохо - срочно, и потом отбой доезжает
|
|
mem.verdict(first)
|
|
late = mem.after_hold(first, still_bad=True)
|
|
assert (late.action, late.urgency) == ("inject", "urgent")
|
|
mem.mark_told(first)
|
|
assert (
|
|
mem.verdict(parse(alert("ServerUnreachable", resolved=True))).action
|
|
== "resolved"
|
|
)
|
|
assert not mem.told
|
|
|
|
|
|
def test_informational_never_reaches_master_and_failures_go_normal():
|
|
mem = AlertMemory()
|
|
assert mem.verdict(parse(alert("StackImageUpdateAvailable"))).action == "drop"
|
|
assert mem.verdict(parse(alert("ScheduleRun"))).action == "drop"
|
|
failed = mem.verdict(parse(alert("ProcedureFailed", name="Fleet Backup")))
|
|
assert (failed.action, failed.urgency) == ("inject", "normal")
|
|
|
|
|
|
def test_describe_mentions_hold():
|
|
a = parse(alert("ServerUnreachable", err={"error": "Timed out waiting for Ping"}))
|
|
text = describe(a, held=HOLD)
|
|
assert "ms-agents-proxy-aeza" in text and "5 мин" in text and "Timed out" in text
|
|
assert describe(
|
|
parse(alert("ServerDisk", name="dell", path="/", used_gb=90, total_gb=100))
|
|
)
|
|
|
|
|
|
async def test_still_bad_reads_current_state():
|
|
async def read(kind, params):
|
|
assert kind == "ListServers"
|
|
return [{"name": "ms-agents-proxy-aeza", "info": {"state": "Ok"}}]
|
|
|
|
a = parse(alert("ServerUnreachable"))
|
|
assert await still_bad(read, a) is False
|
|
|
|
async def boom(kind, params):
|
|
raise RuntimeError("komodo down")
|
|
|
|
assert await still_bad(boom, a) is True
|
|
assert HOLD == timedelta(minutes=5)
|