feat(hands,jobs): журнал прогонов дописывается, шторм новых видов глушится громко
This commit is contained in:
1 parent
f3e4d51e4e
commit
58338f132a
3 files changed
+391
-45
No files matched your search
@@ -1,8 +1,11 @@
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from beaver_agent.hands.komodo import Komodo
|
||||
from beaver_agent.hands.komodo_logs import (
|
||||
KEEP,
|
||||
MUTE_KINDS,
|
||||
MUTE_SHOW,
|
||||
SELF_MARK,
|
||||
SURGE_CEILING,
|
||||
SURGE_FACTOR,
|
||||
@@ -14,7 +17,9 @@ from beaver_agent.hands.komodo_logs import (
|
||||
daily,
|
||||
digest,
|
||||
grouped,
|
||||
note,
|
||||
read_stack,
|
||||
record,
|
||||
)
|
||||
from beaver_agent.jobs.logs import sweep as sweep_fleet
|
||||
|
||||
@@ -728,3 +733,81 @@ def test_the_ceiling_fires_without_any_relative_norm(tmp_path):
|
||||
assert item.gain < SURGE_FACTOR * item.norm
|
||||
assert item.gain >= SURGE_CEILING
|
||||
assert "Всплеск знакомого (1)" in brief(spike)
|
||||
|
||||
|
||||
def test_the_journal_is_appended_and_survives_the_passes(tmp_path):
|
||||
# Состояние - снимок, истории в нём нет: пороги пересматривать по журналу.
|
||||
state = seen(tmp_path)
|
||||
path = tmp_path / "логи-флота.jsonl"
|
||||
times = stamps(20, 8, 2)
|
||||
for i, hour in enumerate((10, 13, 16)):
|
||||
times = times + stamps(20, hour - 1, 2, start=20)
|
||||
report = digest(state, swept([burst("gitea", times)], at=at(20, hour)))
|
||||
told(state, report)
|
||||
note(path, record(report, UTC) | {"pass_of_day": i + 1})
|
||||
|
||||
rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
|
||||
assert len(rows) == 3
|
||||
assert [r["pass_of_day"] for r in rows] == [1, 2, 3]
|
||||
assert rows[0]["stacks"] == 1 and rows[0]["in_scope"] == 1
|
||||
assert rows[0]["new"] == 1 and rows[0]["misses"] == 0
|
||||
(trouble,) = rows[2]["troubles"]
|
||||
assert trouble["where"] == "gitea/svc" and trouble["gain"] == 2
|
||||
assert trouble["total"] == 8 and trouble["day"] == 8
|
||||
|
||||
|
||||
def test_the_journal_turns_over_instead_of_growing_forever(tmp_path):
|
||||
path = tmp_path / "логи-флота.jsonl"
|
||||
for i in range(4):
|
||||
note(path, {"n": i, "padding": "x" * 200}, max_bytes=300)
|
||||
fresh = [json.loads(ln) for ln in path.read_text(encoding="utf-8").splitlines()]
|
||||
turned = [
|
||||
json.loads(ln)
|
||||
for ln in path.with_suffix(".jsonl.1").read_text(encoding="utf-8").splitlines()
|
||||
]
|
||||
assert [r["n"] for r in turned] == [0, 1] # старое уехало в .1, не пропало
|
||||
assert [r["n"] for r in fresh] == [2, 3]
|
||||
|
||||
|
||||
def test_a_storm_of_new_kinds_is_folded_into_one_loud_line(tmp_path):
|
||||
state = seen(tmp_path)
|
||||
flood = [
|
||||
burst("gitea", stamps(20, 9, 1, start=i), text=f"ERROR сбой {chr(1072 + i)}щ")
|
||||
for i in range(MUTE_KINDS + 5)
|
||||
]
|
||||
report = digest(state, swept(flood, at=at(20, 10)))
|
||||
(storm,) = report.storms
|
||||
assert storm.kinds == MUTE_KINDS + 5 and storm.shown == MUTE_SHOW
|
||||
assert len(report.new) == MUTE_SHOW # остальное проглочено, но посчитано
|
||||
text = brief(report)
|
||||
assert "глушу" in text
|
||||
assert f"Стек gitea: {MUTE_KINDS + 5} новых ВИДОВ строк за проход" in text
|
||||
assert report.worth_telling
|
||||
|
||||
# Пока не сказали - повторяем; сказали - в суточной сводке.
|
||||
told(state, report)
|
||||
assert state.mutes["gitea"].told
|
||||
assert state.mutes["gitea"].kinds == 5 + MUTE_KINDS - MUTE_SHOW
|
||||
|
||||
|
||||
def test_the_muted_stack_is_named_once_a_day_until_it_calms_down(tmp_path):
|
||||
state = seen(tmp_path)
|
||||
flood = [
|
||||
burst("gitea", stamps(20, 9, 1, start=i), text=f"ERROR сбой {chr(1072 + i)}щ")
|
||||
for i in range(MUTE_KINDS + 5)
|
||||
]
|
||||
told(state, digest(state, swept(flood, at=at(20, 10))))
|
||||
night = digest(state, swept(flood, at=at(21, 1, 40)))
|
||||
assert "Приглушён gitea" in daily(night, UTC)
|
||||
assert "глушу с 20.09 10:00" in daily(night, UTC)
|
||||
|
||||
calm = digest(state, swept([burst("gitea", stamps(21, 9, 2))], at=at(21, 10)))
|
||||
assert calm.unmuted == ("gitea",) and "глушение снял: gitea" in brief(calm)
|
||||
assert "gitea" not in state.mutes
|
||||
|
||||
|
||||
def test_the_first_sighting_is_never_a_surge(tmp_path):
|
||||
# В хвосте лежит суточный запас: первый счёт - про длину хвоста, не про беду.
|
||||
state = seen(tmp_path, fresh=True)
|
||||
loud = digest(state, swept([burst("gitea", stamps(20, 9, 55))], at=at(20, 10)))
|
||||
assert loud.surge == () and loud.cold and loud.baseline == 1
|
||||
Reference in new issue
Block a user