from datetime import UTC, datetime, timedelta from types import SimpleNamespace from typing import Any, cast import pytest from beaver_agent.hands.runs import ( STREAK, Run, Runs, alarm_text, clear_text, empty_text, first_line, look, mask, ) NOW = datetime(2026, 9, 22, 15, 42, tzinfo=UTC) BOOM = ( "HubError: vibegram 530 bad_json: \n" "Traceback (most recent call last):\n" ' File "/app/src/beaver_gateway/jobs/scheduler.py", line 170, in _dispatch\n' "HubError: vibegram 530 bad_json" ) def run(rid, job, status, *, ago, error=None): return Run( id=rid, job=job, status=status, started_at=NOW - timedelta(minutes=ago), error=error if error is not None or status != "failed" else BOOM, ) def series(job, statuses, *, step=10, start=1000): """Прогоны джоба от старых к новым: statuses[-1] - последний прогон.""" ago = step * len(statuses) out = [] for i, status in enumerate(statuses): out.append(run(start + i, job, status, ago=ago - step * i)) return out def state(tmp_path, **kw): return Runs(path=tmp_path / "прогоны.json", **kw) def test_three_in_a_row_is_an_alarm(tmp_path): rows = series("вайбграм", ["done", "failed", "failed", "failed"]) report = look(state(tmp_path), rows, now=NOW, me="прогоны") assert not report.empty and report.seen == 4 (tail,) = report.alarms assert tail.job == "вайбграм" and tail.count == 3 assert tail.since == NOW - timedelta(minutes=30) assert not tail.capped text = alarm_text(report) assert "джоб `вайбграм` лежит 30 минут, 3 прогона подряд" in text assert "с 22.09 15:12 UTC" in text # Первая строка ошибки, а не трейсбек assert "HubError: vibegram 530 bad_json" in text assert "Traceback" not in text and "scheduler.py" not in text def test_two_in_a_row_stays_quiet(tmp_path): rows = series("вайбграм", ["done", "failed", "failed"]) report = look(state(tmp_path), rows, now=NOW, me="прогоны") assert not report.alarms and not report.clears and not report.empty # Одиночное падение - тем более молчание once = look(state(tmp_path), series("логи", ["failed"]), now=NOW, me="прогоны") assert not once.alarms def test_only_the_tail_counts(tmp_path): # Падало много, но последний прогон прошёл - джоб не лежит rows = series("логи", ["failed", "failed", "failed", "failed", "done"]) assert not look(state(tmp_path), rows, now=NOW, me="прогоны").alarms def test_alarm_not_repeated_within_a_day(tmp_path): rows = series("вайбграм", ["done", "failed", "failed", "failed", "failed"]) told = state(tmp_path, alerted={"вайбграм": NOW - timedelta(hours=5)}) quiet = look(told, rows, now=NOW, me="прогоны") assert not quiet.alarms assert "уже говорили меньше суток назад" in quiet.note older = state(tmp_path, alerted={"вайбграм": NOW - timedelta(hours=25)}) assert look(older, rows, now=NOW, me="прогоны").alarms def test_repair_clears_and_resets(tmp_path): rows = series("вайбграм", ["failed", "failed", "failed", "done"]) told = state(tmp_path, alerted={"вайбграм": NOW - timedelta(hours=2)}) report = look(told, rows, now=NOW, me="прогоны") assert not report.alarms (clear,) = report.clears assert clear.job == "вайбграм" and clear.count == 0 and clear.status == "done" assert "отбой: джоб `вайбграм` снова проходит (done)" in clear_text(report) # Сброс делает джоб; после него тот же вход снова может поднять тревогу del told.alerted["вайбграм"] again = look(told, series("вайбграм", ["failed"] * 3), now=NOW, me="прогоны") assert again.alarms def test_repair_is_told_once_not_every_run(tmp_path): rows = series("вайбграм", ["failed", "failed", "failed", "done"]) clean = state(tmp_path) assert not look(clean, rows, now=NOW, me="прогоны").clears def test_flapping_job_is_a_clear_not_an_alarm(tmp_path): # После тревоги джоб моргает: хвост короче порога - он больше не лежит rows = series("вайбграм", ["done", "done", "failed", "failed"]) told = state(tmp_path, alerted={"вайбграм": NOW - timedelta(hours=2)}) report = look(told, rows, now=NOW, me="прогоны") (clear,) = report.clears assert clear.count == 2 assert f"порог {STREAK}" in clear_text(report) def test_streak_running_past_the_window_says_at_least(tmp_path): rows = series("вайбграм", ["failed"] * 144, step=10) report = look(state(tmp_path), rows, now=NOW, me="прогоны") (tail,) = report.alarms assert tail.capped and tail.count == 144 text = alarm_text(report) assert "лежит 24 часа и дольше, 144 прогона подряд" in text assert "это всё окно суток, серия началась раньше" in text def test_empty_window_is_its_own_alarm(tmp_path): report = look(state(tmp_path), [], now=NOW, me="прогоны") assert report.empty and not report.alarms and report.seen == 0 assert "не отработал ни один джоб" in empty_text(report) assert "планировщике" in empty_text(report) # И повторяется не чаще раза в сутки told = state(tmp_path, empty_alerted=NOW - timedelta(hours=5)) assert not look(told, [], now=NOW, me="прогоны").empty stale = state(tmp_path, empty_alerted=NOW - timedelta(hours=25)) assert look(stale, [], now=NOW, me="прогоны").empty def test_own_runs_do_not_count_as_life_or_as_illness(tmp_path): mine = series("прогоны", ["failed", "failed", "failed", "done"]) report = look(state(tmp_path), mine, now=NOW, me="прогоны") # Свои прогоны в окне есть, чужих нет: это всё равно пустое окно assert report.empty and not report.alarms and report.mine == 4 assert "их 4" in empty_text(report) def test_forgotten_job_leaves_the_state_silently(tmp_path): rows = series("логи", ["done", "done"]) told = state( tmp_path, alerted={ "исчезнувший": NOW - timedelta(days=8), "недавний": NOW - timedelta(hours=3), }, ) report = look(told, rows, now=NOW, me="прогоны") assert not report.clears # починки мы не видели, «отбой» врать не о чем assert "исчезнувший" not in told.alerted assert "недавний" in told.alerted def test_several_broken_jobs_come_in_one_alarm(tmp_path): rows = [ *series("вайбграм", ["failed"] * 4, start=100), *series("логи", ["failed"] * 3, start=200), *series("досуг", ["done"], start=300), ] report = look(state(tmp_path), rows, now=NOW, me="прогоны") assert len(report.alarms) == 2 assert report.alarms[0].job == "вайбграм" # длиннее серия - выше в тексте assert alarm_text(report).count("\n") == 1 def test_first_line_masks_secrets_and_cuts_the_tail(): assert first_line(None) == "без текста ошибки" assert first_line("") == "без текста ошибки" # Токен в строке ошибки не уезжает человеку assert "s3cr3tvalue" not in first_line("RuntimeError: MSOS_MCP=s3cr3tvalue died") assert "MSOS_MCP=<…>" in first_line("RuntimeError: MSOS_MCP=s3cr3tvalue died") assert "hunter2" not in first_line("ошибка: password=hunter2") assert "abc123" not in first_line("GET https://hub/api?t=abc123 -> 500") assert "https://hub/api?<…>" in first_line("GET https://hub/api?t=abc123 -> 500") assert "Bearer abcdefghij" not in first_line("Authorization: Bearer abcdefghij") # Длинная строка режется long = first_line("E: " + "x" * 500) assert len(long) == 160 and long.endswith("…") # Имя без запаха секрета остаётся читаемым assert mask("status=530 job=вайбграм") == "status=530 job=вайбграм" def test_state_survives_a_round_trip(tmp_path): saved = state( tmp_path, alerted={"вайбграм": NOW, "логи": NOW - timedelta(hours=3)}, empty_alerted=NOW, ) saved.save() back = Runs.load(saved.path) assert back.alerted["вайбграм"] == NOW assert back.alerted["логи"] == NOW - timedelta(hours=3) assert back.empty_alerted == NOW assert Runs.load(tmp_path / "нет.json").alerted == {} (tmp_path / "мусор.json").write_text("не json", encoding="utf-8") assert Runs.load(tmp_path / "мусор.json").empty_alerted is None class FakeRun: def __init__(self, database=None, *, name="прогоны"): self.job = SimpleNamespace(name=name, cron="27 * * * *") self.conversations = SimpleNamespace(db=database) self.spawned = [] async def spawn_job(self, *, agent, text, title=None): self.spawned.append((agent, title, text)) return None async def make_db(tmp_path, rows): from beaver_gateway.storage.db import Database from beaver_gateway.storage.models import JobRunRecord database = Database(f"sqlite+aiosqlite:///{tmp_path / 'journal.db'}") await database.create_all() async with database.session() as session: for row in rows: session.add(row) await session.commit() return database def record(job, status, *, ago, error=None): from beaver_gateway.storage.models import JobRunRecord started = (datetime.now(UTC) - timedelta(minutes=ago)).replace(tzinfo=None) return JobRunRecord( job=job, trigger="cron", started_at=started, finished_at=started, status=status, error=error if error is not None or status != "failed" else BOOM, ) async def test_job_reads_the_real_journal_and_spawns(tmp_path, monkeypatch): from beaver_agent.jobs import runs as job monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json") database = await make_db( tmp_path, [ record("вайбграм", "failed", ago=30), record("вайбграм", "failed", ago=20), record("вайбграм", "failed", ago=10), record("досуг", "done", ago=5), record("прогоны", "done", ago=3), # Старше суток - в окно не попадает record("логи", "failed", ago=60 * 30), ], ) run_ = FakeRun(database) await job.watch_runs(cast("Any", run_)) assert len(run_.spawned) == 1 agent, title, text = run_.spawned[0] assert agent == "beaver-triage" and title == "прогоны" assert "джоб `вайбграм` лежит" in text and "3 прогона подряд" in text assert "логи" not in text # за окном assert "прогоны" not in text.split("\n")[1] # сам себя не считает # Тревога записана - следующий прогон в те же сутки молчит assert Runs.load(tmp_path / "прогоны.json").alerted["вайбграм"] is not None await job.watch_runs(cast("Any", run_)) assert len(run_.spawned) == 1 await database.dispose() async def test_job_says_otboy_when_the_job_recovers(tmp_path, monkeypatch): from beaver_agent.jobs import runs as job monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json") Runs( path=tmp_path / "прогоны.json", alerted={"вайбграм": datetime.now(UTC) - timedelta(hours=2)}, ).save() database = await make_db( tmp_path, [ record("вайбграм", "failed", ago=30), record("вайбграм", "failed", ago=20), record("вайбграм", "done", ago=10), ], ) run_ = FakeRun(database) await job.watch_runs(cast("Any", run_)) assert len(run_.spawned) == 1 _, _, text = run_.spawned[0] assert "отбой: джоб `вайбграм` снова проходит" in text assert 'urgency="normal"' in text # отбой не будит assert Runs.load(tmp_path / "прогоны.json").alerted == {} # Сброшено - второй «отбой» не уходит await job.watch_runs(cast("Any", run_)) assert len(run_.spawned) == 1 await database.dispose() async def test_job_alarms_on_an_empty_journal(tmp_path, monkeypatch): from beaver_agent.jobs import runs as job monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json") database = await make_db(tmp_path, [record("прогоны", "done", ago=5)]) run_ = FakeRun(database) await job.watch_runs(cast("Any", run_)) assert len(run_.spawned) == 1 _, _, text = run_.spawned[0] assert "нет ни одной записи" in text and "не отработал ни один джоб" in text assert Runs.load(tmp_path / "прогоны.json").empty_alerted is not None await database.dispose() async def test_unreadable_journal_is_not_a_reading(tmp_path, monkeypatch): from beaver_agent.jobs import runs as job class Dead: def session(self): msg = "база не отвечает" raise RuntimeError(msg) monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json") run_ = FakeRun(Dead()) await job.watch_runs(cast("Any", run_)) assert not run_.spawned assert not (tmp_path / "прогоны.json").exists() @pytest.mark.parametrize("name", ["прогоны", "иначе-названный"]) async def test_the_watcher_never_reports_itself(tmp_path, monkeypatch, name): from beaver_agent.jobs import runs as job monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json") database = await make_db( tmp_path, [ record(name, "failed", ago=30), record(name, "failed", ago=20), record(name, "failed", ago=10), record("досуг", "done", ago=5), ], ) run_ = FakeRun(database, name=name) await job.watch_runs(cast("Any", run_)) assert not run_.spawned await database.dispose()