feat(hands,jobs): job-run watchdog reads job_runs and alarms on three failures in a row

tests/test_runs.py

@@ -0,0 +1,366 @@
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from typing import Any, cast
+
+import pytest
+
+from beaver_agent.hands.runs import (
+    STREAK,
+    Run,
+    Runs,
+    alarm_text,
+    clear_text,
+    empty_text,
+    first_line,
+    look,
+    mask,
+)
+
+NOW = datetime(2026, 9, 22, 15, 42, tzinfo=UTC)
+BOOM = (
+    "HubError: vibegram 530 bad_json: <!doctype html>\n"
+    "Traceback (most recent call last):\n"
+    '  File "/app/src/beaver_gateway/jobs/scheduler.py", line 170, in _dispatch\n'
+    "HubError: vibegram 530 bad_json"
+)
+
+
+def run(rid, job, status, *, ago, error=None):
+    return Run(
+        id=rid,
+        job=job,
+        status=status,
+        started_at=NOW - timedelta(minutes=ago),
+        error=error if error is not None or status != "failed" else BOOM,
+    )
+
+
+def series(job, statuses, *, step=10, start=1000):
+    """Прогоны джоба от старых к новым: statuses[-1] - последний прогон."""
+    ago = step * len(statuses)
+    out = []
+    for i, status in enumerate(statuses):
+        out.append(run(start + i, job, status, ago=ago - step * i))
+    return out
+
+
+def state(tmp_path, **kw):
+    return Runs(path=tmp_path / "прогоны.json", **kw)
+
+
+def test_three_in_a_row_is_an_alarm(tmp_path):
+    rows = series("вайбграм", ["done", "failed", "failed", "failed"])
+    report = look(state(tmp_path), rows, now=NOW, me="прогоны")
+    assert not report.empty and report.seen == 4
+    (tail,) = report.alarms
+    assert tail.job == "вайбграм" and tail.count == 3
+    assert tail.since == NOW - timedelta(minutes=30)
+    assert not tail.capped
+    text = alarm_text(report)
+    assert "джоб `вайбграм` лежит 30 минут, 3 прогона подряд" in text
+    assert "с 22.09 15:12 UTC" in text
+    # Первая строка ошибки, а не трейсбек
+    assert "HubError: vibegram 530 bad_json" in text
+    assert "Traceback" not in text and "scheduler.py" not in text
+
+
+def test_two_in_a_row_stays_quiet(tmp_path):
+    rows = series("вайбграм", ["done", "failed", "failed"])
+    report = look(state(tmp_path), rows, now=NOW, me="прогоны")
+    assert not report.alarms and not report.clears and not report.empty
+    # Одиночное падение - тем более молчание
+    once = look(state(tmp_path), series("логи", ["failed"]), now=NOW, me="прогоны")
+    assert not once.alarms
+
+
+def test_only_the_tail_counts(tmp_path):
+    # Падало много, но последний прогон прошёл - джоб не лежит
+    rows = series("логи", ["failed", "failed", "failed", "failed", "done"])
+    assert not look(state(tmp_path), rows, now=NOW, me="прогоны").alarms
+
+
+def test_alarm_not_repeated_within_a_day(tmp_path):
+    rows = series("вайбграм", ["done", "failed", "failed", "failed", "failed"])
+    told = state(tmp_path, alerted={"вайбграм": NOW - timedelta(hours=5)})
+    quiet = look(told, rows, now=NOW, me="прогоны")
+    assert not quiet.alarms
+    assert "уже говорили меньше суток назад" in quiet.note
+    older = state(tmp_path, alerted={"вайбграм": NOW - timedelta(hours=25)})
+    assert look(older, rows, now=NOW, me="прогоны").alarms
+
+
+def test_repair_clears_and_resets(tmp_path):
+    rows = series("вайбграм", ["failed", "failed", "failed", "done"])
+    told = state(tmp_path, alerted={"вайбграм": NOW - timedelta(hours=2)})
+    report = look(told, rows, now=NOW, me="прогоны")
+    assert not report.alarms
+    (clear,) = report.clears
+    assert clear.job == "вайбграм" and clear.count == 0 and clear.status == "done"
+    assert "отбой: джоб `вайбграм` снова проходит (done)" in clear_text(report)
+    # Сброс делает джоб; после него тот же вход снова может поднять тревогу
+    del told.alerted["вайбграм"]
+    again = look(told, series("вайбграм", ["failed"] * 3), now=NOW, me="прогоны")
+    assert again.alarms
+
+
+def test_repair_is_told_once_not_every_run(tmp_path):
+    rows = series("вайбграм", ["failed", "failed", "failed", "done"])
+    clean = state(tmp_path)
+    assert not look(clean, rows, now=NOW, me="прогоны").clears
+
+
+def test_flapping_job_is_a_clear_not_an_alarm(tmp_path):
+    # После тревоги джоб моргает: хвост короче порога - он больше не лежит
+    rows = series("вайбграм", ["done", "done", "failed", "failed"])
+    told = state(tmp_path, alerted={"вайбграм": NOW - timedelta(hours=2)})
+    report = look(told, rows, now=NOW, me="прогоны")
+    (clear,) = report.clears
+    assert clear.count == 2
+    assert f"порог {STREAK}" in clear_text(report)
+
+
+def test_streak_running_past_the_window_says_at_least(tmp_path):
+    rows = series("вайбграм", ["failed"] * 144, step=10)
+    report = look(state(tmp_path), rows, now=NOW, me="прогоны")
+    (tail,) = report.alarms
+    assert tail.capped and tail.count == 144
+    text = alarm_text(report)
+    assert "лежит 24 часа и дольше, 144 прогона подряд" in text
+    assert "это всё окно суток, серия началась раньше" in text
+
+
+def test_empty_window_is_its_own_alarm(tmp_path):
+    report = look(state(tmp_path), [], now=NOW, me="прогоны")
+    assert report.empty and not report.alarms and report.seen == 0
+    assert "не отработал ни один джоб" in empty_text(report)
+    assert "планировщике" in empty_text(report)
+    # И повторяется не чаще раза в сутки
+    told = state(tmp_path, empty_alerted=NOW - timedelta(hours=5))
+    assert not look(told, [], now=NOW, me="прогоны").empty
+    stale = state(tmp_path, empty_alerted=NOW - timedelta(hours=25))
+    assert look(stale, [], now=NOW, me="прогоны").empty
+
+
+def test_own_runs_do_not_count_as_life_or_as_illness(tmp_path):
+    mine = series("прогоны", ["failed", "failed", "failed", "done"])
+    report = look(state(tmp_path), mine, now=NOW, me="прогоны")
+    # Свои прогоны в окне есть, чужих нет: это всё равно пустое окно
+    assert report.empty and not report.alarms and report.mine == 4
+    assert "их 4" in empty_text(report)
+
+
+def test_forgotten_job_leaves_the_state_silently(tmp_path):
+    rows = series("логи", ["done", "done"])
+    told = state(
+        tmp_path,
+        alerted={
+            "исчезнувший": NOW - timedelta(days=8),
+            "недавний": NOW - timedelta(hours=3),
+        },
+    )
+    report = look(told, rows, now=NOW, me="прогоны")
+    assert not report.clears  # починки мы не видели, «отбой» врать не о чем
+    assert "исчезнувший" not in told.alerted
+    assert "недавний" in told.alerted
+
+
+def test_several_broken_jobs_come_in_one_alarm(tmp_path):
+    rows = [
+        *series("вайбграм", ["failed"] * 4, start=100),
+        *series("логи", ["failed"] * 3, start=200),
+        *series("досуг", ["done"], start=300),
+    ]
+    report = look(state(tmp_path), rows, now=NOW, me="прогоны")
+    assert len(report.alarms) == 2
+    assert report.alarms[0].job == "вайбграм"  # длиннее серия - выше в тексте
+    assert alarm_text(report).count("\n") == 1
+
+
+def test_first_line_masks_secrets_and_cuts_the_tail():
+    assert first_line(None) == "без текста ошибки"
+    assert first_line("") == "без текста ошибки"
+    # Токен в строке ошибки не уезжает человеку
+    assert "s3cr3tvalue" not in first_line("RuntimeError: MSOS_MCP=s3cr3tvalue died")
+    assert "MSOS_MCP=<…>" in first_line("RuntimeError: MSOS_MCP=s3cr3tvalue died")
+    assert "hunter2" not in first_line("ошибка: password=hunter2")
+    assert "abc123" not in first_line("GET https://hub/api?t=abc123 -> 500")
+    assert "https://hub/api?<…>" in first_line("GET https://hub/api?t=abc123 -> 500")
+    assert "Bearer abcdefghij" not in first_line("Authorization: Bearer abcdefghij")
+    # Длинная строка режется
+    long = first_line("E: " + "x" * 500)
+    assert len(long) == 160 and long.endswith("…")
+    # Имя без запаха секрета остаётся читаемым
+    assert mask("status=530 job=вайбграм") == "status=530 job=вайбграм"
+
+
+def test_state_survives_a_round_trip(tmp_path):
+    saved = state(
+        tmp_path,
+        alerted={"вайбграм": NOW, "логи": NOW - timedelta(hours=3)},
+        empty_alerted=NOW,
+    )
+    saved.save()
+    back = Runs.load(saved.path)
+    assert back.alerted["вайбграм"] == NOW
+    assert back.alerted["логи"] == NOW - timedelta(hours=3)
+    assert back.empty_alerted == NOW
+    assert Runs.load(tmp_path / "нет.json").alerted == {}
+    (tmp_path / "мусор.json").write_text("не json", encoding="utf-8")
+    assert Runs.load(tmp_path / "мусор.json").empty_alerted is None
+
+
+class FakeRun:
+    def __init__(self, database=None, *, name="прогоны"):
+        self.job = SimpleNamespace(name=name, cron="27 * * * *")
+        self.conversations = SimpleNamespace(db=database)
+        self.spawned = []
+
+    async def spawn_job(self, *, agent, text, title=None):
+        self.spawned.append((agent, title, text))
+        return None
+
+
+async def make_db(tmp_path, rows):
+    from beaver_gateway.storage.db import Database
+    from beaver_gateway.storage.models import JobRunRecord
+
+    database = Database(f"sqlite+aiosqlite:///{tmp_path / 'journal.db'}")
+    await database.create_all()
+    async with database.session() as session:
+        for row in rows:
+            session.add(row)
+        await session.commit()
+    return database
+
+
+def record(job, status, *, ago, error=None):
+    from beaver_gateway.storage.models import JobRunRecord
+
+    started = (datetime.now(UTC) - timedelta(minutes=ago)).replace(tzinfo=None)
+    return JobRunRecord(
+        job=job,
+        trigger="cron",
+        started_at=started,
+        finished_at=started,
+        status=status,
+        error=error if error is not None or status != "failed" else BOOM,
+    )
+
+
+async def test_job_reads_the_real_journal_and_spawns(tmp_path, monkeypatch):
+    from beaver_agent.jobs import runs as job
+
+    monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json")
+    database = await make_db(
+        tmp_path,
+        [
+            record("вайбграм", "failed", ago=30),
+            record("вайбграм", "failed", ago=20),
+            record("вайбграм", "failed", ago=10),
+            record("досуг", "done", ago=5),
+            record("прогоны", "done", ago=3),
+            # Старше суток - в окно не попадает
+            record("логи", "failed", ago=60 * 30),
+        ],
+    )
+    run_ = FakeRun(database)
+
+    await job.watch_runs(cast("Any", run_))
+
+    assert len(run_.spawned) == 1
+    agent, title, text = run_.spawned[0]
+    assert agent == "beaver-triage" and title == "прогоны"
+    assert "джоб `вайбграм` лежит" in text and "3 прогона подряд" in text
+    assert "логи" not in text  # за окном
+    assert "прогоны" not in text.split("\n")[1]  # сам себя не считает
+    # Тревога записана - следующий прогон в те же сутки молчит
+    assert Runs.load(tmp_path / "прогоны.json").alerted["вайбграм"] is not None
+    await job.watch_runs(cast("Any", run_))
+    assert len(run_.spawned) == 1
+    await database.dispose()
+
+
+async def test_job_says_otboy_when_the_job_recovers(tmp_path, monkeypatch):
+    from beaver_agent.jobs import runs as job
+
+    monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json")
+    Runs(
+        path=tmp_path / "прогоны.json",
+        alerted={"вайбграм": datetime.now(UTC) - timedelta(hours=2)},
+    ).save()
+    database = await make_db(
+        tmp_path,
+        [
+            record("вайбграм", "failed", ago=30),
+            record("вайбграм", "failed", ago=20),
+            record("вайбграм", "done", ago=10),
+        ],
+    )
+    run_ = FakeRun(database)
+
+    await job.watch_runs(cast("Any", run_))
+
+    assert len(run_.spawned) == 1
+    _, _, text = run_.spawned[0]
+    assert "отбой: джоб `вайбграм` снова проходит" in text
+    assert 'urgency="normal"' in text  # отбой не будит
+    assert Runs.load(tmp_path / "прогоны.json").alerted == {}
+    # Сброшено - второй «отбой» не уходит
+    await job.watch_runs(cast("Any", run_))
+    assert len(run_.spawned) == 1
+    await database.dispose()
+
+
+async def test_job_alarms_on_an_empty_journal(tmp_path, monkeypatch):
+    from beaver_agent.jobs import runs as job
+
+    monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json")
+    database = await make_db(tmp_path, [record("прогоны", "done", ago=5)])
+    run_ = FakeRun(database)
+
+    await job.watch_runs(cast("Any", run_))
+
+    assert len(run_.spawned) == 1
+    _, _, text = run_.spawned[0]
+    assert "нет ни одной записи" in text and "не отработал ни один джоб" in text
+    assert Runs.load(tmp_path / "прогоны.json").empty_alerted is not None
+    await database.dispose()
+
+
+async def test_unreadable_journal_is_not_a_reading(tmp_path, monkeypatch):
+    from beaver_agent.jobs import runs as job
+
+    class Dead:
+        def session(self):
+            msg = "база не отвечает"
+            raise RuntimeError(msg)
+
+    monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json")
+    run_ = FakeRun(Dead())
+
+    await job.watch_runs(cast("Any", run_))
+
+    assert not run_.spawned
+    assert not (tmp_path / "прогоны.json").exists()
+
+
+@pytest.mark.parametrize("name", ["прогоны", "иначе-названный"])
+async def test_the_watcher_never_reports_itself(tmp_path, monkeypatch, name):
+    from beaver_agent.jobs import runs as job
+
+    monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json")
+    database = await make_db(
+        tmp_path,
+        [
+            record(name, "failed", ago=30),
+            record(name, "failed", ago=20),
+            record(name, "failed", ago=10),
+            record("досуг", "done", ago=5),
+        ],
+    )
+    run_ = FakeRun(database, name=name)
+
+    await job.watch_runs(cast("Any", run_))
+
+    assert not run_.spawned
+    await database.dispose()