593 lines
24 KiB
Python
593 lines
24 KiB
Python
from datetime import UTC, datetime, timedelta
|
||
from types import SimpleNamespace
|
||
from typing import Any, cast
|
||
|
||
import pytest
|
||
|
||
from beaver_agent.hands.runs import (
|
||
STREAK,
|
||
Run,
|
||
Runs,
|
||
alarm_text,
|
||
awake,
|
||
clear_text,
|
||
empty_text,
|
||
first_line,
|
||
look,
|
||
mask,
|
||
quiet_text,
|
||
)
|
||
|
||
NOW = datetime(2026, 9, 22, 15, 42, tzinfo=UTC)
|
||
BOOM = (
|
||
"HubError: vibegram 530 bad_json: <!doctype html>\n"
|
||
"Traceback (most recent call last):\n"
|
||
' File "/app/src/beaver_gateway/jobs/scheduler.py", line 170, in _dispatch\n'
|
||
"HubError: vibegram 530 bad_json"
|
||
)
|
||
|
||
|
||
def run(rid, job, status, *, ago, error=None):
|
||
return Run(
|
||
id=rid,
|
||
job=job,
|
||
status=status,
|
||
started_at=NOW - timedelta(minutes=ago),
|
||
error=error if error is not None or status != "failed" else BOOM,
|
||
)
|
||
|
||
|
||
def series(job, statuses, *, step=10, start=1000):
|
||
"""Прогоны джоба от старых к новым: statuses[-1] - последний прогон."""
|
||
ago = step * len(statuses)
|
||
out = []
|
||
for i, status in enumerate(statuses):
|
||
out.append(run(start + i, job, status, ago=ago - step * i))
|
||
return out
|
||
|
||
|
||
def state(tmp_path, **kw):
|
||
return Runs(path=tmp_path / "прогоны.json", **kw)
|
||
|
||
|
||
def test_three_in_a_row_is_an_alarm(tmp_path):
|
||
rows = series("вайбграм", ["done", "failed", "failed", "failed"])
|
||
report = look(state(tmp_path), rows, now=NOW, me="прогоны")
|
||
assert not report.empty and report.seen == 4
|
||
(tail,) = report.alarms
|
||
assert tail.job == "вайбграм" and tail.count == 3
|
||
assert tail.since == NOW - timedelta(minutes=30)
|
||
assert not tail.capped
|
||
text = alarm_text(report)
|
||
assert "джоб `вайбграм` лежит 30 минут, 3 прогона подряд" in text
|
||
assert "с 22.09 15:12 UTC" in text
|
||
# Первая строка ошибки, а не трейсбек
|
||
assert "HubError: vibegram 530 bad_json" in text
|
||
assert "Traceback" not in text and "scheduler.py" not in text
|
||
|
||
|
||
def test_two_in_a_row_stays_quiet(tmp_path):
|
||
rows = series("вайбграм", ["done", "failed", "failed"])
|
||
report = look(state(tmp_path), rows, now=NOW, me="прогоны")
|
||
assert not report.alarms and not report.clears and not report.empty
|
||
# Одиночное падение - тем более молчание
|
||
once = look(state(tmp_path), series("логи", ["failed"]), now=NOW, me="прогоны")
|
||
assert not once.alarms
|
||
|
||
|
||
def test_only_the_tail_counts(tmp_path):
|
||
# Падало много, но последний прогон прошёл - джоб не лежит
|
||
rows = series("логи", ["failed", "failed", "failed", "failed", "done"])
|
||
assert not look(state(tmp_path), rows, now=NOW, me="прогоны").alarms
|
||
|
||
|
||
def test_alarm_not_repeated_within_a_day(tmp_path):
|
||
rows = series("вайбграм", ["done", "failed", "failed", "failed", "failed"])
|
||
told = state(tmp_path, alerted={"вайбграм": NOW - timedelta(hours=5)})
|
||
quiet = look(told, rows, now=NOW, me="прогоны")
|
||
assert not quiet.alarms
|
||
assert "уже говорили меньше суток назад" in quiet.note
|
||
older = state(tmp_path, alerted={"вайбграм": NOW - timedelta(hours=25)})
|
||
assert look(older, rows, now=NOW, me="прогоны").alarms
|
||
|
||
|
||
def test_repair_clears_and_resets(tmp_path):
|
||
rows = series("вайбграм", ["failed", "failed", "failed", "done"])
|
||
told = state(tmp_path, alerted={"вайбграм": NOW - timedelta(hours=2)})
|
||
report = look(told, rows, now=NOW, me="прогоны")
|
||
assert not report.alarms
|
||
(clear,) = report.clears
|
||
assert clear.job == "вайбграм" and clear.count == 0 and clear.status == "done"
|
||
assert "отбой: джоб `вайбграм` снова проходит (done)" in clear_text(report)
|
||
# Сброс делает джоб; после него тот же вход снова может поднять тревогу
|
||
del told.alerted["вайбграм"]
|
||
again = look(told, series("вайбграм", ["failed"] * 3), now=NOW, me="прогоны")
|
||
assert again.alarms
|
||
|
||
|
||
def test_repair_is_told_once_not_every_run(tmp_path):
|
||
rows = series("вайбграм", ["failed", "failed", "failed", "done"])
|
||
clean = state(tmp_path)
|
||
assert not look(clean, rows, now=NOW, me="прогоны").clears
|
||
|
||
|
||
def test_flapping_job_is_a_clear_not_an_alarm(tmp_path):
|
||
# После тревоги джоб моргает: хвост короче порога - он больше не лежит
|
||
rows = series("вайбграм", ["done", "done", "failed", "failed"])
|
||
told = state(tmp_path, alerted={"вайбграм": NOW - timedelta(hours=2)})
|
||
report = look(told, rows, now=NOW, me="прогоны")
|
||
(clear,) = report.clears
|
||
assert clear.count == 2
|
||
assert f"порог {STREAK}" in clear_text(report)
|
||
|
||
|
||
def test_streak_running_past_the_window_says_at_least(tmp_path):
|
||
rows = series("вайбграм", ["failed"] * 144, step=10)
|
||
report = look(state(tmp_path), rows, now=NOW, me="прогоны")
|
||
(tail,) = report.alarms
|
||
assert tail.capped and tail.count == 144
|
||
text = alarm_text(report)
|
||
assert "лежит 24 часа и дольше, 144 прогона подряд" in text
|
||
assert "это всё окно недели, серия началась раньше" in text
|
||
|
||
|
||
def test_empty_window_is_its_own_alarm(tmp_path):
|
||
report = look(state(tmp_path), [], now=NOW, me="прогоны")
|
||
assert report.empty and not report.alarms and report.seen == 0
|
||
assert "не отработал ни один джоб" in empty_text(report)
|
||
assert "планировщике" in empty_text(report)
|
||
# И повторяется не чаще раза в сутки
|
||
told = state(tmp_path, empty_alerted=NOW - timedelta(hours=5))
|
||
assert not look(told, [], now=NOW, me="прогоны").empty
|
||
stale = state(tmp_path, empty_alerted=NOW - timedelta(hours=25))
|
||
assert look(stale, [], now=NOW, me="прогоны").empty
|
||
|
||
|
||
def test_own_runs_do_not_count_as_life_or_as_illness(tmp_path):
|
||
mine = series("прогоны", ["failed", "failed", "failed", "done"])
|
||
report = look(state(tmp_path), mine, now=NOW, me="прогоны")
|
||
# Свои прогоны в окне есть, чужих нет: это всё равно пустое окно
|
||
assert report.empty and not report.alarms and report.mine == 4
|
||
assert "их 4" in empty_text(report)
|
||
|
||
|
||
def test_forgotten_job_leaves_the_state_silently(tmp_path):
|
||
rows = series("логи", ["done", "done"])
|
||
told = state(
|
||
tmp_path,
|
||
alerted={
|
||
"исчезнувший": NOW - timedelta(days=8),
|
||
"недавний": NOW - timedelta(hours=3),
|
||
},
|
||
)
|
||
report = look(told, rows, now=NOW, me="прогоны")
|
||
assert not report.clears # починки мы не видели, «отбой» врать не о чем
|
||
assert "исчезнувший" not in told.alerted
|
||
assert "недавний" in told.alerted
|
||
|
||
|
||
def test_several_broken_jobs_come_in_one_alarm(tmp_path):
|
||
rows = [
|
||
*series("вайбграм", ["failed"] * 4, start=100),
|
||
*series("логи", ["failed"] * 3, start=200),
|
||
*series("досуг", ["done"], start=300),
|
||
]
|
||
report = look(state(tmp_path), rows, now=NOW, me="прогоны")
|
||
assert len(report.alarms) == 2
|
||
assert report.alarms[0].job == "вайбграм" # длиннее серия - выше в тексте
|
||
assert alarm_text(report).count("\n") == 1
|
||
|
||
|
||
def test_first_line_masks_secrets_and_cuts_the_tail():
|
||
assert first_line(None) == "без текста ошибки"
|
||
assert first_line("") == "без текста ошибки"
|
||
# Токен в строке ошибки не уезжает человеку
|
||
assert "s3cr3tvalue" not in first_line("RuntimeError: MSOS_MCP=s3cr3tvalue died")
|
||
assert "MSOS_MCP=<…>" in first_line("RuntimeError: MSOS_MCP=s3cr3tvalue died")
|
||
assert "hunter2" not in first_line("ошибка: password=hunter2")
|
||
assert "abc123" not in first_line("GET https://hub/api?t=abc123 -> 500")
|
||
assert "https://hub/api?<…>" in first_line("GET https://hub/api?t=abc123 -> 500")
|
||
assert "Bearer abcdefghij" not in first_line("Authorization: Bearer abcdefghij")
|
||
# Длинная строка режется
|
||
long = first_line("E: " + "x" * 500)
|
||
assert len(long) == 160 and long.endswith("…")
|
||
# Имя без запаха секрета остаётся читаемым
|
||
assert mask("status=530 job=вайбграм") == "status=530 job=вайбграм"
|
||
|
||
|
||
def test_state_survives_a_round_trip(tmp_path):
|
||
saved = state(
|
||
tmp_path,
|
||
alerted={"вайбграм": NOW, "логи": NOW - timedelta(hours=3)},
|
||
empty_alerted=NOW,
|
||
)
|
||
saved.save()
|
||
back = Runs.load(saved.path)
|
||
assert back.alerted["вайбграм"] == NOW
|
||
assert back.alerted["логи"] == NOW - timedelta(hours=3)
|
||
assert back.empty_alerted == NOW
|
||
assert Runs.load(tmp_path / "нет.json").alerted == {}
|
||
(tmp_path / "мусор.json").write_text("не json", encoding="utf-8")
|
||
assert Runs.load(tmp_path / "мусор.json").empty_alerted is None
|
||
|
||
|
||
class FakeRun:
|
||
def __init__(self, database=None, *, name="прогоны", jobs=()):
|
||
self.job = SimpleNamespace(name=name, cron="27 * * * *")
|
||
self.conversations = SimpleNamespace(db=database)
|
||
self.scheduler = SimpleNamespace(
|
||
jobs=[self.job, *(SimpleNamespace(name=n, cron=c) for n, c in jobs)], tz=UTC
|
||
)
|
||
self.spawned = []
|
||
|
||
async def spawn_job(self, *, agent, text, title=None):
|
||
self.spawned.append((agent, title, text))
|
||
return None
|
||
|
||
|
||
async def make_db(tmp_path, rows):
|
||
from beaver_gateway.storage.db import Database
|
||
from beaver_gateway.storage.models import JobRunRecord
|
||
|
||
database = Database(f"sqlite+aiosqlite:///{tmp_path / 'journal.db'}")
|
||
await database.create_all()
|
||
async with database.session() as session:
|
||
for row in rows:
|
||
session.add(row)
|
||
await session.commit()
|
||
return database
|
||
|
||
|
||
def record(job, status, *, ago, error=None):
|
||
from beaver_gateway.storage.models import JobRunRecord
|
||
|
||
started = (datetime.now(UTC) - timedelta(minutes=ago)).replace(tzinfo=None)
|
||
return JobRunRecord(
|
||
job=job,
|
||
trigger="cron",
|
||
started_at=started,
|
||
finished_at=started,
|
||
status=status,
|
||
error=error if error is not None or status != "failed" else BOOM,
|
||
)
|
||
|
||
|
||
async def test_job_reads_the_real_journal_and_spawns(tmp_path, monkeypatch):
|
||
from beaver_agent.jobs import runs as job
|
||
|
||
monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json")
|
||
database = await make_db(
|
||
tmp_path,
|
||
[
|
||
record("вайбграм", "failed", ago=30),
|
||
record("вайбграм", "failed", ago=20),
|
||
record("вайбграм", "failed", ago=10),
|
||
record("досуг", "done", ago=5),
|
||
record("прогоны", "done", ago=3),
|
||
# Одно падение - не серия, сколько бы окно ни было
|
||
record("логи", "failed", ago=60 * 30),
|
||
],
|
||
)
|
||
run_ = FakeRun(database)
|
||
|
||
await job.watch_runs(cast("Any", run_))
|
||
|
||
assert len(run_.spawned) == 1
|
||
agent, title, text = run_.spawned[0]
|
||
assert agent == "beaver-triage" and title == "прогоны"
|
||
assert "джоб `вайбграм` лежит" in text and "3 прогона подряд" in text
|
||
assert "логи" not in text # одно падение
|
||
assert "прогоны" not in text.split("\n")[1] # сам себя не считает
|
||
# Тревога записана - следующий прогон в те же сутки молчит
|
||
assert Runs.load(tmp_path / "прогоны.json").alerted["вайбграм"] is not None
|
||
await job.watch_runs(cast("Any", run_))
|
||
assert len(run_.spawned) == 1
|
||
await database.dispose()
|
||
|
||
|
||
async def test_job_says_otboy_when_the_job_recovers(tmp_path, monkeypatch):
|
||
from beaver_agent.jobs import runs as job
|
||
|
||
monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json")
|
||
Runs(
|
||
path=tmp_path / "прогоны.json",
|
||
alerted={"вайбграм": datetime.now(UTC) - timedelta(hours=2)},
|
||
).save()
|
||
database = await make_db(
|
||
tmp_path,
|
||
[
|
||
record("вайбграм", "failed", ago=30),
|
||
record("вайбграм", "failed", ago=20),
|
||
record("вайбграм", "done", ago=10),
|
||
],
|
||
)
|
||
run_ = FakeRun(database)
|
||
|
||
await job.watch_runs(cast("Any", run_))
|
||
|
||
assert len(run_.spawned) == 1
|
||
_, _, text = run_.spawned[0]
|
||
assert "отбой: джоб `вайбграм` снова проходит" in text
|
||
assert 'urgency="normal"' in text # отбой не будит
|
||
assert Runs.load(tmp_path / "прогоны.json").alerted == {}
|
||
# Сброшено - второй «отбой» не уходит
|
||
await job.watch_runs(cast("Any", run_))
|
||
assert len(run_.spawned) == 1
|
||
await database.dispose()
|
||
|
||
|
||
async def test_job_alarms_on_an_empty_journal(tmp_path, monkeypatch):
|
||
from beaver_agent.jobs import runs as job
|
||
|
||
monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json")
|
||
database = await make_db(tmp_path, [record("прогоны", "done", ago=5)])
|
||
run_ = FakeRun(database)
|
||
|
||
await job.watch_runs(cast("Any", run_))
|
||
|
||
assert len(run_.spawned) == 1
|
||
_, _, text = run_.spawned[0]
|
||
assert "нет ни одной записи" in text and "не отработал ни один джоб" in text
|
||
assert Runs.load(tmp_path / "прогоны.json").empty_alerted is not None
|
||
await database.dispose()
|
||
|
||
|
||
async def test_unreadable_journal_is_not_a_reading(tmp_path, monkeypatch):
|
||
from beaver_agent.jobs import runs as job
|
||
|
||
class Dead:
|
||
def session(self):
|
||
msg = "база не отвечает"
|
||
raise RuntimeError(msg)
|
||
|
||
monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json")
|
||
run_ = FakeRun(Dead())
|
||
|
||
await job.watch_runs(cast("Any", run_))
|
||
|
||
assert not run_.spawned
|
||
assert not (tmp_path / "прогоны.json").exists()
|
||
|
||
|
||
@pytest.mark.parametrize("name", ["прогоны", "иначе-названный"])
|
||
async def test_the_watcher_never_reports_itself(tmp_path, monkeypatch, name):
|
||
from beaver_agent.jobs import runs as job
|
||
|
||
monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json")
|
||
database = await make_db(
|
||
tmp_path,
|
||
[
|
||
record(name, "failed", ago=30),
|
||
record(name, "failed", ago=20),
|
||
record(name, "failed", ago=10),
|
||
record("досуг", "done", ago=5),
|
||
],
|
||
)
|
||
run_ = FakeRun(database, name=name)
|
||
|
||
await job.watch_runs(cast("Any", run_))
|
||
|
||
assert not run_.spawned
|
||
await database.dispose()
|
||
|
||
|
||
# --- Окно неделя: суточные джобы тоже набирают серию ---------------------------
|
||
|
||
|
||
def test_daily_job_failing_three_days_in_a_row_is_an_alarm(tmp_path):
|
||
day = 24 * 60
|
||
rows = [
|
||
run(1, "закрытие", "done", ago=4 * day - 60),
|
||
run(2, "закрытие", "failed", ago=3 * day - 60),
|
||
run(3, "закрытие", "failed", ago=2 * day - 60),
|
||
run(4, "закрытие", "failed", ago=day - 60),
|
||
]
|
||
report = look(state(tmp_path), rows, now=NOW, me="прогоны")
|
||
(tail,) = report.alarms
|
||
assert tail.job == "закрытие" and tail.count == 3 and not tail.capped
|
||
assert "джоб `закрытие` лежит 2 суток, 3 прогона подряд" in alarm_text(report)
|
||
|
||
|
||
async def test_job_reads_a_week_back(tmp_path, monkeypatch):
|
||
from beaver_agent.jobs import runs as job
|
||
|
||
monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json")
|
||
day = 24 * 60
|
||
database = await make_db(
|
||
tmp_path,
|
||
[
|
||
record("голосовой", "failed", ago=3 * day - 30),
|
||
record("голосовой", "failed", ago=2 * day - 30),
|
||
record("голосовой", "failed", ago=day - 30),
|
||
# Старше недели - в окно не попадает
|
||
record("память", "failed", ago=9 * day),
|
||
],
|
||
)
|
||
run_ = FakeRun(database)
|
||
|
||
await job.watch_runs(cast("Any", run_))
|
||
|
||
(spawned,) = run_.spawned
|
||
assert "джоб `голосовой` лежит" in spawned[2]
|
||
assert "память" not in spawned[2]
|
||
await database.dispose()
|
||
|
||
|
||
# --- Давность: джоб перестал запускаться ---------------------------------------
|
||
|
||
CRONS = {"прогоны": "27 * * * *", "закрытие": "20 4 * * *", "вайбграм": "*/10 * * * *"}
|
||
|
||
|
||
def hourly_me(hours=72):
|
||
"""Непрерывная цепочка своих прогонов: планировщик жив всё это время."""
|
||
return [
|
||
run(5000 + i, "прогоны", "done", ago=60 * (hours - i) - 15)
|
||
for i in range(hours)
|
||
]
|
||
|
||
|
||
def alive(ago=5):
|
||
return run(9000, "вайбграм", "done", ago=ago)
|
||
|
||
|
||
def silent(tmp_path, *, closing_ago_h, mine=None, boot=None, **kw):
|
||
last = run(1, "закрытие", "done", ago=closing_ago_h * 60)
|
||
return look(
|
||
kw.pop("runs", None) or state(tmp_path),
|
||
[alive(), *(hourly_me() if mine is None else mine)],
|
||
now=NOW,
|
||
me="прогоны",
|
||
crons=kw.pop("crons", CRONS),
|
||
latest=kw.pop("latest", {"закрытие": last, "вайбграм": alive()}),
|
||
tz=UTC,
|
||
boot=boot,
|
||
)
|
||
|
||
|
||
def test_job_silent_longer_than_two_periods_is_its_own_alarm(tmp_path):
|
||
report = silent(tmp_path, closing_ago_h=50)
|
||
(quiet,) = report.quiet
|
||
assert quiet.job == "закрытие" and quiet.period == timedelta(hours=24)
|
||
assert not report.alarms # это не «лежит»
|
||
text = quiet_text(report)
|
||
assert "джоб `закрытие` не запускался 2 суток при периоде 24 часа" in text
|
||
assert "лежит" not in text
|
||
|
||
|
||
def test_one_missed_fire_is_not_silence(tmp_path):
|
||
# 40 часов: пропущено одно срабатывание, не два
|
||
assert not silent(tmp_path, closing_ago_h=40).quiet
|
||
|
||
|
||
def test_silence_is_told_once_a_day_and_cleared_by_a_run(tmp_path):
|
||
told = state(tmp_path, quiet={"закрытие": NOW - timedelta(hours=5)})
|
||
held = silent(tmp_path, closing_ago_h=50, runs=told)
|
||
assert not held.quiet
|
||
assert "о молчании уже говорили меньше суток назад: закрытие" in held.note
|
||
older = state(tmp_path, quiet={"закрытие": NOW - timedelta(hours=25)})
|
||
assert silent(tmp_path, closing_ago_h=50, runs=older).quiet
|
||
# Запуск появился после тревоги - «отбой», а не молчание
|
||
back = silent(tmp_path, closing_ago_h=1, runs=told)
|
||
(resumed,) = back.resumed
|
||
assert resumed.job == "закрытие" and not back.clears
|
||
assert "отбой: джоб `закрытие` снова запускается (done)" in clear_text(back)
|
||
|
||
|
||
def test_downtime_of_the_scheduler_is_not_a_missed_fire(tmp_path):
|
||
# Гейтвей только что поднялся: после старта срабатывания ещё не было
|
||
fresh = silent(tmp_path, closing_ago_h=50, boot=NOW - timedelta(minutes=30))
|
||
assert not fresh.quiet
|
||
# Сторож сам не бегал сутки (бюджет зажат, некритичное откладывается):
|
||
# пропуски за это время - общий простой, а не молчание одного джоба
|
||
gap = [run(1, "прогоны", "done", ago=60 * 30)]
|
||
assert not silent(tmp_path, closing_ago_h=50, mine=gap).quiet
|
||
assert awake(gap, now=NOW, step=timedelta(hours=2), boot=None) == NOW
|
||
|
||
|
||
def test_short_period_job_uses_the_watchers_resolution(tmp_path):
|
||
latest = {"закрытие": run(1, "закрытие", "done", ago=60)}
|
||
# Вайбграм раз в 10 минут, молчит 70 минут: сторож часовой, и такой
|
||
# пропуск неотличим от его же слепоты между прогонами
|
||
latest["вайбграм"] = run(2, "вайбграм", "done", ago=70)
|
||
assert not silent(tmp_path, closing_ago_h=1, latest=latest).quiet
|
||
latest["вайбграм"] = run(2, "вайбграм", "done", ago=150)
|
||
(quiet,) = silent(tmp_path, closing_ago_h=1, latest=latest).quiet
|
||
assert quiet.job == "вайбграм"
|
||
assert "не запускался 2 часа при периоде 10 минут" in quiet_text(
|
||
silent(tmp_path, closing_ago_h=1, latest=latest)
|
||
)
|
||
|
||
|
||
def test_job_without_cron_is_not_checked(tmp_path):
|
||
# deploy - вебхук: крона у него нет, в `crons` он не попадает вовсе
|
||
latest = {
|
||
"deploy": run(1, "deploy", "done", ago=60 * 24 * 30),
|
||
"закрытие": run(2, "закрытие", "done", ago=60),
|
||
}
|
||
report = silent(tmp_path, closing_ago_h=1, latest=latest)
|
||
assert not report.quiet and "deploy" not in report.note
|
||
|
||
|
||
def test_job_never_seen_in_the_journal_waits_silently(tmp_path):
|
||
report = silent(tmp_path, closing_ago_h=0, latest={"вайбграм": alive()})
|
||
assert not report.quiet and not report.resumed
|
||
assert "ещё ни разу не запускались: закрытие" in report.note
|
||
|
||
|
||
def test_irregular_schedule_is_skipped_and_named(tmp_path):
|
||
crons = {**CRONS, "куратор": "0 2,8,12,16,20 * * *"}
|
||
report = silent(tmp_path, closing_ago_h=1, crons=crons, latest={})
|
||
assert report.unguarded == ("куратор",)
|
||
assert "давностью не охраняются" in report.note and "куратор" in report.note
|
||
assert not report.quiet
|
||
|
||
|
||
def test_the_watcher_does_not_measure_its_own_silence(tmp_path):
|
||
latest = {
|
||
"прогоны": run(1, "прогоны", "done", ago=60 * 24 * 5),
|
||
"закрытие": run(2, "закрытие", "done", ago=60),
|
||
}
|
||
report = silent(tmp_path, closing_ago_h=1, latest=latest)
|
||
assert not report.quiet
|
||
|
||
|
||
def test_job_removed_from_the_schedule_leaves_silence_state(tmp_path):
|
||
told = state(tmp_path, quiet={"старый": NOW - timedelta(hours=3)})
|
||
report = silent(tmp_path, closing_ago_h=1, runs=told)
|
||
assert not report.resumed and "старый" not in told.quiet
|
||
assert "сняты с давности (нет в расписании): старый" in report.note
|
||
|
||
|
||
def test_real_schedule_names_only_the_curator_unguarded(tmp_path):
|
||
from beaver_agent.jobs import jobs
|
||
|
||
crons = {job.name: job.cron for job in jobs if job.cron}
|
||
report = silent(tmp_path, closing_ago_h=1, crons=crons, latest={})
|
||
assert report.unguarded == ("куратор",)
|
||
# Вебхуки в расписание давности не попадают
|
||
assert {"deploy", "komodo", "t3code"}.isdisjoint(crons)
|
||
|
||
|
||
async def test_job_alarms_on_a_silent_job_and_logs_the_unguarded(
|
||
tmp_path, monkeypatch, caplog
|
||
):
|
||
from beaver_agent.jobs import runs as job
|
||
|
||
monkeypatch.setattr(job, "STATE_FILE", tmp_path / "прогоны.json")
|
||
monkeypatch.setattr(job, "BOOT", datetime.now(UTC) - timedelta(days=10))
|
||
database = await make_db(
|
||
tmp_path,
|
||
[
|
||
record("закрытие", "done", ago=60 * 50),
|
||
*(record("прогоны", "done", ago=60 * h - 15) for h in range(1, 73)),
|
||
record("вайбграм", "done", ago=5),
|
||
record("deploy", "done", ago=60 * 24 * 20),
|
||
],
|
||
)
|
||
run_ = FakeRun(
|
||
database,
|
||
jobs=[
|
||
("закрытие", "20 4 * * *"),
|
||
("вайбграм", "*/10 * * * *"),
|
||
("куратор", "0 2,8,12,16,20 * * *"),
|
||
("deploy", None),
|
||
("новый", "0 3 * * *"),
|
||
],
|
||
)
|
||
|
||
with caplog.at_level("INFO", logger=job.__name__):
|
||
await job.watch_runs(cast("Any", run_))
|
||
|
||
(spawned,) = run_.spawned
|
||
text = spawned[2]
|
||
assert "ПЕРЕСТАЛИ ЗАПУСКАТЬСЯ" in text
|
||
assert "джоб `закрытие` не запускался 2 суток при периоде 24 часа" in text
|
||
assert "deploy" not in text and "новый" not in text
|
||
assert "давностью не охраняются" in caplog.text and "куратор" in caplog.text
|
||
assert "ещё ни разу не запускались: новый" in caplog.text
|
||
assert "закрытие" in Runs.load(tmp_path / "прогоны.json").quiet
|
||
# В те же сутки не повторяется
|
||
await job.watch_runs(cast("Any", run_))
|
||
assert len(run_.spawned) == 1
|
||
await database.dispose()
|